fix: set the QTI item language on the real root past prolog comments

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>

Author: rtibblesbot

contentcuration/contentcuration/tests/utils/qti/test_media.py

@@ -1,3 +1,5 @@
+import pytest
+
 from contentcuration.tests.utils.qti.test_validation import VALID_CHOICE_ITEM
 from contentcuration.utils.assessment.qti.media import get_qti_media_references
 from contentcuration.utils.assessment.qti.media import rewrite_qti_media_paths
@@ -95,6 +97,11 @@ def test_rewrite_ignores_values_not_in_mapping():
 )
 
 
+ITEM_WITH_ES = ITEM_WITHOUT_LANGUAGE.replace(
+    'time-dependent="false">', 'time-dependent="false" xml:lang="es">'
+)
+
+
 def test_set_language_adds_it_when_the_item_has_none():
     result = set_qti_item_language(ITEM_WITHOUT_LANGUAGE, "es")
     assert 'xml:lang="es"' in result
@@ -187,3 +194,59 @@ def test_strip_studio_attributes_ignores_lookalikes_in_values_and_text():
         '<p title="a > b data-studio-prompt=\'\'" data-other="1">data-studio-prompt=""</p>',
     )
     assert strip_studio_attributes(tricky) == tricky
+
+
+def test_set_language_skips_a_comment_quoting_the_root_tag_after_the_declaration():
+    declaration, _, root = VALID_CHOICE_ITEM.partition("?>")
+    comment = '<!-- <qti-assessment-item foo="1"> -->'
+    result = set_qti_item_language(f"{declaration}?>{comment}{root}", "fr")
+    assert comment in result
+    assert result.count('xml:lang="fr"') == 1
+    assert 'xml:lang="en-US"' not in result
+    validation = validate_qti_item(result)
+    assert validation.is_valid, validation.errors
+
+
+def test_set_language_skips_a_comment_quoting_the_root_tag_with_its_own_language():
+    comment = '<!-- <qti-assessment-item xml:lang="de"> -->\n'
+    result = set_qti_item_language(comment + ITEM_WITHOUT_LANGUAGE, "es")
+    assert result == comment + ITEM_WITH_ES
+
+
+@pytest.mark.parametrize(
+    "prolog",
+    [
+        '<!--\n  <qti-assessment-item a="1">\n  > still a comment\n-->\n',
+        '<?note <qti-assessment-item a="1"> ?>\n',
+        "<!DOCTYPE x [<!ENTITY e \"<qti-assessment-item a='1'>\">]>\n",
+        '<!DOCTYPE x SYSTEM "a>b">\n',
+        "<!DOCTYPE x SYSTEM 'a[b'>\n",
+        "<!DOCTYPE x [\n<!-- don't -->\n]>\n",
+        "<!DOCTYPE x [\n<!-- a ] b -->\n]>\n",
+        "<!DOCTYPE x [\n<?p it's ?>\n]>\n",
+    ],
+)
+def test_set_language_skips_other_prolog_markup_quoting_the_root_tag(prolog):
+    result = set_qti_item_language(prolog + ITEM_WITHOUT_LANGUAGE, "es")
+    assert result == prolog + ITEM_WITH_ES
+
+
+@pytest.mark.parametrize(
+    "prolog",
+    [
+        "<!-- <qti-assessment-item a='1'>\n",
+        "<?note <qti-assessment-item a='1'>\n",
+        "<!DOCTYPE x [<!ENTITY e \"<qti-assessment-item a='1'>\n",
+        "<!DOCTYPE x [<!-- <qti-assessment-item a='1'>\n",
+    ],
+)
+def test_set_language_leaves_an_item_with_unterminated_prolog_markup_unchanged(
+    prolog,
+):
+    raw_data = prolog + ITEM_WITHOUT_LANGUAGE
+    assert set_qti_item_language(raw_data, "es") == raw_data
+
+
+def test_set_language_stamps_an_item_with_a_byte_order_mark():
+    result = set_qti_item_language("\ufeff" + ITEM_WITHOUT_LANGUAGE, "es")
+    assert result == "\ufeff" + ITEM_WITH_ES

contentcuration/contentcuration/utils/assessment/qti/media.py

@@ -25,13 +25,68 @@
 STUDIO_ATTRIBUTE_OR_QUOTED_VALUE_REGEX = re.compile(
     r"""(?P<studio>\s+data-studio-[\w.-]+\s*=\s*(?:"[^"]*"|'[^']*'))|"[^"]*"|'[^']*'"""
 )
+_WHITESPACE_REGEX = re.compile(r"\s*")
+_DOCTYPE_PLAIN_REGEX = re.compile(r"[^\"'<>\[\]]*")
 
 QTI_MEDIA_ATTRIBUTE_VALUE_REGEX = re.compile(
     r"(?P<attr>" + "|".join(QTI_REFERENCE_ATTRIBUTES + ("srcset",)) + r")"
     r'(?P<eq>\s*=\s*)(?P<quote>["\'])(?P<value>[^"\']*)(?P=quote)'
 )
 
 
+def _skip_past(raw_data, terminator, start):
+    end = raw_data.find(terminator, start)
+    return None if end == -1 else end + len(terminator)
+
+
+def _skip_doctype(raw_data, position):
+    """Return the offset past a DOCTYPE starting at ``position``, or None if unterminated."""
+    in_subset = False
+    position += len("<!DOCTYPE")
+    while position is not None:
+        position = _DOCTYPE_PLAIN_REGEX.match(raw_data, position).end()
+        if position >= len(raw_data):
+            break
+        char = raw_data[position]
+        if char in "\"'":
+            position = _skip_past(raw_data, char, position + 1)
+        elif in_subset and raw_data.startswith("<!--", position):
+            position = _skip_past(raw_data, "-->", position + 4)
+        elif in_subset and raw_data.startswith("<?", position):
+            position = _skip_past(raw_data, "?>", position + 2)
+        elif char == "[":
+            in_subset = True
+            position += 1
+        elif char == "]":
+            in_subset = False
+            position += 1
+        elif char == ">" and not in_subset:
+            return position + 1
+        else:
+            position += 1
+    return None
+
+
+def _find_root_start_tag(raw_data):
+    """
+    Locate the item root start tag by stepping over the prolog (comments, processing
+    instructions, DOCTYPE) that may quote it. Scans with ``str.find`` so memory stays
+    constant however large the prolog is.
+    """
+    position = 1 if raw_data.startswith("\ufeff") else 0
+    while position is not None:
+        position = _WHITESPACE_REGEX.match(raw_data, position).end()
+        if raw_data.startswith("<!--", position):
+            position = _skip_past(raw_data, "-->", position + 4)
+        elif raw_data.startswith("<?", position):
+            position = _skip_past(raw_data, "?>", position + 2)
+        elif raw_data.startswith("<!DOCTYPE", position):
+            position = _skip_doctype(raw_data, position)
+        else:
+            return ITEM_ROOT_START_TAG_REGEX.match(raw_data, position)
+    return None
+
+
 def get_qti_media_references(raw_data):
     """
     Scan QTI item XML for <checksum>.<ext> media references in src/href/data/srcset
@@ -115,7 +170,10 @@ def _replace_root(match):
                 return tag[: attribute.start()] + replacement + tag[attribute.end() :]
         return f"{tag[:-1].rstrip()}{replacement}>"
 
-    return ITEM_ROOT_START_TAG_REGEX.sub(_replace_root, raw_data, count=1)
+    match = _find_root_start_tag(raw_data)
+    if match is None:
+        return raw_data
+    return raw_data[: match.start()] + _replace_root(match) + raw_data[match.end() :]
 
 
 def strip_studio_attributes(raw_data):