SF.net SVN: docutils:[9737 ] trunk/docutils

Revision: 9737
          http://sourceforge.net/p/docutils/code/9737
Author:   milde
Date:     2024-06-06 14:02:08 +0000 (Thu, 06 Jun 2024)
Log Message:
-----------
xml-parser: test and fix handling of invalid input

Gracefully handle invalid attributes and text inserts:
Just generate (invalid) nodes without complaining.

With, e.g., `docutils --parser=xml myfile.xml`, the document tree
is validated by the `universal.Validate` transform by default.

Nodes returned from `parse_elememt()` can be easily validated
via their `validate()` method.

+ Allows parsing XML documents with "extended" document model
  for special applications.
+ Allows for cleanup operations between parsing and validation.
+ Simpler implementation (no duplicating of the reporting framework
  of the validator).

Modified Paths:
--------------
    trunk/docutils/docutils/parsers/docutils_xml.py
    trunk/docutils/test/test_parsers/test_docutils_xml/test_parse.py
    trunk/docutils/test/test_parsers/test_docutils_xml/test_parse_element.py

Modified: trunk/docutils/docutils/parsers/docutils_xml.py
===================================================================
--- trunk/docutils/docutils/parsers/docutils_xml.py	2024-06-06 14:01:54 UTC (rev 9736)
+++ trunk/docutils/docutils/parsers/docutils_xml.py	2024-06-06 14:02:08 UTC (rev 9737)
@@ -52,6 +52,11 @@
         self.finish_parse()
 
 
+class Unknown(nodes.Special, nodes.Inline, nodes.Element):
+    """An unknown element found by the XML parser."""
+    content_model = (((nodes.Element, nodes.Text), '*'),)  # no restrictions
+
+
 def parse_element(inputstring, document=None):
     """
     Parse `inputstring` as "Docutils XML", return `nodes.Element` instance.
@@ -76,11 +81,19 @@
     root = None
     parser = ET.XMLPullParser(events=('start',))
     for i, line in enumerate(inputstring.splitlines(keepends=True)):
-        parser.feed(line)
-        for event, element in parser.read_events():
-            if root is None:
-                root = element
-            element.attrib['source line'] = str(i+1)
+        try:
+            parser.feed(line)
+            for event, element in parser.read_events():
+                if root is None:
+                    root = element
+                element.attrib['source line'] = str(i+1)
+        except ET.ParseError as e:
+            if document is None:
+                raise
+            document.reporter.error(f'XML parse error: {e}.',
+                                    source=document.settings._source,
+                                    line=e.position[0])
+            break
     return element2node(root, document)
 
 
@@ -95,21 +108,38 @@
     if document is None:
         document = utils.new_document('xml input',
                                       frontend.get_default_settings(Parser))
+        document.source == 'xml input'
 
     # Get the corresponding `nodes.Element` instance:
-    nodeclass = getattr(nodes, element.tag)
+    try:
+        nodeclass = getattr(nodes, element.tag)
+        if not issubclass(nodeclass, nodes.Element):
+            nodeclass = Unknown
+    except AttributeError:
+        nodeclass = Unknown
     if nodeclass == nodes.document:
         node = document
+        document.source = document.source or document.settings._source
     else:
         node = nodeclass()
 
     node.line = int(element.get('source line'))
+    if isinstance(node, Unknown):
+        node.tagname = element.tag
+        document.reporter.warning(
+            f'Unknown element type <{element.tag}>.',
+            base_node=node)
 
     # Attributes: convert and add to `node.attributes`.
     for key, value in element.items():
         if key.startswith('{') or key == 'source line':
             continue  # skip duplicate attributes with namespace URL
-        node.attributes[key] = nodes.ATTRIBUTE_VALIDATORS[key](value)
+        try:
+            node.attributes[key] = nodes.ATTRIBUTE_VALIDATORS[key](value)
+        except (ValueError, KeyError):
+            if key in node.list_attributes:
+                value = value.split()
+            node.attributes[key] = value  # node becomes invalid!
 
     # Append text (wrapped in a `nodes.Text` instance)
     append_text(node, element.text)

Modified: trunk/docutils/test/test_parsers/test_docutils_xml/test_parse.py
===================================================================
--- trunk/docutils/test/test_parsers/test_docutils_xml/test_parse.py	2024-06-06 14:01:54 UTC (rev 9736)
+++ trunk/docutils/test/test_parsers/test_docutils_xml/test_parse.py	2024-06-06 14:02:08 UTC (rev 9737)
@@ -131,6 +131,63 @@
 """],
 ]
 
+totest['invalid'] = [
+["""\
+<document>
+    <tip>
+        spurious text
+        <paragraph>A paragraph.</paragraph>
+    </tip>
+</document>
+""",
+"""\
+<document source="test data">
+    <tip>
+        spurious text
+        <paragraph>
+            A paragraph.
+"""],
+["""\
+<document>
+    spurious text
+    <paragraph>A paragraph.</paragraph>
+</document>
+""",
+"""\
+<document source="test data">
+    spurious text
+    <paragraph>
+        A paragraph.
+"""],
+["""\
+<document>
+    <tip>
+        <paragraph>A paragraph.</paragraph>
+        spurious tailing text
+    </tip>
+</document>
+""",
+"""\
+<document source="test data">
+    <tip>
+        <paragraph>
+            A paragraph.
+        spurious tailing text
+"""],
+["""\
+<document>
+    <paragraph>A paragraph.</paragraph>
+    spurious tailing text
+</document>
+""",
+"""\
+<document source="test data">
+    <paragraph>
+        A paragraph.
+    spurious tailing text
+"""],
+]
 
+
 if __name__ == '__main__':
     unittest.main()

Modified: trunk/docutils/test/test_parsers/test_docutils_xml/test_parse_element.py
===================================================================
--- trunk/docutils/test/test_parsers/test_docutils_xml/test_parse_element.py	2024-06-06 14:01:54 UTC (rev 9736)
+++ trunk/docutils/test/test_parsers/test_docutils_xml/test_parse_element.py	2024-06-06 14:02:08 UTC (rev 9737)
@@ -21,23 +21,82 @@
     # so we import the local `docutils` package.
     sys.path.insert(0, str(Path(__file__).resolve().parents[3]))
 
+from docutils import frontend, utils
 from docutils.parsers import docutils_xml
 
 
 class ParseElementTestCase(unittest.TestCase):
+    """Test the `docutils.xml.parse_element()` function."""
 
+    # supress warnings when passing `document` to `parse_element()`
+    settings = frontend.get_default_settings(docutils_xml.Parser)
+    settings.warning_stream = ''  # comment out to see warnings
+    document = utils.new_document('xml input', settings)
+
     def test_element_with_child_with_text(self):
         xml = '<tip><paragraph>some text</paragraph></tip>'
         node = docutils_xml.parse_element(xml)
         self.assertEqual(xml, str(node))
 
-    def test_tailing_text(self):
-        xml = '<strong>text</strong>trailing text'
-        with self.assertRaisesRegex(ET.ParseError,
-                                    'junk after document element'):
+    def test_tailing_text_after_root(self):
+        """etree.ElementTree does not accept tailing text in the input.
+        """
+        xml = '<strong>text</strong>tailing text'
+        with self.assertRaisesRegex(ET.ParseError, 'junk after document '):
             docutils_xml.parse_element(xml)
+        # If a document is provided, report via a "loose" error system message
+        # comment out ``settings.warning_stream = ''`` above to see it).
+        node = docutils_xml.parse_element(xml, self.document)
+        self.assertEqual('<strong>text</strong>', str(node))
 
+    def test_nonexistent_element_type(self):
+        xml = '<tip><p>some text</p></tip>'
+        node = docutils_xml.parse_element(xml, self.document)
+        self.assertEqual(xml, str(node))
+        # see test_misc.py for the warning
 
+    def test_junk_text(self):
+        # insert text also in nodes that are not TextElement instances
+        xml = '<tip>some text</tip>'
+        node = docutils_xml.parse_element(xml)
+        self.assertEqual(xml, str(node))
+        with self.assertRaisesRegex(ValueError,
+                                    'Expecting child of type <Body>,'
+                                    ' not text data "some text"'):
+            node.validate()
+
+    def test_tailing_junk_text(self):
+        # insert text also in nodes that are not TextElement instances
+        xml = '<tip><paragraph>some text</paragraph>tailing text</tip>'
+        node = docutils_xml.parse_element(xml)
+        self.assertEqual(xml, str(node))
+        with self.assertRaisesRegex(
+            ValueError, 'Spurious text: "tailing text"'):
+            node.validate()
+
+    def test_element_with_attributes(self):
+        xml = ('<image align="left" alt="a barking dog" height="3ex"'
+               ' loading="embed" scale="3" uri="dog.jpg" width="4cm"/>')
+        node = docutils_xml.parse_element(xml)
+        self.assertEqual(xml, str(node))
+
+    def test_element_with_invalid_attributes(self):
+        """Silently accept invalid attribute names and values.
+
+        Validation reports problems.
+        """
+        xml = ('<image breadth="3 cm" height="3 inch"/>')
+        node = docutils_xml.parse_element(xml)
+        self.assertEqual(xml, str(node))
+        with self.assertRaisesRegex(ValueError,
+                                    'Element <image breadth="3 cm".*invalid:\n'
+                                    '.*"breadth" not one of "ids",.*\n'
+                                    '.*"height" has invalid value "3 inch".\n'
+                                    '.*Valid units: em ex px in cm mm pt '
+                                    ):
+            node.validate()
+
+
 class XmlAttributesTestCase(unittest.TestCase):
     """
     Test correct parsing of the `supported element attributes`_.

This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.