SF.net SVN: docutils:[10205] trunk/docutils/test/test_parsers/test_rst
milde--- via Docutils-checkins <[email protected]> Tue, 19 Aug 2025 15:07:47 +0000
| Newsgroups | gmane.text.docutils.cvs |
|---|---|
| Message-ID | <[email protected]> |
Revision: 10205
http://sourceforge.net/p/docutils/code/10205
Author: milde
Date: 2025-08-19 15:07:47 +0000 (Tue, 19 Aug 2025)
Log Message:
-----------
Test nested parsing in a directive.
Define/use sample directives to test nested parsing with section support.
Advantages:
* The tests are simpler and easier to comprehend.
* Emulate possible use cases. Provide template for extension developers.
* More comprehensive testing.
* Allows test/comparision with the legacy section parsing algorithm
(used up to 0.22).
Tests revealed problems with section styles matching existing styles:
legacy section parsing:
DATA LOSS:
Sections with a level outside the nested parsing are silently dropped!
This happens also with a base node attached to the document.
new section parsing:
After nested parsing, the "insertion point" is restored to what it was
before. If sibling sections or parent sections are attached according to
their level, the insertion point after parsing is before these sections
(the order of the nodes is mixed up).
In a block-quote or other body element the current node is not attached to
the document because the content is parsed with nested_parse into node that
is appended after parsing.
If the current node is used as base node, the section style hierarchy may be
document-wide or local, depending on the placement of the directive.
Modified Paths:
--------------
trunk/docutils/test/test_parsers/test_rst/test_misc.py
Added Paths:
-----------
trunk/docutils/test/test_parsers/test_rst/test_nested_parsing.py
Modified: trunk/docutils/test/test_parsers/test_rst/test_misc.py
===================================================================
--- trunk/docutils/test/test_parsers/test_rst/test_misc.py 2025-08-15 22:28:45 UTC (rev 10204)
+++ trunk/docutils/test/test_parsers/test_rst/test_misc.py 2025-08-19 15:07:47 UTC (rev 10205)
@@ -9,7 +9,6 @@
from pathlib import Path
import sys
-import types
import unittest
if __name__ == '__main__':
@@ -16,9 +15,8 @@
# prepend the local "docutils root" to the Python library path
sys.path.insert(0, Path(__file__).resolve().parents[2].as_posix())
-from docutils import frontend, nodes, statemachine, utils
+from docutils import frontend, utils
import docutils.parsers.rst
-from docutils.parsers.rst import states
class RstParserTests(unittest.TestCase):
@@ -32,172 +30,5 @@
parser.parse(b'hol', document)
-class RSTStateTests(unittest.TestCase):
-
- # state machine
- machine = states.RSTStateMachine(state_classes=states.state_classes,
- initial_state='Body')
- # "generic" state
- state = states.RSTState(machine)
-
- def title_markup(self, text, adornment='-'):
- underline = adornment * len(text)
- return statemachine.StringList([text, underline], 'section block')
-
- def setUp(self):
- # state machine runtime initialization (cf. RSTStateMachine.run())
- # Only for test:
- # don't use this low-level approach in production code!
-
- # empty <document> and settings:
- settings = frontend.get_default_settings(docutils.parsers.rst.Parser)
- settings.halt_level = 2
- settings.warning_stream = ''
- document = self.document = utils.new_document('test data', settings)
- # language module (localized directive and role names)
- # self.machine.language = languages.get_language(
- # document.settings.language_code)
- # self.machine.match_titles = True # support sections
- # "memo": Container for document-wide auxiliary data
- inliner = states.Inliner() # rST parser for inline markup
- inliner.init_customizations(document.settings)
- self.machine.memo = types.SimpleNamespace(document=document,
- reporter=document.reporter,
- language='en',
- title_styles=[],
- inliner=inliner)
- # self.machine.document = document
- # self.machine.reporter = document.reporter
- self.machine.node = document
- # initialize `state` object
- self.state.runtime_init()
-
- def test_nested_parse(self):
- # parse a text block in a nested parser
- text = statemachine.StringList(['test input'], 'nested block')
- tip = nodes.tip('') # base `node`
-
- # plain text -> attach <paragraph> to `node`:
- self.state.nested_parse(text, input_offset=0, node=tip)
- self.assertEqual('<tip><paragraph>test input</paragraph></tip>',
- str(tip))
-
- # by default, section titles are not supported:
- title = self.title_markup('top', '-')
- with self.assertRaisesRegex(utils.SystemMessage,
- 'Unexpected section title.'):
- self.state.nested_parse(title, input_offset=0, node=tip)
-
- # Nested parsing with section markup is supported with
- # ``match_titles=True`` but not used in Docutils.
- # (However, Sphinx "autodoc" and some contributed extensions use it.)
- def test_nested_parse_with_sections_base_attached(self):
- # If the base `node` is attached to the `document`,
- # the document-wide section style hierarchy is used.
-
- # base`node` is a <section>
- section = nodes.section('')
- self.document += section # attach to document
- self.machine.memo.title_styles.append('-') # register title style
- # parse level-2 section title
- title = self.title_markup('sub', '~')
- # -> append new <section> to the parsers base `node`
- self.state.nested_parse(title, 0, node=section, match_titles=True)
- self.assertEqual('<document source="test data">\n'
- ' <section>\n'
- ' <section ids="sub" names="sub">\n'
- ' <title>\n'
- ' sub\n',
- str(self.document.pformat()))
- self.assertEqual(['-', '~'], self.machine.memo.title_styles)
-
- # parse top-level section title
- title = self.title_markup('top', '-')
- # -> move 1 level up and attach <section> to document
- self.state.nested_parse(title, 0, node=section, match_titles=True)
- self.assertEqual('<document source="test data">\n'
- ' <section>\n'
- ' <section ids="sub" names="sub">\n'
- ' <title>\n'
- ' sub\n'
- ' <section ids="top" names="top">\n'
- ' <title>\n'
- ' top\n',
- str(self.document.pformat()))
-
- # base`node` is a <paragraph>
- paragraph = nodes.paragraph('', 'base node')
- section += paragraph # attach (indirectly)
- # parse top-level section title
- title = self.title_markup('top 2', '-')
- # -> move 1 level up and attach <section> to document
- self.state.nested_parse(title, 0, node=paragraph, match_titles=True)
- self.assertEqual('<section ids="top-2" names="top\\ 2">\n'
- ' <title>\n'
- ' top 2\n',
- self.document[-1].pformat())
-
- # new (2nd-level) section title
- title = self.title_markup('sub 2', '~')
- self.state.nested_parse(title, 0, node=paragraph, match_titles=True)
- self.assertEqual('<section>\n'
- ' <section ids="sub" names="sub">\n'
- ' <title>\n'
- ' sub\n'
- ' <paragraph>\n'
- ' base node\n'
- ' <section ids="sub-2" names="sub\\ 2">\n'
- ' <title>\n'
- ' sub 2\n',
- section.pformat())
-
- def test_nested_parse_with_sections_detached(self):
- # The base `node` does not need to be attached to the document.
- # "global" title style hierarchy (ignored with detached base node)
- self.machine.memo.title_styles = ['-', '~']
-
- # base `node` is a <paragraph> without parents
- base = nodes.paragraph('')
- base.document = self.document # this is not "attaching"
- # level-2 title style
- title = self.title_markup('sub', '~')
- # new hierarchy -> attach <section> to base `node`
- self.state.nested_parse(title, 0, node=base, match_titles=True)
- self.assertEqual('<paragraph>\n'
- ' <section ids="sub" names="sub">\n'
- ' <title>\n'
- ' sub\n',
- base.pformat())
- # It is the users responsibility to ensure that the base node
- # may contain a <section> (or move the section after parsing).
- # You may check with `validate()`:
- with self.assertRaises(nodes.ValidationError):
- base.validate()
-
- # a new hierarchy is used in every call of nested_parse()
- # parse 2 section titles
- title = self.title_markup('new', '*') + self.title_markup('top', '-')
- # new hierarchy -> attach section and sub-section to base node
- self.state.nested_parse(title, 0, node=base, match_titles=True)
- self.assertEqual(
- '<paragraph>\n'
- ' <section ids="sub" names="sub">\n'
- ' <title>\n'
- ' sub\n'
- ' <section ids="new" names="new">\n'
- ' <title>\n'
- ' new\n'
- ' <section ids="top" names="top">\n'
- ' <title>\n'
- ' top\n',
- base.pformat())
-
- # document-wide style hierarchy unchanged:
- self.assertEqual(['-', '~'], self.machine.memo.title_styles)
-
- # print(self.document.pformat())
- # print(base.pformat())
-
-
if __name__ == '__main__':
unittest.main()
Added: trunk/docutils/test/test_parsers/test_rst/test_nested_parsing.py
===================================================================
--- trunk/docutils/test/test_parsers/test_rst/test_nested_parsing.py (rev 0)
+++ trunk/docutils/test/test_parsers/test_rst/test_nested_parsing.py 2025-08-19 15:07:47 UTC (rev 10205)
@@ -0,0 +1,327 @@
+#! /usr/bin/env python3
+# $Id$
+# Author: David Goodger <[email protected]>
+# Copyright: This module has been placed in the public domain.
+
+"""
+Tests for nested parsing with support for sections (cf. states.py).
+
+The method states.RSTState.nested_parse() provides the argument `match_titles`.
+However, in Docutils, it is only used with `match_titles=False`.
+None of the standard Docutils directives supports section titles in the
+directive content. (Directives supporting sections in the content are,
+e.g., defined by the "autodoc" and "kerneldoc" Sphinx extensions.)
+
+Up to Docutils 0.22, the section title styles were document-wide enforced and
+sections with current level or higher were silently dropped!
+
+Sphinx uses the `sphinx.util.parsing._fresh_title_style_context` context
+manager to provide a separate title style hierarchy for nested parsing.
+"""
+
+from pathlib import Path
+import sys
+import unittest
+
+if __name__ == '__main__':
+ # prepend the "docutils root" to the Python library path
+ # so we import the local `docutils` package.
+ sys.path.insert(0, str(Path(__file__).resolve().parents[3]))
+
+from docutils import nodes
+from docutils.frontend import get_default_settings
+from docutils.parsers import rst
+from docutils.utils import new_document
+
+
+class ParseIntoDetachedNode(rst.Directive):
+ """A directive implementing nested parsing with support for sections.
+ """
+ final_argument_whitespace = True
+ has_content = True
+
+ def run(self):
+ # similar to sphinx.util.parsing.nested_parse_to_nodes()
+ node = nodes.Element()
+ node.document = self.state.document
+ self.state.nested_parse(self.content, input_offset=0,
+ node=node, match_titles=True)
+ return node.children
+
+
+class ParseIntoCurrentNode(ParseIntoDetachedNode):
+ def run(self):
+ node = self.state.parent # the current "insertion point"
+ self.state.nested_parse(self.content, 0, node, match_titles=True)
+ return [] # nodes already attached to document
+
+
+class ParseIntoAttachedNode(ParseIntoDetachedNode):
+ def run(self):
+ node = nodes.sidebar('')
+ self.state.parent.append(node)
+ self.state.nested_parse(self.content, 0, node, match_titles=True)
+ return [] # nodes already attached to document
+
+
+class ParserTestCase(unittest.TestCase):
+ maxDiff = None
+
+ def test_parser(self):
+ rst.directives.register_directive('nested-detached',
+ ParseIntoDetachedNode)
+ rst.directives.register_directive('nested-current',
+ ParseIntoCurrentNode)
+ rst.directives.register_directive('nested-attached',
+ ParseIntoAttachedNode)
+ parser = rst.Parser()
+ settings = get_default_settings(rst.Parser)
+ settings.warning_stream = ''
+ settings.halt_level = 5
+ for name, cases in totest.items():
+ for casenum, (case_input, case_expected) in enumerate(cases):
+ with self.subTest(id=f'totest[{name!r}][{casenum}]'):
+ document = new_document('test data', settings.copy())
+ parser.parse(case_input, document)
+ output = document.pformat()
+ self.assertEqual(case_expected, output)
+
+
+totest = {}
+
+# Parse into the base node:
+totest['nested_parsing'] = [
+["""\
+Preceding paragraph.
+
+.. nested-attached::
+
+ .. hint:: this is nested.
+
+Succeeding paragraph.
+""",
+"""\
+<document source="test data">
+ <paragraph>
+ Preceding paragraph.
+ <sidebar>
+ <hint>
+ <paragraph>
+ this is nested.
+ <paragraph>
+ Succeeding paragraph.
+"""],
+# detached base node -> start new section hierarchy with every nested parse
+["""\
+sec1
+====
+sec1.1
+------
+.. nested-detached::
+
+ detached1
+ *********
+ detached1.1
+ -----------
+ detached1.1.1
+ =============
+
+.. nested-detached::
+
+ detached2
+ ---------
+ detached2.1
+ ***********
+
+Succeeding paragraph.
+
+sec2
+====
+The document-wide section title styles are kept.
+""",
+"""\
+<document source="test data">
+ <section ids="sec1" names="sec1">
+ <title>
+ sec1
+ <section ids="sec1-1" names="sec1.1">
+ <title>
+ sec1.1
+ <section ids="detached1" names="detached1">
+ <title>
+ detached1
+ <section ids="detached1-1" names="detached1.1">
+ <title>
+ detached1.1
+ <section ids="detached1-1-1" names="detached1.1.1">
+ <title>
+ detached1.1.1
+ <section ids="detached2" names="detached2">
+ <title>
+ detached2
+ <section ids="detached2-1" names="detached2.1">
+ <title>
+ detached2.1
+ <paragraph>
+ Succeeding paragraph.
+ <section ids="sec2" names="sec2">
+ <title>
+ sec2
+ <paragraph>
+ The document-wide section title styles are kept.
+"""],
+# base node == current node -> keep section hierarchy
+["""\
+sec1
+====
+sec1.1
+------
+.. nested-current::
+
+ current1
+ ********
+ sec1.2
+ -----------
+ Sibling section appended 1 level up.
+
+ sec2
+ =========
+ Top-level section appended to document.
+
+Succeeding paragraph. TODO: currently misplaced!
+""",
+"""\
+<document source="test data">
+ <section ids="sec1" names="sec1">
+ <title>
+ sec1
+ <section ids="sec1-1" names="sec1.1">
+ <title>
+ sec1.1
+ <section ids="current1" names="current1">
+ <title>
+ current1
+ <paragraph>
+ Succeeding paragraph. TODO: currently misplaced!
+ <section ids="sec1-2" names="sec1.2">
+ <title>
+ sec1.2
+ <paragraph>
+ Sibling section appended 1 level up.
+ <section ids="sec2" names="sec2">
+ <title>
+ sec2
+ <paragraph>
+ Top-level section appended to document.
+"""],
+# parse into attached wrapper node:
+["""\
+sec1
+====
+sec1.1
+------
+.. nested-attached::
+
+ attached1
+ *********
+ sec2
+ =========
+ Nested top-level section appended to document.
+
+Succeeding paragraph. TODO: currently misplaced!
+""",
+"""\
+<document source="test data">
+ <section ids="sec1" names="sec1">
+ <title>
+ sec1
+ <section ids="sec1-1" names="sec1.1">
+ <title>
+ sec1.1
+ <sidebar>
+ <section ids="attached1" names="attached1">
+ <title>
+ attached1
+ <paragraph>
+ Succeeding paragraph. TODO: currently misplaced!
+ <section ids="sec2" names="sec2">
+ <title>
+ sec2
+ <paragraph>
+ Nested top-level section appended to document.
+"""],
+# detached base node -> start new section hierarchy
+["""\
+sec1
+====
+sec1.1
+------
+sec2
+====
+.. nested-detached::
+ detached1
+ ~~~~~~~~~
+ detached1.1
+ -----------
+
+Succeeding paragraph.
+""",
+"""\
+<document source="test data">
+ <section ids="sec1" names="sec1">
+ <title>
+ sec1
+ <section ids="sec1-1" names="sec1.1">
+ <title>
+ sec1.1
+ <section ids="sec2" names="sec2">
+ <title>
+ sec2
+ <section ids="detached1" names="detached1">
+ <title>
+ detached1
+ <section ids="detached1-1" names="detached1.1">
+ <title>
+ detached1.1
+ <paragraph>
+ Succeeding paragraph.
+"""],
+# base node == <blockquote>
+["""\
+sec1
+====
+
+ A block-quote is parsed into a detached <blockquote> element.
+
+ .. nested-current::
+
+ nested section
+ ==============
+
+ The nested <section> becomes a child of the <blockquote> (sic.)!
+
+The calling directive should move the nested <section> or report
+a validity violation.
+""",
+"""\
+<document source="test data">
+ <section ids="sec1" names="sec1">
+ <title>
+ sec1
+ <block_quote>
+ <paragraph>
+ A block-quote is parsed into a detached <blockquote> element.
+ <section ids="nested-section" names="nested\\ section">
+ <title>
+ nested section
+ <paragraph>
+ The nested <section> becomes a child of the <blockquote> (sic.)!
+ <paragraph>
+ The calling directive should move the nested <section> or report
+ a validity violation.
+"""],
+]
+
+
+if __name__ == '__main__':
+ unittest.main()
Property changes on: trunk/docutils/test/test_parsers/test_rst/test_nested_parsing.py
___________________________________________________________________
Added: svn:eol-style
## -0,0 +1 ##
+native
\ No newline at end of property
Added: svn:keywords
## -0,0 +1 ##
+Author Date Id Revision
\ No newline at end of property
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.