Update tests for Pygments 2.14

Dear all,

Pygments 2.14, released in January__, contains changes__ in the Python
lexer which break ``test_code``​​, ``test_code_long``, and ``test_include``.

__ https://pygments.org/docs/changelog/#version-2-14-0
__ https://github.com/pygments/pygments/commit/147b22face65617514ccfa8512c6b097b07cad4c

I have attached a sample patch to fix these and maintain support in the
tests for Pygments 2.13 and earlier. A different approach would be to
declare that the tests only support the latest version of Pygments (it
is currently unclear which version of Pygments is our minimum supported).

Thanks,
Adam


-------------


From 8ad1babfdab6b3ba715a1aa8712e2044740fd2ba Mon Sep 17 00:00:00 2001
From: Adam Turner <[email protected]>
Date: Thu, 6 Apr 2023 17:24:40 +0100
Subject: [PATCH] Support Pygments 2.14 and newer in tests

---
 .../test_rst/test_directives/test_code.py     |  62 +++++++
 .../test_directives/test_code_long.py         |  62 +++++++
 .../test_rst/test_directives/test_include.py  | 154 ++++++++++++++++++
 3 files changed, 278 insertions(+)

diff --git a/docutils/test/test_parsers/test_rst/test_directives/test_code.py b/docutils/test/test_parsers/test_rst/test_directives/test_code.py
index 81efaaefa..7985d1b65 100755
--- a/docutils/test/test_parsers/test_rst/test_directives/test_code.py
+++ b/docutils/test/test_parsers/test_rst/test_directives/test_code.py
@@ -22,6 +22,14 @@
 from docutils.utils import new_document
 from docutils.utils.code_analyzer import with_pygments
 
+try:
+    from pygments import __version__ as _pygments_ver
+except ImportError:
+    _pygments_ver = ''
+    PYGMENTS_2_14_PLUS = False
+else:
+    PYGMENTS_2_14_PLUS = tuple(map(int, _pygments_ver.split('.'))) >= (2, 14)
+
 
 class ParserTestCase(unittest.TestCase):
     def test_parser(self):
@@ -153,6 +161,60 @@ def my_function():
       print(8/2)
 """,
 """\
+<document source="test data">
+    <literal_block classes="code python3 testclass" ids="my-function" names="my_function" xml:space="preserve">
+        <inline classes="ln">
+             7 \n\
+        <inline classes="keyword">
+            def
+         \n\
+        <inline classes="name function">
+            my_function
+        <inline classes="punctuation">
+            ():
+        <inline classes="whitespace">
+            \n\
+        <inline classes="ln">
+             8 \n\
+        <inline classes="whitespace">
+                \n\
+        <inline classes="literal string doc">
+            \'\'\'Test the lexer.
+        <inline classes="ln">
+             9 \n\
+        <inline classes="literal string doc">
+                \'\'\'
+        <inline classes="whitespace">
+            \n\
+        <inline classes="ln">
+            10 \n\
+        <inline classes="whitespace">
+            \n\
+        <inline classes="ln">
+            11 \n\
+        <inline classes="whitespace">
+            \n\
+        <inline classes="comment single">
+            # and now for something completely different
+        <inline classes="whitespace">
+            \n\
+        <inline classes="ln">
+            12 \n\
+        <inline classes="whitespace">
+            \n\
+        <inline classes="name builtin">
+            print
+        <inline classes="punctuation">
+            (
+        <inline classes="literal number integer">
+            8
+        <inline classes="operator">
+            /
+        <inline classes="literal number integer">
+            2
+        <inline classes="punctuation">
+            )
+""" if PYGMENTS_2_14_PLUS else """\
 <document source="test data">
     <literal_block classes="code python3 testclass" ids="my-function" names="my_function" xml:space="preserve">
         <inline classes="ln">
diff --git a/docutils/test/test_parsers/test_rst/test_directives/test_code_long.py b/docutils/test/test_parsers/test_rst/test_directives/test_code_long.py
index 07f1b9f36..641a5f1d9 100755
--- a/docutils/test/test_parsers/test_rst/test_directives/test_code_long.py
+++ b/docutils/test/test_parsers/test_rst/test_directives/test_code_long.py
@@ -22,6 +22,14 @@
 from docutils.utils import new_document
 from docutils.utils.code_analyzer import with_pygments
 
+try:
+    from pygments import __version__ as _pygments_ver
+except ImportError:
+    _pygments_ver = ''
+    PYGMENTS_2_14_PLUS = False
+else:
+    PYGMENTS_2_14_PLUS = tuple(map(int, _pygments_ver.split('.'))) >= (2, 14)
+
 
 @unittest.skipUnless(with_pygments, 'needs Pygments')
 class ParserTestCase(unittest.TestCase):
@@ -54,6 +62,60 @@ def my_function():
       print(8/2)
 """,
 """\
+<document source="test data">
+    <literal_block classes="code python3" xml:space="preserve">
+        <inline classes="ln">
+             7 \n\
+        <inline classes="keyword">
+            def
+         \n\
+        <inline classes="name function">
+            my_function
+        <inline classes="punctuation">
+            ():
+        <inline classes="whitespace">
+            \n\
+        <inline classes="ln">
+             8 \n\
+        <inline classes="whitespace">
+                \n\
+        <inline classes="literal string doc">
+            \'\'\'Test the lexer.
+        <inline classes="ln">
+             9 \n\
+        <inline classes="literal string doc">
+                \'\'\'
+        <inline classes="whitespace">
+            \n\
+        <inline classes="ln">
+            10 \n\
+        <inline classes="whitespace">
+            \n\
+        <inline classes="ln">
+            11 \n\
+        <inline classes="whitespace">
+            \n\
+        <inline classes="comment single">
+            # and now for something completely different
+        <inline classes="whitespace">
+            \n\
+        <inline classes="ln">
+            12 \n\
+        <inline classes="whitespace">
+            \n\
+        <inline classes="name builtin">
+            print
+        <inline classes="punctuation">
+            (
+        <inline classes="literal number integer">
+            8
+        <inline classes="operator">
+            /
+        <inline classes="literal number integer">
+            2
+        <inline classes="punctuation">
+            )
+""" if PYGMENTS_2_14_PLUS else """\
 <document source="test data">
     <literal_block classes="code python3" xml:space="preserve">
         <inline classes="ln">
diff --git a/docutils/test/test_parsers/test_rst/test_directives/test_include.py b/docutils/test/test_parsers/test_rst/test_directives/test_include.py
index d5db404c5..0667e0e55 100755
--- a/docutils/test/test_parsers/test_rst/test_directives/test_include.py
+++ b/docutils/test/test_parsers/test_rst/test_directives/test_include.py
@@ -23,6 +23,14 @@
 from docutils.utils import new_document
 from docutils.utils.code_analyzer import with_pygments
 
+try:
+    from pygments import __version__ as _pygments_ver
+except ImportError:
+    _pygments_ver = ''
+    PYGMENTS_2_14_PLUS = False
+else:
+    PYGMENTS_2_14_PLUS = tuple(map(int, _pygments_ver.split('.'))) >= (2, 14)
+
 TEST_ROOT = Path(__file__).resolve().parents[3]
 
 
@@ -1097,6 +1105,23 @@ def mydir(path):
    :code: rst
 """,
 f"""\
+<document source="test data">
+    <paragraph>
+        Included code
+    <literal_block classes="code rst" source="{include1}" xml:space="preserve">
+        <inline classes="generic heading">
+            Inclusion 1
+        \n\
+        <inline classes="generic heading">
+            -----------
+        \n\
+        <inline classes="whitespace">
+            \n\
+        This file is used by \n\
+        <inline classes="literal string">
+            ``test_include.py``
+        .
+""" if PYGMENTS_2_14_PLUS else f"""\
 <document source="test data">
     <paragraph>
         Included code
@@ -1121,6 +1146,32 @@ def mydir(path):
    :number-lines:
 """,
 f"""\
+<document source="test data">
+    <paragraph>
+        Included code
+    <literal_block classes="code rst" source="{include1}" xml:space="preserve">
+        <inline classes="ln">
+            1 \n\
+        <inline classes="generic heading">
+            Inclusion 1
+        \n\
+        <inline classes="ln">
+            2 \n\
+        <inline classes="generic heading">
+            -----------
+        \n\
+        <inline classes="ln">
+            3 \n\
+        <inline classes="whitespace">
+            \n\
+        <inline classes="ln">
+            4 \n\
+        <inline classes="whitespace">
+        This file is used by \n\
+        <inline classes="literal string">
+            ``test_include.py``
+        .
+""" if PYGMENTS_2_14_PLUS else f"""\
 <document source="test data">
     <paragraph>
         Included code
@@ -1152,6 +1203,34 @@ def mydir(path):
    :code: rst
 """,
 f"""\
+<document source="test data">
+    <paragraph>
+        TAB expansion with included code:
+    <literal_block classes="code rst" source="{include_literal}" xml:space="preserve">
+        Literal included this should \n\
+        <inline classes="generic strong">
+            **not**
+         be \n\
+        <inline classes="generic emph">
+            *marked*
+         \n\
+        <inline classes="name variable">
+            `up`
+        .
+        <inline classes="whitespace">
+            \n\
+                <- leading raw tab.
+        <inline classes="whitespace">
+            \n\
+            \n\
+        Newlines
+        <inline classes="whitespace">
+            \n\
+        are
+        <inline classes="whitespace">
+            \n\
+        normalized.
+""" if PYGMENTS_2_14_PLUS else f"""\
 <document source="test data">
     <paragraph>
         TAB expansion with included code:
@@ -1180,6 +1259,34 @@ def mydir(path):
    :tab-width: 2
 """,
 f"""\
+<document source="test data">
+    <paragraph>
+        Custom TAB expansion with included code:
+    <literal_block classes="code rst" source="{include_literal}" xml:space="preserve">
+        Literal included this should \n\
+        <inline classes="generic strong">
+            **not**
+         be \n\
+        <inline classes="generic emph">
+            *marked*
+         \n\
+        <inline classes="name variable">
+            `up`
+        .
+        <inline classes="whitespace">
+            \n\
+          <- leading raw tab.
+        <inline classes="whitespace">
+            \n\
+            \n\
+        Newlines
+        <inline classes="whitespace">
+            \n\
+        are
+        <inline classes="whitespace">
+            \n\
+        normalized.
+""" if PYGMENTS_2_14_PLUS else f"""\
 <document source="test data">
     <paragraph>
         Custom TAB expansion with included code:
@@ -1208,6 +1315,34 @@ def mydir(path):
    :tab-width: -1
 """,
 f"""\
+<document source="test data">
+    <paragraph>
+        Custom TAB expansion with included code:
+    <literal_block classes="code rst" source="{include_literal}" xml:space="preserve">
+        Literal included this should \n\
+        <inline classes="generic strong">
+            **not**
+         be \n\
+        <inline classes="generic emph">
+            *marked*
+         \n\
+        <inline classes="name variable">
+            `up`
+        .
+        <inline classes="whitespace">
+            \n\
+        \t<- leading raw tab.
+        <inline classes="whitespace">
+            \n\
+            \n\
+        Newlines
+        <inline classes="whitespace">
+            \n\
+        are
+        <inline classes="whitespace">
+            \n\
+        normalized.
+""" if PYGMENTS_2_14_PLUS else f"""\
 <document source="test data">
     <paragraph>
         Custom TAB expansion with included code:
@@ -1234,6 +1369,25 @@ def mydir(path):
 .. include:: {include14}
 """,
 f"""\
+<document source="test data">
+    <paragraph>
+        Including includes/include14.txt
+    <paragraph>
+        Including more/include6.txt as rst-code from includes/include14.txt:
+    <literal_block classes="code rst" source="{include6}" xml:space="preserve">
+        In includes/more/include6.txt
+        <inline classes="whitespace">
+            \n\
+            \n\
+        <inline classes="punctuation">
+            ..
+         \n\
+        <inline classes="operator word">
+            include
+        <inline classes="punctuation">
+            ::
+         ../sibling/include7.txt
+""" if PYGMENTS_2_14_PLUS else f"""\
 <document source="test data">
     <paragraph>
         Including includes/include14.txt
-- 
2.40.0.windows.1

From 8ad1babfdab6b3ba715a1aa8712e2044740fd2ba Mon Sep 17 00:00:00 2001
From: Adam Turner <[email protected]>
Date: Thu, 6 Apr 2023 17:24:40 +0100
Subject: [PATCH] Support Pygments 2.14 and newer in tests

---
 .../test_rst/test_directives/test_code.py     |  62 +++++++
 .../test_directives/test_code_long.py         |  62 +++++++
 .../test_rst/test_directives/test_include.py  | 154 ++++++++++++++++++
 3 files changed, 278 insertions(+)

diff --git a/docutils/test/test_parsers/test_rst/test_directives/test_code.py b/docutils/test/test_parsers/test_rst/test_directives/test_code.py
index 81efaaefa..7985d1b65 100755
--- a/docutils/test/test_parsers/test_rst/test_directives/test_code.py
+++ b/docutils/test/test_parsers/test_rst/test_directives/test_code.py
@@ -22,6 +22,14 @@
 from docutils.utils import new_document
 from docutils.utils.code_analyzer import with_pygments
 
+try:
+    from pygments import __version__ as _pygments_ver
+except ImportError:
+    _pygments_ver = ''
+    PYGMENTS_2_14_PLUS = False
+else:
+    PYGMENTS_2_14_PLUS = tuple(map(int, _pygments_ver.split('.'))) >= (2, 14)
+
 
 class ParserTestCase(unittest.TestCase):
     def test_parser(self):
@@ -153,6 +161,60 @@ def my_function():
       print(8/2)
 """,
 """\
+<document source="test data">
+    <literal_block classes="code python3 testclass" ids="my-function" names="my_function" xml:space="preserve">
+        <inline classes="ln">
+             7 \n\
+        <inline classes="keyword">
+            def
+         \n\
+        <inline classes="name function">
+            my_function
+        <inline classes="punctuation">
+            ():
+        <inline classes="whitespace">
+            \n\
+        <inline classes="ln">
+             8 \n\
+        <inline classes="whitespace">
+                \n\
+        <inline classes="literal string doc">
+            \'\'\'Test the lexer.
+        <inline classes="ln">
+             9 \n\
+        <inline classes="literal string doc">
+                \'\'\'
+        <inline classes="whitespace">
+            \n\
+        <inline classes="ln">
+            10 \n\
+        <inline classes="whitespace">
+            \n\
+        <inline classes="ln">
+            11 \n\
+        <inline classes="whitespace">
+            \n\
+        <inline classes="comment single">
+            # and now for something completely different
+        <inline classes="whitespace">
+            \n\
+        <inline classes="ln">
+            12 \n\
+        <inline classes="whitespace">
+            \n\
+        <inline classes="name builtin">
+            print
+        <inline classes="punctuation">
+            (
+        <inline classes="literal number integer">
+            8
+        <inline classes="operator">
+            /
+        <inline classes="literal number integer">
+            2
+        <inline classes="punctuation">
+            )
+""" if PYGMENTS_2_14_PLUS else """\
 <document source="test data">
     <literal_block classes="code python3 testclass" ids="my-function" names="my_function" xml:space="preserve">
         <inline classes="ln">
diff --git a/docutils/test/test_parsers/test_rst/test_directives/test_code_long.py b/docutils/test/test_parsers/test_rst/test_directives/test_code_long.py
index 07f1b9f36..641a5f1d9 100755
--- a/docutils/test/test_parsers/test_rst/test_directives/test_code_long.py
+++ b/docutils/test/test_parsers/test_rst/test_directives/test_code_long.py
@@ -22,6 +22,14 @@
 from docutils.utils import new_document
 from docutils.utils.code_analyzer import with_pygments
 
+try:
+    from pygments import __version__ as _pygments_ver
+except ImportError:
+    _pygments_ver = ''
+    PYGMENTS_2_14_PLUS = False
+else:
+    PYGMENTS_2_14_PLUS = tuple(map(int, _pygments_ver.split('.'))) >= (2, 14)
+
 
 @unittest.skipUnless(with_pygments, 'needs Pygments')
 class ParserTestCase(unittest.TestCase):
@@ -54,6 +62,60 @@ def my_function():
       print(8/2)
 """,
 """\
+<document source="test data">
+    <literal_block classes="code python3" xml:space="preserve">
+        <inline classes="ln">
+             7 \n\
+        <inline classes="keyword">
+            def
+         \n\
+        <inline classes="name function">
+            my_function
+        <inline classes="punctuation">
+            ():
+        <inline classes="whitespace">
+            \n\
+        <inline classes="ln">
+             8 \n\
+        <inline classes="whitespace">
+                \n\
+        <inline classes="literal string doc">
+            \'\'\'Test the lexer.
+        <inline classes="ln">
+             9 \n\
+        <inline classes="literal string doc">
+                \'\'\'
+        <inline classes="whitespace">
+            \n\
+        <inline classes="ln">
+            10 \n\
+        <inline classes="whitespace">
+            \n\
+        <inline classes="ln">
+            11 \n\
+        <inline classes="whitespace">
+            \n\
+        <inline classes="comment single">
+            # and now for something completely different
+        <inline classes="whitespace">
+            \n\
+        <inline classes="ln">
+            12 \n\
+        <inline classes="whitespace">
+            \n\
+        <inline classes="name builtin">
+            print
+        <inline classes="punctuation">
+            (
+        <inline classes="literal number integer">
+            8
+        <inline classes="operator">
+            /
+        <inline classes="literal number integer">
+            2
+        <inline classes="punctuation">
+            )
+""" if PYGMENTS_2_14_PLUS else """\
 <document source="test data">
     <literal_block classes="code python3" xml:space="preserve">
         <inline classes="ln">
diff --git a/docutils/test/test_parsers/test_rst/test_directives/test_include.py b/docutils/test/test_parsers/test_rst/test_directives/test_include.py
index d5db404c5..0667e0e55 100755
--- a/docutils/test/test_parsers/test_rst/test_directives/test_include.py
+++ b/docutils/test/test_parsers/test_rst/test_directives/test_include.py
@@ -23,6 +23,14 @@
 from docutils.utils import new_document
 from docutils.utils.code_analyzer import with_pygments
 
+try:
+    from pygments import __version__ as _pygments_ver
+except ImportError:
+    _pygments_ver = ''
+    PYGMENTS_2_14_PLUS = False
+else:
+    PYGMENTS_2_14_PLUS = tuple(map(int, _pygments_ver.split('.'))) >= (2, 14)
+
 TEST_ROOT = Path(__file__).resolve().parents[3]
 
 
@@ -1097,6 +1105,23 @@ def mydir(path):
    :code: rst
 """,
 f"""\
+<document source="test data">
+    <paragraph>
+        Included code
+    <literal_block classes="code rst" source="{include1}" xml:space="preserve">
+        <inline classes="generic heading">
+            Inclusion 1
+        \n\
+        <inline classes="generic heading">
+            -----------
+        \n\
+        <inline classes="whitespace">
+            \n\
+        This file is used by \n\
+        <inline classes="literal string">
+            ``test_include.py``
+        .
+""" if PYGMENTS_2_14_PLUS else f"""\
 <document source="test data">
     <paragraph>
         Included code
@@ -1121,6 +1146,32 @@ def mydir(path):
    :number-lines:
 """,
 f"""\
+<document source="test data">
+    <paragraph>
+        Included code
+    <literal_block classes="code rst" source="{include1}" xml:space="preserve">
+        <inline classes="ln">
+            1 \n\
+        <inline classes="generic heading">
+            Inclusion 1
+        \n\
+        <inline classes="ln">
+            2 \n\
+        <inline classes="generic heading">
+            -----------
+        \n\
+        <inline classes="ln">
+            3 \n\
+        <inline classes="whitespace">
+            \n\
+        <inline classes="ln">
+            4 \n\
+        <inline classes="whitespace">
+        This file is used by \n\
+        <inline classes="literal string">
+            ``test_include.py``
+        .
+""" if PYGMENTS_2_14_PLUS else f"""\
 <document source="test data">
     <paragraph>
         Included code
@@ -1152,6 +1203,34 @@ def mydir(path):
    :code: rst
 """,
 f"""\
+<document source="test data">
+    <paragraph>
+        TAB expansion with included code:
+    <literal_block classes="code rst" source="{include_literal}" xml:space="preserve">
+        Literal included this should \n\
+        <inline classes="generic strong">
+            **not**
+         be \n\
+        <inline classes="generic emph">
+            *marked*
+         \n\
+        <inline classes="name variable">
+            `up`
+        .
+        <inline classes="whitespace">
+            \n\
+                <- leading raw tab.
+        <inline classes="whitespace">
+            \n\
+            \n\
+        Newlines
+        <inline classes="whitespace">
+            \n\
+        are
+        <inline classes="whitespace">
+            \n\
+        normalized.
+""" if PYGMENTS_2_14_PLUS else f"""\
 <document source="test data">
     <paragraph>
         TAB expansion with included code:
@@ -1180,6 +1259,34 @@ def mydir(path):
    :tab-width: 2
 """,
 f"""\
+<document source="test data">
+    <paragraph>
+        Custom TAB expansion with included code:
+    <literal_block classes="code rst" source="{include_literal}" xml:space="preserve">
+        Literal included this should \n\
+        <inline classes="generic strong">
+            **not**
+         be \n\
+        <inline classes="generic emph">
+            *marked*
+         \n\
+        <inline classes="name variable">
+            `up`
+        .
+        <inline classes="whitespace">
+            \n\
+          <- leading raw tab.
+        <inline classes="whitespace">
+            \n\
+            \n\
+        Newlines
+        <inline classes="whitespace">
+            \n\
+        are
+        <inline classes="whitespace">
+            \n\
+        normalized.
+""" if PYGMENTS_2_14_PLUS else f"""\
 <document source="test data">
     <paragraph>
         Custom TAB expansion with included code:
@@ -1208,6 +1315,34 @@ def mydir(path):
    :tab-width: -1
 """,
 f"""\
+<document source="test data">
+    <paragraph>
+        Custom TAB expansion with included code:
+    <literal_block classes="code rst" source="{include_literal}" xml:space="preserve">
+        Literal included this should \n\
+        <inline classes="generic strong">
+            **not**
+         be \n\
+        <inline classes="generic emph">
+            *marked*
+         \n\
+        <inline classes="name variable">
+            `up`
+        .
+        <inline classes="whitespace">
+            \n\
+        \t<- leading raw tab.
+        <inline classes="whitespace">
+            \n\
+            \n\
+        Newlines
+        <inline classes="whitespace">
+            \n\
+        are
+        <inline classes="whitespace">
+            \n\
+        normalized.
+""" if PYGMENTS_2_14_PLUS else f"""\
 <document source="test data">
     <paragraph>
         Custom TAB expansion with included code:
@@ -1234,6 +1369,25 @@ def mydir(path):
 .. include:: {include14}
 """,
 f"""\
+<document source="test data">
+    <paragraph>
+        Including includes/include14.txt
+    <paragraph>
+        Including more/include6.txt as rst-code from includes/include14.txt:
+    <literal_block classes="code rst" source="{include6}" xml:space="preserve">
+        In includes/more/include6.txt
+        <inline classes="whitespace">
+            \n\
+            \n\
+        <inline classes="punctuation">
+            ..
+         \n\
+        <inline classes="operator word">
+            include
+        <inline classes="punctuation">
+            ::
+         ../sibling/include7.txt
+""" if PYGMENTS_2_14_PLUS else f"""\
 <document source="test data">
     <paragraph>
         Including includes/include14.txt
-- 
2.40.0.windows.1