SF.net SVN: docutils:[9772] trunk/docutils

Revision: 9772
          http://sourceforge.net/p/docutils/code/9772
Author:   milde
Date:     2024-06-29 21:23:01 +0000 (Sat, 29 Jun 2024)
Log Message:
-----------
Change the default input encoding from ``None`` to "utf-8" in io.py.

Fixup for [r9749]. See also [bugs:#490].

Change the default encoding from ``None`` (auto-detect) to "utf-8"
in `docutils.io.Input` and `docutils.io.FileInput`.

Modified Paths:
--------------
    trunk/docutils/HISTORY.txt
    trunk/docutils/docutils/io.py
    trunk/docutils/test/test_io.py

Modified: trunk/docutils/HISTORY.txt
===================================================================
--- trunk/docutils/HISTORY.txt	2024-06-18 08:00:10 UTC (rev 9771)
+++ trunk/docutils/HISTORY.txt	2024-06-29 21:23:01 UTC (rev 9772)
@@ -40,6 +40,10 @@
     and ``--output-encoding``.
   - Change the default input encoding from ``None`` (auto-detect) to "utf-8".
 
+* docutils/io.py
+
+  - Change the default input encoding from ``None`` (auto-detect) to "utf-8".
+
 * docutils/nodes.py
 
   - Raise TypeError if the "rawsource" argument in `Element.__init__()`

Modified: trunk/docutils/docutils/io.py
===================================================================
--- trunk/docutils/docutils/io.py	2024-06-18 08:00:10 UTC (rev 9771)
+++ trunk/docutils/docutils/io.py	2024-06-29 21:23:01 UTC (rev 9772)
@@ -82,7 +82,7 @@
 
     default_source_path = None
 
-    def __init__(self, source=None, source_path=None, encoding=None,
+    def __init__(self, source=None, source_path=None, encoding='utf-8',
                  error_handler='strict'):
         self.encoding = encoding
         """Text encoding for the input source."""
@@ -357,7 +357,7 @@
     Input for single, simple file-like objects.
     """
     def __init__(self, source=None, source_path=None,
-                 encoding=None, error_handler='strict',
+                 encoding='utf-8', error_handler='strict',
                  autoclose=True, mode='r'):
         """
         :Parameters:

Modified: trunk/docutils/test/test_io.py
===================================================================
--- trunk/docutils/test/test_io.py	2024-06-18 08:00:10 UTC (rev 9771)
+++ trunk/docutils/test/test_io.py	2024-06-29 21:23:01 UTC (rev 9772)
@@ -74,7 +74,8 @@
         """Cases where the comparison fails."""
         # stream.encoding is None:
         self.assertEqual(None,
-                         du_io.check_encoding(du_io.FileInput(), 'ascii'))
+                         du_io.check_encoding(du_io.FileInput(encoding=None),
+                                              'ascii'))
         # stream.encoding does not exist:
         self.assertEqual(None, du_io.check_encoding(BBuf, 'ascii'))
         # encoding is None or empty string:
@@ -102,11 +103,14 @@
         # default input encoding will change to UTF-8 in Docutils 0.22
         source = '\ufeffdata\n\ufeff blah\n'
         expected = 'data\n\ufeff blah\n'  # only leading ZWNBSP removed
-        input = du_io.StringInput(source=source.encode('utf-16-be'))
+        input = du_io.StringInput(source=source.encode('utf-16-be'),
+                                  encoding=None)
         self.assertEqual(expected, input.read())
-        input = du_io.StringInput(source=source.encode('utf-16-le'))
+        input = du_io.StringInput(source=source.encode('utf-16-le'),
+                                  encoding=None)
         self.assertEqual(expected, input.read())
-        input = du_io.StringInput(source=source.encode('utf-8'))
+        input = du_io.StringInput(source=source.encode('utf-8'),
+                                  encoding=None)
         self.assertEqual(expected, input.read())
         # With `str` input all ZWNBSPs are still there.
         input = du_io.StringInput(source=source)
@@ -117,7 +121,7 @@
 .. -*- coding: ascii -*-
 data
 blah
-""")
+""", encoding=None)
         data = input.read()  # noqa: F841
         self.assertEqual('ascii', input.successful_encoding)
         input = du_io.StringInput(source=b"""\
@@ -124,7 +128,7 @@
 #! python
 # -*- coding: ascii -*-
 print("hello world")
-""")
+""", encoding=None)
         data = input.read()  # noqa: F841
         self.assertEqual('ascii', input.successful_encoding)
         input = du_io.StringInput(source=b"""\
@@ -257,7 +261,8 @@
         """Drop optional BOM from utf-8 encoded files.
         """
         source = du_io.FileInput(
-            source_path=os.path.join(DATA_ROOT, 'utf-8-sig.txt'))
+            source_path=os.path.join(DATA_ROOT, 'utf-8-sig.txt'),
+            encoding=None)
         self.assertTrue(source.read().startswith('Grüße'))
 
     def test_bom_utf_16(self):
@@ -265,7 +270,8 @@
         """
         # Assert correct decoding, BOM is gone.
         source = du_io.FileInput(
-            source_path=os.path.join(DATA_ROOT, 'utf-16-le-sig.txt'))
+            source_path=os.path.join(DATA_ROOT, 'utf-16-le-sig.txt'),
+            encoding=None)
         self.assertTrue(source.read().startswith('Grüße'))
 
     def test_coding_slug(self):
@@ -272,7 +278,8 @@
         """Use self-declared encoding.
         """
         source = du_io.FileInput(
-            source_path=os.path.join(DATA_ROOT, 'latin2.txt'))
+            source_path=os.path.join(DATA_ROOT, 'latin2.txt'),
+            encoding=None)
         self.assertTrue(source.read().endswith('škoda\n'))
 
     def test_fallback_utf8(self):

This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.



_______________________________________________
Docutils-checkins mailing list
[email protected]
https://lists.sourceforge.net/lists/listinfo/docutils-checkins