Re: I/O uses default encoding argument

From 23b0d3593e4d9dd07ee4b69f3f5f8cc60de5617d Mon Sep 17 00:00:00 2001
From: milde <[email protected]>
Date: Tue, 21 Jun 2022 11:07:02 +0200
Subject: [PATCH 1/4] Cosmetics.

Fix/expand help string.
Disambiguate name "encodings".
Use f-string instead of %-replacements.
---
 docutils/docutils/io.py | 29 ++++++++++++++++++-----------
 1 file changed, 18 insertions(+), 11 deletions(-)

diff --git a/docutils/docutils/io.py b/docutils/docutils/io.py
index 932ef2296..f5013d125 100644
--- a/docutils/docutils/io.py
+++ b/docutils/docutils/io.py
@@ -110,38 +110,45 @@ class Input(TransformSpec):
 
     def decode(self, data):
         """
-        Decode a string, `data`, heuristically.
-        Raise UnicodeError if unsuccessful.
+        Decode `data` if required.
+
+        Return Unicode `str` instances unchanged (nothing to decode).
 
+        If `self.encoding` is None, determine encoding from data
+        or try UTF-8, locale encoding, and (as last ressort) 'latin-1'.
         The client application should call ``locale.setlocale`` at the
         beginning of processing::
 
             locale.setlocale(locale.LC_ALL, '')
+
+        Raise UnicodeError if unsuccessful.
         """
         if self.encoding and self.encoding.lower() == 'unicode':
             assert isinstance(data, str), ('input encoding is "unicode" '
                                            'but input is not a `str` object')
         if isinstance(data, str):
-            # Accept unicode string even if self.encoding != 'unicode'.
+            # nothing to decode
             return data
         if self.encoding:
             # We believe the user/application when the encoding is
             # explicitly given.
-            encodings = [self.encoding]
+            encoding_candidates = [self.encoding]
         else:
             data_encoding = self.determine_encoding_from_data(data)
             if data_encoding:
                 # If the data declares its encoding (explicitly or via a BOM),
                 # we believe it.
-                encodings = [data_encoding]
+                encoding_candidates = [data_encoding]
             else:
                 # Apply heuristics only if no encoding is explicitly given and
                 # no BOM found.  Start with UTF-8, because that only matches
                 # data that *IS* UTF-8:
-                encodings = ['utf-8', 'latin-1']
-                if _locale_encoding:
-                    encodings.insert(1, _locale_encoding)
-        for enc in encodings:
+                encoding_candidates = ['utf-8']
+                if _locale_encoding and _locale_encoding != 'utf-8':
+                    encoding_candidates.append(_locale_encoding)
+                # TODO: fall back to 'latin-1' or report error? (API change)
+                encoding_candidates.append('latin-1')
+        for enc in encoding_candidates:
             try:
                 decoded = str(data, enc, self.error_handler)
                 self.successful_encoding = enc
@@ -152,8 +159,8 @@ class Input(TransformSpec):
                 error = err
         raise UnicodeError(
             'Unable to decode input data.  Tried the following encodings: '
-            '%s.\n(%s)' % (', '.join(repr(enc) for enc in encodings),
-                           error_string(error)))
+            f'{", ".join(repr(enc) for enc in encoding_candidates)}.\n'
+            f'({error_string(error)})')
 
     coding_slug = re.compile(br"coding[:=]\s*([-\w.]+)")
     """Encoding declaration pattern."""
-- 
2.30.2

From 22da7ea2cd2fc7f65ce2cb31ee7c200e4f5040d2 Mon Sep 17 00:00:00 2001
From: milde <[email protected]>
Date: Tue, 21 Jun 2022 12:52:31 +0200
Subject: [PATCH 2/4] Test input encoding auto-detection.

Auto-detection should always kick in if io.FileInput.encoding is None.
---
 docutils/test/test_io.py | 42 ++++++++++++++++++++++++++++++++++++++++
 1 file changed, 42 insertions(+)

diff --git a/docutils/test/test_io.py b/docutils/test/test_io.py
index 8ccf567f0..1a3a7ac87 100755
--- a/docutils/test/test_io.py
+++ b/docutils/test/test_io.py
@@ -256,5 +256,47 @@ class ErrorOutputTests(unittest.TestCase):
         self.assertEqual(buf.getvalue(), 'b\ufffd u\xfc e\xfc b\xfc')
 
 
+class FileInputTests(unittest.TestCase):
+
+    def test_readlines(self):
+        source = io.FileInput(source_path='data/include.txt')
+        data = source.readlines()
+        self.assertEqual(data, ['Some include text.\n'])
+
+    # test input encoding auto-detection:
+
+    def test_bom(self):
+        source = io.FileInput(
+            source_path='test_parsers/test_rst/test_directives/utf-16.csv')
+        # Assert correct decoding, BOMs are gone.
+        self.assertTrue(source.read().startswith('"Treat", "Quantity"'))
+        self.assertEqual(source.successful_encoding, 'utf-16-be')
+
+    def test_coding_slug(self):
+        source = io.FileInput(source_path='data/latin2.txt')
+        # print(source.read())
+        self.assertTrue(source.read().endswith('škoda\n'))
+        self.assertEqual(source.successful_encoding, 'latin2')
+
+    def test_fallback_utf8(self):
+        # if encoding is not given to FileInput nor specified in the source
+        # try decoding with 'utf-8'
+        source = io.FileInput(source_path='data/include.txt')
+        self.assertEqual(source.read(), 'Some include text.\n')
+        self.assertEqual(source.successful_encoding, 'utf-8')
+
+    def test_fallback_no_utf8(self):
+        # if decoding with 'utf-8' fails,
+        # use either the locale encoding (if not None) or 'latin-1'
+        # TODO: don't fall back to latin1?
+        probed_encodings = (io._locale_encoding, 'latin-1')  # noqa
+
+        source = io.FileInput(source_path='data/latin1.txt')
+        data = source.read()
+        self.assertTrue(source.successful_encoding in probed_encodings)
+        if source.successful_encoding == 'latin-1':
+            self.assertEqual(data, 'Gr\xfc\xdfe\n')
+
+
 if __name__ == '__main__':
     unittest.main()
-- 
2.30.2

From 1851f6abccef1a1419a2a6057b7dfc6280382c3d Mon Sep 17 00:00:00 2001
From: milde <[email protected]>
Date: Tue, 21 Jun 2022 12:22:04 +0200
Subject: [PATCH 3/4] Always use heuristics, if input encoding is not
 specified.

Don't rely on Python's default encoding when opening the input stream.

This restores the behaviour under Python2.
---
 docutils/docutils/io.py  | 34 +++++++++++-----------------------
 docutils/test/test_io.py | 22 ----------------------
 2 files changed, 11 insertions(+), 45 deletions(-)

diff --git a/docutils/docutils/io.py b/docutils/docutils/io.py
index f5013d125..a890fd4a5 100644
--- a/docutils/docutils/io.py
+++ b/docutils/docutils/io.py
@@ -342,7 +342,7 @@ class FileInput(Input):
         :Parameters:
             - `source`: either a file-like object (which is read directly), or
               `None` (which implies `sys.stdin` if no `source_path` given).
-            - `source_path`: a path to a file, which is opened and then read.
+            - `source_path`: a path to a file, which is opened for reading.
             - `encoding`: the expected text encoding of the input file.
             - `error_handler`: the encoding error handler to use.
             - `autoclose`: close automatically after read (except when
@@ -379,28 +379,16 @@ class FileInput(Input):
         """
         Read and decode a single file and return the data (Unicode string).
         """
-        try:
-            if self.source is sys.stdin:
-                # read as binary data to circumvent auto-decoding
-                data = self.source.buffer.read()
-                # normalize newlines
-                data = b'\n'.join(data.splitlines()+[b''])
-            else:
-                data = self.source.read()
-        except (UnicodeError, LookupError):
-            if not self.encoding and self.source_path:
-                # re-read in binary mode and decode with heuristics
-                b_source = open(self.source_path, 'rb')
-                data = b_source.read()
-                b_source.close()
-                # normalize newlines
-                data = b'\n'.join(data.splitlines()+[b''])
-            else:
-                raise
-        finally:
-            if self.autoclose:
-                self.close()
-        return self.decode(data)
+        if self.encoding is None:
+            # read as binary data to circumvent auto-decoding
+            data = self.source.buffer.read()
+            # normalize newlines and decode with heuristics
+            data = self.decode(b'\n'.join(data.splitlines()+[b'']))
+        else:
+            data = self.source.read()
+        if self.autoclose:
+            self.close()
+        return data
 
     def readlines(self):
         """
diff --git a/docutils/test/test_io.py b/docutils/test/test_io.py
index 1a3a7ac87..f9e2f483e 100755
--- a/docutils/test/test_io.py
+++ b/docutils/test/test_io.py
@@ -124,28 +124,6 @@ print("hello world")
         data = input.read()  # noqa: F841
         self.assertEqual(input.successful_encoding, 'utf-8')
 
-    def test_readlines(self):
-        input = io.FileInput(source_path='data/include.txt')
-        data = input.readlines()
-        self.assertEqual(data, ['Some include text.\n'])
-
-    def test_heuristics_no_utf8(self):
-        # if no encoding is given and decoding with 'utf-8' fails,
-        # use either the locale encoding (if specified) or 'latin-1':
-        if io._locale_encoding not in ('utf-8', 'utf8'):  # noqa
-            # in Py3k, the locale encoding is used without --input-encoding
-            # skipping the heuristic unless decoding fails.
-            return
-        probed_encodings = (io._locale_encoding, 'latin-1')  # noqa
-        input = io.FileInput(source_path='data/latin1.txt')
-        data = input.read()
-        if input.successful_encoding not in probed_encodings:
-            raise AssertionError(
-                "guessed encoding '%s' differs from probed encodings %r"
-                % (input.successful_encoding, probed_encodings))
-        if input.successful_encoding == 'latin-1':
-            self.assertEqual(data, 'Gr\xfc\xdfe\n')
-
     def test_decode_unicode(self):
         # With the special value "unicode" or "Unicode":
         uniinput = io.Input(encoding='unicode')
-- 
2.30.2

From 536f9189627050c453ef311d07c090f234b009b5 Mon Sep 17 00:00:00 2001
From: milde <[email protected]>
Date: Tue, 21 Jun 2022 13:33:24 +0200
Subject: [PATCH 4/4] Variant: open `source_path` in binary mode if encoding is
 not specified.

Avoids warnings for unspecified encoding.
---
 docutils/docutils/io.py | 20 +++++++++++---------
 1 file changed, 11 insertions(+), 9 deletions(-)

diff --git a/docutils/docutils/io.py b/docutils/docutils/io.py
index a890fd4a5..7de31b143 100644
--- a/docutils/docutils/io.py
+++ b/docutils/docutils/io.py
@@ -357,9 +357,13 @@ class FileInput(Input):
         if source is None:
             if source_path:
                 try:
-                    self.source = open(source_path, mode,
-                                       encoding=self.encoding,
-                                       errors=self.error_handler)
+                    if self.encoding is None and mode == 'r':
+                        # read as binary data to allow encoding detection
+                        self.source = open(source_path, 'rb')
+                    else:
+                        self.source = open(source_path, mode,
+                                           encoding=self.encoding,
+                                           errors=self.error_handler)
                 except OSError as error:
                     raise InputError(error.errno, error.strerror, source_path)
             else:
@@ -377,15 +381,13 @@ class FileInput(Input):
 
     def read(self):
         """
-        Read and decode a single file and return the data (Unicode string).
+        Read a single file and return the data as Unicode string.
         """
-        if self.encoding is None:
-            # read as binary data to circumvent auto-decoding
-            data = self.source.buffer.read()
+        data = self.source.read()
+
+        if isinstance(data, bytes):
             # normalize newlines and decode with heuristics
             data = self.decode(b'\n'.join(data.splitlines()+[b'']))
-        else:
-            data = self.source.read()
         if self.autoclose:
             self.close()
         return data
-- 
2.30.2