SF.net SVN: docutils:[9994] trunk/docutils
milde--- via Docutils-checkins <[email protected]>
| Newsgroups | gmane.text.docutils.cvs |
|---|---|
| Message-ID | <[email protected]> |
Revision: 9994
http://sourceforge.net/p/docutils/code/9994
Author: milde
Date: 2024-12-04 13:32:39 +0000 (Wed, 04 Dec 2024)
Log Message:
-----------
Update "auto" input encoding warning.
Use FutureWarning instead of DeprecationWarning in front end tools if the
"input_encoding" setting uses the special values "" or None (auto-detect).
Add DeprecationWarning in `docutils.io.Input.decode()`.
Adapt tests.
Modified Paths:
--------------
trunk/docutils/docutils/frontend.py
trunk/docutils/docutils/io.py
trunk/docutils/test/test_io.py
trunk/docutils/test/test_publisher.py
Modified: trunk/docutils/docutils/frontend.py
===================================================================
--- trunk/docutils/docutils/frontend.py 2024-12-04 13:32:20 UTC (rev 9993)
+++ trunk/docutils/docutils/frontend.py 2024-12-04 13:32:39 UTC (rev 9994)
@@ -135,8 +135,9 @@
if value is None:
value = setting
if value == '':
- warnings.warn('Input encoding detection will be removed '
- 'in Docutils 1.0.', DeprecationWarning, stacklevel=2)
+ warnings.warn('Input encoding detection will be removed and the '
+ 'special encoding values None and "" become invalid '
+ 'in Docutils 1.0.', FutureWarning, stacklevel=2)
return None
try:
codecs.lookup(value)
Modified: trunk/docutils/docutils/io.py
===================================================================
--- trunk/docutils/docutils/io.py 2024-12-04 13:32:20 UTC (rev 9993)
+++ trunk/docutils/docutils/io.py 2024-12-04 13:32:39 UTC (rev 9994)
@@ -152,7 +152,9 @@
# explicitly given.
encoding_candidates = [self.encoding]
else:
- data_encoding = self.determine_encoding_from_data(data)
+ with warnings.catch_warnings():
+ warnings.filterwarnings('ignore', category=DeprecationWarning)
+ data_encoding = self.determine_encoding_from_data(data)
if data_encoding:
# `data` declares its encoding with "magic comment" or BOM,
encoding_candidates = [data_encoding]
@@ -168,6 +170,10 @@
fallback = locale.getpreferredencoding(do_setlocale=False)
if fallback and fallback.lower() != 'utf-8':
encoding_candidates.append(fallback)
+ if not self.encoding and encoding_candidates[0] != 'utf-8':
+ warnings.warn('Input encoding auto-detection will be removed and '
+ 'the encoding values None and "" become invalid '
+ 'in Docutils 1.0.', DeprecationWarning, stacklevel=2)
for enc in encoding_candidates:
try:
decoded = str(data, enc, self.error_handler)
@@ -195,13 +201,21 @@
)
"""Sequence of (start_bytes, encoding) tuples for encoding detection.
The first bytes of input data are checked against the start_bytes strings.
- A match indicates the given encoding."""
+ A match indicates the given encoding.
+ Internal. Will be removed in Docutils 1.0.
+ """
+
def determine_encoding_from_data(self, data: bytes) -> str | None:
"""
Try to determine the encoding of `data` by looking *in* `data`.
Check for a byte order mark (BOM) or an encoding declaration.
+
+ Deprecated. Will be removed in Docutils 1.0.
"""
+ warnings.warn('docutils.io.Input.determine_encoding_from_data() '
+ 'will be removed in Docutils 1.0.',
+ DeprecationWarning, stacklevel=2)
# check for a byte order mark:
for start_bytes, encoding in self.byte_order_marks:
if data.startswith(start_bytes):
Modified: trunk/docutils/test/test_io.py
===================================================================
--- trunk/docutils/test/test_io.py 2024-12-04 13:32:20 UTC (rev 9993)
+++ trunk/docutils/test/test_io.py 2024-12-04 13:32:39 UTC (rev 9994)
@@ -10,7 +10,6 @@
import codecs
import locale
-import os.path
import sys
import unittest
import warnings
@@ -34,7 +33,7 @@
EncodingWarning = UnicodeWarning # NoQA: A001 (builtin in Py > 0.9)
# DATA_ROOT is ./test/data/ from the docutils root
-DATA_ROOT = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'data')
+DATA_ROOT = Path(__file__).parent / 'data'
# normalize the preferred encoding's name:
with warnings.catch_warnings():
@@ -118,13 +117,16 @@
expected = 'data\n\ufeff blah\n' # only leading ZWNBSP removed
input_ = du_io.StringInput(source=source.encode('utf-16-be'),
encoding=None)
- self.assertEqual(expected, input_.read())
+ with self.assertWarnsRegex(DeprecationWarning, 'auto-detection'):
+ self.assertEqual(expected, input_.read())
input_ = du_io.StringInput(source=source.encode('utf-16-le'),
encoding=None)
- self.assertEqual(expected, input_.read())
+ with self.assertWarnsRegex(DeprecationWarning, 'auto-detection'):
+ self.assertEqual(expected, input_.read())
input_ = du_io.StringInput(source=source.encode('utf-8'),
encoding=None)
- self.assertEqual(expected, input_.read())
+ with self.assertWarnsRegex(DeprecationWarning, 'auto-detection'):
+ self.assertEqual(expected, input_.read())
# With `str` input all ZWNBSPs are still there.
input_ = du_io.StringInput(source=source)
self.assertEqual(source, input_.read())
@@ -135,7 +137,8 @@
data
blah
""", encoding=None)
- data = input_.read() # noqa: F841
+ with self.assertWarnsRegex(DeprecationWarning, 'auto-detection'):
+ data = input_.read() # noqa: F841
self.assertEqual('ascii', input_.successful_encoding)
input_ = du_io.StringInput(source=b"""\
#! python
@@ -142,7 +145,8 @@
# -*- coding: ascii -*-
print("hello world")
""", encoding=None)
- data = input_.read() # noqa: F841
+ with self.assertWarnsRegex(DeprecationWarning, 'auto-detection'):
+ data = input_.read() # noqa: F841
self.assertEqual('ascii', input_.successful_encoding)
input_ = du_io.StringInput(source=b"""\
#! python
@@ -276,10 +280,10 @@
with warnings.catch_warnings():
if SUPPRESS_ENCODING_WARNING:
warnings.filterwarnings('ignore', category=EncodingWarning)
- source = du_io.FileInput(
- source_path=os.path.join(DATA_ROOT, 'utf-8-sig.rst'),
- encoding=None)
- self.assertTrue(source.read().startswith('Grüße'))
+ source = du_io.FileInput(source_path=DATA_ROOT/'utf-8-sig.rst',
+ encoding=None)
+ with self.assertWarnsRegex(DeprecationWarning, 'auto-detection'):
+ self.assertTrue(source.read().startswith('Grüße'))
def test_bom_utf_16(self):
"""Drop BOM from utf-16 encoded files, use correct encoding.
@@ -288,10 +292,10 @@
with warnings.catch_warnings():
if SUPPRESS_ENCODING_WARNING:
warnings.filterwarnings('ignore', category=EncodingWarning)
- source = du_io.FileInput(
- source_path=os.path.join(DATA_ROOT, 'utf-16-le-sig.rst'),
- encoding=None)
- self.assertTrue(source.read().startswith('Grüße'))
+ source = du_io.FileInput(source_path=DATA_ROOT/'utf-16-le-sig.rst',
+ encoding=None)
+ with self.assertWarnsRegex(DeprecationWarning, 'auto-detection'):
+ self.assertTrue(source.read().startswith('Grüße'))
def test_coding_slug(self):
"""Use self-declared encoding.
@@ -299,10 +303,10 @@
with warnings.catch_warnings():
if SUPPRESS_ENCODING_WARNING:
warnings.filterwarnings('ignore', category=EncodingWarning)
- source = du_io.FileInput(
- source_path=os.path.join(DATA_ROOT, 'latin2.rst'),
- encoding=None)
- self.assertTrue(source.read().endswith('škoda\n'))
+ source = du_io.FileInput(source_path=DATA_ROOT/'latin2.rst',
+ encoding=None)
+ with self.assertWarnsRegex(DeprecationWarning, 'auto-detection'):
+ self.assertTrue(source.read().endswith('škoda\n'))
def test_fallback_utf8(self):
"""Try 'utf-8', if encoding is not specified in the source."""
@@ -309,14 +313,12 @@
with warnings.catch_warnings():
if SUPPRESS_ENCODING_WARNING:
warnings.filterwarnings('ignore', category=EncodingWarning)
- source = du_io.FileInput(
- source_path=os.path.join(DATA_ROOT, 'utf8.rst'),
- encoding=None)
+ source = du_io.FileInput(source_path=DATA_ROOT/'utf8.rst',
+ encoding=None)
self.assertEqual('Grüße\n', source.read())
def test_readlines(self):
- source = du_io.FileInput(
- source_path=os.path.join(DATA_ROOT, 'include.rst'))
+ source = du_io.FileInput(source_path=DATA_ROOT/'include.rst')
data = source.readlines()
self.assertEqual(['Some include text.\n'], data)
Modified: trunk/docutils/test/test_publisher.py
===================================================================
--- trunk/docutils/test/test_publisher.py 2024-12-04 13:32:20 UTC (rev 9993)
+++ trunk/docutils/test/test_publisher.py 2024-12-04 13:32:39 UTC (rev 9994)
@@ -207,8 +207,9 @@
# input encoding detection will be removed in Docutils 1.0
source = '.. encoding: latin1\n\nGrüße'
settings['input_encoding'] = None
- output = core.publish_string(source.encode('latin1'),
- settings_overrides=settings)
+ with self.assertWarnsRegex(DeprecationWarning, 'auto-detection'):
+ output = core.publish_string(source.encode('latin1'),
+ settings_overrides=settings)
self.assertTrue(output.endswith('Grüße\n'))
def test_publish_string_output_encoding(self):
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
_______________________________________________
Docutils-checkins mailing list
[email protected]
https://lists.sourceforge.net/lists/listinfo/docutils-checkins