Re: Release 0.20
Guenter Milde via Docutils-develop <[email protected]>
| Newsgroups | gmane.text.docutils.devel |
|---|---|
| Message-ID | <[email protected]> |
Dear Docutils developers,
an update and request for comments...
On 2023-01-10, Guenter Milde via Docutils-develop wrote:
...
> it is time for a new release.
...
> * Future of ``core.publish_string()`` API function:
> a) Keep current behaviour ::
> def publish_string(source: Union[bytes, str],
> [...]
> enable_exit_status=False) -> Union[bytes, str]
> as "wart", just improve documentation?
> b) Deprecate ``publish_string()`` and provide new ``publish_str()``
> and ``publish_bytes()`` functions?
> c) Return a sub-class of ``str`` with ``__bytes__()`` method that
> encodes with ``encoding`` and ``encoding_errors`` set to the
> "output_encoding" and "output_encoding_errors" setting values?
> - as "subtly" changed behaviour, or
> - with a new function replacing ``publish_string()``
> (find a good name!)?
> Proposal [GM]
> - explore c)
> - remove ``core.publish_bytes()`` before releasing 0.20.
I prepared a patch for option c) with a new function attribute
'auto_encode' for `core.publish_string()`. This would allow to
keep the name and switch to a behaviour matching it (returning a string, not
bytes) gradually (by switching the default value and eventually removing the
option later). See below.
> Are there other open issues that should be adressed before the next release?
* Implement the patch for a configurable include_root?
https://sourceforge.net/p/docutils/feature-requests/91/
Thanks,
Günter
Subject: [PATCH] Define and use new `str` sub-class for string output.
New class `io.OutString` adds "encoding" and "errors" attributes
to `str`.
Use it for `io.StringOutput`. Allows storing the "output_encoding"
and "output_encoding_error_handler" settings in a transparent
and easy to process way.
Add "auto_encode" argument to publish_string() and
publish_programatically() to give the user an option to select the
output type (`bytes` or `str`) in a way that does not interfere with the
intended encoding of the output.
---
docutils/docutils/core.py | 34 ++++++-------
docutils/docutils/io.py | 84 +++++++++++++++++++++++++++++++--
docutils/test/test_io.py | 59 +++++++++++++++++++++++
docutils/test/test_publisher.py | 21 ++++++++-
4 files changed, 177 insertions(+), 21 deletions(-)
diff --git a/docutils/docutils/core.py b/docutils/docutils/core.py
index 03c60279f..5d4793f5d 100644
--- a/docutils/docutils/core.py
+++ b/docutils/docutils/core.py
@@ -427,26 +427,21 @@ def publish_string(source, source_path=None, destination_path=None,
writer=None, writer_name='pseudoxml',
settings=None, settings_spec=None,
settings_overrides=None, config_section=None,
- enable_exit_status=False):
+ enable_exit_status=False,
+ auto_encode=True):
"""
Set up & run a `Publisher` for programmatic use with string I/O.
Accepts a `bytes` or `str` instance as `source`.
- The output is encoded according to the "output_encoding" setting;
- the return value is a `bytes` instance (unless `output_encoding`_
- is "unicode", see below).
- To get Docutils output as `str` instance, use `publish_parts()`::
+ If `auto_encode` is True, the output is encoded according to the
+ `output_encoding`_ setting; the return value is a `bytes` instance
+ (unless `output_encoding`_ is "unicode",
+ cf. `docutils.io.StringOutput.write()`).
- output = publish_parts(...)['whole']
-
- or set `output_encoding`_ to the pseudo encoding name "unicode", e.g.::
-
- publish_string(..., settings_overrides={'output_encoding': 'unicode'})
-
- Beware that the `output_encoding`_ setting may affect the content
- of the output (e.g. an encoding declaration in HTML or XML or the
- representation of characters as LaTeX macro vs. literal character).
+ If `auto_encode` is False, the output is an instance of a `str`
+ sub-class with "output_encoding" and "output_encoding_error_handler"
+ settings stored as `encoding` and `errors` attributes.
Parameters: see `publish_programmatically()`.
@@ -463,7 +458,8 @@ def publish_string(source, source_path=None, destination_path=None,
settings=settings, settings_spec=settings_spec,
settings_overrides=settings_overrides,
config_section=config_section,
- enable_exit_status=enable_exit_status)
+ enable_exit_status=enable_exit_status,
+ auto_encode=auto_encode)
return output
@@ -617,7 +613,8 @@ def publish_programmatically(source_class, source, source_path,
writer, writer_name,
settings, settings_spec,
settings_overrides, config_section,
- enable_exit_status):
+ enable_exit_status,
+ auto_encode=True):
"""
Set up & run a `Publisher` for custom programmatic use.
@@ -709,6 +706,9 @@ def publish_programmatically(source_class, source, source_path,
defined by `settings_spec`. Used only if no `settings` specified.
* `enable_exit_status`: Boolean; enable exit status at end of processing?
+
+ * `auto_encode`: Boolean; encode string output and return `bytes`?
+ Ignored with `io.FileOutput`.
"""
publisher = Publisher(reader, parser, writer, settings=settings,
source_class=source_class,
@@ -718,5 +718,7 @@ def publish_programmatically(source_class, source, source_path,
settings_spec, settings_overrides, config_section)
publisher.set_source(source, source_path)
publisher.set_destination(destination, destination_path)
+ if not auto_encode and isinstance(publisher.destination, io.StringOutput):
+ publisher.destination.auto_encode = auto_encode
output = publisher.publish(enable_exit_status=enable_exit_status)
return output, publisher
diff --git a/docutils/docutils/io.py b/docutils/docutils/io.py
index 2007a5cef..2162db2b3 100644
--- a/docutils/docutils/io.py
+++ b/docutils/docutils/io.py
@@ -74,6 +74,57 @@ def error_string(err):
return f'{err.__class__.__name__}: {err}'
+class OutString(str):
+ """Return a string representation of `object` with known encoding.
+
+ Differences to `str()`:
+
+ If the `encoding` is given, both `str` instances and byte-like objects
+ are stored as text string, the latter decoded with `encoding` and
+ `errors` (defaulting to 'strict').
+
+ The encoding is never guessed. If `encoding` is None (the default),
+ an informal string representation is used, also if `errors` are given.
+
+ The original or intended encoding and error handler are stored in the
+ attributes `encoding` and `errors`.
+ Typecasting to `bytes` uses the stored values.
+ """
+
+ def __new__(cls, object, encoding=None, errors='strict'):
+ """Return a new OutString object.
+
+ Provisional.
+ """
+ try:
+ # decode bytes-like objects if encoding is known
+ return super().__new__(cls, object, encoding, errors)
+ except TypeError:
+ return super().__new__(cls, object)
+
+ def __init__(self, object, encoding=None, errors='strict'):
+ """Set "encoding" and "errors" attributes."""
+ self.encoding = encoding
+ self.errors = errors
+
+ def __bytes__(self):
+ try:
+ return super().encode(self.encoding, self.errors)
+ except TypeError:
+ raise TypeError('OutString instance without known encoding')
+
+ def __repr__(self):
+ if self.errors != 'strict':
+ errors_arg = f', errors={self.errors!r}'
+ else:
+ errors_arg = ''
+ return (f'{self.__class__.__name__}({super().__repr__()}, '
+ f'encoding={self.encoding!r}{errors_arg})')
+
+ def encode(self, encoding=None, errors=None):
+ return super().encode(encoding or self.encoding, errors or self.errors)
+
+
class Input(TransformSpec):
"""
Abstract base class for input wrappers.
@@ -264,14 +315,14 @@ class Output(TransformSpec):
raise NotImplementedError
def encode(self, data):
- """Encode and return `data`.
+ """
+ Encode and return `data`.
If `data` is a `bytes` instance, it is returned unchanged.
Otherwise it is encoded with `self.encoding`.
If `self.encoding` is set to the pseudo encoding name "unicode",
`data` must be a `str` instance and is returned unchanged.
-
"""
if self.encoding and self.encoding.lower() == 'unicode':
assert isinstance(data, str), ('output encoding is "unicode" '
@@ -596,14 +647,39 @@ class StringOutput(Output):
default_destination_path = '<string>'
+ def __init__(self, destination=None, destination_path=None,
+ encoding=None, error_handler='strict', auto_encode=True):
+ self.auto_encode = auto_encode
+ """Let `write()` encode the output document and return `bytes`."""
+ super().__init__(destination, destination_path,
+ encoding, error_handler)
+
def write(self, data):
- """Encode `data`, store it in `self.destination`, and return it.
+ """Store `data` in `self.destination`, and return it.
+ If `self.auto_encode` is False, store and return a `str`
+ sub-class instance with "encoding" and "errors" attributes
+ set to `self.encoding` and `self.error_handler`.
+
+ If `self.auto_encode` is True, encode `data` with `self.encoding`
+ and `self.error_handler` and store/return a `bytes` instance.
+ Exception:
If `self.encoding` is set to the pseudo encoding name "unicode",
`data` must be a `str` instance and is returned unchanged
(cf. `Output.encode`).
+ Beware that the `output_encoding`_ setting may affect the content
+ of the output (e.g. an encoding declaration in HTML or XML or the
+ representation of characters as LaTeX macro vs. literal character).
"""
- self.destination = self.encode(data)
+ if self.auto_encode:
+ self.destination = self.encode(data)
+ return self.destination
+
+ if not self.encoding or self.encoding.lower() == 'unicode':
+ encoding = None
+ else:
+ encoding = self.encoding
+ self.destination = OutString(data, encoding, self.error_handler)
return self.destination
diff --git a/docutils/test/test_io.py b/docutils/test/test_io.py
index 17b77eaa1..a1485ce0a 100755
--- a/docutils/test/test_io.py
+++ b/docutils/test/test_io.py
@@ -189,6 +189,19 @@ class OutputTests(unittest.TestCase):
fo.write(self.udata)
self.assertEqual(self.udrain.getvalue(), self.udata)
+ def test_write_auto_encode_false(self):
+ so = io.StringOutput(encoding='latin1', error_handler='replace',
+ auto_encode=False)
+ output = so.write(self.udata)
+ # store output in self.destination and also return it
+ self.assertEqual(output, self.udata)
+ self.assertEqual(so.destination, self.udata)
+ # store also encoding and encoding error handler ...
+ self.assertEqual(output.encoding, 'latin1')
+ self.assertEqual(output.errors, 'replace')
+ # ... to allow easy conversion to `bytes`:
+ self.assertEqual(bytes(output), self.bdata)
+
def test_FileOutput_hande_io_errors_deprection_warning(self):
with self.assertWarnsRegex(DeprecationWarning,
'"handle_io_errors" is ignored'):
@@ -224,6 +237,52 @@ class OutputTests(unittest.TestCase):
self.assertRaises(ValueError, fo.write, self.udata)
+class OutStringTests(unittest.TestCase):
+
+ def test__init__defaults(self):
+ """Test `__new__()` and `__init__()` with default values."""
+
+ os = io.OutString('Grüße')
+ self.assertEqual(str(os), 'Grüße')
+ self.assertEqual(os.encoding, None)
+ self.assertEqual(os.errors, 'strict')
+ # converting to `bytes` fails if the encoding is not known:
+ with self.assertRaises(TypeError):
+ self.assertEqual(bytes(os), 'Grüße')
+ # without known encoding, `bytes` and other incompatible types
+ # are converted to their string representation ...
+ bos = io.OutString(b'gut')
+ self.assertEqual(str(bos), "b'gut'")
+ bos_e = io.OutString('Grüße'.encode('latin1'), errors='ignore')
+ self.assertEqual(str(bos_e), r"b'Gr\xfc\xdfe'")
+ bos = io.OutString(b'gut', encoding=None)
+ self.assertEqual(str(bos), "b'gut'")
+
+ def test__init__custom_attributes(self):
+ """Test `__new__()` and `__init__()` with custom encoding."""
+ os8 = io.OutString('Grüße', encoding='utf-8')
+ self.assertEqual(str(os8), 'Grüße')
+ self.assertEqual(bytes(os8), b'Gr\xc3\xbc\xc3\x9fe')
+ self.assertEqual(repr(os8), "OutString('Grüße', encoding='utf-8')")
+ # With known encoding, "bytes-like" objects are decoded
+ bos1 = io.OutString(b'Gr\xfc\xdfe', encoding='latin1')
+ self.assertEqual(str(bos1), 'Grüße')
+ self.assertEqual(bytes(bos1), b'Gr\xfc\xdfe')
+ # Invalid encodings (including the empty string) raise an error
+ with self.assertRaises(LookupError):
+ io.OutString(b'Gr\xfc\xdfe', encoding='')
+
+ def test__init__custom_errors(self):
+ """Test `__new__()` and `__init__()` with custom `errors`."""
+ ts8_r = io.OutString('Grüße', encoding='utf-8', errors='replace')
+ # Encoding uses the stored error handler:
+ self.assertEqual(ts8_r.encode('ascii'), b'Gr??e')
+ # Initialization with a `bytes` object uses the error handler, too:
+ bts8_r = io.OutString(b'Gr\xfc\xdfe', encoding='utf-8',
+ errors='replace')
+ self.assertEqual(str(bts8_r), 'Gr��e')
+
+
class ErrorOutputTests(unittest.TestCase):
def test_defaults(self):
e = io.ErrorOutput()
diff --git a/docutils/test/test_publisher.py b/docutils/test/test_publisher.py
index 6177ad6d2..a731d2434 100755
--- a/docutils/test/test_publisher.py
+++ b/docutils/test/test_publisher.py
@@ -80,7 +80,8 @@ class PublisherTests(unittest.TestCase):
'nonexisting/path'],
settings_overrides={'traceback': True})
- def test_publish_string(self):
+ def test_publish_string_input_encoding(self):
+ """Test handling of encoded input."""
# Transparently decode `bytes` source (with "input_encoding" setting)
# default: auto-detect, fallback utf-8
# Output is encoded according to "output_encoding" setting.
@@ -102,6 +103,24 @@ class PublisherTests(unittest.TestCase):
settings_overrides=settings)
self.assertTrue(output.endswith('Grüße\n'))
+ def test_publish_string_output_encoding(self):
+ settings = {'_disable_config': True,
+ 'datestamp': False,
+ 'output_encoding': 'latin1',
+ 'output_encoding_error_handler': 'replace'}
+ source = 'Grüß → dich'
+ expected = ('<document source="<string>">\n'
+ ' <paragraph>\n'
+ ' Grüß → dich\n')
+ # current default: encode output, return `bytes`
+ output = core.publish_string(source, settings_overrides=settings)
+ self.assertEqual(output, expected.encode('latin1', 'replace'))
+ # no encoding if `auto_encode` is False:
+ output = core.publish_string(source, settings_overrides=settings,
+ auto_encode=False)
+ self.assertEqual(output, expected)
+ # self.assertEqual(output.encoding, 'latin1')
+
class PublishDoctreeTestCase(unittest.TestCase, docutils.SettingsSpec):
--
2.30.2
_______________________________________________
Docutils-develop mailing list
[email protected]
https://lists.sourceforge.net/lists/listinfo/docutils-develop
Please use "Reply All" to reply to the list.