[PATCH 1/9] string2lines(): Do not convert some whitespace into space

To allow tabs and other whitespace to be part of the output, we must not
remove it prior to parsing.  The handling of whitespace is done during
parsing.

We remove the stripping of whitespace "to the right" as some listing,
e.g. for the Programming Language Whitespace might relay on that.

Also note that there is more whitespace in Unicode than just Space
(`` ``), Tab (``\t``), Vertical Tab (``\v``), and Form Feed (``\f``).
---
 docutils/docutils/parsers/rst/__init__.py       |  4 +---
 .../docutils/parsers/rst/directives/misc.py     |  3 +--
 docutils/docutils/statemachine.py               | 17 +++--------------
 3 files changed, 5 insertions(+), 19 deletions(-)

diff --git a/docutils/docutils/parsers/rst/__init__.py b/docutils/docutils/parsers/rst/__init__.py
index 1eadbfe33..b8f45a45f 100644
--- a/docutils/docutils/parsers/rst/__init__.py
+++ b/docutils/docutils/parsers/rst/__init__.py
@@ -171,9 +171,7 @@ def parse(self, inputstring, document):
               state_classes=self.state_classes,
               initial_state=self.initial_state,
               debug=document.reporter.debug_flag)
-        inputlines = docutils.statemachine.string2lines(
-              inputstring, tab_width=document.settings.tab_width,
-              convert_whitespace=True)
+        inputlines = docutils.statemachine.string2lines(inputstring)
         for i, line in enumerate(inputlines):
             if len(line) > self.document.settings.line_length_limit:
                 error = self.document.reporter.error(
diff --git a/docutils/docutils/parsers/rst/directives/misc.py b/docutils/docutils/parsers/rst/directives/misc.py
index 85032d8e2..4b8043a5b 100644
--- a/docutils/docutils/parsers/rst/directives/misc.py
+++ b/docutils/docutils/parsers/rst/directives/misc.py
@@ -117,8 +117,7 @@ def run(self):
                                   'directive:\nText not found.' % self.name)
             rawtext = rawtext[:before_index]
 
-        include_lines = statemachine.string2lines(rawtext, tab_width,
-                                                  convert_whitespace=True)
+        include_lines = statemachine.string2lines(rawtext)
         for i, line in enumerate(include_lines):
             if len(line) > settings.line_length_limit:
                 raise self.warning('"%s": line %d exceeds the'
diff --git a/docutils/docutils/statemachine.py b/docutils/docutils/statemachine.py
index e3ea892bc..890572564 100644
--- a/docutils/docutils/statemachine.py
+++ b/docutils/docutils/statemachine.py
@@ -1485,26 +1485,15 @@ class StateCorrection(Exception):
     """
 
 
-def string2lines(astring, tab_width=8, convert_whitespace=False,
-                 whitespace=re.compile('[\v\f]')):
+def string2lines(astring):
     """
-    Return a list of one-line strings with tabs expanded, no newlines, and
-    trailing whitespace stripped.
-
-    Each tab is expanded with between 1 and `tab_width` spaces, so that the
-    next character's index becomes a multiple of `tab_width` (8 by default).
+    Return a list of one-line strings.
 
     Parameters:
 
     - `astring`: a multi-line string.
-    - `tab_width`: the number of columns between tab stops.
-    - `convert_whitespace`: convert form feeds and vertical tabs to spaces?
-    - `whitespace`: pattern object with the to-be-converted
-      whitespace characters (default [\\v\\f]).
     """
-    if convert_whitespace:
-        astring = whitespace.sub(' ', astring)
-    return [s.expandtabs(tab_width).rstrip() for s in astring.splitlines()]
+    return astring.splitlines()
 
 
 def _exception_data():