Re: createrepo/utils.py
Toshio Kuratomi <[email protected]>
| Newsgroups | gmane.linux.rpm.metadata |
|---|---|
| Message-ID | <[email protected]> |
Toshio Kuratomi wrote: > James Antill wrote: >> The big problem here being that a bunch of the "small bytes" like 0x01 >> are valid utf8 but aren't valid XML data. Hence the patches. >> >> After looking again, it's now obvious that we still screw up if we pass >> a unicode() object in that has 0x01 bytes in it ... so we should >> probably fix that too (although I'm not sure if that's possible). >> I took a second look at the function and realized you were getting rid of the control characters rather than replacing them with '?'. So here's an updated patch that just removes the control characters. -Toshio _______________________________________________ Rpm-metadata mailing list [email protected] https://lists.dulug.duke.edu/mailman/listinfo/rpm-metadata
createrepo-utf8string-fix2.patch
(text/x-patch, 2.5 KB)
diff --git a/createrepo/utils.py b/createrepo/utils.py
index 1dc3b0c..8c3eaa9 100644
--- a/createrepo/utils.py
+++ b/createrepo/utils.py
@@ -74,39 +74,34 @@ def returnFD(filename):
return fdno
def utf8String(string):
- """hands back a unicoded string"""
- if string is None:
+ """hands back a utf8 encoded byte string"""
+ if not string:
+ # Small optimization
return ''
- elif isinstance(string, unicode):
- return string
- du = False
- try:
- x = unicode(string, 'ascii')
- du = True
- except UnicodeError:
- encodings = ['utf-8', 'iso-8859-1', 'iso-8859-15', 'iso-8859-2']
- for enc in encodings:
- try:
- x = unicode(string, enc)
- except UnicodeError:
- pass
- else:
- if x.encode(enc) == string:
- return x.encode('utf-8')
- newstring = ''
- # Kill bytes (or libxml will die) not in the small byte portion of:
- # http://www.w3.org/TR/REC-xml/#NT-Char
- # we allow high bytes, if it passed the utf8 check above. Eg.
- # good chars = #x9 | #xA | #xD | [#x20-...]
- bad_small_bytes = range(0, 8) + [11, 12] + range(14, 32)
- for char in string:
- if ord(char) in bad_small_bytes:
- pass # Just ignore these bytes...
- elif not du and ord(char) > 127:
- newstring = newstring + '?'
- else:
- newstring = newstring + char
- return newstring
+ elif not isinstance(string, unicode):
+ # By the end of this block, have a unicode string
+ try:
+ string = unicode(string, 'utf-8')
+ except UnicodeError:
+ # Try several encodings
+ encodings = ('iso-8859-1', 'iso-8859-15', 'iso-8859-2')
+ for enc in encodings:
+ try:
+ string = unicode(string, enc)
+ break
+ except UnicodeError:
+ continue
+ if not isinstance(string, unicode):
+ string = unicode(string, 'utf-8', 'replace')
+
+ # Remove control characters. These are valid unicode and ASCII but
+ # not valid xml.
+ control_codes = range(0, 8) + [11, 12] + range(14, 32)
+ control_table = dict(zip(control_codes, [None] * len(control_codes)))
+ string = string.translate(control_table)
+
+ # Finally return a utf-8 encoded byte string representation of the unicode
+ return string.encode('utf-8')
def checkAndMakeDir(dir):
"""
signature.asc
(application/pgp-signature, 189 B)
-----BEGIN PGP SIGNATURE----- Version: GnuPG v1.4.7 (GNU/Linux) iD8DBQFIBrmeX6yAic2E7kgRAhpGAJ9pmR/Tmp7b4Z337IXGXPAbA/u1IQCeIxgX JxBbU9WtTu21MClnrr/oyAg= =Ql6m -----END PGP SIGNATURE-----