createrepo: initial comments and a UTF-8 patch

Ville Skyttä <[email protected]>
Newsgroups gmane.linux.rpm.metadata
Message-ID <[email protected]> 
I tried out createrepo yesterday somewhat, here's initial comments:

Issue 1:

It does not currently do a decent job in UTF-8'ifying content.  Not that
it would generate broken XML, but for example the UTF-8 "ä" in my
surname turns in to two "?"s, when it's already UTF-8 in a RPM header!

Patch attached.  This is a simplified version of what I use in fancix,
and the idea originates to decode() Skip Montanaro's query.py at
http://manatee.mojam.com/~skip/python/query.py

The other gotcha in the patch is that when adding content to a libxml2
tree, one does not need to XML escape it.  AFAIK that happens
automatically correctly at serialization time.  XML escaping would be
only needed when printing stuff directly somewhere outside of the libxml
objects; that is not currently done so I nuked xmlCleanString()
altogether.  While at it, I added explicit encodings to serialize()
calls.

With this patch applied, the output is improved quite a bit here.  Add
new encodings to the list in utf8String() if you like.

Issue 2:

The name "author" attribute in <changelog> is not a very good choice
IMO.  RPM defines it as the "name" of the changelog entry.  It is very
common that for RPMs the author attribute will contain stuff like "John
Doe &lt;john at doe dot com&gt; - 2.6.8-0.1", ie. it's not only the
author -> suggesting changing "author" to "name" unless it causes too
much problems.

Issue 3:

  $ createrepo .
  [...]
  Saving Primary metadata
  Saving file lists metadata
  Saving other metadata
  $ echo foo > repodata/foo.txt
  $ createrepo .
  [...]
  Saving Primary metadata
  Saving file lists metadata
  Saving other metadata
  Could not remove old metadata dir: .olddata
  Error was [Errno 39] Directory not empty: '.olddata'
  $ createrepo .
  Old data directory exists, please remove: .olddata

Bug or feature?

_______________________________________________
Rpm-metadata mailing list
[email protected]
https://lists.dulug.duke.edu/mailman/listinfo/rpm-metadata
createrepo-utf8.patch (text/x-patch, 5.4 KB) 
Index: dumpMetadata.py
===================================================================
RCS file: /cvsroot/metadata/cvs-root/generate/dumpMetadata.py,v
retrieving revision 1.21
diff -a -u -r1.21 dumpMetadata.py
--- dumpMetadata.py	23 Jul 2004 18:45:02 -0000	1.21
+++ dumpMetadata.py	24 Jul 2004 09:03:32 -0000
@@ -86,28 +86,29 @@
 
 def utf8String(string):
     """hands back a unicoded string"""
+    if isinstance(string, unicode):
+        return string
     try:
-        string = unicode(string)
+        x = unicode(string, 'ascii')
+        return string
     except UnicodeError:
-        newstring = ''
-        for char in string:
-            if ord(char) > 127:
-                newstring = newstring + '?'
+        encodings = ['utf-8', 'iso-8859-1', 'iso-8859-15', 'iso-8859-2']
+        for enc in encodings:
+            try:
+                x = unicode(string, enc)
+            except UnicodeError:
+                pass
             else:
-                newstring = newstring + char
-        return unicode(newstring)
-    else:
-        return string
+                if x.encode(enc) == string:
+                    return x.encode('utf-8')
+    newstring = ''
+    for char in string:
+        if ord(char) > 127:
+            newstring = newstring + '?'
+        else:
+            newstring = newstring + char
+    return newstring
 
-def xmlCleanString(doc, string):
-    """hands back a special-char encoded and utf8 cleaned string
-       Takes a libxml2 document object and the string to clean
-       document object is needed to not require expensive get_doc function
-    """
-    string = utf8String(string)
-    string = doc.encodeSpecialChars(string)
-    return string
-    
         
 def byteranges(file):
     """takes an rpm file or fileobject and returns byteranges for location of the header"""
@@ -506,7 +507,6 @@
         value = utf8String(value)
         value = re.sub("\n$", '', value)
         entry = pkgNode.newChild(None, tag, None)
-        value = xmlCleanString(doc, value)
         entry.addContent(value)
         
     time = pkgNode.newChild(None, 'time', None)
@@ -526,7 +526,6 @@
         value = utf8String(value)
         value = re.sub("\n$", '', value)
         entry = format.newChild(formatns, tag, None)
-        value = xmlCleanString(doc, value)
         entry.addContent(value)
         
     hr = format.newChild(formatns, 'header-range', None)
@@ -580,11 +579,11 @@
         
     for file in rpmObj.usefulFiles():
         files = format.newChild(None, 'file', None)
-        file = xmlCleanString(doc, file)
+        file = utf8String(file)
         files.addContent(file)
     for directory in rpmObj.usefulDirs():
         files = format.newChild(None, 'file', None)
-        directory = xmlCleanString(doc, directory)
+        directory = utf8String(directory)
         files.addContent(directory)
         files.newProp('type', 'dir')
     
@@ -601,16 +600,16 @@
     version.newProp('rel', str(rpmObj.tagByName('release')))
     for file in rpmObj.filenames:
         files = pkg.newChild(None, 'file', None)
-        file = xmlCleanString(doc, file)
+        file = utf8String(file)
         files.addContent(file)
     for directory in rpmObj.dirnames:
         files = pkg.newChild(None, 'file', None)
-        directory = xmlCleanString(doc, directory)
+        directory = utf8String(directory)
         files.addContent(directory)
         files.newProp('type', 'dir')
     for ghost in rpmObj.ghostnames:
         files = pkg.newChild(None, 'file', None)
-        ghost = xmlCleanString(doc, ghost)
+        ghost = utf8String(ghost)
         files.addContent(ghost)
         files.newProp('type', 'ghost')
     return pkg
@@ -627,8 +626,7 @@
     clogs = rpmObj.changelogLists()
     for (name, time, text) in clogs:
         clog = pkg.newChild(None, 'changelog', None)
-        text = xmlCleanString(doc, text)
-        clog.addContent(text)
+        clog.addContent(utf8String(text))
         clog.newProp('author', utf8String(name))
         clog.newProp('date', str(time))
     return pkg
Index: genpkgmetadata.py
===================================================================
RCS file: /cvsroot/metadata/cvs-root/generate/genpkgmetadata.py,v
retrieving revision 1.23
diff -a -u -r1.23 genpkgmetadata.py
--- genpkgmetadata.py	23 Jul 2004 19:41:28 -0000	1.23
+++ genpkgmetadata.py	24 Jul 2004 09:03:33 -0000
@@ -288,7 +288,7 @@
                 errorprint(_('\nAn error occurred creating primary metadata: %s') % e)
                 continue
             else:
-                output = node.serialize(None, cmds['pretty'])
+                output = node.serialize('UTF-8', cmds['pretty'])
                 basefile.write(output)
                 basefile.write('\n')
                 node.unlinkNode()
@@ -301,7 +301,7 @@
                 errorprint(_('\nAn error occurred creating filelists: %s') % e)
                 continue
             else:
-                output = node.serialize(None, cmds['pretty'])
+                output = node.serialize('UTF-8', cmds['pretty'])
                 flfile.write(output)
                 flfile.write('\n')
                 node.unlinkNode()
@@ -314,7 +314,7 @@
                 errorprint(_('\nAn error occurred: %s') % e)
                 continue
             else:
-                output = node.serialize(None, cmds['pretty'])
+                output = node.serialize('UTF-8', cmds['pretty'])
                 otherfile.write(output)
                 otherfile.write('\n')
                 node.unlinkNode()
lmpx.com only provides a reader for public news (NNTP) servers. It is not affiliated with the servers or forums shown here and is not responsible for the content of articles, which is written by their respective authors.