Re: createrepo patch to optionally use /usr/bin/file to detect rpms

--- /Users/jay/cvsroot/generate/genpkgmetadata.py	2006-08-11 16:01:38.000000000 -0400
+++ genpkgmetadata.py	2006-09-12 14:26:59.000000000 -0400
@@ -1,4 +1,4 @@
-#!/usr/bin/python -t
+#!/usr/bin/env python -t
 # primary functions and glue for generating the repository metadata
 #
 
@@ -42,26 +42,51 @@
 
 def usage(retval=1):
     print _("""
-    createrepo [options] directory-of-packages
-
-    Options:
-     -u, --baseurl <url> = optional base url location for all files
-     -o, --outputdir <dir> = optional directory to output to
-     -x, --exclude = files globs to exclude, can be specified multiple times
-     -q, --quiet = run quietly
-     -n, --noepoch = don't add zero epochs for non-existent epochs
-                    (incompatible with yum and smart but required for
-                     systems with rpm < 4.2.1)
-     -g, --groupfile <filename> to point to for group information (precreated)
-                    (<filename> relative to directory-of-packages)
-     -v, --verbose = run verbosely
-     -c, --cachedir <dir> = specify which dir to use for the checksum cache
-     -C, --checkts = don't generate repo metadata, if their ctimes are newer
-                     than the rpm ctimes.
-     -h, --help = show this help
-     -V, --version = output version
-     -p, --pretty = output xml files in pretty format.
-    """)
+Usage:
+  createrepo [options] <pkgdir> | -
+  createrepo [options] --split <pkgdir1> [<pkgdir2> ...]
+
+The createrepo command creates a repomd (xml-based metadata) repository from a
+set of packages. It creates/updates a directory named "repodata" containing a
+set of XML files.
+
+If <pkgdir> is "-", then the list of packages is taken from stdin, one
+filename per line. Each filename must be an absolute path under the
+current-working-directory or --basedir or a relative path under the
+current-working-directory or --basedir. In this mode, repodata is created
+under the CWD or --basedir unless overridden with -o.
+
+If <pkgdir> is not "-", then it is taken as the name of a directory (absolute
+or relative to CWD or relative to --basedir) that createrepo should recurse
+over, looking for files with an extension of '.rpm' (case insensitive). In
+this mode, repodata is created under <pkgdir> or --basedir unless overridden
+with "-o"
+
+With the "--split" option, createrepo runs in split media mode; rather than
+operate on a single top-level directory, createrepo take a list of directories
+corresponding to volumes in a media set.
+
+Options:
+     --split           enable split media mode
+ -b, --basedir <dir>   use <dir> as basedir instead of CWD. Ignored if 
+                       <pkgdir> is absolute.
+ -u, --baseurl <url>   optional base url
+ -c, --cachedir <dir>  specify checksum cache directory
+ -C, --checkts         don't generate repo metadata if metadata files' ctimes
+                       are newer than rpms' ctimes.
+ -x, --exclude         files globs to exclude, can be specified multiple times
+ -g, --groupfile <filename> precreated <filename> to point to for group 
+                       information; <filename> should be relative to <pkgdir>
+ -n, --noepoch         don't add zero epochs for non-existent epochs
+                       (incompatible with yum and smart but required for
+                       systems with rpm < 4.2.1)
+ -o, --outputdir <dir> optional output directory for repodata
+ -p, --pretty          use pretty format for xml metadata files
+ -q, --quiet           run quietly
+ -v, --verbose         run verbosely
+ -h, --help            show this help
+ -V, --version         show version
+""")
 
     sys.exit(retval)
 
@@ -72,7 +97,14 @@
         self.pkgcount = 0
         self.files = []
 
+
     def getFileList(self, basepath, directory, ext):
+        if self.cmds["filesfromstdin"]:
+            return self.getFileListFromStdin(basepath, directory)
+        else:
+            return self.getFileListFromPath(basepath, directory, ext)
+        
+    def getFileListFromPath(self, basepath, directory, ext):
         """Return all files in path matching ext, store them in filelist,
         recurse dirs. Returns a list object"""
 
@@ -91,6 +123,27 @@
         startdir = os.path.join(basepath, directory) + '/'
         os.path.walk(startdir, extension_visitor, filelist)
         return filelist
+        
+    def getFileListFromStdin(self, basepath, directory):
+        startdir = os.path.join(basepath, directory)
+        startdir = os.path.normpath(startdir) + "/"
+        filelist = []
+        while 1:
+            line = sys.stdin.readline()
+            if not line:
+                break
+            path = line.rstrip("\n\r")
+            if not os.path.isabs(path):
+                path = os.path.abspath(path)
+            if not os.path.exists(path):
+                errorprint(_('cannot get to file: %s') % path)
+                continue
+            if path.startswith(startdir):
+                path = path.replace(startdir, "", 1)
+                filelist.append(path)
+            else:
+                errorprint(_('file not underneath %s: %s') % (startdir, path))
+        return filelist
 
     def checkTimeStamps(self, directory):
         if self.cmds['checkts']:
@@ -377,11 +430,12 @@
     cmds['mdtimestamp'] = 0
     cmds['split'] = False
     cmds['outputdir'] = ""
+    cmds['filesfromstdin'] = False
     cmds['file-pattern-match'] = ['.*bin\/.*', '^\/etc\/.*', '^\/usr\/lib\/sendmail$']
     cmds['dir-pattern-match'] = ['.*bin\/.*', '^\/etc\/.*']
 
     try:
-        gopts, argsleft = getopt.getopt(args, 'phqVvng:s:x:u:c:o:C', ['help', 'exclude=',
+        gopts, argsleft = getopt.getopt(args, 'phqVvng:s:x:u:c:o:Cb:', ['help', 'exclude=',
                                                                   'quiet', 'verbose', 'cachedir=', 'basedir=',
                                                                   'baseurl=', 'groupfile=', 'checksum=',
                                                                   'version', 'pretty', 'split', 'outputdir=',
@@ -409,8 +463,10 @@
         errorprint(_('Error: Only one directory allowed per run.'))
         usage()
     elif len(argsleft) == 0:
-        errorprint(_('Error: Must specify a directory to index.'))
+        errorprint(_('Error: Must specify a directory to index or "-".'))
         usage()
+    elif "-" in argsleft and cmds['split']:
+        errorprint(_('--split and "-" are mutually exclusive'))
     else:
         directories = argsleft
 
@@ -445,7 +501,7 @@
                 cmds['cachedir'] = a
             elif arg in ['-C', '--checkts']:
                 cmds['checkts'] = True
-            elif arg == '--basedir':
+            elif arg in ['-b','--basedir']:
                 cmds['basedir'] = a
             elif arg in ['-o','--outputdir']:
                 cmds['outputdir'] = a
@@ -461,6 +517,9 @@
         sys.exit(1)
 
     directory = directories[0]
+    if directory == "-":
+        directory = cmds['basedir']
+        cmds['filesfromstdin'] = True
 # 
     directory = os.path.normpath(directory)
     if cmds['split']:

#!/bin/sh
outdir="/Users/jay/Work/WIP/Patch Management/ThirdParty/generate"
repodir="/Users/jay/Downloads/Yum/lca-yum-tutorial/demo/repo"
cmd="$outdir/genpkgmetadata.py"

rm -rf "$outdir/"repodata?
echo "Test 1"
"$cmd" -o "$outdir" "$repodir"
mv "$outdir/repodata"  "$outdir/repodata1"

echo "Test 2"
(cd "$repodir" && find . -name '*.rpm' | "$cmd" -o "$outdir" -)
mv "$outdir/repodata"  "$outdir/repodata2"

echo "Test 3"
(cd "$repodir" && find . -name '*.rpm' | "$cmd" -b . -o "$outdir" -)
mv "$outdir/repodata"  "$outdir/repodata3"

echo "Test 4"
(cd "$repodir" && find . -name '*.rpm' | "$cmd" -b "$repodir" -o "$outdir" -)
mv "$outdir/repodata"  "$outdir/repodata4"

echo "Test 5"
find "$repodir" -name '*.rpm' | "$cmd" -b "$repodir" -o "$outdir" -
mv "$outdir/repodata"  "$outdir/repodata5"

echo "Decompressing metadata"
gunzip "$outdir"/repodata?/*.gz
echo "Testing for diffs (should see no output):"
diff -r -I timestamp "$outdir/repodata1" "$outdir/repodata2"
diff -r -I timestamp "$outdir/repodata1" "$outdir/repodata3"
diff -r -I timestamp "$outdir/repodata1" "$outdir/repodata4"
diff -r -I timestamp "$outdir/repodata1" "$outdir/repodata5"
echo "Done."

--- /Users/jay/cvsroot/generate/docs/createrepo.8	2006-08-11 16:01:38.000000000 -0400
+++ createrepo.8	2006-09-12 14:57:04.000000000 -0400
@@ -4,13 +4,22 @@
 createrepo \- Create repomd (xml-rpm-metadata) repository
 
 .SH "SYNOPSIS"
-\fBcreaterepo\fP [options] <directory>
+\fBcreaterepo\fP [options] {<directory> | -}
+.br
+\fBcreaterepo\fP [options] --split <directory> [<directory> ...]
+
 .PP
 
 .SH "DESCRIPTION"
-\fBcreaterepo\fP is a program that creates a repomd (xml-based rpm metadata) repository from a set of rpms.
+\fBcreaterepo\fP is a program that creates a repomd (xml-based rpm metadata)
+repository from a set of rpms. In non-split mode, if <directory> is specified
+as "-", then a list of files is taken from stdin, one filename per line
+(filenames must be absolute and under the current-working-directory/--basedir
+or must be relative to the current-working-directory/--basedir).
 
 .SH "OPTIONS"
+.IP "\fB\-b --basedir\fP <url>"
+Optional base dir that <directory> should be relative to. The default is the current working directory.
 .IP "\fB\-u --baseurl\fP <url>"
 Optional base url location for all files. (not used by any clients at this
 time)