[Patch] Split tree handling for metadata
Paul Nasrat <[email protected]>
| Newsgroups | gmane.linux.rpm.metadata |
|---|---|
| Organization | Red Hat, Inc. |
| Message-ID | <[email protected]> |
This applies on top of my previous patches. This probably could do with
some feedback but it works for my purposes right now. In particular the
basedir/baseurl munging is a little ugly, but it works as demonstrated,
also I'm currently implementing policy for anaconda (repmod on disc1):
python /home/pauln/scm/duke/metadata/generate/genpkgmetadata.py --pretty
--split --baseurl media:/fedora-core-5 i386-disc?
<data type="other">
<location xml:base="media:/fedora-core-5#1"
href="repodata/other.xml.gz"/>
...
</data>
repoquery --repoid cd1 -qa --qf '%{basepath} %{name}' | head
media:/fedora-core-5#1 procps
media:/fedora-core-5#5 jakarta-commons-httpclient-manual
media:/fedora-core-5#3 joe
media:/fedora-core-5#3 carol
media:/fedora-core-5#1 libgpg-error
media:/fedora-core-5#3 kdevelop
media:/fedora-core-5#1 util-linux
media:/fedora-core-5#1 acpid
media:/fedora-core-5#4 gnome-libs-devel
media:/fedora-core-5#2 oro
Identifying media is left as an exercise to the reader[0]
Paul
[0] and also the writer for anaconda...
_______________________________________________
Rpm-metadata mailing list
[email protected]
https://lists.dulug.duke.edu/mailman/listinfo/rpm-metadata
createrepo-genclass-multidirs.patch
(text/x-patch, 9 KB)
Index: generate/genpkgmetadata.py
===================================================================
--- generate.orig/genpkgmetadata.py
+++ generate/genpkgmetadata.py
@@ -62,10 +62,12 @@ class MetaDataGenerator:
def __init__(self, cmds):
self.cmds = cmds
self.ts = rpm.TransactionSet()
+ self.pkgcount = 0
self.files = []
- def getFileList(self, basepath, path, ext):
- """Return all files in path matching ext, store them in filelist, recurse dirs. Modifies self.files"""
+ def getFileList(self, basepath, path, ext, filelist):
+ """Return all files in path matching ext, store them in filelist,
+ recurse dirs. Returns a list object"""
extlen = len(ext)
totalpath = os.path.normpath(os.path.join(basepath, path))
@@ -77,37 +79,41 @@ class MetaDataGenerator:
for d in dir_list:
if os.path.isdir(totalpath + '/' + d):
- self.getFileList(basepath, os.path.join(path, d), ext )
+ filelist = self.getFileList(basepath, os.path.join(path, d), ext, filelist)
else:
if string.lower(d[-extlen:]) == '%s' % (ext):
if totalpath.find(basepath) == 0:
relativepath = totalpath.replace(basepath, "", 1)
relativepath = relativepath.lstrip("/")
- self.files.append(os.path.join(relativepath, d))
+ filelist.append(os.path.join(relativepath, d))
else:
raise "basepath '%s' not found in path '%s'" % (basepath, totalpath)
- def trimRpms(self):
+ return filelist
+
+
+ def trimRpms(self, files):
badrpms = []
- for file in self.files:
+ for file in files:
for glob in self.cmds['excludes']:
if fnmatch.fnmatch(file, glob):
# print 'excluded: %s' % file
if file not in badrpms:
badrpms.append(file)
for file in badrpms:
- if file in self.files:
- self.files.remove(file)
+ if file in files:
+ files.remove(file)
+ return files
def doPkgMetadata(self, directory):
"""all the heavy lifting for the package metadata"""
# rpms we're going to be dealing with
- self.getFileList(self.cmds['basedir'], directory, '.rpm')
- self.trimRpms()
- self.pkgcount = len(self.files)
+ files = self.getFileList(self.cmds['basedir'], directory, '.rpm', [])
+ files = self.trimRpms(files)
+ self.pkgcount = len(files)
self.openMetadataDocs()
- self.writeMetadataDocs()
+ self.writeMetadataDocs(files)
self.closeMetadataDocs()
@@ -153,9 +159,8 @@ class MetaDataGenerator:
self.otherfile.write('<otherdata xmlns="http://linux.duke.edu/metadata/other" packages="%s">\n' %
self.pkgcount)
- def writeMetadataDocs(self):
- current = 0
- for file in self.files:
+ def writeMetadataDocs(self, files, current=0):
+ for file in files:
current+=1
try:
mdobj = dumpMetadata.RpmMetaData(self.ts, self.cmds['basedir'], file, self.cmds)
@@ -164,7 +169,7 @@ class MetaDataGenerator:
print '%d/%d - %s' % (current, len(files), file)
else:
sys.stdout.write('\r' + ' ' * 80)
- sys.stdout.write("\r%d/%d - %s" % (current, len(self.files), file))
+ sys.stdout.write("\r%d/%d - %s" % (current, self.pkgcount, file))
sys.stdout.flush()
except dumpMetadata.MDError, e:
errorprint('\n%s - %s' % (e, file))
@@ -208,6 +213,7 @@ class MetaDataGenerator:
node.unlinkNode()
node.freeNode()
del node
+ return current
def closeMetadataDocs(self):
@@ -255,6 +261,47 @@ class MetaDataGenerator:
del repodoc
+class SplitMetaDataGenerator(MetaDataGenerator):
+
+ def __init__(self, cmds):
+ MetaDataGenerator.__init__(self, cmds)
+ self.initialdir = self.cmds['basedir']
+
+ def _getFragmentUrl(self, url, fragment):
+ import urlparse
+ urlparse.uses_fragment.append('media')
+ if not url:
+ return url
+ (scheme, netloc, path, query, fragid) = urlparse.urlsplit(url)
+ return urlparse.urlunsplit((scheme, netloc, path, query, str(fragment)))
+
+ def doPkgMetadata(self, directories):
+ """all the heavy lifting for the package metadata"""
+ import types
+ if type(directories) == types.StringType:
+ MetaDataGenerator.doPkgMetadata(self, directories)
+ return
+ filematrix = {}
+ for mydir in directories:
+ filematrix[mydir] = self.getFileList(os.path.join(self.initialdir, mydir), '.', '.rpm', [])
+ self.trimRpms(filematrix[mydir])
+ self.pkgcount += len(filematrix[mydir])
+
+ mediano = 1
+ current = 0
+ self.cmds['baseurl'] = self._getFragmentUrl(self.cmds['baseurl'], mediano)
+ self.cmds['basedir'] = os.path.join(self.initialdir, directories[0])
+ self.openMetadataDocs()
+ for mydir in directories:
+ self.cmds['basedir'] = os.path.join(self.initialdir, mydir)
+ self.cmds['baseurl'] = self._getFragmentUrl(self.cmds['baseurl'], mediano)
+ current = self.writeMetadataDocs(filematrix[mydir], current)
+ mediano += 1
+ self.cmds['basedir'] = os.path.join(self.initialdir, directories[0])
+ self.cmds['baseurl'] = self._getFragmentUrl(self.cmds['baseurl'], 1)
+ self.closeMetadataDocs()
+
+
def checkAndMakeDir(dir):
"""
check out the dir and make it, if possible, return 1 if done, else return 0
@@ -296,6 +343,7 @@ def parseArgs(args):
cmds['cachedir'] = None
cmds['basedir'] = os.getcwd()
cmds['cache'] = False
+ cmds['split'] = False
cmds['file-pattern-match'] = ['.*bin\/.*', '^\/etc\/.*', '^\/usr\/lib\/sendmail$']
cmds['dir-pattern-match'] = ['.*bin\/.*', '^\/etc\/.*']
@@ -303,7 +351,7 @@ def parseArgs(args):
gopts, argsleft = getopt.getopt(args, 'phqVvg:s:x:u:c:', ['help', 'exclude=',
'quiet', 'verbose', 'cachedir=', 'basedir=',
'baseurl=', 'groupfile=', 'checksum=',
- 'version', 'pretty'])
+ 'version', 'pretty', 'split'])
except getopt.error, e:
errorprint(_('Options Error: %s.') % e)
usage()
@@ -315,20 +363,22 @@ def parseArgs(args):
elif arg in ['-V', '--version']:
print '%s' % __version__
sys.exit(0)
+ elif arg == '--split':
+ cmds['split'] = True
except ValueError, e:
errorprint(_('Options Error: %s') % e)
usage()
# make sure our dir makes sense before we continue
- if len(argsleft) > 1:
+ if len(argsleft) > 1 and not cmds['split']:
errorprint(_('Error: Only one directory allowed per run.'))
usage()
elif len(argsleft) == 0:
errorprint(_('Error: Must specify a directory to index.'))
usage()
else:
- directory = argsleft[0]
+ directories = argsleft
try:
for arg,a in gopts:
@@ -376,10 +426,11 @@ def parseArgs(args):
usage()
- return cmds, directory
+ return cmds, directories
def main(args):
- cmds, directory = parseArgs(args)
+ cmds, directories = parseArgs(args)
+ directory = directories[0]
#setup some defaults
cmds['primaryfile'] = 'primary.xml.gz'
@@ -403,7 +454,9 @@ def main(args):
errorprint(_('Directory must be writable.'))
sys.exit(1)
-
+ if cmds['split']:
+ oldbase = cmds['basedir']
+ cmds['basedir'] = os.path.join(cmds['basedir'], directory)
if not checkAndMakeDir(os.path.join(cmds['basedir'], cmds['tempdir'])):
sys.exit(1)
@@ -423,8 +476,13 @@ def main(args):
errorprint(_('error in must be able to write to metadata files:\n -> %s') % filepath)
usage()
- mdgen = MetaDataGenerator(cmds)
- mdgen.doPkgMetadata(directory)
+ if cmds['split']:
+ cmds['basedir'] = oldbase
+ mdgen = SplitMetaDataGenerator(cmds)
+ mdgen.doPkgMetadata(directories)
+ else:
+ mdgen = MetaDataGenerator(cmds)
+ mdgen.doPkgMetadata(directory)
mdgen.doRepoMetadata()
if os.path.exists(os.path.join(cmds['basedir'], cmds['finaldir'])):
@@ -460,6 +518,7 @@ def main(args):
errorprint(_('Error was %s') % e)
sys.exit(1)
+#XXX: fix to remove tree as we mung basedir
try:
os.rmdir(os.path.join(cmds['basedir'], cmds['olddir']))
except OSError, e: