Re: replacing os.chdir calls with basedir config option

Dennis Gregorovic <[email protected]>
Newsgroups gmane.linux.rpm.metadata
Message-ID <[email protected]> 
On Fri, 2005-11-11 at 13:17 -0500, Paul Nasrat wrote:
> On Fri, 2005-11-11 at 08:44 -0500, Dennis Gregorovic wrote:
> > I've started using createrepo as a library instead of a standalone app.
> > The only issue that I've run into so far is that the os.chdir() calls
> > can cause problems, especially if you have multiple threads using
> > createrepo in parallel.
> > 
> > Attached is a patch that removes the os.chdir() calls and introduces a
> > 'basedir' config option, settable with --basedir and defaulting to
> > os.getcwd().
> 
> There seems to be some whitespace damage in your patches:
> 
> 
> -    def __init__(self, ts, filename, options):
> +    def __init__(self, ts, basedir, filename, options):
>          try:
> -            stats = os.stat(filename)
> +            stats = os.stat(os.path.join(basedir, filename))
>              self.size = stats[6]
>              self.mtime = stats[8]
>              del stats
> -        except OSError, e:
> -            raise MDError, "Error Stat'ing file %s" % filename
> -
> You remove the exception above yet the following is not indented.
> 
>          self.options = options
>          self.localurl = options['baseurl']
>          self.relativepath = filename
> -        fd = returnFD(filename)
> +            fd = returnFD(os.path.join(basedir, filename))
> This is indented for no reason.
> 
> Please can you rediff.
> 
> Paul

Hi Paul,

Hopefully this patch is better.  I have emacs set up to automatically
remove tabs and trailing whitespace, so I normally diff -w, but that may
have screwed up the first patch.

-- Dennis

_______________________________________________
Rpm-metadata mailing list
[email protected]
https://lists.dulug.duke.edu/mailman/listinfo/rpm-metadata
createrepo-basedir.patch (text/x-patch, 16.9 KB) 
Index: genpkgmetadata.py
===================================================================
RCS file: /cvsroot/metadata/cvs-root/generate/genpkgmetadata.py,v
retrieving revision 1.41
diff -u -r1.41 genpkgmetadata.py
--- genpkgmetadata.py	2 Nov 2005 20:23:56 -0000	1.41
+++ genpkgmetadata.py	11 Nov 2005 18:30:05 -0000
@@ -38,11 +38,11 @@
 def _(args):
     """Stub function for translation"""
     return args
-    
+
 def usage(retval=1):
     print _("""
     createrepo [options] directory-of-packages
-    
+
     Options:
      -u, --baseurl = optional base url location for all files
      -x, --exclude = files globs to exclude, can be specified multiple times
@@ -54,30 +54,35 @@
      -h, --help = show this help
      -V, --version = output version
      -p, --pretty = output xml files in pretty format.
-    """) 
+    """)
 
     sys.exit(retval)
 
 
-def getFileList(path, ext, filelist):
+def getFileList(basepath, path, ext, filelist):
     """Return all files in path matching ext, store them in filelist, recurse dirs
        return list object"""
-    
+
     extlen = len(ext)
+    totalpath = os.path.normpath(os.path.join(basepath, path))
     try:
-        dir_list = os.listdir(path)
+        dir_list = os.listdir(totalpath)
     except OSError, e:
-        errorprint(_('Error accessing directory %s, %s') % (path, e))
+        errorprint(_('Error accessing directory %s, %s') % (totalpath, e))
         sys.exit(1)
-        
+
     for d in dir_list:
-        if os.path.isdir(path + '/' + d):
-            filelist = getFileList(path + '/' + d, ext, filelist)
+        if os.path.isdir(totalpath + '/' + d):
+            filelist = getFileList(basepath, os.path.join(path, d), ext, filelist)
         else:
             if string.lower(d[-extlen:]) == '%s' % (ext):
-               newpath = os.path.normpath(path + '/' + d)
-               filelist.append(newpath)
-                    
+                if totalpath.find(basepath) == 0:
+                    relativepath = totalpath.replace(basepath, "", 1)
+                    relativepath = relativepath.lstrip("/")
+                    filelist.append(os.path.join(relativepath, d))
+                else:
+                    raise "basepath '%s' not found in path '%s'" % (basepath, totalpath)
+
     return filelist
 
 
@@ -92,7 +97,7 @@
                     badrpms.append(file)
     for file in badrpms:
         if file in rpms:
-            rpms.remove(file)            
+            rpms.remove(file)
     # print 'Post-Trim Len: %d' % len(rpms)
     return rpms
 
@@ -136,20 +141,21 @@
     cmds['pretty'] = 0
 #    cmds['updategroupsonly'] = 0
     cmds['cachedir'] = None
+    cmds['basedir'] = os.getcwd()
     cmds['cache'] = False
     cmds['file-pattern-match'] = ['.*bin\/.*', '^\/etc\/.*', '^\/usr\/lib\/sendmail$']
     cmds['dir-pattern-match'] = ['.*bin\/.*', '^\/etc\/.*']
 
     try:
-        gopts, argsleft = getopt.getopt(args, 'phqVvg:s:x:u:c:', ['help', 'exclude=', 
-                                            'quiet', 'verbose', 'cachedir=',
-                                            'baseurl=', 'groupfile=', 'checksum=',
-                                            'version', 'pretty'])
+        gopts, argsleft = getopt.getopt(args, 'phqVvg:s:x:u:c:', ['help', 'exclude=',
+                                                                  'quiet', 'verbose', 'cachedir=', 'basedir=',
+                                                                  'baseurl=', 'groupfile=', 'checksum=',
+                                                                  'version', 'pretty'])
     except getopt.error, e:
         errorprint(_('Options Error: %s.') % e)
         usage()
 
-    try: 
+    try:
         for arg,a in gopts:
             if arg in ['-h','--help']:
                 usage(retval=0)
@@ -159,7 +165,7 @@
     except ValueError, e:
         errorprint(_('Options Error: %s') % e)
         usage()
-    
+
 
     # make sure our dir makes sense before we continue
     if len(argsleft) > 1:
@@ -170,8 +176,8 @@
         usage()
     else:
         directory = argsleft[0]
-   
-    try: 
+
+    try:
         for arg,a in gopts:
             if arg in ['-v', '--verbose']:
                 cmds['verbose'] = 1
@@ -188,7 +194,7 @@
                     errorprint(_('Error: Only one groupfile allowed.'))
                     usage()
                 else:
-                    if os.path.exists(directory + '/' + a):
+                    if os.path.exists(a):
                         cmds['groupfile'] = a
                     else:
                         errorprint(_('Error: groupfile %s cannot be found.' % a))
@@ -209,12 +215,14 @@
                 if not checkAndMakeDir(a):
                     errorprint(_('Error: cannot open/write to cache dir %s' % a))
                     usage()
+            elif arg == '--basedir':
+                cmds['basedir'] = a
                     
     except ValueError, e:
         errorprint(_('Options Error: %s') % e)
         usage()
 
-        
+
     return cmds, directory
 
 def doPkgMetadata(cmds, ts):
@@ -222,20 +230,20 @@
 
     # rpms we're going to be dealing with
     files = []
-    files = getFileList('./', '.rpm', files)
+    files = getFileList(cmds['basedir'], ".", '.rpm', files)
     files = trimRpms(files, cmds['excludes'])
     pkgcount = len(files)
-    
+
     # setup the base metadata doc
     basedoc = libxml2.newDoc("1.0")
     baseroot =  basedoc.newChild(None, "metadata", None)
     basens = baseroot.newNs('http://linux.duke.edu/metadata/common', None)
     formatns = baseroot.newNs('http://linux.duke.edu/metadata/rpm', 'rpm')
     baseroot.setNs(basens)
-    basefilepath = os.path.join(cmds['tempdir'], cmds['primaryfile'])
+    basefilepath = os.path.join(cmds['basedir'], cmds['tempdir'], cmds['primaryfile'])
     basefile = _gzipOpen(basefilepath, 'w')
     basefile.write('<?xml version="1.0" encoding="UTF-8"?>\n')
-    basefile.write('<metadata xmlns="http://linux.duke.edu/metadata/common" xmlns:rpm="http://linux.duke.edu/metadata/rpm" packages="%s">\n' % 
+    basefile.write('<metadata xmlns="http://linux.duke.edu/metadata/common" xmlns:rpm="http://linux.duke.edu/metadata/rpm" packages="%s">\n' %
                    pkgcount)
 
     # setup the file list doc
@@ -243,30 +251,30 @@
     filesroot = filesdoc.newChild(None, "filelists", None)
     filesns = filesroot.newNs('http://linux.duke.edu/metadata/filelists', None)
     filesroot.setNs(filesns)
-    filelistpath = os.path.join(cmds['tempdir'], cmds['filelistsfile'])
-    flfile = _gzipOpen(filelistpath, 'w')    
+    filelistpath = os.path.join(cmds['basedir'], cmds['tempdir'], cmds['filelistsfile'])
+    flfile = _gzipOpen(filelistpath, 'w')
     flfile.write('<?xml version="1.0" encoding="UTF-8"?>\n')
-    flfile.write('<filelists xmlns="http://linux.duke.edu/metadata/filelists" packages="%s">\n' % 
+    flfile.write('<filelists xmlns="http://linux.duke.edu/metadata/filelists" packages="%s">\n' %
                    pkgcount)
-    
-    
+
+
     # setup the other doc
     otherdoc = libxml2.newDoc("1.0")
     otherroot = otherdoc.newChild(None, "otherdata", None)
     otherns = otherroot.newNs('http://linux.duke.edu/metadata/other', None)
     otherroot.setNs(otherns)
-    otherfilepath = os.path.join(cmds['tempdir'], cmds['otherfile'])
+    otherfilepath = os.path.join(cmds['basedir'], cmds['tempdir'], cmds['otherfile'])
     otherfile = _gzipOpen(otherfilepath, 'w')
     otherfile.write('<?xml version="1.0" encoding="UTF-8"?>\n')
-    otherfile.write('<otherdata xmlns="http://linux.duke.edu/metadata/other" packages="%s">\n' % 
+    otherfile.write('<otherdata xmlns="http://linux.duke.edu/metadata/other" packages="%s">\n' %
                    pkgcount)
-    
-    
+
+
     current = 0
     for file in files:
         current+=1
         try:
-            mdobj = dumpMetadata.RpmMetaData(ts, file, cmds)
+            mdobj = dumpMetadata.RpmMetaData(ts, cmds['basedir'], file, cmds)
             if not cmds['quiet']:
                 if cmds['verbose']:
                     print '%d/%d - %s' % (current, len(files), file)
@@ -317,23 +325,23 @@
                 node.freeNode()
                 del node
 
-        
+
     if not cmds['quiet']:
         print ''
-        
+
     # save them up to the tmp locations:
     if not cmds['quiet']:
         print _('Saving Primary metadata')
     basefile.write('\n</metadata>')
     basefile.close()
     basedoc.freeDoc()
-    
+
     if not cmds['quiet']:
         print _('Saving file lists metadata')
     flfile.write('\n</filelists>')
     flfile.close()
     filesdoc.freeDoc()
-    
+
     if not cmds['quiet']:
         print _('Saving other metadata')
     otherfile.write('\n</otherdata>')
@@ -346,27 +354,27 @@
     reporoot = repodoc.newChild(None, "repomd", None)
     repons = reporoot.newNs('http://linux.duke.edu/metadata/repo', None)
     reporoot.setNs(repons)
-    repofilepath = os.path.join(cmds['tempdir'], cmds['repomdfile'])
-    
+    repofilepath = os.path.join(cmds['basedir'], cmds['tempdir'], cmds['repomdfile'])
+
     try:
         dumpMetadata.repoXML(reporoot, cmds)
     except dumpMetadata.MDError, e:
         errorprint(_('Error generating repo xml file: %s') % e)
         sys.exit(1)
-        
-    try:        
+
+    try:
         repodoc.saveFormatFileEnc(repofilepath, 'UTF-8', 1)
     except:
         errorprint(_('Error saving temp file for rep xml: %s') % repofilepath)
         sys.exit(1)
-        
+
     del repodoc
-        
-   
+
+
 
 def main(args):
     cmds, directory = parseArgs(args)
-    
+
     #setup some defaults
     cmds['primaryfile'] = 'primary.xml.gz'
     cmds['filelistsfile'] = 'filelists.xml.gz'
@@ -375,108 +383,84 @@
     cmds['tempdir'] = '.repodata'
     cmds['finaldir'] = 'repodata'
     cmds['olddir'] = '.olddata'
-    
-    # save where we are right now
-    curdir = os.getcwd()
+
     # start the sanity/stupidity checks
-    if not os.path.exists(directory):
+    if not os.path.exists(os.path.join(cmds['basedir'], directory)):
         errorprint(_('Directory must exist'))
         sys.exit(1)
-        
-    if not os.path.isdir(directory):
+
+    if not os.path.isdir(os.path.join(cmds['basedir'], directory)):
         errorprint(_('Directory of packages must be a directory.'))
         sys.exit(1)
-        
-    if not os.access(directory, os.W_OK):
+
+    if not os.access(cmds['basedir'], os.W_OK):
         errorprint(_('Directory must be writable.'))
         sys.exit(1)
 
- 
-    if not checkAndMakeDir(os.path.join(directory, cmds['tempdir'])):
+
+    if not checkAndMakeDir(os.path.join(cmds['basedir'], cmds['tempdir'])):
         sys.exit(1)
-        
-    if not checkAndMakeDir(os.path.join(directory, cmds['finaldir'])):
+
+    if not checkAndMakeDir(os.path.join(cmds['basedir'], cmds['finaldir'])):
         sys.exit(1)
-        
-    if os.path.exists(os.path.join(directory, cmds['olddir'])):
+
+    if os.path.exists(os.path.join(cmds['basedir'], cmds['olddir'])):
         errorprint(_('Old data directory exists, please remove: %s') % cmds['olddir'])
         sys.exit(1)
-        
-    # change to the basedir to work from w/i the path - for relative url paths
-    os.chdir(directory)
 
     # make sure we can write to where we want to write to:
     for direc in ['tempdir', 'finaldir']:
         for file in ['primaryfile', 'filelistsfile', 'otherfile', 'repomdfile']:
-            filepath = os.path.join(cmds[direc], cmds[file])
+            filepath = os.path.join(cmds['basedir'], cmds[direc], cmds[file])
             if os.path.exists(filepath):
                 if not os.access(filepath, os.W_OK):
                     errorprint(_('error in must be able to write to metadata files:\n  -> %s') % filepath)
-                    os.chdir(curdir)
                     usage()
-                    
+
     ts = rpm.TransactionSet()
-    try:
-        doPkgMetadata(cmds, ts)
-    except:
-        # always clean up your messes
-        os.chdir(curdir)
-        raise
-    
-    try:
-        doRepoMetadata(cmds)
-    except:
-        os.chdir(curdir)
-        raise
-        
-    if os.path.exists(cmds['finaldir']):
+    doPkgMetadata(cmds, ts)
+    doRepoMetadata(cmds)
+
+    if os.path.exists(os.path.join(cmds['basedir'], cmds['finaldir'])):
         try:
-            os.rename(cmds['finaldir'], cmds['olddir'])
+            os.rename(os.path.join(cmds['basedir'], cmds['finaldir']),
+                      os.path.join(cmds['basedir'], cmds['olddir']))
         except:
-            errorprint(_('Error moving final to old dir'))
-            os.chdir(curdir)
+            errorprint(_('Error moving final %s to old dir %s' % (os.path.join(cmds['basedir'], cmds['finaldir']),
+                                                                  os.path.join(cmds['basedir'], cmds['olddir']))))
             sys.exit(1)
-        
+
     try:
-        os.rename(cmds['tempdir'], cmds['finaldir'])
+        os.rename(os.path.join(cmds['basedir'], cmds['tempdir']),
+                  os.path.join(cmds['basedir'], cmds['finaldir']))
     except:
         errorprint(_('Error moving final metadata into place'))
         # put the old stuff back
-        os.rename(cmds['olddir'], cmds['finaldir'])
-        os.chdir(curdir)
+        os.rename(os.path.join(cmds['basedir'], cmds['olddir']),
+                  os.path.join(cmds['basedir'], cmds['finaldir']))
         sys.exit(1)
-        
+
     for file in ['primaryfile', 'filelistsfile', 'otherfile', 'repomdfile', 'groupfile']:
         if cmds[file]:
             fn = os.path.basename(cmds[file])
         else:
             continue
-        oldfile = os.path.join(cmds['olddir'], fn)
+        oldfile = os.path.join(cmds['basedir'], cmds['olddir'], fn)
         if os.path.exists(oldfile):
             try:
                 os.remove(oldfile)
             except OSError, e:
                 errorprint(_('Could not remove old metadata file: %s') % oldfile)
                 errorprint(_('Error was %s') % e)
-                os.chdir(curdir)
                 sys.exit(1)
-            
+
     try:
-        os.rmdir(cmds['olddir'])
+        os.rmdir(os.path.join(cmds['basedir'], cmds['olddir']))
     except OSError, e:
         errorprint(_('Could not remove old metadata dir: %s') % cmds['olddir'])
         errorprint(_('Error was %s') % e)
         errorprint(_('Please clean up this directory manually.'))
-        os.chdir(curdir)
-        
-        
-        
-        
-    # take us home mr. data
-    os.chdir(curdir)
-        
 
-        
 if __name__ == "__main__":
     if len(sys.argv) > 1:
         if sys.argv[1] == 'profile':
Index: dumpMetadata.py
===================================================================
RCS file: /cvsroot/metadata/cvs-root/generate/dumpMetadata.py,v
retrieving revision 1.32
diff -u -r1.32 dumpMetadata.py
--- dumpMetadata.py	2 Nov 2005 20:23:56 -0000	1.32
+++ dumpMetadata.py	11 Nov 2005 18:30:05 -0000
@@ -212,19 +212,18 @@
     """each rpm is one object, you pass it an rpm file
        it opens the file, and pulls the information out in bite-sized chunks :)
     """
-    def __init__(self, ts, filename, options):
+    def __init__(self, ts, basedir, filename, options):
         try:
-            stats = os.stat(filename)
+            stats = os.stat(os.path.join(basedir, filename))
             self.size = stats[6]
             self.mtime = stats[8]
             del stats
         except OSError, e:
-            raise MDError, "Error Stat'ing file %s" % filename
-        
+            raise MDError, "Error Stat'ing file %s %s" % (basedir, filename)
         self.options = options
         self.localurl = options['baseurl']
         self.relativepath = filename
-        fd = returnFD(filename)
+        fd = returnFD(os.path.join(basedir, filename))
         self.hdr = returnHdr(ts, fd)
         os.lseek(fd, 0, 0)
         fo = os.fdopen(fd, 'rb')
@@ -258,7 +257,7 @@
             return 'src'
         else:
             return self.tagByName('arch')
-            
+
     def _correctFlags(self, flags):
         returnflags=[]
         if flags is None:
@@ -737,11 +736,11 @@
     
     
     for (file, ftype) in workfiles:
-        zfo = _gzipOpen(os.path.join(cmds['tempdir'], file))
+        zfo = _gzipOpen(os.path.join(cmds['basedir'], cmds['tempdir'], file))
         uncsum = getChecksum(sumtype, zfo)
         zfo.close()
-        csum = getChecksum(sumtype, os.path.join(cmds['tempdir'], file))
-        timestamp = os.stat(os.path.join(cmds['tempdir'], file))[8]
+        csum = getChecksum(sumtype, os.path.join(cmds['basedir'], cmds['tempdir'], file))
+        timestamp = os.stat(os.path.join(cmds['basedir'], cmds['tempdir'], file))[8]
         data = node.newChild(None, 'data', None)
         data.newProp('type', ftype)
         location = data.newChild(None, 'location', None)
@@ -760,7 +759,7 @@
         timestamp = os.stat(grpfile)[8]
         sfile = os.path.basename(grpfile)
         fo = open(grpfile, 'r')
-        output = open(os.path.join(cmds['tempdir'], sfile), 'w')
+        output = open(os.path.join(cmds['basedir'], cmds['tempdir'], sfile), 'w')
         output.write(fo.read())
         output.close()
         fo.seek(0)
lmpx.com only provides a reader for public news (NNTP) servers. It is not affiliated with the servers or forums shown here and is not responsible for the content of articles, which is written by their respective authors.