createrepo/__init__.py createrepo/utils.py createrepo/yumbased.py genpkgmetadata.py

 createrepo/__init__.py |  169 +++++++++++++++++++++++++++++++++++++++++--------
 createrepo/utils.py    |    6 -
 createrepo/yumbased.py |   15 ++--
 genpkgmetadata.py      |  147 ++++--------------------------------------
 4 files changed, 170 insertions(+), 167 deletions(-)

New commits:
commit 77d30d7581dea9adfa5966088eddcf48efacb5c5
Author: Seth Vidal <[email protected]>
Date:   Wed Jan 16 12:12:44 2008 -0500

    clean up api to simplify all of the code calling it
    probably severely break --split for the moment, though

diff --git a/createrepo/__init__.py b/createrepo/__init__.py
index 0421e1e..210d6fe 100644
--- a/createrepo/__init__.py
+++ b/createrepo/__init__.py
@@ -37,7 +37,7 @@ except ImportError:
     pass
 
 
-from utils import _gzipOpen, bzipFile
+from utils import _gzipOpen, bzipFile, checkAndMakeDir
 
 
 __version__ = '0.9.1'
@@ -98,10 +98,7 @@ class SimpleMDCallBack(object):
         sys.stdout.write("\r%d/%d - %s" % (current, total, item))
         sys.stdout.flush()
             
-#FIXME = make it so you pass in a dir to doPkgMetadata() and it
-# parses out basedir and directory for relative dir from there
-# it creates the .repodata directory in the output location, etc
-        
+      
 class MetaDataGenerator:
     def __init__(self, config_obj=None, callback=None):
         self.conf = config_obj
@@ -116,17 +113,69 @@ class MetaDataGenerator:
         self.ts = rpmUtils.transaction.initReadOnlyTransaction()
         self.pkgcount = 0
         self.files = []
+        
+        if not self.conf.directory and not self.conf.directories:
+            raise MDError, "No directory given on which to run."
+        
+        # this does the dir setup we need done
+        self._parse_directory(self.conf.directory)
+        self._test_setup_dirs()        
 
-    def _setup_and_check_repo_dir(self, direc):
+    def _parse_directory(self, direc):
         if os.path.isabs(direc):
             self.conf.basedir = os.path.dirname(direc)
-            self.conf.directory = os.path.basename(direc)
+            self.conf.relative_dir = os.path.basename(direc)
         else:
             self.conf.basedir = os.path.realpath(self.conf.basedir)
+            self.conf.relative_dir = direc
+
+        self.package_dir = os.path.join(self.conf.basedir, self.conf.relative_dir)
+        
+        if not self.conf.outputdir:
+            self.conf.outputdir = os.path.join(self.conf.basedir, self.conf.relative_dir)
+
+
+    def _test_setup_dirs(self):
+        testdir = os.path.realpath(os.path.join(self.conf.basedir, self.conf.relative_dir))
+        # start the sanity/stupidity checks
+        if not os.path.exists(testdir):
+            raise MDError, _('Directory %s must exist') % self.conf.directory
+
+        if not os.path.isdir(testdir):
+            raise MDError, _('%s must be a directory') % self.conf.directory
+
+        if not os.access(self.conf.outputdir, os.W_OK):
+            raise MDError, _('Directory %s must be writable.') % self.conf.outputdir
+
+# SPLIT STUFF?
+#        if self.conf.split:
+#            oldbase = self.conf.basedir
+#            self.conf.basedir = os.path.join(self.conf.basedir, self.conf.directory)
 
-        if not self.conf.opts.outputdir:
-            self.conf.outputdir = os.path.join(self.conf.basedir, direc)
+        temp_output = os.path.join(self.conf.outputdir, self.conf.tempdir)
+        if not checkAndMakeDir(temp_output):
+            raise MDError, _('Cannot create/verify %s') % temp_output
 
+        temp_final = os.path.join(self.conf.outputdir, self.conf.finaldir)
+        if not checkAndMakeDir(temp_final):
+            raise MDError, _('Cannot create/verify %s') % temp_final
+
+        if os.path.exists(os.path.join(self.conf.outputdir, self.conf.olddir)):
+            raise MDError, _('Old data directory exists, please remove: %s') % self.conf.olddir
+
+        # make sure we can write to where we want to write to:
+        # and pickup the mdtimestamps while we're at it
+        for direc in ['tempdir', 'finaldir']:
+            for f in ['primaryfile', 'filelistsfile', 'otherfile', 'repomdfile']:
+                filepath = os.path.join(self.conf.outputdir, direc, f)
+                if os.path.exists(filepath):
+                    if not os.access(filepath, os.W_OK):
+                        raise MDError, _('error in must be able to write to metadata files:\n  -> %s') % filepath
+
+                    if conf.checkts:
+                        timestamp = os.path.getctime(filepath)
+                        if timestamp > self.conf.mdtimestamp:
+                            self.conf.mdtimestamp = timestamp
 
     def _os_path_walk(self, top, func, arg):
         """Directory tree walk with callback function.
@@ -143,7 +192,7 @@ class MetaDataGenerator:
             if os.path.isdir(name):
                 self._os_path_walk(name, func, arg)
     # module
-    def getFileList(self, basepath, directory, ext):
+    def getFileList(self, directory, ext):
         """Return all files in path matching ext, store them in filelist,
         recurse dirs. Returns a list object"""
 
@@ -161,7 +210,7 @@ class MetaDataGenerator:
                     filelist.append(os.path.join(relativepath,fn))
 
         filelist = []
-        startdir = os.path.join(basepath, directory) + '/'
+        startdir = directory + '/'
         self._os_path_walk(startdir, extension_visitor, filelist)
         return filelist
 
@@ -198,11 +247,9 @@ class MetaDataGenerator:
                 files.remove(file)
         return files
 
-    def doPkgMetadata(self, directory=None):
+    def doPkgMetadata(self):
         """all the heavy lifting for the package metadata"""
-        if not directory:
-            directory = self.conf.directory
-            
+        
         # rpms we're going to be dealing with
         if self.conf.update:
             #build the paths
@@ -211,7 +258,7 @@ class MetaDataGenerator:
             otherfile = os.path.join(self.conf.outputdir, self.conf.finaldir, self.conf.otherfile)
             opts = {
                 'verbose' : self.conf.verbose,
-                'pkgdir' : os.path.normpath(os.path.join(self.conf.basedir, directory))
+                'pkgdir' : os.path.normpath(self.package_dir)
             }
             if self.conf.skip_stat:
                 opts['do_stat'] = False
@@ -222,12 +269,12 @@ class MetaDataGenerator:
         if self.conf.pkglist:
             packages = self.conf.pkglist
         else:
-            packages = self.getFileList(self.conf.basedir, directory, '.rpm')
+            packages = self.getFileList(self.package_dir, '.rpm')
             
         packages = self.trimRpms(packages)
         self.pkgcount = len(packages)
         self.openMetadataDocs()
-        self.writeMetadataDocs(packages, directory)
+        self.writeMetadataDocs(packages)
         self.closeMetadataDocs()
 
     # module
@@ -264,19 +311,23 @@ class MetaDataGenerator:
         return fo
         
 
-    def read_in_package(self, directory, rpmfile):
-        # directory is stupid - just make it part of the class
-        rpmfile = '%s/%s/%s' % (self.conf.basedir, directory, rpmfile)
+    def read_in_package(self, rpmfile):
+        """rpmfile == relative path to file from self.packge_dir"""
+        
+        rpmfile = '%s/%s' % (self.package_dir, rpmfile)
         try:
             po = yumbased.CreateRepoPackage(self.ts, rpmfile)
         except Errors.MiscError, e:
             raise MDError, "Unable to open package: %s" % e
         return po
 
-    def writeMetadataDocs(self, pkglist, directory, current=0):
-        # FIXME
-        # directory is unused, kill it, pkglist should come from self
-        # I don't see why current needs to be this way at all
+    def writeMetadataDocs(self, pkglist=[], directory=None, current=0):
+
+        if not pkglist:
+            pkglist = self.conf.pkglist           
+        if not directory:
+            directory=self.conf.directory
+
         for pkg in pkglist:
             current+=1
             recycled = False
@@ -294,7 +345,7 @@ class MetaDataGenerator:
             if not recycled:
                 #scan rpm files
                 try:
-                    po = self.read_in_package(directory, pkg)
+                    po = self.read_in_package(pkg)
                 except MDError, e:
                     # need to say something here
                     self.callback.errorlog("\nError %s: %s\n" % (pkg, e))
@@ -486,6 +537,72 @@ class MetaDataGenerator:
 
         del repodoc
 
+
+    def doFinalMove(self):
+        """move the just-created repodata from .repodata to repodata
+           also make sure to preserve any files we didn't mess with in the 
+           metadata dir"""
+           
+        output_final_dir = os.path.join(self.conf.outputdir, self.conf.finaldir) 
+        output_old_dir = os.path.join(self.conf.outputdir, self.conf.olddir)
+        
+        if os.path.exists(output_final_dir):
+            try:
+                os.rename(output_final_dir, output_old_dir)
+            except:
+                raise MDError, _('Error moving final %s to old dir %s' % (output_final_dir,
+                                                                     output_old_dir))
+
+        output_temp_dir = os.path.join(self.conf.outputdir, self.conf.tempdir)
+
+        try:
+            os.rename(output_temp_dir, output_final_dir)
+        except:
+            # put the old stuff back
+            os.rename(output_old_dir, output_final_dir)
+            raise MDError, _('Error moving final metadata into place')
+
+        for f in ['primaryfile', 'filelistsfile', 'otherfile', 'repomdfile', 'groupfile']:
+            if getattr(self.conf, f):
+                fn = os.path.basename(getattr(self.conf, f))
+            else:
+                continue
+            oldfile = os.path.join(output_old_dir, fn)
+
+            if os.path.exists(oldfile):
+                try:
+                    os.remove(oldfile)
+                except OSError, e:
+                    raise MDError, _('Could not remove old metadata file: %s: %s') % (oldfile, e)
+
+        # Move everything else back from olddir (eg. repoview files)
+        for f in os.listdir(output_old_dir):
+            oldfile = os.path.join(output_old_dir, f)
+            finalfile = os.path.join(output_final_dir, f)
+            if os.path.exists(finalfile):
+                # Hmph?  Just leave it alone, then.
+                try:
+                    if os.path.isdir(oldfile):
+                        shutil.rmtree(oldfile)
+                    else:
+                        os.remove(oldfile)
+                except OSError, e:
+                    raise MDError, _('Could not remove old metadata file: %s: %s') % (oldfile, e)
+            else:
+                try:
+                    os.rename(oldfile, finalfile)
+                except OSError, e:
+                    msg = _('Could not restore old non-metadata file: %s -> %s') % (oldfile, finalfile)
+                    msg += _('Error was %s') % e
+                    raise MDError, msg
+
+        try:
+            os.rmdir(output_old_dir)
+        except OSError, e:
+            self.errorlog(_('Could not remove old metadata dir: %s') % self.conf.olddir)
+            self.errorlog(_('Error was %s') % e)
+            self.errorlog(_('Please clean up this directory manually.'))
+
 class SplitMetaDataGenerator(MetaDataGenerator):
 
     def __init__(self, config_obj=None, callback=None):
diff --git a/createrepo/utils.py b/createrepo/utils.py
index 51e56cd..13d1d67 100644
--- a/createrepo/utils.py
+++ b/createrepo/utils.py
@@ -104,11 +104,11 @@ def checkAndMakeDir(dir):
     """
     if os.path.exists(dir):
         if not os.path.isdir(dir):
-            errorprint(_('%s is not a dir') % dir)
+            #errorprint(_('%s is not a dir') % dir)
             result = False
         else:
             if not os.access(dir, os.W_OK):
-                errorprint(_('%s is not writable') % dir)
+                #errorprint(_('%s is not writable') % dir)
                 result = False
             else:
                 result = True
@@ -116,7 +116,7 @@ def checkAndMakeDir(dir):
         try:
             os.mkdir(dir)
         except OSError, e:
-            errorprint(_('Error creating dir %s: %s') % (dir, e))
+            #errorprint(_('Error creating dir %s: %s') % (dir, e))
             result = False
         else:
             result = True
diff --git a/createrepo/yumbased.py b/createrepo/yumbased.py
index 88fcb0c..e705f6e 100644
--- a/createrepo/yumbased.py
+++ b/createrepo/yumbased.py
@@ -64,6 +64,7 @@ class CreateRepoPackage(YumLocalPackage):
         
     def _xml(self, item):
         item = utils.utf8String(item)
+        item = item.rstrip()
         return xml.sax.saxutils.escape(item)
         
     def _do_checksum(self):
@@ -154,7 +155,7 @@ class CreateRepoPackage(YumLocalPackage):
   <packager>%s</packager>
   <url>%s</url>
   <time file="%s" build="%s"/>
-  <size package="%s" installed="%s" archive="%s"/>""" % (self.name, 
+  <size package="%s" installed="%s" archive="%s"/>\n""" % (self.name, 
          self.arch, self.epoch, self.ver, self.rel, self.checksum, 
          self._xml(self.summary), self._xml(self.description), packager, 
          url, self.filetime, self.buildtime, self.packagesize, self.size, 
@@ -162,9 +163,9 @@ class CreateRepoPackage(YumLocalPackage):
          
 
         if baseurl:
-            msg += """<location xml:base="%s" href="%s"/>""" % (self._xml(baseurl), relpath)
+            msg += """<location xml:base="%s" href="%s"/>\n""" % (self._xml(baseurl), relpath)
         else:
-            msg += """<location href="%s"/>""" % relpath
+            msg += """<location href="%s"/>\n""" % relpath
             
         return msg
 
@@ -173,22 +174,22 @@ class CreateRepoPackage(YumLocalPackage):
         if self.license:
             msg += """    <rpm:license>%s</rpm:license>\n""" % self._xml(self.license)
         else:
-            msg += """    <rpm:license/>"""
+            msg += """    <rpm:license/>\n"""
             
         if self.vendor:
             msg += """    <rpm:vendor>%s</rpm:vendor>\n""" % self._xml(self.vendor)
         else:
-            msg += """    <rpm:vendor/>"""
+            msg += """    <rpm:vendor/>\n"""
             
         if self.group:
             msg += """    <rpm:group>%s</rpm:group>\n""" % self._xml(self.group)
         else:
-            msg += """    <rpm:group/>"""
+            msg += """    <rpm:group/>\n"""
             
         if self.buildhost:
             msg += """    <rpm:buildhost>%s</rpm:buildhost>\n""" % self._xml(self.buildhost)
         else:
-            msg += """    <rpm:buildhost/>"""
+            msg += """    <rpm:buildhost/>\n"""
             
         if self.sourcerpm:
             msg += """    <rpm:sourcerpm>%s</rpm:sourcerpm>\n""" % self._xml(self.sourcerpm)
diff --git a/genpkgmetadata.py b/genpkgmetadata.py
index 38cd151..0449258 100755
--- a/genpkgmetadata.py
+++ b/genpkgmetadata.py
@@ -106,16 +106,10 @@ def parseArgs(args, conf):
         setattr(conf, opt.dest, getattr(opts, opt.dest))
     
     directory = directories[0]
-    directory = os.path.normpath(directory)
-    if conf.split:
-        pass
-    elif os.path.isabs(directory):
-        conf.basedir = os.path.dirname(directory)
-        directory = os.path.basename(directory)
-    else:
-        conf.basedir = os.path.realpath(conf.basedir)
+    conf.directory = directory
+    conf.directories = directories
 
-   
+    # FIXME - I think this is unnecessary
     if not opts.outputdir:
         conf.outputdir = os.path.join(conf.basedir, directory)
     if conf.groupfile:
@@ -124,10 +118,12 @@ def parseArgs(args, conf):
             a = os.path.join(conf.basedir, directory, conf.groupfile)
         elif not os.path.isabs(a):
             a = os.path.join(conf.basedir, directory, conf.groupfile)
+        # FIXME, move this test most likely inside the class
         if not os.path.exists(a):
             errorprint(_('Error: groupfile %s cannot be found.' % a))
             usage()
         conf.groupfile = a
+    # FIXME - move this one inside the class, too
     if conf.cachedir:
         conf.cache = True
         a = conf.cachedir
@@ -148,12 +144,6 @@ def parseArgs(args, conf):
             
         conf.pkglist = lst
         
-    #setup some defaults
-
-    # Fixup first directory
-    directories[0] = directory
-    conf.directory = directory
-    conf.directories = directories
 
     return conf
 
@@ -172,130 +162,25 @@ class MDCallBack(object):
 def main(args):
     conf = createrepo.MetaDataConfig()
     conf = parseArgs(args, conf)
-    # FIXME - some of these should be moved into the module and out of the cli routines        
-    testdir = os.path.realpath(os.path.join(conf.basedir, conf.directory))
-    # start the sanity/stupidity checks
-    if not os.path.exists(testdir):
-        errorprint(_('Directory %s must exist') % (conf.directory,))
-        sys.exit(1)
 
-    if not os.path.isdir(testdir):
-        errorprint(_('%s - must be a directory') 
-                   % (conf.directory,))
-        sys.exit(1)
-
-    if not os.access(conf.outputdir, os.W_OK):
-        errorprint(_('Directory %s must be writable.') % (conf.outputdir,))
-        sys.exit(1)
-
-    if conf.split:
-        oldbase = conf.basedir
-        conf.basedir = os.path.join(conf.basedir, conf.directory)
-    if not checkAndMakeDir(os.path.join(conf.outputdir, conf.tempdir)):
-        sys.exit(1)
-
-    if not checkAndMakeDir(os.path.join(conf.outputdir, conf.finaldir)):
-        sys.exit(1)
-
-    if os.path.exists(os.path.join(conf.outputdir, conf.olddir)):
-        errorprint(_('Old data directory exists, please remove: %s') % conf.olddir)
-        sys.exit(1)
-
-    # make sure we can write to where we want to write to:
-    for direc in ['tempdir', 'finaldir']:
-        for f in ['primaryfile', 'filelistsfile', 'otherfile', 'repomdfile']:
-            filepath = os.path.join(conf.outputdir, direc, f)
-            if os.path.exists(filepath):
-                if not os.access(filepath, os.W_OK):
-                    errorprint(_('error in must be able to write to metadata files:\n  -> %s') % filepath)
-                    usage()
-                if conf.checkts:
-                    timestamp = os.path.getctime(filepath)
-                    if timestamp > conf.mdtimestamp:
-                        conf.mdtimestamp = timestamp
-        
-    if conf.split:
-        conf.basedir = oldbase
-        mdgen = createrepo.SplitMetaDataGenerator(config_obj=conf, callback=MDCallBack())
-    else:
-        mdgen = createrepo.MetaDataGenerator(config_obj=conf, callback=MDCallBack())
-        if mdgen.checkTimeStamps():
-            if mdgen.conf.verbose:
-                print _('repo is up to date')
-            sys.exit(0)
     try:
+        if conf.split:
+            mdgen = createrepo.SplitMetaDataGenerator(config_obj=conf, callback=MDCallBack())
+        else:
+            mdgen = createrepo.MetaDataGenerator(config_obj=conf, callback=MDCallBack())
+            if mdgen.checkTimeStamps():
+                if mdgen.conf.verbose:
+                    print _('repo is up to date')
+                sys.exit(0)
+
         mdgen.doPkgMetadata()
         mdgen.doRepoMetadata()
+        mdgen.doFinalMove()
+        
     except MDError, e:
         errorprint(_('%s') % e)
         sys.exit(1)
 
-    output_final_dir = os.path.join(mdgen.conf.outputdir, mdgen.conf.finaldir) 
-    output_old_dir = os.path.join(mdgen.conf.outputdir, mdgen.conf.olddir)
-    
-    if os.path.exists(output_final_dir):
-        try:
-            os.rename(output_final_dir, output_old_dir)
-        except:
-            errorprint(_('Error moving final %s to old dir %s' % (output_final_dir,
-                                                                 output_old_dir)))
-            sys.exit(1)
-
-    output_temp_dir =os.path.join(mdgen.conf.outputdir, mdgen.conf.tempdir)
-
-    try:
-        os.rename(output_temp_dir, output_final_dir)
-    except:
-        errorprint(_('Error moving final metadata into place'))
-        # put the old stuff back
-        os.rename(output_old_dir, output_final_dir)
-        sys.exit(1)
-
-    for f in ['primaryfile', 'filelistsfile', 'otherfile', 'repomdfile', 'groupfile']:
-        if getattr(mdgen.conf, f):
-            fn = os.path.basename(getattr(mdgen.conf, f))
-        else:
-            continue
-        oldfile = os.path.join(output_old_dir, fn)
-
-        if os.path.exists(oldfile):
-            try:
-                os.remove(oldfile)
-            except OSError, e:
-                errorprint(_('Could not remove old metadata file: %s') % oldfile)
-                errorprint(_('Error was %s') % e)
-                sys.exit(1)
-
-    # Move everything else back from olddir (eg. repoview files)
-    for f in os.listdir(output_old_dir):
-        oldfile = os.path.join(output_old_dir, f)
-        finalfile = os.path.join(output_final_dir, f)
-        if os.path.exists(finalfile):
-            # Hmph?  Just leave it alone, then.
-            try:
-                if os.path.isdir(oldfile):
-                    shutil.rmtree(oldfile)
-                else:
-                    os.remove(oldfile)
-            except OSError, e:
-                errorprint(_('Could not remove old non-metadata file: %s') % oldfile)
-                errorprint(_('Error was %s') % e)
-                sys.exit(1)
-        else:
-            try:
-                os.rename(oldfile, finalfile)
-            except OSError, e:
-                errorprint(_('Could not restore old non-metadata file: %s -> %s') % (oldfile, finalfile))
-                errorprint(_('Error was %s') % e)
-                sys.exit(1)
-
-
-    try:
-        os.rmdir(output_old_dir)
-    except OSError, e:
-        errorprint(_('Could not remove old metadata dir: %s') % mdgen.conf.olddir)
-        errorprint(_('Error was %s') % e)
-        errorprint(_('Please clean up this directory manually.'))
 
 if __name__ == "__main__":
     if len(sys.argv) > 1: