createrepo/__init__.py createrepo/yumbased.py genpkgmetadata.py

 createrepo/__init__.py |   81 ++++++++++++++++++++++++++++++++++---------------
 createrepo/yumbased.py |   18 ++++------
 genpkgmetadata.py      |    5 ++-
 3 files changed, 68 insertions(+), 36 deletions(-)

New commits:
commit bb9a6387107b3ecd356ab995079735fe15bc8ad5
Author: Seth Vidal <[email protected]>
Date:   Wed Feb 20 15:22:49 2008 -0500

    allow --pkglist or self.conf.pkglist to be a list of arbitrary
    urls to packages. createrepo will download the remote urls to
    a tempdir, read them in and add them to the metadata.

diff --git a/createrepo/__init__.py b/createrepo/__init__.py
index 2e5fa6e..e91cf85 100644
--- a/createrepo/__init__.py
+++ b/createrepo/__init__.py
@@ -21,6 +21,8 @@ import fnmatch
 import time
 import yumbased
 import shutil
+from urlgrabber import grabber
+import tempfile
 
 from yum import misc, Errors
 import rpmUtils.transaction
@@ -72,6 +74,7 @@ class MetaDataConfig(object):
         self.directories = []
         self.changelog_limit = None # needs to be an int or None
         self.unique_md_filenames = False
+
         
 
 class SimpleMDCallBack(object):
@@ -278,8 +281,6 @@ class MetaDataGenerator:
         # function for the first dir passed to --split, not all of them
         # this needs to be fixed by some magic in readMetadata.py
         # using opts.pkgdirs as a list, I think.
-        # FIXME - this needs to read the old repomd.xml to figure out where
-        # the files WERE to pass in the right fns.
         if self.conf.update:
             #build the paths
             opts = {
@@ -288,10 +289,19 @@ class MetaDataGenerator:
             }
             if self.conf.skip_stat:
                 opts['do_stat'] = False
-                
+
             #and scan the old repo
             self.oldData = readMetadata.MetadataIndex(self.conf.outputdir, opts)
-           
+
+    def _setup_grabber(self):
+        if not hasattr(self, '_grabber'):
+            self._grabber = grabber.URLGrabber()
+    
+        return self._grabber
+
+    grabber = property(fget = lambda self: self._setup_grabber())
+    
+    
     def doPkgMetadata(self):
         """all the heavy lifting for the package metadata"""
         if self.conf.update:
@@ -342,18 +352,38 @@ class MetaDataGenerator:
         return fo
         
 
-    def read_in_package(self, rpmfile, pkgpath=None):
+    def read_in_package(self, rpmfile, pkgpath=None, reldir=None):
         """rpmfile == relative path to file from self.packge_dir"""
-        # TODO/FIXME
-        # consider adding a routine to download the package from a remote location
-        # to a tempdir, operate on it, then use that location as a the baseurl
-        # for the package. That would make it possible to have repos entirely 
-        # comprised of remote packages.
+        remote_package = False
+        baseurl = self.conf.baseurl
 
         if not pkgpath:
             pkgpath = self.package_dir
 
-        rpmfile = '%s/%s' % (pkgpath, rpmfile)
+        if not rpmfile.strip():
+            raise MDError, "Blank filename passed in, skipping"
+            
+        if rpmfile.find("://") != -1:
+            remote_package = True
+            
+            if not hasattr(self, 'tempdir'):
+                self.tempdir = tempfile.mkdtemp()
+                
+            pkgname = os.path.basename(rpmfile)
+            baseurl = os.path.dirname(rpmfile)
+            reldir = self.tempdir       
+            dest = os.path.join(self.tempdir, pkgname)
+            if not self.conf.quiet:
+                self.callback.log('\nDownloading %s' % rpmfile)                        
+            try:
+                rpmfile = self.grabber.urlgrab(rpmfile, dest)
+            except grabber.URLGrabError, e:
+                raise MDError, "Unable to retrieve remote package %s: %s" %(rpmfile, e)
+
+            
+        else:
+            rpmfile = '%s/%s' % (pkgpath, rpmfile)
+            
         try:
             po = yumbased.CreateRepoPackage(self.ts, rpmfile)
         except Errors.MiscError, e:
@@ -362,11 +392,8 @@ class MetaDataGenerator:
         # you can do it
         po.crp_changelog_limit = self.conf.changelog_limit
         po.crp_cachedir = self.conf.cachedir
-
-        # FIXME if we wanted to put in a baseurl-per-package here is where 
-        # we should do it
-        # it would be easy to have a lookup dict in the MetaDataConfig object
-        # and work down from there for the baseurl
+        po.crp_baseurl = baseurl
+        po.crp_reldir = reldir
 
         if po.checksum in (None, ""):
             raise MDError, "No Package ID found for package %s, not going to add it" % e
@@ -377,6 +404,7 @@ class MetaDataGenerator:
 
         if not pkglist:
             pkglist = self.conf.pkglist           
+
         if not pkgpath:
             directory=self.conf.directory
         else:
@@ -385,13 +413,16 @@ class MetaDataGenerator:
         for pkg in pkglist:
             current+=1
             recycled = False
-
+            
             # look to see if we can get the data from the old repodata
             # if so write this one out that way
             if self.conf.update:
                 #see if we can pull the nodes from the old repo
                 #print self.oldData.basenodes.keys()
-                nodes = self.oldData.getNodes(pkg)
+                old_pkg = pkg
+                if pkg.find("://") != -1:
+                    old_pkg = os.path.basename(pkg)
+                nodes = self.oldData.getNodes(old_pkg)
                 if nodes is not None:
                     recycled = True
 
@@ -399,17 +430,19 @@ class MetaDataGenerator:
             # otherwise do it individually
             if not recycled:
                 #scan rpm files
+                if not pkgpath:   
+                    reldir = os.path.join(self.conf.basedir, directory)
+                else:
+                    reldir = pkgpath
+
                 try:
-                    po = self.read_in_package(pkg, pkgpath=pkgpath)
+                    po = self.read_in_package(pkg, pkgpath=pkgpath, reldir=reldir)
                 except MDError, e:
                     # need to say something here
                     self.callback.errorlog("\nError %s: %s\n" % (pkg, e))
                     continue
-                if not pkgpath:
-                    reldir = os.path.join(self.conf.basedir, directory)
-                else:
-                    reldir = pkgpath
-                self.primaryfile.write(po.do_primary_xml_dump(reldir, baseurl=self.conf.baseurl))
+
+                self.primaryfile.write(po.do_primary_xml_dump())
                 self.flfile.write(po.do_filelists_xml_dump())
                 self.otherfile.write(po.do_other_xml_dump())
             else:
diff --git a/createrepo/yumbased.py b/createrepo/yumbased.py
index 8f424fd..17af4e6 100644
--- a/createrepo/yumbased.py
+++ b/createrepo/yumbased.py
@@ -167,15 +167,11 @@ class CreateRepoPackage(YumLocalPackage):
     hdrend = property(fget=lambda self: self._get_header_byte_range()[1])
     hdrstart = property(fget=lambda self: self._get_header_byte_range()[0])
     
-    def _dump_base_items(self, basedir, baseurl=None):
-        """Takes an optional baseurl and required basedir.
-           basedir is the relative path to remove from the location
-           baseurl is whether or not this package already has a
-           baseurl rather than just '.'"""
+    def _dump_base_items(self):
         
         # if we start seeing fullpaths in the location tag - this is the culprit
-        if self.localpath.startswith(basedir):
-            relpath = self.localpath.replace(basedir, '')
+        if self.crp_reldir and self.localpath.startswith(self.crp_reldir):
+            relpath = self.localpath.replace(self.crp_reldir, '')
             if relpath[0] == '/': relpath = relpath[1:]
         else:
             relpath = self.localpath
@@ -204,8 +200,8 @@ class CreateRepoPackage(YumLocalPackage):
          self.archivesize)
          
 
-        if baseurl:
-            msg += """<location xml:base="%s" href="%s"/>\n""" % (self._xml(baseurl), relpath)
+        if self.crp_baseurl:
+            msg += """<location xml:base="%s" href="%s"/>\n""" % (self._xml(self.crp_baseurl), relpath)
         else:
             msg += """<location href="%s"/>\n""" % relpath
             
@@ -379,9 +375,9 @@ class CreateRepoPackage(YumLocalPackage):
             del c
         return msg                                                 
 
-    def do_primary_xml_dump(self, basedir, baseurl=None):
+    def do_primary_xml_dump(self):
         msg = """\n<package type="rpm">"""
-        msg += self._dump_base_items(basedir, baseurl)
+        msg += self._dump_base_items()
         msg += self._dump_format_items()
         msg += """\n</package>"""
         return msg
diff --git a/genpkgmetadata.py b/genpkgmetadata.py
index ed369da..e9de6b3 100755
--- a/genpkgmetadata.py
+++ b/genpkgmetadata.py
@@ -20,6 +20,7 @@
 
 import os
 import sys
+import re
 from optparse import OptionParser
 
 import createrepo
@@ -118,7 +119,9 @@ def parseArgs(args, conf):
     if conf.pkglist:
         pfo = open(conf.pkglist, 'r')
         for line in pfo.readlines():
-            line = line.replace('\n', '')
+            line = line.strip()
+            if re.match('^\s*\#.*', line) or re.match('^\s*$', line):
+                continue
             lst.append(line)
         pfo.close()