Re: [PATCH 4 of 4] hard link packages between local caches

--- reposync.py	2007-06-05 14:04:02.000000000 -0400
+++ /home/berrange/usr/bin/reposync	2007-07-15 13:37:23.000000000 -0400
@@ -47,6 +47,7 @@
 import rpmUtils.arch
 import logging
 from urlgrabber.progress import TextMeter
+from urlgrabber.grabber import URLGrabError
 
 # for yum 2.4.X compat
 def sortPkgObj(pkg1 ,pkg2):
@@ -103,10 +104,14 @@
         help="Use a temp dir for storing/accessing yum-cache")
     parser.add_option("-d", "--delete", default=False, action="store_true",
         help="delete local packages no longer present in repository")
+    parser.add_option("-k", "--package-cache", default=[], dest='pkgcache', action='append',
+        help="additional directory to search for pre-existing packages")
     parser.add_option("-p", "--download_path", dest='destdir', 
         default=os.getcwd(), help="Path to download packages to: defaults to current dir")
     parser.add_option("-g", "--gpgcheck", default=False, action="store_true",
         help="Remove packages that fail GPG signature checking after downloading")
+    parser.add_option("-s", "--checksum", default=False, action="store_true",
+        help="Checksum local downloaded packages to verify integrity")
     parser.add_option("-u", "--urls", default=False, action="store_true", 
         help="Just list urls of what would be downloaded, don't download")
     parser.add_option("-n", "--newest-only", dest='newest', default=False, action="store_true", 
@@ -118,6 +123,36 @@
     return (opts, args)
 
 
+def checkPkg(opts, my, pkg, local):
+    # Size check first since it is fast
+    if os.path.getsize(local) != int(pkg.returnSimple('packagesize')):
+        return False, "size difference"
+
+    # Next up a checksum
+    if opts.checksum:
+        try:
+            sums = pkg.returnChecksums()
+            if len(sums) > 0:
+                my.verifyChecksum(local, sums[0][0], sums[0][1])
+        except URLGrabError, e:
+            return False, "failed checksum"
+
+    # Finally a gpg check
+    if opts.gpgcheck:
+        # Hack to make sig check go against our localpath
+        pkg.localpath = local
+        result, error = my.sigCheckPkg(pkg)
+        if result != 0:
+            if result == 1:
+                return False, "missing GPG key"
+            elif result == 2:
+                return False, "failed signature"
+            else:
+                return False, "failed signature %s" % error
+
+    return True, ""
+
+
 def main():
     (opts, junk) = parseArgs()
     
@@ -135,6 +170,14 @@
     my = RepoSync(opts=opts)
     my.doConfigSetup(fn=opts.config, init_plugins=False)
 
+    # Populate cache of existing download RPMs from other
+    # repositories we can link to
+    pkgcache = {}
+    for dir in opts.pkgcache:
+        cache = localpkgs(dir)
+        for k in cache.keys():
+            pkgcache[k] = cache[k]
+
     # Force unprivileged users to have a private temporary cachedir
     # if they've not given an explicit cachedir
     if os.getuid() != 0 and not opts.cachedir:
@@ -183,7 +226,18 @@
             download_list = list(reposack)
 
         local_repo_path = opts.destdir + '/' + repo.id
-        if opts.delete and os.path.exists(local_repo_path):
+        # make sure the repo subdir is here before we go on.
+        if not os.path.exists(local_repo_path):
+            try:
+                os.makedirs(local_repo_path)
+            except IOError, e:
+                my.logger.error("Could not make repo subdir: %s" % e)
+                my.closeRpmDB()
+                sys.exit(1)
+
+        # Check if there's any local files no longer on the remote
+        # repo which need purging
+        if opts.delete:
             current_pkgs = localpkgs(local_repo_path)
 
             download_set = {}
@@ -200,62 +254,65 @@
                     my.logger.info("Removing obsolete %s", pkg)
                 os.unlink(current_pkgs[pkg]['path'])
 
+
         download_list.sort(sortPkgObj)
         n = 0
         for pkg in download_list:
             n = n + 1
             repo = my.repos.getRepo(pkg.repoid)
             remote = pkg.returnSimple('relativepath')
+            rpmname = os.path.basename(remote)
             local = local_repo_path + '/' + remote
             localdir = os.path.dirname(local)
+
             if not os.path.exists(localdir):
                 os.makedirs(localdir)
 
-            if (os.path.exists(local) and 
-                os.path.getsize(local) == int(pkg.returnSimple('packagesize'))):
-                
-                
-                if not opts.quiet:
-                    my.logger.error("[%s: %-5d of %-5d ] Skipping existing %s" % (repo.id, n, len(download_list), remote))
-                continue
-    
+            # If there's already a local copy, check it
+            if os.path.exists(local):
+                matching, err = checkPkg(opts, my, pkg, local)
+                if matching:
+                    my.logger.info("[%s: %-5d of %-5d ] Skipping existing %s" % (repo.id, n, len(download_list), remote))
+                    continue
+                else:
+                    if not opts.quiet:
+                        my.logger.info("[%s: %-5d of %-5d ] Removing existing %s due to %s" % (repo.id, n, len(download_list), remote, err))
+                    os.unlink(local)
+
+            # If another dir has a local copy, and it is on
+            # the same storage device, and verification passes,
+            # then we can hardlink it into local dir.
+            if not os.path.exists(local) and \
+               pkgcache.has_key(rpmname) and \
+               os.stat(local_repo_path).st_dev == pkgcache[rpmname]['device']:
+                matching, err = checkPkg(opts, my, pkg, pkgcache[rpmname]['path'])
+                if matching:
+                    if not opts.quiet:
+                        my.logger.info("[%s: %-5d of %-5d ] Linking existing %s" % (repo.id, n, len(download_list), remote))
+                    os.link(pkgcache[rpmname]['path'], local)
+                    continue
+
+            # If we're just printing URLs, skip to next repo
             if opts.urls:
                 url = urljoin(repo.urls[0],remote)
                 print '%s' % url
                 continue
-    
-            # make sure the repo subdir is here before we go on.
-            if not os.path.exists(local_repo_path):
-                try:
-                    os.makedirs(local_repo_path)
-                except IOError, e:
-                    my.logger.error("Could not make repo subdir: %s" % e)
-                    my.closeRpmDB()
-                    sys.exit(1)
-            
-            # Disable cache otherwise things won't download            
+
+            # Disable cache otherwise things won't download
             repo.cache = 0
             if not opts.quiet:
                 my.logger.info( '[%s: %-5d of %-5d ] Downloading %s' % (repo.id, n, len(download_list), remote))
-            pkg.localpath = local # Hack: to set the localpath we want.
-            path = repo.getPackage(pkg)
-            if opts.gpgcheck:
-                result, error = my.sigCheckPkg(pkg)
-                if result != 0:
-                    if result == 1:
-                        my.logger.warning('Removing %s, due to missing GPG key.' % os.path.basename(remote))
-                    elif result == 2:
-                        my.logger.warning('Removing %s due to failed signature check.' % os.path.basename(remote))
-                    else:
-                        my.logger.warning('Removing %s due to failed signature check: %s' % (os.path.basename(remote), error))
-                    os.unlink(path)
-                    continue
 
-            if not os.path.exists(local) or not os.path.samefile(path, local):
-                shutil.copy2(path, local)
+            pkg.localpath = local # Hack: to set the localpath we want.
+            repo.getPackage(pkg)
+            matching, err = checkPkg(opts, my, pkg, local)
+            if not matching:
+                my.logger.warning("[%s: %-5d of %-5d ] Removing failed download %s due to %s", (repo.id, n, len(download_list), remote, err))
+                os.unlink(path)
+                continue
 
     my.closeRpmDB()
 
 if __name__ == "__main__":
     main()
-    
+