createrepo/__init__.py | 130 ++++++++++++++++++++++++++++++++++++++++++--
createrepo/deltarpms.py | 140 ++++++++++++++++++++++++++++++++++++++++++++++++
genpkgmetadata.py | 5 +
3 files changed, 269 insertions(+), 6 deletions(-)
New commits:
commit 36f5e7664cc12e39fad050f7d5fba513ec27a7a1
Merge: 5a54792... e1a9ec9...
Author: Seth Vidal <[email protected]>
Date: Thu Jan 29 00:58:00 2009 -0500
Merge branch 'master' of ssh://createrepo.baseurl.org/srv/projects/createrepo/git/createrepo
* 'master' of ssh://createrepo.baseurl.org/srv/projects/createrepo/git/createrepo:
Add missing documentation on --checksum and --profile
commit 5a5479258a496e6a60cde1bafdd1fa8d3c709a4a
Author: Seth Vidal <[email protected]>
Date: Thu Jan 29 00:55:57 2009 -0500
--deltas, enable the creation and metadata-creation for presto/deltarpms
diff --git a/createrepo/__init__.py b/createrepo/__init__.py
index b3a91cc..2b3acac 100644
--- a/createrepo/__init__.py
+++ b/createrepo/__init__.py
@@ -25,7 +25,7 @@ from bz2 import BZ2File
from urlgrabber import grabber
import tempfile
-from yum import misc, Errors
+from yum import misc, Errors, to_unicode
from yum.sqlutils import executeSQL
from yum.packageSack import MetaSack
from yum.packages import YumAvailablePackage
@@ -44,6 +44,7 @@ except ImportError:
pass
from utils import _gzipOpen, bzipFile, checkAndMakeDir, GzipFile, checksum_and_rename
+import deltarpms
__version__ = '0.9.6'
@@ -64,7 +65,13 @@ class MetaDataConfig(object):
self.checkts = False
self.split = False
self.update = False
- self.update_md_path = None
+ self.deltas = False # do the deltarpm thing
+ self.deltadir = None # where to put the .drpms - defaults to 'drpms' inside 'repodata'
+ self.delta_relative = 'drpms/'
+ self.oldpackage_paths = [] # where to look for the old packages -
+ self.deltafile = 'prestodelta.xml.gz'
+ self.num_deltas = 1 # number of older versions to delta (max)
+ self.update_md_path = None
self.skip_stat = False
self.database = False
self.outputdir = None
@@ -151,7 +158,6 @@ class MetaDataGenerator:
if not self.conf.outputdir:
self.conf.outputdir = os.path.join(self.conf.basedir, self.conf.relative_dir)
-
def _test_setup_dirs(self):
# start the sanity/stupidity checks
for mydir in self.conf.directories:
@@ -180,13 +186,23 @@ class MetaDataGenerator:
if not checkAndMakeDir(temp_final):
raise MDError, _('Cannot create/verify %s') % temp_final
+ if self.conf.deltas:
+ temp_delta = os.path.join(self.conf.outputdir, self.conf.delta_relative)
+ if not checkAndMakeDir(temp_delta):
+ raise MDError, _('Cannot create/verify %s') % temp_delta
+ self.conf.deltadir = temp_delta
+
if os.path.exists(os.path.join(self.conf.outputdir, self.conf.olddir)):
raise MDError, _('Old data directory exists, please remove: %s') % self.conf.olddir
# make sure we can write to where we want to write to:
# and pickup the mdtimestamps while we're at it
- for direc in ['tempdir', 'finaldir']:
- filepath = os.path.join(self.conf.outputdir, direc)
+ direcs = ['tempdir' , 'finaldir']
+ if self.conf.deltas:
+ direcs.append('deltadir')
+
+ for direc in direcs:
+ filepath = os.path.join(self.conf.outputdir, getattr(self.conf, direc))
if os.path.exists(filepath):
if not os.access(filepath, os.W_OK):
raise MDError, _('error in must be able to write to metadata dir:\n -> %s') % filepath
@@ -348,6 +364,8 @@ class MetaDataGenerator:
self.primaryfile = self._setupPrimary()
self.flfile = self._setupFilelists()
self.otherfile = self._setupOther()
+ if self.conf.deltas:
+ self.deltafile = self._setupDelta()
def _setupPrimary(self):
# setup the primary metadata file
@@ -375,6 +393,14 @@ class MetaDataGenerator:
fo.write('<otherdata xmlns="http://linux.duke.edu/metadata/other" packages="%s">' %
self.pkgcount)
return fo
+
+ def _setupDelta(self):
+ # setup the other file
+ deltafilepath = os.path.join(self.conf.outputdir, self.conf.tempdir, self.conf.deltafile)
+ fo = _gzipOpen(deltafilepath, 'w')
+ fo.write('<?xml version="1.0" encoding="UTF-8"?>\n')
+ fo.write('<prestodelta>\n')
+ return fo
def read_in_package(self, rpmfile, pkgpath=None, reldir=None):
@@ -452,7 +478,8 @@ class MetaDataGenerator:
nodes = self.oldData.getNodes(old_pkg)
if nodes is not None:
recycled = True
-
+
+ # FIXME also open up the delta file
# otherwise do it individually
if not recycled:
@@ -470,6 +497,11 @@ class MetaDataGenerator:
# need to say something here
self.callback.errorlog("\nError %s: %s\n" % (pkg, e))
continue
+ # we can use deltas:
+ presto_md = self._do_delta_rpm_package(po)
+ if presto_md:
+ self.deltafile.write(presto_md)
+
else:
po = pkg
@@ -500,6 +532,9 @@ class MetaDataGenerator:
outfile.write('\n')
self.oldData.freeNodes(pkg)
+ #FIXME - if we're in update and we have deltas enabled
+ # check the presto data for this pkg and write its info back out
+ # to our deltafile
if not self.conf.quiet:
if self.conf.verbose:
@@ -540,6 +575,87 @@ class MetaDataGenerator:
self.otherfile.write('\n</otherdata>')
self.otherfile.close()
+ if not self.conf.quiet:
+ self.callback.log(_('Saving delta metadata'))
+ self.deltafile.write('\n</prestodelta>')
+ self.deltafile.close()
+
+ def _do_delta_rpm_package(self, pkg):
+ """makes the drpms, if possible, for this package object.
+ returns the presto/delta xml metadata as a string
+ """
+
+ results = u""
+ thisdeltastart = u""" <newpackage name="%s" epoch="%s" version="%s" release="%s" arch="%s">\n""" % (pkg.name,
+ pkg.epoch, pkg.ver, pkg.release, pkg.arch)
+ thisdeltaend = u""" </newpackage>\n"""
+
+ # generate a list of all the potential 'old rpms'
+ opl = self._get_old_package_list()
+ # get list of potential candidates which are likely to match
+ pot_cand = []
+ for fn in opl:
+ if os.path.basename(fn).startswith(pkg.name):
+ pot_cand.append(fn)
+
+ candidates = []
+ for fn in pot_cand:
+ try:
+ thispo = yumbased.CreateRepoPackage(self.ts, fn)
+ except Errors.MiscError, e:
+ continue
+ if (thispo.name, thispo.arch) != (pkg.name, pkg.arch):
+ # not the same, doesn't matter
+ continue
+ if thispo == pkg: #exactly the same, doesn't matter
+ continue
+ if thispo.EVR >= pkg.EVR: # greater or equal, doesn't matter
+ continue
+ candidates.append(thispo)
+ candidates.sort()
+ candidates.reverse()
+
+ drpm_results = u""
+ for delta_p in candidates[0:self.conf.num_deltas]:
+ #make drpm of pkg and delta_p
+ drpmfn = deltarpms.create_drpm(delta_p, pkg, self.conf.deltadir)
+
+ if drpmfn:
+ # TODO more sanity check the drpm for size, etc
+ # make xml of drpm
+ try:
+ drpm_po = yumbased.CreateRepoPackage(self.ts, drpmfn)
+ except Errors.MiscError, e:
+ os.unlink(drpmfn)
+ continue
+ rel_drpmfn = drpmfn.replace(self.conf.outputdir, '')
+ if rel_drpmfn[0] == '/':
+ rel_drpmfn = rel_drpmfn[1:]
+ if not self.conf.quiet:
+ if self.conf.verbose:
+ self.callback.log('created drpm from %s to %s: %s' % (
+ delta_p, pkg, drpmfn))
+
+ drpm = deltarpms.DeltaRPMPackage(drpm_po, self.conf.outputdir, rel_drpmfn)
+ drpm_results += to_unicode(drpm.xml_dump_metadata())
+
+ if drpm_results:
+ results = thisdeltastart + drpm_results + thisdeltaend
+
+ return results
+
+ def _get_old_package_list(self):
+ if hasattr(self, '_old_package_list'):
+ return self._old_package_list
+
+ opl = []
+ for d in self.conf.oldpackage_paths:
+ for f in self.getFileList(d, 'rpm'):
+ opl.append(d + '/' + f)
+
+ self._old_package_list = opl
+ return self._old_package_list
+
def addArbitraryMetadata(self, mdfile, mdtype, xml_node, compress=True,
compress_type='gzip', attribs={}):
"""add random metadata to the repodata dir and repomd.xml
@@ -638,6 +754,8 @@ class MetaDataGenerator:
db_workfiles = []
repoid='garbageid'
+ if self.conf.deltas:
+ workfiles.append((self.conf.deltafile, 'deltainfo'))
if self.conf.database:
if not self.conf.quiet: self.callback.log('Generating sqlite DBs')
try:
diff --git a/createrepo/deltarpms.py b/createrepo/deltarpms.py
new file mode 100644
index 0000000..4b4acaf
--- /dev/null
+++ b/createrepo/deltarpms.py
@@ -0,0 +1,140 @@
+#!/usr/bin/python -tt
+# util functions for deltarpms
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Library General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+# copyright 2009 - Red Hat
+
+import os.path
+import commands
+from yum import misc
+import gzip
+
+class DeltaRPMPackage:
+ """each drpm is one object, you pass it a drpm file
+ it opens the file, and pulls the information out in bite-sized chunks :)
+ """
+
+ mode_cache = {}
+
+ def __init__(self, pkgobj, basedir, filename):
+ try:
+ stats = os.stat(os.path.join(basedir, filename))
+ self.size = stats[6]
+ self.mtime = stats[8]
+ del stats
+ except OSError, e:
+ raise MDError, "Error Stat'ing file %s %s" % (basedir, filename)
+ self.csum_type = 'sha256'
+ self.relativepath = filename
+ self.po = pkgobj
+
+ fd = os.open(self.po.localpath, os.O_RDONLY)
+ os.lseek(fd, 0, 0)
+ fo = os.fdopen(fd, 'rb')
+ self.csum = misc.checksum(self.csum_type, fo)
+ fo.seek(int(self.po.hdrend))
+ self._getOldInfo(fo)
+ del fo
+ del fd
+
+ def _stringToNEVR(self, string):
+ i = string.rfind("-", 0, string.rfind("-")-1)
+ name = string[:i]
+ (epoch, ver, rel) = self._stringToVersion(string[i+1:])
+ return (name, epoch, ver, rel)
+
+ def _getLength(self, in_data):
+ length = 0
+ for val in in_data:
+ length = length * 256
+ length += ord(val)
+ return length
+
+ def _getOldInfo(self, fo):
+ try:
+ compobj = gzip.GzipFile("", "rb", 9, fo)
+ except:
+ raise zlibError("Data not stored in gzip format")
+
+ if compobj.read(4)[:3] != "DLT":
+ raise Exception("Not a deltarpm")
+
+ nevr_length = self._getLength(compobj.read(4))
+ nevr = compobj.read(nevr_length).strip("\x00")
+ seq_length = self._getLength(compobj.read(4))
+ seq = compobj.read(seq_length)
+ hex_seq = ""
+ for char in seq:
+ hex_seq += str("%02x" % ord(char))
+ self.oldnevrstring = nevr
+ self.oldnevr = self._stringToNEVR(nevr)
+ self.sequence = hex_seq
+ compobj.close()
+
+ def _stringToVersion(self, strng):
+ i = strng.find(':')
+ if i != -1:
+ epoch = strng[:i]
+ else:
+ epoch = '0'
+ j = strng.find('-')
+ if j != -1:
+ if strng[i + 1:j] == '':
+ version = None
+ else:
+ version = strng[i + 1:j]
+ release = strng[j + 1:]
+ else:
+ if strng[i + 1:] == '':
+ version = None
+ else:
+ version = strng[i + 1:]
+ release = None
+ return (epoch, version, release)
+
+ def xml_dump_metadata(self):
+ """takes an xml doc object and a package metadata entry node, populates a
+ package node with the md information"""
+
+ (oldname, oldepoch, oldver, oldrel) = self.oldnevr
+ sequence = "%s-%s" % (self.oldnevrstring, self.sequence)
+
+ delta_tag = """
+ <delta oldepoch="%s" oldversion="%s" oldrelease="%s">
+ <filename>%s</filename>
+ <sequence>%s</sequence>
+ <size>%s</size>
+ <checksum type="%s">%s</checksum>
+ </delta>\n""" % (oldepoch, oldver, oldrel, self.relativepath, sequence,
+ self.size, self.csum_type, self.csum)
+ return delta_tag
+
+def create_drpm(old_pkg, new_pkg, destdir):
+ """make a drpm file, if possible. returns None if nothing could
+ be created"""
+ drpmfn = '%s-%s-%s_%s-%s.%s.drpm' % (old_pkg.name, old_pkg.ver,
+ old_pkg.release, new_pkg.ver, new_pkg.release,
+ old_pkg.arch)
+ delta_rpm_path = os.path.join(destdir, drpmfn)
+ delta_command = '/usr/bin/makedeltarpm %s %s %s' % (old_pkg.localpath,
+ new_pkg.localpath,
+ delta_rpm_path)
+ if not os.path.exists(delta_rpm_path):
+ #TODO - check/verify the existing one a bit?
+ (code, out) = commands.getstatusoutput(delta_command)
+ if code:
+ print "Error genDeltaRPM for %s: exitcode was %s - Reported Error: %s" % (old_pkg.name, code, out)
+ return None
+
+ return delta_rpm_path
diff --git a/genpkgmetadata.py b/genpkgmetadata.py
index ec69cb8..033cce6 100755
--- a/genpkgmetadata.py
+++ b/genpkgmetadata.py
@@ -93,6 +93,11 @@ def parseArgs(args, conf):
help="tags for the content in the repository")
parser.add_option("--revision", default=None,
help="user-specified revision for this repository")
+ parser.add_option("--deltas", default=False, action="store_true",
+ help="create delta rpms and metadata")
+ parser.add_option("--oldpackagedirs", default=[], dest="oldpackage_paths",
+ action="append", help="paths to look for older pkgs to delta against")
+
(opts, argsleft) = parser.parse_args(args)
if len(argsleft) > 1 and not opts.split:
lmpx.com only provides a reader for public news (NNTP) servers. It is not
affiliated with the servers or forums shown here and is not responsible for
the content of articles, which is written by their respective authors.