2 commits - bin/Makefile bin/mergerepo createrepo/__init__.py createrepo/merge.py createrepo.spec mergerepo.py
[email protected] (Seth Vidal)
| Newsgroups | gmane.linux.rpm.metadata |
|---|---|
| Message-ID | <[email protected]> |
bin/Makefile | 3 + bin/mergerepo | 2 createrepo.spec | 8 ++- createrepo/__init__.py | 2 createrepo/merge.py | 127 +++++++++++++++++++++++++++++++++++++++++++++++++ mergerepo.py | 79 ++++++++++++++++++++++++++++++ 6 files changed, 218 insertions(+), 3 deletions(-) New commits: commit bba27a038266ae17d8936c8f7bffd1a3e361bc35 Author: Seth Vidal <[email protected]> Date: Fri Oct 17 15:56:16 2008 -0400 bump version to 0.9.6 - change the dep to yum 3.2.20 - since it is what WILL be needed diff --git a/createrepo.spec b/createrepo.spec index b7cb49f..ac4573c 100644 --- a/createrepo.spec +++ b/createrepo.spec @@ -2,7 +2,7 @@ Summary: Creates a common metadata repository Name: createrepo -Version: 0.9.5 +Version: 0.9.6 Release: 1 License: GPL Group: System Environment/Base @@ -11,7 +11,7 @@ URL: http://linux.duke.edu/metadata/ BuildRoot: %{_tmppath}/%{name}-%{version}root BuildArchitectures: noarch Requires: python >= 2.1, rpm-python, rpm >= 0:4.1.1, libxml2-python -Requires: yum-metadata-parser, yum >= 3.2.19 +Requires: yum-metadata-parser, yum >= 3.2.20 %description This utility will generate a common metadata repository from a directory of @@ -35,11 +35,15 @@ make DESTDIR=$RPM_BUILD_ROOT install %{_datadir}/%{name}/* %{_bindir}/%{name} %{_bindir}/modifyrepo +%{_bindir}/mergerepo %{_mandir}/man8/createrepo.8* %{_mandir}/man1/modifyrepo.1* %{python_sitelib}/createrepo %changelog +* Fri Oct 17 2008 Seth Vidal <skvidal at fedoraproject.org> +- add mergerepo - 0.9.6 + * Mon Feb 18 2008 Seth Vidal <skvidal at fedoraproject.org> - 0.9.5 diff --git a/createrepo/__init__.py b/createrepo/__init__.py index b10c1e2..488abb4 100644 --- a/createrepo/__init__.py +++ b/createrepo/__init__.py @@ -45,7 +45,7 @@ except ImportError: from utils import _gzipOpen, bzipFile, checkAndMakeDir, GzipFile, checksum_and_rename -__version__ = '0.9.5' +__version__ = '0.9.6' class MetaDataConfig(object): commit 92e76bcb739b343484ce9a70bbaa969220d45da1 Author: Seth Vidal <[email protected]> Date: Fri Oct 17 15:54:19 2008 -0400 add mergerepo diff --git a/bin/Makefile b/bin/Makefile index 4497230..25437f9 100644 --- a/bin/Makefile +++ b/bin/Makefile @@ -42,6 +42,7 @@ all: $(srcdir)/$(PKGNAME) install: all installdirs $(INSTALL_BIN) $(srcdir)/$(PKGNAME) $(DESTDIR)$(bindir)/$(PKGNAME) $(INSTALL_BIN) $(srcdir)/modifyrepo $(DESTDIR)$(bindir)/modifyrepo + $(INSTALL_BIN) $(srcdir)/mergerepo $(DESTDIR)$(bindir)/mergerepo uninstall: @@ -73,6 +74,7 @@ distfiles: $(srcdir)/$(PKGNAME) \ $(srcdir)/Makefile \ $(srcdir)/modifyrepo \ + $(srcdir)/mergerepo \ $(top_srcdir)/.disttmp/$$distdir/bin dailyfiles: @@ -82,6 +84,7 @@ dailyfiles: $(srcdir)/$(PKGNAME) \ $(srcdir)/Makefile \ $(srcdir)/modifyrepo \ + $(srcdir)/mergerepo \ $(top_srcdir)/.disttmp/$$distdir/bin installdirs: diff --git a/bin/mergerepo b/bin/mergerepo new file mode 100755 index 0000000..df6e2f1 --- /dev/null +++ b/bin/mergerepo @@ -0,0 +1,2 @@ +#!/bin/sh +exec /usr/share/createrepo/mergerepo.py "$@" diff --git a/createrepo/merge.py b/createrepo/merge.py new file mode 100644 index 0000000..d096c61 --- /dev/null +++ b/createrepo/merge.py @@ -0,0 +1,127 @@ +#!/usr/bin/python -tt +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# Copyright 2008 Red Hat, Inc - written by seth vidal skvidal at fedoraproject.org + +# merge repos from arbitrary repo urls + +import os +import shutil +import yum +import yum.Errors +from yum.misc import unique, getCacheDir +import yum.update_md +import rpmUtils.arch +import operator +import createrepo +import tempfile + +# take repo paths from cli +# produce new repo metadata from merging the two together. + +#TODO: +# excludes? + + +class RepoMergeBase(): + def __init__(self, repolist=[]): + self.repolist = repolist + self.outputdir = '%s/merged_repo' % os.getcwd() + self.exclude_tuples = [] + self.sort_func = self._sort_func # callback function to magically sort pkgs + self.mdconf = createrepo.MetaDataConfig() + self.yumbase = yum.YumBase() + self.yumbase.conf.cachedir = getCacheDir() + self.yumbase.conf.cache = 0 + # default to all arches + self.archlist = unique(rpmUtils.arch.arches.keys() + rpmUtils.arch.arches.values()) + self.groups = True + self.updateinfo = True + + def _sort_func(self, repos): + """Default sort func for repomerge. Takes a list of repository objects + any package which is not to be included in the merged repo should be + delPackage()'d""" + # sort the repos by _merge_rank + # - lowest number is the highest rank (1st place, 2ndplace, etc) + repos.sort(key=operator.attrgetter('_merge_rank')) + + for repo in repos: + for pkg in repo.sack: + others = self.yumbase.pkgSack.searchNevra(name=pkg.name, arch=pkg.arch) + # NOTE the above is definitely going to catch other versions which may + # be an invalid comparison + if len(others) > 1: + for thatpkg in others: + if pkg.repoid == thatpkg.repoid: continue + if pkg.repo._merge_rank < thatpkg.repo._merge_rank: + thatpkg.repo.sack.delPackage(thatpkg) + + def merge_repos(self): + self.yumbase.repos.disableRepo('*') + # add our repos and give them a merge rank in the order they appear in + # in the repolist + count = 0 + for r in self.repolist: + count +=1 + rid = 'repo%s' % count + n = self.yumbase.add_enable_repo(rid, baseurls=[r]) + n._merge_rank = count + + #setup our sacks + self.yumbase._getSacks(archlist=self.archlist) + + myrepos = self.yumbase.repos.listEnabled() + + self.sort_func(myrepos) + + + def write_metadata(self, outputdir=None): + mytempdir = tempfile.mkdtemp() + if self.groups: + comps_fn = mytempdir + '/groups.xml' + compsfile = open(comps_fn, 'w') + compsfile.write(self.yumbase.comps.xml()) + compsfile.close() + self.mdconf.groupfile=comps_fn + + if self.updateinfo: + ui_fn = mytempdir + '/updateinfo.xml' + uifile = open(ui_fn, 'w') + umd = yum.update_md.UpdateMetadata() + for repo in self.yumbase.repos.listEnabled(): + try: # attempt to grab the updateinfo.xml.gz from the repodata + umd.add(repo) + except yum.Errors.RepoMDError: + continue + umd.xml(fileobj=uifile) + uifile.close() + self.mdconf.additional_metadata['updateinfo'] = ui_fn + + + self.mdconf.pkglist = self.yumbase.pkgSack + self.mdconf.directory = self.outputdir + if outputdir: + self.mdconf.directory = outputdir + # clean out what was there + if os.path.exists(self.mdconf.directory + '/repodata'): + shutil.rmtree(self.mdconf.directory + '/repodata') + + if not os.path.exists(self.mdconf.directory): + os.makedirs(self.mdconf.directory) + + mdgen = createrepo.MetaDataGenerator(config_obj=self.mdconf) + mdgen.doPkgMetadata() + mdgen.doRepoMetadata() + mdgen.doFinalMove() diff --git a/mergerepo.py b/mergerepo.py new file mode 100755 index 0000000..9b8aeab --- /dev/null +++ b/mergerepo.py @@ -0,0 +1,79 @@ +#!/usr/bin/python -tt +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# Copyright 2008 Red Hat, Inc - written by seth vidal skvidal at fedoraproject.org + +# merge repos from arbitrary repo urls + +import sys +import createrepo.merge +from optparse import OptionParser + +#TODO: +# excludes? + +def parse_args(args): + usage = """ + mergerepo: take 2 or more repositories and merge their metadata into a new repo + + mergerepo --repo=url --repo=url --outputdir=/some/path""" + + parser = OptionParser(version = "mergerepo 0.1", usage=usage) + # query options + parser.add_option("-r", "--repo", dest='repos', default=[], action="append", + help="repo url") + parser.add_option("-a", "--archlist", default=[], action="append", + help="Defaults to all arches - otherwise specify arches") + parser.add_option("-d", "--database", default=False, action="store_true") + parser.add_option("-o", "--outputdir", default=None, + help="Location to create the repository") + parser.add_option("", "--nogroups", default=False, action="store_true", + help="Do not merge group(comps) metadata") + parser.add_option("", "--noupdateinfo", default=False, action="store_true", + help="Do not merge updateinfo metadata") + (opts, argsleft) = parser.parse_args() + + if len(opts.repos) < 2: + parser.print_usage() + sys.exit(1) + + # sort out the comma-separated crap we somehow inherited. + archlist = [] + for a in opts.archlist: + for arch in a.split(','): + archlist.append(arch) + + opts.archlist = archlist + + return opts + +def main(args): + opts = parse_args(args) + rm = createrepo.merge.RepoMergeBase(opts.repos) + if opts.archlist: + rm.archlist = opts.archlist + if opts.outputdir: + rm.outputdir = opts.outputdir + if opts.database: + rm.mdconf.database = True + if opts.nogroups: + rm.groups = False + if opts.noupdateinfo: + rm.updateinfo = False + + rm.merge_repos() + rm.write_metadata() + +if __name__ == "__main__": + main(sys.argv[1:])