AUTHORS | 4 +
Makefile | 3 -
README | 3 -
bin/Makefile | 3 +
bin/mergerepo | 2
createrepo.spec | 10 ++-
createrepo/__init__.py | 31 +++++++++--
createrepo/merge.py | 127 +++++++++++++++++++++++++++++++++++++++++++++++++
docs/createrepo.8 | 15 +++--
genpkgmetadata.py | 17 ++++++
mergerepo.py | 81 +++++++++++++++++++++++++++++++
11 files changed, 278 insertions(+), 18 deletions(-)
New commits:
commit daba8c66119acd5ed4b55d816dd9593d063e1134
Merge: 1020057... e4edc4a...
Author: Seth Vidal <[email protected]>
Date: Tue Oct 21 14:35:19 2008 -0400
Merge branch 'master' of ssh://createrepo.baseurl.org/srv/projects/createrepo/git/createrepo
* 'master' of ssh://createrepo.baseurl.org/srv/projects/createrepo/git/createrepo:
Change the NamedTemporaryFile() usage to mkstemp(), stupid API
Fix parallel updates to the cachedir, thx to Michael Schwendt for spotting it
commit 1020057de0fbaad469bd86927ac7622fce7a19d8
Author: Seth Vidal <[email protected]>
Date: Tue Oct 21 14:22:12 2008 -0400
- document --content, --distro and --revision
- update urls in spec and docs
- add Authors file
diff --git a/AUTHORS b/AUTHORS
new file mode 100644
index 0000000..d18ef02
--- /dev/null
+++ b/AUTHORS
@@ -0,0 +1,4 @@
+Seth Vidal
+Luke Macken
+James Antill
+Paul Nasrat
diff --git a/README b/README
index b35c668..99bd0dd 100644
--- a/README
+++ b/README
@@ -4,7 +4,8 @@ other package-repository-related tools.
run createrepo -h for usage syntax
-http://linux.duke.edu/createrepo/
+http://createrepo.baseurl.org/
+
diff --git a/createrepo.spec b/createrepo.spec
index ac4573c..fc49870 100644
--- a/createrepo.spec
+++ b/createrepo.spec
@@ -7,7 +7,7 @@ Release: 1
License: GPL
Group: System Environment/Base
Source: %{name}-%{version}.tar.gz
-URL: http://linux.duke.edu/metadata/
+URL: http://createrepo.baseurl.org/
BuildRoot: %{_tmppath}/%{name}-%{version}root
BuildArchitectures: noarch
Requires: python >= 2.1, rpm-python, rpm >= 0:4.1.1, libxml2-python
diff --git a/createrepo/__init__.py b/createrepo/__init__.py
index 488abb4..744bb67 100644
--- a/createrepo/__init__.py
+++ b/createrepo/__init__.py
@@ -84,8 +84,9 @@ class MetaDataConfig(object):
self.changelog_limit = None # needs to be an int or None
self.unique_md_filenames = False
self.additional_metadata = {} # dict of 'type':'filename'
-
-
+ self.revision = str(int(time.time()))
+ self.content_tags = [] # flat list of strings (like web 2.0 tags)
+ self.distro_tags = []# [(cpeid(None allowed), human-readable-string)]
class SimpleMDCallBack(object):
def errorlog(self, thing):
@@ -604,6 +605,16 @@ class MetaDataGenerator:
rpmns = reporoot.newNs("http://linux.duke.edu/metadata/rpm", 'rpm')
repopath = os.path.join(self.conf.outputdir, self.conf.tempdir)
repofilepath = os.path.join(repopath, self.conf.repomdfile)
+
+ revision = reporoot.newChild(None, 'revision', self.conf.revision)
+ if self.conf.content_tags or self.conf.distro_tags:
+ tags = reporoot.newChild(None, 'tags', None)
+ for item in self.conf.content_tags:
+ c_tags = tags.newChild(None, 'content', item)
+ for (cpeid,item) in self.conf.distro_tags:
+ d_tags = tags.newChild(None, 'distro', item)
+ if cpeid:
+ d_tags.newProp('cpeid', cpeid)
sumtype = self.conf.sumtype
if self.conf.database_only:
diff --git a/docs/createrepo.8 b/docs/createrepo.8
index 719edff..1a31190 100644
--- a/docs/createrepo.8
+++ b/docs/createrepo.8
@@ -73,9 +73,13 @@ Ignore symlinks of packages
Only import the last N changelog entries, from each rpm, into the metadata
.IP "\fB\--unique-md-filenames\fP"
Include the file's checksum in the metadata filename, helps HTTP caching
-
-.IP "\fB\--database-only\fP"
-Generate only the sqlite dbs. Do not create the xml metadata at all.
+.IP "\--distro\fP"
+Specify distro tags. Can be specified more than once. Optional syntax specifying a
+cpeid(http://cpe.mitre.org/) --distro=cpeid,distrotag
+.IP "\--content\fP"
+Specify keyword/tags about the content of the repository. Can be specified more than once.
+.IP "\--revision\fP"
+Arbitrary string for a repository revision.
.SH "EXAMPLES"
Here is an example of a repository with a groups file. Note that the
@@ -99,11 +103,12 @@ repodata/repomd.xml
.PP
.SH "AUTHORS"
.nf
-Seth Vidal <[email protected]>
+See the Authors file
.fi
.PP
.SH "BUGS"
Any bugs which are found should be emailed to the mailing list:
[email protected][email protected]
+or reported in trac at: http://createrepo.baseurl.org
.fi
diff --git a/genpkgmetadata.py b/genpkgmetadata.py
index 5fe9a9a..77b4095 100755
--- a/genpkgmetadata.py
+++ b/genpkgmetadata.py
@@ -83,7 +83,13 @@ def parseArgs(args, conf):
default=False, action="store_true",
help="include the file's checksum in the filename, helps" \
"with proxies")
-
+ parser.add_option("--distro", default=[], action="append",
+ help="distro tag and optional cpeid: --distro 'cpeid,textname'")
+ parser.add_option("--content", default=[], dest='content_tags', action="append",
+ help="tags for the content in the repository")
+ parser.add_option("--revision", default=None,
+ help="user-specified revision for this repository")
+
(opts, argsleft) = parser.parse_args(args)
if len(argsleft) > 1 and not opts.split:
errorprint(_('Error: Only one directory allowed per run.'))
@@ -115,6 +121,15 @@ def parseArgs(args, conf):
conf.directory = directory
conf.directories = directories
+ # distro tag parsing
+
+ for spec in opts.distro:
+ if spec.find(',') == -1:
+ conf.distro_tags.append((None,spec))
+ else:
+ splitspec = spec.split(',')
+ conf.distro_tags.append((splitspec[0], splitspec[1]))
+
lst = []
if conf.pkglist:
pfo = open(conf.pkglist, 'r')
commit 3c7d8ceec7c6e4e7ec573aa9f48afbf13c5fa1a5
Author: Seth Vidal <[email protected]>
Date: Fri Oct 17 16:23:48 2008 -0400
add merge repo here, too
diff --git a/Makefile b/Makefile
index e084a13..facbc5b 100644
--- a/Makefile
+++ b/Makefile
@@ -45,7 +45,8 @@ INSTALL_MODULES = $(INSTALL) -m 755 -D
RM = rm -f
MODULES = $(srcdir)/genpkgmetadata.py \
- $(srcdir)/modifyrepo.py
+ $(srcdir)/modifyrepo.py \
+ $(srcdir)/mergerepo.py
.SUFFIXES: .py .pyc
.py.pyc:
commit 8b1fa94028f51d2ddcecb62014a596eff6db77d0
Author: Seth Vidal <[email protected]>
Date: Fri Oct 17 16:04:43 2008 -0400
pylintian cleanups
diff --git a/mergerepo.py b/mergerepo.py
index 9b8aeab..f1225c6 100755
--- a/mergerepo.py
+++ b/mergerepo.py
@@ -12,7 +12,7 @@
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
-# Copyright 2008 Red Hat, Inc - written by seth vidal skvidal at fedoraproject.org
+# Copyright 2008 Red Hat, Inc - written by skvidal at fedoraproject.org
# merge repos from arbitrary repo urls
@@ -24,6 +24,7 @@ from optparse import OptionParser
# excludes?
def parse_args(args):
+ """Parse our opts/args"""
usage = """
mergerepo: take 2 or more repositories and merge their metadata into a new repo
@@ -42,7 +43,7 @@ def parse_args(args):
help="Do not merge group(comps) metadata")
parser.add_option("", "--noupdateinfo", default=False, action="store_true",
help="Do not merge updateinfo metadata")
- (opts, argsleft) = parser.parse_args()
+ (opts, argsleft) = parser.parse_args(args)
if len(opts.repos) < 2:
parser.print_usage()
@@ -50,30 +51,31 @@ def parse_args(args):
# sort out the comma-separated crap we somehow inherited.
archlist = []
- for a in opts.archlist:
- for arch in a.split(','):
- archlist.append(arch)
+ for archs in opts.archlist:
+ for arch in archs.split(','):
+ archlist.append(arch)
opts.archlist = archlist
return opts
def main(args):
+ """main"""
opts = parse_args(args)
- rm = createrepo.merge.RepoMergeBase(opts.repos)
+ rmbase = createrepo.merge.RepoMergeBase(opts.repos)
if opts.archlist:
- rm.archlist = opts.archlist
+ rmbase.archlist = opts.archlist
if opts.outputdir:
- rm.outputdir = opts.outputdir
+ rmbase.outputdir = opts.outputdir
if opts.database:
- rm.mdconf.database = True
+ rmbase.mdconf.database = True
if opts.nogroups:
- rm.groups = False
+ rmbase.groups = False
if opts.noupdateinfo:
- rm.updateinfo = False
+ rmbase.updateinfo = False
- rm.merge_repos()
- rm.write_metadata()
+ rmbase.merge_repos()
+ rmbase.write_metadata()
if __name__ == "__main__":
main(sys.argv[1:])
commit bba27a038266ae17d8936c8f7bffd1a3e361bc35
Author: Seth Vidal <[email protected]>
Date: Fri Oct 17 15:56:16 2008 -0400
bump version to 0.9.6 - change the dep to yum 3.2.20 - since it is
what WILL be needed
diff --git a/createrepo.spec b/createrepo.spec
index b7cb49f..ac4573c 100644
--- a/createrepo.spec
+++ b/createrepo.spec
@@ -2,7 +2,7 @@
Summary: Creates a common metadata repository
Name: createrepo
-Version: 0.9.5
+Version: 0.9.6
Release: 1
License: GPL
Group: System Environment/Base
@@ -11,7 +11,7 @@ URL: http://linux.duke.edu/metadata/
BuildRoot: %{_tmppath}/%{name}-%{version}root
BuildArchitectures: noarch
Requires: python >= 2.1, rpm-python, rpm >= 0:4.1.1, libxml2-python
-Requires: yum-metadata-parser, yum >= 3.2.19
+Requires: yum-metadata-parser, yum >= 3.2.20
%description
This utility will generate a common metadata repository from a directory of
@@ -35,11 +35,15 @@ make DESTDIR=$RPM_BUILD_ROOT install
%{_datadir}/%{name}/*
%{_bindir}/%{name}
%{_bindir}/modifyrepo
+%{_bindir}/mergerepo
%{_mandir}/man8/createrepo.8*
%{_mandir}/man1/modifyrepo.1*
%{python_sitelib}/createrepo
%changelog
+* Fri Oct 17 2008 Seth Vidal <skvidal at fedoraproject.org>
+- add mergerepo - 0.9.6
+
* Mon Feb 18 2008 Seth Vidal <skvidal at fedoraproject.org>
- 0.9.5
diff --git a/createrepo/__init__.py b/createrepo/__init__.py
index b10c1e2..488abb4 100644
--- a/createrepo/__init__.py
+++ b/createrepo/__init__.py
@@ -45,7 +45,7 @@ except ImportError:
from utils import _gzipOpen, bzipFile, checkAndMakeDir, GzipFile, checksum_and_rename
-__version__ = '0.9.5'
+__version__ = '0.9.6'
class MetaDataConfig(object):
commit 92e76bcb739b343484ce9a70bbaa969220d45da1
Author: Seth Vidal <[email protected]>
Date: Fri Oct 17 15:54:19 2008 -0400
add mergerepo
diff --git a/bin/Makefile b/bin/Makefile
index 4497230..25437f9 100644
--- a/bin/Makefile
+++ b/bin/Makefile
@@ -42,6 +42,7 @@ all: $(srcdir)/$(PKGNAME)
install: all installdirs
$(INSTALL_BIN) $(srcdir)/$(PKGNAME) $(DESTDIR)$(bindir)/$(PKGNAME)
$(INSTALL_BIN) $(srcdir)/modifyrepo $(DESTDIR)$(bindir)/modifyrepo
+ $(INSTALL_BIN) $(srcdir)/mergerepo $(DESTDIR)$(bindir)/mergerepo
uninstall:
@@ -73,6 +74,7 @@ distfiles:
$(srcdir)/$(PKGNAME) \
$(srcdir)/Makefile \
$(srcdir)/modifyrepo \
+ $(srcdir)/mergerepo \
$(top_srcdir)/.disttmp/$$distdir/bin
dailyfiles:
@@ -82,6 +84,7 @@ dailyfiles:
$(srcdir)/$(PKGNAME) \
$(srcdir)/Makefile \
$(srcdir)/modifyrepo \
+ $(srcdir)/mergerepo \
$(top_srcdir)/.disttmp/$$distdir/bin
installdirs:
diff --git a/bin/mergerepo b/bin/mergerepo
new file mode 100755
index 0000000..df6e2f1
--- /dev/null
+++ b/bin/mergerepo
@@ -0,0 +1,2 @@
+#!/bin/sh
+exec /usr/share/createrepo/mergerepo.py "$@"
diff --git a/createrepo/merge.py b/createrepo/merge.py
new file mode 100644
index 0000000..d096c61
--- /dev/null
+++ b/createrepo/merge.py
@@ -0,0 +1,127 @@
+#!/usr/bin/python -tt
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Library General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+# Copyright 2008 Red Hat, Inc - written by seth vidal skvidal at fedoraproject.org
+
+# merge repos from arbitrary repo urls
+
+import os
+import shutil
+import yum
+import yum.Errors
+from yum.misc import unique, getCacheDir
+import yum.update_md
+import rpmUtils.arch
+import operator
+import createrepo
+import tempfile
+
+# take repo paths from cli
+# produce new repo metadata from merging the two together.
+
+#TODO:
+# excludes?
+
+
+class RepoMergeBase():
+ def __init__(self, repolist=[]):
+ self.repolist = repolist
+ self.outputdir = '%s/merged_repo' % os.getcwd()
+ self.exclude_tuples = []
+ self.sort_func = self._sort_func # callback function to magically sort pkgs
+ self.mdconf = createrepo.MetaDataConfig()
+ self.yumbase = yum.YumBase()
+ self.yumbase.conf.cachedir = getCacheDir()
+ self.yumbase.conf.cache = 0
+ # default to all arches
+ self.archlist = unique(rpmUtils.arch.arches.keys() + rpmUtils.arch.arches.values())
+ self.groups = True
+ self.updateinfo = True
+
+ def _sort_func(self, repos):
+ """Default sort func for repomerge. Takes a list of repository objects
+ any package which is not to be included in the merged repo should be
+ delPackage()'d"""
+ # sort the repos by _merge_rank
+ # - lowest number is the highest rank (1st place, 2ndplace, etc)
+ repos.sort(key=operator.attrgetter('_merge_rank'))
+
+ for repo in repos:
+ for pkg in repo.sack:
+ others = self.yumbase.pkgSack.searchNevra(name=pkg.name, arch=pkg.arch)
+ # NOTE the above is definitely going to catch other versions which may
+ # be an invalid comparison
+ if len(others) > 1:
+ for thatpkg in others:
+ if pkg.repoid == thatpkg.repoid: continue
+ if pkg.repo._merge_rank < thatpkg.repo._merge_rank:
+ thatpkg.repo.sack.delPackage(thatpkg)
+
+ def merge_repos(self):
+ self.yumbase.repos.disableRepo('*')
+ # add our repos and give them a merge rank in the order they appear in
+ # in the repolist
+ count = 0
+ for r in self.repolist:
+ count +=1
+ rid = 'repo%s' % count
+ n = self.yumbase.add_enable_repo(rid, baseurls=[r])
+ n._merge_rank = count
+
+ #setup our sacks
+ self.yumbase._getSacks(archlist=self.archlist)
+
+ myrepos = self.yumbase.repos.listEnabled()
+
+ self.sort_func(myrepos)
+
+
+ def write_metadata(self, outputdir=None):
+ mytempdir = tempfile.mkdtemp()
+ if self.groups:
+ comps_fn = mytempdir + '/groups.xml'
+ compsfile = open(comps_fn, 'w')
+ compsfile.write(self.yumbase.comps.xml())
+ compsfile.close()
+ self.mdconf.groupfile=comps_fn
+
+ if self.updateinfo:
+ ui_fn = mytempdir + '/updateinfo.xml'
+ uifile = open(ui_fn, 'w')
+ umd = yum.update_md.UpdateMetadata()
+ for repo in self.yumbase.repos.listEnabled():
+ try: # attempt to grab the updateinfo.xml.gz from the repodata
+ umd.add(repo)
+ except yum.Errors.RepoMDError:
+ continue
+ umd.xml(fileobj=uifile)
+ uifile.close()
+ self.mdconf.additional_metadata['updateinfo'] = ui_fn
+
+
+ self.mdconf.pkglist = self.yumbase.pkgSack
+ self.mdconf.directory = self.outputdir
+ if outputdir:
+ self.mdconf.directory = outputdir
+ # clean out what was there
+ if os.path.exists(self.mdconf.directory + '/repodata'):
+ shutil.rmtree(self.mdconf.directory + '/repodata')
+
+ if not os.path.exists(self.mdconf.directory):
+ os.makedirs(self.mdconf.directory)
+
+ mdgen = createrepo.MetaDataGenerator(config_obj=self.mdconf)
+ mdgen.doPkgMetadata()
+ mdgen.doRepoMetadata()
+ mdgen.doFinalMove()
diff --git a/mergerepo.py b/mergerepo.py
new file mode 100755
index 0000000..9b8aeab
--- /dev/null
+++ b/mergerepo.py
@@ -0,0 +1,79 @@
+#!/usr/bin/python -tt
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Library General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+# Copyright 2008 Red Hat, Inc - written by seth vidal skvidal at fedoraproject.org
+
+# merge repos from arbitrary repo urls
+
+import sys
+import createrepo.merge
+from optparse import OptionParser
+
+#TODO:
+# excludes?
+
+def parse_args(args):
+ usage = """
+ mergerepo: take 2 or more repositories and merge their metadata into a new repo
+
+ mergerepo --repo=url --repo=url --outputdir=/some/path"""
+
+ parser = OptionParser(version = "mergerepo 0.1", usage=usage)
+ # query options
+ parser.add_option("-r", "--repo", dest='repos', default=[], action="append",
+ help="repo url")
+ parser.add_option("-a", "--archlist", default=[], action="append",
+ help="Defaults to all arches - otherwise specify arches")
+ parser.add_option("-d", "--database", default=False, action="store_true")
+ parser.add_option("-o", "--outputdir", default=None,
+ help="Location to create the repository")
+ parser.add_option("", "--nogroups", default=False, action="store_true",
+ help="Do not merge group(comps) metadata")
+ parser.add_option("", "--noupdateinfo", default=False, action="store_true",
+ help="Do not merge updateinfo metadata")
+ (opts, argsleft) = parser.parse_args()
+
+ if len(opts.repos) < 2:
+ parser.print_usage()
+ sys.exit(1)
+
+ # sort out the comma-separated crap we somehow inherited.
+ archlist = []
+ for a in opts.archlist:
+ for arch in a.split(','):
+ archlist.append(arch)
+
+ opts.archlist = archlist
+
+ return opts
+
+def main(args):
+ opts = parse_args(args)
+ rm = createrepo.merge.RepoMergeBase(opts.repos)
+ if opts.archlist:
+ rm.archlist = opts.archlist
+ if opts.outputdir:
+ rm.outputdir = opts.outputdir
+ if opts.database:
+ rm.mdconf.database = True
+ if opts.nogroups:
+ rm.groups = False
+ if opts.noupdateinfo:
+ rm.updateinfo = False
+
+ rm.merge_repos()
+ rm.write_metadata()
+
+if __name__ == "__main__":
+ main(sys.argv[1:])
commit b732307670bf0c9a5198d02c93e512c6c22287a1
Author: Seth Vidal <[email protected]>
Date: Fri Oct 17 15:31:31 2008 -0400
add arbitrary metadata to config options for api callers
diff --git a/createrepo/__init__.py b/createrepo/__init__.py
index e9f1e08..b10c1e2 100644
--- a/createrepo/__init__.py
+++ b/createrepo/__init__.py
@@ -83,6 +83,7 @@ class MetaDataConfig(object):
self.directories = []
self.changelog_limit = None # needs to be an int or None
self.unique_md_filenames = False
+ self.additional_metadata = {} # dict of 'type':'filename'
@@ -742,10 +743,15 @@ class MetaDataGenerator:
self.addArbitraryMetadata(self.conf.groupfile, 'group_gz', reporoot)
self.addArbitraryMetadata(self.conf.groupfile, 'group', reporoot, compress=False)
- if self.rpmlib_reqs:
- rpmlib = reporoot.newChild(rpmns, 'lib', None)
- for r in self.rpmlib_reqs.keys():
- req = rpmlib.newChild(rpmns, 'requires', r)
+ if self.conf.additional_metadata:
+ for md_type, mdfile in self.conf.additional_metadata.items():
+ self.addArbitraryMetadata(mdfile, md_type, reporoot)
+
+ # FIXME - disabled until we decide how best to use this
+ #if self.rpmlib_reqs:
+ # rpmlib = reporoot.newChild(rpmns, 'lib', None)
+ # for r in self.rpmlib_reqs.keys():
+ # req = rpmlib.newChild(rpmns, 'requires', r)
# save it down
lmpx.com only provides a reader for public news (NNTP) servers. It is not
affiliated with the servers or forums shown here and is not responsible for
the content of articles, which is written by their respective authors.