[calm - Cygwin server-side packaging maintenance script] branch master, updated. 20160705-95-g63037d2
jturney-9JcytcrH/[email protected]
| Newsgroups | gmane.os.cygwin.cvs.apps |
|---|---|
| Message-ID | <[email protected]> |
https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=63037d2cfeb1caf0929d30fd170927dcb1d0a84e commit 63037d2cfeb1caf0929d30fd170927dcb1d0a84e Author: Jon Turney <[email protected]> Date: Tue Oct 3 21:48:58 2017 +0100 Fix a problem with version sorting Non-alphanumeric separators (i.e. '.') were not ignored as per rpmvercmp. This led to incorrect sorting for e.g. 15.8b-1 vs 15.8.0.1-2 https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=1edd7bfc022bc1d1e16517ee3af0d6c3765e1e73 commit 1edd7bfc022bc1d1e16517ee3af0d6c3765e1e73 Author: Jon Turney <[email protected]> Date: Mon Oct 2 23:10:38 2017 +0100 Remove unused SetupVersion._warn_ambiguous_compare https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=484dafa7e4ff07fb4fb423697a96b46532ebaaf3 commit 484dafa7e4ff07fb4fb423697a96b46532ebaaf3 Author: Jon Turney <[email protected]> Date: Thu Jun 8 16:02:12 2017 +0100 Add a tool for de-duplicating a source package https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=7cb33c04ecb1ddd0aee793864a95d89d5221fa49 commit 7cb33c04ecb1ddd0aee793864a95d89d5221fa49 Author: Jon Turney <[email protected]> Date: Sat Apr 8 19:52:27 2017 +0100 Generate package listing page for source-only packages Note that we may not have a good sdesc for these packages https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=d51e7d75d48e1940840a000b50473175e0748e35 commit d51e7d75d48e1940840a000b50473175e0748e35 Author: Jon Turney <[email protected]> Date: Fri Jan 20 15:43:50 2017 +0000 Make source packages a thing Look for source packages in src/ Write Source: lines in setup.ini to reference them Future work: could do with an indication that a package is a source package, rather than a heuristic based on package name? Diff: --- calm/calm.py | 30 ++++++----- calm/dedupsrc.py | 156 +++++++++++++++++++++++++++++++++++++++++++++++++++++ calm/package.py | 21 +++++--- calm/pkg2html.py | 27 ++++++--- calm/version.py | 38 ++----------- setup.py | 1 + test/test_calm.py | 18 +++++- 7 files changed, 225 insertions(+), 66 deletions(-) diff --git a/calm/calm.py b/calm/calm.py index cfffb2f..72325f7 100755 --- a/calm/calm.py +++ b/calm/calm.py @@ -112,7 +112,7 @@ def process_relarea(args): if args.stale: stale_to_vault = remove_stale_packages(args, packages) if stale_to_vault: - for arch in common_constants.ARCHES + ['noarch']: + for arch in common_constants.ARCHES + ['noarch', 'src']: logging.info("vaulting %d old package(s) for arch %s, which are no longer accessible by installer" % (len(stale_to_vault[arch]), arch)) uploads.move_to_vault(args, stale_to_vault[arch]) else: @@ -143,7 +143,7 @@ def process_uploads(args, state): # for each arch and noarch scan_result = {} skip_maintainer = False - for arch in common_constants.ARCHES + ['noarch']: + for arch in common_constants.ARCHES + ['noarch', 'src']: logging.debug("reading uploaded arch %s packages from maintainer %s" % (arch, name)) # read uploads @@ -176,7 +176,7 @@ def process_uploads(args, state): logging.debug("merging %s package set with uploads from maintainer %s" % (arch, name)) # merge package sets - merged_packages[arch] = package.merge(state.packages[arch], scan_result[arch].packages, scan_result['noarch'].packages) + merged_packages[arch] = package.merge(state.packages[arch], scan_result[arch].packages, scan_result['noarch'].packages, scan_result['src'].packages) if not merged_packages[arch]: logging.error("error while merging uploaded %s packages for %s" % (arch, name)) valid = False @@ -211,7 +211,7 @@ def process_uploads(args, state): # check for conflicting movelists conflicts = False - for arch in common_constants.ARCHES + ['noarch']: + for arch in common_constants.ARCHES + ['noarch', 'src']: conflicts = conflicts or report_movelist_conflicts(scan_result[arch].to_relarea, scan_result[arch].to_vault, "manually") if args.stale: conflicts = conflicts or report_movelist_conflicts(scan_result[arch].to_relarea, stale_to_vault[arch], "automatically") @@ -223,7 +223,7 @@ def process_uploads(args, state): continue # for each arch and noarch - for arch in common_constants.ARCHES + ['noarch']: + for arch in common_constants.ARCHES + ['noarch', 'src']: logging.debug("moving %s packages for maintainer %s" % (arch, name)) # process the move lists @@ -237,7 +237,7 @@ def process_uploads(args, state): # for each arch if args.stale: - for arch in common_constants.ARCHES + ['noarch']: + for arch in common_constants.ARCHES + ['noarch', 'src']: if stale_to_vault[arch]: logging.info("vaulting %d old package(s) for arch %s, which are no longer accessible by installer" % (len(stale_to_vault[arch]), arch)) uploads.move_to_vault(args, stale_to_vault[arch]) @@ -246,7 +246,7 @@ def process_uploads(args, state): for arch in common_constants.ARCHES: # use merged package list state.packages[arch] = merged_packages[arch] - logging.debug("added %d + %d packages from maintainer %s" % (len(scan_result[arch].packages), len(scan_result['noarch'].packages), name)) + logging.debug("added %d (%s) + %d (noarch) + %d (src) packages from maintainer %s" % (len(scan_result[arch].packages), arch, len(scan_result['noarch'].packages), len(scan_result['src'].packages), name)) # record updated reminder times for maintainers maintainers.Maintainer.update_reminder_times(mlist) @@ -280,6 +280,7 @@ def process(args, state): def remove_stale_packages(args, packages): to_vault = {} to_vault['noarch'] = defaultdict(list) + to_vault['src'] = defaultdict(list) for arch in common_constants.ARCHES: logging.debug("checking for stale packages for arch %s" % (arch)) @@ -308,16 +309,17 @@ def remove_stale_packages(args, packages): if error: return None - # since noarch packages are included in the package set for both arch, we - # will build (hopefully) identical move lists for those packages for each - # arch. + # since noarch and src packages are included in the package set for both + # arch, we will build (hopefully) identical move lists for those packages + # for each arch. # # de-duplicate these package moves, as rather awkward workaround for that for path in list(to_vault[common_constants.ARCHES[0]]): - if path.startswith('noarch'): - to_vault['noarch'][path] = to_vault[common_constants.ARCHES[0]][path] - for arch in common_constants.ARCHES: - del to_vault[arch][path] + for prefix in ['noarch', 'src']: + if path.startswith(prefix): + to_vault[prefix][path] = to_vault[common_constants.ARCHES[0]][path] + for arch in common_constants.ARCHES: + del to_vault[arch][path] return to_vault diff --git a/calm/dedupsrc.py b/calm/dedupsrc.py new file mode 100755 index 0000000..f531a2e --- /dev/null +++ b/calm/dedupsrc.py @@ -0,0 +1,156 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2017 Jon Turney +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +# + +# +# Move a given source archive to src/ (assuming it is indentical in x86/ and +# x86_64/) and adjust hints appropriately. +# + +import argparse +import copy +import os +import re +import sys + +from . import common_constants +from . import hint + +# +# +# + + +def hint_file_write(fn, hints): + with open(fn, 'w') as f: + for k, v in hints.items(): + print("%s: %s" % (k, v), file=f) + +# +# +# + + +def dedup(archive, relarea): + # split path and filename + (path, filename) = os.path.split(archive) + + # parse tarfile name + match = re.match(r'^(.+?)-(\d.*)-src\.tar\.(bz2|gz|lzma|xz)$', filename) + + if not match: + print('tarfile name %s does not meet expectations' % (filename)) + sys.exit(1) + + p = match.group(1) + vr = match.group(2) + ext = match.group(3) + + # compute filenames + to_filename = p + '-src-' + vr + '.tar.' + ext + hint_filename = p + '-' + vr + '.hint' + to_hint_filename = p + '-src-' + vr + '.hint' + + # read hints for both arches + hints = {} + for arch in ['x86', 'x86_64']: + hint_pathname = os.path.join(relarea, arch, path, hint_filename) + + if not os.path.exists(hint_pathname): + print('%s not found' % (hint_pathname)) + return 1 + + hints[arch] = hint.hint_file_parse(hint_pathname, hint.pvr) + + if hints['x86'] != hints['x86_64']: + print('hints for %s-%s differ between arches' % (p, vr)) + return 1 + + # ensure target directory exists + try: + os.makedirs(os.path.join(relarea, 'src', path, p + '-src')) + except FileExistsError: + pass + + # move the src files to src/ + for arch in ['x86', 'x86_64']: + print('%s -> %s' % (os.path.join(relarea, arch, path, filename), os.path.join(relarea, 'src', path, p + '-src', to_filename))) + os.rename(os.path.join(relarea, arch, path, filename), os.path.join(relarea, 'src', path, p + '-src', to_filename)) + + # write .hint file for new -src package + src_hints = copy.copy(hints['x86']) + + if 'source' not in src_hints['sdesc']: + sdesc = re.sub(r'"(.*)"', r'\1', src_hints['sdesc']) + sdesc += ' (source code)' + src_hints['sdesc'] = '"' + sdesc + '"' + + if 'requires' in src_hints: + del src_hints['requires'] + + if 'external-source' in src_hints: + del src_hints['external-source'] + + to_hint_pathname = os.path.join(relarea, 'src', path, p + '-src', to_hint_filename) + print('writing %s' % (to_hint_pathname)) + hint_file_write(to_hint_pathname, src_hints) + + # adjust external-source in .hint for all subpackages + for arch in ['x86', 'x86_64']: + for (dirpath, subdirs, files) in os.walk(os.path.join(relarea, arch, path)): + subpkg = os.path.basename(dirpath) + filename = subpkg + '-' + vr + '.hint' + if filename in files: + hint_pathname = os.path.join(dirpath, filename) + hints = hint.hint_file_parse(hint_pathname, hint.pvr) + if ('skip' in hints): + # p was source only, so no package remains + print('removing %s' % (hint_pathname)) + os.remove(hint_pathname) + elif ('external-source' not in hints) or (hints['external-source'] == p): + hints['external-source'] = p + '-src' + print('writing %s' % (hint_pathname)) + hint_file_write(hint_pathname, hints) + + return 0 + +# +# +# + + +def main(): + relarea_default = common_constants.FTP + + parser = argparse.ArgumentParser(description='Source package deduplicator') + parser.add_argument('archive', metavar='ARCHIVE', nargs=1, help="source archive to deduplicate") + parser.add_argument('--releasearea', action='store', metavar='DIR', help="release directory (default: " + relarea_default + ")", default=relarea_default, dest='rel_area') + (args) = parser.parse_args() + + return dedup(args.archive[0], args.rel_area) + +# +# +# + +if __name__ == "__main__": + sys.exit(main()) diff --git a/calm/package.py b/calm/package.py index 07ef02b..23954dd 100755 --- a/calm/package.py +++ b/calm/package.py @@ -82,8 +82,8 @@ class Tar(object): def read_packages(rel_area, arch): packages = defaultdict(Package) - # both noarch/ and <arch>/ directories are considered - for root in ['noarch', arch]: + # <arch>/ noarch/ and src/ directories are considered + for root in ['noarch', 'src', arch]: releasedir = os.path.join(rel_area, root) logging.debug('reading packages from %s' % releasedir) @@ -423,7 +423,7 @@ def validate_packages(args, packages): for (t, tar) in packages[p].tars.items(): # categorize each tarfile as either 'source' or 'install' - if re.search(r'-src\.tar', t): + if re.search(r'-src.*\.tar', t): category = 'source' else: category = 'install' @@ -736,7 +736,7 @@ def write_setup_ini(args, packages, arch): # for each package for p in sorted(packages.keys(), key=sort_key): # do nothing if 'skip' - if packages[p].skip: + if packages[p].skip and not p.endswith('-src'): continue # write package data @@ -784,11 +784,16 @@ def write_setup_ini(args, packages, arch): # if that doesn't exist, follow external-source elif 'external-source' in packages[p].version_hints[version]: s = packages[p].version_hints[version]['external-source'] - if 'source' in packages[s].vermap[version]: - t = packages[s].vermap[version]['source'] - tar_line('source', packages[s], t, f) + # external-source points to a real source package (-src) + if s.endswith('-src'): + print("Source: %s" % (s), file=f) + # external-source points to a source file in another package else: - logging.warning("package '%s' version '%s' has no source in external-source '%s'" % (p, version, s)) + if 'source' in packages[s].vermap[version]: + t = packages[s].vermap[version]['source'] + tar_line('source', packages[s], t, f) + else: + logging.warning("package '%s' version '%s' has no source in external-source '%s'" % (p, version, s)) # helper function to output details for a particular tar file diff --git a/calm/pkg2html.py b/calm/pkg2html.py index 9b23a94..83308b6 100755 --- a/calm/pkg2html.py +++ b/calm/pkg2html.py @@ -55,6 +55,22 @@ from . import package # +# get sdesc for a package +# +# some source-only packages don't have an sdesc, since they consist of just +# 'skip':', in which case we try to make a reasonable one +# + +def desc(packages, p, bv): + if 'sdesc' in packages[p].version_hints[bv]: + header = packages[p].version_hints[bv]['sdesc'] + else: + header = p + + return header.replace('"', '') + + +# # # @@ -88,10 +104,6 @@ def update_package_listings(args, packages, arch): for p in packages: - # do nothing for packages marked 'skip' - if packages[p].skip: - continue - dir = os.path.join(base, p) if not args.dryrun: try: @@ -140,7 +152,7 @@ def update_package_listings(args, packages, arch): if not args.dryrun: with open(listing, 'w') as f: bv = packages[p].best_version - header = p + ": " + packages[p].version_hints[bv]['sdesc'].replace('"', '') + header = p + ": " + desc(packages, p, bv) if fver.endswith('-src'): header = header + " (source code)" @@ -210,12 +222,9 @@ def update_package_listings(args, packages, arch): <table class="pkglist">''') % (arch, arch), file=index) for p in sorted(packages.keys(), key=package.sort_key): - # don't write anything if 'skip' - if packages[p].skip: - continue bv = packages[p].best_version - header = packages[p].version_hints[bv]['sdesc'].replace('"', '') + header = desc(packages, p, bv) print('<tr><td><a href="' + arch + '/' + p + '">' + p + '</a></td><td>' + html.escape(header, quote=False) + '</td></tr>', file=index) diff --git a/calm/version.py b/calm/version.py index 027372f..f00ddbb 100644 --- a/calm/version.py +++ b/calm/version.py @@ -22,7 +22,6 @@ # import itertools -import logging import re @@ -45,10 +44,14 @@ class SetupVersion: # split version into [V, R], on the last '-', if any split = list(itertools.chain(version_string.rsplit('-', 1), ['']))[:2] - # then split each part into numeric and non-numeric sequences. + # then split each part into numeric and alphabetic sequences + # non-alphanumeric separators are discarded # numeric sequences have leading zeroes discarded for j, i in enumerate(['V', 'R']): - setattr(self, '_' + i, [re.sub(r'^0+(\d)', r'\1', m.group(1), 1) for m in re.finditer(r'(\d+|\D+)', split[j])]) + sequences = re.finditer(r'(\d+|[a-zA-Z]+|[^a-zA-Z\d]+)', split[j]) + sequences = [m for m in sequences if not re.match(r'[^a-zA-Z\d]+', m.group(1))] + sequences = [re.sub(r'^0+(\d)', r'\1', m.group(1), 1) for m in sequences] + setattr(self, '_' + i, sequences) def __str__(self): return '%s (V=%s R=%s)' % (self._version_string, str(self._V), str(self._R)) @@ -63,8 +66,6 @@ class SetupVersion: return self.__cmp__(other) == -1 def __cmp__(self, other): - # warn about ill-specified comparisons - # SetupVersion._warn_ambiguous_compare(self, other) # compare V c = SetupVersion._compare(self._V, other._V) @@ -101,30 +102,3 @@ class SetupVersion: # if equal length, all components have matched, so equal # otherwise, the version with a suffix remaining is greater return cmp(len(a), len(b)) - - # warn if the comparison of these versions is historically under-specified - @staticmethod - def _warn_ambiguous_compare(a, b): - def classify(s): - if len(s) == 0: - return 'e' - elif s[0].isdigit(): - return 'n' - elif s[0] in '.-_': - return s[0] - elif s[0].isalpha(): - return 'a' - return 'o' - - def is_ambiguous(a, b): - ambiguous = False - - for i in range(0, min(len(a), len(b))): - if classify(a[i]) != classify(b[i]): - ambiguous = True - break - - return ambiguous - - if is_ambiguous(a._V, b._V) or is_ambiguous(a._R, b._R): - logging.warning("ordering of versions '%s' and '%s' may not be what you expect" % (a._version_string, b._version_string)) diff --git a/setup.py b/setup.py index 2941fa4..cdc8121 100644 --- a/setup.py +++ b/setup.py @@ -14,6 +14,7 @@ setup( 'calm = calm.calm:main', 'mksetupini = calm.mksetupini:main', 'calm-mkgitoliteconf = calm.mkgitoliteconf:main', + 'dedup-source = calm.dedupsrc:main', ], }, url='https://cygwin.com/git/?p=cygwin-apps/calm.git', diff --git a/test/test_calm.py b/test/test_calm.py index 30ada6f..d1cf3af 100755 --- a/test/test_calm.py +++ b/test/test_calm.py @@ -153,13 +153,25 @@ class CalmTest(unittest.TestCase): ["1.3.30c-2", "1.3.30c-10", -1], ["2.24.51-1", "2.25-1", -1], ["2.1.5+20120813+gitdcbe778-1", "2.1.5-3", 1], - ["3.4.1-1", "3.4b1-1", -1], + ["3.4.1-1", "3.4b1-1", 1], ["041206-1", "200090325-1", -1], ["0.6.2+git20130413-2", "0.6.2-1", 1], ["2.6.0+bzr6602-1", "2.6.0-2", 1], - ["2.6.0-2", "2.6b2-1", -1], - ["2.6.0+bzr6602-1", "2.6b2-1", -1], + ["2.6.0-2", "2.6b2-1", 1], + ["2.6.0+bzr6602-1", "2.6b2-1", 1], ["0.6.7+20150214+git3a710f9-1", "0.6.7-1", 1], + ["15.8b-1", "15.8.0.1-2", -1], + ["1.2rc1-1","1.2.0-2", -1], + # examples from https://fedoraproject.org/wiki/Archive:Tools/RPM/VersionComparison + ["1.0010", "1.9", 1], + ["1.05", "1.5", 0], + ["1.0", "1", 1], + ["2.50", "2.5", 1], + ["fc4", "fc.4", 0], + ["FC5", "fc4", -1], + ["2a", "2.0", -1], + ["1.0", "1.fc4", 1], + ["3.0.0_fc", "3.0.0.fc", 0], ] for d in test_data: