[calm - Cygwin server-side packaging maintenance script] branch master, updated. 20190530-15-g8118f97
jturney-9JcytcrH/[email protected]
| Newsgroups | gmane.os.cygwin.cvs.apps |
|---|---|
| Message-ID | <[email protected]> |
https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=8118f97177a0ffe74681ebe32ca05436eb3c0c78 commit 8118f97177a0ffe74681ebe32ca05436eb3c0c78 Author: Jon Turney <[email protected]> Date: Mon Jun 3 18:15:06 2019 +0100 Allow a package to appear in arch and noarch Use the existing package-dict merge() function to combine them, reporting non-permitted duplications. Improve merge() so it can handle a package appearing in more than one of the merged-in package-dicts Handle the merge reporting an error (by returning None) in validate_packages() https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=cd973f84cae0ec9598591eda8ffafa0e263fd713 commit cd973f84cae0ec9598591eda8ffafa0e263fd713 Author: Jon Turney <[email protected]> Date: Tue May 28 21:07:37 2019 +0100 Stop storing relarea path in package The 'in package list' constraint becomes simply that path relative to <arch>/release starts with a package name in the package list Somewhat unfortunately, items in the movelist don't know what package they belong to, which makes things a bit awkward https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=67c8cfe73eb665685f4bc82ba4297d0c48d81e93 commit 67c8cfe73eb665685f4bc82ba4297d0c48d81e93 Author: Jon Turney <[email protected]> Date: Sun Jun 2 14:08:33 2019 +0100 Store path to tar file in Tar object https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=9d0d2ba77b13b7a95755c631ddda65065a121d50 commit 9d0d2ba77b13b7a95755c631ddda65065a121d50 Author: Jon Turney <[email protected]> Date: Sun Jun 2 13:13:46 2019 +0100 Add hint object to store path to hint file https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=5fbe4c85050c291b8976675e8cd9b36f004f1672 commit 5fbe4c85050c291b8976675e8cd9b36f004f1672 Author: Jon Turney <[email protected]> Date: Wed May 29 13:18:30 2019 +0100 Factor out movelist as a separate class Wrap movelist handling up in an object. This shouldn't make any functional change. https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=a5d2e1a6877e12ad43affb312722a12bff11ff35 commit a5d2e1a6877e12ad43affb312722a12bff11ff35 Author: Jon Turney <[email protected]> Date: Thu May 30 18:49:21 2019 +0100 Add some test coverage for movelist conflict Diff: --- calm/calm.py | 48 +++++--- calm/movelist.py | 112 ++++++++++++++++++ calm/package.py | 115 ++++++++++-------- calm/pkg2html.py | 6 +- calm/uploads.py | 85 ++------------ test/test_calm.py | 43 ++++++- test/testdata/conflict/homedir.expected | 7 + test/testdata/conflict/rel_area.expected | 124 ++++++++++++++++++++ test/testdata/conflict/vault.expected | 12 ++ .../staleversion/staleversion-230-1-src.tar.xz | Bin 0 -> 228 bytes .../release/staleversion/staleversion-230-1.hint | 4 + .../release/staleversion/staleversion-230-1.tar.xz | Bin 0 -> 228 bytes test/testdata/uploads/pkglist.expected | 8 +- 13 files changed, 405 insertions(+), 159 deletions(-) diff --git a/calm/calm.py b/calm/calm.py index 8627d1a..dc63eea 100755 --- a/calm/calm.py +++ b/calm/calm.py @@ -66,6 +66,7 @@ import time from .abeyance_handler import AbeyanceHandler from .buffering_smtp_handler import BufferingSMTPHandler +from .movelist import MoveList from . import common_constants from . import irk from . import maintainers @@ -116,7 +117,7 @@ def process_relarea(args): if stale_to_vault: for arch in common_constants.ARCHES + ['noarch', 'src']: logging.info("vaulting %d old package(s) for arch %s" % (len(stale_to_vault[arch]), arch)) - uploads.move_to_vault(args, stale_to_vault[arch]) + stale_to_vault[arch].move_to_vault(args) else: logging.error("error while evaluating stale packages") return None @@ -185,9 +186,7 @@ def process_uploads(args, state): break # remove files which are to be removed - for p in scan_result[arch].to_vault: - for f in scan_result[arch].to_vault[p]: - package.delete(merged_packages[arch], p, f) + scan_result[arch].to_vault.map(lambda p, f: package.delete(merged_packages[arch], p, f)) # validate the package set logging.debug("validating merged %s package set for maintainer %s" % (arch, name)) @@ -231,18 +230,21 @@ def process_uploads(args, state): # process the move lists if scan_result[arch].to_vault: logging.info("vaulting %d package(s) for arch %s, by request" % (len(scan_result[arch].to_vault), arch)) - uploads.move_to_vault(args, scan_result[arch].to_vault) + scan_result[arch].to_vault.move_to_vault(args) uploads.remove(args, scan_result[arch].remove_success) if scan_result[arch].to_relarea: logging.info("adding %d package(s) for arch %s" % (len(scan_result[arch].to_relarea), arch)) - uploads.move_to_relarea(m, args, scan_result[arch].to_relarea) + scan_result[arch].to_relarea.move_to_relarea(m, args) + # XXX: Note that there seems to be a separate process, not run + # from cygwin-admin's crontab, which changes the ownership of + # files in the release area to cyguser:cygwin # for each arch if args.stale: for arch in common_constants.ARCHES + ['noarch', 'src']: if stale_to_vault[arch]: logging.info("vaulting %d old package(s) for arch %s" % (len(stale_to_vault[arch]), arch)) - uploads.move_to_vault(args, stale_to_vault[arch]) + stale_to_vault[arch].move_to_vault(args) # for each arch for arch in common_constants.ARCHES: @@ -288,8 +290,8 @@ def process(args, state): def remove_stale_packages(args, packages): to_vault = {} - to_vault['noarch'] = defaultdict(list) - to_vault['src'] = defaultdict(list) + to_vault['noarch'] = MoveList() + to_vault['src'] = MoveList() for arch in common_constants.ARCHES: logging.debug("checking for stale packages for arch %s" % (arch)) @@ -298,9 +300,7 @@ def remove_stale_packages(args, packages): to_vault[arch] = package.stale_packages(packages[arch]) # remove stale packages from package set - for p in to_vault[arch]: - for f in to_vault[arch][p]: - package.delete(packages[arch], p, f) + to_vault[arch].map(lambda p, f: package.delete(packages[arch], p, f)) # if there are no stale packages, we don't have anything to do if not any([to_vault[a] for a in to_vault]): @@ -323,12 +323,19 @@ def remove_stale_packages(args, packages): # for each arch. # # de-duplicate these package moves, as rather awkward workaround for that - for path in list(to_vault[common_constants.ARCHES[0]]): + moved_list = [] + + def dedup(path, f): for prefix in ['noarch', 'src']: if path.startswith(prefix): - to_vault[prefix][path] = to_vault[common_constants.ARCHES[0]][path] - for arch in common_constants.ARCHES: - del to_vault[arch][path] + to_vault[prefix].add(path, f) + moved_list.append(path) + + to_vault[common_constants.ARCHES[0]].map(dedup) + + for path in moved_list: + for arch in common_constants.ARCHES: + to_vault[arch].remove(path) return to_vault @@ -340,11 +347,12 @@ def remove_stale_packages(args, packages): def report_movelist_conflicts(a, b, reason): conflicts = False - n = uploads.movelist_intersect(a, b) + n = MoveList.intersect(a, b) if n: - for p in n: - for f in n[p]: - logging.error("%s/%s is both uploaded and %s vaulted" % (p, f, reason)) + def report_conflict(p, f): + logging.error("%s/%s is both uploaded and %s vaulted" % (p, f, reason)) + + n.map(report_conflict) conflicts = True return conflicts diff --git a/calm/movelist.py b/calm/movelist.py new file mode 100644 index 0000000..8fb682a --- /dev/null +++ b/calm/movelist.py @@ -0,0 +1,112 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2015 Jon Turney +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +# + +import logging +import os + +from collections import defaultdict + + +# +# movelist class +# + +class MoveList(object): + def __init__(self): + # a movelist is a dict with relative directory paths for keys and a list + # of filenames for each value + self.movelist = defaultdict(list) + + def __len__(self): + return len(self.movelist) + + def __bool__(self): + # empty movelists are false + return len(self.movelist) > 0 + + def add(self, relpath, f): + self.movelist[relpath].append(f) + + def remove(self, relpath): + del self.movelist[relpath] + + def _move(self, args, fromdir, todir): + for p in sorted(self.movelist): + logging.debug("mkdir %s" % os.path.join(todir, p)) + if not args.dryrun: + try: + os.makedirs(os.path.join(todir, p), exist_ok=True) + except FileExistsError: + pass + logging.debug("move from '%s' to '%s':" % (os.path.join(fromdir, p), os.path.join(todir, p))) + for f in sorted(self.movelist[p]): + if os.path.exists(os.path.join(fromdir, p, f)): + logging.info("%s" % os.path.join(p, f)) + if not args.dryrun: + os.rename(os.path.join(fromdir, p, f), os.path.join(todir, p, f)) + else: + logging.error("%s can't be moved as it doesn't exist" % (f)) + + def move_to_relarea(self, m, args): + if self.movelist: + logging.info("move from %s's upload area to release area:" % (m.name)) + self._move(args, m.homedir(), args.rel_area) + + def move_to_vault(self, args): + if self.movelist: + logging.info("move from release area to vault:") + self._move(args, args.rel_area, args.vault) + + # apply a function to all files in the movelists + def map(self, function): + for p in self.movelist: + for f in self.movelist[p]: + function(p, f) + + # compute the intersection of a pair of movelists + @staticmethod + def intersect(a, b): + i = MoveList() + for p in a.movelist.keys() & b.movelist.keys(): + pi = set(a.movelist[p]) & set(b.movelist[p]) + if pi: + i.movelist[p] = pi + return i + + # copy the files in a movelist + def copy(args, fromdir, todir): + for p in sorted(self.movelist): + logging.debug("mkdir %s" % os.path.join(todir, p)) + if not args.dryrun: + try: + os.makedirs(os.path.join(todir, p), exist_ok=True) + except FileExistsError: + pass + logging.debug("copy from '%s' to '%s':" % (os.path.join(fromdir, p), os.path.join(todir, p))) + for f in sorted(self.movelist[p]): + if os.path.exists(os.path.join(fromdir, p, f)): + logging.debug("%s" % (f)) + if not args.dryrun: + shutil.copy2(os.path.join(fromdir, p, f), os.path.join(todir, p, f)) + else: + logging.error("%s can't be copied as it doesn't exist" % (f)) diff --git a/calm/package.py b/calm/package.py index 76ddb85..838e4fc 100755 --- a/calm/package.py +++ b/calm/package.py @@ -39,6 +39,7 @@ import textwrap import time from .version import SetupVersion +from .movelist import MoveList from . import common_constants from . import hint from . import maintainers @@ -48,9 +49,9 @@ from . import past_mistakes # information we keep about a package class Package(object): def __init__(self): - self.path = '' # path to package, relative to release area + self.pkgpath = '' # path to package, relative to arch self.tars = {} - self.hint_files = {} + self.hints = {} self.is_used_by = set() self.version_hints = {} self.override_hints = {} @@ -59,7 +60,7 @@ class Package(object): def __repr__(self): return "Package('%s', %s, %s, %s, %s)" % ( - self.path, + self.pkgpath, pprint.pformat(self.tars), pprint.pformat(self.version_hints), pprint.pformat(self.override_hints), @@ -72,32 +73,44 @@ class Package(object): # information we keep about a tar file class Tar(object): def __init__(self): + self.path = None # path to tar, relative to release area + self.fn = None # filename self.sha512 = '' self.size = 0 self.is_empty = False self.is_used = False def __repr__(self): - return "Tar('%s', %d, %s)" % (self.sha512, self.size, self.is_empty) + return "Tar('%s', '%s', '%s', %d, %s)" % (self.fn, self.path, self.sha512, self.size, self.is_empty) + + +# information we keep about a hint file +class Hint(object): + def __init__(self): + self.path = None # path to hint, relative to release area + self.fn = None # filename of hint + self.hints = {} # XXX: duplicates version_hints, for the moment # # read a packages from a directory hierarchy # def read_packages(rel_area, arch): - packages = defaultdict(Package) + packages = {} # <arch>/ noarch/ and src/ directories are considered for root in ['noarch', 'src', arch]: + packages[root] = defaultdict(Package) + releasedir = os.path.join(rel_area, root) logging.debug('reading packages from %s' % releasedir) for (dirpath, subdirs, files) in os.walk(releasedir, followlinks=True): - read_package(packages, rel_area, dirpath, files) + read_package(packages[root], rel_area, dirpath, files) - logging.debug("%d packages read" % len(packages)) + logging.debug("%d packages read" % len(packages[root])) - return packages + return merge({}, *packages.values()) # helper function to compute sha512 for a particular file @@ -175,9 +188,10 @@ def read_package(packages, basedir, dirpath, files, remove=[]): return True # check for duplicate package names at different paths + (_, _, pkgpath) = relpath.split(os.sep, 2) if p in packages: logging.error("duplicate package name at paths %s and %s" % - (dirpath, packages[p].path)) + (relpath, packages[p].pkgpath)) return True # determine version overrides @@ -267,6 +281,8 @@ def read_package(packages, basedir, dirpath, files, remove=[]): if not f.endswith('.hint'): # collect the attributes for each tar file t = Tar() + t.path = relpath + t.fn = f t.size = os.path.getsize(os.path.join(dirpath, f)) t.is_empty = tarfile_is_empty(os.path.join(dirpath, f)) t.mtime = os.path.getmtime(os.path.join(dirpath, f)) @@ -281,7 +297,7 @@ def read_package(packages, basedir, dirpath, files, remove=[]): # determine hints for each version we've encountered version_hints = {} - hint_files = {} + hints = {} actual_tars = {} for vr in vr_list: hint_fn = '%s-%s.hint' % (p, vr) @@ -304,8 +320,13 @@ def read_package(packages, basedir, dirpath, files, remove=[]): else: ovr = vr + hintobj = Hint() + hintobj.path = relpath + hintobj.fn = hint_fn + hintobj.hints = pvr_hint + version_hints[ovr] = pvr_hint - hint_files[ovr] = hint_fn + hints[ovr] = hintobj actual_tars[ovr] = tars[vr] # ignore dotfiles @@ -322,8 +343,8 @@ def read_package(packages, basedir, dirpath, files, remove=[]): packages[p].version_hints = version_hints packages[p].override_hints = override_hints packages[p].tars = actual_tars - packages[p].hint_files = hint_files - packages[p].path = relpath + packages[p].hints = hints + packages[p].pkgpath = pkgpath packages[p].skip = any(['skip' in version_hints[vr] for vr in version_hints]) elif (relpath.count(os.path.sep) > 1): @@ -391,6 +412,9 @@ def sort_key(k): def validate_packages(args, packages): error = False + if not packages: + return False + for p in sorted(packages.keys()): logging.log(5, "validating package '%s'" % (p)) has_requires = False @@ -789,7 +813,7 @@ def validate_package_maintainers(args, packages): # ignore obsolete packages if any(['_obsolete' in packages[p].version_hints[vr].get('category', '') for vr in packages[p].version_hints]): continue - if not is_in_package_list(packages[p].path, all_packages): + if not is_in_package_list(packages[p].pkgpath, all_packages): logging.error("package '%s' is not in the package list" % (p)) error = True @@ -975,10 +999,10 @@ def write_setup_ini(args, packages, arch): # helper function to output details for a particular tar file def tar_line(p, category, v, f): - t = p.vermap[v][category] - fn = os.path.join(p.path, t) - sha512 = p.tar(v, category).sha512 - size = p.tar(v, category).size + to = p.tar(v, category) + fn = os.path.join(to.path, to.fn) + sha512 = to.sha512 + size = to.size print("%s: %s %d %s" % (category, fn, size, sha512), file=f) @@ -1054,10 +1078,6 @@ def write_repo_json(args, packages, f): # - we combine the list of tarfiles, duplicates are not permitted # - we use the hints from b, and warn if they are different to the hints for a # -# (XXX: this implementation possibly assumes that a package is at most in a and -# one of b, which is currently true, but it could be written with more -# generality) -# def merge(a, *l): # start with a copy of a c = copy.deepcopy(a) @@ -1065,13 +1085,13 @@ def merge(a, *l): for b in l: for p in b: # if the package is in b but not in a, add it to the copy - if p not in a: + if p not in c: c[p] = b[p] # else, if the package is both in a and b, we have to do a merge else: # package must exist at same relative path - if a[p].path != b[p].path: - logging.error("package '%s' is at paths %s and %s" % (p, a[p].path, b[p].path)) + if c[p].pkgpath != b[p].pkgpath: + logging.error("package '%s' is at paths %s and %s" % (p, c[p].path, b[p].path)) return None else: for vr in b[p].tars: @@ -1087,13 +1107,12 @@ def merge(a, *l): # hints from b override hints from a, but warn if they have # changed - c[p].version_hints = a[p].version_hints for vr in b[p].version_hints: c[p].version_hints[vr] = b[p].version_hints[vr] - if vr in a[p].version_hints: - if a[p].version_hints[vr] != b[p].version_hints[vr]: + if vr in c[p].version_hints: + if c[p].version_hints[vr] != b[p].version_hints[vr]: diff = '\n'.join(difflib.ndiff( - pprint.pformat(a[p].version_hints[vr]).splitlines(), + pprint.pformat(c[p].version_hints[vr]).splitlines(), pprint.pformat(b[p].version_hints[vr]).splitlines())) logging.warning("package '%s' version '%s' hints changed\n%s" % (p, vr, diff)) @@ -1102,10 +1121,10 @@ def merge(a, *l): c[p].override_hints.update(b[p].override_hints) # merge hint file lists - c[p].hint_files.update(b[p].hint_files) + c[p].hints.update(b[p].hints) # skip if both a and b are skip - c[p].skip = a[p].skip and b[p].skip + c[p].skip = c[p].skip and b[p].skip return c @@ -1115,8 +1134,9 @@ def merge(a, *l): # def delete(packages, path, fn): + (_, _, pkgpath) = path.split(os.sep, 2) for p in packages: - if packages[p].path == path: + if packages[p].pkgpath == pkgpath: for vr in packages[p].tars: for t in packages[p].tars[vr]: if t == fn: @@ -1127,14 +1147,14 @@ def delete(packages, path, fn): if not packages[p].tars[vr]: packages[p].vermap.pop(vr, None) - for h in packages[p].hint_files: - if packages[p].hint_files[h] == fn: - del packages[p].hint_files[h] + for h in packages[p].hints: + if packages[p].hints[h].fn == fn: + del packages[p].hints[h] break # -# verify that the package ppath is in the list of packages plist +# verify that the package path starts with a package in the list of packages # # (This means that a maintainer can upload a package with any name, provided the # path contains one allowed for that maintainer) @@ -1147,20 +1167,10 @@ def delete(packages, path, fn): # arbitrary package upload. # -def package_list_re(plist): - if getattr(package_list_re, "_plist", []) != plist: - pattern = '|'.join(map(lambda p: r'/' + re.escape(p) + r'(?:/|$)', plist)) - package_list_re._regex = re.compile(pattern, re.IGNORECASE) - package_list_re._plist = plist - - return package_list_re._regex - def is_in_package_list(ppath, plist): - if package_list_re(plist).search(ppath): - return True - - return False + superpackage = ppath.split(os.sep, 1)[0] + return superpackage in plist # @@ -1231,14 +1241,15 @@ def stale_packages(packages): mark_package_fresh(packages, pn, v) # build a move list of stale versions - stale = defaultdict(list) + stale = MoveList() for pn, po in packages.items(): for v in sorted(po.vermap.keys(), key=lambda v: SetupVersion(v)): all_stale = True for category in ['source', 'install']: if category in po.vermap[v]: if not getattr(po.tar(v, category), 'fresh', False): - stale[po.path].append(po.vermap[v][category]) + to = po.tar(v, category) + stale.add(to.path, to.fn) logging.debug("package '%s' version '%s' %s is stale" % (pn, v, category)) else: all_stale = False @@ -1246,8 +1257,8 @@ def stale_packages(packages): # if there's a pvr.hint without a fresh source or install of the # same version, move it as well if all_stale: - if v in po.hint_files: - stale[po.path].append(po.hint_files[v]) + if v in po.hints: + stale.add(po.hints[v].path, po.hints[v].fn) logging.debug("package '%s' version '%s' hint is stale" % (pn, v)) # clean up freshness mark diff --git a/calm/pkg2html.py b/calm/pkg2html.py index c12d811..9370092 100755 --- a/calm/pkg2html.py +++ b/calm/pkg2html.py @@ -333,8 +333,8 @@ def write_arch_listing(args, packages, arch): listings = os.listdir(dir) listings.remove('.htaccess') - for t in itertools.chain.from_iterable([packages[p].tars[vr] for vr in packages[p].tars]): - fver = re.sub(r'\.tar.*$', '', t) + for tn, to in itertools.chain.from_iterable([packages[p].tars[vr].items() for vr in packages[p].tars]): + fver = re.sub(r'\.tar.*$', '', tn) listing = os.path.join(dir, fver) # ... if it doesn't already exist, or force @@ -365,7 +365,7 @@ def write_arch_listing(args, packages, arch): <h1>%s</h1> <pre>''' % (header, header)), file=f) - tf = os.path.join(args.rel_area, packages[p].path, t) + tf = os.path.join(args.rel_area, to.path, to.fn) if not os.path.exists(tf): # this shouldn't happen with a full mirror logging.error("tarfile %s not found" % (tf)) diff --git a/calm/uploads.py b/calm/uploads.py index 5b765cb..587aa9b 100644 --- a/calm/uploads.py +++ b/calm/uploads.py @@ -34,6 +34,7 @@ import shutil import tarfile import time +from .movelist import MoveList from . import common_constants from . import package @@ -54,8 +55,8 @@ def scan(m, all_packages, arch, args): basedir = os.path.join(m.homedir(), arch) packages = defaultdict(package.Package) - move = defaultdict(list) - vault = defaultdict(list) + move = MoveList() + vault = MoveList() remove = [] remove_success = [] error = False @@ -129,12 +130,13 @@ def scan(m, all_packages, arch, args): continue # package doesn't appear in package list at all - if not package.is_in_package_list(relpath, all_packages): + (_, _, pkgpath) = relpath.split(os.sep, 2) + if not package.is_in_package_list(pkgpath, all_packages): logging.error("package '%s' is not in the package list" % dirpath) continue # only process packages for which we are listed as a maintainer - if not package.is_in_package_list(relpath, m.pkgs): + if not package.is_in_package_list(pkgpath, m.pkgs): logging.warning("package '%s' is not in the package list for maintainer %s" % (dirpath, m.name)) continue @@ -196,7 +198,7 @@ def scan(m, all_packages, arch, args): logging.error("remove file %s is not empty" % fn) error = True else: - vault[relpath].append(f[1:]) + vault.add(relpath, f[1:]) remove_success.append(fn) removed_files.append(f[1:]) files.remove(f) @@ -249,9 +251,9 @@ def scan(m, all_packages, arch, args): logging.debug("different %s is already in release area" % fn) # we always consider .hint files as needing to be moved, as # we currently can't have a valid package without one - move[relpath].append(f) + move.add(relpath, f) else: - move[relpath].append(f) + move.add(relpath, f) # read and validate package if files: @@ -286,72 +288,3 @@ def remove(args, remove): os.unlink(f) except FileNotFoundError: logging.error("%s can't be deleted as it doesn't exist" % (f)) - - -# -# -# - -def move(args, movelist, fromdir, todir): - for p in sorted(movelist): - logging.debug("mkdir %s" % os.path.join(todir, p)) - if not args.dryrun: - try: - os.makedirs(os.path.join(todir, p), exist_ok=True) - except FileExistsError: - pass - logging.debug("move from '%s' to '%s':" % (os.path.join(fromdir, p), os.path.join(todir, p))) - for f in sorted(movelist[p]): - if os.path.exists(os.path.join(fromdir, p, f)): - logging.info("%s" % os.path.join(p, f)) - if not args.dryrun: - os.rename(os.path.join(fromdir, p, f), os.path.join(todir, p, f)) - else: - logging.error("%s can't be moved as it doesn't exist" % (f)) - - -def move_to_relarea(m, args, movelist): - if movelist: - logging.info("move from %s's upload area to release area:" % (m.name)) - move(args, movelist, m.homedir(), args.rel_area) - # XXX: Note that there seems to be a separate process, not run from - # cygwin-admin's crontab, which changes the ownership of files in the - # release area to cyguser:cygwin - - -def move_to_vault(args, movelist): - if movelist: - logging.info("move from release area to vault:") - move(args, movelist, args.rel_area, args.vault) - - -# compute the intersection of a pair of movelists -def movelist_intersect(a, b): - i = defaultdict(list) - for p in a.keys() & b.keys(): - pi = set(a[p]) & set(b[p]) - if pi: - i[p] = pi - return i - - -# -# -# - -def copy(args, movelist, fromdir, todir): - for p in sorted(movelist): - logging.debug("mkdir %s" % os.path.join(todir, p)) - if not args.dryrun: - try: - os.makedirs(os.path.join(todir, p), exist_ok=True) - except FileExistsError: - pass - logging.debug("copy from '%s' to '%s':" % (os.path.join(fromdir, p), os.path.join(todir, p))) - for f in sorted(movelist[p]): - if os.path.exists(os.path.join(fromdir, p, f)): - logging.debug("%s" % (f)) - if not args.dryrun: - shutil.copy2(os.path.join(fromdir, p, f), os.path.join(todir, p, f)) - else: - logging.error("%s can't be copied as it doesn't exist" % (f)) diff --git a/test/test_calm.py b/test/test_calm.py index a44517b..569c81c 100755 --- a/test/test_calm.py +++ b/test/test_calm.py @@ -333,8 +333,8 @@ class CalmTest(unittest.TestCase): shutil.rmtree(test_root) self.assertEqual(scan_result.error, False) - compare_with_expected_file(self, 'testdata/uploads', dict(scan_result.to_relarea), 'move') - self.assertCountEqual(scan_result.to_vault, {'x86/release/testpackage': ['x86/release/testpackage/testpackage-0.1-1.tar.bz2']}) + compare_with_expected_file(self, 'testdata/uploads', dict(scan_result.to_relarea.movelist), 'move') + self.assertCountEqual(scan_result.to_vault.movelist, {'x86/release/testpackage': ['x86/release/testpackage/testpackage-0.1-1.tar.bz2']}) self.assertCountEqual(scan_result.remove_always, [f for (f, t) in ready_fns]) self.assertEqual(scan_result.remove_success, ['testdata/homes/Blooey McFooey/x86/release/testpackage/-testpackage-0.1-1-src.tar.bz2', 'testdata/homes/Blooey McFooey/x86/release/testpackage/-testpackage-0.1-1.tar.bz2']) with pprint_patch(): @@ -366,6 +366,37 @@ class CalmTest(unittest.TestCase): # XXX: delete a needed package, and check validate fails + def test_process_uploads_conflict(self): + args = types.SimpleNamespace() + + for d in ['rel_area', 'homedir', 'vault']: + setattr(args, d, tempfile.mktemp()) + logging.info('%s = %s', d, getattr(args, d)) + + shutil.copytree('testdata/relarea', getattr(args, 'rel_area')) + shutil.copytree('testdata/homes.conflict', getattr(args, 'homedir')) + + setattr(args, 'dryrun', False) + setattr(args, 'email', None) + setattr(args, 'force', False) + setattr(args, 'pkglist', 'testdata/pkglist/cygwin-pkg-maint') + setattr(args, 'stale', True) + + # set appropriate !ready + m_homedir = os.path.join(getattr(args, 'homedir'), 'Blooey McFooey') + os.system('touch "%s"' % (os.path.join(m_homedir, 'x86', 'release', 'staleversion', '!ready'))) + + state = calm.calm.CalmState() + state.packages = calm.calm.process_relarea(args) + state.packages = calm.calm.process_uploads(args, state) + self.assertTrue(state.packages) + + for d in ['rel_area', 'homedir', 'vault']: + with self.subTest(directory=d): + dirlist = capture_dirtree(getattr(args, d)) + compare_with_expected_file(self, 'testdata/conflict', dirlist, d) + shutil.rmtree(getattr(args, d)) + def test_process(self): self.maxDiff = None @@ -436,6 +467,7 @@ class CalmTest(unittest.TestCase): # (git doesn't store timestamps, so they will all be dated the time of checkout) relarea_x86 = os.path.join('testdata', 'relarea', 'x86', 'release') relarea_noarch = os.path.join('testdata', 'relarea', 'noarch', 'release') + home_conflict = os.path.join('testdata', 'homes.conflict', 'Blooey McFooey', 'x86', 'release') touches = [(os.path.join(relarea_x86, 'cygwin', 'cygwin-2.2.0-1.tar.xz'), '2016-11-01'), (os.path.join(relarea_x86, 'cygwin', 'cygwin-2.2.0-1-src.tar.xz'), '2016-11-01'), (os.path.join(relarea_x86, 'cygwin', 'cygwin-2.2.1-1.tar.xz'), '2016-11-02'), @@ -464,9 +496,12 @@ class CalmTest(unittest.TestCase): (os.path.join(relarea_x86, 'keychain', 'keychain-2.6.8-1.tar.bz2'), '2016-11-02'), (os.path.join(relarea_x86, 'keychain', 'keychain-2.6.8-1-src.tar.bz2'), '2016-11-02'), (os.path.join(relarea_noarch, 'perl-Net-SMTP-SSL', 'perl-Net-SMTP-SSL-1.03-1.tar.xz'), '2016-11-01'), - (os.path.join(relarea_noarch, 'perl-Net-SMTP-SSL', 'perl-Net-SMTP-SSL-1.03-1-src.tar.xz'), '2016-11-01')] + (os.path.join(relarea_noarch, 'perl-Net-SMTP-SSL', 'perl-Net-SMTP-SSL-1.03-1-src.tar.xz'), '2016-11-01'), + (os.path.join(home_conflict, 'staleversion', 'staleversion-230-1.hint'), '2017-04-06'), + (os.path.join(home_conflict, 'staleversion', 'staleversion-230-1.tar.xz'), '2017-04-06'), + (os.path.join(home_conflict, 'staleversion', 'staleversion-230-1-src.tar.xz'), '2017-04-06')] for (f, t) in touches: - os.system('touch %s -d %s' % (f, t)) + os.system('touch "%s" -d %s' % (f, t)) # ensure !reminder-timestamp is created for uploads home = os.path.join('testdata', 'homes', 'Blooey McFooey') diff --git a/test/testdata/conflict/homedir.expected b/test/testdata/conflict/homedir.expected new file mode 100644 index 0000000..729c6b9 --- /dev/null +++ b/test/testdata/conflict/homedir.expected @@ -0,0 +1,7 @@ +{'.': [], + 'Blooey McFooey': [], + 'Blooey McFooey/x86': [], + 'Blooey McFooey/x86/release': [], + 'Blooey McFooey/x86/release/staleversion': ['staleversion-230-1-src.tar.xz', + 'staleversion-230-1.hint', + 'staleversion-230-1.tar.xz']} diff --git a/test/testdata/conflict/rel_area.expected b/test/testdata/conflict/rel_area.expected new file mode 100644 index 0000000..a23f44d --- /dev/null +++ b/test/testdata/conflict/rel_area.expected @@ -0,0 +1,124 @@ +{'.': [], + 'noarch': ['sha512.sum'], + 'noarch/release': ['sha512.sum'], + 'noarch/release/obs-a': ['obs-a-1.0-1-src.tar.xz', 'obs-a-1.0-1.hint', 'obs-a-1.0-1.tar.xz', 'sha512.sum'], + 'noarch/release/obs-b': ['obs-b-1.0-1-src.tar.xz', 'obs-b-1.0-1.hint', 'obs-b-1.0-1.tar.xz', 'sha512.sum'], + 'noarch/release/perl-Net-SMTP-SSL': ['perl-Net-SMTP-SSL-1.03-1-src.tar.xz', + 'perl-Net-SMTP-SSL-1.03-1.hint', + 'perl-Net-SMTP-SSL-1.03-1.tar.xz', + 'sha512.sum'], + 'noarch/release/test-c': ['sha512.sum', 'test-c-1.0-1-src.tar.xz', 'test-c-1.0-1.hint', 'test-c-1.0-1.tar.xz'], + 'noarch/release/test-d': ['sha512.sum', 'test-d-1.0-1-src.tar.xz', 'test-d-1.0-1.hint', 'test-d-1.0-1.tar.xz'], + 'noarch/release/test-e': ['sha512.sum', 'test-e-1.0-1-src.tar.xz', 'test-e-1.0-1.hint', 'test-e-1.0-1.tar.xz'], + 'x86': ['sha512.sum'], + 'x86/release': ['sha512.sum'], + 'x86/release/arc': ['arc-4.32.7-10-src.tar.bz2', 'arc-4.32.7-10.hint', 'arc-4.32.7-10.tar.bz2'], + 'x86/release/base-cygwin': ['base-cygwin-3.6-1.hint', + 'base-cygwin-3.6-1.tar.xz', + 'base-cygwin-3.8-1.hint', + 'base-cygwin-3.8-1.tar.xz', + 'sha512.sum'], + 'x86/release/corrupt': ['corrupt-2.0.0-1-src.tar.xz', 'corrupt-2.0.0-1.hint', 'corrupt-2.0.0-1.tar.xz', 'sha512.sum'], + 'x86/release/cygwin': ['.this-should-be-ignored', + 'cygwin-2.2.0-1-src.tar.xz', + 'cygwin-2.2.0-1.hint', + 'cygwin-2.2.0-1.tar.xz', + 'cygwin-2.2.1-1-src.tar.xz', + 'cygwin-2.2.1-1.hint', + 'cygwin-2.2.1-1.tar.xz', + 'cygwin-2.3.0-0.3-src.tar.xz', + 'cygwin-2.3.0-0.3.hint', + 'cygwin-2.3.0-0.3.tar.xz', + 'override.hint', + 'sha512.sum'], + 'x86/release/cygwin/cygwin-debuginfo': ['cygwin-debuginfo-2.2.0-1.hint', + 'cygwin-debuginfo-2.2.0-1.tar.xz', + 'cygwin-debuginfo-2.2.1-1.hint', + 'cygwin-debuginfo-2.2.1-1.tar.xz', + 'cygwin-debuginfo-2.3.0-0.3.hint', + 'cygwin-debuginfo-2.3.0-0.3.tar.xz', + 'override.hint', + 'sha512.sum'], + 'x86/release/cygwin/cygwin-devel': ['cygwin-devel-2.2.0-1.hint', + 'cygwin-devel-2.2.0-1.tar.xz', + 'cygwin-devel-2.2.1-1.hint', + 'cygwin-devel-2.2.1-1.tar.xz', + 'cygwin-devel-2.3.0-0.3.hint', + 'cygwin-devel-2.3.0-0.3.tar.xz', + 'override.hint', + 'sha512.sum'], + 'x86/release/invalid': ['invalid-0.hint', 'sha512.sum'], + 'x86/release/keychain': ['keychain-2.6.8-1-src.tar.bz2', + 'keychain-2.6.8-1.hint', + 'keychain-2.6.8-1.tar.bz2', + 'keychain-2.7.1-1-src.tar.bz2', + 'keychain-2.7.1-1.hint', + 'keychain-2.7.1-1.tar.bz2', + 'sha512.sum'], + 'x86/release/libspiro': ['libspiro-20071029-1.hint', 'sha512.sum'], + 'x86/release/libspiro/libspiro-devel': ['libspiro-devel-20071029-1.hint', 'sha512.sum'], + 'x86/release/libspiro/libspiro0': ['libspiro0-20071029-1.hint', 'sha512.sum'], + 'x86/release/libtextcat': ['libtextcat-2.2-2-src.tar.bz2', + 'libtextcat-2.2-2.hint', + 'libtextcat-2.2-2.tar.bz2', + 'sha512.sum'], + 'x86/release/libtextcat/libtextcat-devel': ['libtextcat-devel-2.2-2.hint', + 'libtextcat-devel-2.2-2.tar.bz2', + 'sha512.sum'], + 'x86/release/libtextcat/libtextcat0': ['libtextcat0-2.2-2.hint', 'libtextcat0-2.2-2.tar.bz2', 'sha512.sum'], + 'x86/release/mDNSResponder': ['mDNSResponder-379.32.1-1-src.tar.bz2', + 'mDNSResponder-379.32.1-1.hint', + 'mDNSResponder-379.32.1-1.tar.bz2', + 'sha512.sum'], + 'x86/release/mDNSResponder/libdns_sd-devel': ['libdns_sd-devel-379.32.1-1.hint', + 'libdns_sd-devel-379.32.1-1.tar.bz2', + 'sha512.sum'], + 'x86/release/mDNSResponder/libdns_sd1': ['libdns_sd1-379.32.1-1.hint', 'libdns_sd1-379.32.1-1.tar.bz2', 'sha512.sum'], + 'x86/release/mingw64-i686-binutils': ['mingw64-i686-binutils-2.29.1.787c9873-1.hint', 'sha512.sum'], + 'x86/release/mingw64-i686-binutils/mingw64-i686-binutils-debuginfo': ['mingw64-i686-binutils-debuginfo-2.29.1.787c9873-1.hint', + 'sha512.sum'], + 'x86/release/openssh': ['openssh-7.2p2-1-src.tar.xz', 'openssh-7.2p2-1.hint', 'openssh-7.2p2-1.tar.xz', 'sha512.sum'], + 'x86/release/per-version': ['override.hint', + 'per-version-4.0-1-src.tar.xz', + 'per-version-4.0-1.hint', + 'per-version-4.0-1.tar.xz', + 'per-version-4.8-1-src.tar.xz', + 'per-version-4.8-1.hint', + 'per-version-4.8-1.tar.xz', + 'sha512.sum'], + 'x86/release/per-version-incomplete': ['override.hint', + 'per-version-incomplete-36-1-src.tar.xz', + 'per-version-incomplete-36-1.hint', + 'per-version-incomplete-36-1.tar.xz', + 'per-version-incomplete-39-1-src.tar.xz', + 'per-version-incomplete-39-1.tar.xz', + 'sha512.sum'], + 'x86/release/per-version-replacement-hint-only': ['per-version-replacement-hint-only-1.0-1-src.tar.xz', + 'per-version-replacement-hint-only-1.0-1.hint', + 'per-version-replacement-hint-only-1.0-1.tar.xz', + 'sha512.sum'], + 'x86/release/proj': ['proj-4.8.0-1.hint', 'sha512.sum'], + 'x86/release/proj/libproj-devel': ['libproj-devel-4.8.0-1.hint', 'sha512.sum'], + 'x86/release/proj/libproj1': ['libproj1-4.8.0-1.hint', 'sha512.sum'], + 'x86/release/rpm-doc': ['rpm-doc-4.1-2-src.tar.bz2', + 'rpm-doc-4.1-2.hint', + 'rpm-doc-4.1-2.tar.bz2', + 'rpm-doc-999-1.hint', + 'rpm-doc-999-1.tar.bz2', + 'sha512.sum'], + 'x86/release/splint': ['sha512.sum', 'splint-3.1.2-1.hint'], + 'x86/release/staleversion': ['override.hint', + 'sha512.sum', + 'staleversion-243-0-src.tar.xz', + 'staleversion-243-0.hint', + 'staleversion-243-0.tar.xz', + 'staleversion-250-0-src.tar.xz', + 'staleversion-250-0.hint', + 'staleversion-250-0.tar.xz', + 'staleversion-260-0-src.tar.xz', + 'staleversion-260-0.hint', + 'staleversion-260-0.tar.xz'], + 'x86/release/testpackage': ['sha512.sum', + 'testpackage-0.1-1-src.tar.bz2', + 'testpackage-0.1-1.hint', + 'testpackage-0.1-1.tar.bz2']} diff --git a/test/testdata/conflict/vault.expected b/test/testdata/conflict/vault.expected new file mode 100644 index 0000000..6f418e2 --- /dev/null +++ b/test/testdata/conflict/vault.expected @@ -0,0 +1,12 @@ +{'.': [], + 'x86': [], + 'x86/release': [], + 'x86/release/staleversion': ['staleversion-240-1-src.tar.xz', + 'staleversion-240-1.hint', + 'staleversion-240-1.tar.xz', + 'staleversion-242-0-src.tar.xz', + 'staleversion-242-0.hint', + 'staleversion-242-0.tar.xz', + 'staleversion-251-0-src.tar.xz', + 'staleversion-251-0.hint', + 'staleversion-251-0.tar.xz']} diff --git a/test/testdata/homes.conflict/Blooey McFooey/x86/release/staleversion/staleversion-230-1-src.tar.xz b/test/testdata/homes.conflict/Blooey McFooey/x86/release/staleversion/staleversion-230-1-src.tar.xz new file mode 100644 index 0000000..0e6f1e8 Binary files /dev/null and b/test/testdata/homes.conflict/Blooey McFooey/x86/release/staleversion/staleversion-230-1-src.tar.xz differ diff --git a/test/testdata/homes.conflict/Blooey McFooey/x86/release/staleversion/staleversion-230-1.hint b/test/testdata/homes.conflict/Blooey McFooey/x86/release/staleversion/staleversion-230-1.hint new file mode 100644 index 0000000..7f7f48a --- /dev/null +++ b/test/testdata/homes.conflict/Blooey McFooey/x86/release/staleversion/staleversion-230-1.hint @@ -0,0 +1,4 @@ +sdesc: "Test package for stale version removal" +ldesc: "Test package for stale version removal" +category: Shells Base +requires: diff --git a/test/testdata/homes.conflict/Blooey McFooey/x86/release/staleversion/staleversion-230-1.tar.xz b/test/testdata/homes.conflict/Blooey McFooey/x86/release/staleversion/staleversion-230-1.tar.xz new file mode 100644 index 0000000..0e6f1e8 Binary files /dev/null and b/test/testdata/homes.conflict/Blooey McFooey/x86/release/staleversion/staleversion-230-1.tar.xz differ diff --git a/test/testdata/uploads/pkglist.expected b/test/testdata/uploads/pkglist.expected index 8498ce9..a9d9bfd 100644 --- a/test/testdata/uploads/pkglist.expected +++ b/test/testdata/uploads/pkglist.expected @@ -1,5 +1,5 @@ -{'testpackage': Package('x86/release/testpackage', {'1.0-1': {'testpackage-1.0-1-src.tar.bz2': Tar('aff488008bee3486e25b539fe6ccd1397bd3c5c0ba2ee2cf34af279554baa195af7493ee51d6f8510735c9a2ea54436d776a71e768165716762aec286abbbf83', 195, False), - 'testpackage-1.0-1.tar.bz2': Tar('aff488008bee3486e25b539fe6ccd1397bd3c5c0ba2ee2cf34af279554baa195af7493ee51d6f8510735c9a2ea54436d776a71e768165716762aec286abbbf83', 195, False)}}, {'1.0-1': {'sdesc': '"A test package"', +{'testpackage': Package('testpackage', {'1.0-1': {'testpackage-1.0-1-src.tar.bz2': Tar('testpackage-1.0-1-src.tar.bz2', 'x86/release/testpackage', 'aff488008bee3486e25b539fe6ccd1397bd3c5c0ba2ee2cf34af279554baa195af7493ee51d6f8510735c9a2ea54436d776a71e768165716762aec286abbbf83', 195, False), + 'testpackage-1.0-1.tar.bz2': Tar('testpackage-1.0-1.tar.bz2', 'x86/release/testpackage', 'aff488008bee3486e25b539fe6ccd1397bd3c5c0ba2ee2cf34af279554baa195af7493ee51d6f8510735c9a2ea54436d776a71e768165716762aec286abbbf83', 195, False)}}, {'1.0-1': {'sdesc': '"A test package"', 'ldesc': '"A test package\n' "It's description might contains some unicode junk\n" 'Like it’s you’re Markup Language™ Nokogiri’s tool―that ' @@ -7,10 +7,10 @@ 'category': 'Devel', 'requires': 'cygwin', 'depends': 'cygwin'}}, {}, False), - 'testpackage-subpackage': Package('x86/release/testpackage/testpackage-subpackage', {'1.0-1': {'testpackage-subpackage-1.0-1.tar.bz2': Tar('aff488008bee3486e25b539fe6ccd1397bd3c5c0ba2ee2cf34af279554baa195af7493ee51d6f8510735c9a2ea54436d776a71e768165716762aec286abbbf83', 195, False)}}, {'1.0-1': {'sdesc': '"A test subpackage"', + 'testpackage-subpackage': Package('testpackage/testpackage-subpackage', {'1.0-1': {'testpackage-subpackage-1.0-1.tar.bz2': Tar('testpackage-subpackage-1.0-1.tar.bz2', 'x86/release/testpackage/testpackage-subpackage', 'aff488008bee3486e25b539fe6ccd1397bd3c5c0ba2ee2cf34af279554baa195af7493ee51d6f8510735c9a2ea54436d776a71e768165716762aec286abbbf83', 195, False)}}, {'1.0-1': {'sdesc': '"A test subpackage"', 'ldesc': '"A test subpackage"', 'category': 'Devel', 'external-source': 'testpackage'}}, {}, False), - 'testpackage2-subpackage': Package('x86/release/testpackage2/testpackage2-subpackage', {'1.0-1': {'testpackage2-subpackage-1.0-1.tar.bz2': Tar('6de201dfed1d45412509c65deb34690dc2d09c6aafccfe491fd2f440f92842b9c755b61dc7bcdd4cc0c9f18cf46c2b3a1241e99c4c2a33fff5555e7b2f0b6348', 14, True)}}, {'1.0-1': {'sdesc': '"A test subpackage 2"', + 'testpackage2-subpackage': Package('testpackage2/testpackage2-subpackage', {'1.0-1': {'testpackage2-subpackage-1.0-1.tar.bz2': Tar('testpackage2-subpackage-1.0-1.tar.bz2', 'x86/release/testpackage2/testpackage2-subpackage', '6de201dfed1d45412509c65deb34690dc2d09c6aafccfe491fd2f440f92842b9c755b61dc7bcdd4cc0c9f18cf46c2b3a1241e99c4c2a33fff5555e7b2f0b6348', 14, True)}}, {'1.0-1': {'sdesc': '"A test subpackage 2"', 'ldesc': '"A test subpackage 2"', 'category': 'Devel'}}, {}, False)}