[calm - Cygwin server-side packaging maintenance script] branch master, updated. 20180131-17-g1add34c
jturney-9JcytcrH/[email protected]
| Newsgroups | gmane.os.cygwin.cvs.apps |
|---|---|
| Message-ID | <[email protected]> |
https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=1add34cdb5d4713f808fa9e11805ec8d5a3b7f75 commit 1add34cdb5d4713f808fa9e11805ec8d5a3b7f75 Author: Jon Turney <[email protected]> Date: Mon May 14 19:46:29 2018 +0100 Add support for 'missing-curr' token in disabled-check: key in pvr.hint in pvr.hint, disable-check can contain: missing-curr: check that package has a current version Move identification of best-version earlier to support this. Also fix looking for empty-obsolete in disable-check. https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=360ce8ab8fad46abe3611abf207c643595ca5fd8 commit 360ce8ab8fad46abe3611abf207c643595ca5fd8 Author: Jon Turney <[email protected]> Date: Mon May 14 19:44:59 2018 +0100 Ignore test-only packages in non-unique current versions check https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=df1e7872cba3de9b6f564c585d4e423aa1452581 commit df1e7872cba3de9b6f564c585d4e423aa1452581 Author: Jon Turney <[email protected]> Date: Mon May 14 18:13:48 2018 +0100 Fix some escaping issues reported by latest pycodestyle Also disable W504 (line break after binary operator). The existing code is formatted to satisify W503 (Line break occurred before a binary operator). https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=7ee154d67d14272433a7ab2dd87e2e7378e763f2 commit 7ee154d67d14272433a7ab2dd87e2e7378e763f2 Author: Jon Turney <[email protected]> Date: Mon May 14 17:54:03 2018 +0100 Also remove versions which have no packages left, after expiry https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=61b82600b59f89ff0894da5357b837e502d04815 commit 61b82600b59f89ff0894da5357b837e502d04815 Author: Jon Turney <[email protected]> Date: Tue Jan 30 18:53:54 2018 +0000 Drop strict option to read_package() We no longer need the ability to not issue warnings about problems in the relarea, since the problems are all fixed. Diff: --- calm/package.py | 64 +++++++++++++++++++++++++++++------------------------ calm/uploads.py | 2 +- pep8 | 3 +- test/test_calm.py | 4 +- 4 files changed, 39 insertions(+), 34 deletions(-) diff --git a/calm/package.py b/calm/package.py index 85d5368..cbae949 100755 --- a/calm/package.py +++ b/calm/package.py @@ -136,7 +136,7 @@ def read_hints(p, fn, kind): # helper function to clean up hints -def clean_hints(p, hints, strict_lvl, warnings): +def clean_hints(p, hints, warnings): # # fix some common defects in the hints # @@ -152,7 +152,7 @@ def clean_hints(p, hints, strict_lvl, warnings): if colon: package_basename = re.sub(r'^lib(.*?)(|-devel|\d*)$', r'\1', p) if package_basename.upper().startswith(colon.group(1).upper()): - logging.log(strict_lvl, "package '%s' sdesc starts with '%s'; this is redundant as the UI will show both the package name and sdesc" % (p, ''.join(colon.group(1, 2)))) + logging.error("package '%s' sdesc starts with '%s'; this is redundant as the UI will show both the package name and sdesc" % (p, ''.join(colon.group(1, 2)))) warnings = True return warnings @@ -161,8 +161,7 @@ def clean_hints(p, hints, strict_lvl, warnings): # # read a single package # -def read_package(packages, basedir, dirpath, files, strict=False, remove=[], upload=False): - strict_lvl = logging.ERROR if strict else logging.WARNING +def read_package(packages, basedir, dirpath, files, remove=[], upload=False): relpath = os.path.relpath(dirpath, basedir) warnings = False @@ -187,7 +186,7 @@ def read_package(packages, basedir, dirpath, files, strict=False, remove=[], upl hints = read_hints(p, os.path.join(dirpath, 'setup.hint'), hint.setup) if not hints: return True - warnings = clean_hints(p, hints, strict_lvl, warnings) + warnings = clean_hints(p, hints, warnings) files.remove('setup.hint') else: hints = {} @@ -260,7 +259,7 @@ def read_package(packages, basedir, dirpath, files, strict=False, remove=[], upl # # P must match the package name, V can contain anything, R must # start with a number - match = re.match(r'^' + re.escape(p) + '-(.+)-(\d[0-9a-zA-Z.]*)(-src|)\.' + match.group(1) + '$', f) + match = re.match(r'^' + re.escape(p) + r'-(.+)-(\d[0-9a-zA-Z.]*)(-src|)\.' + match.group(1) + '$', f) if not match: logging.error("file '%s' in package '%s' doesn't follow naming convention" % (f, p)) return True @@ -273,14 +272,14 @@ def read_package(packages, basedir, dirpath, files, strict=False, remove=[], upl # idea. if '-' in v: if p not in past_mistakes.hyphen_in_version: - lvl = strict_lvl + lvl = logging.ERROR warnings = True else: lvl = logging.INFO logging.log(lvl, "file '%s' in package '%s' contains '-' in version" % (f, p)) if not v[0].isdigit(): - logging.log(strict_lvl, "file '%s' in package '%s' has a version which doesn't start with a digit" % (f, p)) + logging.error("file '%s' in package '%s' has a version which doesn't start with a digit" % (f, p)) warnings = True # if not there already, add to version-release list @@ -313,7 +312,7 @@ def read_package(packages, basedir, dirpath, files, strict=False, remove=[], upl pvr_hint = read_hints(p, os.path.join(dirpath, hint_fn), hint.pvr) if not pvr_hint: return True - warnings = clean_hints(p, pvr_hint, strict_lvl, warnings) + warnings = clean_hints(p, pvr_hint, warnings) files.remove(hint_fn) elif legacy: # otherwise, use setup.hint @@ -344,7 +343,7 @@ def read_package(packages, basedir, dirpath, files, strict=False, remove=[], upl # warn about unexpected files, including tarfiles which don't match the # package name if files: - logging.log(strict_lvl, "unexpected files in %s: %s" % (p, ', '.join(files))) + logging.error("unexpected files in %s: %s" % (p, ', '.join(files))) warnings = True if not upload and legacy and not legacy_used: @@ -365,13 +364,10 @@ def read_package(packages, basedir, dirpath, files, strict=False, remove=[], upl files.remove(s) if len(files) > 0: - logging.log(strict_lvl, "no .hint files in %s but has files: %s" % (dirpath, ', '.join(files))) + logging.error("no .hint files in %s but has files: %s" % (dirpath, ', '.join(files))) warnings = True - if strict: - return warnings - - return False + return warnings # @@ -602,12 +598,25 @@ def validate_packages(args, packages): packages[p].version_hints[v][l] = '' break + # identify a 'best' version to take certain information from: this is + # the curr version, if we have one, otherwise, the highest version. + if ('curr' in packages[p].stability) and (packages[p].stability['curr'] in packages[p].vermap): + packages[p].best_version = packages[p].stability['curr'] + elif len(packages[p].vermap): + packages[p].best_version = sorted(packages[p].vermap.keys(), key=lambda v: SetupVersion(v), reverse=True)[0] + else: + logging.error("package '%s' doesn't have any versions" % (p)) + packages[p].best_version = None + error = True + # the package must have some versions if not packages[p].stability: logging.error("no versions at any stability level for package '%s'" % (p)) error = True # it's also probably a really good idea if a curr version exists - elif 'curr' not in packages[p].stability and 'missing-curr' not in getattr(args, 'disable_check', []): + elif (('curr' not in packages[p].stability) and + ('missing-curr' not in packages[p].version_hints[packages[p].best_version].get('disable-check', '')) and + ('missing-curr' not in getattr(args, 'disable_check', []))): logging.warning("package '%s' doesn't have a curr version" % (p)) # error if the curr: version isn't the most recent non-test: version @@ -636,17 +645,6 @@ def validate_packages(args, packages): break - # identify a 'best' version to take certain information from: this is - # the curr version, if we have one, otherwise, the highest version. - if ('curr' in packages[p].stability) and (packages[p].stability['curr'] in packages[p].vermap): - packages[p].best_version = packages[p].stability['curr'] - elif len(packages[p].vermap): - packages[p].best_version = sorted(packages[p].vermap.keys(), key=lambda v: SetupVersion(v), reverse=True)[0] - else: - logging.error("package '%s' doesn't have any versions" % (p)) - packages[p].best_version = None - error = True - if 'replace-versions' in packages[p].override_hints: for rv in packages[p].override_hints['replace-versions'].split(): # warn if replace-versions lists a version which is less than @@ -670,7 +668,7 @@ def validate_packages(args, packages): if 'install' in packages[p].vermap[vr]: if packages[p].tar(vr, 'install').is_empty: if ((p in past_mistakes.empty_but_not_obsolete) or - ('empty-obsolete' in packages[p].version_hints.get('disable-check', ''))): + ('empty-obsolete' in packages[p].version_hints[vr].get('disable-check', ''))): lvl = logging.DEBUG else: lvl = logging.ERROR @@ -760,6 +758,11 @@ def validate_packages(args, packages): if re.match(r'^lib.*\d', install_p): continue + # ignore packages which don't have a current version (i.e. are test + # only) + if 'curr' not in packages[install_p].stability: + continue + # ignore specific packages we disable this check for if ((install_p in past_mistakes.nonunique_versions) or ('unique-version' in packages[install_p].version_hints[packages[install_p].best_version].get('disable-check', ''))): @@ -1079,9 +1082,12 @@ def delete(packages, path, fn): for t in packages[p].tars[vr]: if t == fn: del packages[p].tars[vr][t] - # XXX: should also remove from vermap break + # if no packages remain for this vr, also remove from vermap + if not packages[p].tars[vr]: + packages[p].vermap.pop(vr, None) + for h in packages[p].hint_files: if packages[p].hint_files[h] == fn: del packages[p].hint_files[h] diff --git a/calm/uploads.py b/calm/uploads.py index 679fb9a..79a9e7f 100644 --- a/calm/uploads.py +++ b/calm/uploads.py @@ -247,7 +247,7 @@ def scan(m, all_packages, arch, args): # read and validate package if files: # strict means we consider warnings as fatal for upload - if package.read_package(packages, m.homedir(), dirpath, files, strict=True, remove=removed_files, upload=True): + if package.read_package(packages, m.homedir(), dirpath, files, remove=removed_files, upload=True): error = True # always consider timestamp as checked during a dry-run, so it is never diff --git a/pep8 b/pep8 index db9c9c7..6230d52 100755 --- a/pep8 +++ b/pep8 @@ -1,4 +1,3 @@ #!/bin/sh set -e -grep -s -l '^#!/usr/bin/env python' * calm/* test/* | xargs python3 -m pycodestyle --count --show-source --max-line-length=240 --ignore=E129,E741 -flake8 --ignore E129,E501,E741,F841 +grep -s -l '^#!/usr/bin/env python' * calm/* test/* | xargs python3 -m pycodestyle --count --show-source --max-line-length=240 --ignore=E129,E741,W504 diff --git a/test/test_calm.py b/test/test_calm.py index f514b7d..322d776 100755 --- a/test/test_calm.py +++ b/test/test_calm.py @@ -415,11 +415,11 @@ class CalmTest(unittest.TestCase): os.chdir(os.path.dirname(os.path.abspath(__file__))) # ensure sha512.sum files exist - os.system("find testdata/relarea/x86 testdata/relarea/noarch -type d -exec sh -c 'cd {} ; sha512sum * >sha512.sum 2>/dev/null' \;") + os.system("find testdata/relarea/x86 testdata/relarea/noarch -type d -exec sh -c 'cd {} ; sha512sum * >sha512.sum 2>/dev/null' \\;") # should remove a sha512.sum file so that we test functioning when it's absent os.unlink('testdata/relarea/x86/release/arc/sha512.sum') # remove !ready files - os.system("find testdata/homes -name !ready -exec rm {} \;") + os.system("find testdata/homes -name !ready -exec rm {} \\;") # fix up package timestamps so highest version is also latest # (git doesn't store timestamps, so they will all be dated the time of checkout)