[calm - Cygwin server-side packaging maintenance script] branch master, updated. 20210408-19-ga5cf1fb
Jon TURNEY via Cygwin-apps-cvs <[email protected]>
| Newsgroups | gmane.os.cygwin.cvs.apps |
|---|---|
| Message-ID | <[email protected]> |
https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=a5cf1fbb3bc0b4bf5ce602f03eb2c0c4b8a94774 commit a5cf1fbb3bc0b4bf5ce602f03eb2c0c4b8a94774 Author: Jon Turney <[email protected]> Date: Tue May 11 16:08:09 2021 +0100 Ignore 'skip' hint Ignore the 'skip' hint, as it's meaningless, now we properly classify packages as source or install. Instead, if installing a non-obsolete package will do nothing, mark it as not_for_output. Future work: also allow the 'does nothing' check to apply to '_obsolete' packages. At the moment this would catch a few more packages which probably need to be obsoleted: by something before they can be safely removed. (It's not safe to simply vault them as we need to ensure any existing installed files owned by that package are removed when a system with them installed is updated.) https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=aece4eb9b30d107c1f30aed6e86f170c943c16a0 commit aece4eb9b30d107c1f30aed6e86f170c943c16a0 Author: Jon Turney <[email protected]> Date: Wed May 12 22:48:29 2021 +0100 Add a tool to fix hint files with invalid hints Add a tool to fix .hint files containing keys which are no longer valid for that type of package (source or install). Also fix invalid keys in -src.hint created at upload. https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=0f66095073bb39eb8a2f28c41d65ddfa76e43f22 commit 0f66095073bb39eb8a2f28c41d65ddfa76e43f22 Author: Jon Turney <[email protected]> Date: Thu May 20 15:48:25 2021 +0100 Move 'packaging repository' up page to be with the rest of the details Diff: --- calm/fix-invalid-key-hint.py | 81 ++++++++++++++++++++++++++++++++++ calm/fix-missing-src-hint-info.py | 2 +- calm/fixes.py | 58 +++++++++++++++++++----- calm/hint.py | 39 ++++++++-------- calm/package.py | 52 +++++++--------------- calm/pkg2html.py | 17 ++++--- calm/uploads.py | 4 +- test/testdata/uploads/pkglist.expected | 4 +- 8 files changed, 178 insertions(+), 79 deletions(-) diff --git a/calm/fix-invalid-key-hint.py b/calm/fix-invalid-key-hint.py new file mode 100644 index 0000000..5f3340c --- /dev/null +++ b/calm/fix-invalid-key-hint.py @@ -0,0 +1,81 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2021 Jon Turney +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +# + +import argparse +import logging +import os +import sys + +from . import common_constants +from . import fixes + + +# +# +# + +def fix_hints(relarea, packages): + for (dirpath, _subdirs, files) in os.walk(relarea): + + # only apply to listed packages, if specified + if packages: + relpath = os.path.relpath(dirpath, relarea) + relpath = relpath.split(os.path.sep) + if (len(relpath) < 3) or (relpath[2] not in packages): + continue + + for f in files: + if f.endswith('.hint') and f != 'override.hint': + if fixes.fix_hint(dirpath, f, '', ['invalid_keys']): + logging.warning('fixed hints %s' % f) + + +# +# +# + +def main(): + relarea_default = common_constants.FTP + + parser = argparse.ArgumentParser(description='fix invalid keys in hint') + + parser.add_argument('package', nargs='*', metavar='PACKAGE') + parser.add_argument('-v', '--verbose', action='count', dest='verbose', help='verbose output', default=0) + parser.add_argument('--releasearea', action='store', metavar='DIR', help="release directory (default: " + relarea_default + ")", default=relarea_default, dest='relarea') + # XXX: should take an argument listing fixes to apply + + (args) = parser.parse_args() + if args.verbose: + logging.getLogger().setLevel(logging.INFO) + + logging.basicConfig(format=os.path.basename(sys.argv[0]) + ': %(message)s') + + fix_hints(args.relarea, args.package) + + +# +# +# + +if __name__ == "__main__": + sys.exit(main()) diff --git a/calm/fix-missing-src-hint-info.py b/calm/fix-missing-src-hint-info.py index c1c9d28..88142a6 100644 --- a/calm/fix-missing-src-hint-info.py +++ b/calm/fix-missing-src-hint-info.py @@ -50,7 +50,7 @@ def fix_hints(relarea, packages): logging.error('hint %s missing' % hf) continue - fixes.fix_homepage_src_hint(dirpath, hf, f) + fixes.fix_hint(dirpath, hf, f, ['homepage']) # diff --git a/calm/fixes.py b/calm/fixes.py index e7a9839..613e361 100644 --- a/calm/fixes.py +++ b/calm/fixes.py @@ -84,16 +84,7 @@ def follow_redirect(homepage): return homepage -def fix_homepage_src_hint(dirpath, hf, tf): - pn = os.path.basename(dirpath) - hintfile = os.path.join(dirpath, hf) - hints = hint.hint_file_parse(hintfile, hint.spvr) - - hints.pop('parse-warnings', None) - if 'parse-errors' in hints: - logging.error('invalid hints %s' % hf) - return - +def _fix_homepage_src_hint(hints, dirpath, _hf, tf): # already present? if 'homepage' in hints: homepage = hints['homepage'] @@ -109,6 +100,7 @@ def fix_homepage_src_hint(dirpath, hf, tf): if match: if homepage: logging.warning('multiple HOMEPAGE lines in .cygport in srcpkg %s', tf) + pn = os.path.basename(dirpath) homepage = match.group(2) homepage = re.sub(r'\$({|)(PN|ORIG_PN|NAME)(}|)', pn, homepage) @@ -118,10 +110,11 @@ def fix_homepage_src_hint(dirpath, hf, tf): if not homepage: logging.info('cannot determine homepage: from srcpkg %s' % tf) - return + return False logging.info('adding homepage:%s to hints for srcpkg %s' % (homepage, tf)) + # check if redirect? redirect_homepage = follow_redirect(homepage) # trivial URL transformations aren't interesting @@ -138,8 +131,49 @@ def fix_homepage_src_hint(dirpath, hf, tf): if not redirect_homepage.startswith(homepage): logging.warning('homepage:%s permanently redirects to %s' % (homepage, redirect_homepage)) - # write updated hints + # changed? if homepage != hints.get('homepage', None): hints['homepage'] = homepage + return True + + return False + + +def _fix_invalid_keys_hint(hints, _dirpath, hf, _tf): + # eliminate keys that aren't appropriate to the package type + if hf.endswith('-src.hint'): + valid_keys = hint.hintkeys[hint.spvr] + else: + valid_keys = hint.hintkeys[hint.pvr] + + changed = False + for k in list(hints.keys()): + if k not in valid_keys: + logging.debug("discarding invalid key '%s:' from hint '%s'" % (k, hf)) + hints.pop(k, None) + changed = True + + return changed + + +def fix_hint(dirpath, hf, tf, problems): + hintfile = os.path.join(dirpath, hf) + hints = hint.hint_file_parse(hintfile, None) + + hints.pop('parse-warnings', None) + if 'parse-errors' in hints: + logging.error('invalid hints %s' % hf) + return + + changed = False + if 'homepage' in problems: + changed = _fix_homepage_src_hint(hints, dirpath, hf, tf) + if 'invalid_keys' in problems: + changed = _fix_invalid_keys_hint(hints, dirpath, hf, tf) or changed + + # write updated hints + if changed: shutil.copy2(hintfile, hintfile + '.bak') hint.hint_file_write(hintfile, hints) + + return changed diff --git a/calm/hint.py b/calm/hint.py index 2c31714..6cabea4 100755 --- a/calm/hint.py +++ b/calm/hint.py @@ -45,7 +45,6 @@ commonkeys = { 'ldesc': 'multilineval', 'category': 'val', 'sdesc': 'val', - 'skip': 'noval', 'requires': 'optval', 'obsoletes': 'optval', 'test': 'noval', # mark the package as a test version @@ -65,6 +64,7 @@ hintkeys[pvr].update({ hintkeys[spvr] = commonkeys.copy() hintkeys[spvr].update({ + 'skip': 'noval', # in all spvr hints, but ignored 'homepage': 'val', 'build-depends': 'optval', }) @@ -197,7 +197,7 @@ def hint_file_parse(fn, kind): errors = [] warnings = [] - assert(kind in hintkeys) + assert((kind in hintkeys) or (kind is None)) with open(fn, 'rb') as f: c = f.read() @@ -221,21 +221,26 @@ def hint_file_parse(fn, kind): key = match.group(1) value = match.group(2) - if key not in hintkeys[kind]: - errors.append('unknown key %s at line %d' % (key, i)) - continue - valtype = hintkeys[kind][key] + if kind is not None: + if key not in hintkeys[kind]: + errors.append('unknown key %s at line %d' % (key, i)) + continue + valtype = hintkeys[kind][key] - # check if the key occurs more than once - if key in hints: - errors.append('duplicate key %s' % (key)) + # check if the key occurs more than once + if key in hints: + errors.append('duplicate key %s' % (key)) - # check the value meets any key-specific constraints - if (valtype == 'val') and (len(value) == 0): - errors.append('%s has empty value' % (key)) + # check the value meets any key-specific constraints + if (valtype == 'val') and (len(value) == 0): + errors.append('%s has empty value' % (key)) - if (valtype == 'noval') and (len(value) != 0): - errors.append("%s has non-empty value '%s'" % (key, value)) + if (valtype == 'noval') and (len(value) != 0): + errors.append("%s has non-empty value '%s'" % (key, value)) + + # only 'ldesc' and 'message' are allowed a multi-line value + if (valtype != 'multilineval') and (len(value.splitlines()) > 1): + errors.append("key %s has multi-line value" % (key)) # validate all categories are in the category list (case-insensitively) if key == 'category': @@ -266,10 +271,6 @@ def hint_file_parse(fn, kind): warnings.append("sdesc contains ' '") value = value.replace(' ', ' ') - # only 'ldesc' and 'message' are allowed a multi-line value - if (valtype != 'multilineval') and (len(value.splitlines()) > 1): - errors.append("key %s has multi-line value" % (key)) - # message must have an id and some text if key == 'message': if not re.match(r'(\S+)\s+(\S.*)', value): @@ -289,7 +290,7 @@ def hint_file_parse(fn, kind): # for the pvr kind, 'category' and 'sdesc' must be present # XXX: genini also requires 'requires' but that seems wrong - if kind != override: + if (kind == pvr) or (kind == spvr): mandatory = ['category', 'sdesc'] for k in mandatory: if k not in hints: diff --git a/calm/package.py b/calm/package.py index 1e5b0f1..75dfc4c 100755 --- a/calm/package.py +++ b/calm/package.py @@ -63,7 +63,7 @@ class Package(object): self.is_used_by = set() self.version_hints = {} self.override_hints = {} - self.skip = False + self.not_for_output = False def __repr__(self): return "Package('%s', %s, %s, %s, %s)" % ( @@ -71,7 +71,7 @@ class Package(object): pprint.pformat(self.tarfiles), pprint.pformat(self.version_hints), pprint.pformat(self.override_hints), - self.skip) + self.not_for_output) def tar(self, vr): return self.tarfiles[vr] @@ -428,10 +428,6 @@ def read_one_package(packages, p, relpath, dirpath, files, remove, kind): packages[pn].tarfiles = actual_tars packages[pn].hints = hints packages[pn].pkgpath = pkgpath - if kind == Kind.source: - packages[pn].skip = True - else: - packages[pn].skip = any(['skip' in version_hints[vr] for vr in version_hints]) packages[pn].kind = kind # since we are kind of inventing the source package names, and don't # want to report them, keep track of the real name @@ -546,9 +542,9 @@ def validate_packages(args, packages): error = True continue - # hint referencing a source-only package makes no sense - if r in packages and packages[r].skip: - logging.error("package '%s' version '%s' %s source-only package '%s'" % (p, v, c, r)) + # package relation hint referencing a source package makes no sense + if r in packages and packages[r].kind == Kind.source: + logging.error("package '%s' version '%s' %s source package '%s'" % (p, v, c, r)) error = True # if external-source is used, the package must exist @@ -583,37 +579,23 @@ def validate_packages(args, packages): else: logging.debug("can't ensure package '%s' doesn't depends: on obsoleting '%s'" % (o, p)) - has_install = False has_nonempty_install = False if packages[p].kind == Kind.binary: for vr in packages[p].versions(): - has_install = True if not packages[p].tar(vr).is_empty: has_nonempty_install = True obsolete = any(['_obsolete' in packages[p].version_hints[vr].get('category', '') for vr in packages[p].version_hints]) - # if the package has no install tarfiles (i.e. is source only), make - # sure it is marked as 'skip' (which really means 'source-only' at the - # moment) - # # if the package has no non-empty install tarfiles, and no dependencies # installing it will do nothing (and making it appear in the package - # list is just confusing), so if it's not obsolete, mark it as 'skip' - # - # (this needs to take place after uploads have been merged into the - # package set, so that an upload containing just a replacement - # setup.hint is not considered a source-only package) - # - if not packages[p].skip: - if not has_install: - packages[p].skip = True - logging.info("package '%s' appears to be source-only as it has no install tarfiles, marking as 'skip'" % (p)) - - elif not has_nonempty_install and not has_requires and not obsolete: - packages[p].skip = True - logging.info("package '%s' appears to be source-only as it has no non-empty install tarfiles and no dependencies, marking as 'skip'" % (p)) + # list is just confusing), so if it's not obsolete, mark it as + # 'not_for_output' + if packages[p].kind == Kind.binary: + if not has_nonempty_install and not has_requires and not obsolete: + packages[p].not_for_output = True + logging.info("package '%s' has no non-empty install tarfiles and no dependencies, marking as 'not_for_output'" % (p)) levels = ['test', 'curr', 'prev'] @@ -743,7 +725,7 @@ def validate_packages(args, packages): # If the install tarball is empty, we should probably either be marked # obsolete (if we have no dependencies) or virtual (if we do) - if packages[p].kind == Kind.binary and not packages[p].skip: + if packages[p].kind == Kind.binary and not packages[p].not_for_output: for vr in packages[p].versions(): if packages[p].tar(vr).is_empty: # this classification relies on obsoleting packages @@ -980,8 +962,11 @@ def write_setup_ini(args, packages, arch): for pn in sorted(packages, key=sort_key): po = packages[pn] - # do nothing if 'skip' - if po.skip: + if po.kind == Kind.source: + continue + + # do nothing if not_for_output + if po.not_for_output: continue # write package data @@ -1263,9 +1248,6 @@ def merge(a, *l): # merge hint file lists c[p].hints.update(b[p].hints) - # skip if both a and b are skip - c[p].skip = c[p].skip and b[p].skip - return c diff --git a/calm/pkg2html.py b/calm/pkg2html.py index 80bc393..5113d36 100755 --- a/calm/pkg2html.py +++ b/calm/pkg2html.py @@ -256,6 +256,12 @@ def update_package_listings(args, packages): <a href="/problems.html#personal-email">Do not contact the maintainer(s) directly</a>.)</span>'''), file=f) print('<br><br>', file=f) + if po.kind == package.Kind.source: + repo = 'git/cygwin-packages/%s.git' % pn + if os.path.exists('/' + repo): + repo_browse_url = '/git-cygwin-packages/?p=%s' % repo + print('<span class="detail">packaging repository</span>: <a href="%s">%s.git</a>' % (repo_browse_url, pn), file=f) + print('<ul>', file=f) for arch in sorted(packages): if p in packages[arch]: @@ -283,12 +289,6 @@ def update_package_listings(args, packages): print('</table><br>', file=f) print('</ul>', file=f) - if po.kind == package.Kind.source: - repo = 'git/cygwin-packages/%s.git' % pn - if os.path.exists('/' + repo): - repo_browse_url = '/git-cygwin-packages/?p=%s' % repo - print('<span class="detail">packaging repository</span>: <a href="%s">%s.git</a>' % (repo_browse_url, pn), file=f) - print(textwrap.dedent('''\ </div> </body> @@ -329,9 +329,8 @@ def write_packages_inc(args, packages, name, kind, includer): if p.endswith('-debuginfo'): continue - if packages[arch][p].kind == package.Kind.binary: - if packages[arch][p].skip: - continue + if packages[arch][p].not_for_output: + continue if packages[arch][p].kind == kind: package_list[packages[arch][p].orig_name] = p diff --git a/calm/uploads.py b/calm/uploads.py index 9998e46..961b397 100644 --- a/calm/uploads.py +++ b/calm/uploads.py @@ -193,8 +193,10 @@ def scan(m, all_packages, arch, args): remove.append(os.path.join(dirpath, old)) # see if we can fix-up missing homepage: in -src.hint file + # check homepage: for liveliness and redirection + # discard any keys which are invalid in a -src.hint if (new in files): - fixes.fix_homepage_src_hint(dirpath, new, f) + fixes.fix_hint(dirpath, new, f, ['homepage', 'invalid_keys']) # filter out files we don't need to consider for f in sorted(files): diff --git a/test/testdata/uploads/pkglist.expected b/test/testdata/uploads/pkglist.expected index 3b61113..fb3d59d 100644 --- a/test/testdata/uploads/pkglist.expected +++ b/test/testdata/uploads/pkglist.expected @@ -14,7 +14,7 @@ 'category': 'Devel', 'requires': 'cygwin', 'homepage': 'http://homepage.url', - 'depends': 'cygwin'}}, {}, True), + 'depends': 'cygwin'}}, {}, False), 'testpackage-subpackage': Package('testpackage/testpackage-subpackage', {'1.0-1': Tar('testpackage-subpackage-1.0-1.tar.bz2', 'x86/release/testpackage/testpackage-subpackage', 'aff488008bee3486e25b539fe6ccd1397bd3c5c0ba2ee2cf34af279554baa195af7493ee51d6f8510735c9a2ea54436d776a71e768165716762aec286abbbf83', 195, False)}, {'1.0-1': {'sdesc': '"A test subpackage"', 'ldesc': '"A test subpackage"', 'category': 'Devel', @@ -28,7 +28,7 @@ 'build-depends': 'cygport', 'sdesc': '"test package (zstd compressed)"', 'ldesc': '"test package (zstd compressed)"', - 'skip': ''}}, {}, True), + 'skip': ''}}, {}, False), 'testpackage2-subpackage': Package('testpackage2/testpackage2-subpackage', {'1.0-1': Tar('testpackage2-subpackage-1.0-1.tar.bz2', 'x86/release/testpackage2/testpackage2-subpackage', 'c4bf8e28d71b532e2b741e2931906dec0f0a70d4d051c0503476f864a5228f43765ae3342aafcebfd5a1738073537726b2bfbbd89c6da939a5f46d95aca3feaf', 46, True)}, {'1.0-1': {'sdesc': '"A test subpackage 2"', 'ldesc': '"A test subpackage 2"', 'category': 'Devel'}}, {}, False)}