[calm - Cygwin server-side packaging maintenance script] branch master, updated. 20210626-15-g8b4211b
Jon TURNEY via Cygwin-apps-cvs <[email protected]>
| Newsgroups | gmane.os.cygwin.cvs.apps |
|---|---|
| Message-ID | <[email protected]> |
https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=8b4211b3a103128ef6743f205418457745a4fc02 commit 8b4211b3a103128ef6743f205418457745a4fc02 Author: Jon Turney <[email protected]> Date: Mon Jan 31 18:59:52 2022 +0000 Include upstream version in 'unmaintained packages' report Fetch the upstream version from repology, and in include it in the 'unmaintained packages' report. https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=e2f6ae6c8e084e4eb77ec55f96b32409314dd563 commit e2f6ae6c8e084e4eb77ec55f96b32409314dd563 Author: Jon Turney <[email protected]> Date: Mon Jan 31 15:36:57 2022 +0000 Add 'deprecated shared libraries' report https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=f81674c2f385bb6803dea509a8463bcb0abd789e commit f81674c2f385bb6803dea509a8463bcb0abd789e Author: Jon Turney <[email protected]> Date: Sat Jan 29 18:11:46 2022 +0000 Add 'unmaintained packages' report https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=88b57728790ae1556d545697c15f315dfa182e75 commit 88b57728790ae1556d545697c15f315dfa182e75 Author: Jon Turney <[email protected]> Date: Sat Jan 29 15:56:21 2022 +0000 Don't emit depends: etc. when there's no install package Don't emit useless depends:, obsoletes:, provides: or conflicts: lines when there's no install package for this version, only a source package. Diff: --- calm/calm.py | 17 +++-- calm/package.py | 41 ++++++++---- calm/pkg2html.py | 9 ++- calm/repology.py | 97 +++++++++++++++++++++++++++ calm/reports.py | 196 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 344 insertions(+), 16 deletions(-) diff --git a/calm/calm.py b/calm/calm.py index cfc05dd..c76ed41 100755 --- a/calm/calm.py +++ b/calm/calm.py @@ -71,6 +71,8 @@ from . import irk from . import maintainers from . import package from . import pkg2html +from . import repology +from . import reports from . import setup_exe from . import uploads from . import utils @@ -278,6 +280,8 @@ def process(args, state): state.packages = process_uploads(args, state) + repology.annotate_packages(args, state.packages) + return state.packages @@ -382,10 +386,6 @@ def do_output(args, state): # update packages listings # XXX: perhaps we need a --[no]listing command line option to disable this from being run? pkg2html.update_package_listings(args, state.packages) - # if we are daemonized, allow force regeneration of static content in htdocs - # initially (in case the generation code has changed), but update that - # static content only as needed on subsequent loops - args.force = 0 update_json = False @@ -482,6 +482,15 @@ def do_output(args, state): except (OSError): pass + # write reports + if update_json or args.force: + reports.do_reports(args, state.packages) + + # if we are daemonized, allow force regeneration of static content in htdocs + # initially (in case the generation code has changed), but update that + # static content only as needed on subsequent loops + args.force = 0 + # # daemonization loop diff --git a/calm/package.py b/calm/package.py index 947d06b..9fd734c 100755 --- a/calm/package.py +++ b/calm/package.py @@ -495,8 +495,10 @@ def validate_packages(args, packages): for hints in packages[p].version_hints.values(): valid_requires.update(hints.get('provides', '').split()) - # reset obsolete:d by some other package state + # reset computed package state packages[p].obsolete = False + packages[p].rdepends = set() + packages[p].build_rdepends = set() # perform various package validations for p in sorted(packages.keys()): @@ -762,6 +764,22 @@ def validate_packages(args, packages): lvl = logging.ERROR logging.log(lvl, "package '%s' version '%s' has empty source tar file" % (p, vr)) + # build the set of packages which depends: on this package (rdepends), and + # the set of packages which build-depends: on it (build_rdepends) + for p in packages: + for hints in packages[p].version_hints.values(): + for k, a in [ + ('depends', 'rdepends'), + ('build-depends', 'build_rdepends') + ]: + if k in hints: + dpl = hints[k].split(',') + for dp in dpl: + dp = dp.strip() + dp = re.sub(r'(.*)\s+\(.*\)', r'\1', dp) + if dp in packages: + getattr(packages[dp], a).add(p) + # make another pass to verify a source tarfile exists for every install # tarfile version for p in packages.keys(): @@ -1100,11 +1118,18 @@ def write_setup_ini(args, packages, arch): # which case we should emit a 'Source:' line, and the package is # also itself emitted. - if hints.get('depends', '') or requires: - print("depends2: %s" % hints.get('depends', ''), file=f) + if version in po.versions(): + if hints.get('depends', '') or requires: + print("depends2: %s" % hints.get('depends', ''), file=f) - if hints.get('obsoletes', ''): - print("obsoletes: %s" % hints['obsoletes'], file=f) + if hints.get('obsoletes', ''): + print("obsoletes: %s" % hints['obsoletes'], file=f) + + if hints.get('provides', ''): + print("provides: %s" % hints['provides'], file=f) + + if hints.get('conflicts', ''): + print("conflicts: %s" % hints['conflicts'], file=f) if s: src_hints = packages[s].version_hints.get(version, {}) @@ -1120,12 +1145,6 @@ def write_setup_ini(args, packages, arch): if bd: print("build-depends: %s" % ', '.join(bd), file=f) - if hints.get('provides', ''): - print("provides: %s" % hints['provides'], file=f) - - if hints.get('conflicts', ''): - print("conflicts: %s" % hints['conflicts'], file=f) - # helper function to output details for a particular tar file def tar_line(p, category, v, f): diff --git a/calm/pkg2html.py b/calm/pkg2html.py index b43ed71..b2080dc 100755 --- a/calm/pkg2html.py +++ b/calm/pkg2html.py @@ -119,6 +119,13 @@ def ensure_dir_exists(args, path): os.chmod(path, 0o755) +# +# format a unix epoch time (UTC) +# +def tsformat(ts): + return time.strftime('%Y-%m-%d %H:%M', time.gmtime(ts)) + + # # # @@ -280,7 +287,7 @@ def update_package_listings(args, packages): name = v + ' (source)' target = "%s-%s-src" % (p.orig_name, v) test = 'test' if 'test' in p.version_hints[v] else 'stable' - ts = time.strftime('%Y-%m-%d %H:%M', time.gmtime(p.tar(v).mtime)) + ts = tsformat(p.tar(v).mtime) print('<tr><td>%s</td><td class="right">%d KiB</td><td>%s</td><td>[<a href="../%s/%s/%s">list of files</a>]</td><td>%s</td></tr>' % (name, size, ts, arch, pn, target, test), file=f) for version in sorted(packages[arch][p].versions(), key=lambda v: SetupVersion(v)): diff --git a/calm/repology.py b/calm/repology.py new file mode 100644 index 0000000..2e19aba --- /dev/null +++ b/calm/repology.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2022 Jon Turney +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +# + +# +# get 'latest upstream version' information from repology.org (note that this +# may not exist for some packages, e.g. where upstream doesn't do releases) +# + +import json +import logging +import time +import urllib.request + +REPOLOGY_API_URL = 'https://repology.org/api/v1/projects/' +last_check = 0 + + +def repology_fetch_versions(): + upstream_versions = {} + last_pn = '' + + while True: + url = REPOLOGY_API_URL + if last_pn: + url = url + last_pn + '/' + url += '?inrepo=cygwin' + + r = urllib.request.urlopen(url) + j = json.loads(r.read().decode('utf-8')) + + for pn in sorted(j.keys()): + p = j[pn] + + # first, pick out the version which repology has called newest + newest_version = None + for i in p: + if i['status'] == 'newest': + newest_version = i['version'] + break + else: + continue + + # next, assign that version to all the corresponding cygwin source + # packages + # + # (multiple cygwin source packages can correspond to a single + # canonical repology package name, e.g. foo and mingww64-arch-foo) + for i in p: + if i['repo'] == "cygwin": + source_pn = i['srcname'] + upstream_versions[source_pn] = newest_version + + if pn == last_pn: + break + else: + last_pn = pn + + return upstream_versions + + +def annotate_packages(args, packages): + # rate limit to daily + global last_check + if (time.time() - last_check) < (24 * 60 * 60): + logging.info("not consulting %s due to ratelimit" % (REPOLOGY_API_URL)) + return + + logging.info("consulting %s" % (REPOLOGY_API_URL)) + uv = repology_fetch_versions() + + for pn in uv: + spn = pn + '-src' + for arch in packages: + if spn in packages[arch]: + packages[arch][spn].upstream_version = uv[pn] + + last_check = time.time() diff --git a/calm/reports.py b/calm/reports.py new file mode 100644 index 0000000..bf77f3f --- /dev/null +++ b/calm/reports.py @@ -0,0 +1,196 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2022 Jon Turney +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +# + +import io +import os +import re +import textwrap +import types + +from . import maintainers +from . import package +from . import pkg2html +from . import utils +from .version import SetupVersion + + +def template(title, body, f): + os.fchmod(f.fileno(), 0o755) + + print(textwrap.dedent('''\ + <!DOCTYPE html> + <html> + <head> + <link rel="stylesheet" type="text/css" href="/static/builds/style.css"/> + <title>{0}</title> + </head> + <body> + <div id="main"> + <h1>{0}</h1>''').format(title), file=f) + + print(body, file=f) + + print(textwrap.dedent('''\ + </div> + </body> + </html>'''), file=f) + + +def linkify(pn): + return '<a href="/packages/summary/{0}-src.html">{0}</a>'.format(pn) + + +# +# produce a report of unmaintained packages +# +def unmaintained(args, packages, reportsdir): + mlist = maintainers.read(args, None) + pkg_maintainers = maintainers.invert(mlist) + + um_list = [] + + arch = 'x86_64' + # XXX: look into how we can make this 'src', after x86 is dropped + for p in packages[arch]: + po = packages[arch][p] + + if po.kind != package.Kind.source: + continue + + if 'ORPHANED' not in pkg_maintainers[po.orig_name]: + continue + + # the highest version we have + v = sorted(po.versions(), key=lambda v: SetupVersion(v), reverse=True)[0] + + # determine the number of unique rdepends over all subpackages (and + # likewise build_rdepends) + # + # zero rdepends makes this package a candidate for removal, whereas lots + # means it's important to update it. + rdepends = set() + build_rdepends = set() + for subp in po.is_used_by: + rdepends.update(packages[arch][subp].rdepends) + build_rdepends.update(packages[arch][subp].build_rdepends) + + up = types.SimpleNamespace() + up.po = po + up.v = v + up.upstream_v = getattr(po, 'upstream_version', 'unknown') + up.ts = po.tar(v).mtime + up.rdepends = len(rdepends) + up.build_rdepends = len(build_rdepends) + + # some packages are mature. If 'v' is still latest upstream version, + # then maybe we don't need to worry about this package quite as much... + if SetupVersion(v)._V == SetupVersion(up.upstream_v)._V: + up.upstream_v += " (unchanged)" + + um_list.append(up) + + body = io.StringIO() + print('<p>Packages without a maintainer.</p>', file=body) + + print('<table class="grid">', file=body) + print('<tr><th>last updated</th><th>package</th><th>version</th><th>upstream version</th><th>rdepends</th><th>build_rdepends</th></tr>', file=body) + + for up in sorted(um_list, key=lambda i: (i.rdepends + i.build_rdepends, i.ts), reverse=True): + po = up.po + print('<tr><td>%s</td><td>%s</td><td>%s</td><td>%s</td><td>%s</td><td>%s</td></tr>' % + (pkg2html.tsformat(up.ts), linkify(po.orig_name), up.v, up.upstream_v, up.rdepends, up.build_rdepends), file=body) + + print('</table>', file=body) + + unmaintained = os.path.join(reportsdir, 'unmaintained.html') + with utils.open_amifc(unmaintained) as f: + template('Unmaintained packages', body.getvalue(), f) + + +# produce a report of deprecated packages +# +def deprecated(args, packages, reportsdir): + dep_list = [] + + arch = 'x86_64' + # XXX: look into how we can make this 'src', after x86 is dropped + for p in packages[arch]: + po = packages[arch][p] + + if po.kind != package.Kind.binary: + continue + + if not re.match(r'^lib.*\d', p): + continue + + bv = po.best_version + es = po.version_hints[bv].get('external-source', None) + if not es: + continue + + if packages[arch][es].best_version == bv: + continue + + if po.tar(bv).is_empty: + continue + + # an old version of a shared library + depp = types.SimpleNamespace() + depp.pn = p + depp.po = po + depp.v = bv + depp.ts = po.tar(bv).mtime + depp.rdepends = len(po.rdepends) + + dep_list.append(depp) + + body = io.StringIO() + print(textwrap.dedent('''\ + <p>Packages for old soversions. (The corresponding source package produces a + newer soversion, or has stopped producing this soversion).</p>'''), file=body) + + print('<table class="grid">', file=body) + print('<tr><th>package</th><th>version</th><th>rdepends</th></tr>', file=body) + + for depp in sorted(dep_list, key=lambda i: (i.rdepends, i.ts), reverse=True): + po = depp.po + print('<tr><td>%s</td><td>%s</td><td>%s</td></tr>' % + (linkify(depp.pn), depp.v, depp.rdepends), file=body) + + print('</table>', file=body) + + deprecated = os.path.join(reportsdir, 'deprecated_so.html') + with utils.open_amifc(deprecated) as f: + template('Deprecated shared library packages', body.getvalue(), f) + + +# +def do_reports(args, packages): + if args.dryrun: + return + + reportsdir = os.path.join(args.htdocs, 'reports') + pkg2html.ensure_dir_exists(args, reportsdir) + + unmaintained(args, packages, reportsdir) + deprecated(args, packages, reportsdir)