[calm - Cygwin server-side packaging maintenance script] branch master, updated. 20160705-101-g6aa625a
jturney-9JcytcrH/[email protected]
| Newsgroups | gmane.os.cygwin.cvs.apps |
|---|---|
| Message-ID | <[email protected]> |
https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=6aa625a6a83ae615a859f61993696a1665340690 commit 6aa625a6a83ae615a859f61993696a1665340690 Author: Jon Turney <[email protected]> Date: Thu Oct 5 12:23:19 2017 +0100 Upgrade some checks from a warning to an error Make "curr: version isn't the most recent non-test: version" and "empty but not in _obsolete category" errors. Fix tests by fixing up package timestamps https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=4a10c12553a645bd719c056fdc9b463f1065702f commit 4a10c12553a645bd719c056fdc9b463f1065702f Author: Jon Turney <[email protected]> Date: Thu Oct 5 14:25:56 2017 +0100 Update past_mistakes Clean out things no longer needed https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=900929448bf1c7852598ebe9ea632faa2a475276 commit 900929448bf1c7852598ebe9ea632faa2a475276 Author: Jon Turney <[email protected]> Date: Thu Oct 5 13:09:40 2017 +0100 Add strict handling for some more warnings in read_package In particular, the hyphen-in-version warning is now treated as an error in strict (upload) mode. https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=52f0a5f18d2173c710eb603ee0675a17f91b9457 commit 52f0a5f18d2173c710eb603ee0675a17f91b9457 Author: Jon Turney <[email protected]> Date: Wed Oct 4 16:07:22 2017 +0100 Also use irker to notify of important events Also, don't fail if irkerd isn't running. https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=3673949b6259839db7526584efbe8de432d41db4 commit 3673949b6259839db7526584efbe8de432d41db4 Author: Jon Turney <[email protected]> Date: Wed Oct 4 15:41:41 2017 +0100 Add more version comparision tests Test data taken from RPM Diff: --- TODO | 1 - calm/calm.py | 9 ++++- calm/irk.py | 13 ++++-- calm/package.py | 26 ++++++++++--- calm/past_mistakes.py | 47 +---------------------- test/test_calm.py | 99 +++++++++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 138 insertions(+), 57 deletions(-) diff --git a/TODO b/TODO index e82b165..07510d5 100644 --- a/TODO +++ b/TODO @@ -1,5 +1,4 @@ * more than 2 versions possible -* use irkerd to report when calm failed due to an error? * upload a hash at the same time as package, and pass that through to setup.ini * mksetupini should have an okmissing option for override.hint which names non-existent versions * use ./setup.hint inside the tar file, avoiding all the hint/tar coherence problems diff --git a/calm/calm.py b/calm/calm.py index 72325f7..14d4317 100755 --- a/calm/calm.py +++ b/calm/calm.py @@ -66,6 +66,7 @@ import time from .abeyance_handler import AbeyanceHandler from .buffering_smtp_handler import BufferingSMTPHandler from . import common_constants +from . import irk from . import maintainers from . import package from . import pkg2html @@ -246,7 +247,9 @@ def process_uploads(args, state): for arch in common_constants.ARCHES: # use merged package list state.packages[arch] = merged_packages[arch] - logging.debug("added %d (%s) + %d (noarch) + %d (src) packages from maintainer %s" % (len(scan_result[arch].packages), arch, len(scan_result['noarch'].packages), len(scan_result['src'].packages), name)) + msg = "added %d (%s) + %d (noarch) + %d (src) packages from maintainer %s" % (len(scan_result[arch].packages), arch, len(scan_result['noarch'].packages), len(scan_result['src'].packages), name) + logging.debug(msg) + irk.irk(msg) # record updated reminder times for maintainers maintainers.Maintainer.update_reminder_times(mlist) @@ -488,6 +491,7 @@ def do_daemon(args, state): with context: logging_setup(args) logging.info("calm daemon started, pid %d" % (os.getpid())) + irk.irk("calm daemon started") state.packages = {} @@ -531,6 +535,9 @@ def do_daemon(args, state): except Exception as e: with mail_logs(args.email, toaddrs=args.email, subject='calm stopping due to unhandled exception', thresholdLevel=logging.ERROR) as leads_email: logging.error("exception %s" % (type(e).__name__), exc_info=True) + irk.irk("calm daemon stopped due to unhandled exception") + else: + irk.irk("calm daemon stopped") logging.info("calm daemon stopped") diff --git a/calm/irk.py b/calm/irk.py index b9f8957..47cb112 100755 --- a/calm/irk.py +++ b/calm/irk.py @@ -26,13 +26,16 @@ def send(s, target, message): def irk(message, target=DEFAULT_TARGET, server=DEFAULT_SERVER): - s = connect(server) - if "irc:" not in target and "ircs:" not in target: - target = "irc://chat.freenode.net/{0}".format(target) + try: + s = connect(server) + if "irc:" not in target and "ircs:" not in target: + target = "irc://chat.freenode.net/{0}".format(target) - send(s, target, message) + send(s, target, message) - s.close() + s.close() + except ConnectionRefusedError: + pass def main(): diff --git a/calm/package.py b/calm/package.py index cf9d242..ed4cf00 100755 --- a/calm/package.py +++ b/calm/package.py @@ -259,11 +259,16 @@ def read_package(packages, basedir, dirpath, files, strict=False, remove=[], upl # we already know P to split unambiguously), but this is a bad # idea. if '-' in v: - lvl = logging.WARNING if p not in past_mistakes.hyphen_in_version else logging.INFO + if p not in past_mistakes.hyphen_in_version: + lvl = strict_lvl + warnings = True + else: + lvl = logging.INFO logging.log(lvl, "file '%s' in package '%s' contains '-' in version" % (f, p)) if not v[0].isdigit(): - logging.warning("file '%s' in package '%s' has a version which doesn't start with a digit" % (f, p)) + logging.log(strict_lvl, "file '%s' in package '%s' has a version which doesn't start with a digit" % (f, p)) + warnings = True # if not there already, add to version-release list vr_list.add('%s-%s' % (v, r)) @@ -329,7 +334,8 @@ def read_package(packages, basedir, dirpath, files, strict=False, remove=[], upl packages[p].skip = any(['skip' in version_hints[vr] for vr in vr_list]) elif (len(files) > 0) and (relpath.count(os.path.sep) > 1): - logging.warning("no .hint files in %s but has files: %s" % (dirpath, ', '.join(files))) + logging.log(strict_lvl, "no .hint files in %s but has files: %s" % (dirpath, ', '.join(files))) + warnings = True if strict: return warnings @@ -553,13 +559,17 @@ def validate_packages(args, packages): elif 'curr' not in packages[p].stability and 'curr' not in getattr(args, 'okmissing', []): logging.warning("package '%s' doesn't have a curr version" % (p)) - # warn if the curr: version isn't the most recent non-test: version + # error if the curr: version isn't the most recent non-test: version for v in sorted(packages[p].vermap.keys(), key=lambda v: packages[p].vermap[v]['mtime'], reverse=True): if 'test' in packages[p].version_hints[v]: continue if packages[p].stability['curr'] != v: - lvl = logging.WARNING if p not in past_mistakes.mtime_anomalies else logging.DEBUG + if p in past_mistakes.mtime_anomalies: + lvl = logging.DEBUG + else: + lvl = logging.ERROR + error = True logging.log(lvl, "package '%s' version '%s' is most recent non-test version, but version '%s' is curr:" % (p, v, packages[p].stability['curr'])) break @@ -584,7 +594,11 @@ def validate_packages(args, packages): if 'source' not in packages[p].vermap[vr]: if 'install' in packages[p].vermap[vr]: if packages[p].tars[packages[p].vermap[vr]['install']].is_empty: - lvl = logging.WARNING if p not in past_mistakes.empty_but_not_obsolete else logging.DEBUG + if p in past_mistakes.empty_but_not_obsolete: + lvl = logging.DEBUG + else: + lvl = logging.ERROR + error = True logging.log(lvl, "package '%s' version '%s' has empty install tar file and no source, but it's not in the _obsolete category" % (p, vr)) # make another pass to verify a source tarfile exists for every install diff --git a/calm/past_mistakes.py b/calm/past_mistakes.py index ab925a1..de99439 100644 --- a/calm/past_mistakes.py +++ b/calm/past_mistakes.py @@ -114,19 +114,15 @@ nonunique_versions = [ 'gnome-panel-doc', 'gtk2.0-engines-svg', 'guile-doc', - 'guile-gv', # dropped pending guile-2 - 'info', # something went wrong with package build? 'kdepasswd', # dropped from split kde-baseapps 'kdialog', # split out from kde-baseapps 'keditbookmarks', # split out from kde-baseapps 'kexi', # split out from calligra 'kfind', # split out from kde-baseapps 'kfilereplace', # split out from kdewebdev - 'khelpcenter', # split out from kde-runtime 'kimagemapeditor', # split out from kdewebdev 'klinkstatus', # split out from kdewebdev 'konqueror', # split out from kde-baseapps - 'konqueror-devel', # split out from kde-baseapps 'libatomic_ops-devel', # separated out from libgc 'libcaca-doc', # dropped pending fix for current doxygen 'libfltk-doc', @@ -139,68 +135,31 @@ nonunique_versions = [ 'minizip', 'mutter-doc', 'ocaml-camlp4', # ocaml-camlp4 removed from ocaml distribution after 4.01.0 - 'ocaml-gv', # dropped pending ocaml cleanup 'okular4-part', # changed to okular5-part in 17.04 - 'python-clang', # split out from clang - 'python3-clang', # split out from clang 'python-spiceclientgtk', # gtk2 dropped from spice-gtk 'sng-debuginfo', 'socat-debuginfo', # debuginfo for test version when curr has no debuginfo 'sqlite3-zlib', # sqlite3-zlib removed in 3.8.10, use sqlite3-compress instead - 'texinfo-debuginfo', # something went wrong with package build? 'w3m-img', ] # packages with an empty install file, no source, but aren't obsolete empty_but_not_obsolete = [ - 'gcc-java', # gcc-java will be obsolete in test version 6.3 - 'gdal-debuginfo', # version 2.1.2-1 has empty debuginfo - 'libgcj-common', - 'libgcj16', - 'libpopt0', # version 1.16-1 was empty (x8_64) - 'libpopt-devel', # - 'mutt-debuginfo', # version 1.7.2-1 has empty debuginfo + 'libpopt0', # version 1.16-1 was empty + 'libpopt-devel', # version 1.16-1 was empty (x86_64) ] # packages with timestamp anomalies mtime_anomalies = [ - 'gcc-ada', - 'gcc-cilkplus', - 'gcc-core', - 'gcc-debuginfo', - 'gcc-fortran', - 'gcc-g++', 'gcc-java', - 'gcc-objc', - 'gcc-objc++', 'gcc-tools-epoch2-autoconf', 'glproto', 'gv-debuginfo', - 'info', - 'libatomic1', - 'libcilkrts5', - 'libgcc1', 'libgcj-common', 'libgcj16', - 'libgfortran3', - 'libgomp1', - 'libobjc4', - 'libquadmath0', - 'libssp0', - 'libstdc++6', - 'opencv', # 2.4.11-1 was created when changed from source-only to install 'python-gtk2.0', 'python-gtk2.0-demo', 'python-gtk2.0-devel', - 'subversion', - 'subversion-debuginfo', - 'subversion-devel', - 'subversion-gnome', - 'subversion-httpd', - 'subversion-perl', - 'subversion-python', - 'subversion-ruby', - 'subversion-tools', - 'texinfo-debuginfo', + 'python-wx', # timestamps reset when split out from wxWidgets 'xextproto', ] diff --git a/test/test_calm.py b/test/test_calm.py index d1cf3af..0f30e85 100755 --- a/test/test_calm.py +++ b/test/test_calm.py @@ -172,6 +172,68 @@ class CalmTest(unittest.TestCase): ["2a", "2.0", -1], ["1.0", "1.fc4", 1], ["3.0.0_fc", "3.0.0.fc", 0], + # from RPM tests + ["1.0", "1.0", 0], + ["1.0", "2.0", -1], + ["2.0", "1.0", 1], + ["2.0.1", "2.0.1", 0], + ["2.0", "2.0.1", -1], + ["2.0.1", "2.0", 1], + ["2.0.1a", "2.0.1a", 0], + ["2.0.1a", "2.0.1", 1], + ["2.0.1", "2.0.1a", -1], + ["5.5p1", "5.5p1", 0], + ["5.5p1", "5.5p2", -1], + ["5.5p2", "5.5p1", 1], + ["5.5p10", "5.5p10", 0], + ["5.5p1", "5.5p10", -1], + ["5.5p10", "5.5p1", 1], + ["10xyz", "10.1xyz", -1], + ["10.1xyz", "10xyz", 1], + ["xyz10", "xyz10", 0], + ["xyz10", "xyz10.1", -1], + ["xyz10.1", "xyz10", 1], + ["xyz.4", "xyz.4", 0], + ["xyz.4", "8", -1], + ["8", "xyz.4", 1], + ["xyz.4", "2", -1], + ["2", "xyz.4", 1], + ["5.5p2", "5.6p1", -1], + ["5.6p1", "5.5p2", 1], + ["5.6p1", "6.5p1", -1], + ["6.5p1", "5.6p1", 1], + ["6.0.rc1", "6.0", 1], + ["6.0", "6.0.rc1", -1], + ["10b2", "10a1", 1], + ["10a2", "10b2", -1], + ["1.0aa", "1.0aa", 0], + ["1.0a", "1.0aa", -1], + ["1.0aa", "1.0a", 1], + ["10.0001", "10.0001", 0], + ["10.0001", "10.1", 0], + ["10.1", "10.0001", 0], + ["10.0001", "10.0039", -1], + ["10.0039", "10.0001", 1], + ["4.999.9", "5.0", -1], + ["5.0", "4.999.9", 1], + ["20101121", "20101121", 0], + ["20101121", "20101122", -1], + ["20101122", "20101121", 1], + ["2_0", "2_0", 0], + ["2.0", "2_0", 0], + ["2_0", "2.0", 0], + ["a", "a", 0], + ["a+", "a+", 0], + ["a+", "a_", 0], + ["a_", "a+", 0], + ["+a", "+a", 0], + ["+a", "_a", 0], + ["_a", "+a", 0], + ["+_", "+_", 0], + ["_+", "+_", 0], + ["_+", "_+", 0], + ["+", "_", 0], + ["_", "+", 0], ] for d in test_data: @@ -314,6 +376,43 @@ class CalmTest(unittest.TestCase): # remove !ready files os.system("find testdata/homes -name !ready -exec rm {} \;") + # fix up package timestamps so highest version is also latest + # (git doesn't store timestamps, so they will all be dated the time of checkout) + relarea_x86 = os.path.join('testdata', 'relarea', 'x86', 'release') + relarea_noarch = os.path.join('testdata', 'relarea', 'noarch', 'release') + touches = [(os.path.join(relarea_x86, 'cygwin', 'cygwin-2.2.0-1.tar.xz'), '2016-11-01'), + (os.path.join(relarea_x86, 'cygwin', 'cygwin-2.2.0-1-src.tar.xz'), '2016-11-01'), + (os.path.join(relarea_x86, 'cygwin', 'cygwin-2.2.1-1.tar.xz'), '2016-11-02'), + (os.path.join(relarea_x86, 'cygwin', 'cygwin-2.2.1-1-src.tar.xz'), '2016-11-02'), + (os.path.join(relarea_x86, 'cygwin', 'cygwin-debuginfo', 'cygwin-debuginfo-2.2.0-1.tar.xz'), '2016-11-01'), + (os.path.join(relarea_x86, 'cygwin', 'cygwin-debuginfo', 'cygwin-debuginfo-2.2.1-1.tar.xz'), '2016-11-02'), + (os.path.join(relarea_x86, 'cygwin', 'cygwin-devel', 'cygwin-devel-2.2.0-1.tar.xz'), '2016-11-01'), + (os.path.join(relarea_x86, 'cygwin', 'cygwin-devel', 'cygwin-devel-2.2.1-1.tar.xz'), '2016-11-02'), + (os.path.join(relarea_x86, 'base-cygwin', 'base-cygwin-3.6-1.tar.xz'), '2016-11-02'), + (os.path.join(relarea_x86, 'per-version', 'per-version-4.0-1.tar.xz'), '2017-04-09'), + (os.path.join(relarea_x86, 'per-version', 'per-version-4.0-1-src.tar.xz'), '2017-04-09'), + (os.path.join(relarea_x86, 'rpm-doc', 'rpm-doc-4.1-2.tar.bz2'), '2016-11-02'), + (os.path.join(relarea_x86, 'rpm-doc', 'rpm-doc-4.1-2-src.tar.bz2'), '2016-11-02'), + (os.path.join(relarea_x86, 'staleversion', 'staleversion-240-1.tar.xz'), '2017-04-07'), + (os.path.join(relarea_x86, 'staleversion', 'staleversion-240-1-src.tar.xz'), '2017-04-07'), + (os.path.join(relarea_x86, 'staleversion', 'staleversion-242-0.tar.xz'), '2017-04-08'), + (os.path.join(relarea_x86, 'staleversion', 'staleversion-242-0-src.tar.xz'), '2017-04-08'), + (os.path.join(relarea_x86, 'staleversion', 'staleversion-243-0.tar.xz'), '2017-04-09'), + (os.path.join(relarea_x86, 'staleversion', 'staleversion-243-0-src.tar.xz'), '2017-04-09'), + (os.path.join(relarea_x86, 'staleversion', 'staleversion-250-0.tar.xz'), '2017-04-10'), + (os.path.join(relarea_x86, 'staleversion', 'staleversion-250-0-src.tar.xz'), '2017-04-10'), + (os.path.join(relarea_x86, 'staleversion', 'staleversion-251-0.tar.xz'), '2017-04-09'), + (os.path.join(relarea_x86, 'staleversion', 'staleversion-251-0-src.tar.xz'), '2017-04-09'), + (os.path.join(relarea_x86, 'staleversion', 'staleversion-260-0.tar.xz'), '2017-04-12'), + (os.path.join(relarea_x86, 'staleversion', 'staleversion-260-0-src.tar.xz'), '2017-04-12'), + (os.path.join(relarea_x86, 'keychain', 'keychain-2.6.8-1.tar.bz2'), '2016-11-02'), + (os.path.join(relarea_x86, 'keychain', 'keychain-2.6.8-1-src.tar.bz2'), '2016-11-02'), + (os.path.join(relarea_noarch, 'perl-Net-SMTP-SSL', 'perl-Net-SMTP-SSL-1.03-1.tar.xz'), '2016-11-01'), + (os.path.join(relarea_noarch, 'perl-Net-SMTP-SSL', 'perl-Net-SMTP-SSL-1.03-1-src.tar.xz'), '2016-11-01'), + ] + for (f, t) in touches: + os.system('touch %s -d %s' % (f, t)) + if __name__ == '__main__': logging.getLogger().setLevel(logging.INFO)