[calm - Cygwin server-side packaging maintenance script] branch master, updated. 20200401-34-g4f7df6c
Jon TURNEY via Cygwin-apps-cvs <[email protected]>
| Newsgroups | gmane.os.cygwin.cvs.apps |
|---|---|
| Message-ID | <[email protected]> |
https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=4f7df6cac9c32c6b2357b245d0b9c4069369f5a3 commit 4f7df6cac9c32c6b2357b245d0b9c4069369f5a3 Author: Jon Turney <[email protected]> Date: Thu May 28 15:06:04 2020 +0100 Allow maintainers to suppress informative-only mail If the !email file contains a line saying 'quiet', the log output will not be mailed if the log only contains INFO severity messages. https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=2e66e6c8d57f6c637ce2cd896a5fe1b8a713e9a7 commit 2e66e6c8d57f6c637ce2cd896a5fe1b8a713e9a7 Author: Jon Turney <[email protected]> Date: Thu May 28 13:50:05 2020 +0100 Move static methods out of Maintainers class We're not writing C++, they are still scoped to the module namespace, so making them @staticmethod just makes using them require more verbiage. https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=23c99abfb263c41f3d386ae392e78a4d00d84b7b commit 23c99abfb263c41f3d386ae392e78a4d00d84b7b Author: Jon Turney <[email protected]> Date: Wed May 27 15:20:23 2020 +0100 Ignore extra whitespace in cygwin-pkg-maint file https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=5c406d9fd9d815d8b6869474146a2754bfcd5157 commit 5c406d9fd9d815d8b6869474146a2754bfcd5157 Author: Jon Turney <[email protected]> Date: Tue May 26 20:36:19 2020 +0100 Try to send email to Bcc: list, even if To: is empty Also log sendmail exit status https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=b155c6f4a5ac611f109ee814d348a282b006478c commit b155c6f4a5ac611f109ee814d348a282b006478c Author: Jon Turney <[email protected]> Date: Wed May 27 13:58:13 2020 +0100 Log an error if maintainer doesn't have an email address Diff: --- calm/buffering_smtp_handler.py | 4 +- calm/calm.py | 9 +- calm/compare-arches | 4 +- calm/maintainers.py | 216 +++++++++++++++++++++-------------------- calm/mkgitoliteconf.py | 4 +- calm/mkmaintdir | 4 +- calm/package.py | 8 +- calm/pkg2html.py | 4 +- test/test_calm.py | 6 +- 9 files changed, 133 insertions(+), 126 deletions(-) diff --git a/calm/buffering_smtp_handler.py b/calm/buffering_smtp_handler.py index fcc5290..c826d34 100644 --- a/calm/buffering_smtp_handler.py +++ b/calm/buffering_smtp_handler.py @@ -84,12 +84,12 @@ class BufferingSMTPHandler(logging.handlers.BufferingHandler): print('-' * 40) print(msg) print('-' * 40) - elif len(self.toaddrs) > 0: + else: with subprocess.Popen(['/usr/sbin/sendmail', '-t', '-oi', '-f', self.fromaddr], stdin=subprocess.PIPE) as p: p.communicate(m.as_bytes()) + logging.debug('sendmail: msgid %s, exit status %d' % (m['Message-Id'], p.returncode)) self.buffer = [] - logging.debug('sent mail with msgid %s' % (m['Message-Id'])) def shouldFlush(self, record): # the capacity we pass to BufferingHandler is irrelevant since we diff --git a/calm/calm.py b/calm/calm.py index dc5df16..88ebdd7 100755 --- a/calm/calm.py +++ b/calm/calm.py @@ -131,17 +131,18 @@ def process_relarea(args): def process_uploads(args, state): # read maintainer list - mlist = maintainers.Maintainer.read(args, getattr(args, 'orphanmaint', None)) + mlist = maintainers.read(args, getattr(args, 'orphanmaint', None)) # make the list of all packages - all_packages = maintainers.Maintainer.all_packages(mlist) + all_packages = maintainers.all_packages(mlist) # for each maintainer for name in sorted(mlist.keys()): m = mlist[name] # also send a mail to each maintainer about their packages - with mail_logs(args.email, toaddrs=m.email, subject='%s for %s' % (state.subject, name), thresholdLevel=logging.INFO) as maint_email: # noqa: F841 + threshold = logging.WARNING if m.quiet else logging.INFO + with mail_logs(args.email, toaddrs=m.email, subject='%s for %s' % (state.subject, name), thresholdLevel=threshold, retainLevel=logging.INFO) as maint_email: # noqa: F841 # for each arch and noarch scan_result = {} @@ -260,7 +261,7 @@ def process_uploads(args, state): irk.irk("calm %s" % msg) # record updated reminder times for maintainers - maintainers.Maintainer.update_reminder_times(mlist) + maintainers.update_reminder_times(mlist) return state.packages diff --git a/calm/compare-arches b/calm/compare-arches index 37e1861..4e2dd19 100755 --- a/calm/compare-arches +++ b/calm/compare-arches @@ -91,8 +91,8 @@ def main(args): print("%s is only in arch %s" % (p, [a for a in exists if exists[a]])) # are there any packages which have a maintainer, but don't exist? - mlist = maintainers.Maintainer.read(args, getattr(args, 'orphanmaint', None)) - all_packages = maintainers.Maintainer.all_packages(mlist) + mlist = maintainers.read(args, getattr(args, 'orphanmaint', None)) + all_packages = maintainers.all_packages(mlist) for p in sorted(all_packages): if p not in union: diff --git a/calm/maintainers.py b/calm/maintainers.py index 7ffc12b..9d7bdee 100644 --- a/calm/maintainers.py +++ b/calm/maintainers.py @@ -65,6 +65,7 @@ class Maintainer(object): self.name = name self.email = email self.pkgs = pkgs + self.quiet = False # the mtime of this file records the timestamp reminder_file = os.path.join(self.homedir(), '!reminder-timestamp') @@ -102,122 +103,127 @@ class Maintainer(object): mlist.setdefault(name, Maintainer(name)) return mlist[name] - # add maintainers which have existing directories - @classmethod - def add_directories(self, mlist, homedirs): - self._homedirs = homedirs - - for n in os.listdir(homedirs): - if not os.path.isdir(os.path.join(homedirs, n)): - continue - - m = Maintainer._find(mlist, n) - - for e in ['!email', '!mail']: - email = os.path.join(homedirs, m.name, e) - if os.path.isfile(email): - with open(email) as f: - for l in f: - # one address per line, ignore blank and comment lines - if l.startswith('#'): - continue - l = l.strip() - if l: - m.email.append(l) - - return mlist - - # add maintainers from the package maintainers list, with the packages they - # maintain - @staticmethod - def add_packages(mlist, pkglist, orphanMaint=None): - with open(pkglist) as f: - for (i, l) in enumerate(f): - l = l.rstrip() - - # match lines of the form '<package> <maintainer(s)|status>' - match = re.match(r'^(\S+)\s+(.+)$', l) - if match: - pkg = match.group(1) - rest = match.group(2) - - # does rest starts with a status in all caps? - status_match = re.match(r'^([A-Z]+)\b.*$', rest) - if status_match: - status = status_match.group(1) - - # ignore packages marked as 'OBSOLETE' - if status == 'OBSOLETE': - continue - # orphaned packages get the default maintainer if we - # have one, otherwise they are assigned to 'ORPHANED' - elif status == 'ORPHANED': - if orphanMaint is not None: - m = orphanMaint - else: - m = status +# add maintainers which have existing directories +def add_directories(mlist, homedirs): + Maintainer._homedirs = homedirs - # also add any previous maintainer(s) listed - prevm = re.match(r'^ORPHANED\s\((.*)\)', rest) - if prevm: - m = m + '/' + prevm.group(1) + for n in os.listdir(homedirs): + if not os.path.isdir(os.path.join(homedirs, n)): + continue - else: - logging.error("unknown package status '%s' in line %s:%d: '%s'" % (status, pkglist, i, l)) + m = Maintainer._find(mlist, n) + + # !mail is the deprecated historical alternative + for e in ['!email', '!mail']: + email = os.path.join(homedirs, m.name, e) + if os.path.isfile(email): + with open(email) as f: + for l in f: + # one address per line, ignore blank and comment lines + if l.startswith('#'): continue + l = l.strip() + if l.lower() == 'quiet': + m.quiet = True + elif l: + m.email.append(l) + if not m.email: + logging.error("no email address known for maintainer '%s'" % (m.name)) + + return mlist + + +# add maintainers from the package maintainers list, with the packages they +# maintain +def add_packages(mlist, pkglist, orphanMaint=None): + with open(pkglist) as f: + for (i, l) in enumerate(f): + l = l.rstrip() + + # match lines of the form '<package> <maintainer(s)|status>' + match = re.match(r'^(\S+)\s+(.+)$', l) + if match: + pkg = match.group(1) + rest = match.group(2) + + # does rest starts with a status in all caps? + status_match = re.match(r'^([A-Z]+)\b.*$', rest) + if status_match: + status = status_match.group(1) + + # ignore packages marked as 'OBSOLETE' + if status == 'OBSOLETE': + continue + + # orphaned packages get the default maintainer if we + # have one, otherwise they are assigned to 'ORPHANED' + elif status == 'ORPHANED': + if orphanMaint is not None: + m = orphanMaint + else: + m = status + + # also add any previous maintainer(s) listed + prevm = re.match(r'^ORPHANED\s\((.*)\)', rest) + if prevm: + m = m + '/' + prevm.group(1) else: - m = rest - - # joint maintainers are separated by '/' - for name in m.split('/'): - - # is the maintainer name ascii? - # - # (despite containing spaces, think of these as an account - # name, rather than a display name) - try: - name.encode('ascii') - except UnicodeError: - logging.error("non-ascii maintainer name '%s' in line %s:%d, skipped" % (rest, pkglist, i)) - continue + logging.error("unknown package status '%s' in line %s:%d: '%s'" % (status, pkglist, i, l)) + continue + else: + m = rest - m = Maintainer._find(mlist, name) - m.pkgs.append(pkg) + # joint maintainers are separated by '/' + for name in m.split('/'): + name = name.strip() - else: - logging.error("unrecognized line in %s:%d: '%s'" % (pkglist, i, l)) + # is the maintainer name ascii? + # + # (despite containing spaces, think of these as an account + # name, rather than a display name) + try: + name.encode('ascii') + except UnicodeError: + logging.error("non-ascii maintainer name '%s' in line %s:%d, skipped" % (rest, pkglist, i)) + continue - return mlist + m = Maintainer._find(mlist, name) + m.pkgs.append(pkg) - # create maintainer list - @staticmethod - def read(args, orphanmaint=None): - mlist = {} - mlist = Maintainer.add_directories(mlist, args.homedir) - mlist = Maintainer.add_packages(mlist, args.pkglist, orphanmaint) + else: + logging.error("unrecognized line in %s:%d: '%s'" % (pkglist, i, l)) - return mlist + return mlist - # invert to a per-package list of maintainers - @staticmethod - def invert(mlist): - _pkgs = defaultdict(list) - # for each maintainer - for m in mlist.values(): - # for each package - for p in m.pkgs: - # add the maintainer name - _pkgs[p].append(m.name) - return _pkgs +# create maintainer list +def read(args, orphanmaint=None): + mlist = {} + mlist = add_directories(mlist, args.homedir) + mlist = add_packages(mlist, args.pkglist, orphanmaint) - @staticmethod - def update_reminder_times(mlist): - for m in mlist.values(): - m._update_reminder_time() + return mlist - # a list of all packages - @staticmethod - def all_packages(mlist): - return list(itertools.chain.from_iterable(mlist[m].pkgs for m in mlist)) + +# invert to a per-package list of maintainers +def invert(mlist): + _pkgs = defaultdict(list) + # for each maintainer + for m in mlist.values(): + # for each package + for p in m.pkgs: + # add the maintainer name + _pkgs[p].append(m.name) + + return _pkgs + + +def update_reminder_times(mlist): + for m in mlist.values(): + m._update_reminder_time() + + +# a list of all packages +def all_packages(mlist): + return list(itertools.chain.from_iterable(mlist[m].pkgs for m in mlist)) diff --git a/calm/mkgitoliteconf.py b/calm/mkgitoliteconf.py index 39c185c..0e12c6e 100755 --- a/calm/mkgitoliteconf.py +++ b/calm/mkgitoliteconf.py @@ -51,10 +51,10 @@ def transform_username(name): def do_main(args): # read maintainer list mlist = {} - mlist = maintainers.Maintainer.add_packages(mlist, args.pkglist, getattr(args, 'orphanmaint', None)) + mlist = maintainers.add_packages(mlist, args.pkglist, getattr(args, 'orphanmaint', None)) # make the list of all packages - maintainers.Maintainer.all_packages(mlist) + maintainers.all_packages(mlist) # invert to a per-package list of maintainers pkgs = defaultdict(list) diff --git a/calm/mkmaintdir b/calm/mkmaintdir index 2023ae6..88c385a 100755 --- a/calm/mkmaintdir +++ b/calm/mkmaintdir @@ -71,8 +71,8 @@ def main(args): # create maintainer list mlist = {} - mlist = maintainers.Maintainer.add_directories(mlist, args.homedir) - mlist = maintainers.Maintainer.add_packages(mlist, args.pkglist, args.orphanmaint) + mlist = maintainers.add_directories(mlist, args.homedir) + mlist = maintainers.add_packages(mlist, args.pkglist, args.orphanmaint) # create or suggest removal for each maintainer directory for name in sorted(mlist.keys()): diff --git a/calm/package.py b/calm/package.py index 66d03df..3ad4252 100755 --- a/calm/package.py +++ b/calm/package.py @@ -915,10 +915,10 @@ def validate_package_maintainers(args, packages): # read maintainer list mlist = {} - mlist = maintainers.Maintainer.add_packages(mlist, args.pkglist) + mlist = maintainers.add_packages(mlist, args.pkglist) # make the list of all packages - all_packages = maintainers.Maintainer.all_packages(mlist) + all_packages = maintainers.all_packages(mlist) # validate that all packages are in the package list for p in sorted(packages): @@ -1165,8 +1165,8 @@ def write_repo_json(args, packages, f): for arch in packages: package_list.update(packages[arch]) - mlist = maintainers.Maintainer.read(args, None) - pkg_maintainers = maintainers.Maintainer.invert(mlist) + mlist = maintainers.read(args, None) + pkg_maintainers = maintainers.invert(mlist) pl = [] for pn in sorted(package_list): diff --git a/calm/pkg2html.py b/calm/pkg2html.py index f554f39..3750e81 100755 --- a/calm/pkg2html.py +++ b/calm/pkg2html.py @@ -135,8 +135,8 @@ def update_package_listings(args, packages): summaries = os.path.join(args.htdocs, 'summary') ensure_dir_exists(args, summaries) - mlist = maintainers.Maintainer.read(args, None) - pkg_maintainers = maintainers.Maintainer.invert(mlist) + mlist = maintainers.read(args, None) + pkg_maintainers = maintainers.invert(mlist) toremove = glob.glob(os.path.join(summaries, '*')) diff --git a/test/test_calm.py b/test/test_calm.py index 17aeb86..8c926bf 100755 --- a/test/test_calm.py +++ b/test/test_calm.py @@ -306,8 +306,8 @@ class CalmTest(unittest.TestCase): self.maxDiff = None mlist = {} - mlist = maintainers.Maintainer.add_directories(mlist, 'testdata/homes') - mlist = maintainers.Maintainer.add_packages(mlist, 'testdata/pkglist/cygwin-pkg-maint', None) + mlist = maintainers.add_directories(mlist, 'testdata/homes') + mlist = maintainers.add_packages(mlist, 'testdata/pkglist/cygwin-pkg-maint', None) compare_with_expected_file(self, 'testdata/pkglist', mlist) @@ -329,7 +329,7 @@ class CalmTest(unittest.TestCase): pkglist = ['after-ready', 'not-ready', 'testpackage', 'testpackage2'] mlist = {} - mlist = maintainers.Maintainer.add_directories(mlist, 'testdata/homes') + mlist = maintainers.add_directories(mlist, 'testdata/homes') m = mlist['Blooey McFooey'] m.pkgs.extend(pkglist + ['not-on-package-list'])