[calm - Cygwin server-side packaging maintenance script] branch master, updated. 20210408-8-g4c8d71e

https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=4c8d71e3a30bb0e342ad8c3da8e30f74554cfa28

commit 4c8d71e3a30bb0e342ad8c3da8e30f74554cfa28
Author: Jon Turney <[email protected]>
Date:   Sun May 2 20:22:26 2021 +0100

    Improve empty package checks
    
    An empty install package without depends should be in the '_obsolete'
    category.
    
    (We're now checking all install packages, not just ones without
    external-source: (which we ignored here for somewhat bogus reasons), so
    add additional exceptions to past_mistakes.)
    
    An empty install package with depends should be in the 'virtual'
    category (but there are many old obsoletion packages which look like
    that, so allow '_obsolete' as well).
    
    Also check for empty source packages (and then fix the empty rpm-doc-src
    we have in test data).

https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=242bfe789a63d420308397ccea6703a245267018

commit 242bfe789a63d420308397ccea6703a245267018
Author: Jon Turney <[email protected]>
Date:   Thu May 6 15:14:35 2021 +0100

    Update past_mistakes
    
    Drop versions which have been expired.

https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=cc898aa0954e7a615cba59b2f126dd271eced643

commit cc898aa0954e7a615cba59b2f126dd271eced643
Author: Jon Turney <[email protected]>
Date:   Tue May 4 18:28:11 2021 +0100

    Drop unused is_empty dict in validate_packages()

https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=727b539a9c0c4b9b8913e09c26bac21fd289596f

commit 727b539a9c0c4b9b8913e09c26bac21fd289596f
Author: Jon Turney <[email protected]>
Date:   Tue May 4 14:04:45 2021 +0100

    Clean test_html_writer output directory before test
    
    html files aren't generated if they already exist, so clean the
    test_html_writer output directory before the test, to ensure files are
    always generated.


Diff:
---
 calm/package.py                                    |  58 +++++++++++++--------
 calm/past_mistakes.py                              |  41 ++++++++++-----
 test/test_calm.py                                  |   5 ++
 .../x86/rpm-doc-src/rpm-doc-4.1-2-src              |   3 ++
 test/testdata/inifile/setup.ini.expected           |   4 +-
 test/testdata/process_arch/setup.ini.expected      |   4 +-
 .../x86/release/rpm-doc/rpm-doc-4.1-2-src.tar.bz2  | Bin 42 -> 188 bytes
 7 files changed, 77 insertions(+), 38 deletions(-)

diff --git a/calm/package.py b/calm/package.py
index ba1dec8..b223edf 100755
--- a/calm/package.py
+++ b/calm/package.py
@@ -574,7 +574,6 @@ def validate_packages(args, packages):
                         logging.debug("can't ensure package '%s' doesn't depends: on obsoleting '%s'" % (o, p))
 
         packages[p].vermap = {}
-        is_empty = {}
         has_source = False
         has_install = False
         has_nonempty_install = False
@@ -588,8 +587,7 @@ def validate_packages(args, packages):
                 else:
                     category = 'install'
                     has_install = True
-                    is_empty[t] = packages[p].tars[vr][t].is_empty
-                    if not is_empty[t]:
+                    if not packages[p].tars[vr][t].is_empty:
                         has_nonempty_install = True
 
                 if vr not in packages[p].vermap:
@@ -783,24 +781,42 @@ def validate_packages(args, packages):
                 if rv in packages[p].version_hints:
                     logging.warning("package '%s' replace-versions: lists version '%s', which is also available to install" % (p, rv))
 
-        # If the install tarball is empty and there is no source tarball, we
-        # should probably be marked obsolete
-        if not packages[p].skip:
-            for vr in packages[p].version_hints:
-                if '_obsolete' not in packages[p].version_hints[vr].get('category', '') and vr in packages[p].vermap:
-                    if ('source' not in packages[p].vermap[vr]) and ('external-source' not in packages[p].version_hints[vr]):
-                        if 'install' in packages[p].vermap[vr]:
-                            if packages[p].tar(vr, 'install').is_empty:
-                                if ((vr in past_mistakes.empty_but_not_obsolete.get(p, [])) or
-                                    ('empty-obsolete' in packages[p].version_hints[vr].get('disable-check', ''))):
-                                    lvl = logging.DEBUG
-                                else:
-                                    if 'external-source' in packages[p].version_hints[vr]:
-                                        lvl = logging.ERROR
-                                        error = True
-                                    else:
-                                        lvl = logging.WARNING
-                                logging.log(lvl, "package '%s' version '%s' has empty install tar file, but it's not in the _obsolete category" % (p, vr))
+        # If the install tarball is empty, we should probably either be marked
+        # obsolete (if we have no dependencies) or virtual (if we do)
+        if packages[p].kind == Kind.binary and not packages[p].skip:
+            for vr in packages[p].vermap:
+                if 'install' in packages[p].vermap[vr]:
+                    if packages[p].tar(vr, 'install').is_empty:
+                        # this classification relies on obsoleting packages
+                        # not being present in depends
+                        if packages[p].version_hints[vr].get('depends', ''):
+                            # also allow '_obsolete' because old obsoletion
+                            # packages depend on their replacement, but are not
+                            # obsoleted by it
+                            expected_categories = ['virtual', '_obsolete']
+                        else:
+                            expected_categories = ['_obsolete']
+
+                        if all(c not in packages[p].version_hints[vr].get('category', '').lower() for c in expected_categories):
+                            if ((vr in past_mistakes.empty_but_not_obsolete.get(p, [])) or
+                                ('empty-obsolete' in packages[p].version_hints[vr].get('disable-check', ''))):
+                                lvl = logging.DEBUG
+                            else:
+                                lvl = logging.ERROR
+                                error = True
+                            logging.log(lvl, "package '%s' version '%s' has empty install tar file, but it's not in %s category" % (p, vr, expected_categories))
+        # If the source tarball is empty, that can't be right!
+        elif packages[p].kind == Kind.source:
+            for vr in packages[p].vermap:
+                if 'source' in packages[p].vermap[vr]:
+                    if packages[p].tar(vr, 'source').is_empty:
+                        if ((vr in past_mistakes.empty_source.get(p, [])) and
+                            '_obsolete' in packages[p].version_hints[vr].get('category', '')):
+                            lvl = logging.DEBUG
+                        else:
+                            error = True
+                            lvl = logging.ERROR
+                        logging.log(lvl, "package '%s' version '%s' has empty source tar file" % (p, vr))
 
     # make another pass to verify a source tarfile exists for every install
     # tarfile version
diff --git a/calm/past_mistakes.py b/calm/past_mistakes.py
index d8f2084..064ad27 100644
--- a/calm/past_mistakes.py
+++ b/calm/past_mistakes.py
@@ -54,8 +54,6 @@ hyphen_in_version = {
     'mingw64-x86_64-hidapi-debuginfo': ['0.8.0-rc1'],
     'recode': ['3.7-beta2'],
     'recode-debuginfo': ['3.7-beta2'],
-    'socat': ['2.0.0-b8', '2.0.0-b9'],
-    'socat-debuginfo': ['2.0.0-b8', '2.0.0-b9'],
     'tack': ['1.07-20130713', '1.07-20150606'],
     'tack-debuginfo': ['1.07-20130713', '1.07-20150606'],
     'xemacs-mule-sumo': ['2007-04-27'],
@@ -137,22 +135,18 @@ nonunique_versions = [
     'w3m-img',
 ]
 
-# packages with an empty install file, no source, but aren't obsolete
+# empty install packages, that aren't obsolete
 #
 # don't add to this list, use 'disable-check: empty-obsolete' in pvr.hint instead
 empty_but_not_obsolete = {
-    'gambas3': ['3.12.0-1', '3.12.2-1', '3.13.0-1'],       # a metapackage
-    'gmp': ['6.1.0-3p1', '6.1.1-1', '6.1.2-1'],            # useless empty package, not autosupressed as it has depends
-    'isl': ['0.14.1-1', '0.16.1-1'],                       # useless empty package, not autosupressed as it has depends
-    'kdegames3': ['3.5.10-11'],                            # a metapackage
-    'kdewebdev': ['15.04.3-1', '16.08.3-1', '16.08.3-2'],  # a metapackage
-    'libao': ['1.1.0-1'],                                  # useless empty package, not autosupressed as it has depends (used to contain doc)
+    'freeglut-doc': ['3.0.0-1', '3.2.1-1'],                # should be obsoleted by libglut-devel which contains doc now
+    'isl': ['0.16.1-1'],                                   # useless empty package, not autosupressed as it has depends
     'libpopt-devel': ['1.16-1'],                           # version 1.16-1 was empty (x86_64)
     'libpopt0': ['1.16-1'],                                # version 1.16-1 was empty
-    'mbedtls': ['2.12.0-1', '2.14.1-1', '2.16.0-1'],       # useless empty package, not autosupressed as it has depends
-    'mpclib': ['1.0.2-2', '1.0.3-1', '1.1.0-1'],           # useless empty package, not autosupressed as it has depends
-    'mpfr': ['4.0.1-3p6', '4.0.1-4p11', '4.0.2-1'],        # useless empty package, not autosupressed as it has depends
-    'ocaml': ['4.02.3-2'],                                 # a metapackage
+    'mbedtls': ['2.16.0-1'],                               # useless empty package, not autosupressed as it has depends
+    'mpclib': ['1.1.0-1'],                                 # useless empty package, not autosupressed as it has depends
+    'mpfr': ['4.0.2-1'],                                   # useless empty package, not autosupressed as it has depends
+    'serf-debuginfo': ['1.3.8-1', '1.3.9-1'],              # empty presumably due to build problems
 }
 
 # packages with timestamp anomalies
@@ -195,3 +189,24 @@ maint_anomalies = {
 nonexistent_provides = [
     'perl5_026',
 ]
+
+# empty source packages
+#
+# (these usually have a corresponding hand-built empty install package, which
+# depends on it's replacement, and so are a lingering remnant of something not
+# properly obsoleted)
+empty_source = {
+    'SuiteSparse-src': ['4.0.2-1'],
+    'ash-src': ['20040127-5'],                   # obsoleted by dash
+    'checkx-src': ['0.2.1-1'],                   # obsoleted by run2
+    'db4.8-src': ['4.8.30-2'],                   # obsoleted by db
+    'gcc-tools-autoconf-src': ['2.59-11'],       # obsoleted by gcc-tools-epoch{1,2}-autoconf
+    'gcc-tools-automake-src': ['1.9.6-11'],      # obsoleted by gcc-tools-epoch{1,2}-automake
+    'lzma-src': ['4.32.7-10'],                   # obsoleted by xz
+    'mlcscope-src': ['99-1'],                    # obsoleted by cscope
+    'octave-forge-src': ['20140215-1'],
+    'octave-octcdf-src': ['1.1.7-99'],
+    'perl-File-Slurp-Unicode-src': ['0.7.1-2'],  # obsoleted by perl-File-Slurp
+    'pinentry-qt3-src': ['0.7.6-3'],             # obsoleted by pinentry-qt
+    'xerces-c-devel-src': ['2.8.0-1'],           # obsoleted by libxerces-c-devel
+}
diff --git a/test/test_calm.py b/test/test_calm.py
index 7ea9718..1703e39 100755
--- a/test/test_calm.py
+++ b/test/test_calm.py
@@ -170,6 +170,11 @@ class CalmTest(unittest.TestCase):
         args.force = True
         args.pkglist = 'testdata/pkglist/cygwin-pkg-maint'
 
+        try:
+            shutil.rmtree(htdocs)
+        except FileNotFoundError:
+            pass
+
         packages = {}
         for arch in common_constants.ARCHES:
             packages[arch] = {}
diff --git a/test/testdata/htdocs.expected/x86/rpm-doc-src/rpm-doc-4.1-2-src b/test/testdata/htdocs.expected/x86/rpm-doc-src/rpm-doc-4.1-2-src
index fb29610..aef3d74 100644
--- a/test/testdata/htdocs.expected/x86/rpm-doc-src/rpm-doc-4.1-2-src
+++ b/test/testdata/htdocs.expected/x86/rpm-doc-src/rpm-doc-4.1-2-src
@@ -6,6 +6,9 @@
 <body>
 <h1>rpm-doc-src: Obsolete package for RPM package management system manual pages (extra text to so repr is not one line) (source)</h1>
 <pre>
+    2021-05-03 19:35           0 rpm-4.1.tar.gz
+    2021-05-03 19:36           0 rpm-4.1-1.patch
+    2021-05-03 19:36           0 rpm-4.1-1.sh
 </pre>
 </body>
 </html>
diff --git a/test/testdata/inifile/setup.ini.expected b/test/testdata/inifile/setup.ini.expected
index ff9dc85..3f583d7 100644
--- a/test/testdata/inifile/setup.ini.expected
+++ b/test/testdata/inifile/setup.ini.expected
@@ -294,8 +294,8 @@
  'version: 4.1-2\n'
  'install: x86/release/rpm-doc/rpm-doc-4.1-2.tar.bz2 50941 '
  '7cc9db802364252e3206ce9f75c8ca53813d8308a22a425b50ef695dd8e51568740b06739d3aa3399a83fb3d3e1345ab7e2ad03a1e9d47c02dded3363bf4f493\n'
- 'source: x86/release/rpm-doc/rpm-doc-4.1-2-src.tar.bz2 42 '
- '28c70b843fe01d90a3eeab4a3617551d236cd0b7d69668d1b1b6c8b14a9fd050e4039c192894c93bdf31575771c58c1fea2a41c24c8da22d10080d8b032b6369\n'
+ 'source: x86/release/rpm-doc/rpm-doc-4.1-2-src.tar.bz2 188 '
+ 'a4ee9121cfff2ccd5b4d84ffd18d2cff0ea071cef3c5416b8825d7b7f7cff8a8873c71832f3f3cb8742d004f9db34e9d160ad613b5df839723a3f986f0901402\n'
  '\n'
  '@ staleversion\n'
  'sdesc: "Test package for stale version removal"\n'
diff --git a/test/testdata/process_arch/setup.ini.expected b/test/testdata/process_arch/setup.ini.expected
index e7aa93c..6705bfb 100644
--- a/test/testdata/process_arch/setup.ini.expected
+++ b/test/testdata/process_arch/setup.ini.expected
@@ -301,8 +301,8 @@
  'version: 4.1-2\n'
  'install: x86/release/rpm-doc/rpm-doc-4.1-2.tar.bz2 50941 '
  '7cc9db802364252e3206ce9f75c8ca53813d8308a22a425b50ef695dd8e51568740b06739d3aa3399a83fb3d3e1345ab7e2ad03a1e9d47c02dded3363bf4f493\n'
- 'source: x86/release/rpm-doc/rpm-doc-4.1-2-src.tar.bz2 42 '
- '28c70b843fe01d90a3eeab4a3617551d236cd0b7d69668d1b1b6c8b14a9fd050e4039c192894c93bdf31575771c58c1fea2a41c24c8da22d10080d8b032b6369\n'
+ 'source: x86/release/rpm-doc/rpm-doc-4.1-2-src.tar.bz2 188 '
+ 'a4ee9121cfff2ccd5b4d84ffd18d2cff0ea071cef3c5416b8825d7b7f7cff8a8873c71832f3f3cb8742d004f9db34e9d160ad613b5df839723a3f986f0901402\n'
  '\n'
  '@ staleversion\n'
  'sdesc: "Test package for stale version removal"\n'
diff --git a/test/testdata/relarea/x86/release/rpm-doc/rpm-doc-4.1-2-src.tar.bz2 b/test/testdata/relarea/x86/release/rpm-doc/rpm-doc-4.1-2-src.tar.bz2
index cbf838c..f90a445 100644
Binary files a/test/testdata/relarea/x86/release/rpm-doc/rpm-doc-4.1-2-src.tar.bz2 and b/test/testdata/relarea/x86/release/rpm-doc/rpm-doc-4.1-2-src.tar.bz2 differ