git: 8d3555f0b084 - main - Tools/scripts/update-plist.py: Updates pkg-plist

The branch main has been updated by jhale:

URL: https://cgit.FreeBSD.org/ports/commit/?id=8d3555f0b08491e45a41766041e7b71ed6787c6b

commit 8d3555f0b08491e45a41766041e7b71ed6787c6b
Author:     Jason E. Hale <[email protected]>
AuthorDate: 2026-08-08 08:33:44 +0000
Commit:     Jason E. Hale <[email protected]>
CommitDate: 2026-08-08 08:59:31 +0000

    Tools/scripts/update-plist.py: Updates pkg-plist
    
    Small script using base Python to update pkg-plist based upon poudriere
    error logs.
    
    Inspired by: https://codeberg.org/tcberner/poudlist, by tcberner@. This
    just uses base Python instead of C++ and flaky Boost libs, runs way
    faster and requires no config file.
    
    This is mainly intendend for simple ports and it's output should be
    manually reviewed.
---
 Tools/scripts/update-plist.py | 263 ++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 263 insertions(+)

diff --git a/Tools/scripts/update-plist.py b/Tools/scripts/update-plist.py
new file mode 100644
index 000000000000..b689997be25f
--- /dev/null
+++ b/Tools/scripts/update-plist.py
@@ -0,0 +1,263 @@
+#!/usr/bin/env python
+
+# update-plist.py - Adds and removes lines in pkg-plist(s) based on
+# poudriere error logs.
+
+# Inspired by: https://codeberg.org/tcberner/poudlist, by tcberner@,
+# but this just requires base Python and doesn't use a config file.
+
+# ----------------------------------------------------------------------------
+# "THE BEER-WARE LICENSE" (Revision 42, (c) Poul-Henning Kamp):
+# Jason E. Hale <[email protected]> wrote this file. As long as you retain
+# this notice you can do whatever you want with this stuff. If we meet some
+# day, and you think this stuff is worth it, you can buy me a beer in return.
+#
+# Jason E. Hale
+# ----------------------------------------------------------------------------
+
+# MAINTAINER=    [email protected]
+
+# SYNOPSIS
+#   update-plist.py [-h] [-E] [-p </path/to/portsdir>] </path/to/poudriere/error/log>
+
+# DESCRIPTION
+#   Adds and removes lines in pkg-plist(s) based on poudriere error logs.
+
+#   It is recommend to run this tool after building a port with all options
+#   enabled with either `poudriere bulk -t` or `poudriere testport` as it
+#   will prune all stale plist entires, including those prefixed by '%%',
+#   but it does recognize where it has already been run and won't try to
+#   duplicate the previous effort.
+
+#   Use ports-mgmt/hs-panopticum for a more in-depth and CPU-intensive
+#   analysis than this little script could ever provide, but with some
+#   human brains behind it, using this takes a fraction of the time for
+#   most ports.
+
+#   You will still need to manually verify that all of the PLIST_SUBs
+#   are correct. This tool follows the poudriere logs to a tee, but
+#   sometimes the PLIST_SUBs for a port are more aggressive than they
+#   really should be as generated by 'make makeplist'.
+
+# OPTIONS
+#   -h, --help      print helps and then exits
+#
+#   -p, --portsdir  path to the ports tree to operate upon
+#                   Overrides the PORTSDIR environment variable
+#                   if defined, required otherwise.
+#
+#   -E, --extended  (experimental) try to work with ports with no pkg-plist
+#                   or more complicated pkg-plist structures (PLIST+=foo).
+#                   Writes pkg-plist.<pkgname>.add~ and pkg-plist.<pkgname>.rm~
+#                   files to the port directory  manual intervention instead of
+#                   skipping. The appended '~' is meant to prevent accidental
+#                   commital of these temporary files.
+
+# ENVIRONMENT
+#   PORTSDIR        path to the ports tree to operate upon
+
+# EXAMPLES
+#   update-plist.py /usr/local/poudriere/data/logs/bulk/150amd64-area51/2026-03-19_22h36m33s/logs/errors
+#   update-plist.py -p ~/src/area51/ports /usr/local/poudriere/data/logs/bulk/150amd64-area51/2026-03-19_22h36m33s/logs/errors/qt6-webengine-6.11.0.log
+
+# BUGS
+#   - Assumes that the existing pkg-plist is already somewhat properly sorted.
+
+import sys
+import re
+import os
+import argparse
+import subprocess
+
+def extract_plist_sub(log_content):
+    sub_map = {}
+    match = re.search(r"--PLIST_SUB--\n(.*?)\n--End PLIST_SUB--", log_content, re.DOTALL)
+    if match:
+        raw_subs = match.group(1)
+        pairs = re.findall(r'(\w+)=("(?:[^"\\]|\\.)*"|[^\s]+)', raw_subs)
+        for key, val in pairs:
+            clean_val = val.strip('"')
+            sub_map[f"%%{key}%%"] = "" if "@comment" in clean_val else clean_val
+    return sub_map
+
+def get_expanded_key(line, sub_map):
+    expanded = line.strip()
+    for macro in sorted(sub_map.keys(), key=len, reverse=True):
+        if macro in expanded:
+            expanded = expanded.replace(macro, sub_map[macro]).replace("//", "/")
+    clean = re.sub(r"%%.*?%%", "", expanded)
+    clean = re.sub(r"@\w+\s+", "", clean)
+    return clean.strip().lstrip('/')
+
+def get_plist_paths(ports_tree_root, origin):
+    """Uses make to resolve all PLIST paths, handling multiple space-separated plists."""
+    port_dir = os.path.join(ports_tree_root, origin)
+    try:
+        result = subprocess.check_output(
+            ['make', '-C', port_dir, '-V', 'PLIST'],
+            text=True,
+            stderr=subprocess.DEVNULL
+        ).strip()
+
+        if result:
+            paths = result.split()
+            return [p if os.path.isabs(p) else os.path.join(port_dir, p) for p in paths]
+    except Exception:
+        pass
+
+    return [os.path.join(port_dir, "pkg-plist")]
+
+def process_log(log_path, ports_tree_root, extended=False):
+    try:
+        with open(log_path, 'r', encoding='utf-8', errors='ignore') as f:
+            first_lines = []
+            for _ in range(15):
+                line = f.readline()
+                if not line: break
+                first_lines.append(line)
+            f.seek(0)
+            log_content = f.read()
+    except Exception as e:
+        sys.stderr.write(f"![Error] Could not read {log_path}: {e}\n")
+        return
+
+    if "Error: Plist issues found" not in log_content:
+        return
+
+    origin = None
+    for line in first_lines:
+        if "port directory:" in line:
+            parts = line.split(":", 1)[1].strip().rstrip('/').split('/')
+            if len(parts) >= 2:
+                origin = os.path.join(parts[-2], parts[-1])
+            break
+
+    if not origin: return
+
+    plist_paths = get_plist_paths(ports_tree_root, origin)
+    existing_plists = [p for p in plist_paths if os.path.exists(p)]
+
+    is_fallback_mode = False
+    fallback_rm_path = None
+
+    # Handle the case where no valid plists exist on disk
+    if not existing_plists:
+        if not extended:
+            sys.stderr.write(f"![Skip] {origin}: No valid plist files found (port likely relies on PLIST_FILES). Try using with the '-E' flag.\n")
+            return
+        else:
+            is_fallback_mode = True
+            log_basename = os.path.basename(log_path)
+            if log_basename.endswith(".log"):
+                log_basename = log_basename[:-4]
+
+            fallback_add_name = f"pkg-plist.{log_basename}.add~"
+            fallback_rm_name = f"pkg-plist.{log_basename}.rm~"
+            fallback_add_path = os.path.join(ports_tree_root, origin, fallback_add_name)
+            fallback_rm_path = os.path.join(ports_tree_root, origin, fallback_rm_name)
+
+            sys.stderr.write(f"-> Warning: {origin} has no existing plist. Manual intervention required.\n")
+            existing_plists = [fallback_add_path]
+
+    plist_data = {}
+    all_existing_lines = set()
+
+    for path in existing_plists:
+        if os.path.exists(path):
+            with open(path, 'r', encoding='utf-8') as f:
+                lines = [l.strip() for l in f.readlines() if l.strip()]
+                plist_data[path] = lines
+                all_existing_lines.update(lines)
+        else:
+            # For the newly created pkg-plist.*.add~
+            plist_data[path] = []
+
+    sub_map = extract_plist_sub(log_content)
+    orphans = re.findall(r"Error: Orphaned: (.*)", log_content)
+    missing_from_log = set(re.findall(r"Error: Missing: (.*)", log_content))
+
+    removed_count = 0
+    added_count = 0
+
+    if is_fallback_mode:
+        # We don't have a source plist to remove from, so dump the missing lines to the rm file
+        if missing_from_log:
+            plist_data[fallback_rm_path] = sorted(list(missing_from_log))
+            removed_count = len(missing_from_log)
+    else:
+        for path, lines in plist_data.items():
+            new_lines = []
+            for line in lines:
+                clean_line = re.sub(r"%%.*?%%", "", line).strip().lstrip('/')
+                if line in missing_from_log or clean_line in missing_from_log:
+                    removed_count += 1
+                    continue
+                new_lines.append(line)
+            plist_data[path] = new_lines
+
+    # Target the FIRST file in our list for orphans (either the base plist or the unique .add~ file)
+    base_plist = existing_plists[0]
+
+    for orphan in sorted(orphans, key=lambda x: get_expanded_key(x, sub_map)):
+        if orphan in all_existing_lines:
+            continue
+
+        orphan_key = get_expanded_key(orphan, sub_map)
+        inserted = False
+
+        for i, line in enumerate(plist_data[base_plist]):
+            if get_expanded_key(line, sub_map) > orphan_key:
+                plist_data[base_plist].insert(i, orphan)
+                inserted = True
+                added_count += 1
+                break
+
+        if not inserted:
+            plist_data[base_plist].append(orphan)
+            added_count += 1
+
+        all_existing_lines.add(orphan)
+
+    if added_count == 0 and removed_count == 0:
+        sys.stderr.write(f"-> {origin}: Already up to date.\n")
+        return
+
+    # Suppress the multi-plist warning if we are writing to a fallback intervention file
+    if len(existing_plists) > 1 and added_count > 0 and "pkg-plist.add" not in base_plist:
+        base_name = os.path.basename(base_plist)
+        sys.stderr.write(f"-> Warning: {origin} uses multiple plists. Orphans appended to {base_name}. Please verify manually.\n")
+
+    sys.stderr.write(f"-> Updating {origin}... Done (+{added_count}/-{removed_count})\n")
+
+    for path, lines in plist_data.items():
+        with open(path, 'w', encoding='utf-8') as f:
+            f.write("\n".join(lines) + "\n")
+
+if __name__ == "__main__":
+    env_portsdir = os.environ.get("PORTSDIR")
+    is_required = not bool(env_portsdir)
+
+    parser = argparse.ArgumentParser(description="Updates pkg-plist based on poudriere error logs.")
+    parser.add_argument(
+        "logpath",
+        help="Poudriere log file or directory of logs"
+    )
+    parser.add_argument(
+        "-p", "--portsdir",
+        required=is_required,
+        default=env_portsdir,
+        help="Path to ports tree to operate upon (Overrides PORTSDIR env var if set)"
+    )
+    parser.add_argument(
+        "-E", "--extended",
+        action="store_true",
+        help="(experimental) If a port has no pkg-plist or a more complicated pkg-plist structure, write orphaned/missing files to pkg-plist.<pkgname>.add~ and pkg-plist.<pkgname>.rm~ for manual review instead of skipping"
+    )
+    args = parser.parse_args()
+
+    if os.path.isdir(args.logpath):
+        for filename in sorted(os.listdir(args.logpath)):
+            if filename.endswith(".log"):
+                process_log(os.path.join(args.logpath, filename), args.portsdir, args.extended)
+    else:
+        process_log(args.logpath, args.portsdir, args.extended)