Re: [AUH][PATCH v2 4/9] upgrade-helper.py: add changelog flag

Can you add examples please? This implements three different
'strategies', all of them are doing 'guessing', so it would be really
beneficial to see what they actually produce.

It would also be super nice to have tests for this module, especially
if people start tweaking it further. It looks like something that can
break super easily. I know AUH has no tests (I inherited it in that
condition), but at least this particular part could.

Alex

On Fri, 24 Apr 2026 at 13:46, <[email protected]> wrote:
>
> From: Daniel Turull <[email protected]>
>
> Add --changelog option to extract git log between old and new versions,
> highlighting CVE references. The changelog summary is appended to the
> commit message and included in the email notification.
>
> Signed-off-by: Daniel Turull <[email protected]>
> Assisted-by: Claude, Anthropic
> ---
>  modules/changelog.py | 256 +++++++++++++++++++++++++++++++++++++++++++
>  modules/steps.py     |  20 ++++
>  upgrade-helper.py    |  16 +++
>  3 files changed, 292 insertions(+)
>  create mode 100644 modules/changelog.py
>
> diff --git a/modules/changelog.py b/modules/changelog.py
> new file mode 100644
> index 0000000..c1cc14b
> --- /dev/null
> +++ b/modules/changelog.py
> @@ -0,0 +1,256 @@
> +# SPDX-License-Identifier: GPL-2.0-or-later
> +
> +import os
> +import re
> +import glob
> +import functools
> +import subprocess
> +
> +from logging import info as I
> +from logging import debug as D
> +
> +import bb.utils
> +
> +CHANGELOG_FILENAMES = [
> +    'ChangeLog', 'CHANGELOG', 'CHANGELOG.md', 'CHANGELOG.txt',
> +    'Changes', 'CHANGES', 'NEWS', 'NEWS.md', 'NEWS.txt',
> +    'RELEASE_NOTES', 'RELEASE_NOTES.md', 'RELEASE-NOTES',
> +    'HISTORY', 'HISTORY.md',
> +    'debian/changelog',
> +]
> +
> +CVE_PATTERN = re.compile(r'(CVE-\d{4}-\d{4,})', re.IGNORECASE)
> +
> +
> +def _find_changelog_files(srcdir):
> +    found = []
> +    for name in CHANGELOG_FILENAMES:
> +        for f in glob.glob(os.path.join(srcdir, '**', name), recursive=True):
> +            if os.path.isfile(f) and f not in found:
> +                found.append(f)
> +    return found
> +
> +
> +RST_INCLUDE_RE = re.compile(r'^\.\.\s+include::\s+(.+)$')
> +RST_COMMENT_RE = re.compile(r'^\.\.\s*$|^\.\.\s')
> +
> +
> +def _strip_rst_comments(text):
> +    """Remove RST comment blocks (license headers etc.)."""
> +    lines = text.split('\n')
> +    result = []
> +    in_comment = False
> +    for line in lines:
> +        if RST_COMMENT_RE.match(line):
> +            in_comment = True
> +            continue
> +        if in_comment:
> +            if line.startswith('   ') or line.strip() == '':
> +                continue
> +            in_comment = False
> +        result.append(line)
> +    return '\n'.join(result)
> +
> +
> +def _resolve_rst_includes(content, base_dir, srcdir):
> +    """Inline RST .. include:: directives."""
> +    lines = content.split('\n')
> +    result = []
> +    for line in lines:
> +        m = RST_INCLUDE_RE.match(line.strip())
> +        if m:
> +            inc_rel = m.group(1).strip()
> +            inc_path = os.path.normpath(os.path.join(base_dir, inc_rel))
> +            if not os.path.isfile(inc_path):
> +                # Search for the filename within the source tree
> +                fname = os.path.basename(inc_rel)
> +                for f in glob.glob(os.path.join(srcdir, '**', fname),
> +                                   recursive=True):
> +                    if os.path.isfile(f):
> +                        inc_path = f
> +                        break
> +            if os.path.isfile(inc_path):
> +                try:
> +                    with open(inc_path, 'r', encoding='utf-8',
> +                              errors='replace') as f:
> +                        result.append(f.read())
> +                    continue
> +                except OSError:
> +                    pass
> +        result.append(line)
> +    return '\n'.join(result)
> +
> +
> +GIT_LOG_RE = re.compile(r'^commit [0-9a-f]{7,}', re.MULTILINE)
> +
> +
> +def _condense_git_log(text):
> +    """Condense git-log-style content to subject lines only."""
> +    if not GIT_LOG_RE.search(text):
> +        return text
> +    lines = text.split('\n')
> +    subjects = []
> +    i = 0
> +    while i < len(lines):
> +        if GIT_LOG_RE.match(lines[i]):
> +            short = lines[i].split()[1][:12]
> +            # Skip Author/Date, blank line, then grab subject
> +            i += 1
> +            while i < len(lines) and (lines[i].startswith('Author:') or
> +                    lines[i].startswith('Date:') or not lines[i].strip()):
> +                i += 1
> +            if i < len(lines):
> +                subjects.append('%s %s' % (short, lines[i].strip()))
> +            continue
> +        i += 1
> +    return '\n'.join(subjects) if subjects else text
> +
> +
> +def _extract_entries_between_versions(content, old_ver, new_ver):
> +    lines = content.split('\n')
> +    new_pattern = re.compile(re.escape(new_ver))
> +    old_pattern = re.compile(re.escape(old_ver))
> +
> +    start_idx = None
> +    end_idx = None
> +
> +    for i, line in enumerate(lines):
> +        if start_idx is None and new_pattern.search(line):
> +            start_idx = i
> +        elif start_idx is not None and old_pattern.search(line):
> +            end_idx = i
> +            break
> +
> +    if start_idx is not None:
> +        return '\n'.join(lines[start_idx:end_idx])
> +    return None
> +
> +
> +def _find_per_version_files(srcdir, old_ver, new_ver):
> +    """Find individual per-version changelog files (e.g. changelog-1.2.3.rst)."""
> +    ver_file_re = re.compile(
> +        r'(?:changelog|changes|news|release|relnotes)[-_./\\]?'
> +        r'v?(?P<pver>(\d+[\.\-_])*\d+)\.\w+$',
> +        re.IGNORECASE)
> +
> +    candidates = {}
> +    for f in glob.glob(os.path.join(srcdir, '**', '*'), recursive=True):
> +        if not os.path.isfile(f):
> +            continue
> +        # Match against last two path components (e.g. RelNotes/v1.47.4.txt)
> +        tail = os.sep.join(f.rsplit(os.sep, 2)[-2:])
> +        m = ver_file_re.search(tail)
> +        if m:
> +            ver = m.group('pver').replace('_', '.').replace('-', '.')
> +            candidates[ver] = f
> +
> +    # Select versions > old_ver and <= new_ver
> +    selected = []
> +    for ver, path in candidates.items():
> +        if bb.utils.vercmp_string(ver, old_ver) > 0 and \
> +           bb.utils.vercmp_string(ver, new_ver) <= 0:
> +            selected.append((ver, path))
> +
> +    selected.sort(key=lambda x: functools.cmp_to_key(
> +        bb.utils.vercmp_string)(x[0]))
> +    return [path for _, path in selected]
> +
> +
> +def extract_changelog(srcdir, pn, old_ver, new_ver, workdir):
> +    if not srcdir or not os.path.isdir(srcdir):
> +        D(" %s: source directory %s not available" % (pn, srcdir))
> +        return None
> +
> +    entries = []
> +    cves = []
> +
> +    # Strategy 1: extract sections between version markers in changelog files
> +    changelog_files = _find_changelog_files(srcdir)
> +    for fpath in changelog_files:
> +        try:
> +            with open(fpath, 'r', encoding='utf-8', errors='replace') as f:
> +                content = f.read()
> +        except OSError:
> +            continue
> +        content = _resolve_rst_includes(content, os.path.dirname(fpath), srcdir)
> +        section = _extract_entries_between_versions(content, old_ver, new_ver)
> +        if section:
> +            section = _condense_git_log(section)
> +            entries.append(section)
> +            cves.extend(CVE_PATTERN.findall(section))
> +            break
> +
> +    # Strategy 2: concatenate per-version files (e.g. changelog-9.18.42.rst)
> +    if not entries:
> +        ver_files = _find_per_version_files(srcdir, old_ver, new_ver)
> +        for fpath in ver_files:
> +            try:
> +                with open(fpath, 'r', encoding='utf-8', errors='replace') as f:
> +                    content = f.read()
> +            except OSError:
> +                continue
> +            entries.append(content)
> +            cves.extend(CVE_PATTERN.findall(content))
> +
> +    # Strategy 3: git log between version tags
> +    if not entries and os.path.isdir(os.path.join(srcdir, '.git')):
> +        tag_prefixes = ['v', '', pn + '-']
> +        old_tag = new_tag = None
> +        for prefix in tag_prefixes:
> +            try:
> +                subprocess.check_output(
> +                    ['git', 'rev-parse', prefix + old_ver],
> +                    cwd=srcdir, stderr=subprocess.DEVNULL)
> +                subprocess.check_output(
> +                    ['git', 'rev-parse', prefix + new_ver],
> +                    cwd=srcdir, stderr=subprocess.DEVNULL)
> +                old_tag, new_tag = prefix + old_ver, prefix + new_ver
> +                break
> +            except (subprocess.CalledProcessError, OSError):
> +                continue
> +        if old_tag and new_tag:
> +            try:
> +                out = subprocess.check_output(
> +                    ['git', 'log', '--oneline', '%s..%s' % (old_tag, new_tag)],
> +                    cwd=srcdir, stderr=subprocess.DEVNULL).decode('utf-8', errors='replace')
> +                if out.strip():
> +                    entries.append(out.strip())
> +                    cves.extend(CVE_PATTERN.findall(out))
> +            except (subprocess.CalledProcessError, OSError):
> +                pass
> +
> +    if not entries:
> +        D(" %s: no changelog entries found" % pn)
> +        return None
> +
> +    I(" %s: found %d changelog entries" % (pn, len(entries)))
> +
> +    cves = sorted(set(cves))
> +    text = "Changelog for %s: %s -> %s\n" % (pn, old_ver, new_ver)
> +    text += "=" * 60 + "\n\n"
> +    if cves:
> +        text += "SECURITY FIXES / CVEs FOUND:\n"
> +        for cve in cves:
> +            text += "  - %s\n" % cve
> +        text += "\n" + "-" * 60 + "\n\n"
> +    text += CVE_PATTERN.sub(r'*** \1 ***', _strip_rst_comments('\n\n'.join(entries)))
> +
> +    # Collapse multiple blank lines into one
> +    text = re.sub(r'\n{3,}', '\n\n', text)
> +
> +    changelog_path = os.path.join(workdir, "changelog-%s.txt" % pn)
> +    with open(changelog_path, 'w', encoding='utf-8') as f:
> +        f.write(text)
> +
> +    I(" %s: changelog saved to %s" % (pn, os.path.basename(changelog_path)))
> +    if cves:
> +        I(" %s: CVEs found: %s" % (pn, ', '.join(cves)))
> +
> +    commit_text = text
> +    if len(commit_text) > 3000:
> +        commit_text = commit_text[:3000] + "\n\n[... changelog truncated ...]\n"
> +    # Sanitize for shell-safe git commit -m "..."
> +    for ch in '"', '`', '$', '\\':
> +        commit_text = commit_text.replace(ch, '')
> +
> +    return {'text': text, 'commit_text': commit_text, 'cves': cves, 'file': changelog_path}
> diff --git a/modules/steps.py b/modules/steps.py
> index b3ec61c..78fcbbe 100644
> --- a/modules/steps.py
> +++ b/modules/steps.py
> @@ -29,6 +29,7 @@ from logging import warning as W
>
>  from errors import Error, DevtoolError, CompilationError
>  from buildhistory import BuildHistory
> +from changelog import extract_changelog
>
>  def load_env(devtool, bb, git, opts, group):
>      group['workdir'] = os.path.join(group['base_dir'], group['name'])
> @@ -150,10 +151,29 @@ def devtool_finish(devtool, bb, git, opts, group):
>              pass
>          raise e1
>
> +def changelog_extract(devtool, bb, git, opts, group):
> +    if not opts.get('changelog'):
> +        return
> +    for p in group['pkgs']:
> +        # After devtool_upgrade, source is in workspace/sources/<pn>/
> +        srcdir = os.path.join(os.environ.get('BUILDDIR', ''),
> +                              'workspace', 'sources', p['PN'])
> +        if not os.path.isdir(srcdir):
> +            # Fallback: derive from env S, replacing old version
> +            srcdir = p['env'].get('S', '')
> +            if p['PV'] in srcdir:
> +                srcdir = srcdir.replace(p['PV'], p['NPV'])
> +        result = extract_changelog(srcdir, p['PN'], p['PV'],
> +                                   p['NPV'], group['workdir'])
> +        if result:
> +            p['changelog'] = result
> +            group['commit_msg'] += "\n\n" + result['commit_text']
> +
>  upgrade_steps = [
>      (load_env, "Loading environment ..."),
>      (buildhistory_init, None),
>      (devtool_upgrade, "Running 'devtool upgrade' ..."),
> +    (changelog_extract, "Extracting changelog ..."),
>      (devtool_finish, "Running 'devtool finish' ..."),
>      (compile, None),
>  ]
> diff --git a/upgrade-helper.py b/upgrade-helper.py
> index df927d1..327bb6d 100755
> --- a/upgrade-helper.py
> +++ b/upgrade-helper.py
> @@ -95,6 +95,8 @@ def parse_cmdline():
>
>      parser.add_argument("-t", "--to_version",
>                          help="version to upgrade the recipe to")
> +    parser.add_argument("--changelog", action="store_true", default=False,
> +                        help="extract changelog between old and new versions, highlighting CVEs")
>
>      parser.add_argument("-d", "--debug-level", type=int, default=4, choices=range(1, 6),
>                          help="set the debug level: CRITICAL=1, ERROR=2, WARNING=3, INFO=4, DEBUG=5")
> @@ -198,6 +200,7 @@ class Updater(object):
>          self.opts['skip_compilation'] = self.args.skip_compilation
>          self.opts['buildhistory'] = self._buildhistory_is_enabled()
>          self.opts['testimage'] = self._testimage_is_enabled()
> +        self.opts['changelog'] = self.args.changelog
>
>      def _make_dirs(self, build_dir):
>          self.uh_dir = os.path.join(build_dir, "upgrade-helper")
> @@ -358,6 +361,19 @@ class Updater(object):
>          if 'patch_file' in g and g['patch_file'] is not None:
>              msg_body += next_steps_info % (os.path.basename(g['patch_file']))
>
> +        # Add changelog summary if available
> +        for pkg_ctx in g['pkgs']:
> +            if 'changelog' in pkg_ctx:
> +                cl = pkg_ctx['changelog']
> +                msg_body += ("\n--- Changelog Summary for %s ---\n"
> +                             % pkg_ctx['PN'])
> +                if cl['cves']:
> +                    msg_body += "\nSECURITY FIXES / CVEs:\n"
> +                    for cve in cl['cves']:
> +                        msg_body += "  - %s\n" % cve
> +                    msg_body += "\n"
> +                msg_body += cl['text'] + "\n"
> +
>          msg_body += mail_footer
>
>          # Add possible attachments to email
> --
> 2.34.1
>