proj/pkgcore/pkgcore:master commit in: src/pkgcore/ebuild/, tests/bugzilla/, /, tests/ebuild/, src/pkgcore/bugzilla/
"Arthur Zamarin" <[email protected]>
| Newsgroups | gmane.linux.gentoo.cvs |
|---|---|
| Message-ID | <1786793933.7f018f5751de318d885c49b57fd1ad0b6d7e75a0.arthurzam@gentoo> |
commit: 7f018f5751de318d885c49b57fd1ad0b6d7e75a0
Author: Arthur Zamarin <arthurzam <AT> gentoo <DOT> org>
AuthorDate: Sat Aug 15 11:29:05 2026 +0000
Commit: Arthur Zamarin <arthurzam <AT> gentoo <DOT> org>
CommitDate: Sat Aug 15 11:38:53 2026 +0000
URL: https://gitweb.gentoo.org/proj/pkgcore/pkgcore.git/commit/?id=7f018f57
keywording: resolve arch requests (adapted from nattka)
The code is very much based, in some ways even copied, from nattka, code
which was written by Michał Górny <mgorny <AT> gentoo.org>.
Turning a bug's package list into concrete packages and arches lived in
nattka.package.match_package_list, so anything else wanting it had to depend
on nattka. pkgdev tatt did exactly that, importing it behind a try/except
ImportError purely to answer which packages a bug asks this arch to test,
and pkgdev bugs had already given up and grown its own verbatim copy of
get_suggested_keywords.
Move the matcher to pkgcore.ebuild.keywording. It takes plain (atom,
keywords) pairs and explicit policy rather than a Bug, so a request that
never came from Bugzilla resolves the same way. The Bug-derived arguments
become Bug.match_packages, sitting next to Bug.arches which already turns CC
into arch names: the category picks stabilizing from keywording, CC picks
the arches, and the ALLARCHES keyword is honoured only when the caller opts
in. The three keyword sentinels get a single definition here, which
bugzilla.pkglist imports rather than keeping its own copy.
The exceptions move with it, rooted at PkgcoreException. KeywordNoneLeft
stays deliberately outside PackageMatchException, since callers catch that
broadly to report a request as unusable and "nothing left to do" is not one.
Based-on-code-by: Michał Górny <mgorny <AT> gentoo.org>
Signed-off-by: Arthur Zamarin <arthurzam <AT> gentoo.org>
NEWS.rst | 21 ++
src/pkgcore/bugzilla/__init__.py | 3 +-
src/pkgcore/bugzilla/bug.py | 46 ++++
src/pkgcore/bugzilla/pkglist.py | 73 +++++--
src/pkgcore/ebuild/keywording.py | 300 +++++++++++++++++++++++++
src/pkgcore/ebuild/misc.py | 5 +-
tests/bugzilla/test_pkglist.py | 43 ++++
tests/ebuild/test_keywording.py | 458 +++++++++++++++++++++++++++++++++++++++
8 files changed, 931 insertions(+), 18 deletions(-)
diff --git a/NEWS.rst b/NEWS.rst
index fed9eb8de..aba7c1372 100644
--- a/NEWS.rst
+++ b/NEWS.rst
@@ -2,6 +2,27 @@
Release Notes
=============
+----------------------------
+pkgcore 0.12.40 (unreleased)
+----------------------------
+
+Features
+~~~~~~~~
+
+- ``pkgcore.ebuild.keywording``: new module resolving a keywording or
+ stabilization request into concrete packages and arches, moved out of nattka
+ so that pkgcore, pkgdev and nattka share one implementation (Arthur Zamarin)
+
+Fixes
+~~~~~
+
+- ``pkgcore.bugzilla.pkglist``: expanding a package list no longer rewrites the
+ lines it touches (Arthur Zamarin)
+
+- ``pkgcore.bugzilla.pkglist.parse_atom`` now also rejects a repo id, as in
+ ``=cat/pkg-1::gentoo`` (Arthur Zamarin)
+
+
----------------------------
pkgcore 0.12.39 (2026-08-14)
----------------------------
diff --git a/src/pkgcore/bugzilla/__init__.py b/src/pkgcore/bugzilla/__init__.py
index 63aa3f5e5..e7df2c8e8 100644
--- a/src/pkgcore/bugzilla/__init__.py
+++ b/src/pkgcore/bugzilla/__init__.py
@@ -33,6 +33,7 @@ __all__ = (
"NewComment",
"PackageList",
"PackageListEntry",
+ "PackageListError",
"Product",
"Resolution",
"RuntimeTesting",
@@ -54,7 +55,7 @@ from .enums import (
Severity,
Status,
)
-from .errors import BugzillaError
+from .errors import BugzillaError, PackageListError
from .pkglist import PackageList, PackageListEntry
from .query import BugQuery, Criterion
from .transport import AuthMode
diff --git a/src/pkgcore/bugzilla/bug.py b/src/pkgcore/bugzilla/bug.py
index dcda976e2..bee100fa6 100644
--- a/src/pkgcore/bugzilla/bug.py
+++ b/src/pkgcore/bugzilla/bug.py
@@ -6,6 +6,7 @@ one declaration and can't drift apart.
"""
__all__ = (
+ "ALLARCHES_KEYWORD",
"INCLUDE_FIELDS",
"Bug",
"BugChanges",
@@ -23,8 +24,10 @@ import dataclasses
import datetime
import typing
+from ..ebuild.keywording import KeywordRequest, PackageInvalid, match_packages
from ..log import logger
from .enums import BugCategory, FlagStatus, Product, RuntimeTesting
+from .errors import PackageListError
from .pkglist import PackageList
from .wire import (
BugId,
@@ -38,6 +41,9 @@ from .wire import (
RawWhoami,
)
+# the Bugzilla keyword marking a stabilization the whole arch set may do at once
+ALLARCHES_KEYWORD: typing.Final = "ALLARCHES"
+
_EPOCH: typing.Final = datetime.datetime.fromtimestamp(0, datetime.UTC)
_NO_FLAG_ID: typing.Final = FlagId(0)
_NO_FLAG_TYPE_ID: typing.Final = FlagTypeId(0)
@@ -188,6 +194,46 @@ class Bug:
and (name := entry.split("@", 1)[0]) in known_arches
)
+ def match_packages(
+ self,
+ repo: typing.Any,
+ *,
+ only_new: bool = False,
+ filter_arch: typing.Iterable[str] = (),
+ permit_allarches: bool = False,
+ ) -> typing.Iterator[KeywordRequest]:
+ """Resolve this bug's package list against ``repo``.
+
+ Bug's category decides stabilizing from keywording and CC decides
+ which arches the request is addressed to; see
+ :func:`pkgcore.ebuild.keywording.match_packages` for the rest.
+
+ :param repo: repo to resolve the package list against
+ :param only_new: drop the arches the package already carries
+ :param filter_arch: keep only the arches listed
+ :param permit_allarches: honour the ALLARCHES keyword when the bug
+ carries it
+ :return: the matched packages, each with the arches to request for it
+ :raises PackageInvalid: if the package list can't be parsed
+ """
+ try:
+ requested = [
+ (entry.pkg, entry.keywords)
+ for entry in self.package_list.entries
+ if entry.pkg is not None
+ ]
+ except PackageListError as exc:
+ raise PackageInvalid(str(exc)) from exc
+ return match_packages(
+ repo,
+ requested,
+ stable=self.category is BugCategory.STABLEREQ,
+ cc_arches=self.arches(repo.known_arches),
+ only_new=only_new,
+ filter_arch=filter_arch,
+ allarches=permit_allarches and ALLARCHES_KEYWORD in self.keywords,
+ )
+
@property
def url(self) -> str:
return f"https://bugs.gentoo.org/{self.id}"
diff --git a/src/pkgcore/bugzilla/pkglist.py b/src/pkgcore/bugzilla/pkglist.py
index e64a00baa..9ab895d57 100644
--- a/src/pkgcore/bugzilla/pkglist.py
+++ b/src/pkgcore/bugzilla/pkglist.py
@@ -27,14 +27,12 @@ from snakeoil.klass.properties import jit_attr_none
from ..ebuild.atom import atom
from ..ebuild.errors import MalformedAtom
+from ..ebuild.keywording import ALL_KEYWORDS, NO_KEYWORDS, SAME_KEYWORDS
from .errors import PackageListError
from .wire import BugId
-ALL_KEYWORDS: typing.Final = "*"
-SAME_KEYWORDS: typing.Final = "^"
-NO_KEYWORDS: typing.Final = "-"
-
_COMMENT_RE: typing.Final = re.compile(r"(?:^|\s)#")
+_TOKEN_RE: typing.Final = re.compile(r"\S+")
def parse_atom(token: str) -> atom:
@@ -43,6 +41,12 @@ def parse_atom(token: str) -> atom:
Stabilization lines carry a bare ``cat/pkg-1.2.3`` rather than the
``=cat/pkg-1.2.3`` an atom needs, so the versioned form is tried first.
+ A package list names packages to act on, so the qualifiers that would make
+ that ambiguous are rejected: blockers, use deps, slot operators and repo
+ ids all mean something the list has no way to honour.
+
+ :param token: the first whitespace separated field of a list line
+ :return: the atom the token names
:raises MalformedAtom: if the token isn't a usable package spec
"""
for candidate in (f"={token}", token):
@@ -50,8 +54,10 @@ def parse_atom(token: str) -> atom:
pkg = atom(candidate)
except MalformedAtom:
continue
- if pkg.blocks or pkg.use or pkg.slot_operator:
- raise MalformedAtom(token, "blockers, use deps and slot operators")
+ if pkg.blocks or pkg.use or pkg.slot_operator or pkg.repo_id:
+ raise MalformedAtom(
+ token, "blockers, use deps, slot operators and repo ids"
+ )
return pkg
raise MalformedAtom(token)
@@ -72,14 +78,36 @@ class PackageListEntry:
return self.pkg is None
def with_keywords(self, keywords: typing.Iterable[str]) -> "PackageListEntry":
- """Return a copy with new keywords, keeping indentation and comment"""
+ """Return a copy with new keywords, rewriting only the keyword text.
+
+ The rest of the line survives as written: the spec in its original
+ spelling, the alignment around it, and the comment.
+
+ :param keywords: the keywords to write in place of the current ones
+ :return: the rewritten entry, or ``self`` if the line holds no spec
+ """
if self.pkg is None:
return self
keywords = tuple(keywords)
- indent = self.raw[: len(self.raw) - len(self.raw.lstrip())]
- body = " ".join((str(self.pkg), *keywords))
- tail = f" {self.comment}" if self.comment else ""
- return dataclasses.replace(self, keywords=keywords, raw=f"{indent}{body}{tail}")
+ comment_at = len(self.raw)
+ if match := _COMMENT_RE.search(self.raw):
+ comment_at = match.end() - 1
+ body = self.raw[:comment_at]
+ if not (tokens := list(_TOKEN_RE.finditer(body))):
+ return self
+ # anchoring on the tokens keeps the whitespace on either side of them
+ if len(tokens) > 1:
+ head = body[: tokens[1].start()]
+ else:
+ head = body[: tokens[0].end()] + (" " if keywords else "")
+ return dataclasses.replace(
+ self,
+ keywords=keywords,
+ raw=(
+ f"{head}{' '.join(keywords)}"
+ f"{body[tokens[-1].end() :]}{self.raw[comment_at:]}"
+ ),
+ )
class PackageList(immutable.Simple):
@@ -146,11 +174,16 @@ class PackageList(immutable.Simple):
) -> "PackageList":
"""Resolve the ``*`` and ``^`` sentinels.
- ``suggest`` returns the keywords a package should be requested for, in
- the order they should be written; returning nothing collapses the line
- to ``-``.
+ Only the lines that change are rewritten, and each keeps everything but
+ its keywords, so the result can go straight back onto the bug.
- :raises PackageListError: on ``^`` with nothing above it to copy
+ :param suggest: returns the keywords a package should be requested
+ for, in the order they should be written; returning nothing
+ collapses the line to ``-``. Which version to suggest for is the
+ caller's to decide
+ :return: the expanded list, or ``self`` if nothing needed rewriting
+ :raises PackageListError: on a ``^`` with nothing above it to copy, or
+ one that would copy nothing onto a line asking for keywords
"""
expanded: list[PackageListEntry] = []
previous: tuple[str, ...] | None = None
@@ -171,6 +204,16 @@ class PackageList(immutable.Simple):
lineno=entry.lineno,
line=entry.raw,
)
+ # copying nothing onto a line that asks for something else
+ # is a request no rewriting can satisfy
+ if not previous and len(entry.keywords) > 1:
+ raise PackageListError(
+ f"{SAME_KEYWORDS!r} keyword copies an empty line, "
+ "but the line has keywords of its own",
+ bug_id=self.bug_id,
+ lineno=entry.lineno,
+ line=entry.raw,
+ )
keywords.extend(previous)
else:
keywords.append(keyword)
diff --git a/src/pkgcore/ebuild/keywording.py b/src/pkgcore/ebuild/keywording.py
new file mode 100644
index 000000000..cd89bfe4a
--- /dev/null
+++ b/src/pkgcore/ebuild/keywording.py
@@ -0,0 +1,300 @@
+"""Resolving a keywording or stabilization request against a repo.
+
+A request is a list of package specs, each with the arches it asks for, in the
+form Gentoo's arch teams use::
+
+ =dev-libs/foo-1.2.3 amd64 x86 # stabilize this exact version
+ dev-libs/bar ~ppc64 # keyword any suitable version
+
+Three keyword sentinels are understood: :data:`ALL_KEYWORDS` expands to the
+arches the package could plausibly be requested for, :data:`SAME_KEYWORDS`
+repeats the line above, and :data:`NO_KEYWORDS` skips the line.
+
+:func:`match_packages` turns that into concrete packages and arches. It knows
+nothing about where the request came from; :meth:`pkgcore.bugzilla.Bug.match_packages`
+is the binding for one that came off a bug.
+"""
+
+__all__ = (
+ "ALL_KEYWORDS",
+ "NO_KEYWORDS",
+ "SAME_KEYWORDS",
+ "KeywordNoMatch",
+ "KeywordNoneLeft",
+ "KeywordNotSpecified",
+ "KeywordRequest",
+ "PackageInvalid",
+ "PackageListDoneAlready",
+ "PackageListEmpty",
+ "PackageMatchException",
+ "PackageNoMatch",
+ "can_stabilize_allarches",
+ "filter_prefix_keywords",
+ "match_packages",
+ "select_best_version",
+ "suggested_keywords",
+)
+
+import typing
+
+from ..exceptions import PkgcoreException
+from .misc import sort_keywords
+
+if typing.TYPE_CHECKING:
+ from .atom import atom
+ from .ebuild_src import package
+ from .repository import UnconfiguredTree
+
+ALL_KEYWORDS: typing.Final = "*"
+SAME_KEYWORDS: typing.Final = "^"
+NO_KEYWORDS: typing.Final = "-"
+
+
+class PackageMatchException(PkgcoreException):
+ """The request can't be resolved as written.
+
+ Callers routinely catch this broadly to report a request as unusable, which
+ is why :class:`KeywordNoneLeft` deliberately sits outside it.
+ """
+
+
+class PackageInvalid(PackageMatchException):
+ """A spec isn't one this kind of request may carry"""
+
+
+class PackageNoMatch(PackageMatchException):
+ """A spec parses but matches nothing in the repo"""
+
+
+class KeywordNoMatch(PackageMatchException):
+ """A keyword isn't known to the repo, or a sentinel can't be resolved"""
+
+
+class KeywordNotSpecified(PackageMatchException):
+ """Some packages were left without keywords"""
+
+ def __init__(self, packages: typing.Iterable[str], message: str = "") -> None:
+ self.packages = tuple(packages)
+ super().__init__(
+ message or f"incomplete keywords for packages: {' '.join(self.packages)}"
+ )
+
+
+class PackageListEmpty(PackageMatchException):
+ """Nothing was left to act on"""
+
+
+class PackageListDoneAlready(PackageListEmpty):
+ """Nothing was left because every package is keyworded already"""
+
+
+class KeywordNoneLeft(PkgcoreException):
+ """No keywords were given and there are none left to suggest.
+
+ Deliberately not a :class:`PackageMatchException`: the request is fine, it
+ simply has nothing to do, and a caller reporting broken requests must not
+ treat this as one.
+ """
+
+
+class KeywordRequest(typing.NamedTuple):
+ """A matched package and the arches requested for it"""
+
+ pkg: "package"
+ keywords: list[str]
+
+
+def select_best_version(matches: typing.Iterable["package"]) -> "package | None":
+ """Pick the version a request without an exact one should act on.
+
+ The newest keyworded version wins; failing that the newest non-live one,
+ since a live ebuild is never what a keywording request means.
+
+ :param matches: candidate packages, in any order
+ :return: the version to act on, or None if there were no candidates
+ """
+ ordered = sorted(matches, reverse=True)
+ for suitable in (lambda p: bool(p.keywords), lambda p: not p.live, lambda p: True):
+ for pkg in ordered:
+ if suitable(pkg):
+ return pkg
+ return None
+
+
+def filter_prefix_keywords(keywords: typing.Iterable[str]) -> list[str]:
+ """Drop prefix keywords, e.g. ``x86-macos`` or ``*-fbsd``"""
+ return [x for x in keywords if "-" not in x]
+
+
+def suggested_keywords(
+ repo: "UnconfiguredTree", pkg: "package", *, stable: bool
+) -> frozenset[str]:
+ """The arches :data:`ALL_KEYWORDS` expands to for ``pkg``.
+
+ Stabilizing, that is the arches already stable on some other version and
+ ``~arch`` on this one. Keywording, it is the arches present on some other
+ version and missing here. Prefix keywords are never suggested.
+
+ :param repo: repo to look for the package's other versions in
+ :param pkg: the version the request names
+ :param stable: whether the request is a stabilization
+ :return: the arches to request, unordered
+ """
+ disallowed = "-~" if stable else "-"
+ candidates = {
+ x.lstrip("~")
+ for other in repo.match(pkg.unversioned_atom)
+ for x in other.keywords
+ if x[0] not in disallowed
+ }
+ if stable:
+ # a version can only go stable where it is currently testing
+ candidates &= {x.lstrip("~") for x in pkg.keywords if x[0] == "~"}
+ else:
+ candidates -= {x.lstrip("~-") for x in pkg.keywords}
+ return frozenset(filter_prefix_keywords(candidates))
+
+
+def can_stabilize_allarches(
+ repo: "UnconfiguredTree",
+ requests: typing.Iterable[tuple["package", typing.Iterable[str]]],
+) -> bool:
+ """Whether every package already has a stable version on every arch asked for.
+
+ That is the precondition for one arch team stabilizing on behalf of all of
+ them: nobody is being asked to take on an arch they never had.
+
+ :param repo: repo to look for the packages' other versions in
+ :param requests: packages paired with the arches asked for
+ :return: whether an all-arches stabilization is permissible
+ """
+ for pkg, keywords in requests:
+ left = set(keywords)
+ for other in repo.itermatch(pkg.unversioned_atom):
+ # ~arch and -arch simply won't match, so they need no filtering
+ left.difference_update(other.keywords)
+ if left:
+ return False
+ return True
+
+
+def match_packages(
+ repo: "UnconfiguredTree",
+ requested: typing.Iterable[tuple["atom", typing.Sequence[str]]],
+ *,
+ stable: bool,
+ cc_arches: typing.Sequence[str] = (),
+ only_new: bool = False,
+ filter_arch: typing.Iterable[str] = (),
+ allarches: bool = False,
+) -> typing.Iterator[KeywordRequest]:
+ """Match ``requested`` specs against ``repo``, yielding packages and arches.
+
+ :param repo: repo to resolve the specs against
+ :param requested: each package spec paired with the keywords written for
+ it, in order, since :data:`SAME_KEYWORDS` refers to the line above
+ :param stable: select stabilization semantics: only ``=`` specs are
+ allowed, and :data:`ALL_KEYWORDS` means "arches stable elsewhere"
+ rather than "arches keyworded elsewhere"
+ :param cc_arches: the arches the request is addressed to. A line with no
+ keywords of its own inherits them; a line with keywords is narrowed to
+ them, and one left with nothing is skipped
+ :param only_new: drop the arches the package already carries
+ :param filter_arch: keep only the arches listed
+ :param allarches: re-add every candidate arch on top of ``filter_arch``,
+ for an all-arches stabilization
+ :return: the matched packages, each with the arches to request for it
+ :raises PackageMatchException: if the request can't be resolved
+ :raises KeywordNoneLeft: if nothing was specified and nothing is left
+ """
+ valid_arches = frozenset(repo.known_arches)
+ cc_arches = tuple(cc_arches)
+ filter_arch = frozenset(filter_arch)
+
+ keyworded_already = filtered = yielded = False
+ no_potential_keywords: list[str] = []
+ no_keywords: list[str] = []
+ previous: list[str] | None = None
+
+ for dep, written in requested:
+ if stable and (dep.op != "=" or dep.slot):
+ raise PackageInvalid(f"disallowed package spec (only = allowed): {dep}")
+ # a stabilization spec is an exact =cpv, so it matches at most one
+ matched = repo.match(dep)
+ pkg = matched[0] if stable and matched else select_best_version(matched)
+ if pkg is None:
+ raise PackageNoMatch(f"no match for package: {dep}")
+
+ keywords = [x.strip().lstrip("~") for x in written]
+ if NO_KEYWORDS in keywords:
+ continue
+ if ALL_KEYWORDS in keywords:
+ keywords = sort_keywords(suggested_keywords(repo, pkg, stable=stable)) + [
+ x for x in keywords if x != ALL_KEYWORDS
+ ]
+ if SAME_KEYWORDS in keywords:
+ if previous is None:
+ raise KeywordNoMatch(f"invalid use of {SAME_KEYWORDS} on first line")
+ keywords = previous + [x for x in keywords if x != SAME_KEYWORDS]
+
+ if unknown := frozenset(keywords) - valid_arches:
+ raise KeywordNoMatch(f"incorrect keywords: {' '.join(sorted(unknown))}")
+
+ if not keywords:
+ keywords = list(cc_arches)
+ elif cc_arches:
+ keywords = [x for x in keywords if x in cc_arches]
+ # the line is no longer addressed to anyone
+ if not keywords:
+ continue
+
+ if not keywords:
+ if suggested_keywords(repo, pkg, stable=stable):
+ no_keywords.append(str(dep))
+ else:
+ no_potential_keywords.append(str(dep))
+ yield KeywordRequest(pkg, keywords)
+ continue
+ previous = keywords
+
+ # still filtered by arch, since the arches asked for may be disjoint
+ # with the all-arches candidates
+ allarches_kw: list[str] = []
+ if allarches and stable and filter_arch:
+ allarches_kw = sort_keywords(suggested_keywords(repo, pkg, stable=True))
+
+ if only_new:
+ keywords = [
+ k
+ for k in keywords
+ if k not in pkg.keywords and (stable or f"~{k}" not in pkg.keywords)
+ ]
+ if not keywords:
+ keyworded_already = True
+ continue
+
+ if filter_arch:
+ keywords = [k for k in keywords if k in filter_arch]
+ keywords += [k for k in allarches_kw if k not in keywords]
+ if not keywords:
+ filtered = True
+ continue
+
+ yield KeywordRequest(pkg, keywords)
+ yielded = True
+
+ if no_keywords:
+ raise KeywordNotSpecified(no_keywords)
+ if no_potential_keywords:
+ # only worth reporting as "nothing left" if nothing else came out
+ if yielded:
+ raise KeywordNotSpecified(no_potential_keywords)
+ raise KeywordNoneLeft(
+ "package keywords in line with other versions and none specified"
+ )
+ if not yielded:
+ if filtered:
+ raise PackageListEmpty("no packages match requested arch")
+ if keyworded_already:
+ raise PackageListDoneAlready("all packages keyworded already")
+ raise PackageListEmpty("empty package list")
diff --git a/src/pkgcore/ebuild/misc.py b/src/pkgcore/ebuild/misc.py
index d71edfb61..3b9bff574 100644
--- a/src/pkgcore/ebuild/misc.py
+++ b/src/pkgcore/ebuild/misc.py
@@ -19,6 +19,7 @@ __all__ = (
import os.path
from collections import defaultdict, namedtuple
+from collections.abc import Iterable, Reversible
from functools import partial
from itertools import chain
@@ -33,7 +34,7 @@ restrict_payload = namedtuple("restrict_data", ["restrict", "data"])
chunked_data = namedtuple("chunked_data", ("key", "neg", "pos"))
-def sort_keywords(keywords):
+def sort_keywords(keywords: Iterable[str]):
"""Sort keywords in the proper order: i.e. glob-arches, arch, prefix-arches."""
def _sort_kwds(kw):
@@ -43,7 +44,7 @@ def sort_keywords(keywords):
return sorted(keywords, key=_sort_kwds)
-def optimize_incrementals(sequence):
+def optimize_incrementals(sequence: Reversible[str]):
# roughly the algorithm walks sequences right->left,
# identifying terminal points for incrementals; aka, -x x, 'x'
# is the terminal point- no point in having -x.
diff --git a/tests/bugzilla/test_pkglist.py b/tests/bugzilla/test_pkglist.py
index 0d5d2effa..52dd91cdf 100644
--- a/tests/bugzilla/test_pkglist.py
+++ b/tests/bugzilla/test_pkglist.py
@@ -27,8 +27,13 @@ class TestParseAtom:
"",
"not-an-atom",
"!dev-libs/foo",
+ "<>dev-libs/foo-1",
+ "dev-libs/foo-*",
"dev-libs/foo[bar]",
"dev-libs/foo:*",
+ "=dev-libs/foo-1:=",
+ "dev-libs/foo::gentoo",
+ "=dev-libs/foo-1::gentoo",
),
)
def test_invalid(self, token):
@@ -181,6 +186,31 @@ class TestExpand:
pkglist = PackageList("dev-libs/a amd64\n")
assert pkglist.expand(lambda pkg: ("arm",)) is pkglist
+ def test_keeps_the_spec_as_written(self):
+ # a bare cat/pkg-1 parses as =cat/pkg-1, but it has to go back the way the reporter wrote it
+ pkglist = PackageList("dev-libs/a-1.2.3 *\n")
+ assert str(pkglist.expand(lambda pkg: ("arm",))) == "dev-libs/a-1.2.3 arm\n"
+
+ def test_keeps_the_equals_prefix_when_it_was_written(self):
+ pkglist = PackageList("=dev-libs/a-1.2.3 *\n")
+ assert str(pkglist.expand(lambda pkg: ("arm",))) == "=dev-libs/a-1.2.3 arm\n"
+
+ def test_keeps_column_alignment(self):
+ pkglist = PackageList("dev-libs/a-1 * # aligned\n")
+ assert str(pkglist.expand(lambda pkg: ("amd64", "x86"))) == (
+ "dev-libs/a-1 amd64 x86 # aligned\n"
+ )
+
+ def test_same_keywords_copying_an_empty_line(self):
+ pkglist = PackageList("dev-libs/a -\ndev-libs/b ^")
+ assert str(pkglist.expand(lambda pkg: ())) == "dev-libs/a -\ndev-libs/b -"
+
+ def test_same_keywords_copying_empty_onto_a_line_with_keywords(self):
+ # nothing to copy, yet the line asks for something: unanswerable
+ pkglist = PackageList("dev-libs/a\ndev-libs/b ^ amd64", bug_id=7)
+ with pytest.raises(PackageListError, match="copies an empty line"):
+ pkglist.expand(lambda pkg: ())
+
def test_blank_lines_do_not_reset_previous(self):
pkglist = PackageList("dev-libs/a amd64\n\n# note\ndev-libs/b ^")
assert str(pkglist.expand(lambda pkg: ())) == (
@@ -196,6 +226,19 @@ class TestPackageListEntry:
assert updated.keywords == ("arm", "ppc")
assert updated.lineno == entry.lineno
+ def test_with_keywords_keeps_the_spec_verbatim(self):
+ (entry,) = PackageList("dev-libs/a-1.2.3 amd64").entries
+ assert entry.pkg is not None and str(entry.pkg) == "=dev-libs/a-1.2.3"
+ assert entry.with_keywords(("arm",)).raw == "dev-libs/a-1.2.3 arm"
+
+ def test_with_keywords_on_a_line_that_had_none(self):
+ (entry,) = PackageList("dev-libs/a-1 # note").entries
+ assert entry.with_keywords(("arm",)).raw == "dev-libs/a-1 arm # note"
+
+ def test_with_keywords_emptied(self):
+ (entry,) = PackageList("dev-libs/a-1 amd64 # note").entries
+ assert entry.with_keywords(()).raw == "dev-libs/a-1 # note"
+
def test_with_keywords_on_blank_is_a_noop(self):
(entry,) = PackageList("# just a comment").entries
assert entry.with_keywords(("arm",)) is entry
diff --git a/tests/ebuild/test_keywording.py b/tests/ebuild/test_keywording.py
new file mode 100644
index 000000000..b6e1932a3
--- /dev/null
+++ b/tests/ebuild/test_keywording.py
@@ -0,0 +1,458 @@
+import pytest
+
+from pkgcore.bugzilla import BugCategory
+from pkgcore.bugzilla.pkglist import PackageList
+from pkgcore.ebuild.keywording import (
+ KeywordNoMatch,
+ KeywordNoneLeft,
+ KeywordNotSpecified,
+ PackageInvalid,
+ PackageListDoneAlready,
+ PackageListEmpty,
+ PackageMatchException,
+ PackageNoMatch,
+ can_stabilize_allarches,
+ filter_prefix_keywords,
+ match_packages,
+ select_best_version,
+ suggested_keywords,
+)
+from pkgcore.exceptions import PkgcoreException
+from pkgcore.pytest.plugin import EbuildRepo
+
+# prefix arches are here on purpose: they are what exercises the prefix filter
+# through the ``*`` sentinel, and the unknown-keyword check reads arch.list
+ARCHES = ("alpha", "amd64", "hppa", "amd64-linux", "sparc-freebsd", "x86-macos")
+
+PKGS = {
+ "test/amd64-testing-1": {"keywords": ["~amd64"]},
+ "test/amd64-testing-2": {"keywords": ["~amd64"]},
+ "test/amd64-testing-10": {"keywords": []},
+ "test/amd64-testing-9999": {"keywords": [], "properties": "live"},
+ "test/amd64-stable-1": {"keywords": ["amd64"]},
+ "test/amd64-stable-10": {"keywords": ["~amd64"]},
+ "test/amd64-stable-hppa-testing-1": {"keywords": ["~alpha", "amd64", "~hppa"]},
+ "test/amd64-stable-hppa-testing-2": {"keywords": ["~alpha", "amd64", "~hppa"]},
+ "test/mixed-keywords-1": {"keywords": ["~alpha", "amd64", "~hppa"]},
+ "test/mixed-keywords-2": {"keywords": ["~alpha", "~amd64", "hppa"]},
+ "test/mixed-keywords-3": {"keywords": ["~alpha", "~amd64", "~hppa"]},
+ "test/mixed-keywords-4": {"keywords": ["~amd64"]},
+ "test/prefix-keywords-1": {"keywords": ["amd64", "x86-macos"]},
+ "test/prefix-keywords-2": {"keywords": ["~amd64", "~x86-macos"]},
+}
+
+
[email protected](scope="session")
+def repo(tmp_path_factory):
+ """A repo shared by the whole module; metadata regenerates once."""
+ # the shipped repo/make_repo fixtures are function scoped, and regenerating
+ # metadata per test is the dominant cost here
+ tree = EbuildRepo(
+ str(tmp_path_factory.mktemp("keywording")), repo_id="test", arches=ARCHES
+ )
+ for cpvstr, attrs in PKGS.items():
+ tree.create_ebuild(cpvstr, **attrs)
+ tree.sync()
+ return tree
+
+
+def match(repo, text, **kwargs):
+ """Resolve a package list written the way a bug would carry it"""
+ kwargs.setdefault("stable", True)
+ requested = [
+ (entry.pkg, entry.keywords)
+ for entry in PackageList(text).entries
+ if entry.pkg is not None
+ ]
+ return [(pkg.cpvstr, kw) for pkg, kw in match_packages(repo, requested, **kwargs)]
+
+
+class TestSelectBestVersion:
+ def test_prefers_newest_keyworded(self, repo):
+ matched = repo.match(PackageList("test/amd64-testing").atoms[0])
+ assert select_best_version(matched).cpvstr == "test/amd64-testing-2"
+
+ def test_ignores_live(self, repo):
+ matched = [
+ p
+ for p in repo.match(PackageList("test/amd64-testing").atoms[0])
+ if not p.keywords
+ ]
+ assert select_best_version(matched).cpvstr == "test/amd64-testing-10"
+
+ def test_falls_back_to_newest(self, repo):
+ matched = [
+ p for p in repo.match(PackageList("test/amd64-testing").atoms[0]) if p.live
+ ]
+ assert select_best_version(matched).cpvstr == "test/amd64-testing-9999"
+
+ def test_empty(self):
+ assert select_best_version([]) is None
+
+
+class TestFilterPrefixKeywords:
+ def test_drops_prefix_arches(self):
+ assert filter_prefix_keywords(["amd64", "x86-macos", "hppa"]) == [
+ "amd64",
+ "hppa",
+ ]
+
+ def test_keeps_plain_arches(self):
+ assert filter_prefix_keywords(["amd64"]) == ["amd64"]
+
+
+class TestSuggestedKeywords:
+ def test_stable_is_limited_to_current_testing(self, repo):
+ pkg = repo.match(PackageList("=test/mixed-keywords-3").atoms[0])[0]
+ # amd64 and hppa are stable on other versions and ~arch here
+ assert suggested_keywords(repo, pkg, stable=True) == frozenset(
+ {"amd64", "hppa"}
+ )
+
+ def test_keywording_is_what_is_missing(self, repo):
+ pkg = repo.match(PackageList("=test/mixed-keywords-4").atoms[0])[0]
+ assert suggested_keywords(repo, pkg, stable=False) == frozenset(
+ {"alpha", "hppa"}
+ )
+
+ def test_prefix_arches_are_never_suggested(self, repo):
+ pkg = repo.match(PackageList("=test/prefix-keywords-2").atoms[0])[0]
+ assert "x86-macos" not in suggested_keywords(repo, pkg, stable=True)
+
+
+class TestSpecValidation:
+ """Only the category-dependent rule lives here.
+
+ Specs that simply aren't package atoms are rejected while parsing, by
+ :func:`~pkgcore.bugzilla.pkglist.parse_atom`, and are tested there.
+ """
+
+ # a stabilization names one exact version; keywording may name a range
+ STABLE_ONLY_REJECTS = (
+ ">=test/amd64-testing-1",
+ "test/amd64-testing",
+ "test/amd64-testing:0",
+ )
+
+ @pytest.mark.parametrize("spec", STABLE_ONLY_REJECTS)
+ def test_rejected_when_stabilizing(self, repo, spec):
+ with pytest.raises(PackageInvalid):
+ match(repo, f"{spec} amd64", stable=True)
+
+ @pytest.mark.parametrize("spec", STABLE_ONLY_REJECTS)
+ def test_accepted_when_keywording(self, repo, spec):
+ assert match(repo, f"{spec} amd64", stable=False)
+
+ def test_no_match(self, repo):
+ with pytest.raises(PackageNoMatch, match="no match for package"):
+ match(repo, "=test/no-such-package-1 amd64")
+
+ def test_no_match_outranks_a_pending_keyword_complaint(self, repo):
+ # an earlier line with no keywords is only reported at the end, so a
+ # later unmatchable line must still raise
+ with pytest.raises(PackageNoMatch):
+ match(repo, "test/mixed-keywords-3\n=test/no-such-package-1\n")
+
+
+class TestMatching:
+ def test_versioned_list(self, repo):
+ assert match(
+ repo,
+ """
+ test/amd64-testing-1 amd64
+ =test/amd64-testing-2 amd64
+ test/amd64-stable-hppa-testing-1 hppa
+ """,
+ ) == [
+ ("test/amd64-testing-1", ["amd64"]),
+ ("test/amd64-testing-2", ["amd64"]),
+ ("test/amd64-stable-hppa-testing-1", ["hppa"]),
+ ]
+
+ def test_keywording_picks_the_best_version(self, repo):
+ assert match(repo, "test/amd64-testing alpha", stable=False) == [
+ ("test/amd64-testing-2", ["alpha"])
+ ]
+
+ def test_tilde_prefixes_are_stripped(self, repo):
+ assert match(repo, "test/amd64-testing ~alpha", stable=False) == [
+ ("test/amd64-testing-2", ["alpha"])
+ ]
+
+ def test_comment_is_ignored(self, repo):
+ assert match(repo, "test/amd64-testing-1 amd64 # why") == [
+ ("test/amd64-testing-1", ["amd64"])
+ ]
+
+ def test_unknown_keyword(self, repo):
+ with pytest.raises(KeywordNoMatch, match="incorrect keywords"):
+ match(repo, "test/amd64-testing-1 nosucharch")
+
+
+class TestSentinels:
+ def test_all_keywords_when_stabilizing(self, repo):
+ assert match(repo, "test/mixed-keywords-3 *") == [
+ ("test/mixed-keywords-3", ["amd64", "hppa"])
+ ]
+
+ def test_all_keywords_when_keywording(self, repo):
+ assert match(repo, "=test/mixed-keywords-4 *", stable=False) == [
+ ("test/mixed-keywords-4", ["alpha", "hppa"])
+ ]
+
+ def test_all_keywords_alongside_explicit(self, repo):
+ ((_, keywords),) = match(repo, "test/mixed-keywords-3 * alpha")
+ assert keywords == ["amd64", "hppa", "alpha"]
+
+ def test_same_keywords(self, repo):
+ assert match(
+ repo,
+ """
+ test/amd64-stable-hppa-testing-1 hppa
+ test/mixed-keywords-3 ^
+ """,
+ ) == [
+ ("test/amd64-stable-hppa-testing-1", ["hppa"]),
+ ("test/mixed-keywords-3", ["hppa"]),
+ ]
+
+ def test_same_keywords_survives_only_new(self, repo):
+ # ^ copies the keywords as written, before only_new prunes them, so
+ # each line drops only what that version already carries
+ assert match(
+ repo,
+ """
+ test/amd64-testing-1 amd64 alpha
+ test/amd64-testing-10 ^
+ test/amd64-testing-2 ^
+ """,
+ stable=False,
+ only_new=True,
+ ) == [
+ ("test/amd64-testing-1", ["alpha"]),
+ ("test/amd64-testing-10", ["amd64", "alpha"]),
+ ("test/amd64-testing-2", ["alpha"]),
+ ]
+
+ def test_same_keywords_on_first_line(self, repo):
+ with pytest.raises(KeywordNoMatch, match="first line"):
+ match(repo, "test/amd64-testing-1 ^")
+
+ def test_no_keywords_skips_the_line(self, repo):
+ assert match(
+ repo,
+ """
+ test/amd64-testing-1 -
+ test/amd64-testing-2 amd64
+ """,
+ ) == [("test/amd64-testing-2", ["amd64"])]
+
+
+class TestCcArches:
+ def test_empty_keywords_inherit_cc(self, repo):
+ assert match(repo, "test/mixed-keywords-3", cc_arches=("amd64",)) == [
+ ("test/mixed-keywords-3", ["amd64"])
+ ]
+
+ def test_keywords_are_narrowed_to_cc(self, repo):
+ assert match(
+ repo, "test/mixed-keywords-3 amd64 hppa", cc_arches=("amd64",)
+ ) == [("test/mixed-keywords-3", ["amd64"])]
+
+ def test_line_disjoint_from_cc_is_dropped(self, repo):
+ assert match(
+ repo,
+ """
+ test/mixed-keywords-3 hppa
+ test/amd64-stable-hppa-testing-1 amd64
+ """,
+ cc_arches=("amd64",),
+ ) == [("test/amd64-stable-hppa-testing-1", ["amd64"])]
+
+
+class TestOnlyNew:
+ def test_stable_drops_arches_already_stable(self, repo):
+ assert match(
+ repo, "test/amd64-stable-hppa-testing-1 amd64 hppa", only_new=True
+ ) == [("test/amd64-stable-hppa-testing-1", ["hppa"])]
+
+ def test_keywording_also_drops_testing_arches(self, repo):
+ # ~amd64 already satisfies a keywording request for amd64
+ with pytest.raises(PackageListDoneAlready):
+ match(repo, "test/amd64-testing-1 amd64", stable=False, only_new=True)
+
+ def test_stabilizing_does_not_treat_testing_as_done(self, repo):
+ assert match(repo, "test/mixed-keywords-3 amd64", only_new=True) == [
+ ("test/mixed-keywords-3", ["amd64"])
+ ]
+
+
+class TestFilterArch:
+ def test_keeps_only_the_listed_arch(self, repo):
+ assert match(
+ repo, "test/mixed-keywords-3 amd64 hppa", filter_arch=("amd64",)
+ ) == [("test/mixed-keywords-3", ["amd64"])]
+
+ def test_everything_filtered_away(self, repo):
+ with pytest.raises(PackageListEmpty, match="no packages match"):
+ match(repo, "test/mixed-keywords-3 hppa", filter_arch=("amd64",))
+
+
+class TestAllarches:
+ def test_readds_candidates_past_the_filter(self, repo):
+ # one arch team stabilizes on behalf of the rest
+ ((_, keywords),) = match(
+ repo,
+ "test/mixed-keywords-3 amd64 hppa",
+ filter_arch=("amd64",),
+ allarches=True,
+ )
+ assert keywords == ["amd64", "hppa"]
+
+ def test_readds_candidates_the_filter_excluded(self, repo):
+ # the requested arch comes first, then whatever allarches adds
+ ((_, keywords),) = match(
+ repo,
+ "test/mixed-keywords-4 amd64 hppa",
+ filter_arch=("hppa",),
+ allarches=True,
+ )
+ assert keywords == ["hppa", "amd64"]
+
+ def test_ignored_without_a_filter(self, repo):
+ assert match(repo, "test/mixed-keywords-3 amd64", allarches=True) == [
+ ("test/mixed-keywords-3", ["amd64"])
+ ]
+
+ def test_ignored_when_keywording(self, repo):
+ assert match(
+ repo,
+ "=test/mixed-keywords-4 alpha hppa",
+ stable=False,
+ filter_arch=("alpha",),
+ allarches=True,
+ ) == [("test/mixed-keywords-4", ["alpha"])]
+
+
+class TestTerminalOutcomes:
+ def test_empty_list(self, repo):
+ with pytest.raises(PackageListEmpty, match="empty package list"):
+ match(repo, "")
+
+ def test_all_done_already(self, repo):
+ with pytest.raises(PackageListDoneAlready):
+ match(repo, "test/amd64-stable-1 amd64", only_new=True)
+
+ def test_keywords_not_specified(self, repo):
+ with pytest.raises(KeywordNotSpecified) as excinfo:
+ match(repo, "test/mixed-keywords-3")
+ assert excinfo.value.packages == ("=test/mixed-keywords-3",)
+
+ def test_nothing_left_to_suggest(self, repo):
+ # stable everywhere it is testing, and nothing was asked for
+ with pytest.raises(KeywordNoneLeft):
+ match(repo, "test/amd64-stable-1")
+
+ def test_nothing_left_but_others_yielded(self, repo):
+ # a partial result keeps the request interesting, so it is reported as
+ # incomplete rather than as nothing-to-do
+ with pytest.raises(KeywordNotSpecified):
+ match(
+ repo,
+ """
+ test/mixed-keywords-3 amd64
+ test/amd64-stable-1
+ """,
+ )
+
+
+class TestCanStabilizeAllarches:
+ def test_all_arches_have_a_stable_version(self, repo):
+ pkg = repo.match(PackageList("=test/mixed-keywords-3").atoms[0])[0]
+ assert can_stabilize_allarches(repo, [(pkg, ["amd64", "hppa"])])
+
+ def test_an_arch_has_never_been_stable(self, repo):
+ pkg = repo.match(PackageList("=test/mixed-keywords-3").atoms[0])[0]
+ assert not can_stabilize_allarches(repo, [(pkg, ["amd64", "alpha"])])
+
+ def test_no_keywords_requested(self, repo):
+ pkg = repo.match(PackageList("=test/mixed-keywords-3").atoms[0])[0]
+ assert can_stabilize_allarches(repo, [(pkg, [])])
+
+
+def test_notice_escapes_the_broad_handler():
+ # a caller reporting broken requests must not swallow "nothing to do"
+ assert issubclass(KeywordNoneLeft, PkgcoreException)
+ assert not issubclass(KeywordNoneLeft, PackageMatchException)
+
+
+class TestBugBinding:
+ def bug(self, category=BugCategory.STABLEREQ, packages="", cc=()):
+ from pkgcore.bugzilla import Bug
+
+ return Bug(
+ product=str(category.product),
+ component=str(category.component),
+ package_list=PackageList(packages),
+ cc=tuple(cc),
+ )
+
+ def test_category_selects_stabilizing(self, repo):
+ bug = self.bug(packages="test/amd64-testing-1 amd64")
+ assert [(p.cpvstr, k) for p, k in bug.match_packages(repo)] == [
+ ("test/amd64-testing-1", ["amd64"])
+ ]
+
+ def test_keywording_bug_allows_a_bare_atom(self, repo):
+ bug = self.bug(BugCategory.KEYWORDREQ, "test/amd64-testing alpha")
+ assert [(p.cpvstr, k) for p, k in bug.match_packages(repo)] == [
+ ("test/amd64-testing-2", ["alpha"])
+ ]
+
+ def test_cc_supplies_the_arches(self, repo):
+ bug = self.bug(
+ packages="test/mixed-keywords-3", cc=("[email protected]", "someone")
+ )
+ assert [(p.cpvstr, k) for p, k in bug.match_packages(repo)] == [
+ ("test/mixed-keywords-3", ["amd64"])
+ ]
+
+ def test_cc_arches_survive_anonymous_truncation(self, repo):
+ # without an api key bugzilla cuts every address at the @
+ bug = self.bug(
+ packages="test/mixed-keywords-3", cc=("amd64", "hppa", "someone")
+ )
+ assert [(p.cpvstr, k) for p, k in bug.match_packages(repo)] == [
+ ("test/mixed-keywords-3", ["amd64", "hppa"])
+ ]
+
+ def test_allarches_keyword_is_honoured(self, repo):
+ from pkgcore.bugzilla import Bug
+
+ bug = Bug(
+ product="Gentoo Linux",
+ component="Stabilization",
+ package_list=PackageList("test/mixed-keywords-3 amd64 hppa"),
+ keywords=("ALLARCHES",),
+ )
+ ((_, keywords),) = bug.match_packages(
+ repo, filter_arch=("amd64",), permit_allarches=True
+ )
+ assert keywords == ["amd64", "hppa"]
+
+ def test_allarches_needs_opting_in(self, repo):
+ from pkgcore.bugzilla import Bug
+
+ bug = Bug(
+ product="Gentoo Linux",
+ component="Stabilization",
+ package_list=PackageList("test/mixed-keywords-3 amd64 hppa"),
+ keywords=("ALLARCHES",),
+ )
+ ((_, keywords),) = bug.match_packages(repo, filter_arch=("amd64",))
+ assert keywords == ["amd64"]
+
+ def test_malformed_list_surfaces_as_package_invalid(self, repo):
+ bug = self.bug(packages="not an atom")
+ with pytest.raises(PackageInvalid):
+ list(bug.match_packages(repo))