proj/pkgcore/pkgcore:master commit in: /, src/pkgcore/ebuild/, tests/ebuild/

commit:     e16d4f3e043741390af59c476497a3026aae0c1b
Author:     Arthur Zamarin <arthurzam <AT> gentoo <DOT> org>
AuthorDate: Fri Aug  7 20:46:51 2026 +0000
Commit:     Arthur Zamarin <arthurzam <AT> gentoo <DOT> org>
CommitDate: Fri Aug  7 20:46:51 2026 +0000
URL:        https://gitweb.gentoo.org/proj/pkgcore/pkgcore.git/commit/?id=e16d4f3e

misc: intern chunked_data when optimizing a ChunkedDataDict

Profiles inherit from shared parents, so the same chunk is rebuilt for
every profile that pulls it in. Those are equal but distinct objects,
and pickle memoizes on identity, so each copy is written out in full and
read back as it's own object. In the gentoo tree 519456 chunk slots hold
only 166998 distinct values, and their neg/pos tuples are 394130 objects
for 97981 distinct values.

Canonicalize both in _build_cp_atom_payload, keyed off the cache dict
already threaded through optimize() - pkgcheck passes one spanning every
profile in a repo. That cache only memoized whole result tuples keyed on
the entire input sequence, so two profiles differing in one entry shared
nothing; interning the individual chunks catches what it can't.

Regenerating the pkgcheck profiles cache for the gentoo tree:

    on disk (zstd)   4.53 MB -> 3.30 MB
    serialized      226.1 MB -> 114.9 MB
    load              0.652s -> 0.326s
    dump              0.860s -> 0.488s
    live chunks       230262 -> 167024

Cache generation itself is unchanged at ~9.5s, and a scan gives byte
identical results either way.

Signed-off-by: Arthur Zamarin <arthurzam <AT> gentoo.org>

 NEWS.rst                   |  8 +++++++
 src/pkgcore/ebuild/misc.py | 36 +++++++++++++++++++++++++++----
 tests/ebuild/test_misc.py  | 53 +++++++++++++++++++++++++++++++++++++++++++++-
 3 files changed, 92 insertions(+), 5 deletions(-)

diff --git a/NEWS.rst b/NEWS.rst
index 7c9ad54c8..ebc16f818 100644
--- a/NEWS.rst
+++ b/NEWS.rst
@@ -20,6 +20,14 @@ Fixes
 Changes
 ~~~~~~~
 
+- ``pkgcore.ebuild.misc.ChunkedDataDict.optimize``: collapse equal
+  ``chunked_data``, and their neg/pos tuples, onto a single object.  Profiles
+  inherit from shared parents, so the same chunk was rebuilt per profile;
+  pickle memoizes on identity, so each copy was written out in full.  For the
+  ::gentoo tree this halves the pkgcheck profiles cache (226MB -> 115MB
+  serialized, 4.5MB -> 3.3MB on disk), loads it 2x faster, and drops 63k live
+  objects.  Cache generation time is unchanged (Arthur Zamarin)
+
 - ``pkgcore.package.base`` and ``pkgcore.restrictions.restriction.base``: drop
   the deprecated ``snakeoil.klass.SlotsPicklingMixin``.  Python pickles
   ``__slots__`` natively, making pickling of restrictions ~4x faster.  Note

diff --git a/src/pkgcore/ebuild/misc.py b/src/pkgcore/ebuild/misc.py
index 05fdb0df4..d71edfb61 100644
--- a/src/pkgcore/ebuild/misc.py
+++ b/src/pkgcore/ebuild/misc.py
@@ -304,18 +304,43 @@ class non_incremental_collapsed_restrict_to_data(collapsed_restrict_to_data):
         return iflatten_instance(l)
 
 
+class _interner:
+    """Collapse equal chunked_data, and their neg/pos tuples, onto one object.
+
+    Profiles inherit from shared parents, so the same chunk is rebuilt for every
+    profile that pulls it in.  Those are equal but distinct objects; pickle memoizes
+    on identity, so each is written out in full and read back as it's own object.
+    """
+
+    __slots__ = ("chunks", "tuples")
+
+    def __init__(self):
+        self.chunks = {}
+        self.tuples = {}
+
+    def __call__(self, key, neg, pos):
+        t = self.tuples
+        item = chunked_data(key, t.setdefault(neg, neg), t.setdefault(pos, pos))
+        return self.chunks.setdefault(item, item)
+
+
+_interner_cache_key = "__chunked_data_interner__"
+
+
 def _cached_build_cp_atom_payload(cache, sequence, restrict, payload_form=False):
     sequence = list(sequence)
     key = (payload_form, restrict, tuple(sequence))
     val = cache.get(key)
     if val is None:
+        if (interner := cache.get(_interner_cache_key)) is None:
+            interner = cache[_interner_cache_key] = _interner()
         val = cache[key] = _build_cp_atom_payload(
-            sequence, restrict, payload_form=payload_form
+            sequence, restrict, payload_form=payload_form, interner=interner
         )
     return val
 
 
-def _build_cp_atom_payload(sequence, restrict, payload_form=False):
+def _build_cp_atom_payload(sequence, restrict, payload_form=False, interner=None):
     locked = {}
     ldefault = locked.setdefault
 
@@ -327,7 +352,7 @@ def _build_cp_atom_payload(sequence, restrict, payload_form=False):
             return restrict_payload(r, tuple(chain(("-" + x for x in neg), pos)))
 
     else:
-        f = chunked_data
+        f = chunked_data if interner is None else interner
 
     i = list(sequence)
     if len(i) <= 1:
@@ -493,14 +518,17 @@ class ChunkedDataDict(GenericEquality):
 
     def optimize(self, cache=None):
         if cache is None:
+            # no cross instance cache, but the keys of this one still share chunks.
+            interner = _interner()
             d_stream = (
-                (k, _build_cp_atom_payload(v, atom.atom(k), False))
+                (k, _build_cp_atom_payload(v, atom.atom(k), False, interner))
                 for k, v in self._dict.items()
             )
             g_stream = _build_cp_atom_payload(
                 self._global_settings,
                 packages.AlwaysTrue,
                 payload_form=isinstance(self, PayloadDict),
+                interner=interner,
             )
         else:
             d_stream = (

diff --git a/tests/ebuild/test_misc.py b/tests/ebuild/test_misc.py
index 106a26929..b9c7468a1 100644
--- a/tests/ebuild/test_misc.py
+++ b/tests/ebuild/test_misc.py
@@ -1,6 +1,6 @@
 import pytest
 
-from pkgcore.ebuild import misc
+from pkgcore.ebuild import atom, misc
 from pkgcore.restrictions import packages
 
 AlwaysTrue = packages.AlwaysTrue
@@ -79,6 +79,57 @@ def test_IncrementalsDict():
     assert len(d) == 0
 
 
+class TestChunkedDataInterning:
+    def mk_dict(self, *keys):
+        # via variables; a tuple literal is folded into a shared constant, which
+        # would hide whether interning did anything.
+        neg, pos = ["-x"], ["y", "z"]
+        d = misc.ChunkedDataDict()
+        for key in keys:
+            d.update_from_stream(
+                [misc.chunked_data(atom.atom(key), tuple(neg), tuple(pos))]
+            )
+        return d
+
+    @staticmethod
+    def chunks(d):
+        return [c for v in d.render_to_dict().values() for c in v]
+
+    def test_interner(self):
+        interner = misc._interner()
+        # via a variable; a tuple literal is folded into a shared constant.
+        values = ["y", "z"]
+        pos, same_pos = tuple(values), tuple(values)
+        assert pos is not same_pos
+        a = interner(atom.atom("dev-util/diffball"), (), pos)
+        b = interner(atom.atom("dev-util/diffball"), (), same_pos)
+        assert a is b
+        assert a.pos is pos
+
+    def test_shared_within_a_dict(self):
+        d = self.mk_dict("dev-util/diffball", "dev-util/foo")
+        assert len({id(c.pos) for c in self.chunks(d)}) == 2
+        d.optimize()
+        assert len({id(c.pos) for c in self.chunks(d)}) == 1
+
+    def test_shared_across_dicts_via_cache(self):
+        # distinct keys, so the existing whole sequence cache can't be what shares them
+        cache = {}
+        first, second = self.mk_dict("dev-util/diffball"), self.mk_dict("dev-util/foo")
+        first.optimize(cache=cache)
+        second.optimize(cache=cache)
+        assert len({id(c.pos) for c in self.chunks(first) + self.chunks(second)}) == 1
+
+    def test_interning_does_not_alter_values(self):
+        plain, interned = (
+            self.mk_dict("dev-util/diffball"),
+            self.mk_dict("dev-util/diffball"),
+        )
+        plain.optimize()
+        interned.optimize(cache={})
+        assert plain.render_to_dict() == interned.render_to_dict()
+
+
 @pytest.mark.parametrize(
     "expected,source,target",
     [