proj/portage:master commit in: bin/, lib/portage/dbapi/
"Matt Turner" <[email protected]>
| Newsgroups | gmane.linux.gentoo.cvs |
|---|---|
| Message-ID | <1786672955.841717804deeb41f47b2c067161d4bda20a155ea.mattst88@gentoo> |
commit: 841717804deeb41f47b2c067161d4bda20a155ea
Author: Matt Turner <mattst88 <AT> gentoo <DOT> org>
AuthorDate: Wed Jul 1 02:40:25 2026 +0000
Commit: Matt Turner <mattst88 <AT> gentoo <DOT> org>
CommitDate: Fri Aug 14 02:02:35 2026 +0000
URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=84171780
vartree: replace vdb_metadata.pickle with per-package metadata file
Remove the cross-session pickle cache (vdb_metadata.pickle +
vdb_metadata_delta.json) and VdbMetadataDelta entirely. The metadata file
covers the cross-session case: it is per-package state that needs no delta
bookkeeping, no whole-file rewrite, and no lock.
The in-session _aux_cache dict is retained for deduplicating repeated
aux_get() calls within a session.
Measured on this machine: 1742 installed packages, 23 keys, best of 7
full-VDB passes through _aux_get() in a fresh process, syscalls via
strace -f -c.
master 301.7 ms, 41012 openat
this series 32.9 ms, 2689 openat
The pickle existed to serve exactly this case, a process starting with no
in-memory cache. On this machine a warm pickle got master to 137.6 ms; the
metadata file is faster than that with no cross-session state to
invalidate. That 137.6 ms is an earlier measurement of master's own path,
not re-derived here, since warming the pickle needs write access to
/var/cache/edb.
aux_get() sourced _mtime_ from the pickle whenever the cache was warm, and
only fell back to the truncated int from a stat() on a miss. With the
pickle gone that int would have become the value callers always see, so
_aux_get() returns the float st_mtime instead and the observable value is
unchanged. Verified field-for-field across every installed package and 24
keys against a warm pickle: no differences.
Remove:
- VdbMetadataDelta (_VdbMetadataDelta.py deleted)
- pickle load/save in _aux_cache_init() and flush_cache()
- _aux_cache_version, _aux_cache_threshold, _flush_cache_enabled,
_aux_cache_filename, _cache_delta_filename, _cache_delta attrs
- _cache_delta.recordEvent() calls on merge/unmerge
Simplify IndexedVardb.cp_all() to always use _iter_cp_all() since the
delta-shortcut path depended on the now-removed pickle.
Signed-off-by: Matt Turner <mattst88 <AT> gentoo.org>
bin/vdb-benchmark | 42 ++++----
lib/portage/dbapi/IndexedVardb.py | 38 +------
lib/portage/dbapi/_MergeProcess.py | 2 -
lib/portage/dbapi/_VdbMetadataDelta.py | 180 ---------------------------------
lib/portage/dbapi/meson.build | 1 -
lib/portage/dbapi/vartree.py | 140 +++----------------------
6 files changed, 37 insertions(+), 366 deletions(-)
diff --git a/bin/vdb-benchmark b/bin/vdb-benchmark
index fbc55a96a..9e54a6e11 100755
--- a/bin/vdb-benchmark
+++ b/bin/vdb-benchmark
@@ -1,5 +1,5 @@
#!/usr/bin/env python
-# Copyright 2025 Gentoo Authors
+# Copyright 2026 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
"""Benchmark VDB metadata read performance.
@@ -17,6 +17,7 @@ import argparse
import os as _os
import subprocess
import sys
+import textwrap
import time
from os import path as osp
@@ -41,15 +42,13 @@ def _count_metadata_files(dbroot):
with_meta = 0
without_meta = 0
try:
- for cat in _os.listdir(dbroot):
- catdir = _os.path.join(dbroot, cat)
- if not _os.path.isdir(catdir):
+ for cat in _os.scandir(dbroot):
+ if not cat.is_dir():
continue
- for pkg in _os.listdir(catdir):
- pkgdir = _os.path.join(catdir, pkg)
- if not _os.path.isdir(pkgdir):
+ for pkg in _os.scandir(cat.path):
+ if not pkg.is_dir():
continue
- if _os.path.exists(_os.path.join(pkgdir, _METADATA_FILE)):
+ if _os.path.exists(_os.path.join(pkg.path, _METADATA_FILE)):
with_meta += 1
else:
without_meta += 1
@@ -61,8 +60,8 @@ def _count_metadata_files(dbroot):
def _run_read_benchmark(vardb, keys, iterations):
"""
Read all metadata keys for every installed package, repeated
- `iterations` times. Calls _aux_get() directly to bypass the pickle
- cache so results reflect actual file-read performance.
+ `iterations` times. Calls _aux_get() directly to bypass the
+ in-session cache so each pass reflects actual file-read performance.
Returns (cpvs, list-of-per-iteration-durations-in-seconds).
"""
cpvs = vardb.cpv_all()
@@ -159,16 +158,19 @@ def main(argv):
if opts.strace:
print("\nCounting openat() syscalls via strace (single pass)…")
# Build a self-contained script for strace to execute.
- script = f"""\
-import sys
-sys.path.insert(0, {repr(_os.path.join(_os.path.dirname(_os.path.dirname(_os.path.realpath(__file__))), "lib"))})
-import portage
-portage._internal_caller = True
-vardb = portage.db[{repr(eroot)}]["vartree"].dbapi
-keys = list(vardb._aux_cache_keys)
-for cpv in vardb.cpv_all():
- vardb._aux_get(cpv, keys)
-"""
+ libdir = _os.path.join(
+ _os.path.dirname(_os.path.dirname(_os.path.realpath(__file__))), "lib"
+ )
+ script = textwrap.dedent(f"""
+ import sys
+ sys.path.insert(0, {libdir!r})
+ import portage
+ portage._internal_caller = True
+ vardb = portage.db[{eroot!r}]["vartree"].dbapi
+ keys = list(vardb._aux_cache_keys)
+ for cpv in vardb.cpv_all():
+ vardb._aux_get(cpv, keys)
+ """)
count = _strace_open_count(script)
if count is None:
print("strace not available or failed to parse output.")
diff --git a/lib/portage/dbapi/IndexedVardb.py b/lib/portage/dbapi/IndexedVardb.py
index f9dec947e..6a0c4a9c9 100644
--- a/lib/portage/dbapi/IndexedVardb.py
+++ b/lib/portage/dbapi/IndexedVardb.py
@@ -3,23 +3,13 @@
import portage
from portage.dep import Atom
-from portage.exception import InvalidData
-from portage.versions import _pkg_str
class IndexedVardb:
"""
A vardbapi interface that sacrifices validation in order to
- improve performance. It takes advantage of vardbdbapi._aux_cache,
- which is backed by vdb_metadata.pickle. Since _aux_cache is
- not updated for every single merge/unmerge (see
- _aux_cache_threshold), the list of packages is obtained directly
- from the real vardbapi instance. If a package is missing from
- _aux_cache, then its metadata is obtained using the normal
- (validated) vardbapi.aux_get method.
-
- For performance reasons, the match method only supports package
- name and version constraints.
+ improve performance. For performance reasons, the match method
+ only supports package name and version constraints.
"""
# Match returns unordered results.
@@ -42,27 +32,7 @@ class IndexedVardb:
"""
if self._cp_map is not None:
return iter(sorted(self._cp_map)) if sort else iter(self._cp_map)
-
- delta_data = self._vardb._cache_delta.loadRace()
- if delta_data is None:
- return self._iter_cp_all()
-
- self._vardb._cache_delta.applyDelta(delta_data)
-
- self._cp_map = cp_map = {}
- for cpv in self._vardb._aux_cache["packages"]:
- try:
- cpv = _pkg_str(cpv, db=self._vardb)
- except InvalidData:
- continue
-
- cp_list = cp_map.get(cpv.cp)
- if cp_list is None:
- cp_list = []
- cp_map[cpv.cp] = cp_list
- cp_list.append(cpv)
-
- return iter(sorted(self._cp_map)) if sort else iter(self._cp_map)
+ return self._iter_cp_all()
def _iter_cp_all(self):
self._cp_map = cp_map = {}
@@ -107,8 +77,6 @@ class IndexedVardb:
):
pkg_data = None
if pkg_data is None:
- # It may be missing from _aux_cache due to
- # _aux_cache_threshold.
return self._vardb.aux_get(cpv, attrs)
metadata = pkg_data[1]
return [metadata.get(k, "") for k in attrs]
diff --git a/lib/portage/dbapi/_MergeProcess.py b/lib/portage/dbapi/_MergeProcess.py
index bb1848641..e87666420 100644
--- a/lib/portage/dbapi/_MergeProcess.py
+++ b/lib/portage/dbapi/_MergeProcess.py
@@ -246,8 +246,6 @@ class MergeProcess(ForkProcess):
# when not using the multiprocessing fork start method.
QueryCommand._db = db
portage.output.havecolor = not no_color(settings)
- # Avoid wastful updates of the vdb cache.
- vardb._flush_cache_enabled = False
# In this subprocess we don't want PORTAGE_BACKGROUND to
# suppress stdout/stderr output since they are pipes. We
diff --git a/lib/portage/dbapi/_VdbMetadataDelta.py b/lib/portage/dbapi/_VdbMetadataDelta.py
deleted file mode 100644
index 1362deea2..000000000
--- a/lib/portage/dbapi/_VdbMetadataDelta.py
+++ /dev/null
@@ -1,180 +0,0 @@
-# Copyright 2014-2015 Gentoo Foundation
-# Distributed under the terms of the GNU General Public License v2
-
-import errno
-import json
-import os
-
-from portage.util import atomic_ofstream
-from portage.versions import cpv_getkey
-
-
-class VdbMetadataDelta:
- _format_version = "1"
-
- def __init__(self, vardb):
- self._vardb = vardb
-
- def initialize(self, timestamp):
- with atomic_ofstream(
- self._vardb._cache_delta_filename,
- "w",
- encoding="utf-8",
- errors="strict",
- ) as f:
- json.dump(
- {"version": self._format_version, "timestamp": timestamp},
- f,
- ensure_ascii=False,
- )
-
- def load(self):
- if not os.path.exists(self._vardb._aux_cache_filename):
- # If the primary cache doesn't exist yet, then
- # we can't record a delta against it.
- return None
-
- try:
- with open(
- self._vardb._cache_delta_filename,
- encoding="utf-8",
- errors="strict",
- ) as f:
- cache_obj = json.load(f)
- except OSError as e:
- if e.errno not in (errno.ENOENT, errno.ESTALE):
- raise
- except (SystemExit, KeyboardInterrupt):
- raise
- except Exception:
- # Corrupt, or not json format.
- pass
- else:
- try:
- version = cache_obj["version"]
- except KeyError:
- pass
- else:
- # Verify that the format version is compatible,
- # since a newer version of portage may have
- # written an incompatible file.
- if version == self._format_version:
- try:
- deltas = cache_obj["deltas"]
- except KeyError:
- cache_obj["deltas"] = deltas = []
-
- if isinstance(deltas, list):
- return cache_obj
-
- return None
-
- def loadRace(self):
- """
- This calls self.load() and validates the timestamp
- against the currently loaded self._vardb._aux_cache. If a
- concurrent update causes the timestamps to be inconsistent,
- then it reloads the caches and tries one more time before
- it aborts. In practice, the race is very unlikely, so
- this will usually succeed on the first try.
- """
-
- tries = 2
- while tries:
- tries -= 1
- cache_delta = self.load()
- if cache_delta is not None and cache_delta.get(
- "timestamp"
- ) != self._vardb._aux_cache.get("timestamp", False):
- self._vardb._aux_cache_obj = None
- else:
- return cache_delta
-
- return None
-
- def recordEvent(self, event, cpv, slot, counter):
- self._vardb.lock()
- try:
- deltas_obj = self.load()
-
- if deltas_obj is None:
- # We can't record meaningful deltas without
- # a pre-existing state.
- return
-
- delta_node = {
- "event": event,
- "package": cpv.cp,
- "version": cpv.version,
- "slot": slot,
- "counter": f"{counter}",
- }
-
- deltas_obj["deltas"].append(delta_node)
-
- # Eliminate earlier nodes cancelled out by later nodes
- # that have identical package and slot attributes.
- filtered_list = []
- slot_keys = set()
- version_keys = set()
- for delta_node in reversed(deltas_obj["deltas"]):
- slot_key = (delta_node["package"], delta_node["slot"])
- version_key = (delta_node["package"], delta_node["version"])
- if not (slot_key in slot_keys or version_key in version_keys):
- filtered_list.append(delta_node)
- slot_keys.add(slot_key)
- version_keys.add(version_key)
-
- filtered_list.reverse()
- deltas_obj["deltas"] = filtered_list
-
- f = atomic_ofstream(
- self._vardb._cache_delta_filename,
- mode="w",
- encoding="utf-8",
- )
- json.dump(deltas_obj, f, ensure_ascii=False)
- f.close()
-
- finally:
- self._vardb.unlock()
-
- def applyDelta(self, data):
- packages = self._vardb._aux_cache["packages"]
- deltas = {}
- for delta in data["deltas"]:
- cpv = delta["package"] + "-" + delta["version"]
- deltas[cpv] = delta
- event = delta["event"]
- if event == "add":
- # Use aux_get to populate the cache
- # for this cpv.
- if cpv not in packages:
- try:
- self._vardb.aux_get(cpv, ["DESCRIPTION"])
- except KeyError:
- pass
- elif event == "remove":
- packages.pop(cpv, None)
-
- if deltas:
- # Delete removed or replaced versions from affected slots
- for cached_cpv, (mtime, metadata) in list(packages.items()):
- if cached_cpv in deltas:
- continue
-
- removed = False
- for cpv, delta in deltas.items():
- if (
- cached_cpv.startswith(delta["package"])
- and metadata.get("SLOT") == delta["slot"]
- and cpv_getkey(cached_cpv) == delta["package"]
- ):
- removed = True
- break
-
- if removed:
- del packages[cached_cpv]
- del deltas[cpv]
- if not deltas:
- break
diff --git a/lib/portage/dbapi/meson.build b/lib/portage/dbapi/meson.build
index 6b6a94c47..24406b41a 100644
--- a/lib/portage/dbapi/meson.build
+++ b/lib/portage/dbapi/meson.build
@@ -12,7 +12,6 @@ py.install_sources(
'_ContentsCaseSensitivityManager.py',
'_MergeProcess.py',
'_SyncfsProcess.py',
- '_VdbMetadataDelta.py',
'_expand_new_virt.py',
'_similar_name_search.py',
'__init__.py',
diff --git a/lib/portage/dbapi/vartree.py b/lib/portage/dbapi/vartree.py
index 64fca6ea8..6b808ad27 100644
--- a/lib/portage/dbapi/vartree.py
+++ b/lib/portage/dbapi/vartree.py
@@ -15,7 +15,6 @@ import logging
import multiprocessing
import operator
import os
-import pickle
import platform
import pwd
import re
@@ -62,14 +61,11 @@ from portage.exception import (
from portage.localization import _
from portage.util.futures import asyncio
from portage.util.futures.executor.fork import ForkExecutor
-from portage.util.pickle import NoGlobalsUnpickler
-
from ._ContentsCaseSensitivityManager import ContentsCaseSensitivityManager
# Made global to fix importing on Python version upgrade:
# https://bugs.gentoo.org/970375
from ._SyncfsProcess import SyncfsProcess
-from ._VdbMetadataDelta import VdbMetadataDelta
_METADATA_FILE = "metadata"
# The exact set of fields the consolidated metadata file carries, and the set
@@ -296,13 +292,8 @@ class vardbapi(dbapi):
r"^(\..*|" + MERGING_IDENTIFIER + ".*|" + "|".join(_excluded_dirs) + r")$"
)
- _aux_cache_version = "1"
_owners_cache_version = "1"
- # Number of uncached packages to trigger cache update, since
- # it's wasteful to update it for every vdb change.
- _aux_cache_threshold = 5
-
_aux_cache_keys_re = re.compile(r"^NEEDED\..*$")
_aux_multi_line_re = re.compile(r"^(CONTENTS|NEEDED\..*)$")
_pkg_str_aux_keys = dbapi._pkg_str_aux_keys + ("BUILD_ID", "BUILD_TIME", "_mtime_")
@@ -326,11 +317,6 @@ class vardbapi(dbapi):
# have been added or removed.
self._pkgs_changed = False
- # The _aux_cache_threshold doesn't work as designed
- # if the cache is flushed from a subprocess, so we
- # use this to avoid waste vdb cache updates.
- self._flush_cache_enabled = True
-
# cache for category directory mtimes
self.mtdircache = {}
@@ -373,13 +359,6 @@ class vardbapi(dbapi):
# because callers such as FakeVartree replace it per instance.
self._aux_cache_keys = set(_METADATA_FILE_FIELDS)
self._aux_cache_obj = None
- self._aux_cache_filename = os.path.join(
- self._eroot, CACHE_PATH, "vdb_metadata.pickle"
- )
- self._cache_delta_filename = os.path.join(
- self._eroot, CACHE_PATH, "vdb_metadata_delta.json"
- )
- self._cache_delta = VdbMetadataDelta(self)
self._counter_path = os.path.join(self._eroot, CACHE_PATH, "counter")
self._plib_registry = PreservedLibsRegistry(
@@ -830,45 +809,7 @@ class vardbapi(dbapi):
return self.getpath(str(mycpv), filename=catsplit(mycpv)[1] + ".ebuild")
def flush_cache(self):
- """If the current user has permission and the internal aux_get cache has
- been updated, save it to disk and mark it unmodified. This is called
- by emerge after it has loaded the full vdb for use in dependency
- calculations. Currently, the cache is only written if the user has
- superuser privileges (since that's required to obtain a lock), but all
- users have read access and benefit from faster metadata lookups (as
- long as at least part of the cache is still valid)."""
- from portage.data import secpass
- from portage.util import apply_secpass_permissions, atomic_ofstream, ensure_dirs
-
- if (
- self._flush_cache_enabled
- and self._aux_cache is not None
- and secpass >= 2
- and (
- len(self._aux_cache["modified"]) >= self._aux_cache_threshold
- or not os.path.exists(self._cache_delta_filename)
- )
- ):
- ensure_dirs(os.path.dirname(self._aux_cache_filename))
-
- self._owners.populate() # index any unindexed contents
- valid_nodes = set(self.cpv_all())
- for cpv in list(self._aux_cache["packages"]):
- if cpv not in valid_nodes:
- del self._aux_cache["packages"][cpv]
- del self._aux_cache["modified"]
- timestamp = time.time()
- self._aux_cache["timestamp"] = timestamp
-
- with atomic_ofstream(self._aux_cache_filename, "wb") as f:
- pickle.dump(self._aux_cache, f, protocol=2)
-
- apply_secpass_permissions(self._aux_cache_filename, mode=0o644)
-
- self._cache_delta.initialize(timestamp)
- apply_secpass_permissions(self._cache_delta_filename, mode=0o644)
-
- self._aux_cache["modified"] = set()
+ pass
@property
def _aux_cache(self):
@@ -877,58 +818,11 @@ class vardbapi(dbapi):
return self._aux_cache_obj
def _aux_cache_init(self):
- from portage.util import writemsg
-
- aux_cache = None
- open_kwargs = {}
- try:
- with open(
- self._aux_cache_filename,
- mode="rb",
- **open_kwargs,
- ) as f:
- aux_cache = NoGlobalsUnpickler(f).load()
- except (SystemExit, KeyboardInterrupt):
- raise
- except Exception as e:
- if isinstance(e, EnvironmentError) and getattr(e, "errno", None) in (
- errno.ENOENT,
- errno.EACCES,
- ):
- pass
- else:
- writemsg(
- _("!!! Error loading '%s': %s\n") % (self._aux_cache_filename, e),
- noiselevel=-1,
- )
- del e
-
- if (
- not aux_cache
- or not isinstance(aux_cache, dict)
- or aux_cache.get("version") != self._aux_cache_version
- or not aux_cache.get("packages")
- ):
- aux_cache = {"version": self._aux_cache_version}
- aux_cache["packages"] = {}
-
- owners = aux_cache.get("owners")
- if owners is not None:
- if (
- not isinstance(owners, dict)
- or "version" not in owners
- or owners["version"] != self._owners_cache_version
- or "base_names" not in owners
- or not isinstance(owners["base_names"], dict)
- ):
- owners = None
-
- if owners is None:
- owners = {"base_names": {}, "version": self._owners_cache_version}
- aux_cache["owners"] = owners
-
- aux_cache["modified"] = set()
- self._aux_cache_obj = aux_cache
+ self._aux_cache_obj = {
+ "packages": {},
+ "owners": {"base_names": {}, "version": self._owners_cache_version},
+ "modified": set(),
+ }
def aux_get(self, mycpv, wants, myrepo=None):
"""This automatically caches selected keys that are frequently needed
@@ -1056,7 +950,12 @@ class vardbapi(dbapi):
env_keys = []
for x in wants:
if x == "_mtime_":
- results[x] = st[stat.ST_MTIME]
+ # Float, matching the value aux_get() seeds itself with. The
+ # pickle cache used to supply this on a warm cache, so reading
+ # it from disk had only ever produced the truncated int on a
+ # cache miss; with the pickle gone that would have become the
+ # value callers always see.
+ results[x] = st.st_mtime
continue
# _read_metadata_file only returns a dict for a file whose format
@@ -1444,12 +1343,6 @@ class vardbapi(dbapi):
self.settings._init_dirs()
write_atomic(self._counter_path, str(counter))
self._cached_counter = counter
-
- # Since we hold a lock, this is a good opportunity
- # to flush the cache. Note that this will only
- # flush the cache periodically in the main process
- # when _aux_cache_threshold is exceeded.
- self.flush_cache()
finally:
self.unlock()
@@ -2224,12 +2117,6 @@ class dblink:
)
return
- if self.dbdir is self.dbpkgdir:
- (counter,) = self.vartree.dbapi.aux_get(self.mycpv, ["COUNTER"])
- self.vartree.dbapi._cache_delta.recordEvent(
- "remove", self.mycpv, self.settings["SLOT"].split("/")[0], counter
- )
-
shutil.rmtree(self.dbdir)
# If empty, remove parent category directory.
try:
@@ -5303,9 +5190,6 @@ class dblink:
self.delete()
_movefile(self.dbtmpdir, self.dbpkgdir, mysettings=self.settings)
self._merged_path(self.dbpkgdir, os.lstat(self.dbpkgdir))
- self.vartree.dbapi._cache_delta.recordEvent(
- "add", self.mycpv, slot, counter
- )
finally:
self.unlockdb()