proj/portage:master commit in: lib/portage/dbapi/, lib/portage/tests/dbapi/, lib/portage/tests/resolver/
"Matt Turner" <[email protected]>
| Newsgroups | gmane.linux.gentoo.cvs |
|---|---|
| Message-ID | <1786672955.c1472dbae4bd6cd980ed2f6c29eb4824d1a6ffb7.mattst88@gentoo> |
commit: c1472dbae4bd6cd980ed2f6c29eb4824d1a6ffb7
Author: Matt Turner <mattst88 <AT> gentoo <DOT> org>
AuthorDate: Thu Jun 18 18:48:26 2026 +0000
Commit: Matt Turner <mattst88 <AT> gentoo <DOT> org>
CommitDate: Fri Aug 14 02:02:35 2026 +0000
URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=c1472dba
vartree: add consolidated metadata file as read optimization
The VDB stores one file per metadata field per installed package (~35
files). Reading all fields for a package takes ~30 open/read/close
syscalls, multiplied across every installed package during operations like
`emerge -p @world`.
Add a supplemental `metadata` file (KEY=value per line) alongside the
existing individual files. _aux_get() prefers it for the fields it
contains, cutting most per-field reads to a single file read. The
individual files are kept, so quickpkg, xpak, bintree and external readers
of the VDB are unaffected.
_METADATA_FILE_FIELDS names the fields the file carries, and vardbapi
shares it as _aux_cache_keys. The two have to stay identical: _aux_get()
may serve a field the file omits as "" only where a missing individual
file means empty, and outside _aux_cache_keys it means "look in
environment.bz2" instead (bug 395463). SRC_URI is the case to keep in
mind: no VDB file has ever held it, so a wider rule would have the file
answer "" for a field whose value is in the environment.
Values are single-line, which also excludes CONTENTS and the NEEDED.*
files; those stay in their own files and are read from there.
A field missing from the metadata file still falls back to its individual
file. Nothing yet guarantees the file lists every field that existed when
it was written, so absence cannot be read as "empty". The next commit adds
a format version, which makes it a complete snapshot and removes the
fallback.
New installs get the file at merge time.
Bug: https://bugs.gentoo.org/321317
Signed-off-by: Matt Turner <mattst88 <AT> gentoo.org>
lib/portage/dbapi/vartree.py | 178 ++++++++++++++++---
lib/portage/tests/dbapi/test_vdb_metadata.py | 211 +++++++++++++++++++++++
lib/portage/tests/resolver/ResolverPlayground.py | 12 ++
3 files changed, 376 insertions(+), 25 deletions(-)
diff --git a/lib/portage/dbapi/vartree.py b/lib/portage/dbapi/vartree.py
index ffb449942..d31923b94 100644
--- a/lib/portage/dbapi/vartree.py
+++ b/lib/portage/dbapi/vartree.py
@@ -71,6 +71,110 @@ from ._ContentsCaseSensitivityManager import ContentsCaseSensitivityManager
from ._SyncfsProcess import SyncfsProcess
from ._VdbMetadataDelta import VdbMetadataDelta
+_METADATA_FILE = "metadata"
+# The exact set of fields the consolidated metadata file carries, and the set
+# vardbapi caches. Membership is bounded by what _aux_get() may serve as "" on
+# a missing individual file: a field outside this set and outside
+# _aux_cache_keys_re falls back to an environment.bz2 search instead (see
+# bug 395463), which the file must not silently replace with "". Keeping the
+# two sets identical is what makes that bound hold by construction.
+#
+# Line-oriented fields (CONTENTS, NEEDED, NEEDED.ELF.2) are absent, which the
+# one-line-per-field format requires anyway.
+_METADATA_FILE_FIELDS = frozenset(
+ (
+ "BDEPEND",
+ "BUILD_ID",
+ "BUILD_TIME",
+ "CHOST",
+ "COUNTER",
+ "DEFINED_PHASES",
+ "DEPEND",
+ "DESCRIPTION",
+ "EAPI",
+ "HOMEPAGE",
+ "IDEPEND",
+ "IUSE",
+ "KEYWORDS",
+ "LICENSE",
+ "PDEPEND",
+ "PROPERTIES",
+ "PROVIDES",
+ "RDEPEND",
+ "REQUIRES",
+ "RESTRICT",
+ "SLOT",
+ "USE",
+ "repository",
+ )
+)
+
+
+def _in_metadata_file(fname):
+ """True if fname is a field the consolidated metadata file carries."""
+ return fname in _METADATA_FILE_FIELDS
+
+
+def _read_metadata_file(path):
+ """Parse KEY=value\\n metadata file. Returns dict[str, str]."""
+ from portage import _encodings
+
+ result = {}
+ with open(path, encoding=_encodings["repo.content"], errors="replace") as f:
+ for line in f:
+ line = line.rstrip("\n")
+ if "=" in line:
+ k, v = line.split("=", 1)
+ result[k] = v
+ return result
+
+
+def _write_metadata_file(dbdir, data):
+ """Atomically write metadata dict to dbdir/metadata.
+
+ The one-line-per-field format cannot represent an embedded newline, so
+ values are whitespace-normalized here (the same normalization _aux_get
+ applies to single-line fields). Doing it here stops a caller that passes a
+ raw multi-line value from silently truncating the file.
+ """
+ from portage import _encodings
+ from portage.util import write_atomic
+
+ content = "".join(f"{k}={' '.join(v.split())}\n" for k, v in sorted(data.items()))
+ write_atomic(
+ os.path.join(dbdir, _METADATA_FILE),
+ content,
+ mode="w",
+ encoding=_encodings["repo.content"],
+ )
+
+
+def _consolidate_to_metadata_file(dbdir):
+ """Build the metadata file from individual per-field VDB files.
+
+ Reads every file in dbdir that _in_metadata_file() accepts and writes them
+ to the metadata file. Individual files are kept for backward compatibility
+ with tools that read the VDB directly.
+ """
+ from portage import _encodings
+
+ data = {}
+ for fname in os.listdir(dbdir):
+ if not _in_metadata_file(fname):
+ continue
+ fpath = os.path.join(dbdir, fname)
+ try:
+ with open(
+ fpath, encoding=_encodings["repo.content"], errors="replace"
+ ) as f:
+ # Normalize whitespace to match what _aux_get previously did
+ # for single-line fields via " ".join(myd.split()).
+ data[fname] = " ".join(f.read().split())
+ except OSError:
+ pass
+ if data:
+ _write_metadata_file(dbdir, data)
+
class vardbapi(dbapi):
_excluded_dirs = ["CVS", "lost+found"]
@@ -151,31 +255,10 @@ class vardbapi(dbapi):
if vartree is None:
vartree = portage.db[settings["EROOT"]]["vartree"]
self.vartree = vartree
- self._aux_cache_keys = {
- "BDEPEND",
- "BUILD_TIME",
- "CHOST",
- "COUNTER",
- "DEPEND",
- "DESCRIPTION",
- "EAPI",
- "HOMEPAGE",
- "BUILD_ID",
- "IDEPEND",
- "IUSE",
- "KEYWORDS",
- "LICENSE",
- "PDEPEND",
- "PROPERTIES",
- "RDEPEND",
- "repository",
- "RESTRICT",
- "SLOT",
- "USE",
- "DEFINED_PHASES",
- "PROVIDES",
- "REQUIRES",
- }
+ # Same set as the consolidated metadata file carries; see
+ # _METADATA_FILE_FIELDS for why the two must not drift apart. Copied
+ # because callers such as FakeVartree replace it per instance.
+ self._aux_cache_keys = set(_METADATA_FILE_FIELDS)
self._aux_cache_obj = None
self._aux_cache_filename = os.path.join(
self._eroot, CACHE_PATH, "vdb_metadata.pickle"
@@ -847,12 +930,35 @@ class vardbapi(dbapi):
raise
if not stat.S_ISDIR(st.st_mode):
raise KeyError(mycpv)
+
+ metadata_data = None
+ try:
+ metadata_data = _read_metadata_file(os.path.join(mydir, _METADATA_FILE))
+ except OSError:
+ pass
+
results = {}
env_keys = []
for x in wants:
if x == "_mtime_":
results[x] = st[stat.ST_MTIME]
continue
+
+ # Only fields actually present in the metadata file may be served
+ # from it. A field missing there is not known to be empty. The file
+ # is written at merge time and is not updated by later writes to the
+ # individual files, and one written by an older portage may predate
+ # the field entirely. In both cases the individual file holds the
+ # real value, so fall through to the per-field read and let those
+ # keep resolving exactly as they do without a metadata file.
+ if (
+ metadata_data is not None
+ and x in metadata_data
+ and _in_metadata_file(x)
+ ):
+ results[x] = metadata_data[x]
+ continue
+
try:
with open(
os.path.join(mydir, x),
@@ -977,6 +1083,15 @@ class vardbapi(dbapi):
os.unlink(os.path.join(self.getpath(cpv), k))
except OSError:
pass
+ # Remove from metadata file if present.
+ metadata_path = os.path.join(self.getpath(cpv), _METADATA_FILE)
+ try:
+ existing = _read_metadata_file(metadata_path)
+ if k in existing:
+ del existing[k]
+ _write_metadata_file(self.getpath(cpv), existing)
+ except OSError:
+ pass
self._bump_mtime(cpv)
@staticmethod
@@ -4923,6 +5038,9 @@ class dblink:
) as f:
f.write(f"{counter}")
+ # Consolidate all per-field metadata files into a single metadata file.
+ _consolidate_to_metadata_file(self.dbtmpdir)
+
self.updateprotect()
# if we have a file containing previously-merged config file md5sums, grab it.
@@ -6083,6 +6201,16 @@ class dblink:
kwargs["encoding"] = "utf-8"
write_atomic(os.path.join(self.dbdir, fname), data, **kwargs)
+ # Keep the metadata file in sync if it exists.
+ if isinstance(data, str) and _in_metadata_file(fname):
+ metadata_path = os.path.join(self.dbdir, _METADATA_FILE)
+ try:
+ existing = _read_metadata_file(metadata_path)
+ except OSError:
+ return
+ existing[fname] = " ".join(data.split())
+ _write_metadata_file(self.dbdir, existing)
+
def getelements(self, ename):
if not os.path.exists(self.dbdir + "/" + ename):
return []
diff --git a/lib/portage/tests/dbapi/test_vdb_metadata.py b/lib/portage/tests/dbapi/test_vdb_metadata.py
new file mode 100644
index 000000000..3a477981c
--- /dev/null
+++ b/lib/portage/tests/dbapi/test_vdb_metadata.py
@@ -0,0 +1,211 @@
+# Copyright 2026 Gentoo Authors
+# Distributed under the terms of the GNU General Public License v2
+
+import os
+import tempfile
+
+from portage.tests import TestCase
+from portage.dbapi.vartree import (
+ _METADATA_FILE,
+ _METADATA_FILE_FIELDS,
+ _consolidate_to_metadata_file,
+ _in_metadata_file,
+ _read_metadata_file,
+ _write_metadata_file,
+ vardbapi,
+)
+from portage.tests.resolver.ResolverPlayground import ResolverPlayground
+
+
+class VdbInMetadataFileTestCase(TestCase):
+ def test_accepts_cached_fields(self):
+ for name in ("EAPI", "SLOT", "USE", "DEPEND", "repository"):
+ self.assertTrue(_in_metadata_file(name), name)
+
+ def test_rejects_multi_line_fields(self):
+ # One line per field cannot represent these, so they stay in their own
+ # file and are read from there.
+ for name in ("CONTENTS", "NEEDED", "NEEDED.ELF.2"):
+ self.assertFalse(_in_metadata_file(name), name)
+
+ def test_rejects_non_fields(self):
+ for name in ("environment.bz2", "foo-1.ebuild", "counter"):
+ self.assertFalse(_in_metadata_file(name), name)
+
+ def test_rejects_uncached_vdb_fields(self):
+ # These have individual VDB files and look like fields, but vardbapi
+ # does not cache them. Serving them from the file would claim a
+ # completeness it cannot have: a field outside _aux_cache_keys falls
+ # back to environment.bz2 when its individual file is missing
+ # (bug 395463), and "" is not that.
+ for name in ("FEATURES", "IUSE_EFFECTIVE", "CFLAGS", "SRC_URI", "INHERITED"):
+ self.assertFalse(_in_metadata_file(name), name)
+
+ def test_no_multi_line_fields(self):
+ # One line per field, so no field the file carries may be one _aux_get
+ # preserves newlines for.
+ for name in _METADATA_FILE_FIELDS:
+ self.assertIsNone(vardbapi._aux_multi_line_re.match(name), name)
+
+
+class VdbMetadataReadWriteTestCase(TestCase):
+ def setUp(self):
+ self._tmpdir = tempfile.mkdtemp()
+
+ def tearDown(self):
+ import shutil
+
+ shutil.rmtree(self._tmpdir, ignore_errors=True)
+
+ def test_write_then_read_roundtrip(self):
+ data = {"EAPI": "8", "SLOT": "0/0", "USE": "foo bar", "repository": "gentoo"}
+ _write_metadata_file(self._tmpdir, data)
+ path = os.path.join(self._tmpdir, _METADATA_FILE)
+ self.assertTrue(os.path.exists(path))
+ result = _read_metadata_file(path)
+ self.assertEqual(result, data)
+
+ def test_keys_sorted_in_file(self):
+ data = {"SLOT": "0", "EAPI": "8", "USE": "foo"}
+ _write_metadata_file(self._tmpdir, data)
+ path = os.path.join(self._tmpdir, _METADATA_FILE)
+ with open(path) as f:
+ lines = [l.rstrip("\n") for l in f if not l.startswith("#")]
+ keys = [l.split("=", 1)[0] for l in lines if "=" in l]
+ self.assertEqual(keys, sorted(keys))
+
+ def test_value_with_equals_sign(self):
+ data = {"HOMEPAGE": "https://example.com/?foo=bar"}
+ _write_metadata_file(self._tmpdir, data)
+ path = os.path.join(self._tmpdir, _METADATA_FILE)
+ result = _read_metadata_file(path)
+ self.assertEqual(result["HOMEPAGE"], "https://example.com/?foo=bar")
+
+ def test_empty_value(self):
+ data = {"IUSE": "", "EAPI": "8"}
+ _write_metadata_file(self._tmpdir, data)
+ path = os.path.join(self._tmpdir, _METADATA_FILE)
+ result = _read_metadata_file(path)
+ self.assertEqual(result["IUSE"], "")
+ self.assertEqual(result["EAPI"], "8")
+
+
+class VdbConsolidateTestCase(TestCase):
+ def setUp(self):
+ self._tmpdir = tempfile.mkdtemp()
+
+ def tearDown(self):
+ import shutil
+
+ shutil.rmtree(self._tmpdir, ignore_errors=True)
+
+ def _write_field(self, name, value):
+ with open(os.path.join(self._tmpdir, name), "w") as f:
+ f.write(value + "\n")
+
+ def test_basic_consolidation(self):
+ self._write_field("EAPI", "8")
+ self._write_field("SLOT", "0/0")
+ self._write_field("USE", "foo bar")
+ _consolidate_to_metadata_file(self._tmpdir)
+ path = os.path.join(self._tmpdir, _METADATA_FILE)
+ result = _read_metadata_file(path)
+ self.assertEqual(result["EAPI"], "8")
+ self.assertEqual(result["SLOT"], "0/0")
+ self.assertEqual(result["USE"], "foo bar")
+
+ def test_contents_excluded(self):
+ self._write_field("EAPI", "8")
+ with open(os.path.join(self._tmpdir, "CONTENTS"), "w") as f:
+ f.write("obj /usr/bin/foo abc123 1234567890\n")
+ _consolidate_to_metadata_file(self._tmpdir)
+ path = os.path.join(self._tmpdir, _METADATA_FILE)
+ result = _read_metadata_file(path)
+ self.assertNotIn("CONTENTS", result)
+
+ def test_dotted_files_excluded(self):
+ self._write_field("EAPI", "8")
+ with open(os.path.join(self._tmpdir, "NEEDED.ELF.2"), "w") as f:
+ f.write("/usr/lib/libfoo.so\n")
+ _consolidate_to_metadata_file(self._tmpdir)
+ path = os.path.join(self._tmpdir, _METADATA_FILE)
+ result = _read_metadata_file(path)
+ self.assertNotIn("NEEDED.ELF.2", result)
+
+ def test_uncached_field_excluded(self):
+ # An all-caps VDB field vardbapi does not cache. It stays in its own
+ # file, and consolidation must not claim it.
+ self._write_field("EAPI", "8")
+ self._write_field("FEATURES", "buildpkg parallel-install")
+ _consolidate_to_metadata_file(self._tmpdir)
+ path = os.path.join(self._tmpdir, _METADATA_FILE)
+ result = _read_metadata_file(path)
+ self.assertNotIn("FEATURES", result)
+ self.assertTrue(os.path.exists(os.path.join(self._tmpdir, "FEATURES")))
+
+ def test_bare_needed_excluded(self):
+ # NEEDED has no dot, so a name-based rule would accept it; it is
+ # multi-line like CONTENTS and is not a cached field either.
+ self._write_field("EAPI", "8")
+ with open(os.path.join(self._tmpdir, "NEEDED"), "w") as f:
+ f.write("/usr/bin/foo libc.so.6\n/usr/bin/bar libm.so.6\n")
+ _consolidate_to_metadata_file(self._tmpdir)
+ path = os.path.join(self._tmpdir, _METADATA_FILE)
+ result = _read_metadata_file(path)
+ self.assertNotIn("NEEDED", result)
+ self.assertTrue(os.path.exists(os.path.join(self._tmpdir, "NEEDED")))
+
+ def test_whitespace_normalized(self):
+ self._write_field("USE", " foo bar baz ")
+ _consolidate_to_metadata_file(self._tmpdir)
+ path = os.path.join(self._tmpdir, _METADATA_FILE)
+ result = _read_metadata_file(path)
+ self.assertEqual(result["USE"], "foo bar baz")
+
+ def test_individual_files_kept_by_default(self):
+ self._write_field("EAPI", "8")
+ _consolidate_to_metadata_file(self._tmpdir)
+ self.assertTrue(os.path.exists(os.path.join(self._tmpdir, "EAPI")))
+
+ def test_empty_dir_no_metadata_file(self):
+ _consolidate_to_metadata_file(self._tmpdir)
+ self.assertFalse(os.path.exists(os.path.join(self._tmpdir, _METADATA_FILE)))
+
+
+class VdbMetadataAuxGetTestCase(TestCase):
+ def testUncachedFieldStillComesFromEnvironment(self):
+ """A field the metadata file does not carry still reaches
+ environment.bz2 (bug 395463). Serving it as "" because the file is a
+ complete snapshot would only be right for fields the file carries."""
+ ebuilds = {
+ "dev-libs/A-1": {
+ "EAPI": "7",
+ "SRC_URI": "https://example.com/A-1.tar.gz",
+ },
+ }
+ installed = {
+ "dev-libs/A-1": {
+ "EAPI": "7",
+ "SRC_URI": "https://example.com/A-1.tar.gz",
+ },
+ }
+ playground = ResolverPlayground(ebuilds=ebuilds, installed=installed)
+ try:
+ vardb = playground.trees[playground.eroot]["vartree"].dbapi
+ pkgdir = vardb.getpath("dev-libs/A-1")
+ # A real merge writes no SRC_URI file; the playground writes one
+ # for every key it is given, so drop it and rebuild the metadata
+ # file, whose recorded dir mtime the unlink would otherwise stale.
+ os.unlink(os.path.join(pkgdir, "SRC_URI"))
+ _consolidate_to_metadata_file(pkgdir)
+ # The optimization under test has to actually be in play, or the
+ # per-field fallback would serve SRC_URI and hide the bug.
+ self.assertIsNotNone(
+ _read_metadata_file(os.path.join(pkgdir, _METADATA_FILE))
+ )
+ self.assertEqual(
+ vardb.aux_get("dev-libs/A-1", ["SRC_URI"])[0],
+ "https://example.com/A-1.tar.gz",
+ )
+ finally:
+ playground.cleanup()
diff --git a/lib/portage/tests/resolver/ResolverPlayground.py b/lib/portage/tests/resolver/ResolverPlayground.py
index 7ef52761e..4c1d628cd 100644
--- a/lib/portage/tests/resolver/ResolverPlayground.py
+++ b/lib/portage/tests/resolver/ResolverPlayground.py
@@ -35,6 +35,7 @@ from portage.const import (
USER_CONFIG_PATH,
)
from portage.dbapi.bintree import binarytree
+from portage.dbapi.vartree import _in_metadata_file, _write_metadata_file
from portage.dep import Atom, _repo_separator
from portage.exception import InvalidBinaryPackageFormat
from portage.gpg import GPG
@@ -513,9 +514,20 @@ class ResolverPlayground:
)
metadata["repository"] = repo
+ metadata_kv = {}
for k, v in metadata.items():
+ # Write the individual file for every field, the way a real
+ # merge does. The metadata file is a read optimization layered
+ # on top of them, not a replacement: both portage consumers
+ # (e.g. quickpkg, xpak binpkgs) and non-portage consumers
+ # (e.g. portage-utils, pkgcore) still read the per-field files
+ # directly.
with open(os.path.join(vdb_pkg_dir, k), "w") as f:
f.write(f"{v}\n")
+ if _in_metadata_file(k):
+ metadata_kv[k] = str(v)
+ if metadata_kv:
+ _write_metadata_file(vdb_pkg_dir, metadata_kv)
ebuild_path = os.path.join(vdb_pkg_dir, a.cpv.split("/")[1] + ".ebuild")
with open(ebuild_path, "w") as f: