proj/portage:master commit in: lib/portage/dbapi/, lib/portage/emaint/modules/, lib/portage/emaint/modules/vdb/, ...
"Matt Turner" <[email protected]>
| Newsgroups | gmane.linux.gentoo.cvs |
|---|---|
| Message-ID | <1786672955.eec739fcb158fee4173da2d8d91d1eb90b708847.mattst88@gentoo> |
commit: eec739fcb158fee4173da2d8d91d1eb90b708847
Author: Matt Turner <mattst88 <AT> gentoo <DOT> org>
AuthorDate: Thu Jun 18 16:28:16 2026 +0000
Commit: Matt Turner <mattst88 <AT> gentoo <DOT> org>
CommitDate: Fri Aug 14 02:02:35 2026 +0000
URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=eec739fc
emaint: add vdb module for consolidated metadata file maintenance
'emaint vdb' reports how many installed packages have the consolidated
metadata file and how many are missing, stale, or written in a format this
portage version does not read.
'emaint vdb --fix' populates the file for those packages.
'emaint vdb --fix --delete-individual-files' also removes the per-field
files afterwards, reducing VDB disk usage. This breaks tools that read
individual VDB files directly (portage-utils, pkgcore, shell scripts), so
it is opt-in and documented as such.
'emaint vdb --remove' removes the consolidated files again.
_consolidate_to_metadata_file() gains delete_individual=False. The
deletions change the package directory, so with that flag the metadata
file is stamped after them rather than as part of writing it; stamping
first would record an mtime the unlinks immediately invalidate, leaving
the package with neither its individual files nor a usable metadata file.
The body is written before the unlinks so no field is ever absent from
disk.
--fix skips a package that already has a current metadata file only on the
plain populate path. With --delete-individual-files there are still
per-field files to remove on such a package, and skipping it would make
the flag a no-op on an already-populated VDB, which is the VDB it is meant
for.
Signed-off-by: Matt Turner <mattst88 <AT> gentoo.org>
lib/portage/dbapi/vartree.py | 70 ++++++++++++++----
lib/portage/emaint/modules/meson.build | 1 +
lib/portage/emaint/modules/vdb/__init__.py | 40 +++++++++++
lib/portage/emaint/modules/vdb/meson.build | 8 +++
lib/portage/emaint/modules/vdb/vdb.py | 102 +++++++++++++++++++++++++++
lib/portage/tests/dbapi/test_vdb_metadata.py | 66 ++++++++++++++++-
6 files changed, 273 insertions(+), 14 deletions(-)
diff --git a/lib/portage/dbapi/vartree.py b/lib/portage/dbapi/vartree.py
index 3b5338a41..64fca6ea8 100644
--- a/lib/portage/dbapi/vartree.py
+++ b/lib/portage/dbapi/vartree.py
@@ -191,7 +191,27 @@ def _read_metadata_file(path, dir_st=None):
return result
-def _write_metadata_file(dbdir, data):
+def _stamp_metadata_file(dbdir):
+ """Append the "#dir_mtime=" line the reader validates against.
+
+ Kept separate from writing the body because it has to happen after the
+ last change to dbdir's contents: write_atomic() renames into place, and
+ that rename bumps dbdir's mtime, so a value recorded before it would never
+ match. Appending does not create or remove a directory entry, so it leaves
+ dbdir's mtime alone and the recorded value stays true.
+
+ A caller that changes dbdir further between the body write and this call
+ must call it afterwards, or it will stamp an mtime its own later change
+ invalidates.
+ """
+ from portage import _encodings
+
+ path = os.path.join(dbdir, _METADATA_FILE)
+ with open(path, mode="a", encoding=_encodings["repo.content"]) as f:
+ f.write(f"{_METADATA_DIR_MTIME_PREFIX}{os.stat(dbdir).st_mtime_ns}\n")
+
+
+def _write_metadata_file(dbdir, data, stamp=True):
"""Atomically write metadata dict to dbdir/metadata.
The one-line-per-field format cannot represent an embedded newline, so
@@ -199,11 +219,10 @@ def _write_metadata_file(dbdir, data):
applies to single-line fields). Doing it here stops a caller that passes a
raw multi-line value from silently truncating the file.
- The "#dir_mtime=" line the reader validates against is appended after the
- atomic write rather than included in it: write_atomic() renames into
- place, and that rename bumps dbdir's mtime, so a value recorded before it
- would never match. Appending does not create or remove a directory entry,
- so it leaves dbdir's mtime alone and the recorded value stays true.
+ Pass stamp=False when the caller still has to change dbdir before the file
+ can be stamped; it must then call _stamp_metadata_file() itself. Until it
+ does, the file lacks "#dir_mtime=" and the reader rejects it, so an
+ interrupted sequence falls back rather than serving a stale snapshot.
"""
from portage import _encodings
from portage.util import write_atomic
@@ -212,19 +231,39 @@ def _write_metadata_file(dbdir, data):
content = f"{_METADATA_FORMAT_PREFIX}{_METADATA_FILE_FORMAT_VERSION}\n"
content += "".join(f"{k}={' '.join(v.split())}\n" for k, v in sorted(data.items()))
write_atomic(path, content, mode="w", encoding=_encodings["repo.content"])
- with open(path, mode="a", encoding=_encodings["repo.content"]) as f:
- f.write(f"{_METADATA_DIR_MTIME_PREFIX}{os.stat(dbdir).st_mtime_ns}\n")
+ if stamp:
+ _stamp_metadata_file(dbdir)
-def _consolidate_to_metadata_file(dbdir):
+def _consolidate_to_metadata_file(dbdir, delete_individual=False):
"""Build the metadata file from individual per-field VDB files.
Reads every file in dbdir that _in_metadata_file() accepts and writes them
- to the metadata file. Individual files are kept for backward compatibility
- with tools that read the VDB directly.
+ to the metadata file. By default individual files are kept for backward
+ compatibility with tools that read the VDB directly. Pass
+ delete_individual=True to remove them after writing.
+
+ The deletions change dbdir, so the metadata file is stamped after them
+ rather than as part of writing it; stamping first would record an mtime
+ the unlinks immediately invalidate, leaving the package with neither its
+ individual files nor a usable metadata file. The body is written before
+ the unlinks so no field is ever absent from disk.
+
+ A metadata file that still validates is an accurate snapshot of dbdir, so
+ rewriting it would produce the same content and there is nothing to do.
+ That shortcut does not apply to delete_individual, which has work left
+ whenever the per-field files are still present.
"""
from portage import _encodings
+ if not delete_individual:
+ try:
+ current = _read_metadata_file(os.path.join(dbdir, _METADATA_FILE))
+ except OSError:
+ current = None
+ if current is not None:
+ return
+
data = {}
for fname in os.listdir(dbdir):
if not _in_metadata_file(fname):
@@ -240,7 +279,14 @@ def _consolidate_to_metadata_file(dbdir):
except OSError:
pass
if data:
- _write_metadata_file(dbdir, data)
+ _write_metadata_file(dbdir, data, stamp=not delete_individual)
+ if delete_individual:
+ for fname in data:
+ try:
+ os.unlink(os.path.join(dbdir, fname))
+ except OSError:
+ pass
+ _stamp_metadata_file(dbdir)
class vardbapi(dbapi):
diff --git a/lib/portage/emaint/modules/meson.build b/lib/portage/emaint/modules/meson.build
index 33b396be9..7344c489e 100644
--- a/lib/portage/emaint/modules/meson.build
+++ b/lib/portage/emaint/modules/meson.build
@@ -14,4 +14,5 @@ subdir('move')
subdir('resume')
subdir('revisions')
subdir('sync')
+subdir('vdb')
subdir('world')
diff --git a/lib/portage/emaint/modules/vdb/__init__.py b/lib/portage/emaint/modules/vdb/__init__.py
new file mode 100644
index 000000000..0fd7e65df
--- /dev/null
+++ b/lib/portage/emaint/modules/vdb/__init__.py
@@ -0,0 +1,40 @@
+# Copyright 2026 Gentoo Authors
+# Distributed under the terms of the GNU General Public License v2
+
+doc = """Manage the VDB consolidated metadata file."""
+__doc__ = doc
+
+
+module_spec = {
+ "name": "vdb",
+ "description": doc,
+ "provides": {
+ "module1": {
+ "name": "vdb",
+ "sourcefile": "vdb",
+ "class": "VdbMetadata",
+ "description": doc,
+ "functions": ["check", "fix", "remove"],
+ "func_desc": {
+ "delete_individual_files": {
+ "long": "--delete-individual-files",
+ "help": "(fix only): also remove per-field files after writing "
+ "the metadata file. WARNING: breaks tools that read "
+ "individual VDB files directly (portage-utils, pkgcore, "
+ "shell scripts). Only use when all VDB consumers support the "
+ "consolidated format.",
+ "action": "store_true",
+ "func": "fix",
+ },
+ "remove": {
+ "short": "-R",
+ "long": "--remove",
+ "help": "Remove consolidated metadata files",
+ "status": "Removing VDB metadata files for %s",
+ "action": "store_true",
+ "func": "remove",
+ },
+ },
+ }
+ },
+}
diff --git a/lib/portage/emaint/modules/vdb/meson.build b/lib/portage/emaint/modules/vdb/meson.build
new file mode 100644
index 000000000..20b9556a7
--- /dev/null
+++ b/lib/portage/emaint/modules/vdb/meson.build
@@ -0,0 +1,8 @@
+py.install_sources(
+ [
+ 'vdb.py',
+ '__init__.py',
+ ],
+ subdir : 'portage/emaint/modules/vdb',
+ pure : not native_extensions
+)
diff --git a/lib/portage/emaint/modules/vdb/vdb.py b/lib/portage/emaint/modules/vdb/vdb.py
new file mode 100644
index 000000000..ffe32d83a
--- /dev/null
+++ b/lib/portage/emaint/modules/vdb/vdb.py
@@ -0,0 +1,102 @@
+# Copyright 2026 Gentoo Authors
+# Distributed under the terms of the GNU General Public License v2
+
+import portage
+from portage import os
+from portage.dbapi.vartree import (
+ _METADATA_FILE,
+ _consolidate_to_metadata_file,
+ _read_metadata_file,
+)
+
+
+def _has_usable_metadata(pkgdir):
+ """True if pkgdir has a metadata file this portage version can read.
+
+ A file is unusable if its format version is not ours or if the package
+ directory has changed since it was written; either way it counts as
+ missing and gets rewritten by --fix.
+ """
+ try:
+ return _read_metadata_file(os.path.join(pkgdir, _METADATA_FILE)) is not None
+ except OSError:
+ return False
+
+
+def _iter_pkg_dirs(settings):
+ """Yield (cpv, package directory) for every installed package."""
+ vardb = portage.db[settings.get("EROOT", "/")]["vartree"].dbapi
+ for cpv in sorted(vardb.cpv_all()):
+ yield cpv, vardb.getpath(cpv)
+
+
+class VdbMetadata:
+ short_desc = "Manage consolidated VDB metadata files"
+
+ @staticmethod
+ def name():
+ return "vdb"
+
+ def can_progressbar(self, func):
+ return False
+
+ def check(self, **kwargs):
+ """Report how many packages have/lack the consolidated metadata file."""
+ settings = kwargs.get("settings", getattr(portage, "settings", {}))
+
+ with_meta = 0
+ without_meta = 0
+ for _cpv, pkgdir in _iter_pkg_dirs(settings):
+ if _has_usable_metadata(pkgdir):
+ with_meta += 1
+ else:
+ without_meta += 1
+
+ total = with_meta + without_meta
+ msgs = [
+ f"{total} packages in VDB",
+ f" {with_meta} have consolidated metadata file",
+ f" {without_meta} are missing, stale, or use an older format",
+ ]
+ if without_meta:
+ msgs.append("Run 'emaint vdb --fix' to populate missing metadata files.")
+ return (without_meta == 0, msgs if without_meta else None)
+
+ def fix(self, **kwargs):
+ """Populate the consolidated metadata file for packages that lack it."""
+ settings = kwargs.get("settings", getattr(portage, "settings", {}))
+ options = kwargs.get("options") or {}
+ delete_individual = options.get("delete_individual_files", False)
+
+ errors = []
+ for cpv, pkgdir in _iter_pkg_dirs(settings):
+ # _consolidate_to_metadata_file() skips a package whose metadata
+ # file is already current, so no check is needed here.
+ try:
+ _consolidate_to_metadata_file(
+ pkgdir, delete_individual=delete_individual
+ )
+ except Exception as e:
+ errors.append(f"{cpv}: {e}")
+
+ if errors:
+ return (False, errors)
+ return (True, None)
+
+ def remove(self, **kwargs):
+ """Remove consolidated metadata files from all VDB package directories."""
+ settings = kwargs.get("settings", getattr(portage, "settings", {}))
+
+ errors = []
+ for cpv, pkgdir in _iter_pkg_dirs(settings):
+ metadata_path = os.path.join(pkgdir, _METADATA_FILE)
+ try:
+ os.unlink(metadata_path)
+ except FileNotFoundError:
+ pass
+ except OSError as e:
+ errors.append(f"{cpv}: {e}")
+
+ if errors:
+ return (False, errors)
+ return (True, None)
diff --git a/lib/portage/tests/dbapi/test_vdb_metadata.py b/lib/portage/tests/dbapi/test_vdb_metadata.py
index 1824a524f..df6244b3f 100644
--- a/lib/portage/tests/dbapi/test_vdb_metadata.py
+++ b/lib/portage/tests/dbapi/test_vdb_metadata.py
@@ -240,10 +240,10 @@ class VdbConsolidateTestCase(TestCase):
def test_uncached_field_excluded(self):
# An all-caps VDB field vardbapi does not cache. It stays in its own
- # file, and consolidation must not claim it.
+ # file, and consolidation must not claim or remove it.
self._write_field("EAPI", "8")
self._write_field("FEATURES", "buildpkg parallel-install")
- _consolidate_to_metadata_file(self._tmpdir)
+ _consolidate_to_metadata_file(self._tmpdir, delete_individual=True)
path = os.path.join(self._tmpdir, _METADATA_FILE)
result = _read_metadata_file(path)
self.assertNotIn("FEATURES", result)
@@ -273,10 +273,72 @@ class VdbConsolidateTestCase(TestCase):
_consolidate_to_metadata_file(self._tmpdir)
self.assertTrue(os.path.exists(os.path.join(self._tmpdir, "EAPI")))
+ def test_delete_individual_files(self):
+ self._write_field("EAPI", "8")
+ self._write_field("SLOT", "0")
+ _consolidate_to_metadata_file(self._tmpdir, delete_individual=True)
+ self.assertFalse(os.path.exists(os.path.join(self._tmpdir, "EAPI")))
+ self.assertFalse(os.path.exists(os.path.join(self._tmpdir, "SLOT")))
+ # metadata file itself must exist
+ self.assertTrue(os.path.exists(os.path.join(self._tmpdir, _METADATA_FILE)))
+
+ def test_delete_individual_leaves_readable_file(self):
+ # The unlinks change the directory, so the file has to be stamped
+ # after them. Stamping first would leave the package with neither its
+ # individual files nor a metadata file the reader accepts.
+ self._write_field("EAPI", "8")
+ self._write_field("SLOT", "0")
+ _consolidate_to_metadata_file(self._tmpdir, delete_individual=True)
+ result = _read_metadata_file(os.path.join(self._tmpdir, _METADATA_FILE))
+ self.assertIsNotNone(result)
+ self.assertEqual(result["EAPI"], "8")
+ self.assertEqual(result["SLOT"], "0")
+
+ def test_delete_individual_preserves_contents(self):
+ self._write_field("EAPI", "8")
+ contents = "obj /usr/bin/foo abc123 1234567890\n"
+ with open(os.path.join(self._tmpdir, "CONTENTS"), "w") as f:
+ f.write(contents)
+ _consolidate_to_metadata_file(self._tmpdir, delete_individual=True)
+ # CONTENTS not in metadata file, not deleted
+ self.assertTrue(os.path.exists(os.path.join(self._tmpdir, "CONTENTS")))
+
def test_empty_dir_no_metadata_file(self):
_consolidate_to_metadata_file(self._tmpdir)
self.assertFalse(os.path.exists(os.path.join(self._tmpdir, _METADATA_FILE)))
+ def test_skips_when_file_is_already_current(self):
+ # A file that still validates describes dbdir exactly, so a second
+ # call must not rewrite it. write_atomic() renames a new file into
+ # place, so a rewrite would change the inode.
+ self._write_field("EAPI", "8")
+ path = os.path.join(self._tmpdir, _METADATA_FILE)
+ _consolidate_to_metadata_file(self._tmpdir)
+ ino = os.stat(path).st_ino
+ _consolidate_to_metadata_file(self._tmpdir)
+ self.assertEqual(os.stat(path).st_ino, ino)
+
+ def test_rebuilds_when_file_is_stale(self):
+ self._write_field("EAPI", "8")
+ _consolidate_to_metadata_file(self._tmpdir)
+ # A changed directory invalidates the file, so it is rebuilt rather
+ # than skipped, and picks up the field added along the way.
+ self._write_field("SLOT", "0/0")
+ _consolidate_to_metadata_file(self._tmpdir)
+ result = _read_metadata_file(os.path.join(self._tmpdir, _METADATA_FILE))
+ self.assertEqual(result["SLOT"], "0/0")
+
+ def test_delete_individual_not_skipped_by_current_file(self):
+ # The plain call leaves the individual files in place, so the
+ # delete_individual call after it still has work despite finding a
+ # metadata file that validates.
+ self._write_field("EAPI", "8")
+ _consolidate_to_metadata_file(self._tmpdir)
+ _consolidate_to_metadata_file(self._tmpdir, delete_individual=True)
+ self.assertFalse(os.path.exists(os.path.join(self._tmpdir, "EAPI")))
+ result = _read_metadata_file(os.path.join(self._tmpdir, _METADATA_FILE))
+ self.assertEqual(result["EAPI"], "8")
+
class VdbMetadataAuxGetTestCase(TestCase):
def testUncachedFieldStillComesFromEnvironment(self):