proj/portage:master commit in: lib/portage/emaint/modules/vdb/, lib/portage/tests/dbapi/, lib/portage/dbapi/
"Matt Turner" <[email protected]>
| Newsgroups | gmane.linux.gentoo.cvs |
|---|---|
| Message-ID | <1786672956.33d003cc76ed187f076547ea6dee9e13e50029ca.mattst88@gentoo> |
commit: 33d003cc76ed187f076547ea6dee9e13e50029ca
Author: Matt Turner <mattst88 <AT> gentoo <DOT> org>
AuthorDate: Wed Aug 12 18:09:21 2026 +0000
Commit: Matt Turner <mattst88 <AT> gentoo <DOT> org>
CommitDate: Fri Aug 14 02:02:36 2026 +0000
URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=33d003cc
emaint: make 'vdb --remove' the reverse of --fix
--remove unlinked the metadata file and nothing else. That is fine when the
individual per-field files are still there, but --fix
--delete-individual-files removes them, and after that the metadata file is
the only copy of every field it carries. Unlinking it then left the package
with no metadata at all, so aux_get() would report every field as empty for
a package whose VDB entry had been perfectly intact.
Add _explode_metadata_file(), the inverse of _consolidate_to_metadata_file():
write back any field the metadata file alone still holds, then remove the
file. Restoring before unlinking keeps every field present on disk, and an
interrupted run leaves the metadata file behind, stale, so the reader
rejects it and falls back to the individual files that now exist.
A metadata file that is the only copy of some field but is not a snapshot
this portage version can trust is refused rather than removed: deleting it
would destroy the field and restoring from it could write a stale value.
That means a corrupted or downgraded VDB, so it reports the package and
leaves it alone. A stale file whose fields all still have their own files
is just garbage and is removed as before.
Restored values are whitespace-normalized, since that is the form the
metadata file stores. _aux_get() applies the same normalization to every
single-line field, so the value it serves does not change.
Signed-off-by: Matt Turner <mattst88 <AT> gentoo.org>
lib/portage/dbapi/vartree.py | 62 +++++++++++++
lib/portage/emaint/modules/vdb/__init__.py | 4 +-
lib/portage/emaint/modules/vdb/vdb.py | 18 ++--
lib/portage/tests/dbapi/test_vdb_metadata.py | 134 +++++++++++++++++++++++++++
4 files changed, 211 insertions(+), 7 deletions(-)
diff --git a/lib/portage/dbapi/vartree.py b/lib/portage/dbapi/vartree.py
index bf3f6902e..7347b3dd7 100644
--- a/lib/portage/dbapi/vartree.py
+++ b/lib/portage/dbapi/vartree.py
@@ -285,6 +285,68 @@ def _consolidate_to_metadata_file(dbdir, delete_individual=False):
_stamp_metadata_file(dbdir)
+def _explode_metadata_file(dbdir):
+ """Remove the metadata file, restoring any field it alone still holds.
+
+ The inverse of _consolidate_to_metadata_file(). Normally the individual
+ files are still there and this just unlinks the metadata file, but after a
+ delete_individual=True run the metadata file is the only copy of the
+ fields it carries, so those are written back to their own files first.
+
+ Restoring before unlinking means a field is never absent from disk. An
+ interrupted run leaves the metadata file in place; it is stale by then, so
+ the reader rejects it and falls back to the individual files that now
+ exist.
+
+ Refuses to unlink a metadata file that is the only copy of some field but
+ is not a snapshot this portage version can trust, since deleting it would
+ destroy that field and restoring from it could write a stale value. That
+ means a corrupted or downgraded VDB, and guessing is worse than stopping.
+
+ Returns the sorted list of field names restored. Raises PortageException
+ if the metadata file cannot be safely removed.
+ """
+ from portage import _encodings
+ from portage.exception import PortageException
+
+ path = os.path.join(dbdir, _METADATA_FILE)
+
+ # Parsed without validating format version or dir mtime, only to learn
+ # which fields the file claims. Values from it are used solely to restore
+ # fields the validated read below also vouches for.
+ claimed = {}
+ try:
+ with open(path, encoding=_encodings["repo.content"], errors="replace") as f:
+ for line in f:
+ line = line.rstrip("\n")
+ if line.startswith("#") or "=" not in line:
+ continue
+ k, v = line.split("=", 1)
+ claimed[k] = v
+ except FileNotFoundError:
+ return []
+
+ missing = sorted(k for k in claimed if not os.path.exists(os.path.join(dbdir, k)))
+
+ if missing:
+ trusted = _read_metadata_file(path)
+ if trusted is None:
+ raise PortageException(
+ f"{path}: refusing to remove, it is the only copy of "
+ f"{', '.join(missing)} and is not a usable snapshot"
+ )
+ for fname in missing:
+ with open(
+ os.path.join(dbdir, fname),
+ mode="w",
+ encoding=_encodings["repo.content"],
+ ) as f:
+ f.write(f"{trusted.get(fname, '')}\n")
+
+ os.unlink(path)
+ return missing
+
+
class vardbapi(dbapi):
_excluded_dirs = ["CVS", "lost+found"]
_excluded_dirs = [re.escape(x) for x in _excluded_dirs]
diff --git a/lib/portage/emaint/modules/vdb/__init__.py b/lib/portage/emaint/modules/vdb/__init__.py
index 0fd7e65df..918e3bb83 100644
--- a/lib/portage/emaint/modules/vdb/__init__.py
+++ b/lib/portage/emaint/modules/vdb/__init__.py
@@ -29,7 +29,9 @@ module_spec = {
"remove": {
"short": "-R",
"long": "--remove",
- "help": "Remove consolidated metadata files",
+ "help": "Undo --fix: restore any per-field file that only "
+ "the metadata file still holds, then remove the metadata "
+ "files",
"status": "Removing VDB metadata files for %s",
"action": "store_true",
"func": "remove",
diff --git a/lib/portage/emaint/modules/vdb/vdb.py b/lib/portage/emaint/modules/vdb/vdb.py
index ffe32d83a..96577cf8b 100644
--- a/lib/portage/emaint/modules/vdb/vdb.py
+++ b/lib/portage/emaint/modules/vdb/vdb.py
@@ -6,6 +6,7 @@ from portage import os
from portage.dbapi.vartree import (
_METADATA_FILE,
_consolidate_to_metadata_file,
+ _explode_metadata_file,
_read_metadata_file,
)
@@ -84,19 +85,24 @@ class VdbMetadata:
return (True, None)
def remove(self, **kwargs):
- """Remove consolidated metadata files from all VDB package directories."""
+ """Undo --fix: restore individual per-field files, drop the metadata file.
+
+ The reverse of what fix() did, including the --delete-individual-files
+ case: a field the metadata file is the only remaining copy of is
+ written back to its own file before the metadata file goes away.
+ """
settings = kwargs.get("settings", getattr(portage, "settings", {}))
errors = []
+ restored = 0
for cpv, pkgdir in _iter_pkg_dirs(settings):
- metadata_path = os.path.join(pkgdir, _METADATA_FILE)
try:
- os.unlink(metadata_path)
- except FileNotFoundError:
- pass
- except OSError as e:
+ restored += len(_explode_metadata_file(pkgdir))
+ except Exception as e:
errors.append(f"{cpv}: {e}")
if errors:
return (False, errors)
+ if restored:
+ return (True, [f"Restored {restored} individual VDB files."])
return (True, None)
diff --git a/lib/portage/tests/dbapi/test_vdb_metadata.py b/lib/portage/tests/dbapi/test_vdb_metadata.py
index df6244b3f..9e9abcaa0 100644
--- a/lib/portage/tests/dbapi/test_vdb_metadata.py
+++ b/lib/portage/tests/dbapi/test_vdb_metadata.py
@@ -4,12 +4,14 @@
import os
import tempfile
+import portage
from portage.tests import TestCase
from portage.dbapi.vartree import (
_METADATA_FILE,
_METADATA_FILE_FIELDS,
_METADATA_FILE_FORMAT_VERSION,
_consolidate_to_metadata_file,
+ _explode_metadata_file,
_in_metadata_file,
_read_metadata_file,
_write_metadata_file,
@@ -340,6 +342,138 @@ class VdbConsolidateTestCase(TestCase):
self.assertEqual(result["EAPI"], "8")
+class VdbExplodeTestCase(TestCase):
+ def setUp(self):
+ self._tmpdir = tempfile.mkdtemp()
+
+ def tearDown(self):
+ import shutil
+
+ shutil.rmtree(self._tmpdir, ignore_errors=True)
+
+ def _write_field(self, name, value):
+ with open(os.path.join(self._tmpdir, name), "w") as f:
+ f.write(value + "\n")
+
+ def _read_field(self, name):
+ with open(os.path.join(self._tmpdir, name)) as f:
+ return f.read()
+
+ def test_no_metadata_file_is_not_an_error(self):
+ self.assertEqual(_explode_metadata_file(self._tmpdir), [])
+
+ def test_removes_file_and_keeps_individual_files(self):
+ self._write_field("EAPI", "8")
+ self._write_field("SLOT", "0")
+ _consolidate_to_metadata_file(self._tmpdir)
+ # Nothing to restore: the individual files were never removed.
+ self.assertEqual(_explode_metadata_file(self._tmpdir), [])
+ self.assertFalse(os.path.exists(os.path.join(self._tmpdir, _METADATA_FILE)))
+ self.assertEqual(self._read_field("EAPI"), "8\n")
+ self.assertEqual(self._read_field("SLOT"), "0\n")
+
+ def test_round_trip_through_delete_individual(self):
+ # The case that makes this the inverse rather than an unlink: after
+ # --delete-individual-files the metadata file is the only copy.
+ self._write_field("EAPI", "8")
+ self._write_field("SLOT", "0/0")
+ self._write_field("USE", "foo bar")
+ _consolidate_to_metadata_file(self._tmpdir, delete_individual=True)
+ self.assertFalse(os.path.exists(os.path.join(self._tmpdir, "EAPI")))
+
+ self.assertEqual(_explode_metadata_file(self._tmpdir), ["EAPI", "SLOT", "USE"])
+ self.assertFalse(os.path.exists(os.path.join(self._tmpdir, _METADATA_FILE)))
+ self.assertEqual(self._read_field("EAPI"), "8\n")
+ self.assertEqual(self._read_field("SLOT"), "0/0\n")
+ self.assertEqual(self._read_field("USE"), "foo bar\n")
+
+ def test_non_metadata_files_untouched(self):
+ self._write_field("EAPI", "8")
+ with open(os.path.join(self._tmpdir, "CONTENTS"), "w") as f:
+ f.write("obj /usr/bin/foo abc123 1234567890\n")
+ _consolidate_to_metadata_file(self._tmpdir, delete_individual=True)
+ _explode_metadata_file(self._tmpdir)
+ self.assertEqual(
+ self._read_field("CONTENTS"), "obj /usr/bin/foo abc123 1234567890\n"
+ )
+
+ def test_refuses_when_sole_copy_is_unusable(self):
+ # Individual files gone and the snapshot no longer validates: the
+ # values cannot be trusted and deleting the file would lose them.
+ from portage.exception import PortageException
+
+ self._write_field("EAPI", "8")
+ self._write_field("SLOT", "0")
+ _consolidate_to_metadata_file(self._tmpdir, delete_individual=True)
+ os.utime(self._tmpdir, ns=(0, 0))
+
+ self.assertRaises(PortageException, _explode_metadata_file, self._tmpdir)
+ # Nothing destroyed: the file is still there to be recovered from.
+ self.assertTrue(os.path.exists(os.path.join(self._tmpdir, _METADATA_FILE)))
+
+ def test_removes_unusable_file_when_individual_files_present(self):
+ # A stale file is just garbage when every field it names still has its
+ # own file, so it is removed rather than refused.
+ self._write_field("EAPI", "8")
+ _consolidate_to_metadata_file(self._tmpdir)
+ os.utime(self._tmpdir, ns=(0, 0))
+
+ self.assertEqual(_explode_metadata_file(self._tmpdir), [])
+ self.assertFalse(os.path.exists(os.path.join(self._tmpdir, _METADATA_FILE)))
+ self.assertEqual(self._read_field("EAPI"), "8\n")
+
+
+class VdbEmaintRoundTripTestCase(TestCase):
+ def testFixDeleteIndividualThenRemoveRestoresVdb(self):
+ """'emaint vdb --remove' undoes --fix --delete-individual-files.
+
+ Without the restore step it would unlink the only remaining copy of
+ every field and leave an unreadable VDB behind.
+ """
+ from portage.emaint.modules.vdb.vdb import VdbMetadata
+
+ pkgs = {
+ "dev-libs/A-1": {"EAPI": "7", "SLOT": "0", "RDEPEND": "dev-libs/B"},
+ "dev-libs/B-1": {"EAPI": "7", "SLOT": "0"},
+ }
+ playground = ResolverPlayground(ebuilds=pkgs, installed=pkgs)
+ # The module resolves the vardb through the global portage.db, the way
+ # it does under a real emaint run.
+ had_db = hasattr(portage, "db")
+ saved_db = getattr(portage, "db", None)
+ portage.db = playground.trees
+ try:
+ settings = playground.settings
+ vardb = playground.trees[playground.eroot]["vartree"].dbapi
+ pkgdir = vardb.getpath("dev-libs/A-1")
+ module = VdbMetadata()
+
+ status, _msgs = module.fix(
+ settings=settings, options={"delete_individual_files": True}
+ )
+ self.assertTrue(status)
+ self.assertFalse(os.path.exists(os.path.join(pkgdir, "RDEPEND")))
+ self.assertTrue(os.path.exists(os.path.join(pkgdir, _METADATA_FILE)))
+
+ status, _msgs = module.remove(settings=settings)
+ self.assertTrue(status)
+ self.assertFalse(os.path.exists(os.path.join(pkgdir, _METADATA_FILE)))
+ self.assertTrue(os.path.exists(os.path.join(pkgdir, "RDEPEND")))
+
+ # The values have to survive the round trip, not just the files.
+ vardb._aux_cache_obj = None
+ self.assertEqual(
+ vardb.aux_get("dev-libs/A-1", ["RDEPEND", "SLOT", "EAPI"]),
+ ["dev-libs/B", "0", "7"],
+ )
+ finally:
+ if had_db:
+ portage.db = saved_db
+ else:
+ del portage.db
+ playground.cleanup()
+
+
class VdbMetadataAuxGetTestCase(TestCase):
def testUncachedFieldStillComesFromEnvironment(self):
"""A field the metadata file does not carry still reaches