[PATCH 1/2] Add binrepos.conf to replace PORTAGE_BINHOST (bug 668334)
Zac Medico <[email protected]>
| Newsgroups | gmane.linux.gentoo.portage.devel |
|---|---|
| Message-ID | <[email protected]> |
Support /etc/portage/binrepos.conf as a replacement for the
PORTAGE_BINHOST variable. Behavior is similar to repos.conf,
initially supporting just the sync-uri attribute. Both binrepos.conf
and PORTAGE_BINHOST can be used simultaneously, in the same way that
repos.conf and PORTDIR_OVERLAY can be used simultaneously.
The emerge --info output for binrepos.conf looks like this:
Binary Repositories:
example-binhost
sync-uri: https://example.com/packages
Bug: https://bugs.gentoo.org/668334
Signed-off-by: Zac Medico <[email protected]>
---
lib/_emerge/actions.py | 13 ++-
lib/portage/binrepo/__init__.py | 0
lib/portage/binrepo/config.py | 131 ++++++++++++++++++++++++
lib/portage/const.py | 1 +
lib/portage/dbapi/bintree.py | 14 ++-
lib/portage/tests/emerge/test_simple.py | 14 ++-
man/make.conf.5 | 3 +-
man/portage.5 | 38 +++++++
8 files changed, 206 insertions(+), 8 deletions(-)
create mode 100644 lib/portage/binrepo/__init__.py
create mode 100644 lib/portage/binrepo/config.py
diff --git a/lib/_emerge/actions.py b/lib/_emerge/actions.py
index a4ecfe43d..34344f046 100644
--- a/lib/_emerge/actions.py
+++ b/lib/_emerge/actions.py
@@ -32,7 +32,8 @@ portage.proxy.lazyimport.lazyimport(globals(),
from portage import os
from portage import shutil
from portage import _encodings, _unicode_decode
-from portage.const import _DEPCLEAN_LIB_CHECK_DEFAULT
+from portage.binrepo.config import BinRepoConfigLoader
+from portage.const import BINREPOS_CONF_FILE, _DEPCLEAN_LIB_CHECK_DEFAULT
from portage.dbapi.dep_expand import dep_expand
from portage.dbapi._expand_new_virt import expand_new_virt
from portage.dbapi.IndexedPortdb import IndexedPortdb
@@ -1836,6 +1837,16 @@ def action_info(settings, trees, myopts, myfiles):
for repo in repos:
append(repo.info_string())
+ binrepos_conf_path = os.path.join(settings['PORTAGE_CONFIGROOT'], BINREPOS_CONF_FILE)
+ binrepos_conf = BinRepoConfigLoader((binrepos_conf_path,), settings)
+ if binrepos_conf and any(repo.name for repo in binrepos_conf.values()):
+ append("Binary Repositories:\n")
+ for repo in reversed(list(binrepos_conf.values())):
+ # Omit repos from the PORTAGE_BINHOST variable, since they
+ # do not have a name to label them with.
+ if repo.name:
+ append(repo.info_string())
+
installed_sets = sorted(s for s in
root_config.sets['selected'].getNonAtoms() if s.startswith(SETPREFIX))
if installed_sets:
diff --git a/lib/portage/binrepo/__init__.py b/lib/portage/binrepo/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/lib/portage/binrepo/config.py b/lib/portage/binrepo/config.py
new file mode 100644
index 000000000..aa3ff7a77
--- /dev/null
+++ b/lib/portage/binrepo/config.py
@@ -0,0 +1,131 @@
+# Copyright 2020 Gentoo Authors
+# Distributed under the terms of the GNU General Public License v2
+
+from collections import OrderedDict
+from collections.abc import Mapping
+from hashlib import md5
+
+from portage.localization import _
+from portage.util import _recursive_file_list, writemsg
+from portage.util.configparser import (SafeConfigParser, ConfigParserError,
+ read_configs)
+
+
+class BinRepoConfig:
+ __slots__ = (
+ 'name',
+ 'name_fallback',
+ 'priority',
+ 'sync_uri',
+ )
+ def __init__(self, opts):
+ """
+ Create a BinRepoConfig with options in opts.
+ """
+ for k in self.__slots__:
+ setattr(self, k, opts.get(k.replace('_', '-')))
+
+ def info_string(self):
+ """
+ Returns a formatted string containing informations about the repository.
+ Used by emerge --info.
+ """
+ indent = " " * 4
+ repo_msg = []
+ repo_msg.append(self.name or self.name_fallback)
+ if self.priority is not None:
+ repo_msg.append(indent + "priority: " + str(self.priority))
+ repo_msg.append(indent + "sync-uri: " + self.sync_uri)
+ repo_msg.append("")
+ return "\n".join(repo_msg)
+
+
+class BinRepoConfigLoader(Mapping):
+ def __init__(self, paths, settings):
+ """Load config from files in paths"""
+
+ # Defaults for value interpolation.
+ parser_defaults = {
+ "EPREFIX" : settings["EPREFIX"],
+ "EROOT" : settings["EROOT"],
+ "PORTAGE_CONFIGROOT" : settings["PORTAGE_CONFIGROOT"],
+ "ROOT" : settings["ROOT"],
+ }
+
+ try:
+ parser = self._parse(paths, parser_defaults)
+ except ConfigParserError as e:
+ writemsg(
+ _("!!! Error while reading binrepo config file: %s\n") % e,
+ noiselevel=-1)
+ parser = SafeConfigParser(defaults=parser_defaults)
+
+ repos = []
+ sync_uris = []
+ for section_name in parser.sections():
+ repo_data = dict(parser[section_name].items())
+ repo_data['name'] = section_name
+ repo = BinRepoConfig(repo_data)
+ if repo.sync_uri is None:
+ writemsg(_("!!! Missing sync-uri setting for binrepo %s\n") % (repo.name,), noiselevel=-1)
+ continue
+
+ sync_uri = self._normalize_uri(repo.sync_uri)
+ sync_uris.append(sync_uri)
+ repo.sync_uri = sync_uri
+ if repo.priority is not None:
+ try:
+ repo.priority = int(repo.priority)
+ except ValueError:
+ repo.priority = None
+ repos.append(repo)
+
+ sync_uris = set(sync_uris)
+ current_priority = 0
+ for sync_uri in reversed(settings.get("PORTAGE_BINHOST", "").split()):
+ sync_uri = self._normalize_uri(sync_uri)
+ if sync_uri not in sync_uris:
+ current_priority += 1
+ sync_uris.add(sync_uri)
+ repos.append(BinRepoConfig({
+ 'name-fallback': self._digest_uri(sync_uri),
+ 'name': None,
+ 'priority': current_priority,
+ 'sync-uri': sync_uri,
+ }))
+
+ self._data = OrderedDict((repo.name, repo) for repo in
+ sorted(repos, key=lambda repo: (repo.priority or 0, repo.name or repo.name_fallback)))
+
+ @staticmethod
+ def _digest_uri(uri):
+ return md5(uri.encode('utf_8')).hexdigest()
+
+ @staticmethod
+ def _normalize_uri(uri):
+ return uri.rstrip('/')
+
+ @staticmethod
+ def _parse(paths, defaults):
+ parser = SafeConfigParser(defaults=defaults)
+ recursive_paths = []
+ for p in paths:
+ if isinstance(p, str):
+ recursive_paths.extend(_recursive_file_list(p))
+ else:
+ recursive_paths.append(p)
+
+ read_configs(parser, recursive_paths)
+ return parser
+
+ def __iter__(self):
+ return iter(self._data)
+
+ def __contains__(self, key):
+ return key in self._data
+
+ def __getitem__(self, key):
+ return self._data[key]
+
+ def __len__(self):
+ return len(self._data)
diff --git a/lib/portage/const.py b/lib/portage/const.py
index 9a7ea23bd..b895f0fa9 100644
--- a/lib/portage/const.py
+++ b/lib/portage/const.py
@@ -28,6 +28,7 @@ import os
# variables used with config_root (these need to be relative)
USER_CONFIG_PATH = "etc/portage"
+BINREPOS_CONF_FILE = USER_CONFIG_PATH + "/binrepos.conf"
MAKE_CONF_FILE = USER_CONFIG_PATH + "/make.conf"
MODULES_FILE_PATH = USER_CONFIG_PATH + "/modules"
CUSTOM_PROFILE_PATH = USER_CONFIG_PATH + "/profile"
diff --git a/lib/portage/dbapi/bintree.py b/lib/portage/dbapi/bintree.py
index 620865a79..a96183561 100644
--- a/lib/portage/dbapi/bintree.py
+++ b/lib/portage/dbapi/bintree.py
@@ -22,8 +22,9 @@ portage.proxy.lazyimport.lazyimport(globals(),
'portage.versions:best,catpkgsplit,catsplit,_pkg_str',
)
+from portage.binrepo.config import BinRepoConfigLoader
from portage.cache.mappings import slot_dict_class
-from portage.const import CACHE_PATH, SUPPORTED_XPAK_EXTENSIONS
+from portage.const import BINREPOS_CONF_FILE, CACHE_PATH, SUPPORTED_XPAK_EXTENSIONS
from portage.dbapi.virtual import fakedbapi
from portage.dep import Atom, use_reduce, paren_enclose
from portage.exception import AlarmSignal, InvalidData, InvalidPackageName, \
@@ -364,6 +365,7 @@ class binarytree:
self.move_slot_ent = self.dbapi.move_slot_ent
self.populated = 0
self.tree = {}
+ self._binrepos_conf = None
self._remote_has_index = False
self._remotepkgs = None # remote metadata indexed by cpv
self._additional_pkgs = {}
@@ -628,8 +630,10 @@ class binarytree:
self._populate_additional(add_repos)
if getbinpkgs:
- if not self.settings.get("PORTAGE_BINHOST"):
- writemsg(_("!!! PORTAGE_BINHOST unset, but use is requested.\n"),
+ config_path = os.path.join(self.settings['PORTAGE_CONFIGROOT'], BINREPOS_CONF_FILE)
+ self._binrepos_conf = BinRepoConfigLoader((config_path,), self.settings)
+ if not self._binrepos_conf:
+ writemsg(_("!!! %s is missing (or PORTAGE_BINHOST is unset), but use is requested.\n") % (config_path,),
noiselevel=-1)
else:
self._populate_remote(getbinpkg_refresh=getbinpkg_refresh)
@@ -903,7 +907,9 @@ class binarytree:
self._remote_has_index = False
self._remotepkgs = {}
- for base_url in self.settings["PORTAGE_BINHOST"].split():
+ # Order by descending priority.
+ for repo in reversed(list(self._binrepos_conf.values())):
+ base_url = repo.sync_uri
parsed_url = urlparse(base_url)
host = parsed_url.netloc
port = parsed_url.port
diff --git a/lib/portage/tests/emerge/test_simple.py b/lib/portage/tests/emerge/test_simple.py
index c24f5c603..8635b70e4 100644
--- a/lib/portage/tests/emerge/test_simple.py
+++ b/lib/portage/tests/emerge/test_simple.py
@@ -7,7 +7,7 @@ import sys
import portage
from portage import shutil, os
from portage import _unicode_decode
-from portage.const import (BASH_BINARY, PORTAGE_PYM_PATH, USER_CONFIG_PATH)
+from portage.const import (BASH_BINARY, BINREPOS_CONF_FILE, PORTAGE_PYM_PATH, USER_CONFIG_PATH)
from portage.cache.mappings import Mapping
from portage.process import find_binary
from portage.tests import TestCase
@@ -419,13 +419,23 @@ call_has_and_best_version() {
)
# Test binhost support if FETCHCOMMAND is available.
+ binrepos_conf_file = os.path.join(os.sep, eprefix, BINREPOS_CONF_FILE)
+ with open(binrepos_conf_file, 'wt') as f:
+ f.write('[test-binhost]\n')
+ f.write('sync-uri = {}\n'.format(binhost_uri))
fetchcommand = portage.util.shlex_split(playground.settings['FETCHCOMMAND'])
fetch_bin = portage.process.find_binary(fetchcommand[0])
if fetch_bin is not None:
test_commands = test_commands + (
+ lambda: os.rename(pkgdir, binhost_dir),
+ emerge_cmd + ("-e", "--getbinpkgonly", "dev-libs/A"),
+ lambda: shutil.rmtree(pkgdir),
+ lambda: os.rename(binhost_dir, pkgdir),
+ # Remove binrepos.conf and test PORTAGE_BINHOST.
+ lambda: os.unlink(binrepos_conf_file),
lambda: os.rename(pkgdir, binhost_dir),
({"PORTAGE_BINHOST": binhost_uri},) + \
- emerge_cmd + ("-e", "--getbinpkgonly", "dev-libs/A"),
+ emerge_cmd + ("-fe", "--getbinpkgonly", "dev-libs/A"),
lambda: shutil.rmtree(pkgdir),
lambda: os.rename(binhost_dir, pkgdir),
)
diff --git a/man/make.conf.5 b/man/make.conf.5
index eb812150f..f7b2460e5 100644
--- a/man/make.conf.5
+++ b/man/make.conf.5
@@ -853,7 +853,8 @@ Each entry in the list must specify the full address of a directory
serving tbz2's for your system (this directory must contain a 'Packages' index
file). This is only used when running with
the get binary pkg options are given to \fBemerge\fR. Review \fBemerge\fR(1)
-for more information.
+for more information. The \fBPORTAGE_BINHOST\fR variable is deprecated in
+favor of the \fBbinrepos.conf\fR configuration file (see \fBportage\fR(5)).
.TP
\fBPORTAGE_BINHOST_HEADER_URI\fR = \
\fI"ftp://login:[email protected]/pub/grp/i686/athlon\-xp/"\fR
diff --git a/man/portage.5 b/man/portage.5
index 7472972cc..890a22adb 100644
--- a/man/portage.5
+++ b/man/portage.5
@@ -47,6 +47,7 @@ virtuals
.BR /etc/portage/
.nf
bashrc
+binrepos.conf
categories
color.map
license_groups
@@ -620,6 +621,43 @@ any other bash script.
Additional package-specific bashrc files can be created in /etc/portage/env.
.TP
+.BR binrepos.conf
+Specifies remote binary package repository configuration information. This
+is intended to be used as a replacement for the \fBmake.conf\fR(5)
+\fBPORTAGE_BINHOST\fR variable.
+
+.I Format:
+.nf
+\- comments begin with # (no inline comments)
+\- configuration of each repository is specified in a section starting with \
+"[${repository_name}]"
+\- attributes are specified in "${attribute} = ${value}" format
+.fi
+
+.RS
+.I Attributes supported in sections of repositories:
+.RS
+.TP
+.B priority
+Specifies priority of given repository. When a package exists in multiple
+repositories, those with higher priority are preferred.
+.TP
+.B sync\-uri
+Specifies URI of repository used for `emerge \-\-getbinpkg`.
+.RE
+.RE
+
+.I Example:
+.nf
+[example-binhost]
+# repos with higher priorities are preferred when packages with equal
+# versions are found in multiple repos
+priority = 9999
+# packages are fetched from here
+sync-uri = https://example.com/binhost
+
+.fi
+.TP
.BR categories
A simple list of valid categories that may be used in repositories and PKGDIR
(see \fBmake.conf\fR(5)). This allows for custom categories to be created.
--
2.25.3