[PATCH 1/2] bitbake-setup: support multiple source override files

From: Ernest Van Hoecke <[email protected]>

The bitbake-setup documentation currently states that:
"The --source-overrides option can be passed multiple times, in which
case the overrides are applied in the order specified in the
command-line."

However, this is not currently true and only the last --source-overrides
argument is actually used.

Allow --source-overrides to be specified more than once and follow the
behaviour set out in the documentation, i.e.: merge override files in
command-line order.

Extend the setup test to exercise two override files through init,
checking that both are applied and that they are applied in order.

Signed-off-by: Ernest Van Hoecke <[email protected]>
---
 bin/bitbake-setup     | 17 +++++++-------
 lib/bb/tests/setup.py | 61 ++++++++++++++++++++++++++++++++++-----------------
 2 files changed, 50 insertions(+), 28 deletions(-)

diff --git a/bin/bitbake-setup b/bin/bitbake-setup
index 220540f7f9ca..2e234cccaba9 100755
--- a/bin/bitbake-setup
+++ b/bin/bitbake-setup
@@ -636,21 +636,22 @@ def obtain_config(top_dir, registry, args, source_overrides, d):
     return upstream_config
 
 def obtain_overrides(args):
-    overrides = {'sources':{}}
-    if args.source_overrides:
-        overrides = json.load(open(args.source_overrides))
-        overrides_dir = os.path.dirname(os.path.abspath(args.source_overrides))
+    all_overrides = {'sources':{}}
+    for overrides_file in args.source_overrides or []:
+        overrides = json.load(open(overrides_file))
+        overrides_dir = os.path.dirname(os.path.abspath(overrides_file))
         for s,v in overrides['sources'].items():
             local = v.get('local')
             if local:
                 path = os.path.expanduser(local['path'])
                 if not os.path.isabs(path):
-                    overrides['sources'][s]['local']['path'] = os.path.join(overrides_dir, path)
+                    v['local']['path'] = os.path.join(overrides_dir, path)
+            all_overrides['sources'][s] = v
 
     for local_name, local_path in args.use_local_source:
-        overrides['sources'][local_name] = {'local':{'path':os.path.abspath(os.path.expanduser(local_path))}}
+        all_overrides['sources'][local_name] = {'local':{'path':os.path.abspath(os.path.expanduser(local_path))}}
 
-    return overrides
+    return all_overrides
 
 def configure_vscode(setupdir, layerdir, builddir, init_script):
     """
@@ -1257,7 +1258,7 @@ def main():
     parser_init = subparsers.add_parser('init', help='Select a configuration and initialize a setup from it')
     parser_init.add_argument('config', nargs='*', help="path/URL/id to a configuration file (use 'list' command to get available ids), followed by configuration options. Bitbake-setup will ask to choose from available choices if command line doesn't completely specify them.")
     parser_init.add_argument('--non-interactive', action='store_true', help='Do not ask to interactively choose from available options; if bitbake-setup cannot make a decision it will stop with a failure.')
-    parser_init.add_argument('--source-overrides', action='store', help='Override sources information (repositories/revisions) with values from a local json file.')
+    parser_init.add_argument('--source-overrides', action='append', help='Override sources information (repositories/revisions) with values from a local json file.')
     parser_init.add_argument('--setup-dir-name', action='store', help='A custom setup directory name under the top directory.')
     parser_init.add_argument('--skip-selection', action='append', help='Do not select and set an option/fragment from available choices; the resulting bitbake configuration may be incomplete.')
     parser_init.add_argument('-L', '--use-local-source', default=[], action='append', nargs=2, metavar=('SOURCE_NAME', 'PATH'),
diff --git a/lib/bb/tests/setup.py b/lib/bb/tests/setup.py
index 638d56d3bb32..d425faa1f6b6 100644
--- a/lib/bb/tests/setup.py
+++ b/lib/bb/tests/setup.py
@@ -180,21 +180,23 @@ print("BBPATH is {{}}".format(os.environ["BBPATH"]))
         self.git('commit -m "Adding {}"'.format(name), cwd=self.registrypath)
         return json.loads(config)
 
-    def add_json_config_to_registry(self, name, rev, branch):
-        sources = """
-        "test-repo": {
-            "git-remote": {
-                "remotes": {
-                    "origin": {
-                        "uri": "file://%s"
-                    }
-                },
-                "branch": "%s",
-                "rev": "%s"
+    def add_json_config_to_registry(self, name, rev, branch, source_names=("test-repo",)):
+        sources = []
+        for source_name in source_names:
+            sources.append("""
+            "%s": {
+                "git-remote": {
+                    "remotes": {
+                        "origin": {
+                            "uri": "file://%s"
+                        }
+                    },
+                    "branch": "%s",
+                    "rev": "%s"
+                }
             }
-        }
-""" % (self.testrepopath, branch, rev)
-        return self._add_json_config_to_registry_helper(name, sources)
+""" % (source_name, self.testrepopath, branch, rev))
+        return self._add_json_config_to_registry_helper(name, ",\n".join(sources))
 
     def add_local_json_config_to_registry(self, name, path):
         sources = """
@@ -442,11 +444,12 @@ print("BBPATH is {{}}".format(os.environ["BBPATH"]))
             sums_after = _conf_chksum(f"{setuppath}/build/conf")
             self.assertEqual(sums_before, sums_after)
 
-        def _check_local_sources(custom_setup_dir):
+        def _check_local_sources(custom_setup_dir, sources=("test-repo",)):
             custom_setup_path = os.path.join(self.tempdir, 'bitbake-builds', custom_setup_dir)
-            custom_layer_path = os.path.join(custom_setup_path, 'layers', 'test-repo')
-            self.assertTrue(os.path.islink(custom_layer_path))
-            self.assertEqual(self.testrepopath, os.path.realpath(custom_layer_path))
+            for source in sources:
+                custom_layer_path = os.path.join(custom_setup_path, 'layers', source)
+                self.assertTrue(os.path.islink(custom_layer_path))
+                self.assertEqual(self.testrepopath, os.path.realpath(custom_layer_path))
             self.config_is_unchanged(custom_setup_path)
 
         def _check_layer_backups(layer_path, expected_backups):
@@ -525,14 +528,32 @@ print("BBPATH is {{}}".format(os.environ["BBPATH"]))
             "local": {
                 "path": "."
             }
+        },
+        "test-repo-extra": {
+            "local": {
+                "path": "/does/not/exist"
+            }
+        }
+    }
+}"""
+        second_source_override_content = """
+{
+    "sources": {
+        "test-repo-extra": {
+            "local": {
+                "path": "."
+            }
         }
     }
 }"""
         override_filename = 'source-overrides.json'
+        second_override_filename = 'second-source-overrides.json'
         custom_setup_dir = 'special-setup-dir'
+        self.add_json_config_to_registry('test-config-1.conf.json', branch, branch, ("test-repo", "test-repo-extra"))
         self.add_file_to_testrepo(override_filename, source_override_content)
-        out = self.runbbsetup("init --non-interactive --source-overrides {} --setup-dir-name {} test-config-1 gadget".format(os.path.join(self.testrepopath, override_filename), custom_setup_dir))
-        _check_local_sources(custom_setup_dir)
+        self.add_file_to_testrepo(second_override_filename, second_source_override_content)
+        out = self.runbbsetup("init --non-interactive --source-overrides {} --source-overrides {} --setup-dir-name {} test-config-1 gadget".format(os.path.join(self.testrepopath, override_filename), os.path.join(self.testrepopath, second_override_filename), custom_setup_dir))
+        _check_local_sources(custom_setup_dir, ("test-repo", "test-repo-extra"))
 
         # same but use command line options to specify local overrides
         custom_setup_dir = 'special-setup-dir-with-cmdline-overrides'

-- 
2.43.0