[PATCH v3] bitbake-setup: use bb.fetch for buildtools installer download

When url and sha256sum are configured, download the buildtools
installer via bb.fetch with checksum enforcement. The file is
unpacked to a working directory and passed to install-buildtools
via --local-file.

If no config is present or --default is passed, fall back to
calling install-buildtools directly with its built-in defaults.

CLI overrides available via --url and --sha256.

Depends: [oe-core] install-buildtools: add --local-file option
AI-Generated: Kiro with Claude Opus 4.6
Signed-off-by: Jaipaul Cheernam <[email protected]>
---

Changes since v2:
- Added --default to bypass config and use script defaults
- Use fetcher.unpack() instead of reading from DL_DIR directly
- Removed explicit sha256 verification (fetcher handles it for https)
- Per-configuration install-buildtools config takes precedence over top-level

Changes since v1:
- Use bb.fetch with mandatory sha256sum when configured
- Fall back to install-buildtools defaults when no config present
- Combined url+filename into single url field
- Removed --sdk-name, --filename, --no-check/--check
- Added --url and --sha256 CLI overrides
- Updated schema, docs, and tests
 bin/bitbake-setup                             | 94 ++++++++++++++++++-
 .../bitbake-user-manual-environment-setup.rst | 19 ++++
 lib/bb/tests/setup.py                         | 77 ++++++++++++++-
 setup-schema/bitbake-setup.schema.json        | 21 +++++
 4 files changed, 203 insertions(+), 8 deletions(-)

diff --git a/bin/bitbake-setup b/bin/bitbake-setup
index 97ea08d11..aa084538d 100755
--- a/bin/bitbake-setup
+++ b/bin/bitbake-setup
@@ -1053,10 +1053,93 @@ def install_buildtools(top_dir, settings, args, d):
             return
         shutil.rmtree(buildtools_install_dir)
 
-    install_buildtools = os.path.join(args.setup_dir, 'layers/oe-scripts/install-buildtools')
-    buildtools_download_dir = add_unique_timestamp_to_path(os.path.join(args.setup_dir, 'buildtools-downloads/buildtools'))
-    logger.plain("Buildtools archive is downloaded into {} and its content installed into {}".format(buildtools_download_dir, buildtools_install_dir))
-    subprocess.check_call("{} -d {} --downloads-directory {}".format(install_buildtools, buildtools_install_dir, buildtools_download_dir), shell=True)
+    install_buildtools_script = os.path.join(args.setup_dir, 'layers/oe-scripts/install-buildtools')
+
+    buildtools_config = {}
+    upstream_config_path = os.path.join(args.setup_dir, 'config', 'config-upstream.json')
+    try:
+        with open(upstream_config_path) as f:
+            upstream_config = json.load(f)
+        # Per-configuration takes precedence over top-level
+        buildtools_config = upstream_config.get('bitbake-config', {}).get('install-buildtools', {})
+        if not buildtools_config:
+            buildtools_config = upstream_config.get('data', {}).get(
+                'bitbake-setup', {}).get('install-buildtools', {})
+    except FileNotFoundError:
+        pass
+    except json.JSONDecodeError as e:
+        logger.error("Failed to parse %s: %s" % (upstream_config_path, e))
+        sys.exit(1)
+
+    url = args.url if args.url is not None else buildtools_config.get('url')
+    sha256 = args.sha256 if args.sha256 is not None else buildtools_config.get('sha256sum')
+
+    if args.default or (not url and not sha256):
+        # No config or --default: fall back to install-buildtools defaults
+        logger.plain("Using install-buildtools script defaults")
+        buildtools_download_dir = add_unique_timestamp_to_path(
+            os.path.join(args.setup_dir, 'buildtools-downloads/buildtools'))
+        subprocess.check_call([install_buildtools_script,
+                               '-d', buildtools_install_dir,
+                               '--downloads-directory', buildtools_download_dir])
+        return
+
+    if not url:
+        logger.error("No buildtools URL configured. "
+                     "Set 'url' in the install-buildtools config section or pass --url.")
+        sys.exit(1)
+
+    if not sha256:
+        logger.error("No sha256sum configured for buildtools installer. "
+                     "Add 'sha256sum' to the install-buildtools config section or pass --sha256.\n"
+                     "You can obtain it with: sha256sum <installer-file>")
+        sys.exit(1)
+
+    # Download using bb.fetch with sha256 validation
+    src_uri = "{};sha256sum={}".format(url, sha256)
+
+    logger.plain("Fetching buildtools installer using bitbake fetcher")
+    logger.plain("    {}".format(url))
+
+    fetcher = bb.fetch.Fetch([src_uri], d)
+    try:
+        fetcher.download()
+    except bb.fetch2.ChecksumError as e:
+        logger.error("Checksum mismatch for buildtools installer. "
+                     "Verify the URL and downloaded file.\n%s" % str(e))
+        sys.exit(1)
+    except bb.fetch2.FetchError as e:
+        logger.error("Failed to download buildtools installer: %s" % str(e))
+        sys.exit(1)
+
+    # Unpack into a working directory (avoid touching DL_DIR directly)
+    import tempfile
+    unpack_base = os.path.join(args.setup_dir, 'buildtools-downloads')
+    os.makedirs(unpack_base, exist_ok=True)
+    unpackdir = tempfile.mkdtemp(dir=unpack_base)
+    fetcher.unpack(unpackdir)
+
+    filename = os.path.basename(url.rstrip('/'))
+    matches = glob.glob(os.path.join(unpackdir, '**', filename), recursive=True)
+    if not matches:
+        logger.error("Unpacked file not found: %s" % filename)
+        sys.exit(1)
+    local_path = matches[0]
+
+    logger.plain("Buildtools installer will be installed into {}".format(
+        buildtools_install_dir))
+
+    cmd = [
+        install_buildtools_script,
+        '--local-file', local_path,
+        '-d', buildtools_install_dir,
+    ]
+
+    try:
+        subprocess.check_call(cmd)
+    except subprocess.CalledProcessError as e:
+        logger.error("install-buildtools failed with exit code %d" % e.returncode)
+        sys.exit(1)
 
 def create_siteconf(top_dir, non_interactive, settings):
     siteconfpath = os.path.join(top_dir, 'site.conf')
@@ -1286,6 +1369,9 @@ def main():
     parser_install_buildtools = subparsers.add_parser('install-buildtools', help='Install buildtools which can help fulfil missing or incorrect dependencies on the host machine')
     add_setup_dir_arg(parser_install_buildtools)
     parser_install_buildtools.add_argument('--force', action='store_true', help='Force a reinstall of buildtools over the previous installation.')
+    parser_install_buildtools.add_argument('--default', action='store_true', help='Use install-buildtools script defaults, ignoring any config.')
+    parser_install_buildtools.add_argument('--url', help='Full URL to the buildtools SDK installer. Overrides config value.')
+    parser_install_buildtools.add_argument('--sha256', help='SHA256 checksum of the buildtools installer. Overrides config value.')
     parser_install_buildtools.set_defaults(func=install_buildtools)
 
     parser_settings_arg_global = argparse.ArgumentParser(add_help=False)
diff --git a/doc/bitbake-user-manual/bitbake-user-manual-environment-setup.rst b/doc/bitbake-user-manual/bitbake-user-manual-environment-setup.rst
index aa546c30b..e1bf0404d 100644
--- a/doc/bitbake-user-manual/bitbake-user-manual-environment-setup.rst
+++ b/doc/bitbake-user-manual/bitbake-user-manual-environment-setup.rst
@@ -660,6 +660,25 @@ In addition, the command can take the following arguments:
 -  ``--setup-dir``: path to the :term:`Setup` to check to status for. Not
    required if :term:`BBPATH` is already configured.
 
+-  ``--url``: full URL to the buildtools SDK installer. Overrides the value
+   from the configuration file.
+
+-  ``--sha256``: SHA256 checksum of the buildtools installer. Overrides the
+   value from the configuration file.
+
+When ``url`` and ``sha256sum`` are set in the :term:`Configuration File`,
+the installer is downloaded via ``bb.fetch`` (cached in ``DL_DIR``) and its
+checksum is enforced. If no configuration is present, the script falls back
+to its built-in defaults::
+
+   "bitbake-setup": {
+       "install-buildtools": {
+           "url": "https://example.com/buildtools/x86_64-buildtools-extended-nativesdk-standalone-5.0.sh",
+           "sha256sum": "af76648b..."
+       },
+       "configurations": [ ... ]
+   }
+
 .. _ref-bbsetup-command-settings:
 
 ``bitbake-setup settings``
diff --git a/lib/bb/tests/setup.py b/lib/bb/tests/setup.py
index 5592e8196..7f07369f6 100644
--- a/lib/bb/tests/setup.py
+++ b/lib/bb/tests/setup.py
@@ -61,16 +61,31 @@ import getopt
 import sys
 import os
 
-opts, args = getopt.getopt(sys.argv[1:], "d:", ["downloads-directory="])
+opts, args = getopt.getopt(sys.argv[1:], "d:", ["downloads-directory=", "local-file="])
+installdir = None
+local_file = None
 for option, value in opts:
     if option == '-d':
         installdir = value
+    elif option == '--local-file':
+        local_file = value
+        print("install-buildtools local-file={}".format(value))
 
 print("Buildtools installed into {}".format(installdir))
-os.makedirs(installdir)
+os.makedirs(installdir, exist_ok=True)
 """
         self.add_file_to_testrepo('scripts/install-buildtools', installbuildtools, script=True)
 
+        # Dummy buildtools installer for bb.fetch testing
+        self.buildtools_dir = os.path.join(self.tempdir, "buildtools-dl")
+        os.makedirs(self.buildtools_dir)
+        self.buildtools_filename = "x86_64-buildtools-nativesdk-standalone-test.sh"
+        buildtools_filepath = os.path.join(self.buildtools_dir, self.buildtools_filename)
+        with open(buildtools_filepath, 'w') as f:
+            f.write("#!/bin/sh\necho dummy\n")
+        with open(buildtools_filepath, 'rb') as f:
+            self.buildtools_sha256 = hashlib.sha256(f.read()).hexdigest()
+
         bitbakeconfigbuild = """#!/usr/bin/env python3
 import os
 import sys
@@ -172,11 +187,15 @@ print("BBPATH is {{}}".format(os.environ["BBPATH"]))
                 "bb-layers-file-relative": ["layerE/meta-layer"],
                 "oe-fragments": ["test-fragment-2"]
             }
-        ]
+        ],
+        "install-buildtools": {
+            "url": "file://%s/%s",
+            "sha256sum": "%s"
+        }
     },
     "version": "1.0"
 }
-""" % (sources)
+""" % (sources, self.buildtools_dir, self.buildtools_filename, self.buildtools_sha256)
         os.makedirs(os.path.join(self.registrypath, os.path.dirname(name)), exist_ok=True)
         with open(os.path.join(self.registrypath, name), 'w') as f:
             f.write(config)
@@ -644,6 +663,56 @@ print("BBPATH is {{}}".format(os.environ["BBPATH"]))
     def _count_layer_backups(self, layers_path):
         return len([f for f in os.listdir(layers_path) if 'backup' in f])
 
+    def test_install_buildtools_fetch(self):
+        """Test that install-buildtools uses bb.fetch with sha256 and passes --local-file"""
+        import shutil
+
+        if 'BBPATH' in os.environ:
+            del os.environ['BBPATH']
+        os.chdir(self.tempdir)
+
+        registry_uri = "git://{};protocol=file;branch=master;rev=master".format(
+            self.registrypath)
+        self.runbbsetup(["settings", "set", "default", "registry", registry_uri])
+        self.add_file_to_testrepo('test-file', 'initial\n')
+        self.add_json_config_to_registry('test-config-bt.conf.json', 'master', 'master')
+        self.runbbsetup(["init", "--non-interactive", "test-config-bt", "gadget"])
+        setuppath = self.get_setup_path('test-config-bt', 'gadget')
+
+        # test config-driven install (url + sha256sum from config)
+        out = self.runbbsetup(["install-buildtools", "--setup-dir", setuppath])
+        self.assertIn("Buildtools installed into", out[0])
+        self.assertIn("install-buildtools local-file=", out[0])
+        self.assertTrue(os.path.exists(os.path.join(setuppath, 'buildtools')))
+
+        # test CLI overrides (use a different file to prove precedence)
+        shutil.rmtree(os.path.join(setuppath, 'buildtools'))
+        alt_filename = "alt-buildtools-test.sh"
+        alt_filepath = os.path.join(self.buildtools_dir, alt_filename)
+        with open(alt_filepath, 'w') as f:
+            f.write("#!/bin/sh\necho alt\n")
+        with open(alt_filepath, 'rb') as f:
+            alt_sha256 = hashlib.sha256(f.read()).hexdigest()
+        alt_url = "file://{}/{}".format(self.buildtools_dir, alt_filename)
+        out = self.runbbsetup(["install-buildtools", "--setup-dir", setuppath,
+                               "--url", alt_url,
+                               "--sha256", alt_sha256])
+        self.assertIn("Buildtools installed into", out[0])
+        self.assertIn("install-buildtools local-file=", out[0])
+        self.assertIn(alt_filename, out[0])
+
+        # test missing sha256sum is a hard error
+        shutil.rmtree(os.path.join(setuppath, 'buildtools'), ignore_errors=True)
+        with open(os.path.join(setuppath, 'config', 'config-upstream.json')) as f:
+            config_upstream = json.load(f)
+        config_upstream['data']['bitbake-setup']['install-buildtools'] = {
+            'url': 'file://{}/{}'.format(self.buildtools_dir, self.buildtools_filename)
+        }
+        with open(os.path.join(setuppath, 'config', 'config-upstream.json'), 'w') as f:
+            json.dump(config_upstream, f)
+        with self.assertRaises(bb.process.ExecutionError):
+            self.runbbsetup(["install-buildtools", "--setup-dir", setuppath, "--force"])
+
     def test_update_rebase_conflicts_strategy(self):
         """Test the --rebase-conflicts-strategy option for the update command.
 
diff --git a/setup-schema/bitbake-setup.schema.json b/setup-schema/bitbake-setup.schema.json
index 99f47f73d..afd01d5de 100644
--- a/setup-schema/bitbake-setup.schema.json
+++ b/setup-schema/bitbake-setup.schema.json
@@ -141,6 +141,27 @@
                         },
                         "additionalProperties": false
                     }
+                },
+                "install-buildtools": {
+                    "type": "object",
+                    "description": "Settings for downloading buildtools via bb.fetch",
+                    "properties": {
+                        "url": {
+                            "type": "string",
+                            "minLength": 1,
+                            "description": "Full URL to the buildtools SDK installer."
+                        },
+                        "sha256sum": {
+                            "type": "string",
+                            "minLength": 1,
+                            "description": "SHA256 checksum of the buildtools installer."
+                        }
+                    },
+                    "required": [
+                        "url",
+                        "sha256sum"
+                    ],
+                    "additionalProperties": false
                 }
             },
             "additionalProperties": false
-- 
2.34.1