[yocto-autobuilder-helper][PATCH 05/11] scripts: add vcontainer-tarball setup, integration, and publishing

From: Tim Orling <[email protected]>

Introduce the vcontainer-tarball SDK plumbing used by container build
jobs. The vcontainer-tarball is a meta-virtualization-derived SDK
(modelled after buildtools-tarball) that ships the container build
toolchain so worker jobs do not need to rebuild it for every step.

  * scripts/utils.py: add setup_vcontainer_tarball(), and add an
    env_glob keyword argument to setup_tools_tarball() and
    enable_tools_tarball() so the vcontainer-tarball can source its
    specific environment-setup-ci file rather than the universal
    glob.
  * scripts/run-config: source the vcontainer-tarball environment
    for build-targets / cmds / test-targets / plain-cmds steps,
    gated by a new NOVCONTAINER step variable so individual steps
    (such as the dashboard indexing step) can opt out independently
    of NOBUILDTOOLS.
  * scripts/shared-repo-unpack: invoke setup_vcontainer_tarball so
    workers extract the SDK during unpack.
  * scripts/publish-artefacts: publish the vcontainer-tarball
    artefact so downstream test jobs can fetch a stable SDK.

AI-Generated: Claude Cowork Opus 4.7
Signed-off-by: Tim Orling <[email protected]>
---
 config.json                |  1 +
 scripts/publish-artefacts  |  5 +++++
 scripts/run-config         | 19 +++++++++++++++++++
 scripts/shared-repo-unpack |  1 +
 scripts/utils.py           | 22 ++++++++++++++++++----
 5 files changed, 44 insertions(+), 4 deletions(-)

diff --git a/config.json b/config.json
index 8d42bbe..ab0ad7c 100644
--- a/config.json
+++ b/config.json
@@ -1433,6 +1433,7 @@
             "step3" : {
                 "shortname" : "Populate/update dashboard site",
                 "NOBUILDTOOLS" : true,
+                "NOVCONTAINER" : true,
                 "EXTRACMDS" : ["${SCRIPTSDIR}/run-dashboard-index ${HELPERBUILDDIR}/../"]
             }
         },
diff --git a/scripts/publish-artefacts b/scripts/publish-artefacts
index e56e131..0e820e9 100755
--- a/scripts/publish-artefacts
+++ b/scripts/publish-artefacts
@@ -146,5 +146,10 @@ case "$target" in
         sha256sums $TMPDIR/deploy/images/qemux86-64
         cp -R --no-dereference --preserve=links $TMPDIR/deploy/images/qemux86-64/*qemux86* $DEST/patchtest
         ;;
+    "vcontainer-tarball")
+        mkdir -p $DEST/vcontainer-tarball
+        sha256sums $TMPDIR/deploy/sdk
+        cp -R --no-dereference --preserve=links $TMPDIR/deploy/sdk/*vcontainer* $DEST/vcontainer-tarball
+        ;;
 esac
 
diff --git a/scripts/run-config b/scripts/run-config
index e896234..0f5a26a 100755
--- a/scripts/run-config
+++ b/scripts/run-config
@@ -153,6 +153,25 @@ else:
     if args.phase == "init" and args.stepname == "buildtools":
         sys.exit(0)
 
+if jcfg:
+    vcontainer = utils.setup_vcontainer_tarball(ourconfig, args.workername, None, checkonly=True)
+    if vcontainer:
+        addentry("vcontainer", "Setup vcontainer tarball", "init")
+else:
+    # If we're executing a specific step, check whether vcontainer is disabled for it
+    vcontainer = True
+    if args.stepname in ("build-targets", "cmds", "test-targets", "plain-cmds"):
+        try:
+            vcontainer = not utils.getconfigvar("NOVCONTAINER", ourconfig, args.target, int(args.phase))
+        except ValueError:
+            # Not an integer step phase
+            pass
+
+    if vcontainer:
+        utils.setup_vcontainer_tarball(ourconfig, args.workername, args.builddir + "/../vcontainer-tarball")
+    if args.phase == "init" and args.stepname == "vcontainer":
+        sys.exit(0)
+
 extratools = utils.getconfigvar("extratools", ourconfig, args.target)
 if jcfg:
     if extratools:
diff --git a/scripts/shared-repo-unpack b/scripts/shared-repo-unpack
index 797dec6..869b214 100755
--- a/scripts/shared-repo-unpack
+++ b/scripts/shared-repo-unpack
@@ -77,6 +77,7 @@ for repo in sorted(repos.keys()):
     utils.flush()
 
 utils.setup_buildtools_tarball(ourconfig, args.workername, args.abworkdir + "/buildtools")
+utils.setup_vcontainer_tarball(ourconfig, args.workername, args.abworkdir + "/vcontainer-tarball")
 
 if "bitbake" not in repos:
     sys.exit(0)
diff --git a/scripts/utils.py b/scripts/utils.py
index 112ebc2..b020a7b 100644
--- a/scripts/utils.py
+++ b/scripts/utils.py
@@ -456,8 +456,8 @@ def sha256_file(filename):
             pass
     return method.hexdigest()
 
-def enable_tools_tarball(btdir, name):
-    btenv = glob.glob(btdir + "/environment-setup*")
+def enable_tools_tarball(btdir, name, env_glob="/environment-setup*"):
+    btenv = glob.glob(btdir + env_glob)
     print("Using %s %s" % (name, btenv))
     # We either parse or wrap all our execution calls, rock and a hard place :(
     with open(btenv[0], "r") as f:
@@ -474,6 +474,20 @@ def enable_tools_tarball(btdir, name):
                 if line in os.environ:
                     del os.environ[line]
 
+def setup_vcontainer_tarball(ourconfig, workername, vcdir, checkonly=False):
+    vctarball = None
+    if "vcontainer" in ourconfig and workername:
+        vccfg = getconfig("vcontainer", ourconfig)
+        for entry in vccfg:
+            if fnmatch.fnmatch(workername, entry):
+                vctarball = vccfg[entry]
+                break
+
+    if checkonly:
+        return vctarball
+
+    setup_tools_tarball(ourconfig, vcdir, vctarball, name="vcontainer-tarball", env_glob="/environment-setup-ci")
+
 def setup_buildtools_tarball(ourconfig, workername, btdir, checkonly=False):
     bttarball = None
     if "buildtools" in ourconfig and workername:
@@ -488,7 +502,7 @@ def setup_buildtools_tarball(ourconfig, workername, btdir, checkonly=False):
 
     setup_tools_tarball(ourconfig, btdir, bttarball)
 
-def setup_tools_tarball(ourconfig, btdir, bttarball, name="buildtools"):
+def setup_tools_tarball(ourconfig, btdir, bttarball, name="buildtools", env_glob="/environment-setup*"):
 
     btenv = None
     if bttarball:
@@ -548,7 +562,7 @@ def setup_tools_tarball(ourconfig, btdir, bttarball, name="buildtools"):
         if not os.path.exists(btdir):
             print("Extracting %s %s" % (name, bttarball))
             subprocess.check_call(["bash", btdlpath, "-d", btdir, "-y"])
-        enable_tools_tarball(btdir, name)
+        enable_tools_tarball(btdir, name, env_glob)
 
 def get_string_from_version(version, milestone=None, rc=None):
     """ Point releases finishing by 0 (e.g 4.0.0, 4.1.0) do no exists,
-- 
2.43.0