Re: [yocto-patches] [yocto-autobuilder-helper][PATCH v2 07/10] scripts: add run-vcontainer-tests for meta-virtualization

Paul Barker <[email protected]> Tue, 02 Jun 2026 11:50:18 +0100
Newsgroups	org.yoctoproject.lists.yocto-patches
Message-ID	<[email protected]>
On Mon, 2026-06-01 at 16:18 -0700, Tim Orling via lists.yoctoproject.org
wrote:
> From: Tim Orling <[email protected]>
> 
> Add scripts/run-vcontainer-tests, the test runner used by the
> vcontainer test jobs. It sources the vcontainer-tarball SDK,
> discovers the meta-virtualization pytest suite, and runs a
> configurable set of suites (vdkr, vpdmn, memres) against the
> checked-out layers. Suites can be selected per-step so the
> top-level 'vcontainer-tests' job runs the container engine
> agnostic tests:
> 
> - tests/test_container_cross_install.py
> - tests/test_container_registry_script.py
> - tests/test_vcontainer_auth_config.py
> - tests/test_multiarch_oci.py
> - tests/test_multilayer_oci.py
> 
> The 'vdkr-tests' and 'vpdmn-tests' jobs run only their respective
> suites (including memres for each container engine):
> 
> - tests/test_vdkr.py
> - tests/test_vdkr_registry.py
> 
> and
> 
> - tests/test_vpdmn.py
> 
> AI-Generated: Claude Cowork Opus 4.7
> Signed-off-by: Tim Orling <[email protected]>
> ---
>  config.json                  |  34 ++++++++
>  scripts/run-vcontainer-tests | 164 +++++++++++++++++++++++++++++++++++
>  2 files changed, 198 insertions(+)
>  create mode 100755 scripts/run-vcontainer-tests
> 
> diff --git a/config.json b/config.json
> index 7206c41..79a9d10 100644
> --- a/config.json
> +++ b/config.json
> @@ -1890,6 +1890,40 @@
>                      "install -d ${BASE_SHAREDDIR}/pub/vcontainer-tarball-latest && install -m 0755 ${BUILDDIR}/tmp/deploy/sdk/vcontainer-standalone.sh ${BASE_SHAREDDIR}/pub/vcontainer-tarball-latest/vcontainer-standalone.sh.new && mv -f ${BASE_SHAREDDIR}/pub/vcontainer-tarball-latest/vcontainer-standalone.sh.new ${BASE_SHAREDDIR}/pub/vcontainer-tarball-latest/vcontainer-standalone.sh"
>                  ]
>              }
> +        },
> +        "vcontainer-tests": {
> +            "NEEDREPOS" : ["bitbake", "meta-openembedded", "meta-virtualization"],
> +            "ADDLAYER" : [
> +                "${BUILDDIR}/../meta-openembedded/meta-oe",
> +                "${BUILDDIR}/../meta-openembedded/meta-python",
> +                "${BUILDDIR}/../meta-openembedded/meta-networking",
> +                "${BUILDDIR}/../meta-openembedded/meta-filesystems",
> +                "${BUILDDIR}/../meta-virtualization"
> +            ],
> +            "step1" : {
> +                "shortname" : "Run vcontainer pytest suite",
> +                "NOBUILDTOOLS" : 1,
> +                "NOVCONTAINER" : 1,
> +                "EXTRACMDS" : [
> +                    "VCONTAINER_SDK=${BASE_SHAREDDIR}/pub/vcontainer-tarball-latest/vcontainer-standalone.sh RESULTS_DIR=${HELPERRESULTSDIR} ${SCRIPTSDIR}/run-vcontainer-tests vcontainer ${BUILDDIR} ${BUILDDIR}/../meta-virtualization"
> +                ]
> +            },
> +            "step2" : {
> +                "shortname" : "Run vdkr pytest suite",
> +                "NOBUILDTOOLS" : 1,
> +                "NOVCONTAINER" : 1,
> +                "EXTRACMDS" : [
> +                    "VCONTAINER_SDK=${BASE_SHAREDDIR}/pub/vcontainer-tarball-latest/vcontainer-standalone.sh RESULTS_DIR=${HELPERRESULTSDIR} ${SCRIPTSDIR}/run-vcontainer-tests vdkr ${BUILDDIR} ${BUILDDIR}/../meta-virtualization"
> +                ]
> +            },
> +            "step3" : {
> +                "shortname" : "Run vpdmn pytest suite",
> +                "NOBUILDTOOLS" : 1,
> +                "NOVCONTAINER" : 1,
> +                "EXTRACMDS" : [
> +                    "VCONTAINER_SDK=${BASE_SHAREDDIR}/pub/vcontainer-tarball-latest/vcontainer-standalone.sh RESULTS_DIR=${HELPERRESULTSDIR} ${SCRIPTSDIR}/run-vcontainer-tests vpdmn ${BUILDDIR} ${BUILDDIR}/../meta-virtualization"
> +                ]
> +            }
>          }
>      },
>      "repo-defaults" : {
> diff --git a/scripts/run-vcontainer-tests b/scripts/run-vcontainer-tests
> new file mode 100755
> index 0000000..1394c7c
> --- /dev/null
> +++ b/scripts/run-vcontainer-tests
> @@ -0,0 +1,164 @@
> +#!/bin/bash
> +#
> +# SPDX-License-Identifier: GPL-2.0-only
> +#
> +# Run meta-virtualization pytest test suites against the vcontainer
> +# standalone SDK (vdkr/vpdmn) that was built in a previous bitbake
> +# step.
> +#
> +# Arguments:
> +#   $1 - suite name: one of "vcontainer", "vdkr", "vpdmn"
> +#   $2 - bitbake build directory (${BUILDDIR})
> +#   $3 - path to the meta-virtualization layer
> +#
> +# Optional environment variables:
> +#   RESULTS_DIR     - directory to copy pytest artefacts (junit xml / log) to
> +#   VCONTAINER_EXTRACT_DIR - where to extract the standalone SDK tarball
> +#                     (default: ${builddir}/vcontainer-test-extracted)
> +#   TEST_OCI_IMAGE  - path to an OCI image directory (enables vdkr/vpdmn
> +#                     import tests)
> +#   VDKR_ARCH       - target architecture for vdkr/vpdmn tests (default: x86_64)

Taking some arguments on the command line and others via environment
variables is confusing. I wonder if should use getopts to parse arguments.

> +#
> +# The script is intentionally conservative: any pytest tests that cannot run
> +# in the CI environment (those marked "slow", "network", "boot") are skipped
> +# so that the autobuilder step completes without needing network access. Those
> +# can be re-enabled by exporting META_VIRT_PYTEST_MARKERS before invocation.
> +#
> +# It is assumed that /dev/kvm is writable by the CI user running the tests,
> +# since the performance is significantly faster with 'memres'.
> +#
> +
> +set -e
> +set -u
> +set -o pipefail
> +set -x
> +
> +if [ $# -lt 3 ]; then
> +    echo "Usage: $0 <suite> <builddir> <meta-virtualization-dir>" >&2
> +    echo "  suite: vcontainer | vdkr | vpdmn" >&2
> +    exit 2
> +fi
> +
> +suite="$1"
> +builddir=$(realpath "$2")
> +metavirtdir=$(realpath "$3")
> +
> +if [ ! -d "$metavirtdir/tests" ]; then
> +    echo "ERROR: meta-virtualization tests directory not found at $metavirtdir/tests" >&2
> +    exit 1
> +fi
> +
> +# Locate the vcontainer standalone SDK tarball. Prefer an externally-built
> +# SDK passed via VCONTAINER_SDK (the autobuilder -tests jobs share the SDK
> +# produced by the separate vcontainer-tarball builder), and fall back to
> +# looking in the local build's deploy/sdk directory when running stand-alone.
> +sdk_tarball=""
> +if [ -n "${VCONTAINER_SDK:-}" ]; then
> +    if [ -f "$VCONTAINER_SDK" ]; then
> +        sdk_tarball="$VCONTAINER_SDK"
> +    else
> +        echo "ERROR: VCONTAINER_SDK=$VCONTAINER_SDK is set but not a file" >&2
> +        exit 1
> +    fi
> +fi
> +if [ -z "$sdk_tarball" ]; then
> +    sdk_tarball="$builddir/tmp/deploy/sdk/vcontainer-standalone.sh"
> +    if [ ! -f "$sdk_tarball" ]; then
> +        # Try to find any matching tarball in case naming changed (e.g. versioned)
> +        alt=$(ls -1 "$builddir"/tmp/deploy/sdk/vcontainer-*.sh 2>/dev/null | head -n1 || true)
> +        if [ -n "$alt" ]; then
> +            sdk_tarball="$alt"
> +        else
> +            echo "ERROR: vcontainer standalone SDK not found." >&2
> +            echo "       Set VCONTAINER_SDK to an existing SDK installer, or" >&2
> +            echo "       build vcontainer-tarball so $builddir/tmp/deploy/sdk/vcontainer-standalone.sh exists." >&2
> +            exit 1
> +        fi
> +    fi
> +fi
> +
> +extract_dir="${VCONTAINER_EXTRACT_DIR:-$builddir/vcontainer-test-extracted}"
> +rm -rf "$extract_dir"
> +mkdir -p "$(dirname "$extract_dir")"
> +
> +# Self-extracting installer (silent, -y agrees to license, -d picks dir)
> +"$sdk_tarball" -d "$extract_dir" -y

Patch 5 in this series added support for setting up the vcontainer
tarball for a step. I think the commit message in this patch needs to
explain why we can't use that in the steps that call
run-vcontainer-tests.

> +
> +# Prepare a Python venv so we don't pollute the worker's system packages.
> +python3 -m venv "$builddir/meta-virt-test-venv"
> +# shellcheck disable=SC1091
> +source "$builddir/meta-virt-test-venv/bin/activate"

Note that this will shadow the venv we use when running test jobs on the
autobuilder workers. That may be ok, but we need to be careful.

> +# Avoid warnings by upgrading pip; install pytest/pexpect into the venv via pip.
> +python3 -m pip install --quiet --upgrade pip setuptools wheel
> +python3 -m pip install --quiet --upgrade pytest pytest-timeout pexpect

Do we want to automatically install the latest versions of these tools,
via the network, in each build? It's better to pin versions to ensure
that upgrades are planned rather than automagical.

Also, should we store this list of dependencies in a requirements.txt
file somewhere? Perhaps tests/requirements.txt in meta-virtualization.

> +
> +# Default marker filter excludes long running / infrastructure dependent tests.
> +marker_filter="${META_VIRT_PYTEST_MARKERS:-not slow and not network and not boot and not incus and not k3s}"

Would it be better to have a positive marker, e.g. 'autobuilder', to
mark tests that we do want to run? That would allow us to maintain the
filtering solely in meta-virtualization instead of needing updates here
as well when things change.

> +
> +# Per-suite test file selection. Uses -k/-m for fine-grained filtering and
> +# keeps the CLI small for logging clarity.
> +case "$suite" in
> +    vdkr)
> +        test_files=(
> +            "tests/test_vdkr.py"
> +            "tests/test_vdkr_registry.py"
> +        )
> +        ;;
> +    vpdmn)
> +        test_files=(
> +            "tests/test_vpdmn.py"
> +        )
> +        ;;
> +    vcontainer)
> +        # Broad vcontainer/bbclass/tooling coverage that doesn't require the
> +        # vdkr/vpdmn CLI harness to be running.
> +        test_files=(
> +            "tests/test_container_cross_install.py"
> +            "tests/test_container_registry_script.py"
> +            "tests/test_vcontainer_auth_config.py"
> +            "tests/test_multiarch_oci.py"
> +            "tests/test_multilayer_oci.py"
> +        )
> +        ;;
> +    *)
> +        echo "ERROR: unknown suite '$suite' (expected vcontainer|vdkr|vpdmn)" >&2
> +        exit 2
> +        ;;
> +esac
> +
> +pytest_args=(
> +    -v
> +    --tb=short
> +    -m "$marker_filter"
> +    --vdkr-dir "$extract_dir"
> +    --junitxml="$builddir/pytest-$suite-results.xml"
> +)
> +
> +# Allow tests that consume an OCI image (import/save/load) to find one.
> +if [ -n "${TEST_OCI_IMAGE:-}" ] && [ -d "${TEST_OCI_IMAGE}" ]; then
> +    pytest_args+=(--oci-image "$TEST_OCI_IMAGE")
> +fi
> +
> +# Pass architecture through when set in the environment (default is x86_64).
> +if [ -n "${VDKR_ARCH:-}" ]; then
> +    pytest_args+=(--arch "$VDKR_ARCH")
> +fi
> +
> +cd "$metavirtdir"
> +# Don't let a single failing test kill the whole step - collect the junit
> +# report, then surface the exit code via the junit file + exit status.
> +set +e
> +python3 -m pytest "${pytest_args[@]}" "${test_files[@]}"
> +rc=$?
> +set -e
> +
> +# Copy artefacts to the results dir if one was provided.
> +if [ -n "${RESULTS_DIR:-}" ]; then
> +    mkdir -p "$RESULTS_DIR"
> +    cp -f "$builddir/pytest-$suite-results.xml" "$RESULTS_DIR/" 2>/dev/null || true
> +    if [ -f /tmp/pytest-vcontainer.log ]; then
> +        cp -f /tmp/pytest-vcontainer.log "$RESULTS_DIR/pytest-$suite.log" || true
> +    fi
> +fi
> +
> +exit $rc

-- 
Paul Barker
signature.asc (application/pgp-signature, 252 B)