Re: [auh][PATCH 2/3] modules/steps.py: refactor compile() to use parallel per-machine builds

I understand the idea, but unfortunately I do not think we can put
this (and the following patch) on the Yocto autobuilder. The
expectation for the builder jobs is that they are expected to run only
one Yocto build at a time, and the autobuilder will:

- configure each builder to use carefully adjusted limited resources
for that one build (via the fragment in oe-core)
- start several builders on each worker machine to parallelize large
builds (e.g. a-full), but no more than a certain upper limit, and
queue them if the limit is reached on all workers

This patch breaks that expectation, as auh build is not using an
exclusive worker for itself, and shares workers with other builds.

It also isn't clear how this patch was tested and benchmarked. Did you
run a substantial upgrade job with it? Did you see a significant
speedup?

Alex


On Tue, 14 Apr 2026 at 07:32, <[email protected]> wrote:
>
> From: Chen Qi <[email protected]>
>
> Reasons:
>
> 1. Building recipes one by one consumes little build resource. In practice,
>    when building one recipe machine by machine in a serialized way, the
>    machine load is very very low. This means AUH is not making use of
>    the machines' computing resources.
> 2. Yocto project has good and mature technique to control the build to
>    not exhaust machines' resource, e.g., BB_NUMBER_THREADS, BB_PRESSURE_MAX_XXX.
>    So we really don't need to worry about AUH exhausting server resources.
>
> Technical Details:
>
> 1. Each machine gets its own build directory ($BUILDDIR-{machine}) with
>    BUILDDIR and BBPATH env vars set accordingly, and conf/ copied from the
>    original build directory. All machines compile in parallel via
>    multiprocessing.Pool, with errors collected and reported after all
>    workers complete.
>
> 2. Use original build directory for the first machine in parallel build.
>    Skip creating a separate build directory for the first machine and
>    build directly in the original BUILDDIR. Only the remaining machines
>    get their own $BUILDDIR-{machine} copies. The purpose is that if testimage
>    is enabled, then the build can just the previous results.
>
> 3. Always refresh per-machine conf/ directory before compilation. This
>    ensure anything new in the main build directory will get into new
>    build directory.
>
> 4. Add --serial-build option to use sequential per-machine compilation.
>    When --serial-build is passed, compile each machine serially using the
>    original build logic instead of the parallel multiprocessing approach.
>
> Note:
>
> After this change, AUH will use parallel build by default. The '--serial-build'
> is there if users really need to use serial build for some reason.
>
> Signed-off-by: Chen Qi <[email protected]>
> ---
>  modules/steps.py  | 96 +++++++++++++++++++++++++++++++++++++----------
>  upgrade-helper.py |  3 ++
>  2 files changed, 80 insertions(+), 19 deletions(-)
>
> diff --git a/modules/steps.py b/modules/steps.py
> index a6ec341..300ae72 100644
> --- a/modules/steps.py
> +++ b/modules/steps.py
> @@ -22,10 +22,12 @@
>  import os
>  import shutil
>  import re
> +import multiprocessing
>
>  from logging import debug as D
>  from logging import info as I
>  from logging import warning as W
> +from logging import error as E
>
>  from errors import Error, DevtoolError, CompilationError
>  from buildhistory import BuildHistory
> @@ -104,31 +106,87 @@ def devtool_upgrade(devtool, bb, git, opts, group):
>      for p in group['pkgs']:
>          _devtool_upgrade(devtool, bb, git, opts, p)
>
> -def _compile(bb, pkg, machine, workdir):
> -        try:
> -            bb.complete(pkg, machine)
> -        except Error as e:
> -            with open("{}/bitbake-output-{}.txt".format(workdir, machine), 'w') as f:
> -                f.write(e.stdout + e.stderr)
> -            for line in e.stdout.split("\n") + e.stderr.split("\n"):
> -                # version going backwards is not a real error
> -                if re.match(".* went backwards which would break package feeds .*", line):
> -                    break
> -                # 'not in COMPATIBLE_HOST/MACHINE is not a real error
> -                if re.match(".*not in COMPATIBLE.*", line):
> -                    break
> -            else:
> -                raise CompilationError()
> +def _compile_worker(args):
> +    """Worker function for multiprocess compilation. Runs in a separate process."""
> +    pkg, machine, workdir, orig_builddir, use_orig = args
> +    from utils.bitbake import Bitbake
> +
> +    if use_orig:
> +        builddir = orig_builddir
> +    else:
> +        builddir = "{}-{}".format(orig_builddir, machine)
> +        os.makedirs(builddir, exist_ok=True)
> +
> +        new_conf = os.path.join(builddir, "conf")
> +        if os.path.isdir(new_conf):
> +            shutil.rmtree(new_conf)
> +        shutil.copytree(os.path.join(orig_builddir, "conf"), new_conf)
> +
> +    os.environ["BUILDDIR"] = builddir
> +    os.environ["BBPATH"] = builddir
> +
> +    bb = Bitbake(builddir)
> +    try:
> +        bb.complete(pkg, machine)
> +        return (machine, True, None)
> +    except Error as e:
> +        with open("{}/bitbake-output-{}.txt".format(workdir, machine), 'w') as f:
> +            f.write(e.stdout + e.stderr)
> +        for line in e.stdout.split("\n") + e.stderr.split("\n"):
> +            if re.match(".* went backwards which would break package feeds .*", line):
> +                return (machine, True, None)
> +            if re.match(".*not in COMPATIBLE.*", line):
> +                return (machine, True, None)
> +        return (machine, False, e.stdout + e.stderr)
> +
> +def _compile_serial(bb, pkg, machine, workdir):
> +    """Compile a single machine serially (original behaviour)."""
> +    try:
> +        bb.complete(pkg, machine)
> +    except Error as e:
> +        with open("{}/bitbake-output-{}.txt".format(workdir, machine), 'w') as f:
> +            f.write(e.stdout + e.stderr)
> +        for line in e.stdout.split("\n") + e.stderr.split("\n"):
> +            if re.match(".* went backwards which would break package feeds .*", line):
> +                break
> +            if re.match(".*not in COMPATIBLE.*", line):
> +                break
> +        else:
> +            raise CompilationError()
>
>  def compile(devtool, bb, git, opts, group):
>      if opts['skip_compilation']:
>          W(" %s: Compilation was skipped by user choice!" % group['name'])
>          return
>
> -    for machine in opts['machines']:
> -        I(" %s: compiling upgraded version for %s ..." % (group['name'], machine))
> -        _compile(bb, " ".join([pkg_ctx['PN'] for pkg_ctx in group['pkgs']]), machine, group['workdir'])
> -        if opts['buildhistory'] and machine == opts['machines'][0]:
> +    pkg = " ".join([pkg_ctx['PN'] for pkg_ctx in group['pkgs']])
> +    machines = opts['machines']
> +
> +    if opts.get('serial_build'):
> +        for machine in machines:
> +            I(" %s: compiling upgraded version for %s ..." % (group['name'], machine))
> +            _compile_serial(bb, pkg, machine, group['workdir'])
> +            if opts['buildhistory'] and machine == machines[0]:
> +                I(" %s: Checking buildhistory ..." % group['name'])
> +                group['buildhistory'].diff()
> +    else:
> +        orig_builddir = os.environ.get("BUILDDIR", "")
> +        worker_args = [(pkg, m, group['workdir'], orig_builddir, i == 0) for i, m in enumerate(machines)]
> +
> +        I(" %s: compiling upgraded version for %s in parallel ..." % (group['name'], machines))
> +        with multiprocessing.Pool(processes=len(machines)) as pool:
> +            results = pool.map(_compile_worker, worker_args)
> +
> +        os.environ["BUILDDIR"] = orig_builddir
> +        os.environ["BBPATH"] = orig_builddir
> +
> +        failed = [(m, err) for m, ok, err in results if not ok]
> +        if failed:
> +            for m, err in failed:
> +                E(" %s: compilation failed for %s" % (group['name'], m))
> +            raise CompilationError()
> +
> +        if opts['buildhistory'] and machines:
>              I(" %s: Checking buildhistory ..." % group['name'])
>              group['buildhistory'].diff()
>
> diff --git a/upgrade-helper.py b/upgrade-helper.py
> index 2ceb722..1c05333 100755
> --- a/upgrade-helper.py
> +++ b/upgrade-helper.py
> @@ -108,6 +108,8 @@ def parse_cmdline():
>                          help="layers to include in the upgrade research")
>      parser.add_argument("--layer-dir", action="store", default='',
>                          help="the layers root directory")
> +    parser.add_argument("--serial-build", action="store_true", default=False,
> +                        help="compile for each machine serially instead of in parallel")
>      return parser.parse_args()
>
>  def parse_config_file(config_file):
> @@ -193,6 +195,7 @@ class Updater(object):
>          self.opts['author'] = "Upgrade Helper <%s>" % \
>                  settings.get('from', '[email protected]')
>          self.opts['skip_compilation'] = self.args.skip_compilation
> +        self.opts['serial_build'] = self.args.serial_build
>          self.opts['buildhistory'] = self._buildhistory_is_enabled()
>          self.opts['testimage'] = self._testimage_is_enabled()
>
> --
> 2.49.0
>