From: Chen Qi <[email protected]>
Reasons:
1. Building recipes one by one consumes little build resource. In practice,
when building one recipe machine by machine in a serialized way, the
machine load is very very low. This means AUH is not making use of
the machines' computing resources.
2. Yocto project has good and mature technique to control the build to
not exhaust machines' resource, e.g., BB_NUMBER_THREADS, BB_PRESSURE_MAX_XXX.
So we really don't need to worry about AUH exhausting server resources.
Technical Details:
1. Each machine gets its own build directory ($BUILDDIR-{machine}) with
BUILDDIR and BBPATH env vars set accordingly, and conf/ copied from the
original build directory. All machines compile in parallel via
multiprocessing.Pool, with errors collected and reported after all
workers complete.
2. Use original build directory for the first machine in parallel build.
Skip creating a separate build directory for the first machine and
build directly in the original BUILDDIR. Only the remaining machines
get their own $BUILDDIR-{machine} copies. The purpose is that if testimage
is enabled, then the build can just the previous results.
3. Always refresh per-machine conf/ directory before compilation. This
ensure anything new in the main build directory will get into new
build directory.
4. Add --serial-build option to use sequential per-machine compilation.
When --serial-build is passed, compile each machine serially using the
original build logic instead of the parallel multiprocessing approach.
Note:
After this change, AUH will use parallel build by default. The '--serial-build'
is there if users really need to use serial build for some reason.
Signed-off-by: Chen Qi <[email protected]>
---
modules/steps.py | 96 +++++++++++++++++++++++++++++++++++++----------
upgrade-helper.py | 3 ++
2 files changed, 80 insertions(+), 19 deletions(-)
diff --git a/modules/steps.py b/modules/steps.py
index a6ec341..300ae72 100644
--- a/modules/steps.py
+++ b/modules/steps.py
@@ -22,10 +22,12 @@
import os
import shutil
import re
+import multiprocessing
from logging import debug as D
from logging import info as I
from logging import warning as W
+from logging import error as E
from errors import Error, DevtoolError, CompilationError
from buildhistory import BuildHistory
@@ -104,31 +106,87 @@ def devtool_upgrade(devtool, bb, git, opts, group):
for p in group['pkgs']:
_devtool_upgrade(devtool, bb, git, opts, p)
-def _compile(bb, pkg, machine, workdir):
- try:
- bb.complete(pkg, machine)
- except Error as e:
- with open("{}/bitbake-output-{}.txt".format(workdir, machine), 'w') as f:
- f.write(e.stdout + e.stderr)
- for line in e.stdout.split("\n") + e.stderr.split("\n"):
- # version going backwards is not a real error
- if re.match(".* went backwards which would break package feeds .*", line):
- break
- # 'not in COMPATIBLE_HOST/MACHINE is not a real error
- if re.match(".*not in COMPATIBLE.*", line):
- break
- else:
- raise CompilationError()
+def _compile_worker(args):
+ """Worker function for multiprocess compilation. Runs in a separate process."""
+ pkg, machine, workdir, orig_builddir, use_orig = args
+ from utils.bitbake import Bitbake
+
+ if use_orig:
+ builddir = orig_builddir
+ else:
+ builddir = "{}-{}".format(orig_builddir, machine)
+ os.makedirs(builddir, exist_ok=True)
+
+ new_conf = os.path.join(builddir, "conf")
+ if os.path.isdir(new_conf):
+ shutil.rmtree(new_conf)
+ shutil.copytree(os.path.join(orig_builddir, "conf"), new_conf)
+
+ os.environ["BUILDDIR"] = builddir
+ os.environ["BBPATH"] = builddir
+
+ bb = Bitbake(builddir)
+ try:
+ bb.complete(pkg, machine)
+ return (machine, True, None)
+ except Error as e:
+ with open("{}/bitbake-output-{}.txt".format(workdir, machine), 'w') as f:
+ f.write(e.stdout + e.stderr)
+ for line in e.stdout.split("\n") + e.stderr.split("\n"):
+ if re.match(".* went backwards which would break package feeds .*", line):
+ return (machine, True, None)
+ if re.match(".*not in COMPATIBLE.*", line):
+ return (machine, True, None)
+ return (machine, False, e.stdout + e.stderr)
+
+def _compile_serial(bb, pkg, machine, workdir):
+ """Compile a single machine serially (original behaviour)."""
+ try:
+ bb.complete(pkg, machine)
+ except Error as e:
+ with open("{}/bitbake-output-{}.txt".format(workdir, machine), 'w') as f:
+ f.write(e.stdout + e.stderr)
+ for line in e.stdout.split("\n") + e.stderr.split("\n"):
+ if re.match(".* went backwards which would break package feeds .*", line):
+ break
+ if re.match(".*not in COMPATIBLE.*", line):
+ break
+ else:
+ raise CompilationError()
def compile(devtool, bb, git, opts, group):
if opts['skip_compilation']:
W(" %s: Compilation was skipped by user choice!" % group['name'])
return
- for machine in opts['machines']:
- I(" %s: compiling upgraded version for %s ..." % (group['name'], machine))
- _compile(bb, " ".join([pkg_ctx['PN'] for pkg_ctx in group['pkgs']]), machine, group['workdir'])
- if opts['buildhistory'] and machine == opts['machines'][0]:
+ pkg = " ".join([pkg_ctx['PN'] for pkg_ctx in group['pkgs']])
+ machines = opts['machines']
+
+ if opts.get('serial_build'):
+ for machine in machines:
+ I(" %s: compiling upgraded version for %s ..." % (group['name'], machine))
+ _compile_serial(bb, pkg, machine, group['workdir'])
+ if opts['buildhistory'] and machine == machines[0]:
+ I(" %s: Checking buildhistory ..." % group['name'])
+ group['buildhistory'].diff()
+ else:
+ orig_builddir = os.environ.get("BUILDDIR", "")
+ worker_args = [(pkg, m, group['workdir'], orig_builddir, i == 0) for i, m in enumerate(machines)]
+
+ I(" %s: compiling upgraded version for %s in parallel ..." % (group['name'], machines))
+ with multiprocessing.Pool(processes=len(machines)) as pool:
+ results = pool.map(_compile_worker, worker_args)
+
+ os.environ["BUILDDIR"] = orig_builddir
+ os.environ["BBPATH"] = orig_builddir
+
+ failed = [(m, err) for m, ok, err in results if not ok]
+ if failed:
+ for m, err in failed:
+ E(" %s: compilation failed for %s" % (group['name'], m))
+ raise CompilationError()
+
+ if opts['buildhistory'] and machines:
I(" %s: Checking buildhistory ..." % group['name'])
group['buildhistory'].diff()
diff --git a/upgrade-helper.py b/upgrade-helper.py
index 2ceb722..1c05333 100755
--- a/upgrade-helper.py
+++ b/upgrade-helper.py
@@ -108,6 +108,8 @@ def parse_cmdline():
help="layers to include in the upgrade research")
parser.add_argument("--layer-dir", action="store", default='',
help="the layers root directory")
+ parser.add_argument("--serial-build", action="store_true", default=False,
+ help="compile for each machine serially instead of in parallel")
return parser.parse_args()
def parse_config_file(config_file):
@@ -193,6 +195,7 @@ class Updater(object):
self.opts['author'] = "Upgrade Helper <%s>" % \
settings.get('from', '[email protected]')
self.opts['skip_compilation'] = self.args.skip_compilation
+ self.opts['serial_build'] = self.args.serial_build
self.opts['buildhistory'] = self._buildhistory_is_enabled()
self.opts['testimage'] = self._testimage_is_enabled()
--
2.49.0
lmpx.com only provides a reader for public news (NNTP) servers. It is not
affiliated with the servers or forums shown here and is not responsible for
the content of articles, which is written by their respective authors.