[PATCH] rteval: Fix measurement module cpuset migration and debug logging
John Kacur <[email protected]> Fri, 17 Jul 2026 18:59:37 -0400
| Newsgroups | org.kernel.vger.linux-rt-users |
|---|---|
| Message-ID | <[email protected]> |
This commit fixes two issues: 1. Cpuset migration disrupting measurement module initialization Problem: When using --cpusets, measurement modules (timerlat/cyclictest) were launched in the root cgroup, then migrated to the measurement cpuset after a 0.5s delay. This mid-startup migration disrupted per-CPU thread initialization, causing some CPUs to fail initialization. Symptoms included missing CPUs in output and zero samples collected. Solution: - Pass the measurement cpuset path to measurement modules before Start() - Use subprocess preexec_fn to move child process into cpuset before exec - Both timerlat and cyclictest now launch inside the cpuset from the start - Fix is defensive: only activates when cpusets are configured - Works correctly both with and without --cpusets flag Files changed: - rteval/__init__.py: Pass cpuset path to measurement modules - rteval/modules/measurement/timerlat.py: Launch inside cpuset via preexec_fn - rteval/modules/measurement/cyclictest.py: Launch inside cpuset via preexec_fn 2. Excessive debug logging from hackbench Problem: hackbench logs at DEBUG level every time it restarts (continuous loop), flooding debug output with thousands of identical messages. Solution: - Add 'initial' parameter to __starton() method - Only log at DEBUG level for first startup - Subsequent restarts don't spam debug log Files changed: - rteval/modules/loads/hackbench.py: Add initial parameter to reduce logging Tested with: - timerlat with cpusets (5 CPUs, 45M+ samples) - cyclictest with cpusets (5 CPUs, 15M+ samples) - timerlat without cpusets (16 CPUs, 142M+ samples) Assisted-by: Claude:claude-sonnet-4-5 Signed-off-by: John Kacur <[email protected]> --- rteval/__init__.py | 14 ++++++++++++++ rteval/modules/loads/hackbench.py | 10 ++++++---- rteval/modules/measurement/cyclictest.py | 17 ++++++++++++++++- rteval/modules/measurement/timerlat.py | 17 ++++++++++++++++- 4 files changed, 52 insertions(+), 6 deletions(-) diff --git a/rteval/__init__.py b/rteval/__init__.py index 6e49889a6d67..b6d3c39a4b6a 100644 --- a/rteval/__init__.py +++ b/rteval/__init__.py @@ -174,6 +174,20 @@ class RtEval(rtevalReport): print(f"started measurement threads on {onlinecpus} cores") print(f"Run duration: {str(self.__rtevcfg.duration)} seconds") + # Pass cpuset path to measurement modules so they can launch inside the cpuset + if self._cpuset_manager and self._cpuset_manager.measurement_cpuset: + cpuset_path = self._cpuset_manager.measurement_cpuset._cpuset_path + for (modname, mod) in self._measuremods._RtEvalModules__modules: + # Try to find the config attribute - modules use self.__cfg (name-mangled) + cfg = None + for attr in ['_cfg', f'_{mod.__class__.__name__}__cfg']: + if hasattr(mod, attr): + cfg = getattr(mod, attr) + break + if cfg: + cfg.cpuset_path = cpuset_path + self.__logger.log(Log.DEBUG, f"Set cpuset_path for {modname}: {cpuset_path}") + self._measuremods.Start() # Unleash the loads and measurement threads diff --git a/rteval/modules/loads/hackbench.py b/rteval/modules/loads/hackbench.py index 946a2adc2e57..20ebbbc4617c 100644 --- a/rteval/modules/loads/hackbench.py +++ b/rteval/modules/loads/hackbench.py @@ -110,7 +110,7 @@ class Hackbench(CommandLineLoad): self.started = False - def __starton(self, node): + def __starton(self, node, initial=False): if self.__multinodes or self.cpulist: if self.__usenumactl: args = ['numactl', '--cpunodebind', str(node)] + self.args @@ -120,7 +120,9 @@ class Hackbench(CommandLineLoad): else: args = self.args - self._log(Log.DEBUG, f"starting on node {node}: args = {args}") + # Only log initial startup at DEBUG level to avoid spam + if initial: + self._log(Log.DEBUG, f"starting on node {node}: args = {args}") p = subprocess.Popen(args, stdin=self.__nullfp, stdout=self.__out, @@ -137,7 +139,7 @@ class Hackbench(CommandLineLoad): # just do this once if not self.started: for n in self.nodes: - self.tasks[n] = self.__starton(n) + self.tasks[n] = self.__starton(n, initial=True) self.started = True return @@ -145,7 +147,7 @@ class Hackbench(CommandLineLoad): try: if self.tasks[n].poll() is not None: self.tasks[n].wait() - self.tasks[n] = self.__starton(n) + self.tasks[n] = self.__starton(n, initial=False) except OSError as e: if e.errno != errno.ENOMEM: raise e diff --git a/rteval/modules/measurement/cyclictest.py b/rteval/modules/measurement/cyclictest.py index a1bf0d2d7324..eee315d270f8 100644 --- a/rteval/modules/measurement/cyclictest.py +++ b/rteval/modules/measurement/cyclictest.py @@ -302,10 +302,25 @@ class Cyclictest(rtevalModulePrototype): if not self._logging: self.__cyclicoutput.seek(0) + # If cpuset is configured, launch process inside the cpuset + # This is critical for cyclictest - migrating it mid-startup disrupts initialization + preexec_fn = None + if hasattr(self.__cfg, 'cpuset_path') and self.__cfg.cpuset_path: + def move_to_cpuset(): + """Move child process into cpuset before exec""" + try: + cpuset_procs = os.path.join(self.__cfg.cpuset_path, 'cgroup.procs') + with open(cpuset_procs, 'w') as f: + f.write(str(os.getpid())) + except Exception: + pass # Fail silently - parent will attempt migration as fallback + preexec_fn = move_to_cpuset + self.__cyclicprocess = subprocess.Popen(self.__cmd, stdout=self.__cyclicoutput, stderr=self.__nullfp, - stdin=self.__nullfp) + stdin=self.__nullfp, + preexec_fn=preexec_fn) self.__started = True def WorkloadAlive(self): diff --git a/rteval/modules/measurement/timerlat.py b/rteval/modules/measurement/timerlat.py index 7f592ea697af..418f1ba151af 100644 --- a/rteval/modules/measurement/timerlat.py +++ b/rteval/modules/measurement/timerlat.py @@ -285,10 +285,25 @@ class Timerlat(rtevalModulePrototype): self.__timerlat_out.seek(0) self.__timerlat_err.seek(0) + # If cpuset is configured, launch process inside the cpuset + # This is critical for timerlat - migrating it mid-startup disrupts initialization + preexec_fn = None + if hasattr(self.__cfg, 'cpuset_path') and self.__cfg.cpuset_path: + def move_to_cpuset(): + """Move child process into cpuset before exec""" + try: + cpuset_procs = os.path.join(self.__cfg.cpuset_path, 'cgroup.procs') + with open(cpuset_procs, 'w') as f: + f.write(str(os.getpid())) + except Exception: + pass # Fail silently - parent will attempt migration as fallback + preexec_fn = move_to_cpuset + self.__timerlat_process = subprocess.Popen(self.__cmd, stdout=self.__timerlat_out, stderr=self.__timerlat_err, - stdin=None) + stdin=None, + preexec_fn=preexec_fn) self.__started = True def WorkloadAlive(self): -- 2.55.0