[PATCH 2/9] rteval: Fix cyclictest error handling to prevent hangs

Apply the same error handling improvements from timerlat.py to cyclictest.py
to prevent hangs, crashes, and infinite loops (RHEL-140898).

1. Limit SIGINT attempts to avoid infinite loop
   - Send maximum of 5 SIGINT signals with 2s intervals
   - Force SIGKILL if process doesn't respond after 10s
   - Check and log exit status

2. Handle partial output from cyclictest gracefully
   - Wrap histogram parsing in try/except/finally
   - Catch IndexError/ValueError when parsing bucket data
   - Log warnings instead of crashing on malformed output

3. Ensure _setFinished() is always called
   - Use finally block to signal completion even on errors
   - Prevents hang in WaitForCompletion when parsing fails

4. Add exception handling in _parse_max_latencies()
   - Catch IndexError/ValueError when parsing max latencies line
   - Prevents crashes on malformed header data

5. Add exception handling for break value parsing
   - Protect against malformed breaktrace lines

6. Improve exception specificity
   - Changed bare except to catch ValueError specifically

Like timerlat, cyclictest can produce partial output if killed or crashes
mid-execution. These fixes ensure rteval handles such cases gracefully.

Assisted-by: Claude Sonnet 4.5 <[email protected]>
Signed-off-by: John Kacur <[email protected]>
---
 rteval/modules/measurement/cyclictest.py | 124 +++++++++++++++--------
 1 file changed, 79 insertions(+), 45 deletions(-)

diff --git a/rteval/modules/measurement/cyclictest.py b/rteval/modules/measurement/cyclictest.py
index c3ef4a54677d..8a5123e8af33 100644
--- a/rteval/modules/measurement/cyclictest.py
+++ b/rteval/modules/measurement/cyclictest.py
@@ -306,63 +306,97 @@ class Cyclictest(rtevalModulePrototype):
         if not line.startswith('# Max Latencies: '):
             return
 
-        line = line.split(':')[1]
-        vals = [int(x) for x in line.split()]
+        try:
+            line = line.split(':')[1]
+            vals = [int(x) for x in line.split()]
 
-        for i, core in enumerate(self.__cpus):
-            self.__cyclicdata[core].update_max(vals[i])
-            self.__cyclicdata['system'].update_max(vals[i])
+            for i, core in enumerate(self.__cpus):
+                self.__cyclicdata[core].update_max(vals[i])
+                self.__cyclicdata['system'].update_max(vals[i])
+        except (IndexError, ValueError) as e:
+            self._log(Log.WARN, f"Error parsing max latencies: {e}")
 
 
     def _WorkloadCleanup(self):
         if not self.__started:
             return
-        while self.__cyclicprocess.poll() is None:
-            self._log(Log.DEBUG, "Sending SIGINT")
+
+        # Send SIGINT and wait for graceful exit
+        # Limit attempts to avoid infinite loop (RHEL-140898)
+        max_attempts = 5
+        attempt = 0
+        while self.__cyclicprocess.poll() is None and attempt < max_attempts:
+            self._log(Log.DEBUG, f"Sending SIGINT (attempt {attempt + 1}/{max_attempts})")
             os.kill(self.__cyclicprocess.pid, signal.SIGINT)
             time.sleep(2)
+            attempt += 1
+
+        # Check if process exited
+        exit_code = self.__cyclicprocess.returncode
+        if exit_code is None:
+            # Process still running after max attempts, force kill
+            self._log(Log.WARN, "cyclictest did not respond to SIGINT, sending SIGKILL")
+            os.kill(self.__cyclicprocess.pid, signal.SIGKILL)
+            self.__cyclicprocess.wait()
+            exit_code = self.__cyclicprocess.returncode
+        elif exit_code != 0:
+            self._log(Log.WARN, f"cyclictest exited with non-zero status: {exit_code}")
+
+        # Parse histogram output - use try/finally to ensure _setFinished() is always called
+        # This prevents hangs if parsing fails due to partial output (RHEL-140898)
+        try:
+            self.__cyclicoutput.seek(0)
+            for line in self.__cyclicoutput:
+                line = bytes.decode(line)
+                if line.startswith('#'):
+                    # Catch if cyclictest stopped due to a breaktrace
+                    if line.startswith('# Break value: '):
+                        try:
+                            self.__breaktraceval = int(line.split(':')[1])
+                        except (IndexError, ValueError) as e:
+                            self._log(Log.WARN, f"Error parsing break value: {e}")
+                    elif line.startswith('# Max Latencies: '):
+                        self._parse_max_latencies(line)
+                    continue
 
-        # now parse the histogram output
-        self.__cyclicoutput.seek(0)
-        for line in self.__cyclicoutput:
-            line = bytes.decode(line)
-            if line.startswith('#'):
-                # Catch if cyclictest stopped due to a breaktrace
-                if line.startswith('# Break value: '):
-                    self.__breaktraceval = int(line.split(':')[1])
-                elif line.startswith('# Max Latencies: '):
-                    self._parse_max_latencies(line)
-                continue
-
-            # Skipping blank lines
-            if not line:
-                continue
-
-            vals = line.split()
-            if not vals:
-                # If we don't have any values, don't try parsing
-                continue
-
-            try:
-                index = int(vals[0])
-            except:
-                self._log(Log.DEBUG, f"cyclictest: unexpected output: {line}")
-                continue
+                # Skipping blank lines
+                if not line:
+                    continue
 
-            for i, core in enumerate(self.__cpus):
-                self.__cyclicdata[core].bucket(index, int(vals[i+1]))
-                self.__cyclicdata['system'].bucket(index, int(vals[i+1]))
+                vals = line.split()
+                if not vals:
+                    # If we don't have any values, don't try parsing
+                    continue
 
-        # generate statistics for each RunData object
-        for n in list(self.__cyclicdata.keys()):
-            #print "reducing self.__cyclicdata[%s]" % n
-            self.__cyclicdata[n].reduce()
-            #print self.__cyclicdata[n]
+                try:
+                    index = int(vals[0])
+                except (ValueError, IndexError):
+                    self._log(Log.DEBUG, f"cyclictest: unexpected output: {line}")
+                    continue
 
-        self._setFinished()
-        self.__started = False
-        os.close(self.__nullfp)
-        del self.__nullfp
+                for i, core in enumerate(self.__cpus):
+                    try:
+                        self.__cyclicdata[core].bucket(index, int(vals[i+1]))
+                        self.__cyclicdata['system'].bucket(index, int(vals[i+1]))
+                    except (IndexError, ValueError) as e:
+                        # Handle partial output from cyclictest (can happen on SIGINT during cleanup)
+                        self._log(Log.WARN, f"Error parsing cyclictest bucket data for core {core}: {e}")
+                        continue
+
+            # generate statistics for each RunData object
+            for n in list(self.__cyclicdata.keys()):
+                #print "reducing self.__cyclicdata[%s]" % n
+                self.__cyclicdata[n].reduce()
+                #print self.__cyclicdata[n]
+
+        except Exception as e:
+            self._log(Log.ERR, f"Error parsing cyclictest output: {e}")
+        finally:
+            # Always signal completion to avoid hangs
+            self._setFinished()
+            self.__started = False
+            os.close(self.__nullfp)
+            del self.__nullfp
 
 
     def MakeReport(self):
-- 
2.54.0