[PATCH 1/2] monitoring: add NVMe OCP SMART support
Daniel Gomez <[email protected]> Thu, 09 Oct 2025 23:24:35 +0200
| Newsgroups | dev.linux.lists.kdevops |
|---|---|
| Message-ID | <[email protected]> |
From: Daniel Gomez <[email protected]> Adds monitoring support for NVMe OCP extended SMART statistics collected via nvme-cli's OCP plugin (nvme ocp smart-add-log). This enables tracking of cloud SSD-specific metrics like wear leveling, thermal throttling, and reliability counters during workflow execution. The monitor integrates with the existing monitoring framework and works with any workflow that supports it (fstests, blktests, sysbench, etc.). Results are collected as timestamped JSON and plotted via matplotlib for analysis of device health trends over time. Generated-by: Claude AI Signed-off-by: Daniel Gomez <[email protected]> --- defconfigs/configs/monitor_ocp_smart.config | 4 + kconfigs/monitors/Kconfig | 45 +++ playbooks/roles/blktests/tasks/main.yml | 21 +- .../roles/monitoring/files/plot_nvme_ocp_stats.py | 378 +++++++++++++++++++++ .../roles/monitoring/tasks/monitor_collect.yml | 30 +- .../monitoring/tasks/monitor_collect_only.yml | 8 + playbooks/roles/monitoring/tasks/monitor_run.yml | 5 + .../tasks/monitors/nvme_ocp_smart/collect.yml | 115 +++++++ .../tasks/monitors/nvme_ocp_smart/collect_only.yml | 64 ++++ .../tasks/monitors/nvme_ocp_smart/run.yml | 88 +++++ 10 files changed, 726 insertions(+), 32 deletions(-) diff --git a/defconfigs/configs/monitor_ocp_smart.config b/defconfigs/configs/monitor_ocp_smart.config new file mode 100644 index 00000000..45b83cd4 --- /dev/null +++ b/defconfigs/configs/monitor_ocp_smart.config @@ -0,0 +1,4 @@ +CONFIG_ENABLE_MONITORING=y +CONFIG_MONITOR_NVME_OCP_SMART=y +CONFIG_MONITOR_NVME_OCP_SMART_INTERVAL=120 +CONFIG_MONITOR_NVME_OCP_SMART_DEVICES="auto" \ No newline at end of file diff --git a/kconfigs/monitors/Kconfig b/kconfigs/monitors/Kconfig index bd4dc81f..e9ab8f85 100644 --- a/kconfigs/monitors/Kconfig +++ b/kconfigs/monitors/Kconfig @@ -116,6 +116,51 @@ config MONITOR_FRAGMENTATION_OUTPUT_DIR endif # MONITOR_DEVELOPMENTAL_STATS +config MONITOR_NVME_OCP_SMART + bool "Monitor NVMe OCP extended SMART statistics" + output yaml + default n + help + Enable monitoring of NVMe OCP (Open Compute Project) extended SMART + statistics using nvme-cli OCP plugin. + + Collects extended statistics from: + nvme ocp smart-add-log --output-format=json + + The collected data includes cloud SSD specific metrics: + - Advanced wear leveling data (erase counts, bad blocks) + - Extended reliability metrics (ECC errors, XOR recovery) + - Thermal throttling and power management data + - Capacitor health and power loss protection status + - PCIe error counters and endurance estimates + +config MONITOR_NVME_OCP_SMART_INTERVAL + int "NVMe OCP SMART monitoring interval (seconds)" + output yaml + default 120 + depends on MONITOR_NVME_OCP_SMART + range 60 3600 + help + How often to collect NVMe OCP SMART statistics in seconds. + Default is 120 seconds (2 minutes). + + OCP SMART data changes more slowly than standard SMART metrics, + so longer intervals are appropriate. Shorter intervals may not + provide meaningful additional data and increase system overhead. + +config MONITOR_NVME_OCP_SMART_DEVICES + string "NVMe OCP devices to monitor (space-separated)" + output yaml + default "auto" + depends on MONITOR_NVME_OCP_SMART + help + Space-separated list of OCP-compatible NVMe devices to monitor. + + Use "auto" to automatically discover OCP-compatible devices by + testing nvme ocp smart-add-log capability. + + Manual specification: "/dev/nvme0n1 /dev/nvme1n1" + # Future monitoring options can be added here # Examples: # - Memory pressure monitoring diff --git a/playbooks/roles/blktests/tasks/main.yml b/playbooks/roles/blktests/tasks/main.yml index c8329696..d8c6addd 100644 --- a/playbooks/roles/blktests/tasks/main.yml +++ b/playbooks/roles/blktests/tasks/main.yml @@ -152,6 +152,13 @@ ansible.builtin.reboot: post_reboot_delay: 10 +# Start monitoring services before running tests +- ansible.builtin.import_tasks: ../../monitoring/tasks/monitor_run.yml + when: + - kdevops_run_blktests|bool + - enable_monitoring|default(false)|bool + tags: ["blktests", "run_tests", "monitoring", "monitor_run"] + - name: Set the path to blktests workflow ansible.builtin.set_fact: blktests_workflow_target: "../workflows/blktests/" @@ -383,6 +390,13 @@ when: - kdevops_run_blktests|bool +# Stop monitoring services and collect data after running tests +- ansible.builtin.import_tasks: ../../monitoring/tasks/monitor_collect.yml + when: + - kdevops_run_blktests|bool + - enable_monitoring|default(false)|bool + tags: ["blktests", "run_tests", "monitoring", "monitor_collect"] + - name: Remove watchdog hint that tests have started ansible.builtin.file: path: "{{ blktests_workflow_target }}/.begin" @@ -393,13 +407,6 @@ - kdevops_run_blktests|bool run_once: true -# Stop monitoring services and collect data after running tests -- ansible.builtin.import_tasks: ../../monitoring/tasks/monitor_collect.yml - when: - - kdevops_run_blktests|bool - - enable_monitoring|default(false)|bool - tags: ["blktests", "run_tests", "monitoring", "monitor_collect"] - - name: Clean up our localhost results/last-run directory ansible.builtin.file: path: "{{ blktests_results_target }}/{{ item }}" diff --git a/playbooks/roles/monitoring/files/plot_nvme_ocp_stats.py b/playbooks/roles/monitoring/files/plot_nvme_ocp_stats.py new file mode 100644 index 00000000..683b14fb --- /dev/null +++ b/playbooks/roles/monitoring/files/plot_nvme_ocp_stats.py @@ -0,0 +1,378 @@ +#!/usr/bin/env python3 + +import argparse +import os +import re +import json +import matplotlib.pyplot as plt +from matplotlib.ticker import FuncFormatter +from datetime import datetime +from mpl_toolkits.axes_grid1.inset_locator import inset_axes + + +def human_format(num): + """Format numbers in human-readable format (K, M, B, T).""" + if num >= 1_000_000_000_000: + return f"{num//1_000_000_000_000:,}T" + elif num >= 1_000_000_000: + return f"{num//1_000_000_000:,}B" + elif num >= 1_000_000: + return f"{num//1_000_000:,}M" + elif num >= 1_000: + return f"{num//1_000:,}K" + return f"{num:,}" + + +def combine_hi_lo(value_dict): + """Combine hi/lo 64-bit values into single integer.""" + if isinstance(value_dict, dict) and "hi" in value_dict and "lo" in value_dict: + return (value_dict["hi"] << 32) + value_dict["lo"] + return value_dict + + +def parse_nvme_ocp_stats_file(filename): + """Parse timestamped NVMe OCP stats file.""" + timestamps = [] + data_points = [] + + with open(filename, "r") as f: + content = f.read() + + # Split by timestamp pattern: YYYY-MM-DD HH:MM:SS + entries = re.split(r"(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2})", content) + + for i in range(1, len(entries), 2): + if i + 1 < len(entries): + timestamp_str = entries[i] + json_block = entries[i + 1].strip() + + if not json_block: + continue + + try: + # Parse JSON data + data = json.loads(json_block) + + # Skip error entries + if "error" in data: + continue + + # Process the data + processed_data = {} + for key, value in data.items(): + # Handle hi/lo format values + processed_data[key] = combine_hi_lo(value) + + timestamps.append(timestamp_str) + data_points.append(processed_data) + + except json.JSONDecodeError: + # Skip malformed JSON entries + continue + + return timestamps, data_points + + +def extract_metric_series(data_points, metric_key): + """Extract time series for a specific metric.""" + values = [] + for data in data_points: + if metric_key in data: + values.append(data[metric_key]) + else: + values.append(0) # Default to 0 if metric missing + return values + + +def plot_nvme_ocp_stats(stats_files, output_file): + """Plot unified NVMe OCP stats from multiple files.""" + fig, ((ax1, ax2), (ax3, ax4)) = plt.subplots(2, 2, figsize=(16, 12)) + + # Store data for A/B comparison analysis + ab_data = [] + + # Color mapping for multiple devices/hosts + color_map = {} + color_idx = 0 + + for idx, file in enumerate(stats_files): + # Extract device and host info from filename + # Format: hostname_nvme_ocp_smart_devicename_stats.txt + base_name = os.path.splitext(os.path.basename(file))[0] + name_parts = base_name.split("_") + + if len(name_parts) >= 5: + hostname = name_parts[0] + # Device name starts after "nvme_ocp_smart_" prefix + device = "_".join(name_parts[4:-1]) if name_parts[-1] == "stats" else "_".join(name_parts[4:]) + display_name = f"{hostname}:{device}" + else: + display_name = base_name + + # Determine if this is a dev node + is_dev = "-dev" in display_name + + # Assign colors + if len(stats_files) == 2: + # A/B comparison - use different colors + color = plt.cm.tab10(idx) + else: + # Multi-device comparison + if display_name not in color_map: + color_map[display_name] = plt.cm.tab10(color_idx % 10) + color_idx += 1 + color = color_map[display_name] + + # Line style based on dev/baseline + if is_dev: + linestyle = (0, (5, 3)) # Dashed for dev + linewidth = 2.3 + alpha = 0.9 + else: + linestyle = "-" # Solid for baseline + linewidth = 2.0 + alpha = 1.0 + + timestamps, data_points = parse_nvme_ocp_stats_file(file) + + if not data_points: + continue + + # Convert to hours from start + time_hours = list(range(len(data_points))) + time_hours = [ + t * (300 / 3600) for t in time_hours + ] # Convert intervals to hours (default 5min intervals) + + # Extract key metrics + media_written = extract_metric_series( + data_points, "Physical media units written" + ) + media_read = extract_metric_series(data_points, "Physical media units read") + bad_blocks_raw = extract_metric_series( + data_points, "Bad user nand blocks - Raw" + ) + thermal_events = extract_metric_series( + data_points, "Number of Thermal throttling events" + ) + uncorrectable_errors = extract_metric_series( + data_points, "Uncorrectable read error count" + ) + erase_count_max = extract_metric_series( + data_points, "Max User data erase counts" + ) + free_blocks_pct = extract_metric_series(data_points, "Percent free blocks") + + # Plot 1: Media Wear (Written/Read) + ax1.plot( + time_hours, + media_written, + label=f"{display_name} (Written)", + color=color, + linewidth=linewidth, + linestyle=linestyle, + alpha=alpha, + ) + ax1.plot( + time_hours, + media_read, + label=f"{display_name} (Read)", + color=color, + linewidth=linewidth, + linestyle=":", + alpha=alpha * 0.8, + ) + + # Plot 2: Health Metrics + ax2.plot( + time_hours, + bad_blocks_raw, + label=f"{display_name} (Bad Blocks)", + color=color, + linewidth=linewidth, + linestyle=linestyle, + alpha=alpha, + ) + ax2_twin = ax2.twinx() + ax2_twin.plot( + time_hours, + free_blocks_pct, + label=f"{display_name} (Free %)", + color=color, + linewidth=linewidth, + linestyle="--", + alpha=alpha * 0.7, + ) + + # Plot 3: Thermal and Performance + ax3.plot( + time_hours, + thermal_events, + label=f"{display_name} (Thermal)", + color=color, + linewidth=linewidth, + linestyle=linestyle, + alpha=alpha, + marker="o" if not is_dev else "^", + markersize=4, + markevery=max(1, len(time_hours) // 10), + ) + + # Plot 4: Reliability Metrics + ax4.plot( + time_hours, + uncorrectable_errors, + label=f"{display_name} (Uncorrectable)", + color=color, + linewidth=linewidth, + linestyle=linestyle, + alpha=alpha, + ) + ax4_twin = ax4.twinx() + ax4_twin.plot( + time_hours, + erase_count_max, + label=f"{display_name} (Max Erase)", + color=color, + linewidth=linewidth, + linestyle="-.", + alpha=alpha * 0.7, + ) + + # Store data for A/B comparison + if len(stats_files) == 2: + ab_data.append( + { + "name": display_name, + "is_dev": is_dev, + "final_written": media_written[-1] if media_written else 0, + "final_read": media_read[-1] if media_read else 0, + "final_bad_blocks": bad_blocks_raw[-1] if bad_blocks_raw else 0, + "final_thermal": thermal_events[-1] if thermal_events else 0, + } + ) + + # Configure plots + title_suffix = " (A/B Comparison)" if len(stats_files) == 2 else "" + + # Plot 1: Media Wear + ax1.set_title(f"NVMe Media Wear Over Time{title_suffix}", fontsize=14) + ax1.set_xlabel("Time (hours from start)", fontsize=12) + ax1.set_ylabel("Physical Media Units", fontsize=12) + ax1.yaxis.set_major_formatter(FuncFormatter(lambda x, _: human_format(int(x)))) + ax1.grid(True, alpha=0.3) + ax1.legend(loc="best", fontsize=9) + + # Plot 2: Health Metrics + ax2.set_title(f"NVMe Health Metrics{title_suffix}", fontsize=14) + ax2.set_xlabel("Time (hours from start)", fontsize=12) + ax2.set_ylabel("Bad NAND Blocks (Raw)", fontsize=12) + ax2.grid(True, alpha=0.3) + ax2.legend(loc="upper left", fontsize=9) + + # Only configure twin axis if it was created + try: + ax2_twin.set_ylabel("Free Blocks (%)", fontsize=12) + ax2_twin.legend(loc="upper right", fontsize=9) + except NameError: + pass + + # Plot 3: Thermal Events + ax3.set_title(f"Thermal Throttling Events{title_suffix}", fontsize=14) + ax3.set_xlabel("Time (hours from start)", fontsize=12) + ax3.set_ylabel("Thermal Throttling Count", fontsize=12) + ax3.grid(True, alpha=0.3) + ax3.legend(loc="best", fontsize=9) + + # Plot 4: Reliability + ax4.set_title(f"Reliability Metrics{title_suffix}", fontsize=14) + ax4.set_xlabel("Time (hours from start)", fontsize=12) + ax4.set_ylabel("Uncorrectable Errors", fontsize=12) + ax4.grid(True, alpha=0.3) + ax4.legend(loc="upper left", fontsize=9) + + # Only configure twin axis if it was created + try: + ax4_twin.set_ylabel("Max Erase Count", fontsize=12) + ax4_twin.legend(loc="upper right", fontsize=9) + except NameError: + pass + + # Add A/B comparison summary if applicable + if len(stats_files) == 2 and len(ab_data) == 2: + baseline = next((d for d in ab_data if not d["is_dev"]), None) + dev = next((d for d in ab_data if d["is_dev"]), None) + + if baseline and dev: + # Calculate differences + written_diff = ( + ( + (dev["final_written"] - baseline["final_written"]) + / baseline["final_written"] + ) + * 100 + if baseline["final_written"] > 0 + else 0 + ) + read_diff = ( + ((dev["final_read"] - baseline["final_read"]) / baseline["final_read"]) + * 100 + if baseline["final_read"] > 0 + else 0 + ) + + diff_text = ( + f"Dev vs Baseline:\n" + f"Media Written: {written_diff:+.1f}%\n" + f"Media Read: {read_diff:+.1f}%\n" + f"Bad Blocks: {dev['final_bad_blocks'] - baseline['final_bad_blocks']:+d}\n" + f"Thermal Events: {dev['final_thermal'] - baseline['final_thermal']:+d}" + ) + + # Add comparison text to first plot + ax1.text( + 0.02, + 0.98, + diff_text, + transform=ax1.transAxes, + fontsize=10, + verticalalignment="top", + bbox=dict(boxstyle="round", facecolor="wheat", alpha=0.8), + ) + + plt.tight_layout() + fig.savefig(output_file, dpi=150, bbox_inches="tight") + print(f"Saved NVMe OCP plot to: {output_file}") + + +def main(): + parser = argparse.ArgumentParser(description="Plot NVMe OCP SMART statistics.") + parser.add_argument( + "stats_files", nargs="+", help="List of *_nvme_ocp_*_stats.txt files" + ) + parser.add_argument( + "-o", + "--output", + default="nvme-ocp-stats.png", + help="Output PNG file (default: nvme-ocp-stats.png)", + ) + args = parser.parse_args() + + # Validate input files exist + valid_files = [] + for file in args.stats_files: + if os.path.exists(file): + valid_files.append(file) + else: + print(f"Warning: File not found: {file}") + + if not valid_files: + print("Error: No valid stats files found") + return 1 + + plot_nvme_ocp_stats(valid_files, args.output) + return 0 + + +if __name__ == "__main__": + exit(main()) diff --git a/playbooks/roles/monitoring/tasks/monitor_collect.yml b/playbooks/roles/monitoring/tasks/monitor_collect.yml index 88233788..2c513d38 100644 --- a/playbooks/roles/monitoring/tasks/monitor_collect.yml +++ b/playbooks/roles/monitoring/tasks/monitor_collect.yml @@ -12,27 +12,11 @@ - monitor_developmental_stats|default(false)|bool - monitor_folio_migration|default(false)|bool -# Set monitoring results path with support for multiple workflows -- name: Set monitoring results path - ansible.builtin.set_fact: - monitoring_results_path: >- - {%- if monitoring_results_base_path is defined -%} - {{ monitoring_results_base_path }} - {%- elif kdevops_run_fstests|default(false)|bool -%} - {{ topdir_path }}/workflows/fstests/results/monitoring - {%- elif kdevops_workflow_enable_mmtests|default(false)|bool -%} - {{ topdir_path }}/workflows/mmtests/results/monitoring - {%- elif kdevops_workflow_enable_sysbench|default(false)|bool -%} - {{ topdir_path }}/workflows/sysbench/results/monitoring - {%- elif kdevops_workflow_enable_ai|default(false)|bool -%} - {{ topdir_path }}/workflows/ai/results/monitoring - {%- elif kdevops_workflow_enable_minio|default(false)|bool -%} - {{ topdir_path }}/workflows/minio/results/monitoring - {%- elif kdevops_workflow_enable_build_linux|default(false)|bool -%} - {{ topdir_path }}/workflows/build-linux/results/monitoring - {%- else -%} - {{ topdir_path }}/results/monitoring - {%- endif -%} +# Import NVMe OCP SMART collection tasks +- ansible.builtin.include_tasks: monitors/nvme_ocp_smart/collect.yml + when: + - monitor_nvme_ocp_smart|default(false)|bool + # Plot-fragmentation collection tasks - name: Check if fragmentation monitoring was started become: true @@ -111,10 +95,6 @@ - monitor_developmental_stats|default(false)|bool - monitor_memory_fragmentation|default(false)|bool -- name: Set monitoring results path - ansible.builtin.set_fact: - monitoring_results_path: "{{ monitoring_results_base_path | default(topdir_path + '/workflows/fstests/results/monitoring') }}" - - name: List fragmentation monitoring output files become: true become_method: sudo diff --git a/playbooks/roles/monitoring/tasks/monitor_collect_only.yml b/playbooks/roles/monitoring/tasks/monitor_collect_only.yml index 085f3022..f98245b2 100644 --- a/playbooks/roles/monitoring/tasks/monitor_collect_only.yml +++ b/playbooks/roles/monitoring/tasks/monitor_collect_only.yml @@ -13,6 +13,11 @@ - monitor_developmental_stats|default(false)|bool - monitor_folio_migration|default(false)|bool +# Import NVMe OCP SMART interim collection tasks +- ansible.builtin.include_tasks: monitors/nvme_ocp_smart/collect_only.yml + when: + - monitor_nvme_ocp_smart|default(false)|bool + - name: Check if fragmentation tracker is running become: true become_method: sudo @@ -271,6 +276,9 @@ {% if monitor_developmental_stats|default(false)|bool and monitor_memory_fragmentation|default(false)|bool %} - Fragmentation interim data collected {% endif %} + {% if monitor_nvme_ocp_smart|default(false)|bool %} + - NVMe OCP SMART interim data collected + {% endif %} {% if localhost_matplotlib_check is defined and localhost_matplotlib_check.rc is defined and localhost_matplotlib_check.rc == 0 %} Plot generation status: {% if folio_interim_plot_generation is defined and folio_interim_plot_generation.rc is defined and folio_interim_plot_generation.rc == 0 %} diff --git a/playbooks/roles/monitoring/tasks/monitor_run.yml b/playbooks/roles/monitoring/tasks/monitor_run.yml index e3906b08..a20b5e23 100644 --- a/playbooks/roles/monitoring/tasks/monitor_run.yml +++ b/playbooks/roles/monitoring/tasks/monitor_run.yml @@ -7,6 +7,11 @@ - monitor_developmental_stats|default(false)|bool - monitor_folio_migration|default(false)|bool +# Import NVMe OCP SMART monitoring tasks +- ansible.builtin.include_tasks: monitors/nvme_ocp_smart/run.yml + when: + - monitor_nvme_ocp_smart|default(false)|bool + # Fragmentation monitoring tasks - name: Create fragmentation scripts directory become: true diff --git a/playbooks/roles/monitoring/tasks/monitors/nvme_ocp_smart/collect.yml b/playbooks/roles/monitoring/tasks/monitors/nvme_ocp_smart/collect.yml new file mode 100644 index 00000000..284f1916 --- /dev/null +++ b/playbooks/roles/monitoring/tasks/monitors/nvme_ocp_smart/collect.yml @@ -0,0 +1,115 @@ +--- +# NVMe OCP SMART monitoring collection +- name: Check if NVMe OCP SMART monitoring is enabled + ansible.builtin.set_fact: + nvme_ocp_smart_enabled: "{{ monitor_nvme_ocp_smart|default(false)|bool }}" + +- name: Check for NVMe OCP SMART monitoring processes + become: true + become_method: sudo + ansible.builtin.shell: "ls /root/monitoring/nvme_ocp_smart_*.pid 2>/dev/null || true" + register: nvme_ocp_smart_pids + changed_when: false + when: + - enable_monitoring|default(false)|bool + +- name: Stop NVMe OCP SMART monitoring processes + become: true + become_method: sudo + ansible.builtin.shell: | + for pidfile in /root/monitoring/nvme_ocp_smart_*.pid; do + if [ -f "$pidfile" ]; then + device_name=$(basename "$pidfile" .pid | sed 's/nvme_ocp_smart_//') + pid=$(cat "$pidfile") + if ps -p $pid > /dev/null 2>&1; then + kill $pid + echo "Stopped NVMe OCP SMART monitoring for device $device_name (PID $pid)" + else + echo "NVMe OCP SMART monitoring for device $device_name (PID $pid) was not running" + fi + rm -f "$pidfile" + fi + done + register: nvme_ocp_stop_monitor + when: + - enable_monitoring|default(false)|bool + - nvme_ocp_smart_pids.stdout != "" + +- name: Collect NVMe OCP SMART monitoring data files + become: true + become_method: sudo + ansible.builtin.shell: "ls /root/monitoring/nvme_ocp_smart_*_stats.txt 2>/dev/null || true" + register: nvme_ocp_smart_data_files + changed_when: false + when: + - enable_monitoring|default(false)|bool + - nvme_ocp_smart_enabled|bool + +- name: Copy NVMe OCP SMART monitoring data to localhost + become: true + become_method: sudo + ansible.builtin.fetch: + src: "{{ item }}" + dest: "{{ monitoring_results_path }}/{{ ansible_hostname }}_{{ item | basename }}" + flat: true + validate_checksum: false + loop: "{{ nvme_ocp_smart_data_files.stdout_lines|default([]) }}" + when: + - enable_monitoring|default(false)|bool + - nvme_ocp_smart_enabled|bool + - nvme_ocp_smart_data_files.stdout != "" + +- name: Display NVMe OCP SMART data collection summary + ansible.builtin.debug: + msg: | + NVMe OCP SMART monitoring collection complete. + Data files copied: {{ nvme_ocp_smart_data_files.stdout_lines|default([])|length }} + Results directory: {{ monitoring_results_path }} + when: + - enable_monitoring|default(false)|bool + - nvme_ocp_smart_enabled|bool + - nvme_ocp_smart_data_files.stdout != "" + +- name: Find collected NVMe OCP SMART stats files for plotting + ansible.builtin.find: + paths: "{{ monitoring_results_path }}" + patterns: "*nvme_ocp_smart*_stats.txt" + file_type: file + delegate_to: localhost + run_once: true + register: nvme_ocp_stats_files + when: + - enable_monitoring|default(false)|bool + - nvme_ocp_smart_enabled|bool + +- name: Generate NVMe OCP SMART plots on localhost + ansible.builtin.command: + cmd: > + python3 {{ playbook_dir }}/roles/monitoring/files/plot_nvme_ocp_stats.py + -o {{ monitoring_results_path }}/nvme_ocp_smart_plot.png + {{ nvme_ocp_stats_files.files | map(attribute='path') | join(' ') }} + delegate_to: localhost + run_once: true + register: nvme_ocp_plot_generation + ignore_errors: true + when: + - enable_monitoring|default(false)|bool + - nvme_ocp_smart_enabled|bool + - nvme_ocp_stats_files.files is defined + - nvme_ocp_stats_files.files | length > 0 + +- name: Display NVMe OCP SMART plotting status + ansible.builtin.debug: + msg: | + NVMe OCP SMART plotting {% if nvme_ocp_plot_generation.rc == 0 %}succeeded{% else %}failed{% endif %}. + {% if nvme_ocp_plot_generation.rc == 0 %} + Plot saved to: {{ monitoring_results_path }}/nvme_ocp_smart_plot.png + {% else %} + Error: {{ nvme_ocp_plot_generation.stderr | default('Unknown error') }} + {% endif %} + delegate_to: localhost + run_once: true + when: + - enable_monitoring|default(false)|bool + - nvme_ocp_smart_enabled|bool + - nvme_ocp_plot_generation is defined diff --git a/playbooks/roles/monitoring/tasks/monitors/nvme_ocp_smart/collect_only.yml b/playbooks/roles/monitoring/tasks/monitors/nvme_ocp_smart/collect_only.yml new file mode 100644 index 00000000..39945a82 --- /dev/null +++ b/playbooks/roles/monitoring/tasks/monitors/nvme_ocp_smart/collect_only.yml @@ -0,0 +1,64 @@ +--- +# NVMe OCP SMART interim monitoring collection (without stopping monitoring) +- name: Check if NVMe OCP SMART monitoring is enabled + ansible.builtin.set_fact: + nvme_ocp_smart_enabled: "{{ monitor_nvme_ocp_smart|default(false)|bool }}" + +- name: Create NVMe OCP SMART snapshots for interim collection + become: true + become_method: sudo + ansible.builtin.shell: | + for file in /root/monitoring/nvme_ocp_smart_*_stats.txt; do + if [ -f "$file" ]; then + base_name=$(basename "$file" .txt) + cp "$file" "/root/monitoring/${base_name}_snapshot.txt" + fi + done + when: + - enable_monitoring|default(false)|bool + - nvme_ocp_smart_enabled|bool + +- name: Collect NVMe OCP SMART snapshot files + become: true + become_method: sudo + ansible.builtin.shell: "ls /root/monitoring/nvme_ocp_smart_*_snapshot.txt 2>/dev/null || true" + register: nvme_ocp_smart_snapshot_files + changed_when: false + when: + - enable_monitoring|default(false)|bool + - nvme_ocp_smart_enabled|bool + +- name: Copy NVMe OCP SMART snapshots to localhost + become: true + become_method: sudo + ansible.builtin.fetch: + src: "{{ item }}" + dest: "{{ monitoring_results_path }}/{{ ansible_hostname }}_{{ item | basename | replace('_snapshot', '_interim') }}" + flat: true + validate_checksum: false + loop: "{{ nvme_ocp_smart_snapshot_files.stdout_lines|default([]) }}" + when: + - enable_monitoring|default(false)|bool + - nvme_ocp_smart_enabled|bool + - nvme_ocp_smart_snapshot_files.stdout != "" + +- name: Clean up NVMe OCP SMART snapshot files + become: true + become_method: sudo + ansible.builtin.shell: "rm -f /root/monitoring/nvme_ocp_smart_*_snapshot.txt" + when: + - enable_monitoring|default(false)|bool + - nvme_ocp_smart_enabled|bool + - nvme_ocp_smart_snapshot_files.stdout != "" + +- name: Display NVMe OCP SMART interim collection summary + ansible.builtin.debug: + msg: | + NVMe OCP SMART interim collection complete. + Snapshot files copied: {{ nvme_ocp_smart_snapshot_files.stdout_lines|default([])|length }} + Results directory: {{ monitoring_results_path }} + Monitoring continues running... + when: + - enable_monitoring|default(false)|bool + - nvme_ocp_smart_enabled|bool + - nvme_ocp_smart_snapshot_files.stdout != "" diff --git a/playbooks/roles/monitoring/tasks/monitors/nvme_ocp_smart/run.yml b/playbooks/roles/monitoring/tasks/monitors/nvme_ocp_smart/run.yml new file mode 100644 index 00000000..1c4eb681 --- /dev/null +++ b/playbooks/roles/monitoring/tasks/monitors/nvme_ocp_smart/run.yml @@ -0,0 +1,88 @@ +--- +# NVMe OCP SMART monitoring startup +- name: Check if NVMe OCP SMART monitoring is enabled + ansible.builtin.set_fact: + nvme_ocp_smart_enabled: "{{ monitor_nvme_ocp_smart|default(false)|bool }}" + +- name: Create monitoring directory for OCP SMART logs + ansible.builtin.file: + path: /root/monitoring + state: directory + mode: '0755' + become: true + become_method: sudo + when: + - enable_monitoring|default(false)|bool + - nvme_ocp_smart_enabled|bool + +- name: Discover OCP-compatible NVMe devices + become: true + become_method: sudo + ansible.builtin.shell: | + devices=() + for dev in /dev/nvme*n1; do + if [ -e "$dev" ]; then + # Test if nvme ocp command produces any output (even with errors) + # QEMU NVMe devices may fail GUID checks but still produce useful data + if output=$(nvme ocp smart-add-log "$dev" --output-format=json 2>&1); then + # Command succeeded fully (real OCP device) + devices+=("$dev") + elif echo "$output" | grep -q "NVMe Status:Successful Completion"; then + # Command partially succeeded (QEMU NVMe with incomplete OCP support) + devices+=("$dev") + fi + fi + done + echo "${devices[@]}" + args: + executable: /bin/bash + register: discovered_ocp_devices + changed_when: false + when: + - enable_monitoring|default(false)|bool + - nvme_ocp_smart_enabled|bool + - monitor_nvme_ocp_smart_devices|default("auto") == "auto" + +- name: Set OCP device list + ansible.builtin.set_fact: + ocp_device_list: >- + {{ + discovered_ocp_devices.stdout.split() + if monitor_nvme_ocp_smart_devices|default("auto") == "auto" + else monitor_nvme_ocp_smart_devices.split() + }} + when: + - enable_monitoring|default(false)|bool + - nvme_ocp_smart_enabled|bool + +- name: Start NVMe OCP SMART monitoring for each device + become: true + become_method: sudo + ansible.builtin.shell: | + device_name=$(echo "{{ item }}" | sed 's|/dev/||g') + nohup bash -c "while true; do + echo \"\$(date +'%Y-%m-%d %H:%M:%S')\" >> /root/monitoring/nvme_ocp_smart_${device_name}_stats.txt + # Capture output even if command fails (QEMU NVMe may have GUID errors) + nvme ocp smart-add-log {{ item }} --output-format=json 2>&1 >> /root/monitoring/nvme_ocp_smart_${device_name}_stats.txt || true + echo \"\" >> /root/monitoring/nvme_ocp_smart_${device_name}_stats.txt + sleep {{ monitor_nvme_ocp_smart_interval|default(120) }} + done" > /root/monitoring/nvme_ocp_smart_${device_name}.log 2>&1 & + echo $! > /root/monitoring/nvme_ocp_smart_${device_name}.pid + loop: "{{ ocp_device_list }}" + when: + - enable_monitoring|default(false)|bool + - nvme_ocp_smart_enabled|bool + - ocp_device_list|length > 0 + +- name: Display OCP SMART monitoring startup status + ansible.builtin.debug: + msg: | + Started NVMe OCP SMART monitoring for {{ ocp_device_list|length }} devices: + {% for device in ocp_device_list %} + - {{ device }} + {% endfor %} + Monitoring interval: {{ monitor_nvme_ocp_smart_interval|default(120) }} seconds + when: + - enable_monitoring|default(false)|bool + - nvme_ocp_smart_enabled|bool + - ocp_device_list|length > 0 -- 2.51.0