[RFC PATCH 2/7] mm/damon/perf: implement observe API and per-CPU statistics engine

From: Kunwu Chan <[email protected]>

Implement the observe_*() core: per-CPU monotonic counters, the
event-lifecycle hooks with lazily-allocated per-event per-CPU state
alongside the global per-CPU stage, NMI-safe sample classification,
ring enqueue/dequeue/peak, match/miss/update, and framework init.  The
global per-CPU stage advances monotonically so a late create/bind/
enable cannot regress a CPU that is already RUNNING.

damon_perf_observe_sample() records the execution context
(process/softirq/hardirq/NMI) so overflow callbacks can be verified to
fire in the expected context.

Co-developed-by: Lian Wang <[email protected]>
Signed-off-by: Lian Wang <[email protected]>
Signed-off-by: Kunwu Chan <[email protected]>
---
 mm/damon/perf/Makefile |   2 +
 mm/damon/perf/stats.c  | 295 +++++++++++++++++++++++++++++++++++++++++
 2 files changed, 297 insertions(+)
 create mode 100644 mm/damon/perf/stats.c

diff --git a/mm/damon/perf/Makefile b/mm/damon/perf/Makefile
index cc0d4f1d1d28..5c46d3da7ef8 100644
--- a/mm/damon/perf/Makefile
+++ b/mm/damon/perf/Makefile
@@ -1,3 +1,5 @@
 # SPDX-License-Identifier: GPL-2.0
 
 # Observability: per-CPU counters, tracepoints, debugfs perf_stats
+obj-$(CONFIG_DAMON_PERF_OBSERVE)	+= damon-perf.o
+damon-perf-objs			:= stats.o
diff --git a/mm/damon/perf/stats.c b/mm/damon/perf/stats.c
new file mode 100644
index 000000000000..a869f115bb26
--- /dev/null
+++ b/mm/damon/perf/stats.c
@@ -0,0 +1,295 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * DAMON Perf Observability — Per-CPU Statistics & Observe API
+ *
+ * All hardware-sampling backends funnel through the observe functions
+ * defined here.  Counters always increment when
+ * CONFIG_DAMON_PERF_OBSERVE=y; tracepoints are guarded by
+ * trace_*_enabled() for zero overhead when ftrace is not attached.
+ *
+ * Per-CPU counters are best-effort: individual u64 writes are atomic
+ * on 64-bit platforms, but no cross-field consistency is guaranteed.
+ * Snapshot reads may race with concurrent writers.  This is debug data
+ * — do not build policy on it.
+ */
+
+#include <linux/cpu.h>
+#include <linux/percpu.h>
+
+#include <trace/events/damon.h>
+
+#include "perf.h"
+
+/*
+ * Per-CPU statistics
+ */
+static DEFINE_PER_CPU(struct damon_perf_stats, damon_perf_stats);
+
+/*
+ * Per-CPU event state for the state machine — the maximum lifecycle
+ * stage ever reached on each CPU (any event).  Set by the NMI
+ * observe_sample() (ENABLED→RUNNING) and by lifecycle functions as a
+ * monotonic "best CPU so far".  Never reset to UNINIT — destroying one
+ * event must not hide that another event is still running on the same
+ * CPU.  Per-event per-CPU state (event->cpu_state) tracks individual
+ * event lifecycle for correctness when multiple events exist.
+ */
+static DEFINE_PER_CPU(int, damon_perf_cpu_state);
+
+/*
+ * Event lifecycle — process context (kdamond / cpuhp callbacks).
+ *
+ * Each event tracks its own per-CPU lifecycle state via
+ * event->cpu_state (allocated on first CREATED, freed on DESTROYED).
+ * The global damon_perf_cpu_state is also advanced (monotonic) so that
+ * debugfs perf_stats shows the "best" stage any event reached on each
+ * CPU.  Diagnostics are emitted via tracepoints, not dmesg.
+ */
+
+/*
+ * Advance the global per-CPU lifecycle state monotonically.  The global
+ * reflects the best stage any event ever reached on this CPU; a late
+ * create/bind/enable must not regress a CPU that is already RUNNING.
+ */
+static void damon_perf_cpu_state_advance(int cpu, int state)
+{
+	int cur = READ_ONCE(per_cpu(damon_perf_cpu_state, cpu));
+
+	if (state > cur)
+		WRITE_ONCE(per_cpu(damon_perf_cpu_state, cpu), state);
+}
+
+void damon_perf_observe_event_created(struct damon_perf_event *event, int cpu)
+{
+	if (!event->cpu_state) {
+		event->cpu_state = alloc_percpu(int);
+		if (!event->cpu_state)
+			return;
+	}
+	*per_cpu_ptr(event->cpu_state, cpu) = DAMON_PERF_STATE_CREATED;
+	damon_perf_cpu_state_advance(cpu, DAMON_PERF_STATE_CREATED);
+}
+
+void damon_perf_observe_event_bound(struct damon_perf_event *event,
+		int cpu, struct perf_event *perf_event)
+{
+	if (event->cpu_state)
+		*per_cpu_ptr(event->cpu_state, cpu) = DAMON_PERF_STATE_BOUND;
+	damon_perf_cpu_state_advance(cpu, DAMON_PERF_STATE_BOUND);
+}
+
+void damon_perf_observe_event_enabled(struct damon_perf_event *event,
+		int cpu, int state, int oncpu)
+{
+	if (event->cpu_state)
+		*per_cpu_ptr(event->cpu_state, cpu) = DAMON_PERF_STATE_ENABLED;
+	damon_perf_cpu_state_advance(cpu, DAMON_PERF_STATE_ENABLED);
+}
+
+void damon_perf_observe_event_disabled(struct damon_perf_event *event,
+		int cpu, int state)
+{
+	/* State unchanged: the event may be re-enabled later. */
+}
+
+void damon_perf_observe_event_destroyed(struct damon_perf_event *event, int cpu)
+{
+	/*
+	 * Reset only this CPU's slot.  The per-CPU array is owned by the
+	 * event as a whole and must NOT be freed here: destroyed() is
+	 * invoked from the per-CPU CPU-offline callback, so freeing the
+	 * whole array on the first offline CPU would leave every other
+	 * still-online CPU with a dangling pointer.  The array is freed
+	 * once by damon_perf_observe_event_free() at event teardown.
+	 */
+	if (event->cpu_state)
+		*per_cpu_ptr(event->cpu_state, cpu) = DAMON_PERF_STATE_UNINIT;
+}
+
+void damon_perf_observe_event_free(struct damon_perf_event *event)
+{
+	if (event->cpu_state) {
+		free_percpu(event->cpu_state);
+		event->cpu_state = NULL;
+	}
+}
+
+/*
+ * Sample observed — NMI-safe.
+ *
+ * The @cpu argument is the CPU the sample fired on; it is passed
+ * through to the tracepoint but stats are always written on the
+ * current CPU via this_cpu_ptr().
+ */
+
+void damon_perf_observe_sample(unsigned long addr, u64 data_src,
+		u64 period, int cpu, u8 reason,
+		u64 sample_flags, u64 sample_type)
+{
+	this_cpu_inc(damon_perf_stats.callback);
+
+	switch (reason) {
+	case 0:
+		this_cpu_inc(damon_perf_stats.sample_valid);
+		break;
+	case 1:
+		this_cpu_inc(damon_perf_stats.sample_null);
+		break;
+	case 2:
+		this_cpu_inc(damon_perf_stats.sample_addr_zero);
+		break;
+	case 3:
+		this_cpu_inc(damon_perf_stats.sample_kernel);
+		break;
+	case 4:
+		this_cpu_inc(damon_perf_stats.sample_invalid_phys);
+		break;
+	}
+
+	/* First callback advances state to RUNNING */
+	if (this_cpu_read(damon_perf_cpu_state) == DAMON_PERF_STATE_ENABLED)
+		this_cpu_write(damon_perf_cpu_state, DAMON_PERF_STATE_RUNNING);
+
+	/*
+	 * Record the execution context so callers can verify whether
+	 * overflow callbacks actually fire in the expected context
+	 * (e.g. NMI for IBS, process for SPE AUX drain).
+	 *
+	 * 0 = process, 1 = softirq, 2 = hardirq, 3 = NMI
+	 */
+	if (trace_damon_perf_sample_enabled())
+		trace_damon_perf_sample(addr, data_src, period, cpu, reason,
+				sample_flags, sample_type,
+				in_nmi() ? 3 : in_hardirq() ? 2 :
+				in_serving_softirq() ? 1 : 0);
+}
+
+/*
+ * Ring operations — enqueue / overflow are NMI-safe
+ */
+
+void damon_perf_observe_ring_enqueue(void)
+{
+	this_cpu_inc(damon_perf_stats.enqueue);
+}
+
+void damon_perf_observe_ring_overflow(int cpu)
+{
+	this_cpu_inc(damon_perf_stats.overflow);
+	if (trace_damon_perf_ring_overflow_enabled())
+		trace_damon_perf_ring_overflow(cpu);
+}
+
+void damon_perf_observe_ring_dequeue(int cpu)
+{
+	per_cpu_ptr(&damon_perf_stats, cpu)->dequeue++;
+}
+
+void damon_perf_observe_ring_peak(unsigned int occupancy)
+{
+	struct damon_perf_stats *st = this_cpu_ptr(&damon_perf_stats);
+
+	if (occupancy > READ_ONCE(st->ring_peak))
+		WRITE_ONCE(st->ring_peak, occupancy);
+}
+
+/*
+ * Matching — process context (kdamond drain loop)
+ */
+
+void damon_perf_observe_match(unsigned long addr, int cpu)
+{
+	per_cpu_ptr(&damon_perf_stats, cpu)->match++;
+}
+
+void damon_perf_observe_miss(unsigned long addr, int cpu, int reason)
+{
+	struct damon_perf_stats *st = per_cpu_ptr(&damon_perf_stats, cpu);
+
+	switch (reason) {
+	case DAMON_REPORT_MISS_TGID:
+		st->miss_tgid++;
+		break;
+	case DAMON_REPORT_MISS_NOREGION:
+		st->miss_region++;
+		break;
+	case DAMON_REPORT_MISS_BOUNDARY:
+		st->miss_boundary++;
+		break;
+	}
+
+	if (trace_damon_perf_report_missed_enabled())
+		trace_damon_perf_report_missed(addr, cpu, reason);
+}
+
+void damon_perf_observe_update(int cpu)
+{
+	per_cpu_ptr(&damon_perf_stats, cpu)->update++;
+}
+
+void damon_perf_observe_drain(unsigned int total, unsigned int matched)
+{
+	if (trace_damon_perf_drain_enabled())
+		trace_damon_perf_drain(total, matched);
+}
+
+/*
+ * Stats accessors (for debugfs)
+ */
+
+/*
+ * Best-effort per-CPU snapshot.  Individual u64 writes are atomic on
+ * 64-bit platforms; no cross-field consistency is guaranteed.  This is
+ * debug data only — do not build policy on it.
+ */
+void damon_perf_stats_snapshot(int cpu, struct damon_perf_stats *dst)
+{
+	struct damon_perf_stats *st = per_cpu_ptr(&damon_perf_stats, cpu);
+
+	*dst = *st;
+	dst->cpu_state = per_cpu(damon_perf_cpu_state, cpu);
+}
+
+void damon_perf_stats_aggregate(struct damon_perf_stats *dst)
+{
+	int cpu;
+
+	memset(dst, 0, sizeof(*dst));
+	dst->cpu_state = DAMON_PERF_STATE_RUNNING; /* start optimistic, take min */
+	cpus_read_lock();
+	for_each_online_cpu(cpu) {
+		struct damon_perf_stats *st = per_cpu_ptr(&damon_perf_stats, cpu);
+
+		dst->callback		+= st->callback;
+		dst->sample_valid	+= st->sample_valid;
+		dst->sample_null	+= st->sample_null;
+		dst->sample_addr_zero	+= st->sample_addr_zero;
+		dst->sample_kernel	+= st->sample_kernel;
+		dst->sample_invalid_phys += st->sample_invalid_phys;
+		dst->enqueue		+= st->enqueue;
+		dst->dequeue		+= st->dequeue;
+		dst->overflow		+= st->overflow;
+		dst->ring_peak		= max(dst->ring_peak, st->ring_peak);
+		dst->match		+= st->match;
+		dst->miss_tgid		+= st->miss_tgid;
+		dst->miss_region	+= st->miss_region;
+		dst->miss_boundary	+= st->miss_boundary;
+		dst->update		+= st->update;
+		/*
+		 * Aggregate per-CPU state: the pipeline hasn't passed a
+		 * stage until ALL CPUs have passed it, so take the min.
+		 */
+		dst->cpu_state = min_t(int, dst->cpu_state,
+				per_cpu(damon_perf_cpu_state, cpu));
+	}
+	cpus_read_unlock();
+}
+
+/*
+ * Subsystem init
+ */
+
+int damon_perf_framework_init(void)
+{
+	return 0;
+}
-- 
2.43.0