[PATCH 30/34] drm/amd/display: Fix DCN6 peak bandwidth measurement overshoot

From: Wenjing Liu <[email protected]>

[Why]
The out-of-order (peak prefetch) bandwidth probe could report
bandwidth far above the physical clock ceiling. The data and
duration counters could latch on different edges of the shared
stop event, and very short measurement windows amplified refclk
tick quantization into large relative errors.

[How]
Add a dedicated counter (2) that generates the single stop event
both the data counter (1) and duration counter (4) key off of, so
they always latch on the same edge. Reject samples where the data
counter overshoots the generator's target, or where the duration
is too short to trust. Also drop the single-timing-group
restriction on the peak-BW probe.

Reviewed-by: Alvin Lee <[email protected]>
Signed-off-by: Wenjing Liu <[email protected]>
Signed-off-by: Tom Chung <[email protected]>
---
 .../display/dc/hubbub/dcn60/dcn60_hubbub.c    | 96 +++++++++++++++----
 .../amd/display/dc/hwss/dcn60/dcn60_hwseq.c   | 10 --
 2 files changed, 75 insertions(+), 31 deletions(-)

diff --git a/drivers/gpu/drm/amd/display/dc/hubbub/dcn60/dcn60_hubbub.c b/drivers/gpu/drm/amd/display/dc/hubbub/dcn60/dcn60_hubbub.c
index 820b156474e1..f1e6b2a1000d 100644
--- a/drivers/gpu/drm/amd/display/dc/hubbub/dcn60/dcn60_hubbub.c
+++ b/drivers/gpu/drm/amd/display/dc/hubbub/dcn60/dcn60_hubbub.c
@@ -1374,21 +1374,33 @@ static uint32_t hubbub60_perfmon_get_urgent_ramp_latency_ns(
  * @hubbub: pointer to the hubbub hardware instance
  *
  * Configures the performance monitoring counters to measure out-of-order
- * (peak prefetch) bandwidth using hardware counters 0, 1, and 4:
+ * (peak prefetch) bandwidth using hardware counters 0, 1, 2, and 4:
  *
  * Counter 0: count-off counter — counts response-valid events and gates the
  *            measurement window once it reaches its target value.
  * Counter 1: data counter — tracks total data received during the measurement
- *            period; generates an interrupt when measurement completes.
+ *            period.
+ * Counter 2: target-reached generator — duplicates counter 1's event/target
+ *            and generates the interrupt that stops counters 1 and 4.
  * Counter 4: duration timer — measures elapsed time in refclk cycles.
  *
  * UTM_FILTER_SEL is set to 0 so that isolation comes from OTG-vblank gating
- * rather than the silicon-broken HW filter.  The first 200 prefetch requests
- * are skipped (ramp-up), and the subsequent 200 are the measurement window.
+ * rather than the silicon-broken HW filter.
  *
  * This function configures counters only; call
  * hubbub60_perfmon_start_measuring_out_of_order_bandwidth() to enable them.
  */
+
+/* Counter0's ramp-up target and counter1's measurement-window target,
+ * packed into PERFMON_CVALUE_LOW's low/high words respectively. Tunable via
+ * windbg (ed dcn60_debug_peak_bw_ramp_size / ed dcn60_debug_peak_bw_window_size).
+ */
+static unsigned int dcn60_debug_peak_bw_ramp_size = 0x100;
+static unsigned int dcn60_debug_peak_bw_window_size = 0x2500;
+
+/* Minimum count4 (refclk ticks) for a peak-BW sample to be trusted. */
+#define DCN60_OUT_OF_ORDER_BW_MIN_DURATION_TICKS 20
+
 static void hubbub60_perfmon_arm_measuring_out_of_order_bandwidth(
 		struct hubbub *hubbub)
 {
@@ -1422,7 +1434,8 @@ static void hubbub60_perfmon_arm_measuring_out_of_order_bandwidth(
 			PERFCOUNTER_HW_STOP1_SEL, 0x1, // the stop trigger is that perfcounter meet the target CVALUE
 			PERFCOUNTER_HW_STOP2_SEL, 0x0); // ignored
 
-	/* Program counter 1 to count total data received */
+	/* Program counter 1 as a passive data accumulator, slaved to the same
+	 * external stop signal as counter 4 (generated by counter 2 below). */
 	REG_SET_9(DC_PERFMON5_PERFCOUNTER_CNTL, 0,
 			PERFCOUNTER_CNTL_SEL, 0x1, // select counter 1
 			PERFCOUNTER_EVENT_SEL, 259, // response vld
@@ -1431,13 +1444,33 @@ static void hubbub60_perfmon_arm_measuring_out_of_order_bandwidth(
 			PERFCOUNTER_HW_CNTL_SEL, 0x1, // independent mode
 			PERFCOUNTER_RUNEN_MODE, 0x0, // counter runs as long as run_enable is high
 			PERFCOUNTER_RESTART_EN, 0x0, // stop after counting is done
-			PERFCOUNTER_INT_EN, 1, // signal when the measurement is complete
+			PERFCOUNTER_INT_EN, 0, // counter 2 now generates the interrupt
 			PERFCOUNTER_ACTIVE, 0x1);
 	REG_SET_5(DC_PERFMON5_PERFCOUNTER_CNTL2, 0,
 			PERFCOUNTER_CNTL2_SEL, 0x1,
 			PERFCOUNTER_CNTOFF_SEL, 0, // start when count0 stops
 			PERFCOUNTER_COUNTED_VALUE_TYPE, 0x0, // count the accumulated value
 			PERFCOUNTER_HW_STOP1_SEL, 0x0, // ignored
+			PERFCOUNTER_HW_STOP2_SEL, 0x1); // stop via the same external signal as counter 4
+
+	/* Counter 2 is a dedicated "target reached" generator: it duplicates
+	 * counter 1's event/target and is the sole source of the completion
+	 * interrupt driving PERFMON_RUN_ENABLE_STOP_SEL. */
+	REG_SET_9(DC_PERFMON5_PERFCOUNTER_CNTL, 0,
+			PERFCOUNTER_CNTL_SEL, 0x2, // select counter 2
+			PERFCOUNTER_EVENT_SEL, 259, // response vld
+			PERFCOUNTER_CVALUE_SEL, 0x2, // use cvalue bits 31-16 (same window target as counter 1)
+			PERFCOUNTER_INC_MODE, 0x2, // count LSB level
+			PERFCOUNTER_HW_CNTL_SEL, 0x1, // independent mode
+			PERFCOUNTER_RUNEN_MODE, 0x0, // counter runs as long as run_enable is high
+			PERFCOUNTER_RESTART_EN, 0x0, // stop after counting is done
+			PERFCOUNTER_INT_EN, 1, // signal when the measurement is complete
+			PERFCOUNTER_ACTIVE, 0x1);
+	REG_SET_5(DC_PERFMON5_PERFCOUNTER_CNTL2, 0,
+			PERFCOUNTER_CNTL2_SEL, 0x2,
+			PERFCOUNTER_CNTOFF_SEL, 0, // start when count0 stops
+			PERFCOUNTER_COUNTED_VALUE_TYPE, 0x0, // count the accumulated value
+			PERFCOUNTER_HW_STOP1_SEL, 0x0, // ignored
 			PERFCOUNTER_HW_STOP2_SEL, 0x0); // the stop trigger is that perfcounter meet the target CVALUE
 
 	/* Program counter 4 to count the measuring time */
@@ -1458,21 +1491,21 @@ static void hubbub60_perfmon_arm_measuring_out_of_order_bandwidth(
 			PERFCOUNTER_HW_STOP2_SEL, 0x1); // the stop trigger is from the external 64 pairs of start/stop events
 
 	/* Program perfcounter states */
-	REG_SET_6(DC_PERFMON5_PERFCOUNTER_STATE, 0,
+	REG_SET_8(DC_PERFMON5_PERFCOUNTER_STATE, 0,
 			PERFCOUNTER_STATE_SEL0, 0x1, // independent state mode
 			PERFCOUNTER_CNT0_STATE, 0x3, // hw mode
 			PERFCOUNTER_STATE_SEL1, 0x1, // independent state mode
 			PERFCOUNTER_CNT1_STATE, 0x3, // hw mode
+			PERFCOUNTER_STATE_SEL2, 0x1, // independent state mode
+			PERFCOUNTER_CNT2_STATE, 0x3, // hw mode
 			PERFCOUNTER_STATE_SEL4, 0x1, // independent state mode
 			PERFCOUNTER_CNT4_STATE, 0x3); // hw mode
 
-	/*
-	 * The cvalue is derived based on experimental results at the lowest
-	 * clock state. Out-of-order bandwidth requires ~200 prefetch requests
-	 * to ramp up to full speed; the subsequent 200 requests form the
-	 * measurement window.
-	 */
-	REG_SET(DC_PERFMON5_PERFMON_CVALUE_LOW, 0, PERFMON_CVALUE_LOW, 200 | 200 << 16);
+	/* Ramp/window sizes are derived from experimental results at the
+	 * lowest clock state. */
+	REG_SET(DC_PERFMON5_PERFMON_CVALUE_LOW, 0, PERFMON_CVALUE_LOW,
+			(dcn60_debug_peak_bw_ramp_size & 0xFFFF)
+					| (dcn60_debug_peak_bw_window_size & 0xFFFF) << 16);
 
 	REG_SET_2(DC_PERFMON5_PERFMON_CNTL2, 0,
 			PERFMON_RUN_ENABLE_START_SEL, 0x0,
@@ -1504,28 +1537,49 @@ static void hubbub60_perfmon_start_measuring_out_of_order_bandwidth(
  * @refclk_mhz: reference clock frequency in MHz, used for duration conversion
  * @duration_ns: output parameter; receives the measured duration in nanoseconds
  *
- * Reads hardware counters 1 (data) and 4 (duration) and converts the raw
- * refclk-cycle count into bandwidth in Mbps.
+ * Reads hardware counters 1 (data), 2 (target-reached generator), and 4
+ * (duration), validates the sample, and converts the raw refclk-cycle count
+ * into bandwidth in Mbps.
  *
- * Return: out-of-order bandwidth in Mbps
+ * Return: out-of-order bandwidth in Mbps, or 0 if the sample is invalid
  */
 static uint32_t hubbub60_perfmon_get_out_of_order_bandwidth_mbps(
 		struct hubbub *hubbub, uint32_t refclk_mhz, uint32_t *duration_ns)
 {
 	struct dcn20_hubbub *hubbub2 = TO_DCN20_HUBBUB(hubbub);
-	uint32_t count0 = 0, count1 = 0, count4 = 0,
+	uint32_t count1 = 0, count2 = 0, count4 = 0,
 			out_of_order_bandwidth_mbps = 0, measuring_duration_ns = 0;
 	struct fixed31_32 temp;
 
-	REG_SET(DC_PERFMON5_PERFMON_HI, 0, PERFMON_READ_SEL, 0x0);
-	REG_GET(DC_PERFMON5_PERFMON_LOW, PERFMON_LOW, &count0);
-
 	REG_SET(DC_PERFMON5_PERFMON_HI, 0, PERFMON_READ_SEL, 0x1);
 	REG_GET(DC_PERFMON5_PERFMON_LOW, PERFMON_LOW, &count1);
 
+	REG_SET(DC_PERFMON5_PERFMON_HI, 0, PERFMON_READ_SEL, 0x2);
+	REG_GET(DC_PERFMON5_PERFMON_LOW, PERFMON_LOW, &count2);
+
 	REG_SET(DC_PERFMON5_PERFMON_HI, 0, PERFMON_READ_SEL, 0x4);
 	REG_GET(DC_PERFMON5_PERFMON_LOW, PERFMON_LOW, &count4);
 
+	/* count1 should never exceed count2's target, doing so means it
+	 * missed the shared stop edge and the sample is corrupted. */
+	if (count1 > count2 + (count2 / 20)) {
+		DC_LOG_WARNING("hubbub60_perfmon_get_out_of_order_bandwidth_mbps: "
+				"count1=%u exceeds count2=%u target, discarding corrupted sample\n",
+				count1, count2);
+		if (duration_ns)
+			*duration_ns = 0;
+		return 0;
+	}
+
+	if (count4 < DCN60_OUT_OF_ORDER_BW_MIN_DURATION_TICKS) {
+		DC_LOG_WARNING("hubbub60_perfmon_get_out_of_order_bandwidth_mbps: "
+				"count4=%u below minimum %u ticks, discarding sample\n",
+				count4, DCN60_OUT_OF_ORDER_BW_MIN_DURATION_TICKS);
+		if (duration_ns)
+			*duration_ns = 0;
+		return 0;
+	}
+
 	if (refclk_mhz == 0)
 		return 0;
 
diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dcn60/dcn60_hwseq.c b/drivers/gpu/drm/amd/display/dc/hwss/dcn60/dcn60_hwseq.c
index 14adf1a3a1e0..b9a046584740 100644
--- a/drivers/gpu/drm/amd/display/dc/hwss/dcn60/dcn60_hwseq.c
+++ b/drivers/gpu/drm/amd/display/dc/hwss/dcn60/dcn60_hwseq.c
@@ -904,16 +904,6 @@ static void dcn60_build_hubbub_perfmon_sequence(
 	if (probe->target_state != DC_PROBE_MEASURED || !ref_tg)
 		return;
 
-	/* Peak BW needs a single timing group. The out-of-order counter spans one
-	 * prefetch window, which is meaningless when streams in separate timing
-	 * groups have non-overlapping prefetch windows. */
-	if (probe->type == DC_PROBE_PEAK_MEM_BW) {
-		int group_size = context->stream_status[0].timing_sync_info.group_size;
-
-		if (group_size != context->stream_count)
-			return;
-	}
-
 	switch (probe->type) {
 	case DC_PROBE_PEAK_MEM_BW:
 		/* Start at the vblank edge and stop at the next vactive so the counter
-- 
2.43.0