git: cb85c4397bb3 - main - ixv: Preserve statistics across resets

The branch main has been updated by kbowling:

URL: https://cgit.FreeBSD.org/src/commit/?id=cb85c4397bb3f57e60ab20c239b9339c4b752412

commit cb85c4397bb3f57e60ab20c239b9339c4b752412
Author:     Kevin Bowling <[email protected]>
AuthorDate: 2026-08-10 15:12:24 +0000
Commit:     Kevin Bowling <[email protected]>
CommitDate: 2026-08-16 01:23:22 +0000

    ixv: Preserve statistics across resets
    
    The VF statistics registers are free running and are not cleared on
    read.  The existing code records attach time bases and pre-reset totals,
    but never uses either when publishing counters.  It instead replaces
    the low hardware bits directly, so counters can inherit pre-attach
    traffic or jump backward after a reset.
    
    Accumulate modular 32- and 36-bit deltas, following DPDK, while keeping
    the software totals across planned resets.  Establish a fresh hardware
    baseline after each successful reset and invalidate the sampling epoch
    when mailbox state is lost.  Detect unsolicited PF resets explicitly so
    a reset while link is down cannot be mistaken for counter wrap.
    
    Remove the unused base and saved-reset bookkeeping.
    
    On E610, packet and octet counters remained monotonic across a VF FLR
    and a PF down/up cycle.  Traffic after each reset advanced both RX and
    TX counters.
    
    MFC after:      2 weeks
    Sponsored by:   BBOX.io
---
 sys/dev/ixgbe/if_ixv.c   | 115 ++++++++++++++++++++---------------------------
 sys/dev/ixgbe/ixgbe_vf.h |  13 +-----
 2 files changed, 50 insertions(+), 78 deletions(-)

diff --git a/sys/dev/ixgbe/if_ixv.c b/sys/dev/ixgbe/if_ixv.c
index 99d1169254c5..c88c334ee94f 100644
--- a/sys/dev/ixgbe/if_ixv.c
+++ b/sys/dev/ixgbe/if_ixv.c
@@ -149,7 +149,6 @@ static void     ixv_if_unregister_vlan(if_ctx_t, u16);
 static uint64_t ixv_if_get_counter(if_ctx_t, ift_counter);
 static bool	ixv_if_needs_restart(if_ctx_t, enum iflib_restart_event);
 
-static void     ixv_save_stats(struct ixgbe_softc *);
 static void     ixv_init_stats(struct ixgbe_softc *);
 static void     ixv_update_stats(struct ixgbe_softc *);
 static void     ixv_add_stats_sysctls(struct ixgbe_softc *);
@@ -568,8 +567,8 @@ ixv_if_attach_post(if_ctx_t ctx)
 	}
 
 	/* Do the stats setup */
-	ixv_save_stats(sc);
-	ixv_init_stats(sc);
+	if (atomic_load_acq_32(&sc->vf_mbx_ready) != 0)
+		ixv_init_stats(sc);
 	ixv_add_stats_sysctls(sc);
 
 end:
@@ -674,12 +673,14 @@ ixv_if_init(if_ctx_t ctx)
 	/* Reset VF and renegotiate mailbox API version. */
 	error = hw->mac.ops.reset_hw(hw);
 	if (error != IXGBE_SUCCESS) {
+		sc->stats.vf.initialized = false;
 		ixv_log_reset_failure(sc, error, false);
 		hw->mac.ops.stop_adapter(hw);
 		ixv_mbx_retry_failed(ctx);
 		return;
 	}
 	hw->mac.ops.start_hw(hw);
+	ixv_init_stats(sc);
 	error = ixv_negotiate_api(sc);
 	if (error) {
 		/*
@@ -725,9 +726,6 @@ ixv_if_init(if_ctx_t ctx)
 	/* Set moderation on the Link interrupt */
 	IXGBE_WRITE_REG(hw, IXGBE_VTEITR(sc->vector), IXGBE_LINK_ITR);
 
-	/* Stats init */
-	ixv_init_stats(sc);
-
 	/* Config/Enable Link */
 	error = hw->mac.ops.get_link_state(hw, &sc->link_enabled);
 	if (error) {
@@ -1294,7 +1292,7 @@ ixv_if_update_admin_status(if_ctx_t ctx)
 	struct ixgbe_softc *sc = iflib_get_softc(ctx);
 	device_t dev = iflib_get_dev(ctx);
 	if_t ifp = iflib_get_ifp(ctx);
-	bool check_link;
+	bool check_link, reset_seen;
 	s32 status;
 	uint64_t baudrate;
 
@@ -1311,13 +1309,13 @@ ixv_if_update_admin_status(if_ctx_t ctx)
 	if (sc->hw.mac.type != ixgbe_mac_E610_vf ||
 	    sc->hw.api_version != ixgbe_mbox_api_16)
 		check_link = true;
+	reset_seen = ixgbe_check_for_rst(&sc->hw, 0) == IXGBE_SUCCESS;
+	if (reset_seen)
+		sc->hw.mac.get_link_status = true;
 	if (check_link) {
 		sc->hw.mac.get_link_status = true;
 		status = ixgbe_check_link(&sc->hw, &sc->link_speed,
 		    &sc->link_up, false);
-	} else if (ixgbe_check_for_rst(&sc->hw, 0) == IXGBE_SUCCESS) {
-		/* Process an unsolicited PF reset without issuing another query. */
-		status = IXGBE_ERR_MBX;
 	} else
 		status = IXGBE_SUCCESS;
 	if (sc->hw.mac.type == ixgbe_mac_E610_vf &&
@@ -1335,6 +1333,9 @@ ixv_if_update_admin_status(if_ctx_t ctx)
 		status = IXGBE_SUCCESS;
 	} else if (status == IXGBE_SUCCESS)
 		sc->vf_link_mbx_failures = 0;
+	/* Reinitialize after an unsolicited reset, even while link is down. */
+	if (reset_seen)
+		status = IXGBE_ERR_MBX;
 
 	if (status != IXGBE_SUCCESS && sc->hw.adapter_stopped == false) {
 		/* Mailbox's Clear To Send status is lost or timeout occurred.
@@ -1370,8 +1371,11 @@ ixv_if_update_admin_status(if_ctx_t ctx)
 	    atomic_readandclear_32(&sc->vf_vlan_retry_tick) != 0)
 		ixv_vlan_retry_tick(sc);
 
-	/* Stats Update */
-	ixv_update_stats(sc);
+	/* Do not treat a PF reset as a hardware-counter wrap. */
+	if (status == IXGBE_SUCCESS)
+		ixv_update_stats(sc);
+	else
+		sc->stats.vf.initialized = false;
 } /* ixv_if_update_admin_status */
 
 
@@ -1387,7 +1391,7 @@ ixv_if_stop(if_ctx_t ctx)
 	struct ixgbe_softc *sc = iflib_get_softc(ctx);
 	struct ixgbe_hw *hw = &sc->hw;
 	if_t ifp = iflib_get_ifp(ctx);
-	bool mailbox_ready;
+	bool mailbox_ready, reset_seen;
 
 	INIT_DEBUGOUT("ixv_stop: begin\n");
 
@@ -1395,8 +1399,18 @@ ixv_if_stop(if_ctx_t ctx)
 	ixv_if_disable_intr(ctx);
 
 	mailbox_ready = atomic_load_acq_32(&sc->vf_mbx_ready) != 0;
-	if (mailbox_ready && (if_getflags(ifp) & IFF_UP) == 0)
-		hw->mac.ops.reset_hw(hw);
+	reset_seen = mailbox_ready &&
+	    ixgbe_check_for_rst(hw, 0) == IXGBE_SUCCESS;
+	if (reset_seen)
+		sc->stats.vf.initialized = false;
+	else if (mailbox_ready && sc->stats.vf.initialized)
+		ixv_update_stats(sc);
+	if (mailbox_ready && (if_getflags(ifp) & IFF_UP) == 0) {
+		if (hw->mac.ops.reset_hw(hw) == IXGBE_SUCCESS)
+			ixv_init_stats(sc);
+		else
+			sc->stats.vf.initialized = false;
+	}
 	atomic_store_rel_32(&sc->vf_mbx_ready, 0);
 	sc->vf_link_mbx_failures = 0;
 	sc->hw.adapter_stopped = false;
@@ -2290,34 +2304,11 @@ ixv_configure_ivars(struct ixgbe_softc *sc)
 	ixv_set_ivar(sc, 1, sc->vector, -1);
 } /* ixv_configure_ivars */
 
-/************************************************************************
- * ixv_save_stats
- *
- *   The VF stats registers never have a truly virgin
- *   starting point, so this routine tries to make an
- *   artificial one, marking ground zero on attach as
- *   it were.
- ************************************************************************/
-static void
-ixv_save_stats(struct ixgbe_softc *sc)
-{
-	if (sc->stats.vf.vfgprc || sc->stats.vf.vfgptc) {
-		sc->stats.vf.saved_reset_vfgprc +=
-		    sc->stats.vf.vfgprc - sc->stats.vf.base_vfgprc;
-		sc->stats.vf.saved_reset_vfgptc +=
-		    sc->stats.vf.vfgptc - sc->stats.vf.base_vfgptc;
-		sc->stats.vf.saved_reset_vfgorc +=
-		    sc->stats.vf.vfgorc - sc->stats.vf.base_vfgorc;
-		sc->stats.vf.saved_reset_vfgotc +=
-		    sc->stats.vf.vfgotc - sc->stats.vf.base_vfgotc;
-		sc->stats.vf.saved_reset_vfmprc +=
-		    sc->stats.vf.vfmprc - sc->stats.vf.base_vfmprc;
-	}
-} /* ixv_save_stats */
-
 /************************************************************************
  * ixv_init_stats
  ************************************************************************/
+#define IXV_STAT_36_MASK	0xFFFFFFFFFULL
+
 static void
 ixv_init_stats(struct ixgbe_softc *sc)
 {
@@ -2327,42 +2318,31 @@ ixv_init_stats(struct ixgbe_softc *sc)
 	sc->stats.vf.last_vfgorc = IXGBE_READ_REG(hw, IXGBE_VFGORC_LSB);
 	sc->stats.vf.last_vfgorc |=
 	    (((u64)(IXGBE_READ_REG(hw, IXGBE_VFGORC_MSB))) << 32);
+	sc->stats.vf.last_vfgorc &= IXV_STAT_36_MASK;
 
 	sc->stats.vf.last_vfgptc = IXGBE_READ_REG(hw, IXGBE_VFGPTC);
 	sc->stats.vf.last_vfgotc = IXGBE_READ_REG(hw, IXGBE_VFGOTC_LSB);
 	sc->stats.vf.last_vfgotc |=
 	    (((u64)(IXGBE_READ_REG(hw, IXGBE_VFGOTC_MSB))) << 32);
+	sc->stats.vf.last_vfgotc &= IXV_STAT_36_MASK;
 
 	sc->stats.vf.last_vfmprc = IXGBE_READ_REG(hw, IXGBE_VFMPRC);
-
-	sc->stats.vf.base_vfgprc = sc->stats.vf.last_vfgprc;
-	sc->stats.vf.base_vfgorc = sc->stats.vf.last_vfgorc;
-	sc->stats.vf.base_vfgptc = sc->stats.vf.last_vfgptc;
-	sc->stats.vf.base_vfgotc = sc->stats.vf.last_vfgotc;
-	sc->stats.vf.base_vfmprc = sc->stats.vf.last_vfmprc;
+	sc->stats.vf.initialized = true;
 } /* ixv_init_stats */
 
-#define UPDATE_STAT_32(reg, last, count)                \
-{                                                       \
-	u32 current = IXGBE_READ_REG(hw, reg);          \
-	if (current < last)                             \
-		count += 0x100000000LL;                 \
-	last = current;                                 \
-	count &= 0xFFFFFFFF00000000LL;                  \
-	count |= current;                               \
-}
+#define UPDATE_STAT_32(reg, last, count) do {                         \
+	u32 current = IXGBE_READ_REG(hw, reg);                         \
+	count += (u32)(current - (u32)last);                            \
+	last = current;                                                 \
+} while (0)
 
-#define UPDATE_STAT_36(lsb, msb, last, count)           \
-{                                                       \
-	u64 cur_lsb = IXGBE_READ_REG(hw, lsb);          \
-	u64 cur_msb = IXGBE_READ_REG(hw, msb);          \
-	u64 current = ((cur_msb << 32) | cur_lsb);      \
-	if (current < last)                             \
-		count += 0x1000000000LL;                \
-	last = current;                                 \
-	count &= 0xFFFFFFF000000000LL;                  \
-	count |= current;                               \
-}
+#define UPDATE_STAT_36(lsb, msb, last, count) do {                    \
+	u64 current = IXGBE_READ_REG(hw, lsb);                         \
+	current |= (u64)IXGBE_READ_REG(hw, msb) << 32;                 \
+	current &= IXV_STAT_36_MASK;                                   \
+	count += (current - last) & IXV_STAT_36_MASK;                  \
+	last = current;                                                 \
+} while (0)
 
 /************************************************************************
  * ixv_update_stats - Update the board statistics counters.
@@ -2373,6 +2353,9 @@ ixv_update_stats(struct ixgbe_softc *sc)
 	struct ixgbe_hw *hw = &sc->hw;
 	struct ixgbevf_hw_stats *stats = &sc->stats.vf;
 
+	if (!stats->initialized)
+		return;
+
 	UPDATE_STAT_32(IXGBE_VFGPRC, sc->stats.vf.last_vfgprc,
 	    sc->stats.vf.vfgprc);
 	UPDATE_STAT_32(IXGBE_VFGPTC, sc->stats.vf.last_vfgptc,
diff --git a/sys/dev/ixgbe/ixgbe_vf.h b/sys/dev/ixgbe/ixgbe_vf.h
index 7f5a8c2e85cf..b55bb745aa95 100644
--- a/sys/dev/ixgbe/ixgbe_vf.h
+++ b/sys/dev/ixgbe/ixgbe_vf.h
@@ -90,12 +90,6 @@
 
 
 struct ixgbevf_hw_stats {
-	u64 base_vfgprc;
-	u64 base_vfgptc;
-	u64 base_vfgorc;
-	u64 base_vfgotc;
-	u64 base_vfmprc;
-
 	u64 last_vfgprc;
 	u64 last_vfgptc;
 	u64 last_vfgorc;
@@ -107,12 +101,7 @@ struct ixgbevf_hw_stats {
 	u64 vfgorc;
 	u64 vfgotc;
 	u64 vfmprc;
-
-	u64 saved_reset_vfgprc;
-	u64 saved_reset_vfgptc;
-	u64 saved_reset_vfgorc;
-	u64 saved_reset_vfgotc;
-	u64 saved_reset_vfmprc;
+	bool initialized;
 };
 
 s32 ixgbe_init_ops_vf(struct ixgbe_hw *hw);