[PATCH v2 09/13] net/sxe2: optimize vectorized Tx/Rx path
[email protected] Sun, 16 Aug 2026 10:52:29 +0800
Newsgroups
org.dpdk.dev
Message-ID
<[email protected] >
From: Jie Liu <[email protected] >
This patch optimizes vectorized packet processing with improved buffer
management and unified buffer structure:
- Introduce unified Tx buffer structure:
* Add union in sxe2_tx_queue for buffer_ring/buffer_ring_vec
* Use sxe2_tx_buffer_vec for vectorized Tx path
* Eliminate runtime type checking and branching
- Refactor Tx queue reset operations:
* Extract desc ring reset to sxe2_tx_queue_desc_ring_reset
* Add sxe2_tx_queue_reset_vec for vectorized queues
* Simplify buffer initialization in vector mode
- Optimize Tx vector path mbuf release:
* Remove conditional AVX512 branching
* Use direct buffer_vec access without casting
* Simplify loop logic with consistent pattern
* Remove NULL checks after initialization validation
- Refactor Tx queue operations:
* Export sxe2_tx_buffer_ring_free as public API
* Add sxe2_tx_vec_ops_get() for vector operations
* Use operation table instead of direct function calls
- Improve VSI management:
* Initialize other_vsi_list before main VSI creation
* Ignore -EPERM errors when destroying VSI in uninit
* Set main_vsi to NULL after successful destroy
* Prevent dangling pointer references
- Enhance Tx mode function selection:
* Use rte_eth_tx_pkt_prepare_dummy for vectorized paths
* Split NEON simple/offload mode selection logic
* Clean up conditional compilation structure
Signed-off-by: Jie Liu <[email protected] >
---
drivers/net/sxe2/sxe2_queue.h | 5 +-
drivers/net/sxe2/sxe2_rx.c | 5 +-
drivers/net/sxe2/sxe2_switchdev.c | 8 +-
drivers/net/sxe2/sxe2_tx.c | 42 +++--
drivers/net/sxe2/sxe2_tx.h | 4 +
drivers/net/sxe2/sxe2_txrx.c | 19 ++-
drivers/net/sxe2/sxe2_txrx_poll.h | 2 -
drivers/net/sxe2/sxe2_txrx_vec.c | 77 +++------
drivers/net/sxe2/sxe2_txrx_vec.h | 1 +
drivers/net/sxe2/sxe2_txrx_vec_avx2.c | 10 +-
drivers/net/sxe2/sxe2_txrx_vec_avx512.c | 123 +-------------
drivers/net/sxe2/sxe2_txrx_vec_common.h | 5 +-
drivers/net/sxe2/sxe2_txrx_vec_neon.c | 215 ++++++++++++++++--------
drivers/net/sxe2/sxe2_txrx_vec_sse.c | 10 +-
drivers/net/sxe2/sxe2_vsi.c | 8 +-
15 files changed, 247 insertions(+), 287 deletions(-)
diff --git a/drivers/net/sxe2/sxe2_queue.h b/drivers/net/sxe2/sxe2_queue.h
index 10bdaf5b8d..e53a1ce852 100644
--- a/drivers/net/sxe2/sxe2_queue.h
+++ b/drivers/net/sxe2/sxe2_queue.h
@@ -62,7 +62,10 @@ struct sxe2_txq_ops {
};
struct sxe2_tx_queue {
volatile union sxe2_tx_data_desc *desc_ring;
- struct sxe2_tx_buffer *buffer_ring;
+ union {
+ struct sxe2_tx_buffer *buffer_ring;
+ struct sxe2_tx_buffer_vec *buffer_ring_vec;
+ };
volatile uint32_t *tdt_reg_addr;
uint64_t offloads;
diff --git a/drivers/net/sxe2/sxe2_rx.c b/drivers/net/sxe2/sxe2_rx.c
index d700c60083..6340ed933a 100644
--- a/drivers/net/sxe2/sxe2_rx.c
+++ b/drivers/net/sxe2/sxe2_rx.c
@@ -319,7 +319,8 @@ int32_t __rte_cold sxe2_rx_queue_setup(struct rte_eth_dev *dev,
rxq->mb_pool = mp;
}
- rxq->rx_free_thresh = rx_conf->rx_free_thresh;
+ rxq->rx_free_thresh = (rx_conf->rx_free_thresh == 0) ?
+ SXE2_DEFAULT_RX_FREE_THRESH : rx_conf->rx_free_thresh;
rxq->port_id = dev->data->port_id;
rxq->offloads = offloads;
if (offloads & RTE_ETH_RX_OFFLOAD_KEEP_CRC)
@@ -550,7 +551,7 @@ void __rte_cold sxe2_rxqs_all_stop(struct rte_eth_dev *dev)
static int32_t sxe2_monitor_callback(const uint64_t value,
const uint64_t arg[RTE_POWER_MONITOR_OPAQUE_SZ] __rte_unused)
{
- const uint64_t dd_state = rte_cpu_to_le_64(SXE2_RX_DESC_STATUS_DD_MASK);
+ const uint64_t dd_state = rte_cpu_to_le_64(SXE2_RX_DESC_STATUS_DD_SHIFT);
return (value & dd_state) == dd_state ? -1 : 0;
}
diff --git a/drivers/net/sxe2/sxe2_switchdev.c b/drivers/net/sxe2/sxe2_switchdev.c
index efb1468b91..374cc4e223 100644
--- a/drivers/net/sxe2/sxe2_switchdev.c
+++ b/drivers/net/sxe2/sxe2_switchdev.c
@@ -316,13 +316,7 @@ int32_t sxe2_switchdev_repr_private_data_init(struct rte_eth_dev *dev,
parent_adapter->repr_ctxt.repr_vf_id[repr_id].kernel_vsi_id;
repr_priv_data->repr_vf_backup_vsi_id =
parent_adapter->repr_ctxt.repr_vf_id[repr_id].dpdk_vsi_id;
-
- repr_priv_data->repr_vf_vsi_id =
- parent_adapter->repr_ctxt.repr_vf_id[repr_id].kernel_vsi_id !=
- SXE2_INVALID_VSI_ID ?
- parent_adapter->repr_ctxt.repr_vf_id[repr_id].kernel_vsi_id :
- parent_adapter->repr_ctxt.repr_vf_id[repr_id].dpdk_vsi_id;
-
+ repr_priv_data->repr_vf_vsi_id = repr_priv_data->repr_vf_primary_vsi_id;
adapter->repr_priv_data = repr_priv_data;
goto l_end;
l_free:
diff --git a/drivers/net/sxe2/sxe2_tx.c b/drivers/net/sxe2/sxe2_tx.c
index f49238ceef..94a6e9afc7 100644
--- a/drivers/net/sxe2/sxe2_tx.c
+++ b/drivers/net/sxe2/sxe2_tx.c
@@ -19,6 +19,17 @@ static void *sxe2_tx_doorbell_addr_get(struct sxe2_adapter *adapter, uint16_t qu
queue_id);
}
+static void sxe2_tx_queue_desc_ring_reset(struct sxe2_tx_queue *txq)
+{
+ uint16_t i;
+ static const union sxe2_tx_data_desc zeroed_desc = {{0}};
+
+ for (i = 0; i < txq->ring_depth; i++) {
+ txq->desc_ring[i] = zeroed_desc;
+ txq->desc_ring[i].wb.dd = rte_cpu_to_le_64(SXE2_TX_DESC_DTYPE_DESC_DONE);
+ }
+}
+
static void sxe2_tx_tail_init(struct sxe2_adapter *adapter, struct sxe2_tx_queue *txq)
{
txq->tdt_reg_addr = sxe2_tx_doorbell_addr_get(adapter, txq->queue_id);
@@ -28,20 +39,12 @@ static void sxe2_tx_tail_init(struct sxe2_adapter *adapter, struct sxe2_tx_queue
void __rte_cold sxe2_tx_queue_reset(struct sxe2_tx_queue *txq)
{
uint16_t prev, i;
- volatile union sxe2_tx_data_desc *txd;
- static const union sxe2_tx_data_desc zeroed_desc = {{0}};
struct sxe2_tx_buffer *tx_buffer = txq->buffer_ring;
- for (i = 0; i < txq->ring_depth; i++)
- txq->desc_ring[i] = zeroed_desc;
+ sxe2_tx_queue_desc_ring_reset(txq);
prev = txq->ring_depth - 1;
for (i = 0; i < txq->ring_depth; i++) {
- txd = &txq->desc_ring[i];
- if (txd == NULL)
- continue;
-
- txd->wb.dd = rte_cpu_to_le_64(SXE2_TX_DESC_DTYPE_DESC_DONE);
tx_buffer[i].mbuf = NULL;
tx_buffer[i].last_id = i;
tx_buffer[prev].next_id = i;
@@ -56,6 +59,21 @@ void __rte_cold sxe2_tx_queue_reset(struct sxe2_tx_queue *txq)
txq->next_rs = txq->rs_thresh - 1;
}
+void __rte_cold sxe2_tx_queue_reset_vec(struct sxe2_tx_queue *txq)
+{
+ sxe2_tx_queue_desc_ring_reset(txq);
+
+ memset(txq->buffer_ring, 0,
+ sizeof(struct sxe2_tx_buffer) * txq->ring_depth);
+
+ txq->desc_used_num = 0;
+ txq->desc_free_num = txq->ring_depth - 1;
+ txq->next_use = 0;
+ txq->next_clean = txq->ring_depth - 1;
+ txq->next_dd = txq->rs_thresh - 1;
+ txq->next_rs = txq->rs_thresh - 1;
+}
+
void __rte_cold sxe2_tx_queue_mbufs_release(struct sxe2_tx_queue *txq)
{
uint32_t i;
@@ -70,10 +88,12 @@ void __rte_cold sxe2_tx_queue_mbufs_release(struct sxe2_tx_queue *txq)
}
}
-static void sxe2_tx_buffer_ring_free(struct sxe2_tx_queue *txq)
+void __rte_cold sxe2_tx_buffer_ring_free(struct sxe2_tx_queue *txq)
{
- if (txq != NULL && txq->buffer_ring != NULL)
+ if (txq != NULL && txq->buffer_ring != NULL) {
rte_free(txq->buffer_ring);
+ txq->buffer_ring = NULL;
+ }
}
const struct sxe2_txq_ops sxe2_default_txq_ops = {
diff --git a/drivers/net/sxe2/sxe2_tx.h b/drivers/net/sxe2/sxe2_tx.h
index f4823126b3..bc5ff1c2bc 100644
--- a/drivers/net/sxe2/sxe2_tx.h
+++ b/drivers/net/sxe2/sxe2_tx.h
@@ -9,6 +9,10 @@
void __rte_cold sxe2_tx_queue_reset(struct sxe2_tx_queue *txq);
+void __rte_cold sxe2_tx_queue_reset_vec(struct sxe2_tx_queue *txq);
+
+void __rte_cold sxe2_tx_buffer_ring_free(struct sxe2_tx_queue *txq);
+
int32_t __rte_cold sxe2_tx_queue_start(struct rte_eth_dev *dev, uint16_t queue_id);
void sxe2_tx_queue_mbufs_release(struct sxe2_tx_queue *txq);
diff --git a/drivers/net/sxe2/sxe2_txrx.c b/drivers/net/sxe2/sxe2_txrx.c
index 79870866d1..d27d2ce630 100644
--- a/drivers/net/sxe2/sxe2_txrx.c
+++ b/drivers/net/sxe2/sxe2_txrx.c
@@ -358,7 +358,7 @@ void sxe2_tx_mode_func_set(struct rte_eth_dev *dev)
}
if (tx_mode_flags & SXE2_TX_MODE_VEC_SET_MASK) {
- dev->tx_pkt_prepare = NULL;
+ dev->tx_pkt_prepare = rte_eth_tx_pkt_prepare_dummy;
#ifdef RTE_ARCH_X86
if (tx_mode_flags & SXE2_TX_MODE_VEC_AVX512) {
#ifdef CC_AVX512_SUPPORT
@@ -386,21 +386,25 @@ void sxe2_tx_mode_func_set(struct rte_eth_dev *dev)
}
#elif defined(RTE_ARCH_ARM64)
if (tx_mode_flags & SXE2_TX_MODE_VEC_NEON) {
- dev->tx_pkt_prepare = sxe2_tx_pkts_prepare;
- dev->tx_pkt_burst = sxe2_tx_pkts_vec_neon;
- } else {
- dev->tx_pkt_burst = sxe2_tx_pkts_vec_neon_simple;
+ if (tx_mode_flags & SXE2_TX_MODE_VEC_OFFLOAD) {
+ dev->tx_pkt_prepare = sxe2_tx_pkts_prepare;
+ dev->tx_pkt_burst = sxe2_tx_pkts_vec_neon;
+ } else {
+ dev->tx_pkt_burst = sxe2_tx_pkts_vec_neon_simple;
+ }
}
#endif
} else {
if (tx_mode_flags & SXE2_TX_MODE_SIMPLE_BATCH) {
- dev->tx_pkt_prepare = NULL;
+ dev->tx_pkt_prepare = rte_eth_tx_pkt_prepare_dummy;
dev->tx_pkt_burst = sxe2_tx_pkts_simple;
} else {
dev->tx_pkt_prepare = sxe2_tx_pkts_prepare;
dev->tx_pkt_burst = sxe2_tx_pkts;
}
}
+ PMD_LOG_DEBUG(TX, "Tx mode flags:0x%016x port_id:%u.",
+ tx_mode_flags, dev->data->port_id);
}
static const struct {
@@ -582,6 +586,9 @@ void sxe2_rx_mode_func_set(struct rte_eth_dev *dev)
dev->rx_pkt_burst = sxe2_rx_pkts_scattered_split;
else
dev->rx_pkt_burst = sxe2_rx_pkts_scattered;
+
+ PMD_LOG_DEBUG(RX, "Rx mode flags:0x%016x port_id:%u.",
+ rx_mode_flags, dev->data->port_id);
}
static const struct {
diff --git a/drivers/net/sxe2/sxe2_txrx_poll.h b/drivers/net/sxe2/sxe2_txrx_poll.h
index 708e3839d7..bfa099c097 100644
--- a/drivers/net/sxe2/sxe2_txrx_poll.h
+++ b/drivers/net/sxe2/sxe2_txrx_poll.h
@@ -13,8 +13,6 @@ uint16_t sxe2_tx_pkts_simple(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t
uint16_t sxe2_rx_pkts_scattered(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts);
-uint16_t sxe2_rx_pkts_scattered(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts);
-
uint16_t sxe2_rx_pkts_scattered_split(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts);
#endif /* SXE2_TXRX_POLL_H */
diff --git a/drivers/net/sxe2/sxe2_txrx_vec.c b/drivers/net/sxe2/sxe2_txrx_vec.c
index cf004f5eb2..31ab66708c 100644
--- a/drivers/net/sxe2/sxe2_txrx_vec.c
+++ b/drivers/net/sxe2/sxe2_txrx_vec.c
@@ -8,6 +8,19 @@
#include "sxe2_ethdev.h"
#include "sxe2_common_log.h"
+static void sxe2_tx_queue_mbufs_release_vec(struct sxe2_tx_queue *txq);
+
+struct sxe2_txq_ops sxe2_tx_vec_ops_get(void)
+{
+ static const struct sxe2_txq_ops ops = {
+ .queue_reset = sxe2_tx_queue_reset_vec,
+ .mbufs_release = sxe2_tx_queue_mbufs_release_vec,
+ .buffer_ring_free = sxe2_tx_buffer_ring_free,
+ };
+
+ return ops;
+}
+
int32_t __rte_cold sxe2_rx_vec_support_check(struct rte_eth_dev *dev, uint32_t *vec_flags)
{
struct sxe2_rx_queue *rxq;
@@ -157,67 +170,28 @@ int32_t __rte_cold sxe2_tx_vec_support_check(struct rte_eth_dev *dev, uint32_t *
static void sxe2_tx_queue_mbufs_release_vec(struct sxe2_tx_queue *txq)
{
- struct sxe2_tx_buffer *buffer;
+ struct sxe2_tx_buffer_vec *buffer_vec;
uint16_t i;
if (unlikely(txq == NULL || txq->buffer_ring == NULL)) {
PMD_LOG_ERR(TX, "Tx release mbufs vec, invalid params.");
return;
}
- i = txq->next_dd - (txq->rs_thresh - 1);
-#ifdef CC_AVX512_SUPPORT
- struct rte_eth_dev *dev;
- struct sxe2_tx_buffer_vec *buffer_vec;
- dev = &rte_eth_devices[txq->port_id];
-
- if (dev->tx_pkt_burst == sxe2_tx_pkts_vec_avx512 ||
- dev->tx_pkt_burst == sxe2_tx_pkts_vec_avx512_simple) {
- buffer_vec = (struct sxe2_tx_buffer_vec *)txq->buffer_ring;
+ i = txq->next_dd - (txq->rs_thresh - 1);
+ buffer_vec = txq->buffer_ring_vec;
- if (txq->next_use < i) {
- for ( ; i < txq->ring_depth; ++i) {
- if (buffer_vec[i].mbuf != NULL) {
- rte_pktmbuf_free_seg(buffer_vec[i].mbuf);
- buffer_vec[i].mbuf = NULL;
- }
- }
- i = 0;
- }
- for ( ; i < txq->next_use; ++i) {
- if (buffer_vec[i].mbuf != NULL) {
- rte_pktmbuf_free_seg(buffer_vec[i].mbuf);
- buffer_vec[i].mbuf = NULL;
- }
+ if (txq->next_use < i) {
+ for ( ; i < txq->ring_depth; ++i) {
+ rte_pktmbuf_free_seg(buffer_vec[i].mbuf);
+ buffer_vec[i].mbuf = NULL;
}
- } else {
-#endif
- buffer = txq->buffer_ring;
- buffer = txq->buffer_ring;
- if (txq->next_use < i) {
- for ( ; i < txq->ring_depth; ++i) {
- if (buffer[i].mbuf != NULL) {
- rte_pktmbuf_free_seg(buffer[i].mbuf);
- buffer[i].mbuf = NULL;
- }
- }
- i = 0;
- }
- for (; i < txq->next_use; ++i) {
- if (buffer[i].mbuf != NULL) {
- rte_pktmbuf_free_seg(buffer[i].mbuf);
- buffer[i].mbuf = NULL;
- }
- }
-#ifdef CC_AVX512_SUPPORT
+ i = 0;
}
-#endif
- for (; i < txq->next_use; ++i) {
- if (buffer[i].mbuf != NULL) {
- rte_pktmbuf_free_seg(buffer[i].mbuf);
- buffer[i].mbuf = NULL;
- }
+ for ( ; i < txq->next_use; ++i) {
+ rte_pktmbuf_free_seg(buffer_vec[i].mbuf);
+ buffer_vec[i].mbuf = NULL;
}
}
@@ -233,7 +207,8 @@ int32_t __rte_cold sxe2_tx_queues_vec_prepare(struct rte_eth_dev *dev)
PMD_LOG_INFO(TX, "Failed to prepare tx queue, txq[%d] is NULL", i);
continue;
}
- txq->ops.mbufs_release = sxe2_tx_queue_mbufs_release_vec;
+ txq->ops = sxe2_tx_vec_ops_get();
+ txq->ops.queue_reset(txq);
}
return ret;
}
diff --git a/drivers/net/sxe2/sxe2_txrx_vec.h b/drivers/net/sxe2/sxe2_txrx_vec.h
index c139aed776..b9bc4f9c27 100644
--- a/drivers/net/sxe2/sxe2_txrx_vec.h
+++ b/drivers/net/sxe2/sxe2_txrx_vec.h
@@ -89,6 +89,7 @@ uint16_t sxe2_rx_pkts_scattered_vec_neon_offload(void *rx_queue, struct rte_mbuf
uint16_t sxe2_tx_pkts_vec_neon_simple(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts);
uint16_t sxe2_tx_pkts_vec_neon(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts);
#endif
+struct sxe2_txq_ops sxe2_tx_vec_ops_get(void);
int32_t __rte_cold sxe2_tx_vec_support_check(struct rte_eth_dev *dev, uint32_t *vec_flags);
int32_t __rte_cold sxe2_tx_queues_vec_prepare(struct rte_eth_dev *dev);
int32_t __rte_cold sxe2_rx_vec_support_check(struct rte_eth_dev *dev, uint32_t *vec_flags);
diff --git a/drivers/net/sxe2/sxe2_txrx_vec_avx2.c b/drivers/net/sxe2/sxe2_txrx_vec_avx2.c
index 0618e6d988..da96ca3064 100644
--- a/drivers/net/sxe2/sxe2_txrx_vec_avx2.c
+++ b/drivers/net/sxe2/sxe2_txrx_vec_avx2.c
@@ -115,7 +115,7 @@ sxe2_tx_pkts_vec_avx2_batch(struct sxe2_tx_queue *txq, struct rte_mbuf **tx_pkts
uint16_t nb_pkts, bool with_offloads)
{
volatile union sxe2_tx_data_desc *desc;
- struct sxe2_tx_buffer *buffer;
+ struct sxe2_tx_buffer_vec *buffer;
uint16_t next_use;
uint16_t res_num;
uint16_t tx_num;
@@ -134,14 +134,14 @@ sxe2_tx_pkts_vec_avx2_batch(struct sxe2_tx_queue *txq, struct rte_mbuf **tx_pkts
next_use = txq->next_use;
desc = &txq->desc_ring[next_use];
- buffer = &txq->buffer_ring[next_use];
+ buffer = &txq->buffer_ring_vec[next_use];
txq->desc_free_num -= nb_pkts;
res_num = txq->ring_depth - txq->next_use;
if (tx_num >= res_num) {
- sxe2_tx_pkts_mbuf_fill(buffer, tx_pkts, res_num);
+ sxe2_tx_pkts_mbuf_fill_vec(buffer, tx_pkts, res_num);
sxe2_tx_desc_fill_avx2(desc, tx_pkts, res_num,
SXE2_TX_DATA_DESC_CMD_EOP, with_offloads);
@@ -157,10 +157,10 @@ sxe2_tx_pkts_vec_avx2_batch(struct sxe2_tx_queue *txq, struct rte_mbuf **tx_pkts
next_use = 0;
txq->next_rs = txq->rs_thresh - 1;
desc = &txq->desc_ring[next_use];
- buffer = &txq->buffer_ring[next_use];
+ buffer = &txq->buffer_ring_vec[next_use];
}
- sxe2_tx_pkts_mbuf_fill(buffer, tx_pkts, tx_num);
+ sxe2_tx_pkts_mbuf_fill_vec(buffer, tx_pkts, tx_num);
sxe2_tx_desc_fill_avx2(desc, tx_pkts, tx_num,
SXE2_TX_DATA_DESC_CMD_EOP, with_offloads);
diff --git a/drivers/net/sxe2/sxe2_txrx_vec_avx512.c b/drivers/net/sxe2/sxe2_txrx_vec_avx512.c
index a830c7a33b..deea4c2720 100644
--- a/drivers/net/sxe2/sxe2_txrx_vec_avx512.c
+++ b/drivers/net/sxe2/sxe2_txrx_vec_avx512.c
@@ -1,8 +1,6 @@
/* SPDX-License-Identifier: BSD-3-Clause
* Copyright (C), 2025, Wuxi Stars Micro System Technologies Co., Ltd.
*/
-
-#ifndef SXE2_TEST
#include <rte_vect.h>
#include "sxe2_ethdev.h"
@@ -12,114 +10,6 @@
#include "sxe2_txrx_vec_common.h"
#include "sxe2_vsi.h"
-static __rte_always_inline int32_t sxe2_tx_bufs_free_vec_avx512(struct sxe2_tx_queue *txq)
-{
- struct sxe2_tx_buffer_vec *buffer;
- struct rte_mbuf *mbuf;
- struct rte_mbuf *mbuf_free_arr[SXE2_TX_FREE_BUFFER_SIZE_MAX_VEC];
- struct rte_mempool *mp;
- struct rte_mempool_cache *cache;
- void **cache_objs;
- uint32_t copied;
- uint32_t i;
- int32_t ret;
- uint16_t rs_thresh;
- uint16_t free_num;
-
- if (rte_cpu_to_le_64(SXE2_TX_DESC_DTYPE_DESC_DONE) !=
- (txq->desc_ring[txq->next_dd].wb.dd &
- rte_cpu_to_le_64(SXE2_TX_DESC_DTYPE_MASK))) {
- ret = 0;
- goto l_end;
- }
-
- rs_thresh = txq->rs_thresh;
-
- buffer = (struct sxe2_tx_buffer_vec *)txq->buffer_ring;
- buffer += txq->next_dd - (rs_thresh - 1);
-
- if ((txq->offloads & RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE) &&
- (rs_thresh & 31) == 0) {
- mp = buffer[0].mbuf->pool;
- cache = rte_mempool_default_cache(mp, rte_lcore_id());
-
- if (cache == NULL || cache->len)
- goto normal;
-
- if (rs_thresh > RTE_MEMPOOL_CACHE_MAX_SIZE) {
- (void)rte_mempool_ops_enqueue_bulk(mp, (void *)buffer, rs_thresh);
- goto done;
- }
- cache_objs = &cache->objs[cache->len];
-
- copied = 0;
- while (copied < rs_thresh) {
- const __m512i objs0 = _mm512_loadu_si512(&buffer[copied]);
- const __m512i objs1 = _mm512_loadu_si512(&buffer[copied + 8]);
- const __m512i objs2 = _mm512_loadu_si512(&buffer[copied + 16]);
- const __m512i objs3 = _mm512_loadu_si512(&buffer[copied + 24]);
-
- _mm512_storeu_si512(&cache_objs[copied], objs0);
- _mm512_storeu_si512(&cache_objs[copied + 8], objs1);
- _mm512_storeu_si512(&cache_objs[copied + 16], objs2);
- _mm512_storeu_si512(&cache_objs[copied + 24], objs3);
- copied += 32;
- }
- cache->len += rs_thresh;
-
- if (cache->len >= cache->flushthresh) {
- (void)rte_mempool_ops_enqueue_bulk(mp,
- &cache->objs[cache->size], cache->len - cache->size);
- cache->len = cache->size;
- }
- goto done;
- }
-
-normal:
- mbuf = rte_pktmbuf_prefree_seg(buffer[0].mbuf);
-
- if (likely(mbuf)) {
- mbuf_free_arr[0] = mbuf;
- free_num = 1;
-
- for (i = 1; i < rs_thresh; ++i) {
- mbuf = rte_pktmbuf_prefree_seg(buffer[i].mbuf);
-
- if (likely(mbuf)) {
- if (likely(mbuf->pool == mbuf_free_arr[0]->pool)) {
- mbuf_free_arr[free_num] = mbuf;
- free_num++;
- } else {
- rte_mempool_put_bulk(mbuf_free_arr[0]->pool,
- (void *)mbuf_free_arr, free_num);
-
- mbuf_free_arr[0] = mbuf;
- free_num = 1;
- }
- }
- }
-
- rte_mempool_put_bulk(mbuf_free_arr[0]->pool,
- (void *)mbuf_free_arr, free_num);
- } else {
- for (i = 1; i < rs_thresh; ++i) {
- mbuf = rte_pktmbuf_prefree_seg(buffer[i].mbuf);
- if (mbuf != NULL)
- rte_mempool_put(mbuf->pool, mbuf);
- }
- }
-
-done:
- txq->desc_free_num += txq->rs_thresh;
- txq->next_dd += txq->rs_thresh;
- if (txq->next_dd >= txq->ring_depth)
- txq->next_dd = txq->rs_thresh - 1;
- ret = rs_thresh;
-
-l_end:
- return ret;
-}
-
static __rte_always_inline void
sxe2_tx_desc_fill_one_avx512(volatile union sxe2_tx_data_desc *desc, struct rte_mbuf *pkt,
uint64_t desc_cmd, bool with_offloads)
@@ -228,7 +118,7 @@ sxe2_tx_pkts_vec_avx512_batch(struct sxe2_tx_queue *txq, struct rte_mbuf **tx_pk
uint16_t tx_num;
if (txq->desc_free_num < txq->free_thresh)
- (void)sxe2_tx_bufs_free_vec_avx512(txq);
+ (void)sxe2_tx_bufs_free_vec(txq);
nb_pkts = RTE_MIN(txq->desc_free_num, nb_pkts);
if (unlikely(nb_pkts == 0)) {
@@ -241,15 +131,14 @@ sxe2_tx_pkts_vec_avx512_batch(struct sxe2_tx_queue *txq, struct rte_mbuf **tx_pk
next_use = txq->next_use;
desc = &txq->desc_ring[next_use];
- buffer = (struct sxe2_tx_buffer_vec *)txq->buffer_ring;
- buffer += next_use;
+ buffer = &txq->buffer_ring_vec[next_use];
txq->desc_free_num -= nb_pkts;
res_num = txq->ring_depth - txq->next_use;
if (tx_num >= res_num) {
- sxe2_tx_pkts_mbuf_fill_avx512(buffer, tx_pkts, res_num);
+ sxe2_tx_pkts_mbuf_fill_vec(buffer, tx_pkts, res_num);
sxe2_tx_desc_fill_avx512(desc, tx_pkts, res_num,
SXE2_TX_DATA_DESC_CMD_EOP, with_offloads);
@@ -265,10 +154,10 @@ sxe2_tx_pkts_vec_avx512_batch(struct sxe2_tx_queue *txq, struct rte_mbuf **tx_pk
next_use = 0;
txq->next_rs = txq->rs_thresh - 1;
desc = txq->desc_ring;
- buffer = (struct sxe2_tx_buffer_vec *)txq->buffer_ring;
+ buffer = &txq->buffer_ring_vec[next_use];
}
- sxe2_tx_pkts_mbuf_fill_avx512(buffer, tx_pkts, tx_num);
+ sxe2_tx_pkts_mbuf_fill_vec(buffer, tx_pkts, tx_num);
sxe2_tx_desc_fill_avx512(desc, tx_pkts, tx_num,
SXE2_TX_DATA_DESC_CMD_EOP, with_offloads);
@@ -863,5 +752,3 @@ uint16_t sxe2_rx_pkts_scattered_vec_avx512_offload(void *rx_queue,
return sxe2_rx_pkts_scattered_common_vec_avx512(rx_queue,
rx_pkts, nb_pkts, true);
}
-
-#endif
diff --git a/drivers/net/sxe2/sxe2_txrx_vec_common.h b/drivers/net/sxe2/sxe2_txrx_vec_common.h
index 9ac99cf0fa..d16d0a5a5a 100644
--- a/drivers/net/sxe2/sxe2_txrx_vec_common.h
+++ b/drivers/net/sxe2/sxe2_txrx_vec_common.h
@@ -25,10 +25,11 @@
#define SXE2_TX_FREE_BUFFER_SIZE_MAX_VEC 64
static __rte_always_inline void
-sxe2_tx_pkts_mbuf_fill(struct sxe2_tx_buffer *buffer,
- struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
+sxe2_tx_pkts_mbuf_fill_vec(struct sxe2_tx_buffer_vec *buffer,
+ struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
{
uint16_t i;
+
for (i = 0; i < nb_pkts; ++i)
buffer[i].mbuf = tx_pkts[i];
}
diff --git a/drivers/net/sxe2/sxe2_txrx_vec_neon.c b/drivers/net/sxe2/sxe2_txrx_vec_neon.c
index 4e5cb87cd5..c39e4ad81c 100644
--- a/drivers/net/sxe2/sxe2_txrx_vec_neon.c
+++ b/drivers/net/sxe2/sxe2_txrx_vec_neon.c
@@ -34,6 +34,51 @@ sxe2_tx_desc_fill_one_neon(volatile union sxe2_tx_data_desc *desc,
vst1q_u64(RTE_CAST_PTR(uint64_t *, desc), data_desc);
}
+static __rte_always_inline void
+sxe2_tx_desc_fill_4_neon_simple(volatile union sxe2_tx_data_desc *desc,
+ struct rte_mbuf **pkts)
+{
+ uint64x2_t d0, d1, d2, d3;
+ uint64x2x4_t v;
+ const u64 cmd_base = ((u64)SXE2_TX_DESC_DTYPE_DATA) |
+ ((u64)SXE2_TX_DATA_DESC_CMD_EOP) << SXE2_TX_DATA_DESC_CMD_SHIFT;
+
+ d0 = (uint64x2_t){
+ rte_pktmbuf_iova(pkts[0]),
+ cmd_base |
+ ((u64)pkts[0]->data_len) << SXE2_TX_DATA_DESC_BUF_SZ_SHIFT |
+ ((u64)SXE2_TX_DATA_DESC_MACLEN_VAL(pkts[0]->l2_len))
+ << SXE2_TX_DATA_DESC_OFFSET_SHIFT
+ };
+ d1 = (uint64x2_t){
+ rte_pktmbuf_iova(pkts[1]),
+ cmd_base |
+ ((u64)pkts[1]->data_len) << SXE2_TX_DATA_DESC_BUF_SZ_SHIFT |
+ ((u64)SXE2_TX_DATA_DESC_MACLEN_VAL(pkts[1]->l2_len))
+ << SXE2_TX_DATA_DESC_OFFSET_SHIFT
+ };
+ d2 = (uint64x2_t){
+ rte_pktmbuf_iova(pkts[2]),
+ cmd_base |
+ ((u64)pkts[2]->data_len) << SXE2_TX_DATA_DESC_BUF_SZ_SHIFT |
+ ((u64)SXE2_TX_DATA_DESC_MACLEN_VAL(pkts[2]->l2_len))
+ << SXE2_TX_DATA_DESC_OFFSET_SHIFT
+ };
+ d3 = (uint64x2_t){
+ rte_pktmbuf_iova(pkts[3]),
+ cmd_base |
+ ((u64)pkts[3]->data_len) << SXE2_TX_DATA_DESC_BUF_SZ_SHIFT |
+ ((u64)SXE2_TX_DATA_DESC_MACLEN_VAL(pkts[3]->l2_len))
+ << SXE2_TX_DATA_DESC_OFFSET_SHIFT
+ };
+
+ v.val[0] = d0;
+ v.val[1] = d1;
+ v.val[2] = d2;
+ v.val[3] = d3;
+ vst1q_u64_x4((u64 *)desc, v);
+}
+
static __rte_always_inline uint16_t
sxe2_tx_pkts_vec_neon_batch(struct sxe2_tx_queue *txq, struct rte_mbuf **tx_pkts,
uint16_t nb_pkts, bool with_offloads)
@@ -66,11 +111,19 @@ sxe2_tx_pkts_vec_neon_batch(struct sxe2_tx_queue *txq, struct rte_mbuf **tx_pkts
res_num = txq->ring_depth - txq->next_use;
if (tx_num >= res_num) {
- sxe2_tx_pkts_mbuf_fill(buffer, tx_pkts, res_num);
-
- for (i = 0; i < res_num - 1; ++i, ++tx_pkts, ++desc) {
- sxe2_tx_desc_fill_one_neon(desc, *tx_pkts,
- SXE2_TX_DATA_DESC_CMD_EOP, with_offloads);
+ sxe2_tx_pkts_mbuf_fill_vec(buffer, tx_pkts, res_num);
+ if (with_offloads) {
+ for (i = 0; i < res_num - 1; ++i, ++tx_pkts, ++desc) {
+ sxe2_tx_desc_fill_one_neon(desc, *tx_pkts,
+ SXE2_TX_DATA_DESC_CMD_EOP, with_offloads);
+ }
+ } else {
+ for (i = 0; i + 3 < res_num - 1; i += 4, tx_pkts += 4, desc += 4)
+ sxe2_tx_desc_fill_4_neon_simple(desc, tx_pkts);
+ for (; i < res_num - 1; ++i, ++tx_pkts, ++desc) {
+ sxe2_tx_desc_fill_one_neon(desc, *tx_pkts,
+ SXE2_TX_DATA_DESC_CMD_EOP, false);
+ }
}
sxe2_tx_desc_fill_one_neon(desc, *tx_pkts++,
@@ -82,14 +135,23 @@ sxe2_tx_pkts_vec_neon_batch(struct sxe2_tx_queue *txq, struct rte_mbuf **tx_pkts
next_use = 0;
txq->next_rs = txq->rs_thresh - 1;
desc = &txq->desc_ring[next_use];
- buffer = &txq->buffer_ring[next_use];
+ buffer = &txq->buffer_ring_vec[next_use];
}
- sxe2_tx_pkts_mbuf_fill(buffer, tx_pkts, tx_num);
+ sxe2_tx_pkts_mbuf_fill_vec(buffer, tx_pkts, tx_num);
- for (i = 0; i < tx_num; ++i, ++tx_pkts, ++desc) {
- sxe2_tx_desc_fill_one_neon(desc, *tx_pkts,
- SXE2_TX_DATA_DESC_CMD_EOP, with_offloads);
+ if (with_offloads) {
+ for (i = 0; i < tx_num; ++i, ++tx_pkts, ++desc) {
+ sxe2_tx_desc_fill_one_neon(desc, *tx_pkts,
+ SXE2_TX_DATA_DESC_CMD_EOP, true);
+ }
+ } else {
+ for (i = 0; i + 3 < tx_num; i += 4, tx_pkts += 4, desc += 4)
+ sxe2_tx_desc_fill_4_neon_simple(desc, tx_pkts);
+ for (; i < tx_num; ++i, ++tx_pkts, ++desc) {
+ sxe2_tx_desc_fill_one_neon(desc, *tx_pkts,
+ SXE2_TX_DATA_DESC_CMD_EOP, false);
+ }
}
next_use += tx_num;
@@ -150,22 +212,24 @@ uint16_t sxe2_tx_pkts_vec_neon(void *tx_queue,
}
static __rte_always_inline void
-sxe2_rx_desc_ptype_fill_neon(uint16x8_t staterr, struct rte_mbuf **__rte_restrict rx_pkts)
+sxe2_rx_desc_ptype_fill_neon(uint32x4_t desc_lo,
+ struct rte_mbuf **__rte_restrict rx_pkts,
+ const u32 *__rte_restrict ptype_tbl)
{
- uint16x8_t ptype_mask = {
- 0, 0x3FFULL,
- 0, 0x3FFULL,
- 0, 0x3FFULL,
- 0, 0x3FFULL,
+ const uint32x4_t ptype_mask = {
+ SXE2_RX_DESC_PTYPE_MASK_NO_SHIFT << 16,
+ SXE2_RX_DESC_PTYPE_MASK_NO_SHIFT << 16,
+ SXE2_RX_DESC_PTYPE_MASK_NO_SHIFT << 16,
+ SXE2_RX_DESC_PTYPE_MASK_NO_SHIFT << 16,
};
uint16x8_t ptype_all;
- ptype_all = vandq_u16(staterr, ptype_mask);
+ ptype_all = vreinterpretq_u16_u32(vandq_u32(desc_lo, ptype_mask));
- rx_pkts[3]->packet_type = sxe2_ptype_tbl[vgetq_lane_u16(ptype_all, 3)];
- rx_pkts[2]->packet_type = sxe2_ptype_tbl[vgetq_lane_u16(ptype_all, 7)];
- rx_pkts[1]->packet_type = sxe2_ptype_tbl[vgetq_lane_u16(ptype_all, 1)];
- rx_pkts[0]->packet_type = sxe2_ptype_tbl[vgetq_lane_u16(ptype_all, 5)];
+ rx_pkts[0]->packet_type = ptype_tbl[vgetq_lane_u16(ptype_all, 1)];
+ rx_pkts[1]->packet_type = ptype_tbl[vgetq_lane_u16(ptype_all, 3)];
+ rx_pkts[2]->packet_type = ptype_tbl[vgetq_lane_u16(ptype_all, 5)];
+ rx_pkts[3]->packet_type = ptype_tbl[vgetq_lane_u16(ptype_all, 7)];
}
static __rte_always_inline uint32x4_t
@@ -208,9 +272,10 @@ sxe2_rx_desc_fnav_flags_neon(uint64x2_t descs_arr[4])
static __rte_always_inline void
sxe2_rx_desc_offloads_para_fill_neon(struct sxe2_rx_queue *rxq,
volatile union sxe2_rx_desc *desc,
- uint64x2_t descs[4], struct rte_mbuf **rx_pkts)
+ uint64x2_t descs[4], uint32x4_t desc_lo, uint32x4_t desc_hi,
+ struct rte_mbuf **rx_pkts)
{
- uint32x4_t desc_lo, desc_hi, flags, tmp_flags;
+ uint32x4_t flags, tmp_flags;
const uint64x2_t mbuf_init = {rxq->mbuf_init_value, 0};
uint64x2_t rearm0, rearm1, rearm2, rearm3;
@@ -267,23 +332,6 @@ sxe2_rx_desc_offloads_para_fill_neon(struct sxe2_rx_queue *rxq,
0, 0, 0, 0, 0, 0, 0, 0
};
- {
- uint32x4_t d0 = vreinterpretq_u32_u64(descs[0]);
- uint32x4_t d1 = vreinterpretq_u32_u64(descs[1]);
- uint32x4_t d2 = vreinterpretq_u32_u64(descs[2]);
- uint32x4_t d3 = vreinterpretq_u32_u64(descs[3]);
- uint64x2_t f64, t64;
-
- flags = vzip2q_u32(d1, d0);
- tmp_flags = vzip2q_u32(d3, d2);
- f64 = vreinterpretq_u64_u32(flags);
- t64 = vreinterpretq_u64_u32(tmp_flags);
- desc_lo = vreinterpretq_u32_u64(vcombine_u64(vget_low_u64(f64),
- vget_low_u64(t64)));
- desc_hi = vreinterpretq_u32_u64(vcombine_u64(vget_high_u64(f64),
- vget_high_u64(t64)));
- }
-
desc_lo = vandq_u32(desc_lo, desc_msk);
desc_hi = vandq_u32(desc_hi, rss_msk);
@@ -407,6 +455,7 @@ sxe2_rx_pkts_common_vec_neon(struct sxe2_rx_queue *rxq, struct rte_mbuf **rx_pkt
struct rte_mbuf **buffer;
uint32_t i;
uint16_t done_num = 0;
+ const u32 *ptype_tbl = rxq->vsi->adapter->ptype_tbl;
uint8x16_t rvp_shuf_mask = {
0xFF, 0xFF, 0xFF, 0xFF,
@@ -442,25 +491,39 @@ sxe2_rx_pkts_common_vec_neon(struct sxe2_rx_queue *rxq, struct rte_mbuf **rx_pkt
uint64x2_t descs[SXE2_RX_NUM_PER_LOOP_NEON];
uint8x16_t pkt_mb1, pkt_mb2, pkt_mb3, pkt_mb4;
uint64x2_t mbp1, mbp2;
+ uint32x4_t desc_lo, desc_hi;
uint16x8_t staterr;
uint16x8_t tmp;
uint16_t bit_num;
descs[3] = vld1q_u64(RTE_CAST_PTR(uint64_t *, desc + 3));
- rte_atomic_thread_fence(rte_memory_order_acquire);
descs[2] = vld1q_u64(RTE_CAST_PTR(uint64_t *, desc + 2));
- rte_atomic_thread_fence(rte_memory_order_acquire);
descs[1] = vld1q_u64(RTE_CAST_PTR(uint64_t *, desc + 1));
- rte_atomic_thread_fence(rte_memory_order_acquire);
descs[0] = vld1q_u64(RTE_CAST_PTR(uint64_t *, desc));
rte_atomic_thread_fence(rte_memory_order_acquire);
-
descs[3] = vld1q_lane_u64(RTE_CAST_PTR(uint64_t *, desc + 3), descs[3], 0);
descs[2] = vld1q_lane_u64(RTE_CAST_PTR(uint64_t *, desc + 2), descs[2], 0);
descs[1] = vld1q_lane_u64(RTE_CAST_PTR(uint64_t *, desc + 1), descs[1], 0);
descs[0] = vld1q_lane_u64(RTE_CAST_PTR(uint64_t *, desc), descs[0], 0);
+ {
+ uint32x4_t d0 = vreinterpretq_u32_u64(descs[0]);
+ uint32x4_t d1 = vreinterpretq_u32_u64(descs[1]);
+ uint32x4_t d2 = vreinterpretq_u32_u64(descs[2]);
+ uint32x4_t d3 = vreinterpretq_u32_u64(descs[3]);
+
+ uint32x4_t q1_01 = vzip2q_u32(d0, d1);
+ uint32x4_t q1_23 = vzip2q_u32(d2, d3);
+ uint64x2_t q1_01_64 = vreinterpretq_u64_u32(q1_01);
+ uint64x2_t q1_23_64 = vreinterpretq_u64_u32(q1_23);
+
+ desc_lo = vreinterpretq_u32_u64(vcombine_u64(vget_low_u64(q1_01_64),
+ vget_low_u64(q1_23_64)));
+ desc_hi = vreinterpretq_u32_u64(vcombine_u64(vget_high_u64(q1_01_64),
+ vget_high_u64(q1_23_64)));
+ }
+
mbp1 = vld1q_u64((uint64_t *)&buffer[i]);
mbp2 = vld1q_u64((uint64_t *)&buffer[i + 2]);
@@ -480,7 +543,8 @@ sxe2_rx_pkts_common_vec_neon(struct sxe2_rx_queue *rxq, struct rte_mbuf **rx_pkt
pkt_mb1 = vqtbl1q_u8(vreinterpretq_u8_u64(descs[0]), rvp_shuf_mask);
if (do_offload) {
- sxe2_rx_desc_offloads_para_fill_neon(rxq, desc, descs, &rx_pkts[i]);
+ sxe2_rx_desc_offloads_para_fill_neon(rxq, desc, descs, desc_lo,
+ desc_hi, &rx_pkts[i]);
} else {
const uint64x2_t mbuf_init = {
rxq->mbuf_init_value,
@@ -515,55 +579,48 @@ sxe2_rx_pkts_common_vec_neon(struct sxe2_rx_queue *rxq, struct rte_mbuf **rx_pkt
rte_prefetch_non_temporal(desc + SXE2_RX_NUM_PER_LOOP_NEON);
{
- uint32x4_t d0 = vreinterpretq_u32_u64(descs[0]);
- uint32x4_t d1 = vreinterpretq_u32_u64(descs[1]);
- uint32x4_t d2 = vreinterpretq_u32_u64(descs[2]);
- uint32x4_t d3 = vreinterpretq_u32_u64(descs[3]);
- uint32x4_t sterr_tmp1 = vzip2q_u32(d1, d0);
- uint32x4_t sterr_tmp2 = vzip2q_u32(d3, d2);
- uint32x4_t sterr_u32 = vzip1q_u32(sterr_tmp1, sterr_tmp2);
-
- staterr = vreinterpretq_u16_u32(sterr_u32);
+ uint16x8_t sterr_tmp1 = vzip2q_u16(vreinterpretq_u16_u64(descs[0]),
+ vreinterpretq_u16_u64(descs[2]));
+ uint16x8_t sterr_tmp2 = vzip2q_u16(vreinterpretq_u16_u64(descs[1]),
+ vreinterpretq_u16_u64(descs[3]));
+ staterr = vzip1q_u16(sterr_tmp1, sterr_tmp2);
}
- sxe2_rx_desc_ptype_fill_neon(staterr, &rx_pkts[i]);
+ sxe2_rx_desc_ptype_fill_neon(desc_lo, &rx_pkts[i], ptype_tbl);
if (umbcast_flags != NULL) {
- uint32x4_t umbcast_mask = {
- SXE2_RX_DESC_STATUS_UMBCAST_MASK, SXE2_RX_DESC_STATUS_UMBCAST_MASK,
- SXE2_RX_DESC_STATUS_UMBCAST_MASK, SXE2_RX_DESC_STATUS_UMBCAST_MASK,
- };
-
+ const uint32x4_t umbcast_mask =
+ vdupq_n_u32(SXE2_RX_DESC_STATUS_UMBCAST_MASK);
uint8x16_t umbcast_shuf_mask = {
- 0x0B, 0x03, 0x0F, 0x07,
+ 3, 7, 11, 15,
0xFF, 0xFF, 0xFF, 0xFF,
0xFF, 0xFF, 0xFF, 0xFF,
0xFF, 0xFF, 0xFF, 0xFF,
};
uint8x16_t umbcast_bits =
- vreinterpretq_u8_u32(vandq_u32(vreinterpretq_u32_u16(staterr),
- umbcast_mask));
+ vreinterpretq_u8_u32(vandq_u32(desc_lo, umbcast_mask));
umbcast_bits = vqtbl1q_u8(umbcast_bits, umbcast_shuf_mask);
- vst1q_lane_u32((uint32_t *)umbcast_flags,
- vreinterpretq_u32_u8(umbcast_bits), 0);
+ *(u32 *)umbcast_flags =
+ vgetq_lane_u32(vreinterpretq_u32_u8(umbcast_bits), 0);
umbcast_flags += SXE2_RX_NUM_PER_LOOP_NEON;
}
if (split_rxe_flags) {
uint8x16_t eop_shuf_mask = {
- 0x08, 0x00, 0x0C, 0x04,
+ 0, 2, 4, 6,
0xFF, 0xFF, 0xFF, 0xFF,
0xFF, 0xFF, 0xFF, 0xFF,
0xFF, 0xFF, 0xFF, 0xFF};
uint8x16_t eop_bits;
uint32x4_t rxe_mask = {
- 0x2080, 0x2080, 0x2080, 0x2080
+ 0x20802080, 0x20802080, 0x20802080, 0x20802080
};
uint32x4_t rxe_bits;
uint32x4_t eop_mask;
- eop_mask = vshlq_n_u32(vdupq_n_u32(1), SXE2_RX_DESC_STATUS_EOP_SHIFT);
+ eop_mask = vdupq_n_u32((1U << SXE2_RX_DESC_STATUS_EOP_SHIFT) |
+ (1U << (SXE2_RX_DESC_STATUS_EOP_SHIFT + 16)));
eop_bits = vandq_u8(vmvnq_u8(vreinterpretq_u8_u16(staterr)),
vreinterpretq_u8_u32(eop_mask));
@@ -587,12 +644,22 @@ sxe2_rx_pkts_common_vec_neon(struct sxe2_rx_queue *rxq, struct rte_mbuf **rx_pkt
}
{
- uint32x4_t dd_mask = vdupq_n_u32(1);
- uint32x4_t sterr_dd = vandq_u32(vreinterpretq_u32_u16(staterr), dd_mask);
- uint16x4_t packed_lo = vmovn_u32(sterr_dd);
- uint64_t dd64 = vget_lane_u64(vreinterpret_u64_u16(packed_lo), 0);
-
- bit_num = (uint16_t)rte_popcount64(dd64);
+ const uint16x8_t dd_check = {
+ 0x0001, 0x0001, 0x0001, 0x0001,
+ 0, 0, 0, 0
+ };
+ uint16x8_t sterr_dd;
+ uint64_t stat;
+ sterr_dd = vandq_u16(staterr, dd_check);
+ sterr_dd = vshlq_n_u16(sterr_dd, 15);
+ sterr_dd =
+ vreinterpretq_u16_s16(vshrq_n_s16(vreinterpretq_s16_u16(sterr_dd),
+ 15));
+ stat = ~vgetq_lane_u64(vreinterpretq_u64_u16(sterr_dd), 0);
+ if (likely(stat == 0))
+ bit_num = SXE2_RX_NUM_PER_LOOP_NEON;
+ else
+ bit_num = (u16)(rte_ctz64(stat) / 16);
}
done_num += bit_num;
if (likely(bit_num != SXE2_RX_NUM_PER_LOOP_NEON))
diff --git a/drivers/net/sxe2/sxe2_txrx_vec_sse.c b/drivers/net/sxe2/sxe2_txrx_vec_sse.c
index c3e8a2983b..181bb40041 100644
--- a/drivers/net/sxe2/sxe2_txrx_vec_sse.c
+++ b/drivers/net/sxe2/sxe2_txrx_vec_sse.c
@@ -40,7 +40,7 @@ sxe2_tx_pkts_vec_sse_batch(struct sxe2_tx_queue *txq,
uint16_t nb_pkts, bool with_offloads)
{
volatile union sxe2_tx_data_desc *desc;
- struct sxe2_tx_buffer *buffer;
+ struct sxe2_tx_buffer_vec *buffer;
uint16_t next_use;
uint16_t res_num;
uint16_t tx_num;
@@ -57,11 +57,11 @@ sxe2_tx_pkts_vec_sse_batch(struct sxe2_tx_queue *txq,
tx_num = nb_pkts;
next_use = txq->next_use;
desc = &txq->desc_ring[next_use];
- buffer = &txq->buffer_ring[next_use];
+ buffer = &txq->buffer_ring_vec[next_use];
txq->desc_free_num -= nb_pkts;
res_num = txq->ring_depth - txq->next_use;
if (tx_num >= res_num) {
- sxe2_tx_pkts_mbuf_fill(buffer, tx_pkts, res_num);
+ sxe2_tx_pkts_mbuf_fill_vec(buffer, tx_pkts, res_num);
for (i = 0; i < res_num - 1; ++i, ++tx_pkts, ++desc) {
sxe2_tx_desc_fill_one_sse(desc, *tx_pkts,
SXE2_TX_DATA_DESC_CMD_EOP,
@@ -74,9 +74,9 @@ sxe2_tx_pkts_vec_sse_batch(struct sxe2_tx_queue *txq,
next_use = 0;
txq->next_rs = txq->rs_thresh - 1;
desc = &txq->desc_ring[next_use];
- buffer = &txq->buffer_ring[next_use];
+ buffer = &txq->buffer_ring_vec[next_use];
}
- sxe2_tx_pkts_mbuf_fill(buffer, tx_pkts, tx_num);
+ sxe2_tx_pkts_mbuf_fill_vec(buffer, tx_pkts, tx_num);
for (i = 0; i < tx_num; ++i, ++tx_pkts, ++desc) {
sxe2_tx_desc_fill_one_sse(desc, *tx_pkts,
SXE2_TX_DATA_DESC_CMD_EOP,
diff --git a/drivers/net/sxe2/sxe2_vsi.c b/drivers/net/sxe2/sxe2_vsi.c
index d29480b931..ba4cc7414e 100644
--- a/drivers/net/sxe2/sxe2_vsi.c
+++ b/drivers/net/sxe2/sxe2_vsi.c
@@ -230,7 +230,7 @@ int32_t sxe2_vsi_init(struct rte_eth_dev *dev)
uint16_t srcvsi_cnt;
PMD_INIT_FUNC_TRACE();
-
+ TAILQ_INIT(&adapter->vsi_ctxt.other_vsi_list);
ret = sxe2_main_vsi_create(adapter);
if (ret) {
PMD_LOG_ERR(DRV, "Failed to create main VSI, ret=%d", ret);
@@ -283,13 +283,14 @@ void sxe2_vsi_uninit(struct rte_eth_dev *dev)
l_free:
ret = sxe2_vsi_destroy(adapter, adapter->vsi_ctxt.main_vsi);
- if (ret) {
+ if (ret && ret != -EPERM) {
PMD_LOG_ERR(DRV, "Failed to del vsi from fw, ret=%d", ret);
goto l_end;
}
+ adapter->vsi_ctxt.main_vsi = NULL;
RTE_TAILQ_FOREACH_SAFE(var, &adapter->vsi_ctxt.other_vsi_list, next, tvar) {
ret = sxe2_vsi_destroy(adapter, var);
- if (ret) {
+ if (ret && ret != -EPERM) {
PMD_LOG_ERR(DRV, "Failed to del vsi from fw, ret=%d", ret);
break;
}
@@ -357,4 +358,5 @@ void sxe2_vsi_repr_main_vsi_destroy(struct rte_eth_dev *dev)
struct sxe2_adapter *adapter = SXE2_DEV_PRIVATE_TO_ADAPTER(dev);
sxe2_vsi_node_free(adapter->vsi_ctxt.main_vsi);
+ adapter->vsi_ctxt.main_vsi = NULL;
}
--
2.52.0