From: Nazim Amirul <[email protected]>
Use page pool buffers instead of streaming skb buffers in stmmac_xmit
when snps,pagepool-tx-buf-quirk is enabled. As page pool buffers are
recycled, the overhead of dma map/unmap for each transmit can be
avoided. There is a trade-off of copying from the skb into the page
pool buffer, but this helps where the copy is cheaper than dma
map/unmap (for example with IOMMU invalidate cost).
Currently page pool buffers are only used for single-fragment transmit
in the non-TSO path.
Signed-off-by: Nazim Amirul <[email protected]>
---
drivers/net/ethernet/stmicro/stmmac/stmmac.h | 3 +
.../net/ethernet/stmicro/stmmac/stmmac_main.c | 75 ++++++++++++++++---
.../ethernet/stmicro/stmmac/stmmac_platform.c | 2 +
include/linux/stmmac.h | 1 +
4 files changed, 70 insertions(+), 11 deletions(-)
diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac.h b/drivers/net/ethernet/stmicro/stmmac/stmmac.h
index 8ba8f03e1ce0..093c052b4908 100644
--- a/drivers/net/ethernet/stmicro/stmmac/stmmac.h
+++ b/drivers/net/ethernet/stmicro/stmmac/stmmac.h
@@ -41,6 +41,7 @@ struct stmmac_resources {
enum stmmac_txbuf_type {
STMMAC_TXBUF_T_SKB,
+ STMMAC_TXBUF_T_DMA,
STMMAC_TXBUF_T_XDP_TX,
STMMAC_TXBUF_T_XDP_NDO,
STMMAC_TXBUF_T_XSK_TX,
@@ -49,6 +50,7 @@ enum stmmac_txbuf_type {
struct stmmac_tx_info {
struct xsk_tx_metadata_compl xsk_meta;
dma_addr_t buf;
+ struct page *page;
unsigned len;
enum stmmac_txbuf_type buf_type;
bool map_as_page;
@@ -65,6 +67,7 @@ struct stmmac_tx_queue {
int tbs;
struct hrtimer txtimer;
u32 queue_index;
+ struct page_pool *page_pool;
struct stmmac_priv *priv_data;
struct dma_extended_desc *dma_etx ____cacheline_aligned_in_smp;
struct dma_edesc *dma_entx;
diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c
index ee44bd6f4d48..4aa6b1d1507b 100644
--- a/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c
+++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c
@@ -1712,7 +1712,10 @@ static void stmmac_free_tx_buffer(struct stmmac_priv *priv,
if (tx_q->tx_skbuff_dma[i].buf &&
tx_q->tx_skbuff_dma[i].buf_type != STMMAC_TXBUF_T_XDP_TX) {
- if (tx_q->tx_skbuff_dma[i].map_as_page)
+ if (tx_q->tx_skbuff_dma[i].buf_type == STMMAC_TXBUF_T_DMA)
+ page_pool_recycle_direct(tx_q->page_pool,
+ tx_q->tx_skbuff_dma[i].page);
+ else if (tx_q->tx_skbuff_dma[i].map_as_page)
dma_unmap_page(priv->device,
tx_q->tx_skbuff_dma[i].buf,
tx_q->tx_skbuff_dma[i].len,
@@ -1735,7 +1738,8 @@ static void stmmac_free_tx_buffer(struct stmmac_priv *priv,
tx_q->xsk_frames_done++;
if (tx_q->tx_skbuff[i] &&
- tx_q->tx_skbuff_dma[i].buf_type == STMMAC_TXBUF_T_SKB) {
+ (tx_q->tx_skbuff_dma[i].buf_type == STMMAC_TXBUF_T_SKB ||
+ tx_q->tx_skbuff_dma[i].buf_type == STMMAC_TXBUF_T_DMA)) {
dev_kfree_skb_any(tx_q->tx_skbuff[i]);
tx_q->tx_skbuff[i] = NULL;
}
@@ -2201,6 +2205,8 @@ static void __free_dma_tx_desc_resources(struct stmmac_priv *priv,
kfree(tx_q->tx_skbuff_dma);
kfree(tx_q->tx_skbuff);
+ if (tx_q->page_pool)
+ page_pool_destroy(tx_q->page_pool);
}
static void free_dma_tx_desc_resources(struct stmmac_priv *priv,
@@ -2335,12 +2341,32 @@ static int __alloc_dma_tx_desc_resources(struct stmmac_priv *priv,
u32 queue)
{
struct stmmac_tx_queue *tx_q = &dma_conf->tx_queue[queue];
+ struct page_pool_params pp_params = { 0 };
+ unsigned int num_pages;
size_t size;
void *addr;
+ int ret;
tx_q->queue_index = queue;
tx_q->priv_data = priv;
+ pp_params.flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV;
+ pp_params.pool_size = dma_conf->dma_tx_size;
+ num_pages = DIV_ROUND_UP(dma_conf->dma_buf_sz, PAGE_SIZE);
+ pp_params.order = ilog2(num_pages);
+ pp_params.nid = dev_to_node(priv->device);
+ pp_params.dev = priv->device;
+ pp_params.dma_dir = DMA_BIDIRECTIONAL;
+ pp_params.offset = 0;
+ pp_params.max_len = num_pages * PAGE_SIZE;
+
+ tx_q->page_pool = page_pool_create(&pp_params);
+ if (IS_ERR(tx_q->page_pool)) {
+ ret = PTR_ERR(tx_q->page_pool);
+ tx_q->page_pool = NULL;
+ return ret;
+ }
+
tx_q->tx_skbuff_dma = kzalloc_objs(*tx_q->tx_skbuff_dma,
dma_conf->dma_tx_size);
if (!tx_q->tx_skbuff_dma)
@@ -2836,7 +2862,8 @@ static int stmmac_tx_clean(struct stmmac_priv *priv, int budget, u32 queue,
tx_q->tx_skbuff_dma[entry].buf_type == STMMAC_TXBUF_T_XDP_NDO) {
xdpf = tx_q->xdpf[entry];
skb = NULL;
- } else if (tx_q->tx_skbuff_dma[entry].buf_type == STMMAC_TXBUF_T_SKB) {
+ } else if (tx_q->tx_skbuff_dma[entry].buf_type == STMMAC_TXBUF_T_SKB ||
+ tx_q->tx_skbuff_dma[entry].buf_type == STMMAC_TXBUF_T_DMA) {
xdpf = NULL;
skb = tx_q->tx_skbuff[entry];
} else {
@@ -2884,7 +2911,10 @@ static int stmmac_tx_clean(struct stmmac_priv *priv, int budget, u32 queue,
if (likely(tx_q->tx_skbuff_dma[entry].buf &&
tx_q->tx_skbuff_dma[entry].buf_type != STMMAC_TXBUF_T_XDP_TX)) {
- if (tx_q->tx_skbuff_dma[entry].map_as_page)
+ if (tx_q->tx_skbuff_dma[entry].buf_type == STMMAC_TXBUF_T_DMA)
+ page_pool_recycle_direct(tx_q->page_pool,
+ tx_q->tx_skbuff_dma[entry].page);
+ else if (tx_q->tx_skbuff_dma[entry].map_as_page)
dma_unmap_page(priv->device,
tx_q->tx_skbuff_dma[entry].buf,
tx_q->tx_skbuff_dma[entry].len,
@@ -2922,7 +2952,8 @@ static int stmmac_tx_clean(struct stmmac_priv *priv, int budget, u32 queue,
if (tx_q->tx_skbuff_dma[entry].buf_type == STMMAC_TXBUF_T_XSK_TX)
tx_q->xsk_frames_done++;
- if (tx_q->tx_skbuff_dma[entry].buf_type == STMMAC_TXBUF_T_SKB) {
+ if (tx_q->tx_skbuff_dma[entry].buf_type == STMMAC_TXBUF_T_SKB ||
+ tx_q->tx_skbuff_dma[entry].buf_type == STMMAC_TXBUF_T_DMA) {
if (likely(skb)) {
pkts_compl++;
bytes_compl += skb->len;
@@ -4742,12 +4773,14 @@ static netdev_tx_t stmmac_xmit(struct sk_buff *skb, struct net_device *dev)
bool enh_desc, has_vlan, set_ic, is_jumbo = false;
struct stmmac_priv *priv = netdev_priv(dev);
unsigned int nopaged_len = skb_headlen(skb);
+ gfp_t gfp = GFP_ATOMIC | __GFP_NOWARN;
u32 queue = skb_get_queue_mapping(skb);
int nfrags = skb_shinfo(skb)->nr_frags;
unsigned int first_entry, tx_packets;
struct stmmac_txq_stats *txq_stats;
struct dma_desc *desc, *first_desc;
struct stmmac_tx_queue *tx_q;
+ struct page *tx_buf_page;
int i, csum_insertion = 0;
int entry, first_tx;
dma_addr_t dma_addr;
@@ -4814,13 +4847,33 @@ static netdev_tx_t stmmac_xmit(struct sk_buff *skb, struct net_device *dev)
} else {
bool last_segment = (nfrags == 0);
- dma_addr = dma_map_single(priv->device, skb->data,
- nopaged_len, DMA_TO_DEVICE);
- if (dma_mapping_error(priv->device, dma_addr))
- goto dma_map_err;
+ if (!(last_segment && priv->plat->tx_buf_quirk)) {
+ dma_addr = dma_map_single(priv->device, skb->data,
+ nopaged_len, DMA_TO_DEVICE);
+ if (dma_mapping_error(priv->device, dma_addr))
+ goto dma_map_err;
- stmmac_set_tx_skb_dma_entry(tx_q, first_entry, dma_addr,
- nopaged_len, false);
+ stmmac_set_tx_skb_dma_entry(tx_q, first_entry, dma_addr,
+ nopaged_len, false);
+ } else {
+ tx_buf_page = page_pool_alloc_pages(tx_q->page_pool, gfp);
+ if (!tx_buf_page)
+ goto dma_map_err;
+
+ dma_addr = page_pool_get_dma_addr(tx_buf_page);
+ stmmac_set_tx_dma_entry(tx_q, first_entry,
+ STMMAC_TXBUF_T_DMA, dma_addr,
+ nopaged_len, false);
+ tx_q->tx_skbuff_dma[first_entry].page = tx_buf_page;
+
+ dma_sync_single_for_cpu(priv->device, dma_addr,
+ nopaged_len, DMA_BIDIRECTIONAL);
+ skb_copy_from_linear_data(skb, page_address(tx_buf_page),
+ nopaged_len);
+ dma_sync_single_for_device(priv->device, dma_addr,
+ nopaged_len,
+ DMA_BIDIRECTIONAL);
+ }
stmmac_set_desc_addr(priv, first_desc, dma_addr);
diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_platform.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_platform.c
index dc5f951a311d..0f8d2fc61ce4 100644
--- a/drivers/net/ethernet/stmicro/stmmac/stmmac_platform.c
+++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_platform.c
@@ -575,6 +575,8 @@ stmmac_probe_config_dt(struct platform_device *pdev, u8 *mac)
of_property_read_u32(np, "snps,ps-speed", &plat->mac_port_sel_speed);
+ plat->tx_buf_quirk = of_property_read_bool(np, "snps,pagepool-tx-buf-quirk");
+
plat->axi = stmmac_axi_setup(pdev);
rc = stmmac_mtl_setup(pdev, plat);
diff --git a/include/linux/stmmac.h b/include/linux/stmmac.h
index 4430b967abde..d4c9d77a6f81 100644
--- a/include/linux/stmmac.h
+++ b/include/linux/stmmac.h
@@ -268,6 +268,7 @@ struct plat_stmmacenet_data {
u8 tx_queues_to_use;
u8 rx_sched_algorithm;
u8 tx_sched_algorithm;
+ bool tx_buf_quirk;
struct stmmac_rxq_cfg rx_queues_cfg[MTL_MAX_RX_QUEUES];
struct stmmac_txq_cfg tx_queues_cfg[MTL_MAX_TX_QUEUES];
void (*get_interfaces)(struct stmmac_priv *priv, void *bsp_priv,
--
2.43.7
lmpx.com only provides a reader for public news (NNTP) servers. It is not
affiliated with the servers or forums shown here and is not responsible for
the content of articles, which is written by their respective authors.