From cde6d65ea5063367fa782fcd1ea9fc54cdad4deb Mon Sep 17 00:00:00 2001 From: Jonathan Toppins Date: Wed, 2 Oct 2019 18:22:46 -0400 Subject: [PATCH 31/96] [netdrv] bnxt_en: add page_pool support Message-id: <5af2e26517985727f11c26b4580740d97a57abce.1570027456.git.jtoppins@redhat.com> Patchwork-id: 276452 O-Subject: [RHEL-8.2 PATCH 24/78] bnxt_en: add page_pool support Bugzilla: 1724766 RH-Acked-by: John Linville RH-Acked-by: Jarod Wilson This removes contention over page allocation for XDP_REDIRECT actions by adding page_pool support per queue for the driver. The performance for XDP_REDIRECT actions scales linearly with the number of cores performing redirect actions when using the page pools instead of the standard page allocator. v2: Fix up the error path from XDP registration, noted by Ilias Apalodimas. Signed-off-by: Andy Gospodarek Signed-off-by: Michael Chan Signed-off-by: David S. Miller (cherry picked from commit 322b87ca55f2f3936ec9f9de438ef9b2115b5c9b) Bugzilla: 1724766 Build Info: https://brewweb.engineering.redhat.com/brew/taskinfo?taskID=23809532 Tested: build, boot, basic ping Signed-off-by: Jonathan Toppins Signed-off-by: Bruno Meneguele --- drivers/net/ethernet/broadcom/bnxt/bnxt.c | 47 +++++++++++++++++++++++---- drivers/net/ethernet/broadcom/bnxt/bnxt.h | 3 ++ drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.c | 3 +- 3 files changed, 46 insertions(+), 7 deletions(-) Index: src/drivers/net/ethernet/broadcom/bnxt/bnxt.c =================================================================== --- src.orig/drivers/net/ethernet/broadcom/bnxt/bnxt.c 2020-02-06 16:23:14.390525268 +0100 +++ src/drivers/net/ethernet/broadcom/bnxt/bnxt.c 2020-02-06 16:23:14.547523827 +0100 @@ -54,6 +54,7 @@ #include #include #include +#include #include "bnxt_hsi.h" #include "bnxt.h" @@ -670,19 +671,20 @@ } static struct page *__bnxt_alloc_rx_page(struct bnxt *bp, dma_addr_t *mapping, + struct bnxt_rx_ring_info *rxr, gfp_t gfp) { struct device *dev = &bp->pdev->dev; struct page *page; - page = alloc_page(gfp); + page = page_pool_dev_alloc_pages(rxr->page_pool); if (!page) return NULL; *mapping = dma_map_page_attrs(dev, page, 0, PAGE_SIZE, bp->rx_dir, DMA_ATTR_WEAK_ORDERING); if (dma_mapping_error(dev, *mapping)) { - __free_page(page); + page_pool_recycle_direct(rxr->page_pool, page); return NULL; } *mapping += bp->rx_dma_offset; @@ -718,7 +720,8 @@ dma_addr_t mapping; if (BNXT_RX_PAGE_MODE(bp)) { - struct page *page = __bnxt_alloc_rx_page(bp, &mapping, gfp); + struct page *page = + __bnxt_alloc_rx_page(bp, &mapping, rxr, gfp); if (!page) return -ENOMEM; @@ -2363,7 +2366,7 @@ dma_unmap_page_attrs(&pdev->dev, mapping, PAGE_SIZE, bp->rx_dir, DMA_ATTR_WEAK_ORDERING); - __free_page(data); + page_pool_recycle_direct(rxr->page_pool, data); } else { dma_unmap_single_attrs(&pdev->dev, mapping, bp->rx_buf_use_size, @@ -2500,6 +2503,8 @@ if (xdp_rxq_info_is_reg(&rxr->xdp_rxq)) xdp_rxq_info_unreg(&rxr->xdp_rxq); + rxr->page_pool = NULL; + kfree(rxr->rx_tpa); rxr->rx_tpa = NULL; @@ -2514,6 +2519,26 @@ } } +static int bnxt_alloc_rx_page_pool(struct bnxt *bp, + struct bnxt_rx_ring_info *rxr) +{ + struct page_pool_params pp = { 0 }; + + pp.pool_size = bp->rx_ring_size; + pp.nid = dev_to_node(&bp->pdev->dev); + pp.dev = &bp->pdev->dev; + pp.dma_dir = DMA_BIDIRECTIONAL; + + rxr->page_pool = page_pool_create(&pp); + if (IS_ERR(rxr->page_pool)) { + int err = PTR_ERR(rxr->page_pool); + + rxr->page_pool = NULL; + return err; + } + return 0; +} + static int bnxt_alloc_rx_rings(struct bnxt *bp) { int i, rc, agg_rings = 0, tpa_rings = 0; @@ -2533,14 +2558,24 @@ ring = &rxr->rx_ring_struct; + rc = bnxt_alloc_rx_page_pool(bp, rxr); + if (rc) + return rc; + rc = xdp_rxq_info_reg(&rxr->xdp_rxq, bp->dev, i); - if (rc < 0) + if (rc < 0) { + page_pool_free(rxr->page_pool); + rxr->page_pool = NULL; return rc; + } rc = xdp_rxq_info_reg_mem_model(&rxr->xdp_rxq, - MEM_TYPE_PAGE_SHARED, NULL); + MEM_TYPE_PAGE_POOL, + rxr->page_pool); if (rc) { xdp_rxq_info_unreg(&rxr->xdp_rxq); + page_pool_free(rxr->page_pool); + rxr->page_pool = NULL; return rc; } Index: src/drivers/net/ethernet/broadcom/bnxt/bnxt.h =================================================================== --- src.orig/drivers/net/ethernet/broadcom/bnxt/bnxt.h 2020-02-06 16:23:14.390525268 +0100 +++ src/drivers/net/ethernet/broadcom/bnxt/bnxt.h 2020-02-06 16:23:14.548523818 +0100 @@ -26,6 +26,8 @@ #include #include +struct page_pool; + struct tx_bd { __le32 tx_bd_len_flags_type; #define TX_BD_TYPE (0x3f << 0) @@ -799,6 +801,7 @@ struct bnxt_ring_struct rx_ring_struct; struct bnxt_ring_struct rx_agg_ring_struct; struct xdp_rxq_info xdp_rxq; + struct page_pool *page_pool; }; struct bnxt_cp_ring_info { Index: src/drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.c =================================================================== --- src.orig/drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.c 2020-02-06 16:23:14.390525268 +0100 +++ src/drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.c 2020-02-06 16:23:14.548523818 +0100 @@ -15,6 +15,7 @@ #include #include #include +#include #include "bnxt_hsi.h" #include "bnxt.h" #include "bnxt_xdp.h" @@ -191,7 +192,7 @@ if (xdp_do_redirect(bp->dev, &xdp, xdp_prog)) { trace_xdp_exception(bp->dev, xdp_prog, act); - __free_page(page); + page_pool_recycle_direct(rxr->page_pool, page); return true; }