summaryrefslogtreecommitdiff
path: root/include/linux
diff options
context:
space:
mode:
authorJakub Kicinski2024-04-02 18:13:51 -0700
committerJakub Kicinski2024-04-02 18:13:51 -0700
commiteb05529a106a2803f1360952041e9bf5a94183e2 (patch)
tree78701c158045e79d05961247fd068dad9fe12d3c /include/linux
parent8db2509faa331865903a81a92f15c449e821b1d7 (diff)
parent39806b96c89ae5d52092c8f86393ecbfaae26697 (diff)
Merge branch 'page_pool-allow-direct-bulk-recycling'
Alexander Lobakin says: ==================== page_pool: allow direct bulk recycling Previously, there was no reliable way to check whether it's safe to use direct PP cache. The drivers were passing @allow_direct to the PP recycling functions and that was it. Bulk recycling is used by xdp_return_frame_bulk() on .ndo_xdp_xmit() frames completion where the page origin is unknown, thus the direct recycling has never been tried. Now that we have at least 2 ways of checking if we're allowed to perform direct recycling -- pool->p.napi (Jakub) and pool->cpuid (Lorenzo), we can use them when doing bulk recycling as well. Just move that logic from the skb core to the PP core and call it before __page_pool_put_page() every time @allow_direct is false. Under high .ndo_xdp_xmit() traffic load, the win is 2-3% Pps assuming the sending driver uses xdp_return_frame_bulk() on Tx completion. ==================== Link: https://lore.kernel.org/r/20240329165507.3240110-1-aleksander.lobakin@intel.com Signed-off-by: Jakub Kicinski <kuba@kernel.org>
Diffstat (limited to 'include/linux')
-rw-r--r--include/linux/skbuff.h12
1 files changed, 6 insertions, 6 deletions
diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h
index b7f1ecdaec38..03ea36a82cdd 100644
--- a/include/linux/skbuff.h
+++ b/include/linux/skbuff.h
@@ -3510,25 +3510,25 @@ int skb_pp_cow_data(struct page_pool *pool, struct sk_buff **pskb,
unsigned int headroom);
int skb_cow_data_for_xdp(struct page_pool *pool, struct sk_buff **pskb,
struct bpf_prog *prog);
-bool napi_pp_put_page(struct page *page, bool napi_safe);
+bool napi_pp_put_page(struct page *page);
static inline void
-skb_page_unref(const struct sk_buff *skb, struct page *page, bool napi_safe)
+skb_page_unref(const struct sk_buff *skb, struct page *page)
{
#ifdef CONFIG_PAGE_POOL
- if (skb->pp_recycle && napi_pp_put_page(page, napi_safe))
+ if (skb->pp_recycle && napi_pp_put_page(page))
return;
#endif
put_page(page);
}
static inline void
-napi_frag_unref(skb_frag_t *frag, bool recycle, bool napi_safe)
+napi_frag_unref(skb_frag_t *frag, bool recycle)
{
struct page *page = skb_frag_page(frag);
#ifdef CONFIG_PAGE_POOL
- if (recycle && napi_pp_put_page(page, napi_safe))
+ if (recycle && napi_pp_put_page(page))
return;
#endif
put_page(page);
@@ -3544,7 +3544,7 @@ napi_frag_unref(skb_frag_t *frag, bool recycle, bool napi_safe)
*/
static inline void __skb_frag_unref(skb_frag_t *frag, bool recycle)
{
- napi_frag_unref(frag, recycle, false);
+ napi_frag_unref(frag, recycle);
}
/**