* [PATCH net 1/4] net: skbuff: limit folio refcount when copying skb frag references
2026-11-18 8:32 [PATCH net 0/4] net: skbuff: limit pagecache folio refcount David Hildenbrand (Arm)
@ 2026-11-18 8:32 ` David Hildenbrand (Arm)
2026-11-18 8:32 ` [PATCH net 2/4] net: skbuff: limit folio refcount when rearranging skb frags David Hildenbrand (Arm)
` (2 subsequent siblings)
3 siblings, 0 replies; 5+ messages in thread
From: David Hildenbrand (Arm) @ 2026-11-18 8:32 UTC (permalink / raw)
To: David S. Miller, Eric Dumazet, Jakub Kicinski, Paolo Abeni,
Simon Horman, Andrew Morton, Kiryl Shutsemau, Song Liu,
Johannes Weiner, Rik van Riel, Matthew Wilcox (Oracle),
Neal Cardwell, Kuniyuki Iwashima, Pedro Falcato
Cc: netdev, linux-kernel, linux-mm, David Hildenbrand (Arm), stable
Various networking features copy frag references of an skb, for example
when unsharing a cloned skb (AF_PACKET trimming a clone), when copying an
skb (nft dup) or when building NFQUEUE messages. This is an effective
way for multiplying folio references held by an skb, allowing for
exceeding folio refcount limits.
This is not specific to large pagecache folios, but with them it is a
lot easier to reach the refcount limit, as page table mappings can cheaply
create many references.
Let's introduce skb_frag_try_ref() and skb_frags_try_ref(), which use
try_get_page(), and use them in pskb_expand_head(), __pskb_copy_fclone(),
skb_zerocopy() and pskb_carve_inside_*(). All of these can already fail,
and their callers handle that.
Fixes: 800d8c63b2e9 ("shmem: add huge pages support")
Fixes: 99cb0dbd47a1 ("mm,thp: add read-only THP support for (non-shmem) FS")
Fixes: 793917d997df ("mm/readahead: Add large folio readahead")
Cc: stable@vger.kernel.org
Acked-by: Pedro Falcato <pfalcato@suse.de>
Signed-off-by: David Hildenbrand (Arm) <david@kernel.org>
---
include/linux/skbuff_ref.h | 53 ++++++++++++++++++++++++++++++++++++++++++++++
net/core/skbuff.c | 35 +++++++++++++++++++-----------
2 files changed, 76 insertions(+), 12 deletions(-)
diff --git a/include/linux/skbuff_ref.h b/include/linux/skbuff_ref.h
index b6f6da556ee90..560ddc863bbec 100644
--- a/include/linux/skbuff_ref.h
+++ b/include/linux/skbuff_ref.h
@@ -44,6 +44,33 @@ static __always_inline void skb_frag_ref(struct sk_buff *skb, int f)
skb->pp_recycle);
}
+/**
+ * skb_frag_try_ref - try to acquire a reference on a paged fragment of an skb
+ * @skb: the buffer
+ * @f: the fragment offset
+ *
+ * Like skb_frag_ref(), but acquires non-page_pool page references using
+ * try_get_page(), refusing additional references to a page whose refcount is
+ * at its limit.
+ *
+ * Return: True if a reference was acquired, otherwise false.
+ */
+static __always_inline __must_check bool skb_frag_try_ref(struct sk_buff *skb,
+ int f)
+{
+ netmem_ref netmem = skb_frag_netmem(&skb_shinfo(skb)->frags[f]);
+
+#ifdef CONFIG_PAGE_POOL
+ if (skb->pp_recycle && napi_pp_get_page(netmem))
+ return true;
+#endif
+ if (netmem_is_net_iov(netmem)) {
+ net_iov_inc_ref(netmem_to_net_iov(netmem));
+ return true;
+ }
+ return try_get_page(netmem_to_page(netmem));
+}
+
bool napi_pp_put_page(netmem_ref netmem);
/**
@@ -83,4 +110,30 @@ static __always_inline void skb_frag_unref(struct sk_buff *skb, int f)
skb->pp_recycle);
}
+/**
+ * skb_frags_try_ref - try to acquire references on all paged fragments
+ * @skb: the buffer
+ *
+ * Try to acquire a reference on each paged fragment of @skb using
+ * skb_frag_try_ref(). On failure, acquired references are dropped.
+ *
+ * Must not be used on skbs with managed frag references.
+ *
+ * Return: True if all references were acquired, otherwise false.
+ */
+static inline __must_check bool skb_frags_try_ref(struct sk_buff *skb)
+{
+ int i;
+
+ DEBUG_NET_WARN_ON_ONCE(skb_zcopy_managed(skb));
+ for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
+ if (unlikely(!skb_frag_try_ref(skb, i))) {
+ while (i--)
+ skb_frag_unref(skb, i);
+ return false;
+ }
+ }
+ return true;
+}
+
#endif /* _LINUX_SKBUFF_REF_H */
diff --git a/net/core/skbuff.c b/net/core/skbuff.c
index 1ac1e327ac4c6..447099b39deeb 100644
--- a/net/core/skbuff.c
+++ b/net/core/skbuff.c
@@ -2266,7 +2266,13 @@ struct sk_buff *__pskb_copy_fclone(struct sk_buff *skb, int headroom,
n->pp_recycle = skb->pp_recycle;
for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
skb_shinfo(n)->frags[i] = skb_shinfo(skb)->frags[i];
- skb_frag_ref(n, i);
+ if (!skb_frag_try_ref(n, i)) {
+ /* Drop the references acquired so far. */
+ skb_shinfo(n)->nr_frags = i;
+ kfree_skb(n);
+ n = NULL;
+ goto out;
+ }
}
skb_shinfo(n)->nr_frags = i;
skb_shinfo(n)->flags |= skb_shinfo(skb)->flags & SKBFL_SHARED_FRAG;
@@ -2310,7 +2316,6 @@ int pskb_expand_head(struct sk_buff *skb, int nhead, int ntail,
unsigned int size = osize + nhead + ntail;
long off;
u8 *data;
- int i;
if (unlikely(nhead < 0)) {
DEBUG_NET_WARN_ON_ONCE(1);
@@ -2347,12 +2352,10 @@ int pskb_expand_head(struct sk_buff *skb, int nhead, int ntail,
* be since all we did is relocate the values
*/
if (skb_cloned(skb)) {
- if (skb_orphan_frags(skb, gfp_mask))
+ if (skb_orphan_frags(skb, gfp_mask) || !skb_frags_try_ref(skb))
goto nofrags;
if (skb_zcopy(skb))
net_zcopy_get(skb_uarg(skb));
- for (i = 0; i < skb_shinfo(skb)->nr_frags; i++)
- skb_frag_ref(skb, i);
if (skb_has_frag_list(skb))
skb_clone_fraglist(skb);
@@ -3908,7 +3911,8 @@ EXPORT_SYMBOL_GPL(skb_zerocopy_headlen);
*
* Return value:
* 0: everything is OK
- * -ENOMEM: couldn't orphan frags of @from due to lack of memory
+ * -ENOMEM: couldn't orphan frags of @from due to lack of memory, or
+ * couldn't take additional references to the frags of @from
* -EFAULT: skb_copy_bits() found some problem with skb geometry, or readable head
* payload would be mixed with unreadable frags.
*/
@@ -3973,7 +3977,11 @@ skb_zerocopy(struct sk_buff *to, struct sk_buff *from, int len, int hlen)
len);
skb_frag_size_set(&skb_shinfo(to)->frags[j], size);
len -= size;
- skb_frag_ref(to, j);
+ if (!skb_frag_try_ref(to, j)) {
+ /* The caller frees @to, dropping the acquired refs. */
+ skb_shinfo(to)->nr_frags = j;
+ return -ENOMEM;
+ }
j++;
}
skb_shinfo(to)->nr_frags = j;
@@ -7004,7 +7012,6 @@ static void skb_carve_reset_headers(struct sk_buff *skb)
static int pskb_carve_inside_header(struct sk_buff *skb, const u32 off,
const int headlen, gfp_t gfp_mask)
{
- int i;
unsigned int size = skb_end_offset(skb);
int new_hlen = headlen - off;
u8 *data;
@@ -7032,14 +7039,12 @@ static int pskb_carve_inside_header(struct sk_buff *skb, const u32 off,
frags[skb_shinfo(skb)->nr_frags]));
if (skb_cloned(skb)) {
/* drop the old head gracefully */
- if (skb_orphan_frags(skb, gfp_mask)) {
+ if (skb_orphan_frags(skb, gfp_mask) || !skb_frags_try_ref(skb)) {
skb_kfree_head(data);
return -ENOMEM;
}
if (skb_zcopy(skb))
net_zcopy_get(skb_uarg(skb));
- for (i = 0; i < skb_shinfo(skb)->nr_frags; i++)
- skb_frag_ref(skb, i);
if (skb_has_frag_list(skb))
skb_clone_fraglist(skb);
skb_release_data(skb, SKB_CONSUMED);
@@ -7169,7 +7174,13 @@ static int pskb_carve_inside_nonlinear(struct sk_buff *skb, const u32 off,
skb_frag_off_add(&shinfo->frags[0], off - pos);
skb_frag_size_sub(&shinfo->frags[0], off - pos);
}
- skb_frag_ref(skb, i);
+ if (!skb_frag_try_ref(skb, i)) {
+ /* We referenced frags i - k ... i - 1 so far. */
+ while (k--)
+ skb_frag_unref(skb, --i);
+ skb_kfree_head(data);
+ return -ENOMEM;
+ }
k++;
}
pos += fsize;
--
2.43.0
^ permalink raw reply [flat|nested] 5+ messages in thread* [PATCH net 2/4] net: skbuff: limit folio refcount when rearranging skb frags
2026-11-18 8:32 [PATCH net 0/4] net: skbuff: limit pagecache folio refcount David Hildenbrand (Arm)
2026-11-18 8:32 ` [PATCH net 1/4] net: skbuff: limit folio refcount when copying skb frag references David Hildenbrand (Arm)
@ 2026-11-18 8:32 ` David Hildenbrand (Arm)
2026-11-18 8:32 ` [PATCH net 3/4] net: skbuff: limit folio refcount when splicing pages into skbs David Hildenbrand (Arm)
2026-11-18 8:32 ` [PATCH net 4/4] net: skbuff: limit folio refcount when splicing from sockets David Hildenbrand (Arm)
3 siblings, 0 replies; 5+ messages in thread
From: David Hildenbrand (Arm) @ 2026-11-18 8:32 UTC (permalink / raw)
To: David S. Miller, Eric Dumazet, Jakub Kicinski, Paolo Abeni,
Simon Horman, Andrew Morton, Kiryl Shutsemau, Song Liu,
Johannes Weiner, Rik van Riel, Matthew Wilcox (Oracle),
Neal Cardwell, Kuniyuki Iwashima, Pedro Falcato
Cc: netdev, linux-kernel, linux-mm, David Hildenbrand (Arm), stable
Various networking features take additional references to frag pages
that are still referenced elsewhere when rearranging skb frags, for
example software GSO segmentation, TCP receive coalescing of clones, SACK
shifting and MTU probing. This is an effective way for multiplying folio
references held by an skb, allowing for exceeding folio refcount limits.
This is not specific to large pagecache folios, but with them it is a
lot easier to reach the refcount limit, as page table mappings can cheaply
create many references.
Let's use skb_frag_try_ref() and skb_frags_try_ref() in skb_segment(),
skb_try_coalesce(), skb_shift() and tcp_clone_payload(). All of these can
already fail, and their callers handle that.
skb_split() can still add one reference when splitting a frag. But each
split also allocates a new skb that is charged to the socket, reaching
the refcount limit that way would require a lot of socket memory, and the
references go away once the data was acked, so it's not that relevant in
practice. Making skb_split() fail is more involved, so let's leave it alone
for now.
Fixes: 800d8c63b2e9 ("shmem: add huge pages support")
Fixes: 99cb0dbd47a1 ("mm,thp: add read-only THP support for (non-shmem) FS")
Fixes: 793917d997df ("mm/readahead: Add large folio readahead")
Cc: stable@vger.kernel.org
Acked-by: Pedro Falcato <pfalcato@suse.de>
Signed-off-by: David Hildenbrand (Arm) <david@kernel.org>
---
net/core/skbuff.c | 32 +++++++++++++++++++++-----------
net/ipv4/tcp_output.c | 8 +++++---
2 files changed, 26 insertions(+), 14 deletions(-)
diff --git a/net/core/skbuff.c b/net/core/skbuff.c
index 447099b39deeb..bed1c8c4c6b99 100644
--- a/net/core/skbuff.c
+++ b/net/core/skbuff.c
@@ -4338,7 +4338,8 @@ static int skb_prepare_for_shift(struct sk_buff *skb)
* the length of the skb, from skb to tgt. Returns number bytes shifted.
* It's up to caller to free skb if everything was shifted.
*
- * If @tgt runs out of frags, the whole operation is aborted.
+ * If @tgt runs out of frags, or we cannot take an additional frag reference,
+ * the whole operation is aborted.
*
* Skb cannot include anything else but paged data while tgt is allowed
* to have non-paged data as well.
@@ -4421,7 +4422,9 @@ int skb_shift(struct sk_buff *tgt, struct sk_buff *skb, int shiftlen)
} else {
skb_frag_page_copy(fragto, fragfrom);
- skb_frag_ref(tgt, to);
+ /* Nothing has been committed yet. */
+ if (!skb_frag_try_ref(tgt, to))
+ return 0;
skb_frag_off_copy(fragto, fragfrom);
skb_frag_size_set(fragto, todo);
@@ -5160,7 +5163,10 @@ struct sk_buff *skb_segment(struct sk_buff *head_skb,
if (!skb_shinfo(nskb)->nr_frags)
nskb->pp_recycle = frag_skb->pp_recycle;
*nskb_frag = (i < 0) ? skb_head_frag_to_page_desc(frag_skb) : *frag;
- skb_frag_ref(nskb, skb_shinfo(nskb)->nr_frags);
+ if (!skb_frag_try_ref(nskb, skb_shinfo(nskb)->nr_frags)) {
+ err = -ENOMEM;
+ goto err;
+ }
size = skb_frag_size(nskb_frag);
if (pos < offset) {
@@ -6301,7 +6307,8 @@ bool skb_try_coalesce(struct sk_buff *to, struct sk_buff *from,
bool *fragstolen, int *delta_truesize)
{
struct skb_shared_info *to_shinfo, *from_shinfo;
- int i, delta, len = from->len;
+ int delta, len = from->len;
+ bool cloned;
*fragstolen = false;
@@ -6374,6 +6381,15 @@ bool skb_try_coalesce(struct sk_buff *to, struct sk_buff *from,
WARN_ON_ONCE(delta < len);
+ /*
+ * A clone keeps referencing its frags, so we need our own references.
+ * Nothing was added to @to yet: we only steal the head of @from above
+ * if it is not locked, and the head of a clone is always locked.
+ */
+ cloned = skb_cloned(from);
+ if (cloned && !skb_frags_try_ref(from))
+ return false;
+
memcpy(to_shinfo->frags + to_shinfo->nr_frags,
from_shinfo->frags,
from_shinfo->nr_frags * sizeof(skb_frag_t));
@@ -6381,15 +6397,9 @@ bool skb_try_coalesce(struct sk_buff *to, struct sk_buff *from,
if (from_shinfo->nr_frags)
to_shinfo->flags |= from_shinfo->flags & SKBFL_SHARED_FRAG;
- if (!skb_cloned(from))
+ if (!cloned)
from_shinfo->nr_frags = 0;
- /* if the skb is not cloned this does nothing
- * since we set nr_frags to 0.
- */
- for (i = 0; i < from_shinfo->nr_frags; i++)
- skb_frag_ref(from, i);
-
to->truesize += delta;
to->len += len;
to->data_len += len;
diff --git a/net/ipv4/tcp_output.c b/net/ipv4/tcp_output.c
index cf3ddc055d48d..043838d996404 100644
--- a/net/ipv4/tcp_output.c
+++ b/net/ipv4/tcp_output.c
@@ -2710,10 +2710,12 @@ static int tcp_clone_payload(struct sock *sk, struct sk_buff *to,
}
commit:
WARN_ON_ONCE(len != probe_size);
- for (i = 0; i < nr_frags; i++)
- skb_frag_ref(to, i);
-
skb_shinfo(to)->nr_frags = nr_frags;
+ if (!skb_frags_try_ref(to)) {
+ skb_shinfo(to)->nr_frags = 0;
+ return -ENOMEM;
+ }
+
to->truesize += probe_size;
to->len += probe_size;
to->data_len += probe_size;
--
2.43.0
^ permalink raw reply [flat|nested] 5+ messages in thread* [PATCH net 3/4] net: skbuff: limit folio refcount when splicing pages into skbs
2026-11-18 8:32 [PATCH net 0/4] net: skbuff: limit pagecache folio refcount David Hildenbrand (Arm)
2026-11-18 8:32 ` [PATCH net 1/4] net: skbuff: limit folio refcount when copying skb frag references David Hildenbrand (Arm)
2026-11-18 8:32 ` [PATCH net 2/4] net: skbuff: limit folio refcount when rearranging skb frags David Hildenbrand (Arm)
@ 2026-11-18 8:32 ` David Hildenbrand (Arm)
2026-11-18 8:32 ` [PATCH net 4/4] net: skbuff: limit folio refcount when splicing from sockets David Hildenbrand (Arm)
3 siblings, 0 replies; 5+ messages in thread
From: David Hildenbrand (Arm) @ 2026-11-18 8:32 UTC (permalink / raw)
To: David S. Miller, Eric Dumazet, Jakub Kicinski, Paolo Abeni,
Simon Horman, Andrew Morton, Kiryl Shutsemau, Song Liu,
Johannes Weiner, Rik van Riel, Matthew Wilcox (Oracle),
Neal Cardwell, Kuniyuki Iwashima, Pedro Falcato
Cc: netdev, linux-kernel, linux-mm, David Hildenbrand (Arm), stable
Various networking features use MSG_SPLICE_PAGES to add pages to skbs,
whereby skb_append_pagefrags() takes a page reference for each fragment it
adds. Pages spliced from pipes were already checked when entering the
pipe, but in-kernel users like sunrpc pass pagecache pages directly. This
allows for exceeding folio refcount limits.
This is not specific to large pagecache folios, but with them it is a
lot easier to reach the refcount limit, as page table mappings can cheaply
create many references.
Let's use try_get_page() in skb_append_pagefrags() and fail with -EFAULT.
Don't use -EMSGSIZE, as TCP would just retry with a new skb. The only
caller, skb_splice_from_iter(), handles that.
Fixes: 800d8c63b2e9 ("shmem: add huge pages support")
Fixes: 99cb0dbd47a1 ("mm,thp: add read-only THP support for (non-shmem) FS")
Fixes: 793917d997df ("mm/readahead: Add large folio readahead")
Cc: stable@vger.kernel.org
Acked-by: Pedro Falcato <pfalcato@suse.de>
Signed-off-by: David Hildenbrand (Arm) <david@kernel.org>
---
net/core/skbuff.c | 12 +++++++++---
1 file changed, 9 insertions(+), 3 deletions(-)
diff --git a/net/core/skbuff.c b/net/core/skbuff.c
index bed1c8c4c6b99..c937d74eed6ac 100644
--- a/net/core/skbuff.c
+++ b/net/core/skbuff.c
@@ -4714,8 +4714,13 @@ int skb_append_pagefrags(struct sk_buff *skb, struct page *page,
if (skb_can_coalesce(skb, i, page, offset)) {
skb_frag_size_add(&skb_shinfo(skb)->frags[i - 1], size);
} else if (i < max_frags) {
+ /*
+ * Don't use -EMSGSIZE, as callers would just retry with a new
+ * skb.
+ */
+ if (!try_get_page(page))
+ return -EFAULT;
skb_zcopy_downgrade_managed(skb);
- get_page(page);
skb_fill_page_desc_noacc(skb, i, page, offset, size);
} else {
return -EMSGSIZE;
@@ -7613,8 +7618,9 @@ static void skb_splice_csum_page(struct sk_buff *skb, struct page *page,
* possible, copying them to fragments if not possible (such as if they're slab
* pages).
*
- * Returns the amount of data spliced/copied or -EMSGSIZE if there's
- * insufficient space in the buffer to transfer anything.
+ * Returns the amount of data spliced/copied, -EMSGSIZE if there's
+ * insufficient space in the buffer to transfer anything, or -EFAULT if we
+ * could not take a page reference.
*/
ssize_t skb_splice_from_iter(struct sk_buff *skb, struct iov_iter *iter,
ssize_t maxsize)
--
2.43.0
^ permalink raw reply [flat|nested] 5+ messages in thread* [PATCH net 4/4] net: skbuff: limit folio refcount when splicing from sockets
2026-11-18 8:32 [PATCH net 0/4] net: skbuff: limit pagecache folio refcount David Hildenbrand (Arm)
` (2 preceding siblings ...)
2026-11-18 8:32 ` [PATCH net 3/4] net: skbuff: limit folio refcount when splicing pages into skbs David Hildenbrand (Arm)
@ 2026-11-18 8:32 ` David Hildenbrand (Arm)
3 siblings, 0 replies; 5+ messages in thread
From: David Hildenbrand (Arm) @ 2026-11-18 8:32 UTC (permalink / raw)
To: David S. Miller, Eric Dumazet, Jakub Kicinski, Paolo Abeni,
Simon Horman, Andrew Morton, Kiryl Shutsemau, Song Liu,
Johannes Weiner, Rik van Riel, Matthew Wilcox (Oracle),
Neal Cardwell, Kuniyuki Iwashima, Pedro Falcato
Cc: netdev, linux-kernel, linux-mm, David Hildenbrand (Arm), stable
Various networking features support splicing data from sockets into
pipes (TCP, AF_UNIX, KCM, kTLS), whereby spd_fill_page() takes an
additional page reference for each fragment, while the skb keeps its own
reference until it was fully consumed. This allows for exceeding folio
refcount limits.
This is not specific to large pagecache folios, but with them it is a
lot easier to reach the refcount limit, as page table mappings can cheaply
create many references.
Let's use try_get_page() in spd_fill_page() and stop splicing with
-EFAULT if that fails. To propagate the error, let spd_fill_page(),
__splice_segment() and __skb_splice_bits() return an int instead of a
bool. Keep returning true/false instead of 1/0 to reduce the churn in
this fix.
If we could not splice anything, return the error from skb_splice_bits():
returning 0 would make the callers (TCP, AF_UNIX, KCM, kTLS) report EOF,
although data is still queued. The callers already handle errors.
Use -EFAULT, consistent with splice_folio_into_pipe().
Fixes: 800d8c63b2e9 ("shmem: add huge pages support")
Fixes: 99cb0dbd47a1 ("mm,thp: add read-only THP support for (non-shmem) FS")
Fixes: 793917d997df ("mm/readahead: Add large folio readahead")
Cc: stable@vger.kernel.org
Acked-by: Pedro Falcato <pfalcato@suse.de>
Signed-off-by: David Hildenbrand (Arm) <david@kernel.org>
---
net/core/skbuff.c | 90 +++++++++++++++++++++++++++++++------------------------
1 file changed, 51 insertions(+), 39 deletions(-)
diff --git a/net/core/skbuff.c b/net/core/skbuff.c
index c937d74eed6ac..b73fef7a1b460 100644
--- a/net/core/skbuff.c
+++ b/net/core/skbuff.c
@@ -3162,11 +3162,13 @@ static bool spd_can_coalesce(const struct splice_pipe_desc *spd,
}
/*
- * Fill page/offset/length into spd, if it can hold more pages.
+ * Fill page/offset/length into spd, if it can hold more pages and we can
+ * take a page reference. Returns non-zero to stop, or a negative error
+ * if we cannot take a page reference.
*/
-static bool spd_fill_page(struct splice_pipe_desc *spd, struct page *page,
- unsigned int *len, unsigned int offset, bool linear,
- struct sock *sk)
+static int spd_fill_page(struct splice_pipe_desc *spd, struct page *page,
+ unsigned int *len, unsigned int offset, bool linear,
+ struct sock *sk)
{
if (unlikely(spd->nr_pages == MAX_SKB_FRAGS))
return true;
@@ -3180,7 +3182,8 @@ static bool spd_fill_page(struct splice_pipe_desc *spd, struct page *page,
spd->partial[spd->nr_pages - 1].len += *len;
return false;
}
- get_page(page);
+ if (!try_get_page(page))
+ return -EFAULT;
spd->pages[spd->nr_pages] = page;
spd->partial[spd->nr_pages].len = *len;
spd->partial[spd->nr_pages].offset = offset;
@@ -3189,12 +3192,14 @@ static bool spd_fill_page(struct splice_pipe_desc *spd, struct page *page,
return false;
}
-static bool __splice_segment(struct page *page, unsigned int poff,
- unsigned int plen, unsigned int *off,
- unsigned int *len,
- struct splice_pipe_desc *spd, bool linear,
- struct sock *sk)
+static int __splice_segment(struct page *page, unsigned int poff,
+ unsigned int plen, unsigned int *off,
+ unsigned int *len,
+ struct splice_pipe_desc *spd, bool linear,
+ struct sock *sk)
{
+ int ret;
+
if (!*len)
return true;
@@ -3212,8 +3217,9 @@ static bool __splice_segment(struct page *page, unsigned int poff,
do {
unsigned int flen = min(*len, plen);
- if (spd_fill_page(spd, page, &flen, poff, linear, sk))
- return true;
+ ret = spd_fill_page(spd, page, &flen, poff, linear, sk);
+ if (ret)
+ return ret;
poff += flen;
plen -= flen;
*len -= flen;
@@ -3225,28 +3231,30 @@ static bool __splice_segment(struct page *page, unsigned int poff,
}
/*
- * Map linear and fragment data from the skb to spd. It reports true if the
- * pipe is full or if we already spliced the requested length.
+ * Map linear and fragment data from the skb to spd. It returns non-zero to
+ * stop: true if the pipe is full or if we already spliced the requested
+ * length, or a negative error if we cannot take a page reference.
*/
-static bool __skb_splice_bits(struct sk_buff *skb, struct pipe_inode_info *pipe,
- unsigned int *offset, unsigned int *len,
- struct splice_pipe_desc *spd, struct sock *sk)
+static int __skb_splice_bits(struct sk_buff *skb, struct pipe_inode_info *pipe,
+ unsigned int *offset, unsigned int *len,
+ struct splice_pipe_desc *spd, struct sock *sk)
{
struct sk_buff *iter;
- int seg;
+ int seg, ret;
/* map the linear part :
* If skb->head_frag is set, this 'linear' part is backed by a
* fragment, and if the head is not shared with any clones then
* we can avoid a copy since we own the head portion of this page.
*/
- if (__splice_segment(virt_to_page(skb->data),
- (unsigned long) skb->data & (PAGE_SIZE - 1),
- skb_headlen(skb),
- offset, len, spd,
- skb_head_is_locked(skb),
- sk))
- return true;
+ ret = __splice_segment(virt_to_page(skb->data),
+ (unsigned long)skb->data & (PAGE_SIZE - 1),
+ skb_headlen(skb),
+ offset, len, spd,
+ skb_head_is_locked(skb),
+ sk);
+ if (ret)
+ return ret;
/*
* then map the fragments
@@ -3260,10 +3268,11 @@ static bool __skb_splice_bits(struct sk_buff *skb, struct pipe_inode_info *pipe,
if (WARN_ON_ONCE(!skb_frag_page(f)))
return false;
- if (__splice_segment(skb_frag_page(f),
- skb_frag_off(f), skb_frag_size(f),
- offset, len, spd, false, sk))
- return true;
+ ret = __splice_segment(skb_frag_page(f),
+ skb_frag_off(f), skb_frag_size(f),
+ offset, len, spd, false, sk);
+ if (ret)
+ return ret;
}
skb_walk_frags(skb, iter) {
@@ -3272,11 +3281,12 @@ static bool __skb_splice_bits(struct sk_buff *skb, struct pipe_inode_info *pipe,
continue;
}
/* __skb_splice_bits() only fails if the output has no room
- * left, so no point in going over the frag_list for the error
- * case.
+ * left or on error, so no point in going over the frag_list
+ * for the error case.
*/
- if (__skb_splice_bits(iter, pipe, offset, len, spd, sk))
- return true;
+ ret = __skb_splice_bits(iter, pipe, offset, len, spd, sk);
+ if (ret)
+ return ret;
}
return false;
@@ -3284,7 +3294,9 @@ static bool __skb_splice_bits(struct sk_buff *skb, struct pipe_inode_info *pipe,
/*
* Map data from the skb to a pipe. Should handle both the linear part,
- * the fragments, and the frag list.
+ * the fragments, and the frag list. Returns the number of bytes spliced,
+ * or an error if we could not splice anything because we could not take
+ * a page reference.
*/
int skb_splice_bits(struct sk_buff *skb, struct sock *sk, unsigned int offset,
struct pipe_inode_info *pipe, unsigned int tlen,
@@ -3299,14 +3311,14 @@ int skb_splice_bits(struct sk_buff *skb, struct sock *sk, unsigned int offset,
.ops = &default_pipe_buf_ops,
.spd_release = sock_spd_release,
};
- int ret = 0;
+ int ret;
- __skb_splice_bits(skb, pipe, &offset, &tlen, &spd, sk);
+ ret = __skb_splice_bits(skb, pipe, &offset, &tlen, &spd, sk);
if (spd.nr_pages)
- ret = splice_to_pipe(pipe, &spd);
-
- return ret;
+ return splice_to_pipe(pipe, &spd);
+ /* Returning 0 would make callers report EOF. */
+ return ret < 0 ? ret : 0;
}
EXPORT_SYMBOL_GPL(skb_splice_bits);
--
2.43.0
^ permalink raw reply [flat|nested] 5+ messages in thread