bianbu-repo-mirror/bianbu-linux-6.6
1
0
Fork 0
mirror of https://gitee.com/bianbu-linux/linux-6.6 synced 2025-04-24 14:07:52 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

net: shrink struct ubuf_info

Browse source 
We can benefit from a smaller struct ubuf_info, so leave only mandatory
fields and let users to decide how they want to extend it. Convert
MSG_ZEROCOPY to struct ubuf_info_msgzc and remove duplicated fields.
This reduces the size from 48 bytes to just 16.

Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
Acked-by: Paolo Abeni <pabeni@redhat.com>
Signed-off-by: Jakub Kicinski <kuba@kernel.org>
This commit is contained in:
Pavel Begunkov 2022-09-23 17:39:04 +01:00 • committed by Jakub Kicinski
parent dfff202be5
commit e7d2b51016
5 changed files with 28 additions and 38 deletions
Download patch file Download diff file Expand all files Collapse all files

22
include/linux/skbuff.h
View file

@ -533,25 +533,8 @@ enum {
struct ubuf_info { struct ubuf_info {
void (*callback)(struct sk_buff *, struct ubuf_info *, void (*callback)(struct sk_buff *, struct ubuf_info *,
bool zerocopy_success); bool zerocopy_success);
union {
struct {
unsigned long desc;
void *ctx;
};
struct {
u32 id;
u16 len;
u16 zerocopy:1;
u32 bytelen;
};
};
refcount_t refcnt; refcount_t refcnt;
u8 flags; u8 flags;
struct mmpin {
struct user_struct *user;
unsigned int num_pg;
} mmp;
}; };
struct ubuf_info_msgzc { struct ubuf_info_msgzc {
@ -570,7 +553,10 @@ struct ubuf_info_msgzc {
}; };
}; };
struct mmpin mmp; struct mmpin {
struct user_struct *user;
unsigned int num_pg;
} mmp;
}; };
#define skb_uarg(SKB) ((struct ubuf_info *)(skb_shinfo(SKB)->destructor_arg)) #define skb_uarg(SKB) ((struct ubuf_info *)(skb_shinfo(SKB)->destructor_arg))

38
net/core/skbuff.c
View file

@ -1188,7 +1188,7 @@ EXPORT_SYMBOL_GPL(mm_unaccount_pinned_pages);
static struct ubuf_info *msg_zerocopy_alloc(struct sock *sk, size_t size) static struct ubuf_info *msg_zerocopy_alloc(struct sock *sk, size_t size)
{ {
struct ubuf_info *uarg; struct ubuf_info_msgzc *uarg;
struct sk_buff *skb; struct sk_buff *skb;
WARN_ON_ONCE(!in_task()); WARN_ON_ONCE(!in_task());
@ -1206,19 +1206,19 @@ static struct ubuf_info *msg_zerocopy_alloc(struct sock *sk, size_t size)
return NULL; return NULL;
} }
uarg->callback = msg_zerocopy_callback; uarg->ubuf.callback = msg_zerocopy_callback;
uarg->id = ((u32)atomic_inc_return(&sk->sk_zckey)) - 1; uarg->id = ((u32)atomic_inc_return(&sk->sk_zckey)) - 1;
uarg->len = 1; uarg->len = 1;
uarg->bytelen = size; uarg->bytelen = size;
uarg->zerocopy = 1; uarg->zerocopy = 1;
uarg->flags = SKBFL_ZEROCOPY_FRAG | SKBFL_DONT_ORPHAN; uarg->ubuf.flags = SKBFL_ZEROCOPY_FRAG | SKBFL_DONT_ORPHAN;
refcount_set(&uarg->refcnt, 1); refcount_set(&uarg->ubuf.refcnt, 1);
sock_hold(sk); sock_hold(sk);
return uarg; return &uarg->ubuf;
} }
static inline struct sk_buff *skb_from_uarg(struct ubuf_info *uarg) static inline struct sk_buff *skb_from_uarg(struct ubuf_info_msgzc *uarg)
{ {
return container_of((void *)uarg, struct sk_buff, cb); return container_of((void *)uarg, struct sk_buff, cb);
} }
@ -1227,6 +1227,7 @@ struct ubuf_info *msg_zerocopy_realloc(struct sock *sk, size_t size,
struct ubuf_info *uarg) struct ubuf_info *uarg)
{ {
if (uarg) { if (uarg) {
struct ubuf_info_msgzc *uarg_zc;
const u32 byte_limit = 1 << 19; /* limit to a few TSO */ const u32 byte_limit = 1 << 19; /* limit to a few TSO */
u32 bytelen, next; u32 bytelen, next;
@ -1242,8 +1243,9 @@ struct ubuf_info *msg_zerocopy_realloc(struct sock *sk, size_t size,
return NULL; return NULL;
} }
bytelen = uarg->bytelen + size; uarg_zc = uarg_to_msgzc(uarg);
if (uarg->len == USHRT_MAX - 1 || bytelen > byte_limit) { bytelen = uarg_zc->bytelen + size;
if (uarg_zc->len == USHRT_MAX - 1 || bytelen > byte_limit) {
/* TCP can create new skb to attach new uarg */ /* TCP can create new skb to attach new uarg */
if (sk->sk_type == SOCK_STREAM) if (sk->sk_type == SOCK_STREAM)
goto new_alloc; goto new_alloc;
@ -1251,11 +1253,11 @@ struct ubuf_info *msg_zerocopy_realloc(struct sock *sk, size_t size,
} }
next = (u32)atomic_read(&sk->sk_zckey); next = (u32)atomic_read(&sk->sk_zckey);
if ((u32)(uarg->id + uarg->len) == next) { if ((u32)(uarg_zc->id + uarg_zc->len) == next) {
if (mm_account_pinned_pages(&uarg->mmp, size)) if (mm_account_pinned_pages(&uarg_zc->mmp, size))
return NULL; return NULL;
uarg->len++; uarg_zc->len++;
uarg->bytelen = bytelen; uarg_zc->bytelen = bytelen;
atomic_set(&sk->sk_zckey, ++next); atomic_set(&sk->sk_zckey, ++next);
/* no extra ref when appending to datagram (MSG_MORE) */ /* no extra ref when appending to datagram (MSG_MORE) */
@ -1291,7 +1293,7 @@ static bool skb_zerocopy_notify_extend(struct sk_buff *skb, u32 lo, u16 len)
return true; return true;
} }
static void __msg_zerocopy_callback(struct ubuf_info *uarg) static void __msg_zerocopy_callback(struct ubuf_info_msgzc *uarg)
{ {
struct sk_buff *tail, *skb = skb_from_uarg(uarg); struct sk_buff *tail, *skb = skb_from_uarg(uarg);
struct sock_exterr_skb *serr; struct sock_exterr_skb *serr;
@ -1344,19 +1346,21 @@ release:
void msg_zerocopy_callback(struct sk_buff *skb, struct ubuf_info *uarg, void msg_zerocopy_callback(struct sk_buff *skb, struct ubuf_info *uarg,
bool success) bool success)
{ {
uarg->zerocopy = uarg->zerocopy & success; struct ubuf_info_msgzc *uarg_zc = uarg_to_msgzc(uarg);
uarg_zc->zerocopy = uarg_zc->zerocopy & success;
if (refcount_dec_and_test(&uarg->refcnt)) if (refcount_dec_and_test(&uarg->refcnt))
__msg_zerocopy_callback(uarg); __msg_zerocopy_callback(uarg_zc);
} }
EXPORT_SYMBOL_GPL(msg_zerocopy_callback); EXPORT_SYMBOL_GPL(msg_zerocopy_callback);
void msg_zerocopy_put_abort(struct ubuf_info *uarg, bool have_uref) void msg_zerocopy_put_abort(struct ubuf_info *uarg, bool have_uref)
{ {
struct sock *sk = skb_from_uarg(uarg)->sk; struct sock *sk = skb_from_uarg(uarg_to_msgzc(uarg))->sk;
atomic_dec(&sk->sk_zckey); atomic_dec(&sk->sk_zckey);
uarg->len--; uarg_to_msgzc(uarg)->len--;
if (have_uref) if (have_uref)
msg_zerocopy_callback(NULL, uarg, true); msg_zerocopy_callback(NULL, uarg, true);

2
net/ipv4/ip_output.c
View file

@ -1043,7 +1043,7 @@ static int __ip_append_data(struct sock *sk,
paged = true; paged = true;
zc = true; zc = true;
} else { } else {
uarg->zerocopy = 0; uarg_to_msgzc(uarg)->zerocopy = 0;
skb_zcopy_set(skb, uarg, &extra_uref); skb_zcopy_set(skb, uarg, &extra_uref);
} }
} }

2
net/ipv4/tcp.c
View file

@ -1239,7 +1239,7 @@ int tcp_sendmsg_locked(struct sock *sk, struct msghdr *msg, size_t size)
} }
zc = sk->sk_route_caps & NETIF_F_SG; zc = sk->sk_route_caps & NETIF_F_SG;
if (!zc) if (!zc)
uarg->zerocopy = 0; uarg_to_msgzc(uarg)->zerocopy = 0;
} }
} }

2
net/ipv6/ip6_output.c
View file

@ -1567,7 +1567,7 @@ emsgsize:
paged = true; paged = true;
zc = true; zc = true;
} else { } else {
uarg->zerocopy = 0; uarg_to_msgzc(uarg)->zerocopy = 0;
skb_zcopy_set(skb, uarg, &extra_uref); skb_zcopy_set(skb, uarg, &extra_uref);
} }
} }