* [PATCH v8 net-next 1/4] net/udp: Add a new struct for hash2 slot
2024-11-08 5:48 [PATCH v8 net-next 0/4] udp: Add 4-tuple hash for connected sockets Philo Lu
@ 2024-11-08 5:48 ` Philo Lu
2024-11-08 5:48 ` [PATCH v8 net-next 2/4] net/udp: Add 4-tuple hash list basis Philo Lu
` (3 subsequent siblings)
4 siblings, 0 replies; 9+ messages in thread
From: Philo Lu @ 2024-11-08 5:48 UTC (permalink / raw)
To: netdev
Cc: willemdebruijn.kernel, davem, edumazet, kuba, pabeni, dsahern,
horms, antony.antony, steffen.klassert, linux-kernel, dust.li,
jakub, fred.cc, yubing.qiuyubing
Preparing for udp 4-tuple hash (uhash4 for short).
To implement uhash4 without cache line missing when lookup, hslot2 is
used to record the number of hashed sockets in hslot4. Thus adding a new
struct udp_hslot_main with field hash4_cnt, which is used by hash2. The
new struct is used to avoid doubling the size of udp_hslot.
Before uhash4 lookup, firstly checking hash4_cnt to see if there are
hashed sks in hslot4. Because hslot2 is always used in lookup, there is
no cache line miss.
Related helpers are updated, and use the helpers as possible.
uhash4 is implemented in following patches.
Signed-off-by: Philo Lu <lulie@linux.alibaba.com>
---
include/net/udp.h | 38 ++++++++++++++++++++++++++++++++++----
net/ipv4/udp.c | 44 +++++++++++++++++++++++---------------------
net/ipv6/udp.c | 15 ++++++---------
3 files changed, 63 insertions(+), 34 deletions(-)
diff --git a/include/net/udp.h b/include/net/udp.h
index 61222545ab1c..62a7207e65f2 100644
--- a/include/net/udp.h
+++ b/include/net/udp.h
@@ -50,7 +50,7 @@ struct udp_skb_cb {
#define UDP_SKB_CB(__skb) ((struct udp_skb_cb *)((__skb)->cb))
/**
- * struct udp_hslot - UDP hash slot
+ * struct udp_hslot - UDP hash slot used by udp_table.hash
*
* @head: head of list of sockets
* @count: number of sockets in 'head' list
@@ -60,7 +60,22 @@ struct udp_hslot {
struct hlist_head head;
int count;
spinlock_t lock;
-} __attribute__((aligned(2 * sizeof(long))));
+} __aligned(2 * sizeof(long));
+
+/**
+ * struct udp_hslot_main - UDP hash slot used by udp_table.hash2
+ *
+ * @hslot: basic hash slot
+ * @hash4_cnt: number of sockets in hslot4 of the same
+ * (local port, local address)
+ */
+struct udp_hslot_main {
+ struct udp_hslot hslot; /* must be the first member */
+#if !IS_ENABLED(CONFIG_BASE_SMALL)
+ u32 hash4_cnt;
+#endif
+} __aligned(2 * sizeof(long));
+#define UDP_HSLOT_MAIN(__hslot) ((struct udp_hslot_main *)(__hslot))
/**
* struct udp_table - UDP table
@@ -72,7 +87,7 @@ struct udp_hslot {
*/
struct udp_table {
struct udp_hslot *hash;
- struct udp_hslot *hash2;
+ struct udp_hslot_main *hash2;
unsigned int mask;
unsigned int log;
};
@@ -84,6 +99,7 @@ static inline struct udp_hslot *udp_hashslot(struct udp_table *table,
{
return &table->hash[udp_hashfn(net, num, table->mask)];
}
+
/*
* For secondary hash, net_hash_mix() is performed before calling
* udp_hashslot2(), this explains difference with udp_hashslot()
@@ -91,8 +107,22 @@ static inline struct udp_hslot *udp_hashslot(struct udp_table *table,
static inline struct udp_hslot *udp_hashslot2(struct udp_table *table,
unsigned int hash)
{
- return &table->hash2[hash & table->mask];
+ return &table->hash2[hash & table->mask].hslot;
+}
+
+#if IS_ENABLED(CONFIG_BASE_SMALL)
+static inline void udp_table_hash4_init(struct udp_table *table)
+{
+}
+#else /* !CONFIG_BASE_SMALL */
+
+/* Must be called with table->hash2 initialized */
+static inline void udp_table_hash4_init(struct udp_table *table)
+{
+ for (int i = 0; i <= table->mask; i++)
+ table->hash2[i].hash4_cnt = 0;
}
+#endif /* CONFIG_BASE_SMALL */
extern struct proto udp_prot;
diff --git a/net/ipv4/udp.c b/net/ipv4/udp.c
index 0e24916b39d4..2fdac5fae2a8 100644
--- a/net/ipv4/udp.c
+++ b/net/ipv4/udp.c
@@ -486,13 +486,12 @@ struct sock *__udp4_lib_lookup(const struct net *net, __be32 saddr,
int sdif, struct udp_table *udptable, struct sk_buff *skb)
{
unsigned short hnum = ntohs(dport);
- unsigned int hash2, slot2;
struct udp_hslot *hslot2;
struct sock *result, *sk;
+ unsigned int hash2;
hash2 = ipv4_portaddr_hash(net, daddr, hnum);
- slot2 = hash2 & udptable->mask;
- hslot2 = &udptable->hash2[slot2];
+ hslot2 = udp_hashslot2(udptable, hash2);
/* Lookup connected or non-wildcard socket */
result = udp4_lib_lookup2(net, saddr, sport,
@@ -519,8 +518,7 @@ struct sock *__udp4_lib_lookup(const struct net *net, __be32 saddr,
/* Lookup wildcard sockets */
hash2 = ipv4_portaddr_hash(net, htonl(INADDR_ANY), hnum);
- slot2 = hash2 & udptable->mask;
- hslot2 = &udptable->hash2[slot2];
+ hslot2 = udp_hashslot2(udptable, hash2);
result = udp4_lib_lookup2(net, saddr, sport,
htonl(INADDR_ANY), hnum, dif, sdif,
@@ -2268,7 +2266,7 @@ static int __udp4_lib_mcast_deliver(struct net *net, struct sk_buff *skb,
udptable->mask;
hash2 = ipv4_portaddr_hash(net, daddr, hnum) & udptable->mask;
start_lookup:
- hslot = &udptable->hash2[hash2];
+ hslot = &udptable->hash2[hash2].hslot;
offset = offsetof(typeof(*sk), __sk_common.skc_portaddr_node);
}
@@ -2539,14 +2537,13 @@ static struct sock *__udp4_lib_demux_lookup(struct net *net,
struct udp_table *udptable = net->ipv4.udp_table;
INET_ADDR_COOKIE(acookie, rmt_addr, loc_addr);
unsigned short hnum = ntohs(loc_port);
- unsigned int hash2, slot2;
struct udp_hslot *hslot2;
+ unsigned int hash2;
__portpair ports;
struct sock *sk;
hash2 = ipv4_portaddr_hash(net, loc_addr, hnum);
- slot2 = hash2 & udptable->mask;
- hslot2 = &udptable->hash2[slot2];
+ hslot2 = udp_hashslot2(udptable, hash2);
ports = INET_COMBINED_PORTS(rmt_port, hnum);
udp_portaddr_for_each_entry_rcu(sk, &hslot2->head) {
@@ -3187,7 +3184,7 @@ static struct sock *bpf_iter_udp_batch(struct seq_file *seq)
batch_sks = 0;
for (; state->bucket <= udptable->mask; state->bucket++) {
- struct udp_hslot *hslot2 = &udptable->hash2[state->bucket];
+ struct udp_hslot *hslot2 = &udptable->hash2[state->bucket].hslot;
if (hlist_empty(&hslot2->head))
continue;
@@ -3428,10 +3425,11 @@ __setup("uhash_entries=", set_uhash_entries);
void __init udp_table_init(struct udp_table *table, const char *name)
{
- unsigned int i;
+ unsigned int i, slot_size;
+ slot_size = sizeof(struct udp_hslot) + sizeof(struct udp_hslot_main);
table->hash = alloc_large_system_hash(name,
- 2 * sizeof(struct udp_hslot),
+ slot_size,
uhash_entries,
21, /* one slot per 2 MB */
0,
@@ -3440,17 +3438,18 @@ void __init udp_table_init(struct udp_table *table, const char *name)
UDP_HTABLE_SIZE_MIN,
UDP_HTABLE_SIZE_MAX);
- table->hash2 = table->hash + (table->mask + 1);
+ table->hash2 = (void *)(table->hash + (table->mask + 1));
for (i = 0; i <= table->mask; i++) {
INIT_HLIST_HEAD(&table->hash[i].head);
table->hash[i].count = 0;
spin_lock_init(&table->hash[i].lock);
}
for (i = 0; i <= table->mask; i++) {
- INIT_HLIST_HEAD(&table->hash2[i].head);
- table->hash2[i].count = 0;
- spin_lock_init(&table->hash2[i].lock);
+ INIT_HLIST_HEAD(&table->hash2[i].hslot.head);
+ table->hash2[i].hslot.count = 0;
+ spin_lock_init(&table->hash2[i].hslot.lock);
}
+ udp_table_hash4_init(table);
}
u32 udp_flow_hashrnd(void)
@@ -3476,18 +3475,20 @@ static void __net_init udp_sysctl_init(struct net *net)
static struct udp_table __net_init *udp_pernet_table_alloc(unsigned int hash_entries)
{
struct udp_table *udptable;
+ unsigned int slot_size;
int i;
udptable = kmalloc(sizeof(*udptable), GFP_KERNEL);
if (!udptable)
goto out;
- udptable->hash = vmalloc_huge(hash_entries * 2 * sizeof(struct udp_hslot),
+ slot_size = sizeof(struct udp_hslot) + sizeof(struct udp_hslot_main);
+ udptable->hash = vmalloc_huge(hash_entries * slot_size,
GFP_KERNEL_ACCOUNT);
if (!udptable->hash)
goto free_table;
- udptable->hash2 = udptable->hash + hash_entries;
+ udptable->hash2 = (void *)(udptable->hash + hash_entries);
udptable->mask = hash_entries - 1;
udptable->log = ilog2(hash_entries);
@@ -3496,10 +3497,11 @@ static struct udp_table __net_init *udp_pernet_table_alloc(unsigned int hash_ent
udptable->hash[i].count = 0;
spin_lock_init(&udptable->hash[i].lock);
- INIT_HLIST_HEAD(&udptable->hash2[i].head);
- udptable->hash2[i].count = 0;
- spin_lock_init(&udptable->hash2[i].lock);
+ INIT_HLIST_HEAD(&udptable->hash2[i].hslot.head);
+ udptable->hash2[i].hslot.count = 0;
+ spin_lock_init(&udptable->hash2[i].hslot.lock);
}
+ udp_table_hash4_init(udptable);
return udptable;
diff --git a/net/ipv6/udp.c b/net/ipv6/udp.c
index 0cef8ae5d1ea..0d7aac9d44e5 100644
--- a/net/ipv6/udp.c
+++ b/net/ipv6/udp.c
@@ -224,13 +224,12 @@ struct sock *__udp6_lib_lookup(const struct net *net,
struct sk_buff *skb)
{
unsigned short hnum = ntohs(dport);
- unsigned int hash2, slot2;
struct udp_hslot *hslot2;
struct sock *result, *sk;
+ unsigned int hash2;
hash2 = ipv6_portaddr_hash(net, daddr, hnum);
- slot2 = hash2 & udptable->mask;
- hslot2 = &udptable->hash2[slot2];
+ hslot2 = udp_hashslot2(udptable, hash2);
/* Lookup connected or non-wildcard sockets */
result = udp6_lib_lookup2(net, saddr, sport,
@@ -257,8 +256,7 @@ struct sock *__udp6_lib_lookup(const struct net *net,
/* Lookup wildcard sockets */
hash2 = ipv6_portaddr_hash(net, &in6addr_any, hnum);
- slot2 = hash2 & udptable->mask;
- hslot2 = &udptable->hash2[slot2];
+ hslot2 = udp_hashslot2(udptable, hash2);
result = udp6_lib_lookup2(net, saddr, sport,
&in6addr_any, hnum, dif, sdif,
@@ -859,7 +857,7 @@ static int __udp6_lib_mcast_deliver(struct net *net, struct sk_buff *skb,
udptable->mask;
hash2 = ipv6_portaddr_hash(net, daddr, hnum) & udptable->mask;
start_lookup:
- hslot = &udptable->hash2[hash2];
+ hslot = &udptable->hash2[hash2].hslot;
offset = offsetof(typeof(*sk), __sk_common.skc_portaddr_node);
}
@@ -1065,14 +1063,13 @@ static struct sock *__udp6_lib_demux_lookup(struct net *net,
{
struct udp_table *udptable = net->ipv4.udp_table;
unsigned short hnum = ntohs(loc_port);
- unsigned int hash2, slot2;
struct udp_hslot *hslot2;
+ unsigned int hash2;
__portpair ports;
struct sock *sk;
hash2 = ipv6_portaddr_hash(net, loc_addr, hnum);
- slot2 = hash2 & udptable->mask;
- hslot2 = &udptable->hash2[slot2];
+ hslot2 = udp_hashslot2(udptable, hash2);
ports = INET_COMBINED_PORTS(rmt_port, hnum);
udp_portaddr_for_each_entry_rcu(sk, &hslot2->head) {
--
2.32.0.3.g01195cf9f
^ permalink raw reply [flat|nested] 9+ messages in thread* [PATCH v8 net-next 2/4] net/udp: Add 4-tuple hash list basis
2024-11-08 5:48 [PATCH v8 net-next 0/4] udp: Add 4-tuple hash for connected sockets Philo Lu
2024-11-08 5:48 ` [PATCH v8 net-next 1/4] net/udp: Add a new struct for hash2 slot Philo Lu
@ 2024-11-08 5:48 ` Philo Lu
2024-11-08 5:48 ` [PATCH v8 net-next 3/4] ipv4/udp: Add 4-tuple hash for connected socket Philo Lu
` (2 subsequent siblings)
4 siblings, 0 replies; 9+ messages in thread
From: Philo Lu @ 2024-11-08 5:48 UTC (permalink / raw)
To: netdev
Cc: willemdebruijn.kernel, davem, edumazet, kuba, pabeni, dsahern,
horms, antony.antony, steffen.klassert, linux-kernel, dust.li,
jakub, fred.cc, yubing.qiuyubing
Add a new hash list, hash4, in udp table. It will be used to implement
4-tuple hash for connected udp sockets. This patch adds the hlist to
table, and implements helpers and the initialization. 4-tuple hash is
implemented in the following patch.
hash4 uses hlist_nulls to avoid moving wrongly onto another hlist due to
concurrent rehash, because rehash() can happen with lookup().
Signed-off-by: Philo Lu <lulie@linux.alibaba.com>
Signed-off-by: Cambda Zhu <cambda@linux.alibaba.com>
Signed-off-by: Fred Chen <fred.cc@alibaba-inc.com>
Signed-off-by: Yubing Qiu <yubing.qiuyubing@alibaba-inc.com>
---
include/linux/udp.h | 11 ++++++
include/net/udp.h | 85 +++++++++++++++++++++++++++++++++++++++++++--
net/ipv4/udp.c | 6 ++--
3 files changed, 97 insertions(+), 5 deletions(-)
diff --git a/include/linux/udp.h b/include/linux/udp.h
index 3eb3f2b9a2a0..0807e21cfec9 100644
--- a/include/linux/udp.h
+++ b/include/linux/udp.h
@@ -56,6 +56,12 @@ struct udp_sock {
int pending; /* Any pending frames ? */
__u8 encap_type; /* Is this an Encapsulation socket? */
+#if !IS_ENABLED(CONFIG_BASE_SMALL)
+ /* For UDP 4-tuple hash */
+ __u16 udp_lrpa_hash;
+ struct hlist_nulls_node udp_lrpa_node;
+#endif
+
/*
* Following member retains the information to create a UDP header
* when the socket is uncorked.
@@ -206,6 +212,11 @@ static inline void udp_allow_gso(struct sock *sk)
#define udp_portaddr_for_each_entry_rcu(__sk, list) \
hlist_for_each_entry_rcu(__sk, list, __sk_common.skc_portaddr_node)
+#if !IS_ENABLED(CONFIG_BASE_SMALL)
+#define udp_lrpa_for_each_entry_rcu(__up, node, list) \
+ hlist_nulls_for_each_entry_rcu(__up, node, list, udp_lrpa_node)
+#endif
+
#define IS_UDPLITE(__sk) (__sk->sk_protocol == IPPROTO_UDPLITE)
#endif /* _LINUX_UDP_H */
diff --git a/include/net/udp.h b/include/net/udp.h
index 62a7207e65f2..edb669967130 100644
--- a/include/net/udp.h
+++ b/include/net/udp.h
@@ -50,14 +50,21 @@ struct udp_skb_cb {
#define UDP_SKB_CB(__skb) ((struct udp_skb_cb *)((__skb)->cb))
/**
- * struct udp_hslot - UDP hash slot used by udp_table.hash
+ * struct udp_hslot - UDP hash slot used by udp_table.hash/hash4
*
* @head: head of list of sockets
+ * @nulls_head: head of list of sockets, only used by hash4
* @count: number of sockets in 'head' list
* @lock: spinlock protecting changes to head/count
*/
struct udp_hslot {
- struct hlist_head head;
+ union {
+ struct hlist_head head;
+ /* hash4 uses hlist_nulls to avoid moving wrongly onto another
+ * hlist, because rehash() can happen with lookup().
+ */
+ struct hlist_nulls_head nulls_head;
+ };
int count;
spinlock_t lock;
} __aligned(2 * sizeof(long));
@@ -82,12 +89,17 @@ struct udp_hslot_main {
*
* @hash: hash table, sockets are hashed on (local port)
* @hash2: hash table, sockets are hashed on (local port, local address)
+ * @hash4: hash table, connected sockets are hashed on
+ * (local port, local address, remote port, remote address)
* @mask: number of slots in hash tables, minus 1
* @log: log2(number of slots in hash table)
*/
struct udp_table {
struct udp_hslot *hash;
struct udp_hslot_main *hash2;
+#if !IS_ENABLED(CONFIG_BASE_SMALL)
+ struct udp_hslot *hash4;
+#endif
unsigned int mask;
unsigned int log;
};
@@ -114,13 +126,80 @@ static inline struct udp_hslot *udp_hashslot2(struct udp_table *table,
static inline void udp_table_hash4_init(struct udp_table *table)
{
}
+
+static inline struct udp_hslot *udp_hashslot4(struct udp_table *table,
+ unsigned int hash)
+{
+ BUILD_BUG();
+ return NULL;
+}
+
+static inline bool udp_hashed4(const struct sock *sk)
+{
+ return false;
+}
+
+static inline unsigned int udp_hash4_slot_size(void)
+{
+ return 0;
+}
+
+static inline bool udp_has_hash4(const struct udp_hslot *hslot2)
+{
+ return false;
+}
+
+static inline void udp_hash4_inc(struct udp_hslot *hslot2)
+{
+}
+
+static inline void udp_hash4_dec(struct udp_hslot *hslot2)
+{
+}
#else /* !CONFIG_BASE_SMALL */
/* Must be called with table->hash2 initialized */
static inline void udp_table_hash4_init(struct udp_table *table)
{
- for (int i = 0; i <= table->mask; i++)
+ table->hash4 = (void *)(table->hash2 + (table->mask + 1));
+ for (int i = 0; i <= table->mask; i++) {
table->hash2[i].hash4_cnt = 0;
+
+ INIT_HLIST_NULLS_HEAD(&table->hash4[i].nulls_head, i);
+ table->hash4[i].count = 0;
+ spin_lock_init(&table->hash4[i].lock);
+ }
+}
+
+static inline struct udp_hslot *udp_hashslot4(struct udp_table *table,
+ unsigned int hash)
+{
+ return &table->hash4[hash & table->mask];
+}
+
+static inline bool udp_hashed4(const struct sock *sk)
+{
+ return !hlist_nulls_unhashed(&udp_sk(sk)->udp_lrpa_node);
+}
+
+static inline unsigned int udp_hash4_slot_size(void)
+{
+ return sizeof(struct udp_hslot);
+}
+
+static inline bool udp_has_hash4(const struct udp_hslot *hslot2)
+{
+ return UDP_HSLOT_MAIN(hslot2)->hash4_cnt;
+}
+
+static inline void udp_hash4_inc(struct udp_hslot *hslot2)
+{
+ UDP_HSLOT_MAIN(hslot2)->hash4_cnt++;
+}
+
+static inline void udp_hash4_dec(struct udp_hslot *hslot2)
+{
+ UDP_HSLOT_MAIN(hslot2)->hash4_cnt--;
}
#endif /* CONFIG_BASE_SMALL */
diff --git a/net/ipv4/udp.c b/net/ipv4/udp.c
index 2fdac5fae2a8..0bc0881d6569 100644
--- a/net/ipv4/udp.c
+++ b/net/ipv4/udp.c
@@ -3427,7 +3427,8 @@ void __init udp_table_init(struct udp_table *table, const char *name)
{
unsigned int i, slot_size;
- slot_size = sizeof(struct udp_hslot) + sizeof(struct udp_hslot_main);
+ slot_size = sizeof(struct udp_hslot) + sizeof(struct udp_hslot_main) +
+ udp_hash4_slot_size();
table->hash = alloc_large_system_hash(name,
slot_size,
uhash_entries,
@@ -3482,7 +3483,8 @@ static struct udp_table __net_init *udp_pernet_table_alloc(unsigned int hash_ent
if (!udptable)
goto out;
- slot_size = sizeof(struct udp_hslot) + sizeof(struct udp_hslot_main);
+ slot_size = sizeof(struct udp_hslot) + sizeof(struct udp_hslot_main) +
+ udp_hash4_slot_size();
udptable->hash = vmalloc_huge(hash_entries * slot_size,
GFP_KERNEL_ACCOUNT);
if (!udptable->hash)
--
2.32.0.3.g01195cf9f
^ permalink raw reply [flat|nested] 9+ messages in thread* [PATCH v8 net-next 3/4] ipv4/udp: Add 4-tuple hash for connected socket
2024-11-08 5:48 [PATCH v8 net-next 0/4] udp: Add 4-tuple hash for connected sockets Philo Lu
2024-11-08 5:48 ` [PATCH v8 net-next 1/4] net/udp: Add a new struct for hash2 slot Philo Lu
2024-11-08 5:48 ` [PATCH v8 net-next 2/4] net/udp: Add 4-tuple hash list basis Philo Lu
@ 2024-11-08 5:48 ` Philo Lu
2024-11-12 14:58 ` Paolo Abeni
2024-11-08 5:48 ` [PATCH v8 net-next 4/4] ipv6/udp: " Philo Lu
2024-11-11 17:58 ` [PATCH v8 net-next 0/4] udp: Add 4-tuple hash for connected sockets Willem de Bruijn
4 siblings, 1 reply; 9+ messages in thread
From: Philo Lu @ 2024-11-08 5:48 UTC (permalink / raw)
To: netdev
Cc: willemdebruijn.kernel, davem, edumazet, kuba, pabeni, dsahern,
horms, antony.antony, steffen.klassert, linux-kernel, dust.li,
jakub, fred.cc, yubing.qiuyubing
Currently, the udp_table has two hash table, the port hash and portaddr
hash. Usually for UDP servers, all sockets have the same local port and
addr, so they are all on the same hash slot within a reuseport group.
In some applications, UDP servers use connect() to manage clients. In
particular, when firstly receiving from an unseen 4 tuple, a new socket
is created and connect()ed to the remote addr:port, and then the fd is
used exclusively by the client.
Once there are connected sks in a reuseport group, udp has to score all
sks in the same hash2 slot to find the best match. This could be
inefficient with a large number of connections, resulting in high
softirq overhead.
To solve the problem, this patch implement 4-tuple hash for connected
udp sockets. During connect(), hash4 slot is updated, as well as a
corresponding counter, hash4_cnt, in hslot2. In __udp4_lib_lookup(),
hslot4 will be searched firstly if the counter is non-zero. Otherwise,
hslot2 is used like before. Note that only connected sockets enter this
hash4 path, while un-connected ones are not affected.
hlist_nulls is used for hash4, because we probably move to another hslot
wrongly when lookup with concurrent rehash. Then we check nulls at the
list end to see if we should restart lookup. Because udp does not use
SLAB_TYPESAFE_BY_RCU, we don't need to touch sk_refcnt when lookup.
Stress test results (with 1 cpu fully used) are shown below, in pps:
(1) _un-connected_ socket as server
[a] w/o hash4: 1,825176
[b] w/ hash4: 1,831750 (+0.36%)
(2) 500 _connected_ sockets as server
[c] w/o hash4: 290860 (only 16% of [a])
[d] w/ hash4: 1,889658 (+3.1% compared with [b])
With hash4, compute_score is skipped when lookup, so [d] is slightly
better than [b].
Signed-off-by: Philo Lu <lulie@linux.alibaba.com>
Signed-off-by: Cambda Zhu <cambda@linux.alibaba.com>
Signed-off-by: Fred Chen <fred.cc@alibaba-inc.com>
Signed-off-by: Yubing Qiu <yubing.qiuyubing@alibaba-inc.com>
---
include/net/udp.h | 16 +++-
net/ipv4/udp.c | 197 +++++++++++++++++++++++++++++++++++++++++++++-
net/ipv6/udp.c | 2 +-
3 files changed, 210 insertions(+), 5 deletions(-)
diff --git a/include/net/udp.h b/include/net/udp.h
index edb669967130..feb06c0e48fb 100644
--- a/include/net/udp.h
+++ b/include/net/udp.h
@@ -302,13 +302,27 @@ static inline int udp_lib_hash(struct sock *sk)
}
void udp_lib_unhash(struct sock *sk);
-void udp_lib_rehash(struct sock *sk, u16 new_hash);
+void udp_lib_rehash(struct sock *sk, u16 new_hash, u16 new_hash4);
static inline void udp_lib_close(struct sock *sk, long timeout)
{
sk_common_release(sk);
}
+/* hash4 routines shared between UDPv4/6 */
+#if IS_ENABLED(CONFIG_BASE_SMALL)
+static inline void udp_lib_hash4(struct sock *sk, u16 hash)
+{
+}
+
+static inline void udp4_hash4(struct sock *sk)
+{
+}
+#else /* !CONFIG_BASE_SMALL */
+void udp_lib_hash4(struct sock *sk, u16 hash);
+void udp4_hash4(struct sock *sk);
+#endif /* CONFIG_BASE_SMALL */
+
int udp_lib_get_port(struct sock *sk, unsigned short snum,
unsigned int hash2_nulladdr);
diff --git a/net/ipv4/udp.c b/net/ipv4/udp.c
index 0bc0881d6569..b6c5edd7ff48 100644
--- a/net/ipv4/udp.c
+++ b/net/ipv4/udp.c
@@ -478,6 +478,159 @@ static struct sock *udp4_lib_lookup2(const struct net *net,
return result;
}
+#if IS_ENABLED(CONFIG_BASE_SMALL)
+static struct sock *udp4_lib_lookup4(const struct net *net,
+ __be32 saddr, __be16 sport,
+ __be32 daddr, unsigned int hnum,
+ int dif, int sdif,
+ struct udp_table *udptable)
+{
+ return NULL;
+}
+
+static void udp_rehash4(struct udp_table *udptable, struct sock *sk,
+ u16 newhash4)
+{
+}
+
+static void udp_unhash4(struct udp_table *udptable, struct sock *sk)
+{
+}
+#else /* !CONFIG_BASE_SMALL */
+static struct sock *udp4_lib_lookup4(const struct net *net,
+ __be32 saddr, __be16 sport,
+ __be32 daddr, unsigned int hnum,
+ int dif, int sdif,
+ struct udp_table *udptable)
+{
+ const __portpair ports = INET_COMBINED_PORTS(sport, hnum);
+ const struct hlist_nulls_node *node;
+ struct udp_hslot *hslot4;
+ unsigned int hash4, slot;
+ struct udp_sock *up;
+ struct sock *sk;
+
+ hash4 = udp_ehashfn(net, daddr, hnum, saddr, sport);
+ slot = hash4 & udptable->mask;
+ hslot4 = &udptable->hash4[slot];
+ INET_ADDR_COOKIE(acookie, saddr, daddr);
+
+begin:
+ /* SLAB_TYPESAFE_BY_RCU not used, so we don't need to touch sk_refcnt */
+ udp_lrpa_for_each_entry_rcu(up, node, &hslot4->nulls_head) {
+ sk = (struct sock *)up;
+ if (inet_match(net, sk, acookie, ports, dif, sdif))
+ return sk;
+ }
+
+ /* if the nulls value we got at the end of this lookup is not the
+ * expected one, we must restart lookup. We probably met an item that
+ * was moved to another chain due to rehash.
+ */
+ if (get_nulls_value(node) != slot)
+ goto begin;
+
+ return NULL;
+}
+
+/* In hash4, rehash can happen in connect(), where hash4_cnt keeps unchanged. */
+static void udp_rehash4(struct udp_table *udptable, struct sock *sk,
+ u16 newhash4)
+{
+ struct udp_hslot *hslot4, *nhslot4;
+
+ hslot4 = udp_hashslot4(udptable, udp_sk(sk)->udp_lrpa_hash);
+ nhslot4 = udp_hashslot4(udptable, newhash4);
+ udp_sk(sk)->udp_lrpa_hash = newhash4;
+
+ if (hslot4 != nhslot4) {
+ spin_lock_bh(&hslot4->lock);
+ hlist_nulls_del_init_rcu(&udp_sk(sk)->udp_lrpa_node);
+ hslot4->count--;
+ spin_unlock_bh(&hslot4->lock);
+
+ spin_lock_bh(&nhslot4->lock);
+ hlist_nulls_add_head_rcu(&udp_sk(sk)->udp_lrpa_node,
+ &nhslot4->nulls_head);
+ nhslot4->count++;
+ spin_unlock_bh(&nhslot4->lock);
+ }
+}
+
+static void udp_unhash4(struct udp_table *udptable, struct sock *sk)
+{
+ struct udp_hslot *hslot2, *hslot4;
+
+ if (udp_hashed4(sk)) {
+ hslot2 = udp_hashslot2(udptable, udp_sk(sk)->udp_portaddr_hash);
+ hslot4 = udp_hashslot4(udptable, udp_sk(sk)->udp_lrpa_hash);
+
+ spin_lock(&hslot4->lock);
+ hlist_nulls_del_init_rcu(&udp_sk(sk)->udp_lrpa_node);
+ hslot4->count--;
+ spin_unlock(&hslot4->lock);
+
+ spin_lock(&hslot2->lock);
+ udp_hash4_dec(hslot2);
+ spin_unlock(&hslot2->lock);
+ }
+}
+
+void udp_lib_hash4(struct sock *sk, u16 hash)
+{
+ struct udp_hslot *hslot, *hslot2, *hslot4;
+ struct net *net = sock_net(sk);
+ struct udp_table *udptable;
+
+ /* Connected udp socket can re-connect to another remote address,
+ * so rehash4 is needed.
+ */
+ udptable = net->ipv4.udp_table;
+ if (udp_hashed4(sk)) {
+ udp_rehash4(udptable, sk, hash);
+ return;
+ }
+
+ hslot = udp_hashslot(udptable, net, udp_sk(sk)->udp_port_hash);
+ hslot2 = udp_hashslot2(udptable, udp_sk(sk)->udp_portaddr_hash);
+ hslot4 = udp_hashslot4(udptable, hash);
+ udp_sk(sk)->udp_lrpa_hash = hash;
+
+ spin_lock_bh(&hslot->lock);
+ if (rcu_access_pointer(sk->sk_reuseport_cb))
+ reuseport_detach_sock(sk);
+
+ spin_lock(&hslot4->lock);
+ hlist_nulls_add_head_rcu(&udp_sk(sk)->udp_lrpa_node,
+ &hslot4->nulls_head);
+ hslot4->count++;
+ spin_unlock(&hslot4->lock);
+
+ spin_lock(&hslot2->lock);
+ udp_hash4_inc(hslot2);
+ spin_unlock(&hslot2->lock);
+
+ spin_unlock_bh(&hslot->lock);
+}
+EXPORT_SYMBOL(udp_lib_hash4);
+
+/* call with sock lock */
+void udp4_hash4(struct sock *sk)
+{
+ struct net *net = sock_net(sk);
+ unsigned int hash;
+
+ if (sk_unhashed(sk) || sk->sk_rcv_saddr == htonl(INADDR_ANY))
+ return;
+
+ hash = udp_ehashfn(net, sk->sk_rcv_saddr, sk->sk_num,
+ sk->sk_daddr, sk->sk_dport);
+
+ udp_lib_hash4(sk, hash);
+}
+EXPORT_SYMBOL(udp4_hash4);
+#endif /* CONFIG_BASE_SMALL */
+
/* UDP is nearly always wildcards out the wazoo, it makes no sense to try
* harder than this. -DaveM
*/
@@ -493,6 +646,13 @@ struct sock *__udp4_lib_lookup(const struct net *net, __be32 saddr,
hash2 = ipv4_portaddr_hash(net, daddr, hnum);
hslot2 = udp_hashslot2(udptable, hash2);
+ if (udp_has_hash4(hslot2)) {
+ result = udp4_lib_lookup4(net, saddr, sport, daddr, hnum,
+ dif, sdif, udptable);
+ if (result) /* udp4_lib_lookup4 return sk or NULL */
+ return result;
+ }
+
/* Lookup connected or non-wildcard socket */
result = udp4_lib_lookup2(net, saddr, sport,
daddr, hnum, dif, sdif,
@@ -1933,6 +2093,18 @@ int udp_pre_connect(struct sock *sk, struct sockaddr *uaddr, int addr_len)
}
EXPORT_SYMBOL(udp_pre_connect);
+static int udp_connect(struct sock *sk, struct sockaddr *uaddr, int addr_len)
+{
+ int res;
+
+ lock_sock(sk);
+ res = __ip4_datagram_connect(sk, uaddr, addr_len);
+ if (!res)
+ udp4_hash4(sk);
+ release_sock(sk);
+ return res;
+}
+
int __udp_disconnect(struct sock *sk, int flags)
{
struct inet_sock *inet = inet_sk(sk);
@@ -1992,6 +2164,8 @@ void udp_lib_unhash(struct sock *sk)
hlist_del_init_rcu(&udp_sk(sk)->udp_portaddr_node);
hslot2->count--;
spin_unlock(&hslot2->lock);
+
+ udp_unhash4(udptable, sk);
}
spin_unlock_bh(&hslot->lock);
}
@@ -2001,7 +2175,7 @@ EXPORT_SYMBOL(udp_lib_unhash);
/*
* inet_rcv_saddr was changed, we must rehash secondary hash
*/
-void udp_lib_rehash(struct sock *sk, u16 newhash)
+void udp_lib_rehash(struct sock *sk, u16 newhash, u16 newhash4)
{
if (sk_hashed(sk)) {
struct udp_table *udptable = udp_get_table_prot(sk);
@@ -2033,6 +2207,19 @@ void udp_lib_rehash(struct sock *sk, u16 newhash)
spin_unlock(&nhslot2->lock);
}
+ if (udp_hashed4(sk)) {
+ udp_rehash4(udptable, sk, newhash4);
+
+ if (hslot2 != nhslot2) {
+ spin_lock(&hslot2->lock);
+ udp_hash4_dec(hslot2);
+ spin_unlock(&hslot2->lock);
+
+ spin_lock(&nhslot2->lock);
+ udp_hash4_inc(nhslot2);
+ spin_unlock(&nhslot2->lock);
+ }
+ }
spin_unlock_bh(&hslot->lock);
}
}
@@ -2044,7 +2231,11 @@ void udp_v4_rehash(struct sock *sk)
u16 new_hash = ipv4_portaddr_hash(sock_net(sk),
inet_sk(sk)->inet_rcv_saddr,
inet_sk(sk)->inet_num);
- udp_lib_rehash(sk, new_hash);
+ u16 new_hash4 = udp_ehashfn(sock_net(sk),
+ sk->sk_rcv_saddr, sk->sk_num,
+ sk->sk_daddr, sk->sk_dport);
+
+ udp_lib_rehash(sk, new_hash, new_hash4);
}
static int __udp_queue_rcv_skb(struct sock *sk, struct sk_buff *skb)
@@ -2937,7 +3128,7 @@ struct proto udp_prot = {
.owner = THIS_MODULE,
.close = udp_lib_close,
.pre_connect = udp_pre_connect,
- .connect = ip4_datagram_connect,
+ .connect = udp_connect,
.disconnect = udp_disconnect,
.ioctl = udp_ioctl,
.init = udp_init_sock,
diff --git a/net/ipv6/udp.c b/net/ipv6/udp.c
index 0d7aac9d44e5..1ea99d704e31 100644
--- a/net/ipv6/udp.c
+++ b/net/ipv6/udp.c
@@ -111,7 +111,7 @@ void udp_v6_rehash(struct sock *sk)
&sk->sk_v6_rcv_saddr,
inet_sk(sk)->inet_num);
- udp_lib_rehash(sk, new_hash);
+ udp_lib_rehash(sk, new_hash, 0); /* 4-tuple hash not implemented */
}
static int compute_score(struct sock *sk, const struct net *net,
--
2.32.0.3.g01195cf9f
^ permalink raw reply [flat|nested] 9+ messages in thread* Re: [PATCH v8 net-next 3/4] ipv4/udp: Add 4-tuple hash for connected socket
2024-11-08 5:48 ` [PATCH v8 net-next 3/4] ipv4/udp: Add 4-tuple hash for connected socket Philo Lu
@ 2024-11-12 14:58 ` Paolo Abeni
2024-11-13 1:50 ` Philo Lu
0 siblings, 1 reply; 9+ messages in thread
From: Paolo Abeni @ 2024-11-12 14:58 UTC (permalink / raw)
To: Philo Lu, netdev
Cc: willemdebruijn.kernel, davem, edumazet, kuba, dsahern, horms,
antony.antony, steffen.klassert, linux-kernel, dust.li, jakub,
fred.cc, yubing.qiuyubing
On 11/8/24 06:48, Philo Lu wrote:
[...]
> Signed-off-by: Philo Lu <lulie@linux.alibaba.com>
> Signed-off-by: Cambda Zhu <cambda@linux.alibaba.com>
> Signed-off-by: Fred Chen <fred.cc@alibaba-inc.com>
> Signed-off-by: Yubing Qiu <yubing.qiuyubing@alibaba-inc.com>
[...]
> @@ -2937,7 +3128,7 @@ struct proto udp_prot = {
> .owner = THIS_MODULE,
> .close = udp_lib_close,
> .pre_connect = udp_pre_connect,
> - .connect = ip4_datagram_connect,
> + .connect = udp_connect,
> .disconnect = udp_disconnect,
> .ioctl = udp_ioctl,
> .init = udp_init_sock,
2 minor notes, possibly not needing a repost:
- The SoB chain looks strange, do you mean co-developed-by actually?
- udplite is not touched. AFAICS should not be a problem - just the
feature will not be available for udplite. I'm wondering if syzbot could
prove me wrong about "not being a problem" (usually it's able to do that;)
/P
^ permalink raw reply [flat|nested] 9+ messages in thread* Re: [PATCH v8 net-next 3/4] ipv4/udp: Add 4-tuple hash for connected socket
2024-11-12 14:58 ` Paolo Abeni
@ 2024-11-13 1:50 ` Philo Lu
2024-11-14 9:12 ` Paolo Abeni
0 siblings, 1 reply; 9+ messages in thread
From: Philo Lu @ 2024-11-13 1:50 UTC (permalink / raw)
To: Paolo Abeni, netdev
Cc: willemdebruijn.kernel, davem, edumazet, kuba, dsahern, horms,
antony.antony, steffen.klassert, linux-kernel, dust.li, jakub,
fred.cc, yubing.qiuyubing
On 2024/11/12 22:58, Paolo Abeni wrote:
> On 11/8/24 06:48, Philo Lu wrote:
> [...]
>> Signed-off-by: Philo Lu <lulie@linux.alibaba.com>
>> Signed-off-by: Cambda Zhu <cambda@linux.alibaba.com>
>> Signed-off-by: Fred Chen <fred.cc@alibaba-inc.com>
>> Signed-off-by: Yubing Qiu <yubing.qiuyubing@alibaba-inc.com>
>
> [...]
>> @@ -2937,7 +3128,7 @@ struct proto udp_prot = {
>> .owner = THIS_MODULE,
>> .close = udp_lib_close,
>> .pre_connect = udp_pre_connect,
>> - .connect = ip4_datagram_connect,
>> + .connect = udp_connect,
>> .disconnect = udp_disconnect,
>> .ioctl = udp_ioctl,
>> .init = udp_init_sock,
>
> 2 minor notes, possibly not needing a repost:
>
> - The SoB chain looks strange, do you mean co-developed-by actually?
Yes, we're all involved in the development. I think it could be
indicated by SoBs (and all of us agree with this). Please let me know if
I'm wrong :)
Or strictly as [1], it should be:
Co-developed-by: Cambda Zhu <cambda@linux.alibaba.com>
Signed-off-by: Cambda Zhu <cambda@linux.alibaba.com>
Co-developed-by: Fred Chen <fred.cc@alibaba-inc.com>
Signed-off-by: Fred Chen <fred.cc@alibaba-inc.com>
Co-developed-by: Yubing Qiu <yubing.qiuyubing@alibaba-inc.com>
Signed-off-by: Yubing Qiu <yubing.qiuyubing@alibaba-inc.com>
Signed-off-by: Philo Lu <lulie@linux.alibaba.com>
[1]
https://www.kernel.org/doc/html/latest/process/submitting-patches.html#when-to-use-acked-by-cc-and-co-developed-by
> - udplite is not touched. AFAICS should not be a problem - just the
> feature will not be available for udplite.
Agreed. Theoretically, the feature relies on udp4_hash4/udp6_hash4 when
connecting, and all other functions including lookup/unhash/rehash
always check "hashed4" firstly, and do nothing if it's false (which is
the case for udplite).
AFAICT, the effects to udplite include:
- Additional memory consumption in udp_sock and udptable
- Control path: udp_hashed4 checking when unhash/rehash
- Data path: udp_has_hash4 checking when lookup
(like unconnected sks in udp)
> I'm wondering if syzbot could
> prove me wrong about "not being a problem" (usually it's able to do that;)
>
> /P
>
Thanks.
--
Philo
^ permalink raw reply [flat|nested] 9+ messages in thread* Re: [PATCH v8 net-next 3/4] ipv4/udp: Add 4-tuple hash for connected socket
2024-11-13 1:50 ` Philo Lu
@ 2024-11-14 9:12 ` Paolo Abeni
0 siblings, 0 replies; 9+ messages in thread
From: Paolo Abeni @ 2024-11-14 9:12 UTC (permalink / raw)
To: Philo Lu, netdev
Cc: willemdebruijn.kernel, davem, edumazet, kuba, dsahern, horms,
antony.antony, steffen.klassert, linux-kernel, dust.li, jakub,
fred.cc, yubing.qiuyubing
On 11/13/24 02:50, Philo Lu wrote:
> On 2024/11/12 22:58, Paolo Abeni wrote:
>> On 11/8/24 06:48, Philo Lu wrote:
>> [...]
>>> Signed-off-by: Philo Lu <lulie@linux.alibaba.com>
>>> Signed-off-by: Cambda Zhu <cambda@linux.alibaba.com>
>>> Signed-off-by: Fred Chen <fred.cc@alibaba-inc.com>
>>> Signed-off-by: Yubing Qiu <yubing.qiuyubing@alibaba-inc.com>
>>
>> [...]
>>> @@ -2937,7 +3128,7 @@ struct proto udp_prot = {
>>> .owner = THIS_MODULE,
>>> .close = udp_lib_close,
>>> .pre_connect = udp_pre_connect,
>>> - .connect = ip4_datagram_connect,
>>> + .connect = udp_connect,
>>> .disconnect = udp_disconnect,
>>> .ioctl = udp_ioctl,
>>> .init = udp_init_sock,
>>
>> 2 minor notes, possibly not needing a repost:
>>
>> - The SoB chain looks strange, do you mean co-developed-by actually?
>
> Yes, we're all involved in the development. I think it could be
> indicated by SoBs (and all of us agree with this). Please let me know if
> I'm wrong :)
>
> Or strictly as [1], it should be:
>
> Co-developed-by: Cambda Zhu <cambda@linux.alibaba.com>
> Signed-off-by: Cambda Zhu <cambda@linux.alibaba.com>
> Co-developed-by: Fred Chen <fred.cc@alibaba-inc.com>
> Signed-off-by: Fred Chen <fred.cc@alibaba-inc.com>
> Co-developed-by: Yubing Qiu <yubing.qiuyubing@alibaba-inc.com>
> Signed-off-by: Yubing Qiu <yubing.qiuyubing@alibaba-inc.com>
> Signed-off-by: Philo Lu <lulie@linux.alibaba.com>
>
> [1]
> https://www.kernel.org/doc/html/latest/process/submitting-patches.html#when-to-use-acked-by-cc-and-co-developed-by
The latter (with co-dev-by tags) I think would be better, it will give
full attribution to all of you. Since the scope of this work is relevant
I think it would be better a complete recognition.
>> - udplite is not touched. AFAICS should not be a problem - just the
>> feature will not be available for udplite.
>
> Agreed. Theoretically, the feature relies on udp4_hash4/udp6_hash4 when
> connecting, and all other functions including lookup/unhash/rehash
> always check "hashed4" firstly, and do nothing if it's false (which is
> the case for udplite).
>
> AFAICT, the effects to udplite include:
> - Additional memory consumption in udp_sock and udptable
> - Control path: udp_hashed4 checking when unhash/rehash
> - Data path: udp_has_hash4 checking when lookup
> (like unconnected sks in udp)
Looks good. Please note explicitly in the cover letter that udplite is
intentionally excluded here, the reason why such choice is safe
(basically a synopsis of the above) and repost with the update tags
chain including my Acked-by tag
(you should retain Willem acked-by tag, too.)
Thanks!
Paolo
^ permalink raw reply [flat|nested] 9+ messages in thread
* [PATCH v8 net-next 4/4] ipv6/udp: Add 4-tuple hash for connected socket
2024-11-08 5:48 [PATCH v8 net-next 0/4] udp: Add 4-tuple hash for connected sockets Philo Lu
` (2 preceding siblings ...)
2024-11-08 5:48 ` [PATCH v8 net-next 3/4] ipv4/udp: Add 4-tuple hash for connected socket Philo Lu
@ 2024-11-08 5:48 ` Philo Lu
2024-11-11 17:58 ` [PATCH v8 net-next 0/4] udp: Add 4-tuple hash for connected sockets Willem de Bruijn
4 siblings, 0 replies; 9+ messages in thread
From: Philo Lu @ 2024-11-08 5:48 UTC (permalink / raw)
To: netdev
Cc: willemdebruijn.kernel, davem, edumazet, kuba, pabeni, dsahern,
horms, antony.antony, steffen.klassert, linux-kernel, dust.li,
jakub, fred.cc, yubing.qiuyubing
Implement ipv6 udp hash4 like that in ipv4. The major difference is that
the hash value should be calculated with udp6_ehashfn(). Besides,
ipv4-mapped ipv6 address is handled before hash() and rehash(). Export
udp_ehashfn because now we use it in udpv6 rehash.
Core procedures of hash/unhash/rehash are same as ipv4, and udpv4 and
udpv6 share the same udptable, so some functions in ipv4 hash4 can also
be shared.
Signed-off-by: Philo Lu <lulie@linux.alibaba.com>
Signed-off-by: Cambda Zhu <cambda@linux.alibaba.com>
Signed-off-by: Fred Chen <fred.cc@alibaba-inc.com>
Signed-off-by: Yubing Qiu <yubing.qiuyubing@alibaba-inc.com>
---
include/net/udp.h | 2 +
net/ipv4/udp.c | 2 +-
net/ipv6/udp.c | 102 +++++++++++++++++++++++++++++++++++++++++++++-
3 files changed, 103 insertions(+), 3 deletions(-)
diff --git a/include/net/udp.h b/include/net/udp.h
index feb06c0e48fb..6e89520e100d 100644
--- a/include/net/udp.h
+++ b/include/net/udp.h
@@ -303,6 +303,8 @@ static inline int udp_lib_hash(struct sock *sk)
void udp_lib_unhash(struct sock *sk);
void udp_lib_rehash(struct sock *sk, u16 new_hash, u16 new_hash4);
+u32 udp_ehashfn(const struct net *net, const __be32 laddr, const __u16 lport,
+ const __be32 faddr, const __be16 fport);
static inline void udp_lib_close(struct sock *sk, long timeout)
{
diff --git a/net/ipv4/udp.c b/net/ipv4/udp.c
index b6c5edd7ff48..6a01905d379f 100644
--- a/net/ipv4/udp.c
+++ b/net/ipv4/udp.c
@@ -410,7 +410,6 @@ static int compute_score(struct sock *sk, const struct net *net,
return score;
}
-INDIRECT_CALLABLE_SCOPE
u32 udp_ehashfn(const struct net *net, const __be32 laddr, const __u16 lport,
const __be32 faddr, const __be16 fport)
{
@@ -419,6 +418,7 @@ u32 udp_ehashfn(const struct net *net, const __be32 laddr, const __u16 lport,
return __inet_ehashfn(laddr, lport, faddr, fport,
udp_ehash_secret + net_hash_mix(net));
}
+EXPORT_SYMBOL(udp_ehashfn);
/* called with rcu_read_lock() */
static struct sock *udp4_lib_lookup2(const struct net *net,
diff --git a/net/ipv6/udp.c b/net/ipv6/udp.c
index 1ea99d704e31..d766fd798ecf 100644
--- a/net/ipv6/udp.c
+++ b/net/ipv6/udp.c
@@ -110,8 +110,19 @@ void udp_v6_rehash(struct sock *sk)
u16 new_hash = ipv6_portaddr_hash(sock_net(sk),
&sk->sk_v6_rcv_saddr,
inet_sk(sk)->inet_num);
+ u16 new_hash4;
- udp_lib_rehash(sk, new_hash, 0); /* 4-tuple hash not implemented */
+ if (ipv6_addr_v4mapped(&sk->sk_v6_rcv_saddr)) {
+ new_hash4 = udp_ehashfn(sock_net(sk),
+ sk->sk_rcv_saddr, sk->sk_num,
+ sk->sk_daddr, sk->sk_dport);
+ } else {
+ new_hash4 = udp6_ehashfn(sock_net(sk),
+ &sk->sk_v6_rcv_saddr, sk->sk_num,
+ &sk->sk_v6_daddr, sk->sk_dport);
+ }
+
+ udp_lib_rehash(sk, new_hash, new_hash4);
}
static int compute_score(struct sock *sk, const struct net *net,
@@ -216,6 +227,74 @@ static struct sock *udp6_lib_lookup2(const struct net *net,
return result;
}
+#if IS_ENABLED(CONFIG_BASE_SMALL)
+static struct sock *udp6_lib_lookup4(const struct net *net,
+ const struct in6_addr *saddr, __be16 sport,
+ const struct in6_addr *daddr,
+ unsigned int hnum, int dif, int sdif,
+ struct udp_table *udptable)
+{
+ return NULL;
+}
+
+static void udp6_hash4(struct sock *sk)
+{
+}
+#else /* !CONFIG_BASE_SMALL */
+static struct sock *udp6_lib_lookup4(const struct net *net,
+ const struct in6_addr *saddr, __be16 sport,
+ const struct in6_addr *daddr,
+ unsigned int hnum, int dif, int sdif,
+ struct udp_table *udptable)
+{
+ const __portpair ports = INET_COMBINED_PORTS(sport, hnum);
+ const struct hlist_nulls_node *node;
+ struct udp_hslot *hslot4;
+ unsigned int hash4, slot;
+ struct udp_sock *up;
+ struct sock *sk;
+
+ hash4 = udp6_ehashfn(net, daddr, hnum, saddr, sport);
+ slot = hash4 & udptable->mask;
+ hslot4 = &udptable->hash4[slot];
+
+begin:
+ udp_lrpa_for_each_entry_rcu(up, node, &hslot4->nulls_head) {
+ sk = (struct sock *)up;
+ if (inet6_match(net, sk, saddr, daddr, ports, dif, sdif))
+ return sk;
+ }
+
+ /* if the nulls value we got at the end of this lookup is not the
+ * expected one, we must restart lookup. We probably met an item that
+ * was moved to another chain due to rehash.
+ */
+ if (get_nulls_value(node) != slot)
+ goto begin;
+
+ return NULL;
+}
+
+static void udp6_hash4(struct sock *sk)
+{
+ struct net *net = sock_net(sk);
+ unsigned int hash;
+
+ if (ipv6_addr_v4mapped(&sk->sk_v6_rcv_saddr)) {
+ udp4_hash4(sk);
+ return;
+ }
+
+ if (sk_unhashed(sk) || ipv6_addr_any(&sk->sk_v6_rcv_saddr))
+ return;
+
+ hash = udp6_ehashfn(net, &sk->sk_v6_rcv_saddr, sk->sk_num,
+ &sk->sk_v6_daddr, sk->sk_dport);
+
+ udp_lib_hash4(sk, hash);
+}
+#endif /* CONFIG_BASE_SMALL */
+
/* rcu_read_lock() must be held */
struct sock *__udp6_lib_lookup(const struct net *net,
const struct in6_addr *saddr, __be16 sport,
@@ -231,6 +310,13 @@ struct sock *__udp6_lib_lookup(const struct net *net,
hash2 = ipv6_portaddr_hash(net, daddr, hnum);
hslot2 = udp_hashslot2(udptable, hash2);
+ if (udp_has_hash4(hslot2)) {
+ result = udp6_lib_lookup4(net, saddr, sport, daddr, hnum,
+ dif, sdif, udptable);
+ if (result) /* udp6_lib_lookup4 return sk or NULL */
+ return result;
+ }
+
/* Lookup connected or non-wildcard sockets */
result = udp6_lib_lookup2(net, saddr, sport,
daddr, hnum, dif, sdif,
@@ -1166,6 +1252,18 @@ static int udpv6_pre_connect(struct sock *sk, struct sockaddr *uaddr,
return BPF_CGROUP_RUN_PROG_INET6_CONNECT_LOCK(sk, uaddr, &addr_len);
}
+static int udpv6_connect(struct sock *sk, struct sockaddr *uaddr, int addr_len)
+{
+ int res;
+
+ lock_sock(sk);
+ res = __ip6_datagram_connect(sk, uaddr, addr_len);
+ if (!res)
+ udp6_hash4(sk);
+ release_sock(sk);
+ return res;
+}
+
/**
* udp6_hwcsum_outgoing - handle outgoing HW checksumming
* @sk: socket we are sending on
@@ -1761,7 +1859,7 @@ struct proto udpv6_prot = {
.owner = THIS_MODULE,
.close = udp_lib_close,
.pre_connect = udpv6_pre_connect,
- .connect = ip6_datagram_connect,
+ .connect = udpv6_connect,
.disconnect = udp_disconnect,
.ioctl = udp_ioctl,
.init = udpv6_init_sock,
--
2.32.0.3.g01195cf9f
^ permalink raw reply [flat|nested] 9+ messages in thread* Re: [PATCH v8 net-next 0/4] udp: Add 4-tuple hash for connected sockets
2024-11-08 5:48 [PATCH v8 net-next 0/4] udp: Add 4-tuple hash for connected sockets Philo Lu
` (3 preceding siblings ...)
2024-11-08 5:48 ` [PATCH v8 net-next 4/4] ipv6/udp: " Philo Lu
@ 2024-11-11 17:58 ` Willem de Bruijn
4 siblings, 0 replies; 9+ messages in thread
From: Willem de Bruijn @ 2024-11-11 17:58 UTC (permalink / raw)
To: Philo Lu, netdev
Cc: willemdebruijn.kernel, davem, edumazet, kuba, pabeni, dsahern,
horms, antony.antony, steffen.klassert, linux-kernel, dust.li,
jakub, fred.cc, yubing.qiuyubing
Philo Lu wrote:
> This patchset introduces 4-tuple hash for connected udp sockets, to make
> connected udp lookup faster.
>
> Stress test results (with 1 cpu fully used) are shown below, in pps:
> (1) _un-connected_ socket as server
> [a] w/o hash4: 1,825176
> [b] w/ hash4: 1,831750 (+0.36%)
>
> (2) 500 _connected_ sockets as server
> [c] w/o hash4: 290860 (only 16% of [a])
> [d] w/ hash4: 1,889658 (+3.1% compared with [b])
> With hash4, compute_score is skipped when lookup, so [d] is slightly
> better than [b].
>
> Patch1: Add a new counter for hslot2 named hash4_cnt, to avoid cache line
> miss when lookup.
> Patch2: Add hslot/hlist_nulls for 4-tuple hash.
> Patch3 and 4: Implement 4-tuple hash for ipv4 and ipv6.
>
> The detailed motivation is described in Patch 3.
>
> The 4-tuple hash increases the size of udp_sock and udp_hslot. Thus add it
> with CONFIG_BASE_SMALL, i.e., it's a no op with CONFIG_BASE_SMALL.
>
> changelogs:
> v7 -> v8:
> - add EXPORT_SYMBOL for ipv6.ko build
>
> v6 -> v7 (Kuniyuki Iwashima):
> - export udp_ehashfn to be used by udpv6 rehash
>
> v5 -> v6 (Paolo Abeni):
> - move udp_table_hash4_init from patch2 to patch1
> - use hlist_nulls for lookup-rehash race
> - add test results in commit log
> - add more comment, e.g., for rehash4 used in hash4
> - add ipv6 support (Patch4), and refactor some functions for better
> sharing, without functionality change
>
> v4 -> v5 (Paolo Abeni):
> - add CONFIG_BASE_SMALL with which udp hash4 does nothing
>
> v3 -> v4 (Willem de Bruijn):
> - fix mistakes in udp_pernet_table_alloc()
>
> RFCv2 -> v3 (Gur Stavi):
> - minor fix in udp_hashslot2() and udp_table_init()
> - add rcu sync in rehash4()
>
> RFCv1 -> RFCv2:
> - add a new struct for hslot2
> - remove the sockopt UDP_HASH4 because it has little side effect for
> unconnected sockets
> - add rehash in connect()
> - re-organize the patch into 3 smaller ones
> - other minor fix
>
> v7:
> https://lore.kernel.org/all/20241105121225.12513-1-lulie@linux.alibaba.com/
> v6:
> https://lore.kernel.org/all/20241031124550.20227-1-lulie@linux.alibaba.com/
> v5:
> https://lore.kernel.org/all/20241018114535.35712-1-lulie@linux.alibaba.com/
> v4:
> https://lore.kernel.org/all/20241012012918.70888-1-lulie@linux.alibaba.com/
> v3:
> https://lore.kernel.org/all/20241010090351.79698-1-lulie@linux.alibaba.com/
> RFCv2:
> https://lore.kernel.org/all/20240924110414.52618-1-lulie@linux.alibaba.com/
> RFCv1:
> https://lore.kernel.org/all/20240913100941.8565-1-lulie@linux.alibaba.com/
>
> Philo Lu (4):
> net/udp: Add a new struct for hash2 slot
> net/udp: Add 4-tuple hash list basis
> ipv4/udp: Add 4-tuple hash for connected socket
> ipv6/udp: Add 4-tuple hash for connected socket
>
> include/linux/udp.h | 11 ++
> include/net/udp.h | 137 +++++++++++++++++++++++--
> net/ipv4/udp.c | 245 +++++++++++++++++++++++++++++++++++++++-----
> net/ipv6/udp.c | 117 +++++++++++++++++++--
> 4 files changed, 468 insertions(+), 42 deletions(-)
Acked-by: Willem de Bruijn <willemb@google.com>
My expertise in routing is limited, fair warning.
But I see no significant remaining issues.
^ permalink raw reply [flat|nested] 9+ messages in thread