* [PATCH net-next v2 1/4] net: use indirect call helpers for dst_input
2020-12-11 23:33 [PATCH net-next v2 0/4] net: avoid indirect calls in dst functions Brian Vazquez
@ 2020-12-11 23:33 ` Brian Vazquez
2020-12-11 23:33 ` [PATCH net-next v2 2/4] net: use indirect call helpers for dst_output Brian Vazquez
` (3 subsequent siblings)
4 siblings, 0 replies; 6+ messages in thread
From: Brian Vazquez @ 2020-12-11 23:33 UTC (permalink / raw)
To: Brian Vazquez, Brian Vazquez, Eric Dumazet, David S . Miller
Cc: linux-kernel, netdev
From: brianvv <brianvv@google.com>
This patch avoids the indirect call for the common case:
ip_local_deliver and ip6_input
Signed-off-by: brianvv <brianvv@google.com>
---
include/net/dst.h | 6 +++++-
net/ipv4/ip_input.c | 1 +
2 files changed, 6 insertions(+), 1 deletion(-)
diff --git a/include/net/dst.h b/include/net/dst.h
index 10f0a8399867..98cf6e8c06c4 100644
--- a/include/net/dst.h
+++ b/include/net/dst.h
@@ -18,6 +18,7 @@
#include <linux/refcount.h>
#include <net/neighbour.h>
#include <asm/processor.h>
+#include <linux/indirect_call_wrapper.h>
struct sk_buff;
@@ -441,10 +442,13 @@ static inline int dst_output(struct net *net, struct sock *sk, struct sk_buff *s
return skb_dst(skb)->output(net, sk, skb);
}
+INDIRECT_CALLABLE_DECLARE(int ip6_input(struct sk_buff *));
+INDIRECT_CALLABLE_DECLARE(int ip_local_deliver(struct sk_buff *));
/* Input packet from network to transport. */
static inline int dst_input(struct sk_buff *skb)
{
- return skb_dst(skb)->input(skb);
+ return INDIRECT_CALL_INET(skb_dst(skb)->input,
+ ip6_input, ip_local_deliver, skb);
}
static inline struct dst_entry *dst_check(struct dst_entry *dst, u32 cookie)
diff --git a/net/ipv4/ip_input.c b/net/ipv4/ip_input.c
index b0c244af1e4d..3a025c011971 100644
--- a/net/ipv4/ip_input.c
+++ b/net/ipv4/ip_input.c
@@ -253,6 +253,7 @@ int ip_local_deliver(struct sk_buff *skb)
net, NULL, skb, skb->dev, NULL,
ip_local_deliver_finish);
}
+EXPORT_SYMBOL(ip_local_deliver);
static inline bool ip_rcv_options(struct sk_buff *skb, struct net_device *dev)
{
--
2.29.2.576.ga3fc446d84-goog
^ permalink raw reply [flat|nested] 6+ messages in thread* [PATCH net-next v2 2/4] net: use indirect call helpers for dst_output
2020-12-11 23:33 [PATCH net-next v2 0/4] net: avoid indirect calls in dst functions Brian Vazquez
2020-12-11 23:33 ` [PATCH net-next v2 1/4] net: use indirect call helpers for dst_input Brian Vazquez
@ 2020-12-11 23:33 ` Brian Vazquez
2020-12-11 23:33 ` [PATCH net-next v2 3/4] net: use indirect call helpers for dst_mtu Brian Vazquez
` (2 subsequent siblings)
4 siblings, 0 replies; 6+ messages in thread
From: Brian Vazquez @ 2020-12-11 23:33 UTC (permalink / raw)
To: Brian Vazquez, Brian Vazquez, Eric Dumazet, David S . Miller
Cc: linux-kernel, netdev
From: brianvv <brianvv@google.com>
This patch avoids the indirect call for the common case:
ip6_output and ip_output
Signed-off-by: brianvv <brianvv@google.com>
---
include/net/dst.h | 8 +++++++-
net/ipv4/ip_output.c | 1 +
net/ipv6/ip6_output.c | 1 +
3 files changed, 9 insertions(+), 1 deletion(-)
diff --git a/include/net/dst.h b/include/net/dst.h
index 98cf6e8c06c4..3932e9931f08 100644
--- a/include/net/dst.h
+++ b/include/net/dst.h
@@ -436,10 +436,16 @@ static inline void dst_set_expires(struct dst_entry *dst, int timeout)
dst->expires = expires;
}
+INDIRECT_CALLABLE_DECLARE(int ip6_output(struct net *, struct sock *,
+ struct sk_buff *));
+INDIRECT_CALLABLE_DECLARE(int ip_output(struct net *, struct sock *,
+ struct sk_buff *));
/* Output packet to network from transport. */
static inline int dst_output(struct net *net, struct sock *sk, struct sk_buff *skb)
{
- return skb_dst(skb)->output(net, sk, skb);
+ return INDIRECT_CALL_INET(skb_dst(skb)->output,
+ ip6_output, ip_output,
+ net, sk, skb);
}
INDIRECT_CALLABLE_DECLARE(int ip6_input(struct sk_buff *));
diff --git a/net/ipv4/ip_output.c b/net/ipv4/ip_output.c
index 879b76ae4435..356c89575b08 100644
--- a/net/ipv4/ip_output.c
+++ b/net/ipv4/ip_output.c
@@ -434,6 +434,7 @@ int ip_output(struct net *net, struct sock *sk, struct sk_buff *skb)
ip_finish_output,
!(IPCB(skb)->flags & IPSKB_REROUTED));
}
+EXPORT_SYMBOL(ip_output);
/*
* copy saddr and daddr, possibly using 64bit load/stores
diff --git a/net/ipv6/ip6_output.c b/net/ipv6/ip6_output.c
index 749ad72386b2..1260c0cac592 100644
--- a/net/ipv6/ip6_output.c
+++ b/net/ipv6/ip6_output.c
@@ -178,6 +178,7 @@ int ip6_output(struct net *net, struct sock *sk, struct sk_buff *skb)
ip6_finish_output,
!(IP6CB(skb)->flags & IP6SKB_REROUTED));
}
+EXPORT_SYMBOL(ip6_output);
bool ip6_autoflowlabel(struct net *net, const struct ipv6_pinfo *np)
{
--
2.29.2.576.ga3fc446d84-goog
^ permalink raw reply [flat|nested] 6+ messages in thread* [PATCH net-next v2 3/4] net: use indirect call helpers for dst_mtu
2020-12-11 23:33 [PATCH net-next v2 0/4] net: avoid indirect calls in dst functions Brian Vazquez
2020-12-11 23:33 ` [PATCH net-next v2 1/4] net: use indirect call helpers for dst_input Brian Vazquez
2020-12-11 23:33 ` [PATCH net-next v2 2/4] net: use indirect call helpers for dst_output Brian Vazquez
@ 2020-12-11 23:33 ` Brian Vazquez
2020-12-11 23:33 ` [PATCH net-next v2 4/4] net: indirect call helpers for ipv4/ipv6 dst_check functions Brian Vazquez
2020-12-15 3:27 ` [PATCH net-next v2 0/4] net: avoid indirect calls in dst functions Jakub Kicinski
4 siblings, 0 replies; 6+ messages in thread
From: Brian Vazquez @ 2020-12-11 23:33 UTC (permalink / raw)
To: Brian Vazquez, Brian Vazquez, Eric Dumazet, David S . Miller
Cc: linux-kernel, netdev
From: brianvv <brianvv@google.com>
This patch avoids the indirect call for the common case:
ip6_mtu and ipv4_mtu
Signed-off-by: brianvv <brianvv@google.com>
---
include/net/dst.h | 4 +++-
net/ipv4/route.c | 6 ++++--
net/ipv6/route.c | 6 ++++--
3 files changed, 11 insertions(+), 5 deletions(-)
diff --git a/include/net/dst.h b/include/net/dst.h
index 3932e9931f08..9f474a79ed7d 100644
--- a/include/net/dst.h
+++ b/include/net/dst.h
@@ -194,9 +194,11 @@ dst_feature(const struct dst_entry *dst, u32 feature)
return dst_metric(dst, RTAX_FEATURES) & feature;
}
+INDIRECT_CALLABLE_DECLARE(unsigned int ip6_mtu(const struct dst_entry *));
+INDIRECT_CALLABLE_DECLARE(unsigned int ipv4_mtu(const struct dst_entry *));
static inline u32 dst_mtu(const struct dst_entry *dst)
{
- return dst->ops->mtu(dst);
+ return INDIRECT_CALL_INET(dst->ops->mtu, ip6_mtu, ipv4_mtu, dst);
}
/* RTT metrics are stored in milliseconds for user ABI, but used as jiffies */
diff --git a/net/ipv4/route.c b/net/ipv4/route.c
index e26652ff7059..4fac91f8bd6c 100644
--- a/net/ipv4/route.c
+++ b/net/ipv4/route.c
@@ -135,7 +135,8 @@ static int ip_rt_gc_timeout __read_mostly = RT_GC_TIMEOUT;
static struct dst_entry *ipv4_dst_check(struct dst_entry *dst, u32 cookie);
static unsigned int ipv4_default_advmss(const struct dst_entry *dst);
-static unsigned int ipv4_mtu(const struct dst_entry *dst);
+INDIRECT_CALLABLE_SCOPE
+unsigned int ipv4_mtu(const struct dst_entry *dst);
static struct dst_entry *ipv4_negative_advice(struct dst_entry *dst);
static void ipv4_link_failure(struct sk_buff *skb);
static void ip_rt_update_pmtu(struct dst_entry *dst, struct sock *sk,
@@ -1311,7 +1312,7 @@ static unsigned int ipv4_default_advmss(const struct dst_entry *dst)
return min(advmss, IPV4_MAX_PMTU - header_size);
}
-static unsigned int ipv4_mtu(const struct dst_entry *dst)
+INDIRECT_CALLABLE_SCOPE unsigned int ipv4_mtu(const struct dst_entry *dst)
{
const struct rtable *rt = (const struct rtable *)dst;
unsigned int mtu = rt->rt_pmtu;
@@ -1333,6 +1334,7 @@ static unsigned int ipv4_mtu(const struct dst_entry *dst)
return mtu - lwtunnel_headroom(dst->lwtstate, mtu);
}
+EXPORT_SYMBOL(ipv4_mtu);
static void ip_del_fnhe(struct fib_nh_common *nhc, __be32 daddr)
{
diff --git a/net/ipv6/route.c b/net/ipv6/route.c
index 188e114b29b4..22caee290b6c 100644
--- a/net/ipv6/route.c
+++ b/net/ipv6/route.c
@@ -83,7 +83,8 @@ enum rt6_nud_state {
static struct dst_entry *ip6_dst_check(struct dst_entry *dst, u32 cookie);
static unsigned int ip6_default_advmss(const struct dst_entry *dst);
-static unsigned int ip6_mtu(const struct dst_entry *dst);
+INDIRECT_CALLABLE_SCOPE
+unsigned int ip6_mtu(const struct dst_entry *dst);
static struct dst_entry *ip6_negative_advice(struct dst_entry *);
static void ip6_dst_destroy(struct dst_entry *);
static void ip6_dst_ifdown(struct dst_entry *,
@@ -3089,7 +3090,7 @@ static unsigned int ip6_default_advmss(const struct dst_entry *dst)
return mtu;
}
-static unsigned int ip6_mtu(const struct dst_entry *dst)
+INDIRECT_CALLABLE_SCOPE unsigned int ip6_mtu(const struct dst_entry *dst)
{
struct inet6_dev *idev;
unsigned int mtu;
@@ -3111,6 +3112,7 @@ static unsigned int ip6_mtu(const struct dst_entry *dst)
return mtu - lwtunnel_headroom(dst->lwtstate, mtu);
}
+EXPORT_SYMBOL(ip6_mtu);
/* MTU selection:
* 1. mtu on route is locked - use it
--
2.29.2.576.ga3fc446d84-goog
^ permalink raw reply [flat|nested] 6+ messages in thread* [PATCH net-next v2 4/4] net: indirect call helpers for ipv4/ipv6 dst_check functions
2020-12-11 23:33 [PATCH net-next v2 0/4] net: avoid indirect calls in dst functions Brian Vazquez
` (2 preceding siblings ...)
2020-12-11 23:33 ` [PATCH net-next v2 3/4] net: use indirect call helpers for dst_mtu Brian Vazquez
@ 2020-12-11 23:33 ` Brian Vazquez
2020-12-15 3:27 ` [PATCH net-next v2 0/4] net: avoid indirect calls in dst functions Jakub Kicinski
4 siblings, 0 replies; 6+ messages in thread
From: Brian Vazquez @ 2020-12-11 23:33 UTC (permalink / raw)
To: Brian Vazquez, Brian Vazquez, Eric Dumazet, David S . Miller
Cc: linux-kernel, netdev
From: brianvv <brianvv@google.com>
This patch avoids the indirect call for the common case:
ip6_dst_check and ipv4_dst_check
Signed-off-by: brianvv <brianvv@google.com>
---
include/net/dst.h | 7 ++++++-
net/core/sock.c | 12 ++++++++++--
net/ipv4/route.c | 7 +++++--
net/ipv4/tcp_ipv4.c | 5 ++++-
net/ipv6/route.c | 7 +++++--
net/ipv6/tcp_ipv6.c | 5 ++++-
6 files changed, 34 insertions(+), 9 deletions(-)
diff --git a/include/net/dst.h b/include/net/dst.h
index 9f474a79ed7d..26f134ad3a25 100644
--- a/include/net/dst.h
+++ b/include/net/dst.h
@@ -459,10 +459,15 @@ static inline int dst_input(struct sk_buff *skb)
ip6_input, ip_local_deliver, skb);
}
+INDIRECT_CALLABLE_DECLARE(struct dst_entry *ip6_dst_check(struct dst_entry *,
+ u32));
+INDIRECT_CALLABLE_DECLARE(struct dst_entry *ipv4_dst_check(struct dst_entry *,
+ u32));
static inline struct dst_entry *dst_check(struct dst_entry *dst, u32 cookie)
{
if (dst->obsolete)
- dst = dst->ops->check(dst, cookie);
+ dst = INDIRECT_CALL_INET(dst->ops->check, ip6_dst_check,
+ ipv4_dst_check, dst, cookie);
return dst;
}
diff --git a/net/core/sock.c b/net/core/sock.c
index 4fd7e785f177..753b831a9d70 100644
--- a/net/core/sock.c
+++ b/net/core/sock.c
@@ -526,11 +526,17 @@ int __sk_receive_skb(struct sock *sk, struct sk_buff *skb,
}
EXPORT_SYMBOL(__sk_receive_skb);
+INDIRECT_CALLABLE_DECLARE(struct dst_entry *ip6_dst_check(struct dst_entry *,
+ u32));
+INDIRECT_CALLABLE_DECLARE(struct dst_entry *ipv4_dst_check(struct dst_entry *,
+ u32));
struct dst_entry *__sk_dst_check(struct sock *sk, u32 cookie)
{
struct dst_entry *dst = __sk_dst_get(sk);
- if (dst && dst->obsolete && dst->ops->check(dst, cookie) == NULL) {
+ if (dst && dst->obsolete &&
+ INDIRECT_CALL_INET(dst->ops->check, ip6_dst_check, ipv4_dst_check,
+ dst, cookie) == NULL) {
sk_tx_queue_clear(sk);
sk->sk_dst_pending_confirm = 0;
RCU_INIT_POINTER(sk->sk_dst_cache, NULL);
@@ -546,7 +552,9 @@ struct dst_entry *sk_dst_check(struct sock *sk, u32 cookie)
{
struct dst_entry *dst = sk_dst_get(sk);
- if (dst && dst->obsolete && dst->ops->check(dst, cookie) == NULL) {
+ if (dst && dst->obsolete &&
+ INDIRECT_CALL_INET(dst->ops->check, ip6_dst_check, ipv4_dst_check,
+ dst, cookie) == NULL) {
sk_dst_reset(sk);
dst_release(dst);
return NULL;
diff --git a/net/ipv4/route.c b/net/ipv4/route.c
index 4fac91f8bd6c..9e6537709794 100644
--- a/net/ipv4/route.c
+++ b/net/ipv4/route.c
@@ -133,7 +133,8 @@ static int ip_rt_gc_timeout __read_mostly = RT_GC_TIMEOUT;
* Interface to generic destination cache.
*/
-static struct dst_entry *ipv4_dst_check(struct dst_entry *dst, u32 cookie);
+INDIRECT_CALLABLE_SCOPE
+struct dst_entry *ipv4_dst_check(struct dst_entry *dst, u32 cookie);
static unsigned int ipv4_default_advmss(const struct dst_entry *dst);
INDIRECT_CALLABLE_SCOPE
unsigned int ipv4_mtu(const struct dst_entry *dst);
@@ -1188,7 +1189,8 @@ void ipv4_sk_redirect(struct sk_buff *skb, struct sock *sk)
}
EXPORT_SYMBOL_GPL(ipv4_sk_redirect);
-static struct dst_entry *ipv4_dst_check(struct dst_entry *dst, u32 cookie)
+INDIRECT_CALLABLE_SCOPE struct dst_entry *ipv4_dst_check(struct dst_entry *dst,
+ u32 cookie)
{
struct rtable *rt = (struct rtable *) dst;
@@ -1204,6 +1206,7 @@ static struct dst_entry *ipv4_dst_check(struct dst_entry *dst, u32 cookie)
return NULL;
return dst;
}
+EXPORT_SYMBOL(ipv4_dst_check);
static void ipv4_send_dest_unreach(struct sk_buff *skb)
{
diff --git a/net/ipv4/tcp_ipv4.c b/net/ipv4/tcp_ipv4.c
index af2338294598..aba5061024c7 100644
--- a/net/ipv4/tcp_ipv4.c
+++ b/net/ipv4/tcp_ipv4.c
@@ -1646,6 +1646,8 @@ u16 tcp_v4_get_syncookie(struct sock *sk, struct iphdr *iph,
return mss;
}
+INDIRECT_CALLABLE_DECLARE(struct dst_entry *ipv4_dst_check(struct dst_entry *,
+ u32));
/* The socket must have it's spinlock held when we get
* here, unless it is a TCP_LISTEN socket.
*
@@ -1665,7 +1667,8 @@ int tcp_v4_do_rcv(struct sock *sk, struct sk_buff *skb)
sk_mark_napi_id(sk, skb);
if (dst) {
if (inet_sk(sk)->rx_dst_ifindex != skb->skb_iif ||
- !dst->ops->check(dst, 0)) {
+ !INDIRECT_CALL_1(dst->ops->check, ipv4_dst_check,
+ dst, 0)) {
dst_release(dst);
sk->sk_rx_dst = NULL;
}
diff --git a/net/ipv6/route.c b/net/ipv6/route.c
index 22caee290b6c..e074fb5964e2 100644
--- a/net/ipv6/route.c
+++ b/net/ipv6/route.c
@@ -81,7 +81,8 @@ enum rt6_nud_state {
RT6_NUD_SUCCEED = 1
};
-static struct dst_entry *ip6_dst_check(struct dst_entry *dst, u32 cookie);
+INDIRECT_CALLABLE_SCOPE
+struct dst_entry *ip6_dst_check(struct dst_entry *dst, u32 cookie);
static unsigned int ip6_default_advmss(const struct dst_entry *dst);
INDIRECT_CALLABLE_SCOPE
unsigned int ip6_mtu(const struct dst_entry *dst);
@@ -2612,7 +2613,8 @@ static struct dst_entry *rt6_dst_from_check(struct rt6_info *rt,
return NULL;
}
-static struct dst_entry *ip6_dst_check(struct dst_entry *dst, u32 cookie)
+INDIRECT_CALLABLE_SCOPE struct dst_entry *ip6_dst_check(struct dst_entry *dst,
+ u32 cookie)
{
struct dst_entry *dst_ret;
struct fib6_info *from;
@@ -2642,6 +2644,7 @@ static struct dst_entry *ip6_dst_check(struct dst_entry *dst, u32 cookie)
return dst_ret;
}
+EXPORT_SYMBOL(ip6_dst_check);
static struct dst_entry *ip6_negative_advice(struct dst_entry *dst)
{
diff --git a/net/ipv6/tcp_ipv6.c b/net/ipv6/tcp_ipv6.c
index 1a1510513739..9e61e4fda03e 100644
--- a/net/ipv6/tcp_ipv6.c
+++ b/net/ipv6/tcp_ipv6.c
@@ -1417,6 +1417,8 @@ static struct sock *tcp_v6_syn_recv_sock(const struct sock *sk, struct sk_buff *
return NULL;
}
+INDIRECT_CALLABLE_DECLARE(struct dst_entry *ipv4_dst_check(struct dst_entry *,
+ u32));
/* The socket must have it's spinlock held when we get
* here, unless it is a TCP_LISTEN socket.
*
@@ -1470,7 +1472,8 @@ static int tcp_v6_do_rcv(struct sock *sk, struct sk_buff *skb)
sk_mark_napi_id(sk, skb);
if (dst) {
if (inet_sk(sk)->rx_dst_ifindex != skb->skb_iif ||
- dst->ops->check(dst, np->rx_dst_cookie) == NULL) {
+ INDIRECT_CALL_1(dst->ops->check, ip6_dst_check,
+ dst, np->rx_dst_cookie) == NULL) {
dst_release(dst);
sk->sk_rx_dst = NULL;
}
--
2.29.2.576.ga3fc446d84-goog
^ permalink raw reply [flat|nested] 6+ messages in thread* Re: [PATCH net-next v2 0/4] net: avoid indirect calls in dst functions
2020-12-11 23:33 [PATCH net-next v2 0/4] net: avoid indirect calls in dst functions Brian Vazquez
` (3 preceding siblings ...)
2020-12-11 23:33 ` [PATCH net-next v2 4/4] net: indirect call helpers for ipv4/ipv6 dst_check functions Brian Vazquez
@ 2020-12-15 3:27 ` Jakub Kicinski
4 siblings, 0 replies; 6+ messages in thread
From: Jakub Kicinski @ 2020-12-15 3:27 UTC (permalink / raw)
To: Brian Vazquez
Cc: Brian Vazquez, Eric Dumazet, David S . Miller, linux-kernel, netdev
On Fri, 11 Dec 2020 23:33:36 +0000 Brian Vazquez wrote:
> From: brianvv <brianvv@google.com>
We'd prefer you to use your normal name rather than just "brianvv".
> Use of the indirect call wrappers in some dst related functions for the
> ipv6/ipv4 case. This is a small improvent for CONFIG_RETPOLINE=y
Any numbers you can provide?
^ permalink raw reply [flat|nested] 6+ messages in thread