Commit 09c75704 authored by Florian Westphal's avatar Florian Westphal Committed by David S. Miller

xfrm: remove flow cache

After rcu conversions performance degradation in forward tests isn't that
noticeable anymore.

See next patch for some numbers.

A followup patcg could then also remove genid from the policies
as we do not cache bundles anymore.
Signed-off-by: default avatarFlorian Westphal <fw@strlen.de>
Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parent bd45c539
...@@ -218,40 +218,6 @@ static inline unsigned int flow_key_size(u16 family) ...@@ -218,40 +218,6 @@ static inline unsigned int flow_key_size(u16 family)
return 0; return 0;
} }
#define FLOW_DIR_IN 0
#define FLOW_DIR_OUT 1
#define FLOW_DIR_FWD 2
struct net;
struct sock;
struct flow_cache_ops;
struct flow_cache_object {
const struct flow_cache_ops *ops;
};
struct flow_cache_ops {
struct flow_cache_object *(*get)(struct flow_cache_object *);
int (*check)(struct flow_cache_object *);
void (*delete)(struct flow_cache_object *);
};
typedef struct flow_cache_object *(*flow_resolve_t)(
struct net *net, const struct flowi *key, u16 family,
u8 dir, struct flow_cache_object *oldobj, void *ctx);
struct flow_cache_object *flow_cache_lookup(struct net *net,
const struct flowi *key, u16 family,
u8 dir, flow_resolve_t resolver,
void *ctx);
int flow_cache_init(struct net *net);
void flow_cache_fini(struct net *net);
void flow_cache_hp_init(void);
void flow_cache_flush(struct net *net);
void flow_cache_flush_deferred(struct net *net);
extern atomic_t flow_cache_genid;
__u32 __get_hash_from_flowi6(const struct flowi6 *fl6, struct flow_keys *keys); __u32 __get_hash_from_flowi6(const struct flowi6 *fl6, struct flow_keys *keys);
static inline __u32 get_hash_from_flowi6(const struct flowi6 *fl6) static inline __u32 get_hash_from_flowi6(const struct flowi6 *fl6)
......
#ifndef _NET_FLOWCACHE_H
#define _NET_FLOWCACHE_H
#include <linux/interrupt.h>
#include <linux/types.h>
#include <linux/timer.h>
#include <linux/notifier.h>
struct flow_cache_percpu {
struct hlist_head *hash_table;
unsigned int hash_count;
u32 hash_rnd;
int hash_rnd_recalc;
struct tasklet_struct flush_tasklet;
};
struct flow_cache {
u32 hash_shift;
struct flow_cache_percpu __percpu *percpu;
struct hlist_node node;
unsigned int low_watermark;
unsigned int high_watermark;
struct timer_list rnd_timer;
};
#endif /* _NET_FLOWCACHE_H */
...@@ -6,7 +6,6 @@ ...@@ -6,7 +6,6 @@
#include <linux/workqueue.h> #include <linux/workqueue.h>
#include <linux/xfrm.h> #include <linux/xfrm.h>
#include <net/dst_ops.h> #include <net/dst_ops.h>
#include <net/flowcache.h>
struct ctl_table_header; struct ctl_table_header;
...@@ -73,16 +72,6 @@ struct netns_xfrm { ...@@ -73,16 +72,6 @@ struct netns_xfrm {
spinlock_t xfrm_state_lock; spinlock_t xfrm_state_lock;
spinlock_t xfrm_policy_lock; spinlock_t xfrm_policy_lock;
struct mutex xfrm_cfg_mutex; struct mutex xfrm_cfg_mutex;
/* flow cache part */
struct flow_cache flow_cache_global;
atomic_t flow_cache_genid;
struct list_head flow_cache_gc_list;
atomic_t flow_cache_gc_count;
spinlock_t flow_cache_gc_lock;
struct work_struct flow_cache_gc_work;
struct work_struct flow_cache_flush_work;
struct mutex flow_flush_sem;
}; };
#endif #endif
...@@ -563,7 +563,6 @@ struct xfrm_policy { ...@@ -563,7 +563,6 @@ struct xfrm_policy {
refcount_t refcnt; refcount_t refcnt;
struct timer_list timer; struct timer_list timer;
struct flow_cache_object flo;
atomic_t genid; atomic_t genid;
u32 priority; u32 priority;
u32 index; u32 index;
...@@ -978,7 +977,6 @@ struct xfrm_dst { ...@@ -978,7 +977,6 @@ struct xfrm_dst {
struct rt6_info rt6; struct rt6_info rt6;
} u; } u;
struct dst_entry *route; struct dst_entry *route;
struct flow_cache_object flo;
struct xfrm_policy *pols[XFRM_POLICY_TYPE_MAX]; struct xfrm_policy *pols[XFRM_POLICY_TYPE_MAX];
int num_pols, num_xfrms; int num_pols, num_xfrms;
u32 xfrm_genid; u32 xfrm_genid;
...@@ -1226,9 +1224,6 @@ static inline void xfrm_sk_free_policy(struct sock *sk) ...@@ -1226,9 +1224,6 @@ static inline void xfrm_sk_free_policy(struct sock *sk)
} }
} }
void xfrm_garbage_collect(struct net *net);
void xfrm_garbage_collect_deferred(struct net *net);
#else #else
static inline void xfrm_sk_free_policy(struct sock *sk) {} static inline void xfrm_sk_free_policy(struct sock *sk) {}
...@@ -1263,9 +1258,6 @@ static inline int xfrm6_policy_check_reverse(struct sock *sk, int dir, ...@@ -1263,9 +1258,6 @@ static inline int xfrm6_policy_check_reverse(struct sock *sk, int dir,
{ {
return 1; return 1;
} }
static inline void xfrm_garbage_collect(struct net *net)
{
}
#endif #endif
static __inline__ static __inline__
......
...@@ -11,7 +11,6 @@ obj-y += dev.o ethtool.o dev_addr_lists.o dst.o netevent.o \ ...@@ -11,7 +11,6 @@ obj-y += dev.o ethtool.o dev_addr_lists.o dst.o netevent.o \
neighbour.o rtnetlink.o utils.o link_watch.o filter.o \ neighbour.o rtnetlink.o utils.o link_watch.o filter.o \
sock_diag.o dev_ioctl.o tso.o sock_reuseport.o sock_diag.o dev_ioctl.o tso.o sock_reuseport.o
obj-$(CONFIG_XFRM) += flow.o
obj-y += net-sysfs.o obj-y += net-sysfs.o
obj-$(CONFIG_PROC_FS) += net-procfs.o obj-$(CONFIG_PROC_FS) += net-procfs.o
obj-$(CONFIG_NET_PKTGEN) += pktgen.o obj-$(CONFIG_NET_PKTGEN) += pktgen.o
......
This diff is collapsed.
...@@ -213,14 +213,6 @@ _decode_session4(struct sk_buff *skb, struct flowi *fl, int reverse) ...@@ -213,14 +213,6 @@ _decode_session4(struct sk_buff *skb, struct flowi *fl, int reverse)
fl4->flowi4_tos = iph->tos; fl4->flowi4_tos = iph->tos;
} }
static inline int xfrm4_garbage_collect(struct dst_ops *ops)
{
struct net *net = container_of(ops, struct net, xfrm.xfrm4_dst_ops);
xfrm_garbage_collect_deferred(net);
return (dst_entries_get_slow(ops) > ops->gc_thresh * 2);
}
static void xfrm4_update_pmtu(struct dst_entry *dst, struct sock *sk, static void xfrm4_update_pmtu(struct dst_entry *dst, struct sock *sk,
struct sk_buff *skb, u32 mtu) struct sk_buff *skb, u32 mtu)
{ {
...@@ -259,7 +251,6 @@ static void xfrm4_dst_ifdown(struct dst_entry *dst, struct net_device *dev, ...@@ -259,7 +251,6 @@ static void xfrm4_dst_ifdown(struct dst_entry *dst, struct net_device *dev,
static struct dst_ops xfrm4_dst_ops_template = { static struct dst_ops xfrm4_dst_ops_template = {
.family = AF_INET, .family = AF_INET,
.gc = xfrm4_garbage_collect,
.update_pmtu = xfrm4_update_pmtu, .update_pmtu = xfrm4_update_pmtu,
.redirect = xfrm4_redirect, .redirect = xfrm4_redirect,
.cow_metrics = dst_cow_metrics_generic, .cow_metrics = dst_cow_metrics_generic,
......
...@@ -214,14 +214,6 @@ _decode_session6(struct sk_buff *skb, struct flowi *fl, int reverse) ...@@ -214,14 +214,6 @@ _decode_session6(struct sk_buff *skb, struct flowi *fl, int reverse)
} }
} }
static inline int xfrm6_garbage_collect(struct dst_ops *ops)
{
struct net *net = container_of(ops, struct net, xfrm.xfrm6_dst_ops);
xfrm_garbage_collect_deferred(net);
return dst_entries_get_fast(ops) > ops->gc_thresh * 2;
}
static void xfrm6_update_pmtu(struct dst_entry *dst, struct sock *sk, static void xfrm6_update_pmtu(struct dst_entry *dst, struct sock *sk,
struct sk_buff *skb, u32 mtu) struct sk_buff *skb, u32 mtu)
{ {
...@@ -279,7 +271,6 @@ static void xfrm6_dst_ifdown(struct dst_entry *dst, struct net_device *dev, ...@@ -279,7 +271,6 @@ static void xfrm6_dst_ifdown(struct dst_entry *dst, struct net_device *dev,
static struct dst_ops xfrm6_dst_ops_template = { static struct dst_ops xfrm6_dst_ops_template = {
.family = AF_INET6, .family = AF_INET6,
.gc = xfrm6_garbage_collect,
.update_pmtu = xfrm6_update_pmtu, .update_pmtu = xfrm6_update_pmtu,
.redirect = xfrm6_redirect, .redirect = xfrm6_redirect,
.cow_metrics = dst_cow_metrics_generic, .cow_metrics = dst_cow_metrics_generic,
......
...@@ -2398,8 +2398,6 @@ static int pfkey_spddelete(struct sock *sk, struct sk_buff *skb, const struct sa ...@@ -2398,8 +2398,6 @@ static int pfkey_spddelete(struct sock *sk, struct sk_buff *skb, const struct sa
out: out:
xfrm_pol_put(xp); xfrm_pol_put(xp);
if (err == 0)
xfrm_garbage_collect(net);
return err; return err;
} }
...@@ -2650,8 +2648,6 @@ static int pfkey_spdget(struct sock *sk, struct sk_buff *skb, const struct sadb_ ...@@ -2650,8 +2648,6 @@ static int pfkey_spdget(struct sock *sk, struct sk_buff *skb, const struct sadb_
out: out:
xfrm_pol_put(xp); xfrm_pol_put(xp);
if (delete && err == 0)
xfrm_garbage_collect(net);
return err; return err;
} }
...@@ -2751,8 +2747,6 @@ static int pfkey_spdflush(struct sock *sk, struct sk_buff *skb, const struct sad ...@@ -2751,8 +2747,6 @@ static int pfkey_spdflush(struct sock *sk, struct sk_buff *skb, const struct sad
int err, err2; int err, err2;
err = xfrm_policy_flush(net, XFRM_POLICY_TYPE_MAIN, true); err = xfrm_policy_flush(net, XFRM_POLICY_TYPE_MAIN, true);
if (!err)
xfrm_garbage_collect(net);
err2 = unicast_flush_resp(sk, hdr); err2 = unicast_flush_resp(sk, hdr);
if (err || err2) { if (err || err2) {
if (err == -ESRCH) /* empty table - old silent behavior */ if (err == -ESRCH) /* empty table - old silent behavior */
......
...@@ -175,8 +175,6 @@ static int xfrm_dev_down(struct net_device *dev) ...@@ -175,8 +175,6 @@ static int xfrm_dev_down(struct net_device *dev)
if (dev->features & NETIF_F_HW_ESP) if (dev->features & NETIF_F_HW_ESP)
xfrm_dev_state_flush(dev_net(dev), dev, true); xfrm_dev_state_flush(dev_net(dev), dev, true);
xfrm_garbage_collect(dev_net(dev));
return NOTIFY_DONE; return NOTIFY_DONE;
} }
......
...@@ -246,36 +246,6 @@ static void xfrm_policy_timer(unsigned long data) ...@@ -246,36 +246,6 @@ static void xfrm_policy_timer(unsigned long data)
xfrm_pol_put(xp); xfrm_pol_put(xp);
} }
static struct flow_cache_object *xfrm_policy_flo_get(struct flow_cache_object *flo)
{
struct xfrm_policy *pol = container_of(flo, struct xfrm_policy, flo);
if (unlikely(pol->walk.dead))
flo = NULL;
else
xfrm_pol_hold(pol);
return flo;
}
static int xfrm_policy_flo_check(struct flow_cache_object *flo)
{
struct xfrm_policy *pol = container_of(flo, struct xfrm_policy, flo);
return !pol->walk.dead;
}
static void xfrm_policy_flo_delete(struct flow_cache_object *flo)
{
xfrm_pol_put(container_of(flo, struct xfrm_policy, flo));
}
static const struct flow_cache_ops xfrm_policy_fc_ops = {
.get = xfrm_policy_flo_get,
.check = xfrm_policy_flo_check,
.delete = xfrm_policy_flo_delete,
};
/* Allocate xfrm_policy. Not used here, it is supposed to be used by pfkeyv2 /* Allocate xfrm_policy. Not used here, it is supposed to be used by pfkeyv2
* SPD calls. * SPD calls.
*/ */
...@@ -298,7 +268,6 @@ struct xfrm_policy *xfrm_policy_alloc(struct net *net, gfp_t gfp) ...@@ -298,7 +268,6 @@ struct xfrm_policy *xfrm_policy_alloc(struct net *net, gfp_t gfp)
(unsigned long)policy); (unsigned long)policy);
setup_timer(&policy->polq.hold_timer, xfrm_policy_queue_process, setup_timer(&policy->polq.hold_timer, xfrm_policy_queue_process,
(unsigned long)policy); (unsigned long)policy);
policy->flo.ops = &xfrm_policy_fc_ops;
} }
return policy; return policy;
} }
...@@ -798,7 +767,6 @@ int xfrm_policy_insert(int dir, struct xfrm_policy *policy, int excl) ...@@ -798,7 +767,6 @@ int xfrm_policy_insert(int dir, struct xfrm_policy *policy, int excl)
else else
hlist_add_head(&policy->bydst, chain); hlist_add_head(&policy->bydst, chain);
__xfrm_policy_link(policy, dir); __xfrm_policy_link(policy, dir);
atomic_inc(&net->xfrm.flow_cache_genid);
/* After previous checking, family can either be AF_INET or AF_INET6 */ /* After previous checking, family can either be AF_INET or AF_INET6 */
if (policy->family == AF_INET) if (policy->family == AF_INET)
...@@ -1490,58 +1458,6 @@ static int xfrm_get_tos(const struct flowi *fl, int family) ...@@ -1490,58 +1458,6 @@ static int xfrm_get_tos(const struct flowi *fl, int family)
return tos; return tos;
} }
static struct flow_cache_object *xfrm_bundle_flo_get(struct flow_cache_object *flo)
{
struct xfrm_dst *xdst = container_of(flo, struct xfrm_dst, flo);
struct dst_entry *dst = &xdst->u.dst;
if (xdst->route == NULL) {
/* Dummy bundle - if it has xfrms we were not
* able to build bundle as template resolution failed.
* It means we need to try again resolving. */
if (xdst->num_xfrms > 0)
return NULL;
} else if (dst->flags & DST_XFRM_QUEUE) {
return NULL;
} else {
/* Real bundle */
if (stale_bundle(dst))
return NULL;
}
dst_hold(dst);
return flo;
}
static int xfrm_bundle_flo_check(struct flow_cache_object *flo)
{
struct xfrm_dst *xdst = container_of(flo, struct xfrm_dst, flo);
struct dst_entry *dst = &xdst->u.dst;
if (!xdst->route)
return 0;
if (stale_bundle(dst))
return 0;
return 1;
}
static void xfrm_bundle_flo_delete(struct flow_cache_object *flo)
{
struct xfrm_dst *xdst = container_of(flo, struct xfrm_dst, flo);
struct dst_entry *dst = &xdst->u.dst;
/* Mark DST_OBSOLETE_DEAD to fail the next xfrm_dst_check() */
dst->obsolete = DST_OBSOLETE_DEAD;
dst_release_immediate(dst);
}
static const struct flow_cache_ops xfrm_bundle_fc_ops = {
.get = xfrm_bundle_flo_get,
.check = xfrm_bundle_flo_check,
.delete = xfrm_bundle_flo_delete,
};
static inline struct xfrm_dst *xfrm_alloc_dst(struct net *net, int family) static inline struct xfrm_dst *xfrm_alloc_dst(struct net *net, int family)
{ {
const struct xfrm_policy_afinfo *afinfo = xfrm_policy_get_afinfo(family); const struct xfrm_policy_afinfo *afinfo = xfrm_policy_get_afinfo(family);
...@@ -1569,7 +1485,6 @@ static inline struct xfrm_dst *xfrm_alloc_dst(struct net *net, int family) ...@@ -1569,7 +1485,6 @@ static inline struct xfrm_dst *xfrm_alloc_dst(struct net *net, int family)
struct dst_entry *dst = &xdst->u.dst; struct dst_entry *dst = &xdst->u.dst;
memset(dst + 1, 0, sizeof(*xdst) - sizeof(*dst)); memset(dst + 1, 0, sizeof(*xdst) - sizeof(*dst));
xdst->flo.ops = &xfrm_bundle_fc_ops;
} else } else
xdst = ERR_PTR(-ENOBUFS); xdst = ERR_PTR(-ENOBUFS);
...@@ -2521,11 +2436,9 @@ static struct dst_entry *xfrm_dst_check(struct dst_entry *dst, u32 cookie) ...@@ -2521,11 +2436,9 @@ static struct dst_entry *xfrm_dst_check(struct dst_entry *dst, u32 cookie)
* notice. That's what we are validating here via the * notice. That's what we are validating here via the
* stale_bundle() check. * stale_bundle() check.
* *
* When an xdst is removed from flow cache, DST_OBSOLETE_DEAD will
* be marked on it.
* When a dst is removed from the fib tree, DST_OBSOLETE_DEAD will * When a dst is removed from the fib tree, DST_OBSOLETE_DEAD will
* be marked on it. * be marked on it.
* Both will force stable_bundle() to fail on any xdst bundle with * This will force stale_bundle() to fail on any xdst bundle with
* this dst linked in it. * this dst linked in it.
*/ */
if (dst->obsolete < 0 && !stale_bundle(dst)) if (dst->obsolete < 0 && !stale_bundle(dst))
...@@ -2565,18 +2478,6 @@ static struct dst_entry *xfrm_negative_advice(struct dst_entry *dst) ...@@ -2565,18 +2478,6 @@ static struct dst_entry *xfrm_negative_advice(struct dst_entry *dst)
return dst; return dst;
} }
void xfrm_garbage_collect(struct net *net)
{
flow_cache_flush(net);
}
EXPORT_SYMBOL(xfrm_garbage_collect);
void xfrm_garbage_collect_deferred(struct net *net)
{
flow_cache_flush_deferred(net);
}
EXPORT_SYMBOL(xfrm_garbage_collect_deferred);
static void xfrm_init_pmtu(struct dst_entry *dst) static void xfrm_init_pmtu(struct dst_entry *dst)
{ {
do { do {
...@@ -2914,14 +2815,9 @@ static int __net_init xfrm_net_init(struct net *net) ...@@ -2914,14 +2815,9 @@ static int __net_init xfrm_net_init(struct net *net)
rv = xfrm_sysctl_init(net); rv = xfrm_sysctl_init(net);
if (rv < 0) if (rv < 0)
goto out_sysctl; goto out_sysctl;
rv = flow_cache_init(net);
if (rv < 0)
goto out;
return 0; return 0;
out:
xfrm_sysctl_fini(net);
out_sysctl: out_sysctl:
xfrm_policy_fini(net); xfrm_policy_fini(net);
out_policy: out_policy:
...@@ -2934,7 +2830,6 @@ static int __net_init xfrm_net_init(struct net *net) ...@@ -2934,7 +2830,6 @@ static int __net_init xfrm_net_init(struct net *net)
static void __net_exit xfrm_net_exit(struct net *net) static void __net_exit xfrm_net_exit(struct net *net)
{ {
flow_cache_fini(net);
xfrm_sysctl_fini(net); xfrm_sysctl_fini(net);
xfrm_policy_fini(net); xfrm_policy_fini(net);
xfrm_state_fini(net); xfrm_state_fini(net);
...@@ -2948,7 +2843,6 @@ static struct pernet_operations __net_initdata xfrm_net_ops = { ...@@ -2948,7 +2843,6 @@ static struct pernet_operations __net_initdata xfrm_net_ops = {
void __init xfrm_init(void) void __init xfrm_init(void)
{ {
flow_cache_hp_init();
register_pernet_subsys(&xfrm_net_ops); register_pernet_subsys(&xfrm_net_ops);
seqcount_init(&xfrm_policy_hash_generation); seqcount_init(&xfrm_policy_hash_generation);
xfrm_input_init(); xfrm_input_init();
......
...@@ -1815,8 +1815,6 @@ static int xfrm_get_policy(struct sk_buff *skb, struct nlmsghdr *nlh, ...@@ -1815,8 +1815,6 @@ static int xfrm_get_policy(struct sk_buff *skb, struct nlmsghdr *nlh,
out: out:
xfrm_pol_put(xp); xfrm_pol_put(xp);
if (delete && err == 0)
xfrm_garbage_collect(net);
return err; return err;
} }
...@@ -2027,7 +2025,6 @@ static int xfrm_flush_policy(struct sk_buff *skb, struct nlmsghdr *nlh, ...@@ -2027,7 +2025,6 @@ static int xfrm_flush_policy(struct sk_buff *skb, struct nlmsghdr *nlh,
return 0; return 0;
return err; return err;
} }
xfrm_garbage_collect(net);
c.data.type = type; c.data.type = type;
c.event = nlh->nlmsg_type; c.event = nlh->nlmsg_type;
......
...@@ -47,10 +47,8 @@ static inline void selinux_xfrm_notify_policyload(void) ...@@ -47,10 +47,8 @@ static inline void selinux_xfrm_notify_policyload(void)
struct net *net; struct net *net;
rtnl_lock(); rtnl_lock();
for_each_net(net) { for_each_net(net)
atomic_inc(&net->xfrm.flow_cache_genid);
rt_genid_bump_all(net); rt_genid_bump_all(net);
}
rtnl_unlock(); rtnl_unlock();
} }
#else #else
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment