Commit 960632ec authored by Aaron Conole's avatar Aaron Conole Committed by Pablo Neira Ayuso

netfilter: convert hook list to an array

This converts the storage and layout of netfilter hook entries from a
linked list to an array.  After this commit, hook entries will be
stored adjacent in memory.  The next pointer is no longer required.

The ops pointers are stored at the end of the array as they are only
used in the register/unregister path and in the legacy br_netfilter code.

nf_unregister_net_hooks() is slower than needed as it just calls
nf_unregister_net_hook in a loop (i.e. at least n synchronize_net()
calls), this will be addressed in followup patch.

Test setup:
 - ixgbe 10gbit
 - netperf UDP_STREAM, 64 byte packets
 - 5 hooks: (raw + mangle prerouting, mangle+filter input, inet filter):
empty mangle and raw prerouting, mangle and filter input hooks:
353.9
this patch:
364.2
Signed-off-by: default avatarAaron Conole <aconole@bytheb.org>
Signed-off-by: default avatarFlorian Westphal <fw@strlen.de>
Signed-off-by: default avatarPablo Neira Ayuso <pablo@netfilter.org>
parent 5fd02ebe
...@@ -1811,7 +1811,7 @@ struct net_device { ...@@ -1811,7 +1811,7 @@ struct net_device {
#endif #endif
struct netdev_queue __rcu *ingress_queue; struct netdev_queue __rcu *ingress_queue;
#ifdef CONFIG_NETFILTER_INGRESS #ifdef CONFIG_NETFILTER_INGRESS
struct nf_hook_entry __rcu *nf_hooks_ingress; struct nf_hook_entries __rcu *nf_hooks_ingress;
#endif #endif
unsigned char broadcast[MAX_ADDR_LEN]; unsigned char broadcast[MAX_ADDR_LEN];
......
...@@ -72,25 +72,32 @@ struct nf_hook_ops { ...@@ -72,25 +72,32 @@ struct nf_hook_ops {
}; };
struct nf_hook_entry { struct nf_hook_entry {
struct nf_hook_entry __rcu *next;
nf_hookfn *hook; nf_hookfn *hook;
void *priv; void *priv;
const struct nf_hook_ops *orig_ops;
}; };
static inline void struct nf_hook_entries {
nf_hook_entry_init(struct nf_hook_entry *entry, const struct nf_hook_ops *ops) u16 num_hook_entries;
{ /* padding */
entry->next = NULL; struct nf_hook_entry hooks[];
entry->hook = ops->hook;
entry->priv = ops->priv;
entry->orig_ops = ops;
}
static inline int /* trailer: pointers to original orig_ops of each hook.
nf_hook_entry_priority(const struct nf_hook_entry *entry) *
* This is not part of struct nf_hook_entry since its only
* needed in slow path (hook register/unregister).
*
* const struct nf_hook_ops *orig_ops[]
*/
};
static inline struct nf_hook_ops **nf_hook_entries_get_hook_ops(const struct nf_hook_entries *e)
{ {
return entry->orig_ops->priority; unsigned int n = e->num_hook_entries;
const void *hook_end;
hook_end = &e->hooks[n]; /* this is *past* ->hooks[]! */
return (struct nf_hook_ops **)hook_end;
} }
static inline int static inline int
...@@ -100,12 +107,6 @@ nf_hook_entry_hookfn(const struct nf_hook_entry *entry, struct sk_buff *skb, ...@@ -100,12 +107,6 @@ nf_hook_entry_hookfn(const struct nf_hook_entry *entry, struct sk_buff *skb,
return entry->hook(entry->priv, skb, state); return entry->hook(entry->priv, skb, state);
} }
static inline const struct nf_hook_ops *
nf_hook_entry_ops(const struct nf_hook_entry *entry)
{
return entry->orig_ops;
}
static inline void nf_hook_state_init(struct nf_hook_state *p, static inline void nf_hook_state_init(struct nf_hook_state *p,
unsigned int hook, unsigned int hook,
u_int8_t pf, u_int8_t pf,
...@@ -168,7 +169,7 @@ extern struct static_key nf_hooks_needed[NFPROTO_NUMPROTO][NF_MAX_HOOKS]; ...@@ -168,7 +169,7 @@ extern struct static_key nf_hooks_needed[NFPROTO_NUMPROTO][NF_MAX_HOOKS];
#endif #endif
int nf_hook_slow(struct sk_buff *skb, struct nf_hook_state *state, int nf_hook_slow(struct sk_buff *skb, struct nf_hook_state *state,
struct nf_hook_entry *entry); const struct nf_hook_entries *e, unsigned int i);
/** /**
* nf_hook - call a netfilter hook * nf_hook - call a netfilter hook
...@@ -182,7 +183,7 @@ static inline int nf_hook(u_int8_t pf, unsigned int hook, struct net *net, ...@@ -182,7 +183,7 @@ static inline int nf_hook(u_int8_t pf, unsigned int hook, struct net *net,
struct net_device *indev, struct net_device *outdev, struct net_device *indev, struct net_device *outdev,
int (*okfn)(struct net *, struct sock *, struct sk_buff *)) int (*okfn)(struct net *, struct sock *, struct sk_buff *))
{ {
struct nf_hook_entry *hook_head; struct nf_hook_entries *hook_head;
int ret = 1; int ret = 1;
#ifdef HAVE_JUMP_LABEL #ifdef HAVE_JUMP_LABEL
...@@ -200,7 +201,7 @@ static inline int nf_hook(u_int8_t pf, unsigned int hook, struct net *net, ...@@ -200,7 +201,7 @@ static inline int nf_hook(u_int8_t pf, unsigned int hook, struct net *net,
nf_hook_state_init(&state, hook, pf, indev, outdev, nf_hook_state_init(&state, hook, pf, indev, outdev,
sk, net, okfn); sk, net, okfn);
ret = nf_hook_slow(skb, &state, hook_head); ret = nf_hook_slow(skb, &state, hook_head, 0);
} }
rcu_read_unlock(); rcu_read_unlock();
......
...@@ -17,7 +17,7 @@ static inline bool nf_hook_ingress_active(const struct sk_buff *skb) ...@@ -17,7 +17,7 @@ static inline bool nf_hook_ingress_active(const struct sk_buff *skb)
/* caller must hold rcu_read_lock */ /* caller must hold rcu_read_lock */
static inline int nf_hook_ingress(struct sk_buff *skb) static inline int nf_hook_ingress(struct sk_buff *skb)
{ {
struct nf_hook_entry *e = rcu_dereference(skb->dev->nf_hooks_ingress); struct nf_hook_entries *e = rcu_dereference(skb->dev->nf_hooks_ingress);
struct nf_hook_state state; struct nf_hook_state state;
int ret; int ret;
...@@ -30,7 +30,7 @@ static inline int nf_hook_ingress(struct sk_buff *skb) ...@@ -30,7 +30,7 @@ static inline int nf_hook_ingress(struct sk_buff *skb)
nf_hook_state_init(&state, NF_NETDEV_INGRESS, nf_hook_state_init(&state, NF_NETDEV_INGRESS,
NFPROTO_NETDEV, skb->dev, NULL, NULL, NFPROTO_NETDEV, skb->dev, NULL, NULL,
dev_net(skb->dev), NULL); dev_net(skb->dev), NULL);
ret = nf_hook_slow(skb, &state, e); ret = nf_hook_slow(skb, &state, e, 0);
if (ret == 0) if (ret == 0)
return -1; return -1;
......
...@@ -10,9 +10,9 @@ struct nf_queue_entry { ...@@ -10,9 +10,9 @@ struct nf_queue_entry {
struct list_head list; struct list_head list;
struct sk_buff *skb; struct sk_buff *skb;
unsigned int id; unsigned int id;
unsigned int hook_index; /* index in hook_entries->hook[] */
struct nf_hook_state state; struct nf_hook_state state;
struct nf_hook_entry *hook;
u16 size; /* sizeof(entry) + saved route keys */ u16 size; /* sizeof(entry) + saved route keys */
/* extra space to store route keys */ /* extra space to store route keys */
......
...@@ -16,7 +16,7 @@ struct netns_nf { ...@@ -16,7 +16,7 @@ struct netns_nf {
#ifdef CONFIG_SYSCTL #ifdef CONFIG_SYSCTL
struct ctl_table_header *nf_log_dir_header; struct ctl_table_header *nf_log_dir_header;
#endif #endif
struct nf_hook_entry __rcu *hooks[NFPROTO_NUMPROTO][NF_MAX_HOOKS]; struct nf_hook_entries __rcu *hooks[NFPROTO_NUMPROTO][NF_MAX_HOOKS];
#if IS_ENABLED(CONFIG_NF_DEFRAG_IPV4) #if IS_ENABLED(CONFIG_NF_DEFRAG_IPV4)
bool defrag_ipv4; bool defrag_ipv4;
#endif #endif
......
...@@ -985,22 +985,25 @@ int br_nf_hook_thresh(unsigned int hook, struct net *net, ...@@ -985,22 +985,25 @@ int br_nf_hook_thresh(unsigned int hook, struct net *net,
int (*okfn)(struct net *, struct sock *, int (*okfn)(struct net *, struct sock *,
struct sk_buff *)) struct sk_buff *))
{ {
struct nf_hook_entry *elem; const struct nf_hook_entries *e;
struct nf_hook_state state; struct nf_hook_state state;
struct nf_hook_ops **ops;
unsigned int i;
int ret; int ret;
for (elem = rcu_dereference(net->nf.hooks[NFPROTO_BRIDGE][hook]); e = rcu_dereference(net->nf.hooks[NFPROTO_BRIDGE][hook]);
elem && nf_hook_entry_priority(elem) <= NF_BR_PRI_BRNF; if (!e)
elem = rcu_dereference(elem->next))
;
if (!elem)
return okfn(net, sk, skb); return okfn(net, sk, skb);
ops = nf_hook_entries_get_hook_ops(e);
for (i = 0; i < e->num_hook_entries &&
ops[i]->priority <= NF_BR_PRI_BRNF; i++)
;
nf_hook_state_init(&state, hook, NFPROTO_BRIDGE, indev, outdev, nf_hook_state_init(&state, hook, NFPROTO_BRIDGE, indev, outdev,
sk, net, okfn); sk, net, okfn);
ret = nf_hook_slow(skb, &state, elem); ret = nf_hook_slow(skb, &state, e, i);
if (ret == 1) if (ret == 1)
ret = okfn(net, sk, skb); ret = okfn(net, sk, skb);
......
This diff is collapsed.
...@@ -13,7 +13,8 @@ ...@@ -13,7 +13,8 @@
/* nf_queue.c */ /* nf_queue.c */
int nf_queue(struct sk_buff *skb, struct nf_hook_state *state, int nf_queue(struct sk_buff *skb, struct nf_hook_state *state,
struct nf_hook_entry **entryp, unsigned int verdict); const struct nf_hook_entries *entries, unsigned int index,
unsigned int verdict);
unsigned int nf_queue_nf_hook_drop(struct net *net); unsigned int nf_queue_nf_hook_drop(struct net *net);
/* nf_log.c */ /* nf_log.c */
......
...@@ -112,7 +112,8 @@ unsigned int nf_queue_nf_hook_drop(struct net *net) ...@@ -112,7 +112,8 @@ unsigned int nf_queue_nf_hook_drop(struct net *net)
EXPORT_SYMBOL_GPL(nf_queue_nf_hook_drop); EXPORT_SYMBOL_GPL(nf_queue_nf_hook_drop);
static int __nf_queue(struct sk_buff *skb, const struct nf_hook_state *state, static int __nf_queue(struct sk_buff *skb, const struct nf_hook_state *state,
struct nf_hook_entry *hook_entry, unsigned int queuenum) const struct nf_hook_entries *entries,
unsigned int index, unsigned int queuenum)
{ {
int status = -ENOENT; int status = -ENOENT;
struct nf_queue_entry *entry = NULL; struct nf_queue_entry *entry = NULL;
...@@ -140,7 +141,7 @@ static int __nf_queue(struct sk_buff *skb, const struct nf_hook_state *state, ...@@ -140,7 +141,7 @@ static int __nf_queue(struct sk_buff *skb, const struct nf_hook_state *state,
*entry = (struct nf_queue_entry) { *entry = (struct nf_queue_entry) {
.skb = skb, .skb = skb,
.state = *state, .state = *state,
.hook = hook_entry, .hook_index = index,
.size = sizeof(*entry) + afinfo->route_key_size, .size = sizeof(*entry) + afinfo->route_key_size,
}; };
...@@ -163,18 +164,16 @@ static int __nf_queue(struct sk_buff *skb, const struct nf_hook_state *state, ...@@ -163,18 +164,16 @@ static int __nf_queue(struct sk_buff *skb, const struct nf_hook_state *state,
/* Packets leaving via this function must come back through nf_reinject(). */ /* Packets leaving via this function must come back through nf_reinject(). */
int nf_queue(struct sk_buff *skb, struct nf_hook_state *state, int nf_queue(struct sk_buff *skb, struct nf_hook_state *state,
struct nf_hook_entry **entryp, unsigned int verdict) const struct nf_hook_entries *entries, unsigned int index,
unsigned int verdict)
{ {
struct nf_hook_entry *entry = *entryp;
int ret; int ret;
ret = __nf_queue(skb, state, entry, verdict >> NF_VERDICT_QBITS); ret = __nf_queue(skb, state, entries, index, verdict >> NF_VERDICT_QBITS);
if (ret < 0) { if (ret < 0) {
if (ret == -ESRCH && if (ret == -ESRCH &&
(verdict & NF_VERDICT_FLAG_QUEUE_BYPASS)) { (verdict & NF_VERDICT_FLAG_QUEUE_BYPASS))
*entryp = rcu_dereference(entry->next);
return 1; return 1;
}
kfree_skb(skb); kfree_skb(skb);
} }
...@@ -183,33 +182,56 @@ int nf_queue(struct sk_buff *skb, struct nf_hook_state *state, ...@@ -183,33 +182,56 @@ int nf_queue(struct sk_buff *skb, struct nf_hook_state *state,
static unsigned int nf_iterate(struct sk_buff *skb, static unsigned int nf_iterate(struct sk_buff *skb,
struct nf_hook_state *state, struct nf_hook_state *state,
struct nf_hook_entry **entryp) const struct nf_hook_entries *hooks,
unsigned int *index)
{ {
unsigned int verdict; const struct nf_hook_entry *hook;
unsigned int verdict, i = *index;
do { while (i < hooks->num_hook_entries) {
hook = &hooks->hooks[i];
repeat: repeat:
verdict = nf_hook_entry_hookfn((*entryp), skb, state); verdict = nf_hook_entry_hookfn(hook, skb, state);
if (verdict != NF_ACCEPT) { if (verdict != NF_ACCEPT) {
if (verdict != NF_REPEAT) if (verdict != NF_REPEAT)
return verdict; return verdict;
goto repeat; goto repeat;
} }
*entryp = rcu_dereference((*entryp)->next); i++;
} while (*entryp); }
*index = i;
return NF_ACCEPT; return NF_ACCEPT;
} }
/* Caller must hold rcu read-side lock */
void nf_reinject(struct nf_queue_entry *entry, unsigned int verdict) void nf_reinject(struct nf_queue_entry *entry, unsigned int verdict)
{ {
struct nf_hook_entry *hook_entry = entry->hook; const struct nf_hook_entry *hook_entry;
const struct nf_hook_entries *hooks;
struct sk_buff *skb = entry->skb; struct sk_buff *skb = entry->skb;
const struct nf_afinfo *afinfo; const struct nf_afinfo *afinfo;
const struct net *net;
unsigned int i;
int err; int err;
u8 pf;
net = entry->state.net;
pf = entry->state.pf;
hooks = rcu_dereference(net->nf.hooks[pf][entry->state.hook]);
nf_queue_entry_release_refs(entry); nf_queue_entry_release_refs(entry);
i = entry->hook_index;
if (WARN_ON_ONCE(i >= hooks->num_hook_entries)) {
kfree_skb(skb);
kfree(entry);
return;
}
hook_entry = &hooks->hooks[i];
/* Continue traversal iff userspace said ok... */ /* Continue traversal iff userspace said ok... */
if (verdict == NF_REPEAT) if (verdict == NF_REPEAT)
verdict = nf_hook_entry_hookfn(hook_entry, skb, &entry->state); verdict = nf_hook_entry_hookfn(hook_entry, skb, &entry->state);
...@@ -221,27 +243,22 @@ void nf_reinject(struct nf_queue_entry *entry, unsigned int verdict) ...@@ -221,27 +243,22 @@ void nf_reinject(struct nf_queue_entry *entry, unsigned int verdict)
} }
if (verdict == NF_ACCEPT) { if (verdict == NF_ACCEPT) {
hook_entry = rcu_dereference(hook_entry->next);
if (hook_entry)
next_hook: next_hook:
verdict = nf_iterate(skb, &entry->state, &hook_entry); ++i;
verdict = nf_iterate(skb, &entry->state, hooks, &i);
} }
switch (verdict & NF_VERDICT_MASK) { switch (verdict & NF_VERDICT_MASK) {
case NF_ACCEPT: case NF_ACCEPT:
case NF_STOP: case NF_STOP:
okfn:
local_bh_disable(); local_bh_disable();
entry->state.okfn(entry->state.net, entry->state.sk, skb); entry->state.okfn(entry->state.net, entry->state.sk, skb);
local_bh_enable(); local_bh_enable();
break; break;
case NF_QUEUE: case NF_QUEUE:
err = nf_queue(skb, &entry->state, &hook_entry, verdict); err = nf_queue(skb, &entry->state, hooks, i, verdict);
if (err == 1) { if (err == 1)
if (hook_entry)
goto next_hook; goto next_hook;
goto okfn;
}
break; break;
case NF_STOLEN: case NF_STOLEN:
break; break;
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment