Commit c3bd8549 authored by Eric Dumazet's avatar Eric Dumazet Committed by David S. Miller

pkt_sched: fq: more robust memory allocation

This patch brings NUMA support and automatic fallback to vmalloc()
in case kmalloc() failed to allocate FQ hash table.

NUMA support depends on XPS being setup for the device before
qdisc allocation. After a XPS change, it might be worth creating
qdisc hierarchy again.
Signed-off-by: default avatarEric Dumazet <edumazet@google.com>
Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parent e7ef941d
...@@ -47,6 +47,7 @@ ...@@ -47,6 +47,7 @@
#include <linux/rbtree.h> #include <linux/rbtree.h>
#include <linux/hash.h> #include <linux/hash.h>
#include <linux/prefetch.h> #include <linux/prefetch.h>
#include <linux/vmalloc.h>
#include <net/netlink.h> #include <net/netlink.h>
#include <net/pkt_sched.h> #include <net/pkt_sched.h>
#include <net/sock.h> #include <net/sock.h>
...@@ -578,15 +579,36 @@ static void fq_rehash(struct fq_sched_data *q, ...@@ -578,15 +579,36 @@ static void fq_rehash(struct fq_sched_data *q,
q->stat_gc_flows += fcnt; q->stat_gc_flows += fcnt;
} }
static int fq_resize(struct fq_sched_data *q, u32 log) static void *fq_alloc_node(size_t sz, int node)
{ {
void *ptr;
ptr = kmalloc_node(sz, GFP_KERNEL | __GFP_REPEAT | __GFP_NOWARN, node);
if (!ptr)
ptr = vmalloc_node(sz, node);
return ptr;
}
static void fq_free(void *addr)
{
if (addr && is_vmalloc_addr(addr))
vfree(addr);
else
kfree(addr);
}
static int fq_resize(struct Qdisc *sch, u32 log)
{
struct fq_sched_data *q = qdisc_priv(sch);
struct rb_root *array; struct rb_root *array;
u32 idx; u32 idx;
if (q->fq_root && log == q->fq_trees_log) if (q->fq_root && log == q->fq_trees_log)
return 0; return 0;
array = kmalloc(sizeof(struct rb_root) << log, GFP_KERNEL); /* If XPS was setup, we can allocate memory on right NUMA node */
array = fq_alloc_node(sizeof(struct rb_root) << log,
netdev_queue_numa_node_read(sch->dev_queue));
if (!array) if (!array)
return -ENOMEM; return -ENOMEM;
...@@ -595,7 +617,7 @@ static int fq_resize(struct fq_sched_data *q, u32 log) ...@@ -595,7 +617,7 @@ static int fq_resize(struct fq_sched_data *q, u32 log)
if (q->fq_root) { if (q->fq_root) {
fq_rehash(q, q->fq_root, q->fq_trees_log, array, log); fq_rehash(q, q->fq_root, q->fq_trees_log, array, log);
kfree(q->fq_root); fq_free(q->fq_root);
} }
q->fq_root = array; q->fq_root = array;
q->fq_trees_log = log; q->fq_trees_log = log;
...@@ -676,7 +698,7 @@ static int fq_change(struct Qdisc *sch, struct nlattr *opt) ...@@ -676,7 +698,7 @@ static int fq_change(struct Qdisc *sch, struct nlattr *opt)
} }
if (!err) if (!err)
err = fq_resize(q, fq_log); err = fq_resize(sch, fq_log);
while (sch->q.qlen > sch->limit) { while (sch->q.qlen > sch->limit) {
struct sk_buff *skb = fq_dequeue(sch); struct sk_buff *skb = fq_dequeue(sch);
...@@ -697,7 +719,7 @@ static void fq_destroy(struct Qdisc *sch) ...@@ -697,7 +719,7 @@ static void fq_destroy(struct Qdisc *sch)
struct fq_sched_data *q = qdisc_priv(sch); struct fq_sched_data *q = qdisc_priv(sch);
fq_reset(sch); fq_reset(sch);
kfree(q->fq_root); fq_free(q->fq_root);
qdisc_watchdog_cancel(&q->watchdog); qdisc_watchdog_cancel(&q->watchdog);
} }
...@@ -723,7 +745,7 @@ static int fq_init(struct Qdisc *sch, struct nlattr *opt) ...@@ -723,7 +745,7 @@ static int fq_init(struct Qdisc *sch, struct nlattr *opt)
if (opt) if (opt)
err = fq_change(sch, opt); err = fq_change(sch, opt);
else else
err = fq_resize(q, q->fq_trees_log); err = fq_resize(sch, q->fq_trees_log);
return err; return err;
} }
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment