Commit 4a8c7bb5 authored by Nathan Zimmer's avatar Nathan Zimmer Committed by Linus Torvalds

mm/mempolicy.c: convert the shared_policy lock to a rwlock

When running the SPECint_rate gcc on some very large boxes it was
noticed that the system was spending lots of time in
mpol_shared_policy_lookup().  The gamess benchmark can also show it and
is what I mostly used to chase down the issue since the setup for that I
found to be easier.

To be clear the binaries were on tmpfs because of disk I/O requirements.
We then used text replication to avoid icache misses and having all the
copies banging on the memory where the instruction code resides.  This
results in us hitting a bottleneck in mpol_shared_policy_lookup() since
lookup is serialised by the shared_policy lock.

I have only reproduced this on very large (3k+ cores) boxes.  The
problem starts showing up at just a few hundred ranks getting worse
until it threatens to livelock once it gets large enough.  For example
on the gamess benchmark at 128 ranks this area consumes only ~1% of
time, at 512 ranks it consumes nearly 13%, and at 2k ranks it is over
90%.

To alleviate the contention in this area I converted the spinlock to an
rwlock.  This allows a large number of lookups to happen simultaneously.
The results were quite good reducing this consumtion at max ranks to
around 2%.

[akpm@linux-foundation.org: tidy up code comments]
Signed-off-by: default avatarNathan Zimmer <nzimmer@sgi.com>
Acked-by: default avatarDavid Rientjes <rientjes@google.com>
Acked-by: default avatarVlastimil Babka <vbabka@suse.cz>
Cc: Nadia Yvette Chambers <nyc@holomorphy.com>
Cc: Naoya Horiguchi <n-horiguchi@ah.jp.nec.com>
Cc: Mel Gorman <mgorman@suse.de>
Cc: "Aneesh Kumar K.V" <aneesh.kumar@linux.vnet.ibm.com>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent 8f235d1a
...@@ -738,7 +738,7 @@ static struct inode *hugetlbfs_get_inode(struct super_block *sb, ...@@ -738,7 +738,7 @@ static struct inode *hugetlbfs_get_inode(struct super_block *sb,
/* /*
* The policy is initialized here even if we are creating a * The policy is initialized here even if we are creating a
* private inode because initialization simply creates an * private inode because initialization simply creates an
* an empty rb tree and calls spin_lock_init(), later when we * an empty rb tree and calls rwlock_init(), later when we
* call mpol_free_shared_policy() it will just return because * call mpol_free_shared_policy() it will just return because
* the rb tree will still be empty. * the rb tree will still be empty.
*/ */
......
...@@ -122,7 +122,7 @@ struct sp_node { ...@@ -122,7 +122,7 @@ struct sp_node {
struct shared_policy { struct shared_policy {
struct rb_root root; struct rb_root root;
spinlock_t lock; rwlock_t lock;
}; };
int vma_dup_policy(struct vm_area_struct *src, struct vm_area_struct *dst); int vma_dup_policy(struct vm_area_struct *src, struct vm_area_struct *dst);
......
...@@ -2142,12 +2142,14 @@ bool __mpol_equal(struct mempolicy *a, struct mempolicy *b) ...@@ -2142,12 +2142,14 @@ bool __mpol_equal(struct mempolicy *a, struct mempolicy *b)
* *
* Remember policies even when nobody has shared memory mapped. * Remember policies even when nobody has shared memory mapped.
* The policies are kept in Red-Black tree linked from the inode. * The policies are kept in Red-Black tree linked from the inode.
* They are protected by the sp->lock spinlock, which should be held * They are protected by the sp->lock rwlock, which should be held
* for any accesses to the tree. * for any accesses to the tree.
*/ */
/* lookup first element intersecting start-end */ /*
/* Caller holds sp->lock */ * lookup first element intersecting start-end. Caller holds sp->lock for
* reading or for writing
*/
static struct sp_node * static struct sp_node *
sp_lookup(struct shared_policy *sp, unsigned long start, unsigned long end) sp_lookup(struct shared_policy *sp, unsigned long start, unsigned long end)
{ {
...@@ -2178,8 +2180,10 @@ sp_lookup(struct shared_policy *sp, unsigned long start, unsigned long end) ...@@ -2178,8 +2180,10 @@ sp_lookup(struct shared_policy *sp, unsigned long start, unsigned long end)
return rb_entry(n, struct sp_node, nd); return rb_entry(n, struct sp_node, nd);
} }
/* Insert a new shared policy into the list. */ /*
/* Caller holds sp->lock */ * Insert a new shared policy into the list. Caller holds sp->lock for
* writing.
*/
static void sp_insert(struct shared_policy *sp, struct sp_node *new) static void sp_insert(struct shared_policy *sp, struct sp_node *new)
{ {
struct rb_node **p = &sp->root.rb_node; struct rb_node **p = &sp->root.rb_node;
...@@ -2211,13 +2215,13 @@ mpol_shared_policy_lookup(struct shared_policy *sp, unsigned long idx) ...@@ -2211,13 +2215,13 @@ mpol_shared_policy_lookup(struct shared_policy *sp, unsigned long idx)
if (!sp->root.rb_node) if (!sp->root.rb_node)
return NULL; return NULL;
spin_lock(&sp->lock); read_lock(&sp->lock);
sn = sp_lookup(sp, idx, idx+1); sn = sp_lookup(sp, idx, idx+1);
if (sn) { if (sn) {
mpol_get(sn->policy); mpol_get(sn->policy);
pol = sn->policy; pol = sn->policy;
} }
spin_unlock(&sp->lock); read_unlock(&sp->lock);
return pol; return pol;
} }
...@@ -2360,7 +2364,7 @@ static int shared_policy_replace(struct shared_policy *sp, unsigned long start, ...@@ -2360,7 +2364,7 @@ static int shared_policy_replace(struct shared_policy *sp, unsigned long start,
int ret = 0; int ret = 0;
restart: restart:
spin_lock(&sp->lock); write_lock(&sp->lock);
n = sp_lookup(sp, start, end); n = sp_lookup(sp, start, end);
/* Take care of old policies in the same range. */ /* Take care of old policies in the same range. */
while (n && n->start < end) { while (n && n->start < end) {
...@@ -2393,7 +2397,7 @@ static int shared_policy_replace(struct shared_policy *sp, unsigned long start, ...@@ -2393,7 +2397,7 @@ static int shared_policy_replace(struct shared_policy *sp, unsigned long start,
} }
if (new) if (new)
sp_insert(sp, new); sp_insert(sp, new);
spin_unlock(&sp->lock); write_unlock(&sp->lock);
ret = 0; ret = 0;
err_out: err_out:
...@@ -2405,7 +2409,7 @@ static int shared_policy_replace(struct shared_policy *sp, unsigned long start, ...@@ -2405,7 +2409,7 @@ static int shared_policy_replace(struct shared_policy *sp, unsigned long start,
return ret; return ret;
alloc_new: alloc_new:
spin_unlock(&sp->lock); write_unlock(&sp->lock);
ret = -ENOMEM; ret = -ENOMEM;
n_new = kmem_cache_alloc(sn_cache, GFP_KERNEL); n_new = kmem_cache_alloc(sn_cache, GFP_KERNEL);
if (!n_new) if (!n_new)
...@@ -2431,7 +2435,7 @@ void mpol_shared_policy_init(struct shared_policy *sp, struct mempolicy *mpol) ...@@ -2431,7 +2435,7 @@ void mpol_shared_policy_init(struct shared_policy *sp, struct mempolicy *mpol)
int ret; int ret;
sp->root = RB_ROOT; /* empty tree == default mempolicy */ sp->root = RB_ROOT; /* empty tree == default mempolicy */
spin_lock_init(&sp->lock); rwlock_init(&sp->lock);
if (mpol) { if (mpol) {
struct vm_area_struct pvma; struct vm_area_struct pvma;
...@@ -2497,14 +2501,14 @@ void mpol_free_shared_policy(struct shared_policy *p) ...@@ -2497,14 +2501,14 @@ void mpol_free_shared_policy(struct shared_policy *p)
if (!p->root.rb_node) if (!p->root.rb_node)
return; return;
spin_lock(&p->lock); write_lock(&p->lock);
next = rb_first(&p->root); next = rb_first(&p->root);
while (next) { while (next) {
n = rb_entry(next, struct sp_node, nd); n = rb_entry(next, struct sp_node, nd);
next = rb_next(&n->nd); next = rb_next(&n->nd);
sp_delete(p, n); sp_delete(p, n);
} }
spin_unlock(&p->lock); write_unlock(&p->lock);
} }
#ifdef CONFIG_NUMA_BALANCING #ifdef CONFIG_NUMA_BALANCING
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment