Commit d5a187da authored by Vlastimil Babka's avatar Vlastimil Babka Committed by Linus Torvalds

mm, rmap: handle anon_vma_prepare() common case inline

anon_vma_prepare() is mostly a large "if (unlikely(...))" block, as the
expected common case is that an anon_vma already exists.  We could turn
the condition around and return 0, but it also makes sense to do it
inline and avoid a call for the common case.

Bloat-o-meter naturally shows that inlining the check has some code size
costs:

add/remove: 1/1 grow/shrink: 4/0 up/down: 475/-373 (102)
function                                     old     new   delta
__anon_vma_prepare                             -     359    +359
handle_mm_fault                             2744    2796     +52
hugetlb_cow                                 1146    1170     +24
hugetlb_fault                               2123    2145     +22
wp_page_copy                                1469    1487     +18
anon_vma_prepare                             373       -    -373

Checking the asm however confirms that the hot paths now avoid a call,
which is moved away.

[akpm@linux-foundation.org: coding-style fixes]
Link: http://lkml.kernel.org/r/20161116074005.22768-1-vbabka@suse.czSigned-off-by: default avatarVlastimil Babka <vbabka@suse.cz>
Cc: "Kirill A. Shutemov" <kirill.shutemov@linux.intel.com>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: Konstantin Khlebnikov <koct9i@gmail.com>
Cc: Rik van Riel <riel@redhat.com>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent 46e8a3a0
...@@ -137,11 +137,19 @@ static inline void anon_vma_unlock_read(struct anon_vma *anon_vma) ...@@ -137,11 +137,19 @@ static inline void anon_vma_unlock_read(struct anon_vma *anon_vma)
* anon_vma helper functions. * anon_vma helper functions.
*/ */
void anon_vma_init(void); /* create anon_vma_cachep */ void anon_vma_init(void); /* create anon_vma_cachep */
int anon_vma_prepare(struct vm_area_struct *); int __anon_vma_prepare(struct vm_area_struct *);
void unlink_anon_vmas(struct vm_area_struct *); void unlink_anon_vmas(struct vm_area_struct *);
int anon_vma_clone(struct vm_area_struct *, struct vm_area_struct *); int anon_vma_clone(struct vm_area_struct *, struct vm_area_struct *);
int anon_vma_fork(struct vm_area_struct *, struct vm_area_struct *); int anon_vma_fork(struct vm_area_struct *, struct vm_area_struct *);
static inline int anon_vma_prepare(struct vm_area_struct *vma)
{
if (likely(vma->anon_vma))
return 0;
return __anon_vma_prepare(vma);
}
static inline void anon_vma_merge(struct vm_area_struct *vma, static inline void anon_vma_merge(struct vm_area_struct *vma,
struct vm_area_struct *next) struct vm_area_struct *next)
{ {
......
...@@ -141,14 +141,15 @@ static void anon_vma_chain_link(struct vm_area_struct *vma, ...@@ -141,14 +141,15 @@ static void anon_vma_chain_link(struct vm_area_struct *vma,
} }
/** /**
* anon_vma_prepare - attach an anon_vma to a memory region * __anon_vma_prepare - attach an anon_vma to a memory region
* @vma: the memory region in question * @vma: the memory region in question
* *
* This makes sure the memory mapping described by 'vma' has * This makes sure the memory mapping described by 'vma' has
* an 'anon_vma' attached to it, so that we can associate the * an 'anon_vma' attached to it, so that we can associate the
* anonymous pages mapped into it with that anon_vma. * anonymous pages mapped into it with that anon_vma.
* *
* The common case will be that we already have one, but if * The common case will be that we already have one, which
* is handled inline by anon_vma_prepare(). But if
* not we either need to find an adjacent mapping that we * not we either need to find an adjacent mapping that we
* can re-use the anon_vma from (very common when the only * can re-use the anon_vma from (very common when the only
* reason for splitting a vma has been mprotect()), or we * reason for splitting a vma has been mprotect()), or we
...@@ -167,48 +168,46 @@ static void anon_vma_chain_link(struct vm_area_struct *vma, ...@@ -167,48 +168,46 @@ static void anon_vma_chain_link(struct vm_area_struct *vma,
* *
* This must be called with the mmap_sem held for reading. * This must be called with the mmap_sem held for reading.
*/ */
int anon_vma_prepare(struct vm_area_struct *vma) int __anon_vma_prepare(struct vm_area_struct *vma)
{ {
struct anon_vma *anon_vma = vma->anon_vma; struct mm_struct *mm = vma->vm_mm;
struct anon_vma *anon_vma, *allocated;
struct anon_vma_chain *avc; struct anon_vma_chain *avc;
might_sleep(); might_sleep();
if (unlikely(!anon_vma)) {
struct mm_struct *mm = vma->vm_mm;
struct anon_vma *allocated;
avc = anon_vma_chain_alloc(GFP_KERNEL); avc = anon_vma_chain_alloc(GFP_KERNEL);
if (!avc) if (!avc)
goto out_enomem; goto out_enomem;
anon_vma = find_mergeable_anon_vma(vma);
allocated = NULL;
if (!anon_vma) {
anon_vma = anon_vma_alloc();
if (unlikely(!anon_vma))
goto out_enomem_free_avc;
allocated = anon_vma;
}
anon_vma = find_mergeable_anon_vma(vma); anon_vma_lock_write(anon_vma);
/* page_table_lock to protect against threads */
spin_lock(&mm->page_table_lock);
if (likely(!vma->anon_vma)) {
vma->anon_vma = anon_vma;
anon_vma_chain_link(vma, avc, anon_vma);
/* vma reference or self-parent link for new root */
anon_vma->degree++;
allocated = NULL; allocated = NULL;
if (!anon_vma) { avc = NULL;
anon_vma = anon_vma_alloc(); }
if (unlikely(!anon_vma)) spin_unlock(&mm->page_table_lock);
goto out_enomem_free_avc; anon_vma_unlock_write(anon_vma);
allocated = anon_vma;
}
anon_vma_lock_write(anon_vma); if (unlikely(allocated))
/* page_table_lock to protect against threads */ put_anon_vma(allocated);
spin_lock(&mm->page_table_lock); if (unlikely(avc))
if (likely(!vma->anon_vma)) { anon_vma_chain_free(avc);
vma->anon_vma = anon_vma;
anon_vma_chain_link(vma, avc, anon_vma);
/* vma reference or self-parent link for new root */
anon_vma->degree++;
allocated = NULL;
avc = NULL;
}
spin_unlock(&mm->page_table_lock);
anon_vma_unlock_write(anon_vma);
if (unlikely(allocated))
put_anon_vma(allocated);
if (unlikely(avc))
anon_vma_chain_free(avc);
}
return 0; return 0;
out_enomem_free_avc: out_enomem_free_avc:
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment