Commit 6df46865 authored by Dave Hansen's avatar Dave Hansen Committed by Linus Torvalds

mm: vmstats: track TLB flush stats on UP too

The previous patch doing vmstats for TLB flushes ("mm: vmstats: tlb flush
counters") effectively missed UP since arch/x86/mm/tlb.c is only compiled
for SMP.

UP systems do not do remote TLB flushes, so compile those counters out on
UP.

arch/x86/kernel/cpu/mtrr/generic.c calls __flush_tlb() directly.  This is
probably an optimization since both the mtrr code and __flush_tlb() write
cr4.  It would probably be safe to make that a flush_tlb_all() (and then
get these statistics), but the mtrr code is ancient and I'm hesitant to
touch it other than to just stick in the counters.

[akpm@linux-foundation.org: tweak comments]
Signed-off-by: default avatarDave Hansen <dave.hansen@linux.intel.com>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Ingo Molnar <mingo@elte.hu>
Cc: "H. Peter Anvin" <hpa@zytor.com>
Cc: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent 9824cf97
...@@ -62,6 +62,7 @@ static inline void __flush_tlb_all(void) ...@@ -62,6 +62,7 @@ static inline void __flush_tlb_all(void)
static inline void __flush_tlb_one(unsigned long addr) static inline void __flush_tlb_one(unsigned long addr)
{ {
count_vm_event(NR_TLB_LOCAL_FLUSH_ONE);
__flush_tlb_single(addr); __flush_tlb_single(addr);
} }
...@@ -84,14 +85,38 @@ static inline void __flush_tlb_one(unsigned long addr) ...@@ -84,14 +85,38 @@ static inline void __flush_tlb_one(unsigned long addr)
#ifndef CONFIG_SMP #ifndef CONFIG_SMP
#define flush_tlb() __flush_tlb() /* "_up" is for UniProcessor.
#define flush_tlb_all() __flush_tlb_all() *
#define local_flush_tlb() __flush_tlb() * This is a helper for other header functions. *Not* intended to be called
* directly. All global TLB flushes need to either call this, or to bump the
* vm statistics themselves.
*/
static inline void __flush_tlb_up(void)
{
count_vm_event(NR_TLB_LOCAL_FLUSH_ALL);
__flush_tlb();
}
static inline void flush_tlb_all(void)
{
count_vm_event(NR_TLB_LOCAL_FLUSH_ALL);
__flush_tlb_all();
}
static inline void flush_tlb(void)
{
__flush_tlb_up();
}
static inline void local_flush_tlb(void)
{
__flush_tlb_up();
}
static inline void flush_tlb_mm(struct mm_struct *mm) static inline void flush_tlb_mm(struct mm_struct *mm)
{ {
if (mm == current->active_mm) if (mm == current->active_mm)
__flush_tlb(); __flush_tlb_up();
} }
static inline void flush_tlb_page(struct vm_area_struct *vma, static inline void flush_tlb_page(struct vm_area_struct *vma,
...@@ -105,14 +130,14 @@ static inline void flush_tlb_range(struct vm_area_struct *vma, ...@@ -105,14 +130,14 @@ static inline void flush_tlb_range(struct vm_area_struct *vma,
unsigned long start, unsigned long end) unsigned long start, unsigned long end)
{ {
if (vma->vm_mm == current->active_mm) if (vma->vm_mm == current->active_mm)
__flush_tlb(); __flush_tlb_up();
} }
static inline void flush_tlb_mm_range(struct mm_struct *mm, static inline void flush_tlb_mm_range(struct mm_struct *mm,
unsigned long start, unsigned long end, unsigned long vmflag) unsigned long start, unsigned long end, unsigned long vmflag)
{ {
if (mm == current->active_mm) if (mm == current->active_mm)
__flush_tlb(); __flush_tlb_up();
} }
static inline void native_flush_tlb_others(const struct cpumask *cpumask, static inline void native_flush_tlb_others(const struct cpumask *cpumask,
......
...@@ -683,6 +683,7 @@ static void prepare_set(void) __acquires(set_atomicity_lock) ...@@ -683,6 +683,7 @@ static void prepare_set(void) __acquires(set_atomicity_lock)
} }
/* Flush all TLBs via a mov %cr3, %reg; mov %reg, %cr3 */ /* Flush all TLBs via a mov %cr3, %reg; mov %reg, %cr3 */
count_vm_event(NR_TLB_LOCAL_FLUSH_ALL);
__flush_tlb(); __flush_tlb();
/* Save MTRR state */ /* Save MTRR state */
...@@ -696,6 +697,7 @@ static void prepare_set(void) __acquires(set_atomicity_lock) ...@@ -696,6 +697,7 @@ static void prepare_set(void) __acquires(set_atomicity_lock)
static void post_set(void) __releases(set_atomicity_lock) static void post_set(void) __releases(set_atomicity_lock)
{ {
/* Flush TLBs (no need to flush caches - they are disabled) */ /* Flush TLBs (no need to flush caches - they are disabled) */
count_vm_event(NR_TLB_LOCAL_FLUSH_ALL);
__flush_tlb(); __flush_tlb();
/* Intel (P6) standard MTRRs */ /* Intel (P6) standard MTRRs */
......
...@@ -280,10 +280,8 @@ static void do_kernel_range_flush(void *info) ...@@ -280,10 +280,8 @@ static void do_kernel_range_flush(void *info)
unsigned long addr; unsigned long addr;
/* flush range by one by one 'invlpg' */ /* flush range by one by one 'invlpg' */
for (addr = f->flush_start; addr < f->flush_end; addr += PAGE_SIZE) { for (addr = f->flush_start; addr < f->flush_end; addr += PAGE_SIZE)
count_vm_event(NR_TLB_LOCAL_FLUSH_ONE_KERNEL);
__flush_tlb_single(addr); __flush_tlb_single(addr);
}
} }
void flush_tlb_kernel_range(unsigned long start, unsigned long end) void flush_tlb_kernel_range(unsigned long start, unsigned long end)
......
...@@ -70,11 +70,12 @@ enum vm_event_item { PGPGIN, PGPGOUT, PSWPIN, PSWPOUT, ...@@ -70,11 +70,12 @@ enum vm_event_item { PGPGIN, PGPGOUT, PSWPIN, PSWPOUT,
THP_ZERO_PAGE_ALLOC, THP_ZERO_PAGE_ALLOC,
THP_ZERO_PAGE_ALLOC_FAILED, THP_ZERO_PAGE_ALLOC_FAILED,
#endif #endif
#ifdef CONFIG_SMP
NR_TLB_REMOTE_FLUSH, /* cpu tried to flush others' tlbs */ NR_TLB_REMOTE_FLUSH, /* cpu tried to flush others' tlbs */
NR_TLB_REMOTE_FLUSH_RECEIVED,/* cpu received ipi for flush */ NR_TLB_REMOTE_FLUSH_RECEIVED,/* cpu received ipi for flush */
#endif
NR_TLB_LOCAL_FLUSH_ALL, NR_TLB_LOCAL_FLUSH_ALL,
NR_TLB_LOCAL_FLUSH_ONE, NR_TLB_LOCAL_FLUSH_ONE,
NR_TLB_LOCAL_FLUSH_ONE_KERNEL,
NR_VM_EVENT_ITEMS NR_VM_EVENT_ITEMS
}; };
......
...@@ -817,11 +817,12 @@ const char * const vmstat_text[] = { ...@@ -817,11 +817,12 @@ const char * const vmstat_text[] = {
"thp_zero_page_alloc", "thp_zero_page_alloc",
"thp_zero_page_alloc_failed", "thp_zero_page_alloc_failed",
#endif #endif
#ifdef CONFIG_SMP
"nr_tlb_remote_flush", "nr_tlb_remote_flush",
"nr_tlb_remote_flush_received", "nr_tlb_remote_flush_received",
#endif
"nr_tlb_local_flush_all", "nr_tlb_local_flush_all",
"nr_tlb_local_flush_one", "nr_tlb_local_flush_one",
"nr_tlb_local_flush_one_kernel",
#endif /* CONFIG_VM_EVENTS_COUNTERS */ #endif /* CONFIG_VM_EVENTS_COUNTERS */
}; };
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment