[PATCH] Inefficient TLB flush fix

From: Martin Hicks <mort@wildopensource.com>

This is a patch based on one that Jack Steiner sent to the ia64 list in
November.  The original thread can be found at:

http://marc.theaimsgroup.com/?l=linux-ia64&m=106869606922555&w=2

I created the little wrapper function that was requested.  I think the only
other arch, other than ia64, that doesn't at least include asm-generic/tlb.h
is arm.


Something appears broken in TLB flushing on IA64 (& possibly other
architectures).  Functionally, it works but performance is bad on systems
with large cpu counts.

The result is that TLB flushing in exit_mmap() is frequently being done via
IPIs to all cpus rather than with a "ptc" instruction or with a new
context..

include/asm-arm/tlb.h

@@ -70,6 +70,12 @@ tlb_finish_mmu(struct mmu_gather *tlb, unsigned long start, unsigned long end)
 	check_pgt_cache();
 }
 
+static inline unsigned int
+tlb_is_full_mm(struct mmu_gather *tlb)
+{
+     return tlb->fullmm;
+}
+
 #define tlb_remove_tlb_entry(tlb,ptep,address)	do { } while (0)
 
 #define tlb_start_vma(tlb,vma)						\

include/asm-generic/tlb.h

@@ -98,6 +98,11 @@ tlb_finish_mmu(struct mmu_gather *tlb, unsigned long start, unsigned long end)
 	check_pgt_cache();
 }
 
+static inline unsigned int
+tlb_is_full_mm(struct mmu_gather *tlb)
+{
+	return tlb->fullmm;
+}
 
 /* tlb_remove_page
  *	Must perform the equivalent to __free_pte(pte_get_and_clear(ptep)), while

include/asm-ia64/tlb.h

@@ -173,6 +173,12 @@ tlb_finish_mmu (struct mmu_gather *tlb, unsigned long start, unsigned long end)
 	check_pgt_cache();
 }
 
+static inline unsigned int
+tlb_is_full_mm(struct mmu_gather *tlb)
+{
+     return tlb->fullmm;
+}
+
 /*
  * Logically, this routine frees PAGE.  On MP machines, the actual freeing of the page
  * must be delayed until after the TLB has been flushed (see comments at the beginning of

mm/memory.c

@@ -574,9 +574,10 @@ int unmap_vmas(struct mmu_gather **tlbp, struct mm_struct *mm,
 			if ((long)zap_bytes > 0)
 				continue;
 			if (need_resched()) {
+				int fullmm = tlb_is_full_mm(*tlbp);
 				tlb_finish_mmu(*tlbp, tlb_start, start);
 				cond_resched_lock(&mm->page_table_lock);
-				*tlbp = tlb_gather_mmu(mm, 0);
+				*tlbp = tlb_gather_mmu(mm, fullmm);
 				tlb_start_valid = 0;
 			}
 			zap_bytes = ZAP_BLOCK_SIZE;