[PATCH] vm: per-zone vmscan instrumentation

To check on zone balancing, split the /proc/vmstat:pgsteal, pgreclaim pgalloc
and pgscan stats into per-zone counters.

Additionally, split the pgscan stats into pgscan_direct and pgscan_kswapd to
see who's doing how much scanning.

And add a metric for the number of slab objects which were scanned.

include/linux/mmzone.h

@@ -288,6 +288,11 @@ static inline int is_highmem(struct zone *zone)
 	return (zone - zone->zone_pgdat->node_zones == ZONE_HIGHMEM);
 }
 
+static inline int is_normal(struct zone *zone)
+{
+	return (zone - zone->zone_pgdat->node_zones == ZONE_NORMAL);
+}
+
 /* These two functions are used to setup the per zone pages min values */
 struct ctl_table;
 struct file;

include/linux/page-flags.h

@@ -98,23 +98,38 @@ struct page_state {
 	unsigned long pgpgout;		/* Disk writes */
 	unsigned long pswpin;		/* swap reads */
 	unsigned long pswpout;		/* swap writes */
-	unsigned long pgalloc;		/* page allocations */
+	unsigned long pgalloc_high;	/* page allocations */
 
+	unsigned long pgalloc_normal;
+	unsigned long pgalloc_dma;
 	unsigned long pgfree;		/* page freeings */
 	unsigned long pgactivate;	/* pages moved inactive->active */
 	unsigned long pgdeactivate;	/* pages moved active->inactive */
+
 	unsigned long pgfault;		/* faults (major+minor) */
 	unsigned long pgmajfault;	/* faults (major only) */
-
-	unsigned long pgscan;		/* pages scanned by page reclaim */
-	unsigned long pgrefill;		/* inspected in refill_inactive_zone */
-	unsigned long pgsteal;		/* total pages reclaimed */
+	unsigned long pgrefill_high;	/* inspected in refill_inactive_zone */
+	unsigned long pgrefill_normal;
+	unsigned long pgrefill_dma;
+
+	unsigned long pgsteal_high;	/* total highmem pages reclaimed */
+	unsigned long pgsteal_normal;
+	unsigned long pgsteal_dma;
+	unsigned long pgscan_kswapd_high;/* total highmem pages scanned */
+	unsigned long pgscan_kswapd_normal;
+
+	unsigned long pgscan_kswapd_dma;
+	unsigned long pgscan_direct_high;/* total highmem pages scanned */
+	unsigned long pgscan_direct_normal;
+	unsigned long pgscan_direct_dma;
 	unsigned long pginodesteal;	/* pages reclaimed via inode freeing */
-	unsigned long kswapd_steal;	/* pages reclaimed by kswapd */
 
+	unsigned long slabs_scanned;	/* slab objects scanned */
+	unsigned long kswapd_steal;	/* pages reclaimed by kswapd */
 	unsigned long kswapd_inodesteal;/* reclaimed via kswapd inode freeing */
 	unsigned long pageoutrun;	/* kswapd's calls to page reclaim */
 	unsigned long allocstall;	/* direct reclaim calls */
+
 	unsigned long pgrotated;	/* pages rotated to tail of the LRU */
 } ____cacheline_aligned;
 
@@ -131,11 +146,24 @@ extern void get_full_page_state(struct page_state *ret);
 		local_irq_restore(flags);				\
 	} while (0)
 
+
 #define inc_page_state(member)	mod_page_state(member, 1UL)
 #define dec_page_state(member)	mod_page_state(member, 0UL - 1)
 #define add_page_state(member,delta) mod_page_state(member, (delta))
 #define sub_page_state(member,delta) mod_page_state(member, 0UL - (delta))
 
+#define mod_page_state_zone(zone, member, delta)			\
+	do {								\
+		unsigned long flags;					\
+		local_irq_save(flags);					\
+		if (is_highmem(zone))					\
+			__get_cpu_var(page_states).member##_high += (delta);\
+		else if (is_normal(zone))				\
+			__get_cpu_var(page_states).member##_normal += (delta);\
+		else							\
+			__get_cpu_var(page_states).member##_dma += (delta);\
+		local_irq_restore(flags);				\
+	} while (0)
 
 /*
  * Manipulation of page state flags

mm/page_alloc.c

@@ -518,7 +518,7 @@ static struct page *buffered_rmqueue(struct zone *zone, int order, int cold)
 
 	if (page != NULL) {
 		BUG_ON(bad_range(zone, page));
-		mod_page_state(pgalloc, 1 << order);
+		mod_page_state_zone(zone, pgalloc, 1 << order);
 		prep_new_page(page, order);
 	}
 	return page;
@@ -1021,6 +1021,7 @@ void show_free_areas(void)
 			" high:%lukB"
 			" active:%lukB"
 			" inactive:%lukB"
+			" present:%lukB"
 			"\n",
 			zone->name,
 			K(zone->free_pages),
@@ -1028,7 +1029,8 @@ void show_free_areas(void)
 			K(zone->pages_low),
 			K(zone->pages_high),
 			K(zone->nr_active),
-			K(zone->nr_inactive)
+			K(zone->nr_inactive),
+			K(zone->present_pages)
 			);
 	}
 
@@ -1586,23 +1588,38 @@ static char *vmstat_text[] = {
 	"pgpgout",
 	"pswpin",
 	"pswpout",
-	"pgalloc",
+	"pgalloc_high",
 
+	"pgalloc_normal",
+	"pgalloc_dma",
 	"pgfree",
 	"pgactivate",
 	"pgdeactivate",
+
 	"pgfault",
 	"pgmajfault",
-
-	"pgscan",
-	"pgrefill",
-	"pgsteal",
+	"pgrefill_high",
+	"pgrefill_normal",
+	"pgrefill_dma",
+
+	"pgsteal_high",
+	"pgsteal_normal",
+	"pgsteal_dma",
+	"pgscan_kswapd_high",
+	"pgscan_kswapd_normal",
+
+	"pgscan_kswapd_dma",
+	"pgscan_direct_high",
+	"pgscan_direct_normal",
+	"pgscan_direct_dma",
 	"pginodesteal",
-	"kswapd_steal",
 
+	"slabs_scanned",
+	"kswapd_steal",
 	"kswapd_inodesteal",
 	"pageoutrun",
 	"allocstall",
+
 	"pgrotated",
 };
 

mm/vmscan.c

@@ -155,6 +155,7 @@ static int shrink_slab(long scanned, unsigned int gfp_mask)
 			long nr_to_scan = shrinker->nr;
 
 			shrinker->nr = 0;
+			mod_page_state(slabs_scanned, nr_to_scan);
 			while (nr_to_scan) {
 				long this_scan = nr_to_scan;
 
@@ -459,9 +460,6 @@ shrink_list(struct list_head *page_list, unsigned int gfp_mask,
 	list_splice(&ret_pages, page_list);
 	if (pagevec_count(&freed_pvec))
 		__pagevec_release_nonlru(&freed_pvec);
-	mod_page_state(pgsteal, ret);
-	if (current_is_kswapd())
-		mod_page_state(kswapd_steal, ret);
 	mod_page_state(pgactivate, pgactivate);
 	return ret;
 }
@@ -532,9 +530,16 @@ shrink_cache(const int nr_pages, struct zone *zone,
 			goto done;
 
 		max_scan -= nr_scan;
-		mod_page_state(pgscan, nr_scan);
+		if (current_is_kswapd())
+			mod_page_state_zone(zone, pgscan_kswapd, nr_scan);
+		else
+			mod_page_state_zone(zone, pgscan_direct, nr_scan);
 		nr_freed = shrink_list(&page_list, gfp_mask,
 					&max_scan, nr_mapped);
+		if (current_is_kswapd())
+			mod_page_state(kswapd_steal, nr_freed);
+		mod_page_state_zone(zone, pgsteal, nr_freed);
+
 		ret += nr_freed;
 		if (nr_freed <= 0 && list_empty(&page_list))
 			goto done;
@@ -733,7 +738,7 @@ refill_inactive_zone(struct zone *zone, const int nr_pages_in,
 	spin_unlock_irq(&zone->lru_lock);
 	pagevec_release(&pvec);
 
-	mod_page_state(pgrefill, nr_pages_in - nr_pages);
+	mod_page_state_zone(zone, pgrefill, nr_pages_in - nr_pages);
 	mod_page_state(pgdeactivate, pgdeactivate);
 }