[PATCH] Speed freeing memory for suspend.

Here's a patch I've prepared which improves the speed at which memory is freed
prior to suspend.  It should be a big gain for swsusp.  For suspend2, it isn't
used much, but has shown big improvements when I set a very low image size
limit and had memory quite full.

1GB P4, 2.6.11+Suspend2 2.1.8.
Soft image size limit set to 2MB to emulate Pavel's implementation (eat
as much memory as we can).

Without patch:
Freed  16545 pages in  4000 jiffies = 16.16 MB/s
Freed  83281 pages in 14060 jiffies = 23.14 MB/s
Freed 237754 pages in 41482 jiffies = 22.39 MB/s

With patch:
Freed  52257 pages in  6700 jiffies = 30.46 MB/s
Freed 105693 pages in 11035 jiffies = 37.41 MB/s
Freed 239007 pages in 18284 jiffies = 51.06 MB/s

With a less aggressive image size limit (200MB):

Without the patch:
Freed  14600 pages in  1749 jiffies = 32.61 MB/s (Anomolous!)
Freed  88563 pages in 14719 jiffies = 23.50 MB/s
Freed 205734 pages in 32389 jiffies = 24.81 MB/s

With the patch:
Freed  68252 pages in   496 jiffies = 537.52 MB/s
Freed 116464 pages in   569 jiffies = 798.54 MB/s
Freed 209699 pages in   705 jiffies = 1161.89 MB/s

The later pages take more work to get, which accounts for the slower MB/s with
smaller numbers of pages to free.  Without the patch, though, getting the
easier pages also takes longer because we do a far greater number of
invocations of shrink_all_memory in order to get the same number of pages.
Signed-Off-By: Nigel Cunningham <ncunningham@cyclades.com>
Acked-By: Pavel Machek <pavel@ucw.cz>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>

mm/vmscan.c

@@ -72,6 +72,12 @@ struct scan_control {
 	unsigned int gfp_mask;
 
 	int may_writepage;
+
+	/* This context's SWAP_CLUSTER_MAX. If freeing memory for
+	 * suspend, we effectively ignore SWAP_CLUSTER_MAX.
+	 * In this context, it doesn't matter that we scan the
+	 * whole list at once. */
+	int swap_cluster_max;
 };
 
 /*
@@ -552,7 +558,7 @@ static void shrink_cache(struct zone *zone, struct scan_control *sc)
 		int nr_scan = 0;
 		int nr_freed;
 
-		while (nr_scan++ < SWAP_CLUSTER_MAX &&
+		while (nr_scan++ < sc->swap_cluster_max &&
 				!list_empty(&zone->inactive_list)) {
 			page = lru_to_page(&zone->inactive_list);
 
@@ -797,31 +803,31 @@ shrink_zone(struct zone *zone, struct scan_control *sc)
 	 */
 	zone->nr_scan_active += (zone->nr_active >> sc->priority) + 1;
 	nr_active = zone->nr_scan_active;
-	if (nr_active >= SWAP_CLUSTER_MAX)
+	if (nr_active >= sc->swap_cluster_max)
 		zone->nr_scan_active = 0;
 	else
 		nr_active = 0;
 
 	zone->nr_scan_inactive += (zone->nr_inactive >> sc->priority) + 1;
 	nr_inactive = zone->nr_scan_inactive;
-	if (nr_inactive >= SWAP_CLUSTER_MAX)
+	if (nr_inactive >= sc->swap_cluster_max)
 		zone->nr_scan_inactive = 0;
 	else
 		nr_inactive = 0;
 
-	sc->nr_to_reclaim = SWAP_CLUSTER_MAX;
+	sc->nr_to_reclaim = sc->swap_cluster_max;
 
 	while (nr_active || nr_inactive) {
 		if (nr_active) {
 			sc->nr_to_scan = min(nr_active,
-					(unsigned long)SWAP_CLUSTER_MAX);
+					(unsigned long)sc->swap_cluster_max);
 			nr_active -= sc->nr_to_scan;
 			refill_inactive_zone(zone, sc);
 		}
 
 		if (nr_inactive) {
 			sc->nr_to_scan = min(nr_inactive,
-					(unsigned long)SWAP_CLUSTER_MAX);
+					(unsigned long)sc->swap_cluster_max);
 			nr_inactive -= sc->nr_to_scan;
 			shrink_cache(zone, sc);
 			if (sc->nr_to_reclaim <= 0)
@@ -911,6 +917,7 @@ int try_to_free_pages(struct zone **zones,
 		sc.nr_scanned = 0;
 		sc.nr_reclaimed = 0;
 		sc.priority = priority;
+		sc.swap_cluster_max = SWAP_CLUSTER_MAX;
 		shrink_caches(zones, &sc);
 		shrink_slab(sc.nr_scanned, gfp_mask, lru_pages);
 		if (reclaim_state) {
@@ -919,7 +926,7 @@ int try_to_free_pages(struct zone **zones,
 		}
 		total_scanned += sc.nr_scanned;
 		total_reclaimed += sc.nr_reclaimed;
-		if (total_reclaimed >= SWAP_CLUSTER_MAX) {
+		if (total_reclaimed >= sc.swap_cluster_max) {
 			ret = 1;
 			goto out;
 		}
@@ -931,7 +938,7 @@ int try_to_free_pages(struct zone **zones,
 		 * that's undesirable in laptop mode, where we *want* lumpy
 		 * writeout.  So in laptop mode, write out the whole world.
 		 */
-		if (total_scanned > SWAP_CLUSTER_MAX + SWAP_CLUSTER_MAX/2) {
+		if (total_scanned > sc.swap_cluster_max + sc.swap_cluster_max/2) {
 			wakeup_bdflush(laptop_mode ? 0 : total_scanned);
 			sc.may_writepage = 1;
 		}
@@ -1063,6 +1070,7 @@ static int balance_pgdat(pg_data_t *pgdat, int nr_pages, int order)
 			sc.nr_scanned = 0;
 			sc.nr_reclaimed = 0;
 			sc.priority = priority;
+			sc.swap_cluster_max = nr_pages? nr_pages : SWAP_CLUSTER_MAX;
 			shrink_zone(zone, &sc);
 			reclaim_state->reclaimed_slab = 0;
 			shrink_slab(sc.nr_scanned, GFP_KERNEL, lru_pages);
@@ -1100,7 +1108,7 @@ static int balance_pgdat(pg_data_t *pgdat, int nr_pages, int order)
 		 * matches the direct reclaim path behaviour in terms of impact
 		 * on zone->*_priority.
 		 */
-		if (total_reclaimed >= SWAP_CLUSTER_MAX)
+		if ((total_reclaimed >= SWAP_CLUSTER_MAX) && (!nr_pages))
 			break;
 	}
 out: