Commit 46453a6e authored by Nick Piggin's avatar Nick Piggin Committed by Linus Torvalds

[PATCH] mm: never ClearPageLRU released pages

If vmscan finds a zero refcount page on the lru list, never ClearPageLRU
it.  This means the release code need not hold ->lru_lock to stabilise
PageLRU, so that lock may be skipped entirely when releasing !PageLRU pages
(because we know PageLRU won't have been temporarily cleared by vmscan,
which was previously guaranteed by holding the lock to synchronise against
vmscan).
Signed-off-by: default avatarNick Piggin <npiggin@suse.de>
Signed-off-by: default avatarAndrew Morton <akpm@osdl.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@osdl.org>
parent 2492ecc1
...@@ -209,19 +209,18 @@ int lru_add_drain_all(void) ...@@ -209,19 +209,18 @@ int lru_add_drain_all(void)
*/ */
void fastcall __page_cache_release(struct page *page) void fastcall __page_cache_release(struct page *page)
{ {
unsigned long flags; if (PageLRU(page)) {
struct zone *zone = page_zone(page); unsigned long flags;
struct zone *zone = page_zone(page);
spin_lock_irqsave(&zone->lru_lock, flags); spin_lock_irqsave(&zone->lru_lock, flags);
if (TestClearPageLRU(page)) if (!TestClearPageLRU(page))
BUG();
del_page_from_lru(zone, page); del_page_from_lru(zone, page);
if (page_count(page) != 0) spin_unlock_irqrestore(&zone->lru_lock, flags);
page = NULL; }
spin_unlock_irqrestore(&zone->lru_lock, flags); free_hot_page(page);
if (page)
free_hot_page(page);
} }
EXPORT_SYMBOL(__page_cache_release); EXPORT_SYMBOL(__page_cache_release);
/* /*
...@@ -245,7 +244,6 @@ void release_pages(struct page **pages, int nr, int cold) ...@@ -245,7 +244,6 @@ void release_pages(struct page **pages, int nr, int cold)
pagevec_init(&pages_to_free, cold); pagevec_init(&pages_to_free, cold);
for (i = 0; i < nr; i++) { for (i = 0; i < nr; i++) {
struct page *page = pages[i]; struct page *page = pages[i];
struct zone *pagezone;
if (unlikely(PageCompound(page))) { if (unlikely(PageCompound(page))) {
if (zone) { if (zone) {
...@@ -259,23 +257,27 @@ void release_pages(struct page **pages, int nr, int cold) ...@@ -259,23 +257,27 @@ void release_pages(struct page **pages, int nr, int cold)
if (!put_page_testzero(page)) if (!put_page_testzero(page))
continue; continue;
pagezone = page_zone(page); if (PageLRU(page)) {
if (pagezone != zone) { struct zone *pagezone = page_zone(page);
if (zone) if (pagezone != zone) {
spin_unlock_irq(&zone->lru_lock); if (zone)
zone = pagezone; spin_unlock_irq(&zone->lru_lock);
spin_lock_irq(&zone->lru_lock); zone = pagezone;
} spin_lock_irq(&zone->lru_lock);
if (TestClearPageLRU(page)) }
if (!TestClearPageLRU(page))
BUG();
del_page_from_lru(zone, page); del_page_from_lru(zone, page);
if (page_count(page) == 0) { }
if (!pagevec_add(&pages_to_free, page)) {
if (!pagevec_add(&pages_to_free, page)) {
if (zone) {
spin_unlock_irq(&zone->lru_lock); spin_unlock_irq(&zone->lru_lock);
__pagevec_free(&pages_to_free); zone = NULL;
pagevec_reinit(&pages_to_free);
zone = NULL; /* No lock is held */
} }
} __pagevec_free(&pages_to_free);
pagevec_reinit(&pages_to_free);
}
} }
if (zone) if (zone)
spin_unlock_irq(&zone->lru_lock); spin_unlock_irq(&zone->lru_lock);
......
...@@ -1085,21 +1085,25 @@ static int isolate_lru_pages(int nr_to_scan, struct list_head *src, ...@@ -1085,21 +1085,25 @@ static int isolate_lru_pages(int nr_to_scan, struct list_head *src,
page = lru_to_page(src); page = lru_to_page(src);
prefetchw_prev_lru_page(page, src, flags); prefetchw_prev_lru_page(page, src, flags);
if (!TestClearPageLRU(page))
BUG();
list_del(&page->lru); list_del(&page->lru);
if (get_page_testone(page)) { if (unlikely(get_page_testone(page))) {
/* /*
* It is being freed elsewhere * It is being freed elsewhere
*/ */
__put_page(page); __put_page(page);
SetPageLRU(page);
list_add(&page->lru, src); list_add(&page->lru, src);
continue; continue;
} else {
list_add(&page->lru, dst);
nr_taken++;
} }
/*
* Be careful not to clear PageLRU until after we're sure
* the page is not being freed elsewhere -- the page release
* code relies on it.
*/
if (!TestClearPageLRU(page))
BUG();
list_add(&page->lru, dst);
nr_taken++;
} }
*scanned = scan; *scanned = scan;
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment