Commit f1e61557 authored by Kirill A. Shutemov's avatar Kirill A. Shutemov Committed by Linus Torvalds

mm: pack compound_dtor and compound_order into one word in struct page

The patch halves space occupied by compound_dtor and compound_order in
struct page.

For compound_order, it's trivial long -> short conversion.

For get_compound_page_dtor(), we now use hardcoded table for destructor
lookup and store its index in the struct page instead of direct pointer
to destructor. It shouldn't be a big trouble to maintain the table: we
have only two destructor and NULL currently.

This patch free up one word in tail pages for reuse. This is preparation
for the next patch.
Signed-off-by: default avatarKirill A. Shutemov <kirill.shutemov@linux.intel.com>
Reviewed-by: default avatarMichal Hocko <mhocko@suse.com>
Acked-by: default avatarVlastimil Babka <vbabka@suse.cz>
Reviewed-by: default avatarAndrea Arcangeli <aarcange@redhat.com>
Cc: "Paul E. McKenney" <paulmck@linux.vnet.ibm.com>
Cc: Andi Kleen <ak@linux.intel.com>
Cc: Aneesh Kumar K.V <aneesh.kumar@linux.vnet.ibm.com>
Cc: Christoph Lameter <cl@linux.com>
Cc: David Rientjes <rientjes@google.com>
Cc: Hugh Dickins <hughd@google.com>
Cc: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Cc: Sergey Senozhatsky <sergey.senozhatsky@gmail.com>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent 32e7ba1e
...@@ -568,18 +568,32 @@ int split_free_page(struct page *page); ...@@ -568,18 +568,32 @@ int split_free_page(struct page *page);
/* /*
* Compound pages have a destructor function. Provide a * Compound pages have a destructor function. Provide a
* prototype for that function and accessor functions. * prototype for that function and accessor functions.
* These are _only_ valid on the head of a PG_compound page. * These are _only_ valid on the head of a compound page.
*/ */
typedef void compound_page_dtor(struct page *);
/* Keep the enum in sync with compound_page_dtors array in mm/page_alloc.c */
enum compound_dtor_id {
NULL_COMPOUND_DTOR,
COMPOUND_PAGE_DTOR,
#ifdef CONFIG_HUGETLB_PAGE
HUGETLB_PAGE_DTOR,
#endif
NR_COMPOUND_DTORS,
};
extern compound_page_dtor * const compound_page_dtors[];
static inline void set_compound_page_dtor(struct page *page, static inline void set_compound_page_dtor(struct page *page,
compound_page_dtor *dtor) enum compound_dtor_id compound_dtor)
{ {
page[1].compound_dtor = dtor; VM_BUG_ON_PAGE(compound_dtor >= NR_COMPOUND_DTORS, page);
page[1].compound_dtor = compound_dtor;
} }
static inline compound_page_dtor *get_compound_page_dtor(struct page *page) static inline compound_page_dtor *get_compound_page_dtor(struct page *page)
{ {
return page[1].compound_dtor; VM_BUG_ON_PAGE(page[1].compound_dtor >= NR_COMPOUND_DTORS, page);
return compound_page_dtors[page[1].compound_dtor];
} }
static inline int compound_order(struct page *page) static inline int compound_order(struct page *page)
...@@ -589,7 +603,7 @@ static inline int compound_order(struct page *page) ...@@ -589,7 +603,7 @@ static inline int compound_order(struct page *page)
return page[1].compound_order; return page[1].compound_order;
} }
static inline void set_compound_order(struct page *page, unsigned long order) static inline void set_compound_order(struct page *page, unsigned int order)
{ {
page[1].compound_order = order; page[1].compound_order = order;
} }
......
...@@ -28,8 +28,6 @@ struct mem_cgroup; ...@@ -28,8 +28,6 @@ struct mem_cgroup;
IS_ENABLED(CONFIG_ARCH_ENABLE_SPLIT_PMD_PTLOCK)) IS_ENABLED(CONFIG_ARCH_ENABLE_SPLIT_PMD_PTLOCK))
#define ALLOC_SPLIT_PTLOCKS (SPINLOCK_SIZE > BITS_PER_LONG/8) #define ALLOC_SPLIT_PTLOCKS (SPINLOCK_SIZE > BITS_PER_LONG/8)
typedef void compound_page_dtor(struct page *);
/* /*
* Each physical page in the system has a struct page associated with * Each physical page in the system has a struct page associated with
* it to keep track of whatever it is we are using the page for at the * it to keep track of whatever it is we are using the page for at the
...@@ -136,8 +134,8 @@ struct page { ...@@ -136,8 +134,8 @@ struct page {
*/ */
/* First tail page of compound page */ /* First tail page of compound page */
struct { struct {
compound_page_dtor *compound_dtor; unsigned short int compound_dtor;
unsigned long compound_order; unsigned short int compound_order;
}; };
#if defined(CONFIG_TRANSPARENT_HUGEPAGE) && USE_SPLIT_PMD_PTLOCKS #if defined(CONFIG_TRANSPARENT_HUGEPAGE) && USE_SPLIT_PMD_PTLOCKS
......
...@@ -1146,7 +1146,7 @@ static void update_and_free_page(struct hstate *h, struct page *page) ...@@ -1146,7 +1146,7 @@ static void update_and_free_page(struct hstate *h, struct page *page)
1 << PG_writeback); 1 << PG_writeback);
} }
VM_BUG_ON_PAGE(hugetlb_cgroup_from_page(page), page); VM_BUG_ON_PAGE(hugetlb_cgroup_from_page(page), page);
set_compound_page_dtor(page, NULL); set_compound_page_dtor(page, NULL_COMPOUND_DTOR);
set_page_refcounted(page); set_page_refcounted(page);
if (hstate_is_gigantic(h)) { if (hstate_is_gigantic(h)) {
destroy_compound_gigantic_page(page, huge_page_order(h)); destroy_compound_gigantic_page(page, huge_page_order(h));
...@@ -1242,7 +1242,7 @@ void free_huge_page(struct page *page) ...@@ -1242,7 +1242,7 @@ void free_huge_page(struct page *page)
static void prep_new_huge_page(struct hstate *h, struct page *page, int nid) static void prep_new_huge_page(struct hstate *h, struct page *page, int nid)
{ {
INIT_LIST_HEAD(&page->lru); INIT_LIST_HEAD(&page->lru);
set_compound_page_dtor(page, free_huge_page); set_compound_page_dtor(page, HUGETLB_PAGE_DTOR);
spin_lock(&hugetlb_lock); spin_lock(&hugetlb_lock);
set_hugetlb_cgroup(page, NULL); set_hugetlb_cgroup(page, NULL);
h->nr_huge_pages++; h->nr_huge_pages++;
...@@ -1294,7 +1294,7 @@ int PageHuge(struct page *page) ...@@ -1294,7 +1294,7 @@ int PageHuge(struct page *page)
return 0; return 0;
page = compound_head(page); page = compound_head(page);
return get_compound_page_dtor(page) == free_huge_page; return page[1].compound_dtor == HUGETLB_PAGE_DTOR;
} }
EXPORT_SYMBOL_GPL(PageHuge); EXPORT_SYMBOL_GPL(PageHuge);
...@@ -1568,7 +1568,7 @@ static struct page *__alloc_buddy_huge_page(struct hstate *h, ...@@ -1568,7 +1568,7 @@ static struct page *__alloc_buddy_huge_page(struct hstate *h,
if (page) { if (page) {
INIT_LIST_HEAD(&page->lru); INIT_LIST_HEAD(&page->lru);
r_nid = page_to_nid(page); r_nid = page_to_nid(page);
set_compound_page_dtor(page, free_huge_page); set_compound_page_dtor(page, HUGETLB_PAGE_DTOR);
set_hugetlb_cgroup(page, NULL); set_hugetlb_cgroup(page, NULL);
/* /*
* We incremented the global counters already * We incremented the global counters already
......
...@@ -229,6 +229,15 @@ static char * const zone_names[MAX_NR_ZONES] = { ...@@ -229,6 +229,15 @@ static char * const zone_names[MAX_NR_ZONES] = {
#endif #endif
}; };
static void free_compound_page(struct page *page);
compound_page_dtor * const compound_page_dtors[] = {
NULL,
free_compound_page,
#ifdef CONFIG_HUGETLB_PAGE
free_huge_page,
#endif
};
int min_free_kbytes = 1024; int min_free_kbytes = 1024;
int user_min_free_kbytes = -1; int user_min_free_kbytes = -1;
...@@ -458,7 +467,7 @@ void prep_compound_page(struct page *page, unsigned long order) ...@@ -458,7 +467,7 @@ void prep_compound_page(struct page *page, unsigned long order)
int i; int i;
int nr_pages = 1 << order; int nr_pages = 1 << order;
set_compound_page_dtor(page, free_compound_page); set_compound_page_dtor(page, COMPOUND_PAGE_DTOR);
set_compound_order(page, order); set_compound_order(page, order);
__SetPageHead(page); __SetPageHead(page);
for (i = 1; i < nr_pages; i++) { for (i = 1; i < nr_pages; i++) {
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment