Commit ed7588d5 authored by Kirill A. Shutemov's avatar Kirill A. Shutemov Committed by Ingo Molnar

x86/mm: Stop pretending pgtable_l5_enabled is a variable

pgtable_l5_enabled is defined using cpu_feature_enabled() but we refer
to it as a variable. This is misleading.

Make pgtable_l5_enabled() a function.

We cannot literally define it as a function due to circular dependencies
between header files. Function-alike macros is close enough.
Signed-off-by: default avatarKirill A. Shutemov <kirill.shutemov@linux.intel.com>
Reviewed-by: default avatarThomas Gleixner <tglx@linutronix.de>
Cc: Hugh Dickins <hughd@google.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Link: http://lkml.kernel.org/r/20180518103528.59260-4-kirill.shutemov@linux.intel.comSigned-off-by: default avatarIngo Molnar <mingo@kernel.org>
parent ad3fe525
...@@ -53,7 +53,7 @@ ...@@ -53,7 +53,7 @@
#define __PHYSICAL_MASK_SHIFT 52 #define __PHYSICAL_MASK_SHIFT 52
#ifdef CONFIG_X86_5LEVEL #ifdef CONFIG_X86_5LEVEL
#define __VIRTUAL_MASK_SHIFT (pgtable_l5_enabled ? 56 : 47) #define __VIRTUAL_MASK_SHIFT (pgtable_l5_enabled() ? 56 : 47)
#else #else
#define __VIRTUAL_MASK_SHIFT 47 #define __VIRTUAL_MASK_SHIFT 47
#endif #endif
......
...@@ -574,14 +574,14 @@ static inline void __set_pgd(pgd_t *pgdp, pgd_t pgd) ...@@ -574,14 +574,14 @@ static inline void __set_pgd(pgd_t *pgdp, pgd_t pgd)
} }
#define set_pgd(pgdp, pgdval) do { \ #define set_pgd(pgdp, pgdval) do { \
if (pgtable_l5_enabled) \ if (pgtable_l5_enabled()) \
__set_pgd(pgdp, pgdval); \ __set_pgd(pgdp, pgdval); \
else \ else \
set_p4d((p4d_t *)(pgdp), (p4d_t) { (pgdval).pgd }); \ set_p4d((p4d_t *)(pgdp), (p4d_t) { (pgdval).pgd }); \
} while (0) } while (0)
#define pgd_clear(pgdp) do { \ #define pgd_clear(pgdp) do { \
if (pgtable_l5_enabled) \ if (pgtable_l5_enabled()) \
set_pgd(pgdp, __pgd(0)); \ set_pgd(pgdp, __pgd(0)); \
} while (0) } while (0)
......
...@@ -167,7 +167,7 @@ static inline void __pud_free_tlb(struct mmu_gather *tlb, pud_t *pud, ...@@ -167,7 +167,7 @@ static inline void __pud_free_tlb(struct mmu_gather *tlb, pud_t *pud,
#if CONFIG_PGTABLE_LEVELS > 4 #if CONFIG_PGTABLE_LEVELS > 4
static inline void pgd_populate(struct mm_struct *mm, pgd_t *pgd, p4d_t *p4d) static inline void pgd_populate(struct mm_struct *mm, pgd_t *pgd, p4d_t *p4d)
{ {
if (!pgtable_l5_enabled) if (!pgtable_l5_enabled())
return; return;
paravirt_alloc_p4d(mm, __pa(p4d) >> PAGE_SHIFT); paravirt_alloc_p4d(mm, __pa(p4d) >> PAGE_SHIFT);
set_pgd(pgd, __pgd(_PAGE_TABLE | __pa(p4d))); set_pgd(pgd, __pgd(_PAGE_TABLE | __pa(p4d)));
...@@ -193,7 +193,7 @@ extern void ___p4d_free_tlb(struct mmu_gather *tlb, p4d_t *p4d); ...@@ -193,7 +193,7 @@ extern void ___p4d_free_tlb(struct mmu_gather *tlb, p4d_t *p4d);
static inline void __p4d_free_tlb(struct mmu_gather *tlb, p4d_t *p4d, static inline void __p4d_free_tlb(struct mmu_gather *tlb, p4d_t *p4d,
unsigned long address) unsigned long address)
{ {
if (pgtable_l5_enabled) if (pgtable_l5_enabled())
___p4d_free_tlb(tlb, p4d); ___p4d_free_tlb(tlb, p4d);
} }
......
...@@ -65,7 +65,7 @@ extern pmdval_t early_pmd_flags; ...@@ -65,7 +65,7 @@ extern pmdval_t early_pmd_flags;
#ifndef __PAGETABLE_P4D_FOLDED #ifndef __PAGETABLE_P4D_FOLDED
#define set_pgd(pgdp, pgd) native_set_pgd(pgdp, pgd) #define set_pgd(pgdp, pgd) native_set_pgd(pgdp, pgd)
#define pgd_clear(pgd) (pgtable_l5_enabled ? native_pgd_clear(pgd) : 0) #define pgd_clear(pgd) (pgtable_l5_enabled() ? native_pgd_clear(pgd) : 0)
#endif #endif
#ifndef set_p4d #ifndef set_p4d
...@@ -881,7 +881,7 @@ static inline unsigned long p4d_index(unsigned long address) ...@@ -881,7 +881,7 @@ static inline unsigned long p4d_index(unsigned long address)
#if CONFIG_PGTABLE_LEVELS > 4 #if CONFIG_PGTABLE_LEVELS > 4
static inline int pgd_present(pgd_t pgd) static inline int pgd_present(pgd_t pgd)
{ {
if (!pgtable_l5_enabled) if (!pgtable_l5_enabled())
return 1; return 1;
return pgd_flags(pgd) & _PAGE_PRESENT; return pgd_flags(pgd) & _PAGE_PRESENT;
} }
...@@ -900,7 +900,7 @@ static inline unsigned long pgd_page_vaddr(pgd_t pgd) ...@@ -900,7 +900,7 @@ static inline unsigned long pgd_page_vaddr(pgd_t pgd)
/* to find an entry in a page-table-directory. */ /* to find an entry in a page-table-directory. */
static inline p4d_t *p4d_offset(pgd_t *pgd, unsigned long address) static inline p4d_t *p4d_offset(pgd_t *pgd, unsigned long address)
{ {
if (!pgtable_l5_enabled) if (!pgtable_l5_enabled())
return (p4d_t *)pgd; return (p4d_t *)pgd;
return (p4d_t *)pgd_page_vaddr(*pgd) + p4d_index(address); return (p4d_t *)pgd_page_vaddr(*pgd) + p4d_index(address);
} }
...@@ -909,7 +909,7 @@ static inline int pgd_bad(pgd_t pgd) ...@@ -909,7 +909,7 @@ static inline int pgd_bad(pgd_t pgd)
{ {
unsigned long ignore_flags = _PAGE_USER; unsigned long ignore_flags = _PAGE_USER;
if (!pgtable_l5_enabled) if (!pgtable_l5_enabled())
return 0; return 0;
if (IS_ENABLED(CONFIG_PAGE_TABLE_ISOLATION)) if (IS_ENABLED(CONFIG_PAGE_TABLE_ISOLATION))
...@@ -920,7 +920,7 @@ static inline int pgd_bad(pgd_t pgd) ...@@ -920,7 +920,7 @@ static inline int pgd_bad(pgd_t pgd)
static inline int pgd_none(pgd_t pgd) static inline int pgd_none(pgd_t pgd)
{ {
if (!pgtable_l5_enabled) if (!pgtable_l5_enabled())
return 0; return 0;
/* /*
* There is no need to do a workaround for the KNL stray * There is no need to do a workaround for the KNL stray
......
...@@ -15,7 +15,7 @@ ...@@ -15,7 +15,7 @@
# include <asm/pgtable-2level_types.h> # include <asm/pgtable-2level_types.h>
#endif #endif
#define pgtable_l5_enabled 0 #define pgtable_l5_enabled() 0
#define PGDIR_SIZE (1UL << PGDIR_SHIFT) #define PGDIR_SIZE (1UL << PGDIR_SHIFT)
#define PGDIR_MASK (~(PGDIR_SIZE - 1)) #define PGDIR_MASK (~(PGDIR_SIZE - 1))
......
...@@ -220,7 +220,7 @@ static inline void native_set_p4d(p4d_t *p4dp, p4d_t p4d) ...@@ -220,7 +220,7 @@ static inline void native_set_p4d(p4d_t *p4dp, p4d_t p4d)
{ {
pgd_t pgd; pgd_t pgd;
if (pgtable_l5_enabled || !IS_ENABLED(CONFIG_PAGE_TABLE_ISOLATION)) { if (pgtable_l5_enabled() || !IS_ENABLED(CONFIG_PAGE_TABLE_ISOLATION)) {
*p4dp = p4d; *p4dp = p4d;
return; return;
} }
......
...@@ -28,12 +28,16 @@ extern unsigned int __pgtable_l5_enabled; ...@@ -28,12 +28,16 @@ extern unsigned int __pgtable_l5_enabled;
* cpu_feature_enabled() is not available in early boot code. * cpu_feature_enabled() is not available in early boot code.
* Use variable instead. * Use variable instead.
*/ */
#define pgtable_l5_enabled __pgtable_l5_enabled static inline bool pgtable_l5_enabled(void)
{
return __pgtable_l5_enabled;
}
#else #else
#define pgtable_l5_enabled cpu_feature_enabled(X86_FEATURE_LA57) #define pgtable_l5_enabled() cpu_feature_enabled(X86_FEATURE_LA57)
#endif /* USE_EARLY_PGTABLE_L5 */ #endif /* USE_EARLY_PGTABLE_L5 */
#else #else
#define pgtable_l5_enabled 0 #define pgtable_l5_enabled() 0
#endif /* CONFIG_X86_5LEVEL */ #endif /* CONFIG_X86_5LEVEL */
extern unsigned int pgdir_shift; extern unsigned int pgdir_shift;
...@@ -109,7 +113,7 @@ extern unsigned int ptrs_per_p4d; ...@@ -109,7 +113,7 @@ extern unsigned int ptrs_per_p4d;
#define LDT_PGD_ENTRY_L4 -3UL #define LDT_PGD_ENTRY_L4 -3UL
#define LDT_PGD_ENTRY_L5 -112UL #define LDT_PGD_ENTRY_L5 -112UL
#define LDT_PGD_ENTRY (pgtable_l5_enabled ? LDT_PGD_ENTRY_L5 : LDT_PGD_ENTRY_L4) #define LDT_PGD_ENTRY (pgtable_l5_enabled() ? LDT_PGD_ENTRY_L5 : LDT_PGD_ENTRY_L4)
#define LDT_BASE_ADDR (LDT_PGD_ENTRY << PGDIR_SHIFT) #define LDT_BASE_ADDR (LDT_PGD_ENTRY << PGDIR_SHIFT)
#define __VMALLOC_BASE_L4 0xffffc90000000000UL #define __VMALLOC_BASE_L4 0xffffc90000000000UL
...@@ -123,7 +127,7 @@ extern unsigned int ptrs_per_p4d; ...@@ -123,7 +127,7 @@ extern unsigned int ptrs_per_p4d;
#ifdef CONFIG_DYNAMIC_MEMORY_LAYOUT #ifdef CONFIG_DYNAMIC_MEMORY_LAYOUT
# define VMALLOC_START vmalloc_base # define VMALLOC_START vmalloc_base
# define VMALLOC_SIZE_TB (pgtable_l5_enabled ? VMALLOC_SIZE_TB_L5 : VMALLOC_SIZE_TB_L4) # define VMALLOC_SIZE_TB (pgtable_l5_enabled() ? VMALLOC_SIZE_TB_L5 : VMALLOC_SIZE_TB_L4)
# define VMEMMAP_START vmemmap_base # define VMEMMAP_START vmemmap_base
#else #else
# define VMALLOC_START __VMALLOC_BASE_L4 # define VMALLOC_START __VMALLOC_BASE_L4
......
...@@ -27,8 +27,8 @@ ...@@ -27,8 +27,8 @@
# endif # endif
#else /* CONFIG_X86_32 */ #else /* CONFIG_X86_32 */
# define SECTION_SIZE_BITS 27 /* matt - 128 is convenient right now */ # define SECTION_SIZE_BITS 27 /* matt - 128 is convenient right now */
# define MAX_PHYSADDR_BITS (pgtable_l5_enabled ? 52 : 44) # define MAX_PHYSADDR_BITS (pgtable_l5_enabled() ? 52 : 44)
# define MAX_PHYSMEM_BITS (pgtable_l5_enabled ? 52 : 46) # define MAX_PHYSMEM_BITS (pgtable_l5_enabled() ? 52 : 46)
#endif #endif
#endif /* CONFIG_SPARSEMEM */ #endif /* CONFIG_SPARSEMEM */
......
...@@ -279,7 +279,7 @@ int __init __early_make_pgtable(unsigned long address, pmdval_t pmd) ...@@ -279,7 +279,7 @@ int __init __early_make_pgtable(unsigned long address, pmdval_t pmd)
* critical -- __PAGE_OFFSET would point us back into the dynamic * critical -- __PAGE_OFFSET would point us back into the dynamic
* range and we might end up looping forever... * range and we might end up looping forever...
*/ */
if (!pgtable_l5_enabled) if (!pgtable_l5_enabled())
p4d_p = pgd_p; p4d_p = pgd_p;
else if (pgd) else if (pgd)
p4d_p = (p4dval_t *)((pgd & PTE_PFN_MASK) + __START_KERNEL_map - phys_base); p4d_p = (p4dval_t *)((pgd & PTE_PFN_MASK) + __START_KERNEL_map - phys_base);
......
...@@ -354,7 +354,8 @@ void arch_crash_save_vmcoreinfo(void) ...@@ -354,7 +354,8 @@ void arch_crash_save_vmcoreinfo(void)
{ {
VMCOREINFO_NUMBER(phys_base); VMCOREINFO_NUMBER(phys_base);
VMCOREINFO_SYMBOL(init_top_pgt); VMCOREINFO_SYMBOL(init_top_pgt);
VMCOREINFO_NUMBER(pgtable_l5_enabled); vmcoreinfo_append_str("NUMBER(pgtable_l5_enabled)=%d\n",
pgtable_l5_enabled());
#ifdef CONFIG_NUMA #ifdef CONFIG_NUMA
VMCOREINFO_SYMBOL(node_data); VMCOREINFO_SYMBOL(node_data);
......
...@@ -360,7 +360,7 @@ static inline bool kasan_page_table(struct seq_file *m, struct pg_state *st, ...@@ -360,7 +360,7 @@ static inline bool kasan_page_table(struct seq_file *m, struct pg_state *st,
void *pt) void *pt)
{ {
if (__pa(pt) == __pa(kasan_zero_pmd) || if (__pa(pt) == __pa(kasan_zero_pmd) ||
(pgtable_l5_enabled && __pa(pt) == __pa(kasan_zero_p4d)) || (pgtable_l5_enabled() && __pa(pt) == __pa(kasan_zero_p4d)) ||
__pa(pt) == __pa(kasan_zero_pud)) { __pa(pt) == __pa(kasan_zero_pud)) {
pgprotval_t prot = pte_flags(kasan_zero_pte[0]); pgprotval_t prot = pte_flags(kasan_zero_pte[0]);
note_page(m, st, __pgprot(prot), 0, 5); note_page(m, st, __pgprot(prot), 0, 5);
...@@ -476,8 +476,8 @@ static void walk_p4d_level(struct seq_file *m, struct pg_state *st, pgd_t addr, ...@@ -476,8 +476,8 @@ static void walk_p4d_level(struct seq_file *m, struct pg_state *st, pgd_t addr,
} }
} }
#define pgd_large(a) (pgtable_l5_enabled ? pgd_large(a) : p4d_large(__p4d(pgd_val(a)))) #define pgd_large(a) (pgtable_l5_enabled() ? pgd_large(a) : p4d_large(__p4d(pgd_val(a))))
#define pgd_none(a) (pgtable_l5_enabled ? pgd_none(a) : p4d_none(__p4d(pgd_val(a)))) #define pgd_none(a) (pgtable_l5_enabled() ? pgd_none(a) : p4d_none(__p4d(pgd_val(a))))
static inline bool is_hypervisor_range(int idx) static inline bool is_hypervisor_range(int idx)
{ {
......
...@@ -439,7 +439,7 @@ static noinline int vmalloc_fault(unsigned long address) ...@@ -439,7 +439,7 @@ static noinline int vmalloc_fault(unsigned long address)
if (pgd_none(*pgd_k)) if (pgd_none(*pgd_k))
return -1; return -1;
if (pgtable_l5_enabled) { if (pgtable_l5_enabled()) {
if (pgd_none(*pgd)) { if (pgd_none(*pgd)) {
set_pgd(pgd, *pgd_k); set_pgd(pgd, *pgd_k);
arch_flush_lazy_mmu_mode(); arch_flush_lazy_mmu_mode();
...@@ -454,7 +454,7 @@ static noinline int vmalloc_fault(unsigned long address) ...@@ -454,7 +454,7 @@ static noinline int vmalloc_fault(unsigned long address)
if (p4d_none(*p4d_k)) if (p4d_none(*p4d_k))
return -1; return -1;
if (p4d_none(*p4d) && !pgtable_l5_enabled) { if (p4d_none(*p4d) && !pgtable_l5_enabled()) {
set_p4d(p4d, *p4d_k); set_p4d(p4d, *p4d_k);
arch_flush_lazy_mmu_mode(); arch_flush_lazy_mmu_mode();
} else { } else {
......
...@@ -123,7 +123,7 @@ int kernel_ident_mapping_init(struct x86_mapping_info *info, pgd_t *pgd_page, ...@@ -123,7 +123,7 @@ int kernel_ident_mapping_init(struct x86_mapping_info *info, pgd_t *pgd_page,
result = ident_p4d_init(info, p4d, addr, next); result = ident_p4d_init(info, p4d, addr, next);
if (result) if (result)
return result; return result;
if (pgtable_l5_enabled) { if (pgtable_l5_enabled()) {
set_pgd(pgd, __pgd(__pa(p4d) | info->kernpg_flag)); set_pgd(pgd, __pgd(__pa(p4d) | info->kernpg_flag));
} else { } else {
/* /*
......
...@@ -180,7 +180,7 @@ static void sync_global_pgds_l4(unsigned long start, unsigned long end) ...@@ -180,7 +180,7 @@ static void sync_global_pgds_l4(unsigned long start, unsigned long end)
*/ */
void sync_global_pgds(unsigned long start, unsigned long end) void sync_global_pgds(unsigned long start, unsigned long end)
{ {
if (pgtable_l5_enabled) if (pgtable_l5_enabled())
sync_global_pgds_l5(start, end); sync_global_pgds_l5(start, end);
else else
sync_global_pgds_l4(start, end); sync_global_pgds_l4(start, end);
...@@ -643,7 +643,7 @@ phys_p4d_init(p4d_t *p4d_page, unsigned long paddr, unsigned long paddr_end, ...@@ -643,7 +643,7 @@ phys_p4d_init(p4d_t *p4d_page, unsigned long paddr, unsigned long paddr_end,
unsigned long vaddr = (unsigned long)__va(paddr); unsigned long vaddr = (unsigned long)__va(paddr);
int i = p4d_index(vaddr); int i = p4d_index(vaddr);
if (!pgtable_l5_enabled) if (!pgtable_l5_enabled())
return phys_pud_init((pud_t *) p4d_page, paddr, paddr_end, page_size_mask); return phys_pud_init((pud_t *) p4d_page, paddr, paddr_end, page_size_mask);
for (; i < PTRS_PER_P4D; i++, paddr = paddr_next) { for (; i < PTRS_PER_P4D; i++, paddr = paddr_next) {
...@@ -723,7 +723,7 @@ kernel_physical_mapping_init(unsigned long paddr_start, ...@@ -723,7 +723,7 @@ kernel_physical_mapping_init(unsigned long paddr_start,
page_size_mask); page_size_mask);
spin_lock(&init_mm.page_table_lock); spin_lock(&init_mm.page_table_lock);
if (pgtable_l5_enabled) if (pgtable_l5_enabled())
pgd_populate(&init_mm, pgd, p4d); pgd_populate(&init_mm, pgd, p4d);
else else
p4d_populate(&init_mm, p4d_offset(pgd, vaddr), (pud_t *) p4d); p4d_populate(&init_mm, p4d_offset(pgd, vaddr), (pud_t *) p4d);
...@@ -1100,7 +1100,7 @@ remove_p4d_table(p4d_t *p4d_start, unsigned long addr, unsigned long end, ...@@ -1100,7 +1100,7 @@ remove_p4d_table(p4d_t *p4d_start, unsigned long addr, unsigned long end,
* 5-level case we should free them. This code will have to change * 5-level case we should free them. This code will have to change
* to adapt for boot-time switching between 4 and 5 level page tables. * to adapt for boot-time switching between 4 and 5 level page tables.
*/ */
if (pgtable_l5_enabled) if (pgtable_l5_enabled())
free_pud_table(pud_base, p4d); free_pud_table(pud_base, p4d);
} }
......
...@@ -180,7 +180,7 @@ static void __init clear_pgds(unsigned long start, ...@@ -180,7 +180,7 @@ static void __init clear_pgds(unsigned long start,
* With folded p4d, pgd_clear() is nop, use p4d_clear() * With folded p4d, pgd_clear() is nop, use p4d_clear()
* instead. * instead.
*/ */
if (pgtable_l5_enabled) if (pgtable_l5_enabled())
pgd_clear(pgd); pgd_clear(pgd);
else else
p4d_clear(p4d_offset(pgd, start)); p4d_clear(p4d_offset(pgd, start));
...@@ -195,7 +195,7 @@ static inline p4d_t *early_p4d_offset(pgd_t *pgd, unsigned long addr) ...@@ -195,7 +195,7 @@ static inline p4d_t *early_p4d_offset(pgd_t *pgd, unsigned long addr)
{ {
unsigned long p4d; unsigned long p4d;
if (!pgtable_l5_enabled) if (!pgtable_l5_enabled())
return (p4d_t *)pgd; return (p4d_t *)pgd;
p4d = __pa_nodebug(pgd_val(*pgd)) & PTE_PFN_MASK; p4d = __pa_nodebug(pgd_val(*pgd)) & PTE_PFN_MASK;
...@@ -282,7 +282,7 @@ void __init kasan_early_init(void) ...@@ -282,7 +282,7 @@ void __init kasan_early_init(void)
for (i = 0; i < PTRS_PER_PUD; i++) for (i = 0; i < PTRS_PER_PUD; i++)
kasan_zero_pud[i] = __pud(pud_val); kasan_zero_pud[i] = __pud(pud_val);
for (i = 0; pgtable_l5_enabled && i < PTRS_PER_P4D; i++) for (i = 0; pgtable_l5_enabled() && i < PTRS_PER_P4D; i++)
kasan_zero_p4d[i] = __p4d(p4d_val); kasan_zero_p4d[i] = __p4d(p4d_val);
kasan_map_early_shadow(early_top_pgt); kasan_map_early_shadow(early_top_pgt);
...@@ -313,7 +313,7 @@ void __init kasan_init(void) ...@@ -313,7 +313,7 @@ void __init kasan_init(void)
* bunch of things like kernel code, modules, EFI mapping, etc. * bunch of things like kernel code, modules, EFI mapping, etc.
* We need to take extra steps to not overwrite them. * We need to take extra steps to not overwrite them.
*/ */
if (pgtable_l5_enabled) { if (pgtable_l5_enabled()) {
void *ptr; void *ptr;
ptr = (void *)pgd_page_vaddr(*pgd_offset_k(KASAN_SHADOW_END)); ptr = (void *)pgd_page_vaddr(*pgd_offset_k(KASAN_SHADOW_END));
......
...@@ -78,7 +78,7 @@ void __init kernel_randomize_memory(void) ...@@ -78,7 +78,7 @@ void __init kernel_randomize_memory(void)
struct rnd_state rand_state; struct rnd_state rand_state;
unsigned long remain_entropy; unsigned long remain_entropy;
vaddr_start = pgtable_l5_enabled ? __PAGE_OFFSET_BASE_L5 : __PAGE_OFFSET_BASE_L4; vaddr_start = pgtable_l5_enabled() ? __PAGE_OFFSET_BASE_L5 : __PAGE_OFFSET_BASE_L4;
vaddr = vaddr_start; vaddr = vaddr_start;
/* /*
...@@ -124,7 +124,7 @@ void __init kernel_randomize_memory(void) ...@@ -124,7 +124,7 @@ void __init kernel_randomize_memory(void)
*/ */
entropy = remain_entropy / (ARRAY_SIZE(kaslr_regions) - i); entropy = remain_entropy / (ARRAY_SIZE(kaslr_regions) - i);
prandom_bytes_state(&rand_state, &rand, sizeof(rand)); prandom_bytes_state(&rand_state, &rand, sizeof(rand));
if (pgtable_l5_enabled) if (pgtable_l5_enabled())
entropy = (rand % (entropy + 1)) & P4D_MASK; entropy = (rand % (entropy + 1)) & P4D_MASK;
else else
entropy = (rand % (entropy + 1)) & PUD_MASK; entropy = (rand % (entropy + 1)) & PUD_MASK;
...@@ -136,7 +136,7 @@ void __init kernel_randomize_memory(void) ...@@ -136,7 +136,7 @@ void __init kernel_randomize_memory(void)
* randomization alignment. * randomization alignment.
*/ */
vaddr += get_padding(&kaslr_regions[i]); vaddr += get_padding(&kaslr_regions[i]);
if (pgtable_l5_enabled) if (pgtable_l5_enabled())
vaddr = round_up(vaddr + 1, P4D_SIZE); vaddr = round_up(vaddr + 1, P4D_SIZE);
else else
vaddr = round_up(vaddr + 1, PUD_SIZE); vaddr = round_up(vaddr + 1, PUD_SIZE);
...@@ -212,7 +212,7 @@ void __meminit init_trampoline(void) ...@@ -212,7 +212,7 @@ void __meminit init_trampoline(void)
return; return;
} }
if (pgtable_l5_enabled) if (pgtable_l5_enabled())
init_trampoline_p4d(); init_trampoline_p4d();
else else
init_trampoline_pud(); init_trampoline_pud();
......
...@@ -157,7 +157,7 @@ static void sync_current_stack_to_mm(struct mm_struct *mm) ...@@ -157,7 +157,7 @@ static void sync_current_stack_to_mm(struct mm_struct *mm)
unsigned long sp = current_stack_pointer; unsigned long sp = current_stack_pointer;
pgd_t *pgd = pgd_offset(mm, sp); pgd_t *pgd = pgd_offset(mm, sp);
if (pgtable_l5_enabled) { if (pgtable_l5_enabled()) {
if (unlikely(pgd_none(*pgd))) { if (unlikely(pgd_none(*pgd))) {
pgd_t *pgd_ref = pgd_offset_k(sp); pgd_t *pgd_ref = pgd_offset_k(sp);
......
...@@ -225,7 +225,7 @@ int __init efi_alloc_page_tables(void) ...@@ -225,7 +225,7 @@ int __init efi_alloc_page_tables(void)
pud = pud_alloc(&init_mm, p4d, EFI_VA_END); pud = pud_alloc(&init_mm, p4d, EFI_VA_END);
if (!pud) { if (!pud) {
if (pgtable_l5_enabled) if (pgtable_l5_enabled())
free_page((unsigned long) pgd_page_vaddr(*pgd)); free_page((unsigned long) pgd_page_vaddr(*pgd));
free_pages((unsigned long)efi_pgd, PGD_ALLOCATION_ORDER); free_pages((unsigned long)efi_pgd, PGD_ALLOCATION_ORDER);
return -ENOMEM; return -ENOMEM;
......
...@@ -72,7 +72,7 @@ static int set_up_temporary_text_mapping(pgd_t *pgd) ...@@ -72,7 +72,7 @@ static int set_up_temporary_text_mapping(pgd_t *pgd)
* tables used by the image kernel. * tables used by the image kernel.
*/ */
if (pgtable_l5_enabled) { if (pgtable_l5_enabled()) {
p4d = (p4d_t *)get_safe_page(GFP_ATOMIC); p4d = (p4d_t *)get_safe_page(GFP_ATOMIC);
if (!p4d) if (!p4d)
return -ENOMEM; return -ENOMEM;
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment