Skip to content

L
linux
Commit 8d2c59fd authored by Russell King's avatar Russell King
Browse files
Options

[ARM] Optimise set_pmd 

Since we store two hardware pte tables contiguously, a common
operation is to set two pmd entries.  Rather than call set_pmd()
with the associated overhead twice, we set the two PMD entries,
and then call cpu_flush_pmd() to perform any CPU specific handling.
parent bd51d9ad
Hide whitespace changes
Inline Side-by-side
Showing with 60 additions and 93 deletions
arch/arm/mm/mm-armv.c
View file @ 8d2c59fd
......@@ -165,14 +165,13 @@ void free_pgd_slow(pgd_t *pgd)
static inline void
alloc_init_section(unsigned long virt, unsigned long phys, int prot)
{
pmd_t *pmdp, pmd;
pmd_t *pmdp;
pmdp = pmd_offset(pgd_offset_k(virt), virt);
if (virt & (1 << 20))
pmdp++;
pmd_val(pmd) = phys | prot;
set_pmd(pmdp, pmd);
set_pmd(pmdp, __pmd(phys | prot));
}
/*
......@@ -185,19 +184,20 @@ alloc_init_section(unsigned long virt, unsigned long phys, int prot)
static inline void
alloc_init_page(unsigned long virt, unsigned long phys, unsigned int prot_l1, pgprot_t prot)
{
pmd_t *pmdp, pmd;
pmd_t *pmdp;
pte_t *ptep;
pmdp = pmd_offset(pgd_offset_k(virt), virt);
if (pmd_none(*pmdp)) {
unsigned long pmdval;
ptep = alloc_bootmem_low_pages(2 * PTRS_PER_PTE *
sizeof(pte_t));
pmd_val(pmd) = __pa(ptep) | prot_l1;
set_pmd(pmdp, pmd);
pmd_val(pmd) += 256 * sizeof(pte_t);
set_pmd(pmdp + 1, pmd);
pmdval = __pa(ptep) | prot_l1;
pmdp[0] = __pmd(pmdval);
pmdp[1] = __pmd(pmdval + 256 * sizeof(pte_t));
cpu_flush_pmd(pmdp);
}
ptep = pte_offset_kernel(pmdp, virt);
......@@ -359,8 +359,9 @@ static void __init create_mapping(struct map_desc *md)
*/
void setup_mm_for_reboot(char mode)
{
unsigned long pmdval;
pgd_t *pgd;
pmd_t pmd;
pmd_t *pmd;
int i;
if (current->mm && current->mm->pgd)
......@@ -369,10 +370,11 @@ void setup_mm_for_reboot(char mode)
pgd = init_mm.pgd;
for (i = 0; i < FIRST_USER_PGD_NR + USER_PTRS_PER_PGD; i++) {
pmd_val(pmd) = (i << PGDIR_SHIFT) |
PMD_SECT_AP_WRITE | PMD_SECT_AP_READ |
PMD_TYPE_SECT;
set_pmd(pmd_offset(pgd + i, i << PGDIR_SHIFT), pmd);
pmdval = (i << PGDIR_SHIFT) |
PMD_SECT_AP_WRITE | PMD_SECT_AP_READ |
PMD_BIT4 | PMD_TYPE_SECT;
pmd = pmd_offset(pgd + i, i << PGDIR_SHIFT);
set_pmd(pmd, __pmd(pmdval));
}
}
......
arch/arm/mm/proc-arm1020.S
View file @ 8d2c59fd
......@@ -365,17 +365,15 @@ ENTRY(cpu_arm1020_set_pgd)
mov pc, lr
/*
* cpu_arm1020_set_pmd(pmdp, pmd)
* cpu_arm1020_flush_pmd(pmdp)
*
* Set a level 1 translation table entry, and clean it out of
* any caches such that the MMUs can load it correctly.
*
* pmdp: pointer to PMD entry
* pmd: PMD value to store
*/
.align 5
ENTRY(cpu_arm1020_set_pmd)
str r1, [r0]
ENTRY(cpu_arm1020_flush_pmd)
#ifndef CONFIG_CPU_DCACHE_DISABLE
mcr p15, 0, r0, c7, c10, 4
mcr p15, 0, r0, c7, c10, 1 @ clean D entry (drain is done by TLB fns)
......@@ -515,7 +513,7 @@ arm1020_processor_functions:
/* pgtable */
.word cpu_arm1020_set_pgd
.word cpu_arm1020_set_pmd
.word cpu_arm1020_flush_pmd
.word cpu_arm1020_set_pte
.size arm1020_processor_functions, . - arm1020_processor_functions
......
arch/arm/mm/proc-arm6_7.S
View file @ 8d2c59fd
......@@ -234,34 +234,14 @@ ENTRY(cpu_arm7_set_pgd)
mov pc, lr
/*
* Function: arm6_set_pmd ()
* Function: arm6_flush_pmd(pmdp)
*
* Params : r0 = Address to set
* : r1 = value to set
*
* Purpose : Set a PMD and flush it out of any WB cache
*/
ENTRY(cpu_arm6_set_pmd)
and r2, r1, #11
teq r2, #1
teqne r2, #9
teqne r2, #10
orreq r1, r1, #16 @ Updatable = 1 if Page table/Cacheable section
str r1, [r0]
mov pc, lr
/*
* Function: arm7_set_pmd ()
*
* Params : r0 = Address to set
* : r1 = value to set
*
* Purpose : Set a PMD and flush it out of any WB cache
*/
ENTRY(cpu_arm7_set_pmd)
tst r1, #3
orrne r1, r1, #16 @ Updatable bit is always set on ARM7
str r1, [r0]
ENTRY(cpu_arm6_flush_pmd)
ENTRY(cpu_arm7_flush_pmd)
mov pc, lr
/*
......@@ -368,7 +348,7 @@ ENTRY(arm6_processor_functions)
/* pgtable */
.word cpu_arm6_set_pgd
.word cpu_arm6_set_pmd
.word cpu_arm6_flush_pmd
.word cpu_arm6_set_pte
.size arm6_processor_functions, . - arm6_processor_functions
......@@ -402,7 +382,7 @@ ENTRY(arm7_processor_functions)
/* pgtable */
.word cpu_arm7_set_pgd
.word cpu_arm7_set_pmd
.word cpu_arm7_flush_pmd
.word cpu_arm7_set_pte
.size arm7_processor_functions, . - arm7_processor_functions
......
arch/arm/mm/proc-arm720.S
View file @ 8d2c59fd
......@@ -115,17 +115,13 @@ ENTRY(cpu_arm720_set_pgd)
mov pc, lr
/*
* Function: arm720_set_pmd ()
* Function: arm720_flush_pmd(pmdp)
*
* Params : r0 = Address to set
* : r1 = value to set
*
* Purpose : Set a PMD and flush it out of any WB cache
*/
ENTRY(cpu_arm720_set_pmd)
tst r1, #3
orrne r1, r1, #16 @ Updatable bit is
str r1, [r0] @ always set on ARM720
ENTRY(cpu_arm720_flush_pmd)
mov pc, lr
/*
......@@ -222,7 +218,7 @@ ENTRY(arm720_processor_functions)
/* pgtable */
.word cpu_arm720_set_pgd
.word cpu_arm720_set_pmd
.word cpu_arm720_flush_pmd
.word cpu_arm720_set_pte
.size arm720_processor_functions, . - arm720_processor_functions
......
arch/arm/mm/proc-arm920.S
View file @ 8d2c59fd
......@@ -368,17 +368,15 @@ ENTRY(cpu_arm920_set_pgd)
mov pc, lr
/*
* cpu_arm920_set_pmd(pmdp, pmd)
* cpu_arm920_flush_pmd(pmdp)
*
* Set a level 1 translation table entry, and clean it out of
* any caches such that the MMUs can load it correctly.
*
* pmdp: pointer to PMD entry
* pmd: PMD value to store
*/
.align 5
ENTRY(cpu_arm920_set_pmd)
str r1, [r0]
ENTRY(cpu_arm920_flush_pmd)
mcr p15, 0, r0, c7, c10, 1 @ clean D entry
mcr p15, 0, r0, c7, c10, 4 @ drain WB
mov pc, lr
......@@ -503,7 +501,7 @@ arm920_processor_functions:
/* pgtable */
.word cpu_arm920_set_pgd
.word cpu_arm920_set_pmd
.word cpu_arm920_flush_pmd
.word cpu_arm920_set_pte
.size arm920_processor_functions, . - arm920_processor_functions
......
arch/arm/mm/proc-arm922.S
View file @ 8d2c59fd
......@@ -369,17 +369,15 @@ ENTRY(cpu_arm922_set_pgd)
mov pc, lr
/*
* cpu_arm922_set_pmd(pmdp, pmd)
* cpu_arm922_flush_pmd(pmdp)
*
* Set a level 1 translation table entry, and clean it out of
* any caches such that the MMUs can load it correctly.
*
* pmdp: pointer to PMD entry
* pmd: PMD value to store
*/
.align 5
ENTRY(cpu_arm922_set_pmd)
str r1, [r0]
ENTRY(cpu_arm922_flush_pmd)
mcr p15, 0, r0, c7, c10, 1 @ clean D entry
mcr p15, 0, r0, c7, c10, 4 @ drain WB
mov pc, lr
......@@ -502,7 +500,7 @@ arm922_processor_functions:
/* pgtable */
.word cpu_arm922_set_pgd
.word cpu_arm922_set_pmd
.word cpu_arm922_flush_pmd
.word cpu_arm922_set_pte
.size arm922_processor_functions, . - arm922_processor_functions
......
arch/arm/mm/proc-arm926.S
View file @ 8d2c59fd
......@@ -350,17 +350,15 @@ ENTRY(cpu_arm926_set_pgd)
mov pc, lr
/*
* cpu_arm926_set_pmd(pmdp, pmd)
* cpu_arm926_flush_pmd(pmdp)
*
* Set a level 1 translation table entry, and clean it out of
* any caches such that the MMUs can load it correctly.
*
* pmdp: pointer to PMD entry
* pmd: PMD value to store
*/
.align 5
ENTRY(cpu_arm926_set_pmd)
str r1, [r0]
ENTRY(cpu_arm926_flush_pmd)
#ifndef CONFIG_CPU_DCACHE_WRITETHROUGH
mcr p15, 0, r0, c7, c10, 1 @ clean D entry
#endif
......@@ -500,7 +498,7 @@ arm926_processor_functions:
/* pgtable */
.word cpu_arm926_set_pgd
.word cpu_arm926_set_pmd
.word cpu_arm926_flush_pmd
.word cpu_arm926_set_pte
.size arm926_processor_functions, . - arm926_processor_functions
......
arch/arm/mm/proc-sa110.S
View file @ 8d2c59fd
......@@ -425,18 +425,16 @@ ENTRY(cpu_sa1100_set_pgd)
mov pc, lr
/*
* cpu_sa110_set_pmd(pmdp, pmd)
* cpu_sa110_flush_pmd(pmdp)
*
* Set a level 1 translation table entry, and clean it out of
* any caches such that the MMUs can load it correctly.
*
* pmdp: pointer to PMD entry
* pmd: PMD value to store
*/
.align 5
ENTRY(cpu_sa110_set_pmd)
ENTRY(cpu_sa1100_set_pmd)
str r1, [r0]
ENTRY(cpu_sa110_flush_pmd)
ENTRY(cpu_sa1100_flush_pmd)
mcr p15, 0, r0, c7, c10, 1 @ clean D entry
mcr p15, 0, r0, c7, c10, 4 @ drain WB
mov pc, lr
......@@ -544,7 +542,7 @@ ENTRY(sa110_processor_functions)
/* pgtable */
.word cpu_sa110_set_pgd
.word cpu_sa110_set_pmd
.word cpu_sa110_flush_pmd
.word cpu_sa110_set_pte
.size sa110_processor_functions, . - sa110_processor_functions
......@@ -584,7 +582,7 @@ ENTRY(sa1100_processor_functions)
/* pgtable */
.word cpu_sa1100_set_pgd
.word cpu_sa1100_set_pmd
.word cpu_sa1100_flush_pmd
.word cpu_sa1100_set_pte
.size sa1100_processor_functions, . - sa1100_processor_functions
......
arch/arm/mm/proc-syms.c
View file @ 8d2c59fd
......@@ -27,7 +27,7 @@ EXPORT_SYMBOL(cpu_dcache_invalidate_range);
EXPORT_SYMBOL(cpu_icache_invalidate_range);
EXPORT_SYMBOL(cpu_icache_invalidate_page);
EXPORT_SYMBOL(cpu_set_pgd);
EXPORT_SYMBOL(cpu_set_pmd);
EXPORT_SYMBOL(cpu_flush_pmd);
EXPORT_SYMBOL(cpu_set_pte);
#else
EXPORT_SYMBOL(processor);
......
arch/arm/mm/proc-xscale.S
View file @ 8d2c59fd
......@@ -573,17 +573,15 @@ ENTRY(cpu_xscale_set_pgd)
cpwait_ret lr, ip
/*
* cpu_xscale_set_pmd(pmdp, pmd)
* cpu_xscale_flush_pmd(pmdp)
*
* Set a level 1 translation table entry, and clean it out of
* any caches such that the MMUs can load it correctly.
*
* pmdp: pointer to PMD entry
* pmd: PMD value to store
*/
.align 5
ENTRY(cpu_xscale_set_pmd)
str r1, [r0]
ENTRY(cpu_xscale_flush_pmd)
mov ip, #0
mcr p15, 0, r0, c7, c10, 1 @ Clean D cache line
mcr p15, 0, ip, c7, c10, 4 @ Drain Write (& Fill) Buffer
......@@ -721,7 +719,7 @@ ENTRY(xscale_processor_functions)
/* pgtable */
.word cpu_xscale_set_pgd
.word cpu_xscale_set_pmd
.word cpu_xscale_flush_pmd
.word cpu_xscale_set_pte
.size xscale_processor_functions, . - xscale_processor_functions
......
include/asm-arm/proc-armv/pgalloc.h
View file @ 8d2c59fd
......@@ -96,7 +96,7 @@ static inline void
pmd_populate_kernel(struct mm_struct *mm, pmd_t *pmdp, pte_t *ptep)
{
unsigned long pte_ptr = (unsigned long)ptep;
pmd_t pmd;
unsigned long pmdval;
BUG_ON(mm != &init_mm);
......@@ -105,21 +105,21 @@ pmd_populate_kernel(struct mm_struct *mm, pmd_t *pmdp, pte_t *ptep)
* address of the PTE table
*/
pte_ptr -= PTRS_PER_PTE * sizeof(void *);
pmd_val(pmd) = __pa(pte_ptr) | _PAGE_KERNEL_TABLE;
set_pmd(pmdp, pmd);
pmd_val(pmd) += 256 * sizeof(pte_t);
set_pmd(pmdp + 1, pmd);
pmdval = __pa(pte_ptr) | _PAGE_KERNEL_TABLE;
pmdp[0] = __pmd(pmdval);
pmdp[1] = __pmd(pmdval + 256 * sizeof(pte_t));
cpu_flush_pmd(pmdp);
}
static inline void
pmd_populate(struct mm_struct *mm, pmd_t *pmdp, struct page *ptep)
{
pmd_t pmd;
unsigned long pmdval;
BUG_ON(mm == &init_mm);
pmd_val(pmd) = __pa(page_address(ptep)) | _PAGE_USER_TABLE;
set_pmd(pmdp, pmd);
pmd_val(pmd) += 256 * sizeof(pte_t);
set_pmd(pmdp + 1, pmd);
pmdval = page_to_pfn(ptep) << PAGE_SHIFT | _PAGE_USER_TABLE;
pmdp[0] = __pmd(pmdval);
pmdp[1] = __pmd(pmdval + 256 * sizeof(pte_t));
cpu_flush_pmd(pmdp);
}
include/asm-arm/proc-armv/pgtable.h
View file @ 8d2c59fd
......@@ -115,16 +115,17 @@
#include <asm/proc/domain.h>
#define _PAGE_USER_TABLE (PMD_TYPE_TABLE | PMD_DOMAIN(DOMAIN_USER))
#define _PAGE_KERNEL_TABLE (PMD_TYPE_TABLE | PMD_DOMAIN(DOMAIN_KERNEL))
#define _PAGE_USER_TABLE (PMD_TYPE_TABLE | PMD_BIT4 | PMD_DOMAIN(DOMAIN_USER))
#define _PAGE_KERNEL_TABLE (PMD_TYPE_TABLE | PMD_BIT4 | PMD_DOMAIN(DOMAIN_KERNEL))
#define pmd_bad(pmd) (pmd_val(pmd) & 2)
#define set_pmd(pmdp,pmd) cpu_set_pmd(pmdp, pmd)
#define set_pmd(pmdp,pmd) do { *pmdp = pmd; cpu_flush_pmd(pmdp); } while (0)
static inline void pmd_clear(pmd_t *pmdp)
{
set_pmd(pmdp, __pmd(0));
set_pmd(pmdp + 1, __pmd(0));
pmdp[0], __pmd(0));
pmdp[1], __pmd(0));
cpu_flush_pmd(pmdp);
}
static inline pte_t *pmd_page_kernel(pmd_t pmd)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7