Commit fecb0199 authored by Linus Torvalds's avatar Linus Torvalds

Merge bk://ppc.bkbits.net/for-linus-ppc

into penguin.transmeta.com:/home/penguin/torvalds/repositories/kernel/linux
parents a291da91 2fc01c94
......@@ -44,6 +44,7 @@ SECTIONS
{
*(.data)
*(.data1)
*(.data.boot)
*(.sdata)
*(.sdata2)
*(.got.plt) *(.got)
......
......@@ -59,7 +59,9 @@ char netroot_string[] = "root=/dev/nfs rw ip=auto";
/* Serial port to use. */
unsigned long com_port;
bd_t hold_resid_buf;
/* We need to make sure that this is before the images to ensure
* that it's in a mapped location. - Tom */
bd_t hold_resid_buf __attribute__ ((__section__ (".data.boot")));
bd_t *hold_residual = &hold_resid_buf;
extern unsigned long serial_init(int chan, bd_t *bp);
......
......@@ -258,8 +258,9 @@ endmenu
mainmenu_option next_comment
comment 'General setup'
bool 'Prompt for advanced kernel configuration options' CONFIG_ADVANCED_OPTIONS
bool 'High memory support' CONFIG_HIGHMEM
dep_bool ' Support for PTEs in high memory' CONFIG_HIGHPTE $CONFIG_HIGHMEM
bool 'Prompt for advanced kernel configuration options' CONFIG_ADVANCED_OPTIONS
if [ "$CONFIG_ADVANCED_OPTIONS" = "y" ]; then
if [ "$CONFIG_HIGHMEM" = "y" ]; then
bool " Set high memory pool address" CONFIG_HIGHMEM_START_BOOL
......
......@@ -288,15 +288,12 @@ ret_from_syscall_2:
*/
_GLOBAL(_switch)
stwu r1,-INT_FRAME_SIZE(r1)
stw r0,GPR0(r1)
lwz r0,0(r1)
stw r0,GPR1(r1)
mflr r0
stw r0,INT_FRAME_SIZE+4(r1)
/* r3-r13 are caller saved -- Cort */
SAVE_GPR(2, r1)
SAVE_8GPRS(14, r1)
SAVE_10GPRS(22, r1)
mflr r20 /* Return to switch caller */
stw r20,INT_FRAME_SIZE+4(r1)
stw r0,_NIP(r1) /* Return to switch caller */
mfmsr r22
li r0,MSR_FP /* Disable floating-point */
#ifdef CONFIG_ALTIVEC
......@@ -309,17 +306,9 @@ END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
andc r22,r22,r0
mtmsr r22
isync
1: stw r20,_NIP(r1)
stw r22,_MSR(r1)
stw r20,_LINK(r1)
1: stw r22,_MSR(r1)
mfcr r20
mfctr r22
mfspr r23,XER
stw r20,_CCR(r1)
stw r22,_CTR(r1)
stw r23,_XER(r1)
li r0,0x0ff0
stw r0,TRAP(r1)
stw r1,KSP(r3) /* Set old stack pointer */
tophys(r0,r4)
......
......@@ -839,7 +839,7 @@ hash_page:
/*
* We hard enable here (but first soft disable) so that the hash_page
* code can spin on the hash_table_lock without problem on a shared
* code can spin on the mmu_hash_lock without problem on a shared
* processor
*/
li r0,0
......
......@@ -29,7 +29,7 @@
.align 5
#ifdef CONFIG_SMP
.comm hash_table_lock,4
.comm mmu_hash_lock,4
#endif /* CONFIG_SMP */
_GLOBAL(is_msr_enabled)
......
......@@ -47,17 +47,10 @@ extern unsigned long yield_count;
#define run_light_on(x) do { } while (0)
#endif /* CONFIG_PPC_ISERIES */
void zero_paged(void);
void power_save(void);
unsigned long zero_paged_on = 0;
unsigned long powersave_nap = 0;
unsigned long *zero_cache; /* head linked list of pre-zero'd pages */
atomic_t zerototal; /* # pages zero'd over time */
atomic_t zeropage_hits; /* # zero'd pages request that we've done */
atomic_t zero_sz; /* # currently pre-zero'd pages */
atomic_t zeropage_calls; /* # zero'd pages request that've been made */
unsigned long zero_paged_on;
unsigned long powersave_nap;
int idled(void)
{
......@@ -92,7 +85,6 @@ int idled(void)
if (need_resched()) {
run_light_on(1);
schedule();
check_pgt_cache();
}
#ifdef CONFIG_PPC_ISERIES
else {
......@@ -115,141 +107,6 @@ int cpu_idle(void)
return 0;
}
#if 0
/*
* Returns a pre-zero'd page from the list otherwise returns
* NULL.
*/
unsigned long get_zero_page_fast(void)
{
unsigned long page = 0;
atomic_inc(&zero_cache_calls);
if ( zero_quicklist )
{
/* atomically remove this page from the list */
register unsigned long tmp;
asm ( "101:lwarx %1,0,%3\n" /* reserve zero_cache */
" lwz %0,0(%1)\n" /* get next -- new zero_cache */
PPC405_ERR77(0,%3)
" stwcx. %0,0,%3\n" /* update zero_cache */
" bne- 101b\n" /* if lost reservation try again */
: "=&r" (tmp), "=&r" (page), "+m" (zero_cache)
: "r" (&zero_quicklist)
: "cc" );
#ifdef CONFIG_SMP
/* if another cpu beat us above this can happen -- Cort */
if ( page == 0 )
return 0;
#endif /* CONFIG_SMP */
/* we can update zerocount after the fact since it is not
* used for anything but control of a loop which doesn't
* matter since it won't affect anything if it zeros one
* less page -- Cort
*/
atomic_inc((atomic_t *)&zero_cache_hits);
atomic_dec((atomic_t *)&zero_cache_sz);
/* zero out the pointer to next in the page */
*(unsigned long *)page = 0;
return page;
}
return 0;
}
/*
* Experimental stuff to zero out pages in the idle task
* to speed up get_free_pages(). Zero's out pages until
* we've reached the limit of zero'd pages. We handle
* reschedule()'s in here so when we return we know we've
* zero'd all we need to for now.
*/
int zero_cache_water[2] = { 25, 96 }; /* high and low water marks for zero cache */
void zero_paged(void)
{
unsigned long pageptr = 0; /* current page being zero'd */
unsigned long bytecount = 0;
register unsigned long tmp;
pte_t *pte;
if ( atomic_read(&zero_cache_sz) >= zero_cache_water[0] )
return;
while ( (atomic_read(&zero_cache_sz) < zero_cache_water[1]) && !need_resched() )
{
/*
* Mark a page as reserved so we can mess with it
* If we're interrupted we keep this page and our place in it
* since we validly hold it and it's reserved for us.
*/
pageptr = __get_free_pages(GFP_ATOMIC, 0);
if ( !pageptr )
return;
cond_resched();
/*
* Make the page no cache so we don't blow our cache with 0's
*/
pte = find_pte(&init_mm, pageptr);
if ( !pte )
{
printk("pte NULL in zero_paged()\n");
return;
}
pte_uncache(*pte);
flush_tlb_page(find_vma(&init_mm,pageptr),pageptr);
/*
* Important here to not take time away from real processes.
*/
for ( bytecount = 0; bytecount < PAGE_SIZE ; bytecount += 4 )
{
cond_resched();
*(unsigned long *)(bytecount + pageptr) = 0;
}
/*
* If we finished zero-ing out a page add this page to
* the zero_cache atomically -- we can't use
* down/up since we can't sleep in idle.
* Disabling interrupts is also a bad idea since we would
* steal time away from real processes.
* We can also have several zero_paged's running
* on different processors so we can't interfere with them.
* So we update the list atomically without locking it.
* -- Cort
*/
/* turn cache on for this page */
pte_cache(*pte);
flush_tlb_page(find_vma(&init_mm,pageptr),pageptr);
/* atomically add this page to the list */
asm ( "101:lwarx %0,0,%2\n" /* reserve zero_cache */
" stw %0,0(%3)\n" /* update *pageptr */
#ifdef CONFIG_SMP
" sync\n" /* let store settle */
#endif
PPC405_ERR77(0,%2)
" stwcx. %3,0,%2\n" /* update zero_cache in mem */
" bne- 101b\n" /* if lost reservation try again */
: "=&r" (tmp), "+m" (zero_quicklist)
: "r" (&zero_quicklist), "r" (pageptr)
: "cc" );
/*
* This variable is used in the above loop and nowhere
* else so the worst that could happen is we would
* zero out one more or one less page than we want
* per processor on the machine. This is because
* we could add our page to the list but not have
* zerocount updated yet when another processor
* reads it. -- Cort
*/
atomic_inc((atomic_t *)&zero_cache_sz);
atomic_inc((atomic_t *)&zero_cache_total);
}
}
#endif /* 0 */
void power_save(void)
{
unsigned long hid0;
......
......@@ -369,34 +369,39 @@ _GLOBAL(_tlbia)
isync
#else
#if defined(CONFIG_SMP)
rlwinm r8,r1,0,0,18
lwz r8,TI_CPU(r8)
oris r8,r8,10
mfmsr r10
SYNC
rlwinm r0,r10,0,17,15 /* clear bit 16 (MSR_EE) */
rlwinm r0,r0,0,28,26 /* clear DR */
mtmsr r0
SYNC
lis r9,hash_table_lock@h
ori r9,r9,hash_table_lock@l
rlwinm r8,r1,0,0,18
lwz r8,TI_CPU(r8)
oris r8,r8,10
SYNC_601
isync
lis r9,mmu_hash_lock@h
ori r9,r9,mmu_hash_lock@l
tophys(r9,r9)
10: lwarx r7,0,r9
cmpi 0,r7,0
bne- 10b
/* No 405 Erratum 77 fix needed here, because 4xx can't do SMP */
stwcx. r8,0,r9
bne- 10b
#endif /* CONFIG_SMP */
sync
tlbia
sync
#ifdef CONFIG_SMP
TLBSYNC
li r0,0
stw r0,0(r9) /* clear hash_table_lock */
stw r0,0(r9) /* clear mmu_hash_lock */
mtmsr r10
SYNC
#endif
#endif
SYNC_601
isync
#else /* CONFIG_SMP */
sync
tlbia
sync
#endif /* CONFIG_SMP */
#endif /* CONFIG_4xx */
blr
/*
......@@ -415,33 +420,37 @@ _GLOBAL(_tlbie)
10:
#else
#if defined(CONFIG_SMP)
rlwinm r8,r1,0,0,18
lwz r8,TI_CPU(r8)
oris r8,r8,11
mfmsr r10
SYNC
rlwinm r0,r10,0,17,15 /* clear bit 16 (MSR_EE) */
rlwinm r0,r0,0,28,26 /* clear DR */
mtmsr r0
SYNC
lis r9,hash_table_lock@h
ori r9,r9,hash_table_lock@l
rlwinm r8,r1,0,0,18
lwz r8,TI_CPU(r8)
oris r8,r8,11
SYNC_601
isync
lis r9,mmu_hash_lock@h
ori r9,r9,mmu_hash_lock@l
tophys(r9,r9)
10: lwarx r7,0,r9
cmpi 0,r7,0
bne- 10b
PPC405_ERR77(0,r9)
stwcx. r8,0,r9
bne- 10b
eieio
#endif /* CONFIG_SMP */
tlbie r3
sync
#ifdef CONFIG_SMP
TLBSYNC
li r0,0
stw r0,0(r9) /* clear hash_table_lock */
stw r0,0(r9) /* clear mmu_hash_lock */
mtmsr r10
SYNC
#endif
SYNC_601
isync
#else /* CONFIG_SMP */
tlbie r3
sync
#endif /* CONFIG_SMP */
#endif /* CONFIG_4xx */
blr
......@@ -630,6 +639,40 @@ _GLOBAL(__flush_dcache_icache)
isync
blr
/*
* Flush a particular page from the data cache to RAM, identified
* by its physical address. We turn off the MMU so we can just use
* the physical address (this may be a highmem page without a kernel
* mapping).
*
* void __flush_dcache_icache_phys(unsigned long physaddr)
*/
_GLOBAL(__flush_dcache_icache_phys)
mfspr r5,PVR
rlwinm r5,r5,16,16,31
cmpi 0,r5,1
beqlr /* for 601, do nothing */
mfmsr r10
rlwinm r0,r10,0,28,26 /* clear DR */
mtmsr r0
isync
rlwinm r3,r3,0,0,19 /* Get page base address */
li r4,4096/L1_CACHE_LINE_SIZE /* Number of lines in a page */
mtctr r4
mr r6,r3
0: dcbst 0,r3 /* Write line to ram */
addi r3,r3,L1_CACHE_LINE_SIZE
bdnz 0b
sync
mtctr r4
1: icbi 0,r6
addi r6,r6,L1_CACHE_LINE_SIZE
bdnz 1b
sync
mtmsr r10 /* restore DR */
isync
blr
/*
* Clear a page using the dcbz instruction, which doesn't cause any
* memory traffic (except to write out any cache lines which get
......
......@@ -218,13 +218,13 @@ EXPORT_SYMBOL(__global_sti);
EXPORT_SYMBOL(__global_save_flags);
EXPORT_SYMBOL(__global_restore_flags);
#ifdef SPINLOCK_DEBUG
EXPORT_SYMBOL(_spin_lock);
EXPORT_SYMBOL(_spin_unlock);
EXPORT_SYMBOL(spin_trylock);
EXPORT_SYMBOL(_read_lock);
EXPORT_SYMBOL(_read_unlock);
EXPORT_SYMBOL(_write_lock);
EXPORT_SYMBOL(_write_unlock);
EXPORT_SYMBOL(_raw_spin_lock);
EXPORT_SYMBOL(_raw_spin_unlock);
EXPORT_SYMBOL(_raw_spin_trylock);
EXPORT_SYMBOL(_raw_read_lock);
EXPORT_SYMBOL(_raw_read_unlock);
EXPORT_SYMBOL(_raw_write_lock);
EXPORT_SYMBOL(_raw_write_unlock);
#endif
EXPORT_SYMBOL(smp_call_function);
EXPORT_SYMBOL(smp_hw_index);
......@@ -361,7 +361,7 @@ EXPORT_SYMBOL(set_context);
EXPORT_SYMBOL(handle_mm_fault); /* For MOL */
EXPORT_SYMBOL_NOVERS(disarm_decr);
#ifdef CONFIG_PPC_STD_MMU
EXPORT_SYMBOL(flush_hash_page); /* For MOL */
EXPORT_SYMBOL(flush_hash_pages); /* For MOL */
extern long *intercept_table;
EXPORT_SYMBOL(intercept_table);
#endif
......
......@@ -291,26 +291,6 @@ void smp_call_function_interrupt(void)
atomic_inc(&call_data->finished);
}
/*
* Task migration callback.
*/
void smp_task_migration_interrupt(void *new_task)
{
task_t *p;
p = new_task;
sched_task_migrated(p);
}
/*
* This function sends a 'task migration' IPI to another CPU.
* Must be called from syscall contexts, with interrupts *enabled*.
*/
void smp_migrate_task(int cpu, task_t *p)
{
__smp_call_function(smp_task_migration_interrupt, p, 0, cpu);
}
void __init smp_boot_cpus(void)
{
int i, cpu_nr;
......
......@@ -28,7 +28,7 @@
* since they may inhibit forward progress by other CPUs in getting
* a lock.
*/
static unsigned long __spin_trylock(volatile unsigned long *lock)
unsigned long __spin_trylock(volatile unsigned long *lock)
{
unsigned long ret;
......
......@@ -50,8 +50,6 @@
#include <asm/smp.h>
#include <asm/machdep.h>
extern int get_pteptr(struct mm_struct *mm, unsigned long addr, pte_t **ptep);
/* This function will allocate the requested contiguous pages and
* map them into the kernel's vmalloc() space. This is done so we
* get unique mapping for these pages, outside of the kernel's 1:1
......@@ -157,6 +155,6 @@ size_t size, int direction)
{
unsigned long start;
start = (unsigned long)(page->virtual) + offset;
start = page_address(page) + offset;
consistent_sync(start, size, direction);
}
......@@ -154,6 +154,7 @@ void do_page_fault(struct pt_regs *regs, unsigned long address,
/* Since 4xx supports per-page execute permission,
* we lazily flush dcache to icache. */
ptep = NULL;
if (get_pteptr(mm, address, &ptep) && pte_present(*ptep)) {
struct page *page = pte_page(*ptep);
......@@ -164,9 +165,12 @@ void do_page_fault(struct pt_regs *regs, unsigned long address,
}
pte_update(ptep, 0, _PAGE_HWEXEC);
_tlbie(address);
pte_unmap(ptep);
up_read(&mm->mmap_sem);
return;
}
if (ptep != NULL)
pte_unmap(ptep);
#endif
/* a read */
} else {
......@@ -289,27 +293,18 @@ pte_t *va_to_pte(unsigned long address)
struct mm_struct *mm;
if (address < TASK_SIZE)
mm = current->mm;
else
mm = &init_mm;
return NULL;
dir = pgd_offset(mm, address & PAGE_MASK);
dir = pgd_offset(&init_mm, address);
if (dir) {
pmd = pmd_offset(dir, address & PAGE_MASK);
if (pmd && pmd_present(*pmd)) {
pte = pte_offset(pmd, address & PAGE_MASK);
if (pte && pte_present(*pte)) {
pte = pte_offset_kernel(pmd, address & PAGE_MASK);
if (pte && pte_present(*pte))
return(pte);
}
}
else {
return (0);
}
}
else {
return (0);
}
return (0);
return NULL;
}
unsigned long va_to_phys(unsigned long address)
......@@ -334,7 +329,7 @@ print_8xx_pte(struct mm_struct *mm, unsigned long addr)
if (pgd) {
pmd = pmd_offset(pgd, addr & PAGE_MASK);
if (pmd && pmd_present(*pmd)) {
pte = pte_offset(pmd, addr & PAGE_MASK);
pte = pte_offset_kernel(pmd, addr & PAGE_MASK);
if (pte) {
printk(" (0x%08lx)->(0x%08lx)->0x%08lx\n",
(long)pgd, (long)pte, (long)pte_val(*pte));
......@@ -375,7 +370,7 @@ get_8xx_pte(struct mm_struct *mm, unsigned long addr)
if (pgd) {
pmd = pmd_offset(pgd, addr & PAGE_MASK);
if (pmd && pmd_present(*pmd)) {
pte = pte_offset(pmd, addr & PAGE_MASK);
pte = pte_offset_kernel(pmd, addr & PAGE_MASK);
if (pte) {
retval = (int)pte_val(*pte);
}
......
......@@ -36,7 +36,7 @@
#include <kernel/ppc_defs.h>
#ifdef CONFIG_SMP
.comm hash_table_lock,4
.comm mmu_hash_lock,4
#endif /* CONFIG_SMP */
/*
......@@ -62,8 +62,8 @@ hash_page:
#endif
tophys(r7,0) /* gets -KERNELBASE into r7 */
#ifdef CONFIG_SMP
addis r2,r7,hash_table_lock@h
ori r2,r2,hash_table_lock@l
addis r2,r7,mmu_hash_lock@h
ori r2,r2,mmu_hash_lock@l
lis r0,0x0fff
b 10f
11: lwz r6,0(r2)
......@@ -88,8 +88,8 @@ hash_page:
rlwimi r3,r23,32-12,29,29 /* MSR_PR -> _PAGE_USER */
112: add r5,r5,r7 /* convert to phys addr */
rlwimi r5,r4,12,20,29 /* insert top 10 bits of address */
lwz r5,0(r5) /* get pmd entry */
rlwinm. r5,r5,0,0,19 /* extract address of pte page */
lwz r2,0(r5) /* get pmd entry */
rlwinm. r2,r2,0,0,19 /* extract address of pte page */
#ifdef CONFIG_SMP
beq- hash_page_out /* return if no mapping */
#else
......@@ -99,7 +99,6 @@ hash_page:
to the address following the rfi. */
beqlr-
#endif
add r2,r5,r7 /* convert to phys addr */
rlwimi r2,r4,22,20,29 /* insert next 10 bits of address */
rlwinm r0,r3,32-3,24,24 /* _PAGE_RW access -> _PAGE_DIRTY */
ori r0,r0,_PAGE_ACCESSED|_PAGE_HASHPTE
......@@ -142,9 +141,9 @@ retry:
#ifdef CONFIG_SMP
eieio
addis r2,r7,hash_table_lock@ha
addis r2,r7,mmu_hash_lock@ha
li r0,0
stw r0,hash_table_lock@l(r2)
stw r0,mmu_hash_lock@l(r2)
#endif
/* Return from the exception */
......@@ -174,16 +173,16 @@ retry:
#ifdef CONFIG_SMP
hash_page_out:
eieio
addis r2,r7,hash_table_lock@ha
addis r2,r7,mmu_hash_lock@ha
li r0,0
stw r0,hash_table_lock@l(r2)
stw r0,mmu_hash_lock@l(r2)
blr
#endif /* CONFIG_SMP */
/*
* Add an entry for a particular page to the hash table.
*
* add_hash_page(unsigned context, unsigned long va, pte_t pte)
* add_hash_page(unsigned context, unsigned long va, unsigned long pmdval)
*
* We assume any necessary modifications to the pte (e.g. setting
* the accessed bit) have already been done and that there is actually
......@@ -199,31 +198,41 @@ _GLOBAL(add_hash_page)
mulli r0,r0,0x111 /* multiply by ESID skew */
add r3,r3,r0 /* note create_hpte trims to 24 bits */
#ifdef CONFIG_SMP
rlwinm r8,r1,0,0,18 /* use cpu number to make tag */
lwz r8,TI_CPU(r8) /* to go in mmu_hash_lock */
oris r8,r8,12
#endif /* CONFIG_SMP */
/*
* We disable interrupts here, even on UP, because we don't
* want to race with hash_page, and because we want the
* _PAGE_HASHPTE bit to be a reliable indication of whether
* the HPTE exists (or at least whether one did once). -- paulus
* the HPTE exists (or at least whether one did once).
* We also turn off the MMU for data accesses so that we
* we can't take a hash table miss (assuming the code is
* covered by a BAT). -- paulus
*/
mfmsr r10
SYNC
rlwinm r0,r10,0,17,15 /* clear bit 16 (MSR_EE) */
rlwinm r0,r0,0,28,26 /* clear MSR_DR */
mtmsr r0
SYNC
SYNC_601
isync
tophys(r7,0)
#ifdef CONFIG_SMP
lis r9,hash_table_lock@h
ori r9,r9,hash_table_lock@l
rlwinm r8,r1,0,0,18
lwz r8,TI_CPU(r8)
oris r8,r8,12
10: lwarx r7,0,r9
cmpi 0,r7,0
addis r9,r7,mmu_hash_lock@ha
addi r9,r9,mmu_hash_lock@l
10: lwarx r0,0,r9 /* take the mmu_hash_lock */
cmpi 0,r0,0
bne- 11f
stwcx. r8,0,r9
beq+ 12f
11: lwz r7,0(r9)
cmpi 0,r7,0
11: lwz r0,0(r9)
cmpi 0,r0,0
beq 10b
b 11b
12: isync
......@@ -234,18 +243,18 @@ _GLOBAL(add_hash_page)
* If _PAGE_HASHPTE was already set, we don't replace the existing
* HPTE, so we just unlock and return.
*/
mr r7,r5
1: lwarx r6,0,r7
mr r8,r5
rlwimi r8,r4,22,20,29
1: lwarx r6,0,r8
andi. r0,r6,_PAGE_HASHPTE
bne 9f /* if HASHPTE already set, done */
ori r5,r6,_PAGE_ACCESSED|_PAGE_HASHPTE
stwcx. r5,0,r7
ori r5,r6,_PAGE_HASHPTE
stwcx. r5,0,r8
bne- 1b
li r7,0 /* no address offset needed */
bl create_hpte
lis r8,htab_preloads@ha
addis r8,r7,htab_preloads@ha
lwz r3,htab_preloads@l(r8)
addi r3,r3,1
stw r3,htab_preloads@l(r8)
......@@ -254,15 +263,16 @@ _GLOBAL(add_hash_page)
#ifdef CONFIG_SMP
eieio
li r0,0
stw r0,0(r9) /* clear hash_table_lock */
stw r0,0(r9) /* clear mmu_hash_lock */
#endif
/* reenable interrupts and DR */
mtmsr r10
SYNC_601
isync
lwz r0,4(r1)
mtlr r0
/* reenable interrupts */
mtmsr r10
SYNC
blr
/*
......@@ -273,7 +283,7 @@ _GLOBAL(add_hash_page)
* linux PTE (before setting _PAGE_HASHPTE) and r7 contains the
* offset to be added to addresses (0 if the MMU is on,
* -KERNELBASE if it is off).
* On SMP, the caller should have the hash_table_lock held.
* On SMP, the caller should have the mmu_hash_lock held.
* We assume that the caller has (or will) set the _PAGE_HASHPTE
* bit in the linux PTE in memory. The value passed in r6 should
* be the old linux PTE value; if it doesn't have _PAGE_HASHPTE set
......@@ -486,41 +496,73 @@ found_slot:
/*
* Flush the entry for a particular page from the hash table.
*
* flush_hash_page(unsigned context, unsigned long va, pte_t *ptep)
* flush_hash_pages(unsigned context, unsigned long va, unsigned long pmdval,
* int count)
*
* We assume that there is a hash table in use (Hash != 0).
*/
_GLOBAL(flush_hash_page)
/* Convert context and va to VSID */
mulli r3,r3,897*16 /* multiply context by context skew */
rlwinm r0,r4,4,28,31 /* get ESID (top 4 bits of va) */
mulli r0,r0,0x111 /* multiply by ESID skew */
add r3,r3,r0 /* note code below trims to 24 bits */
_GLOBAL(flush_hash_pages)
tophys(r7,0)
/*
* We disable interrupts here, even on UP, because we want
* the _PAGE_HASHPTE bit to be a reliable indication of
* whether the HPTE exists. -- paulus
* whether the HPTE exists (or at least whether one did once).
* We also turn off the MMU for data accesses so that we
* we can't take a hash table miss (assuming the code is
* covered by a BAT). -- paulus
*/
mfmsr r10
rlwinm r0,r10,0,17,15 /* clear bit 16 (MSR_EE) */
SYNC
rlwinm r0,r10,0,17,15 /* clear bit 16 (MSR_EE) */
rlwinm r0,r0,0,28,26 /* clear MSR_DR */
mtmsr r0
SYNC
SYNC_601
isync
/* First find a PTE in the range that has _PAGE_HASHPTE set */
rlwimi r5,r4,22,20,29
1: lwz r0,0(r5)
cmpwi cr1,r6,1
andi. r0,r0,_PAGE_HASHPTE
bne 2f
ble cr1,19f
addi r4,r4,0x1000
addi r5,r5,4
addi r6,r6,-1
b 1b
/* Convert context and va to VSID */
2: mulli r3,r3,897*16 /* multiply context by context skew */
rlwinm r0,r4,4,28,31 /* get ESID (top 4 bits of va) */
mulli r0,r0,0x111 /* multiply by ESID skew */
add r3,r3,r0 /* note code below trims to 24 bits */
/* Construct the high word of the PPC-style PTE (r11) */
#ifndef CONFIG_PPC64BRIDGE
rlwinm r11,r3,7,1,24 /* put VSID in 0x7fffff80 bits */
rlwimi r11,r4,10,26,31 /* put in API (abbrev page index) */
#else /* CONFIG_PPC64BRIDGE */
clrlwi r3,r3,8 /* reduce vsid to 24 bits */
sldi r11,r3,12 /* shift vsid into position */
rlwimi r11,r4,16,20,24 /* put in API (abbrev page index) */
#endif /* CONFIG_PPC64BRIDGE */
SET_V(r11) /* set V (valid) bit */
#ifdef CONFIG_SMP
lis r9,hash_table_lock@h
ori r9,r9,hash_table_lock@l
addis r9,r7,mmu_hash_lock@ha
addi r9,r9,mmu_hash_lock@l
rlwinm r8,r1,0,0,18
add r8,r8,r7
lwz r8,TI_CPU(r8)
oris r8,r8,9
10: lwarx r7,0,r9
cmpi 0,r7,0
10: lwarx r0,0,r9
cmpi 0,r0,0
bne- 11f
stwcx. r8,0,r9
beq+ 12f
11: lwz r7,0(r9)
cmpi 0,r7,0
11: lwz r0,0(r9)
cmpi 0,r0,0
beq 10b
b 11b
12: isync
......@@ -528,69 +570,72 @@ _GLOBAL(flush_hash_page)
/*
* Check the _PAGE_HASHPTE bit in the linux PTE. If it is
* already clear, we're done. If not, clear it (atomically)
* and proceed. -- paulus.
* already clear, we're done (for this pte). If not,
* clear it (atomically) and proceed. -- paulus.
*/
1: lwarx r6,0,r5 /* fetch the pte */
andi. r0,r6,_PAGE_HASHPTE
beq 9f /* done if HASHPTE is already clear */
rlwinm r6,r6,0,31,29 /* clear HASHPTE bit */
stwcx. r6,0,r5 /* update the pte */
bne- 1b
/* Construct the high word of the PPC-style PTE (r5) */
#ifndef CONFIG_PPC64BRIDGE
rlwinm r5,r3,7,1,24 /* put VSID in 0x7fffff80 bits */
rlwimi r5,r4,10,26,31 /* put in API (abbrev page index) */
#else /* CONFIG_PPC64BRIDGE */
clrlwi r3,r3,8 /* reduce vsid to 24 bits */
sldi r5,r3,12 /* shift vsid into position */
rlwimi r5,r4,16,20,24 /* put in API (abbrev page index) */
#endif /* CONFIG_PPC64BRIDGE */
SET_V(r5) /* set V (valid) bit */
33: lwarx r8,0,r5 /* fetch the pte */
andi. r0,r8,_PAGE_HASHPTE
beq 8f /* done if HASHPTE is already clear */
rlwinm r8,r8,0,31,29 /* clear HASHPTE bit */
stwcx. r8,0,r5 /* update the pte */
bne- 33b
/* Get the address of the primary PTE group in the hash table (r3) */
.globl flush_hash_patch_A
flush_hash_patch_A:
lis r8,Hash_base@h /* base address of hash table */
addis r8,r7,Hash_base@h /* base address of hash table */
rlwimi r8,r3,LG_PTEG_SIZE,HASH_LEFT,HASH_RIGHT /* VSID -> hash */
rlwinm r3,r4,20+LG_PTEG_SIZE,HASH_LEFT,HASH_RIGHT /* PI -> hash */
xor r3,r3,r8 /* make primary hash */
li r8,8 /* PTEs/group */
rlwinm r0,r4,20+LG_PTEG_SIZE,HASH_LEFT,HASH_RIGHT /* PI -> hash */
xor r8,r0,r8 /* make primary hash */
/* Search the primary PTEG for a PTE whose 1st (d)word matches r5 */
mtctr r8
addi r7,r3,-PTE_SIZE
1: LDPTEu r0,PTE_SIZE(r7) /* get next PTE */
CMPPTE 0,r0,r5
li r0,8 /* PTEs/group */
mtctr r0
addi r12,r8,-PTE_SIZE
1: LDPTEu r0,PTE_SIZE(r12) /* get next PTE */
CMPPTE 0,r0,r11
bdnzf 2,1b /* loop while ctr != 0 && !cr0.eq */
beq+ 3f
/* Search the secondary PTEG for a matching PTE */
ori r5,r5,PTE_H /* set H (secondary hash) bit */
ori r11,r11,PTE_H /* set H (secondary hash) bit */
li r0,8 /* PTEs/group */
.globl flush_hash_patch_B
flush_hash_patch_B:
xoris r7,r3,Hash_msk>>16 /* compute secondary hash */
xori r7,r7,(-PTEG_SIZE & 0xffff)
addi r7,r7,-PTE_SIZE
mtctr r8
2: LDPTEu r0,PTE_SIZE(r7)
CMPPTE 0,r0,r5
xoris r12,r8,Hash_msk>>16 /* compute secondary hash */
xori r12,r12,(-PTEG_SIZE & 0xffff)
addi r12,r12,-PTE_SIZE
mtctr r0
2: LDPTEu r0,PTE_SIZE(r12)
CMPPTE 0,r0,r11
bdnzf 2,2b
bne- 4f /* should never fail to find it */
xori r11,r11,PTE_H /* clear H again */
bne- 4f /* should rarely fail to find it */
3: li r0,0
STPTE r0,0(r7) /* invalidate entry */
STPTE r0,0(r12) /* invalidate entry */
4: sync
tlbie r4 /* in hw tlb too */
sync
8: ble cr1,9f /* if all ptes checked */
81: addi r6,r6,-1
addi r5,r5,4 /* advance to next pte */
addi r4,r4,0x1000
lwz r0,0(r5) /* check next pte */
cmpwi cr1,r6,1
andi. r0,r0,_PAGE_HASHPTE
bne 33b
bgt cr1,81b
9:
#ifdef CONFIG_SMP
TLBSYNC
9: li r0,0
stw r0,0(r9) /* clear hash_table_lock */
li r0,0
stw r0,0(r9) /* clear mmu_hash_lock */
#endif
9: mtmsr r10
SYNC
19: mtmsr r10
SYNC_601
isync
blr
......@@ -39,7 +39,7 @@
int iSeries_hpt_loaded;
static spinlock_t hash_table_lock = SPIN_LOCK_UNLOCKED;
static spinlock_t mmu_hash_lock = SPIN_LOCK_UNLOCKED;
extern unsigned long htab_reloads; // Defined in ppc/kernel/ppc_htab.c
extern unsigned long htab_evicts;
......@@ -159,10 +159,10 @@ int iSeries_create_hpte( unsigned long access, unsigned long va )
access |= _PAGE_PRESENT; // _PAGE_PRESENT also needed
spin_lock( &hash_table_lock );
spin_lock( &mmu_hash_lock );
// check if pte is in the required state
if ( ( access & ~(pte_val(*pt)) ) ) {
spin_unlock( &hash_table_lock );
spin_unlock( &mmu_hash_lock );
return 1;
}
......@@ -177,18 +177,18 @@ int iSeries_create_hpte( unsigned long access, unsigned long va )
va,
pte_val(*pt));
spin_unlock( &hash_table_lock );
spin_unlock( &mmu_hash_lock );
return 0;
}
void add_hash_page(unsigned context, unsigned long va, pte_t *ptep)
{
spin_lock( &hash_table_lock );
spin_lock( &mmu_hash_lock );
pte_update(ptep,0,_PAGE_HASHPTE);
__create_hpte(CTX_TO_VSID(context, va),
va,
pte_val(*ptep));
spin_unlock( &hash_table_lock );
spin_unlock( &mmu_hash_lock );
}
int flush_hash_page(unsigned context, unsigned long va, pte_t *ptep)
......@@ -208,7 +208,7 @@ int flush_hash_page(unsigned context, unsigned long va, pte_t *ptep)
hpte1Ptr = hpte0Ptr + 1;
*hpte0Ptr = *hpte1Ptr = 0;
spin_lock( &hash_table_lock );
spin_lock( &mmu_hash_lock );
rtnIndex = HvCallHpt_findValid( &hpte, vpn );
if ( hpte.v ) {
......@@ -217,7 +217,7 @@ int flush_hash_page(unsigned context, unsigned long va, pte_t *ptep)
rc = 0;
} else
rc = 1;
spin_unlock( &hash_table_lock );
spin_unlock( &mmu_hash_lock );
return rc;
}
......@@ -175,12 +175,13 @@ void update_mmu_cache(struct vm_area_struct *vma, unsigned long address,
pte_t *ptep;
static int nopreload;
if (nopreload)
if (nopreload || address >= TASK_SIZE)
return;
mm = (address < TASK_SIZE)? vma->vm_mm: &init_mm;
mm = vma->vm_mm;
pmd = pmd_offset(pgd_offset(mm, address), address);
if (!pmd_none(*pmd)) {
ptep = pte_offset(pmd, address);
ptep = pte_offset_map(pmd, address);
add_hash_page(mm->context, address, ptep);
pte_unmap(ptep);
}
}
......@@ -113,24 +113,6 @@ unsigned long __max_memory;
/* max amount of low RAM to map in */
unsigned long __max_low_memory = MAX_LOW_MEM;
int do_check_pgt_cache(int low, int high)
{
int freed = 0;
if (pgtable_cache_size > high) {
do {
if (pgd_quicklist) {
free_pgd_slow(get_pgd_fast());
freed++;
}
if (pte_quicklist) {
pte_free_slow(pte_alloc_one_fast(NULL, 0));
freed++;
}
} while (pgtable_cache_size > low);
}
return freed;
}
void show_mem(void)
{
int i,free = 0,total = 0,reserved = 0;
......@@ -160,7 +142,6 @@ void show_mem(void)
printk("%d reserved pages\n",reserved);
printk("%d pages shared\n",shared);
printk("%d pages swap cached\n",cached);
printk("%d pages in page table cache\n",(int)pgtable_cache_size);
show_buffers();
}
......@@ -396,9 +377,11 @@ void __init paging_init(void)
#ifdef CONFIG_HIGHMEM
map_page(PKMAP_BASE, 0, 0); /* XXX gross */
pkmap_page_table = pte_offset(pmd_offset(pgd_offset_k(PKMAP_BASE), PKMAP_BASE), PKMAP_BASE);
pkmap_page_table = pte_offset_kernel(pmd_offset(pgd_offset_k
(PKMAP_BASE), PKMAP_BASE), PKMAP_BASE);
map_page(KMAP_FIX_BEGIN, 0, 0); /* XXX gross */
kmap_pte = pte_offset(pmd_offset(pgd_offset_k(KMAP_FIX_BEGIN), KMAP_FIX_BEGIN), KMAP_FIX_BEGIN);
kmap_pte = pte_offset_kernel(pmd_offset(pgd_offset_k
(KMAP_FIX_BEGIN), KMAP_FIX_BEGIN), KMAP_FIX_BEGIN);
kmap_prot = PAGE_KERNEL;
#endif /* CONFIG_HIGHMEM */
......@@ -588,10 +571,12 @@ void flush_dcache_page(struct page *page)
void flush_icache_page(struct vm_area_struct *vma, struct page *page)
{
unsigned long phys;
if (page->mapping && !PageReserved(page)
&& !test_bit(PG_arch_1, &page->flags)) {
__flush_dcache_icache(kmap(page));
kunmap(page);
phys = ((page - mem_map) << PAGE_SHIFT) + PPC_MEMSTART;
__flush_dcache_icache_phys(phys);
set_bit(PG_arch_1, &page->flags);
}
}
......
......@@ -61,11 +61,12 @@ extern void MMU_init_hw(void);
* which includes all new 82xx processors. We need tlbie/tlbsync here
* in that case (I think). -- Dan.
*/
static inline void flush_HPTE(unsigned context, unsigned long va, pte_t *pg)
static inline void flush_HPTE(unsigned context, unsigned long va,
unsigned long pdval)
{
if ((Hash != 0) &&
(cur_cpu_spec[0]->cpu_features & CPU_FTR_HPTE_TABLE))
flush_hash_page(0, va, pg);
flush_hash_pages(0, va, pdval, 1);
else
_tlbie(va);
}
......
......@@ -39,10 +39,6 @@ unsigned long ioremap_base;
unsigned long ioremap_bot;
int io_bat_index;
#ifndef CONFIG_SMP
struct pgtable_cache_struct quicklists;
#endif
#if defined(CONFIG_6xx) || defined(CONFIG_POWER3)
#define HAVE_BATS 1
#endif
......@@ -173,12 +169,12 @@ map_page(unsigned long va, unsigned long pa, int flags)
/* Use upper 10 bits of VA to index the first level map */
pd = pmd_offset(pgd_offset_k(va), va);
/* Use middle 10 bits of VA to index the second-level map */
pg = pte_alloc(&init_mm, pd, va);
pg = pte_alloc_kernel(&init_mm, pd, va);
if (pg != 0) {
err = 0;
set_pte(pg, mk_pte_phys(pa & PAGE_MASK, __pgprot(flags)));
if (mem_init_done)
flush_HPTE(0, va, pg);
flush_HPTE(0, va, pmd_val(*pd));
}
spin_unlock(&init_mm.page_table_lock);
return err;
......@@ -272,10 +268,11 @@ get_pteptr(struct mm_struct *mm, unsigned long addr, pte_t **ptep)
if (pgd) {
pmd = pmd_offset(pgd, addr & PAGE_MASK);
if (pmd_present(*pmd)) {
pte = pte_offset(pmd, addr & PAGE_MASK);
pte = pte_offset_map(pmd, addr & PAGE_MASK);
if (pte) {
retval = 1;
*ptep = pte;
/* XXX caller needs to do pte_unmap, yuck */
}
}
}
......@@ -312,8 +309,10 @@ unsigned long iopa(unsigned long addr)
mm = &init_mm;
pa = 0;
if (get_pteptr(mm, addr, &pte))
if (get_pteptr(mm, addr, &pte)) {
pa = (pte_val(*pte) & PAGE_MASK) | (addr & ~PAGE_MASK);
pte_unmap(pte);
}
return(pa);
}
......
......@@ -33,6 +33,7 @@
#include <linux/kernel.h>
#include <linux/mm.h>
#include <linux/init.h>
#include <linux/highmem.h>
#include <asm/prom.h>
#include <asm/mmu.h>
......@@ -289,7 +290,6 @@ void update_mmu_cache(struct vm_area_struct *vma, unsigned long address,
{
struct mm_struct *mm;
pmd_t *pmd;
pte_t *ptep;
static int nopreload;
if (Hash == 0 || nopreload)
......@@ -299,8 +299,6 @@ void update_mmu_cache(struct vm_area_struct *vma, unsigned long address,
return;
mm = (address < TASK_SIZE)? vma->vm_mm: &init_mm;
pmd = pmd_offset(pgd_offset(mm, address), address);
if (!pmd_none(*pmd)) {
ptep = pte_offset(pmd, address);
add_hash_page(mm->context, address, ptep);
}
if (!pmd_none(*pmd))
add_hash_page(mm->context, address, pmd_val(*pmd));
}
......@@ -30,6 +30,7 @@
#include <linux/kernel.h>
#include <linux/mm.h>
#include <linux/init.h>
#include <linux/highmem.h>
#include "mmu_decl.h"
......@@ -104,7 +105,6 @@ local_flush_tlb_page(struct vm_area_struct *vma, unsigned long vmaddr)
{
struct mm_struct *mm;
pmd_t *pmd;
pte_t *pte;
if (Hash == 0) {
_tlbie(vmaddr);
......@@ -112,11 +112,8 @@ local_flush_tlb_page(struct vm_area_struct *vma, unsigned long vmaddr)
}
mm = (vmaddr < TASK_SIZE)? vma->vm_mm: &init_mm;
pmd = pmd_offset(pgd_offset(mm, vmaddr), vmaddr);
if (!pmd_none(*pmd)) {
pte = pte_offset(pmd, vmaddr);
if (pte_val(*pte) & _PAGE_HASHPTE)
flush_hash_page(mm->context, vmaddr, pte);
}
if (!pmd_none(*pmd))
flush_hash_pages(mm->context, vmaddr, pmd_val(*pmd), 1);
#ifdef CONFIG_SMP
smp_send_tlb_invalidate(0);
#endif
......@@ -133,8 +130,8 @@ local_flush_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned
{
struct mm_struct *mm = vma->vm_mm;
pmd_t *pmd;
pte_t *pte;
unsigned long pmd_end;
int count;
unsigned int ctx = mm->context;
if (Hash == 0) {
......@@ -144,24 +141,21 @@ local_flush_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned
start &= PAGE_MASK;
if (start >= end)
return;
end = (end - 1) | ~PAGE_MASK;
pmd = pmd_offset(pgd_offset(mm, start), start);
do {
pmd_end = (start + PGDIR_SIZE) & PGDIR_MASK;
if (!pmd_none(*pmd)) {
if (!pmd_end || pmd_end > end)
for (;;) {
pmd_end = ((start + PGDIR_SIZE) & PGDIR_MASK) - 1;
if (pmd_end > end)
pmd_end = end;
pte = pte_offset(pmd, start);
do {
if ((pte_val(*pte) & _PAGE_HASHPTE) != 0)
flush_hash_page(ctx, start, pte);
start += PAGE_SIZE;
++pte;
} while (start && start < pmd_end);
} else {
start = pmd_end;
if (!pmd_none(*pmd)) {
count = ((pmd_end - start) >> PAGE_SHIFT) + 1;
flush_hash_pages(ctx, start, pmd_val(*pmd), count);
}
if (pmd_end == end)
break;
start = pmd_end + 1;
++pmd;
} while (start && start < end);
}
#ifdef CONFIG_SMP
smp_send_tlb_invalidate(0);
......
......@@ -106,7 +106,7 @@ static inline void *kmap_atomic(struct page *page, enum km_type type)
static inline void kunmap_atomic(void *kvaddr, enum km_type type)
{
#if HIGHMEM_DEBUG
unsigned long vaddr = (unsigned long) kvaddr;
unsigned long vaddr = (unsigned long) kvaddr & PAGE_MASK;
unsigned int idx = type + KM_TYPE_NR*smp_processor_id();
if (vaddr < KMAP_FIX_BEGIN) // FIXME
......
/*
* BK Id: SCCS/s.kmap_types.h 1.9 08/29/01 14:03:05 paulus
* BK Id: %F% %I% %G% %U% %#%
*/
#ifdef __KERNEL__
#ifndef _ASM_KMAP_TYPES_H
......@@ -12,6 +12,8 @@ enum km_type {
KM_USER0,
KM_USER1,
KM_BIO_IRQ,
KM_PTE0,
KM_PTE1,
KM_TYPE_NR
};
......
/*
* BK Id: SCCS/s.pgalloc.h 1.9 05/17/01 18:14:25 cort
* BK Id: %F% %I% %G% %U% %#%
*/
#ifdef __KERNEL__
#ifndef _PPC_PGALLOC_H
......@@ -7,55 +7,12 @@
#include <linux/config.h>
#include <linux/threads.h>
#include <linux/highmem.h>
#include <asm/processor.h>
/*
* This is handled very differently on the PPC since out page tables
* are all 0's and I want to be able to use these zero'd pages elsewhere
* as well - it gives us quite a speedup.
*
* Note that the SMP/UP versions are the same but we don't need a
* per cpu list of zero pages because we do the zero-ing with the cache
* off and the access routines are lock-free but the pgt cache stuff
* is per-cpu since it isn't done with any lock-free access routines
* (although I think we need arch-specific routines so I can do lock-free).
*
* I need to generalize this so we can use it for other arch's as well.
* -- Cort
*/
#ifdef CONFIG_SMP
#define quicklists cpu_data[smp_processor_id()]
#else
extern struct pgtable_cache_struct {
unsigned long *pgd_cache;
unsigned long *pte_cache;
unsigned long pgtable_cache_sz;
} quicklists;
#endif
#define pgd_quicklist (quicklists.pgd_cache)
#define pmd_quicklist ((unsigned long *)0)
#define pte_quicklist (quicklists.pte_cache)
#define pgtable_cache_size (quicklists.pgtable_cache_sz)
extern unsigned long *zero_cache; /* head linked list of pre-zero'd pages */
extern atomic_t zero_sz; /* # currently pre-zero'd pages */
extern atomic_t zeropage_hits; /* # zero'd pages request that we've done */
extern atomic_t zeropage_calls; /* # zero'd pages request that've been made */
extern atomic_t zerototal; /* # pages zero'd over time */
#define zero_quicklist (zero_cache)
#define zero_cache_sz (zero_sz)
#define zero_cache_calls (zeropage_calls)
#define zero_cache_hits (zeropage_hits)
#define zero_cache_total (zerototal)
/* return a pre-zero'd page from the list, return NULL if none available -- Cort */
extern unsigned long get_zero_page_fast(void);
extern void __bad_pte(pmd_t *pmd);
extern __inline__ pgd_t *get_pgd_slow(void)
static inline pgd_t *pgd_alloc(struct mm_struct *mm)
{
pgd_t *ret;
......@@ -64,85 +21,75 @@ extern __inline__ pgd_t *get_pgd_slow(void)
return ret;
}
extern __inline__ pgd_t *get_pgd_fast(void)
{
unsigned long *ret;
if ((ret = pgd_quicklist) != NULL) {
pgd_quicklist = (unsigned long *)(*ret);
ret[0] = 0;
pgtable_cache_size--;
} else
ret = (unsigned long *)get_pgd_slow();
return (pgd_t *)ret;
}
extern __inline__ void free_pgd_fast(pgd_t *pgd)
{
*(unsigned long **)pgd = pgd_quicklist;
pgd_quicklist = (unsigned long *) pgd;
pgtable_cache_size++;
}
extern __inline__ void free_pgd_slow(pgd_t *pgd)
extern __inline__ void pgd_free(pgd_t *pgd)
{
free_page((unsigned long)pgd);
}
#define pgd_free(pgd) free_pgd_fast(pgd)
#define pgd_alloc(mm) get_pgd_fast()
/*
* We don't have any real pmd's, and this code never triggers because
* the pgd will always be present..
*/
#define pmd_alloc_one_fast(mm, address) ({ BUG(); ((pmd_t *)1); })
#define pmd_alloc_one(mm,address) ({ BUG(); ((pmd_t *)2); })
#define pmd_free(x) do { } while (0)
#define pgd_populate(mm, pmd, pte) BUG()
static inline pte_t *pte_alloc_one(struct mm_struct *mm, unsigned long address)
static inline pte_t *
pte_alloc_one_kernel(struct mm_struct *mm, unsigned long address)
{
pte_t *pte;
extern int mem_init_done;
extern void *early_get_page(void);
int timeout = 0;
if (mem_init_done)
pte = (pte_t *) __get_free_page(GFP_KERNEL);
else
if (mem_init_done) {
while ((pte = (pte_t *) __get_free_page(GFP_KERNEL)) == NULL
&& ++timeout < 10) {
set_current_state(TASK_UNINTERRUPTIBLE);
schedule_timeout(HZ);
}
} else
pte = (pte_t *) early_get_page();
if (pte != NULL)
clear_page(pte);
return pte;
}
static inline pte_t *pte_alloc_one_fast(struct mm_struct *mm, unsigned long address)
static inline struct page *
pte_alloc_one(struct mm_struct *mm, unsigned long address)
{
unsigned long *ret;
struct page *pte;
int timeout = 0;
#ifdef CONFIG_HIGHPTE
int flags = GFP_KERNEL | __GFP_HIGHMEM;
#else
int flags = GFP_KERNEL;
#endif
if ((ret = pte_quicklist) != NULL) {
pte_quicklist = (unsigned long *)(*ret);
ret[0] = 0;
pgtable_cache_size--;
while ((pte = alloc_pages(flags, 0)) == NULL) {
if (++timeout >= 10)
return NULL;
set_current_state(TASK_UNINTERRUPTIBLE);
schedule_timeout(HZ);
}
return (pte_t *)ret;
clear_highpage(pte);
return pte;
}
extern __inline__ void pte_free_fast(pte_t *pte)
static inline void pte_free_kernel(pte_t *pte)
{
*(unsigned long **)pte = pte_quicklist;
pte_quicklist = (unsigned long *) pte;
pgtable_cache_size++;
free_page((unsigned long)pte);
}
extern __inline__ void pte_free_slow(pte_t *pte)
static inline void pte_free(struct page *pte)
{
free_page((unsigned long)pte);
__free_page(pte);
}
#define pte_free(pte) pte_free_slow(pte)
#define pmd_populate(mm, pmd, pte) (pmd_val(*(pmd)) = (unsigned long) (pte))
#define pmd_populate_kernel(mm, pmd, pte) \
(pmd_val(*(pmd)) = __pa(pte))
#define pmd_populate(mm, pmd, pte) \
(pmd_val(*(pmd)) = ((pte) - mem_map) << PAGE_SHIFT)
extern int do_check_pgt_cache(int, int);
......
......@@ -13,6 +13,7 @@
#include <asm/processor.h> /* For TASK_SIZE */
#include <asm/mmu.h>
#include <asm/page.h>
#include <asm/kmap_types.h>
extern void _tlbie(unsigned long address);
extern void _tlbia(void);
......@@ -98,6 +99,7 @@ extern void flush_icache_user_range(struct vm_area_struct *vma,
struct page *page, unsigned long addr, int len);
extern void flush_icache_range(unsigned long, unsigned long);
extern void __flush_dcache_icache(void *page_va);
extern void __flush_dcache_icache_phys(unsigned long physaddr);
extern void flush_dcache_page(struct page *page);
extern void flush_icache_page(struct vm_area_struct *vma, struct page *page);
......@@ -274,7 +276,6 @@ extern unsigned long ioremap_bot, ioremap_base;
#define _PAGE_HWWRITE 0x0100 /* h/w write enable: never set in Linux PTE */
#define _PAGE_USER 0x0800 /* One of the PP bits, the other is USER&~RW */
#define _PMD_PRESENT 0x0001
#define _PMD_PAGE_MASK 0x000c
#define _PMD_PAGE_8M 0x000c
......@@ -385,8 +386,8 @@ extern unsigned long empty_zero_page[1024];
#define pte_clear(ptep) do { set_pte((ptep), __pte(0)); } while (0)
#define pmd_none(pmd) (!pmd_val(pmd))
#define pmd_bad(pmd) ((pmd_val(pmd) & ~PAGE_MASK) != 0)
#define pmd_present(pmd) ((pmd_val(pmd) & PAGE_MASK) != 0)
#define pmd_bad(pmd) (0)
#define pmd_present(pmd) (pmd_val(pmd) != 0)
#define pmd_clear(pmdp) do { pmd_val(*(pmdp)) = 0; } while (0)
#define pte_page(x) (mem_map+(unsigned long)((pte_val(x)-PPC_MEMSTART) >> PAGE_SHIFT))
......@@ -530,7 +531,10 @@ static inline void ptep_mkdirty(pte_t *ptep)
#define pte_same(A,B) (((pte_val(A) ^ pte_val(B)) & ~_PAGE_HASHPTE) == 0)
#define pmd_page(pmd) (pmd_val(pmd) & PAGE_MASK)
#define pmd_page_kernel(pmd) \
((unsigned long) __va(pmd_val(pmd) & PAGE_MASK))
#define pmd_page(pmd) \
(mem_map + (pmd_val(pmd) >> PAGE_SHIFT))
/* to find an entry in a kernel page-table-directory */
#define pgd_offset_k(address) pgd_offset(&init_mm, address)
......@@ -546,10 +550,17 @@ static inline pmd_t * pmd_offset(pgd_t * dir, unsigned long address)
}
/* Find an entry in the third-level page table.. */
static inline pte_t * pte_offset(pmd_t * dir, unsigned long address)
{
return (pte_t *) pmd_page(*dir) + ((address >> PAGE_SHIFT) & (PTRS_PER_PTE - 1));
}
#define __pte_offset(address) \
((address >> PAGE_SHIFT) & (PTRS_PER_PTE - 1))
#define pte_offset_kernel(dir, addr) \
((pte_t *) pmd_page_kernel(*(dir)) + __pte_offset(addr))
#define pte_offset_map(dir, addr) \
((pte_t *) kmap_atomic(pmd_page(*(dir)), KM_PTE0) + __pte_offset(addr))
#define pte_offset_map_nested(dir, addr) \
((pte_t *) kmap_atomic(pmd_page(*(dir)), KM_PTE1) + __pte_offset(addr))
#define pte_unmap(pte) kunmap_atomic(pte, KM_PTE0)
#define pte_unmap_nested(pte) kunmap_atomic(pte, KM_PTE1)
extern pgd_t swapper_pg_dir[1024];
extern void paging_init(void);
......@@ -558,10 +569,12 @@ extern void paging_init(void);
* When flushing the tlb entry for a page, we also need to flush the hash
* table entry. flush_hash_page is assembler (for speed) in hashtable.S.
*/
extern int flush_hash_page(unsigned context, unsigned long va, pte_t *ptep);
extern int flush_hash_pages(unsigned context, unsigned long va,
unsigned long pmdval, int count);
/* Add an HPTE to the hash table */
extern void add_hash_page(unsigned context, unsigned long va, pte_t *ptep);
extern void add_hash_page(unsigned context, unsigned long va,
unsigned long pmdval);
/*
* Encode and decode a swap entry.
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment