Commit fecb0199 authored by Linus Torvalds's avatar Linus Torvalds

Merge bk://ppc.bkbits.net/for-linus-ppc

into penguin.transmeta.com:/home/penguin/torvalds/repositories/kernel/linux
parents a291da91 2fc01c94
......@@ -44,6 +44,7 @@ SECTIONS
{
*(.data)
*(.data1)
*(.data.boot)
*(.sdata)
*(.sdata2)
*(.got.plt) *(.got)
......
......@@ -59,7 +59,9 @@ char netroot_string[] = "root=/dev/nfs rw ip=auto";
/* Serial port to use. */
unsigned long com_port;
bd_t hold_resid_buf;
/* We need to make sure that this is before the images to ensure
* that it's in a mapped location. - Tom */
bd_t hold_resid_buf __attribute__ ((__section__ (".data.boot")));
bd_t *hold_residual = &hold_resid_buf;
extern unsigned long serial_init(int chan, bd_t *bp);
......
......@@ -258,8 +258,9 @@ endmenu
mainmenu_option next_comment
comment 'General setup'
bool 'Prompt for advanced kernel configuration options' CONFIG_ADVANCED_OPTIONS
bool 'High memory support' CONFIG_HIGHMEM
dep_bool ' Support for PTEs in high memory' CONFIG_HIGHPTE $CONFIG_HIGHMEM
bool 'Prompt for advanced kernel configuration options' CONFIG_ADVANCED_OPTIONS
if [ "$CONFIG_ADVANCED_OPTIONS" = "y" ]; then
if [ "$CONFIG_HIGHMEM" = "y" ]; then
bool " Set high memory pool address" CONFIG_HIGHMEM_START_BOOL
......
......@@ -288,15 +288,12 @@ ret_from_syscall_2:
*/
_GLOBAL(_switch)
stwu r1,-INT_FRAME_SIZE(r1)
stw r0,GPR0(r1)
lwz r0,0(r1)
stw r0,GPR1(r1)
mflr r0
stw r0,INT_FRAME_SIZE+4(r1)
/* r3-r13 are caller saved -- Cort */
SAVE_GPR(2, r1)
SAVE_8GPRS(14, r1)
SAVE_10GPRS(22, r1)
mflr r20 /* Return to switch caller */
stw r20,INT_FRAME_SIZE+4(r1)
stw r0,_NIP(r1) /* Return to switch caller */
mfmsr r22
li r0,MSR_FP /* Disable floating-point */
#ifdef CONFIG_ALTIVEC
......@@ -309,17 +306,9 @@ END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
andc r22,r22,r0
mtmsr r22
isync
1: stw r20,_NIP(r1)
stw r22,_MSR(r1)
stw r20,_LINK(r1)
1: stw r22,_MSR(r1)
mfcr r20
mfctr r22
mfspr r23,XER
stw r20,_CCR(r1)
stw r22,_CTR(r1)
stw r23,_XER(r1)
li r0,0x0ff0
stw r0,TRAP(r1)
stw r1,KSP(r3) /* Set old stack pointer */
tophys(r0,r4)
......
......@@ -839,7 +839,7 @@ hash_page:
/*
* We hard enable here (but first soft disable) so that the hash_page
* code can spin on the hash_table_lock without problem on a shared
* code can spin on the mmu_hash_lock without problem on a shared
* processor
*/
li r0,0
......
......@@ -29,7 +29,7 @@
.align 5
#ifdef CONFIG_SMP
.comm hash_table_lock,4
.comm mmu_hash_lock,4
#endif /* CONFIG_SMP */
_GLOBAL(is_msr_enabled)
......
......@@ -47,17 +47,10 @@ extern unsigned long yield_count;
#define run_light_on(x) do { } while (0)
#endif /* CONFIG_PPC_ISERIES */
void zero_paged(void);
void power_save(void);
unsigned long zero_paged_on = 0;
unsigned long powersave_nap = 0;
unsigned long *zero_cache; /* head linked list of pre-zero'd pages */
atomic_t zerototal; /* # pages zero'd over time */
atomic_t zeropage_hits; /* # zero'd pages request that we've done */
atomic_t zero_sz; /* # currently pre-zero'd pages */
atomic_t zeropage_calls; /* # zero'd pages request that've been made */
unsigned long zero_paged_on;
unsigned long powersave_nap;
int idled(void)
{
......@@ -92,7 +85,6 @@ int idled(void)
if (need_resched()) {
run_light_on(1);
schedule();
check_pgt_cache();
}
#ifdef CONFIG_PPC_ISERIES
else {
......@@ -115,141 +107,6 @@ int cpu_idle(void)
return 0;
}
#if 0
/*
* Returns a pre-zero'd page from the list otherwise returns
* NULL.
*/
unsigned long get_zero_page_fast(void)
{
unsigned long page = 0;
atomic_inc(&zero_cache_calls);
if ( zero_quicklist )
{
/* atomically remove this page from the list */
register unsigned long tmp;
asm ( "101:lwarx %1,0,%3\n" /* reserve zero_cache */
" lwz %0,0(%1)\n" /* get next -- new zero_cache */
PPC405_ERR77(0,%3)
" stwcx. %0,0,%3\n" /* update zero_cache */
" bne- 101b\n" /* if lost reservation try again */
: "=&r" (tmp), "=&r" (page), "+m" (zero_cache)
: "r" (&zero_quicklist)
: "cc" );
#ifdef CONFIG_SMP
/* if another cpu beat us above this can happen -- Cort */
if ( page == 0 )
return 0;
#endif /* CONFIG_SMP */
/* we can update zerocount after the fact since it is not
* used for anything but control of a loop which doesn't
* matter since it won't affect anything if it zeros one
* less page -- Cort
*/
atomic_inc((atomic_t *)&zero_cache_hits);
atomic_dec((atomic_t *)&zero_cache_sz);
/* zero out the pointer to next in the page */
*(unsigned long *)page = 0;
return page;
}
return 0;
}
/*
* Experimental stuff to zero out pages in the idle task
* to speed up get_free_pages(). Zero's out pages until
* we've reached the limit of zero'd pages. We handle
* reschedule()'s in here so when we return we know we've
* zero'd all we need to for now.
*/
int zero_cache_water[2] = { 25, 96 }; /* high and low water marks for zero cache */
void zero_paged(void)
{
unsigned long pageptr = 0; /* current page being zero'd */
unsigned long bytecount = 0;
register unsigned long tmp;
pte_t *pte;
if ( atomic_read(&zero_cache_sz) >= zero_cache_water[0] )
return;
while ( (atomic_read(&zero_cache_sz) < zero_cache_water[1]) && !need_resched() )
{
/*
* Mark a page as reserved so we can mess with it
* If we're interrupted we keep this page and our place in it
* since we validly hold it and it's reserved for us.
*/
pageptr = __get_free_pages(GFP_ATOMIC, 0);
if ( !pageptr )
return;
cond_resched();
/*
* Make the page no cache so we don't blow our cache with 0's
*/
pte = find_pte(&init_mm, pageptr);
if ( !pte )
{
printk("pte NULL in zero_paged()\n");
return;
}
pte_uncache(*pte);
flush_tlb_page(find_vma(&init_mm,pageptr),pageptr);
/*
* Important here to not take time away from real processes.
*/
for ( bytecount = 0; bytecount < PAGE_SIZE ; bytecount += 4 )
{
cond_resched();
*(unsigned long *)(bytecount + pageptr) = 0;
}
/*
* If we finished zero-ing out a page add this page to
* the zero_cache atomically -- we can't use
* down/up since we can't sleep in idle.
* Disabling interrupts is also a bad idea since we would
* steal time away from real processes.
* We can also have several zero_paged's running
* on different processors so we can't interfere with them.
* So we update the list atomically without locking it.
* -- Cort
*/
/* turn cache on for this page */
pte_cache(*pte);
flush_tlb_page(find_vma(&init_mm,pageptr),pageptr);
/* atomically add this page to the list */
asm ( "101:lwarx %0,0,%2\n" /* reserve zero_cache */
" stw %0,0(%3)\n" /* update *pageptr */
#ifdef CONFIG_SMP
" sync\n" /* let store settle */
#endif
PPC405_ERR77(0,%2)
" stwcx. %3,0,%2\n" /* update zero_cache in mem */
" bne- 101b\n" /* if lost reservation try again */
: "=&r" (tmp), "+m" (zero_quicklist)
: "r" (&zero_quicklist), "r" (pageptr)
: "cc" );
/*
* This variable is used in the above loop and nowhere
* else so the worst that could happen is we would
* zero out one more or one less page than we want
* per processor on the machine. This is because
* we could add our page to the list but not have
* zerocount updated yet when another processor
* reads it. -- Cort
*/
atomic_inc((atomic_t *)&zero_cache_sz);
atomic_inc((atomic_t *)&zero_cache_total);
}
}
#endif /* 0 */
void power_save(void)
{
unsigned long hid0;
......
......@@ -369,34 +369,39 @@ _GLOBAL(_tlbia)
isync
#else
#if defined(CONFIG_SMP)
rlwinm r8,r1,0,0,18
lwz r8,TI_CPU(r8)
oris r8,r8,10
mfmsr r10
SYNC
rlwinm r0,r10,0,17,15 /* clear bit 16 (MSR_EE) */
rlwinm r0,r0,0,28,26 /* clear DR */
mtmsr r0
SYNC
lis r9,hash_table_lock@h
ori r9,r9,hash_table_lock@l
rlwinm r8,r1,0,0,18
lwz r8,TI_CPU(r8)
oris r8,r8,10
SYNC_601
isync
lis r9,mmu_hash_lock@h
ori r9,r9,mmu_hash_lock@l
tophys(r9,r9)
10: lwarx r7,0,r9
cmpi 0,r7,0
bne- 10b
/* No 405 Erratum 77 fix needed here, because 4xx can't do SMP */
stwcx. r8,0,r9
bne- 10b
#endif /* CONFIG_SMP */
sync
tlbia
sync
#ifdef CONFIG_SMP
TLBSYNC
li r0,0
stw r0,0(r9) /* clear hash_table_lock */
stw r0,0(r9) /* clear mmu_hash_lock */
mtmsr r10
SYNC
#endif
#endif
SYNC_601
isync
#else /* CONFIG_SMP */
sync
tlbia
sync
#endif /* CONFIG_SMP */
#endif /* CONFIG_4xx */
blr
/*
......@@ -415,33 +420,37 @@ _GLOBAL(_tlbie)
10:
#else
#if defined(CONFIG_SMP)
rlwinm r8,r1,0,0,18
lwz r8,TI_CPU(r8)
oris r8,r8,11
mfmsr r10
SYNC
rlwinm r0,r10,0,17,15 /* clear bit 16 (MSR_EE) */
rlwinm r0,r0,0,28,26 /* clear DR */
mtmsr r0
SYNC
lis r9,hash_table_lock@h
ori r9,r9,hash_table_lock@l
rlwinm r8,r1,0,0,18
lwz r8,TI_CPU(r8)
oris r8,r8,11
SYNC_601
isync
lis r9,mmu_hash_lock@h
ori r9,r9,mmu_hash_lock@l
tophys(r9,r9)
10: lwarx r7,0,r9
cmpi 0,r7,0
bne- 10b
PPC405_ERR77(0,r9)
stwcx. r8,0,r9
bne- 10b
eieio
#endif /* CONFIG_SMP */
tlbie r3
sync
#ifdef CONFIG_SMP
TLBSYNC
li r0,0
stw r0,0(r9) /* clear hash_table_lock */
stw r0,0(r9) /* clear mmu_hash_lock */
mtmsr r10
SYNC
#endif
SYNC_601
isync
#else /* CONFIG_SMP */
tlbie r3
sync
#endif /* CONFIG_SMP */
#endif /* CONFIG_4xx */
blr
......@@ -630,6 +639,40 @@ _GLOBAL(__flush_dcache_icache)
isync
blr
/*
* Flush a particular page from the data cache to RAM, identified
* by its physical address. We turn off the MMU so we can just use
* the physical address (this may be a highmem page without a kernel
* mapping).
*
* void __flush_dcache_icache_phys(unsigned long physaddr)
*/
_GLOBAL(__flush_dcache_icache_phys)
mfspr r5,PVR
rlwinm r5,r5,16,16,31
cmpi 0,r5,1
beqlr /* for 601, do nothing */
mfmsr r10
rlwinm r0,r10,0,28,26 /* clear DR */
mtmsr r0
isync
rlwinm r3,r3,0,0,19 /* Get page base address */
li r4,4096/L1_CACHE_LINE_SIZE /* Number of lines in a page */
mtctr r4
mr r6,r3
0: dcbst 0,r3 /* Write line to ram */
addi r3,r3,L1_CACHE_LINE_SIZE
bdnz 0b
sync
mtctr r4
1: icbi 0,r6
addi r6,r6,L1_CACHE_LINE_SIZE
bdnz 1b
sync
mtmsr r10 /* restore DR */
isync
blr
/*
* Clear a page using the dcbz instruction, which doesn't cause any
* memory traffic (except to write out any cache lines which get
......
......@@ -218,13 +218,13 @@ EXPORT_SYMBOL(__global_sti);
EXPORT_SYMBOL(__global_save_flags);
EXPORT_SYMBOL(__global_restore_flags);
#ifdef SPINLOCK_DEBUG
EXPORT_SYMBOL(_spin_lock);
EXPORT_SYMBOL(_spin_unlock);
EXPORT_SYMBOL(spin_trylock);
EXPORT_SYMBOL(_read_lock);
EXPORT_SYMBOL(_read_unlock);
EXPORT_SYMBOL(_write_lock);
EXPORT_SYMBOL(_write_unlock);
EXPORT_SYMBOL(_raw_spin_lock);
EXPORT_SYMBOL(_raw_spin_unlock);
EXPORT_SYMBOL(_raw_spin_trylock);
EXPORT_SYMBOL(_raw_read_lock);
EXPORT_SYMBOL(_raw_read_unlock);
EXPORT_SYMBOL(_raw_write_lock);
EXPORT_SYMBOL(_raw_write_unlock);
#endif
EXPORT_SYMBOL(smp_call_function);
EXPORT_SYMBOL(smp_hw_index);
......@@ -361,7 +361,7 @@ EXPORT_SYMBOL(set_context);
EXPORT_SYMBOL(handle_mm_fault); /* For MOL */
EXPORT_SYMBOL_NOVERS(disarm_decr);
#ifdef CONFIG_PPC_STD_MMU
EXPORT_SYMBOL(flush_hash_page); /* For MOL */
EXPORT_SYMBOL(flush_hash_pages); /* For MOL */
extern long *intercept_table;
EXPORT_SYMBOL(intercept_table);
#endif
......
......@@ -291,26 +291,6 @@ void smp_call_function_interrupt(void)
atomic_inc(&call_data->finished);
}
/*
* Task migration callback.
*/
void smp_task_migration_interrupt(void *new_task)
{
task_t *p;
p = new_task;
sched_task_migrated(p);
}
/*
* This function sends a 'task migration' IPI to another CPU.
* Must be called from syscall contexts, with interrupts *enabled*.
*/
void smp_migrate_task(int cpu, task_t *p)
{
__smp_call_function(smp_task_migration_interrupt, p, 0, cpu);
}
void __init smp_boot_cpus(void)
{
int i, cpu_nr;
......
......@@ -28,7 +28,7 @@
* since they may inhibit forward progress by other CPUs in getting
* a lock.
*/
static unsigned long __spin_trylock(volatile unsigned long *lock)
unsigned long __spin_trylock(volatile unsigned long *lock)
{
unsigned long ret;
......
......@@ -50,8 +50,6 @@
#include <asm/smp.h>
#include <asm/machdep.h>
extern int get_pteptr(struct mm_struct *mm, unsigned long addr, pte_t **ptep);
/* This function will allocate the requested contiguous pages and
* map them into the kernel's vmalloc() space. This is done so we
* get unique mapping for these pages, outside of the kernel's 1:1
......@@ -157,6 +155,6 @@ size_t size, int direction)
{
unsigned long start;
start = (unsigned long)(page->virtual) + offset;
start = page_address(page) + offset;
consistent_sync(start, size, direction);
}
......@@ -154,6 +154,7 @@ void do_page_fault(struct pt_regs *regs, unsigned long address,
/* Since 4xx supports per-page execute permission,
* we lazily flush dcache to icache. */
ptep = NULL;
if (get_pteptr(mm, address, &ptep) && pte_present(*ptep)) {
struct page *page = pte_page(*ptep);
......@@ -164,9 +165,12 @@ void do_page_fault(struct pt_regs *regs, unsigned long address,
}
pte_update(ptep, 0, _PAGE_HWEXEC);
_tlbie(address);
pte_unmap(ptep);
up_read(&mm->mmap_sem);
return;
}
if (ptep != NULL)
pte_unmap(ptep);
#endif
/* a read */
} else {
......@@ -289,27 +293,18 @@ pte_t *va_to_pte(unsigned long address)
struct mm_struct *mm;
if (address < TASK_SIZE)
mm = current->mm;
else
mm = &init_mm;
return NULL;
dir = pgd_offset(mm, address & PAGE_MASK);
dir = pgd_offset(&init_mm, address);
if (dir) {
pmd = pmd_offset(dir, address & PAGE_MASK);
if (pmd && pmd_present(*pmd)) {
pte = pte_offset(pmd, address & PAGE_MASK);
if (pte && pte_present(*pte)) {
pte = pte_offset_kernel(pmd, address & PAGE_MASK);
if (pte && pte_present(*pte))
return(pte);
}
}
else {
return (0);
}
}
else {
return (0);
}
return (0);
return NULL;
}
unsigned long va_to_phys(unsigned long address)
......@@ -334,7 +329,7 @@ print_8xx_pte(struct mm_struct *mm, unsigned long addr)
if (pgd) {
pmd = pmd_offset(pgd, addr & PAGE_MASK);
if (pmd && pmd_present(*pmd)) {
pte = pte_offset(pmd, addr & PAGE_MASK);
pte = pte_offset_kernel(pmd, addr & PAGE_MASK);
if (pte) {
printk(" (0x%08lx)->(0x%08lx)->0x%08lx\n",
(long)pgd, (long)pte, (long)pte_val(*pte));
......@@ -375,7 +370,7 @@ get_8xx_pte(struct mm_struct *mm, unsigned long addr)
if (pgd) {
pmd = pmd_offset(pgd, addr & PAGE_MASK);
if (pmd && pmd_present(*pmd)) {
pte = pte_offset(pmd, addr & PAGE_MASK);
pte = pte_offset_kernel(pmd, addr & PAGE_MASK);
if (pte) {
retval = (int)pte_val(*pte);
}
......
This diff is collapsed.
......@@ -39,7 +39,7 @@
int iSeries_hpt_loaded;
static spinlock_t hash_table_lock = SPIN_LOCK_UNLOCKED;
static spinlock_t mmu_hash_lock = SPIN_LOCK_UNLOCKED;
extern unsigned long htab_reloads; // Defined in ppc/kernel/ppc_htab.c
extern unsigned long htab_evicts;
......@@ -159,10 +159,10 @@ int iSeries_create_hpte( unsigned long access, unsigned long va )
access |= _PAGE_PRESENT; // _PAGE_PRESENT also needed
spin_lock( &hash_table_lock );
spin_lock( &mmu_hash_lock );
// check if pte is in the required state
if ( ( access & ~(pte_val(*pt)) ) ) {
spin_unlock( &hash_table_lock );
spin_unlock( &mmu_hash_lock );
return 1;
}
......@@ -177,18 +177,18 @@ int iSeries_create_hpte( unsigned long access, unsigned long va )
va,
pte_val(*pt));
spin_unlock( &hash_table_lock );
spin_unlock( &mmu_hash_lock );
return 0;
}
void add_hash_page(unsigned context, unsigned long va, pte_t *ptep)
{
spin_lock( &hash_table_lock );
spin_lock( &mmu_hash_lock );
pte_update(ptep,0,_PAGE_HASHPTE);
__create_hpte(CTX_TO_VSID(context, va),
va,
pte_val(*ptep));
spin_unlock( &hash_table_lock );
spin_unlock( &mmu_hash_lock );
}
int flush_hash_page(unsigned context, unsigned long va, pte_t *ptep)
......@@ -208,7 +208,7 @@ int flush_hash_page(unsigned context, unsigned long va, pte_t *ptep)
hpte1Ptr = hpte0Ptr + 1;
*hpte0Ptr = *hpte1Ptr = 0;
spin_lock( &hash_table_lock );
spin_lock( &mmu_hash_lock );
rtnIndex = HvCallHpt_findValid( &hpte, vpn );
if ( hpte.v ) {
......@@ -217,7 +217,7 @@ int flush_hash_page(unsigned context, unsigned long va, pte_t *ptep)
rc = 0;
} else
rc = 1;
spin_unlock( &hash_table_lock );
spin_unlock( &mmu_hash_lock );
return rc;
}
......@@ -175,12 +175,13 @@ void update_mmu_cache(struct vm_area_struct *vma, unsigned long address,
pte_t *ptep;
static int nopreload;
if (nopreload)
if (nopreload || address >= TASK_SIZE)
return;
mm = (address < TASK_SIZE)? vma->vm_mm: &init_mm;
mm = vma->vm_mm;
pmd = pmd_offset(pgd_offset(mm, address), address);
if (!pmd_none(*pmd)) {
ptep = pte_offset(pmd, address);
ptep = pte_offset_map(pmd, address);
add_hash_page(mm->context, address, ptep);
pte_unmap(ptep);
}
}
......@@ -113,24 +113,6 @@ unsigned long __max_memory;
/* max amount of low RAM to map in */
unsigned long __max_low_memory = MAX_LOW_MEM;
int do_check_pgt_cache(int low, int high)
{
int freed = 0;
if (pgtable_cache_size > high) {
do {
if (pgd_quicklist) {
free_pgd_slow(get_pgd_fast());
freed++;
}
if (pte_quicklist) {
pte_free_slow(pte_alloc_one_fast(NULL, 0));
freed++;
}
} while (pgtable_cache_size > low);
}
return freed;
}
void show_mem(void)
{
int i,free = 0,total = 0,reserved = 0;
......@@ -160,7 +142,6 @@ void show_mem(void)
printk("%d reserved pages\n",reserved);
printk("%d pages shared\n",shared);
printk("%d pages swap cached\n",cached);
printk("%d pages in page table cache\n",(int)pgtable_cache_size);
show_buffers();
}
......@@ -396,9 +377,11 @@ void __init paging_init(void)
#ifdef CONFIG_HIGHMEM
map_page(PKMAP_BASE, 0, 0); /* XXX gross */
pkmap_page_table = pte_offset(pmd_offset(pgd_offset_k(PKMAP_BASE), PKMAP_BASE), PKMAP_BASE);
pkmap_page_table = pte_offset_kernel(pmd_offset(pgd_offset_k
(PKMAP_BASE), PKMAP_BASE), PKMAP_BASE);
map_page(KMAP_FIX_BEGIN, 0, 0); /* XXX gross */
kmap_pte = pte_offset(pmd_offset(pgd_offset_k(KMAP_FIX_BEGIN), KMAP_FIX_BEGIN), KMAP_FIX_BEGIN);
kmap_pte = pte_offset_kernel(pmd_offset(pgd_offset_k
(KMAP_FIX_BEGIN), KMAP_FIX_BEGIN), KMAP_FIX_BEGIN);
kmap_prot = PAGE_KERNEL;
#endif /* CONFIG_HIGHMEM */
......@@ -588,10 +571,12 @@ void flush_dcache_page(struct page *page)
void flush_icache_page(struct vm_area_struct *vma, struct page *page)
{
unsigned long phys;
if (page->mapping && !PageReserved(page)
&& !test_bit(PG_arch_1, &page->flags)) {
__flush_dcache_icache(kmap(page));
kunmap(page);
phys = ((page - mem_map) << PAGE_SHIFT) + PPC_MEMSTART;
__flush_dcache_icache_phys(phys);
set_bit(PG_arch_1, &page->flags);
}
}
......
......@@ -61,11 +61,12 @@ extern void MMU_init_hw(void);
* which includes all new 82xx processors. We need tlbie/tlbsync here
* in that case (I think). -- Dan.
*/
static inline void flush_HPTE(unsigned context, unsigned long va, pte_t *pg)
static inline void flush_HPTE(unsigned context, unsigned long va,
unsigned long pdval)
{
if ((Hash != 0) &&
(cur_cpu_spec[0]->cpu_features & CPU_FTR_HPTE_TABLE))
flush_hash_page(0, va, pg);
flush_hash_pages(0, va, pdval, 1);
else
_tlbie(va);
}
......
......@@ -39,10 +39,6 @@ unsigned long ioremap_base;
unsigned long ioremap_bot;
int io_bat_index;
#ifndef CONFIG_SMP
struct pgtable_cache_struct quicklists;
#endif
#if defined(CONFIG_6xx) || defined(CONFIG_POWER3)
#define HAVE_BATS 1
#endif
......@@ -173,12 +169,12 @@ map_page(unsigned long va, unsigned long pa, int flags)
/* Use upper 10 bits of VA to index the first level map */
pd = pmd_offset(pgd_offset_k(va), va);
/* Use middle 10 bits of VA to index the second-level map */
pg = pte_alloc(&init_mm, pd, va);
pg = pte_alloc_kernel(&init_mm, pd, va);
if (pg != 0) {
err = 0;
set_pte(pg, mk_pte_phys(pa & PAGE_MASK, __pgprot(flags)));
if (mem_init_done)
flush_HPTE(0, va, pg);
flush_HPTE(0, va, pmd_val(*pd));
}
spin_unlock(&init_mm.page_table_lock);
return err;
......@@ -272,10 +268,11 @@ get_pteptr(struct mm_struct *mm, unsigned long addr, pte_t **ptep)
if (pgd) {
pmd = pmd_offset(pgd, addr & PAGE_MASK);
if (pmd_present(*pmd)) {
pte = pte_offset(pmd, addr & PAGE_MASK);
pte = pte_offset_map(pmd, addr & PAGE_MASK);
if (pte) {
retval = 1;
*ptep = pte;
/* XXX caller needs to do pte_unmap, yuck */
}
}
}
......@@ -312,8 +309,10 @@ unsigned long iopa(unsigned long addr)
mm = &init_mm;
pa = 0;
if (get_pteptr(mm, addr, &pte))
if (get_pteptr(mm, addr, &pte)) {
pa = (pte_val(*pte) & PAGE_MASK) | (addr & ~PAGE_MASK);
pte_unmap(pte);
}
return(pa);
}
......
......@@ -33,6 +33,7 @@
#include <linux/kernel.h>
#include <linux/mm.h>
#include <linux/init.h>
#include <linux/highmem.h>
#include <asm/prom.h>
#include <asm/mmu.h>
......@@ -289,7 +290,6 @@ void update_mmu_cache(struct vm_area_struct *vma, unsigned long address,
{
struct mm_struct *mm;
pmd_t *pmd;
pte_t *ptep;
static int nopreload;
if (Hash == 0 || nopreload)
......@@ -299,8 +299,6 @@ void update_mmu_cache(struct vm_area_struct *vma, unsigned long address,
return;
mm = (address < TASK_SIZE)? vma->vm_mm: &init_mm;
pmd = pmd_offset(pgd_offset(mm, address), address);
if (!pmd_none(*pmd)) {
ptep = pte_offset(pmd, address);
add_hash_page(mm->context, address, ptep);
}
if (!pmd_none(*pmd))
add_hash_page(mm->context, address, pmd_val(*pmd));
}
......@@ -30,6 +30,7 @@
#include <linux/kernel.h>
#include <linux/mm.h>
#include <linux/init.h>
#include <linux/highmem.h>
#include "mmu_decl.h"
......@@ -104,7 +105,6 @@ local_flush_tlb_page(struct vm_area_struct *vma, unsigned long vmaddr)
{
struct mm_struct *mm;
pmd_t *pmd;
pte_t *pte;
if (Hash == 0) {
_tlbie(vmaddr);
......@@ -112,11 +112,8 @@ local_flush_tlb_page(struct vm_area_struct *vma, unsigned long vmaddr)
}
mm = (vmaddr < TASK_SIZE)? vma->vm_mm: &init_mm;
pmd = pmd_offset(pgd_offset(mm, vmaddr), vmaddr);
if (!pmd_none(*pmd)) {
pte = pte_offset(pmd, vmaddr);
if (pte_val(*pte) & _PAGE_HASHPTE)
flush_hash_page(mm->context, vmaddr, pte);
}
if (!pmd_none(*pmd))
flush_hash_pages(mm->context, vmaddr, pmd_val(*pmd), 1);
#ifdef CONFIG_SMP
smp_send_tlb_invalidate(0);
#endif
......@@ -133,8 +130,8 @@ local_flush_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned
{
struct mm_struct *mm = vma->vm_mm;
pmd_t *pmd;
pte_t *pte;
unsigned long pmd_end;
int count;
unsigned int ctx = mm->context;
if (Hash == 0) {
......@@ -144,24 +141,21 @@ local_flush_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned
start &= PAGE_MASK;
if (start >= end)
return;
end = (end - 1) | ~PAGE_MASK;
pmd = pmd_offset(pgd_offset(mm, start), start);
do {
pmd_end = (start + PGDIR_SIZE) & PGDIR_MASK;
if (!pmd_none(*pmd)) {
if (!pmd_end || pmd_end > end)
for (;;) {
pmd_end = ((start + PGDIR_SIZE) & PGDIR_MASK) - 1;
if (pmd_end > end)
pmd_end = end;
pte = pte_offset(pmd, start);
do {
if ((pte_val(*pte) & _PAGE_HASHPTE) != 0)
flush_hash_page(ctx, start, pte);
start += PAGE_SIZE;
++pte;
} while (start && start < pmd_end);
} else {
start = pmd_end;
if (!pmd_none(*pmd)) {
count = ((pmd_end - start) >> PAGE_SHIFT) + 1;
flush_hash_pages(ctx, start, pmd_val(*pmd), count);
}
if (pmd_end == end)
break;
start = pmd_end + 1;
++pmd;
} while (start && start < end);
}
#ifdef CONFIG_SMP
smp_send_tlb_invalidate(0);
......
......@@ -106,7 +106,7 @@ static inline void *kmap_atomic(struct page *page, enum km_type type)
static inline void kunmap_atomic(void *kvaddr, enum km_type type)
{
#if HIGHMEM_DEBUG
unsigned long vaddr = (unsigned long) kvaddr;
unsigned long vaddr = (unsigned long) kvaddr & PAGE_MASK;
unsigned int idx = type + KM_TYPE_NR*smp_processor_id();
if (vaddr < KMAP_FIX_BEGIN) // FIXME
......
/*
* BK Id: SCCS/s.kmap_types.h 1.9 08/29/01 14:03:05 paulus
* BK Id: %F% %I% %G% %U% %#%
*/
#ifdef __KERNEL__
#ifndef _ASM_KMAP_TYPES_H
......@@ -12,6 +12,8 @@ enum km_type {
KM_USER0,
KM_USER1,
KM_BIO_IRQ,
KM_PTE0,
KM_PTE1,
KM_TYPE_NR
};
......
/*
* BK Id: SCCS/s.pgalloc.h 1.9 05/17/01 18:14:25 cort
* BK Id: %F% %I% %G% %U% %#%
*/
#ifdef __KERNEL__
#ifndef _PPC_PGALLOC_H
......@@ -7,55 +7,12 @@
#include <linux/config.h>
#include <linux/threads.h>
#include <linux/highmem.h>
#include <asm/processor.h>
/*
* This is handled very differently on the PPC since out page tables
* are all 0's and I want to be able to use these zero'd pages elsewhere
* as well - it gives us quite a speedup.
*
* Note that the SMP/UP versions are the same but we don't need a
* per cpu list of zero pages because we do the zero-ing with the cache
* off and the access routines are lock-free but the pgt cache stuff
* is per-cpu since it isn't done with any lock-free access routines
* (although I think we need arch-specific routines so I can do lock-free).
*
* I need to generalize this so we can use it for other arch's as well.
* -- Cort
*/
#ifdef CONFIG_SMP
#define quicklists cpu_data[smp_processor_id()]
#else
extern struct pgtable_cache_struct {
unsigned long *pgd_cache;
unsigned long *pte_cache;
unsigned long pgtable_cache_sz;
} quicklists;
#endif
#define pgd_quicklist (quicklists.pgd_cache)
#define pmd_quicklist ((unsigned long *)0)
#define pte_quicklist (quicklists.pte_cache)
#define pgtable_cache_size (quicklists.pgtable_cache_sz)
extern unsigned long *zero_cache; /* head linked list of pre-zero'd pages */
extern atomic_t zero_sz; /* # currently pre-zero'd pages */
extern atomic_t zeropage_hits; /* # zero'd pages request that we've done */
extern atomic_t zeropage_calls; /* # zero'd pages request that've been made */
extern atomic_t zerototal; /* # pages zero'd over time */
#define zero_quicklist (zero_cache)
#define zero_cache_sz (zero_sz)
#define zero_cache_calls (zeropage_calls)
#define zero_cache_hits (zeropage_hits)
#define zero_cache_total (zerototal)
/* return a pre-zero'd page from the list, return NULL if none available -- Cort */
extern unsigned long get_zero_page_fast(void);
extern void __bad_pte(pmd_t *pmd);
extern __inline__ pgd_t *get_pgd_slow(void)
static inline pgd_t *pgd_alloc(struct mm_struct *mm)
{
pgd_t *ret;
......@@ -64,85 +21,75 @@ extern __inline__ pgd_t *get_pgd_slow(void)
return ret;
}
extern __inline__ pgd_t *get_pgd_fast(void)
{
unsigned long *ret;
if ((ret = pgd_quicklist) != NULL) {
pgd_quicklist = (unsigned long *)(*ret);
ret[0] = 0;
pgtable_cache_size--;
} else
ret = (unsigned long *)get_pgd_slow();
return (pgd_t *)ret;
}
extern __inline__ void free_pgd_fast(pgd_t *pgd)
{
*(unsigned long **)pgd = pgd_quicklist;
pgd_quicklist = (unsigned long *) pgd;
pgtable_cache_size++;
}
extern __inline__ void free_pgd_slow(pgd_t *pgd)
extern __inline__ void pgd_free(pgd_t *pgd)
{
free_page((unsigned long)pgd);
}
#define pgd_free(pgd) free_pgd_fast(pgd)
#define pgd_alloc(mm) get_pgd_fast()
/*
* We don't have any real pmd's, and this code never triggers because
* the pgd will always be present..
*/
#define pmd_alloc_one_fast(mm, address) ({ BUG(); ((pmd_t *)1); })
#define pmd_alloc_one(mm,address) ({ BUG(); ((pmd_t *)2); })
#define pmd_free(x) do { } while (0)
#define pgd_populate(mm, pmd, pte) BUG()
static inline pte_t *pte_alloc_one(struct mm_struct *mm, unsigned long address)
static inline pte_t *
pte_alloc_one_kernel(struct mm_struct *mm, unsigned long address)
{
pte_t *pte;
extern int mem_init_done;
extern void *early_get_page(void);
int timeout = 0;
if (mem_init_done)
pte = (pte_t *) __get_free_page(GFP_KERNEL);
else
if (mem_init_done) {
while ((pte = (pte_t *) __get_free_page(GFP_KERNEL)) == NULL
&& ++timeout < 10) {
set_current_state(TASK_UNINTERRUPTIBLE);
schedule_timeout(HZ);
}
} else
pte = (pte_t *) early_get_page();
if (pte != NULL)
clear_page(pte);
return pte;
}
static inline pte_t *pte_alloc_one_fast(struct mm_struct *mm, unsigned long address)
static inline struct page *
pte_alloc_one(struct mm_struct *mm, unsigned long address)
{
unsigned long *ret;
struct page *pte;
int timeout = 0;
#ifdef CONFIG_HIGHPTE
int flags = GFP_KERNEL | __GFP_HIGHMEM;
#else
int flags = GFP_KERNEL;
#endif
if ((ret = pte_quicklist) != NULL) {
pte_quicklist = (unsigned long *)(*ret);
ret[0] = 0;
pgtable_cache_size--;
while ((pte = alloc_pages(flags, 0)) == NULL) {
if (++timeout >= 10)
return NULL;
set_current_state(TASK_UNINTERRUPTIBLE);
schedule_timeout(HZ);
}
return (pte_t *)ret;
clear_highpage(pte);
return pte;
}
extern __inline__ void pte_free_fast(pte_t *pte)
static inline void pte_free_kernel(pte_t *pte)
{
*(unsigned long **)pte = pte_quicklist;
pte_quicklist = (unsigned long *) pte;
pgtable_cache_size++;
free_page((unsigned long)pte);
}
extern __inline__ void pte_free_slow(pte_t *pte)
static inline void pte_free(struct page *pte)
{
free_page((unsigned long)pte);
__free_page(pte);
}
#define pte_free(pte) pte_free_slow(pte)
#define pmd_populate(mm, pmd, pte) (pmd_val(*(pmd)) = (unsigned long) (pte))
#define pmd_populate_kernel(mm, pmd, pte) \
(pmd_val(*(pmd)) = __pa(pte))
#define pmd_populate(mm, pmd, pte) \
(pmd_val(*(pmd)) = ((pte) - mem_map) << PAGE_SHIFT)
extern int do_check_pgt_cache(int, int);
......
......@@ -13,6 +13,7 @@
#include <asm/processor.h> /* For TASK_SIZE */
#include <asm/mmu.h>
#include <asm/page.h>
#include <asm/kmap_types.h>
extern void _tlbie(unsigned long address);
extern void _tlbia(void);
......@@ -98,6 +99,7 @@ extern void flush_icache_user_range(struct vm_area_struct *vma,
struct page *page, unsigned long addr, int len);
extern void flush_icache_range(unsigned long, unsigned long);
extern void __flush_dcache_icache(void *page_va);
extern void __flush_dcache_icache_phys(unsigned long physaddr);
extern void flush_dcache_page(struct page *page);
extern void flush_icache_page(struct vm_area_struct *vma, struct page *page);
......@@ -274,7 +276,6 @@ extern unsigned long ioremap_bot, ioremap_base;
#define _PAGE_HWWRITE 0x0100 /* h/w write enable: never set in Linux PTE */
#define _PAGE_USER 0x0800 /* One of the PP bits, the other is USER&~RW */
#define _PMD_PRESENT 0x0001
#define _PMD_PAGE_MASK 0x000c
#define _PMD_PAGE_8M 0x000c
......@@ -385,8 +386,8 @@ extern unsigned long empty_zero_page[1024];
#define pte_clear(ptep) do { set_pte((ptep), __pte(0)); } while (0)
#define pmd_none(pmd) (!pmd_val(pmd))
#define pmd_bad(pmd) ((pmd_val(pmd) & ~PAGE_MASK) != 0)
#define pmd_present(pmd) ((pmd_val(pmd) & PAGE_MASK) != 0)
#define pmd_bad(pmd) (0)
#define pmd_present(pmd) (pmd_val(pmd) != 0)
#define pmd_clear(pmdp) do { pmd_val(*(pmdp)) = 0; } while (0)
#define pte_page(x) (mem_map+(unsigned long)((pte_val(x)-PPC_MEMSTART) >> PAGE_SHIFT))
......@@ -530,7 +531,10 @@ static inline void ptep_mkdirty(pte_t *ptep)
#define pte_same(A,B) (((pte_val(A) ^ pte_val(B)) & ~_PAGE_HASHPTE) == 0)
#define pmd_page(pmd) (pmd_val(pmd) & PAGE_MASK)
#define pmd_page_kernel(pmd) \
((unsigned long) __va(pmd_val(pmd) & PAGE_MASK))
#define pmd_page(pmd) \
(mem_map + (pmd_val(pmd) >> PAGE_SHIFT))
/* to find an entry in a kernel page-table-directory */
#define pgd_offset_k(address) pgd_offset(&init_mm, address)
......@@ -546,10 +550,17 @@ static inline pmd_t * pmd_offset(pgd_t * dir, unsigned long address)
}
/* Find an entry in the third-level page table.. */
static inline pte_t * pte_offset(pmd_t * dir, unsigned long address)
{
return (pte_t *) pmd_page(*dir) + ((address >> PAGE_SHIFT) & (PTRS_PER_PTE - 1));
}
#define __pte_offset(address) \
((address >> PAGE_SHIFT) & (PTRS_PER_PTE - 1))
#define pte_offset_kernel(dir, addr) \
((pte_t *) pmd_page_kernel(*(dir)) + __pte_offset(addr))
#define pte_offset_map(dir, addr) \
((pte_t *) kmap_atomic(pmd_page(*(dir)), KM_PTE0) + __pte_offset(addr))
#define pte_offset_map_nested(dir, addr) \
((pte_t *) kmap_atomic(pmd_page(*(dir)), KM_PTE1) + __pte_offset(addr))
#define pte_unmap(pte) kunmap_atomic(pte, KM_PTE0)
#define pte_unmap_nested(pte) kunmap_atomic(pte, KM_PTE1)
extern pgd_t swapper_pg_dir[1024];
extern void paging_init(void);
......@@ -558,10 +569,12 @@ extern void paging_init(void);
* When flushing the tlb entry for a page, we also need to flush the hash
* table entry. flush_hash_page is assembler (for speed) in hashtable.S.
*/
extern int flush_hash_page(unsigned context, unsigned long va, pte_t *ptep);
extern int flush_hash_pages(unsigned context, unsigned long va,
unsigned long pmdval, int count);
/* Add an HPTE to the hash table */
extern void add_hash_page(unsigned context, unsigned long va, pte_t *ptep);
extern void add_hash_page(unsigned context, unsigned long va,
unsigned long pmdval);
/*
* Encode and decode a swap entry.
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment