stab.c 11.2 KB
Newer Older
1 2 3 4 5
/*
 * PowerPC64 Segment Translation Support.
 *
 * Dave Engebretsen and Mike Corrigan {engebret|mikejc}@us.ibm.com
 *    Copyright (c) 2001 Dave Engebretsen
6 7
 *
 * Copyright (C) 2002 Anton Blanchard <anton@au.ibm.com>, IBM
8 9 10 11 12 13 14
 * 
 *      This program is free software; you can redistribute it and/or
 *      modify it under the terms of the GNU General Public License
 *      as published by the Free Software Foundation; either version
 *      2 of the License, or (at your option) any later version.
 */

15 16
/* XXX Note: Changes for bolted region have not been merged - Anton */

17 18 19 20
#include <linux/config.h>
#include <asm/pgtable.h>
#include <asm/mmu.h>
#include <asm/mmu_context.h>
21 22
#include <asm/paca.h>
#include <asm/naca.h>
23 24
#include <asm/pmc.h>

25
int make_ste(unsigned long stab, unsigned long esid, unsigned long vsid);
26 27
void make_slbe(unsigned long esid, unsigned long vsid, int large,
	       int kernel_segment);
28

29 30 31 32 33 34 35 36 37 38 39 40
/*
 * Build an entry for the base kernel segment and put it into
 * the segment table or SLB.  All other segment table or SLB
 * entries are faulted in.
 */
void stab_initialize(unsigned long stab)
{
	unsigned long esid, vsid; 

	esid = GET_ESID(KERNELBASE);
	vsid = get_kernel_vsid(esid << SID_SHIFT); 

41 42 43 44
	if (cpu_has_slb()) {
		/* Invalidate the entire SLB & all the ERATS */
#ifdef CONFIG_PPC_ISERIES
		asm volatile("isync; slbia; isync":::"memory");
45
#else
46 47 48
		asm volatile("isync":::"memory");
		asm volatile("slbmte  %0,%0"::"r" (0) : "memory");
		asm volatile("isync; slbia; isync":::"memory");
49 50
		make_slbe(esid, vsid, 0, 1);
		asm volatile("isync":::"memory");
51
#endif
52 53 54 55 56 57 58
	} else {
		asm volatile("isync; slbia; isync":::"memory");
		make_ste(stab, esid, vsid);

		/* Order update */
		asm volatile("sync":::"memory"); 
	}
59 60 61 62
}

/*
 * Create a segment table entry for the given esid/vsid pair.
63
 */
64
int make_ste(unsigned long stab, unsigned long esid, unsigned long vsid)
65 66 67 68 69 70 71 72 73
{
	unsigned long entry, group, old_esid, castout_entry, i;
	unsigned int global_entry;
	STE *ste, *castout_ste;

	/* Search the primary group first. */
	global_entry = (esid & 0x1f) << 3;
	ste = (STE *)(stab | ((esid & 0x1f) << 7)); 

74 75 76 77
	/* Find an empty entry, if one exists. */
	for (group = 0; group < 2; group++) {
		for (entry = 0; entry < 8; entry++, ste++) {
			if (!(ste->dw0.dw0.v)) {
78 79 80
				ste->dw1.dw1.vsid = vsid;
				ste->dw0.dw0.esid = esid;
				ste->dw0.dw0.kp = 1;
81 82
				asm volatile("eieio":::"memory");
				ste->dw0.dw0.v = 1;
83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101
				return(global_entry | entry);
			}
		}
		/* Now search the secondary group. */
		global_entry = ((~esid) & 0x1f) << 3;
		ste = (STE *)(stab | (((~esid) & 0x1f) << 7)); 
	}

	/*
	 * Could not find empty entry, pick one with a round robin selection.
	 * Search all entries in the two groups.  Note that the first time
	 * we get here, we start with entry 1 so the initializer
	 * can be common with the SLB castout code.
	 */

	/* This assumes we never castout when initializing the stab. */
	PMC_SW_PROCESSOR(stab_capacity_castouts); 

	castout_entry = get_paca()->xStab_data.next_round_robin;
102 103
	for (i = 0; i < 16; i++) {
		if (castout_entry < 8) {
104 105 106 107 108 109 110 111 112
			global_entry = (esid & 0x1f) << 3;
			ste = (STE *)(stab | ((esid & 0x1f) << 7)); 
			castout_ste = ste + castout_entry;
		} else {
			global_entry = ((~esid) & 0x1f) << 3;
			ste = (STE *)(stab | (((~esid) & 0x1f) << 7)); 
			castout_ste = ste + (castout_entry - 8);
		}

113 114
		/* Dont cast out the first kernel segment */
		if (castout_ste->dw0.dw0.esid != GET_ESID(KERNELBASE))
115 116 117 118 119 120 121 122 123 124
			break;

		castout_entry = (castout_entry + 1) & 0xf;
	}

	get_paca()->xStab_data.next_round_robin = (castout_entry + 1) & 0xf;

	/* Modify the old entry to the new value. */

	/* Force previous translations to complete. DRENG */
125
	asm volatile("isync" : : : "memory" );
126 127

	castout_ste->dw0.dw0.v = 0;
128
	asm volatile("sync" : : : "memory" );    /* Order update */
129 130 131 132
	castout_ste->dw1.dw1.vsid = vsid;
	old_esid = castout_ste->dw0.dw0.esid;
	castout_ste->dw0.dw0.esid = esid;
	castout_ste->dw0.dw0.kp = 1;
133 134 135
	asm volatile("eieio" : : : "memory" );   /* Order update */
	castout_ste->dw0.dw0.v  = 1;
	asm volatile("slbie  %0" : : "r" (old_esid << SID_SHIFT)); 
136
	/* Ensure completion of slbie */
137
	asm volatile("sync" : : : "memory" );
138

139
	return (global_entry | (castout_entry & 0x7));
140 141 142 143
}

/*
 * Create a segment buffer entry for the given esid/vsid pair.
144 145 146
 *
 * NOTE: A context syncronising instruction is required before and after
 * this, in the common case we use exception entry and rfid.
147
 */
148 149
void make_slbe(unsigned long esid, unsigned long vsid, int large,
	       int kernel_segment)
150 151 152 153 154 155 156 157 158 159
{
	unsigned long entry, castout_entry;
	union {
		unsigned long word0;
		slb_dword0    data;
	} esid_data;
	union {
		unsigned long word0;
		slb_dword1    data;
	} vsid_data;
160

161
	/*
162 163
	 * Find an empty entry, if one exists. Must start at 0 because
	 * we use this code to load SLB entry 0 at boot.
164
	 */
165 166 167
	for (entry = 0; entry < naca->slb_size; entry++) {
		asm volatile("slbmfee  %0,%1" 
			     : "=r" (esid_data) : "r" (entry)); 
168 169
		if (!esid_data.data.v)
			goto write_entry;
170
	}
171

172 173 174 175 176 177
	/*
	 * Could not find empty entry, pick one with a round robin selection.
	 */

	PMC_SW_PROCESSOR(stab_capacity_castouts); 

178 179 180 181 182 183 184
	/* 
	 * Never cast out the segment for our kernel stack. Since we
	 * dont invalidate the ERAT we could have a valid translation
	 * for the kernel stack during the first part of exception exit 
	 * which gets invalidated due to a tlbie from another cpu at a
	 * non recoverable point (after setting srr0/1) - Anton
	 */
185
	castout_entry = get_paca()->xStab_data.next_round_robin;
186 187 188 189 190 191
	do {
		entry = castout_entry;
		castout_entry++; 
		if (castout_entry >= naca->slb_size)
			castout_entry = 1; 
		asm volatile("slbmfee  %0,%1" : "=r" (esid_data) : "r" (entry));
192
	} while (esid_data.data.esid == GET_ESID((unsigned long)_get_SP()));
193

194 195 196
	get_paca()->xStab_data.next_round_robin = castout_entry;

	/* slbie not needed as the previous mapping is still valid. */
197 198

write_entry:	
199 200 201 202 203 204 205 206
	/* 
	 * Write the new SLB entry.
	 */
	vsid_data.word0 = 0;
	vsid_data.data.vsid = vsid;
	vsid_data.data.kp = 1;
	if (large)
		vsid_data.data.l = 1;
207 208
	if (kernel_segment)
		vsid_data.data.c = 1;
209

210 211 212 213
	esid_data.word0 = 0;
	esid_data.data.esid = esid;
	esid_data.data.v = 1;
	esid_data.data.index = entry;
214

215 216 217 218 219
	/*
	 * No need for an isync before or after this slbmte. The exception
         * we enter with and the rfid we exit with are context synchronizing.
	 */
	asm volatile("slbmte  %0,%1" : : "r" (vsid_data), "r" (esid_data)); 
220 221 222 223 224 225 226 227
}

static inline void __ste_allocate(unsigned long esid, unsigned long vsid,
				  int kernel_segment)
{
	if (cpu_has_slb()) {
#ifndef CONFIG_PPC_ISERIES
		if (REGION_ID(esid << SID_SHIFT) == KERNEL_REGION_ID)
228
			make_slbe(esid, vsid, 1, kernel_segment); 
229 230
		else
#endif
231
			make_slbe(esid, vsid, 0, kernel_segment);
232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247
	} else {
		unsigned char top_entry, stab_entry, *segments; 

		stab_entry = make_ste(get_paca()->xStab_data.virt, esid, vsid);
		PMC_SW_PROCESSOR_A(stab_entry_use, stab_entry & 0xf); 

		segments = get_paca()->xSegments;		
		top_entry = get_paca()->stab_cache_pointer;
		if (!kernel_segment && top_entry < STAB_CACHE_SIZE) {
			segments[top_entry] = stab_entry;
			if (top_entry == STAB_CACHE_SIZE)
				top_entry = 0xff;
			top_entry++;
			get_paca()->stab_cache_pointer = top_entry;
		}
	}
248 249 250 251 252
}

/*
 * Allocate a segment table entry for the given ea.
 */
253
int ste_allocate(unsigned long ea)
254 255 256 257 258 259 260
{
	unsigned long vsid, esid;
	int kernel_segment = 0;

	PMC_SW_PROCESSOR(stab_faults); 

	/* Check for invalid effective addresses. */
261
	if (!IS_VALID_EA(ea))
262
		return 1;
263

264 265 266
	/* Kernel or user address? */
	if (REGION_ID(ea) >= KERNEL_REGION_ID) {
		kernel_segment = 1;
267
		vsid = get_kernel_vsid(ea);
268 269
	} else {
		struct mm_struct *mm = current->mm;
270 271 272
		if (mm)
			vsid = get_vsid(mm->context, ea);
		else
273 274 275 276
			return 1;
	}

	esid = GET_ESID(ea);
277 278 279 280 281
	__ste_allocate(esid, vsid, kernel_segment);
	if (!cpu_has_slb()) {
		/* Order update */
		asm volatile("sync":::"memory"); 
	}
282

283 284
	return 0;
}
285

286 287 288 289 290 291 292 293 294 295 296
unsigned long ppc64_preload_all_segments;
unsigned long ppc64_stab_preload = 1;
#define STAB_PRESSURE 0
#define USE_SLBIE_ON_STAB 0

/*
 * preload all 16 segments for a 32 bit process and the PC and SP segments
 * for a 64 bit process.
 */
static void preload_stab(struct task_struct *tsk, struct mm_struct *mm)
{
297 298
	if (ppc64_preload_all_segments &&
	    test_tsk_thread_flag(tsk, TIF_32BIT)) {
299 300 301
		unsigned long esid, vsid;

		for (esid = 0; esid < 16; esid++) {
302 303
			unsigned long ea = esid << SID_SHIFT;
			vsid = get_vsid(mm->context, ea);
304 305 306 307 308 309 310 311 312 313
			__ste_allocate(esid, vsid, 0);
		}
	} else {
		unsigned long pc = KSTK_EIP(tsk);
		unsigned long stack = KSTK_ESP(tsk);
		unsigned long pc_segment = pc & ~SID_MASK;
		unsigned long stack_segment = stack & ~SID_MASK;
		unsigned long vsid;

		if (pc) {
314 315 316
			if (!IS_VALID_EA(pc) || 
			    (REGION_ID(pc) >= KERNEL_REGION_ID))
				return;
317 318 319 320 321
			vsid = get_vsid(mm->context, pc);
			__ste_allocate(GET_ESID(pc), vsid, 0);
		}

		if (stack && (pc_segment != stack_segment)) {
322 323 324
			if (!IS_VALID_EA(stack) || 
			    (REGION_ID(stack) >= KERNEL_REGION_ID))
				return;
325 326
			vsid = get_vsid(mm->context, stack);
			__ste_allocate(GET_ESID(stack), vsid, 0);
327 328 329
		}
	}

330 331 332 333 334
	if (!cpu_has_slb()) {
		/* Order update */
		asm volatile("sync" : : : "memory"); 
	}
}
335

336
/* Flush all user entries from the segment table of the current processor. */
337
void flush_stab(struct task_struct *tsk, struct mm_struct *mm)
338
{
339
	if (cpu_has_slb()) {
340 341 342 343 344 345
		/*
		 * XXX disable 32bit slb invalidate optimisation until we fix
		 * the issue where a 32bit app execed out of a 64bit app can
		 * cause segments above 4GB not to be flushed - Anton
		 */
		if (0 && !STAB_PRESSURE && test_thread_flag(TIF_32BIT)) {
346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361
			union {
				unsigned long word0;
				slb_dword0 data;
			} esid_data;
			unsigned long esid;

			asm volatile("isync" : : : "memory");
			for (esid = 0; esid < 16; esid++) {
				esid_data.word0 = 0;
				esid_data.data.esid = esid;
				asm volatile("slbie %0" : : "r" (esid_data));
			}
			asm volatile("isync" : : : "memory");
		} else {
			asm volatile("isync; slbia; isync":::"memory");
		}
362

363 364 365
		PMC_SW_PROCESSOR(stab_invalidations);
	} else {
		STE *stab = (STE *) get_paca()->xStab_data.virt;
366
		STE *ste;
367
		unsigned long flags;
368 369

		/* Force previous translations to complete. DRENG */
370
		asm volatile("isync" : : : "memory");
371

372
		local_irq_save(flags);
373 374 375 376 377
		if (get_paca()->stab_cache_pointer != 0xff && !STAB_PRESSURE) {
			int i;
			unsigned char *segments = get_paca()->xSegments;

			for (i = 0; i < get_paca()->stab_cache_pointer; i++) {
378 379 380 381
				ste = stab + segments[i]; 
				ste->dw0.dw0.v = 0;
				PMC_SW_PROCESSOR(stab_invalidations); 
			}
382 383 384 385 386 387 388 389 390 391 392 393 394

#if USE_SLBIE_ON_STAB
			asm volatile("sync":::"memory");
			for (i = 0; i < get_paca()->stab_cache_pointer; i++) {
				ste = stab + segments[i]; 
				asm volatile("slbie  %0" : :
					"r" (ste->dw0.dw0.esid << SID_SHIFT)); 
			}
			asm volatile("sync":::"memory");
#else
			asm volatile("sync; slbia; sync":::"memory");
#endif

395
		} else {
396 397
			unsigned long entry;

398
			/* Invalidate all entries. */
399
			ste = stab;
400

401 402 403 404 405
			/* Never flush the first entry. */ 
			ste += 1;
			for (entry = 1;
			     entry < (PAGE_SIZE / sizeof(STE)); 
			     entry++, ste++) {
406 407 408 409 410 411 412 413
				unsigned long ea;
				ea = ste->dw0.dw0.esid << SID_SHIFT;
				if (STAB_PRESSURE || ea < KERNELBASE) {
					ste->dw0.dw0.v = 0;
					PMC_SW_PROCESSOR(stab_invalidations); 
				}
			}

414
			asm volatile("sync; slbia; sync":::"memory");
415 416
		}

417
		get_paca()->stab_cache_pointer = 0;
418
		local_irq_restore(flags);
419
	}
420 421 422

	if (ppc64_stab_preload)
		preload_stab(tsk, mm);
423
}