smpboot.c 23 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33
/*
 *	x86 SMP booting functions
 *
 *	(c) 1995 Alan Cox, Building #3 <alan@redhat.com>
 *	(c) 1998, 1999, 2000 Ingo Molnar <mingo@redhat.com>
 *	Copyright 2001 Andi Kleen, SuSE Labs.
 *
 *	Much of the core SMP work is based on previous work by Thomas Radke, to
 *	whom a great many thanks are extended.
 *
 *	Thanks to Intel for making available several different Pentium,
 *	Pentium Pro and Pentium-II/Xeon MP machines.
 *	Original development of Linux SMP code supported by Caldera.
 *
 *	This code is released under the GNU General Public License version 2 or
 *	later.
 *
 *	Fixes
 *		Felix Koop	:	NR_CPUS used properly
 *		Jose Renau	:	Handle single CPU case.
 *		Alan Cox	:	By repeated request 8) - Total BogoMIP report.
 *		Greg Wright	:	Fix for kernel stacks panic.
 *		Erich Boleyn	:	MP v1.4 and additional changes.
 *	Matthias Sattler	:	Changes for 2.1 kernel map.
 *	Michel Lespinasse	:	Changes for 2.1 kernel map.
 *	Michael Chastain	:	Change trampoline.S to gnu as.
 *		Alan Cox	:	Dumb bug: 'B' step PPro's are fine
 *		Ingo Molnar	:	Added APIC timers, based on code
 *					from Jose Renau
 *		Ingo Molnar	:	various cleanups and rewrites
 *		Tigran Aivazian	:	fixed "0.00 in /proc/uptime on SMP" bug.
 *	Maciej W. Rozycki	:	Bits for genuine 82489DX APICs
 *	Andi Kleen		:	Changed for SMP boot into long mode.
34
 *		Rusty Russell	:	Hacked into shape for new "hotplug" boot process. 
35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50
 */

#include <linux/config.h>
#include <linux/init.h>

#include <linux/mm.h>
#include <linux/kernel_stat.h>
#include <linux/smp_lock.h>
#include <linux/irq.h>
#include <linux/bootmem.h>

#include <linux/delay.h>
#include <linux/mc146818rtc.h>
#include <asm/mtrr.h>
#include <asm/pgalloc.h>
#include <asm/desc.h>
51 52
#include <asm/kdebug.h>
#include <asm/tlbflush.h>
53 54 55 56 57 58 59 60 61 62

/* Bitmask of currently online CPUs */
unsigned long cpu_online_map;

/* which CPU (physical APIC ID) maps to which logical CPU number */
volatile int x86_apicid_to_cpu[NR_CPUS];
/* which logical CPU number maps to which CPU (physical APIC ID) */
volatile int x86_cpu_to_apicid[NR_CPUS];

static volatile unsigned long cpu_callin_map;
63 64
volatile unsigned long cpu_callout_map;
static unsigned long smp_commenced_mask;
65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87

/* Per CPU bogomips and other parameters */
struct cpuinfo_x86 cpu_data[NR_CPUS] __cacheline_aligned;

/* Set when the idlers are all forked */
int smp_threads_ready;

/*
 * Trampoline 80x86 program as an array.
 */

extern unsigned char trampoline_data [];
extern unsigned char trampoline_end  [];
static unsigned char *trampoline_base;

/*
 * Currently trivial. Write the real->protected mode
 * bootstrap into the page concerned. The caller
 * has made sure it's suitably aligned.
 */

static unsigned long __init setup_trampoline(void)
{
88
	extern volatile __u32 tramp_gdt_ptr; 
89
	tramp_gdt_ptr = __pa_symbol(&cpu_gdt_table); 
90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113
	memcpy(trampoline_base, trampoline_data, trampoline_end - trampoline_data);
	return virt_to_phys(trampoline_base);
}

/*
 * We are called very early to get the low memory for the
 * SMP bootup trampoline page.
 */
void __init smp_alloc_memory(void)
{
	trampoline_base = (void *) alloc_bootmem_low_pages(PAGE_SIZE);
	/*
	 * Has to be in very low memory so we can execute
	 * real-mode AP code.
	 */
	if (__pa(trampoline_base) >= 0x9F000)
		BUG();
}

/*
 * The bootstrap kernel entry code has set these up. Save them for
 * a given CPU
 */

114
static void __init smp_store_cpu_info(int id)
115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135
{
	struct cpuinfo_x86 *c = cpu_data + id;

	*c = boot_cpu_data;
	identify_cpu(c);
}

/*
 * TSC synchronization.
 *
 * We first check wether all CPUs have their TSC's synchronized,
 * then we print a warning if not, and always resync.
 */

static atomic_t tsc_start_flag = ATOMIC_INIT(0);
static atomic_t tsc_count_start = ATOMIC_INIT(0);
static atomic_t tsc_count_stop = ATOMIC_INIT(0);
static unsigned long long tsc_values[NR_CPUS];

#define NR_LOOPS 5

136
extern unsigned int fast_gettimeoffset_quotient;
137 138 139 140 141 142 143 144 145

static void __init synchronize_tsc_bp (void)
{
	int i;
	unsigned long long t0;
	unsigned long long sum, avg;
	long long delta;
	unsigned long one_usec;
	int buggy = 0;
146
	extern unsigned cpu_khz;
147

148
	printk("checking TSC synchronization across %u CPUs: ",num_booting_cpus());
149

150
	one_usec = cpu_khz; 
151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168

	atomic_set(&tsc_start_flag, 1);
	wmb();

	/*
	 * We loop a few times to get a primed instruction cache,
	 * then the last pass is more or less synchronized and
	 * the BP and APs set their cycle counters to zero all at
	 * once. This reduces the chance of having random offsets
	 * between the processors, and guarantees that the maximum
	 * delay between the cycle counters is never bigger than
	 * the latency of information-passing (cachelines) between
	 * two CPUs.
	 */
	for (i = 0; i < NR_LOOPS; i++) {
		/*
		 * all APs synchronize but they loop on '== num_cpus'
		 */
169
		while (atomic_read(&tsc_count_start) != num_booting_cpus()-1) mb();
170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186
		atomic_set(&tsc_count_stop, 0);
		wmb();
		/*
		 * this lets the APs save their current TSC:
		 */
		atomic_inc(&tsc_count_start);

		rdtscll(tsc_values[smp_processor_id()]);
		/*
		 * We clear the TSC in the last loop:
		 */
		if (i == NR_LOOPS-1)
			write_tsc(0, 0);

		/*
		 * Wait for all APs to leave the synchronization point:
		 */
187
		while (atomic_read(&tsc_count_stop) != num_booting_cpus()-1) mb();
188 189 190 191 192 193
		atomic_set(&tsc_count_start, 0);
		wmb();
		atomic_inc(&tsc_count_stop);
	}

	sum = 0;
194 195
	for (i = 0; i < NR_CPUS; i++) {
		if (test_bit(i, &cpu_callout_map)) {
196 197 198
		t0 = tsc_values[i];
		sum += t0;
	}
199 200
	}
	avg = sum / num_booting_cpus();
201 202

	sum = 0;
203 204 205 206
	for (i = 0; i < NR_CPUS; i++) {
		if (!test_bit(i, &cpu_callout_map))
			continue;

207 208 209 210 211 212 213 214 215 216 217 218
		delta = tsc_values[i] - avg;
		if (delta < 0)
			delta = -delta;
		/*
		 * We report bigger than 2 microseconds clock differences.
		 */
		if (delta > 2*one_usec) {
			long realdelta;
			if (!buggy) {
				buggy = 1;
				printk("\n");
			}
219
			realdelta = delta / one_usec;
220 221 222
			if (tsc_values[i] < avg)
				realdelta = -realdelta;

223 224
			printk("BIOS BUG: CPU#%d improperly initialized, has %ld usecs TSC skew! FIXED.\n",
				i, realdelta);
225 226 227 228 229 230 231 232 233 234 235 236 237
		}

		sum += delta;
	}
	if (!buggy)
		printk("passed.\n");
}

static void __init synchronize_tsc_ap (void)
{
	int i;

	/*
238
	 * Not every cpu is online at the time
239 240 241 242 243 244 245
	 * this gets called, so we first wait for the BP to
	 * finish SMP initialization:
	 */
	while (!atomic_read(&tsc_start_flag)) mb();

	for (i = 0; i < NR_LOOPS; i++) {
		atomic_inc(&tsc_count_start);
246
		while (atomic_read(&tsc_count_start) != num_booting_cpus()) mb();
247 248 249 250 251 252

		rdtscll(tsc_values[smp_processor_id()]);
		if (i == NR_LOOPS-1)
			write_tsc(0, 0);

		atomic_inc(&tsc_count_stop);
253
		while (atomic_read(&tsc_count_stop) != num_booting_cpus()) mb();
254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279
	}
}
#undef NR_LOOPS

extern void calibrate_delay(void);

static atomic_t init_deasserted;

void __init smp_callin(void)
{
	int cpuid, phys_id;
	unsigned long timeout;

	/*
	 * If waken up by an INIT in an 82489DX configuration
	 * we may get here before an INIT-deassert IPI reaches
	 * our local APIC.  We have to wait for the IPI or we'll
	 * lock up on an APIC access.
	 */
	while (!atomic_read(&init_deasserted));

	/*
	 * (This works even if the APIC is not enabled.)
	 */
	phys_id = GET_APIC_ID(apic_read(APIC_ID));
	cpuid = smp_processor_id();
280
	if (test_and_set_bit(cpuid, &cpu_callin_map)) {
281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323
		printk("huh, phys CPU#%d, CPU#%d already present??\n",
					phys_id, cpuid);
		BUG();
	}
	Dprintk("CPU#%d (phys ID: %d) waiting for CALLOUT\n", cpuid, phys_id);

	/*
	 * STARTUP IPIs are fragile beasts as they might sometimes
	 * trigger some glue motherboard logic. Complete APIC bus
	 * silence for 1 second, this overestimates the time the
	 * boot CPU is spending to send the up to 2 STARTUP IPIs
	 * by a factor of two. This should be enough.
	 */

	/*
	 * Waiting 2s total for startup (udelay is not yet working)
	 */
	timeout = jiffies + 2*HZ;
	while (time_before(jiffies, timeout)) {
		/*
		 * Has the boot CPU finished it's STARTUP sequence?
		 */
		if (test_bit(cpuid, &cpu_callout_map))
			break;
		rep_nop();
	}

	if (!time_before(jiffies, timeout)) {
		printk("BUG: CPU%d started up but did not get a callout!\n",
			cpuid);
		BUG();
	}

	/*
	 * the boot CPU has finished the init stage and is spinning
	 * on callin_map until we finish. We are free to set up this
	 * CPU, first the APIC. (this is probably redundant on most
	 * boards)
	 */

	Dprintk("CALLIN, before setup_local_APIC().\n");
	setup_local_APIC();

324
	local_irq_enable();
325 326 327 328 329 330 331

	/*
	 * Get our bogomips.
	 */
	calibrate_delay();
	Dprintk("Stack at about %p\n",&cpuid);

332 333
	disable_APIC_timer();

334 335 336 337 338
	/*
	 * Save our processor parameters
	 */
 	smp_store_cpu_info(cpuid);

339 340
	notify_die(DIE_CPUINIT, "cpuinit", NULL, 0);

Andi Kleen's avatar
Andi Kleen committed
341 342
	local_irq_disable();

343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370
	/*
	 * Allow the master to continue.
	 */
	set_bit(cpuid, &cpu_callin_map);

	/*
	 *      Synchronize the TSC with the BP
	 */
	if (cpu_has_tsc)
		synchronize_tsc_ap();
}

int cpucount;

extern int cpu_idle(void);

/*
 * Activate a secondary processor.
 */
int __init start_secondary(void *unused)
{
	/*
	 * Dont put anything before smp_callin(), SMP
	 * booting is too fragile that we want to limit the
	 * things done here to the most necessary things.
	 */
	cpu_init();
	smp_callin();
371 372 373

	Dprintk("cpu %d: waiting for commence\n", smp_processor_id()); 
	while (!test_bit(smp_processor_id(), &smp_commenced_mask))
374
		rep_nop();
375 376 377 378 379 380 381 382 383 384 385 386 387 388 389

	Dprintk("cpu %d: setting up apic clock\n", smp_processor_id()); 	
	setup_secondary_APIC_clock();

	Dprintk("cpu %d: enabling apic timer\n", smp_processor_id()); 

	if (nmi_watchdog == NMI_IO_APIC) {
		disable_8259A_irq(0);
		enable_NMI_through_LVT0(NULL);
		enable_8259A_irq(0);
	}


	enable_APIC_timer(); 

390 391 392 393 394 395
	/*
	 * low-memory mappings have been cleared, flush them from
	 * the local TLBs too.
	 */
	local_flush_tlb();

396 397 398 399
	Dprintk("cpu %d eSetting cpu_online_map\n", smp_processor_id()); 
	set_bit(smp_processor_id(), &cpu_online_map);
	wmb();
	
400 401 402 403 404 405 406 407 408 409 410
	return cpu_idle();
}

/*
 * Everything has been set up for the secondary
 * CPUs - they just need to reload everything
 * from the task structure
 * This function must not return.
 */
void __init initialize_secondary(void)
{
411 412
	struct task_struct *me = stack_current();

413 414 415 416 417 418 419 420 421
	/*
	 * We don't actually need to load the full TSS,
	 * basically just the stack pointer and the eip.
	 */

	asm volatile(
		"movq %0,%%rsp\n\t"
		"jmp *%1"
		:
422
		:"r" (me->thread.rsp),"r" (me->thread.rip));
423 424
}

425
extern volatile unsigned long init_rsp; 
426 427
extern void (*initial_code)(void);

Rusty Russell's avatar
Rusty Russell committed
428
static struct task_struct * __init fork_by_hand(void)
429 430 431
{
	struct pt_regs regs;
	/*
432
	 * don't care about the rip and regs settings since
433 434
	 * we'll never reschedule the forked task.
	 */
Andi Kleen's avatar
Andi Kleen committed
435
	return do_fork(CLONE_VM|CLONE_IDLETASK, 0, &regs, 0, NULL, NULL);
436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475
}

#if APIC_DEBUG
static inline void inquire_remote_apic(int apicid)
{
	int i, regs[] = { APIC_ID >> 4, APIC_LVR >> 4, APIC_SPIV >> 4 };
	char *names[] = { "ID", "VERSION", "SPIV" };
	int timeout, status;

	printk("Inquiring remote APIC #%d...\n", apicid);

	for (i = 0; i < sizeof(regs) / sizeof(*regs); i++) {
		printk("... APIC #%d %s: ", apicid, names[i]);

		/*
		 * Wait for idle.
		 */
		apic_wait_icr_idle();

		apic_write_around(APIC_ICR2, SET_APIC_DEST_FIELD(apicid));
		apic_write_around(APIC_ICR, APIC_DM_REMRD | regs[i]);

		timeout = 0;
		do {
			udelay(100);
			status = apic_read(APIC_ICR) & APIC_ICR_RR_MASK;
		} while (status == APIC_ICR_RR_INPROG && timeout++ < 1000);

		switch (status) {
		case APIC_ICR_RR_VALID:
			status = apic_read(APIC_RRR);
			printk("%08x\n", status);
			break;
		default:
			printk("failed\n");
		}
	}
}
#endif

476
static int __init wakeup_secondary_via_INIT(int phys_apicid, unsigned int start_rip)
477
{
478 479
	unsigned long send_status = 0, accept_status = 0;
	int maxlvt, timeout, num_starts, j;
480 481 482 483 484 485

	Dprintk("Asserting INIT.\n");

	/*
	 * Turn INIT on target chip
	 */
486
	apic_write_around(APIC_ICR2, SET_APIC_DEST_FIELD(phys_apicid));
487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506

	/*
	 * Send IPI
	 */
	apic_write_around(APIC_ICR, APIC_INT_LEVELTRIG | APIC_INT_ASSERT
				| APIC_DM_INIT);

	Dprintk("Waiting for send to finish...\n");
	timeout = 0;
	do {
		Dprintk("+");
		udelay(100);
		send_status = apic_read(APIC_ICR) & APIC_ICR_BUSY;
	} while (send_status && (timeout++ < 1000));

	mdelay(10);

	Dprintk("Deasserting INIT.\n");

	/* Target chip */
507
	apic_write_around(APIC_ICR2, SET_APIC_DEST_FIELD(phys_apicid));
508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525

	/* Send IPI */
	apic_write_around(APIC_ICR, APIC_INT_LEVELTRIG | APIC_DM_INIT);

	Dprintk("Waiting for send to finish...\n");
	timeout = 0;
	do {
		Dprintk("+");
		udelay(100);
		send_status = apic_read(APIC_ICR) & APIC_ICR_BUSY;
	} while (send_status && (timeout++ < 1000));

	atomic_set(&init_deasserted, 1);

	/*
	 * Should we send STARTUP IPIs ?
	 *
	 * Determine this based on the APIC version.
526
	 * If we don't have an integrated APIC, don't send the STARTUP IPIs.
527
	 */
528
	if (APIC_INTEGRATED(apic_version[phys_apicid]))
529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551
		num_starts = 2;
	else
		num_starts = 0;

	/*
	 * Run STARTUP IPI loop.
	 */
	Dprintk("#startup loops: %d.\n", num_starts);

	maxlvt = get_maxlvt();

	for (j = 1; j <= num_starts; j++) {
		Dprintk("Sending STARTUP #%d.\n",j);
		apic_read_around(APIC_SPIV);
		apic_write(APIC_ESR, 0);
		apic_read(APIC_ESR);
		Dprintk("After apic_write.\n");

		/*
		 * STARTUP IPI
		 */

		/* Target chip */
552
		apic_write_around(APIC_ICR2, SET_APIC_DEST_FIELD(phys_apicid));
553 554 555 556

		/* Boot on the stack */
		/* Kick the second */
		apic_write_around(APIC_ICR, APIC_DM_STARTUP
557
					| (start_rip >> 12));
558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595

		/*
		 * Give the other CPU some time to accept the IPI.
		 */
		udelay(300);

		Dprintk("Startup point 1.\n");

		Dprintk("Waiting for send to finish...\n");
		timeout = 0;
		do {
			Dprintk("+");
			udelay(100);
			send_status = apic_read(APIC_ICR) & APIC_ICR_BUSY;
		} while (send_status && (timeout++ < 1000));

		/*
		 * Give the other CPU some time to accept the IPI.
		 */
		udelay(200);
		/*
		 * Due to the Pentium erratum 3AP.
		 */
		if (maxlvt > 3) {
			apic_read_around(APIC_SPIV);
			apic_write(APIC_ESR, 0);
		}
		accept_status = (apic_read(APIC_ESR) & 0xEF);
		if (send_status || accept_status)
			break;
	}
	Dprintk("After Startup.\n");

	if (send_status)
		printk("APIC never delivered???\n");
	if (accept_status)
		printk("APIC delivery error (%lx).\n", accept_status);

596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678
	return (send_status | accept_status);
}

extern unsigned long cpu_initialized;

static void __init do_boot_cpu (int apicid)
{
	struct task_struct *idle;
	unsigned long boot_error;
	int timeout, cpu;
	unsigned long start_rip;

	cpu = ++cpucount;
	/*
	 * We can't use kernel_thread since we must avoid to
	 * reschedule the child.
	 */
	idle = fork_by_hand();
	if (IS_ERR(idle))
		panic("failed fork for CPU %d", cpu);

	/*
	 * We remove it from the pidhash and the runqueue
	 * once we got the process:
	 */
	init_idle(idle,cpu);

	x86_cpu_to_apicid[cpu] = apicid;
	x86_apicid_to_cpu[apicid] = cpu;
	idle->thread.rip = (unsigned long)start_secondary;
//	idle->thread.rsp = (unsigned long)idle->thread_info + THREAD_SIZE - 512;

	unhash_process(idle);

	cpu_pda[cpu].pcurrent = idle;

	/* start_eip had better be page-aligned! */
	start_rip = setup_trampoline();

	init_rsp = (unsigned long)idle->thread_info + PAGE_SIZE + 1024;
	init_tss[cpu].rsp0 = init_rsp;
	initial_code = initialize_secondary;

	printk("Booting processor %d/%d rip %lx rsp %lx rsp2 %lx\n", cpu, apicid, 
	       start_rip, idle->thread.rsp, init_rsp);

	/*
	 * This grunge runs the startup process for
	 * the targeted processor.
	 */

	atomic_set(&init_deasserted, 0);

	Dprintk("Setting warm reset code and vector.\n");

	CMOS_WRITE(0xa, 0xf);
	local_flush_tlb();
	Dprintk("1.\n");
	*((volatile unsigned short *) phys_to_virt(0x469)) = start_rip >> 4;
	Dprintk("2.\n");
	*((volatile unsigned short *) phys_to_virt(0x467)) = start_rip & 0xf;
	Dprintk("3.\n");

	/*
	 * Be paranoid about clearing APIC errors.
	 */
	if (APIC_INTEGRATED(apic_version[apicid])) {
		apic_read_around(APIC_SPIV);
		apic_write(APIC_ESR, 0);
		apic_read(APIC_ESR);
	}

	/*
	 * Status is now clean
	 */
	boot_error = 0;

	/*
	 * Starting actual IPI sequence...
	 */
	boot_error = wakeup_secondary_via_INIT(apicid, start_rip); 

	if (!boot_error) {
679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701
		/*
		 * allow APs to start initializing.
		 */
		Dprintk("Before Callout %d.\n", cpu);
		set_bit(cpu, &cpu_callout_map);
		Dprintk("After Callout %d.\n", cpu);

		/*
		 * Wait 5s total for a response
		 */
		for (timeout = 0; timeout < 50000; timeout++) {
			if (test_bit(cpu, &cpu_callin_map))
				break;	/* It has booted */
			udelay(100);
		}

		if (test_bit(cpu, &cpu_callin_map)) {
			/* number CPUs logically, starting from 1 (BSP is 0) */
			Dprintk("OK.\n");
			printk("CPU%d: ", cpu);
			print_cpu_info(&cpu_data[cpu]);
			Dprintk("CPU has booted.\n");
		} else {
702
			boot_error = 1;
703 704 705 706 707 708 709 710 711 712 713 714
			if (*((volatile unsigned char *)phys_to_virt(8192))
					== 0xA5)
				/* trampoline started but...? */
				printk("Stuck ??\n");
			else
				/* trampoline code not run */
				printk("Not responding.\n");
#if APIC_DEBUG
			inquire_remote_apic(apicid);
#endif
		}
	}
715
	if (boot_error) {
716 717
		x86_cpu_to_apicid[cpu] = -1;
		x86_apicid_to_cpu[apicid] = -1;
718 719
		clear_bit(cpu, &cpu_callout_map); /* was set here (do_boot_cpu()) */
		clear_bit(cpu, &cpu_initialized); /* was set by cpu_init() */
720 721 722 723
		cpucount--;
	}

	/* mark "stuck" area as not stuck */
724
	*((volatile unsigned *)phys_to_virt(8192)) = 0;
725 726 727 728 729 730 731 732
}

cycles_t cacheflush_time;
unsigned long cache_decay_ticks;

static void smp_tune_scheduling (void)
{
	unsigned long cachesize;       /* kB   */
733
	unsigned long bandwidth = 1000; /* MB/s */
734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774
	/*
	 * Rough estimation for SMP scheduling, this is the number of
	 * cycles it takes for a fully memory-limited process to flush
	 * the SMP-local cache.
	 *
	 * (For a P5 this pretty much means we will choose another idle
	 *  CPU almost always at wakeup time (this is due to the small
	 *  L1 cache), on PIIs it's around 50-100 usecs, depending on
	 *  the cache size)
	 */

	if (!cpu_khz) {
		/*
		 * this basically disables processor-affinity
		 * scheduling on SMP without a TSC.
		 */
		cacheflush_time = 0;
		return;
	} else {
		cachesize = boot_cpu_data.x86_cache_size;
		if (cachesize == -1) {
			cachesize = 16; /* Pentiums, 2x8kB cache */
			bandwidth = 100;
		}

		cacheflush_time = (cpu_khz>>10) * (cachesize<<10) / bandwidth;
	}

	cache_decay_ticks = (long)cacheflush_time/cpu_khz * HZ / 1000;

	printk("per-CPU timeslice cutoff: %ld.%02ld usecs.\n",
		(long)cacheflush_time/(cpu_khz/1000),
		((long)cacheflush_time*100/(cpu_khz/1000)) % 100);
	printk("task migration cache decay timeout: %ld msecs.\n",
		(cache_decay_ticks + 1) * 1000 / HZ);
}

/*
 * Cycle through the processors sending APIC IPIs to boot each.
 */

775
static void __init smp_boot_cpus(unsigned int max_cpus)
776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798
{
	int apicid, cpu;

	/*
	 * Initialize the logical to physical CPU number mapping
	 */

	for (apicid = 0; apicid < NR_CPUS; apicid++) {
		x86_apicid_to_cpu[apicid] = -1;
	}

	/*
	 * Setup boot CPU information
	 */
	smp_store_cpu_info(0); /* Final full version of the data */
	printk("CPU%d: ", 0);
	print_cpu_info(&cpu_data[0]);

	x86_apicid_to_cpu[boot_cpu_id] = 0;
	x86_cpu_to_apicid[0] = boot_cpu_id;
	current_thread_info()->cpu = 0;
	smp_tune_scheduling();

799 800 801 802 803 804
	if (!test_bit(hard_smp_processor_id(), &phys_cpu_present_map)) {
		printk("weird, boot CPU (#%d) not listed by the BIOS.\n",
		       hard_smp_processor_id());
		phys_cpu_present_map |= (1 << hard_smp_processor_id());
	}

805 806 807 808 809 810 811 812
	/*
	 * If we couldnt find an SMP configuration at boot time,
	 * get out of here now!
	 */
	if (!smp_found_config) {
		printk(KERN_NOTICE "SMP motherboard not detected.\n");
		io_apic_irqs = 0;
		cpu_online_map = phys_cpu_present_map = 1;
813
		phys_cpu_present_map = 1;
814 815 816
		if (APIC_init_uniprocessor())
			printk(KERN_NOTICE "Local APIC not detected."
					   " Using dummy APIC emulation.\n");
817
		return;
818 819 820 821 822 823 824 825 826 827 828 829 830 831 832
	}

	/*
	 * Should not be necessary because the MP table should list the boot
	 * CPU too, but we do it for the sake of robustness anyway.
	 */
	if (!test_bit(boot_cpu_id, &phys_cpu_present_map)) {
		printk("weird, boot CPU (#%d) not listed by the BIOS.\n",
								 boot_cpu_id);
		phys_cpu_present_map |= (1 << hard_smp_processor_id());
	}

	/*
	 * If we couldn't find a local APIC, then get out of here now!
	 */
833
	if (APIC_INTEGRATED(apic_version[boot_cpu_id]) && !cpu_has_apic) {
834 835 836 837 838
		printk(KERN_ERR "BIOS bug, local APIC #%d not detected!...\n",
			boot_cpu_id);
		printk(KERN_ERR "... forcing use of dummy APIC emulation. (tell your hw vendor)\n");
		io_apic_irqs = 0;
		cpu_online_map = phys_cpu_present_map = 1;
839 840
		phys_cpu_present_map = 1;
		return;
841 842 843 844 845 846 847 848 849 850 851 852
	}

	verify_local_APIC();

	/*
	 * If SMP should be disabled, then really disable it!
	 */
	if (!max_cpus) {
		smp_found_config = 0;
		printk(KERN_INFO "SMP mode deactivated, forcing use of dummy APIC emulation.\n");
		io_apic_irqs = 0;
		cpu_online_map = phys_cpu_present_map = 1;
853 854
		phys_cpu_present_map = 1;
		return;
855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904
	}

	connect_bsp_APIC();
	setup_local_APIC();

	if (GET_APIC_ID(apic_read(APIC_ID)) != boot_cpu_id)
		BUG();

	/*
	 * Now scan the CPU present map and fire up the other CPUs.
	 */
	Dprintk("CPU present map: %lx\n", phys_cpu_present_map);

	for (apicid = 0; apicid < NR_CPUS; apicid++) {
		/*
		 * Don't even attempt to start the boot CPU!
		 */
		if (apicid == boot_cpu_id)
			continue;

		if (!(phys_cpu_present_map & (1 << apicid)))
			continue;
		if ((max_cpus >= 0) && (max_cpus <= cpucount+1))
			continue;

		do_boot_cpu(apicid);

		/*
		 * Make sure we unmap all failed CPUs
		 */
		if ((x86_apicid_to_cpu[apicid] == -1) &&
				(phys_cpu_present_map & (1 << apicid)))
			printk("phys CPU #%d not responding - cannot use it.\n",apicid);
	}

	/*
	 * Cleanup possible dangling ends...
	 */
	{
		/*
		 * Install writable page 0 entry to set BIOS data area.
		 */
		local_flush_tlb();

		/*
		 * Paranoid:  Set warm reset code and vector here back
		 * to default values.
		 */
		CMOS_WRITE(0, 0xf);

905
		*((volatile int *) phys_to_virt(0x467)) = 0;
906 907 908 909 910 911 912 913
	}

	/*
	 * Allow the user to impress friends.
	 */

	Dprintk("Before bogomips.\n");
	if (!cpucount) {
914
		printk(KERN_INFO "Only one processor found.\n");
915 916 917
	} else {
		unsigned long bogosum = 0;
		for (cpu = 0; cpu < NR_CPUS; cpu++)
918
			if (cpu_callout_map & (1<<cpu))
919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935
				bogosum += cpu_data[cpu].loops_per_jiffy;
		printk(KERN_INFO "Total of %d processors activated (%lu.%02lu BogoMIPS).\n",
			cpucount+1,
			bogosum/(500000/HZ),
			(bogosum/(5000/HZ))%100);
		Dprintk("Before bogocount - setting activated=1.\n");
	}

	Dprintk("Boot done.\n");

	/*
	 * Here we can be sure that there is an IO-APIC in the system. Let's
	 * go and set it up:
	 */
	if (!skip_ioapic_setup && nr_ioapics)
		setup_IO_APIC();

936
	setup_boot_APIC_clock();
937 938 939 940 941 942

	/*
	 * Synchronize the TSC with the AP
	 */
	if (cpu_has_tsc && cpucount)
		synchronize_tsc_bp();
943 944 945 946 947 948 949 950 951
}

/* These are wrappers to interface to the new boot process.  Someone
   who understands all this stuff should rewrite it properly. --RR 15/Jul/02 */
void __init smp_prepare_cpus(unsigned int max_cpus)
{
	smp_boot_cpus(max_cpus);
}

952 953 954 955 956 957
void __devinit smp_prepare_boot_cpu(void)
{
	set_bit(smp_processor_id(), &cpu_online_map);
	set_bit(smp_processor_id(), &cpu_callout_map);
}

958 959 960
int __devinit __cpu_up(unsigned int cpu)
{
	/* This only works at boot for x86.  See "rewrite" above. */
Andi Kleen's avatar
Andi Kleen committed
961 962
	if (test_bit(cpu, &smp_commenced_mask)) { 
		local_irq_enable();
963
		return -ENOSYS;
Andi Kleen's avatar
Andi Kleen committed
964
	}
965

966
	/* In case one didn't come up */
Andi Kleen's avatar
Andi Kleen committed
967 968
	if (!test_bit(cpu, &cpu_callin_map)) { 
		local_irq_enable();
969
		return -EIO;
Andi Kleen's avatar
Andi Kleen committed
970 971
	}
	local_irq_enable();
972 973 974 975 976 977 978 979 980 981 982 983

	/* Unleash the CPU! */
	Dprintk("waiting for cpu %d\n", cpu);

	set_bit(cpu, &smp_commenced_mask);
	while (!test_bit(cpu, &cpu_online_map))
		mb();
	return 0;
}

void __init smp_cpus_done(unsigned int max_cpus)
{
984 985
	zap_low_mappings();
}
986