smp.c 8.26 KB
Newer Older
Linus Torvalds's avatar
Linus Torvalds committed
1 2 3 4
/*
 * SMP Support
 *
 * Copyright (C) 1999 Walt Drummond <drummond@valinux.com>
5
 * Copyright (C) 1999, 2001, 2003 David Mosberger-Tang <davidm@hpl.hp.com>
Linus Torvalds's avatar
Linus Torvalds committed
6
 *
Linus Torvalds's avatar
Linus Torvalds committed
7 8
 * Lots of stuff stolen from arch/alpha/kernel/smp.c
 *
Linus Torvalds's avatar
Linus Torvalds committed
9 10 11 12 13 14 15 16 17 18
 * 01/05/16 Rohit Seth <rohit.seth@intel.com>  IA64-SMP functions. Reorganized
 * the existing code (on the lines of x86 port).
 * 00/09/11 David Mosberger <davidm@hpl.hp.com> Do loops_per_jiffy
 * calibration on each CPU.
 * 00/08/23 Asit Mallick <asit.k.mallick@intel.com> fixed logical processor id
 * 00/03/31 Rohit Seth <rohit.seth@intel.com>	Fixes for Bootstrap Processor
 * & cpu_online_map now gets done here (instead of setup.c)
 * 99/10/05 davidm	Update to bring it in sync with new command-line processing
 *  scheme.
 * 10/13/00 Goutham Rao <goutham.rao@intel.com> Updated smp_call_function and
Linus Torvalds's avatar
Linus Torvalds committed
19 20
 *		smp_call_function_single to resend IPI on timeouts
 */
David Mosberger's avatar
David Mosberger committed
21
#include <linux/module.h>
Linus Torvalds's avatar
Linus Torvalds committed
22 23 24 25 26 27 28
#include <linux/kernel.h>
#include <linux/sched.h>
#include <linux/init.h>
#include <linux/interrupt.h>
#include <linux/smp.h>
#include <linux/kernel_stat.h>
#include <linux/mm.h>
David Mosberger's avatar
David Mosberger committed
29
#include <linux/cache.h>
Linus Torvalds's avatar
Linus Torvalds committed
30
#include <linux/delay.h>
Linus Torvalds's avatar
Linus Torvalds committed
31
#include <linux/cache.h>
32
#include <linux/efi.h>
Linus Torvalds's avatar
Linus Torvalds committed
33 34 35 36 37 38 39 40 41 42

#include <asm/atomic.h>
#include <asm/bitops.h>
#include <asm/current.h>
#include <asm/delay.h>
#include <asm/machvec.h>
#include <asm/io.h>
#include <asm/irq.h>
#include <asm/page.h>
#include <asm/pgalloc.h>
Linus Torvalds's avatar
Linus Torvalds committed
43
#include <asm/pgtable.h>
Linus Torvalds's avatar
Linus Torvalds committed
44 45 46 47
#include <asm/processor.h>
#include <asm/ptrace.h>
#include <asm/sal.h>
#include <asm/system.h>
48
#include <asm/tlbflush.h>
Linus Torvalds's avatar
Linus Torvalds committed
49
#include <asm/unistd.h>
Linus Torvalds's avatar
Linus Torvalds committed
50
#include <asm/mca.h>
Linus Torvalds's avatar
Linus Torvalds committed
51

Linus Torvalds's avatar
Linus Torvalds committed
52 53 54 55
/*
 * Structure and data for smp_call_function(). This is designed to minimise static memory
 * requirements. It also looks cleaner.
 */
David Mosberger's avatar
David Mosberger committed
56
static spinlock_t call_lock __cacheline_aligned = SPIN_LOCK_UNLOCKED;
Linus Torvalds's avatar
Linus Torvalds committed
57

Linus Torvalds's avatar
Linus Torvalds committed
58
struct call_data_struct {
Linus Torvalds's avatar
Linus Torvalds committed
59 60 61
	void (*func) (void *info);
	void *info;
	long wait;
Linus Torvalds's avatar
Linus Torvalds committed
62 63
	atomic_t started;
	atomic_t finished;
Linus Torvalds's avatar
Linus Torvalds committed
64 65
};

Linus Torvalds's avatar
Linus Torvalds committed
66
static volatile struct call_data_struct *call_data;
Linus Torvalds's avatar
Linus Torvalds committed
67

Linus Torvalds's avatar
Linus Torvalds committed
68 69
#define IPI_CALL_FUNC		0
#define IPI_CPU_STOP		1
Linus Torvalds's avatar
Linus Torvalds committed
70

71
/* This needs to be cacheline aligned because it is written to by *other* CPUs.  */
72
static DEFINE_PER_CPU(u64, ipi_operation) ____cacheline_aligned;
73

Linus Torvalds's avatar
Linus Torvalds committed
74 75
static void
stop_this_cpu (void)
Linus Torvalds's avatar
Linus Torvalds committed
76
{
Linus Torvalds's avatar
Linus Torvalds committed
77
	extern void cpu_halt (void);
Linus Torvalds's avatar
Linus Torvalds committed
78 79 80
	/*
	 * Remove this CPU:
	 */
81
	cpu_clear(smp_processor_id(), cpu_online_map);
Linus Torvalds's avatar
Linus Torvalds committed
82
	max_xtp();
83
	local_irq_disable();
Linus Torvalds's avatar
Linus Torvalds committed
84
	cpu_halt();
Linus Torvalds's avatar
Linus Torvalds committed
85 86
}

87
irqreturn_t
Linus Torvalds's avatar
Linus Torvalds committed
88
handle_IPI (int irq, void *dev_id, struct pt_regs *regs)
Linus Torvalds's avatar
Linus Torvalds committed
89
{
90
	int this_cpu = get_cpu();
91
	unsigned long *pending_ipis = &__ia64_per_cpu_var(ipi_operation);
Linus Torvalds's avatar
Linus Torvalds committed
92 93 94 95
	unsigned long ops;

	mb();	/* Order interrupt and bit testing. */
	while ((ops = xchg(pending_ipis, 0)) != 0) {
David Mosberger's avatar
David Mosberger committed
96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138
		mb();	/* Order bit clearing and data access. */
		do {
			unsigned long which;

			which = ffz(~ops);
			ops &= ~(1 << which);

			switch (which) {
			      case IPI_CALL_FUNC:
			      {
				      struct call_data_struct *data;
				      void (*func)(void *info);
				      void *info;
				      int wait;

				      /* release the 'pointer lock' */
				      data = (struct call_data_struct *) call_data;
				      func = data->func;
				      info = data->info;
				      wait = data->wait;

				      mb();
				      atomic_inc(&data->started);
				      /*
				       * At this point the structure may be gone unless
				       * wait is true.
				       */
				      (*func)(info);

				      /* Notify the sending CPU that the task is done.  */
				      mb();
				      if (wait)
					      atomic_inc(&data->finished);
			      }
			      break;

			      case IPI_CPU_STOP:
				stop_this_cpu();
				break;

			      default:
				printk(KERN_CRIT "Unknown IPI on CPU %d: %lu\n", this_cpu, which);
				break;
Linus Torvalds's avatar
Linus Torvalds committed
139
			}
David Mosberger's avatar
David Mosberger committed
140 141
		} while (ops);
		mb();	/* Order data access and bit testing. */
Linus Torvalds's avatar
Linus Torvalds committed
142
	}
143
	put_cpu();
144
	return IRQ_HANDLED;
Linus Torvalds's avatar
Linus Torvalds committed
145 146
}

147
/*
148
 * Called with preeemption disabled.
149
 */
Linus Torvalds's avatar
Linus Torvalds committed
150
static inline void
Linus Torvalds's avatar
Linus Torvalds committed
151
send_IPI_single (int dest_cpu, int op)
Linus Torvalds's avatar
Linus Torvalds committed
152
{
153
	set_bit(op, &per_cpu(ipi_operation, dest_cpu));
Linus Torvalds's avatar
Linus Torvalds committed
154
	platform_send_ipi(dest_cpu, IA64_IPI_VECTOR, IA64_IPI_DM_INT, 0);
Linus Torvalds's avatar
Linus Torvalds committed
155 156
}

157
/*
158
 * Called with preeemption disabled.
159
 */
Linus Torvalds's avatar
Linus Torvalds committed
160
static inline void
Linus Torvalds's avatar
Linus Torvalds committed
161
send_IPI_allbutself (int op)
Linus Torvalds's avatar
Linus Torvalds committed
162
{
163
	unsigned int i;
Linus Torvalds's avatar
Linus Torvalds committed
164

165 166
	for (i = 0; i < NR_CPUS; i++) {
		if (cpu_online(i) && i != smp_processor_id())
Linus Torvalds's avatar
Linus Torvalds committed
167 168 169 170
			send_IPI_single(i, op);
	}
}

171
/*
172
 * Called with preeemption disabled.
173
 */
Linus Torvalds's avatar
Linus Torvalds committed
174
static inline void
Linus Torvalds's avatar
Linus Torvalds committed
175
send_IPI_all (int op)
Linus Torvalds's avatar
Linus Torvalds committed
176 177 178
{
	int i;

179 180 181
	for (i = 0; i < NR_CPUS; i++)
		if (cpu_online(i))
			send_IPI_single(i, op);
Linus Torvalds's avatar
Linus Torvalds committed
182 183
}

184
/*
185
 * Called with preeemption disabled.
186
 */
Linus Torvalds's avatar
Linus Torvalds committed
187
static inline void
Linus Torvalds's avatar
Linus Torvalds committed
188
send_IPI_self (int op)
Linus Torvalds's avatar
Linus Torvalds committed
189 190 191 192
{
	send_IPI_single(smp_processor_id(), op);
}

193
/*
194
 * Called with preeemption disabled.
195
 */
Linus Torvalds's avatar
Linus Torvalds committed
196
void
Linus Torvalds's avatar
Linus Torvalds committed
197
smp_send_reschedule (int cpu)
Linus Torvalds's avatar
Linus Torvalds committed
198
{
Linus Torvalds's avatar
Linus Torvalds committed
199
	platform_send_ipi(cpu, IA64_IPI_RESCHEDULE, IA64_IPI_DM_INT, 0);
Linus Torvalds's avatar
Linus Torvalds committed
200 201
}

Linus Torvalds's avatar
Linus Torvalds committed
202 203 204
void
smp_flush_tlb_all (void)
{
205
	on_each_cpu((void (*)(void *))local_flush_tlb_all, 0, 1, 1);
Linus Torvalds's avatar
Linus Torvalds committed
206
}
David Mosberger's avatar
David Mosberger committed
207
EXPORT_SYMBOL(smp_flush_tlb_all);
Linus Torvalds's avatar
Linus Torvalds committed
208

209 210 211 212 213
void
smp_flush_tlb_mm (struct mm_struct *mm)
{
	/* this happens for the common case of a single-threaded fork():  */
	if (likely(mm == current->active_mm && atomic_read(&mm->mm_users) == 1))
214 215
	{
		local_finish_flush_tlb_mm(mm);
216
		return;
217
	}
218

219 220 221 222 223 224 225
	/*
	 * We could optimize this further by using mm->cpu_vm_mask to track which CPUs
	 * have been running in the address space.  It's not clear that this is worth the
	 * trouble though: to avoid races, we have to raise the IPI on the target CPU
	 * anyhow, and once a CPU is interrupted, the cost of local_flush_tlb_all() is
	 * rather trivial.
	 */
226
	on_each_cpu((void (*)(void *))local_finish_flush_tlb_mm, mm, 1, 1);
227 228
}

Linus Torvalds's avatar
Linus Torvalds committed
229 230 231 232
/*
 * Run a function on another CPU
 *  <func>	The function to run. This must be fast and non-blocking.
 *  <info>	An arbitrary pointer to pass to the function.
Linus Torvalds's avatar
Linus Torvalds committed
233
 *  <nonatomic>	Currently unused.
Linus Torvalds's avatar
Linus Torvalds committed
234 235 236 237 238 239 240 241
 *  <wait>	If true, wait until function has completed on other CPUs.
 *  [RETURNS]   0 on success, else a negative status code.
 *
 * Does not return until the remote CPU is nearly ready to execute <func>
 * or is or has executed.
 */

int
Linus Torvalds's avatar
Linus Torvalds committed
242 243
smp_call_function_single (int cpuid, void (*func) (void *info), void *info, int nonatomic,
			  int wait)
Linus Torvalds's avatar
Linus Torvalds committed
244
{
Linus Torvalds's avatar
Linus Torvalds committed
245
	struct call_data_struct data;
Linus Torvalds's avatar
Linus Torvalds committed
246
	int cpus = 1;
247
	int me = get_cpu(); /* prevent preemption and reschedule on another processor */
Linus Torvalds's avatar
Linus Torvalds committed
248

249
	if (cpuid == me) {
250
		printk("%s: trying to call self\n", __FUNCTION__);
251
		put_cpu();
Linus Torvalds's avatar
Linus Torvalds committed
252 253
		return -EBUSY;
	}
Linus Torvalds's avatar
Linus Torvalds committed
254

Linus Torvalds's avatar
Linus Torvalds committed
255 256
	data.func = func;
	data.info = info;
Linus Torvalds's avatar
Linus Torvalds committed
257
	atomic_set(&data.started, 0);
Linus Torvalds's avatar
Linus Torvalds committed
258
	data.wait = wait;
Linus Torvalds's avatar
Linus Torvalds committed
259 260
	if (wait)
		atomic_set(&data.finished, 0);
Linus Torvalds's avatar
Linus Torvalds committed
261

Linus Torvalds's avatar
Linus Torvalds committed
262
	spin_lock_bh(&call_lock);
Linus Torvalds's avatar
Linus Torvalds committed
263

Linus Torvalds's avatar
Linus Torvalds committed
264 265
	call_data = &data;
	mb();	/* ensure store to call_data precedes setting of IPI_CALL_FUNC */
Linus Torvalds's avatar
Linus Torvalds committed
266 267 268 269 270
  	send_IPI_single(cpuid, IPI_CALL_FUNC);

	/* Wait for response */
	while (atomic_read(&data.started) != cpus)
		barrier();
Linus Torvalds's avatar
Linus Torvalds committed
271

Linus Torvalds's avatar
Linus Torvalds committed
272
	if (wait)
Linus Torvalds's avatar
Linus Torvalds committed
273
		while (atomic_read(&data.finished) != cpus)
Linus Torvalds's avatar
Linus Torvalds committed
274
			barrier();
Linus Torvalds's avatar
Linus Torvalds committed
275 276 277
	call_data = NULL;

	spin_unlock_bh(&call_lock);
278
	put_cpu();
Linus Torvalds's avatar
Linus Torvalds committed
279 280
	return 0;
}
David Mosberger's avatar
David Mosberger committed
281
EXPORT_SYMBOL(smp_call_function_single);
Linus Torvalds's avatar
Linus Torvalds committed
282 283

/*
Linus Torvalds's avatar
Linus Torvalds committed
284 285 286 287 288 289
 * this function sends a 'generic call function' IPI to all other CPUs
 * in the system.
 */

/*
 *  [SUMMARY]	Run a function on all other CPUs.
Linus Torvalds's avatar
Linus Torvalds committed
290 291
 *  <func>	The function to run. This must be fast and non-blocking.
 *  <info>	An arbitrary pointer to pass to the function.
Linus Torvalds's avatar
Linus Torvalds committed
292 293
 *  <nonatomic>	currently unused.
 *  <wait>	If true, wait (atomically) until function has completed on other CPUs.
Linus Torvalds's avatar
Linus Torvalds committed
294 295
 *  [RETURNS]   0 on success, else a negative status code.
 *
Linus Torvalds's avatar
Linus Torvalds committed
296 297 298
 * Does not return until remote CPUs are nearly ready to execute <func> or are or have
 * executed.
 *
299 300
 * You must not call this function with disabled interrupts or from a
 * hardware interrupt handler or from a bottom half handler.
Linus Torvalds's avatar
Linus Torvalds committed
301 302
 */
int
Linus Torvalds's avatar
Linus Torvalds committed
303
smp_call_function (void (*func) (void *info), void *info, int nonatomic, int wait)
Linus Torvalds's avatar
Linus Torvalds committed
304
{
Linus Torvalds's avatar
Linus Torvalds committed
305
	struct call_data_struct data;
306
	int cpus = num_online_cpus()-1;
Linus Torvalds's avatar
Linus Torvalds committed
307

Linus Torvalds's avatar
Linus Torvalds committed
308
	if (!cpus)
Linus Torvalds's avatar
Linus Torvalds committed
309
		return 0;
Linus Torvalds's avatar
Linus Torvalds committed
310

Linus Torvalds's avatar
Linus Torvalds committed
311 312
	data.func = func;
	data.info = info;
Linus Torvalds's avatar
Linus Torvalds committed
313
	atomic_set(&data.started, 0);
Linus Torvalds's avatar
Linus Torvalds committed
314
	data.wait = wait;
Linus Torvalds's avatar
Linus Torvalds committed
315 316
	if (wait)
		atomic_set(&data.finished, 0);
Linus Torvalds's avatar
Linus Torvalds committed
317

318
	spin_lock(&call_lock);
Linus Torvalds's avatar
Linus Torvalds committed
319

Linus Torvalds's avatar
Linus Torvalds committed
320 321
	call_data = &data;
	mb();	/* ensure store to call_data precedes setting of IPI_CALL_FUNC */
Linus Torvalds's avatar
Linus Torvalds committed
322 323 324 325 326 327
	send_IPI_allbutself(IPI_CALL_FUNC);

	/* Wait for response */
	while (atomic_read(&data.started) != cpus)
		barrier();

Linus Torvalds's avatar
Linus Torvalds committed
328
	if (wait)
Linus Torvalds's avatar
Linus Torvalds committed
329
		while (atomic_read(&data.finished) != cpus)
Linus Torvalds's avatar
Linus Torvalds committed
330
			barrier();
Linus Torvalds's avatar
Linus Torvalds committed
331
	call_data = NULL;
Linus Torvalds's avatar
Linus Torvalds committed
332

333
	spin_unlock(&call_lock);
Linus Torvalds's avatar
Linus Torvalds committed
334
	return 0;
Linus Torvalds's avatar
Linus Torvalds committed
335
}
David Mosberger's avatar
David Mosberger committed
336
EXPORT_SYMBOL(smp_call_function);
Linus Torvalds's avatar
Linus Torvalds committed
337

Linus Torvalds's avatar
Linus Torvalds committed
338
/*
Linus Torvalds's avatar
Linus Torvalds committed
339
 * this function calls the 'stop' function on all other CPUs in the system.
Linus Torvalds's avatar
Linus Torvalds committed
340
 */
Linus Torvalds's avatar
Linus Torvalds committed
341 342
void
smp_send_stop (void)
Linus Torvalds's avatar
Linus Torvalds committed
343
{
Linus Torvalds's avatar
Linus Torvalds committed
344
	send_IPI_allbutself(IPI_CPU_STOP);
Linus Torvalds's avatar
Linus Torvalds committed
345 346 347
}

int __init
Linus Torvalds's avatar
Linus Torvalds committed
348
setup_profiling_timer (unsigned int multiplier)
Linus Torvalds's avatar
Linus Torvalds committed
349
{
Linus Torvalds's avatar
Linus Torvalds committed
350
	return -EINVAL;
Linus Torvalds's avatar
Linus Torvalds committed
351
}