Commit b87ac021 authored by Nicholas Piggin's avatar Nicholas Piggin Committed by Michael Ellerman

powerpc: Introduce msgsnd/doorbell barrier primitives

POWER9 changes requirements and adds new instructions for
synchronization.
Signed-off-by: default avatarNicholas Piggin <npiggin@gmail.com>
Signed-off-by: default avatarMichael Ellerman <mpe@ellerman.id.au>
parent b866cc21
...@@ -44,6 +44,17 @@ static inline void _ppc_msgsnd(u32 msg) ...@@ -44,6 +44,17 @@ static inline void _ppc_msgsnd(u32 msg)
__asm__ __volatile__ (PPC_MSGSNDP(%0) : : "r" (msg)); __asm__ __volatile__ (PPC_MSGSNDP(%0) : : "r" (msg));
} }
/* sync before sending message */
static inline void ppc_msgsnd_sync(void)
{
__asm__ __volatile__ ("sync" : : : "memory");
}
/* sync after taking message interrupt */
static inline void ppc_msgsync(void)
{
}
#else /* CONFIG_PPC_BOOK3S */ #else /* CONFIG_PPC_BOOK3S */
#define PPC_DBELL_MSGTYPE PPC_DBELL #define PPC_DBELL_MSGTYPE PPC_DBELL
...@@ -53,6 +64,17 @@ static inline void _ppc_msgsnd(u32 msg) ...@@ -53,6 +64,17 @@ static inline void _ppc_msgsnd(u32 msg)
__asm__ __volatile__ (PPC_MSGSND(%0) : : "r" (msg)); __asm__ __volatile__ (PPC_MSGSND(%0) : : "r" (msg));
} }
/* sync before sending message */
static inline void ppc_msgsnd_sync(void)
{
__asm__ __volatile__ ("sync" : : : "memory");
}
/* sync after taking message interrupt */
static inline void ppc_msgsync(void)
{
}
#endif /* CONFIG_PPC_BOOK3S */ #endif /* CONFIG_PPC_BOOK3S */
extern void doorbell_global_ipi(int cpu); extern void doorbell_global_ipi(int cpu);
......
...@@ -128,6 +128,7 @@ extern const char *smp_ipi_name[]; ...@@ -128,6 +128,7 @@ extern const char *smp_ipi_name[];
extern void smp_muxed_ipi_message_pass(int cpu, int msg); extern void smp_muxed_ipi_message_pass(int cpu, int msg);
extern void smp_muxed_ipi_set_message(int cpu, int msg); extern void smp_muxed_ipi_set_message(int cpu, int msg);
extern irqreturn_t smp_ipi_demux(void); extern irqreturn_t smp_ipi_demux(void);
extern irqreturn_t smp_ipi_demux_relaxed(void);
void smp_init_pSeries(void); void smp_init_pSeries(void);
void smp_init_cell(void); void smp_init_cell(void);
......
...@@ -38,7 +38,7 @@ void doorbell_global_ipi(int cpu) ...@@ -38,7 +38,7 @@ void doorbell_global_ipi(int cpu)
kvmppc_set_host_ipi(cpu, 1); kvmppc_set_host_ipi(cpu, 1);
/* Order previous accesses vs. msgsnd, which is treated as a store */ /* Order previous accesses vs. msgsnd, which is treated as a store */
mb(); ppc_msgsnd_sync();
ppc_msgsnd(PPC_DBELL_MSGTYPE, 0, tag); ppc_msgsnd(PPC_DBELL_MSGTYPE, 0, tag);
} }
...@@ -53,7 +53,7 @@ void doorbell_core_ipi(int cpu) ...@@ -53,7 +53,7 @@ void doorbell_core_ipi(int cpu)
kvmppc_set_host_ipi(cpu, 1); kvmppc_set_host_ipi(cpu, 1);
/* Order previous accesses vs. msgsnd, which is treated as a store */ /* Order previous accesses vs. msgsnd, which is treated as a store */
mb(); ppc_msgsnd_sync();
ppc_msgsnd(PPC_DBELL_MSGTYPE, 0, tag); ppc_msgsnd(PPC_DBELL_MSGTYPE, 0, tag);
} }
...@@ -82,12 +82,14 @@ void doorbell_exception(struct pt_regs *regs) ...@@ -82,12 +82,14 @@ void doorbell_exception(struct pt_regs *regs)
irq_enter(); irq_enter();
ppc_msgsync();
may_hard_irq_enable(); may_hard_irq_enable();
kvmppc_set_host_ipi(smp_processor_id(), 0); kvmppc_set_host_ipi(smp_processor_id(), 0);
__this_cpu_inc(irq_stat.doorbell_irqs); __this_cpu_inc(irq_stat.doorbell_irqs);
smp_ipi_demux(); smp_ipi_demux_relaxed(); /* already performed the barrier */
irq_exit(); irq_exit();
set_irq_regs(old_regs); set_irq_regs(old_regs);
......
...@@ -245,12 +245,18 @@ void smp_muxed_ipi_message_pass(int cpu, int msg) ...@@ -245,12 +245,18 @@ void smp_muxed_ipi_message_pass(int cpu, int msg)
#endif #endif
irqreturn_t smp_ipi_demux(void) irqreturn_t smp_ipi_demux(void)
{
mb(); /* order any irq clear */
return smp_ipi_demux_relaxed();
}
/* sync-free variant. Callers should ensure synchronization */
irqreturn_t smp_ipi_demux_relaxed(void)
{ {
struct cpu_messages *info; struct cpu_messages *info;
unsigned long all; unsigned long all;
mb(); /* order any irq clear */
info = this_cpu_ptr(&ipi_message); info = this_cpu_ptr(&ipi_message);
do { do {
all = xchg(&info->messages, 0); all = xchg(&info->messages, 0);
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment