Commit 93dc6887 authored by Catalin Marinas's avatar Catalin Marinas Committed by Russell King

ARM: 7684/1: errata: Workaround for Cortex-A15 erratum 798181 (TLBI/DSB operations)

On Cortex-A15 (r0p0..r3p2) the TLBI/DSB are not adequately shooting down
all use of the old entries. This patch implements the erratum workaround
which consists of:

1. Dummy TLBIMVAIS and DSB on the CPU doing the TLBI operation.
2. Send IPI to the CPUs that are running the same mm (and ASID) as the
   one being invalidated (or all the online CPUs for global pages).
3. CPU receiving the IPI executes a DMB and CLREX (part of the exception
   return code already).
Signed-off-by: default avatarCatalin Marinas <catalin.marinas@arm.com>
Signed-off-by: default avatarRussell King <rmk+kernel@arm.linux.org.uk>
parent 6e7aceeb
...@@ -1439,6 +1439,16 @@ config ARM_ERRATA_775420 ...@@ -1439,6 +1439,16 @@ config ARM_ERRATA_775420
to deadlock. This workaround puts DSB before executing ISB if to deadlock. This workaround puts DSB before executing ISB if
an abort may occur on cache maintenance. an abort may occur on cache maintenance.
config ARM_ERRATA_798181
bool "ARM errata: TLBI/DSB failure on Cortex-A15"
depends on CPU_V7 && SMP
help
On Cortex-A15 (r0p0..r3p2) the TLBI*IS/DSB operations are not
adequately shooting down all use of the old entries. This
option enables the Linux kernel workaround for this erratum
which sends an IPI to the CPUs that are running the same ASID
as the one being invalidated.
endmenu endmenu
source "arch/arm/common/Kconfig" source "arch/arm/common/Kconfig"
......
...@@ -41,6 +41,13 @@ extern void kunmap_high(struct page *page); ...@@ -41,6 +41,13 @@ extern void kunmap_high(struct page *page);
#endif #endif
#endif #endif
/*
* Needed to be able to broadcast the TLB invalidation for kmap.
*/
#ifdef CONFIG_ARM_ERRATA_798181
#undef ARCH_NEEDS_KMAP_HIGH_GET
#endif
#ifdef ARCH_NEEDS_KMAP_HIGH_GET #ifdef ARCH_NEEDS_KMAP_HIGH_GET
extern void *kmap_high_get(struct page *page); extern void *kmap_high_get(struct page *page);
#else #else
......
...@@ -27,6 +27,8 @@ void __check_vmalloc_seq(struct mm_struct *mm); ...@@ -27,6 +27,8 @@ void __check_vmalloc_seq(struct mm_struct *mm);
void check_and_switch_context(struct mm_struct *mm, struct task_struct *tsk); void check_and_switch_context(struct mm_struct *mm, struct task_struct *tsk);
#define init_new_context(tsk,mm) ({ atomic64_set(&mm->context.id, 0); 0; }) #define init_new_context(tsk,mm) ({ atomic64_set(&mm->context.id, 0); 0; })
DECLARE_PER_CPU(atomic64_t, active_asids);
#else /* !CONFIG_CPU_HAS_ASID */ #else /* !CONFIG_CPU_HAS_ASID */
#ifdef CONFIG_MMU #ifdef CONFIG_MMU
......
...@@ -450,6 +450,21 @@ static inline void local_flush_bp_all(void) ...@@ -450,6 +450,21 @@ static inline void local_flush_bp_all(void)
isb(); isb();
} }
#ifdef CONFIG_ARM_ERRATA_798181
static inline void dummy_flush_tlb_a15_erratum(void)
{
/*
* Dummy TLBIMVAIS. Using the unmapped address 0 and ASID 0.
*/
asm("mcr p15, 0, %0, c8, c3, 1" : : "r" (0));
dsb();
}
#else
static inline void dummy_flush_tlb_a15_erratum(void)
{
}
#endif
/* /*
* flush_pmd_entry * flush_pmd_entry
* *
......
...@@ -12,6 +12,7 @@ ...@@ -12,6 +12,7 @@
#include <asm/smp_plat.h> #include <asm/smp_plat.h>
#include <asm/tlbflush.h> #include <asm/tlbflush.h>
#include <asm/mmu_context.h>
/**********************************************************************/ /**********************************************************************/
...@@ -69,12 +70,72 @@ static inline void ipi_flush_bp_all(void *ignored) ...@@ -69,12 +70,72 @@ static inline void ipi_flush_bp_all(void *ignored)
local_flush_bp_all(); local_flush_bp_all();
} }
#ifdef CONFIG_ARM_ERRATA_798181
static int erratum_a15_798181(void)
{
unsigned int midr = read_cpuid_id();
/* Cortex-A15 r0p0..r3p2 affected */
if ((midr & 0xff0ffff0) != 0x410fc0f0 || midr > 0x413fc0f2)
return 0;
return 1;
}
#else
static int erratum_a15_798181(void)
{
return 0;
}
#endif
static void ipi_flush_tlb_a15_erratum(void *arg)
{
dmb();
}
static void broadcast_tlb_a15_erratum(void)
{
if (!erratum_a15_798181())
return;
dummy_flush_tlb_a15_erratum();
smp_call_function_many(cpu_online_mask, ipi_flush_tlb_a15_erratum,
NULL, 1);
}
static void broadcast_tlb_mm_a15_erratum(struct mm_struct *mm)
{
int cpu;
cpumask_t mask = { CPU_BITS_NONE };
if (!erratum_a15_798181())
return;
dummy_flush_tlb_a15_erratum();
for_each_online_cpu(cpu) {
if (cpu == smp_processor_id())
continue;
/*
* We only need to send an IPI if the other CPUs are running
* the same ASID as the one being invalidated. There is no
* need for locking around the active_asids check since the
* switch_mm() function has at least one dmb() (as required by
* this workaround) in case a context switch happens on
* another CPU after the condition below.
*/
if (atomic64_read(&mm->context.id) ==
atomic64_read(&per_cpu(active_asids, cpu)))
cpumask_set_cpu(cpu, &mask);
}
smp_call_function_many(&mask, ipi_flush_tlb_a15_erratum, NULL, 1);
}
void flush_tlb_all(void) void flush_tlb_all(void)
{ {
if (tlb_ops_need_broadcast()) if (tlb_ops_need_broadcast())
on_each_cpu(ipi_flush_tlb_all, NULL, 1); on_each_cpu(ipi_flush_tlb_all, NULL, 1);
else else
local_flush_tlb_all(); local_flush_tlb_all();
broadcast_tlb_a15_erratum();
} }
void flush_tlb_mm(struct mm_struct *mm) void flush_tlb_mm(struct mm_struct *mm)
...@@ -83,6 +144,7 @@ void flush_tlb_mm(struct mm_struct *mm) ...@@ -83,6 +144,7 @@ void flush_tlb_mm(struct mm_struct *mm)
on_each_cpu_mask(mm_cpumask(mm), ipi_flush_tlb_mm, mm, 1); on_each_cpu_mask(mm_cpumask(mm), ipi_flush_tlb_mm, mm, 1);
else else
local_flush_tlb_mm(mm); local_flush_tlb_mm(mm);
broadcast_tlb_mm_a15_erratum(mm);
} }
void flush_tlb_page(struct vm_area_struct *vma, unsigned long uaddr) void flush_tlb_page(struct vm_area_struct *vma, unsigned long uaddr)
...@@ -95,6 +157,7 @@ void flush_tlb_page(struct vm_area_struct *vma, unsigned long uaddr) ...@@ -95,6 +157,7 @@ void flush_tlb_page(struct vm_area_struct *vma, unsigned long uaddr)
&ta, 1); &ta, 1);
} else } else
local_flush_tlb_page(vma, uaddr); local_flush_tlb_page(vma, uaddr);
broadcast_tlb_mm_a15_erratum(vma->vm_mm);
} }
void flush_tlb_kernel_page(unsigned long kaddr) void flush_tlb_kernel_page(unsigned long kaddr)
...@@ -105,6 +168,7 @@ void flush_tlb_kernel_page(unsigned long kaddr) ...@@ -105,6 +168,7 @@ void flush_tlb_kernel_page(unsigned long kaddr)
on_each_cpu(ipi_flush_tlb_kernel_page, &ta, 1); on_each_cpu(ipi_flush_tlb_kernel_page, &ta, 1);
} else } else
local_flush_tlb_kernel_page(kaddr); local_flush_tlb_kernel_page(kaddr);
broadcast_tlb_a15_erratum();
} }
void flush_tlb_range(struct vm_area_struct *vma, void flush_tlb_range(struct vm_area_struct *vma,
...@@ -119,6 +183,7 @@ void flush_tlb_range(struct vm_area_struct *vma, ...@@ -119,6 +183,7 @@ void flush_tlb_range(struct vm_area_struct *vma,
&ta, 1); &ta, 1);
} else } else
local_flush_tlb_range(vma, start, end); local_flush_tlb_range(vma, start, end);
broadcast_tlb_mm_a15_erratum(vma->vm_mm);
} }
void flush_tlb_kernel_range(unsigned long start, unsigned long end) void flush_tlb_kernel_range(unsigned long start, unsigned long end)
...@@ -130,6 +195,7 @@ void flush_tlb_kernel_range(unsigned long start, unsigned long end) ...@@ -130,6 +195,7 @@ void flush_tlb_kernel_range(unsigned long start, unsigned long end)
on_each_cpu(ipi_flush_tlb_kernel_range, &ta, 1); on_each_cpu(ipi_flush_tlb_kernel_range, &ta, 1);
} else } else
local_flush_tlb_kernel_range(start, end); local_flush_tlb_kernel_range(start, end);
broadcast_tlb_a15_erratum();
} }
void flush_bp_all(void) void flush_bp_all(void)
......
...@@ -48,7 +48,7 @@ static DEFINE_RAW_SPINLOCK(cpu_asid_lock); ...@@ -48,7 +48,7 @@ static DEFINE_RAW_SPINLOCK(cpu_asid_lock);
static atomic64_t asid_generation = ATOMIC64_INIT(ASID_FIRST_VERSION); static atomic64_t asid_generation = ATOMIC64_INIT(ASID_FIRST_VERSION);
static DECLARE_BITMAP(asid_map, NUM_USER_ASIDS); static DECLARE_BITMAP(asid_map, NUM_USER_ASIDS);
static DEFINE_PER_CPU(atomic64_t, active_asids); DEFINE_PER_CPU(atomic64_t, active_asids);
static DEFINE_PER_CPU(u64, reserved_asids); static DEFINE_PER_CPU(u64, reserved_asids);
static cpumask_t tlb_flush_pending; static cpumask_t tlb_flush_pending;
...@@ -215,6 +215,7 @@ void check_and_switch_context(struct mm_struct *mm, struct task_struct *tsk) ...@@ -215,6 +215,7 @@ void check_and_switch_context(struct mm_struct *mm, struct task_struct *tsk)
if (cpumask_test_and_clear_cpu(cpu, &tlb_flush_pending)) { if (cpumask_test_and_clear_cpu(cpu, &tlb_flush_pending)) {
local_flush_bp_all(); local_flush_bp_all();
local_flush_tlb_all(); local_flush_tlb_all();
dummy_flush_tlb_a15_erratum();
} }
atomic64_set(&per_cpu(active_asids, cpu), asid); atomic64_set(&per_cpu(active_asids, cpu), asid);
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment