Commit 04ed7d9c authored by Linus Torvalds's avatar Linus Torvalds

Merge git://git.kernel.org/pub/scm/linux/kernel/git/davem/sparc

Pull sparc fixes from David Miller:
 "Several sparc64 bug fixes here:

  1) Make the user copy routines on sparc64 return a properly accurate
     residual length when an exception occurs.

  2) We can get enormous kernel TLB range flush requests from vmalloc
     unmaps, so handle these more gracefully by doing full flushes
     instead of going page-by-page.

  3) Cope properly with negative branch offsets in sparc jump-label
     support, from James Clarke.

  4) Some old-style decl GCC warning fixups from Tobias Klauser"

* git://git.kernel.org/pub/scm/linux/kernel/git/davem/sparc:
  sparc64: Handle extremely large kernel TLB range flushes more gracefully.
  sparc64: Fix illegal relative branches in hypervisor patched TLB cross-call code.
  sparc64: Fix instruction count in comment for __hypervisor_flush_tlb_pending.
  sparc64: Handle extremely large kernel TSB range flushes sanely.
  sparc: Handle negative offsets in arch_jump_label_transform
  sparc64: Fix illegal relative branches in hypervisor patched TLB code.
  sparc64: Delete now unused user copy fixup functions.
  sparc64: Delete now unused user copy assembler helpers.
  sparc64: Convert U3copy_{from,to}_user to accurate exception reporting.
  sparc64: Convert NG2copy_{from,to}_user to accurate exception reporting.
  sparc64: Convert NGcopy_{from,to}_user to accurate exception reporting.
  sparc64: Convert NG4copy_{from,to}_user to accurate exception reporting.
  sparc64: Convert U1copy_{from,to}_user to accurate exception reporting.
  sparc64: Convert GENcopy_{from,to}_user to accurate exception reporting.
  sparc64: Convert copy_in_user to accurate exception reporting.
  sparc64: Prepare to move to more saner user copy exception handling.
  sparc64: Delete __ret_efault.
  sparc32: Fix old style declaration GCC warnings
  sparc64: Fix old style declaration GCC warnings
  sparc64: Setup a scheduling domain for highest level cache.
parents 2a26d99b a74ad5e6
...@@ -24,9 +24,10 @@ typedef struct { ...@@ -24,9 +24,10 @@ typedef struct {
unsigned int icache_line_size; unsigned int icache_line_size;
unsigned int ecache_size; unsigned int ecache_size;
unsigned int ecache_line_size; unsigned int ecache_line_size;
unsigned short sock_id; unsigned short sock_id; /* physical package */
unsigned short core_id; unsigned short core_id;
int proc_id; unsigned short max_cache_id; /* groupings of highest shared cache */
unsigned short proc_id; /* strand (aka HW thread) id */
} cpuinfo_sparc; } cpuinfo_sparc;
DECLARE_PER_CPU(cpuinfo_sparc, __cpu_data); DECLARE_PER_CPU(cpuinfo_sparc, __cpu_data);
......
...@@ -134,7 +134,7 @@ static inline void arch_write_lock(arch_rwlock_t *rw) ...@@ -134,7 +134,7 @@ static inline void arch_write_lock(arch_rwlock_t *rw)
*(volatile __u32 *)&lp->lock = ~0U; *(volatile __u32 *)&lp->lock = ~0U;
} }
static void inline arch_write_unlock(arch_rwlock_t *lock) static inline void arch_write_unlock(arch_rwlock_t *lock)
{ {
__asm__ __volatile__( __asm__ __volatile__(
" st %%g0, [%0]" " st %%g0, [%0]"
......
...@@ -96,7 +96,7 @@ static inline void arch_spin_lock_flags(arch_spinlock_t *lock, unsigned long fla ...@@ -96,7 +96,7 @@ static inline void arch_spin_lock_flags(arch_spinlock_t *lock, unsigned long fla
/* Multi-reader locks, these are much saner than the 32-bit Sparc ones... */ /* Multi-reader locks, these are much saner than the 32-bit Sparc ones... */
static void inline arch_read_lock(arch_rwlock_t *lock) static inline void arch_read_lock(arch_rwlock_t *lock)
{ {
unsigned long tmp1, tmp2; unsigned long tmp1, tmp2;
...@@ -119,7 +119,7 @@ static void inline arch_read_lock(arch_rwlock_t *lock) ...@@ -119,7 +119,7 @@ static void inline arch_read_lock(arch_rwlock_t *lock)
: "memory"); : "memory");
} }
static int inline arch_read_trylock(arch_rwlock_t *lock) static inline int arch_read_trylock(arch_rwlock_t *lock)
{ {
int tmp1, tmp2; int tmp1, tmp2;
...@@ -140,7 +140,7 @@ static int inline arch_read_trylock(arch_rwlock_t *lock) ...@@ -140,7 +140,7 @@ static int inline arch_read_trylock(arch_rwlock_t *lock)
return tmp1; return tmp1;
} }
static void inline arch_read_unlock(arch_rwlock_t *lock) static inline void arch_read_unlock(arch_rwlock_t *lock)
{ {
unsigned long tmp1, tmp2; unsigned long tmp1, tmp2;
...@@ -156,7 +156,7 @@ static void inline arch_read_unlock(arch_rwlock_t *lock) ...@@ -156,7 +156,7 @@ static void inline arch_read_unlock(arch_rwlock_t *lock)
: "memory"); : "memory");
} }
static void inline arch_write_lock(arch_rwlock_t *lock) static inline void arch_write_lock(arch_rwlock_t *lock)
{ {
unsigned long mask, tmp1, tmp2; unsigned long mask, tmp1, tmp2;
...@@ -181,7 +181,7 @@ static void inline arch_write_lock(arch_rwlock_t *lock) ...@@ -181,7 +181,7 @@ static void inline arch_write_lock(arch_rwlock_t *lock)
: "memory"); : "memory");
} }
static void inline arch_write_unlock(arch_rwlock_t *lock) static inline void arch_write_unlock(arch_rwlock_t *lock)
{ {
__asm__ __volatile__( __asm__ __volatile__(
" stw %%g0, [%0]" " stw %%g0, [%0]"
...@@ -190,7 +190,7 @@ static void inline arch_write_unlock(arch_rwlock_t *lock) ...@@ -190,7 +190,7 @@ static void inline arch_write_unlock(arch_rwlock_t *lock)
: "memory"); : "memory");
} }
static int inline arch_write_trylock(arch_rwlock_t *lock) static inline int arch_write_trylock(arch_rwlock_t *lock)
{ {
unsigned long mask, tmp1, tmp2, result; unsigned long mask, tmp1, tmp2, result;
......
...@@ -44,14 +44,20 @@ int __node_distance(int, int); ...@@ -44,14 +44,20 @@ int __node_distance(int, int);
#define topology_physical_package_id(cpu) (cpu_data(cpu).proc_id) #define topology_physical_package_id(cpu) (cpu_data(cpu).proc_id)
#define topology_core_id(cpu) (cpu_data(cpu).core_id) #define topology_core_id(cpu) (cpu_data(cpu).core_id)
#define topology_core_cpumask(cpu) (&cpu_core_sib_map[cpu]) #define topology_core_cpumask(cpu) (&cpu_core_sib_map[cpu])
#define topology_core_cache_cpumask(cpu) (&cpu_core_sib_cache_map[cpu])
#define topology_sibling_cpumask(cpu) (&per_cpu(cpu_sibling_map, cpu)) #define topology_sibling_cpumask(cpu) (&per_cpu(cpu_sibling_map, cpu))
#endif /* CONFIG_SMP */ #endif /* CONFIG_SMP */
extern cpumask_t cpu_core_map[NR_CPUS]; extern cpumask_t cpu_core_map[NR_CPUS];
extern cpumask_t cpu_core_sib_map[NR_CPUS]; extern cpumask_t cpu_core_sib_map[NR_CPUS];
extern cpumask_t cpu_core_sib_cache_map[NR_CPUS];
/**
* Return cores that shares the last level cache.
*/
static inline const struct cpumask *cpu_coregroup_mask(int cpu) static inline const struct cpumask *cpu_coregroup_mask(int cpu)
{ {
return &cpu_core_map[cpu]; return &cpu_core_sib_cache_map[cpu];
} }
#endif /* _ASM_SPARC64_TOPOLOGY_H */ #endif /* _ASM_SPARC64_TOPOLOGY_H */
...@@ -82,7 +82,6 @@ static inline int access_ok(int type, const void __user * addr, unsigned long si ...@@ -82,7 +82,6 @@ static inline int access_ok(int type, const void __user * addr, unsigned long si
return 1; return 1;
} }
void __ret_efault(void);
void __retl_efault(void); void __retl_efault(void);
/* Uh, these should become the main single-value transfer routines.. /* Uh, these should become the main single-value transfer routines..
...@@ -189,55 +188,34 @@ int __get_user_bad(void); ...@@ -189,55 +188,34 @@ int __get_user_bad(void);
unsigned long __must_check ___copy_from_user(void *to, unsigned long __must_check ___copy_from_user(void *to,
const void __user *from, const void __user *from,
unsigned long size); unsigned long size);
unsigned long copy_from_user_fixup(void *to, const void __user *from,
unsigned long size);
static inline unsigned long __must_check static inline unsigned long __must_check
copy_from_user(void *to, const void __user *from, unsigned long size) copy_from_user(void *to, const void __user *from, unsigned long size)
{ {
unsigned long ret;
check_object_size(to, size, false); check_object_size(to, size, false);
ret = ___copy_from_user(to, from, size); return ___copy_from_user(to, from, size);
if (unlikely(ret))
ret = copy_from_user_fixup(to, from, size);
return ret;
} }
#define __copy_from_user copy_from_user #define __copy_from_user copy_from_user
unsigned long __must_check ___copy_to_user(void __user *to, unsigned long __must_check ___copy_to_user(void __user *to,
const void *from, const void *from,
unsigned long size); unsigned long size);
unsigned long copy_to_user_fixup(void __user *to, const void *from,
unsigned long size);
static inline unsigned long __must_check static inline unsigned long __must_check
copy_to_user(void __user *to, const void *from, unsigned long size) copy_to_user(void __user *to, const void *from, unsigned long size)
{ {
unsigned long ret;
check_object_size(from, size, true); check_object_size(from, size, true);
ret = ___copy_to_user(to, from, size); return ___copy_to_user(to, from, size);
if (unlikely(ret))
ret = copy_to_user_fixup(to, from, size);
return ret;
} }
#define __copy_to_user copy_to_user #define __copy_to_user copy_to_user
unsigned long __must_check ___copy_in_user(void __user *to, unsigned long __must_check ___copy_in_user(void __user *to,
const void __user *from, const void __user *from,
unsigned long size); unsigned long size);
unsigned long copy_in_user_fixup(void __user *to, void __user *from,
unsigned long size);
static inline unsigned long __must_check static inline unsigned long __must_check
copy_in_user(void __user *to, void __user *from, unsigned long size) copy_in_user(void __user *to, void __user *from, unsigned long size)
{ {
unsigned long ret = ___copy_in_user(to, from, size); return ___copy_in_user(to, from, size);
if (unlikely(ret))
ret = copy_in_user_fixup(to, from, size);
return ret;
} }
#define __copy_in_user copy_in_user #define __copy_in_user copy_in_user
......
...@@ -926,48 +926,11 @@ tlb_type: .word 0 /* Must NOT end up in BSS */ ...@@ -926,48 +926,11 @@ tlb_type: .word 0 /* Must NOT end up in BSS */
EXPORT_SYMBOL(tlb_type) EXPORT_SYMBOL(tlb_type)
.section ".fixup",#alloc,#execinstr .section ".fixup",#alloc,#execinstr
.globl __ret_efault, __retl_efault, __ret_one, __retl_one
ENTRY(__ret_efault)
ret
restore %g0, -EFAULT, %o0
ENDPROC(__ret_efault)
EXPORT_SYMBOL(__ret_efault)
ENTRY(__retl_efault) ENTRY(__retl_efault)
retl retl
mov -EFAULT, %o0 mov -EFAULT, %o0
ENDPROC(__retl_efault) ENDPROC(__retl_efault)
ENTRY(__retl_one)
retl
mov 1, %o0
ENDPROC(__retl_one)
ENTRY(__retl_one_fp)
VISExitHalf
retl
mov 1, %o0
ENDPROC(__retl_one_fp)
ENTRY(__ret_one_asi)
wr %g0, ASI_AIUS, %asi
ret
restore %g0, 1, %o0
ENDPROC(__ret_one_asi)
ENTRY(__retl_one_asi)
wr %g0, ASI_AIUS, %asi
retl
mov 1, %o0
ENDPROC(__retl_one_asi)
ENTRY(__retl_one_asi_fp)
wr %g0, ASI_AIUS, %asi
VISExitHalf
retl
mov 1, %o0
ENDPROC(__retl_one_asi_fp)
ENTRY(__retl_o1) ENTRY(__retl_o1)
retl retl
mov %o1, %o0 mov %o1, %o0
......
...@@ -13,19 +13,30 @@ ...@@ -13,19 +13,30 @@
void arch_jump_label_transform(struct jump_entry *entry, void arch_jump_label_transform(struct jump_entry *entry,
enum jump_label_type type) enum jump_label_type type)
{ {
u32 val;
u32 *insn = (u32 *) (unsigned long) entry->code; u32 *insn = (u32 *) (unsigned long) entry->code;
u32 val;
if (type == JUMP_LABEL_JMP) { if (type == JUMP_LABEL_JMP) {
s32 off = (s32)entry->target - (s32)entry->code; s32 off = (s32)entry->target - (s32)entry->code;
bool use_v9_branch = false;
BUG_ON(off & 3);
#ifdef CONFIG_SPARC64 #ifdef CONFIG_SPARC64
/* ba,pt %xcc, . + (off << 2) */ if (off <= 0xfffff && off >= -0x100000)
val = 0x10680000 | ((u32) off >> 2); use_v9_branch = true;
#else
/* ba . + (off << 2) */
val = 0x10800000 | ((u32) off >> 2);
#endif #endif
if (use_v9_branch) {
/* WDISP19 - target is . + immed << 2 */
/* ba,pt %xcc, . + off */
val = 0x10680000 | (((u32) off >> 2) & 0x7ffff);
} else {
/* WDISP22 - target is . + immed << 2 */
BUG_ON(off > 0x7fffff);
BUG_ON(off < -0x800000);
/* ba . + off */
val = 0x10800000 | (((u32) off >> 2) & 0x3fffff);
}
} else { } else {
val = 0x01000000; val = 0x01000000;
} }
......
...@@ -645,13 +645,20 @@ static void __mark_core_id(struct mdesc_handle *hp, u64 node, ...@@ -645,13 +645,20 @@ static void __mark_core_id(struct mdesc_handle *hp, u64 node,
cpu_data(*id).core_id = core_id; cpu_data(*id).core_id = core_id;
} }
static void __mark_sock_id(struct mdesc_handle *hp, u64 node, static void __mark_max_cache_id(struct mdesc_handle *hp, u64 node,
int sock_id) int max_cache_id)
{ {
const u64 *id = mdesc_get_property(hp, node, "id", NULL); const u64 *id = mdesc_get_property(hp, node, "id", NULL);
if (*id < num_possible_cpus()) if (*id < num_possible_cpus()) {
cpu_data(*id).sock_id = sock_id; cpu_data(*id).max_cache_id = max_cache_id;
/**
* On systems without explicit socket descriptions socket
* is max_cache_id
*/
cpu_data(*id).sock_id = max_cache_id;
}
} }
static void mark_core_ids(struct mdesc_handle *hp, u64 mp, static void mark_core_ids(struct mdesc_handle *hp, u64 mp,
...@@ -660,10 +667,11 @@ static void mark_core_ids(struct mdesc_handle *hp, u64 mp, ...@@ -660,10 +667,11 @@ static void mark_core_ids(struct mdesc_handle *hp, u64 mp,
find_back_node_value(hp, mp, "cpu", __mark_core_id, core_id, 10); find_back_node_value(hp, mp, "cpu", __mark_core_id, core_id, 10);
} }
static void mark_sock_ids(struct mdesc_handle *hp, u64 mp, static void mark_max_cache_ids(struct mdesc_handle *hp, u64 mp,
int sock_id) int max_cache_id)
{ {
find_back_node_value(hp, mp, "cpu", __mark_sock_id, sock_id, 10); find_back_node_value(hp, mp, "cpu", __mark_max_cache_id,
max_cache_id, 10);
} }
static void set_core_ids(struct mdesc_handle *hp) static void set_core_ids(struct mdesc_handle *hp)
...@@ -694,14 +702,15 @@ static void set_core_ids(struct mdesc_handle *hp) ...@@ -694,14 +702,15 @@ static void set_core_ids(struct mdesc_handle *hp)
} }
} }
static int set_sock_ids_by_cache(struct mdesc_handle *hp, int level) static int set_max_cache_ids_by_cache(struct mdesc_handle *hp, int level)
{ {
u64 mp; u64 mp;
int idx = 1; int idx = 1;
int fnd = 0; int fnd = 0;
/* Identify unique sockets by looking for cpus backpointed to by /**
* shared level n caches. * Identify unique highest level of shared cache by looking for cpus
* backpointed to by shared level N caches.
*/ */
mdesc_for_each_node_by_name(hp, mp, "cache") { mdesc_for_each_node_by_name(hp, mp, "cache") {
const u64 *cur_lvl; const u64 *cur_lvl;
...@@ -709,8 +718,7 @@ static int set_sock_ids_by_cache(struct mdesc_handle *hp, int level) ...@@ -709,8 +718,7 @@ static int set_sock_ids_by_cache(struct mdesc_handle *hp, int level)
cur_lvl = mdesc_get_property(hp, mp, "level", NULL); cur_lvl = mdesc_get_property(hp, mp, "level", NULL);
if (*cur_lvl != level) if (*cur_lvl != level)
continue; continue;
mark_max_cache_ids(hp, mp, idx);
mark_sock_ids(hp, mp, idx);
idx++; idx++;
fnd = 1; fnd = 1;
} }
...@@ -745,15 +753,17 @@ static void set_sock_ids(struct mdesc_handle *hp) ...@@ -745,15 +753,17 @@ static void set_sock_ids(struct mdesc_handle *hp)
{ {
u64 mp; u64 mp;
/* If machine description exposes sockets data use it. /**
* Otherwise fallback to use shared L3 or L2 caches. * Find the highest level of shared cache which pre-T7 is also
* the socket.
*/ */
if (!set_max_cache_ids_by_cache(hp, 3))
set_max_cache_ids_by_cache(hp, 2);
/* If machine description exposes sockets data use it.*/
mp = mdesc_node_by_name(hp, MDESC_NODE_NULL, "sockets"); mp = mdesc_node_by_name(hp, MDESC_NODE_NULL, "sockets");
if (mp != MDESC_NODE_NULL) if (mp != MDESC_NODE_NULL)
return set_sock_ids_by_socket(hp, mp); set_sock_ids_by_socket(hp, mp);
if (!set_sock_ids_by_cache(hp, 3))
set_sock_ids_by_cache(hp, 2);
} }
static void mark_proc_ids(struct mdesc_handle *hp, u64 mp, int proc_id) static void mark_proc_ids(struct mdesc_handle *hp, u64 mp, int proc_id)
......
...@@ -63,9 +63,13 @@ cpumask_t cpu_core_map[NR_CPUS] __read_mostly = ...@@ -63,9 +63,13 @@ cpumask_t cpu_core_map[NR_CPUS] __read_mostly =
cpumask_t cpu_core_sib_map[NR_CPUS] __read_mostly = { cpumask_t cpu_core_sib_map[NR_CPUS] __read_mostly = {
[0 ... NR_CPUS-1] = CPU_MASK_NONE }; [0 ... NR_CPUS-1] = CPU_MASK_NONE };
cpumask_t cpu_core_sib_cache_map[NR_CPUS] __read_mostly = {
[0 ... NR_CPUS - 1] = CPU_MASK_NONE };
EXPORT_PER_CPU_SYMBOL(cpu_sibling_map); EXPORT_PER_CPU_SYMBOL(cpu_sibling_map);
EXPORT_SYMBOL(cpu_core_map); EXPORT_SYMBOL(cpu_core_map);
EXPORT_SYMBOL(cpu_core_sib_map); EXPORT_SYMBOL(cpu_core_sib_map);
EXPORT_SYMBOL(cpu_core_sib_cache_map);
static cpumask_t smp_commenced_mask; static cpumask_t smp_commenced_mask;
...@@ -1265,6 +1269,10 @@ void smp_fill_in_sib_core_maps(void) ...@@ -1265,6 +1269,10 @@ void smp_fill_in_sib_core_maps(void)
unsigned int j; unsigned int j;
for_each_present_cpu(j) { for_each_present_cpu(j) {
if (cpu_data(i).max_cache_id ==
cpu_data(j).max_cache_id)
cpumask_set_cpu(j, &cpu_core_sib_cache_map[i]);
if (cpu_data(i).sock_id == cpu_data(j).sock_id) if (cpu_data(i).sock_id == cpu_data(j).sock_id)
cpumask_set_cpu(j, &cpu_core_sib_map[i]); cpumask_set_cpu(j, &cpu_core_sib_map[i]);
} }
......
...@@ -3,11 +3,11 @@ ...@@ -3,11 +3,11 @@
* Copyright (C) 2007 David S. Miller (davem@davemloft.net) * Copyright (C) 2007 David S. Miller (davem@davemloft.net)
*/ */
#define EX_LD(x) \ #define EX_LD(x,y) \
98: x; \ 98: x; \
.section __ex_table,"a";\ .section __ex_table,"a";\
.align 4; \ .align 4; \
.word 98b, __retl_one; \ .word 98b, y; \
.text; \ .text; \
.align 4; .align 4;
......
...@@ -3,11 +3,11 @@ ...@@ -3,11 +3,11 @@
* Copyright (C) 2007 David S. Miller (davem@davemloft.net) * Copyright (C) 2007 David S. Miller (davem@davemloft.net)
*/ */
#define EX_ST(x) \ #define EX_ST(x,y) \
98: x; \ 98: x; \
.section __ex_table,"a";\ .section __ex_table,"a";\
.align 4; \ .align 4; \
.word 98b, __retl_one; \ .word 98b, y; \
.text; \ .text; \
.align 4; .align 4;
......
...@@ -4,21 +4,18 @@ ...@@ -4,21 +4,18 @@
*/ */
#ifdef __KERNEL__ #ifdef __KERNEL__
#include <linux/linkage.h>
#define GLOBAL_SPARE %g7 #define GLOBAL_SPARE %g7
#else #else
#define GLOBAL_SPARE %g5 #define GLOBAL_SPARE %g5
#endif #endif
#ifndef EX_LD #ifndef EX_LD
#define EX_LD(x) x #define EX_LD(x,y) x
#endif #endif
#ifndef EX_ST #ifndef EX_ST
#define EX_ST(x) x #define EX_ST(x,y) x
#endif
#ifndef EX_RETVAL
#define EX_RETVAL(x) x
#endif #endif
#ifndef LOAD #ifndef LOAD
...@@ -45,6 +42,29 @@ ...@@ -45,6 +42,29 @@
.register %g3,#scratch .register %g3,#scratch
.text .text
#ifndef EX_RETVAL
#define EX_RETVAL(x) x
ENTRY(GEN_retl_o4_1)
add %o4, %o2, %o4
retl
add %o4, 1, %o0
ENDPROC(GEN_retl_o4_1)
ENTRY(GEN_retl_g1_8)
add %g1, %o2, %g1
retl
add %g1, 8, %o0
ENDPROC(GEN_retl_g1_8)
ENTRY(GEN_retl_o2_4)
retl
add %o2, 4, %o0
ENDPROC(GEN_retl_o2_4)
ENTRY(GEN_retl_o2_1)
retl
add %o2, 1, %o0
ENDPROC(GEN_retl_o2_1)
#endif
.align 64 .align 64
.globl FUNC_NAME .globl FUNC_NAME
...@@ -73,8 +93,8 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ ...@@ -73,8 +93,8 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
sub %g0, %o4, %o4 sub %g0, %o4, %o4
sub %o2, %o4, %o2 sub %o2, %o4, %o2
1: subcc %o4, 1, %o4 1: subcc %o4, 1, %o4
EX_LD(LOAD(ldub, %o1, %g1)) EX_LD(LOAD(ldub, %o1, %g1),GEN_retl_o4_1)
EX_ST(STORE(stb, %g1, %o0)) EX_ST(STORE(stb, %g1, %o0),GEN_retl_o4_1)
add %o1, 1, %o1 add %o1, 1, %o1
bne,pt %XCC, 1b bne,pt %XCC, 1b
add %o0, 1, %o0 add %o0, 1, %o0
...@@ -82,8 +102,8 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ ...@@ -82,8 +102,8 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
andn %o2, 0x7, %g1 andn %o2, 0x7, %g1
sub %o2, %g1, %o2 sub %o2, %g1, %o2
1: subcc %g1, 0x8, %g1 1: subcc %g1, 0x8, %g1
EX_LD(LOAD(ldx, %o1, %g2)) EX_LD(LOAD(ldx, %o1, %g2),GEN_retl_g1_8)
EX_ST(STORE(stx, %g2, %o0)) EX_ST(STORE(stx, %g2, %o0),GEN_retl_g1_8)
add %o1, 0x8, %o1 add %o1, 0x8, %o1
bne,pt %XCC, 1b bne,pt %XCC, 1b
add %o0, 0x8, %o0 add %o0, 0x8, %o0
...@@ -100,8 +120,8 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ ...@@ -100,8 +120,8 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
1: 1:
subcc %o2, 4, %o2 subcc %o2, 4, %o2
EX_LD(LOAD(lduw, %o1, %g1)) EX_LD(LOAD(lduw, %o1, %g1),GEN_retl_o2_4)
EX_ST(STORE(stw, %g1, %o1 + %o3)) EX_ST(STORE(stw, %g1, %o1 + %o3),GEN_retl_o2_4)
bgu,pt %XCC, 1b bgu,pt %XCC, 1b
add %o1, 4, %o1 add %o1, 4, %o1
...@@ -111,8 +131,8 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ ...@@ -111,8 +131,8 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
.align 32 .align 32
90: 90:
subcc %o2, 1, %o2 subcc %o2, 1, %o2
EX_LD(LOAD(ldub, %o1, %g1)) EX_LD(LOAD(ldub, %o1, %g1),GEN_retl_o2_1)
EX_ST(STORE(stb, %g1, %o1 + %o3)) EX_ST(STORE(stb, %g1, %o1 + %o3),GEN_retl_o2_1)
bgu,pt %XCC, 90b bgu,pt %XCC, 90b
add %o1, 1, %o1 add %o1, 1, %o1
retl retl
......
...@@ -38,7 +38,7 @@ lib-$(CONFIG_SPARC64) += NG4patch.o NG4copy_page.o NG4clear_page.o NG4memset.o ...@@ -38,7 +38,7 @@ lib-$(CONFIG_SPARC64) += NG4patch.o NG4copy_page.o NG4clear_page.o NG4memset.o
lib-$(CONFIG_SPARC64) += GENmemcpy.o GENcopy_from_user.o GENcopy_to_user.o lib-$(CONFIG_SPARC64) += GENmemcpy.o GENcopy_from_user.o GENcopy_to_user.o
lib-$(CONFIG_SPARC64) += GENpatch.o GENpage.o GENbzero.o lib-$(CONFIG_SPARC64) += GENpatch.o GENpage.o GENbzero.o
lib-$(CONFIG_SPARC64) += copy_in_user.o user_fixup.o memmove.o lib-$(CONFIG_SPARC64) += copy_in_user.o memmove.o
lib-$(CONFIG_SPARC64) += mcount.o ipcsum.o xor.o hweight.o ffs.o lib-$(CONFIG_SPARC64) += mcount.o ipcsum.o xor.o hweight.o ffs.o
obj-$(CONFIG_SPARC64) += iomap.o obj-$(CONFIG_SPARC64) += iomap.o
......
...@@ -3,19 +3,19 @@ ...@@ -3,19 +3,19 @@
* Copyright (C) 2007 David S. Miller (davem@davemloft.net) * Copyright (C) 2007 David S. Miller (davem@davemloft.net)
*/ */
#define EX_LD(x) \ #define EX_LD(x,y) \
98: x; \ 98: x; \
.section __ex_table,"a";\ .section __ex_table,"a";\
.align 4; \ .align 4; \
.word 98b, __retl_one_asi;\ .word 98b, y; \
.text; \ .text; \
.align 4; .align 4;
#define EX_LD_FP(x) \ #define EX_LD_FP(x,y) \
98: x; \ 98: x; \
.section __ex_table,"a";\ .section __ex_table,"a";\
.align 4; \ .align 4; \
.word 98b, __retl_one_asi_fp;\ .word 98b, y##_fp; \
.text; \ .text; \
.align 4; .align 4;
......
...@@ -3,19 +3,19 @@ ...@@ -3,19 +3,19 @@
* Copyright (C) 2007 David S. Miller (davem@davemloft.net) * Copyright (C) 2007 David S. Miller (davem@davemloft.net)
*/ */
#define EX_ST(x) \ #define EX_ST(x,y) \
98: x; \ 98: x; \
.section __ex_table,"a";\ .section __ex_table,"a";\
.align 4; \ .align 4; \
.word 98b, __retl_one_asi;\ .word 98b, y; \
.text; \ .text; \
.align 4; .align 4;
#define EX_ST_FP(x) \ #define EX_ST_FP(x,y) \
98: x; \ 98: x; \
.section __ex_table,"a";\ .section __ex_table,"a";\
.align 4; \ .align 4; \
.word 98b, __retl_one_asi_fp;\ .word 98b, y##_fp; \
.text; \ .text; \
.align 4; .align 4;
......
This diff is collapsed.
...@@ -3,19 +3,19 @@ ...@@ -3,19 +3,19 @@
* Copyright (C) 2012 David S. Miller (davem@davemloft.net) * Copyright (C) 2012 David S. Miller (davem@davemloft.net)
*/ */
#define EX_LD(x) \ #define EX_LD(x, y) \
98: x; \ 98: x; \
.section __ex_table,"a";\ .section __ex_table,"a";\
.align 4; \ .align 4; \
.word 98b, __retl_one_asi;\ .word 98b, y; \
.text; \ .text; \
.align 4; .align 4;
#define EX_LD_FP(x) \ #define EX_LD_FP(x,y) \
98: x; \ 98: x; \
.section __ex_table,"a";\ .section __ex_table,"a";\
.align 4; \ .align 4; \
.word 98b, __retl_one_asi_fp;\ .word 98b, y##_fp; \
.text; \ .text; \
.align 4; .align 4;
......
...@@ -3,19 +3,19 @@ ...@@ -3,19 +3,19 @@
* Copyright (C) 2012 David S. Miller (davem@davemloft.net) * Copyright (C) 2012 David S. Miller (davem@davemloft.net)
*/ */
#define EX_ST(x) \ #define EX_ST(x,y) \
98: x; \ 98: x; \
.section __ex_table,"a";\ .section __ex_table,"a";\
.align 4; \ .align 4; \
.word 98b, __retl_one_asi;\ .word 98b, y; \
.text; \ .text; \
.align 4; .align 4;
#define EX_ST_FP(x) \ #define EX_ST_FP(x,y) \
98: x; \ 98: x; \
.section __ex_table,"a";\ .section __ex_table,"a";\
.align 4; \ .align 4; \
.word 98b, __retl_one_asi_fp;\ .word 98b, y##_fp; \
.text; \ .text; \
.align 4; .align 4;
......
This diff is collapsed.
...@@ -3,11 +3,11 @@ ...@@ -3,11 +3,11 @@
* Copyright (C) 2006, 2007 David S. Miller (davem@davemloft.net) * Copyright (C) 2006, 2007 David S. Miller (davem@davemloft.net)
*/ */
#define EX_LD(x) \ #define EX_LD(x,y) \
98: x; \ 98: x; \
.section __ex_table,"a";\ .section __ex_table,"a";\
.align 4; \ .align 4; \
.word 98b, __ret_one_asi;\ .word 98b, y; \
.text; \ .text; \
.align 4; .align 4;
......
...@@ -3,11 +3,11 @@ ...@@ -3,11 +3,11 @@
* Copyright (C) 2006, 2007 David S. Miller (davem@davemloft.net) * Copyright (C) 2006, 2007 David S. Miller (davem@davemloft.net)
*/ */
#define EX_ST(x) \ #define EX_ST(x,y) \
98: x; \ 98: x; \
.section __ex_table,"a";\ .section __ex_table,"a";\
.align 4; \ .align 4; \
.word 98b, __ret_one_asi;\ .word 98b, y; \
.text; \ .text; \
.align 4; .align 4;
......
This diff is collapsed.
...@@ -3,19 +3,19 @@ ...@@ -3,19 +3,19 @@
* Copyright (C) 1999, 2000, 2004 David S. Miller (davem@redhat.com) * Copyright (C) 1999, 2000, 2004 David S. Miller (davem@redhat.com)
*/ */
#define EX_LD(x) \ #define EX_LD(x,y) \
98: x; \ 98: x; \
.section __ex_table,"a";\ .section __ex_table,"a";\
.align 4; \ .align 4; \
.word 98b, __retl_one; \ .word 98b, y; \
.text; \ .text; \
.align 4; .align 4;
#define EX_LD_FP(x) \ #define EX_LD_FP(x,y) \
98: x; \ 98: x; \
.section __ex_table,"a";\ .section __ex_table,"a";\
.align 4; \ .align 4; \
.word 98b, __retl_one_fp;\ .word 98b, y; \
.text; \ .text; \
.align 4; .align 4;
......
...@@ -3,19 +3,19 @@ ...@@ -3,19 +3,19 @@
* Copyright (C) 1999, 2000, 2004 David S. Miller (davem@redhat.com) * Copyright (C) 1999, 2000, 2004 David S. Miller (davem@redhat.com)
*/ */
#define EX_ST(x) \ #define EX_ST(x,y) \
98: x; \ 98: x; \
.section __ex_table,"a";\ .section __ex_table,"a";\
.align 4; \ .align 4; \
.word 98b, __retl_one; \ .word 98b, y; \
.text; \ .text; \
.align 4; .align 4;
#define EX_ST_FP(x) \ #define EX_ST_FP(x,y) \
98: x; \ 98: x; \
.section __ex_table,"a";\ .section __ex_table,"a";\
.align 4; \ .align 4; \
.word 98b, __retl_one_fp;\ .word 98b, y; \
.text; \ .text; \
.align 4; .align 4;
......
This diff is collapsed.
...@@ -3,19 +3,19 @@ ...@@ -3,19 +3,19 @@
* Copyright (C) 1999, 2000, 2004 David S. Miller (davem@redhat.com) * Copyright (C) 1999, 2000, 2004 David S. Miller (davem@redhat.com)
*/ */
#define EX_LD(x) \ #define EX_LD(x,y) \
98: x; \ 98: x; \
.section __ex_table,"a";\ .section __ex_table,"a";\
.align 4; \ .align 4; \
.word 98b, __retl_one; \ .word 98b, y; \
.text; \ .text; \
.align 4; .align 4;
#define EX_LD_FP(x) \ #define EX_LD_FP(x,y) \
98: x; \ 98: x; \
.section __ex_table,"a";\ .section __ex_table,"a";\
.align 4; \ .align 4; \
.word 98b, __retl_one_fp;\ .word 98b, y##_fp; \
.text; \ .text; \
.align 4; .align 4;
......
...@@ -3,19 +3,19 @@ ...@@ -3,19 +3,19 @@
* Copyright (C) 1999, 2000, 2004 David S. Miller (davem@redhat.com) * Copyright (C) 1999, 2000, 2004 David S. Miller (davem@redhat.com)
*/ */
#define EX_ST(x) \ #define EX_ST(x,y) \
98: x; \ 98: x; \
.section __ex_table,"a";\ .section __ex_table,"a";\
.align 4; \ .align 4; \
.word 98b, __retl_one; \ .word 98b, y; \
.text; \ .text; \
.align 4; .align 4;
#define EX_ST_FP(x) \ #define EX_ST_FP(x,y) \
98: x; \ 98: x; \
.section __ex_table,"a";\ .section __ex_table,"a";\
.align 4; \ .align 4; \
.word 98b, __retl_one_fp;\ .word 98b, y##_fp; \
.text; \ .text; \
.align 4; .align 4;
......
This diff is collapsed.
...@@ -9,18 +9,33 @@ ...@@ -9,18 +9,33 @@
#define XCC xcc #define XCC xcc
#define EX(x,y) \ #define EX(x,y,z) \
98: x,y; \ 98: x,y; \
.section __ex_table,"a";\ .section __ex_table,"a";\
.align 4; \ .align 4; \
.word 98b, __retl_one; \ .word 98b, z; \
.text; \ .text; \
.align 4; .align 4;
#define EX_O4(x,y) EX(x,y,__retl_o4_plus_8)
#define EX_O2_4(x,y) EX(x,y,__retl_o2_plus_4)
#define EX_O2_1(x,y) EX(x,y,__retl_o2_plus_1)
.register %g2,#scratch .register %g2,#scratch
.register %g3,#scratch .register %g3,#scratch
.text .text
__retl_o4_plus_8:
add %o4, %o2, %o4
retl
add %o4, 8, %o0
__retl_o2_plus_4:
retl
add %o2, 4, %o0
__retl_o2_plus_1:
retl
add %o2, 1, %o0
.align 32 .align 32
/* Don't try to get too fancy here, just nice and /* Don't try to get too fancy here, just nice and
...@@ -45,8 +60,8 @@ ENTRY(___copy_in_user) /* %o0=dst, %o1=src, %o2=len */ ...@@ -45,8 +60,8 @@ ENTRY(___copy_in_user) /* %o0=dst, %o1=src, %o2=len */
andn %o2, 0x7, %o4 andn %o2, 0x7, %o4
and %o2, 0x7, %o2 and %o2, 0x7, %o2
1: subcc %o4, 0x8, %o4 1: subcc %o4, 0x8, %o4
EX(ldxa [%o1] %asi, %o5) EX_O4(ldxa [%o1] %asi, %o5)
EX(stxa %o5, [%o0] %asi) EX_O4(stxa %o5, [%o0] %asi)
add %o1, 0x8, %o1 add %o1, 0x8, %o1
bgu,pt %XCC, 1b bgu,pt %XCC, 1b
add %o0, 0x8, %o0 add %o0, 0x8, %o0
...@@ -54,8 +69,8 @@ ENTRY(___copy_in_user) /* %o0=dst, %o1=src, %o2=len */ ...@@ -54,8 +69,8 @@ ENTRY(___copy_in_user) /* %o0=dst, %o1=src, %o2=len */
be,pt %XCC, 1f be,pt %XCC, 1f
nop nop
sub %o2, 0x4, %o2 sub %o2, 0x4, %o2
EX(lduwa [%o1] %asi, %o5) EX_O2_4(lduwa [%o1] %asi, %o5)
EX(stwa %o5, [%o0] %asi) EX_O2_4(stwa %o5, [%o0] %asi)
add %o1, 0x4, %o1 add %o1, 0x4, %o1
add %o0, 0x4, %o0 add %o0, 0x4, %o0
1: cmp %o2, 0 1: cmp %o2, 0
...@@ -71,8 +86,8 @@ ENTRY(___copy_in_user) /* %o0=dst, %o1=src, %o2=len */ ...@@ -71,8 +86,8 @@ ENTRY(___copy_in_user) /* %o0=dst, %o1=src, %o2=len */
82: 82:
subcc %o2, 4, %o2 subcc %o2, 4, %o2
EX(lduwa [%o1] %asi, %g1) EX_O2_4(lduwa [%o1] %asi, %g1)
EX(stwa %g1, [%o0] %asi) EX_O2_4(stwa %g1, [%o0] %asi)
add %o1, 4, %o1 add %o1, 4, %o1
bgu,pt %XCC, 82b bgu,pt %XCC, 82b
add %o0, 4, %o0 add %o0, 4, %o0
...@@ -83,8 +98,8 @@ ENTRY(___copy_in_user) /* %o0=dst, %o1=src, %o2=len */ ...@@ -83,8 +98,8 @@ ENTRY(___copy_in_user) /* %o0=dst, %o1=src, %o2=len */
.align 32 .align 32
90: 90:
subcc %o2, 1, %o2 subcc %o2, 1, %o2
EX(lduba [%o1] %asi, %g1) EX_O2_1(lduba [%o1] %asi, %g1)
EX(stba %g1, [%o0] %asi) EX_O2_1(stba %g1, [%o0] %asi)
add %o1, 1, %o1 add %o1, 1, %o1
bgu,pt %XCC, 90b bgu,pt %XCC, 90b
add %o0, 1, %o0 add %o0, 1, %o0
......
/* user_fixup.c: Fix up user copy faults.
*
* Copyright (C) 2004 David S. Miller <davem@redhat.com>
*/
#include <linux/compiler.h>
#include <linux/kernel.h>
#include <linux/string.h>
#include <linux/errno.h>
#include <linux/module.h>
#include <asm/uaccess.h>
/* Calculating the exact fault address when using
* block loads and stores can be very complicated.
*
* Instead of trying to be clever and handling all
* of the cases, just fix things up simply here.
*/
static unsigned long compute_size(unsigned long start, unsigned long size, unsigned long *offset)
{
unsigned long fault_addr = current_thread_info()->fault_address;
unsigned long end = start + size;
if (fault_addr < start || fault_addr >= end) {
*offset = 0;
} else {
*offset = fault_addr - start;
size = end - fault_addr;
}
return size;
}
unsigned long copy_from_user_fixup(void *to, const void __user *from, unsigned long size)
{
unsigned long offset;
size = compute_size((unsigned long) from, size, &offset);
if (likely(size))
memset(to + offset, 0, size);
return size;
}
EXPORT_SYMBOL(copy_from_user_fixup);
unsigned long copy_to_user_fixup(void __user *to, const void *from, unsigned long size)
{
unsigned long offset;
return compute_size((unsigned long) to, size, &offset);
}
EXPORT_SYMBOL(copy_to_user_fixup);
unsigned long copy_in_user_fixup(void __user *to, void __user *from, unsigned long size)
{
unsigned long fault_addr = current_thread_info()->fault_address;
unsigned long start = (unsigned long) to;
unsigned long end = start + size;
if (fault_addr >= start && fault_addr < end)
return end - fault_addr;
start = (unsigned long) from;
end = start + size;
if (fault_addr >= start && fault_addr < end)
return end - fault_addr;
return size;
}
EXPORT_SYMBOL(copy_in_user_fixup);
...@@ -27,6 +27,20 @@ static inline int tag_compare(unsigned long tag, unsigned long vaddr) ...@@ -27,6 +27,20 @@ static inline int tag_compare(unsigned long tag, unsigned long vaddr)
return (tag == (vaddr >> 22)); return (tag == (vaddr >> 22));
} }
static void flush_tsb_kernel_range_scan(unsigned long start, unsigned long end)
{
unsigned long idx;
for (idx = 0; idx < KERNEL_TSB_NENTRIES; idx++) {
struct tsb *ent = &swapper_tsb[idx];
unsigned long match = idx << 13;
match |= (ent->tag << 22);
if (match >= start && match < end)
ent->tag = (1UL << TSB_TAG_INVALID_BIT);
}
}
/* TSB flushes need only occur on the processor initiating the address /* TSB flushes need only occur on the processor initiating the address
* space modification, not on each cpu the address space has run on. * space modification, not on each cpu the address space has run on.
* Only the TLB flush needs that treatment. * Only the TLB flush needs that treatment.
...@@ -36,6 +50,9 @@ void flush_tsb_kernel_range(unsigned long start, unsigned long end) ...@@ -36,6 +50,9 @@ void flush_tsb_kernel_range(unsigned long start, unsigned long end)
{ {
unsigned long v; unsigned long v;
if ((end - start) >> PAGE_SHIFT >= 2 * KERNEL_TSB_NENTRIES)
return flush_tsb_kernel_range_scan(start, end);
for (v = start; v < end; v += PAGE_SIZE) { for (v = start; v < end; v += PAGE_SIZE) {
unsigned long hash = tsb_hash(v, PAGE_SHIFT, unsigned long hash = tsb_hash(v, PAGE_SHIFT,
KERNEL_TSB_NENTRIES); KERNEL_TSB_NENTRIES);
......
This diff is collapsed.
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment