Skip to content

L
linux
Commit 2ab82e66 authored by Markos Chandras's avatar Markos Chandras Committed by Ralf Baechle
Browse files
Options

MIPS: lib: csum_partial: Merge EXC and load/store macros 

Each load/store macro always adds an entry to the __ex_table
using the EXC macro. There are cases where a load instruction may
never fail such as when we are sure the load happens in the kernel
address space. Therefore, we merge these the EXC and LOADX/STOREX
macros into a single one. We also expand the argument list in the EXC
macro to make the macro more flexible. The extra 'type' argument is not
used by this commit, but it will be used when EVA support is added to
memcpy.
Signed-off-by: default avatarMarkos Chandras <markos.chandras@imgtec.com>
parent ac85227f
Show whitespace changes
Inline Side-by-side
Showing with 91 additions and 69 deletions
arch/mips/lib/csum_partial.S
View file @ 2ab82e66
...@@ -328,20 +328,39 @@ LEAF(csum_partial) ...@@ -328,20 +328,39 @@ LEAF(csum_partial)
* These handlers do not need to overwrite any data. * These handlers do not need to overwrite any data.
*/ */
#define EXC(inst_reg,addr,handler) \ /* Instruction type */
9: inst_reg, addr; \ #define LD_INSN 1
#define ST_INSN 2
/*
* Wrapper to add an entry in the exception table
* in case the insn causes a memory exception.
* Arguments:
* insn : Load/store instruction
* type : Instruction type
* reg : Register
* addr : Address
* handler : Exception handler
*/
#define EXC(insn, type, reg, addr, handler) \
9: insn reg, addr; \
.section __ex_table,"a"; \ .section __ex_table,"a"; \
PTR 9b, handler; \ PTR 9b, handler; \
.previous .previous
#undef LOAD
#ifdef USE_DOUBLE #ifdef USE_DOUBLE
#define LOAD ld #define LOADK ld /* No exception */
#define LOADL ldl #define LOAD(reg, addr, handler) EXC(ld, LD_INSN, reg, addr, handler)
#define LOADR ldr #define LOADBU(reg, addr, handler) EXC(lbu, LD_INSN, reg, addr, handler)
#define STOREL sdl #define LOADL(reg, addr, handler) EXC(ldl, LD_INSN, reg, addr, handler)
#define STORER sdr #define LOADR(reg, addr, handler) EXC(ldr, LD_INSN, reg, addr, handler)
#define STORE sd #define STOREB(reg, addr, handler) EXC(sb, ST_INSN, reg, addr, handler)
#define STOREL(reg, addr, handler) EXC(sdl, ST_INSN, reg, addr, handler)
#define STORER(reg, addr, handler) EXC(sdr, ST_INSN, reg, addr, handler)
#define STORE(reg, addr, handler) EXC(sd, ST_INSN, reg, addr, handler)
#define ADD daddu #define ADD daddu
#define SUB dsubu #define SUB dsubu
#define SRL dsrl #define SRL dsrl
...@@ -353,12 +372,15 @@ LEAF(csum_partial) ...@@ -353,12 +372,15 @@ LEAF(csum_partial)
#else #else
#define LOAD lw #define LOADK lw /* No exception */
#define LOADL lwl #define LOAD(reg, addr, handler) EXC(lw, LD_INSN, reg, addr, handler)
#define LOADR lwr #define LOADBU(reg, addr, handler) EXC(lbu, LD_INSN, reg, addr, handler)
#define STOREL swl #define LOADL(reg, addr, handler) EXC(lwl, LD_INSN, reg, addr, handler)
#define STORER swr #define LOADR(reg, addr, handler) EXC(lwr, LD_INSN, reg, addr, handler)
#define STORE sw #define STOREB(reg, addr, handler) EXC(sb, ST_INSN, reg, addr, handler)
#define STOREL(reg, addr, handler) EXC(swl, ST_INSN, reg, addr, handler)
#define STORER(reg, addr, handler) EXC(swr, ST_INSN, reg, addr, handler)
#define STORE(reg, addr, handler) EXC(sw, ST_INSN, reg, addr, handler)
#define ADD addu #define ADD addu
#define SUB subu #define SUB subu
#define SRL srl #define SRL srl
...@@ -439,31 +461,31 @@ FEXPORT(csum_partial_copy_nocheck) ...@@ -439,31 +461,31 @@ FEXPORT(csum_partial_copy_nocheck)
SUB len, 8*NBYTES # subtract here for bgez loop SUB len, 8*NBYTES # subtract here for bgez loop
.align 4 .align 4
1: 1:
EXC( LOAD t0, UNIT(0)(src), .Ll_exc) LOAD(t0, UNIT(0)(src), .Ll_exc)
EXC( LOAD t1, UNIT(1)(src), .Ll_exc_copy) LOAD(t1, UNIT(1)(src), .Ll_exc_copy)
EXC( LOAD t2, UNIT(2)(src), .Ll_exc_copy) LOAD(t2, UNIT(2)(src), .Ll_exc_copy)
EXC( LOAD t3, UNIT(3)(src), .Ll_exc_copy) LOAD(t3, UNIT(3)(src), .Ll_exc_copy)
EXC( LOAD t4, UNIT(4)(src), .Ll_exc_copy) LOAD(t4, UNIT(4)(src), .Ll_exc_copy)
EXC( LOAD t5, UNIT(5)(src), .Ll_exc_copy) LOAD(t5, UNIT(5)(src), .Ll_exc_copy)
EXC( LOAD t6, UNIT(6)(src), .Ll_exc_copy) LOAD(t6, UNIT(6)(src), .Ll_exc_copy)
EXC( LOAD t7, UNIT(7)(src), .Ll_exc_copy) LOAD(t7, UNIT(7)(src), .Ll_exc_copy)
SUB len, len, 8*NBYTES SUB len, len, 8*NBYTES
ADD src, src, 8*NBYTES ADD src, src, 8*NBYTES
EXC( STORE t0, UNIT(0)(dst), .Ls_exc) STORE(t0, UNIT(0)(dst), .Ls_exc)
ADDC(sum, t0) ADDC(sum, t0)
EXC( STORE t1, UNIT(1)(dst), .Ls_exc) STORE(t1, UNIT(1)(dst), .Ls_exc)
ADDC(sum, t1) ADDC(sum, t1)
EXC( STORE t2, UNIT(2)(dst), .Ls_exc) STORE(t2, UNIT(2)(dst), .Ls_exc)
ADDC(sum, t2) ADDC(sum, t2)
EXC( STORE t3, UNIT(3)(dst), .Ls_exc) STORE(t3, UNIT(3)(dst), .Ls_exc)
ADDC(sum, t3) ADDC(sum, t3)
EXC( STORE t4, UNIT(4)(dst), .Ls_exc) STORE(t4, UNIT(4)(dst), .Ls_exc)
ADDC(sum, t4) ADDC(sum, t4)
EXC( STORE t5, UNIT(5)(dst), .Ls_exc) STORE(t5, UNIT(5)(dst), .Ls_exc)
ADDC(sum, t5) ADDC(sum, t5)
EXC( STORE t6, UNIT(6)(dst), .Ls_exc) STORE(t6, UNIT(6)(dst), .Ls_exc)
ADDC(sum, t6) ADDC(sum, t6)
EXC( STORE t7, UNIT(7)(dst), .Ls_exc) STORE(t7, UNIT(7)(dst), .Ls_exc)
ADDC(sum, t7) ADDC(sum, t7)
.set reorder /* DADDI_WAR */ .set reorder /* DADDI_WAR */
ADD dst, dst, 8*NBYTES ADD dst, dst, 8*NBYTES
...@@ -483,19 +505,19 @@ EXC( STORE t7, UNIT(7)(dst), .Ls_exc) ...@@ -483,19 +505,19 @@ EXC( STORE t7, UNIT(7)(dst), .Ls_exc)
/* /*
* len >= 4*NBYTES * len >= 4*NBYTES
*/ */
EXC( LOAD t0, UNIT(0)(src), .Ll_exc) LOAD(t0, UNIT(0)(src), .Ll_exc)
EXC( LOAD t1, UNIT(1)(src), .Ll_exc_copy) LOAD(t1, UNIT(1)(src), .Ll_exc_copy)
EXC( LOAD t2, UNIT(2)(src), .Ll_exc_copy) LOAD(t2, UNIT(2)(src), .Ll_exc_copy)
EXC( LOAD t3, UNIT(3)(src), .Ll_exc_copy) LOAD(t3, UNIT(3)(src), .Ll_exc_copy)
SUB len, len, 4*NBYTES SUB len, len, 4*NBYTES
ADD src, src, 4*NBYTES ADD src, src, 4*NBYTES
EXC( STORE t0, UNIT(0)(dst), .Ls_exc) STORE(t0, UNIT(0)(dst), .Ls_exc)
ADDC(sum, t0) ADDC(sum, t0)
EXC( STORE t1, UNIT(1)(dst), .Ls_exc) STORE(t1, UNIT(1)(dst), .Ls_exc)
ADDC(sum, t1) ADDC(sum, t1)
EXC( STORE t2, UNIT(2)(dst), .Ls_exc) STORE(t2, UNIT(2)(dst), .Ls_exc)
ADDC(sum, t2) ADDC(sum, t2)
EXC( STORE t3, UNIT(3)(dst), .Ls_exc) STORE(t3, UNIT(3)(dst), .Ls_exc)
ADDC(sum, t3) ADDC(sum, t3)
.set reorder /* DADDI_WAR */ .set reorder /* DADDI_WAR */
ADD dst, dst, 4*NBYTES ADD dst, dst, 4*NBYTES
...@@ -508,10 +530,10 @@ EXC( STORE t3, UNIT(3)(dst), .Ls_exc) ...@@ -508,10 +530,10 @@ EXC( STORE t3, UNIT(3)(dst), .Ls_exc)
beq rem, len, .Lcopy_bytes beq rem, len, .Lcopy_bytes
nop nop
1: 1:
EXC( LOAD t0, 0(src), .Ll_exc) LOAD(t0, 0(src), .Ll_exc)
ADD src, src, NBYTES ADD src, src, NBYTES
SUB len, len, NBYTES SUB len, len, NBYTES
EXC( STORE t0, 0(dst), .Ls_exc) STORE(t0, 0(dst), .Ls_exc)
ADDC(sum, t0) ADDC(sum, t0)
.set reorder /* DADDI_WAR */ .set reorder /* DADDI_WAR */
ADD dst, dst, NBYTES ADD dst, dst, NBYTES
...@@ -534,10 +556,10 @@ EXC( STORE t0, 0(dst), .Ls_exc) ...@@ -534,10 +556,10 @@ EXC( STORE t0, 0(dst), .Ls_exc)
ADD t1, dst, len # t1 is just past last byte of dst ADD t1, dst, len # t1 is just past last byte of dst
li bits, 8*NBYTES li bits, 8*NBYTES
SLL rem, len, 3 # rem = number of bits to keep SLL rem, len, 3 # rem = number of bits to keep
EXC( LOAD t0, 0(src), .Ll_exc) LOAD(t0, 0(src), .Ll_exc)
SUB bits, bits, rem # bits = number of bits to discard SUB bits, bits, rem # bits = number of bits to discard
SHIFT_DISCARD t0, t0, bits SHIFT_DISCARD t0, t0, bits
EXC( STREST t0, -1(t1), .Ls_exc) STREST(t0, -1(t1), .Ls_exc)
SHIFT_DISCARD_REVERT t0, t0, bits SHIFT_DISCARD_REVERT t0, t0, bits
.set reorder .set reorder
ADDC(sum, t0) ADDC(sum, t0)
...@@ -554,12 +576,12 @@ EXC( STREST t0, -1(t1), .Ls_exc) ...@@ -554,12 +576,12 @@ EXC( STREST t0, -1(t1), .Ls_exc)
* Set match = (src and dst have same alignment) * Set match = (src and dst have same alignment)
*/ */
#define match rem #define match rem
EXC( LDFIRST t3, FIRST(0)(src), .Ll_exc) LDFIRST(t3, FIRST(0)(src), .Ll_exc)
ADD t2, zero, NBYTES ADD t2, zero, NBYTES
EXC( LDREST t3, REST(0)(src), .Ll_exc_copy) LDREST(t3, REST(0)(src), .Ll_exc_copy)
SUB t2, t2, t1 # t2 = number of bytes copied SUB t2, t2, t1 # t2 = number of bytes copied
xor match, t0, t1 xor match, t0, t1
EXC( STFIRST t3, FIRST(0)(dst), .Ls_exc) STFIRST(t3, FIRST(0)(dst), .Ls_exc)
SLL t4, t1, 3 # t4 = number of bits to discard SLL t4, t1, 3 # t4 = number of bits to discard
SHIFT_DISCARD t3, t3, t4 SHIFT_DISCARD t3, t3, t4
/* no SHIFT_DISCARD_REVERT to handle odd buffer properly */ /* no SHIFT_DISCARD_REVERT to handle odd buffer properly */
...@@ -581,26 +603,26 @@ EXC( STFIRST t3, FIRST(0)(dst), .Ls_exc) ...@@ -581,26 +603,26 @@ EXC( STFIRST t3, FIRST(0)(dst), .Ls_exc)
* It's OK to load FIRST(N+1) before REST(N) because the two addresses * It's OK to load FIRST(N+1) before REST(N) because the two addresses
* are to the same unit (unless src is aligned, but it's not). * are to the same unit (unless src is aligned, but it's not).
*/ */
EXC( LDFIRST t0, FIRST(0)(src), .Ll_exc) LDFIRST(t0, FIRST(0)(src), .Ll_exc)
EXC( LDFIRST t1, FIRST(1)(src), .Ll_exc_copy) LDFIRST(t1, FIRST(1)(src), .Ll_exc_copy)
SUB len, len, 4*NBYTES SUB len, len, 4*NBYTES
EXC( LDREST t0, REST(0)(src), .Ll_exc_copy) LDREST(t0, REST(0)(src), .Ll_exc_copy)
EXC( LDREST t1, REST(1)(src), .Ll_exc_copy) LDREST(t1, REST(1)(src), .Ll_exc_copy)
EXC( LDFIRST t2, FIRST(2)(src), .Ll_exc_copy) LDFIRST(t2, FIRST(2)(src), .Ll_exc_copy)
EXC( LDFIRST t3, FIRST(3)(src), .Ll_exc_copy) LDFIRST(t3, FIRST(3)(src), .Ll_exc_copy)
EXC( LDREST t2, REST(2)(src), .Ll_exc_copy) LDREST(t2, REST(2)(src), .Ll_exc_copy)
EXC( LDREST t3, REST(3)(src), .Ll_exc_copy) LDREST(t3, REST(3)(src), .Ll_exc_copy)
ADD src, src, 4*NBYTES ADD src, src, 4*NBYTES
#ifdef CONFIG_CPU_SB1 #ifdef CONFIG_CPU_SB1
nop # improves slotting nop # improves slotting
#endif #endif
EXC( STORE t0, UNIT(0)(dst), .Ls_exc) STORE(t0, UNIT(0)(dst), .Ls_exc)
ADDC(sum, t0) ADDC(sum, t0)
EXC( STORE t1, UNIT(1)(dst), .Ls_exc) STORE(t1, UNIT(1)(dst), .Ls_exc)
ADDC(sum, t1) ADDC(sum, t1)
EXC( STORE t2, UNIT(2)(dst), .Ls_exc) STORE(t2, UNIT(2)(dst), .Ls_exc)
ADDC(sum, t2) ADDC(sum, t2)
EXC( STORE t3, UNIT(3)(dst), .Ls_exc) STORE(t3, UNIT(3)(dst), .Ls_exc)
ADDC(sum, t3) ADDC(sum, t3)
.set reorder /* DADDI_WAR */ .set reorder /* DADDI_WAR */
ADD dst, dst, 4*NBYTES ADD dst, dst, 4*NBYTES
...@@ -613,11 +635,11 @@ EXC( STORE t3, UNIT(3)(dst), .Ls_exc) ...@@ -613,11 +635,11 @@ EXC( STORE t3, UNIT(3)(dst), .Ls_exc)
beq rem, len, .Lcopy_bytes beq rem, len, .Lcopy_bytes
nop nop
1: 1:
EXC( LDFIRST t0, FIRST(0)(src), .Ll_exc) LDFIRST(t0, FIRST(0)(src), .Ll_exc)
EXC( LDREST t0, REST(0)(src), .Ll_exc_copy) LDREST(t0, REST(0)(src), .Ll_exc_copy)
ADD src, src, NBYTES ADD src, src, NBYTES
SUB len, len, NBYTES SUB len, len, NBYTES
EXC( STORE t0, 0(dst), .Ls_exc) STORE(t0, 0(dst), .Ls_exc)
ADDC(sum, t0) ADDC(sum, t0)
.set reorder /* DADDI_WAR */ .set reorder /* DADDI_WAR */
ADD dst, dst, NBYTES ADD dst, dst, NBYTES
...@@ -640,9 +662,9 @@ EXC( STORE t0, 0(dst), .Ls_exc) ...@@ -640,9 +662,9 @@ EXC( STORE t0, 0(dst), .Ls_exc)
li t3, SHIFT_START # shift li t3, SHIFT_START # shift
/* use .Ll_exc_copy here to return correct sum on fault */ /* use .Ll_exc_copy here to return correct sum on fault */
#define COPY_BYTE(N) \ #define COPY_BYTE(N) \
EXC( lbu t0, N(src), .Ll_exc_copy); \ LOADBU(t0, N(src), .Ll_exc_copy); \
SUB len, len, 1; \ SUB len, len, 1; \
EXC( sb t0, N(dst), .Ls_exc); \ STOREB(t0, N(dst), .Ls_exc); \
SLLV t0, t0, t3; \ SLLV t0, t0, t3; \
addu t3, SHIFT_INC; \ addu t3, SHIFT_INC; \
beqz len, .Lcopy_bytes_done; \ beqz len, .Lcopy_bytes_done; \
...@@ -656,9 +678,9 @@ EXC( sb t0, N(dst), .Ls_exc); \ ...@@ -656,9 +678,9 @@ EXC( sb t0, N(dst), .Ls_exc); \
COPY_BYTE(4) COPY_BYTE(4)
COPY_BYTE(5) COPY_BYTE(5)
#endif #endif
EXC( lbu t0, NBYTES-2(src), .Ll_exc_copy) LOADBU(t0, NBYTES-2(src), .Ll_exc_copy)
SUB len, len, 1 SUB len, len, 1
EXC( sb t0, NBYTES-2(dst), .Ls_exc) STOREB(t0, NBYTES-2(dst), .Ls_exc)
SLLV t0, t0, t3 SLLV t0, t0, t3
or t2, t0 or t2, t0
.Lcopy_bytes_done: .Lcopy_bytes_done:
...@@ -703,11 +725,11 @@ EXC( sb t0, NBYTES-2(dst), .Ls_exc) ...@@ -703,11 +725,11 @@ EXC( sb t0, NBYTES-2(dst), .Ls_exc)
* *
* Assumes src < THREAD_BUADDR($28) * Assumes src < THREAD_BUADDR($28)
*/ */
LOAD t0, TI_TASK($28) LOADK t0, TI_TASK($28)
li t2, SHIFT_START li t2, SHIFT_START
LOAD t0, THREAD_BUADDR(t0) LOADK t0, THREAD_BUADDR(t0)
1: 1:
EXC( lbu t1, 0(src), .Ll_exc) LOADBU(t1, 0(src), .Ll_exc)
ADD src, src, 1 ADD src, src, 1
sb t1, 0(dst) # can't fault -- we're copy_from_user sb t1, 0(dst) # can't fault -- we're copy_from_user
SLLV t1, t1, t2 SLLV t1, t1, t2
...@@ -718,9 +740,9 @@ EXC( lbu t1, 0(src), .Ll_exc) ...@@ -718,9 +740,9 @@ EXC( lbu t1, 0(src), .Ll_exc)
bne src, t0, 1b bne src, t0, 1b
.set noreorder .set noreorder
.Ll_exc: .Ll_exc:
LOAD t0, TI_TASK($28) LOADK t0, TI_TASK($28)
nop nop
LOAD t0, THREAD_BUADDR(t0) # t0 is just past last good address LOADK t0, THREAD_BUADDR(t0) # t0 is just past last good address
nop nop
SUB len, AT, t0 # len number of uncopied bytes SUB len, AT, t0 # len number of uncopied bytes
/* /*
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7