Skip to content

L
linux
Commit 325a1de8 authored by Sam Tebbs's avatar Sam Tebbs Committed by Will Deacon
Browse files
Options

arm64: Import updated version of Cortex Strings' strlen 

Import an updated version of the former Cortex Strings - now Arm
Optimized Routines - strcmp function. The latest version introduces
Advanced SIMD usage which rules it out for our purposes, but we can
still pick an intermediate improvement from the previous version,
namely string/aarch64/strlen.S at commit 98e4d6a from
https://github.com/ARM-software/optimized-routines

Note that for simplicity Arm have chosen to contribute this code
to Linux under GPLv2 rather than the original MIT license.
Signed-off-by: default avatarSam Tebbs <sam.tebbs@arm.com>
[ rm: update attribution and commit message ]
Signed-off-by: default avatarRobin Murphy <robin.murphy@arm.com>
Link: https://lore.kernel.org/r/32e3489398a24b23ae6e996935ac4818f8fd9dfd.1622128527.git.robin.murphy@arm.comSigned-off-by: default avatarWill Deacon <will@kernel.org>
parent 758602c0
Hide whitespace changes
Inline Side-by-side
Showing with 173 additions and 85 deletions
arch/arm64/lib/strlen.S
View file @ 325a1de8
/* SPDX-License-Identifier: GPL-2.0-only */ /* SPDX-License-Identifier: GPL-2.0-only */
/* /*
* Copyright (C) 2013 ARM Ltd. * Copyright (c) 2013, Arm Limited.
* Copyright (C) 2013 Linaro.
* *
* This code is based on glibc cortex strings work originally authored by Linaro * Adapted from the original at:
* be found @ * https://github.com/ARM-software/optimized-routines/blob/master/string/aarch64/strlen.S
*
* http://bazaar.launchpad.net/~linaro-toolchain-dev/cortex-strings/trunk/
* files/head:/src/aarch64/
*/ */
#include <linux/linkage.h> #include <linux/linkage.h>
#include <asm/assembler.h> #include <asm/assembler.h>
/* /* Assumptions:
* calculate the length of a string
* *
* Parameters: * ARMv8-a, AArch64, unaligned accesses, min page size 4k.
* x0 - const string pointer
* Returns:
* x0 - the return length of specific string
*/ */
#define L(label) .L ## label
/* Arguments and results. */ /* Arguments and results. */
srcin .req x0 #define srcin x0
len .req x0 #define len x0
/* Locals and temporaries. */ /* Locals and temporaries. */
src .req x1 #define src x1
data1 .req x2 #define data1 x2
data2 .req x3 #define data2 x3
data2a .req x4 #define has_nul1 x4
has_nul1 .req x5 #define has_nul2 x5
has_nul2 .req x6 #define tmp1 x4
tmp1 .req x7 #define tmp2 x5
tmp2 .req x8 #define tmp3 x6
tmp3 .req x9 #define tmp4 x7
tmp4 .req x10 #define zeroones x8
zeroones .req x11
pos .req x12 /* NUL detection works on the principle that (X - 1) & (~X) & 0x80
(=> (X - 1) & ~(X | 0x7f)) is non-zero iff a byte is zero, and
can be done in parallel across the entire word. A faster check
(X - 1) & 0x80 is zero for non-NUL ASCII characters, but gives
false hits for characters 129..255. */
#define REP8_01 0x0101010101010101 #define REP8_01 0x0101010101010101
#define REP8_7f 0x7f7f7f7f7f7f7f7f #define REP8_7f 0x7f7f7f7f7f7f7f7f
#define REP8_80 0x8080808080808080 #define REP8_80 0x8080808080808080
#define MIN_PAGE_SIZE 4096
/* Since strings are short on average, we check the first 16 bytes
of the string for a NUL character. In order to do an unaligned ldp
safely we have to do a page cross check first. If there is a NUL
byte we calculate the length from the 2 8-byte words using
conditional select to reduce branch mispredictions (it is unlikely
strlen will be repeatedly called on strings with the same length).
If the string is longer than 16 bytes, we align src so don't need
further page cross checks, and process 32 bytes per iteration
using the fast NUL check. If we encounter non-ASCII characters,
fallback to a second loop using the full NUL check.
If the page cross check fails, we read 16 bytes from an aligned
address, remove any characters before the string, and continue
in the main loop using aligned loads. Since strings crossing a
page in the first 16 bytes are rare (probability of
16/MIN_PAGE_SIZE ~= 0.4%), this case does not need to be optimized.
AArch64 systems have a minimum page size of 4k. We don't bother
checking for larger page sizes - the cost of setting up the correct
page size is just not worth the extra gain from a small reduction in
the cases taking the slow path. Note that we only care about
whether the first fetch, which may be misaligned, crosses a page
boundary. */
SYM_FUNC_START_WEAK_PI(strlen) SYM_FUNC_START_WEAK_PI(strlen)
mov zeroones, #REP8_01 and tmp1, srcin, MIN_PAGE_SIZE - 1
bic src, srcin, #15 mov zeroones, REP8_01
ands tmp1, srcin, #15 cmp tmp1, MIN_PAGE_SIZE - 16
b.ne .Lmisaligned b.gt L(page_cross)
/* ldp data1, data2, [srcin]
* NUL detection works on the principle that (X - 1) & (~X) & 0x80 #ifdef __AARCH64EB__
* (=> (X - 1) & ~(X | 0x7f)) is non-zero iff a byte is zero, and /* For big-endian, carry propagation (if the final byte in the
* can be done in parallel across the entire word. string is 0x01) means we cannot use has_nul1/2 directly.
*/ Since we expect strings to be small and early-exit,
/* byte-swap the data now so has_null1/2 will be correct. */
* The inner loop deals with two Dwords at a time. This has a rev data1, data1
* slightly higher start-up cost, but we should win quite quickly, rev data2, data2
* especially on cores with a high number of issue slots per #endif
* cycle, as we get much better parallelism out of the operations.
*/
.Lloop:
ldp data1, data2, [src], #16
.Lrealigned:
sub tmp1, data1, zeroones sub tmp1, data1, zeroones
orr tmp2, data1, #REP8_7f orr tmp2, data1, REP8_7f
sub tmp3, data2, zeroones sub tmp3, data2, zeroones
orr tmp4, data2, #REP8_7f orr tmp4, data2, REP8_7f
bic has_nul1, tmp1, tmp2 bics has_nul1, tmp1, tmp2
bics has_nul2, tmp3, tmp4 bic has_nul2, tmp3, tmp4
ccmp has_nul1, #0, #0, eq /* NZCV = 0000 */ ccmp has_nul2, 0, 0, eq
b.eq .Lloop beq L(main_loop_entry)
/* Enter with C = has_nul1 == 0. */
csel has_nul1, has_nul1, has_nul2, cc
mov len, 8
rev has_nul1, has_nul1
clz tmp1, has_nul1
csel len, xzr, len, cc
add len, len, tmp1, lsr 3
ret
/* The inner loop processes 32 bytes per iteration and uses the fast
NUL check. If we encounter non-ASCII characters, use a second
loop with the accurate NUL check. */
.p2align 4
L(main_loop_entry):
bic src, srcin, 15
sub src, src, 16
L(main_loop):
ldp data1, data2, [src, 32]!
L(page_cross_entry):
sub tmp1, data1, zeroones
sub tmp3, data2, zeroones
orr tmp2, tmp1, tmp3
tst tmp2, zeroones, lsl 7
bne 1f
ldp data1, data2, [src, 16]
sub tmp1, data1, zeroones
sub tmp3, data2, zeroones
orr tmp2, tmp1, tmp3
tst tmp2, zeroones, lsl 7
beq L(main_loop)
add src, src, 16
1:
/* The fast check failed, so do the slower, accurate NUL check. */
orr tmp2, data1, REP8_7f
orr tmp4, data2, REP8_7f
bics has_nul1, tmp1, tmp2
bic has_nul2, tmp3, tmp4
ccmp has_nul2, 0, 0, eq
beq L(nonascii_loop)
/* Enter with C = has_nul1 == 0. */
L(tail):
#ifdef __AARCH64EB__
/* For big-endian, carry propagation (if the final byte in the
string is 0x01) means we cannot use has_nul1/2 directly. The
easiest way to get the correct byte is to byte-swap the data
and calculate the syndrome a second time. */
csel data1, data1, data2, cc
rev data1, data1
sub tmp1, data1, zeroones
orr tmp2, data1, REP8_7f
bic has_nul1, tmp1, tmp2
#else
csel has_nul1, has_nul1, has_nul2, cc
#endif
sub len, src, srcin sub len, src, srcin
cbz has_nul1, .Lnul_in_data2 rev has_nul1, has_nul1
CPU_BE( mov data2, data1 ) /*prepare data to re-calculate the syndrome*/ add tmp2, len, 8
sub len, len, #8 clz tmp1, has_nul1
mov has_nul2, has_nul1 csel len, len, tmp2, cc
.Lnul_in_data2: add len, len, tmp1, lsr 3
/*
* For big-endian, carry propagation (if the final byte in the
* string is 0x01) means we cannot use has_nul directly. The
* easiest way to get the correct byte is to byte-swap the data
* and calculate the syndrome a second time.
*/
CPU_BE( rev data2, data2 )
CPU_BE( sub tmp1, data2, zeroones )
CPU_BE( orr tmp2, data2, #REP8_7f )
CPU_BE( bic has_nul2, tmp1, tmp2 )
sub len, len, #8
rev has_nul2, has_nul2
clz pos, has_nul2
add len, len, pos, lsr #3 /* Bits to bytes. */
ret ret
.Lmisaligned: L(nonascii_loop):
cmp tmp1, #8 ldp data1, data2, [src, 16]!
neg tmp1, tmp1 sub tmp1, data1, zeroones
ldp data1, data2, [src], #16 orr tmp2, data1, REP8_7f
lsl tmp1, tmp1, #3 /* Bytes beyond alignment -> bits. */ sub tmp3, data2, zeroones
mov tmp2, #~0 orr tmp4, data2, REP8_7f
/* Big-endian. Early bytes are at MSB. */ bics has_nul1, tmp1, tmp2
CPU_BE( lsl tmp2, tmp2, tmp1 ) /* Shift (tmp1 & 63). */ bic has_nul2, tmp3, tmp4
ccmp has_nul2, 0, 0, eq
bne L(tail)
ldp data1, data2, [src, 16]!
sub tmp1, data1, zeroones
orr tmp2, data1, REP8_7f
sub tmp3, data2, zeroones
orr tmp4, data2, REP8_7f
bics has_nul1, tmp1, tmp2
bic has_nul2, tmp3, tmp4
ccmp has_nul2, 0, 0, eq
beq L(nonascii_loop)
b L(tail)
/* Load 16 bytes from [srcin & ~15] and force the bytes that precede
srcin to 0x7f, so we ignore any NUL bytes before the string.
Then continue in the aligned loop. */
L(page_cross):
bic src, srcin, 15
ldp data1, data2, [src]
lsl tmp1, srcin, 3
mov tmp4, -1
#ifdef __AARCH64EB__
/* Big-endian. Early bytes are at MSB. */
lsr tmp1, tmp4, tmp1 /* Shift (tmp1 & 63). */
#else
/* Little-endian. Early bytes are at LSB. */ /* Little-endian. Early bytes are at LSB. */
CPU_LE( lsr tmp2, tmp2, tmp1 ) /* Shift (tmp1 & 63). */ lsl tmp1, tmp4, tmp1 /* Shift (tmp1 & 63). */
#endif
orr tmp1, tmp1, REP8_80
orn data1, data1, tmp1
orn tmp2, data2, tmp1
tst srcin, 8
csel data1, data1, tmp4, eq
csel data2, data2, tmp2, eq
b L(page_cross_entry)
orr data1, data1, tmp2
orr data2a, data2, tmp2
csinv data1, data1, xzr, le
csel data2, data2, data2a, le
b .Lrealigned
SYM_FUNC_END_PI(strlen) SYM_FUNC_END_PI(strlen)
EXPORT_SYMBOL_NOKASAN(strlen) EXPORT_SYMBOL_NOKASAN(strlen)
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7