arch/arm64/lib/strlen.S

Source file repositories/reference/linux-study-clean/arch/arm64/lib/strlen.S

File Facts

System
Linux kernel
Corpus path
arch/arm64/lib/strlen.S
Extension
.S
Size
6353 bytes
Lines
214
Domain
Architecture Layer
Bucket
arch/arm64
Inferred role
Architecture Layer: exported/initcall integration point
Status
integration implementation candidate

Why This File Exists

CPU and platform-specific kernel glue: boot entry, traps, syscall entry, interrupts, page tables, context switch, and low-level barriers.

Dependency Surface

Detected Declarations

Annotated Snippet

#include <linux/linkage.h>
#include <asm/assembler.h>
#include <asm/mte-def.h>

/* Assumptions:
 *
 * ARMv8-a, AArch64, unaligned accesses, min page size 4k.
 */

#define L(label) .L ## label

/* Arguments and results.  */
#define srcin		x0
#define len		x0

/* Locals and temporaries.  */
#define src		x1
#define data1		x2
#define data2		x3
#define has_nul1	x4
#define has_nul2	x5
#define tmp1		x4
#define tmp2		x5
#define tmp3		x6
#define tmp4		x7
#define zeroones	x8

	/* NUL detection works on the principle that (X - 1) & (~X) & 0x80
	   (=> (X - 1) & ~(X | 0x7f)) is non-zero iff a byte is zero, and
	   can be done in parallel across the entire word. A faster check
	   (X - 1) & 0x80 is zero for non-NUL ASCII characters, but gives
	   false hits for characters 129..255.	*/

#define REP8_01 0x0101010101010101
#define REP8_7f 0x7f7f7f7f7f7f7f7f
#define REP8_80 0x8080808080808080

/*
 * When KASAN_HW_TAGS is in use, memory is checked at MTE_GRANULE_SIZE
 * (16-byte) granularity, and we must ensure that no access straddles this
 * alignment boundary.
 */
#ifdef CONFIG_KASAN_HW_TAGS
#define MIN_PAGE_SIZE MTE_GRANULE_SIZE
#else
#define MIN_PAGE_SIZE 4096
#endif

	/* Since strings are short on average, we check the first 16 bytes
	   of the string for a NUL character.  In order to do an unaligned ldp
	   safely we have to do a page cross check first.  If there is a NUL
	   byte we calculate the length from the 2 8-byte words using
	   conditional select to reduce branch mispredictions (it is unlikely
	   strlen will be repeatedly called on strings with the same length).

	   If the string is longer than 16 bytes, we align src so don't need
	   further page cross checks, and process 32 bytes per iteration
	   using the fast NUL check.  If we encounter non-ASCII characters,
	   fallback to a second loop using the full NUL check.

	   If the page cross check fails, we read 16 bytes from an aligned
	   address, remove any characters before the string, and continue
	   in the main loop using aligned loads.  Since strings crossing a
	   page in the first 16 bytes are rare (probability of
	   16/MIN_PAGE_SIZE ~= 0.4%), this case does not need to be optimized.

	   AArch64 systems have a minimum page size of 4k.  We don't bother
	   checking for larger page sizes - the cost of setting up the correct
	   page size is just not worth the extra gain from a small reduction in
	   the cases taking the slow path.  Note that we only care about

Annotation

Implementation Notes