arch/arm64/include/asm/atomic_ll_sc.h

Source file repositories/reference/linux-study-clean/arch/arm64/include/asm/atomic_ll_sc.h

File Facts

System
Linux kernel
Corpus path
arch/arm64/include/asm/atomic_ll_sc.h
Extension
.h
Size
11012 bytes
Lines
340
Domain
Architecture Layer
Bucket
arch/arm64
Inferred role
Architecture Layer: implementation source
Status
source implementation candidate

Why This File Exists

CPU and platform-specific kernel glue: boot entry, traps, syscall entry, interrupts, page tables, context switch, and low-level barriers.

Dependency Surface

Detected Declarations

Annotated Snippet

#ifndef __ASM_ATOMIC_LL_SC_H
#define __ASM_ATOMIC_LL_SC_H

#include <linux/stringify.h>

#ifndef CONFIG_CC_HAS_K_CONSTRAINT
#define K
#endif

/*
 * AArch64 UP and SMP safe atomic ops.  We use load exclusive and
 * store exclusive to ensure that these are atomic.  We may loop
 * to ensure that the update happens.
 */

#define ATOMIC_OP(op, asm_op, constraint)				\
static __always_inline void						\
__ll_sc_atomic_##op(int i, atomic_t *v)					\
{									\
	unsigned long tmp;						\
	int result;							\
									\
	asm volatile("// atomic_" #op "\n"				\
	"	prfm	pstl1strm, %2\n"				\
	"1:	ldxr	%w0, %2\n"					\
	"	" #asm_op "	%w0, %w0, %w3\n"			\
	"	stxr	%w1, %w0, %2\n"					\
	"	cbnz	%w1, 1b\n"					\
	: "=&r" (result), "=&r" (tmp), "+Q" (v->counter)		\
	: __stringify(constraint) "r" (i));				\
}

#define ATOMIC_OP_RETURN(name, mb, acq, rel, cl, op, asm_op, constraint)\
static __always_inline int						\
__ll_sc_atomic_##op##_return##name(int i, atomic_t *v)			\
{									\
	unsigned long tmp;						\
	int result;							\
									\
	asm volatile("// atomic_" #op "_return" #name "\n"		\
	"	prfm	pstl1strm, %2\n"				\
	"1:	ld" #acq "xr	%w0, %2\n"				\
	"	" #asm_op "	%w0, %w0, %w3\n"			\
	"	st" #rel "xr	%w1, %w0, %2\n"				\
	"	cbnz	%w1, 1b\n"					\
	"	" #mb							\
	: "=&r" (result), "=&r" (tmp), "+Q" (v->counter)		\
	: __stringify(constraint) "r" (i)				\
	: cl);								\
									\
	return result;							\
}

#define ATOMIC_FETCH_OP(name, mb, acq, rel, cl, op, asm_op, constraint) \
static __always_inline int						\
__ll_sc_atomic_fetch_##op##name(int i, atomic_t *v)			\
{									\
	unsigned long tmp;						\
	int val, result;						\
									\
	asm volatile("// atomic_fetch_" #op #name "\n"			\
	"	prfm	pstl1strm, %3\n"				\
	"1:	ld" #acq "xr	%w0, %3\n"				\
	"	" #asm_op "	%w1, %w0, %w4\n"			\
	"	st" #rel "xr	%w2, %w1, %3\n"				\
	"	cbnz	%w2, 1b\n"					\
	"	" #mb							\
	: "=&r" (result), "=&r" (val), "=&r" (tmp), "+Q" (v->counter)	\
	: __stringify(constraint) "r" (i)				\
	: cl);								\
									\
	return result;							\
}

#define ATOMIC_OPS(...)							\
	ATOMIC_OP(__VA_ARGS__)						\
	ATOMIC_OP_RETURN(        , dmb ish,  , l, "memory", __VA_ARGS__)\
	ATOMIC_OP_RETURN(_relaxed,        ,  ,  ,         , __VA_ARGS__)\
	ATOMIC_OP_RETURN(_acquire,        , a,  , "memory", __VA_ARGS__)\
	ATOMIC_OP_RETURN(_release,        ,  , l, "memory", __VA_ARGS__)\
	ATOMIC_FETCH_OP (        , dmb ish,  , l, "memory", __VA_ARGS__)\
	ATOMIC_FETCH_OP (_relaxed,        ,  ,  ,         , __VA_ARGS__)\
	ATOMIC_FETCH_OP (_acquire,        , a,  , "memory", __VA_ARGS__)\
	ATOMIC_FETCH_OP (_release,        ,  , l, "memory", __VA_ARGS__)

ATOMIC_OPS(add, add, I)
ATOMIC_OPS(sub, sub, J)

#undef ATOMIC_OPS
#define ATOMIC_OPS(...)							\

Annotation

Implementation Notes