arch/arm64/include/asm/fpsimd.h

Source file repositories/reference/linux-study-clean/arch/arm64/include/asm/fpsimd.h

File Facts

System
Linux kernel
Corpus path
arch/arm64/include/asm/fpsimd.h
Extension
.h
Size
20275 bytes
Lines
809
Domain
Architecture Layer
Bucket
arch/arm64
Inferred role
Architecture Layer: implementation source
Status
source implementation candidate

Why This File Exists

CPU and platform-specific kernel glue: boot entry, traps, syscall entry, interrupts, page tables, context switch, and low-level barriers.

Dependency Surface

Detected Declarations

Annotated Snippet

struct cpu_fp_state {
	struct user_fpsimd_state *st;
	struct arm64_sve_state *sve_state;
	struct arm64_sme_state *sme_state;
	u64 *svcr;
	u64 *fpmr;
	unsigned int sve_vl;
	unsigned int sme_vl;
	enum fp_type *fp_type;
	enum fp_type to_save;
};

DECLARE_PER_CPU(struct cpu_fp_state, fpsimd_last_state);

extern void fpsimd_bind_state_to_cpu(struct cpu_fp_state *fp_state);

extern void fpsimd_flush_task_state(struct task_struct *target);
extern void fpsimd_save_and_flush_current_state(void);
extern void fpsimd_save_and_flush_cpu_state(void);

static inline bool thread_sm_enabled(struct thread_struct *thread)
{
	return system_supports_sme() && (thread->svcr & SVCR_SM_MASK);
}

static inline bool thread_za_enabled(struct thread_struct *thread)
{
	return system_supports_sme() && (thread->svcr & SVCR_ZA_MASK);
}

extern void task_smstop_sm(struct task_struct *task);

/* Maximum VL that SVE/SME VL-agnostic software can transparently support */
#define VL_ARCH_MAX 0x100

static inline void *thread_zt_state(struct thread_struct *thread)
{
	/* The ZT register state is stored immediately after the ZA state */
	unsigned int sme_vq = sve_vq_from_vl(thread_get_sme_vl(thread));
	return (void *)thread->sme_state + ZA_SIG_REGS_SIZE(sme_vq);
}

static inline unsigned int sve_get_vl(void)
{
	unsigned int vl;

	asm volatile(
	__SVE_PREAMBLE
	"	rdvl %x[vl], #1\n"
	: [vl] "=r" (vl)
	);

	return vl;
}

#define FOR_EACH_Z_REG(idx_str, asm_str)											\
	"	.irp " idx_str ",0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31\n"	\
	asm_str	"\n"														\
	"	.endr\n"

#define FOR_EACH_P_REG(idx_str, asm_str)											\
	"	.irp " idx_str ",0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15\n"	\
	asm_str	"\n"								\
	"	.endr\n"

static inline void __sve_save_z(struct arm64_sve_state *state, unsigned long vl)
{
	instrument_write(state, SVE_NUM_ZREGS * vl);
	asm volatile(
	__SVE_PREAMBLE
	FOR_EACH_Z_REG("n", "str	z\\n, [%[zregs], #\\n, MUL VL]")
	:
	: [zregs] "r" (state)
	: "memory"
	);
}

static inline void __sve_load_z(const struct arm64_sve_state *state, unsigned long vl)
{
	instrument_read(state, SVE_NUM_ZREGS * vl);
	asm volatile(
	__SVE_PREAMBLE
	FOR_EACH_Z_REG("n", "ldr	z\\n, [%[zregs], #\\n, MUL VL]")
	:
	: [zregs] "r" (state)
	: "memory"
	);
}

static inline void __sve_save_p(struct arm64_sve_state *state, unsigned long vl, bool ffr)

Annotation

Implementation Notes