arch/arm64/include/asm/barrier.h

Source file repositories/reference/linux-study-clean/arch/arm64/include/asm/barrier.h

File Facts

System
Linux kernel
Corpus path
arch/arm64/include/asm/barrier.h
Extension
.h
Size
6126 bytes
Lines
227
Domain
Architecture Layer
Bucket
arch/arm64
Inferred role
Architecture Layer: implementation source
Status
source implementation candidate

Why This File Exists

CPU and platform-specific kernel glue: boot entry, traps, syscall entry, interrupts, page tables, context switch, and low-level barriers.

Dependency Surface

Detected Declarations

Annotated Snippet

#ifndef __ASM_BARRIER_H
#define __ASM_BARRIER_H

#ifndef __ASSEMBLER__

#include <linux/kasan-checks.h>

#include <asm/alternative-macros.h>

#define __nops(n)	".rept	" #n "\nnop\n.endr\n"
#define nops(n)		asm volatile(__nops(n))

#define sev()		asm volatile("sev" : : : "memory")
#define wfe()		asm volatile("wfe" : : : "memory")
#define wfet(val)	asm volatile("msr s0_3_c1_c0_0, %0"	\
				     : : "r" (val) : "memory")
#define wfi()		asm volatile("wfi" : : : "memory")
#define wfit(val)	asm volatile("msr s0_3_c1_c0_1, %0"	\
				     : : "r" (val) : "memory")

#define isb()		asm volatile("isb" : : : "memory")
#define dmb(opt)	asm volatile("dmb " #opt : : : "memory")
#define dsb(opt)	asm volatile("dsb " #opt : : : "memory")

#define psb_csync()	asm volatile("hint #17" : : : "memory")
#define __tsb_csync()	asm volatile("hint #18" : : : "memory")
#define csdb()		asm volatile("hint #20" : : : "memory")

/*
 * Data Gathering Hint:
 * This instruction prevents merging memory accesses with Normal-NC or
 * Device-GRE attributes before the hint instruction with any memory accesses
 * appearing after the hint instruction.
 */
#define dgh()		asm volatile("hint #6" : : : "memory")

#define spec_bar()	asm volatile(ALTERNATIVE("dsb nsh\nisb\n",		\
						 SB_BARRIER_INSN"nop\n",	\
						 ARM64_HAS_SB))

#define gsb_ack()	asm volatile(GSB_ACK_BARRIER_INSN : : : "memory")
#define gsb_sys()	asm volatile(GSB_SYS_BARRIER_INSN : : : "memory")

#ifdef CONFIG_ARM64_PSEUDO_NMI
#define pmr_sync()						\
	do {							\
		asm volatile(					\
		ALTERNATIVE_CB("dsb sy",			\
			       ARM64_HAS_GIC_PRIO_RELAXED_SYNC,	\
			       alt_cb_patch_nops)		\
		);						\
	} while(0)
#else
#define pmr_sync()	do {} while (0)
#endif

#define __mb()		dsb(sy)
#define __rmb()		dsb(ld)
#define __wmb()		dsb(st)

#define __dma_mb()	dmb(osh)
#define __dma_rmb()	dmb(oshld)
#define __dma_wmb()	dmb(oshst)

#define io_stop_wc()	dgh()

#define tsb_csync()								\
	do {									\
		/*								\
		 * CPUs affected by Arm Erratum 2054223 or 2067961 needs	\
		 * another TSB to ensure the trace is flushed. The barriers	\
		 * don't have to be strictly back to back, as long as the	\
		 * CPU is in trace prohibited state.				\
		 */								\
		if (cpus_have_final_cap(ARM64_WORKAROUND_TSB_FLUSH_FAILURE))	\
			__tsb_csync();						\
		__tsb_csync();							\
	} while (0)

/*
 * Generate a mask for array_index__nospec() that is ~0UL when 0 <= idx < sz
 * and 0 otherwise.
 */
#define array_index_mask_nospec array_index_mask_nospec
static inline unsigned long array_index_mask_nospec(unsigned long idx,
						    unsigned long sz)
{
	unsigned long mask;

	asm volatile(

Annotation

Implementation Notes