arch/s390/include/asm/percpu.h

Source file repositories/reference/linux-study-clean/arch/s390/include/asm/percpu.h

File Facts

System
Linux kernel
Corpus path
arch/s390/include/asm/percpu.h
Extension
.h
Size
14232 bytes
Lines
364
Domain
Architecture Layer
Bucket
arch/s390
Inferred role
Architecture Layer: implementation source
Status
source implementation candidate

Why This File Exists

CPU and platform-specific kernel glue: boot entry, traps, syscall entry, interrupts, page tables, context switch, and low-level barriers.

Dependency Surface

Detected Declarations

Annotated Snippet

#ifndef __ARCH_S390_PERCPU__
#define __ARCH_S390_PERCPU__

#include <linux/preempt.h>
#include <asm/cmpxchg.h>
#include <asm/march.h>

/*
 * s390 uses its own implementation for per cpu data, the offset of
 * the cpu local data area is cached in the cpu's lowcore memory.
 */
#define __my_cpu_offset get_lowcore()->percpu_offset

#define arch_raw_cpu_ptr(_ptr)						\
({									\
	unsigned long lc_percpu, tcp_ptr__;				\
									\
	tcp_ptr__ = (__force unsigned long)(_ptr);			\
	lc_percpu = offsetof(struct lowcore, percpu_offset);		\
	asm_inline volatile(						\
	ALTERNATIVE("ag		%[__ptr__],%[offzero](%%r0)\n",		\
		    "ag		%[__ptr__],%[offalt](%%r0)\n",		\
		    ALT_FEATURE(MFEATURE_LOWCORE))			\
	: [__ptr__] "+d" (tcp_ptr__)					\
	: [offzero] "i" (lc_percpu),					\
	  [offalt] "i" (lc_percpu + LOWCORE_ALT_ADDRESS),		\
	  "m" (((struct lowcore *)0)->percpu_offset)			\
	: "cc");							\
	(TYPEOF_UNQUAL(*(_ptr)) __force __kernel *)tcp_ptr__;		\
})

/*
 * We use a compare-and-swap loop since that uses less cpu cycles than
 * disabling and enabling interrupts like the generic variant would do.
 */
#define arch_this_cpu_to_op_simple(pcp, val, op)			\
({									\
	typedef typeof(pcp) pcp_op_T__;					\
	pcp_op_T__ old__, new__, prev__;				\
	pcp_op_T__ *ptr__;						\
	preempt_disable_notrace();					\
	ptr__ = raw_cpu_ptr(&(pcp));					\
	prev__ = READ_ONCE(*ptr__);					\
	do {								\
		old__ = prev__;						\
		new__ = old__ op (val);					\
		prev__ = cmpxchg(ptr__, old__, new__);			\
	} while (prev__ != old__);					\
	preempt_enable_notrace();					\
	new__;								\
})

#define this_cpu_add_1(pcp, val)	arch_this_cpu_to_op_simple(pcp, val, +)
#define this_cpu_add_2(pcp, val)	arch_this_cpu_to_op_simple(pcp, val, +)
#define this_cpu_add_return_1(pcp, val) arch_this_cpu_to_op_simple(pcp, val, +)
#define this_cpu_add_return_2(pcp, val) arch_this_cpu_to_op_simple(pcp, val, +)
#define this_cpu_and_1(pcp, val)	arch_this_cpu_to_op_simple(pcp, val, &)
#define this_cpu_and_2(pcp, val)	arch_this_cpu_to_op_simple(pcp, val, &)
#define this_cpu_or_1(pcp, val)		arch_this_cpu_to_op_simple(pcp, val, |)
#define this_cpu_or_2(pcp, val)		arch_this_cpu_to_op_simple(pcp, val, |)

/*
 * Macros to be used for percpu code section based on atomic instructions.
 *
 * Avoid the need to use preempt_disable() / preempt_disable() pairs and the
 * conditional preempt_schedule_notrace() function calls which come with
 * this. The idea is that this_cpu operations based on atomic instructions are
 * guarded with mviy instructions:
 *
 * - The first mviy instruction writes the register number, which contains the
 *   percpu address variable to lowcore. This also indicates that a percpu
 *   code section is executed.
 *
 * - The first mviy instruction following the mviy instruction must be the ag
 *   instruction which adds the percpu offset to the percpu address register.
 *
 * - Afterwards the atomic percpu operation follows.
 *
 * - Then a second mviy instruction writes a zero to lowcore, which indicates
 *   the end of the percpu code section.
 *
 * - In case of an interrupt/exception/nmi the register number which was
 *   written to lowcore is copied to the exception frame (pt_regs), and a zero
 *   is written to lowcore.
 *
 * - On return to the previous context it is checked if a percpu code section
 *   was executed (saved register number not zero), and if the process was
 *   migrated to a different cpu. If the percpu offset was already added to
 *   the percpu address register (instruction address does _not_ point to the
 *   ag instruction) the content of the percpu address register is adjusted so

Annotation

Implementation Notes