arch/arm64/net/bpf_jit_comp.c

Source file repositories/reference/linux-study-clean/arch/arm64/net/bpf_jit_comp.c

File Facts

System
Linux kernel
Corpus path
arch/arm64/net/bpf_jit_comp.c
Extension
.c
Size
89879 bytes
Lines
3250
Domain
Architecture Layer
Bucket
arch/arm64
Inferred role
Architecture Layer: implementation source
Status
source implementation candidate

Why This File Exists

CPU and platform-specific kernel glue: boot entry, traps, syscall entry, interrupts, page tables, context switch, and low-level barriers.

Dependency Surface

Detected Declarations

Annotated Snippet

struct jit_ctx {
	const struct bpf_prog *prog;
	int idx;
	int epilogue_offset;
	int *offset;
	int exentry_idx;
	int nr_used_callee_reg;
	u8 used_callee_reg[8]; /* r6~r9, fp, arena_vm_start */
	__le32 *image;
	__le32 *ro_image;
	u32 stack_size;
	u16 stack_arg_size;
	u64 user_vm_start;
	u64 arena_vm_start;
	bool fp_used;
	bool priv_sp_used;
	bool write;
};

struct bpf_plt {
	u32 insn_ldr; /* load target */
	u32 insn_br;  /* branch to target */
	u64 target;   /* target value */
};

#define PLT_TARGET_SIZE   sizeof_field(struct bpf_plt, target)
#define PLT_TARGET_OFFSET offsetof(struct bpf_plt, target)

/* Memory size/value to protect private stack overflow/underflow */
#define PRIV_STACK_GUARD_SZ    16
#define PRIV_STACK_GUARD_VAL   0xEB9F12345678eb9fULL

static inline void emit(const u32 insn, struct jit_ctx *ctx)
{
	if (ctx->image != NULL && ctx->write)
		ctx->image[ctx->idx] = cpu_to_le32(insn);

	ctx->idx++;
}

static inline void emit_u32_data(const u32 data, struct jit_ctx *ctx)
{
	if (ctx->image != NULL && ctx->write)
		ctx->image[ctx->idx] = (__force __le32)data;

	ctx->idx++;
}

static inline void emit_a64_mov_i(const int is64, const int reg,
				  const s32 val, struct jit_ctx *ctx)
{
	u16 hi = val >> 16;
	u16 lo = val & 0xffff;

	if (hi & 0x8000) {
		if (hi == 0xffff) {
			emit(A64_MOVN(is64, reg, (u16)~lo, 0), ctx);
		} else {
			emit(A64_MOVN(is64, reg, (u16)~hi, 16), ctx);
			if (lo != 0xffff)
				emit(A64_MOVK(is64, reg, lo, 0), ctx);
		}
	} else {
		emit(A64_MOVZ(is64, reg, lo, 0), ctx);
		if (hi)
			emit(A64_MOVK(is64, reg, hi, 16), ctx);
	}
}

static int i64_i16_blocks(const u64 val, bool inverse)
{
	return (((val >>  0) & 0xffff) != (inverse ? 0xffff : 0x0000)) +
	       (((val >> 16) & 0xffff) != (inverse ? 0xffff : 0x0000)) +
	       (((val >> 32) & 0xffff) != (inverse ? 0xffff : 0x0000)) +
	       (((val >> 48) & 0xffff) != (inverse ? 0xffff : 0x0000));
}

static inline void emit_a64_mov_i64(const int reg, const u64 val,
				    struct jit_ctx *ctx)
{
	u64 nrm_tmp = val, rev_tmp = ~val;
	bool inverse;
	int shift;

	if (!(nrm_tmp >> 32))
		return emit_a64_mov_i(0, reg, (u32)val, ctx);

	inverse = i64_i16_blocks(nrm_tmp, true) < i64_i16_blocks(nrm_tmp, false);
	shift = max(round_down((inverse ? (fls64(rev_tmp) - 1) :
					  (fls64(nrm_tmp) - 1)), 16), 0);

Annotation

Implementation Notes