kernel/bpf/trampoline.c

Source file repositories/reference/linux-study-clean/kernel/bpf/trampoline.c

File Facts

System
Linux kernel
Corpus path
kernel/bpf/trampoline.c
Extension
.c
Size
46420 bytes
Lines
1763
Domain
Core OS
Bucket
Scheduler, Processes, Timers, Sync, And Syscalls
Inferred role
Core OS: implementation source
Status
source implementation candidate

Why This File Exists

Core operating-system implementation surface: boot, tasks, memory, VFS, syscall-facing interfaces, synchronization, credentials, and isolation.

Dependency Surface

Detected Declarations

Annotated Snippet

struct bpf_trampoline_ops {
	int (*register_fentry)(struct bpf_trampoline *tr, struct bpf_tramp_image *im, void *data);
	int (*unregister_fentry)(struct bpf_trampoline *tr, u32 orig_flags, void *data);
	int (*modify_fentry)(struct bpf_trampoline *tr, u32 orig_flags, struct bpf_tramp_image *im,
			     bool lock_direct_mutex, void *data);
};

#ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS
static int bpf_trampoline_update(struct bpf_trampoline *tr, bool lock_direct_mutex,
				 const struct bpf_trampoline_ops *ops, void *data);
static const struct bpf_trampoline_ops trampoline_ops;

#ifdef CONFIG_HAVE_SINGLE_FTRACE_DIRECT_OPS
static struct bpf_trampoline *direct_ops_ip_lookup(struct ftrace_ops *ops, unsigned long ip)
{
	struct hlist_head *head_ip;
	struct bpf_trampoline *tr;

	mutex_lock(&trampoline_mutex);
	head_ip = &trampoline_ip_table[hash_64(ip, TRAMPOLINE_HASH_BITS)];
	hlist_for_each_entry(tr, head_ip, hlist_ip) {
		if (tr->ip == ip)
			goto out;
	}
	tr = NULL;
out:
	mutex_unlock(&trampoline_mutex);
	return tr;
}
#else
static struct bpf_trampoline *direct_ops_ip_lookup(struct ftrace_ops *ops, unsigned long ip)
{
	return ops->private;
}
#endif /* CONFIG_HAVE_SINGLE_FTRACE_DIRECT_OPS */

static int bpf_tramp_ftrace_ops_func(struct ftrace_ops *ops, unsigned long ip,
				     enum ftrace_ops_cmd cmd)
{
	struct bpf_trampoline *tr;
	int ret = 0;

	tr = direct_ops_ip_lookup(ops, ip);
	if (!tr)
		return -EINVAL;

	if (cmd == FTRACE_OPS_CMD_ENABLE_SHARE_IPMODIFY_SELF) {
		/* This is called inside register_ftrace_direct_multi(), so
		 * trampoline's mutex is already locked.
		 */
		lockdep_assert_held_once(select_trampoline_lock(tr));

		/* Instead of updating the trampoline here, we propagate
		 * -EAGAIN to register_ftrace_direct(). Then we can
		 * retry register_ftrace_direct() after updating the
		 * trampoline.
		 */
		if ((tr->flags & BPF_TRAMP_F_CALL_ORIG) &&
		    !(tr->flags & BPF_TRAMP_F_ORIG_STACK)) {
			if (WARN_ON_ONCE(tr->flags & BPF_TRAMP_F_SHARE_IPMODIFY))
				return -EBUSY;

			tr->flags |= BPF_TRAMP_F_SHARE_IPMODIFY;
			return -EAGAIN;
		}

		return 0;
	}

	/* The normal locking order is
	 *    select_trampoline_lock(tr) => direct_mutex (ftrace.c) => ftrace_lock (ftrace.c)
	 *
	 * The following two commands are called from
	 *
	 *   prepare_direct_functions_for_ipmodify
	 *   cleanup_direct_functions_after_ipmodify
	 *
	 * In both cases, direct_mutex is already locked. Use
	 * mutex_trylock(select_trampoline_lock(tr)) to avoid deadlock in race condition
	 * (something else holds the same pool lock).
	 */
	if (!mutex_trylock(select_trampoline_lock(tr))) {
		/* sleep 1 ms to make sure whatever holding select_trampoline_lock(tr)
		 * makes some progress.
		 */
		msleep(1);
		return -EAGAIN;
	}

	switch (cmd) {

Annotation

Implementation Notes