kernel/bpf/ringbuf.c

Source file repositories/reference/linux-study-clean/kernel/bpf/ringbuf.c

File Facts

System
Linux kernel
Corpus path
kernel/bpf/ringbuf.c
Extension
.c
Size
25922 bytes
Lines
881
Domain
Core OS
Bucket
Scheduler, Processes, Timers, Sync, And Syscalls
Inferred role
Core OS: implementation source
Status
source implementation candidate

Why This File Exists

Core operating-system implementation surface: boot, tasks, memory, VFS, syscall-facing interfaces, synchronization, credentials, and isolation.

Dependency Surface

Detected Declarations

Annotated Snippet

struct bpf_ringbuf {
	wait_queue_head_t waitq;
	struct irq_work work;
	u64 mask;
	struct page **pages;
	int nr_pages;
	bool overwrite_mode;
	rqspinlock_t spinlock ____cacheline_aligned_in_smp;
	/* For user-space producer ring buffers, an atomic_t busy bit is used
	 * to synchronize access to the ring buffers in the kernel, rather than
	 * the spinlock that is used for kernel-producer ring buffers. This is
	 * done because the ring buffer must hold a lock across a BPF program's
	 * callback:
	 *
	 *    __bpf_user_ringbuf_peek() // lock acquired
	 * -> program callback_fn()
	 * -> __bpf_user_ringbuf_sample_release() // lock released
	 *
	 * It is unsafe and incorrect to hold an IRQ spinlock across what could
	 * be a long execution window, so we instead simply disallow concurrent
	 * access to the ring buffer by kernel consumers, and return -EBUSY from
	 * __bpf_user_ringbuf_peek() if the busy bit is held by another task.
	 */
	atomic_t busy ____cacheline_aligned_in_smp;
	/* Consumer and producer counters are put into separate pages to
	 * allow each position to be mapped with different permissions.
	 * This prevents a user-space application from modifying the
	 * position and ruining in-kernel tracking. The permissions of the
	 * pages depend on who is producing samples: user-space or the
	 * kernel. Note that the pending counter is placed in the same
	 * page as the producer, so that it shares the same cache line.
	 *
	 * Kernel-producer
	 * ---------------
	 * The producer position and data pages are mapped as r/o in
	 * userspace. For this approach, bits in the header of samples are
	 * used to signal to user-space, and to other producers, whether a
	 * sample is currently being written.
	 *
	 * User-space producer
	 * -------------------
	 * Only the page containing the consumer position is mapped r/o in
	 * user-space. User-space producers also use bits of the header to
	 * communicate to the kernel, but the kernel must carefully check and
	 * validate each sample to ensure that they're correctly formatted, and
	 * fully contained within the ring buffer.
	 */
	unsigned long consumer_pos __aligned(PAGE_SIZE);
	unsigned long producer_pos __aligned(PAGE_SIZE);
	unsigned long pending_pos;
	unsigned long overwrite_pos; /* position after the last overwritten record */
	char data[] __aligned(PAGE_SIZE);
};

struct bpf_ringbuf_map {
	struct bpf_map map;
	struct bpf_ringbuf *rb;
};

/* 8-byte ring buffer record header structure */
struct bpf_ringbuf_hdr {
	u32 len;
	u32 pg_off;
};

static struct bpf_ringbuf *bpf_ringbuf_area_alloc(size_t data_sz, int numa_node)
{
	const gfp_t flags = GFP_KERNEL_ACCOUNT | __GFP_RETRY_MAYFAIL |
			    __GFP_NOWARN | __GFP_ZERO;
	int nr_meta_pages = RINGBUF_NR_META_PAGES;
	int nr_data_pages = data_sz >> PAGE_SHIFT;
	int nr_pages = nr_meta_pages + nr_data_pages;
	struct page **pages, *page;
	struct bpf_ringbuf *rb;
	size_t array_size;
	int i;

	/* Each data page is mapped twice to allow "virtual"
	 * continuous read of samples wrapping around the end of ring
	 * buffer area:
	 * ------------------------------------------------------
	 * | meta pages |  real data pages  |  same data pages  |
	 * ------------------------------------------------------
	 * |            | 1 2 3 4 5 6 7 8 9 | 1 2 3 4 5 6 7 8 9 |
	 * ------------------------------------------------------
	 * |            | TA             DA | TA             DA |
	 * ------------------------------------------------------
	 *                               ^^^^^^^
	 *                                  |
	 * Here, no need to worry about special handling of wrapped-around

Annotation

Implementation Notes