kernel/bpf/cpumap.c

Source file repositories/reference/linux-study-clean/kernel/bpf/cpumap.c

File Facts

System
Linux kernel
Corpus path
kernel/bpf/cpumap.c
Extension
.c
Size
22113 bytes
Lines
835
Domain
Core OS
Bucket
Scheduler, Processes, Timers, Sync, And Syscalls
Inferred role
Core OS: implementation source
Status
source implementation candidate

Why This File Exists

Core operating-system implementation surface: boot, tasks, memory, VFS, syscall-facing interfaces, synchronization, credentials, and isolation.

Dependency Surface

Detected Declarations

Annotated Snippet

struct xdp_bulk_queue {
	void *q[CPU_MAP_BULK_SIZE];
	struct list_head flush_node;
	struct bpf_cpu_map_entry *obj;
	unsigned int count;
	local_lock_t bq_lock;
};

/* Struct for every remote "destination" CPU in map */
struct bpf_cpu_map_entry {
	u32 cpu;    /* kthread CPU and map index */
	int map_id; /* Back reference to map */

	/* XDP can run multiple RX-ring queues, need __percpu enqueue store */
	struct xdp_bulk_queue __percpu *bulkq;

	/* Queue with potential multi-producers, and single-consumer kthread */
	struct ptr_ring *queue;
	struct task_struct *kthread;

	struct bpf_cpumap_val value;
	struct bpf_prog *prog;
	struct gro_node gro;

	struct completion kthread_running;
	struct rcu_work free_work;
};

struct bpf_cpu_map {
	struct bpf_map map;
	/* Below members specific for map type */
	struct bpf_cpu_map_entry __rcu **cpu_map;
};

static struct bpf_map *cpu_map_alloc(union bpf_attr *attr)
{
	u32 value_size = attr->value_size;
	struct bpf_cpu_map *cmap;

	/* check sanity of attributes */
	if (attr->max_entries == 0 || attr->key_size != 4 ||
	    (value_size != offsetofend(struct bpf_cpumap_val, qsize) &&
	     value_size != offsetofend(struct bpf_cpumap_val, bpf_prog.fd)) ||
	    attr->map_flags & ~BPF_F_NUMA_NODE)
		return ERR_PTR(-EINVAL);

	/* Pre-limit array size based on NR_CPUS, not final CPU check */
	if (attr->max_entries > NR_CPUS)
		return ERR_PTR(-E2BIG);

	cmap = bpf_map_area_alloc(sizeof(*cmap), NUMA_NO_NODE);
	if (!cmap)
		return ERR_PTR(-ENOMEM);

	bpf_map_init_from_attr(&cmap->map, attr);

	/* Alloc array for possible remote "destination" CPUs */
	cmap->cpu_map = bpf_map_area_alloc(cmap->map.max_entries *
					   sizeof(struct bpf_cpu_map_entry *),
					   cmap->map.numa_node);
	if (!cmap->cpu_map) {
		bpf_map_area_free(cmap);
		return ERR_PTR(-ENOMEM);
	}

	return &cmap->map;
}

static void __cpu_map_ring_cleanup(struct ptr_ring *ring)
{
	/* The tear-down procedure should have made sure that queue is
	 * empty.  See __cpu_map_entry_replace() and work-queue
	 * invoked cpu_map_kthread_stop(). Catch any broken behaviour
	 * gracefully and warn once.
	 */
	void *ptr;

	while ((ptr = ptr_ring_consume(ring))) {
		WARN_ON_ONCE(1);
		if (unlikely(__ptr_test_bit(0, &ptr))) {
			__ptr_clear_bit(0, &ptr);
			kfree_skb(ptr);
			continue;
		}
		xdp_return_frame(ptr);
	}
}

static u32 cpu_map_bpf_prog_run_skb(struct bpf_cpu_map_entry *rcpu,
				    void **skbs, u32 skb_n,

Annotation

Implementation Notes