kernel/bpf/memalloc.c

Source file repositories/reference/linux-study-clean/kernel/bpf/memalloc.c

File Facts

System
Linux kernel
Corpus path
kernel/bpf/memalloc.c
Extension
.c
Size
27814 bytes
Lines
1040
Domain
Core OS
Bucket
Scheduler, Processes, Timers, Sync, And Syscalls
Inferred role
Core OS: implementation source
Status
source implementation candidate

Why This File Exists

Core operating-system implementation surface: boot, tasks, memory, VFS, syscall-facing interfaces, synchronization, credentials, and isolation.

Dependency Surface

Detected Declarations

Annotated Snippet

struct bpf_mem_cache {
	/* per-cpu list of free objects of size 'unit_size'.
	 * All accesses are done with interrupts disabled and 'active' counter
	 * protection with __llist_add() and __llist_del_first().
	 */
	struct llist_head free_llist;
	local_t active;

	/* Operations on the free_list from unit_alloc/unit_free/bpf_mem_refill
	 * are sequenced by per-cpu 'active' counter. But unit_free() cannot
	 * fail. When 'active' is busy the unit_free() will add an object to
	 * free_llist_extra.
	 */
	struct llist_head free_llist_extra;

	struct irq_work refill_work;
	struct obj_cgroup *objcg;
	int unit_size;
	/* count of objects in free_llist */
	int free_cnt;
	int low_watermark, high_watermark, batch;
	int percpu_size;
	bool draining;
	struct bpf_mem_cache *tgt;
	void (*dtor)(void *obj, void *ctx);
	void *dtor_ctx;

	/* list of objects to be freed after RCU GP */
	struct llist_head free_by_rcu;
	struct llist_node *free_by_rcu_tail;
	struct llist_head waiting_for_gp;
	struct llist_node *waiting_for_gp_tail;
	struct rcu_head rcu;
	atomic_t call_rcu_in_progress;
	struct llist_head free_llist_extra_rcu;

	/* list of objects to be freed after RCU tasks trace GP */
	struct llist_head free_by_rcu_ttrace;
	struct llist_head waiting_for_gp_ttrace;
	struct rcu_head rcu_ttrace;
	atomic_t call_rcu_ttrace_in_progress;
};

struct bpf_mem_caches {
	struct bpf_mem_cache cache[NUM_CACHES];
};

static const u16 sizes[NUM_CACHES] = {96, 192, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096};

static struct llist_node notrace *__llist_del_first(struct llist_head *head)
{
	struct llist_node *entry, *next;

	entry = head->first;
	if (!entry)
		return NULL;
	next = entry->next;
	head->first = next;
	return entry;
}

static void *__alloc(struct bpf_mem_cache *c, int node, gfp_t flags)
{
	if (c->percpu_size) {
		void __percpu **obj = kmalloc_node(c->percpu_size, flags, node);
		void __percpu *pptr = __alloc_percpu_gfp(c->unit_size, 8, flags);

		if (!obj || !pptr) {
			free_percpu(pptr);
			kfree(obj);
			return NULL;
		}
		obj[1] = pptr;
		return obj;
	}

	return kmalloc_node(c->unit_size, flags | __GFP_ZERO, node);
}

static struct mem_cgroup *get_memcg(const struct bpf_mem_cache *c)
{
#ifdef CONFIG_MEMCG
	if (c->objcg)
		return get_mem_cgroup_from_objcg(c->objcg);
	return root_mem_cgroup;
#else
	return NULL;
#endif
}

Annotation

Implementation Notes