tools/sched_ext/scx_central.bpf.c

Source file repositories/reference/linux-study-clean/tools/sched_ext/scx_central.bpf.c

File Facts

System
Linux kernel
Corpus path
tools/sched_ext/scx_central.bpf.c
Extension
.c
Size
10903 bytes
Lines
385
Domain
Support Tooling And Documentation
Bucket
tools
Inferred role
Support Tooling And Documentation: implementation source
Status
source implementation candidate

Why This File Exists

Repository support layer: documentation, build tooling, samples, user-space helper tools, generated initramfs support, licenses, and validation utilities.

Dependency Surface

Detected Declarations

Annotated Snippet

struct central_timer {
	struct bpf_timer timer;
};

struct {
	__uint(type, BPF_MAP_TYPE_ARRAY);
	__uint(max_entries, 1);
	__type(key, u32);
	__type(value, struct central_timer);
} central_timer SEC(".maps");

s32 BPF_STRUCT_OPS(central_select_cpu, struct task_struct *p,
		   s32 prev_cpu, u64 wake_flags)
{
	/*
	 * Steer wakeups to the central CPU as much as possible to avoid
	 * disturbing other CPUs. It's safe to blindly return the central cpu as
	 * select_cpu() is a hint and if @p can't be on it, the kernel will
	 * automatically pick a fallback CPU.
	 */
	return central_cpu;
}

void BPF_STRUCT_OPS(central_enqueue, struct task_struct *p, u64 enq_flags)
{
	s32 pid = p->pid;

	__sync_fetch_and_add(&nr_total, 1);

	/*
	 * Push per-cpu kthreads at the head of local dsq's and preempt the
	 * corresponding CPU. This ensures that e.g. ksoftirqd isn't blocked
	 * behind other threads which is necessary for forward progress
	 * guarantee as we depend on the BPF timer which may run from ksoftirqd.
	 */
	if ((p->flags & PF_KTHREAD) && p->nr_cpus_allowed == 1) {
		__sync_fetch_and_add(&nr_locals, 1);
		scx_bpf_dsq_insert(p, SCX_DSQ_LOCAL, SCX_SLICE_INF,
				   enq_flags | SCX_ENQ_PREEMPT);
		return;
	}

	if (bpf_map_push_elem(&central_q, &pid, 0)) {
		__sync_fetch_and_add(&nr_overflows, 1);
		scx_bpf_dsq_insert(p, FALLBACK_DSQ_ID, SCX_SLICE_INF, enq_flags);
		return;
	}

	__sync_fetch_and_add(&nr_queued, 1);

	if (!scx_bpf_task_running(p))
		scx_bpf_kick_cpu(central_cpu, SCX_KICK_PREEMPT);
}

static bool dispatch_to_cpu(s32 cpu)
{
	struct task_struct *p;
	s32 pid;

	bpf_repeat(BPF_MAX_LOOPS) {
		if (bpf_map_pop_elem(&central_q, &pid))
			break;

		__sync_fetch_and_sub(&nr_queued, 1);

		p = bpf_task_from_pid(pid);
		if (!p) {
			__sync_fetch_and_add(&nr_lost_pids, 1);
			continue;
		}

		/*
		 * If we can't run the task at the top for whatever reason,
		 * bounce it to the fallback dsq. Also check
		 * is_migration_disabled() explicitly as p->cpus_ptr may not
		 * reflect the migration-disabled state yet if
		 * migrate_disable_switch() hasn't run.
		 */
		if (!bpf_cpumask_test_cpu(cpu, p->cpus_ptr) ||
		    (is_migration_disabled(p) && scx_bpf_task_cpu(p) != cpu)) {
			__sync_fetch_and_add(&nr_mismatches, 1);
			scx_bpf_dsq_insert(p, FALLBACK_DSQ_ID, SCX_SLICE_INF, 0);
			bpf_task_release(p);
			/*
			 * We might run out of dispatch buffer slots if we continue dispatching
			 * to the fallback DSQ, without dispatching to the local DSQ of the
			 * target CPU. In such a case, break the loop now as will fail the
			 * next dispatch operation.
			 */
			if (!scx_bpf_dispatch_nr_slots())

Annotation

Implementation Notes