io_uring/sqpoll.c

Source file repositories/reference/linux-study-clean/io_uring/sqpoll.c

File Facts

System
Linux kernel
Corpus path
io_uring/sqpoll.c
Extension
.c
Size
13153 bytes
Lines
560
Domain
Kernel Services
Bucket
io_uring
Inferred role
Kernel Services: implementation source
Status
source implementation candidate

Why This File Exists

Shared kernel service surface used by multiple subsystems, including helpers, cryptography, virtualization support, and async I/O infrastructure.

Dependency Surface

Detected Declarations

Annotated Snippet

struct io_sq_time {
	bool started;
	u64 usec;
};

u64 io_sq_cpu_usec(struct task_struct *tsk)
{
	u64 utime, stime;

	task_cputime_adjusted(tsk, &utime, &stime);
	do_div(stime, 1000);
	return stime;
}

static void io_sq_update_worktime(struct io_sq_data *sqd, struct io_sq_time *ist)
{
	if (!ist->started)
		return;
	ist->started = false;
	sqd->work_time += io_sq_cpu_usec(current) - ist->usec;
}

static void io_sq_start_worktime(struct io_sq_time *ist)
{
	if (ist->started)
		return;
	ist->started = true;
	ist->usec = io_sq_cpu_usec(current);
}

static int __io_sq_thread(struct io_ring_ctx *ctx, struct io_sq_data *sqd,
			  bool cap_entries, struct io_sq_time *ist)
{
	unsigned int to_submit;
	int ret = 0;

	to_submit = io_sqring_entries(ctx);
	/* if we're handling multiple rings, cap submit size for fairness */
	if (cap_entries && to_submit > IORING_SQPOLL_CAP_ENTRIES_VALUE)
		to_submit = IORING_SQPOLL_CAP_ENTRIES_VALUE;

	if (to_submit || !list_empty(&ctx->iopoll_list)) {
		const struct cred *creds = NULL;

		io_sq_start_worktime(ist);

		if (ctx->sq_creds != current_cred())
			creds = override_creds(ctx->sq_creds);

		mutex_lock(&ctx->uring_lock);
		if (!list_empty(&ctx->iopoll_list))
			io_do_iopoll(ctx, true);

		/*
		 * Don't submit if refs are dying, good for io_uring_register(),
		 * but also it is relied upon by io_ring_exit_work()
		 */
		if (to_submit && likely(!percpu_ref_is_dying(&ctx->refs)) &&
		    !(ctx->flags & IORING_SETUP_R_DISABLED))
			ret = io_submit_sqes(ctx, to_submit);
		mutex_unlock(&ctx->uring_lock);

		if (to_submit && wq_has_sleeper(&ctx->sqo_sq_wait))
			wake_up(&ctx->sqo_sq_wait);
		if (creds)
			revert_creds(creds);
	}

	return ret;
}

static bool io_sqd_handle_event(struct io_sq_data *sqd)
{
	bool did_sig = false;
	struct ksignal ksig;

	if (test_bit(IO_SQ_THREAD_SHOULD_PARK, &sqd->state) ||
	    signal_pending(current)) {
		mutex_unlock(&sqd->lock);
		if (signal_pending(current))
			did_sig = get_signal(&ksig);
		wait_event(sqd->wait, !atomic_read(&sqd->park_pending));
		mutex_lock(&sqd->lock);
		sqd->sq_cpu = raw_smp_processor_id();
	}
	return did_sig || test_bit(IO_SQ_THREAD_SHOULD_STOP, &sqd->state);
}

/*
 * Run task_work, processing no more than max_entries at a time. If more

Annotation

Implementation Notes