io_uring/io_uring.c

Source file repositories/reference/linux-study-clean/io_uring/io_uring.c

File Facts

System
Linux kernel
Corpus path
io_uring/io_uring.c
Extension
.c
Size
90433 bytes
Lines
3271
Domain
Kernel Services
Bucket
io_uring
Inferred role
Kernel Services: syscall or user/kernel boundary
Status
core implementation candidate

Why This File Exists

Shared kernel service surface used by multiple subsystems, including helpers, cryptography, virtualization support, and async I/O infrastructure.

Dependency Surface

Detected Declarations

Annotated Snippet

SYSCALL_DEFINE6(io_uring_enter, unsigned int, fd, u32, to_submit,
		u32, min_complete, u32, flags, const void __user *, argp,
		size_t, argsz)
{
	struct io_ring_ctx *ctx;
	struct file *file;
	long ret;

	if (unlikely(flags & ~IORING_ENTER_FLAGS))
		return -EINVAL;

	file = io_uring_ctx_get_file(fd, flags & IORING_ENTER_REGISTERED_RING);
	if (IS_ERR(file))
		return PTR_ERR(file);
	ctx = file->private_data;
	ret = -EBADFD;
	/*
	 * Keep IORING_SETUP_R_DISABLED check before submitter_task load
	 * in io_uring_add_tctx_node() -> __io_uring_add_tctx_node_from_submit()
	 */
	if (unlikely(smp_load_acquire(&ctx->flags) & IORING_SETUP_R_DISABLED))
		goto out;

	if (io_has_loop_ops(ctx)) {
		ret = io_run_loop(ctx);
		goto out;
	}

	/*
	 * For SQ polling, the thread will do all submissions and completions.
	 * Just return the requested submit count, and wake the thread if
	 * we were asked to.
	 */
	ret = 0;
	if (ctx->flags & IORING_SETUP_SQPOLL) {
		if (unlikely(ctx->sq_data->thread == NULL)) {
			ret = -EOWNERDEAD;
			goto out;
		}
		if (flags & IORING_ENTER_SQ_WAKEUP)
			wake_up(&ctx->sq_data->wait);
		if (flags & IORING_ENTER_SQ_WAIT)
			io_sqpoll_wait_sq(ctx);

		ret = to_submit;
	} else if (to_submit) {
		ret = io_uring_add_tctx_node(ctx);
		if (unlikely(ret))
			goto out;

		mutex_lock(&ctx->uring_lock);
		ret = io_submit_sqes(ctx, to_submit);
		if (ret != to_submit) {
			mutex_unlock(&ctx->uring_lock);
			goto out;
		}
		if (flags & IORING_ENTER_GETEVENTS) {
			if (ctx->int_flags & IO_RING_F_SYSCALL_IOPOLL)
				goto iopoll_locked;
			/*
			 * Ignore errors, we'll soon call io_cqring_wait() and
			 * it should handle ownership problems if any.
			 */
			if (ctx->flags & IORING_SETUP_DEFER_TASKRUN)
				(void)io_run_local_work_locked(ctx, min_complete);
		}
		mutex_unlock(&ctx->uring_lock);
	}

	if (flags & IORING_ENTER_GETEVENTS) {
		int ret2;

		if (ctx->int_flags & IO_RING_F_SYSCALL_IOPOLL) {
			/*
			 * We disallow the app entering submit/complete with
			 * polling, but we still need to lock the ring to
			 * prevent racing with polled issue that got punted to
			 * a workqueue.
			 */
			mutex_lock(&ctx->uring_lock);
iopoll_locked:
			ret2 = io_validate_ext_arg(ctx, flags, argp, argsz);
			if (likely(!ret2))
				ret2 = io_iopoll_check(ctx, min_complete);
			mutex_unlock(&ctx->uring_lock);
		} else {
			struct ext_arg ext_arg = { .argsz = argsz };

			ret2 = io_get_ext_arg(ctx, flags, argp, &ext_arg);
			if (likely(!ret2))

Annotation

Implementation Notes