io_uring/kbuf.c

Source file repositories/reference/linux-study-clean/io_uring/kbuf.c

File Facts

System
Linux kernel
Corpus path
io_uring/kbuf.c
Extension
.c
Size
19635 bytes
Lines
772
Domain
Kernel Services
Bucket
io_uring
Inferred role
Kernel Services: implementation source
Status
source implementation candidate

Why This File Exists

Shared kernel service surface used by multiple subsystems, including helpers, cryptography, virtualization support, and async I/O infrastructure.

Dependency Surface

Detected Declarations

Annotated Snippet

struct io_provide_buf {
	struct file			*file;
	__u64				addr;
	__u32				len;
	__u32				bgid;
	__u32				nbufs;
	__u16				bid;
};

static bool io_kbuf_inc_commit(struct io_buffer_list *bl, int len)
{
	/* No data consumed, return false early to avoid consuming the buffer */
	if (!len)
		return false;

	while (len) {
		struct io_uring_buf *buf;
		u32 buf_len, this_len;

		buf = io_ring_head_to_buf(bl->buf_ring, bl->head, bl->mask);
		buf_len = READ_ONCE(buf->len);
		this_len = min_t(u32, len, buf_len);
		buf_len -= this_len;
		/* Stop looping for invalid buffer length of 0 */
		if (buf_len > bl->min_left_sub_one || !this_len) {
			WRITE_ONCE(buf->addr, READ_ONCE(buf->addr) + this_len);
			WRITE_ONCE(buf->len, buf_len);
			return false;
		}
		WRITE_ONCE(buf->len, 0);
		bl->head++;
		len -= this_len;
	}
	return true;
}

bool io_kbuf_commit(struct io_kiocb *req,
		    struct io_buffer_list *bl, int len, int nr)
{
	if (unlikely(!(req->flags & REQ_F_BUFFERS_COMMIT)))
		return true;

	req->flags &= ~REQ_F_BUFFERS_COMMIT;

	if (unlikely(len < 0))
		return true;
	if (bl->flags & IOBL_INC)
		return io_kbuf_inc_commit(bl, len);
	bl->head += nr;
	return true;
}

static inline struct io_buffer_list *io_buffer_get_list(struct io_ring_ctx *ctx,
							unsigned int bgid)
{
	lockdep_assert_held(&ctx->uring_lock);

	return xa_load(&ctx->io_bl_xa, bgid);
}

static int io_buffer_add_list(struct io_ring_ctx *ctx,
			      struct io_buffer_list *bl, unsigned int bgid)
{
	/*
	 * Store buffer group ID and finally mark the list as visible.
	 * The normal lookup doesn't care about the visibility as we're
	 * always under the ->uring_lock, but lookups from mmap do.
	 */
	bl->bgid = bgid;
	guard(mutex)(&ctx->mmap_lock);
	return xa_err(xa_store(&ctx->io_bl_xa, bgid, bl, GFP_KERNEL));
}

void io_kbuf_drop_legacy(struct io_kiocb *req)
{
	if (WARN_ON_ONCE(!(req->flags & REQ_F_BUFFER_SELECTED)))
		return;
	req->flags &= ~REQ_F_BUFFER_SELECTED;
	kfree(req->kbuf);
	req->kbuf = NULL;
}

bool io_kbuf_recycle_legacy(struct io_kiocb *req, unsigned issue_flags)
{
	struct io_ring_ctx *ctx = req->ctx;
	struct io_buffer_list *bl;
	struct io_buffer *buf;

	io_ring_submit_lock(ctx, issue_flags);

Annotation

Implementation Notes