drivers/nvme/host/ioctl.c

Source file repositories/reference/linux-study-clean/drivers/nvme/host/ioctl.c

File Facts

System
Linux kernel
Corpus path
drivers/nvme/host/ioctl.c
Extension
.c
Size
23445 bytes
Lines
875
Domain
Representative Device Path
Bucket
PCIe NVMe Storage Path
Inferred role
Representative Device Path: implementation source
Status
source implementation candidate

Why This File Exists

Part of the selected hardware vertical slice: PCI discovery, driver binding, NVMe queues, block requests, DMA, interrupts, and completion.

Dependency Surface

Detected Declarations

Annotated Snippet

struct nvme_uring_data {
	__u64	metadata;
	__u64	addr;
	__u32	data_len;
	__u32	metadata_len;
	__u32	timeout_ms;
};

/*
 * This overlays struct io_uring_cmd pdu.
 * Expect build errors if this grows larger than that.
 */
struct nvme_uring_cmd_pdu {
	struct request *req;
	struct bio *bio;
	u64 result;
	int status;
};

static inline struct nvme_uring_cmd_pdu *nvme_uring_cmd_pdu(
		struct io_uring_cmd *ioucmd)
{
	return io_uring_cmd_to_pdu(ioucmd, struct nvme_uring_cmd_pdu);
}

static void nvme_uring_task_cb(struct io_tw_req tw_req, io_tw_token_t tw)
{
	struct io_uring_cmd *ioucmd = io_uring_cmd_from_tw(tw_req);
	struct nvme_uring_cmd_pdu *pdu = nvme_uring_cmd_pdu(ioucmd);

	if (pdu->bio)
		blk_rq_unmap_user(pdu->bio);
	io_uring_cmd_done32(ioucmd, pdu->status, pdu->result,
			    IO_URING_CMD_TASK_WORK_ISSUE_FLAGS);
}

static enum rq_end_io_ret nvme_uring_cmd_end_io(struct request *req,
						blk_status_t err,
						const struct io_comp_batch *iob)
{
	struct io_uring_cmd *ioucmd = req->end_io_data;
	struct nvme_uring_cmd_pdu *pdu = nvme_uring_cmd_pdu(ioucmd);

	if (nvme_req(req)->flags & NVME_REQ_CANCELLED) {
		pdu->status = -EINTR;
	} else {
		pdu->status = nvme_req(req)->status;
		if (!pdu->status)
			pdu->status = blk_status_to_errno(err);
	}
	pdu->result = le64_to_cpu(nvme_req(req)->result.u64);

	/*
	 * For IOPOLL, check if this completion is happening in the context
	 * of the same io_ring that owns the request (local context). If so,
	 * we can complete inline without task_work overhead. Otherwise, we
	 * must punt to task_work to ensure completion happens in the correct
	 * ring's context.
	 */
	if (blk_rq_is_poll(req) && iob &&
	    iob->poll_ctx == io_uring_cmd_ctx_handle(ioucmd)) {
		if (pdu->bio)
			blk_rq_unmap_user(pdu->bio);
		io_uring_cmd_done32(ioucmd, pdu->status, pdu->result, 0);
	} else {
		io_uring_cmd_do_in_task_lazy(ioucmd, nvme_uring_task_cb);
	}
	return RQ_END_IO_FREE;
}

static int nvme_uring_cmd_io(struct nvme_ctrl *ctrl, struct nvme_ns *ns,
		struct io_uring_cmd *ioucmd, unsigned int issue_flags, bool vec)
{
	struct nvme_uring_cmd_pdu *pdu = nvme_uring_cmd_pdu(ioucmd);
	const struct nvme_uring_cmd *cmd = io_uring_sqe128_cmd(ioucmd->sqe,
							       struct nvme_uring_cmd);
	struct request_queue *q = ns ? ns->queue : ctrl->admin_q;
	struct nvme_uring_data d;
	struct nvme_command c;
	struct iov_iter iter;
	struct iov_iter *map_iter = NULL;
	struct request *req;
	blk_opf_t rq_flags = 0;
	blk_mq_req_flags_t blk_flags = 0;
	int ret;

	c.common.opcode = READ_ONCE(cmd->opcode);
	c.common.flags = READ_ONCE(cmd->flags);
	if (c.common.flags)
		return -EINVAL;

Annotation

Implementation Notes