io_uring/eventfd.c

Source file repositories/reference/linux-study-clean/io_uring/eventfd.c

File Facts

System
Linux kernel
Corpus path
io_uring/eventfd.c
Extension
.c
Size
4223 bytes
Lines
174
Domain
Kernel Services
Bucket
io_uring
Inferred role
Kernel Services: implementation source
Status
source implementation candidate

Why This File Exists

Shared kernel service surface used by multiple subsystems, including helpers, cryptography, virtualization support, and async I/O infrastructure.

Dependency Surface

Detected Declarations

Annotated Snippet

struct io_ev_fd {
	struct eventfd_ctx	*cq_ev_fd;
	unsigned int		eventfd_async;
	/* protected by ->completion_lock */
	unsigned		last_cq_tail;
	refcount_t		refs;
	atomic_t		ops;
	struct rcu_head		rcu;
};

enum {
	IO_EVENTFD_OP_SIGNAL_BIT,
};

static void io_eventfd_free(struct rcu_head *rcu)
{
	struct io_ev_fd *ev_fd = container_of(rcu, struct io_ev_fd, rcu);

	eventfd_ctx_put(ev_fd->cq_ev_fd);
	kfree(ev_fd);
}

static void io_eventfd_put(struct io_ev_fd *ev_fd)
{
	if (refcount_dec_and_test(&ev_fd->refs))
		call_rcu(&ev_fd->rcu, io_eventfd_free);
}

static void io_eventfd_do_signal(struct rcu_head *rcu)
{
	struct io_ev_fd *ev_fd = container_of(rcu, struct io_ev_fd, rcu);

	atomic_andnot(BIT(IO_EVENTFD_OP_SIGNAL_BIT), &ev_fd->ops);
	eventfd_signal_mask(ev_fd->cq_ev_fd, EPOLL_URING_WAKE);
	io_eventfd_put(ev_fd);
}

/*
 * Returns true if the caller should put the ev_fd reference, false if not.
 */
static bool __io_eventfd_signal(struct io_ev_fd *ev_fd)
{
	if (eventfd_signal_allowed()) {
		eventfd_signal_mask(ev_fd->cq_ev_fd, EPOLL_URING_WAKE);
		return true;
	}
	if (!atomic_fetch_or(BIT(IO_EVENTFD_OP_SIGNAL_BIT), &ev_fd->ops)) {
		call_rcu_hurry(&ev_fd->rcu, io_eventfd_do_signal);
		return false;
	}
	return true;
}

/*
 * Trigger if eventfd_async isn't set, or if it's set and the caller is
 * an async worker.
 */
static bool io_eventfd_trigger(struct io_ev_fd *ev_fd)
{
	return !ev_fd->eventfd_async || io_wq_current_is_worker();
}

void io_eventfd_signal(struct io_ring_ctx *ctx, bool cqe_event)
{
	bool skip = false;
	struct io_ev_fd *ev_fd;
	struct io_rings *rings;

	guard(rcu)();

	rings = rcu_dereference(ctx->rings_rcu);
	if (!rings)
		return;
	if (READ_ONCE(rings->cq_flags) & IORING_CQ_EVENTFD_DISABLED)
		return;
	ev_fd = rcu_dereference(ctx->io_ev_fd);
	/*
	 * Check again if ev_fd exists in case an io_eventfd_unregister call
	 * completed between the NULL check of ctx->io_ev_fd at the start of
	 * the function and rcu_read_lock.
	 */
	if (!ev_fd)
		return;
	if (!io_eventfd_trigger(ev_fd) || !refcount_inc_not_zero(&ev_fd->refs))
		return;

	if (cqe_event) {
		/*
		 * Eventfd should only get triggered when at least one event
		 * has been posted. Some applications rely on the eventfd

Annotation

Implementation Notes