block/blk-rq-qos.c

Source file repositories/reference/linux-study-clean/block/blk-rq-qos.c

File Facts

System
Linux kernel
Corpus path
block/blk-rq-qos.c
Extension
.c
Size
9726 bytes
Lines
375
Domain
Representative Device Path
Bucket
PCIe NVMe Storage Path
Inferred role
Representative Device Path: implementation source
Status
source implementation candidate

Why This File Exists

Part of the selected hardware vertical slice: PCI discovery, driver binding, NVMe queues, block requests, DMA, interrupts, and completion.

Dependency Surface

Detected Declarations

Annotated Snippet

struct rq_qos_wait_data {
	struct wait_queue_entry wq;
	struct rq_wait *rqw;
	acquire_inflight_cb_t *cb;
	void *private_data;
	bool got_token;
};

static int rq_qos_wake_function(struct wait_queue_entry *curr,
				unsigned int mode, int wake_flags, void *key)
{
	struct rq_qos_wait_data *data = container_of(curr,
						     struct rq_qos_wait_data,
						     wq);

	/*
	 * If we fail to get a budget, return -1 to interrupt the wake up loop
	 * in __wake_up_common.
	 */
	if (!data->cb(data->rqw, data->private_data))
		return -1;

	data->got_token = true;
	/*
	 * autoremove_wake_function() removes the wait entry only when it
	 * actually changed the task state. We want the wait always removed.
	 * Remove explicitly and use default_wake_function().
	 */
	default_wake_function(curr, mode, wake_flags, key);
	/*
	 * Note that the order of operations is important as finish_wait()
	 * tests whether @curr is removed without grabbing the lock. This
	 * should be the last thing to do to make sure we will not have a
	 * UAF access to @data. And the semantics of memory barrier in it
	 * also make sure the waiter will see the latest @data->got_token
	 * once list_empty_careful() in finish_wait() returns true.
	 */
	list_del_init_careful(&curr->entry);
	return 1;
}

/**
 * rq_qos_wait - throttle on a rqw if we need to
 * @rqw: rqw to throttle on
 * @private_data: caller provided specific data
 * @acquire_inflight_cb: inc the rqw->inflight counter if we can
 * @cleanup_cb: the callback to cleanup in case we race with a waker
 *
 * This provides a uniform place for the rq_qos users to do their throttling.
 * Since you can end up with a lot of things sleeping at once, this manages the
 * waking up based on the resources available.  The acquire_inflight_cb should
 * inc the rqw->inflight if we have the ability to do so, or return false if not
 * and then we will sleep until the room becomes available.
 *
 * cleanup_cb is in case that we race with a waker and need to cleanup the
 * inflight count accordingly.
 */
void rq_qos_wait(struct rq_wait *rqw, void *private_data,
		 acquire_inflight_cb_t *acquire_inflight_cb,
		 cleanup_cb_t *cleanup_cb)
{
	struct rq_qos_wait_data data = {
		.rqw		= rqw,
		.cb		= acquire_inflight_cb,
		.private_data	= private_data,
		.got_token	= false,
	};
	bool first_waiter;

	/*
	 * If there are no waiters in the waiting queue, try to increase the
	 * inflight counter if we can. Otherwise, prepare for adding ourselves
	 * to the waiting queue.
	 */
	if (!waitqueue_active(&rqw->wait) && acquire_inflight_cb(rqw, private_data))
		return;

	init_wait_func(&data.wq, rq_qos_wake_function);
	first_waiter = prepare_to_wait_exclusive(&rqw->wait, &data.wq,
						 TASK_UNINTERRUPTIBLE);
	/*
	 * Make sure there is at least one inflight process; otherwise, waiters
	 * will never be woken up. Since there may be no inflight process before
	 * adding ourselves to the waiting queue above, we need to try to
	 * increase the inflight counter for ourselves. And it is sufficient to
	 * guarantee that at least the first waiter to enter the waiting queue
	 * will re-check the waiting condition before going to sleep, thus
	 * ensuring forward progress.
	 */
	if (!data.got_token && first_waiter && acquire_inflight_cb(rqw, private_data)) {

Annotation

Implementation Notes