kernel/rseq.c

Source file repositories/reference/linux-study-clean/kernel/rseq.c

File Facts

System
Linux kernel
Corpus path
kernel/rseq.c
Extension
.c
Size
25775 bytes
Lines
890
Domain
Core OS
Bucket
Scheduler, Processes, Timers, Sync, And Syscalls
Inferred role
Core OS: syscall or user/kernel boundary
Status
core implementation candidate

Why This File Exists

Core operating-system implementation surface: boot, tasks, memory, VFS, syscall-facing interfaces, synchronization, credentials, and isolation.

Dependency Surface

Detected Declarations

Annotated Snippet

SYSCALL_DEFINE4(rseq, struct rseq __user *, rseq, u32, rseq_len, int, flags, u32, sig)
{
	if (flags & RSEQ_FLAG_UNREGISTER)
		return rseq_unregister(rseq, rseq_len, flags, sig);

	if (unlikely(flags & ~RSEQ_FLAGS_SUPPORTED))
		return -EINVAL;

	if (current->rseq.usrptr)
		return rseq_reregister(rseq, rseq_len, sig);

	if (!rseq_length_valid(rseq, rseq_len))
		return -EINVAL;

	return rseq_register(rseq, rseq_len, flags, sig);
}

#ifdef CONFIG_RSEQ_SLICE_EXTENSION
struct slice_timer {
	struct hrtimer	timer;
	void		*cookie;
};

static const unsigned int rseq_slice_ext_nsecs_min =  5 * NSEC_PER_USEC;
static const unsigned int rseq_slice_ext_nsecs_max = 50 * NSEC_PER_USEC;
unsigned int rseq_slice_ext_nsecs __read_mostly = rseq_slice_ext_nsecs_min;
static DEFINE_PER_CPU(struct slice_timer, slice_timer);
DEFINE_STATIC_KEY_TRUE(rseq_slice_extension_key);

/*
 * When the timer expires and the task is still in user space, the return
 * from interrupt will revoke the grant and schedule. If the task already
 * entered the kernel via a syscall and the timer fires before the syscall
 * work was able to cancel it, then depending on the preemption model this
 * will either reschedule on return from interrupt or in the syscall work
 * below.
 */
static enum hrtimer_restart rseq_slice_expired(struct hrtimer *tmr)
{
	struct slice_timer *st = container_of(tmr, struct slice_timer, timer);

	/*
	 * Validate that the task which armed the timer is still on the
	 * CPU. It could have been scheduled out without canceling the
	 * timer.
	 */
	if (st->cookie == current && current->rseq.slice.state.granted) {
		rseq_stat_inc(rseq_stats.s_expired);
		set_need_resched_current();
	}
	return HRTIMER_NORESTART;
}

bool __rseq_arm_slice_extension_timer(void)
{
	struct slice_timer *st = this_cpu_ptr(&slice_timer);
	struct task_struct *curr = current;

	lockdep_assert_irqs_disabled();

	/*
	 * This check prevents a task, which got a time slice extension
	 * granted, from exceeding the maximum scheduling latency when the
	 * grant expired before going out to user space. Don't bother to
	 * clear the grant here, it will be cleaned up automatically before
	 * going out to user space after being scheduled back in.
	 */
	if ((unlikely(curr->rseq.slice.expires < ktime_get_mono_fast_ns()))) {
		set_need_resched_current();
		return true;
	}

	/*
	 * Store the task pointer as a cookie for comparison in the timer
	 * function. This is safe as the timer is CPU local and cannot be
	 * in the expiry function at this point.
	 */
	st->cookie = curr;
	hrtimer_start(&st->timer, curr->rseq.slice.expires, HRTIMER_MODE_ABS_PINNED_HARD);
	/* Arm the syscall entry work */
	set_task_syscall_work(curr, SYSCALL_RSEQ_SLICE);
	return false;
}

static void rseq_cancel_slice_extension_timer(void)
{
	struct slice_timer *st = this_cpu_ptr(&slice_timer);

	/*
	 * st->cookie can be safely read as preemption is disabled and the

Annotation

Implementation Notes