kernel/locking/rtmutex.c

Source file repositories/reference/linux-study-clean/kernel/locking/rtmutex.c

File Facts

System
Linux kernel
Corpus path
kernel/locking/rtmutex.c
Extension
.c
Size
54084 bytes
Lines
1927
Domain
Core OS
Bucket
Scheduler, Processes, Timers, Sync, And Syscalls
Inferred role
Core OS: implementation source
Status
source implementation candidate

Why This File Exists

Core operating-system implementation surface: boot, tasks, memory, VFS, syscall-facing interfaces, synchronization, credentials, and isolation.

Dependency Surface

Detected Declarations

Annotated Snippet

*		  if (wait_list_empty(l) {
	 *		    l->owner = owner
	 *		    owner = l->owner & ~HAS_WAITERS;
	 *		      ==> l->owner = T1
	 *		  }
	 *				lock(l->lock)
	 * rt_mutex_unlock(l)		fixup_rt_mutex_waiters()
	 *				  if (wait_list_empty(l) {
	 *				    owner = l->owner & ~HAS_WAITERS;
	 * cmpxchg(l->owner, T1, NULL)
	 *  ===> Success (l->owner = NULL)
	 *
	 *				    l->owner = owner
	 *				      ==> l->owner = T1
	 *				  }
	 *
	 * With the check for the waiter bit in place T3 on CPU2 will not
	 * overwrite. All tasks fiddling with the waiters bit are
	 * serialized by l->lock, so nothing else can modify the waiters
	 * bit. If the bit is set then nothing can change l->owner either
	 * so the simple RMW is safe. The cmpxchg() will simply fail if it
	 * happens in the middle of the RMW because the waiters bit is
	 * still set.
	 */
	owner = READ_ONCE(*p);
	if (owner & RT_MUTEX_HAS_WAITERS) {
		/*
		 * See rt_mutex_set_owner() and rt_mutex_clear_owner() on
		 * why xchg_acquire() is used for updating owner for
		 * locking and WRITE_ONCE() for unlocking.
		 *
		 * WRITE_ONCE() would work for the acquire case too, but
		 * in case that the lock acquisition failed it might
		 * force other lockers into the slow path unnecessarily.
		 */
		if (acquire_lock)
			xchg_acquire(p, owner & ~RT_MUTEX_HAS_WAITERS);
		else
			WRITE_ONCE(*p, owner & ~RT_MUTEX_HAS_WAITERS);
	}
}

/*
 * We can speed up the acquire/release, if there's no debugging state to be
 * set up.
 */
#ifndef CONFIG_DEBUG_RT_MUTEXES
static __always_inline bool rt_mutex_cmpxchg_acquire(struct rt_mutex_base *lock,
						     struct task_struct *old,
						     struct task_struct *new)
{
	return try_cmpxchg_acquire(&lock->owner, &old, new);
}

static __always_inline bool rt_mutex_try_acquire(struct rt_mutex_base *lock)
{
	return rt_mutex_cmpxchg_acquire(lock, NULL, current);
}

static __always_inline bool rt_mutex_cmpxchg_release(struct rt_mutex_base *lock,
						     struct task_struct *old,
						     struct task_struct *new)
{
	return try_cmpxchg_release(&lock->owner, &old, new);
}

/*
 * Callers must hold the ->wait_lock -- which is the whole purpose as we force
 * all future threads that attempt to [Rmw] the lock to the slowpath. As such
 * relaxed semantics suffice.
 */
static __always_inline void mark_rt_mutex_waiters(struct rt_mutex_base *lock)
{
	unsigned long *p = (unsigned long *) &lock->owner;
	unsigned long owner, new;

	owner = READ_ONCE(*p);
	do {
		new = owner | RT_MUTEX_HAS_WAITERS;
	} while (!try_cmpxchg_relaxed(p, &owner, new));

	/*
	 * The cmpxchg loop above is relaxed to avoid back-to-back ACQUIRE
	 * operations in the event of contention. Ensure the successful
	 * cmpxchg is visible.
	 */
	smp_mb__after_atomic();
}

/*

Annotation

Implementation Notes