c29204db27
[ Upstream commit 125b0bb95dd6bec81b806b997a4ccb026eeecf8f ]
We really don't want to do atomic_read() or anything like that, since we
already have the value, not the lock. The whole point of this is that
we've loaded the lock from memory, and we want to check whether the
value we loaded was a locked one or not.
The main use of this is the lockref code, which loads both the lock and
the reference count in one atomic operation, and then works on that
combined value. With the atomic_read(), the compiler would pointlessly
spill the value to the stack, in order to then be able to read it back
"atomically".
This is the qspinlock version of commit c6f4a9002252 ("asm-generic:
ticket-lock: Optimize arch_spin_value_unlocked()") which fixed this same
bug for ticket locks.
Cc: Guo Ren <guoren@kernel.org>
Cc: Ingo Molnar <mingo@kernel.org>
Cc: Waiman Long <longman@redhat.com>
Link: https://lore.kernel.org/all/CAHk-=whNRv0v6kQiV5QO6DJhjH4KEL36vWQ6Re8Csrnh4zbRkQ@mail.gmail.com/
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
Signed-off-by: Sasha Levin <sashal@kernel.org>
138 lines
3.8 KiB
C
Executable file
138 lines
3.8 KiB
C
Executable file
/* SPDX-License-Identifier: GPL-2.0-or-later */
|
|
/*
|
|
* Queued spinlock
|
|
*
|
|
* (C) Copyright 2013-2015 Hewlett-Packard Development Company, L.P.
|
|
* (C) Copyright 2015 Hewlett-Packard Enterprise Development LP
|
|
*
|
|
* Authors: Waiman Long <waiman.long@hpe.com>
|
|
*/
|
|
#ifndef __ASM_GENERIC_QSPINLOCK_H
|
|
#define __ASM_GENERIC_QSPINLOCK_H
|
|
|
|
#include <asm-generic/qspinlock_types.h>
|
|
#include <linux/atomic.h>
|
|
|
|
#ifndef queued_spin_is_locked
|
|
/**
|
|
* queued_spin_is_locked - is the spinlock locked?
|
|
* @lock: Pointer to queued spinlock structure
|
|
* Return: 1 if it is locked, 0 otherwise
|
|
*/
|
|
static __always_inline int queued_spin_is_locked(struct qspinlock *lock)
|
|
{
|
|
/*
|
|
* Any !0 state indicates it is locked, even if _Q_LOCKED_VAL
|
|
* isn't immediately observable.
|
|
*/
|
|
return atomic_read(&lock->val);
|
|
}
|
|
#endif
|
|
|
|
/**
|
|
* queued_spin_value_unlocked - is the spinlock structure unlocked?
|
|
* @lock: queued spinlock structure
|
|
* Return: 1 if it is unlocked, 0 otherwise
|
|
*
|
|
* N.B. Whenever there are tasks waiting for the lock, it is considered
|
|
* locked wrt the lockref code to avoid lock stealing by the lockref
|
|
* code and change things underneath the lock. This also allows some
|
|
* optimizations to be applied without conflict with lockref.
|
|
*/
|
|
static __always_inline int queued_spin_value_unlocked(struct qspinlock lock)
|
|
{
|
|
return !lock.val.counter;
|
|
}
|
|
|
|
/**
|
|
* queued_spin_is_contended - check if the lock is contended
|
|
* @lock : Pointer to queued spinlock structure
|
|
* Return: 1 if lock contended, 0 otherwise
|
|
*/
|
|
static __always_inline int queued_spin_is_contended(struct qspinlock *lock)
|
|
{
|
|
return atomic_read(&lock->val) & ~_Q_LOCKED_MASK;
|
|
}
|
|
|
|
#ifdef CONFIG_SEC_DEBUG_QSPIN_OWNER
|
|
static __always_inline u8 __queued_spin_get_locked_val(void)
|
|
{
|
|
int cpu = raw_smp_processor_id();
|
|
return (cpu << _Q_OWNER_OFFSET) | _Q_DBGEN_VAL | _Q_LOCKED_VAL;
|
|
}
|
|
#endif
|
|
|
|
/**
|
|
* queued_spin_trylock - try to acquire the queued spinlock
|
|
* @lock : Pointer to queued spinlock structure
|
|
* Return: 1 if lock acquired, 0 if failed
|
|
*/
|
|
static __always_inline int queued_spin_trylock(struct qspinlock *lock)
|
|
{
|
|
u32 val = atomic_read(&lock->val);
|
|
|
|
if (unlikely(val))
|
|
return 0;
|
|
|
|
#ifdef CONFIG_SEC_DEBUG_QSPIN_OWNER
|
|
return likely(atomic_try_cmpxchg_acquire(&lock->val, &val, __queued_spin_get_locked_val()));
|
|
#else
|
|
return likely(atomic_try_cmpxchg_acquire(&lock->val, &val, _Q_LOCKED_VAL));
|
|
#endif
|
|
}
|
|
|
|
extern void queued_spin_lock_slowpath(struct qspinlock *lock, u32 val);
|
|
|
|
#ifndef queued_spin_lock
|
|
/**
|
|
* queued_spin_lock - acquire a queued spinlock
|
|
* @lock: Pointer to queued spinlock structure
|
|
*/
|
|
static __always_inline void queued_spin_lock(struct qspinlock *lock)
|
|
{
|
|
u32 val = 0;
|
|
|
|
#ifdef CONFIG_SEC_DEBUG_QSPIN_OWNER
|
|
if (likely(atomic_try_cmpxchg_acquire(&lock->val, &val, __queued_spin_get_locked_val())))
|
|
#else
|
|
if (likely(atomic_try_cmpxchg_acquire(&lock->val, &val, _Q_LOCKED_VAL)))
|
|
#endif
|
|
return;
|
|
|
|
queued_spin_lock_slowpath(lock, val);
|
|
}
|
|
#endif
|
|
|
|
#ifndef queued_spin_unlock
|
|
/**
|
|
* queued_spin_unlock - release a queued spinlock
|
|
* @lock : Pointer to queued spinlock structure
|
|
*/
|
|
static __always_inline void queued_spin_unlock(struct qspinlock *lock)
|
|
{
|
|
/*
|
|
* unlock() needs release semantics:
|
|
*/
|
|
smp_store_release(&lock->locked, 0);
|
|
}
|
|
#endif
|
|
|
|
#ifndef virt_spin_lock
|
|
static __always_inline bool virt_spin_lock(struct qspinlock *lock)
|
|
{
|
|
return false;
|
|
}
|
|
#endif
|
|
|
|
/*
|
|
* Remapping spinlock architecture specific functions to the corresponding
|
|
* queued spinlock functions.
|
|
*/
|
|
#define arch_spin_is_locked(l) queued_spin_is_locked(l)
|
|
#define arch_spin_is_contended(l) queued_spin_is_contended(l)
|
|
#define arch_spin_value_unlocked(l) queued_spin_value_unlocked(l)
|
|
#define arch_spin_lock(l) queued_spin_lock(l)
|
|
#define arch_spin_trylock(l) queued_spin_trylock(l)
|
|
#define arch_spin_unlock(l) queued_spin_unlock(l)
|
|
|
|
#endif /* __ASM_GENERIC_QSPINLOCK_H */
|