Ksawlii/kernel_samsung_a53x
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

arm64: lse: Always use LSE atomic instructions

Browse source 
Since we are compiling for a single chipset that is known to support LSE,
the system_uses_lse_atomics() static branch can be eliminated entirely.

Therefore, make system_uses_lse_atomics() always true to always use LSE
atomics, and update ARM64_LSE_ATOMIC_INSN() users to get rid of the extra
nops used for alternatives patching at runtime.

This reduces generated code size by removing LL/SC atomics, which improves
instruction cache footprint.

Signed-off-by: Sultan Alsawaf <sultan@kerneltoast.com>
This commit is contained in:
Sultan Alsawaf 2023-11-18 15:19:37 -08:00 • committed by Ksawlii
parent 6007c2066e
commit 15898055b7
5 changed files with 10 additions and 48 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
arch/arm64/include/asm/cmpxchg.h
View file

@ -25,16 +25,8 @@ static inline u##sz __xchg_case_##name##sz(u##sz x, volatile void *ptr) \
unsigned long tmp; \
\
asm volatile(ARM64_LSE_ATOMIC_INSN( \
/* LL/SC */ \
" prfm pstl1strm, %2\n" \
"1: ld" #acq "xr" #sfx "\t%" #w "0, %2\n" \
" st" #rel "xr" #sfx "\t%w1, %" #w "3, %2\n" \
" cbnz %w1, 1b\n" \
" " #mb, \
/* LSE atomics */ \
" swp" #acq_lse #rel #sfx "\t%" #w "3, %" #w "0, %2\n" \
__nops(3) \
" " #nop_lse) \
" swp" #acq_lse #rel #sfx "\t%" #w "3, %" #w "0, %2\n") \
: "=&r" (ret), "=&r" (tmp), "+Q" (*(u##sz *)ptr) \
: "r" (x) \
: cl); \

12
arch/arm64/include/asm/lse.h
View file

@ -19,11 +19,8 @@
extern struct static_key_false cpu_hwcap_keys[ARM64_NCAPS];
extern struct static_key_false arm64_const_caps_ready;
static inline bool system_uses_lse_atomics(void)
{
return (static_branch_likely(&arm64_const_caps_ready)) &&
static_branch_likely(&cpu_hwcap_keys[ARM64_HAS_LSE_ATOMICS]);
}
/* Always use LSE atomics */
#define system_uses_lse_atomics() true
#define __lse_ll_sc_body(op, ...) \
({ \
@ -32,9 +29,8 @@ static inline bool system_uses_lse_atomics(void)
__ll_sc_##op(__VA_ARGS__); \
})
/* In-line patching at runtime */
#define ARM64_LSE_ATOMIC_INSN(llsc, lse) \
ALTERNATIVE(llsc, __LSE_PREAMBLE lse, ARM64_HAS_LSE_ATOMICS)
/* Always use LSE atomics */
#define ARM64_LSE_ATOMIC_INSN(lse) __LSE_PREAMBLE lse
#else /* CONFIG_ARM64_LSE_ATOMICS */

16
arch/arm64/include/asm/percpu.h
View file

@ -71,14 +71,8 @@ __percpu_##name##_case_##sz(void *ptr, unsigned long val) \
u##sz tmp; \
\
asm volatile (ARM64_LSE_ATOMIC_INSN( \
/* LL/SC */ \
"1: ldxr" #sfx "\t%" #w "[tmp], %[ptr]\n" \
#op_llsc "\t%" #w "[tmp], %" #w "[tmp], %" #w "[val]\n" \
" stxr" #sfx "\t%w[loop], %" #w "[tmp], %[ptr]\n" \
" cbnz %w[loop], 1b", \
/* LSE atomics */ \
#op_lse "\t%" #w "[val], %[ptr]\n" \
__nops(3)) \
#op_lse "\t%" #w "[val], %[ptr]\n") \
: [loop] "=&r" (loop), [tmp] "=&r" (tmp), \
[ptr] "+Q"(*(u##sz *)ptr) \
: [val] "r" ((u##sz)(val))); \
@ -92,15 +86,9 @@ __percpu_##name##_return_case_##sz(void *ptr, unsigned long val) \
u##sz ret; \
\
asm volatile (ARM64_LSE_ATOMIC_INSN( \
/* LL/SC */ \
"1: ldxr" #sfx "\t%" #w "[ret], %[ptr]\n" \
#op_llsc "\t%" #w "[ret], %" #w "[ret], %" #w "[val]\n" \
" stxr" #sfx "\t%w[loop], %" #w "[ret], %[ptr]\n" \
" cbnz %w[loop], 1b", \
/* LSE atomics */ \
#op_lse "\t%" #w "[val], %" #w "[ret], %[ptr]\n" \
#op_llsc "\t%" #w "[ret], %" #w "[ret], %" #w "[val]\n" \
__nops(2)) \
#op_llsc "\t%" #w "[ret], %" #w "[ret], %" #w "[val]\n")\
: [loop] "=&r" (loop), [ret] "=&r" (ret), \
[ptr] "+Q"(*(u##sz *)ptr) \
: [val] "r" ((u##sz)(val))); \

4
arch/arm64/include/asm/processor.h
View file

@ -293,9 +293,7 @@ static inline void prefetchw(const void *ptr)
#define ARCH_HAS_SPINLOCK_PREFETCH
static inline void spin_lock_prefetch(const void *ptr)
{
asm volatile(ARM64_LSE_ATOMIC_INSN(
"prfm pstl1strm, %a0",
"nop") : : "p" (ptr));
/* No prefetch for LSE atomics */
}
extern unsigned long __ro_after_init signal_minsigstksz; /* sigframe size */

16
arch/arm64/kvm/hyp/include/nvhe/spinlock.h
View file

@ -40,16 +40,9 @@ static inline void hyp_spin_lock(hyp_spinlock_t *lock)
asm volatile(
/* Atomically increment the next ticket. */
ARM64_LSE_ATOMIC_INSN(
/* LL/SC */
" prfm pstl1strm, %3\n"
"1: ldaxr %w0, %3\n"
" add %w1, %w0, #(1 << 16)\n"
" stxr %w2, %w1, %3\n"
" cbnz %w2, 1b\n",
/* LSE atomics */
" mov %w2, #(1 << 16)\n"
" ldadda %w2, %w0, %3\n"
__nops(3))
" ldadda %w2, %w0, %3\n")
/* Did we get the lock? */
" eor %w1, %w0, %w0, ror #16\n"
@ -76,14 +69,9 @@ static inline void hyp_spin_unlock(hyp_spinlock_t *lock)
asm volatile(
ARM64_LSE_ATOMIC_INSN(
/* LL/SC */
" ldrh %w1, %0\n"
" add %w1, %w1, #1\n"
" stlrh %w1, %0",
/* LSE atomics */
" mov %w1, #1\n"
" staddlh %w1, %0\n"
__nops(1))
" staddlh %w1, %0\n")
: "=Q" (lock->owner), "=&r" (tmp)
:
: "memory");