riscv/cmpxchg: Deduplicate cmpxchg() asm and macros
In this header every cmpxchg define (_relaxed, _acquire, _release, vanilla) contain it's own asm file, both for 4-byte variables an 8-byte variables, on a total of 8 versions of mostly the same asm. This is usually bad, as it means any change may be done in up to 8 different places. Unify those versions by creating a new define with enough parameters to generate any version of the previous 8. Then unify the result under a more general define, and simplify arch_cmpxchg* generation (This did not cause any change in generated asm) Signed-off-by: Leonardo Bras <leobras@redhat.com> Reviewed-by: Guo Ren <guoren@kernel.org> Reviewed-by: Andrea Parri <parri.andrea@gmail.com> Tested-by: Guo Ren <guoren@kernel.org> Link: https://lore.kernel.org/r/20240103163203.72768-4-leobras@redhat.com Signed-off-by: Palmer Dabbelt <palmer@rivosinc.com>
This commit is contained in:
Leonardo Bras
committed by
Palmer Dabbelt
Palmer Dabbelt
parent
4bfa185fe3
commit
07a0a41cb7
@@ -71,190 +71,61 @@
|
||||
* store NEW in MEM. Return the initial value in MEM. Success is
|
||||
* indicated by comparing RETURN with OLD.
|
||||
*/
|
||||
#define __cmpxchg_relaxed(ptr, old, new, size) \
|
||||
|
||||
|
||||
#define __arch_cmpxchg(lr_sfx, sc_sfx, prepend, append, r, p, co, o, n) \
|
||||
({ \
|
||||
register unsigned int __rc; \
|
||||
\
|
||||
__asm__ __volatile__ ( \
|
||||
prepend \
|
||||
"0: lr" lr_sfx " %0, %2\n" \
|
||||
" bne %0, %z3, 1f\n" \
|
||||
" sc" sc_sfx " %1, %z4, %2\n" \
|
||||
" bnez %1, 0b\n" \
|
||||
append \
|
||||
"1:\n" \
|
||||
: "=&r" (r), "=&r" (__rc), "+A" (*(p)) \
|
||||
: "rJ" (co o), "rJ" (n) \
|
||||
: "memory"); \
|
||||
})
|
||||
|
||||
#define _arch_cmpxchg(ptr, old, new, sc_sfx, prepend, append) \
|
||||
({ \
|
||||
__typeof__(ptr) __ptr = (ptr); \
|
||||
__typeof__(*(ptr)) __old = (old); \
|
||||
__typeof__(*(ptr)) __new = (new); \
|
||||
__typeof__(*(ptr)) __ret; \
|
||||
register unsigned int __rc; \
|
||||
switch (size) { \
|
||||
__typeof__(*(__ptr)) __old = (old); \
|
||||
__typeof__(*(__ptr)) __new = (new); \
|
||||
__typeof__(*(__ptr)) __ret; \
|
||||
\
|
||||
switch (sizeof(*__ptr)) { \
|
||||
case 4: \
|
||||
__asm__ __volatile__ ( \
|
||||
"0: lr.w %0, %2\n" \
|
||||
" bne %0, %z3, 1f\n" \
|
||||
" sc.w %1, %z4, %2\n" \
|
||||
" bnez %1, 0b\n" \
|
||||
"1:\n" \
|
||||
: "=&r" (__ret), "=&r" (__rc), "+A" (*__ptr) \
|
||||
: "rJ" ((long)__old), "rJ" (__new) \
|
||||
: "memory"); \
|
||||
__arch_cmpxchg(".w", ".w" sc_sfx, prepend, append, \
|
||||
__ret, __ptr, (long), __old, __new); \
|
||||
break; \
|
||||
case 8: \
|
||||
__asm__ __volatile__ ( \
|
||||
"0: lr.d %0, %2\n" \
|
||||
" bne %0, %z3, 1f\n" \
|
||||
" sc.d %1, %z4, %2\n" \
|
||||
" bnez %1, 0b\n" \
|
||||
"1:\n" \
|
||||
: "=&r" (__ret), "=&r" (__rc), "+A" (*__ptr) \
|
||||
: "rJ" (__old), "rJ" (__new) \
|
||||
: "memory"); \
|
||||
__arch_cmpxchg(".d", ".d" sc_sfx, prepend, append, \
|
||||
__ret, __ptr, /**/, __old, __new); \
|
||||
break; \
|
||||
default: \
|
||||
BUILD_BUG(); \
|
||||
} \
|
||||
__ret; \
|
||||
(__typeof__(*(__ptr)))__ret; \
|
||||
})
|
||||
|
||||
#define arch_cmpxchg_relaxed(ptr, o, n) \
|
||||
({ \
|
||||
__typeof__(*(ptr)) _o_ = (o); \
|
||||
__typeof__(*(ptr)) _n_ = (n); \
|
||||
(__typeof__(*(ptr))) __cmpxchg_relaxed((ptr), \
|
||||
_o_, _n_, sizeof(*(ptr))); \
|
||||
})
|
||||
|
||||
#define __cmpxchg_acquire(ptr, old, new, size) \
|
||||
({ \
|
||||
__typeof__(ptr) __ptr = (ptr); \
|
||||
__typeof__(*(ptr)) __old = (old); \
|
||||
__typeof__(*(ptr)) __new = (new); \
|
||||
__typeof__(*(ptr)) __ret; \
|
||||
register unsigned int __rc; \
|
||||
switch (size) { \
|
||||
case 4: \
|
||||
__asm__ __volatile__ ( \
|
||||
"0: lr.w %0, %2\n" \
|
||||
" bne %0, %z3, 1f\n" \
|
||||
" sc.w %1, %z4, %2\n" \
|
||||
" bnez %1, 0b\n" \
|
||||
RISCV_ACQUIRE_BARRIER \
|
||||
"1:\n" \
|
||||
: "=&r" (__ret), "=&r" (__rc), "+A" (*__ptr) \
|
||||
: "rJ" ((long)__old), "rJ" (__new) \
|
||||
: "memory"); \
|
||||
break; \
|
||||
case 8: \
|
||||
__asm__ __volatile__ ( \
|
||||
"0: lr.d %0, %2\n" \
|
||||
" bne %0, %z3, 1f\n" \
|
||||
" sc.d %1, %z4, %2\n" \
|
||||
" bnez %1, 0b\n" \
|
||||
RISCV_ACQUIRE_BARRIER \
|
||||
"1:\n" \
|
||||
: "=&r" (__ret), "=&r" (__rc), "+A" (*__ptr) \
|
||||
: "rJ" (__old), "rJ" (__new) \
|
||||
: "memory"); \
|
||||
break; \
|
||||
default: \
|
||||
BUILD_BUG(); \
|
||||
} \
|
||||
__ret; \
|
||||
})
|
||||
_arch_cmpxchg((ptr), (o), (n), "", "", "")
|
||||
|
||||
#define arch_cmpxchg_acquire(ptr, o, n) \
|
||||
({ \
|
||||
__typeof__(*(ptr)) _o_ = (o); \
|
||||
__typeof__(*(ptr)) _n_ = (n); \
|
||||
(__typeof__(*(ptr))) __cmpxchg_acquire((ptr), \
|
||||
_o_, _n_, sizeof(*(ptr))); \
|
||||
})
|
||||
|
||||
#define __cmpxchg_release(ptr, old, new, size) \
|
||||
({ \
|
||||
__typeof__(ptr) __ptr = (ptr); \
|
||||
__typeof__(*(ptr)) __old = (old); \
|
||||
__typeof__(*(ptr)) __new = (new); \
|
||||
__typeof__(*(ptr)) __ret; \
|
||||
register unsigned int __rc; \
|
||||
switch (size) { \
|
||||
case 4: \
|
||||
__asm__ __volatile__ ( \
|
||||
RISCV_RELEASE_BARRIER \
|
||||
"0: lr.w %0, %2\n" \
|
||||
" bne %0, %z3, 1f\n" \
|
||||
" sc.w %1, %z4, %2\n" \
|
||||
" bnez %1, 0b\n" \
|
||||
"1:\n" \
|
||||
: "=&r" (__ret), "=&r" (__rc), "+A" (*__ptr) \
|
||||
: "rJ" ((long)__old), "rJ" (__new) \
|
||||
: "memory"); \
|
||||
break; \
|
||||
case 8: \
|
||||
__asm__ __volatile__ ( \
|
||||
RISCV_RELEASE_BARRIER \
|
||||
"0: lr.d %0, %2\n" \
|
||||
" bne %0, %z3, 1f\n" \
|
||||
" sc.d %1, %z4, %2\n" \
|
||||
" bnez %1, 0b\n" \
|
||||
"1:\n" \
|
||||
: "=&r" (__ret), "=&r" (__rc), "+A" (*__ptr) \
|
||||
: "rJ" (__old), "rJ" (__new) \
|
||||
: "memory"); \
|
||||
break; \
|
||||
default: \
|
||||
BUILD_BUG(); \
|
||||
} \
|
||||
__ret; \
|
||||
})
|
||||
_arch_cmpxchg((ptr), (o), (n), "", "", RISCV_ACQUIRE_BARRIER)
|
||||
|
||||
#define arch_cmpxchg_release(ptr, o, n) \
|
||||
({ \
|
||||
__typeof__(*(ptr)) _o_ = (o); \
|
||||
__typeof__(*(ptr)) _n_ = (n); \
|
||||
(__typeof__(*(ptr))) __cmpxchg_release((ptr), \
|
||||
_o_, _n_, sizeof(*(ptr))); \
|
||||
})
|
||||
|
||||
#define __cmpxchg(ptr, old, new, size) \
|
||||
({ \
|
||||
__typeof__(ptr) __ptr = (ptr); \
|
||||
__typeof__(*(ptr)) __old = (old); \
|
||||
__typeof__(*(ptr)) __new = (new); \
|
||||
__typeof__(*(ptr)) __ret; \
|
||||
register unsigned int __rc; \
|
||||
switch (size) { \
|
||||
case 4: \
|
||||
__asm__ __volatile__ ( \
|
||||
"0: lr.w %0, %2\n" \
|
||||
" bne %0, %z3, 1f\n" \
|
||||
" sc.w.rl %1, %z4, %2\n" \
|
||||
" bnez %1, 0b\n" \
|
||||
" fence rw, rw\n" \
|
||||
"1:\n" \
|
||||
: "=&r" (__ret), "=&r" (__rc), "+A" (*__ptr) \
|
||||
: "rJ" ((long)__old), "rJ" (__new) \
|
||||
: "memory"); \
|
||||
break; \
|
||||
case 8: \
|
||||
__asm__ __volatile__ ( \
|
||||
"0: lr.d %0, %2\n" \
|
||||
" bne %0, %z3, 1f\n" \
|
||||
" sc.d.rl %1, %z4, %2\n" \
|
||||
" bnez %1, 0b\n" \
|
||||
" fence rw, rw\n" \
|
||||
"1:\n" \
|
||||
: "=&r" (__ret), "=&r" (__rc), "+A" (*__ptr) \
|
||||
: "rJ" (__old), "rJ" (__new) \
|
||||
: "memory"); \
|
||||
break; \
|
||||
default: \
|
||||
BUILD_BUG(); \
|
||||
} \
|
||||
__ret; \
|
||||
})
|
||||
_arch_cmpxchg((ptr), (o), (n), "", RISCV_RELEASE_BARRIER, "")
|
||||
|
||||
#define arch_cmpxchg(ptr, o, n) \
|
||||
({ \
|
||||
__typeof__(*(ptr)) _o_ = (o); \
|
||||
__typeof__(*(ptr)) _n_ = (n); \
|
||||
(__typeof__(*(ptr))) __cmpxchg((ptr), \
|
||||
_o_, _n_, sizeof(*(ptr))); \
|
||||
})
|
||||
_arch_cmpxchg((ptr), (o), (n), ".rl", "", " fence rw, rw\n")
|
||||
|
||||
#define arch_cmpxchg_local(ptr, o, n) \
|
||||
(__cmpxchg_relaxed((ptr), (o), (n), sizeof(*(ptr))))
|
||||
arch_cmpxchg_relaxed((ptr), (o), (n))
|
||||
|
||||
#define arch_cmpxchg64(ptr, o, n) \
|
||||
({ \
|
||||
|
||||
Reference in New Issue
Block a user