1 /* 2 * Copyright (c) 2025, Arm Limited. All rights reserved. 3 * 4 * SPDX-License-Identifier: BSD-3-Clause 5 */ 6 7 #include <arch_features.h> 8 #include <lib/spinlock.h> 9 10 /* 11 * Performs a compare-and-swap of 0 -> 1. If the lock is already held, uses 12 * LDAXR/WFE to efficiently wait. 13 */ 14 static void spin_lock_atomic(volatile uint32_t *dst) 15 { 16 uint32_t src = 1; 17 uint32_t tmp; 18 19 __asm__ volatile ( 20 ".arch_extension lse\n" 21 "1: mov %w[tmp], wzr\n" 22 "2: casa %w[tmp], %w[src], [%[dst]]\n" 23 " cbz %w[tmp], 3f\n" 24 " ldxr %w[tmp], [%[dst]]\n" 25 " cbz %w[tmp], 2b\n" 26 " wfe\n" 27 " b 1b\n" 28 "3:\n" 29 : "+m" (*dst), [tmp] "=&r" (tmp), [src] "+r" (src) 30 : [dst] "r" (dst)); 31 } 32 33 /* 34 * Uses the load-acquire (LDAXR) and store-exclusive (STXR) instruction pair. 35 */ 36 static void spin_lock_excl(volatile uint32_t *dst) 37 { 38 uint32_t src = 1; 39 uint32_t tmp; 40 41 __asm__ volatile ( 42 " sevl\n" 43 "1: wfe\n" 44 "2: ldaxr %w[tmp], [%[dst]]\n" 45 " cbnz %w[tmp], 1b\n" 46 " stxr %w[tmp], %w[src], [%[dst]]\n" 47 " cbnz %w[tmp], 2b\n" 48 : "+m" (*dst), [tmp] "=&r" (tmp), [src] "+r" (src) 49 : [dst] "r" (dst)); 50 } 51 52 void spin_lock(spinlock_t *lock) 53 { 54 volatile uint32_t *dst = &(lock->lock); 55 56 if (is_feat_lse_supported()) { 57 spin_lock_atomic(dst); 58 } else { 59 spin_lock_excl(dst); 60 } 61 } 62 63 /* 64 * Use store-release to unconditionally clear the spinlock variable. Store 65 * operation generates an event to all cores waiting in WFE when address is 66 * monitored by the global monitor. 67 */ 68 void spin_unlock(spinlock_t *lock) 69 { 70 volatile uint32_t *dst = &(lock->lock); 71 72 __asm__ volatile ( 73 "stlr wzr, [%[dst]]" 74 : "=m" (dst) 75 : [dst] "r" (dst)); 76 } 77 78 static bool spin_trylock_atomic(volatile uint32_t *dst) 79 { 80 uint32_t src = 1; 81 uint32_t tmp = 0; 82 bool out; 83 84 __asm__ volatile ( 85 ".arch_extension lse\n" 86 "casa %w[tmp], %w[src], [%[dst]]\n" 87 "eor %w[out], %w[tmp], #1\n" /* convert the result to bool */ 88 : "+m" (*dst), [tmp] "+r" (tmp), [out] "=r" (out) 89 : [src] "r" (src), [dst] "r" (dst)); 90 91 return out; 92 } 93 94 static bool spin_trylock_excl(volatile uint32_t *dst) 95 { 96 uint32_t src = 1; 97 uint32_t tmp; 98 bool out; 99 100 __asm__ volatile ( 101 "ldaxr %w[tmp], [%[dst]]\n" 102 "cbnz %w[tmp], 1f\n" 103 "stxr %w[tmp], %w[src], [%[dst]]\n" 104 "cbnz %w[tmp], 1f\n" 105 "mov %w[out], #1\n" 106 "1:\n" /* fail */ 107 "mov %w[out], #0\n" 108 : "+m" (*dst), [tmp] "=&r" (tmp), [out] "=r" (out) 109 : [src] "r" (src), [dst] "r" (dst)); 110 111 return out; 112 } 113 114 /* 115 * Attempts to acquire the spinlock once without spinning. If unlocked (0), 116 * attempts to store 1 to acquire it. 117 */ 118 bool spin_trylock(spinlock_t *lock) 119 { 120 volatile uint32_t *dst = &(lock->lock); 121 122 if (is_feat_lse_supported()) { 123 return spin_trylock_atomic(dst); 124 } else { 125 return spin_trylock_excl(dst); 126 } 127 } 128 129 #if USE_SPINLOCK_CAS 130 /* 131 * Acquire bitlock using atomic bit set on byte. If the original read value 132 * has the bit set, use load exclusive semantics to monitor the address and 133 * enter WFE. 134 */ 135 void bit_lock(bitlock_t *lock, uint8_t mask) 136 { 137 volatile uint8_t *dst = &(lock->lock); 138 uint32_t tmp; 139 140 /* there is no exclusive fallback */ 141 assert(is_feat_lse_supported()); 142 143 __asm__ volatile ( 144 "1: ldsetab %w[mask], %w[tmp], [%[dst]]\n" 145 " tst %w[tmp], %w[mask]\n" 146 " b.eq 2f\n" 147 " ldxrb %w[tmp], [%[dst]]\n" 148 " tst %w[tmp], %w[mask]\n" 149 " b.eq 1b\n" 150 " wfe\n" 151 " b 1b\n" 152 "2:\n" 153 : "+m" (*dst), [tmp] "=&r" (tmp) 154 : [mask] "r" (mask), [dst] "r" (dst)); 155 } 156 157 /* 158 * Use atomic bit clear store-release to unconditionally clear bitlock variable. 159 * Store operation generates an event to all cores waiting in WFE when address 160 * is monitored by the global monitor. 161 */ 162 void bit_unlock(bitlock_t *lock, uint8_t mask) 163 { 164 volatile uint8_t *dst = &(lock->lock); 165 166 /* there is no exclusive fallback */ 167 assert(is_feat_lse_supported()); 168 169 __asm__ volatile ( 170 "stclrlb %w[mask], [%[dst]]" 171 : "=m" (dst) 172 : [mask] "r" (mask), [dst] "r" (dst)); 173 } 174 #endif 175