1 /*
2 * Copyright (c) 2025-2026, Arm Limited. All rights reserved.
3 *
4 * SPDX-License-Identifier: BSD-3-Clause
5 */
6
7 #include <arch_features.h>
8 #include <lib/spinlock.h>
9
10 /*
11 * Performs a compare-and-swap of 0 -> 1. If the lock is already held, uses
12 * LDAXR/WFE to efficiently wait.
13 */
ldaxr_32(volatile uint32_t * dst)14 static uint32_t ldaxr_32(volatile uint32_t *dst)
15 {
16 uint32_t ret;
17
18 __asm__ volatile (
19 "ldaxr %w[ret], %[dst]\n"
20 : [ret] "=r" (ret)
21 : [dst] "Q" (*dst));
22
23 return ret;
24 }
25
stxr_32(uint32_t src,volatile uint32_t * dst)26 static uint32_t stxr_32(uint32_t src, volatile uint32_t *dst)
27 {
28 uint32_t ret;
29
30 __asm__ volatile (
31 "stxr %w[ret], %w[src], %[dst]\n"
32 : [dst] "=Q" (*dst), [ret] "=&r" (ret)
33 : [src] "r" (src));
34
35 return ret;
36 }
37
casa_32(uint32_t src,volatile uint32_t * dst)38 static uint32_t casa_32(uint32_t src, volatile uint32_t *dst)
39 {
40 uint32_t ret = 0;
41
42 __asm__ volatile (
43 ".arch_extension lse\n"
44 " casa %w[ret], %w[src], %[dst]\n"
45 : [dst] "+Q" (*dst), [ret] "+r" (ret)
46 : [src] "r" (src));
47
48 return ret;
49 }
50
51 /*
52 * Check that the lock isn't held. Tries to compare-and-swap (CAS) it if not.
53 * Uses WFE to efficiently wait.
54 */
spin_lock_atomic(volatile uint32_t * dst)55 static void spin_lock_atomic(volatile uint32_t *dst)
56 {
57 for (; 1; wfe()) {
58 /* 1 means lock is held */
59 if (ldaxr_32(dst) != 0) {
60 continue;
61 }
62
63 if (casa_32(1, dst) == 0) {
64 return;
65 }
66 }
67 }
68
69 /*
70 * Uses the load-acquire (LDAXR) and store-exclusive (STXR) instruction pair.
71 */
spin_lock_excl(volatile uint32_t * dst)72 static void spin_lock_excl(volatile uint32_t *dst)
73 {
74 sevl();
75 while (1) {
76 wfe();
77 spinlock_retry:
78 if (ldaxr_32(dst) == 0) {
79 if (stxr_32(1, dst) == 0) {
80 return;
81 } else {
82 goto spinlock_retry;
83 }
84 }
85 }
86 }
87
spin_lock(spinlock_t * lock)88 void spin_lock(spinlock_t *lock)
89 {
90 volatile uint32_t *dst = &(lock->lock);
91
92 if (is_feat_lse_supported()) {
93 spin_lock_atomic(dst);
94 } else {
95 spin_lock_excl(dst);
96 }
97 }
98
spin_unlock(spinlock_t * lock)99 void spin_unlock(spinlock_t *lock)
100 {
101 volatile uint32_t *dst = &(lock->lock);
102
103 /*
104 * Plain store operations generate an event to other cores waiting in
105 * WFE when address is monitored by the global monitor.
106 */
107 __asm__ volatile (
108 "stlr wzr, %[dst]"
109 : [dst] "=Q" (*dst));
110
111 /* atomics don't generate events so wake others manually */
112 if (is_feat_lse_supported()) {
113 sev();
114 }
115 }
116
spin_trylock_atomic(volatile uint32_t * dst)117 static bool spin_trylock_atomic(volatile uint32_t *dst)
118 {
119 uint32_t tmp = 0;
120 bool out;
121
122 __asm__ volatile (
123 ".arch_extension lse\n"
124 "casa %w[tmp], %w[src], %[dst]\n"
125 "eor %w[out], %w[tmp], #1\n" /* convert the result to bool */
126 : [dst] "+Q" (*dst), [tmp] "+r" (tmp), [out] "=r" (out)
127 : [src] "r" (1));
128
129 return out;
130 }
131
spin_trylock_excl(volatile uint32_t * dst)132 static bool spin_trylock_excl(volatile uint32_t *dst)
133 {
134 /*
135 * Loop until we either get the lock or are certain that we don't have
136 * it. The exclusive store can fail due to racing and not because we
137 * don't hold the lock.
138 */
139 while (1) {
140 /* 1 means lock is held */
141 if (ldaxr_32(dst) != 0) {
142 return false;
143 }
144
145 if (stxr_32(1, dst) == 0) {
146 return true;
147 }
148 }
149 }
150
151 /*
152 * Attempts to acquire the spinlock once without spinning. If unlocked (0),
153 * attempts to store 1 to acquire it.
154 */
spin_trylock(spinlock_t * lock)155 bool spin_trylock(spinlock_t *lock)
156 {
157 volatile uint32_t *dst = &(lock->lock);
158
159 if (is_feat_lse_supported()) {
160 return spin_trylock_atomic(dst);
161 } else {
162 return spin_trylock_excl(dst);
163 }
164 }
165
166 #if USE_SPINLOCK_CAS
167 /*
168 * Acquire bitlock using atomic bit set on byte. If the original read value
169 * has the bit set, use load exclusive semantics to monitor the address and
170 * enter WFE.
171 */
bit_lock(bitlock_t * lock,uint8_t mask)172 void bit_lock(bitlock_t *lock, uint8_t mask)
173 {
174 volatile uint8_t *dst = &(lock->lock);
175 uint32_t tmp;
176
177 /* there is no exclusive fallback */
178 assert(is_feat_lse_supported());
179
180 __asm__ volatile (
181 "1: ldsetab %w[mask], %w[tmp], %[dst]\n"
182 " tst %w[tmp], %w[mask]\n"
183 " b.eq 2f\n"
184 " ldxrb %w[tmp], %[dst]\n"
185 " tst %w[tmp], %w[mask]\n"
186 " b.eq 1b\n"
187 " wfe\n"
188 " b 1b\n"
189 "2:\n"
190 : [dst] "+Q" (*dst), [tmp] "=&r" (tmp)
191 : [mask] "r" (mask));
192 }
193
194 /*
195 * Use atomic bit clear store-release to unconditionally clear bitlock variable.
196 * Store operation generates an event to all cores waiting in WFE when address
197 * is monitored by the global monitor.
198 */
bit_unlock(bitlock_t * lock,uint8_t mask)199 void bit_unlock(bitlock_t *lock, uint8_t mask)
200 {
201 volatile uint8_t *dst = &(lock->lock);
202
203 /* there is no exclusive fallback */
204 assert(is_feat_lse_supported());
205
206 __asm__ volatile (
207 "stclrlb %w[mask], %[dst]"
208 : [dst] "+Q" (*dst)
209 : [mask] "r" (mask));
210 }
211 #endif
212