1 /*
2 * Copyright (c) 2025, Arm Limited. All rights reserved.
3 *
4 * SPDX-License-Identifier: BSD-3-Clause
5 */
6
7 #include <arch_features.h>
8 #include <lib/spinlock.h>
9
10 /*
11 * Performs a compare-and-swap of 0 -> 1. If the lock is already held, uses
12 * LDAXR/WFE to efficiently wait.
13 */
spin_lock_atomic(volatile uint32_t * dst)14 static void spin_lock_atomic(volatile uint32_t *dst)
15 {
16 uint32_t src = 1;
17 uint32_t tmp;
18
19 __asm__ volatile (
20 ".arch_extension lse\n"
21 "1: mov %w[tmp], wzr\n"
22 "2: casa %w[tmp], %w[src], [%[dst]]\n"
23 " cbz %w[tmp], 3f\n"
24 " ldxr %w[tmp], [%[dst]]\n"
25 " cbz %w[tmp], 2b\n"
26 " wfe\n"
27 " b 1b\n"
28 "3:\n"
29 : "+m" (*dst), [tmp] "=&r" (tmp), [src] "+r" (src)
30 : [dst] "r" (dst));
31 }
32
33 /*
34 * Uses the load-acquire (LDAXR) and store-exclusive (STXR) instruction pair.
35 */
spin_lock_excl(volatile uint32_t * dst)36 static void spin_lock_excl(volatile uint32_t *dst)
37 {
38 uint32_t src = 1;
39 uint32_t tmp;
40
41 __asm__ volatile (
42 " sevl\n"
43 "1: wfe\n"
44 "2: ldaxr %w[tmp], [%[dst]]\n"
45 " cbnz %w[tmp], 1b\n"
46 " stxr %w[tmp], %w[src], [%[dst]]\n"
47 " cbnz %w[tmp], 2b\n"
48 : "+m" (*dst), [tmp] "=&r" (tmp), [src] "+r" (src)
49 : [dst] "r" (dst));
50 }
51
spin_lock(spinlock_t * lock)52 void spin_lock(spinlock_t *lock)
53 {
54 volatile uint32_t *dst = &(lock->lock);
55
56 if (is_feat_lse_supported()) {
57 spin_lock_atomic(dst);
58 } else {
59 spin_lock_excl(dst);
60 }
61 }
62
63 /*
64 * Use store-release to unconditionally clear the spinlock variable. Store
65 * operation generates an event to all cores waiting in WFE when address is
66 * monitored by the global monitor.
67 */
spin_unlock(spinlock_t * lock)68 void spin_unlock(spinlock_t *lock)
69 {
70 volatile uint32_t *dst = &(lock->lock);
71
72 __asm__ volatile (
73 "stlr wzr, [%[dst]]"
74 : "=m" (dst)
75 : [dst] "r" (dst));
76 }
77
spin_trylock_atomic(volatile uint32_t * dst)78 static bool spin_trylock_atomic(volatile uint32_t *dst)
79 {
80 uint32_t src = 1;
81 uint32_t tmp = 0;
82 bool out;
83
84 __asm__ volatile (
85 ".arch_extension lse\n"
86 "casa %w[tmp], %w[src], [%[dst]]\n"
87 "eor %w[out], %w[tmp], #1\n" /* convert the result to bool */
88 : "+m" (*dst), [tmp] "+r" (tmp), [out] "=r" (out)
89 : [src] "r" (src), [dst] "r" (dst));
90
91 return out;
92 }
93
spin_trylock_excl(volatile uint32_t * dst)94 static bool spin_trylock_excl(volatile uint32_t *dst)
95 {
96 uint32_t src = 1;
97 uint32_t tmp;
98 bool out;
99
100 __asm__ volatile (
101 "ldaxr %w[tmp], [%[dst]]\n"
102 "cbnz %w[tmp], 1f\n"
103 "stxr %w[tmp], %w[src], [%[dst]]\n"
104 "cbnz %w[tmp], 1f\n"
105 "mov %w[out], #1\n"
106 "1:\n" /* fail */
107 "mov %w[out], #0\n"
108 : "+m" (*dst), [tmp] "=&r" (tmp), [out] "=r" (out)
109 : [src] "r" (src), [dst] "r" (dst));
110
111 return out;
112 }
113
114 /*
115 * Attempts to acquire the spinlock once without spinning. If unlocked (0),
116 * attempts to store 1 to acquire it.
117 */
spin_trylock(spinlock_t * lock)118 bool spin_trylock(spinlock_t *lock)
119 {
120 volatile uint32_t *dst = &(lock->lock);
121
122 if (is_feat_lse_supported()) {
123 return spin_trylock_atomic(dst);
124 } else {
125 return spin_trylock_excl(dst);
126 }
127 }
128
129 #if USE_SPINLOCK_CAS
130 /*
131 * Acquire bitlock using atomic bit set on byte. If the original read value
132 * has the bit set, use load exclusive semantics to monitor the address and
133 * enter WFE.
134 */
bit_lock(bitlock_t * lock,uint8_t mask)135 void bit_lock(bitlock_t *lock, uint8_t mask)
136 {
137 volatile uint8_t *dst = &(lock->lock);
138 uint32_t tmp;
139
140 /* there is no exclusive fallback */
141 assert(is_feat_lse_supported());
142
143 __asm__ volatile (
144 "1: ldsetab %w[mask], %w[tmp], [%[dst]]\n"
145 " tst %w[tmp], %w[mask]\n"
146 " b.eq 2f\n"
147 " ldxrb %w[tmp], [%[dst]]\n"
148 " tst %w[tmp], %w[mask]\n"
149 " b.eq 1b\n"
150 " wfe\n"
151 " b 1b\n"
152 "2:\n"
153 : "+m" (*dst), [tmp] "=&r" (tmp)
154 : [mask] "r" (mask), [dst] "r" (dst));
155 }
156
157 /*
158 * Use atomic bit clear store-release to unconditionally clear bitlock variable.
159 * Store operation generates an event to all cores waiting in WFE when address
160 * is monitored by the global monitor.
161 */
bit_unlock(bitlock_t * lock,uint8_t mask)162 void bit_unlock(bitlock_t *lock, uint8_t mask)
163 {
164 volatile uint8_t *dst = &(lock->lock);
165
166 /* there is no exclusive fallback */
167 assert(is_feat_lse_supported());
168
169 __asm__ volatile (
170 "stclrlb %w[mask], [%[dst]]"
171 : "=m" (dst)
172 : [mask] "r" (mask), [dst] "r" (dst));
173 }
174 #endif
175