1 /* 2 * Copyright (c) 2025, Arm Limited. All rights reserved. 3 * 4 * SPDX-License-Identifier: BSD-3-Clause 5 */ 6 7 #include <arch_features.h> 8 #include <lib/spinlock.h> 9 10 /* 11 * Performs a compare-and-swap of 0 -> 1. If the lock is already held, uses 12 * LDAXR/WFE to efficiently wait. 13 */ 14 static void spin_lock_atomic(volatile uint32_t *dst) 15 { 16 uint32_t src = 1; 17 uint32_t tmp; 18 19 __asm__ volatile ( 20 ".arch_extension lse\n" 21 "1: mov %w[tmp], wzr\n" 22 "2: casa %w[tmp], %w[src], [%[dst]]\n" 23 " cbz %w[tmp], 3f\n" 24 " ldxr %w[tmp], [%[dst]]\n" 25 " cbz %w[tmp], 2b\n" 26 " wfe\n" 27 " b 1b\n" 28 "3:\n" 29 : "+m" (*dst), [tmp] "=&r" (tmp), [src] "+r" (src) 30 : [dst] "r" (dst)); 31 } 32 33 /* 34 * Uses the load-acquire (LDAXR) and store-exclusive (STXR) instruction pair. 35 */ 36 static void spin_lock_excl(volatile uint32_t *dst) 37 { 38 uint32_t src = 1; 39 uint32_t tmp; 40 41 __asm__ volatile ( 42 " sevl\n" 43 "1: wfe\n" 44 "2: ldaxr %w[tmp], [%[dst]]\n" 45 " cbnz %w[tmp], 1b\n" 46 " stxr %w[tmp], %w[src], [%[dst]]\n" 47 " cbnz %w[tmp], 2b\n" 48 : "+m" (*dst), [tmp] "=&r" (tmp), [src] "+r" (src) 49 : [dst] "r" (dst)); 50 } 51 52 void spin_lock(spinlock_t *lock) 53 { 54 volatile uint32_t *dst = &(lock->lock); 55 56 if (is_feat_lse_supported()) { 57 spin_lock_atomic(dst); 58 } else { 59 spin_lock_excl(dst); 60 } 61 } 62 63 /* 64 * Use store-release to unconditionally clear the spinlock variable. Store 65 * operation generates an event to all cores waiting in WFE when address is 66 * monitored by the global monitor. 67 */ 68 void spin_unlock(spinlock_t *lock) 69 { 70 volatile uint32_t *dst = &(lock->lock); 71 72 __asm__ volatile ( 73 "stlr wzr, [%[dst]]" 74 : "=m" (dst) 75 : [dst] "r" (dst)); 76 } 77 78 static bool spin_trylock_atomic(volatile uint32_t *dst) 79 { 80 uint32_t src = 1; 81 uint32_t tmp = 0; 82 bool out; 83 84 __asm__ volatile ( 85 ".arch_extension lse\n" 86 "casa %w[tmp], %w[src], [%[dst]]\n" 87 "eor %w[out], %w[tmp], #1\n" /* convert the result to bool */ 88 : "+m" (*dst), [tmp] "+r" (tmp), [out] "=r" (out) 89 : [src] "r" (src), [dst] "r" (dst)); 90 91 return out; 92 } 93 94 static bool spin_trylock_excl(volatile uint32_t *dst) 95 { 96 uint32_t src = 1; 97 uint32_t ret; 98 99 __asm__ volatile ( 100 "ldaxr %w[ret], [%[dst]]\n" 101 : [ret] "=r" (ret) 102 : "m" (*dst), [dst] "r" (dst)); 103 104 /* 1 means lock is held */ 105 if (ret != 0) { 106 return false; 107 } 108 109 __asm__ volatile ( 110 "stxr %w[ret], %w[src], [%[dst]]\n" 111 : "+m" (*dst), [ret] "=r" (ret) 112 : [src] "r" (src), [dst] "r" (dst)); 113 114 if (ret == 0) { 115 return true; 116 } 117 } 118 119 /* 120 * Attempts to acquire the spinlock once without spinning. If unlocked (0), 121 * attempts to store 1 to acquire it. 122 */ 123 bool spin_trylock(spinlock_t *lock) 124 { 125 volatile uint32_t *dst = &(lock->lock); 126 127 if (is_feat_lse_supported()) { 128 return spin_trylock_atomic(dst); 129 } else { 130 return spin_trylock_excl(dst); 131 } 132 } 133 134 #if USE_SPINLOCK_CAS 135 /* 136 * Acquire bitlock using atomic bit set on byte. If the original read value 137 * has the bit set, use load exclusive semantics to monitor the address and 138 * enter WFE. 139 */ 140 void bit_lock(bitlock_t *lock, uint8_t mask) 141 { 142 volatile uint8_t *dst = &(lock->lock); 143 uint32_t tmp; 144 145 /* there is no exclusive fallback */ 146 assert(is_feat_lse_supported()); 147 148 __asm__ volatile ( 149 "1: ldsetab %w[mask], %w[tmp], [%[dst]]\n" 150 " tst %w[tmp], %w[mask]\n" 151 " b.eq 2f\n" 152 " ldxrb %w[tmp], [%[dst]]\n" 153 " tst %w[tmp], %w[mask]\n" 154 " b.eq 1b\n" 155 " wfe\n" 156 " b 1b\n" 157 "2:\n" 158 : "+m" (*dst), [tmp] "=&r" (tmp) 159 : [mask] "r" (mask), [dst] "r" (dst)); 160 } 161 162 /* 163 * Use atomic bit clear store-release to unconditionally clear bitlock variable. 164 * Store operation generates an event to all cores waiting in WFE when address 165 * is monitored by the global monitor. 166 */ 167 void bit_unlock(bitlock_t *lock, uint8_t mask) 168 { 169 volatile uint8_t *dst = &(lock->lock); 170 171 /* there is no exclusive fallback */ 172 assert(is_feat_lse_supported()); 173 174 __asm__ volatile ( 175 "stclrlb %w[mask], [%[dst]]" 176 : "=m" (dst) 177 : [mask] "r" (mask), [dst] "r" (dst)); 178 } 179 #endif 180