1 /* 2 * Copyright (c) 2017, ARM Limited and Contributors. All rights reserved. 3 * 4 * SPDX-License-Identifier: BSD-3-Clause 5 */ 6 7 /* 8 * Exception handlers at EL3, their priority levels, and management. 9 */ 10 11 #include <assert.h> 12 #include <cpu_data.h> 13 #include <debug.h> 14 #include <ehf.h> 15 #include <interrupt_mgmt.h> 16 #include <platform.h> 17 18 /* Output EHF logs as verbose */ 19 #define EHF_LOG(...) VERBOSE("EHF: " __VA_ARGS__) 20 21 #define EHF_INVALID_IDX (-1) 22 23 /* For a valid handler, return the actual function pointer; otherwise, 0. */ 24 #define RAW_HANDLER(h) \ 25 ((ehf_handler_t) ((h & _EHF_PRI_VALID) ? (h & ~_EHF_PRI_VALID) : 0)) 26 27 #define PRI_BIT(idx) (((ehf_pri_bits_t) 1) << idx) 28 29 /* 30 * Convert index into secure priority using the platform-defined priority bits 31 * field. 32 */ 33 #define IDX_TO_PRI(idx) \ 34 ((idx << (7 - exception_data.pri_bits)) & 0x7f) 35 36 /* Check whether a given index is valid */ 37 #define IS_IDX_VALID(idx) \ 38 ((exception_data.ehf_priorities[idx].ehf_handler & _EHF_PRI_VALID) != 0) 39 40 /* Returns whether given priority is in secure priority range */ 41 #define IS_PRI_SECURE(pri) ((pri & 0x80) == 0) 42 43 /* To be defined by the platform */ 44 extern const ehf_priorities_t exception_data; 45 46 /* Translate priority to the index in the priority array */ 47 static int pri_to_idx(unsigned int priority) 48 { 49 int idx; 50 51 idx = EHF_PRI_TO_IDX(priority, exception_data.pri_bits); 52 assert((idx >= 0) && (idx < exception_data.num_priorities)); 53 assert(IS_IDX_VALID(idx)); 54 55 return idx; 56 } 57 58 /* Return whether there are outstanding priority activation */ 59 static int has_valid_pri_activations(pe_exc_data_t *pe_data) 60 { 61 return pe_data->active_pri_bits != 0; 62 } 63 64 static pe_exc_data_t *this_cpu_data(void) 65 { 66 return &get_cpu_data(ehf_data); 67 } 68 69 /* 70 * Return the current priority index of this CPU. If no priority is active, 71 * return EHF_INVALID_IDX. 72 */ 73 static int get_pe_highest_active_idx(pe_exc_data_t *pe_data) 74 { 75 if (!has_valid_pri_activations(pe_data)) 76 return EHF_INVALID_IDX; 77 78 /* Current priority is the right-most bit */ 79 return __builtin_ctz(pe_data->active_pri_bits); 80 } 81 82 /* 83 * Mark priority active by setting the corresponding bit in active_pri_bits and 84 * programming the priority mask. 85 * 86 * This API is to be used as part of delegating to lower ELs other than for 87 * interrupts; e.g. while handling synchronous exceptions. 88 * 89 * This API is expected to be invoked before restoring context (Secure or 90 * Non-secure) in preparation for the respective dispatch. 91 */ 92 void ehf_activate_priority(unsigned int priority) 93 { 94 int idx, cur_pri_idx; 95 unsigned int old_mask, run_pri; 96 pe_exc_data_t *pe_data = this_cpu_data(); 97 98 /* 99 * Query interrupt controller for the running priority, or idle priority 100 * if no interrupts are being handled. The requested priority must be 101 * less (higher priority) than the active running priority. 102 */ 103 run_pri = plat_ic_get_running_priority(); 104 if (priority >= run_pri) { 105 ERROR("Running priority higher (0x%x) than requested (0x%x)\n", 106 run_pri, priority); 107 panic(); 108 } 109 110 /* 111 * If there were priority activations already, the requested priority 112 * must be less (higher priority) than the current highest priority 113 * activation so far. 114 */ 115 cur_pri_idx = get_pe_highest_active_idx(pe_data); 116 idx = pri_to_idx(priority); 117 if ((cur_pri_idx != EHF_INVALID_IDX) && (idx >= cur_pri_idx)) { 118 ERROR("Activation priority mismatch: req=0x%x current=0x%x\n", 119 priority, IDX_TO_PRI(cur_pri_idx)); 120 panic(); 121 } 122 123 /* Set the bit corresponding to the requested priority */ 124 pe_data->active_pri_bits |= PRI_BIT(idx); 125 126 /* 127 * Program priority mask for the activated level. Check that the new 128 * priority mask is setting a higher priority level than the existing 129 * mask. 130 */ 131 old_mask = plat_ic_set_priority_mask(priority); 132 if (priority >= old_mask) { 133 ERROR("Requested priority (0x%x) lower than Priority Mask (0x%x)\n", 134 priority, old_mask); 135 panic(); 136 } 137 138 /* 139 * If this is the first activation, save the priority mask. This will be 140 * restored after the last deactivation. 141 */ 142 if (cur_pri_idx == EHF_INVALID_IDX) 143 pe_data->init_pri_mask = old_mask; 144 145 EHF_LOG("activate prio=%d\n", get_pe_highest_active_idx(pe_data)); 146 } 147 148 /* 149 * Mark priority inactive by clearing the corresponding bit in active_pri_bits, 150 * and programming the priority mask. 151 * 152 * This API is expected to be used as part of delegating to to lower ELs other 153 * than for interrupts; e.g. while handling synchronous exceptions. 154 * 155 * This API is expected to be invoked after saving context (Secure or 156 * Non-secure), having concluded the respective dispatch. 157 */ 158 void ehf_deactivate_priority(unsigned int priority) 159 { 160 int idx, cur_pri_idx; 161 pe_exc_data_t *pe_data = this_cpu_data(); 162 unsigned int old_mask, run_pri; 163 164 /* 165 * Query interrupt controller for the running priority, or idle priority 166 * if no interrupts are being handled. The requested priority must be 167 * less (higher priority) than the active running priority. 168 */ 169 run_pri = plat_ic_get_running_priority(); 170 if (priority >= run_pri) { 171 ERROR("Running priority higher (0x%x) than requested (0x%x)\n", 172 run_pri, priority); 173 panic(); 174 } 175 176 /* 177 * Deactivation is allowed only when there are priority activations, and 178 * the deactivation priority level must match the current activated 179 * priority. 180 */ 181 cur_pri_idx = get_pe_highest_active_idx(pe_data); 182 idx = pri_to_idx(priority); 183 if ((cur_pri_idx == EHF_INVALID_IDX) || (idx != cur_pri_idx)) { 184 ERROR("Deactivation priority mismatch: req=0x%x current=0x%x\n", 185 priority, IDX_TO_PRI(cur_pri_idx)); 186 panic(); 187 } 188 189 /* Clear bit corresponding to highest priority */ 190 pe_data->active_pri_bits &= (pe_data->active_pri_bits - 1); 191 192 /* 193 * Restore priority mask corresponding to the next priority, or the 194 * one stashed earlier if there are no more to deactivate. 195 */ 196 idx = get_pe_highest_active_idx(pe_data); 197 if (idx == EHF_INVALID_IDX) 198 old_mask = plat_ic_set_priority_mask(pe_data->init_pri_mask); 199 else 200 old_mask = plat_ic_set_priority_mask(priority); 201 202 if (old_mask >= priority) { 203 ERROR("Deactivation priority (0x%x) lower than Priority Mask (0x%x)\n", 204 priority, old_mask); 205 panic(); 206 } 207 208 EHF_LOG("deactivate prio=%d\n", get_pe_highest_active_idx(pe_data)); 209 } 210 211 /* 212 * Top-level EL3 interrupt handler. 213 */ 214 static uint64_t ehf_el3_interrupt_handler(uint32_t id, uint32_t flags, 215 void *handle, void *cookie) 216 { 217 int pri, idx, intr, intr_raw, ret = 0; 218 ehf_handler_t handler; 219 220 /* 221 * Top-level interrupt type handler from Interrupt Management Framework 222 * doesn't acknowledge the interrupt; so the interrupt ID must be 223 * invalid. 224 */ 225 assert(id == INTR_ID_UNAVAILABLE); 226 227 /* 228 * Acknowledge interrupt. Proceed with handling only for valid interrupt 229 * IDs. This situation may arise because of Interrupt Management 230 * Framework identifying an EL3 interrupt, but before it's been 231 * acknowledged here, the interrupt was either deasserted, or there was 232 * a higher-priority interrupt of another type. 233 */ 234 intr_raw = plat_ic_acknowledge_interrupt(); 235 intr = plat_ic_get_interrupt_id(intr_raw); 236 if (intr == INTR_ID_UNAVAILABLE) 237 return 0; 238 239 /* Having acknowledged the interrupt, get the running priority */ 240 pri = plat_ic_get_running_priority(); 241 242 /* Check EL3 interrupt priority is in secure range */ 243 assert(IS_PRI_SECURE(pri)); 244 245 /* 246 * Translate the priority to a descriptor index. We do this by masking 247 * and shifting the running priority value (platform-supplied). 248 */ 249 idx = pri_to_idx(pri); 250 251 /* Validate priority */ 252 assert(pri == IDX_TO_PRI(idx)); 253 254 handler = RAW_HANDLER(exception_data.ehf_priorities[idx].ehf_handler); 255 if (!handler) { 256 ERROR("No EL3 exception handler for priority 0x%x\n", 257 IDX_TO_PRI(idx)); 258 panic(); 259 } 260 261 /* 262 * Call registered handler. Pass the raw interrupt value to registered 263 * handlers. 264 */ 265 ret = handler(intr_raw, flags, handle, cookie); 266 267 return ret; 268 } 269 270 /* 271 * Initialize the EL3 exception handling. 272 */ 273 void ehf_init(void) 274 { 275 unsigned int flags = 0; 276 int ret __unused; 277 278 /* Ensure EL3 interrupts are supported */ 279 assert(plat_ic_has_interrupt_type(INTR_TYPE_EL3)); 280 281 /* 282 * Make sure that priority water mark has enough bits to represent the 283 * whole priority array. 284 */ 285 assert(exception_data.num_priorities <= (sizeof(ehf_pri_bits_t) * 8)); 286 287 assert(exception_data.ehf_priorities); 288 289 /* 290 * Bit 7 of GIC priority must be 0 for secure interrupts. This means 291 * platforms must use at least 1 of the remaining 7 bits. 292 */ 293 assert((exception_data.pri_bits >= 1) || (exception_data.pri_bits < 8)); 294 295 /* Route EL3 interrupts when in Secure and Non-secure. */ 296 set_interrupt_rm_flag(flags, NON_SECURE); 297 set_interrupt_rm_flag(flags, SECURE); 298 299 /* Register handler for EL3 interrupts */ 300 ret = register_interrupt_type_handler(INTR_TYPE_EL3, 301 ehf_el3_interrupt_handler, flags); 302 assert(ret == 0); 303 } 304 305 /* 306 * Register a handler at the supplied priority. Registration is allowed only if 307 * a handler hasn't been registered before, or one wasn't provided at build 308 * time. The priority for which the handler is being registered must also accord 309 * with the platform-supplied data. 310 */ 311 void ehf_register_priority_handler(unsigned int pri, ehf_handler_t handler) 312 { 313 int idx; 314 315 /* Sanity check for handler */ 316 assert(handler != NULL); 317 318 /* Handler ought to be 4-byte aligned */ 319 assert((((uintptr_t) handler) & 3) == 0); 320 321 /* Ensure we register for valid priority */ 322 idx = pri_to_idx(pri); 323 assert(idx < exception_data.num_priorities); 324 assert(IDX_TO_PRI(idx) == pri); 325 326 /* Return failure if a handler was already registered */ 327 if (exception_data.ehf_priorities[idx].ehf_handler != _EHF_NO_HANDLER) { 328 ERROR("Handler already registered for priority 0x%x\n", pri); 329 panic(); 330 } 331 332 /* 333 * Install handler, and retain the valid bit. We assume that the handler 334 * is 4-byte aligned, which is usually the case. 335 */ 336 exception_data.ehf_priorities[idx].ehf_handler = 337 (((uintptr_t) handler) | _EHF_PRI_VALID); 338 339 EHF_LOG("register pri=0x%x handler=%p\n", pri, handler); 340 } 341