1 // SPDX-License-Identifier: BSD-2-Clause
2 /*
3 * Copyright (c) 2025-2026, Advanced Micro Devices, Inc. All rights reserved.
4 *
5 */
6
7 #include <assert.h>
8 #include <drivers/amd/asu_client.h>
9 #include <initcall.h>
10 #include <io.h>
11 #include <kernel/delay.h>
12 #include <kernel/dt.h>
13 #include <kernel/interrupt.h>
14 #include <kernel/panic.h>
15 #include <kernel/spinlock.h>
16 #include <libfdt.h>
17 #include <mm/core_mmu.h>
18 #include <stdbool.h>
19 #include <stdint.h>
20 #include <string.h>
21 #include <trace.h>
22 #include <util.h>
23
24 #include "asu_doorbell.h"
25
26 #define ASU_QUEUE_BUFFER_FULL 0xFFU
27 #define ASU_CLIENT_READY 0xFFFFFFFFU
28 #define ASU_TARGET_IPI_INT_MASK 1U
29
30 #define ASU_BASEADDR 0xEBF80000U
31 #define ASU_GLOBAL_CNTRL (ASU_BASEADDR + 0x00000000U)
32
33 #define ASU_BASEADDR_SIZE 0x10000U
34 #define ASU_GLOBAL_ADDR_LIMIT 0x1000U
35
36 #define ASU_GLOBAL_CNTRL_FW_IS_PRESENT_MASK 0x10U
37 #define ASU_ASUFW_BIT_CHECK_TIMEOUT_VALUE 0xFFFFFU
38
39 #define ASU_RESP_TIMEOUT 2000000U /* 2sec */
40
41 #define ASU_CHNL_IPI_BITMASK GENMASK_32(31, 16)
42
43 struct asu_client {
44 struct asu_channel_memory *chnl_memptr;
45 uint32_t is_ready;
46 uint32_t p0_last_index;
47 uint32_t p1_last_index;
48 unsigned int slock; /* chnl_memptr spin lock */
49 void *global_ctrl;
50 void *doorbell;
51 };
52
53 struct asu_ids {
54 uint8_t ids[ASU_UNIQUE_ID_MAX];
55 unsigned int slock; /* id array spin lock */
56 };
57
58 static struct asu_ids asuid;
59 static struct asu_client *asu;
60
61 /*
62 * asu_fwcheck() - Check if ASU firmware is present and ready
63 * Polls the ASU global control register to verify if the HSM firmware
64 * is present and ready for interaction. Uses a timeout of approximately
65 * 1 second for the check.
66 *
67 * Return: TEE_SUCCESS if firmware is ready, TEE_ERROR_BAD_STATE if not present
68 */
asu_fwcheck(void)69 static TEE_Result asu_fwcheck(void)
70 {
71 uint64_t timeout = 0;
72
73 /*
74 * Timeout is set to ~1sec.
75 * This is the worst case time within which ASUFW ready for interaction
76 * with components requests.
77 */
78 timeout = timeout_init_us(ASU_ASUFW_BIT_CHECK_TIMEOUT_VALUE);
79
80 do {
81 if (io_read32((vaddr_t)asu->global_ctrl) &
82 ASU_GLOBAL_CNTRL_FW_IS_PRESENT_MASK) {
83 DMSG("ASU FW is ready!");
84 return TEE_SUCCESS;
85 }
86 } while (!timeout_elapsed(timeout));
87
88 EMSG("ASU FW is not present!");
89
90 return TEE_ERROR_BAD_STATE;
91 }
92
93 /*
94 * asu_get_channelID() - Determine the ASU channel ID for APU communication
95 *
96 * Maps the Runtime Configuration Area (RTCA) to find which ASU channel
97 * should be used by the APU for communication. Searches through available
98 * channels to find one matching the APU's local IPI ID.
99 *
100 * Return: Channel ID (0 to ASU_MAX_IPI_CHANNELS-1) or ASU_MAX_IPI_CHANNELS on
101 * failure
102 */
asu_get_channelID(void)103 static uint32_t asu_get_channelID(void)
104 {
105 TEE_Result ret = TEE_ERROR_GENERIC;
106 void *comm_chnl_info = NULL;
107 uint32_t channel_id = ASU_MAX_IPI_CHANNELS;
108 vaddr_t membase = 0;
109 uint32_t id = 0;
110
111 /*
112 * RTCA is mapped only to find the ASU Channel ID
113 * to be used by APU.
114 * After reading the information RTCA region is unmapped.
115 */
116 comm_chnl_info = core_mmu_add_mapping(MEM_AREA_IO_SEC,
117 ASU_RTCA_BASEADDR,
118 ASU_GLOBAL_ADDR_LIMIT);
119 if (!comm_chnl_info) {
120 EMSG("Failed to map runtime config area");
121 return channel_id;
122 }
123
124 for (id = 0; id < ASU_MAX_IPI_CHANNELS; id++) {
125 membase = (vaddr_t)comm_chnl_info +
126 ASU_RTCA_CHANNEL_BASE_OFFSET +
127 ASU_RTCA_CHANNEL_INFO_LEN * id;
128 if ((io_read32(membase) & ASU_CHNL_IPI_BITMASK) ==
129 (CFG_AMD_APU_LCL_IPI_ID << 16)) {
130 channel_id = id;
131 DMSG("Use ASU channel ID %"PRIu32, channel_id);
132 break;
133 }
134 }
135
136 if (channel_id == ASU_MAX_IPI_CHANNELS)
137 EMSG("Failed to identify ASU channel ID for APU");
138
139 ret = core_mmu_remove_mapping(MEM_AREA_IO_SEC,
140 comm_chnl_info, ASU_GLOBAL_ADDR_LIMIT);
141 if (ret)
142 EMSG("Failed to unmap RTCA");
143
144 return channel_id;
145 }
146
147 /*
148 * asu_alloc_unique_id() - Generate a unique identifier for ASU operations
149 *
150 * Creates a unique ID by cycling through available IDs in the callback
151 * reference array. Ensures no ID collision by checking if the slot is
152 * already in use.
153 *
154 * Return: Unique ID (1 to ASU_UNIQUE_ID_MAX-1) or ASU_UNIQUE_ID_MAX if none
155 * available
156 */
asu_alloc_unique_id(void)157 uint8_t asu_alloc_unique_id(void)
158 {
159 uint8_t unqid = 0;
160 uint32_t state = 0;
161
162 state = cpu_spin_lock_xsave(&asuid.slock);
163 while (unqid < ASU_UNIQUE_ID_MAX) {
164 if (asuid.ids[unqid] == ASU_UNIQUE_ID_MAX) {
165 asuid.ids[unqid] = unqid;
166 DMSG("Got unique ID %"PRIu8, unqid);
167 break;
168 }
169 unqid++;
170 };
171 cpu_spin_unlock_xrestore(&asuid.slock, state);
172
173 return unqid;
174 }
175
176 /**
177 * asu_free_unique_id() - Release a previously allocated unique ID
178 * @uniqueid: The unique ID to be freed
179 *
180 * Marks the specified unique ID as available for reuse.
181 * The released ID is set to ASU_UNIQUE_ID_MAX
182 * to indicate its availability. Intended to be used in environments where
183 * concurrent access to the unique ID pool occurs.
184 */
asu_free_unique_id(uint8_t uniqueid)185 void asu_free_unique_id(uint8_t uniqueid)
186 {
187 uint32_t state = 0;
188
189 state = cpu_spin_lock_xsave(&asuid.slock);
190 asuid.ids[uniqueid] = ASU_UNIQUE_ID_MAX;
191 cpu_spin_unlock_xrestore(&asuid.slock, state);
192 }
193
194 /*
195 * get_free_index() - Find a free buffer index in the specified queue
196 * @qptr: Pointer to the channel queue structure to search
197 * @last_index: Pointer to track the starting search index for
198 * round-robin allocation
199 *
200 * Searches for an available buffer slot in the specified channel queue.
201 * Performs a circular search of ASU_MAX_BUFFERS slots starting from the
202 * provided index, wrapping around if necessary.
203 *
204 * Return: Buffer index (0 to ASU_MAX_BUFFERS-1) or ASU_MAX_BUFFERS if queue
205 * is full
206 */
get_free_index(struct asu_channel_queue * qptr,uint32_t * last_index)207 static uint8_t get_free_index(struct asu_channel_queue *qptr,
208 uint32_t *last_index)
209 {
210 uint8_t index = *last_index;
211 uint8_t iterations = 0;
212
213 while (iterations < ASU_MAX_BUFFERS) {
214 index = (index + 1) % ASU_MAX_BUFFERS;
215 if (qptr->queue_bufs[index].reqbufstatus == 0U)
216 break;
217 iterations++;
218 }
219
220 if (iterations == ASU_MAX_BUFFERS)
221 return ASU_MAX_BUFFERS;
222
223 return index;
224 }
225
put_free_index(struct asu_channel_queue_buf * bufptr)226 static void put_free_index(struct asu_channel_queue_buf *bufptr)
227 {
228 uint32_t state = 0;
229
230 state = cpu_spin_lock_xsave(&asu->slock);
231 bufptr->reqbufstatus = 0;
232 bufptr->respbufstatus = 0;
233 cpu_spin_unlock_xrestore(&asu->slock, state);
234 }
235
236 /*
237 * send_doorbell() - Send IPI doorbell interrupt to ASU
238 *
239 * Triggers an Inter-Processor Interrupt (IPI) to notify the ASU
240 * that a new command is available in the shared memory queue.
241 *
242 * Return: TEE_SUCCESS on successful doorbell trigger
243 */
send_doorbell(void)244 static TEE_Result send_doorbell(void)
245 {
246 io_write32((vaddr_t)asu->doorbell + IPIPSU_TRIG_OFFSET,
247 ASU_TARGET_IPI_INT_MASK);
248
249 return TEE_SUCCESS;
250 }
251
252 /*
253 * asu_update_queue_buffer_n_send_ipi() - Queue command and send IPI
254 * @param: Client parameters including priority
255 * @req_buffer: Request buffer containing command data
256 * @size: Size of request data
257 * @header: Command header information
258 * @status: FW return status
259 *
260 * Places a command in the appropriate priority queue buffer, updates
261 * queue status, and sends an IPI to notify ASU of the pending command.
262 *
263 * Return: TEE_SUCCESS on success, appropriate error code on failure
264 */
asu_update_queue_buffer_n_send_ipi(struct asu_client_params * param,void * req_buffer,uint32_t size,uint32_t header,uint32_t * status)265 TEE_Result asu_update_queue_buffer_n_send_ipi(struct asu_client_params *param,
266 void *req_buffer,
267 uint32_t size,
268 uint32_t header,
269 uint32_t *status)
270 {
271 TEE_Result ret = TEE_ERROR_GENERIC;
272 uint8_t freeindex = 0;
273 uint32_t *last_index;
274 uint32_t state = 0;
275 uint64_t resp_to = 0;
276 struct asu_channel_queue_buf *bufptr = NULL;
277 struct asu_channel_queue *qptr = NULL;
278
279 if (!param || !status) {
280 EMSG("Invalid parameters provided");
281 return TEE_ERROR_BAD_PARAMETERS;
282 }
283
284 if (asu->is_ready != ASU_CLIENT_READY) {
285 EMSG("ASU client is not ready");
286 return TEE_ERROR_BAD_STATE;
287 }
288
289 if (param->priority == ASU_PRIORITY_HIGH) {
290 qptr = &asu->chnl_memptr->p0_chnl_q;
291 last_index = &asu->p0_last_index;
292 } else {
293 qptr = &asu->chnl_memptr->p1_chnl_q;
294 last_index = &asu->p1_last_index;
295 }
296
297 state = cpu_spin_lock_xsave(&asu->slock);
298 freeindex = get_free_index(qptr, last_index);
299 if (freeindex == ASU_MAX_BUFFERS) {
300 EMSG("ASU buffers full");
301 cpu_spin_unlock_xrestore(&asu->slock, state);
302 return TEE_ERROR_SHORT_BUFFER;
303 }
304
305 bufptr = &qptr->queue_bufs[freeindex];
306 bufptr->req.header = header;
307 if (req_buffer && size != 0U)
308 memcpy(bufptr->req.arg, req_buffer, size);
309
310 bufptr->respbufstatus = 0;
311 qptr->queue_bufs[freeindex].reqbufstatus = ASU_COMMAND_IS_PRESENT;
312 *last_index = freeindex;
313 qptr->cmd_is_present = true;
314 qptr->req_sent++;
315 cpu_spin_unlock_xrestore(&asu->slock, state);
316
317 ret = send_doorbell();
318 if (ret != TEE_SUCCESS) {
319 EMSG("Failed to communicate to ASU");
320 return TEE_ERROR_COMMUNICATION;
321 }
322
323 /*
324 * Timeout is armed unconditionally before the loop.
325 * Poll if IRQs are masked, otherwise yield with wfe()
326 */
327 resp_to = timeout_init_us(ASU_RESP_TIMEOUT);
328 while (io_read8((vaddr_t)&bufptr->respbufstatus) !=
329 ASU_RESPONSE_IS_PRESENT) {
330 if (timeout_elapsed(resp_to)) {
331 EMSG("ASU: response timeout");
332 EMSG("last_idx=0x%"PRIx32" req_sent=0x%"PRIx32,
333 *last_index, qptr->req_sent);
334 EMSG("req_served=0x%"PRIx32" header=0x%"PRIx32,
335 qptr->req_served, header);
336 put_free_index(bufptr);
337 return TEE_ERROR_TARGET_DEAD;
338 }
339
340 if (!(thread_get_exceptions() & THREAD_EXCP_NATIVE_INTR))
341 wfe();
342 else
343 udelay(10);
344 }
345
346 *status = bufptr->resp.arg[ASU_RESPONSE_STATUS_INDEX];
347 if (param->cbhandler && !*status)
348 ret = param->cbhandler(param->cbptr, &bufptr->resp);
349 put_free_index(bufptr);
350
351 return ret;
352 }
353
asu_clear_intr(void)354 static void asu_clear_intr(void)
355 {
356 uint32_t status = 0;
357
358 status = io_read32((vaddr_t)asu->doorbell + IPIPSU_ISR_OFFSET);
359 io_write32((vaddr_t)asu->doorbell + IPIPSU_ISR_OFFSET,
360 status & IPIPSU_ALL_MASK);
361 }
362
363 /*
364 * asu_resp_handler() - Interrupt handler for ASU responses
365 * @handler: Interrupt handler structure (unused)
366 *
367 * Interrupt service routine that processes responses from both high
368 * priority (P0) and low priority (P1) queues when ASU completes
369 * command processing.
370 *
371 * Return: ITRR_HANDLED indicating interrupt was handled
372 */
asu_resp_handler(struct itr_handler * handler __unused)373 static enum itr_return asu_resp_handler(struct itr_handler *handler __unused)
374 {
375 sev();
376 asu_clear_intr();
377
378 return ITRR_HANDLED;
379 }
380
381 static struct itr_handler doorbell_handler = {
382 .it = PAR_IPIPSU_0_INT_ID,
383 .handler = asu_resp_handler,
384 };
385
386 /*
387 * setup_doorbell() - Initialize doorbell interrupt handling
388 *
389 * Maps the doorbell register region, configures interrupt settings,
390 * registers the interrupt handler, and enables the interrupt for
391 * receiving ASU response notifications.
392 *
393 * Return: Pointer to mapped doorbell region or NULL on failure
394 */
setup_doorbell(void)395 static void *setup_doorbell(void)
396 {
397 void *dbell = NULL;
398 TEE_Result res = TEE_ERROR_GENERIC;
399
400 dbell = core_mmu_add_mapping(MEM_AREA_IO_SEC,
401 asu_configtable.baseaddr,
402 ASU_BASEADDR_SIZE);
403 if (!dbell) {
404 EMSG("Failed to map doorbell register");
405 return dbell;
406 }
407
408 io_write32((vaddr_t)dbell + IPIPSU_IER_OFFSET, IPIPSU_ALL_MASK);
409 io_write32((vaddr_t)dbell + IPIPSU_ISR_OFFSET, IPIPSU_ALL_MASK);
410
411 doorbell_handler.chip = interrupt_get_main_chip();
412
413 res = interrupt_add_configure_handler(&doorbell_handler,
414 IRQ_TYPE_LEVEL_HIGH, 7);
415 if (res)
416 panic();
417
418 interrupt_enable(doorbell_handler.chip, doorbell_handler.it);
419
420 return dbell;
421 }
422
asu_init_unique_id(void)423 static void asu_init_unique_id(void)
424 {
425 uint32_t idx = 0;
426
427 asuid.slock = SPINLOCK_UNLOCK;
428 for (idx = 0; idx < ARRAY_SIZE(asuid.ids); idx++)
429 asuid.ids[idx] = ASU_UNIQUE_ID_MAX;
430 }
431
432 /*
433 * asu_init() - Initialize the ASU driver and communication channel
434 *
435 * Performs complete ASU driver initialization including memory allocation,
436 * firmware readiness check, channel ID discovery, shared memory mapping,
437 * doorbell setup, and marking the client as ready for operation.
438 *
439 * Return: TEE_SUCCESS on successful initialization, appropriate error code on
440 * failure
441 */
asu_init(void)442 static TEE_Result asu_init(void)
443 {
444 uint32_t channel_id = 0;
445 void *asu_shmem = NULL;
446 uint64_t membase = 0;
447
448 asu = calloc(1, sizeof(struct asu_client));
449 if (!asu) {
450 EMSG("Failed to allocate memory for ASU");
451 return TEE_ERROR_OUT_OF_MEMORY;
452 }
453
454 asu->global_ctrl = core_mmu_add_mapping(MEM_AREA_IO_SEC,
455 ASU_BASEADDR,
456 ASU_BASEADDR_SIZE);
457 if (!asu->global_ctrl) {
458 EMSG("Failed to initialize ASU");
459 goto free;
460 }
461
462 if (asu_fwcheck() != TEE_SUCCESS) {
463 EMSG("ASU FW check failed");
464 goto global_unmap;
465 }
466
467 channel_id = asu_get_channelID();
468
469 if (channel_id == ASU_MAX_IPI_CHANNELS) {
470 EMSG("ASU channel for APU not configured");
471 goto global_unmap;
472 }
473
474 membase = ASU_CHANNEL_MEMORY_BASEADDR +
475 ASU_GLOBAL_ADDR_LIMIT * channel_id;
476 asu_shmem = core_mmu_add_mapping(MEM_AREA_IO_SEC,
477 membase,
478 ASU_GLOBAL_ADDR_LIMIT);
479 if (!asu_shmem) {
480 EMSG("Failed to map ASU SHM");
481 goto global_unmap;
482 }
483 asu_init_unique_id();
484 asu->doorbell = setup_doorbell();
485 if (!asu->doorbell) {
486 EMSG("Failed to set up ASU doorbell");
487 goto sh_unmap;
488 }
489
490 asu->chnl_memptr = asu_shmem;
491 asu->is_ready = ASU_CLIENT_READY;
492 asu->slock = SPINLOCK_UNLOCK;
493 asu->p0_last_index = ASU_MAX_BUFFERS - 1;
494 asu->p1_last_index = ASU_MAX_BUFFERS - 1;
495
496 IMSG("ASU initialization complete");
497
498 return TEE_SUCCESS;
499
500 sh_unmap:
501 core_mmu_remove_mapping(MEM_AREA_IO_SEC, asu_shmem,
502 ASU_GLOBAL_ADDR_LIMIT);
503 global_unmap:
504 core_mmu_remove_mapping(MEM_AREA_IO_SEC, asu->global_ctrl,
505 ASU_BASEADDR_SIZE);
506 free:
507 free(asu);
508
509 EMSG("Failed to initialize ASU");
510
511 return TEE_ERROR_GENERIC;
512 }
513
514 service_init(asu_init);
515