1 // SPDX-License-Identifier: BSD-2-Clause
2 /*
3 * Copyright (c) 2021 Western Digital Corporation or its affiliates.
4 * Copyright (c) 2022 Ventana Micro Systems Inc.
5 * Copyright (c) 2025 Beijing Institute of Open Source Chip (BOSC)
6 *
7 * Authors:
8 * Anup Patel <anup.patel@wdc.com>
9 * Huang Borong <huangborong@bosc.ac.cn>
10 */
11
12 #include <compiler.h>
13 #include <drivers/imsic.h>
14 #include <io.h>
15 #include <kernel/dt.h>
16 #include <kernel/interrupt.h>
17 #include <libfdt.h>
18 #include <mm/core_mmu.h>
19 #include <platform_config.h>
20 #include <riscv.h>
21 #include <string.h>
22 #include <trace.h>
23 #include <util.h>
24
25 #define IMSIC_MMIO_PAGE_SHIFT 12
26 #define IMSIC_MMIO_PAGE_SZ BIT(IMSIC_MMIO_PAGE_SHIFT)
27
28 #define IMSIC_MIN_ID 63
29 #define IMSIC_MAX_ID 2048
30
31 #define IMSIC_TOPEI_ID_SHIFT 16
32
33 #define IMSIC_IPI_ID 1
34
35 #define IMSIC_COMPATIBLE "riscv,imsics"
36
37 /* IMSIC CSRs */
38 #define IMSIC_EIDELIVERY 0x70
39 #define IMSIC_EITHRESHOLD 0x72
40
41 #define IMSIC_EIP0 0x80
42 #define IMSIC_EIP63 0xBF
43 #define IMSIC_EIPx_BITS 32
44
45 #define IMSIC_EIE0 0xC0
46 #define IMSIC_EIE63 0xFF
47 #define IMSIC_EIEx_BITS 32
48
49 #define IMSIC_DISABLE_EIDELIVERY 0
50 #define IMSIC_ENABLE_EIDELIVERY 1
51
52 #define IMSIC_DISABLE_EITHRESHOLD 1
53 #define IMSIC_ENABLE_EITHRESHOLD 0
54
55 static struct imsic_data imsic_data __nex_bss;
56
57 /*
58 * The IMSIC CSRs need to be indirectly accessed through
59 * the *iselect(miselect/siselect) and *ireg(mireg/sireg) CSRs.
60 */
imsic_csr_write(unsigned long reg,unsigned long val)61 static void imsic_csr_write(unsigned long reg, unsigned long val)
62 {
63 write_csr(CSR_XISELECT, reg);
64 write_csr(CSR_XIREG, val);
65 }
66
imsic_csr_read(unsigned long reg)67 static unsigned long __unused imsic_csr_read(unsigned long reg)
68 {
69 write_csr(CSR_XISELECT, reg);
70 return read_csr(CSR_XIREG);
71 }
72
imsic_csr_set(unsigned long reg,unsigned long val)73 static void imsic_csr_set(unsigned long reg, unsigned long val)
74 {
75 write_csr(CSR_XISELECT, reg);
76 set_csr(CSR_XIREG, val);
77 }
78
imsic_csr_clear(unsigned long reg,unsigned long val)79 static void imsic_csr_clear(unsigned long reg, unsigned long val)
80 {
81 write_csr(CSR_XISELECT, reg);
82 clear_csr(CSR_XIREG, val);
83 }
84
imsic_enable_interrupt_delivery(void)85 static void imsic_enable_interrupt_delivery(void)
86 {
87 imsic_csr_write(IMSIC_EIDELIVERY, IMSIC_ENABLE_EIDELIVERY);
88 }
89
imsic_disable_interrupt_delivery(void)90 static void __unused imsic_disable_interrupt_delivery(void)
91 {
92 imsic_csr_write(IMSIC_EIDELIVERY, IMSIC_DISABLE_EIDELIVERY);
93 }
94
imsic_enable_interrupt_threshold(void)95 static void imsic_enable_interrupt_threshold(void)
96 {
97 imsic_csr_write(IMSIC_EITHRESHOLD, IMSIC_ENABLE_EITHRESHOLD);
98 }
99
imsic_disable_interrupt_threshold(void)100 static void __unused imsic_disable_interrupt_threshold(void)
101 {
102 imsic_csr_write(IMSIC_EITHRESHOLD, IMSIC_DISABLE_EITHRESHOLD);
103 }
104
imsic_claim_interrupt(void)105 static uint32_t imsic_claim_interrupt(void)
106 {
107 uint32_t val = swap_csr(CSR_XTOPEI, 0);
108
109 return val >> IMSIC_TOPEI_ID_SHIFT;
110 }
111
imsic_local_eix_update(uint32_t base_id,uint32_t num_id,bool pend,bool val)112 static void imsic_local_eix_update(uint32_t base_id, uint32_t num_id,
113 bool pend, bool val)
114 {
115 uint32_t i = 0;
116 uint32_t isel = 0;
117 uint32_t ireg = 0;
118 uint32_t id = base_id;
119 uint32_t last_id = base_id + num_id;
120
121 while (id < last_id) {
122 isel = ROUNDDOWN(id, RISCV_XLEN_BITS) / IMSIC_EIPx_BITS;
123 isel += (pend) ? IMSIC_EIP0 : IMSIC_EIE0;
124
125 ireg = 0;
126 for (i = id & (RISCV_XLEN_BITS - 1);
127 (id < last_id) && (i < RISCV_XLEN_BITS); i++) {
128 ireg |= BIT(i);
129 id++;
130 }
131
132 if (val)
133 imsic_csr_set(isel, ireg);
134 else
135 imsic_csr_clear(isel, ireg);
136 }
137 }
138
imsic_it_enable(uint32_t id)139 static void imsic_it_enable(uint32_t id)
140 {
141 imsic_local_eix_update(id, 1, false, true);
142 }
143
imsic_it_disable(uint32_t id)144 static void imsic_it_disable(uint32_t id)
145 {
146 imsic_local_eix_update(id, 1, false, false);
147 }
148
imsic_it_set_pending(uint32_t id)149 static void imsic_it_set_pending(uint32_t id)
150 {
151 imsic_local_eix_update(id, 1, true, true);
152 }
153
imsic_it_clear_pending(uint32_t id)154 static void imsic_it_clear_pending(uint32_t id)
155 {
156 imsic_local_eix_update(id, 1, true, false);
157 }
158
imsic_is_bad_it(struct imsic_data * imsic,size_t it)159 static bool imsic_is_bad_it(struct imsic_data *imsic, size_t it)
160 {
161 assert(imsic == &imsic_data);
162 return (!it || it > imsic->num_ids);
163 }
164
imsic_op_configure(struct itr_chip * chip,size_t it,uint32_t type,uint32_t prio __unused)165 static void imsic_op_configure(struct itr_chip *chip, size_t it,
166 uint32_t type, uint32_t prio __unused)
167 {
168 struct imsic_data *imsic = container_of(chip, struct imsic_data, chip);
169 TEE_Result res = TEE_ERROR_GENERIC;
170
171 if (imsic_is_bad_it(imsic, it))
172 panic();
173
174 if (imsic->aplic_chip) {
175 res = interrupt_configure(imsic->aplic_chip, it, type, prio);
176 if (res)
177 panic();
178 }
179 imsic_it_disable(it);
180 imsic_it_clear_pending(it);
181 }
182
imsic_op_enable(struct itr_chip * chip,size_t it)183 static void imsic_op_enable(struct itr_chip *chip, size_t it)
184 {
185 struct imsic_data *imsic = container_of(chip, struct imsic_data, chip);
186
187 if (imsic_is_bad_it(imsic, it))
188 panic();
189
190 if (imsic->aplic_chip)
191 interrupt_enable(imsic->aplic_chip, it);
192 imsic_it_enable(it);
193 }
194
imsic_op_disable(struct itr_chip * chip,size_t it)195 static void imsic_op_disable(struct itr_chip *chip, size_t it)
196 {
197 struct imsic_data *imsic = container_of(chip, struct imsic_data, chip);
198
199 if (imsic_is_bad_it(imsic, it))
200 panic();
201
202 imsic_it_disable(it);
203 if (imsic->aplic_chip)
204 interrupt_disable(imsic->aplic_chip, it);
205 }
206
imsic_op_raise_pi(struct itr_chip * chip,size_t it)207 static void imsic_op_raise_pi(struct itr_chip *chip, size_t it)
208 {
209 struct imsic_data *imsic = container_of(chip, struct imsic_data, chip);
210
211 if (imsic_is_bad_it(imsic, it))
212 panic();
213
214 imsic_it_set_pending(it);
215 }
216
217 static const struct itr_ops imsic_ops = {
218 .configure = imsic_op_configure,
219 .enable = imsic_op_enable,
220 .disable = imsic_op_disable,
221 .mask = imsic_op_disable,
222 .unmask = imsic_op_enable,
223 .raise_pi = imsic_op_raise_pi,
224 };
225
imsic_init_base_addr(paddr_t imsic_base_pa)226 static void imsic_init_base_addr(paddr_t imsic_base_pa)
227 {
228 struct imsic_data *imsic = &imsic_data;
229 vaddr_t imsic_base = 0;
230
231 assert(cpu_mmu_enabled());
232
233 imsic_base = core_mmu_get_va(imsic_base_pa, MEM_AREA_IO_SEC,
234 IMSIC_SIZE);
235 if (!imsic_base)
236 panic();
237
238 imsic->imsic_base = imsic_base;
239 imsic->size = IMSIC_SIZE;
240 imsic->targets_mmode = false;
241 imsic->num_ids = IMSIC_NUM_IDS;
242 imsic->guest_index_bits = IMSIC_GUEST_INDEX_BITS;
243 imsic->hart_index_bits = IMSIC_HART_INDEX_BITS;
244 imsic->group_index_bits = IMSIC_GROUP_INDEX_BITS;
245 imsic->group_index_shift = IMSIC_GROUP_INDEX_SHIFT;
246 }
247
248 #if defined(CFG_DT) && defined(CFG_RISCV_IMSIC)
imisc_parse_fdt_node(const void * fdt,int nodeoff,struct imsic_data * imsic)249 TEE_Result imisc_parse_fdt_node(const void *fdt, int nodeoff,
250 struct imsic_data *imsic)
251 {
252 const fdt32_t *val = NULL;
253 paddr_t reg_addr = 0;
254 size_t reg_size = 0;
255 int i = 0;
256 int rc = -1;
257 int len = 0;
258
259 if (nodeoff < 0 || !imsic || !fdt)
260 return TEE_ERROR_BAD_PARAMETERS;
261
262 rc = fdt_reg_info(fdt, nodeoff, ®_addr, ®_size);
263 if (rc < 0 || !reg_addr || !reg_size)
264 return TEE_ERROR_ITEM_NOT_FOUND;
265 imsic->imsic_base = core_mmu_get_va(reg_addr, MEM_AREA_IO_SEC,
266 reg_size);
267 if (!imsic->imsic_base)
268 return TEE_ERROR_GENERIC;
269 imsic->size = reg_size;
270
271 imsic->targets_mmode = false;
272 val = fdt_getprop(fdt, nodeoff, "interrupts-extended", &len);
273 if (val && (size_t)len >= (2 * sizeof(fdt32_t))) {
274 len = len / sizeof(fdt32_t);
275 for (i = 0; i < len; i += 2) {
276 if (fdt32_to_cpu(val[i + 1]) == IRQ_M_EXT) {
277 imsic->targets_mmode = true;
278 break;
279 }
280 }
281 } else {
282 return TEE_ERROR_ITEM_NOT_FOUND;
283 }
284
285 val = fdt_getprop(fdt, nodeoff, "riscv,guest-index-bits", &len);
286 if (val && len > 0)
287 imsic->guest_index_bits = fdt32_to_cpu(*val);
288 else
289 imsic->guest_index_bits = 0;
290
291 val = fdt_getprop(fdt, nodeoff, "riscv,hart-index-bits", &len);
292 if (val && len > 0)
293 imsic->hart_index_bits = fdt32_to_cpu(*val);
294 else
295 imsic->hart_index_bits =
296 32 - __builtin_clz(CFG_TEE_CORE_NB_CORE - 1);
297
298 val = fdt_getprop(fdt, nodeoff, "riscv,group-index-bits", &len);
299 if (val && len > 0)
300 imsic->group_index_bits = fdt32_to_cpu(*val);
301 else
302 imsic->group_index_bits = 0;
303
304 val = fdt_getprop(fdt, nodeoff, "riscv,group-index-shift", &len);
305 if (val && len > 0)
306 imsic->group_index_shift = fdt32_to_cpu(*val);
307 else
308 imsic->group_index_shift = 2 * IMSIC_MMIO_PAGE_SHIFT;
309
310 val = fdt_getprop(fdt, nodeoff, "riscv,num-ids", &len);
311 if (val && len > 0)
312 imsic->num_ids = fdt32_to_cpu(*val);
313 else
314 return TEE_ERROR_ITEM_NOT_FOUND;
315
316 return TEE_SUCCESS;
317 }
318 #endif
319
imsic_init_from_device_tree(struct imsic_data * imsic)320 static TEE_Result imsic_init_from_device_tree(struct imsic_data *imsic)
321 {
322 void *fdt = NULL;
323 int node = FDT_ERR_NOTFOUND;
324 TEE_Result res = TEE_ERROR_GENERIC;
325
326 fdt = get_dt();
327 if (!fdt) {
328 EMSG("Unable to get DTB, IMSIC init failed");
329 return TEE_ERROR_ITEM_NOT_FOUND;
330 }
331
332 /*
333 * Currently, only the S-level interrupt file is considered.
334 * If the interrupt file is M-level, continue traversing.
335 * If it is S-level, return directly.
336 */
337 node = fdt_node_offset_by_compatible(fdt, -1, IMSIC_COMPATIBLE);
338 while (node != -FDT_ERR_NOTFOUND) {
339 res = imisc_parse_fdt_node(fdt, node, imsic);
340 if (res) {
341 EMSG("Parse IMSIC node failed");
342 return res;
343 }
344 if (!imsic->targets_mmode)
345 return TEE_SUCCESS;
346 node = fdt_node_offset_by_compatible(fdt, node,
347 IMSIC_COMPATIBLE);
348 }
349
350 return TEE_ERROR_ITEM_NOT_FOUND;
351 }
352
imsic_it_handle(void)353 void imsic_it_handle(void)
354 {
355 struct imsic_data *imsic = &imsic_data;
356 uint32_t id = imsic_claim_interrupt();
357
358 if (id == IMSIC_IPI_ID)
359 DMSG("Interprocessor interrupt");
360
361 if (id > IMSIC_IPI_ID && id <= imsic->num_ids)
362 interrupt_call_handlers(&imsic->chip, id);
363 else
364 DMSG("ignoring interrupt %" PRIu32, id);
365 }
366
imsic_init_per_hart(void)367 void imsic_init_per_hart(void)
368 {
369 struct imsic_data *imsic = &imsic_data;
370
371 imsic_local_eix_update(1, imsic->num_ids, false, false);
372 imsic_enable_interrupt_threshold();
373 imsic_enable_interrupt_delivery();
374 }
375
imsic_init(paddr_t imsic_base_pa)376 void imsic_init(paddr_t imsic_base_pa)
377 {
378 struct imsic_data *imsic = &imsic_data;
379 TEE_Result res = TEE_ERROR_GENERIC;
380
381 if (IS_ENABLED(CFG_DT)) {
382 res = imsic_init_from_device_tree(imsic);
383 if (res)
384 panic();
385 } else {
386 imsic_init_base_addr(imsic_base_pa);
387 }
388
389 imsic->chip.ops = &imsic_ops;
390
391 imsic_init_per_hart();
392
393 imsic->aplic_chip = interrupt_get_main_chip();
394
395 interrupt_main_init(&imsic->chip);
396 }
397
imsic_dump_state(void)398 void imsic_dump_state(void)
399 {
400 }
401