1 /* 2 * Copyright (c) 2021-2023, Arm Limited. All rights reserved. 3 * 4 * SPDX-License-Identifier: BSD-3-Clause 5 */ 6 7 #include <stdint.h> 8 #include <stdbool.h> 9 10 #include <common/debug.h> 11 #include <common/runtime_svc.h> 12 #include <drivers/arm/ethosn.h> 13 #include <drivers/delay_timer.h> 14 #include <lib/mmio.h> 15 #include <lib/utils_def.h> 16 #include <plat/arm/common/fconf_ethosn_getter.h> 17 18 #include <platform_def.h> 19 20 #if ARM_ETHOSN_NPU_TZMP1 21 #include "ethosn_big_fw.h" 22 #endif 23 24 /* 25 * Number of Arm(R) Ethos(TM)-N NPU (NPU) devices available 26 */ 27 #define ETHOSN_NUM_DEVICES \ 28 FCONF_GET_PROPERTY(hw_config, ethosn_config, num_devices) 29 30 #define ETHOSN_GET_DEVICE(dev_idx) \ 31 FCONF_GET_PROPERTY(hw_config, ethosn_device, dev_idx) 32 33 /* NPU core sec registry address */ 34 #define ETHOSN_CORE_SEC_REG(core_addr, reg_offset) \ 35 (core_addr + reg_offset) 36 37 #define ETHOSN_FW_VA_BASE 0x20000000UL 38 #define ETHOSN_WORKING_DATA_VA_BASE 0x40000000UL 39 #define ETHOSN_COMMAND_STREAM_VA_BASE 0x60000000UL 40 41 /* Reset timeout in us */ 42 #define ETHOSN_RESET_TIMEOUT_US U(10 * 1000 * 1000) 43 #define ETHOSN_RESET_WAIT_US U(1) 44 45 #define ETHOSN_AUX_FEAT_LEVEL_IRQ U(0x1) 46 #define ETHOSN_AUX_FEAT_STASHING U(0x2) 47 48 #define SEC_AUXCTLR_REG U(0x0024) 49 #define SEC_AUXCTLR_VAL U(0x80) 50 #define SEC_AUXCTLR_LEVEL_IRQ_VAL U(0x04) 51 #define SEC_AUXCTLR_STASHING_VAL U(0xA5000000) 52 53 #define SEC_DEL_REG U(0x0004) 54 #define SEC_DEL_VAL U(0x80C) 55 #define SEC_DEL_EXCC_MASK U(0x20) 56 57 #define SEC_SECCTLR_REG U(0x0010) 58 /* Set bit[10] = 1 to workaround erratum 2838783 */ 59 #define SEC_SECCTLR_VAL U(0x403) 60 61 #define SEC_DEL_ADDR_EXT_REG U(0x201C) 62 #define SEC_DEL_ADDR_EXT_VAL U(0x1) 63 64 #define SEC_SYSCTRL0_REG U(0x0018) 65 #define SEC_SYSCTRL0_SLEEPING U(1U << 4) 66 #define SEC_SYSCTRL0_SOFT_RESET U(3U << 29) 67 #define SEC_SYSCTRL0_HARD_RESET U(1U << 31) 68 69 #define SEC_SYSCTRL1_REG U(0x001C) 70 #define SEC_SYSCTRL1_VAL U(0x180110) 71 72 #define SEC_NSAID_REG_BASE U(0x3004) 73 #define SEC_NSAID_OFFSET U(0x1000) 74 75 #define SEC_MMUSID_REG_BASE U(0x3008) 76 #define SEC_MMUSID_OFFSET U(0x1000) 77 78 #define SEC_ADDR_EXT_REG_BASE U(0x3018) 79 #define SEC_ADDR_EXT_OFFSET U(0x1000) 80 #define SEC_ADDR_EXT_SHIFT U(0x14) 81 #define SEC_ADDR_EXT_MASK U(0x1FFFFE00) 82 83 #define SEC_ATTR_CTLR_REG_BASE U(0x3010) 84 #define SEC_ATTR_CTLR_OFFSET U(0x1000) 85 #define SEC_ATTR_CTLR_NUM U(9) 86 #define SEC_ATTR_CTLR_VAL U(0x1) 87 88 #define SEC_NPU_ID_REG U(0xF000) 89 #define SEC_NPU_ID_ARCH_VER_SHIFT U(0X10) 90 91 #define INPUT_STREAM_INDEX U(0x6) 92 #define INTERMEDIATE_STREAM_INDEX U(0x7) 93 #define OUTPUT_STREAM_INDEX U(0x8) 94 95 #define TO_EXTEND_ADDR(addr) \ 96 ((addr >> SEC_ADDR_EXT_SHIFT) & SEC_ADDR_EXT_MASK) 97 98 #if ARM_ETHOSN_NPU_TZMP1 99 CASSERT(ARM_ETHOSN_NPU_FW_IMAGE_BASE > 0U, assert_ethosn_invalid_fw_image_base); 100 static const struct ethosn_big_fw *big_fw; 101 #endif 102 103 static bool ethosn_get_device_and_core(uintptr_t core_addr, 104 const struct ethosn_device_t **dev_match, 105 const struct ethosn_core_t **core_match) 106 { 107 uint32_t dev_idx; 108 uint32_t core_idx; 109 110 for (dev_idx = 0U; dev_idx < ETHOSN_NUM_DEVICES; ++dev_idx) { 111 const struct ethosn_device_t *dev = ETHOSN_GET_DEVICE(dev_idx); 112 113 for (core_idx = 0U; core_idx < dev->num_cores; ++core_idx) { 114 const struct ethosn_core_t *core = &(dev->cores[core_idx]); 115 116 if (core->addr == core_addr) { 117 *dev_match = dev; 118 *core_match = core; 119 return true; 120 } 121 } 122 } 123 124 WARN("ETHOSN: Unknown core address given to SMC call.\n"); 125 return false; 126 } 127 128 #if ARM_ETHOSN_NPU_TZMP1 129 static uint32_t ethosn_core_read_arch_version(uintptr_t core_addr) 130 { 131 uint32_t npu_id = mmio_read_32(ETHOSN_CORE_SEC_REG(core_addr, 132 SEC_NPU_ID_REG)); 133 134 return (npu_id >> SEC_NPU_ID_ARCH_VER_SHIFT); 135 } 136 137 static void ethosn_configure_stream_nsaid(const struct ethosn_core_t *core, 138 bool is_protected) 139 { 140 size_t i; 141 uint32_t streams[9] = {0, 0, 0, 0, 0, 0, 0, 0, 0}; 142 143 if (is_protected) { 144 streams[INPUT_STREAM_INDEX] = ARM_ETHOSN_NPU_PROT_DATA_NSAID; 145 streams[INTERMEDIATE_STREAM_INDEX] = 146 ARM_ETHOSN_NPU_PROT_DATA_NSAID; 147 streams[OUTPUT_STREAM_INDEX] = ARM_ETHOSN_NPU_PROT_DATA_NSAID; 148 } 149 150 for (i = 0U; i < ARRAY_SIZE(streams); ++i) { 151 const uintptr_t reg_addr = SEC_NSAID_REG_BASE + 152 (SEC_NSAID_OFFSET * i); 153 mmio_write_32(ETHOSN_CORE_SEC_REG(core->addr, reg_addr), 154 streams[i]); 155 } 156 } 157 #endif 158 159 static void ethosn_configure_events(uintptr_t core_addr) 160 { 161 mmio_write_32(ETHOSN_CORE_SEC_REG(core_addr, SEC_SYSCTRL1_REG), SEC_SYSCTRL1_VAL); 162 } 163 164 static bool ethosn_configure_aux_features(const struct ethosn_device_t *device, 165 uintptr_t core_addr, 166 uint32_t features) 167 { 168 uint32_t val = SEC_AUXCTLR_VAL; 169 170 if (features & ETHOSN_AUX_FEAT_LEVEL_IRQ) { 171 val |= SEC_AUXCTLR_LEVEL_IRQ_VAL; 172 } 173 174 if (features & ETHOSN_AUX_FEAT_STASHING) { 175 /* Stashing can't be used with reserved memory */ 176 if (device->has_reserved_memory) { 177 return false; 178 } 179 180 val |= SEC_AUXCTLR_STASHING_VAL; 181 } 182 183 mmio_setbits_32(ETHOSN_CORE_SEC_REG(core_addr, SEC_AUXCTLR_REG), val); 184 185 return true; 186 } 187 188 static void ethosn_configure_smmu_streams(const struct ethosn_device_t *device, 189 const struct ethosn_core_t *core, 190 uint32_t asset_alloc_idx) 191 { 192 const struct ethosn_main_allocator_t *main_alloc = 193 &(core->main_allocator); 194 const struct ethosn_asset_allocator_t *asset_alloc = 195 &(device->asset_allocators[asset_alloc_idx]); 196 const uint32_t streams[9] = { 197 main_alloc->firmware.stream_id, 198 main_alloc->working_data.stream_id, 199 asset_alloc->command_stream.stream_id, 200 0U, /* Not used*/ 201 main_alloc->firmware.stream_id, 202 asset_alloc->weight_data.stream_id, 203 asset_alloc->buffer_data.stream_id, 204 asset_alloc->intermediate_data.stream_id, 205 asset_alloc->buffer_data.stream_id 206 }; 207 size_t i; 208 209 for (i = 0U; i < ARRAY_SIZE(streams); ++i) { 210 const uintptr_t reg_addr = SEC_MMUSID_REG_BASE + 211 (SEC_MMUSID_OFFSET * i); 212 mmio_write_32(ETHOSN_CORE_SEC_REG(core->addr, reg_addr), 213 streams[i]); 214 } 215 } 216 217 static void ethosn_configure_stream_addr_extends(const struct ethosn_device_t *device, 218 uintptr_t core_addr) 219 { 220 uint32_t addr_extends[3] = { 0 }; 221 size_t i; 222 223 if (device->has_reserved_memory) { 224 const uint32_t addr = TO_EXTEND_ADDR(device->reserved_memory_addr); 225 226 addr_extends[0] = addr; 227 addr_extends[1] = addr; 228 addr_extends[2] = addr; 229 } else { 230 addr_extends[0] = TO_EXTEND_ADDR(ETHOSN_FW_VA_BASE); 231 addr_extends[1] = TO_EXTEND_ADDR(ETHOSN_WORKING_DATA_VA_BASE); 232 addr_extends[2] = TO_EXTEND_ADDR(ETHOSN_COMMAND_STREAM_VA_BASE); 233 } 234 235 for (i = 0U; i < ARRAY_SIZE(addr_extends); ++i) { 236 const uintptr_t reg_addr = SEC_ADDR_EXT_REG_BASE + 237 (SEC_ADDR_EXT_OFFSET * i); 238 mmio_write_32(ETHOSN_CORE_SEC_REG(core_addr, reg_addr), 239 addr_extends[i]); 240 } 241 } 242 243 static void ethosn_configure_stream_attr_ctlr(uintptr_t core_addr) 244 { 245 size_t i; 246 247 for (i = 0U; i < SEC_ATTR_CTLR_NUM; ++i) { 248 const uintptr_t reg_addr = SEC_ATTR_CTLR_REG_BASE + 249 (SEC_ATTR_CTLR_OFFSET * i); 250 mmio_write_32(ETHOSN_CORE_SEC_REG(core_addr, reg_addr), 251 SEC_ATTR_CTLR_VAL); 252 } 253 } 254 255 static void ethosn_delegate_to_ns(uintptr_t core_addr) 256 { 257 mmio_setbits_32(ETHOSN_CORE_SEC_REG(core_addr, SEC_SECCTLR_REG), 258 SEC_SECCTLR_VAL); 259 260 mmio_setbits_32(ETHOSN_CORE_SEC_REG(core_addr, SEC_DEL_REG), 261 SEC_DEL_VAL); 262 263 mmio_setbits_32(ETHOSN_CORE_SEC_REG(core_addr, SEC_DEL_ADDR_EXT_REG), 264 SEC_DEL_ADDR_EXT_VAL); 265 } 266 267 static int ethosn_is_sec(uintptr_t core_addr) 268 { 269 if ((mmio_read_32(ETHOSN_CORE_SEC_REG(core_addr, SEC_DEL_REG)) 270 & SEC_DEL_EXCC_MASK) != 0U) { 271 return 0; 272 } 273 274 return 1; 275 } 276 277 static int ethosn_core_is_sleeping(uintptr_t core_addr) 278 { 279 const uintptr_t sysctrl0_reg = 280 ETHOSN_CORE_SEC_REG(core_addr, SEC_SYSCTRL0_REG); 281 const uint32_t sleeping_mask = SEC_SYSCTRL0_SLEEPING; 282 283 return ((mmio_read_32(sysctrl0_reg) & sleeping_mask) == sleeping_mask); 284 } 285 286 static bool ethosn_core_reset(uintptr_t core_addr, bool hard_reset) 287 { 288 unsigned int timeout; 289 const uintptr_t sysctrl0_reg = 290 ETHOSN_CORE_SEC_REG(core_addr, SEC_SYSCTRL0_REG); 291 const uint32_t reset_val = hard_reset ? SEC_SYSCTRL0_HARD_RESET : 292 SEC_SYSCTRL0_SOFT_RESET; 293 294 mmio_write_32(sysctrl0_reg, reset_val); 295 296 /* Wait for reset to complete */ 297 for (timeout = 0U; timeout < ETHOSN_RESET_TIMEOUT_US; 298 timeout += ETHOSN_RESET_WAIT_US) { 299 300 if ((mmio_read_32(sysctrl0_reg) & reset_val) == 0U) { 301 break; 302 } 303 304 udelay(ETHOSN_RESET_WAIT_US); 305 } 306 307 return timeout < ETHOSN_RESET_TIMEOUT_US; 308 } 309 310 static int ethosn_core_full_reset(const struct ethosn_device_t *device, 311 const struct ethosn_core_t *core, 312 bool hard_reset, 313 u_register_t asset_alloc_idx, 314 u_register_t is_protected, 315 u_register_t aux_features) 316 { 317 if (!device->has_reserved_memory && 318 asset_alloc_idx >= device->num_allocators) { 319 WARN("ETHOSN: Unknown asset allocator index given to SMC call.\n"); 320 return ETHOSN_UNKNOWN_ALLOCATOR_IDX; 321 } 322 323 if (!ethosn_core_reset(core->addr, hard_reset)) { 324 return ETHOSN_FAILURE; 325 } 326 327 if (!ethosn_configure_aux_features(device, core->addr, aux_features)) { 328 return ETHOSN_INVALID_CONFIGURATION; 329 } 330 331 ethosn_configure_events(core->addr); 332 333 if (!device->has_reserved_memory) { 334 ethosn_configure_smmu_streams(device, core, asset_alloc_idx); 335 336 #if ARM_ETHOSN_NPU_TZMP1 337 ethosn_configure_stream_nsaid(core, is_protected); 338 #endif 339 } 340 341 ethosn_configure_stream_addr_extends(device, core->addr); 342 ethosn_configure_stream_attr_ctlr(core->addr); 343 344 ethosn_delegate_to_ns(core->addr); 345 346 return ETHOSN_SUCCESS; 347 } 348 349 static uintptr_t ethosn_smc_core_reset_handler(const struct ethosn_device_t *device, 350 const struct ethosn_core_t *core, 351 bool hard_reset, 352 u_register_t asset_alloc_idx, 353 u_register_t reset_type, 354 u_register_t is_protected, 355 u_register_t aux_features, 356 void *handle) 357 { 358 int ret; 359 360 switch (reset_type) { 361 case ETHOSN_RESET_TYPE_FULL: 362 ret = ethosn_core_full_reset(device, core, hard_reset, 363 asset_alloc_idx, is_protected, 364 aux_features); 365 break; 366 case ETHOSN_RESET_TYPE_HALT: 367 ret = ethosn_core_reset(core->addr, hard_reset) ? ETHOSN_SUCCESS : ETHOSN_FAILURE; 368 break; 369 default: 370 WARN("ETHOSN: Invalid reset type given to SMC call.\n"); 371 ret = ETHOSN_INVALID_PARAMETER; 372 break; 373 } 374 375 SMC_RET1(handle, ret); 376 } 377 378 static uintptr_t ethosn_smc_core_handler(uint32_t fid, 379 u_register_t core_addr, 380 u_register_t asset_alloc_idx, 381 u_register_t reset_type, 382 u_register_t is_protected, 383 u_register_t aux_features, 384 void *handle) 385 { 386 bool hard_reset = false; 387 const struct ethosn_device_t *device = NULL; 388 const struct ethosn_core_t *core = NULL; 389 390 if (!ethosn_get_device_and_core(core_addr, &device, &core)) { 391 SMC_RET1(handle, ETHOSN_UNKNOWN_CORE_ADDRESS); 392 } 393 394 switch (fid) { 395 case ETHOSN_FNUM_IS_SEC: 396 SMC_RET1(handle, ethosn_is_sec(core->addr)); 397 case ETHOSN_FNUM_IS_SLEEPING: 398 SMC_RET1(handle, ethosn_core_is_sleeping(core->addr)); 399 case ETHOSN_FNUM_HARD_RESET: 400 hard_reset = true; 401 /* Fallthrough */ 402 case ETHOSN_FNUM_SOFT_RESET: 403 return ethosn_smc_core_reset_handler(device, core, 404 hard_reset, 405 asset_alloc_idx, 406 reset_type, 407 is_protected, 408 aux_features, 409 handle); 410 default: 411 WARN("ETHOSN: Unimplemented SMC call: 0x%x\n", fid); 412 SMC_RET1(handle, SMC_UNK); 413 } 414 } 415 416 static uintptr_t ethosn_smc_fw_prop_handler(u_register_t fw_property, 417 void *handle) 418 { 419 #if ARM_ETHOSN_NPU_TZMP1 420 switch (fw_property) { 421 case ETHOSN_FW_PROP_VERSION: 422 SMC_RET4(handle, ETHOSN_SUCCESS, 423 big_fw->fw_ver_major, 424 big_fw->fw_ver_minor, 425 big_fw->fw_ver_patch); 426 case ETHOSN_FW_PROP_MEM_INFO: 427 SMC_RET3(handle, ETHOSN_SUCCESS, 428 ((void *)big_fw) + big_fw->offset, 429 big_fw->size); 430 case ETHOSN_FW_PROP_OFFSETS: 431 SMC_RET3(handle, ETHOSN_SUCCESS, 432 big_fw->ple_offset, 433 big_fw->unpriv_stack_offset); 434 case ETHOSN_FW_PROP_VA_MAP: 435 SMC_RET4(handle, ETHOSN_SUCCESS, 436 ETHOSN_FW_VA_BASE, 437 ETHOSN_WORKING_DATA_VA_BASE, 438 ETHOSN_COMMAND_STREAM_VA_BASE); 439 default: 440 WARN("ETHOSN: Unknown firmware property\n"); 441 SMC_RET1(handle, ETHOSN_INVALID_PARAMETER); 442 } 443 #else 444 SMC_RET1(handle, ETHOSN_NOT_SUPPORTED); 445 #endif 446 } 447 448 uintptr_t ethosn_smc_handler(uint32_t smc_fid, 449 u_register_t x1, 450 u_register_t x2, 451 u_register_t x3, 452 u_register_t x4, 453 void *cookie, 454 void *handle, 455 u_register_t flags) 456 { 457 const uint32_t fid = smc_fid & FUNCID_NUM_MASK; 458 459 /* Only SiP fast calls are expected */ 460 if ((GET_SMC_TYPE(smc_fid) != SMC_TYPE_FAST) || 461 (GET_SMC_OEN(smc_fid) != OEN_SIP_START)) { 462 SMC_RET1(handle, SMC_UNK); 463 } 464 465 /* Truncate parameters to 32-bits for SMC32 */ 466 if (GET_SMC_CC(smc_fid) == SMC_32) { 467 x1 &= 0xFFFFFFFF; 468 x2 &= 0xFFFFFFFF; 469 x3 &= 0xFFFFFFFF; 470 x4 &= 0xFFFFFFFF; 471 } 472 473 if (!is_ethosn_fid(smc_fid) || (fid > ETHOSN_FNUM_GET_FW_PROP)) { 474 WARN("ETHOSN: Unknown SMC call: 0x%x\n", smc_fid); 475 SMC_RET1(handle, SMC_UNK); 476 } 477 478 switch (fid) { 479 case ETHOSN_FNUM_VERSION: 480 SMC_RET2(handle, ETHOSN_VERSION_MAJOR, ETHOSN_VERSION_MINOR); 481 case ETHOSN_FNUM_GET_FW_PROP: 482 return ethosn_smc_fw_prop_handler(x1, handle); 483 } 484 485 return ethosn_smc_core_handler(fid, x1, x2, x3, x4, 486 SMC_GET_GP(handle, CTX_GPREG_X5), 487 handle); 488 } 489 490 int ethosn_smc_setup(void) 491 { 492 #if ARM_ETHOSN_NPU_TZMP1 493 struct ethosn_device_t *dev; 494 uint32_t arch_ver; 495 #endif 496 497 if (ETHOSN_NUM_DEVICES == 0U) { 498 ERROR("ETHOSN: No NPU found\n"); 499 return ETHOSN_FAILURE; 500 } 501 502 #if ARM_ETHOSN_NPU_TZMP1 503 504 /* Only one NPU core is supported in the TZMP1 setup */ 505 if ((ETHOSN_NUM_DEVICES != 1U) || 506 (ETHOSN_GET_DEVICE(0U)->num_cores != 1U)) { 507 ERROR("ETHOSN: TZMP1 doesn't support multiple NPU cores\n"); 508 return ETHOSN_FAILURE; 509 } 510 511 dev = ETHOSN_GET_DEVICE(0U); 512 arch_ver = ethosn_core_read_arch_version(dev->cores[0U].addr); 513 big_fw = (struct ethosn_big_fw *)ARM_ETHOSN_NPU_FW_IMAGE_BASE; 514 515 if (!ethosn_big_fw_verify_header(big_fw, arch_ver)) { 516 return ETHOSN_FAILURE; 517 } 518 519 NOTICE("ETHOSN: TZMP1 setup succeeded with firmware version %u.%u.%u\n", 520 big_fw->fw_ver_major, big_fw->fw_ver_minor, 521 big_fw->fw_ver_patch); 522 #else 523 NOTICE("ETHOSN: Setup succeeded\n"); 524 #endif 525 526 return 0; 527 } 528