1 /* 2 * Copyright (c) 2021-2023, Arm Limited. All rights reserved. 3 * 4 * SPDX-License-Identifier: BSD-3-Clause 5 */ 6 7 #include <stdint.h> 8 #include <stdbool.h> 9 10 #include <common/debug.h> 11 #include <common/runtime_svc.h> 12 #include <drivers/arm/ethosn.h> 13 #include <drivers/delay_timer.h> 14 #include <lib/mmio.h> 15 #include <lib/utils_def.h> 16 #include <plat/arm/common/fconf_ethosn_getter.h> 17 18 #include <platform_def.h> 19 20 #if ARM_ETHOSN_NPU_TZMP1 21 #include "ethosn_big_fw.h" 22 #endif 23 24 /* 25 * Number of Arm(R) Ethos(TM)-N NPU (NPU) devices available 26 */ 27 #define ETHOSN_NUM_DEVICES \ 28 FCONF_GET_PROPERTY(hw_config, ethosn_config, num_devices) 29 30 #define ETHOSN_GET_DEVICE(dev_idx) \ 31 FCONF_GET_PROPERTY(hw_config, ethosn_device, dev_idx) 32 33 /* NPU core sec registry address */ 34 #define ETHOSN_CORE_SEC_REG(core_addr, reg_offset) \ 35 (core_addr + reg_offset) 36 37 #define ETHOSN_FW_VA_BASE 0x20000000UL 38 #define ETHOSN_WORKING_DATA_VA_BASE 0x40000000UL 39 #define ETHOSN_COMMAND_STREAM_VA_BASE 0x60000000UL 40 41 /* Reset timeout in us */ 42 #define ETHOSN_RESET_TIMEOUT_US U(10 * 1000 * 1000) 43 #define ETHOSN_RESET_WAIT_US U(1) 44 45 #define ETHOSN_AUX_FEAT_LEVEL_IRQ U(0x1) 46 #define ETHOSN_AUX_FEAT_STASHING U(0x2) 47 48 #define SEC_AUXCTLR_REG U(0x0024) 49 #define SEC_AUXCTLR_VAL U(0x80) 50 #define SEC_AUXCTLR_LEVEL_IRQ_VAL U(0x04) 51 #define SEC_AUXCTLR_STASHING_VAL U(0xA5000000) 52 53 #define SEC_DEL_REG U(0x0004) 54 #define SEC_DEL_VAL U(0x80C) 55 #define SEC_DEL_EXCC_MASK U(0x20) 56 57 #define SEC_SECCTLR_REG U(0x0010) 58 /* Set bit[10] = 1 to workaround erratum 2838783 */ 59 #define SEC_SECCTLR_VAL U(0x403) 60 61 #define SEC_DEL_ADDR_EXT_REG U(0x201C) 62 #define SEC_DEL_ADDR_EXT_VAL U(0x15) 63 64 #define SEC_SYSCTRL0_REG U(0x0018) 65 #define SEC_SYSCTRL0_SLEEPING U(1U << 4) 66 #define SEC_SYSCTRL0_SOFT_RESET U(3U << 29) 67 #define SEC_SYSCTRL0_HARD_RESET U(1U << 31) 68 69 #define SEC_SYSCTRL1_REG U(0x001C) 70 #define SEC_SYSCTRL1_VAL U(0x180110) 71 72 #define SEC_NSAID_REG_BASE U(0x3004) 73 #define SEC_NSAID_OFFSET U(0x1000) 74 75 #define SEC_MMUSID_REG_BASE U(0x3008) 76 #define SEC_MMUSID_OFFSET U(0x1000) 77 78 #define SEC_NPU_ID_REG U(0xF000) 79 #define SEC_NPU_ID_ARCH_VER_SHIFT U(0X10) 80 81 #define INPUT_STREAM_INDEX U(0x6) 82 #define INTERMEDIATE_STREAM_INDEX U(0x7) 83 #define OUTPUT_STREAM_INDEX U(0x8) 84 85 #if ARM_ETHOSN_NPU_TZMP1 86 CASSERT(ARM_ETHOSN_NPU_FW_IMAGE_BASE > 0U, assert_ethosn_invalid_fw_image_base); 87 static const struct ethosn_big_fw *big_fw; 88 #endif 89 90 static bool ethosn_get_device_and_core(uintptr_t core_addr, 91 const struct ethosn_device_t **dev_match, 92 const struct ethosn_core_t **core_match) 93 { 94 uint32_t dev_idx; 95 uint32_t core_idx; 96 97 for (dev_idx = 0U; dev_idx < ETHOSN_NUM_DEVICES; ++dev_idx) { 98 const struct ethosn_device_t *dev = ETHOSN_GET_DEVICE(dev_idx); 99 100 for (core_idx = 0U; core_idx < dev->num_cores; ++core_idx) { 101 const struct ethosn_core_t *core = &(dev->cores[core_idx]); 102 103 if (core->addr == core_addr) { 104 *dev_match = dev; 105 *core_match = core; 106 return true; 107 } 108 } 109 } 110 111 WARN("ETHOSN: Unknown core address given to SMC call.\n"); 112 return false; 113 } 114 115 #if ARM_ETHOSN_NPU_TZMP1 116 static uint32_t ethosn_core_read_arch_version(uintptr_t core_addr) 117 { 118 uint32_t npu_id = mmio_read_32(ETHOSN_CORE_SEC_REG(core_addr, 119 SEC_NPU_ID_REG)); 120 121 return (npu_id >> SEC_NPU_ID_ARCH_VER_SHIFT); 122 } 123 124 static void ethosn_configure_stream_nsaid(const struct ethosn_core_t *core, 125 bool is_protected) 126 { 127 size_t i; 128 uint32_t streams[9] = {0, 0, 0, 0, 0, 0, 0, 0, 0}; 129 130 if (is_protected) { 131 streams[INPUT_STREAM_INDEX] = ARM_ETHOSN_NPU_PROT_DATA_NSAID; 132 streams[INTERMEDIATE_STREAM_INDEX] = 133 ARM_ETHOSN_NPU_PROT_DATA_NSAID; 134 streams[OUTPUT_STREAM_INDEX] = ARM_ETHOSN_NPU_PROT_DATA_NSAID; 135 } 136 137 for (i = 0U; i < ARRAY_SIZE(streams); ++i) { 138 const uintptr_t reg_addr = SEC_NSAID_REG_BASE + 139 (SEC_NSAID_OFFSET * i); 140 mmio_write_32(ETHOSN_CORE_SEC_REG(core->addr, reg_addr), 141 streams[i]); 142 } 143 } 144 #endif 145 146 static void ethosn_configure_events(uintptr_t core_addr) 147 { 148 mmio_write_32(ETHOSN_CORE_SEC_REG(core_addr, SEC_SYSCTRL1_REG), SEC_SYSCTRL1_VAL); 149 } 150 151 static bool ethosn_configure_aux_features(const struct ethosn_device_t *device, 152 uintptr_t core_addr, 153 uint32_t features) 154 { 155 uint32_t val = SEC_AUXCTLR_VAL; 156 157 if (features & ETHOSN_AUX_FEAT_LEVEL_IRQ) { 158 val |= SEC_AUXCTLR_LEVEL_IRQ_VAL; 159 } 160 161 if (features & ETHOSN_AUX_FEAT_STASHING) { 162 /* Stashing can't be used with reserved memory */ 163 if (device->has_reserved_memory) { 164 return false; 165 } 166 167 val |= SEC_AUXCTLR_STASHING_VAL; 168 } 169 170 mmio_setbits_32(ETHOSN_CORE_SEC_REG(core_addr, SEC_AUXCTLR_REG), val); 171 172 return true; 173 } 174 175 static void ethosn_configure_smmu_streams(const struct ethosn_device_t *device, 176 const struct ethosn_core_t *core, 177 uint32_t asset_alloc_idx) 178 { 179 const struct ethosn_main_allocator_t *main_alloc = 180 &(core->main_allocator); 181 const struct ethosn_asset_allocator_t *asset_alloc = 182 &(device->asset_allocators[asset_alloc_idx]); 183 const uint32_t streams[9] = { 184 main_alloc->firmware.stream_id, 185 main_alloc->working_data.stream_id, 186 asset_alloc->command_stream.stream_id, 187 0U, /* Not used*/ 188 main_alloc->firmware.stream_id, 189 asset_alloc->weight_data.stream_id, 190 asset_alloc->buffer_data.stream_id, 191 asset_alloc->intermediate_data.stream_id, 192 asset_alloc->buffer_data.stream_id 193 }; 194 size_t i; 195 196 for (i = 0U; i < ARRAY_SIZE(streams); ++i) { 197 const uintptr_t reg_addr = SEC_MMUSID_REG_BASE + 198 (SEC_MMUSID_OFFSET * i); 199 mmio_write_32(ETHOSN_CORE_SEC_REG(core->addr, reg_addr), 200 streams[i]); 201 } 202 } 203 204 static void ethosn_delegate_to_ns(uintptr_t core_addr) 205 { 206 mmio_setbits_32(ETHOSN_CORE_SEC_REG(core_addr, SEC_SECCTLR_REG), 207 SEC_SECCTLR_VAL); 208 209 mmio_setbits_32(ETHOSN_CORE_SEC_REG(core_addr, SEC_DEL_REG), 210 SEC_DEL_VAL); 211 212 mmio_setbits_32(ETHOSN_CORE_SEC_REG(core_addr, SEC_DEL_ADDR_EXT_REG), 213 SEC_DEL_ADDR_EXT_VAL); 214 } 215 216 static int ethosn_is_sec(uintptr_t core_addr) 217 { 218 if ((mmio_read_32(ETHOSN_CORE_SEC_REG(core_addr, SEC_DEL_REG)) 219 & SEC_DEL_EXCC_MASK) != 0U) { 220 return 0; 221 } 222 223 return 1; 224 } 225 226 static int ethosn_core_is_sleeping(uintptr_t core_addr) 227 { 228 const uintptr_t sysctrl0_reg = 229 ETHOSN_CORE_SEC_REG(core_addr, SEC_SYSCTRL0_REG); 230 const uint32_t sleeping_mask = SEC_SYSCTRL0_SLEEPING; 231 232 return ((mmio_read_32(sysctrl0_reg) & sleeping_mask) == sleeping_mask); 233 } 234 235 static bool ethosn_core_reset(uintptr_t core_addr, bool hard_reset) 236 { 237 unsigned int timeout; 238 const uintptr_t sysctrl0_reg = 239 ETHOSN_CORE_SEC_REG(core_addr, SEC_SYSCTRL0_REG); 240 const uint32_t reset_val = hard_reset ? SEC_SYSCTRL0_HARD_RESET : 241 SEC_SYSCTRL0_SOFT_RESET; 242 243 mmio_write_32(sysctrl0_reg, reset_val); 244 245 /* Wait for reset to complete */ 246 for (timeout = 0U; timeout < ETHOSN_RESET_TIMEOUT_US; 247 timeout += ETHOSN_RESET_WAIT_US) { 248 249 if ((mmio_read_32(sysctrl0_reg) & reset_val) == 0U) { 250 break; 251 } 252 253 udelay(ETHOSN_RESET_WAIT_US); 254 } 255 256 return timeout < ETHOSN_RESET_TIMEOUT_US; 257 } 258 259 static int ethosn_core_full_reset(const struct ethosn_device_t *device, 260 const struct ethosn_core_t *core, 261 bool hard_reset, 262 u_register_t asset_alloc_idx, 263 u_register_t is_protected, 264 u_register_t aux_features) 265 { 266 if (!device->has_reserved_memory && 267 asset_alloc_idx >= device->num_allocators) { 268 WARN("ETHOSN: Unknown asset allocator index given to SMC call.\n"); 269 return ETHOSN_UNKNOWN_ALLOCATOR_IDX; 270 } 271 272 if (!ethosn_core_reset(core->addr, hard_reset)) { 273 return ETHOSN_FAILURE; 274 } 275 276 if (!ethosn_configure_aux_features(device, core->addr, aux_features)) { 277 return ETHOSN_INVALID_CONFIGURATION; 278 } 279 280 ethosn_configure_events(core->addr); 281 282 if (!device->has_reserved_memory) { 283 ethosn_configure_smmu_streams(device, core, asset_alloc_idx); 284 285 #if ARM_ETHOSN_NPU_TZMP1 286 ethosn_configure_stream_nsaid(core, is_protected); 287 #endif 288 } 289 290 ethosn_delegate_to_ns(core->addr); 291 292 return ETHOSN_SUCCESS; 293 } 294 295 static uintptr_t ethosn_smc_core_reset_handler(const struct ethosn_device_t *device, 296 const struct ethosn_core_t *core, 297 bool hard_reset, 298 u_register_t asset_alloc_idx, 299 u_register_t reset_type, 300 u_register_t is_protected, 301 u_register_t aux_features, 302 void *handle) 303 { 304 int ret; 305 306 switch (reset_type) { 307 case ETHOSN_RESET_TYPE_FULL: 308 ret = ethosn_core_full_reset(device, core, hard_reset, 309 asset_alloc_idx, is_protected, 310 aux_features); 311 break; 312 case ETHOSN_RESET_TYPE_HALT: 313 ret = ethosn_core_reset(core->addr, hard_reset) ? ETHOSN_SUCCESS : ETHOSN_FAILURE; 314 break; 315 default: 316 WARN("ETHOSN: Invalid reset type given to SMC call.\n"); 317 ret = ETHOSN_INVALID_PARAMETER; 318 break; 319 } 320 321 SMC_RET1(handle, ret); 322 } 323 324 static uintptr_t ethosn_smc_core_handler(uint32_t fid, 325 u_register_t core_addr, 326 u_register_t asset_alloc_idx, 327 u_register_t reset_type, 328 u_register_t is_protected, 329 u_register_t aux_features, 330 void *handle) 331 { 332 bool hard_reset = false; 333 const struct ethosn_device_t *device = NULL; 334 const struct ethosn_core_t *core = NULL; 335 336 if (!ethosn_get_device_and_core(core_addr, &device, &core)) { 337 SMC_RET1(handle, ETHOSN_UNKNOWN_CORE_ADDRESS); 338 } 339 340 switch (fid) { 341 case ETHOSN_FNUM_IS_SEC: 342 SMC_RET1(handle, ethosn_is_sec(core->addr)); 343 case ETHOSN_FNUM_IS_SLEEPING: 344 SMC_RET1(handle, ethosn_core_is_sleeping(core->addr)); 345 case ETHOSN_FNUM_HARD_RESET: 346 hard_reset = true; 347 /* Fallthrough */ 348 case ETHOSN_FNUM_SOFT_RESET: 349 return ethosn_smc_core_reset_handler(device, core, 350 hard_reset, 351 asset_alloc_idx, 352 reset_type, 353 is_protected, 354 aux_features, 355 handle); 356 default: 357 WARN("ETHOSN: Unimplemented SMC call: 0x%x\n", fid); 358 SMC_RET1(handle, SMC_UNK); 359 } 360 } 361 362 static uintptr_t ethosn_smc_fw_prop_handler(u_register_t fw_property, 363 void *handle) 364 { 365 #if ARM_ETHOSN_NPU_TZMP1 366 switch (fw_property) { 367 case ETHOSN_FW_PROP_VERSION: 368 SMC_RET4(handle, ETHOSN_SUCCESS, 369 big_fw->fw_ver_major, 370 big_fw->fw_ver_minor, 371 big_fw->fw_ver_patch); 372 case ETHOSN_FW_PROP_MEM_INFO: 373 SMC_RET3(handle, ETHOSN_SUCCESS, 374 ((void *)big_fw) + big_fw->offset, 375 big_fw->size); 376 case ETHOSN_FW_PROP_OFFSETS: 377 SMC_RET3(handle, ETHOSN_SUCCESS, 378 big_fw->ple_offset, 379 big_fw->unpriv_stack_offset); 380 case ETHOSN_FW_PROP_VA_MAP: 381 SMC_RET4(handle, ETHOSN_SUCCESS, 382 ETHOSN_FW_VA_BASE, 383 ETHOSN_WORKING_DATA_VA_BASE, 384 ETHOSN_COMMAND_STREAM_VA_BASE); 385 default: 386 WARN("ETHOSN: Unknown firmware property\n"); 387 SMC_RET1(handle, ETHOSN_INVALID_PARAMETER); 388 } 389 #else 390 SMC_RET1(handle, ETHOSN_NOT_SUPPORTED); 391 #endif 392 } 393 394 uintptr_t ethosn_smc_handler(uint32_t smc_fid, 395 u_register_t x1, 396 u_register_t x2, 397 u_register_t x3, 398 u_register_t x4, 399 void *cookie, 400 void *handle, 401 u_register_t flags) 402 { 403 const uint32_t fid = smc_fid & FUNCID_NUM_MASK; 404 405 /* Only SiP fast calls are expected */ 406 if ((GET_SMC_TYPE(smc_fid) != SMC_TYPE_FAST) || 407 (GET_SMC_OEN(smc_fid) != OEN_SIP_START)) { 408 SMC_RET1(handle, SMC_UNK); 409 } 410 411 /* Truncate parameters to 32-bits for SMC32 */ 412 if (GET_SMC_CC(smc_fid) == SMC_32) { 413 x1 &= 0xFFFFFFFF; 414 x2 &= 0xFFFFFFFF; 415 x3 &= 0xFFFFFFFF; 416 x4 &= 0xFFFFFFFF; 417 } 418 419 if (!is_ethosn_fid(smc_fid) || (fid > ETHOSN_FNUM_GET_FW_PROP)) { 420 WARN("ETHOSN: Unknown SMC call: 0x%x\n", smc_fid); 421 SMC_RET1(handle, SMC_UNK); 422 } 423 424 switch (fid) { 425 case ETHOSN_FNUM_VERSION: 426 SMC_RET2(handle, ETHOSN_VERSION_MAJOR, ETHOSN_VERSION_MINOR); 427 case ETHOSN_FNUM_GET_FW_PROP: 428 return ethosn_smc_fw_prop_handler(x1, handle); 429 } 430 431 return ethosn_smc_core_handler(fid, x1, x2, x3, x4, 432 SMC_GET_GP(handle, CTX_GPREG_X5), 433 handle); 434 } 435 436 int ethosn_smc_setup(void) 437 { 438 #if ARM_ETHOSN_NPU_TZMP1 439 struct ethosn_device_t *dev; 440 uint32_t arch_ver; 441 #endif 442 443 if (ETHOSN_NUM_DEVICES == 0U) { 444 ERROR("ETHOSN: No NPU found\n"); 445 return ETHOSN_FAILURE; 446 } 447 448 #if ARM_ETHOSN_NPU_TZMP1 449 450 /* Only one NPU core is supported in the TZMP1 setup */ 451 if ((ETHOSN_NUM_DEVICES != 1U) || 452 (ETHOSN_GET_DEVICE(0U)->num_cores != 1U)) { 453 ERROR("ETHOSN: TZMP1 doesn't support multiple NPU cores\n"); 454 return ETHOSN_FAILURE; 455 } 456 457 dev = ETHOSN_GET_DEVICE(0U); 458 arch_ver = ethosn_core_read_arch_version(dev->cores[0U].addr); 459 big_fw = (struct ethosn_big_fw *)ARM_ETHOSN_NPU_FW_IMAGE_BASE; 460 461 if (!ethosn_big_fw_verify_header(big_fw, arch_ver)) { 462 return ETHOSN_FAILURE; 463 } 464 465 NOTICE("ETHOSN: TZMP1 setup succeeded with firmware version %u.%u.%u\n", 466 big_fw->fw_ver_major, big_fw->fw_ver_minor, 467 big_fw->fw_ver_patch); 468 #else 469 NOTICE("ETHOSN: Setup succeeded\n"); 470 #endif 471 472 return 0; 473 } 474