1 /* 2 * Copyright (c) 2021-2023, Arm Limited. All rights reserved. 3 * 4 * SPDX-License-Identifier: BSD-3-Clause 5 */ 6 7 #include <stdint.h> 8 #include <stdbool.h> 9 10 #include <common/debug.h> 11 #include <common/runtime_svc.h> 12 #include <drivers/arm/ethosn.h> 13 #include <drivers/delay_timer.h> 14 #include <lib/mmio.h> 15 #include <lib/utils_def.h> 16 #include <plat/arm/common/fconf_ethosn_getter.h> 17 18 #include <platform_def.h> 19 20 #if ARM_ETHOSN_NPU_TZMP1 21 #include "ethosn_big_fw.h" 22 #endif 23 24 /* 25 * Number of Arm(R) Ethos(TM)-N NPU (NPU) devices available 26 */ 27 #define ETHOSN_NUM_DEVICES \ 28 FCONF_GET_PROPERTY(hw_config, ethosn_config, num_devices) 29 30 #define ETHOSN_GET_DEVICE(dev_idx) \ 31 FCONF_GET_PROPERTY(hw_config, ethosn_device, dev_idx) 32 33 /* NPU core sec registry address */ 34 #define ETHOSN_CORE_SEC_REG(core_addr, reg_offset) \ 35 (core_addr + reg_offset) 36 37 #define ETHOSN_FW_VA_BASE 0x20000000UL 38 #define ETHOSN_WORKING_DATA_VA_BASE 0x40000000UL 39 #define ETHOSN_COMMAND_STREAM_VA_BASE 0x60000000UL 40 41 /* Reset timeout in us */ 42 #define ETHOSN_RESET_TIMEOUT_US U(10 * 1000 * 1000) 43 #define ETHOSN_RESET_WAIT_US U(1) 44 45 #define SEC_DEL_REG U(0x0004) 46 #define SEC_DEL_VAL U(0x81C) 47 #define SEC_DEL_EXCC_MASK U(0x20) 48 49 #define SEC_SECCTLR_REG U(0x0010) 50 /* Set bit[10] = 1 to workaround erratum 2838783 */ 51 #define SEC_SECCTLR_VAL U(0x403) 52 53 #define SEC_DEL_ADDR_EXT_REG U(0x201C) 54 #define SEC_DEL_ADDR_EXT_VAL U(0x15) 55 56 #define SEC_SYSCTRL0_REG U(0x0018) 57 #define SEC_SYSCTRL0_SLEEPING U(1U << 4) 58 #define SEC_SYSCTRL0_SOFT_RESET U(3U << 29) 59 #define SEC_SYSCTRL0_HARD_RESET U(1U << 31) 60 61 #define SEC_NSAID_REG_BASE U(0x3004) 62 #define SEC_NSAID_OFFSET U(0x1000) 63 64 #define SEC_MMUSID_REG_BASE U(0x3008) 65 #define SEC_MMUSID_OFFSET U(0x1000) 66 67 #define SEC_NPU_ID_REG U(0xF000) 68 #define SEC_NPU_ID_ARCH_VER_SHIFT U(0X10) 69 70 #define INPUT_STREAM_INDEX U(0x6) 71 #define INTERMEDIATE_STREAM_INDEX U(0x7) 72 #define OUTPUT_STREAM_INDEX U(0x8) 73 74 #if ARM_ETHOSN_NPU_TZMP1 75 CASSERT(ARM_ETHOSN_NPU_FW_IMAGE_BASE > 0U, assert_ethosn_invalid_fw_image_base); 76 static const struct ethosn_big_fw *big_fw; 77 #endif 78 79 static bool ethosn_get_device_and_core(uintptr_t core_addr, 80 const struct ethosn_device_t **dev_match, 81 const struct ethosn_core_t **core_match) 82 { 83 uint32_t dev_idx; 84 uint32_t core_idx; 85 86 for (dev_idx = 0U; dev_idx < ETHOSN_NUM_DEVICES; ++dev_idx) { 87 const struct ethosn_device_t *dev = ETHOSN_GET_DEVICE(dev_idx); 88 89 for (core_idx = 0U; core_idx < dev->num_cores; ++core_idx) { 90 const struct ethosn_core_t *core = &(dev->cores[core_idx]); 91 92 if (core->addr == core_addr) { 93 *dev_match = dev; 94 *core_match = core; 95 return true; 96 } 97 } 98 } 99 100 WARN("ETHOSN: Unknown core address given to SMC call.\n"); 101 return false; 102 } 103 104 #if ARM_ETHOSN_NPU_TZMP1 105 static uint32_t ethosn_core_read_arch_version(uintptr_t core_addr) 106 { 107 uint32_t npu_id = mmio_read_32(ETHOSN_CORE_SEC_REG(core_addr, 108 SEC_NPU_ID_REG)); 109 110 return (npu_id >> SEC_NPU_ID_ARCH_VER_SHIFT); 111 } 112 113 static void ethosn_configure_stream_nsaid(const struct ethosn_core_t *core, 114 bool is_protected) 115 { 116 size_t i; 117 uint32_t streams[9] = {0, 0, 0, 0, 0, 0, 0, 0, 0}; 118 119 if (is_protected) { 120 streams[INPUT_STREAM_INDEX] = ARM_ETHOSN_NPU_PROT_DATA_NSAID; 121 streams[INTERMEDIATE_STREAM_INDEX] = 122 ARM_ETHOSN_NPU_PROT_DATA_NSAID; 123 streams[OUTPUT_STREAM_INDEX] = ARM_ETHOSN_NPU_PROT_DATA_NSAID; 124 } 125 126 for (i = 0U; i < ARRAY_SIZE(streams); ++i) { 127 const uintptr_t reg_addr = SEC_NSAID_REG_BASE + 128 (SEC_NSAID_OFFSET * i); 129 mmio_write_32(ETHOSN_CORE_SEC_REG(core->addr, reg_addr), 130 streams[i]); 131 } 132 } 133 #endif 134 135 static void ethosn_configure_smmu_streams(const struct ethosn_device_t *device, 136 const struct ethosn_core_t *core, 137 uint32_t asset_alloc_idx) 138 { 139 const struct ethosn_main_allocator_t *main_alloc = 140 &(core->main_allocator); 141 const struct ethosn_asset_allocator_t *asset_alloc = 142 &(device->asset_allocators[asset_alloc_idx]); 143 const uint32_t streams[9] = { 144 main_alloc->firmware.stream_id, 145 main_alloc->working_data.stream_id, 146 asset_alloc->command_stream.stream_id, 147 0U, /* Not used*/ 148 main_alloc->firmware.stream_id, 149 asset_alloc->weight_data.stream_id, 150 asset_alloc->buffer_data.stream_id, 151 asset_alloc->intermediate_data.stream_id, 152 asset_alloc->buffer_data.stream_id 153 }; 154 size_t i; 155 156 for (i = 0U; i < ARRAY_SIZE(streams); ++i) { 157 const uintptr_t reg_addr = SEC_MMUSID_REG_BASE + 158 (SEC_MMUSID_OFFSET * i); 159 mmio_write_32(ETHOSN_CORE_SEC_REG(core->addr, reg_addr), 160 streams[i]); 161 } 162 } 163 164 static void ethosn_delegate_to_ns(uintptr_t core_addr) 165 { 166 mmio_setbits_32(ETHOSN_CORE_SEC_REG(core_addr, SEC_SECCTLR_REG), 167 SEC_SECCTLR_VAL); 168 169 mmio_setbits_32(ETHOSN_CORE_SEC_REG(core_addr, SEC_DEL_REG), 170 SEC_DEL_VAL); 171 172 mmio_setbits_32(ETHOSN_CORE_SEC_REG(core_addr, SEC_DEL_ADDR_EXT_REG), 173 SEC_DEL_ADDR_EXT_VAL); 174 } 175 176 static int ethosn_is_sec(uintptr_t core_addr) 177 { 178 if ((mmio_read_32(ETHOSN_CORE_SEC_REG(core_addr, SEC_DEL_REG)) 179 & SEC_DEL_EXCC_MASK) != 0U) { 180 return 0; 181 } 182 183 return 1; 184 } 185 186 static int ethosn_core_is_sleeping(uintptr_t core_addr) 187 { 188 const uintptr_t sysctrl0_reg = 189 ETHOSN_CORE_SEC_REG(core_addr, SEC_SYSCTRL0_REG); 190 const uint32_t sleeping_mask = SEC_SYSCTRL0_SLEEPING; 191 192 return ((mmio_read_32(sysctrl0_reg) & sleeping_mask) == sleeping_mask); 193 } 194 195 static bool ethosn_core_reset(uintptr_t core_addr, bool hard_reset) 196 { 197 unsigned int timeout; 198 const uintptr_t sysctrl0_reg = 199 ETHOSN_CORE_SEC_REG(core_addr, SEC_SYSCTRL0_REG); 200 const uint32_t reset_val = hard_reset ? SEC_SYSCTRL0_HARD_RESET : 201 SEC_SYSCTRL0_SOFT_RESET; 202 203 mmio_write_32(sysctrl0_reg, reset_val); 204 205 /* Wait for reset to complete */ 206 for (timeout = 0U; timeout < ETHOSN_RESET_TIMEOUT_US; 207 timeout += ETHOSN_RESET_WAIT_US) { 208 209 if ((mmio_read_32(sysctrl0_reg) & reset_val) == 0U) { 210 break; 211 } 212 213 udelay(ETHOSN_RESET_WAIT_US); 214 } 215 216 return timeout < ETHOSN_RESET_TIMEOUT_US; 217 } 218 219 static int ethosn_core_full_reset(const struct ethosn_device_t *device, 220 const struct ethosn_core_t *core, 221 bool hard_reset, 222 u_register_t asset_alloc_idx, 223 u_register_t is_protected) 224 { 225 if (!device->has_reserved_memory && 226 asset_alloc_idx >= device->num_allocators) { 227 WARN("ETHOSN: Unknown asset allocator index given to SMC call.\n"); 228 return ETHOSN_UNKNOWN_ALLOCATOR_IDX; 229 } 230 231 if (!ethosn_core_reset(core->addr, hard_reset)) { 232 return ETHOSN_FAILURE; 233 } 234 235 if (!device->has_reserved_memory) { 236 ethosn_configure_smmu_streams(device, core, asset_alloc_idx); 237 238 #if ARM_ETHOSN_NPU_TZMP1 239 ethosn_configure_stream_nsaid(core, is_protected); 240 #endif 241 } 242 243 ethosn_delegate_to_ns(core->addr); 244 245 return ETHOSN_SUCCESS; 246 } 247 248 static uintptr_t ethosn_smc_core_reset_handler(const struct ethosn_device_t *device, 249 const struct ethosn_core_t *core, 250 bool hard_reset, 251 u_register_t asset_alloc_idx, 252 u_register_t reset_type, 253 u_register_t is_protected, 254 void *handle) 255 { 256 int ret; 257 258 switch (reset_type) { 259 case ETHOSN_RESET_TYPE_FULL: 260 ret = ethosn_core_full_reset(device, core, hard_reset, 261 asset_alloc_idx, is_protected); 262 break; 263 case ETHOSN_RESET_TYPE_HALT: 264 ret = ethosn_core_reset(core->addr, hard_reset) ? ETHOSN_SUCCESS : ETHOSN_FAILURE; 265 break; 266 default: 267 WARN("ETHOSN: Invalid reset type given to SMC call.\n"); 268 ret = ETHOSN_INVALID_PARAMETER; 269 break; 270 } 271 272 SMC_RET1(handle, ret); 273 } 274 275 static uintptr_t ethosn_smc_core_handler(uint32_t fid, 276 u_register_t core_addr, 277 u_register_t asset_alloc_idx, 278 u_register_t reset_type, 279 u_register_t is_protected, 280 void *handle) 281 { 282 bool hard_reset = false; 283 const struct ethosn_device_t *device = NULL; 284 const struct ethosn_core_t *core = NULL; 285 286 if (!ethosn_get_device_and_core(core_addr, &device, &core)) { 287 SMC_RET1(handle, ETHOSN_UNKNOWN_CORE_ADDRESS); 288 } 289 290 switch (fid) { 291 case ETHOSN_FNUM_IS_SEC: 292 SMC_RET1(handle, ethosn_is_sec(core->addr)); 293 case ETHOSN_FNUM_IS_SLEEPING: 294 SMC_RET1(handle, ethosn_core_is_sleeping(core->addr)); 295 case ETHOSN_FNUM_HARD_RESET: 296 hard_reset = true; 297 /* Fallthrough */ 298 case ETHOSN_FNUM_SOFT_RESET: 299 return ethosn_smc_core_reset_handler(device, core, 300 hard_reset, 301 asset_alloc_idx, 302 reset_type, 303 is_protected, 304 handle); 305 default: 306 WARN("ETHOSN: Unimplemented SMC call: 0x%x\n", fid); 307 SMC_RET1(handle, SMC_UNK); 308 } 309 } 310 311 static uintptr_t ethosn_smc_fw_prop_handler(u_register_t fw_property, 312 void *handle) 313 { 314 #if ARM_ETHOSN_NPU_TZMP1 315 switch (fw_property) { 316 case ETHOSN_FW_PROP_VERSION: 317 SMC_RET4(handle, ETHOSN_SUCCESS, 318 big_fw->fw_ver_major, 319 big_fw->fw_ver_minor, 320 big_fw->fw_ver_patch); 321 case ETHOSN_FW_PROP_MEM_INFO: 322 SMC_RET3(handle, ETHOSN_SUCCESS, 323 ((void *)big_fw) + big_fw->offset, 324 big_fw->size); 325 case ETHOSN_FW_PROP_OFFSETS: 326 SMC_RET3(handle, ETHOSN_SUCCESS, 327 big_fw->ple_offset, 328 big_fw->unpriv_stack_offset); 329 case ETHOSN_FW_PROP_VA_MAP: 330 SMC_RET4(handle, ETHOSN_SUCCESS, 331 ETHOSN_FW_VA_BASE, 332 ETHOSN_WORKING_DATA_VA_BASE, 333 ETHOSN_COMMAND_STREAM_VA_BASE); 334 default: 335 WARN("ETHOSN: Unknown firmware property\n"); 336 SMC_RET1(handle, ETHOSN_INVALID_PARAMETER); 337 } 338 #else 339 SMC_RET1(handle, ETHOSN_NOT_SUPPORTED); 340 #endif 341 } 342 343 uintptr_t ethosn_smc_handler(uint32_t smc_fid, 344 u_register_t x1, 345 u_register_t x2, 346 u_register_t x3, 347 u_register_t x4, 348 void *cookie, 349 void *handle, 350 u_register_t flags) 351 { 352 const uint32_t fid = smc_fid & FUNCID_NUM_MASK; 353 354 /* Only SiP fast calls are expected */ 355 if ((GET_SMC_TYPE(smc_fid) != SMC_TYPE_FAST) || 356 (GET_SMC_OEN(smc_fid) != OEN_SIP_START)) { 357 SMC_RET1(handle, SMC_UNK); 358 } 359 360 /* Truncate parameters to 32-bits for SMC32 */ 361 if (GET_SMC_CC(smc_fid) == SMC_32) { 362 x1 &= 0xFFFFFFFF; 363 x2 &= 0xFFFFFFFF; 364 x3 &= 0xFFFFFFFF; 365 x4 &= 0xFFFFFFFF; 366 } 367 368 if (!is_ethosn_fid(smc_fid) || (fid > ETHOSN_FNUM_GET_FW_PROP)) { 369 WARN("ETHOSN: Unknown SMC call: 0x%x\n", smc_fid); 370 SMC_RET1(handle, SMC_UNK); 371 } 372 373 switch (fid) { 374 case ETHOSN_FNUM_VERSION: 375 SMC_RET2(handle, ETHOSN_VERSION_MAJOR, ETHOSN_VERSION_MINOR); 376 case ETHOSN_FNUM_GET_FW_PROP: 377 return ethosn_smc_fw_prop_handler(x1, handle); 378 } 379 380 return ethosn_smc_core_handler(fid, x1, x2, x3, x4, handle); 381 } 382 383 int ethosn_smc_setup(void) 384 { 385 #if ARM_ETHOSN_NPU_TZMP1 386 struct ethosn_device_t *dev; 387 uint32_t arch_ver; 388 #endif 389 390 if (ETHOSN_NUM_DEVICES == 0U) { 391 ERROR("ETHOSN: No NPU found\n"); 392 return ETHOSN_FAILURE; 393 } 394 395 #if ARM_ETHOSN_NPU_TZMP1 396 397 /* Only one NPU core is supported in the TZMP1 setup */ 398 if ((ETHOSN_NUM_DEVICES != 1U) || 399 (ETHOSN_GET_DEVICE(0U)->num_cores != 1U)) { 400 ERROR("ETHOSN: TZMP1 doesn't support multiple NPU cores\n"); 401 return ETHOSN_FAILURE; 402 } 403 404 dev = ETHOSN_GET_DEVICE(0U); 405 arch_ver = ethosn_core_read_arch_version(dev->cores[0U].addr); 406 big_fw = (struct ethosn_big_fw *)ARM_ETHOSN_NPU_FW_IMAGE_BASE; 407 408 if (!ethosn_big_fw_verify_header(big_fw, arch_ver)) { 409 return ETHOSN_FAILURE; 410 } 411 412 NOTICE("ETHOSN: TZMP1 setup succeeded with firmware version %u.%u.%u\n", 413 big_fw->fw_ver_major, big_fw->fw_ver_minor, 414 big_fw->fw_ver_patch); 415 #else 416 NOTICE("ETHOSN: Setup succeeded\n"); 417 #endif 418 419 return 0; 420 } 421