1 /* 2 * Copyright (c) 2021-2023, Arm Limited. All rights reserved. 3 * 4 * SPDX-License-Identifier: BSD-3-Clause 5 */ 6 7 #include <stdint.h> 8 #include <stdbool.h> 9 10 #include <common/debug.h> 11 #include <common/runtime_svc.h> 12 #include <drivers/arm/ethosn.h> 13 #include <drivers/delay_timer.h> 14 #include <lib/mmio.h> 15 #include <lib/utils_def.h> 16 #include <plat/arm/common/fconf_ethosn_getter.h> 17 18 #include <platform_def.h> 19 20 #if ARM_ETHOSN_NPU_TZMP1 21 #include "ethosn_big_fw.h" 22 #endif 23 24 /* 25 * Number of Arm(R) Ethos(TM)-N NPU (NPU) devices available 26 */ 27 #define ETHOSN_NUM_DEVICES \ 28 FCONF_GET_PROPERTY(hw_config, ethosn_config, num_devices) 29 30 #define ETHOSN_GET_DEVICE(dev_idx) \ 31 FCONF_GET_PROPERTY(hw_config, ethosn_device, dev_idx) 32 33 /* NPU core sec registry address */ 34 #define ETHOSN_CORE_SEC_REG(core_addr, reg_offset) \ 35 (core_addr + reg_offset) 36 37 /* Reset timeout in us */ 38 #define ETHOSN_RESET_TIMEOUT_US U(10 * 1000 * 1000) 39 #define ETHOSN_RESET_WAIT_US U(1) 40 41 #define SEC_DEL_REG U(0x0004) 42 #define SEC_DEL_VAL U(0x81C) 43 #define SEC_DEL_EXCC_MASK U(0x20) 44 45 #define SEC_SECCTLR_REG U(0x0010) 46 /* Set bit[10] = 1 to workaround erratum 2838783 */ 47 #define SEC_SECCTLR_VAL U(0x403) 48 49 #define SEC_DEL_ADDR_EXT_REG U(0x201C) 50 #define SEC_DEL_ADDR_EXT_VAL U(0x15) 51 52 #define SEC_SYSCTRL0_REG U(0x0018) 53 #define SEC_SYSCTRL0_SLEEPING U(1U << 4) 54 #define SEC_SYSCTRL0_SOFT_RESET U(3U << 29) 55 #define SEC_SYSCTRL0_HARD_RESET U(1U << 31) 56 57 #define SEC_NSAID_REG_BASE U(0x3004) 58 #define SEC_NSAID_OFFSET U(0x1000) 59 60 #define SEC_MMUSID_REG_BASE U(0x3008) 61 #define SEC_MMUSID_OFFSET U(0x1000) 62 63 #define SEC_NPU_ID_REG U(0xF000) 64 #define SEC_NPU_ID_ARCH_VER_SHIFT U(0X10) 65 66 #define INPUT_STREAM_INDEX U(0x6) 67 #define INTERMEDIATE_STREAM_INDEX U(0x7) 68 #define OUTPUT_STREAM_INDEX U(0x8) 69 70 #if ARM_ETHOSN_NPU_TZMP1 71 CASSERT(ARM_ETHOSN_NPU_FW_IMAGE_BASE > 0U, assert_ethosn_invalid_fw_image_base); 72 static const struct ethosn_big_fw *big_fw; 73 #endif 74 75 static bool ethosn_get_device_and_core(uintptr_t core_addr, 76 const struct ethosn_device_t **dev_match, 77 const struct ethosn_core_t **core_match) 78 { 79 uint32_t dev_idx; 80 uint32_t core_idx; 81 82 for (dev_idx = 0U; dev_idx < ETHOSN_NUM_DEVICES; ++dev_idx) { 83 const struct ethosn_device_t *dev = ETHOSN_GET_DEVICE(dev_idx); 84 85 for (core_idx = 0U; core_idx < dev->num_cores; ++core_idx) { 86 const struct ethosn_core_t *core = &(dev->cores[core_idx]); 87 88 if (core->addr == core_addr) { 89 *dev_match = dev; 90 *core_match = core; 91 return true; 92 } 93 } 94 } 95 96 WARN("ETHOSN: Unknown core address given to SMC call.\n"); 97 return false; 98 } 99 100 #if ARM_ETHOSN_NPU_TZMP1 101 static uint32_t ethosn_core_read_arch_version(uintptr_t core_addr) 102 { 103 uint32_t npu_id = mmio_read_32(ETHOSN_CORE_SEC_REG(core_addr, 104 SEC_NPU_ID_REG)); 105 106 return (npu_id >> SEC_NPU_ID_ARCH_VER_SHIFT); 107 } 108 109 static void ethosn_configure_stream_nsaid(const struct ethosn_core_t *core, 110 bool is_protected) 111 { 112 size_t i; 113 uint32_t streams[9] = {0, 0, 0, 0, 0, 0, 0, 0, 0}; 114 115 if (is_protected) { 116 streams[INPUT_STREAM_INDEX] = ARM_ETHOSN_NPU_PROT_DATA_NSAID; 117 streams[INTERMEDIATE_STREAM_INDEX] = 118 ARM_ETHOSN_NPU_PROT_DATA_NSAID; 119 streams[OUTPUT_STREAM_INDEX] = ARM_ETHOSN_NPU_PROT_DATA_NSAID; 120 } 121 122 for (i = 0U; i < ARRAY_SIZE(streams); ++i) { 123 const uintptr_t reg_addr = SEC_NSAID_REG_BASE + 124 (SEC_NSAID_OFFSET * i); 125 mmio_write_32(ETHOSN_CORE_SEC_REG(core->addr, reg_addr), 126 streams[i]); 127 } 128 } 129 #endif 130 131 static void ethosn_configure_smmu_streams(const struct ethosn_device_t *device, 132 const struct ethosn_core_t *core, 133 uint32_t asset_alloc_idx) 134 { 135 const struct ethosn_main_allocator_t *main_alloc = 136 &(core->main_allocator); 137 const struct ethosn_asset_allocator_t *asset_alloc = 138 &(device->asset_allocators[asset_alloc_idx]); 139 const uint32_t streams[9] = { 140 main_alloc->firmware.stream_id, 141 main_alloc->working_data.stream_id, 142 asset_alloc->command_stream.stream_id, 143 0U, /* Not used*/ 144 main_alloc->firmware.stream_id, 145 asset_alloc->weight_data.stream_id, 146 asset_alloc->buffer_data.stream_id, 147 asset_alloc->intermediate_data.stream_id, 148 asset_alloc->buffer_data.stream_id 149 }; 150 size_t i; 151 152 for (i = 0U; i < ARRAY_SIZE(streams); ++i) { 153 const uintptr_t reg_addr = SEC_MMUSID_REG_BASE + 154 (SEC_MMUSID_OFFSET * i); 155 mmio_write_32(ETHOSN_CORE_SEC_REG(core->addr, reg_addr), 156 streams[i]); 157 } 158 } 159 160 static void ethosn_delegate_to_ns(uintptr_t core_addr) 161 { 162 mmio_setbits_32(ETHOSN_CORE_SEC_REG(core_addr, SEC_SECCTLR_REG), 163 SEC_SECCTLR_VAL); 164 165 mmio_setbits_32(ETHOSN_CORE_SEC_REG(core_addr, SEC_DEL_REG), 166 SEC_DEL_VAL); 167 168 mmio_setbits_32(ETHOSN_CORE_SEC_REG(core_addr, SEC_DEL_ADDR_EXT_REG), 169 SEC_DEL_ADDR_EXT_VAL); 170 } 171 172 static int ethosn_is_sec(uintptr_t core_addr) 173 { 174 if ((mmio_read_32(ETHOSN_CORE_SEC_REG(core_addr, SEC_DEL_REG)) 175 & SEC_DEL_EXCC_MASK) != 0U) { 176 return 0; 177 } 178 179 return 1; 180 } 181 182 static int ethosn_core_is_sleeping(uintptr_t core_addr) 183 { 184 const uintptr_t sysctrl0_reg = 185 ETHOSN_CORE_SEC_REG(core_addr, SEC_SYSCTRL0_REG); 186 const uint32_t sleeping_mask = SEC_SYSCTRL0_SLEEPING; 187 188 return ((mmio_read_32(sysctrl0_reg) & sleeping_mask) == sleeping_mask); 189 } 190 191 static bool ethosn_core_reset(uintptr_t core_addr, bool hard_reset) 192 { 193 unsigned int timeout; 194 const uintptr_t sysctrl0_reg = 195 ETHOSN_CORE_SEC_REG(core_addr, SEC_SYSCTRL0_REG); 196 const uint32_t reset_val = hard_reset ? SEC_SYSCTRL0_HARD_RESET : 197 SEC_SYSCTRL0_SOFT_RESET; 198 199 mmio_write_32(sysctrl0_reg, reset_val); 200 201 /* Wait for reset to complete */ 202 for (timeout = 0U; timeout < ETHOSN_RESET_TIMEOUT_US; 203 timeout += ETHOSN_RESET_WAIT_US) { 204 205 if ((mmio_read_32(sysctrl0_reg) & reset_val) == 0U) { 206 break; 207 } 208 209 udelay(ETHOSN_RESET_WAIT_US); 210 } 211 212 return timeout < ETHOSN_RESET_TIMEOUT_US; 213 } 214 215 static int ethosn_core_full_reset(const struct ethosn_device_t *device, 216 const struct ethosn_core_t *core, 217 bool hard_reset, 218 u_register_t asset_alloc_idx, 219 u_register_t is_protected) 220 { 221 if (!device->has_reserved_memory && 222 asset_alloc_idx >= device->num_allocators) { 223 WARN("ETHOSN: Unknown asset allocator index given to SMC call.\n"); 224 return ETHOSN_UNKNOWN_ALLOCATOR_IDX; 225 } 226 227 if (!ethosn_core_reset(core->addr, hard_reset)) { 228 return ETHOSN_FAILURE; 229 } 230 231 if (!device->has_reserved_memory) { 232 ethosn_configure_smmu_streams(device, core, asset_alloc_idx); 233 234 #if ARM_ETHOSN_NPU_TZMP1 235 ethosn_configure_stream_nsaid(core, is_protected); 236 #endif 237 } 238 239 ethosn_delegate_to_ns(core->addr); 240 241 return ETHOSN_SUCCESS; 242 } 243 244 static uintptr_t ethosn_smc_core_reset_handler(const struct ethosn_device_t *device, 245 const struct ethosn_core_t *core, 246 bool hard_reset, 247 u_register_t asset_alloc_idx, 248 u_register_t reset_type, 249 u_register_t is_protected, 250 void *handle) 251 { 252 int ret; 253 254 switch (reset_type) { 255 case ETHOSN_RESET_TYPE_FULL: 256 ret = ethosn_core_full_reset(device, core, hard_reset, 257 asset_alloc_idx, is_protected); 258 break; 259 case ETHOSN_RESET_TYPE_HALT: 260 ret = ethosn_core_reset(core->addr, hard_reset) ? ETHOSN_SUCCESS : ETHOSN_FAILURE; 261 break; 262 default: 263 WARN("ETHOSN: Invalid reset type given to SMC call.\n"); 264 ret = ETHOSN_INVALID_PARAMETER; 265 break; 266 } 267 268 SMC_RET1(handle, ret); 269 } 270 271 static uintptr_t ethosn_smc_core_handler(uint32_t fid, 272 u_register_t core_addr, 273 u_register_t asset_alloc_idx, 274 u_register_t reset_type, 275 u_register_t is_protected, 276 void *handle) 277 { 278 bool hard_reset = false; 279 const struct ethosn_device_t *device = NULL; 280 const struct ethosn_core_t *core = NULL; 281 282 if (!ethosn_get_device_and_core(core_addr, &device, &core)) { 283 SMC_RET1(handle, ETHOSN_UNKNOWN_CORE_ADDRESS); 284 } 285 286 switch (fid) { 287 case ETHOSN_FNUM_IS_SEC: 288 SMC_RET1(handle, ethosn_is_sec(core->addr)); 289 case ETHOSN_FNUM_IS_SLEEPING: 290 SMC_RET1(handle, ethosn_core_is_sleeping(core->addr)); 291 case ETHOSN_FNUM_HARD_RESET: 292 hard_reset = true; 293 /* Fallthrough */ 294 case ETHOSN_FNUM_SOFT_RESET: 295 return ethosn_smc_core_reset_handler(device, core, 296 hard_reset, 297 asset_alloc_idx, 298 reset_type, 299 is_protected, 300 handle); 301 default: 302 WARN("ETHOSN: Unimplemented SMC call: 0x%x\n", fid); 303 SMC_RET1(handle, SMC_UNK); 304 } 305 } 306 307 uintptr_t ethosn_smc_handler(uint32_t smc_fid, 308 u_register_t x1, 309 u_register_t x2, 310 u_register_t x3, 311 u_register_t x4, 312 void *cookie, 313 void *handle, 314 u_register_t flags) 315 { 316 const uint32_t fid = smc_fid & FUNCID_NUM_MASK; 317 318 /* Only SiP fast calls are expected */ 319 if ((GET_SMC_TYPE(smc_fid) != SMC_TYPE_FAST) || 320 (GET_SMC_OEN(smc_fid) != OEN_SIP_START)) { 321 SMC_RET1(handle, SMC_UNK); 322 } 323 324 /* Truncate parameters to 32-bits for SMC32 */ 325 if (GET_SMC_CC(smc_fid) == SMC_32) { 326 x1 &= 0xFFFFFFFF; 327 x2 &= 0xFFFFFFFF; 328 x3 &= 0xFFFFFFFF; 329 x4 &= 0xFFFFFFFF; 330 } 331 332 if (!is_ethosn_fid(smc_fid) || (fid > ETHOSN_FNUM_IS_SLEEPING)) { 333 WARN("ETHOSN: Unknown SMC call: 0x%x\n", smc_fid); 334 SMC_RET1(handle, SMC_UNK); 335 } 336 337 if (fid == ETHOSN_FNUM_VERSION) { 338 SMC_RET2(handle, ETHOSN_VERSION_MAJOR, ETHOSN_VERSION_MINOR); 339 } 340 341 return ethosn_smc_core_handler(fid, x1, x2, x3, x4, handle); 342 } 343 344 int ethosn_smc_setup(void) 345 { 346 #if ARM_ETHOSN_NPU_TZMP1 347 struct ethosn_device_t *dev; 348 uint32_t arch_ver; 349 #endif 350 351 if (ETHOSN_NUM_DEVICES == 0U) { 352 ERROR("ETHOSN: No NPU found\n"); 353 return ETHOSN_FAILURE; 354 } 355 356 #if ARM_ETHOSN_NPU_TZMP1 357 358 /* Only one NPU core is supported in the TZMP1 setup */ 359 if ((ETHOSN_NUM_DEVICES != 1U) || 360 (ETHOSN_GET_DEVICE(0U)->num_cores != 1U)) { 361 ERROR("ETHOSN: TZMP1 doesn't support multiple NPU cores\n"); 362 return ETHOSN_FAILURE; 363 } 364 365 dev = ETHOSN_GET_DEVICE(0U); 366 arch_ver = ethosn_core_read_arch_version(dev->cores[0U].addr); 367 big_fw = (struct ethosn_big_fw *)ARM_ETHOSN_NPU_FW_IMAGE_BASE; 368 369 if (!ethosn_big_fw_verify_header(big_fw, arch_ver)) { 370 return ETHOSN_FAILURE; 371 } 372 373 NOTICE("ETHOSN: TZMP1 setup succeeded with firmware version %u.%u.%u\n", 374 big_fw->fw_ver_major, big_fw->fw_ver_minor, 375 big_fw->fw_ver_patch); 376 #else 377 NOTICE("ETHOSN: Setup succeeded\n"); 378 #endif 379 380 return 0; 381 } 382