1 /* 2 * Copyright (c) 2021-2023, Arm Limited. All rights reserved. 3 * 4 * SPDX-License-Identifier: BSD-3-Clause 5 */ 6 7 #include <stdint.h> 8 #include <stdbool.h> 9 10 #include <common/debug.h> 11 #include <common/runtime_svc.h> 12 #include <drivers/arm/ethosn.h> 13 #include <drivers/delay_timer.h> 14 #include <lib/mmio.h> 15 #include <lib/utils_def.h> 16 #include <plat/arm/common/fconf_ethosn_getter.h> 17 18 #include <platform_def.h> 19 20 /* 21 * Number of Arm(R) Ethos(TM)-N NPU (NPU) devices available 22 */ 23 #define ETHOSN_NUM_DEVICES \ 24 FCONF_GET_PROPERTY(hw_config, ethosn_config, num_devices) 25 26 #define ETHOSN_GET_DEVICE(dev_idx) \ 27 FCONF_GET_PROPERTY(hw_config, ethosn_device, dev_idx) 28 29 /* NPU core sec registry address */ 30 #define ETHOSN_CORE_SEC_REG(core_addr, reg_offset) \ 31 (core_addr + reg_offset) 32 33 /* Reset timeout in us */ 34 #define ETHOSN_RESET_TIMEOUT_US U(10 * 1000 * 1000) 35 #define ETHOSN_RESET_WAIT_US U(1) 36 37 #define SEC_DEL_REG U(0x0004) 38 #define SEC_DEL_VAL U(0x81C) 39 #define SEC_DEL_EXCC_MASK U(0x20) 40 41 #define SEC_SECCTLR_REG U(0x0010) 42 #define SEC_SECCTLR_VAL U(0x3) 43 44 #define SEC_DEL_ADDR_EXT_REG U(0x201C) 45 #define SEC_DEL_ADDR_EXT_VAL U(0x15) 46 47 #define SEC_SYSCTRL0_REG U(0x0018) 48 #define SEC_SYSCTRL0_SLEEPING U(1U << 4) 49 #define SEC_SYSCTRL0_SOFT_RESET U(3U << 29) 50 #define SEC_SYSCTRL0_HARD_RESET U(1U << 31) 51 52 #define SEC_NSAID_REG_BASE U(0x3004) 53 #define SEC_NSAID_OFFSET U(0x1000) 54 55 #define SEC_MMUSID_REG_BASE U(0x3008) 56 #define SEC_MMUSID_OFFSET U(0x1000) 57 58 #define INPUT_STREAM_INDEX U(0x6) 59 #define INTERMEDIATE_STREAM_INDEX U(0x7) 60 #define OUTPUT_STREAM_INDEX U(0x8) 61 62 static bool ethosn_get_device_and_core(uintptr_t core_addr, 63 const struct ethosn_device_t **dev_match, 64 const struct ethosn_core_t **core_match) 65 { 66 uint32_t dev_idx; 67 uint32_t core_idx; 68 69 for (dev_idx = 0U; dev_idx < ETHOSN_NUM_DEVICES; ++dev_idx) { 70 const struct ethosn_device_t *dev = ETHOSN_GET_DEVICE(dev_idx); 71 72 for (core_idx = 0U; core_idx < dev->num_cores; ++core_idx) { 73 const struct ethosn_core_t *core = &(dev->cores[core_idx]); 74 75 if (core->addr == core_addr) { 76 *dev_match = dev; 77 *core_match = core; 78 return true; 79 } 80 } 81 } 82 83 WARN("ETHOSN: Unknown core address given to SMC call.\n"); 84 return false; 85 } 86 87 #if ARM_ETHOSN_NPU_TZMP1 88 static void ethosn_configure_stream_nsaid(const struct ethosn_core_t *core, 89 bool is_protected) 90 { 91 size_t i; 92 uint32_t streams[9] = {0, 0, 0, 0, 0, 0, 0, 0, 0}; 93 94 if (is_protected) { 95 streams[INPUT_STREAM_INDEX] = ARM_ETHOSN_NPU_PROT_DATA_NSAID; 96 streams[INTERMEDIATE_STREAM_INDEX] = 97 ARM_ETHOSN_NPU_PROT_DATA_NSAID; 98 streams[OUTPUT_STREAM_INDEX] = ARM_ETHOSN_NPU_PROT_DATA_NSAID; 99 } 100 101 for (i = 0U; i < ARRAY_SIZE(streams); ++i) { 102 const uintptr_t reg_addr = SEC_NSAID_REG_BASE + 103 (SEC_NSAID_OFFSET * i); 104 mmio_write_32(ETHOSN_CORE_SEC_REG(core->addr, reg_addr), 105 streams[i]); 106 } 107 } 108 #endif 109 110 static void ethosn_configure_smmu_streams(const struct ethosn_device_t *device, 111 const struct ethosn_core_t *core, 112 uint32_t asset_alloc_idx) 113 { 114 const struct ethosn_main_allocator_t *main_alloc = 115 &(core->main_allocator); 116 const struct ethosn_asset_allocator_t *asset_alloc = 117 &(device->asset_allocators[asset_alloc_idx]); 118 const uint32_t streams[9] = { 119 main_alloc->firmware.stream_id, 120 main_alloc->working_data.stream_id, 121 asset_alloc->command_stream.stream_id, 122 0U, /* Not used*/ 123 main_alloc->firmware.stream_id, 124 asset_alloc->weight_data.stream_id, 125 asset_alloc->buffer_data.stream_id, 126 asset_alloc->intermediate_data.stream_id, 127 asset_alloc->buffer_data.stream_id 128 }; 129 size_t i; 130 131 for (i = 0U; i < ARRAY_SIZE(streams); ++i) { 132 const uintptr_t reg_addr = SEC_MMUSID_REG_BASE + 133 (SEC_MMUSID_OFFSET * i); 134 mmio_write_32(ETHOSN_CORE_SEC_REG(core->addr, reg_addr), 135 streams[i]); 136 } 137 } 138 139 static void ethosn_delegate_to_ns(uintptr_t core_addr) 140 { 141 mmio_setbits_32(ETHOSN_CORE_SEC_REG(core_addr, SEC_SECCTLR_REG), 142 SEC_SECCTLR_VAL); 143 144 mmio_setbits_32(ETHOSN_CORE_SEC_REG(core_addr, SEC_DEL_REG), 145 SEC_DEL_VAL); 146 147 mmio_setbits_32(ETHOSN_CORE_SEC_REG(core_addr, SEC_DEL_ADDR_EXT_REG), 148 SEC_DEL_ADDR_EXT_VAL); 149 } 150 151 static int ethosn_is_sec(uintptr_t core_addr) 152 { 153 if ((mmio_read_32(ETHOSN_CORE_SEC_REG(core_addr, SEC_DEL_REG)) 154 & SEC_DEL_EXCC_MASK) != 0U) { 155 return 0; 156 } 157 158 return 1; 159 } 160 161 static int ethosn_core_is_sleeping(uintptr_t core_addr) 162 { 163 const uintptr_t sysctrl0_reg = 164 ETHOSN_CORE_SEC_REG(core_addr, SEC_SYSCTRL0_REG); 165 const uint32_t sleeping_mask = SEC_SYSCTRL0_SLEEPING; 166 167 return ((mmio_read_32(sysctrl0_reg) & sleeping_mask) == sleeping_mask); 168 } 169 170 static bool ethosn_reset(uintptr_t core_addr, int hard_reset) 171 { 172 unsigned int timeout; 173 const uintptr_t sysctrl0_reg = 174 ETHOSN_CORE_SEC_REG(core_addr, SEC_SYSCTRL0_REG); 175 const uint32_t reset_val = (hard_reset != 0) ? SEC_SYSCTRL0_HARD_RESET 176 : SEC_SYSCTRL0_SOFT_RESET; 177 178 mmio_write_32(sysctrl0_reg, reset_val); 179 180 /* Wait for reset to complete */ 181 for (timeout = 0U; timeout < ETHOSN_RESET_TIMEOUT_US; 182 timeout += ETHOSN_RESET_WAIT_US) { 183 184 if ((mmio_read_32(sysctrl0_reg) & reset_val) == 0U) { 185 break; 186 } 187 188 udelay(ETHOSN_RESET_WAIT_US); 189 } 190 191 return timeout < ETHOSN_RESET_TIMEOUT_US; 192 } 193 194 uintptr_t ethosn_smc_handler(uint32_t smc_fid, 195 u_register_t core_addr, 196 u_register_t asset_alloc_idx, 197 u_register_t reset_type, 198 u_register_t is_protected, 199 void *cookie, 200 void *handle, 201 u_register_t flags) 202 { 203 int hard_reset = 0; 204 const struct ethosn_device_t *device = NULL; 205 const struct ethosn_core_t *core = NULL; 206 const uint32_t fid = smc_fid & FUNCID_NUM_MASK; 207 208 /* Only SiP fast calls are expected */ 209 if ((GET_SMC_TYPE(smc_fid) != SMC_TYPE_FAST) || 210 (GET_SMC_OEN(smc_fid) != OEN_SIP_START)) { 211 SMC_RET1(handle, SMC_UNK); 212 } 213 214 /* Truncate parameters to 32-bits for SMC32 */ 215 if (GET_SMC_CC(smc_fid) == SMC_32) { 216 core_addr &= 0xFFFFFFFF; 217 asset_alloc_idx &= 0xFFFFFFFF; 218 reset_type &= 0xFFFFFFFF; 219 is_protected &= 0xFFFFFFFF; 220 } 221 222 if (!is_ethosn_fid(smc_fid) || (fid > ETHOSN_FNUM_IS_SLEEPING)) { 223 WARN("ETHOSN: Unknown SMC call: 0x%x\n", smc_fid); 224 SMC_RET1(handle, SMC_UNK); 225 } 226 227 /* Commands that do not require a valid core address */ 228 switch (fid) { 229 case ETHOSN_FNUM_VERSION: 230 SMC_RET2(handle, ETHOSN_VERSION_MAJOR, ETHOSN_VERSION_MINOR); 231 } 232 233 if (!ethosn_get_device_and_core(core_addr, &device, &core)) { 234 SMC_RET1(handle, ETHOSN_UNKNOWN_CORE_ADDRESS); 235 } 236 237 /* Commands that require a valid core address */ 238 switch (fid) { 239 case ETHOSN_FNUM_IS_SEC: 240 SMC_RET1(handle, ethosn_is_sec(core->addr)); 241 case ETHOSN_FNUM_IS_SLEEPING: 242 SMC_RET1(handle, ethosn_core_is_sleeping(core->addr)); 243 } 244 245 if (!device->has_reserved_memory && 246 asset_alloc_idx >= device->num_allocators) { 247 WARN("ETHOSN: Unknown asset allocator index given to SMC call.\n"); 248 SMC_RET1(handle, ETHOSN_UNKNOWN_ALLOCATOR_IDX); 249 } 250 251 if (reset_type > ETHOSN_RESET_TYPE_HALT) { 252 WARN("ETHOSN: Invalid reset type given to SMC call.\n"); 253 SMC_RET1(handle, ETHOSN_INVALID_PARAMETER); 254 } 255 256 /* 257 * Commands that require a valid device, reset type, 258 * core and asset allocator 259 */ 260 switch (fid) { 261 case ETHOSN_FNUM_HARD_RESET: 262 hard_reset = 1; 263 /* Fallthrough */ 264 case ETHOSN_FNUM_SOFT_RESET: 265 if (!ethosn_reset(core->addr, hard_reset)) { 266 SMC_RET1(handle, ETHOSN_FAILURE); 267 } 268 269 if (reset_type == ETHOSN_RESET_TYPE_FULL) { 270 if (!device->has_reserved_memory) { 271 ethosn_configure_smmu_streams(device, core, 272 asset_alloc_idx); 273 274 #if ARM_ETHOSN_NPU_TZMP1 275 ethosn_configure_stream_nsaid(core, 276 is_protected); 277 #endif 278 } 279 280 ethosn_delegate_to_ns(core->addr); 281 } 282 SMC_RET1(handle, ETHOSN_SUCCESS); 283 default: 284 WARN("ETHOSN: Unimplemented SMC call: 0x%x\n", fid); 285 SMC_RET1(handle, SMC_UNK); 286 } 287 } 288