1 /* 2 * Copyright (c) 2021-2023, Arm Limited. All rights reserved. 3 * 4 * SPDX-License-Identifier: BSD-3-Clause 5 */ 6 7 #include <stdint.h> 8 #include <stdbool.h> 9 10 #include <common/debug.h> 11 #include <common/runtime_svc.h> 12 #include <drivers/arm/ethosn.h> 13 #include <drivers/delay_timer.h> 14 #include <lib/mmio.h> 15 #include <lib/utils_def.h> 16 #include <plat/arm/common/fconf_ethosn_getter.h> 17 18 #include <platform_def.h> 19 20 /* 21 * Number of Arm(R) Ethos(TM)-N NPU (NPU) devices available 22 */ 23 #define ETHOSN_NUM_DEVICES \ 24 FCONF_GET_PROPERTY(hw_config, ethosn_config, num_devices) 25 26 #define ETHOSN_GET_DEVICE(dev_idx) \ 27 FCONF_GET_PROPERTY(hw_config, ethosn_device, dev_idx) 28 29 /* NPU core sec registry address */ 30 #define ETHOSN_CORE_SEC_REG(core_addr, reg_offset) \ 31 (core_addr + reg_offset) 32 33 /* Reset timeout in us */ 34 #define ETHOSN_RESET_TIMEOUT_US U(10 * 1000 * 1000) 35 #define ETHOSN_RESET_WAIT_US U(1) 36 37 #define SEC_DEL_REG U(0x0004) 38 #define SEC_DEL_VAL U(0x81C) 39 #define SEC_DEL_EXCC_MASK U(0x20) 40 41 #define SEC_SECCTLR_REG U(0x0010) 42 /* Set bit[10] = 1 to workaround erratum 2838783 */ 43 #define SEC_SECCTLR_VAL U(0x403) 44 45 #define SEC_DEL_ADDR_EXT_REG U(0x201C) 46 #define SEC_DEL_ADDR_EXT_VAL U(0x15) 47 48 #define SEC_SYSCTRL0_REG U(0x0018) 49 #define SEC_SYSCTRL0_SLEEPING U(1U << 4) 50 #define SEC_SYSCTRL0_SOFT_RESET U(3U << 29) 51 #define SEC_SYSCTRL0_HARD_RESET U(1U << 31) 52 53 #define SEC_NSAID_REG_BASE U(0x3004) 54 #define SEC_NSAID_OFFSET U(0x1000) 55 56 #define SEC_MMUSID_REG_BASE U(0x3008) 57 #define SEC_MMUSID_OFFSET U(0x1000) 58 59 #define INPUT_STREAM_INDEX U(0x6) 60 #define INTERMEDIATE_STREAM_INDEX U(0x7) 61 #define OUTPUT_STREAM_INDEX U(0x8) 62 63 static bool ethosn_get_device_and_core(uintptr_t core_addr, 64 const struct ethosn_device_t **dev_match, 65 const struct ethosn_core_t **core_match) 66 { 67 uint32_t dev_idx; 68 uint32_t core_idx; 69 70 for (dev_idx = 0U; dev_idx < ETHOSN_NUM_DEVICES; ++dev_idx) { 71 const struct ethosn_device_t *dev = ETHOSN_GET_DEVICE(dev_idx); 72 73 for (core_idx = 0U; core_idx < dev->num_cores; ++core_idx) { 74 const struct ethosn_core_t *core = &(dev->cores[core_idx]); 75 76 if (core->addr == core_addr) { 77 *dev_match = dev; 78 *core_match = core; 79 return true; 80 } 81 } 82 } 83 84 WARN("ETHOSN: Unknown core address given to SMC call.\n"); 85 return false; 86 } 87 88 #if ARM_ETHOSN_NPU_TZMP1 89 static void ethosn_configure_stream_nsaid(const struct ethosn_core_t *core, 90 bool is_protected) 91 { 92 size_t i; 93 uint32_t streams[9] = {0, 0, 0, 0, 0, 0, 0, 0, 0}; 94 95 if (is_protected) { 96 streams[INPUT_STREAM_INDEX] = ARM_ETHOSN_NPU_PROT_DATA_NSAID; 97 streams[INTERMEDIATE_STREAM_INDEX] = 98 ARM_ETHOSN_NPU_PROT_DATA_NSAID; 99 streams[OUTPUT_STREAM_INDEX] = ARM_ETHOSN_NPU_PROT_DATA_NSAID; 100 } 101 102 for (i = 0U; i < ARRAY_SIZE(streams); ++i) { 103 const uintptr_t reg_addr = SEC_NSAID_REG_BASE + 104 (SEC_NSAID_OFFSET * i); 105 mmio_write_32(ETHOSN_CORE_SEC_REG(core->addr, reg_addr), 106 streams[i]); 107 } 108 } 109 #endif 110 111 static void ethosn_configure_smmu_streams(const struct ethosn_device_t *device, 112 const struct ethosn_core_t *core, 113 uint32_t asset_alloc_idx) 114 { 115 const struct ethosn_main_allocator_t *main_alloc = 116 &(core->main_allocator); 117 const struct ethosn_asset_allocator_t *asset_alloc = 118 &(device->asset_allocators[asset_alloc_idx]); 119 const uint32_t streams[9] = { 120 main_alloc->firmware.stream_id, 121 main_alloc->working_data.stream_id, 122 asset_alloc->command_stream.stream_id, 123 0U, /* Not used*/ 124 main_alloc->firmware.stream_id, 125 asset_alloc->weight_data.stream_id, 126 asset_alloc->buffer_data.stream_id, 127 asset_alloc->intermediate_data.stream_id, 128 asset_alloc->buffer_data.stream_id 129 }; 130 size_t i; 131 132 for (i = 0U; i < ARRAY_SIZE(streams); ++i) { 133 const uintptr_t reg_addr = SEC_MMUSID_REG_BASE + 134 (SEC_MMUSID_OFFSET * i); 135 mmio_write_32(ETHOSN_CORE_SEC_REG(core->addr, reg_addr), 136 streams[i]); 137 } 138 } 139 140 static void ethosn_delegate_to_ns(uintptr_t core_addr) 141 { 142 mmio_setbits_32(ETHOSN_CORE_SEC_REG(core_addr, SEC_SECCTLR_REG), 143 SEC_SECCTLR_VAL); 144 145 mmio_setbits_32(ETHOSN_CORE_SEC_REG(core_addr, SEC_DEL_REG), 146 SEC_DEL_VAL); 147 148 mmio_setbits_32(ETHOSN_CORE_SEC_REG(core_addr, SEC_DEL_ADDR_EXT_REG), 149 SEC_DEL_ADDR_EXT_VAL); 150 } 151 152 static int ethosn_is_sec(uintptr_t core_addr) 153 { 154 if ((mmio_read_32(ETHOSN_CORE_SEC_REG(core_addr, SEC_DEL_REG)) 155 & SEC_DEL_EXCC_MASK) != 0U) { 156 return 0; 157 } 158 159 return 1; 160 } 161 162 static int ethosn_core_is_sleeping(uintptr_t core_addr) 163 { 164 const uintptr_t sysctrl0_reg = 165 ETHOSN_CORE_SEC_REG(core_addr, SEC_SYSCTRL0_REG); 166 const uint32_t sleeping_mask = SEC_SYSCTRL0_SLEEPING; 167 168 return ((mmio_read_32(sysctrl0_reg) & sleeping_mask) == sleeping_mask); 169 } 170 171 static bool ethosn_reset(uintptr_t core_addr, int hard_reset) 172 { 173 unsigned int timeout; 174 const uintptr_t sysctrl0_reg = 175 ETHOSN_CORE_SEC_REG(core_addr, SEC_SYSCTRL0_REG); 176 const uint32_t reset_val = (hard_reset != 0) ? SEC_SYSCTRL0_HARD_RESET 177 : SEC_SYSCTRL0_SOFT_RESET; 178 179 mmio_write_32(sysctrl0_reg, reset_val); 180 181 /* Wait for reset to complete */ 182 for (timeout = 0U; timeout < ETHOSN_RESET_TIMEOUT_US; 183 timeout += ETHOSN_RESET_WAIT_US) { 184 185 if ((mmio_read_32(sysctrl0_reg) & reset_val) == 0U) { 186 break; 187 } 188 189 udelay(ETHOSN_RESET_WAIT_US); 190 } 191 192 return timeout < ETHOSN_RESET_TIMEOUT_US; 193 } 194 195 uintptr_t ethosn_smc_handler(uint32_t smc_fid, 196 u_register_t core_addr, 197 u_register_t asset_alloc_idx, 198 u_register_t reset_type, 199 u_register_t is_protected, 200 void *cookie, 201 void *handle, 202 u_register_t flags) 203 { 204 int hard_reset = 0; 205 const struct ethosn_device_t *device = NULL; 206 const struct ethosn_core_t *core = NULL; 207 const uint32_t fid = smc_fid & FUNCID_NUM_MASK; 208 209 /* Only SiP fast calls are expected */ 210 if ((GET_SMC_TYPE(smc_fid) != SMC_TYPE_FAST) || 211 (GET_SMC_OEN(smc_fid) != OEN_SIP_START)) { 212 SMC_RET1(handle, SMC_UNK); 213 } 214 215 /* Truncate parameters to 32-bits for SMC32 */ 216 if (GET_SMC_CC(smc_fid) == SMC_32) { 217 core_addr &= 0xFFFFFFFF; 218 asset_alloc_idx &= 0xFFFFFFFF; 219 reset_type &= 0xFFFFFFFF; 220 is_protected &= 0xFFFFFFFF; 221 } 222 223 if (!is_ethosn_fid(smc_fid) || (fid > ETHOSN_FNUM_IS_SLEEPING)) { 224 WARN("ETHOSN: Unknown SMC call: 0x%x\n", smc_fid); 225 SMC_RET1(handle, SMC_UNK); 226 } 227 228 /* Commands that do not require a valid core address */ 229 switch (fid) { 230 case ETHOSN_FNUM_VERSION: 231 SMC_RET2(handle, ETHOSN_VERSION_MAJOR, ETHOSN_VERSION_MINOR); 232 } 233 234 if (!ethosn_get_device_and_core(core_addr, &device, &core)) { 235 SMC_RET1(handle, ETHOSN_UNKNOWN_CORE_ADDRESS); 236 } 237 238 /* Commands that require a valid core address */ 239 switch (fid) { 240 case ETHOSN_FNUM_IS_SEC: 241 SMC_RET1(handle, ethosn_is_sec(core->addr)); 242 case ETHOSN_FNUM_IS_SLEEPING: 243 SMC_RET1(handle, ethosn_core_is_sleeping(core->addr)); 244 } 245 246 if (!device->has_reserved_memory && 247 asset_alloc_idx >= device->num_allocators) { 248 WARN("ETHOSN: Unknown asset allocator index given to SMC call.\n"); 249 SMC_RET1(handle, ETHOSN_UNKNOWN_ALLOCATOR_IDX); 250 } 251 252 if (reset_type > ETHOSN_RESET_TYPE_HALT) { 253 WARN("ETHOSN: Invalid reset type given to SMC call.\n"); 254 SMC_RET1(handle, ETHOSN_INVALID_PARAMETER); 255 } 256 257 /* 258 * Commands that require a valid device, reset type, 259 * core and asset allocator 260 */ 261 switch (fid) { 262 case ETHOSN_FNUM_HARD_RESET: 263 hard_reset = 1; 264 /* Fallthrough */ 265 case ETHOSN_FNUM_SOFT_RESET: 266 if (!ethosn_reset(core->addr, hard_reset)) { 267 SMC_RET1(handle, ETHOSN_FAILURE); 268 } 269 270 if (reset_type == ETHOSN_RESET_TYPE_FULL) { 271 if (!device->has_reserved_memory) { 272 ethosn_configure_smmu_streams(device, core, 273 asset_alloc_idx); 274 275 #if ARM_ETHOSN_NPU_TZMP1 276 ethosn_configure_stream_nsaid(core, 277 is_protected); 278 #endif 279 } 280 281 ethosn_delegate_to_ns(core->addr); 282 } 283 SMC_RET1(handle, ETHOSN_SUCCESS); 284 default: 285 WARN("ETHOSN: Unimplemented SMC call: 0x%x\n", fid); 286 SMC_RET1(handle, SMC_UNK); 287 } 288 } 289