1 /* 2 * Copyright (c) 2021-2024, Arm Limited and Contributors. All rights reserved. 3 * 4 * SPDX-License-Identifier: BSD-3-Clause 5 */ 6 7 #include <assert.h> 8 #include <errno.h> 9 #include <inttypes.h> 10 #include <stdint.h> 11 #include <string.h> 12 13 #include <arch_helpers.h> 14 #include <arch_features.h> 15 #include <bl31/bl31.h> 16 #include <common/debug.h> 17 #include <common/runtime_svc.h> 18 #include <context.h> 19 #include <lib/el3_runtime/context_mgmt.h> 20 #include <lib/el3_runtime/cpu_data.h> 21 #include <lib/el3_runtime/pubsub.h> 22 #include <lib/extensions/pmuv3.h> 23 #include <lib/extensions/sys_reg_trace.h> 24 #include <lib/gpt_rme/gpt_rme.h> 25 26 #include <lib/spinlock.h> 27 #include <lib/utils.h> 28 #include <lib/xlat_tables/xlat_tables_v2.h> 29 #include <plat/common/common_def.h> 30 #include <plat/common/platform.h> 31 #include <platform_def.h> 32 #include <services/rmmd_svc.h> 33 #include <smccc_helpers.h> 34 #include <lib/extensions/sme.h> 35 #include <lib/extensions/sve.h> 36 #include "rmmd_initial_context.h" 37 #include "rmmd_private.h" 38 39 /******************************************************************************* 40 * RMM boot failure flag 41 ******************************************************************************/ 42 static bool rmm_boot_failed; 43 44 /******************************************************************************* 45 * RMM context information. 46 ******************************************************************************/ 47 rmmd_rmm_context_t rmm_context[PLATFORM_CORE_COUNT]; 48 49 /******************************************************************************* 50 * RMM entry point information. Discovered on the primary core and reused 51 * on secondary cores. 52 ******************************************************************************/ 53 static entry_point_info_t *rmm_ep_info; 54 55 /******************************************************************************* 56 * Static function declaration. 57 ******************************************************************************/ 58 static int32_t rmm_init(void); 59 60 /******************************************************************************* 61 * This function takes an RMM context pointer and performs a synchronous entry 62 * into it. 63 ******************************************************************************/ 64 uint64_t rmmd_rmm_sync_entry(rmmd_rmm_context_t *rmm_ctx) 65 { 66 uint64_t rc; 67 68 assert(rmm_ctx != NULL); 69 70 cm_set_context(&(rmm_ctx->cpu_ctx), REALM); 71 72 /* Restore the realm context assigned above */ 73 cm_el2_sysregs_context_restore(REALM); 74 cm_set_next_eret_context(REALM); 75 76 /* Enter RMM */ 77 rc = rmmd_rmm_enter(&rmm_ctx->c_rt_ctx); 78 79 /* 80 * Save realm context. EL2 Non-secure context will be restored 81 * before exiting Non-secure world, therefore there is no need 82 * to clear EL2 context registers. 83 */ 84 cm_el2_sysregs_context_save(REALM); 85 86 return rc; 87 } 88 89 /******************************************************************************* 90 * This function returns to the place where rmmd_rmm_sync_entry() was 91 * called originally. 92 ******************************************************************************/ 93 __dead2 void rmmd_rmm_sync_exit(uint64_t rc) 94 { 95 rmmd_rmm_context_t *ctx = &rmm_context[plat_my_core_pos()]; 96 97 /* Get context of the RMM in use by this CPU. */ 98 assert(cm_get_context(REALM) == &(ctx->cpu_ctx)); 99 100 /* 101 * The RMMD must have initiated the original request through a 102 * synchronous entry into RMM. Jump back to the original C runtime 103 * context with the value of rc in x0; 104 */ 105 rmmd_rmm_exit(ctx->c_rt_ctx, rc); 106 107 panic(); 108 } 109 110 static void rmm_el2_context_init(el2_sysregs_t *regs) 111 { 112 write_el2_ctx_common(regs, spsr_el2, REALM_SPSR_EL2); 113 write_el2_ctx_common(regs, sctlr_el2, SCTLR_EL2_RES1); 114 } 115 116 /******************************************************************************* 117 * Enable architecture extensions on first entry to Realm world. 118 ******************************************************************************/ 119 120 static void manage_extensions_realm(cpu_context_t *ctx) 121 { 122 pmuv3_enable(ctx); 123 124 /* 125 * Enable access to TPIDR2_EL0 if SME/SME2 is enabled for Non Secure world. 126 */ 127 if (is_feat_sme_supported()) { 128 sme_enable(ctx); 129 } 130 } 131 132 static void manage_extensions_realm_per_world(void) 133 { 134 cm_el3_arch_init_per_world(&per_world_context[CPU_CONTEXT_REALM]); 135 136 if (is_feat_sve_supported()) { 137 /* 138 * Enable SVE and FPU in realm context when it is enabled for NS. 139 * Realm manager must ensure that the SVE and FPU register 140 * contexts are properly managed. 141 */ 142 sve_enable_per_world(&per_world_context[CPU_CONTEXT_REALM]); 143 } 144 145 /* NS can access this but Realm shouldn't */ 146 if (is_feat_sys_reg_trace_supported()) { 147 sys_reg_trace_disable_per_world(&per_world_context[CPU_CONTEXT_REALM]); 148 } 149 150 /* 151 * If SME/SME2 is supported and enabled for NS world, then disable trapping 152 * of SME instructions for Realm world. RMM will save/restore required 153 * registers that are shared with SVE/FPU so that Realm can use FPU or SVE. 154 */ 155 if (is_feat_sme_supported()) { 156 sme_enable_per_world(&per_world_context[CPU_CONTEXT_REALM]); 157 } 158 } 159 160 /******************************************************************************* 161 * Jump to the RMM for the first time. 162 ******************************************************************************/ 163 static int32_t rmm_init(void) 164 { 165 long rc; 166 rmmd_rmm_context_t *ctx = &rmm_context[plat_my_core_pos()]; 167 168 INFO("RMM init start.\n"); 169 170 /* Enable architecture extensions */ 171 manage_extensions_realm(&ctx->cpu_ctx); 172 173 manage_extensions_realm_per_world(); 174 175 /* Initialize RMM EL2 context. */ 176 rmm_el2_context_init(&ctx->cpu_ctx.el2_sysregs_ctx); 177 178 rc = rmmd_rmm_sync_entry(ctx); 179 if (rc != E_RMM_BOOT_SUCCESS) { 180 ERROR("RMM init failed: %ld\n", rc); 181 /* Mark the boot as failed for all the CPUs */ 182 rmm_boot_failed = true; 183 return 0; 184 } 185 186 INFO("RMM init end.\n"); 187 188 return 1; 189 } 190 191 /******************************************************************************* 192 * Load and read RMM manifest, setup RMM. 193 ******************************************************************************/ 194 int rmmd_setup(void) 195 { 196 size_t shared_buf_size __unused; 197 uintptr_t shared_buf_base; 198 uint32_t ep_attr; 199 unsigned int linear_id = plat_my_core_pos(); 200 rmmd_rmm_context_t *rmm_ctx = &rmm_context[linear_id]; 201 struct rmm_manifest *manifest; 202 int rc; 203 204 /* Make sure RME is supported. */ 205 if (is_feat_rme_present() == 0U) { 206 /* Mark the RMM boot as failed for all the CPUs */ 207 rmm_boot_failed = true; 208 return -ENOTSUP; 209 } 210 211 rmm_ep_info = bl31_plat_get_next_image_ep_info(REALM); 212 if ((rmm_ep_info == NULL) || (rmm_ep_info->pc == 0)) { 213 WARN("No RMM image provided by BL2 boot loader, Booting " 214 "device without RMM initialization. SMCs destined for " 215 "RMM will return SMC_UNK\n"); 216 217 /* Mark the boot as failed for all the CPUs */ 218 rmm_boot_failed = true; 219 return -ENOENT; 220 } 221 222 /* Initialise an entrypoint to set up the CPU context */ 223 ep_attr = EP_REALM; 224 if ((read_sctlr_el3() & SCTLR_EE_BIT) != 0U) { 225 ep_attr |= EP_EE_BIG; 226 } 227 228 SET_PARAM_HEAD(rmm_ep_info, PARAM_EP, VERSION_1, ep_attr); 229 rmm_ep_info->spsr = SPSR_64(MODE_EL2, 230 MODE_SP_ELX, 231 DISABLE_ALL_EXCEPTIONS); 232 233 shared_buf_size = 234 plat_rmmd_get_el3_rmm_shared_mem(&shared_buf_base); 235 236 assert((shared_buf_size == SZ_4K) && 237 ((void *)shared_buf_base != NULL)); 238 239 /* Zero out and load the boot manifest at the beginning of the share area */ 240 manifest = (struct rmm_manifest *)shared_buf_base; 241 (void)memset((void *)manifest, 0, sizeof(struct rmm_manifest)); 242 243 rc = plat_rmmd_load_manifest(manifest); 244 if (rc != 0) { 245 ERROR("Error loading RMM Boot Manifest (%i)\n", rc); 246 /* Mark the boot as failed for all the CPUs */ 247 rmm_boot_failed = true; 248 return rc; 249 } 250 flush_dcache_range((uintptr_t)shared_buf_base, shared_buf_size); 251 252 /* 253 * Prepare coldboot arguments for RMM: 254 * arg0: This CPUID (primary processor). 255 * arg1: Version for this Boot Interface. 256 * arg2: PLATFORM_CORE_COUNT. 257 * arg3: Base address for the EL3 <-> RMM shared area. The boot 258 * manifest will be stored at the beginning of this area. 259 */ 260 rmm_ep_info->args.arg0 = linear_id; 261 rmm_ep_info->args.arg1 = RMM_EL3_INTERFACE_VERSION; 262 rmm_ep_info->args.arg2 = PLATFORM_CORE_COUNT; 263 rmm_ep_info->args.arg3 = shared_buf_base; 264 265 /* Initialise RMM context with this entry point information */ 266 cm_setup_context(&rmm_ctx->cpu_ctx, rmm_ep_info); 267 268 INFO("RMM setup done.\n"); 269 270 /* Register init function for deferred init. */ 271 bl31_register_rmm_init(&rmm_init); 272 273 return 0; 274 } 275 276 /******************************************************************************* 277 * Forward SMC to the other security state 278 ******************************************************************************/ 279 static uint64_t rmmd_smc_forward(uint32_t src_sec_state, 280 uint32_t dst_sec_state, uint64_t x0, 281 uint64_t x1, uint64_t x2, uint64_t x3, 282 uint64_t x4, void *handle) 283 { 284 cpu_context_t *ctx = cm_get_context(dst_sec_state); 285 286 /* Save incoming security state */ 287 cm_el2_sysregs_context_save(src_sec_state); 288 289 /* Restore outgoing security state */ 290 cm_el2_sysregs_context_restore(dst_sec_state); 291 cm_set_next_eret_context(dst_sec_state); 292 293 /* 294 * As per SMCCCv1.2, we need to preserve x4 to x7 unless 295 * being used as return args. Hence we differentiate the 296 * onward and backward path. Support upto 8 args in the 297 * onward path and 4 args in return path. 298 * Register x4 will be preserved by RMM in case it is not 299 * used in return path. 300 */ 301 if (src_sec_state == NON_SECURE) { 302 SMC_RET8(ctx, x0, x1, x2, x3, x4, 303 SMC_GET_GP(handle, CTX_GPREG_X5), 304 SMC_GET_GP(handle, CTX_GPREG_X6), 305 SMC_GET_GP(handle, CTX_GPREG_X7)); 306 } 307 308 SMC_RET5(ctx, x0, x1, x2, x3, x4); 309 } 310 311 /******************************************************************************* 312 * This function handles all SMCs in the range reserved for RMI. Each call is 313 * either forwarded to the other security state or handled by the RMM dispatcher 314 ******************************************************************************/ 315 uint64_t rmmd_rmi_handler(uint32_t smc_fid, uint64_t x1, uint64_t x2, 316 uint64_t x3, uint64_t x4, void *cookie, 317 void *handle, uint64_t flags) 318 { 319 uint32_t src_sec_state; 320 321 /* If RMM failed to boot, treat any RMI SMC as unknown */ 322 if (rmm_boot_failed) { 323 WARN("RMMD: Failed to boot up RMM. Ignoring RMI call\n"); 324 SMC_RET1(handle, SMC_UNK); 325 } 326 327 /* Determine which security state this SMC originated from */ 328 src_sec_state = caller_sec_state(flags); 329 330 /* RMI must not be invoked by the Secure world */ 331 if (src_sec_state == SMC_FROM_SECURE) { 332 WARN("RMMD: RMI invoked by secure world.\n"); 333 SMC_RET1(handle, SMC_UNK); 334 } 335 336 /* 337 * Forward an RMI call from the Normal world to the Realm world as it 338 * is. 339 */ 340 if (src_sec_state == SMC_FROM_NON_SECURE) { 341 /* 342 * If SVE hint bit is set in the flags then update the SMC 343 * function id and pass it on to the lower EL. 344 */ 345 if (is_sve_hint_set(flags)) { 346 smc_fid |= (FUNCID_SVE_HINT_MASK << 347 FUNCID_SVE_HINT_SHIFT); 348 } 349 VERBOSE("RMMD: RMI call from non-secure world.\n"); 350 return rmmd_smc_forward(NON_SECURE, REALM, smc_fid, 351 x1, x2, x3, x4, handle); 352 } 353 354 if (src_sec_state != SMC_FROM_REALM) { 355 SMC_RET1(handle, SMC_UNK); 356 } 357 358 switch (smc_fid) { 359 case RMM_RMI_REQ_COMPLETE: { 360 uint64_t x5 = SMC_GET_GP(handle, CTX_GPREG_X5); 361 362 return rmmd_smc_forward(REALM, NON_SECURE, x1, 363 x2, x3, x4, x5, handle); 364 } 365 default: 366 WARN("RMMD: Unsupported RMM call 0x%08x\n", smc_fid); 367 SMC_RET1(handle, SMC_UNK); 368 } 369 } 370 371 /******************************************************************************* 372 * This cpu has been turned on. Enter RMM to initialise R-EL2. Entry into RMM 373 * is done after initialising minimal architectural state that guarantees safe 374 * execution. 375 ******************************************************************************/ 376 static void *rmmd_cpu_on_finish_handler(const void *arg) 377 { 378 long rc; 379 uint32_t linear_id = plat_my_core_pos(); 380 rmmd_rmm_context_t *ctx = &rmm_context[linear_id]; 381 382 if (rmm_boot_failed) { 383 /* RMM Boot failed on a previous CPU. Abort. */ 384 ERROR("RMM Failed to initialize. Ignoring for CPU%d\n", 385 linear_id); 386 return NULL; 387 } 388 389 /* 390 * Prepare warmboot arguments for RMM: 391 * arg0: This CPUID. 392 * arg1 to arg3: Not used. 393 */ 394 rmm_ep_info->args.arg0 = linear_id; 395 rmm_ep_info->args.arg1 = 0ULL; 396 rmm_ep_info->args.arg2 = 0ULL; 397 rmm_ep_info->args.arg3 = 0ULL; 398 399 /* Initialise RMM context with this entry point information */ 400 cm_setup_context(&ctx->cpu_ctx, rmm_ep_info); 401 402 /* Enable architecture extensions */ 403 manage_extensions_realm(&ctx->cpu_ctx); 404 405 /* Initialize RMM EL2 context. */ 406 rmm_el2_context_init(&ctx->cpu_ctx.el2_sysregs_ctx); 407 408 rc = rmmd_rmm_sync_entry(ctx); 409 410 if (rc != E_RMM_BOOT_SUCCESS) { 411 ERROR("RMM init failed on CPU%d: %ld\n", linear_id, rc); 412 /* Mark the boot as failed for any other booting CPU */ 413 rmm_boot_failed = true; 414 } 415 416 return NULL; 417 } 418 419 /* Subscribe to PSCI CPU on to initialize RMM on secondary */ 420 SUBSCRIBE_TO_EVENT(psci_cpu_on_finish, rmmd_cpu_on_finish_handler); 421 422 /* Convert GPT lib error to RMMD GTS error */ 423 static int gpt_to_gts_error(int error, uint32_t smc_fid, uint64_t address) 424 { 425 int ret; 426 427 if (error == 0) { 428 return E_RMM_OK; 429 } 430 431 if (error == -EINVAL) { 432 ret = E_RMM_BAD_ADDR; 433 } else { 434 /* This is the only other error code we expect */ 435 assert(error == -EPERM); 436 ret = E_RMM_BAD_PAS; 437 } 438 439 ERROR("RMMD: PAS Transition failed. GPT ret = %d, PA: 0x%"PRIx64 ", FID = 0x%x\n", 440 error, address, smc_fid); 441 return ret; 442 } 443 444 /******************************************************************************* 445 * This function handles RMM-EL3 interface SMCs 446 ******************************************************************************/ 447 uint64_t rmmd_rmm_el3_handler(uint32_t smc_fid, uint64_t x1, uint64_t x2, 448 uint64_t x3, uint64_t x4, void *cookie, 449 void *handle, uint64_t flags) 450 { 451 uint64_t remaining_len = 0; 452 uint32_t src_sec_state; 453 int ret; 454 455 /* If RMM failed to boot, treat any RMM-EL3 interface SMC as unknown */ 456 if (rmm_boot_failed) { 457 WARN("RMMD: Failed to boot up RMM. Ignoring RMM-EL3 call\n"); 458 SMC_RET1(handle, SMC_UNK); 459 } 460 461 /* Determine which security state this SMC originated from */ 462 src_sec_state = caller_sec_state(flags); 463 464 if (src_sec_state != SMC_FROM_REALM) { 465 WARN("RMMD: RMM-EL3 call originated from secure or normal world\n"); 466 SMC_RET1(handle, SMC_UNK); 467 } 468 469 switch (smc_fid) { 470 case RMM_GTSI_DELEGATE: 471 ret = gpt_delegate_pas(x1, PAGE_SIZE_4KB, SMC_FROM_REALM); 472 SMC_RET1(handle, gpt_to_gts_error(ret, smc_fid, x1)); 473 case RMM_GTSI_UNDELEGATE: 474 ret = gpt_undelegate_pas(x1, PAGE_SIZE_4KB, SMC_FROM_REALM); 475 SMC_RET1(handle, gpt_to_gts_error(ret, smc_fid, x1)); 476 case RMM_ATTEST_GET_PLAT_TOKEN: 477 ret = rmmd_attest_get_platform_token(x1, &x2, x3, &remaining_len); 478 SMC_RET3(handle, ret, x2, remaining_len); 479 case RMM_ATTEST_GET_REALM_KEY: 480 ret = rmmd_attest_get_signing_key(x1, &x2, x3); 481 SMC_RET2(handle, ret, x2); 482 483 case RMM_BOOT_COMPLETE: 484 VERBOSE("RMMD: running rmmd_rmm_sync_exit\n"); 485 rmmd_rmm_sync_exit(x1); 486 487 default: 488 WARN("RMMD: Unsupported RMM-EL3 call 0x%08x\n", smc_fid); 489 SMC_RET1(handle, SMC_UNK); 490 } 491 } 492