1 /* 2 * Copyright (c) 2021-2025, Arm Limited and Contributors. All rights reserved. 3 * 4 * SPDX-License-Identifier: BSD-3-Clause 5 */ 6 7 #include <assert.h> 8 #include <errno.h> 9 #include <inttypes.h> 10 #include <stdint.h> 11 #include <string.h> 12 13 #include <arch_helpers.h> 14 #include <arch_features.h> 15 #include <bl31/bl31.h> 16 #include <common/debug.h> 17 #include <common/runtime_svc.h> 18 #include <context.h> 19 #include <lib/el3_runtime/context_mgmt.h> 20 #include <lib/el3_runtime/cpu_data.h> 21 #include <lib/el3_runtime/pubsub.h> 22 #include <lib/extensions/mpam.h> 23 #include <lib/extensions/pmuv3.h> 24 #include <lib/extensions/sys_reg_trace.h> 25 #include <lib/gpt_rme/gpt_rme.h> 26 27 #include <lib/spinlock.h> 28 #include <lib/utils.h> 29 #include <lib/xlat_tables/xlat_tables_v2.h> 30 #include <plat/common/common_def.h> 31 #include <plat/common/platform.h> 32 #include <platform_def.h> 33 #include <services/rmmd_svc.h> 34 #include <smccc_helpers.h> 35 #include <lib/extensions/sme.h> 36 #include <lib/extensions/sve.h> 37 #include <lib/extensions/spe.h> 38 #include <lib/extensions/trbe.h> 39 #include "rmmd_private.h" 40 41 /******************************************************************************* 42 * RMM boot failure flag 43 ******************************************************************************/ 44 static bool rmm_boot_failed; 45 46 /******************************************************************************* 47 * RMM context information. 48 ******************************************************************************/ 49 rmmd_rmm_context_t rmm_context[PLATFORM_CORE_COUNT]; 50 51 /******************************************************************************* 52 * RMM entry point information. Discovered on the primary core and reused 53 * on secondary cores. 54 ******************************************************************************/ 55 static entry_point_info_t *rmm_ep_info; 56 57 /******************************************************************************* 58 * Static function declaration. 59 ******************************************************************************/ 60 static int32_t rmm_init(void); 61 62 /******************************************************************************* 63 * This function takes an RMM context pointer and performs a synchronous entry 64 * into it. 65 ******************************************************************************/ 66 uint64_t rmmd_rmm_sync_entry(rmmd_rmm_context_t *rmm_ctx) 67 { 68 uint64_t rc; 69 70 assert(rmm_ctx != NULL); 71 72 cm_set_context(&(rmm_ctx->cpu_ctx), REALM); 73 74 /* Restore the realm context assigned above */ 75 cm_el2_sysregs_context_restore(REALM); 76 cm_set_next_eret_context(REALM); 77 78 /* Enter RMM */ 79 rc = rmmd_rmm_enter(&rmm_ctx->c_rt_ctx); 80 81 /* 82 * Save realm context. EL2 Non-secure context will be restored 83 * before exiting Non-secure world, therefore there is no need 84 * to clear EL2 context registers. 85 */ 86 cm_el2_sysregs_context_save(REALM); 87 88 return rc; 89 } 90 91 /******************************************************************************* 92 * This function returns to the place where rmmd_rmm_sync_entry() was 93 * called originally. 94 ******************************************************************************/ 95 __dead2 void rmmd_rmm_sync_exit(uint64_t rc) 96 { 97 rmmd_rmm_context_t *ctx = &rmm_context[plat_my_core_pos()]; 98 99 /* Get context of the RMM in use by this CPU. */ 100 assert(cm_get_context(REALM) == &(ctx->cpu_ctx)); 101 102 /* 103 * The RMMD must have initiated the original request through a 104 * synchronous entry into RMM. Jump back to the original C runtime 105 * context with the value of rc in x0; 106 */ 107 rmmd_rmm_exit(ctx->c_rt_ctx, rc); 108 109 panic(); 110 } 111 112 /******************************************************************************* 113 * Jump to the RMM for the first time. 114 ******************************************************************************/ 115 static int32_t rmm_init(void) 116 { 117 long rc; 118 rmmd_rmm_context_t *ctx = &rmm_context[plat_my_core_pos()]; 119 120 INFO("RMM init start.\n"); 121 122 rc = rmmd_rmm_sync_entry(ctx); 123 if (rc != E_RMM_BOOT_SUCCESS) { 124 ERROR("RMM init failed: %ld\n", rc); 125 /* Mark the boot as failed for all the CPUs */ 126 rmm_boot_failed = true; 127 return 0; 128 } 129 130 INFO("RMM init end.\n"); 131 132 return 1; 133 } 134 135 /******************************************************************************* 136 * Load and read RMM manifest, setup RMM. 137 ******************************************************************************/ 138 int rmmd_setup(void) 139 { 140 size_t shared_buf_size __unused; 141 uintptr_t shared_buf_base; 142 uint32_t ep_attr; 143 unsigned int linear_id = plat_my_core_pos(); 144 rmmd_rmm_context_t *rmm_ctx = &rmm_context[linear_id]; 145 struct rmm_manifest *manifest; 146 int rc; 147 148 /* Make sure RME is supported. */ 149 if (is_feat_rme_present() == 0U) { 150 /* Mark the RMM boot as failed for all the CPUs */ 151 rmm_boot_failed = true; 152 return -ENOTSUP; 153 } 154 155 rmm_ep_info = bl31_plat_get_next_image_ep_info(REALM); 156 if ((rmm_ep_info == NULL) || (rmm_ep_info->pc == 0)) { 157 WARN("No RMM image provided by BL2 boot loader, Booting " 158 "device without RMM initialization. SMCs destined for " 159 "RMM will return SMC_UNK\n"); 160 161 /* Mark the boot as failed for all the CPUs */ 162 rmm_boot_failed = true; 163 return -ENOENT; 164 } 165 166 /* Initialise an entrypoint to set up the CPU context */ 167 ep_attr = EP_REALM; 168 if ((read_sctlr_el3() & SCTLR_EE_BIT) != 0U) { 169 ep_attr |= EP_EE_BIG; 170 } 171 172 SET_PARAM_HEAD(rmm_ep_info, PARAM_EP, VERSION_1, ep_attr); 173 rmm_ep_info->spsr = SPSR_64(MODE_EL2, 174 MODE_SP_ELX, 175 DISABLE_ALL_EXCEPTIONS); 176 177 shared_buf_size = 178 plat_rmmd_get_el3_rmm_shared_mem(&shared_buf_base); 179 180 assert((shared_buf_size == SZ_4K) && 181 ((void *)shared_buf_base != NULL)); 182 183 /* Zero out and load the boot manifest at the beginning of the share area */ 184 manifest = (struct rmm_manifest *)shared_buf_base; 185 (void)memset((void *)manifest, 0, sizeof(struct rmm_manifest)); 186 187 rc = plat_rmmd_load_manifest(manifest); 188 if (rc != 0) { 189 ERROR("Error loading RMM Boot Manifest (%i)\n", rc); 190 /* Mark the boot as failed for all the CPUs */ 191 rmm_boot_failed = true; 192 return rc; 193 } 194 flush_dcache_range((uintptr_t)shared_buf_base, shared_buf_size); 195 196 /* 197 * Prepare coldboot arguments for RMM: 198 * arg0: This CPUID (primary processor). 199 * arg1: Version for this Boot Interface. 200 * arg2: PLATFORM_CORE_COUNT. 201 * arg3: Base address for the EL3 <-> RMM shared area. The boot 202 * manifest will be stored at the beginning of this area. 203 * arg4: opaque activation token, as returned by previous calls 204 */ 205 rmm_ep_info->args.arg0 = linear_id; 206 rmm_ep_info->args.arg1 = RMM_EL3_INTERFACE_VERSION; 207 rmm_ep_info->args.arg2 = PLATFORM_CORE_COUNT; 208 rmm_ep_info->args.arg3 = shared_buf_base; 209 rmm_ep_info->args.arg4 = rmm_ctx->activation_token; 210 211 /* Initialise RMM context with this entry point information */ 212 cm_setup_context(&rmm_ctx->cpu_ctx, rmm_ep_info); 213 214 INFO("RMM setup done.\n"); 215 216 /* Register init function for deferred init. */ 217 bl31_register_rmm_init(&rmm_init); 218 219 return 0; 220 } 221 222 /******************************************************************************* 223 * Forward SMC to the other security state 224 ******************************************************************************/ 225 static uint64_t rmmd_smc_forward(uint32_t src_sec_state, 226 uint32_t dst_sec_state, uint64_t x0, 227 uint64_t x1, uint64_t x2, uint64_t x3, 228 uint64_t x4, void *handle) 229 { 230 cpu_context_t *ctx = cm_get_context(dst_sec_state); 231 232 /* Save incoming security state */ 233 cm_el2_sysregs_context_save(src_sec_state); 234 235 /* Restore outgoing security state */ 236 cm_el2_sysregs_context_restore(dst_sec_state); 237 cm_set_next_eret_context(dst_sec_state); 238 239 /* 240 * As per SMCCCv1.2, we need to preserve x4 to x7 unless 241 * being used as return args. Hence we differentiate the 242 * onward and backward path. Support upto 8 args in the 243 * onward path and 4 args in return path. 244 * Register x4 will be preserved by RMM in case it is not 245 * used in return path. 246 */ 247 if (src_sec_state == NON_SECURE) { 248 SMC_RET8(ctx, x0, x1, x2, x3, x4, 249 SMC_GET_GP(handle, CTX_GPREG_X5), 250 SMC_GET_GP(handle, CTX_GPREG_X6), 251 SMC_GET_GP(handle, CTX_GPREG_X7)); 252 } 253 254 SMC_RET5(ctx, x0, x1, x2, x3, x4); 255 } 256 257 /******************************************************************************* 258 * This function handles all SMCs in the range reserved for RMI. Each call is 259 * either forwarded to the other security state or handled by the RMM dispatcher 260 ******************************************************************************/ 261 uint64_t rmmd_rmi_handler(uint32_t smc_fid, uint64_t x1, uint64_t x2, 262 uint64_t x3, uint64_t x4, void *cookie, 263 void *handle, uint64_t flags) 264 { 265 uint32_t src_sec_state; 266 267 /* If RMM failed to boot, treat any RMI SMC as unknown */ 268 if (rmm_boot_failed) { 269 WARN("RMMD: Failed to boot up RMM. Ignoring RMI call\n"); 270 SMC_RET1(handle, SMC_UNK); 271 } 272 273 /* Determine which security state this SMC originated from */ 274 src_sec_state = caller_sec_state(flags); 275 276 /* RMI must not be invoked by the Secure world */ 277 if (src_sec_state == SMC_FROM_SECURE) { 278 WARN("RMMD: RMI invoked by secure world.\n"); 279 SMC_RET1(handle, SMC_UNK); 280 } 281 282 /* 283 * Forward an RMI call from the Normal world to the Realm world as it 284 * is. 285 */ 286 if (src_sec_state == SMC_FROM_NON_SECURE) { 287 /* 288 * If SVE hint bit is set in the flags then update the SMC 289 * function id and pass it on to the lower EL. 290 */ 291 if (is_sve_hint_set(flags)) { 292 smc_fid |= (FUNCID_SVE_HINT_MASK << 293 FUNCID_SVE_HINT_SHIFT); 294 } 295 VERBOSE("RMMD: RMI call from non-secure world.\n"); 296 return rmmd_smc_forward(NON_SECURE, REALM, smc_fid, 297 x1, x2, x3, x4, handle); 298 } 299 300 if (src_sec_state != SMC_FROM_REALM) { 301 SMC_RET1(handle, SMC_UNK); 302 } 303 304 switch (smc_fid) { 305 case RMM_RMI_REQ_COMPLETE: { 306 uint64_t x5 = SMC_GET_GP(handle, CTX_GPREG_X5); 307 308 return rmmd_smc_forward(REALM, NON_SECURE, x1, 309 x2, x3, x4, x5, handle); 310 } 311 default: 312 WARN("RMMD: Unsupported RMM call 0x%08x\n", smc_fid); 313 SMC_RET1(handle, SMC_UNK); 314 } 315 } 316 317 /******************************************************************************* 318 * This cpu has been turned on. Enter RMM to initialise R-EL2. Entry into RMM 319 * is done after initialising minimal architectural state that guarantees safe 320 * execution. 321 ******************************************************************************/ 322 static void *rmmd_cpu_on_finish_handler(const void *arg) 323 { 324 long rc; 325 uint32_t linear_id = plat_my_core_pos(); 326 rmmd_rmm_context_t *ctx = &rmm_context[linear_id]; 327 328 if (rmm_boot_failed) { 329 /* RMM Boot failed on a previous CPU. Abort. */ 330 ERROR("RMM Failed to initialize. Ignoring for CPU%d\n", 331 linear_id); 332 return NULL; 333 } 334 335 /* 336 * Prepare warmboot arguments for RMM: 337 * arg0: This CPUID. 338 * arg1: opaque activation token, as returned by previous calls 339 * arg2 to arg3: Not used. 340 */ 341 rmm_ep_info->args.arg0 = linear_id; 342 rmm_ep_info->args.arg1 = ctx->activation_token; 343 rmm_ep_info->args.arg2 = 0ULL; 344 rmm_ep_info->args.arg3 = 0ULL; 345 346 /* Initialise RMM context with this entry point information */ 347 cm_setup_context(&ctx->cpu_ctx, rmm_ep_info); 348 349 rc = rmmd_rmm_sync_entry(ctx); 350 351 if (rc != E_RMM_BOOT_SUCCESS) { 352 ERROR("RMM init failed on CPU%d: %ld\n", linear_id, rc); 353 /* Mark the boot as failed for any other booting CPU */ 354 rmm_boot_failed = true; 355 } 356 357 return NULL; 358 } 359 360 /* Subscribe to PSCI CPU on to initialize RMM on secondary */ 361 SUBSCRIBE_TO_EVENT(psci_cpu_on_finish, rmmd_cpu_on_finish_handler); 362 363 /* Convert GPT lib error to RMMD GTS error */ 364 static int gpt_to_gts_error(int error, uint32_t smc_fid, uint64_t address) 365 { 366 int ret; 367 368 if (error == 0) { 369 return E_RMM_OK; 370 } 371 372 if (error == -EINVAL) { 373 ret = E_RMM_BAD_ADDR; 374 } else { 375 /* This is the only other error code we expect */ 376 assert(error == -EPERM); 377 ret = E_RMM_BAD_PAS; 378 } 379 380 ERROR("RMMD: PAS Transition failed. GPT ret = %d, PA: 0x%"PRIx64 ", FID = 0x%x\n", 381 error, address, smc_fid); 382 return ret; 383 } 384 385 static int rmm_el3_ifc_get_feat_register(uint64_t feat_reg_idx, 386 uint64_t *feat_reg) 387 { 388 if (feat_reg_idx != RMM_EL3_FEAT_REG_0_IDX) { 389 ERROR("RMMD: Failed to get feature register %ld\n", feat_reg_idx); 390 return E_RMM_INVAL; 391 } 392 393 *feat_reg = 0UL; 394 #if RMMD_ENABLE_EL3_TOKEN_SIGN 395 *feat_reg |= RMM_EL3_FEAT_REG_0_EL3_TOKEN_SIGN_MASK; 396 #endif 397 return E_RMM_OK; 398 } 399 400 /* 401 * Update encryption key associated with @mecid. 402 */ 403 static int rmmd_mecid_key_update(uint64_t mecid) 404 { 405 uint64_t mecid_width, mecid_width_mask; 406 int ret; 407 408 /* 409 * Check whether FEAT_MEC is supported by the hardware. If not, return 410 * unknown SMC. 411 */ 412 if (is_feat_mec_supported() == false) { 413 return E_RMM_UNK; 414 } 415 416 /* 417 * Check whether the mecid parameter is at most MECIDR_EL2.MECIDWidthm1 + 1 418 * in length. 419 */ 420 mecid_width = ((read_mecidr_el2() >> MECIDR_EL2_MECIDWidthm1_SHIFT) & 421 MECIDR_EL2_MECIDWidthm1_MASK) + 1; 422 mecid_width_mask = ((1 << mecid_width) - 1); 423 if ((mecid & ~mecid_width_mask) != 0U) { 424 return E_RMM_INVAL; 425 } 426 427 ret = plat_rmmd_mecid_key_update(mecid); 428 429 if (ret != 0) { 430 return E_RMM_UNK; 431 } 432 return E_RMM_OK; 433 } 434 435 /******************************************************************************* 436 * This function handles RMM-EL3 interface SMCs 437 ******************************************************************************/ 438 uint64_t rmmd_rmm_el3_handler(uint32_t smc_fid, uint64_t x1, uint64_t x2, 439 uint64_t x3, uint64_t x4, void *cookie, 440 void *handle, uint64_t flags) 441 { 442 uint64_t remaining_len = 0UL; 443 uint32_t src_sec_state; 444 int ret; 445 446 /* If RMM failed to boot, treat any RMM-EL3 interface SMC as unknown */ 447 if (rmm_boot_failed) { 448 WARN("RMMD: Failed to boot up RMM. Ignoring RMM-EL3 call\n"); 449 SMC_RET1(handle, SMC_UNK); 450 } 451 452 /* Determine which security state this SMC originated from */ 453 src_sec_state = caller_sec_state(flags); 454 455 if (src_sec_state != SMC_FROM_REALM) { 456 WARN("RMMD: RMM-EL3 call originated from secure or normal world\n"); 457 SMC_RET1(handle, SMC_UNK); 458 } 459 460 switch (smc_fid) { 461 case RMM_GTSI_DELEGATE: 462 ret = gpt_delegate_pas(x1, PAGE_SIZE_4KB, SMC_FROM_REALM); 463 SMC_RET1(handle, gpt_to_gts_error(ret, smc_fid, x1)); 464 case RMM_GTSI_UNDELEGATE: 465 ret = gpt_undelegate_pas(x1, PAGE_SIZE_4KB, SMC_FROM_REALM); 466 SMC_RET1(handle, gpt_to_gts_error(ret, smc_fid, x1)); 467 case RMM_ATTEST_GET_REALM_KEY: 468 ret = rmmd_attest_get_signing_key(x1, &x2, x3); 469 SMC_RET2(handle, ret, x2); 470 case RMM_ATTEST_GET_PLAT_TOKEN: 471 ret = rmmd_attest_get_platform_token(x1, &x2, x3, &remaining_len); 472 SMC_RET3(handle, ret, x2, remaining_len); 473 case RMM_EL3_FEATURES: 474 ret = rmm_el3_ifc_get_feat_register(x1, &x2); 475 SMC_RET2(handle, ret, x2); 476 #if RMMD_ENABLE_EL3_TOKEN_SIGN 477 case RMM_EL3_TOKEN_SIGN: 478 return rmmd_el3_token_sign(handle, x1, x2, x3, x4); 479 #endif 480 481 #if RMMD_ENABLE_IDE_KEY_PROG 482 case RMM_IDE_KEY_PROG: 483 { 484 rp_ide_key_info_t ide_key_info; 485 486 ide_key_info.keyqw0 = x4; 487 ide_key_info.keyqw1 = SMC_GET_GP(handle, CTX_GPREG_X5); 488 ide_key_info.keyqw2 = SMC_GET_GP(handle, CTX_GPREG_X6); 489 ide_key_info.keyqw3 = SMC_GET_GP(handle, CTX_GPREG_X7); 490 ide_key_info.ifvqw0 = SMC_GET_GP(handle, CTX_GPREG_X8); 491 ide_key_info.ifvqw1 = SMC_GET_GP(handle, CTX_GPREG_X9); 492 uint64_t x10 = SMC_GET_GP(handle, CTX_GPREG_X10); 493 uint64_t x11 = SMC_GET_GP(handle, CTX_GPREG_X11); 494 495 ret = rmmd_el3_ide_key_program(x1, x2, x3, &ide_key_info, x10, x11); 496 SMC_RET1(handle, ret); 497 } 498 case RMM_IDE_KEY_SET_GO: 499 ret = rmmd_el3_ide_key_set_go(x1, x2, x3, x4, SMC_GET_GP(handle, CTX_GPREG_X5)); 500 SMC_RET1(handle, ret); 501 case RMM_IDE_KEY_SET_STOP: 502 ret = rmmd_el3_ide_key_set_stop(x1, x2, x3, x4, SMC_GET_GP(handle, CTX_GPREG_X5)); 503 SMC_RET1(handle, ret); 504 case RMM_IDE_KM_PULL_RESPONSE: { 505 uint64_t req_resp = 0, req_id = 0, cookie_var = 0; 506 507 ret = rmmd_el3_ide_km_pull_response(x1, x2, &req_resp, &req_id, &cookie_var); 508 SMC_RET4(handle, ret, req_resp, req_id, cookie_var); 509 } 510 #endif /* RMMD_ENABLE_IDE_KEY_PROG */ 511 case RMM_RESERVE_MEMORY: 512 ret = rmmd_reserve_memory(x1, &x2); 513 SMC_RET2(handle, ret, x2); 514 515 case RMM_BOOT_COMPLETE: 516 { 517 rmmd_rmm_context_t *ctx = &rmm_context[plat_my_core_pos()]; 518 519 ctx->activation_token = x2; 520 VERBOSE("RMMD: running rmmd_rmm_sync_exit\n"); 521 rmmd_rmm_sync_exit(x1); 522 } 523 case RMM_MECID_KEY_UPDATE: 524 ret = rmmd_mecid_key_update(x1); 525 SMC_RET1(handle, ret); 526 default: 527 WARN("RMMD: Unsupported RMM-EL3 call 0x%08x\n", smc_fid); 528 SMC_RET1(handle, SMC_UNK); 529 } 530 } 531