1 /* 2 * Copyright (c) 2022-2025, Arm Limited. All rights reserved. 3 * 4 * SPDX-License-Identifier: BSD-3-Clause 5 */ 6 7 #include <assert.h> 8 #include <errno.h> 9 #include <inttypes.h> 10 #include <limits.h> 11 #include <stdint.h> 12 13 #include <arch.h> 14 #include <arch_features.h> 15 #include <common/debug.h> 16 #include <lib/gpt_rme/gpt_rme.h> 17 #include <lib/smccc.h> 18 #include <lib/xlat_tables/xlat_tables_v2.h> 19 20 #include "gpt_rme_private.h" 21 22 #if !ENABLE_RME 23 #error "ENABLE_RME must be enabled to use the GPT library" 24 #endif 25 26 /* 27 * Lookup T from PPS 28 * 29 * PPS Size T 30 * 0b000 4GB 32 31 * 0b001 64GB 36 32 * 0b010 1TB 40 33 * 0b011 4TB 42 34 * 0b100 16TB 44 35 * 0b101 256TB 48 36 * 0b110 4PB 52 37 * 38 * See section 15.1.27 of the RME specification. 39 */ 40 static const gpt_t_val_e gpt_t_lookup[] = {PPS_4GB_T, PPS_64GB_T, 41 PPS_1TB_T, PPS_4TB_T, 42 PPS_16TB_T, PPS_256TB_T, 43 PPS_4PB_T}; 44 45 /* 46 * Lookup P from PGS 47 * 48 * PGS Size P 49 * 0b00 4KB 12 50 * 0b10 16KB 14 51 * 0b01 64KB 16 52 * 53 * Note that pgs=0b10 is 16KB and pgs=0b01 is 64KB, this is not a typo. 54 * 55 * See section 15.1.27 of the RME specification. 56 */ 57 static const gpt_p_val_e gpt_p_lookup[] = {PGS_4KB_P, PGS_64KB_P, PGS_16KB_P}; 58 59 static void shatter_2mb(uintptr_t base, const gpi_info_t *gpi_info, 60 uint64_t l1_desc); 61 static void shatter_32mb(uintptr_t base, const gpi_info_t *gpi_info, 62 uint64_t l1_desc); 63 static void shatter_512mb(uintptr_t base, const gpi_info_t *gpi_info, 64 uint64_t l1_desc); 65 66 /* 67 * This structure contains GPT configuration data 68 */ 69 typedef struct { 70 uintptr_t plat_gpt_l0_base; 71 gpccr_pps_e pps; 72 gpt_t_val_e t; 73 gpccr_pgs_e pgs; 74 gpt_p_val_e p; 75 } gpt_config_t; 76 77 static gpt_config_t gpt_config; 78 79 /* 80 * Number of L1 entries in 2MB, depending on GPCCR_EL3.PGS: 81 * +-------+------------+ 82 * | PGS | L1 entries | 83 * +-------+------------+ 84 * | 4KB | 32 | 85 * +-------+------------+ 86 * | 16KB | 8 | 87 * +-------+------------+ 88 * | 64KB | 2 | 89 * +-------+------------+ 90 */ 91 static unsigned int gpt_l1_cnt_2mb; 92 93 /* 94 * Mask for the L1 index field, depending on 95 * GPCCR_EL3.L0GPTSZ and GPCCR_EL3.PGS: 96 * +---------+-------------------------------+ 97 * | | PGS | 98 * +---------+----------+----------+---------+ 99 * | L0GPTSZ | 4KB | 16KB | 64KB | 100 * +---------+----------+----------+---------+ 101 * | 1GB | 0x3FFF | 0xFFF | 0x3FF | 102 * +---------+----------+----------+---------+ 103 * | 16GB | 0x3FFFF | 0xFFFF | 0x3FFF | 104 * +---------+----------+----------+---------+ 105 * | 64GB | 0xFFFFF | 0x3FFFF | 0xFFFF | 106 * +---------+----------+----------+---------+ 107 * | 512GB | 0x7FFFFF | 0x1FFFFF | 0x7FFFF | 108 * +---------+----------+----------+---------+ 109 */ 110 static uint64_t gpt_l1_index_mask; 111 112 /* Number of 128-bit L1 entries in 2MB, 32MB and 512MB */ 113 #define L1_QWORDS_2MB (gpt_l1_cnt_2mb / 2U) 114 #define L1_QWORDS_32MB (L1_QWORDS_2MB * 16U) 115 #define L1_QWORDS_512MB (L1_QWORDS_32MB * 16U) 116 117 /* Size in bytes of L1 entries in 2MB, 32MB */ 118 #define L1_BYTES_2MB (gpt_l1_cnt_2mb * sizeof(uint64_t)) 119 #define L1_BYTES_32MB (L1_BYTES_2MB * 16U) 120 121 /* Get the index into the L1 table from a physical address */ 122 #define GPT_L1_INDEX(_pa) \ 123 (((_pa) >> (unsigned int)GPT_L1_IDX_SHIFT(gpt_config.p)) & gpt_l1_index_mask) 124 125 /* This variable is used during initialization of the L1 tables */ 126 static uintptr_t gpt_l1_tbl; 127 128 /* These variables are used during runtime */ 129 #if (RME_GPT_BITLOCK_BLOCK == 0) 130 /* 131 * The GPTs are protected by a global spinlock to ensure 132 * that multiple CPUs do not attempt to change the descriptors at once. 133 */ 134 static spinlock_t gpt_lock; 135 136 /* Lock/unlock macros for GPT entries 137 * 138 * Access to GPT is controlled by a global lock to ensure 139 * that no more than one CPU is allowed to make changes at any 140 * given time. 141 */ 142 #define GPT_LOCK spin_lock(&gpt_lock) 143 #define GPT_UNLOCK spin_unlock(&gpt_lock) 144 #else 145 146 /* Base address of bitlocks array */ 147 static bitlock_t *gpt_bitlock; 148 149 /* 150 * Access to a block of memory is controlled by a bitlock. 151 * Size of block = RME_GPT_BITLOCK_BLOCK * 512MB. 152 */ 153 #define GPT_LOCK bit_lock(gpi_info.lock, gpi_info.mask) 154 #define GPT_UNLOCK bit_unlock(gpi_info.lock, gpi_info.mask) 155 #endif /* RME_GPT_BITLOCK_BLOCK */ 156 157 static void tlbi_page_dsbosh(uintptr_t base) 158 { 159 /* Look-up table for invalidation TLBs for 4KB, 16KB and 64KB pages */ 160 static const gpt_tlbi_lookup_t tlbi_page_lookup[] = { 161 { tlbirpalos_4k, ~(SZ_4K - 1UL) }, 162 { tlbirpalos_64k, ~(SZ_64K - 1UL) }, 163 { tlbirpalos_16k, ~(SZ_16K - 1UL) } 164 }; 165 166 tlbi_page_lookup[gpt_config.pgs].function( 167 base & tlbi_page_lookup[gpt_config.pgs].mask); 168 dsbosh(); 169 } 170 171 /* 172 * Helper function to fill out GPI entries in a single L1 table 173 * with Granules or Contiguous descriptor. 174 * 175 * Parameters 176 * l1 Pointer to 2MB, 32MB or 512MB aligned L1 table entry to fill out 177 * l1_desc GPT Granules or Contiguous descriptor set this range to 178 * cnt Number of double 128-bit L1 entries to fill 179 * 180 */ 181 static void fill_desc(uint64_t *l1, uint64_t l1_desc, unsigned int cnt) 182 { 183 uint128_t *l1_quad = (uint128_t *)l1; 184 uint128_t l1_quad_desc = (uint128_t)l1_desc | ((uint128_t)l1_desc << 64); 185 186 VERBOSE("GPT: %s(%p 0x%"PRIx64" %u)\n", __func__, l1, l1_desc, cnt); 187 188 for (unsigned int i = 0U; i < cnt; i++) { 189 *l1_quad++ = l1_quad_desc; 190 } 191 } 192 193 static void shatter_2mb(uintptr_t base, const gpi_info_t *gpi_info, 194 uint64_t l1_desc) 195 { 196 unsigned long idx = GPT_L1_INDEX(ALIGN_2MB(base)); 197 198 VERBOSE("GPT: %s(0x%"PRIxPTR" 0x%"PRIx64")\n", 199 __func__, base, l1_desc); 200 201 /* Convert 2MB Contiguous block to Granules */ 202 fill_desc(&gpi_info->gpt_l1_addr[idx], l1_desc, L1_QWORDS_2MB); 203 } 204 205 static void shatter_32mb(uintptr_t base, const gpi_info_t *gpi_info, 206 uint64_t l1_desc) 207 { 208 unsigned long idx = GPT_L1_INDEX(ALIGN_2MB(base)); 209 const uint64_t *l1_gran = &gpi_info->gpt_l1_addr[idx]; 210 uint64_t l1_cont_desc = GPT_L1_CONT_DESC(l1_desc, 2MB); 211 uint64_t *l1; 212 213 VERBOSE("GPT: %s(0x%"PRIxPTR" 0x%"PRIx64")\n", 214 __func__, base, l1_desc); 215 216 /* Get index corresponding to 32MB aligned address */ 217 idx = GPT_L1_INDEX(ALIGN_32MB(base)); 218 l1 = &gpi_info->gpt_l1_addr[idx]; 219 220 /* 16 x 2MB blocks in 32MB */ 221 for (unsigned int i = 0U; i < 16U; i++) { 222 /* Fill with Granules or Contiguous descriptors */ 223 fill_desc(l1, (l1 == l1_gran) ? l1_desc : l1_cont_desc, 224 L1_QWORDS_2MB); 225 l1 = (uint64_t *)((uintptr_t)l1 + L1_BYTES_2MB); 226 } 227 } 228 229 static void shatter_512mb(uintptr_t base, const gpi_info_t *gpi_info, 230 uint64_t l1_desc) 231 { 232 unsigned long idx = GPT_L1_INDEX(ALIGN_32MB(base)); 233 const uint64_t *l1_32mb = &gpi_info->gpt_l1_addr[idx]; 234 uint64_t l1_cont_desc = GPT_L1_CONT_DESC(l1_desc, 32MB); 235 uint64_t *l1; 236 237 VERBOSE("GPT: %s(0x%"PRIxPTR" 0x%"PRIx64")\n", 238 __func__, base, l1_desc); 239 240 /* Get index corresponding to 512MB aligned address */ 241 idx = GPT_L1_INDEX(ALIGN_512MB(base)); 242 l1 = &gpi_info->gpt_l1_addr[idx]; 243 244 /* 16 x 32MB blocks in 512MB */ 245 for (unsigned int i = 0U; i < 16U; i++) { 246 if (l1 == l1_32mb) { 247 /* Shatter this 32MB block */ 248 shatter_32mb(base, gpi_info, l1_desc); 249 } else { 250 /* Fill 32MB with Contiguous descriptors */ 251 fill_desc(l1, l1_cont_desc, L1_QWORDS_32MB); 252 } 253 254 l1 = (uint64_t *)((uintptr_t)l1 + L1_BYTES_32MB); 255 } 256 } 257 258 /* 259 * This function checks to see if a GPI value is valid. 260 * 261 * These are valid GPI values. 262 * GPT_GPI_NO_ACCESS U(0x0) 263 * GPT_GPI_SA U(0x4) 264 * GPT_GPI_NSP U(0x5) 265 * GPT_GPI_SECURE U(0x8) 266 * GPT_GPI_NS U(0x9) 267 * GPT_GPI_ROOT U(0xA) 268 * GPT_GPI_REALM U(0xB) 269 * GPT_GPI_NSO U(0xD) 270 * GPT_GPI_ANY U(0xF) 271 * 272 * Parameters 273 * gpi GPI to check for validity. 274 * 275 * Return 276 * true for a valid GPI, false for an invalid one. 277 */ 278 static bool is_gpi_valid(unsigned int gpi) 279 { 280 switch (gpi) { 281 case GPT_GPI_NO_ACCESS: 282 case GPT_GPI_SECURE: 283 case GPT_GPI_NS: 284 case GPT_GPI_ROOT: 285 case GPT_GPI_REALM: 286 case GPT_GPI_ANY: 287 return true; 288 case GPT_GPI_NSO: 289 return is_feat_rme_gpc2_present(); 290 case GPT_GPI_SA: 291 case GPT_GPI_NSP: 292 return is_feat_rme_gdi_supported(); 293 default: 294 return false; 295 } 296 } 297 298 /* 299 * This function checks to see if two PAS regions overlap. 300 * 301 * Parameters 302 * base_1: base address of first PAS 303 * size_1: size of first PAS 304 * base_2: base address of second PAS 305 * size_2: size of second PAS 306 * 307 * Return 308 * True if PAS regions overlap, false if they do not. 309 */ 310 static bool check_pas_overlap(uintptr_t base_1, size_t size_1, 311 uintptr_t base_2, size_t size_2) 312 { 313 if (((base_1 + size_1) > base_2) && ((base_2 + size_2) > base_1)) { 314 return true; 315 } 316 return false; 317 } 318 319 /* 320 * This helper function checks to see if a PAS region from index 0 to 321 * (pas_idx - 1) occupies the L0 region at index l0_idx in the L0 table. 322 * 323 * Parameters 324 * l0_idx: Index of the L0 entry to check 325 * pas_regions: PAS region array 326 * pas_idx: Upper bound of the PAS array index. 327 * 328 * Return 329 * True if a PAS region occupies the L0 region in question, false if not. 330 */ 331 static bool does_previous_pas_exist_here(unsigned int l0_idx, 332 pas_region_t *pas_regions, 333 unsigned int pas_idx) 334 { 335 /* Iterate over PAS regions up to pas_idx */ 336 for (unsigned int i = 0U; i < pas_idx; i++) { 337 if (check_pas_overlap((GPT_L0GPTSZ_ACTUAL_SIZE * l0_idx), 338 GPT_L0GPTSZ_ACTUAL_SIZE, 339 pas_regions[i].base_pa, pas_regions[i].size)) { 340 return true; 341 } 342 } 343 return false; 344 } 345 346 /* 347 * This function iterates over all of the PAS regions and checks them to ensure 348 * proper alignment of base and size, that the GPI is valid, and that no regions 349 * overlap. As a part of the overlap checks, this function checks existing L0 350 * mappings against the new PAS regions in the event that gpt_init_pas_l1_tables 351 * is called multiple times to place L1 tables in different areas of memory. It 352 * also counts the number of L1 tables needed and returns it on success. 353 * 354 * Parameters 355 * *pas_regions Pointer to array of PAS region structures. 356 * pas_region_cnt Total number of PAS regions in the array. 357 * 358 * Return 359 * Negative Linux error code in the event of a failure, number of L1 regions 360 * required when successful. 361 */ 362 static int validate_pas_mappings(pas_region_t *pas_regions, 363 unsigned int pas_region_cnt) 364 { 365 unsigned int idx; 366 unsigned int l1_cnt = 0U; 367 unsigned int pas_l1_cnt; 368 uint64_t *l0_desc = (uint64_t *)gpt_config.plat_gpt_l0_base; 369 370 assert(pas_regions != NULL); 371 assert(pas_region_cnt != 0U); 372 373 for (idx = 0U; idx < pas_region_cnt; idx++) { 374 /* Check for arithmetic overflow in region */ 375 if ((ULONG_MAX - pas_regions[idx].base_pa) < 376 pas_regions[idx].size) { 377 ERROR("GPT: Address overflow in PAS[%u]!\n", idx); 378 return -EOVERFLOW; 379 } 380 381 /* Initial checks for PAS validity */ 382 if (((pas_regions[idx].base_pa + pas_regions[idx].size) > 383 GPT_PPS_ACTUAL_SIZE(gpt_config.t)) || 384 !is_gpi_valid(GPT_PAS_ATTR_GPI(pas_regions[idx].attrs))) { 385 ERROR("GPT: PAS[%u] is invalid!\n", idx); 386 return -EFAULT; 387 } 388 389 /* 390 * Make sure this PAS does not overlap with another one. We 391 * start from idx + 1 instead of 0 since prior PAS mappings will 392 * have already checked themselves against this one. 393 */ 394 for (unsigned int i = idx + 1U; i < pas_region_cnt; i++) { 395 if (check_pas_overlap(pas_regions[idx].base_pa, 396 pas_regions[idx].size, 397 pas_regions[i].base_pa, 398 pas_regions[i].size)) { 399 ERROR("GPT: PAS[%u] overlaps with PAS[%u]\n", 400 i, idx); 401 return -EFAULT; 402 } 403 } 404 405 /* 406 * Since this function can be called multiple times with 407 * separate L1 tables we need to check the existing L0 mapping 408 * to see if this PAS would fall into one that has already been 409 * initialized. 410 */ 411 for (unsigned int i = 412 (unsigned int)GPT_L0_IDX(pas_regions[idx].base_pa); 413 i <= GPT_L0_IDX(pas_regions[idx].base_pa + 414 pas_regions[idx].size - 1UL); 415 i++) { 416 if ((GPT_L0_TYPE(l0_desc[i]) == GPT_L0_TYPE_BLK_DESC) && 417 (GPT_L0_BLKD_GPI(l0_desc[i]) == GPT_GPI_ANY)) { 418 /* This descriptor is unused so continue */ 419 continue; 420 } 421 422 /* 423 * This descriptor has been initialized in a previous 424 * call to this function so cannot be initialized again. 425 */ 426 ERROR("GPT: PAS[%u] overlaps with previous L0[%u]!\n", 427 idx, i); 428 return -EFAULT; 429 } 430 431 /* Check for block mapping (L0) type */ 432 if (GPT_PAS_ATTR_MAP_TYPE(pas_regions[idx].attrs) == 433 GPT_PAS_ATTR_MAP_TYPE_BLOCK) { 434 /* Make sure base and size are block-aligned */ 435 if (!GPT_IS_L0_ALIGNED(pas_regions[idx].base_pa) || 436 !GPT_IS_L0_ALIGNED(pas_regions[idx].size)) { 437 ERROR("GPT: PAS[%u] is not block-aligned!\n", 438 idx); 439 return -EFAULT; 440 } 441 442 continue; 443 } 444 445 /* Check for granule mapping (L1) type */ 446 if (GPT_PAS_ATTR_MAP_TYPE(pas_regions[idx].attrs) == 447 GPT_PAS_ATTR_MAP_TYPE_GRANULE) { 448 /* Make sure base and size are granule-aligned */ 449 if (!GPT_IS_L1_ALIGNED(gpt_config.p, pas_regions[idx].base_pa) || 450 !GPT_IS_L1_ALIGNED(gpt_config.p, pas_regions[idx].size)) { 451 ERROR("GPT: PAS[%u] is not granule-aligned!\n", 452 idx); 453 return -EFAULT; 454 } 455 456 /* Find how many L1 tables this PAS occupies */ 457 pas_l1_cnt = (GPT_L0_IDX(pas_regions[idx].base_pa + 458 pas_regions[idx].size - 1UL) - 459 GPT_L0_IDX(pas_regions[idx].base_pa) + 1U); 460 461 /* 462 * This creates a situation where, if multiple PAS 463 * regions occupy the same table descriptor, we can get 464 * an artificially high total L1 table count. The way we 465 * handle this is by checking each PAS against those 466 * before it in the array, and if they both occupy the 467 * same PAS we subtract from pas_l1_cnt and only the 468 * first PAS in the array gets to count it. 469 */ 470 471 /* 472 * If L1 count is greater than 1 we know the start and 473 * end PAs are in different L0 regions so we must check 474 * both for overlap against other PAS. 475 */ 476 if (pas_l1_cnt > 1) { 477 if (does_previous_pas_exist_here( 478 GPT_L0_IDX(pas_regions[idx].base_pa + 479 pas_regions[idx].size - 1UL), 480 pas_regions, idx)) { 481 pas_l1_cnt--; 482 } 483 } 484 485 if (does_previous_pas_exist_here( 486 GPT_L0_IDX(pas_regions[idx].base_pa), 487 pas_regions, idx)) { 488 pas_l1_cnt--; 489 } 490 491 l1_cnt += pas_l1_cnt; 492 continue; 493 } 494 495 /* If execution reaches this point, mapping type is invalid */ 496 ERROR("GPT: PAS[%u] has invalid mapping type 0x%x.\n", idx, 497 GPT_PAS_ATTR_MAP_TYPE(pas_regions[idx].attrs)); 498 return -EINVAL; 499 } 500 501 return l1_cnt; 502 } 503 504 /* 505 * This function validates L0 initialization parameters. 506 * 507 * Parameters 508 * l0_mem_base Base address of memory used for L0 table. 509 * l0_mem_size Size of memory available for L0 table. 510 * 511 * Return 512 * Negative Linux error code in the event of a failure, 0 for success. 513 */ 514 static int validate_l0_params(gpccr_pps_e pps, uintptr_t l0_mem_base, 515 size_t l0_mem_size) 516 { 517 size_t l0_alignment; 518 519 /* 520 * Make sure PPS is valid and then store it since macros need this value 521 * to work. 522 */ 523 if (pps > GPT_PPS_MAX) { 524 ERROR("GPT: Invalid PPS: 0x%x\n", pps); 525 return -EINVAL; 526 } 527 gpt_config.pps = pps; 528 gpt_config.t = gpt_t_lookup[pps]; 529 530 /* Alignment must be the greater of 4KB or L0 table size */ 531 l0_alignment = SZ_4K; 532 if (l0_alignment < GPT_L0_TABLE_SIZE(gpt_config.t)) { 533 l0_alignment = GPT_L0_TABLE_SIZE(gpt_config.t); 534 } 535 536 /* Check base address */ 537 if ((l0_mem_base == 0UL) || 538 ((l0_mem_base & (l0_alignment - 1UL)) != 0UL)) { 539 ERROR("GPT: Invalid L0 base address: 0x%lx\n", l0_mem_base); 540 return -EFAULT; 541 } 542 543 /* Check memory size for L0 table */ 544 if (l0_mem_size < GPT_L0_TABLE_SIZE(gpt_config.t)) { 545 ERROR("GPT: Inadequate L0 memory\n"); 546 ERROR(" Expected 0x%lx bytes, got 0x%lx\n", 547 GPT_L0_TABLE_SIZE(gpt_config.t), l0_mem_size); 548 return -ENOMEM; 549 } 550 551 return 0; 552 } 553 554 /* 555 * In the event that L1 tables are needed, this function validates 556 * the L1 table generation parameters. 557 * 558 * Parameters 559 * l1_mem_base Base address of memory used for L1 table allocation. 560 * l1_mem_size Total size of memory available for L1 tables. 561 * l1_gpt_cnt Number of L1 tables needed. 562 * 563 * Return 564 * Negative Linux error code in the event of a failure, 0 for success. 565 */ 566 static int validate_l1_params(uintptr_t l1_mem_base, size_t l1_mem_size, 567 unsigned int l1_gpt_cnt) 568 { 569 size_t l1_gpt_mem_sz; 570 571 /* Check if the granularity is supported */ 572 if (!xlat_arch_is_granule_size_supported( 573 GPT_PGS_ACTUAL_SIZE(gpt_config.p))) { 574 return -EPERM; 575 } 576 577 /* Make sure L1 tables are aligned to their size */ 578 if ((l1_mem_base & (GPT_L1_TABLE_SIZE(gpt_config.p) - 1UL)) != 0UL) { 579 ERROR("GPT: Unaligned L1 GPT base address: 0x%"PRIxPTR"\n", 580 l1_mem_base); 581 return -EFAULT; 582 } 583 584 /* Get total memory needed for L1 tables */ 585 l1_gpt_mem_sz = l1_gpt_cnt * GPT_L1_TABLE_SIZE(gpt_config.p); 586 587 /* Check for overflow */ 588 if ((l1_gpt_mem_sz / GPT_L1_TABLE_SIZE(gpt_config.p)) != l1_gpt_cnt) { 589 ERROR("GPT: Overflow calculating L1 memory size\n"); 590 return -ENOMEM; 591 } 592 593 /* Make sure enough space was supplied */ 594 if (l1_mem_size < l1_gpt_mem_sz) { 595 ERROR("%sL1 GPTs%s", (const char *)"GPT: Inadequate ", 596 (const char *)" memory\n"); 597 ERROR(" Expected 0x%lx bytes, got 0x%lx\n", 598 l1_gpt_mem_sz, l1_mem_size); 599 return -ENOMEM; 600 } 601 602 VERBOSE("GPT: Requested 0x%lx bytes for L1 GPTs\n", l1_gpt_mem_sz); 603 return 0; 604 } 605 606 /* 607 * This function initializes L0 block descriptors (regions that cannot be 608 * transitioned at the granule level) according to the provided PAS. 609 * 610 * Parameters 611 * *pas Pointer to the structure defining the PAS region to 612 * initialize. 613 */ 614 static void generate_l0_blk_desc(pas_region_t *pas) 615 { 616 uint64_t gpt_desc; 617 unsigned long idx, end_idx; 618 uint64_t *l0_gpt_arr; 619 620 assert(gpt_config.plat_gpt_l0_base != 0UL); 621 assert(pas != NULL); 622 623 /* 624 * Checking of PAS parameters has already been done in 625 * validate_pas_mappings so no need to check the same things again. 626 */ 627 628 l0_gpt_arr = (uint64_t *)gpt_config.plat_gpt_l0_base; 629 630 /* Create the GPT Block descriptor for this PAS region */ 631 gpt_desc = GPT_L0_BLK_DESC(GPT_PAS_ATTR_GPI(pas->attrs)); 632 633 /* Start index of this region in L0 GPTs */ 634 idx = GPT_L0_IDX(pas->base_pa); 635 636 /* 637 * Determine number of L0 GPT descriptors covered by 638 * this PAS region and use the count to populate these 639 * descriptors. 640 */ 641 end_idx = GPT_L0_IDX(pas->base_pa + pas->size); 642 643 /* Generate the needed block descriptors */ 644 for (; idx < end_idx; idx++) { 645 l0_gpt_arr[idx] = gpt_desc; 646 VERBOSE("GPT: L0 entry (BLOCK) index %lu [%p]: GPI = 0x%"PRIx64" (0x%"PRIx64")\n", 647 idx, &l0_gpt_arr[idx], 648 (gpt_desc >> GPT_L0_BLK_DESC_GPI_SHIFT) & 649 GPT_L0_BLK_DESC_GPI_MASK, l0_gpt_arr[idx]); 650 } 651 } 652 653 /* 654 * Helper function to determine if the end physical address lies in the same L0 655 * region as the current physical address. If true, the end physical address is 656 * returned else, the start address of the next region is returned. 657 * 658 * Parameters 659 * cur_pa Physical address of the current PA in the loop through 660 * the range. 661 * end_pa Physical address of the end PA in a PAS range. 662 * 663 * Return 664 * The PA of the end of the current range. 665 */ 666 static uintptr_t get_l1_end_pa(uintptr_t cur_pa, uintptr_t end_pa) 667 { 668 uintptr_t cur_idx; 669 uintptr_t end_idx; 670 671 cur_idx = GPT_L0_IDX(cur_pa); 672 end_idx = GPT_L0_IDX(end_pa); 673 674 assert(cur_idx <= end_idx); 675 676 if (cur_idx == end_idx) { 677 return end_pa; 678 } 679 680 return (cur_idx + 1UL) << GPT_L0_IDX_SHIFT; 681 } 682 683 /* 684 * Helper function to fill out GPI entries from 'first' granule address of 685 * the specified 'length' in a single L1 table with 'l1_desc' Contiguous 686 * descriptor. 687 * 688 * Parameters 689 * l1 Pointer to L1 table to fill out 690 * first Address of first granule in range 691 * length Length of the range in bytes 692 * gpi GPI set this range to 693 * 694 * Return 695 * Address of next granule in range. 696 */ 697 __unused static uintptr_t fill_l1_cont_desc(uint64_t *l1, uintptr_t first, 698 size_t length, unsigned int gpi) 699 { 700 /* 701 * Look up table for contiguous blocks and descriptors. 702 * Entries should be defined in descending block sizes: 703 * 512MB, 32MB and 2MB. 704 */ 705 static const gpt_fill_lookup_t gpt_fill_lookup[] = { 706 #if (RME_GPT_MAX_BLOCK == 512) 707 { SZ_512M, GPT_L1_CONT_DESC_512MB }, 708 #endif 709 #if (RME_GPT_MAX_BLOCK >= 32) 710 { SZ_32M, GPT_L1_CONT_DESC_32MB }, 711 #endif 712 #if (RME_GPT_MAX_BLOCK != 0) 713 { SZ_2M, GPT_L1_CONT_DESC_2MB } 714 #endif 715 }; 716 717 /* 718 * Iterate through all block sizes (512MB, 32MB and 2MB) 719 * starting with maximum supported. 720 */ 721 for (unsigned long i = 0UL; i < ARRAY_SIZE(gpt_fill_lookup); i++) { 722 /* Calculate index */ 723 unsigned long idx = GPT_L1_INDEX(first); 724 725 /* Contiguous block size */ 726 size_t cont_size = gpt_fill_lookup[i].size; 727 728 if (GPT_REGION_IS_CONT(length, first, cont_size)) { 729 730 /* Generate Contiguous descriptor */ 731 uint64_t l1_desc = GPT_L1_GPI_CONT_DESC(gpi, 732 gpt_fill_lookup[i].desc); 733 734 /* Number of 128-bit L1 entries in block */ 735 unsigned int cnt; 736 737 switch (cont_size) { 738 case SZ_512M: 739 cnt = L1_QWORDS_512MB; 740 break; 741 case SZ_32M: 742 cnt = L1_QWORDS_32MB; 743 break; 744 default: /* SZ_2MB */ 745 cnt = L1_QWORDS_2MB; 746 } 747 748 VERBOSE("GPT: Contiguous descriptor 0x%"PRIxPTR" %luMB\n", 749 first, cont_size / SZ_1M); 750 751 /* Fill Contiguous descriptors */ 752 fill_desc(&l1[idx], l1_desc, cnt); 753 return (first + cont_size); 754 } 755 } 756 757 return first; 758 } 759 760 /* Build Granules descriptor with the same 'gpi' for every GPI entry */ 761 static uint64_t build_l1_desc(unsigned int gpi) 762 { 763 uint64_t l1_desc = (uint64_t)gpi | ((uint64_t)gpi << 4); 764 765 l1_desc |= (l1_desc << 8); 766 l1_desc |= (l1_desc << 16); 767 return (l1_desc | (l1_desc << 32)); 768 } 769 770 /* 771 * Helper function to fill out GPI entries from 'first' to 'last' granule 772 * address in a single L1 table with 'l1_desc' Granules descriptor. 773 * 774 * Parameters 775 * l1 Pointer to L1 table to fill out 776 * first Address of first granule in range 777 * last Address of last granule in range (inclusive) 778 * gpi GPI set this range to 779 * 780 * Return 781 * Address of next granule in range. 782 */ 783 static uintptr_t fill_l1_gran_desc(uint64_t *l1, uintptr_t first, 784 uintptr_t last, unsigned int gpi) 785 { 786 uint64_t gpi_mask; 787 unsigned long i; 788 789 /* Generate Granules descriptor */ 790 uint64_t l1_desc = build_l1_desc(gpi); 791 792 /* Shift the mask if we're starting in the middle of an L1 entry */ 793 gpi_mask = ULONG_MAX << (GPT_L1_GPI_IDX(gpt_config.p, first) << 2); 794 795 /* Fill out each L1 entry for this region */ 796 for (i = GPT_L1_INDEX(first); i <= GPT_L1_INDEX(last); i++) { 797 798 /* Account for stopping in the middle of an L1 entry */ 799 if (i == GPT_L1_INDEX(last)) { 800 gpi_mask &= (gpi_mask >> ((15U - 801 GPT_L1_GPI_IDX(gpt_config.p, last)) << 2)); 802 } 803 804 assert((l1[i] & gpi_mask) == (GPT_L1_ANY_DESC & gpi_mask)); 805 806 /* Write GPI values */ 807 l1[i] = (l1[i] & ~gpi_mask) | (l1_desc & gpi_mask); 808 809 /* Reset mask */ 810 gpi_mask = ULONG_MAX; 811 } 812 813 return last + GPT_PGS_ACTUAL_SIZE(gpt_config.p); 814 } 815 816 /* 817 * Helper function to fill out GPI entries in a single L1 table. 818 * This function fills out an entire L1 table with either Granules or Contiguous 819 * (RME_GPT_MAX_BLOCK != 0) descriptors depending on region length and alignment. 820 * Note. If RME_GPT_MAX_BLOCK == 0, then the L1 tables are filled with regular 821 * Granules descriptors. 822 * 823 * Parameters 824 * l1 Pointer to L1 table to fill out 825 * first Address of first granule in range 826 * last Address of last granule in range (inclusive) 827 * gpi GPI set this range to 828 */ 829 static void fill_l1_tbl(uint64_t *l1, uintptr_t first, uintptr_t last, 830 unsigned int gpi) 831 { 832 assert(l1 != NULL); 833 assert(first <= last); 834 assert((first & (GPT_PGS_ACTUAL_SIZE(gpt_config.p) - 1UL)) == 0UL); 835 assert((last & (GPT_PGS_ACTUAL_SIZE(gpt_config.p) - 1UL)) == 0UL); 836 assert(GPT_L0_IDX(first) == GPT_L0_IDX(last)); 837 838 #if (RME_GPT_MAX_BLOCK != 0) 839 while (first <= last) { 840 /* Region length */ 841 size_t length = last - first + GPT_PGS_ACTUAL_SIZE(gpt_config.p); 842 843 if (length < SZ_2M) { 844 /* 845 * Fill with Granule descriptors in case of 846 * region length < 2MB. 847 */ 848 first = fill_l1_gran_desc(l1, first, last, gpi); 849 850 } else if ((first & (SZ_2M - UL(1))) == UL(0)) { 851 /* 852 * For region length >= 2MB and at least 2MB aligned 853 * call to fill_l1_cont_desc will iterate through 854 * all block sizes (512MB, 32MB and 2MB) supported and 855 * fill corresponding Contiguous descriptors. 856 */ 857 first = fill_l1_cont_desc(l1, first, length, gpi); 858 } else { 859 /* 860 * For not aligned region >= 2MB fill with Granules 861 * descriptors up to the next 2MB aligned address. 862 */ 863 uintptr_t new_last = ALIGN_2MB(first + SZ_2M) - 864 GPT_PGS_ACTUAL_SIZE(gpt_config.p); 865 866 first = fill_l1_gran_desc(l1, first, new_last, gpi); 867 } 868 } 869 #else 870 /* Fill with Granule descriptors */ 871 first = fill_l1_gran_desc(l1, first, last, gpi); 872 #endif 873 assert(first == (last + GPT_PGS_ACTUAL_SIZE(gpt_config.p))); 874 } 875 876 /* 877 * This function finds the next available unused L1 table and initializes all 878 * granules descriptor entries to GPI_ANY. This ensures that there are no chunks 879 * of GPI_NO_ACCESS (0b0000) memory floating around in the system in the 880 * event that a PAS region stops midway through an L1 table, thus guaranteeing 881 * that all memory not explicitly assigned is GPI_ANY. This function does not 882 * check for overflow conditions, that should be done by the caller. 883 * 884 * Return 885 * Pointer to the next available L1 table. 886 */ 887 static uint64_t *get_new_l1_tbl(void) 888 { 889 /* Retrieve the next L1 table */ 890 uint64_t *l1 = (uint64_t *)gpt_l1_tbl; 891 892 /* Increment L1 GPT address */ 893 gpt_l1_tbl += GPT_L1_TABLE_SIZE(gpt_config.p); 894 895 /* Initialize all GPIs to GPT_GPI_ANY */ 896 for (unsigned int i = 0U; i < GPT_L1_ENTRY_COUNT(gpt_config.p); i++) { 897 l1[i] = GPT_L1_ANY_DESC; 898 } 899 900 return l1; 901 } 902 903 /* 904 * When L1 tables are needed, this function creates the necessary L0 table 905 * descriptors and fills out the L1 table entries according to the supplied 906 * PAS range. 907 * 908 * Parameters 909 * *pas Pointer to the structure defining the PAS region. 910 */ 911 static void generate_l0_tbl_desc(pas_region_t *pas) 912 { 913 uintptr_t end_pa; 914 uintptr_t cur_pa; 915 uintptr_t last_gran_pa; 916 uint64_t *l0_gpt_base; 917 uint64_t *l1_gpt_arr; 918 unsigned int l0_idx, gpi; 919 920 assert(gpt_config.plat_gpt_l0_base != 0UL); 921 assert(pas != NULL); 922 923 /* 924 * Checking of PAS parameters has already been done in 925 * validate_pas_mappings so no need to check the same things again. 926 */ 927 end_pa = pas->base_pa + pas->size; 928 l0_gpt_base = (uint64_t *)gpt_config.plat_gpt_l0_base; 929 930 /* We start working from the granule at base PA */ 931 cur_pa = pas->base_pa; 932 933 /* Get GPI */ 934 gpi = GPT_PAS_ATTR_GPI(pas->attrs); 935 936 /* Iterate over each L0 region in this memory range */ 937 for (l0_idx = (unsigned int)GPT_L0_IDX(pas->base_pa); 938 l0_idx <= (unsigned int)GPT_L0_IDX(end_pa - 1UL); 939 l0_idx++) { 940 /* 941 * See if the L0 entry is already a table descriptor or if we 942 * need to create one. 943 */ 944 if (GPT_L0_TYPE(l0_gpt_base[l0_idx]) == GPT_L0_TYPE_TBL_DESC) { 945 /* Get the L1 array from the L0 entry */ 946 l1_gpt_arr = GPT_L0_TBLD_ADDR(l0_gpt_base[l0_idx]); 947 } else { 948 /* Get a new L1 table from the L1 memory space */ 949 l1_gpt_arr = get_new_l1_tbl(); 950 951 /* Fill out the L0 descriptor and flush it */ 952 l0_gpt_base[l0_idx] = GPT_L0_TBL_DESC(l1_gpt_arr); 953 } 954 955 VERBOSE("GPT: L0 entry (TABLE) index %u [%p] ==> L1 Addr %p (0x%"PRIx64")\n", 956 l0_idx, &l0_gpt_base[l0_idx], l1_gpt_arr, l0_gpt_base[l0_idx]); 957 958 /* 959 * Determine the PA of the last granule in this L0 descriptor. 960 */ 961 last_gran_pa = get_l1_end_pa(cur_pa, end_pa) - 962 GPT_PGS_ACTUAL_SIZE(gpt_config.p); 963 964 /* 965 * Fill up L1 GPT entries between these two addresses. This 966 * function needs the addresses of the first granule and last 967 * granule in the range. 968 */ 969 fill_l1_tbl(l1_gpt_arr, cur_pa, last_gran_pa, gpi); 970 971 /* Advance cur_pa to first granule in next L0 region */ 972 cur_pa = get_l1_end_pa(cur_pa, end_pa); 973 } 974 } 975 976 /* 977 * This function flushes a range of L0 descriptors used by a given PAS region 978 * array. There is a chance that some unmodified L0 descriptors would be flushed 979 * in the case that there are "holes" in an array of PAS regions but overall 980 * this should be faster than individually flushing each modified L0 descriptor 981 * as they are created. 982 * 983 * Parameters 984 * *pas Pointer to an array of PAS regions. 985 * pas_count Number of entries in the PAS array. 986 */ 987 static void flush_l0_for_pas_array(pas_region_t *pas, unsigned int pas_count) 988 { 989 unsigned long idx; 990 unsigned long start_idx; 991 unsigned long end_idx; 992 uint64_t *l0 = (uint64_t *)gpt_config.plat_gpt_l0_base; 993 994 assert(pas != NULL); 995 assert(pas_count != 0U); 996 997 /* Initial start and end values */ 998 start_idx = GPT_L0_IDX(pas[0].base_pa); 999 end_idx = GPT_L0_IDX(pas[0].base_pa + pas[0].size - 1UL); 1000 1001 /* Find lowest and highest L0 indices used in this PAS array */ 1002 for (idx = 1UL; idx < pas_count; idx++) { 1003 if (GPT_L0_IDX(pas[idx].base_pa) < start_idx) { 1004 start_idx = GPT_L0_IDX(pas[idx].base_pa); 1005 } 1006 if (GPT_L0_IDX(pas[idx].base_pa + pas[idx].size - 1UL) > end_idx) { 1007 end_idx = GPT_L0_IDX(pas[idx].base_pa + pas[idx].size - 1UL); 1008 } 1009 } 1010 1011 /* 1012 * Flush all covered L0 descriptors, add 1 because we need to include 1013 * the end index value. 1014 */ 1015 flush_dcache_range((uintptr_t)&l0[start_idx], 1016 ((end_idx + 1UL) - start_idx) * sizeof(uint64_t)); 1017 } 1018 1019 /* 1020 * Public API to enable granule protection checks once the tables have all been 1021 * initialized. This function is called at first initialization and then again 1022 * later during warm boots of CPU cores. 1023 * 1024 * Return 1025 * Negative Linux error code in the event of a failure, 0 for success. 1026 */ 1027 int gpt_enable(void) 1028 { 1029 u_register_t gpccr_el3; 1030 1031 /* 1032 * Granule tables must be initialised before enabling 1033 * granule protection. 1034 */ 1035 if (gpt_config.plat_gpt_l0_base == 0UL) { 1036 ERROR("GPT: Tables have not been initialized!\n"); 1037 return -EPERM; 1038 } 1039 1040 /* Write the base address of the L0 tables into GPTBR */ 1041 write_gptbr_el3(((gpt_config.plat_gpt_l0_base >> GPTBR_BADDR_VAL_SHIFT) 1042 >> GPTBR_BADDR_SHIFT) & GPTBR_BADDR_MASK); 1043 1044 /* GPCCR_EL3.PPS */ 1045 gpccr_el3 = SET_GPCCR_PPS(gpt_config.pps); 1046 1047 /* GPCCR_EL3.PGS */ 1048 gpccr_el3 |= SET_GPCCR_PGS(gpt_config.pgs); 1049 1050 /* 1051 * Since EL3 maps the L1 region as Inner shareable, use the same 1052 * shareability attribute for GPC as well so that 1053 * GPC fetches are visible to PEs 1054 */ 1055 gpccr_el3 |= SET_GPCCR_SH(GPCCR_SH_IS); 1056 1057 /* Outer and Inner cacheability set to Normal memory, WB, RA, WA */ 1058 gpccr_el3 |= SET_GPCCR_ORGN(GPCCR_ORGN_WB_RA_WA); 1059 gpccr_el3 |= SET_GPCCR_IRGN(GPCCR_IRGN_WB_RA_WA); 1060 1061 /* Enable NSP and SA if FEAT_RME_GDI is implemented */ 1062 if (is_feat_rme_gdi_supported()) { 1063 gpccr_el3 |= GPCCR_NSP_BIT; 1064 gpccr_el3 |= GPCCR_SA_BIT; 1065 } 1066 1067 /* Prepopulate GPCCR_EL3 but don't enable GPC yet */ 1068 write_gpccr_el3(gpccr_el3); 1069 isb(); 1070 1071 /* Invalidate any stale TLB entries and any cached register fields */ 1072 tlbipaallos(); 1073 dsb(); 1074 isb(); 1075 1076 /* Enable GPT */ 1077 gpccr_el3 |= GPCCR_GPC_BIT; 1078 1079 /* Enable NSO encoding if FEAT_RME_GPC2 is supported. */ 1080 if (is_feat_rme_gpc2_present()) { 1081 gpccr_el3 |= GPCCR_NSO_BIT; 1082 } 1083 1084 /* TODO: Configure GPCCR_EL3_GPCP for Fault control */ 1085 write_gpccr_el3(gpccr_el3); 1086 isb(); 1087 tlbipaallos(); 1088 dsb(); 1089 isb(); 1090 1091 return 0; 1092 } 1093 1094 /* 1095 * Public API to disable granule protection checks. 1096 */ 1097 void gpt_disable(void) 1098 { 1099 u_register_t gpccr_el3 = read_gpccr_el3(); 1100 1101 write_gpccr_el3(gpccr_el3 & ~GPCCR_GPC_BIT); 1102 dsbsy(); 1103 isb(); 1104 } 1105 1106 /* 1107 * Public API that initializes the entire protected space to GPT_GPI_ANY using 1108 * the L0 tables (block descriptors). Ideally, this function is invoked prior 1109 * to DDR discovery and initialization. The MMU must be initialized before 1110 * calling this function. 1111 * 1112 * Parameters 1113 * pps PPS value to use for table generation 1114 * l0_mem_base Base address of L0 tables in memory. 1115 * l0_mem_size Total size of memory available for L0 tables. 1116 * 1117 * Return 1118 * Negative Linux error code in the event of a failure, 0 for success. 1119 */ 1120 int gpt_init_l0_tables(gpccr_pps_e pps, uintptr_t l0_mem_base, 1121 size_t l0_mem_size) 1122 { 1123 uint64_t gpt_desc; 1124 int ret; 1125 1126 /* Ensure that MMU and Data caches are enabled */ 1127 assert((read_sctlr_el3() & SCTLR_C_BIT) != 0UL); 1128 1129 /* Validate other parameters */ 1130 ret = validate_l0_params(pps, l0_mem_base, l0_mem_size); 1131 if (ret != 0) { 1132 return ret; 1133 } 1134 1135 /* Create the descriptor to initialize L0 entries with */ 1136 gpt_desc = GPT_L0_BLK_DESC(GPT_GPI_ANY); 1137 1138 /* Iterate through all L0 entries */ 1139 for (unsigned int i = 0U; i < GPT_L0_REGION_COUNT(gpt_config.t); i++) { 1140 ((uint64_t *)l0_mem_base)[i] = gpt_desc; 1141 } 1142 1143 /* Flush updated L0 table to memory */ 1144 flush_dcache_range((uintptr_t)l0_mem_base, GPT_L0_TABLE_SIZE(gpt_config.t)); 1145 1146 /* Stash the L0 base address once initial setup is complete */ 1147 gpt_config.plat_gpt_l0_base = l0_mem_base; 1148 1149 return 0; 1150 } 1151 1152 /* 1153 * Public API that carves out PAS regions from the L0 tables and builds any L1 1154 * tables that are needed. This function ideally is run after DDR discovery and 1155 * initialization. The L0 tables must have already been initialized to GPI_ANY 1156 * when this function is called. 1157 * 1158 * This function can be called multiple times with different L1 memory ranges 1159 * and PAS regions if it is desirable to place L1 tables in different locations 1160 * in memory. (ex: you have multiple DDR banks and want to place the L1 tables 1161 * in the DDR bank that they control). 1162 * 1163 * Parameters 1164 * pgs PGS value to use for table generation. 1165 * l1_mem_base Base address of memory used for L1 tables. 1166 * l1_mem_size Total size of memory available for L1 tables. 1167 * *pas_regions Pointer to PAS regions structure array. 1168 * pas_count Total number of PAS regions. 1169 * 1170 * Return 1171 * Negative Linux error code in the event of a failure, 0 for success. 1172 */ 1173 int gpt_init_pas_l1_tables(gpccr_pgs_e pgs, uintptr_t l1_mem_base, 1174 size_t l1_mem_size, pas_region_t *pas_regions, 1175 unsigned int pas_count) 1176 { 1177 int l1_gpt_cnt, ret; 1178 1179 /* Ensure that MMU and Data caches are enabled */ 1180 assert((read_sctlr_el3() & SCTLR_C_BIT) != 0UL); 1181 1182 /* PGS is needed for validate_pas_mappings so check it now */ 1183 if (pgs > GPT_PGS_MAX) { 1184 ERROR("GPT: Invalid PGS: 0x%x\n", pgs); 1185 return -EINVAL; 1186 } 1187 gpt_config.pgs = pgs; 1188 gpt_config.p = gpt_p_lookup[pgs]; 1189 1190 /* Make sure L0 tables have been initialized */ 1191 if (gpt_config.plat_gpt_l0_base == 0UL) { 1192 ERROR("GPT: L0 tables must be initialized first!\n"); 1193 return -EPERM; 1194 } 1195 1196 /* Check if L1 GPTs are required and how many */ 1197 l1_gpt_cnt = validate_pas_mappings(pas_regions, pas_count); 1198 if (l1_gpt_cnt < 0) { 1199 return l1_gpt_cnt; 1200 } 1201 1202 VERBOSE("GPT: %i L1 GPTs requested\n", l1_gpt_cnt); 1203 1204 /* If L1 tables are needed then validate the L1 parameters */ 1205 if (l1_gpt_cnt > 0) { 1206 ret = validate_l1_params(l1_mem_base, l1_mem_size, 1207 (unsigned int)l1_gpt_cnt); 1208 if (ret != 0) { 1209 return ret; 1210 } 1211 1212 /* Set up parameters for L1 table generation */ 1213 gpt_l1_tbl = l1_mem_base; 1214 } 1215 1216 /* Number of L1 entries in 2MB depends on GPCCR_EL3.PGS value */ 1217 gpt_l1_cnt_2mb = (unsigned int)GPT_L1_ENTRY_COUNT_2MB(gpt_config.p); 1218 1219 /* Mask for the L1 index field */ 1220 gpt_l1_index_mask = GPT_L1_IDX_MASK(gpt_config.p); 1221 1222 INFO("GPT: Boot Configuration\n"); 1223 INFO(" PPS/T: 0x%x/%u\n", gpt_config.pps, gpt_config.t); 1224 INFO(" PGS/P: 0x%x/%u\n", gpt_config.pgs, gpt_config.p); 1225 INFO(" L0GPTSZ/S: 0x%x/%u\n", GPT_L0GPTSZ, GPT_S_VAL); 1226 INFO(" PAS count: %u\n", pas_count); 1227 INFO(" L0 base: 0x%"PRIxPTR"\n", gpt_config.plat_gpt_l0_base); 1228 1229 /* Generate the tables in memory */ 1230 for (unsigned int idx = 0U; idx < pas_count; idx++) { 1231 VERBOSE("GPT: PAS[%u]: base 0x%"PRIxPTR"\tsize 0x%lx\tGPI 0x%x\ttype 0x%x\n", 1232 idx, pas_regions[idx].base_pa, pas_regions[idx].size, 1233 GPT_PAS_ATTR_GPI(pas_regions[idx].attrs), 1234 GPT_PAS_ATTR_MAP_TYPE(pas_regions[idx].attrs)); 1235 1236 /* Check if a block or table descriptor is required */ 1237 if (GPT_PAS_ATTR_MAP_TYPE(pas_regions[idx].attrs) == 1238 GPT_PAS_ATTR_MAP_TYPE_BLOCK) { 1239 generate_l0_blk_desc(&pas_regions[idx]); 1240 1241 } else { 1242 generate_l0_tbl_desc(&pas_regions[idx]); 1243 } 1244 } 1245 1246 /* Flush modified L0 tables */ 1247 flush_l0_for_pas_array(pas_regions, pas_count); 1248 1249 /* Flush L1 tables if needed */ 1250 if (l1_gpt_cnt > 0) { 1251 flush_dcache_range(l1_mem_base, 1252 GPT_L1_TABLE_SIZE(gpt_config.p) * 1253 (size_t)l1_gpt_cnt); 1254 } 1255 1256 /* Make sure that all the entries are written to the memory */ 1257 dsbishst(); 1258 tlbipaallos(); 1259 dsb(); 1260 isb(); 1261 1262 return 0; 1263 } 1264 1265 /* 1266 * Public API to initialize the runtime gpt_config structure based on the values 1267 * present in the GPTBR_EL3 and GPCCR_EL3 registers. GPT initialization 1268 * typically happens in a bootloader stage prior to setting up the EL3 runtime 1269 * environment for the granule transition service so this function detects the 1270 * initialization from a previous stage. Granule protection checks must be 1271 * enabled already or this function will return an error. 1272 * 1273 * Parameters 1274 * l1_bitlocks_base Base address of memory for L1 tables bitlocks. 1275 * l1_bitlocks_size Total size of memory available for L1 tables bitlocks. 1276 * 1277 * Return 1278 * Negative Linux error code in the event of a failure, 0 for success. 1279 */ 1280 int gpt_runtime_init(uintptr_t l1_bitlocks_base, size_t l1_bitlocks_size) 1281 { 1282 u_register_t reg; 1283 __unused size_t locks_size; 1284 1285 /* Ensure that MMU and Data caches are enabled */ 1286 assert((read_sctlr_el3() & SCTLR_C_BIT) != 0UL); 1287 1288 /* Ensure GPC are already enabled */ 1289 if ((read_gpccr_el3() & GPCCR_GPC_BIT) == 0UL) { 1290 ERROR("GPT: Granule protection checks are not enabled!\n"); 1291 return -EPERM; 1292 } 1293 1294 /* 1295 * Read the L0 table address from GPTBR, we don't need the L1 base 1296 * address since those are included in the L0 tables as needed. 1297 */ 1298 reg = read_gptbr_el3(); 1299 gpt_config.plat_gpt_l0_base = ((reg >> GPTBR_BADDR_SHIFT) & 1300 GPTBR_BADDR_MASK) << 1301 GPTBR_BADDR_VAL_SHIFT; 1302 1303 /* Read GPCCR to get PGS and PPS values */ 1304 reg = read_gpccr_el3(); 1305 gpt_config.pps = (reg >> GPCCR_PPS_SHIFT) & GPCCR_PPS_MASK; 1306 gpt_config.t = gpt_t_lookup[gpt_config.pps]; 1307 gpt_config.pgs = (reg >> GPCCR_PGS_SHIFT) & GPCCR_PGS_MASK; 1308 gpt_config.p = gpt_p_lookup[gpt_config.pgs]; 1309 1310 /* Number of L1 entries in 2MB depends on GPCCR_EL3.PGS value */ 1311 gpt_l1_cnt_2mb = (unsigned int)GPT_L1_ENTRY_COUNT_2MB(gpt_config.p); 1312 1313 /* Mask for the L1 index field */ 1314 gpt_l1_index_mask = GPT_L1_IDX_MASK(gpt_config.p); 1315 1316 #if (RME_GPT_BITLOCK_BLOCK != 0) 1317 /* 1318 * Size of GPT bitlocks in bytes for the protected address space 1319 * with RME_GPT_BITLOCK_BLOCK * 512MB per bitlock. 1320 */ 1321 locks_size = GPT_PPS_ACTUAL_SIZE(gpt_config.t) / 1322 (RME_GPT_BITLOCK_BLOCK * SZ_512M * 8U); 1323 /* 1324 * If protected space size is less than the size covered 1325 * by 'bitlock' structure, check for a single bitlock. 1326 */ 1327 if (locks_size < LOCK_SIZE) { 1328 locks_size = LOCK_SIZE; 1329 /* Check bitlocks array size */ 1330 } else if (locks_size > l1_bitlocks_size) { 1331 ERROR("GPT: Inadequate GPT bitlocks memory\n"); 1332 ERROR(" Expected 0x%lx bytes, got 0x%lx\n", 1333 locks_size, l1_bitlocks_size); 1334 return -ENOMEM; 1335 } 1336 1337 gpt_bitlock = (bitlock_t *)l1_bitlocks_base; 1338 1339 /* Initialise GPT bitlocks */ 1340 (void)memset((void *)gpt_bitlock, 0, locks_size); 1341 1342 /* Flush GPT bitlocks to memory */ 1343 flush_dcache_range((uintptr_t)gpt_bitlock, locks_size); 1344 #endif /* RME_GPT_BITLOCK_BLOCK */ 1345 1346 VERBOSE("GPT: Runtime Configuration\n"); 1347 VERBOSE(" PPS/T: 0x%x/%u\n", gpt_config.pps, gpt_config.t); 1348 VERBOSE(" PGS/P: 0x%x/%u\n", gpt_config.pgs, gpt_config.p); 1349 VERBOSE(" L0GPTSZ/S: 0x%x/%u\n", GPT_L0GPTSZ, GPT_S_VAL); 1350 VERBOSE(" L0 base: 0x%"PRIxPTR"\n", gpt_config.plat_gpt_l0_base); 1351 #if (RME_GPT_BITLOCK_BLOCK != 0) 1352 VERBOSE(" Bitlocks: 0x%"PRIxPTR"/0x%lx\n", (uintptr_t)gpt_bitlock, 1353 locks_size); 1354 #endif 1355 return 0; 1356 } 1357 1358 /* 1359 * A helper to write the value (target_pas << gpi_shift) to the index of 1360 * the gpt_l1_addr. 1361 */ 1362 static inline void write_gpt(uint64_t *gpt_l1_desc, uint64_t *gpt_l1_addr, 1363 unsigned int gpi_shift, unsigned int idx, 1364 unsigned int target_pas) 1365 { 1366 *gpt_l1_desc &= ~(GPT_L1_GRAN_DESC_GPI_MASK << gpi_shift); 1367 *gpt_l1_desc |= ((uint64_t)target_pas << gpi_shift); 1368 gpt_l1_addr[idx] = *gpt_l1_desc; 1369 1370 dsboshst(); 1371 } 1372 1373 /* 1374 * Helper to retrieve the gpt_l1_* information from the base address 1375 * returned in gpi_info. 1376 */ 1377 static int get_gpi_params(uint64_t base, gpi_info_t *gpi_info) 1378 { 1379 uint64_t gpt_l0_desc, *gpt_l0_base; 1380 __unused unsigned int block_idx; 1381 1382 gpt_l0_base = (uint64_t *)gpt_config.plat_gpt_l0_base; 1383 gpt_l0_desc = gpt_l0_base[GPT_L0_IDX(base)]; 1384 if (GPT_L0_TYPE(gpt_l0_desc) != GPT_L0_TYPE_TBL_DESC) { 1385 VERBOSE("GPT: Granule is not covered by a table descriptor!\n"); 1386 VERBOSE(" Base=0x%"PRIx64"\n", base); 1387 return -EINVAL; 1388 } 1389 1390 /* Get the table index and GPI shift from PA */ 1391 gpi_info->gpt_l1_addr = GPT_L0_TBLD_ADDR(gpt_l0_desc); 1392 gpi_info->idx = (unsigned int)GPT_L1_INDEX(base); 1393 gpi_info->gpi_shift = GPT_L1_GPI_IDX(gpt_config.p, base) << 2; 1394 1395 #if (RME_GPT_BITLOCK_BLOCK != 0) 1396 /* Block index */ 1397 block_idx = (unsigned int)(base / (RME_GPT_BITLOCK_BLOCK * SZ_512M)); 1398 1399 /* Bitlock address and mask */ 1400 gpi_info->lock = &gpt_bitlock[block_idx / LOCK_BITS]; 1401 gpi_info->mask = 1U << (block_idx & (LOCK_BITS - 1U)); 1402 #endif 1403 return 0; 1404 } 1405 1406 /* 1407 * Helper to retrieve the gpt_l1_desc and GPI information from gpi_info. 1408 * This function is called with bitlock or spinlock acquired. 1409 */ 1410 static void read_gpi(gpi_info_t *gpi_info) 1411 { 1412 gpi_info->gpt_l1_desc = (gpi_info->gpt_l1_addr)[gpi_info->idx]; 1413 1414 if ((gpi_info->gpt_l1_desc & GPT_L1_TYPE_CONT_DESC_MASK) == 1415 GPT_L1_TYPE_CONT_DESC) { 1416 /* Read GPI from Contiguous descriptor */ 1417 gpi_info->gpi = (unsigned int)GPT_L1_CONT_GPI(gpi_info->gpt_l1_desc); 1418 } else { 1419 /* Read GPI from Granules descriptor */ 1420 gpi_info->gpi = (unsigned int)((gpi_info->gpt_l1_desc >> gpi_info->gpi_shift) & 1421 GPT_L1_GRAN_DESC_GPI_MASK); 1422 } 1423 } 1424 1425 static void flush_page_to_popa(uintptr_t addr) 1426 { 1427 size_t size = GPT_PGS_ACTUAL_SIZE(gpt_config.p); 1428 1429 if (is_feat_mte2_supported()) { 1430 flush_dcache_to_popa_range_mte2(addr, size); 1431 } else { 1432 flush_dcache_to_popa_range(addr, size); 1433 } 1434 } 1435 1436 /* 1437 * Helper function to check if all L1 entries in 2MB block have 1438 * the same Granules descriptor value. 1439 * 1440 * Parameters 1441 * base Base address of the region to be checked 1442 * gpi_info Pointer to 'gpt_config_t' structure 1443 * l1_desc GPT Granules descriptor with all entries 1444 * set to the same GPI. 1445 * 1446 * Return 1447 * true if L1 all entries have the same descriptor value, false otherwise. 1448 */ 1449 __unused static bool check_fuse_2mb(uint64_t base, const gpi_info_t *gpi_info, 1450 uint64_t l1_desc) 1451 { 1452 /* Last L1 entry index in 2MB block */ 1453 unsigned int long idx = GPT_L1_INDEX(ALIGN_2MB(base)) + 1454 gpt_l1_cnt_2mb - 1UL; 1455 1456 /* Number of L1 entries in 2MB block */ 1457 unsigned int cnt = gpt_l1_cnt_2mb; 1458 1459 /* 1460 * Start check from the last L1 entry and continue until the first 1461 * non-matching to the passed Granules descriptor value is found. 1462 */ 1463 while (cnt-- != 0U) { 1464 if (gpi_info->gpt_l1_addr[idx--] != l1_desc) { 1465 /* Non-matching L1 entry found */ 1466 return false; 1467 } 1468 } 1469 1470 return true; 1471 } 1472 1473 __unused static void fuse_2mb(uint64_t base, const gpi_info_t *gpi_info, 1474 uint64_t l1_desc) 1475 { 1476 /* L1 entry index of the start of 2MB block */ 1477 unsigned long idx_2 = GPT_L1_INDEX(ALIGN_2MB(base)); 1478 1479 /* 2MB Contiguous descriptor */ 1480 uint64_t l1_cont_desc = GPT_L1_CONT_DESC(l1_desc, 2MB); 1481 1482 VERBOSE("GPT: %s(0x%"PRIxPTR" 0x%"PRIx64")\n", __func__, base, l1_desc); 1483 1484 fill_desc(&gpi_info->gpt_l1_addr[idx_2], l1_cont_desc, L1_QWORDS_2MB); 1485 } 1486 1487 /* 1488 * Helper function to check if all 1st L1 entries of 2MB blocks 1489 * in 32MB have the same 2MB Contiguous descriptor value. 1490 * 1491 * Parameters 1492 * base Base address of the region to be checked 1493 * gpi_info Pointer to 'gpt_config_t' structure 1494 * l1_desc GPT Granules descriptor. 1495 * 1496 * Return 1497 * true if all L1 entries have the same descriptor value, false otherwise. 1498 */ 1499 __unused static bool check_fuse_32mb(uint64_t base, const gpi_info_t *gpi_info, 1500 uint64_t l1_desc) 1501 { 1502 /* The 1st L1 entry index of the last 2MB block in 32MB */ 1503 unsigned long idx = GPT_L1_INDEX(ALIGN_32MB(base)) + 1504 (15UL * gpt_l1_cnt_2mb); 1505 1506 /* 2MB Contiguous descriptor */ 1507 uint64_t l1_cont_desc = GPT_L1_CONT_DESC(l1_desc, 2MB); 1508 1509 /* Number of 2MB blocks in 32MB */ 1510 unsigned int cnt = 16U; 1511 1512 /* Set the first L1 entry to 2MB Contiguous descriptor */ 1513 gpi_info->gpt_l1_addr[GPT_L1_INDEX(ALIGN_2MB(base))] = l1_cont_desc; 1514 1515 /* 1516 * Start check from the 1st L1 entry of the last 2MB block and 1517 * continue until the first non-matching to 2MB Contiguous descriptor 1518 * value is found. 1519 */ 1520 while (cnt-- != 0U) { 1521 if (gpi_info->gpt_l1_addr[idx] != l1_cont_desc) { 1522 /* Non-matching L1 entry found */ 1523 return false; 1524 } 1525 idx -= gpt_l1_cnt_2mb; 1526 } 1527 1528 return true; 1529 } 1530 1531 __unused static void fuse_32mb(uint64_t base, const gpi_info_t *gpi_info, 1532 uint64_t l1_desc) 1533 { 1534 /* L1 entry index of the start of 32MB block */ 1535 unsigned long idx_32 = GPT_L1_INDEX(ALIGN_32MB(base)); 1536 1537 /* 32MB Contiguous descriptor */ 1538 uint64_t l1_cont_desc = GPT_L1_CONT_DESC(l1_desc, 32MB); 1539 1540 VERBOSE("GPT: %s(0x%"PRIxPTR" 0x%"PRIx64")\n", __func__, base, l1_desc); 1541 1542 fill_desc(&gpi_info->gpt_l1_addr[idx_32], l1_cont_desc, L1_QWORDS_32MB); 1543 } 1544 1545 /* 1546 * Helper function to check if all 1st L1 entries of 32MB blocks 1547 * in 512MB have the same 32MB Contiguous descriptor value. 1548 * 1549 * Parameters 1550 * base Base address of the region to be checked 1551 * gpi_info Pointer to 'gpt_config_t' structure 1552 * l1_desc GPT Granules descriptor. 1553 * 1554 * Return 1555 * true if all L1 entries have the same descriptor value, false otherwise. 1556 */ 1557 __unused static bool check_fuse_512mb(uint64_t base, const gpi_info_t *gpi_info, 1558 uint64_t l1_desc) 1559 { 1560 /* The 1st L1 entry index of the last 32MB block in 512MB */ 1561 unsigned long idx = GPT_L1_INDEX(ALIGN_512MB(base)) + 1562 (15UL * 16UL * gpt_l1_cnt_2mb); 1563 1564 /* 32MB Contiguous descriptor */ 1565 uint64_t l1_cont_desc = GPT_L1_CONT_DESC(l1_desc, 32MB); 1566 1567 /* Number of 32MB blocks in 512MB */ 1568 unsigned int cnt = 16U; 1569 1570 /* Set the first L1 entry to 2MB Contiguous descriptor */ 1571 gpi_info->gpt_l1_addr[GPT_L1_INDEX(ALIGN_32MB(base))] = l1_cont_desc; 1572 1573 /* 1574 * Start check from the 1st L1 entry of the last 32MB block and 1575 * continue until the first non-matching to 32MB Contiguous descriptor 1576 * value is found. 1577 */ 1578 while (cnt-- != 0U) { 1579 if (gpi_info->gpt_l1_addr[idx] != l1_cont_desc) { 1580 /* Non-matching L1 entry found */ 1581 return false; 1582 } 1583 idx -= 16UL * gpt_l1_cnt_2mb; 1584 } 1585 1586 return true; 1587 } 1588 1589 __unused static void fuse_512mb(uint64_t base, const gpi_info_t *gpi_info, 1590 uint64_t l1_desc) 1591 { 1592 /* L1 entry index of the start of 512MB block */ 1593 unsigned long idx_512 = GPT_L1_INDEX(ALIGN_512MB(base)); 1594 1595 /* 512MB Contiguous descriptor */ 1596 uint64_t l1_cont_desc = GPT_L1_CONT_DESC(l1_desc, 512MB); 1597 1598 VERBOSE("GPT: %s(0x%"PRIxPTR" 0x%"PRIx64")\n", __func__, base, l1_desc); 1599 1600 fill_desc(&gpi_info->gpt_l1_addr[idx_512], l1_cont_desc, L1_QWORDS_512MB); 1601 } 1602 1603 /* 1604 * Helper function to convert GPI entries in a single L1 table 1605 * from Granules to Contiguous descriptor. 1606 * 1607 * Parameters 1608 * base Base address of the region to be written 1609 * gpi_info Pointer to 'gpt_config_t' structure 1610 * l1_desc GPT Granules descriptor with all entries 1611 * set to the same GPI. 1612 */ 1613 __unused static void fuse_block(uint64_t base, const gpi_info_t *gpi_info, 1614 uint64_t l1_desc) 1615 { 1616 /* Start with check for 2MB block */ 1617 if (!check_fuse_2mb(base, gpi_info, l1_desc)) { 1618 /* Check for 2MB fusing failed */ 1619 return; 1620 } 1621 1622 #if (RME_GPT_MAX_BLOCK == 2) 1623 fuse_2mb(base, gpi_info, l1_desc); 1624 #else 1625 /* Check for 32MB block */ 1626 if (!check_fuse_32mb(base, gpi_info, l1_desc)) { 1627 /* Check for 32MB fusing failed, fuse to 2MB */ 1628 fuse_2mb(base, gpi_info, l1_desc); 1629 return; 1630 } 1631 1632 #if (RME_GPT_MAX_BLOCK == 32) 1633 fuse_32mb(base, gpi_info, l1_desc); 1634 #else 1635 /* Check for 512MB block */ 1636 if (!check_fuse_512mb(base, gpi_info, l1_desc)) { 1637 /* Check for 512MB fusing failed, fuse to 32MB */ 1638 fuse_32mb(base, gpi_info, l1_desc); 1639 return; 1640 } 1641 1642 /* Fuse to 512MB */ 1643 fuse_512mb(base, gpi_info, l1_desc); 1644 1645 #endif /* RME_GPT_MAX_BLOCK == 32 */ 1646 #endif /* RME_GPT_MAX_BLOCK == 2 */ 1647 } 1648 1649 /* 1650 * Helper function to convert GPI entries in a single L1 table 1651 * from Contiguous to Granules descriptor. This function updates 1652 * descriptor to Granules in passed 'gpt_config_t' structure as 1653 * the result of shuttering. 1654 * 1655 * Parameters 1656 * base Base address of the region to be written 1657 * gpi_info Pointer to 'gpt_config_t' structure 1658 * l1_desc GPT Granules descriptor set this range to. 1659 */ 1660 __unused static void shatter_block(uint64_t base, gpi_info_t *gpi_info, 1661 uint64_t l1_desc) 1662 { 1663 /* Look-up table for 2MB, 32MB and 512MB locks shattering */ 1664 static const gpt_shatter_func gpt_shatter_lookup[] = { 1665 shatter_2mb, 1666 shatter_32mb, 1667 shatter_512mb 1668 }; 1669 1670 /* Look-up table for invalidation TLBs for 2MB, 32MB and 512MB blocks */ 1671 static const gpt_tlbi_lookup_t tlbi_lookup[] = { 1672 { tlbirpalos_2m, ~(SZ_2M - 1UL) }, 1673 { tlbirpalos_32m, ~(SZ_32M - 1UL) }, 1674 { tlbirpalos_512m, ~(SZ_512M - 1UL) } 1675 }; 1676 1677 /* Get shattering level from Contig field of Contiguous descriptor */ 1678 unsigned long level = GPT_L1_CONT_CONTIG(gpi_info->gpt_l1_desc) - 1UL; 1679 1680 /* Shatter contiguous block */ 1681 gpt_shatter_lookup[level](base, gpi_info, l1_desc); 1682 1683 tlbi_lookup[level].function(base & tlbi_lookup[level].mask); 1684 dsbosh(); 1685 1686 /* 1687 * Update 'gpt_config_t' structure's descriptor to Granules to reflect 1688 * the shattered GPI back to caller. 1689 */ 1690 gpi_info->gpt_l1_desc = l1_desc; 1691 } 1692 1693 /* 1694 * This function is the granule transition delegate service. When a granule 1695 * transition request occurs it is routed to this function to have the request, 1696 * if valid, fulfilled following A1.1.1 Delegate of RME supplement. 1697 * 1698 * TODO: implement support for transitioning multiple granules at once. 1699 * 1700 * Parameters 1701 * base Base address of the region to transition, must be 1702 * aligned to granule size. 1703 * size Size of region to transition, must be aligned to granule 1704 * size. 1705 * src_sec_state Security state of the caller. 1706 * 1707 * Return 1708 * Negative Linux error code in the event of a failure, 0 for success. 1709 */ 1710 int gpt_delegate_pas(uint64_t base, size_t size, unsigned int src_sec_state) 1711 { 1712 gpi_info_t gpi_info; 1713 uint64_t nse, __unused l1_desc; 1714 unsigned int target_pas; 1715 int res; 1716 1717 /* Ensure that the tables have been set up before taking requests */ 1718 assert(gpt_config.plat_gpt_l0_base != 0UL); 1719 1720 /* Ensure that caches are enabled */ 1721 assert((read_sctlr_el3() & SCTLR_C_BIT) != 0UL); 1722 1723 /* See if this is a single or a range of granule transition */ 1724 if (size != GPT_PGS_ACTUAL_SIZE(gpt_config.p)) { 1725 return -EINVAL; 1726 } 1727 1728 /* Check that base and size are valid */ 1729 if ((ULONG_MAX - base) < size) { 1730 VERBOSE("GPT: Transition request address overflow!\n"); 1731 VERBOSE(" Base=0x%"PRIx64"\n", base); 1732 VERBOSE(" Size=0x%lx\n", size); 1733 return -EINVAL; 1734 } 1735 1736 /* Make sure base and size are valid */ 1737 if (((base & (GPT_PGS_ACTUAL_SIZE(gpt_config.p) - 1UL)) != 0UL) || 1738 ((size & (GPT_PGS_ACTUAL_SIZE(gpt_config.p) - 1UL)) != 0UL) || 1739 (size == 0UL) || 1740 ((base + size) >= GPT_PPS_ACTUAL_SIZE(gpt_config.t))) { 1741 VERBOSE("GPT: Invalid granule transition address range!\n"); 1742 VERBOSE(" Base=0x%"PRIx64"\n", base); 1743 VERBOSE(" Size=0x%lx\n", size); 1744 return -EINVAL; 1745 } 1746 1747 /* Delegate request can only come from REALM or SECURE */ 1748 if ((src_sec_state != SMC_FROM_REALM) && 1749 (src_sec_state != SMC_FROM_SECURE)) { 1750 VERBOSE("GPT: Invalid caller security state 0x%x\n", 1751 src_sec_state); 1752 return -EINVAL; 1753 } 1754 1755 if (src_sec_state == SMC_FROM_REALM) { 1756 target_pas = GPT_GPI_REALM; 1757 nse = (uint64_t)GPT_NSE_REALM << GPT_NSE_SHIFT; 1758 l1_desc = GPT_L1_REALM_DESC; 1759 } else { 1760 target_pas = GPT_GPI_SECURE; 1761 nse = (uint64_t)GPT_NSE_SECURE << GPT_NSE_SHIFT; 1762 l1_desc = GPT_L1_SECURE_DESC; 1763 } 1764 1765 res = get_gpi_params(base, &gpi_info); 1766 if (res != 0) { 1767 return res; 1768 } 1769 1770 /* 1771 * Access to GPT is controlled by a lock to ensure that no more 1772 * than one CPU is allowed to make changes at any given time. 1773 */ 1774 GPT_LOCK; 1775 read_gpi(&gpi_info); 1776 1777 /* Check that the current address is in NS state */ 1778 if (gpi_info.gpi != GPT_GPI_NS) { 1779 VERBOSE("GPT: Only Granule in NS state can be delegated.\n"); 1780 VERBOSE(" Caller: %u, Current GPI: %u\n", src_sec_state, 1781 gpi_info.gpi); 1782 GPT_UNLOCK; 1783 return -EPERM; 1784 } 1785 1786 #if (RME_GPT_MAX_BLOCK != 0) 1787 /* Check for Contiguous descriptor */ 1788 if ((gpi_info.gpt_l1_desc & GPT_L1_TYPE_CONT_DESC_MASK) == 1789 GPT_L1_TYPE_CONT_DESC) { 1790 shatter_block(base, &gpi_info, GPT_L1_NS_DESC); 1791 } 1792 #endif 1793 /* 1794 * In order to maintain mutual distrust between Realm and Secure 1795 * states, remove any data speculatively fetched into the target 1796 * physical address space. 1797 * Issue DC CIPAPA or DC_CIGDPAPA on implementations with FEAT_MTE2. 1798 */ 1799 flush_page_to_popa(base | nse); 1800 1801 write_gpt(&gpi_info.gpt_l1_desc, gpi_info.gpt_l1_addr, 1802 gpi_info.gpi_shift, gpi_info.idx, target_pas); 1803 1804 /* Ensure that all agents observe the new configuration */ 1805 tlbi_page_dsbosh(base); 1806 1807 nse = (uint64_t)GPT_NSE_NS << GPT_NSE_SHIFT; 1808 1809 /* Ensure that the scrubbed data have made it past the PoPA */ 1810 flush_page_to_popa(base | nse); 1811 1812 #if (RME_GPT_MAX_BLOCK != 0) 1813 if (gpi_info.gpt_l1_desc == l1_desc) { 1814 /* Try to fuse */ 1815 fuse_block(base, &gpi_info, l1_desc); 1816 } 1817 #endif 1818 1819 /* Unlock the lock to GPT */ 1820 GPT_UNLOCK; 1821 1822 /* 1823 * The isb() will be done as part of context 1824 * synchronization when returning to lower EL. 1825 */ 1826 VERBOSE("GPT: Granule 0x%"PRIx64" GPI 0x%x->0x%x\n", 1827 base, gpi_info.gpi, target_pas); 1828 1829 return 0; 1830 } 1831 1832 /* 1833 * This function is the granule transition undelegate service. When a granule 1834 * transition request occurs it is routed to this function where the request is 1835 * validated then fulfilled if possible. 1836 * 1837 * TODO: implement support for transitioning multiple granules at once. 1838 * 1839 * Parameters 1840 * base Base address of the region to transition, must be 1841 * aligned to granule size. 1842 * size Size of region to transition, must be aligned to granule 1843 * size. 1844 * src_sec_state Security state of the caller. 1845 * 1846 * Return 1847 * Negative Linux error code in the event of a failure, 0 for success. 1848 */ 1849 int gpt_undelegate_pas(uint64_t base, size_t size, unsigned int src_sec_state) 1850 { 1851 gpi_info_t gpi_info; 1852 uint64_t nse, __unused l1_desc; 1853 int res; 1854 1855 /* Ensure that the tables have been set up before taking requests */ 1856 assert(gpt_config.plat_gpt_l0_base != 0UL); 1857 1858 /* Ensure that MMU and caches are enabled */ 1859 assert((read_sctlr_el3() & SCTLR_C_BIT) != 0UL); 1860 1861 /* See if this is a single or a range of granule transition */ 1862 if (size != GPT_PGS_ACTUAL_SIZE(gpt_config.p)) { 1863 return -EINVAL; 1864 } 1865 1866 /* Check that base and size are valid */ 1867 if ((ULONG_MAX - base) < size) { 1868 VERBOSE("GPT: Transition request address overflow!\n"); 1869 VERBOSE(" Base=0x%"PRIx64"\n", base); 1870 VERBOSE(" Size=0x%lx\n", size); 1871 return -EINVAL; 1872 } 1873 1874 /* Make sure base and size are valid */ 1875 if (((base & (GPT_PGS_ACTUAL_SIZE(gpt_config.p) - 1UL)) != 0UL) || 1876 ((size & (GPT_PGS_ACTUAL_SIZE(gpt_config.p) - 1UL)) != 0UL) || 1877 (size == 0UL) || 1878 ((base + size) >= GPT_PPS_ACTUAL_SIZE(gpt_config.t))) { 1879 VERBOSE("GPT: Invalid granule transition address range!\n"); 1880 VERBOSE(" Base=0x%"PRIx64"\n", base); 1881 VERBOSE(" Size=0x%lx\n", size); 1882 return -EINVAL; 1883 } 1884 1885 res = get_gpi_params(base, &gpi_info); 1886 if (res != 0) { 1887 return res; 1888 } 1889 1890 /* 1891 * Access to GPT is controlled by a lock to ensure that no more 1892 * than one CPU is allowed to make changes at any given time. 1893 */ 1894 GPT_LOCK; 1895 read_gpi(&gpi_info); 1896 1897 /* Check that the current address is in the delegated state */ 1898 if ((src_sec_state == SMC_FROM_REALM) && 1899 (gpi_info.gpi == GPT_GPI_REALM)) { 1900 l1_desc = GPT_L1_REALM_DESC; 1901 nse = (uint64_t)GPT_NSE_REALM << GPT_NSE_SHIFT; 1902 } else if ((src_sec_state == SMC_FROM_SECURE) && 1903 (gpi_info.gpi == GPT_GPI_SECURE)) { 1904 l1_desc = GPT_L1_SECURE_DESC; 1905 nse = (uint64_t)GPT_NSE_SECURE << GPT_NSE_SHIFT; 1906 } else { 1907 VERBOSE("GPT: Only Granule in REALM or SECURE state can be undelegated\n"); 1908 VERBOSE(" Caller: %u Current GPI: %u\n", src_sec_state, 1909 gpi_info.gpi); 1910 GPT_UNLOCK; 1911 return -EPERM; 1912 } 1913 1914 #if (RME_GPT_MAX_BLOCK != 0) 1915 /* Check for Contiguous descriptor */ 1916 if ((gpi_info.gpt_l1_desc & GPT_L1_TYPE_CONT_DESC_MASK) == 1917 GPT_L1_TYPE_CONT_DESC) { 1918 shatter_block(base, &gpi_info, l1_desc); 1919 } 1920 #endif 1921 /* 1922 * In order to maintain mutual distrust between Realm and Secure 1923 * states, remove access now, in order to guarantee that writes 1924 * to the currently-accessible physical address space will not 1925 * later become observable. 1926 */ 1927 write_gpt(&gpi_info.gpt_l1_desc, gpi_info.gpt_l1_addr, 1928 gpi_info.gpi_shift, gpi_info.idx, GPT_GPI_NO_ACCESS); 1929 1930 /* Ensure that all agents observe the new NO_ACCESS configuration */ 1931 tlbi_page_dsbosh(base); 1932 1933 /* Ensure that the scrubbed data have made it past the PoPA */ 1934 flush_page_to_popa(base | nse); 1935 1936 /* 1937 * Remove any data loaded speculatively in NS space from before 1938 * the scrubbing. 1939 */ 1940 nse = (uint64_t)GPT_NSE_NS << GPT_NSE_SHIFT; 1941 1942 flush_page_to_popa(base | nse); 1943 1944 /* Clear existing GPI encoding and transition granule */ 1945 write_gpt(&gpi_info.gpt_l1_desc, gpi_info.gpt_l1_addr, 1946 gpi_info.gpi_shift, gpi_info.idx, GPT_GPI_NS); 1947 1948 /* Ensure that all agents observe the new NS configuration */ 1949 tlbi_page_dsbosh(base); 1950 1951 #if (RME_GPT_MAX_BLOCK != 0) 1952 if (gpi_info.gpt_l1_desc == GPT_L1_NS_DESC) { 1953 /* Try to fuse */ 1954 fuse_block(base, &gpi_info, GPT_L1_NS_DESC); 1955 } 1956 #endif 1957 /* Unlock the lock to GPT */ 1958 GPT_UNLOCK; 1959 1960 /* 1961 * The isb() will be done as part of context 1962 * synchronization when returning to lower EL. 1963 */ 1964 VERBOSE("GPT: Granule 0x%"PRIx64" GPI 0x%x->0x%x\n", 1965 base, gpi_info.gpi, GPT_GPI_NS); 1966 1967 return 0; 1968 } 1969