1 /* 2 * Copyright (c) 2022-2025, Arm Limited. All rights reserved. 3 * 4 * SPDX-License-Identifier: BSD-3-Clause 5 */ 6 7 #include <assert.h> 8 #include <errno.h> 9 #include <inttypes.h> 10 #include <limits.h> 11 #include <stdint.h> 12 13 #include <arch.h> 14 #include <arch_features.h> 15 #include <common/debug.h> 16 #include <lib/gpt_rme/gpt_rme.h> 17 #include <lib/smccc.h> 18 #include <lib/xlat_tables/xlat_tables_v2.h> 19 20 #include "gpt_rme_private.h" 21 22 #if !ENABLE_RME 23 #error "ENABLE_RME must be enabled to use the GPT library" 24 #endif 25 26 /* 27 * Lookup T from PPS 28 * 29 * PPS Size T 30 * 0b000 4GB 32 31 * 0b001 64GB 36 32 * 0b010 1TB 40 33 * 0b011 4TB 42 34 * 0b100 16TB 44 35 * 0b101 256TB 48 36 * 0b110 4PB 52 37 * 38 * See section 15.1.27 of the RME specification. 39 */ 40 static const gpt_t_val_e gpt_t_lookup[] = {PPS_4GB_T, PPS_64GB_T, 41 PPS_1TB_T, PPS_4TB_T, 42 PPS_16TB_T, PPS_256TB_T, 43 PPS_4PB_T}; 44 45 /* 46 * Lookup P from PGS 47 * 48 * PGS Size P 49 * 0b00 4KB 12 50 * 0b10 16KB 14 51 * 0b01 64KB 16 52 * 53 * Note that pgs=0b10 is 16KB and pgs=0b01 is 64KB, this is not a typo. 54 * 55 * See section 15.1.27 of the RME specification. 56 */ 57 static const gpt_p_val_e gpt_p_lookup[] = {PGS_4KB_P, PGS_64KB_P, PGS_16KB_P}; 58 59 static void shatter_2mb(uintptr_t base, const gpi_info_t *gpi_info, 60 uint64_t l1_desc); 61 static void shatter_32mb(uintptr_t base, const gpi_info_t *gpi_info, 62 uint64_t l1_desc); 63 static void shatter_512mb(uintptr_t base, const gpi_info_t *gpi_info, 64 uint64_t l1_desc); 65 66 /* 67 * This structure contains GPT configuration data 68 */ 69 typedef struct { 70 uintptr_t plat_gpt_l0_base; 71 gpccr_pps_e pps; 72 gpt_t_val_e t; 73 gpccr_pgs_e pgs; 74 gpt_p_val_e p; 75 } gpt_config_t; 76 77 static gpt_config_t gpt_config; 78 79 /* 80 * Number of L1 entries in 2MB, depending on GPCCR_EL3.PGS: 81 * +-------+------------+ 82 * | PGS | L1 entries | 83 * +-------+------------+ 84 * | 4KB | 32 | 85 * +-------+------------+ 86 * | 16KB | 8 | 87 * +-------+------------+ 88 * | 64KB | 2 | 89 * +-------+------------+ 90 */ 91 static unsigned int gpt_l1_cnt_2mb; 92 93 /* 94 * Mask for the L1 index field, depending on 95 * GPCCR_EL3.L0GPTSZ and GPCCR_EL3.PGS: 96 * +---------+-------------------------------+ 97 * | | PGS | 98 * +---------+----------+----------+---------+ 99 * | L0GPTSZ | 4KB | 16KB | 64KB | 100 * +---------+----------+----------+---------+ 101 * | 1GB | 0x3FFF | 0xFFF | 0x3FF | 102 * +---------+----------+----------+---------+ 103 * | 16GB | 0x3FFFF | 0xFFFF | 0x3FFF | 104 * +---------+----------+----------+---------+ 105 * | 64GB | 0xFFFFF | 0x3FFFF | 0xFFFF | 106 * +---------+----------+----------+---------+ 107 * | 512GB | 0x7FFFFF | 0x1FFFFF | 0x7FFFF | 108 * +---------+----------+----------+---------+ 109 */ 110 static uint64_t gpt_l1_index_mask; 111 112 /* Number of 128-bit L1 entries in 2MB, 32MB and 512MB */ 113 #define L1_QWORDS_2MB (gpt_l1_cnt_2mb / 2U) 114 #define L1_QWORDS_32MB (L1_QWORDS_2MB * 16U) 115 #define L1_QWORDS_512MB (L1_QWORDS_32MB * 16U) 116 117 /* Size in bytes of L1 entries in 2MB, 32MB */ 118 #define L1_BYTES_2MB (gpt_l1_cnt_2mb * sizeof(uint64_t)) 119 #define L1_BYTES_32MB (L1_BYTES_2MB * 16U) 120 121 /* Get the index into the L1 table from a physical address */ 122 #define GPT_L1_INDEX(_pa) \ 123 (((_pa) >> (unsigned int)GPT_L1_IDX_SHIFT(gpt_config.p)) & gpt_l1_index_mask) 124 125 /* This variable is used during initialization of the L1 tables */ 126 static uintptr_t gpt_l1_tbl; 127 128 /* These variables are used during runtime */ 129 #if (RME_GPT_BITLOCK_BLOCK == 0) 130 /* 131 * The GPTs are protected by a global spinlock to ensure 132 * that multiple CPUs do not attempt to change the descriptors at once. 133 */ 134 static spinlock_t gpt_lock; 135 136 /* Lock/unlock macros for GPT entries 137 * 138 * Access to GPT is controlled by a global lock to ensure 139 * that no more than one CPU is allowed to make changes at any 140 * given time. 141 */ 142 #define GPT_LOCK spin_lock(&gpt_lock) 143 #define GPT_UNLOCK spin_unlock(&gpt_lock) 144 #else 145 146 /* Base address of bitlocks array */ 147 static bitlock_t *gpt_bitlock; 148 149 /* 150 * Access to a block of memory is controlled by a bitlock. 151 * Size of block = RME_GPT_BITLOCK_BLOCK * 512MB. 152 */ 153 #define GPT_LOCK bit_lock(gpi_info.lock, gpi_info.mask) 154 #define GPT_UNLOCK bit_unlock(gpi_info.lock, gpi_info.mask) 155 #endif /* RME_GPT_BITLOCK_BLOCK */ 156 157 static void tlbi_page_dsbosh(uintptr_t base) 158 { 159 /* Look-up table for invalidation TLBs for 4KB, 16KB and 64KB pages */ 160 static const gpt_tlbi_lookup_t tlbi_page_lookup[] = { 161 { tlbirpalos_4k, ~(SZ_4K - 1UL) }, 162 { tlbirpalos_64k, ~(SZ_64K - 1UL) }, 163 { tlbirpalos_16k, ~(SZ_16K - 1UL) } 164 }; 165 166 tlbi_page_lookup[gpt_config.pgs].function( 167 base & tlbi_page_lookup[gpt_config.pgs].mask); 168 dsbosh(); 169 } 170 171 /* 172 * Helper function to fill out GPI entries in a single L1 table 173 * with Granules or Contiguous descriptor. 174 * 175 * Parameters 176 * l1 Pointer to 2MB, 32MB or 512MB aligned L1 table entry to fill out 177 * l1_desc GPT Granules or Contiguous descriptor set this range to 178 * cnt Number of double 128-bit L1 entries to fill 179 * 180 */ 181 static void fill_desc(uint64_t *l1, uint64_t l1_desc, unsigned int cnt) 182 { 183 uint128_t *l1_quad = (uint128_t *)l1; 184 uint128_t l1_quad_desc = (uint128_t)l1_desc | ((uint128_t)l1_desc << 64); 185 186 VERBOSE("GPT: %s(%p 0x%"PRIx64" %u)\n", __func__, l1, l1_desc, cnt); 187 188 for (unsigned int i = 0U; i < cnt; i++) { 189 *l1_quad++ = l1_quad_desc; 190 } 191 } 192 193 static void shatter_2mb(uintptr_t base, const gpi_info_t *gpi_info, 194 uint64_t l1_desc) 195 { 196 unsigned long idx = GPT_L1_INDEX(ALIGN_2MB(base)); 197 198 VERBOSE("GPT: %s(0x%"PRIxPTR" 0x%"PRIx64")\n", 199 __func__, base, l1_desc); 200 201 /* Convert 2MB Contiguous block to Granules */ 202 fill_desc(&gpi_info->gpt_l1_addr[idx], l1_desc, L1_QWORDS_2MB); 203 } 204 205 static void shatter_32mb(uintptr_t base, const gpi_info_t *gpi_info, 206 uint64_t l1_desc) 207 { 208 unsigned long idx = GPT_L1_INDEX(ALIGN_2MB(base)); 209 const uint64_t *l1_gran = &gpi_info->gpt_l1_addr[idx]; 210 uint64_t l1_cont_desc = GPT_L1_CONT_DESC(l1_desc, 2MB); 211 uint64_t *l1; 212 213 VERBOSE("GPT: %s(0x%"PRIxPTR" 0x%"PRIx64")\n", 214 __func__, base, l1_desc); 215 216 /* Get index corresponding to 32MB aligned address */ 217 idx = GPT_L1_INDEX(ALIGN_32MB(base)); 218 l1 = &gpi_info->gpt_l1_addr[idx]; 219 220 /* 16 x 2MB blocks in 32MB */ 221 for (unsigned int i = 0U; i < 16U; i++) { 222 /* Fill with Granules or Contiguous descriptors */ 223 fill_desc(l1, (l1 == l1_gran) ? l1_desc : l1_cont_desc, 224 L1_QWORDS_2MB); 225 l1 = (uint64_t *)((uintptr_t)l1 + L1_BYTES_2MB); 226 } 227 } 228 229 static void shatter_512mb(uintptr_t base, const gpi_info_t *gpi_info, 230 uint64_t l1_desc) 231 { 232 unsigned long idx = GPT_L1_INDEX(ALIGN_32MB(base)); 233 const uint64_t *l1_32mb = &gpi_info->gpt_l1_addr[idx]; 234 uint64_t l1_cont_desc = GPT_L1_CONT_DESC(l1_desc, 32MB); 235 uint64_t *l1; 236 237 VERBOSE("GPT: %s(0x%"PRIxPTR" 0x%"PRIx64")\n", 238 __func__, base, l1_desc); 239 240 /* Get index corresponding to 512MB aligned address */ 241 idx = GPT_L1_INDEX(ALIGN_512MB(base)); 242 l1 = &gpi_info->gpt_l1_addr[idx]; 243 244 /* 16 x 32MB blocks in 512MB */ 245 for (unsigned int i = 0U; i < 16U; i++) { 246 if (l1 == l1_32mb) { 247 /* Shatter this 32MB block */ 248 shatter_32mb(base, gpi_info, l1_desc); 249 } else { 250 /* Fill 32MB with Contiguous descriptors */ 251 fill_desc(l1, l1_cont_desc, L1_QWORDS_32MB); 252 } 253 254 l1 = (uint64_t *)((uintptr_t)l1 + L1_BYTES_32MB); 255 } 256 } 257 258 /* 259 * This function checks to see if a GPI value is valid. 260 * 261 * These are valid GPI values. 262 * GPT_GPI_NO_ACCESS U(0x0) 263 * GPT_GPI_SECURE U(0x8) 264 * GPT_GPI_NS U(0x9) 265 * GPT_GPI_ROOT U(0xA) 266 * GPT_GPI_REALM U(0xB) 267 * GPT_GPI_NSO U(0xD) 268 * GPT_GPI_ANY U(0xF) 269 * 270 * Parameters 271 * gpi GPI to check for validity. 272 * 273 * Return 274 * true for a valid GPI, false for an invalid one. 275 */ 276 static bool is_gpi_valid(unsigned int gpi) 277 { 278 switch (gpi) { 279 case GPT_GPI_NO_ACCESS: 280 case GPT_GPI_SECURE: 281 case GPT_GPI_NS: 282 case GPT_GPI_ROOT: 283 case GPT_GPI_REALM: 284 case GPT_GPI_ANY: 285 return true; 286 case GPT_GPI_NSO: 287 return is_feat_rme_gpc2_present(); 288 default: 289 return false; 290 } 291 } 292 293 /* 294 * This function checks to see if two PAS regions overlap. 295 * 296 * Parameters 297 * base_1: base address of first PAS 298 * size_1: size of first PAS 299 * base_2: base address of second PAS 300 * size_2: size of second PAS 301 * 302 * Return 303 * True if PAS regions overlap, false if they do not. 304 */ 305 static bool check_pas_overlap(uintptr_t base_1, size_t size_1, 306 uintptr_t base_2, size_t size_2) 307 { 308 if (((base_1 + size_1) > base_2) && ((base_2 + size_2) > base_1)) { 309 return true; 310 } 311 return false; 312 } 313 314 /* 315 * This helper function checks to see if a PAS region from index 0 to 316 * (pas_idx - 1) occupies the L0 region at index l0_idx in the L0 table. 317 * 318 * Parameters 319 * l0_idx: Index of the L0 entry to check 320 * pas_regions: PAS region array 321 * pas_idx: Upper bound of the PAS array index. 322 * 323 * Return 324 * True if a PAS region occupies the L0 region in question, false if not. 325 */ 326 static bool does_previous_pas_exist_here(unsigned int l0_idx, 327 pas_region_t *pas_regions, 328 unsigned int pas_idx) 329 { 330 /* Iterate over PAS regions up to pas_idx */ 331 for (unsigned int i = 0U; i < pas_idx; i++) { 332 if (check_pas_overlap((GPT_L0GPTSZ_ACTUAL_SIZE * l0_idx), 333 GPT_L0GPTSZ_ACTUAL_SIZE, 334 pas_regions[i].base_pa, pas_regions[i].size)) { 335 return true; 336 } 337 } 338 return false; 339 } 340 341 /* 342 * This function iterates over all of the PAS regions and checks them to ensure 343 * proper alignment of base and size, that the GPI is valid, and that no regions 344 * overlap. As a part of the overlap checks, this function checks existing L0 345 * mappings against the new PAS regions in the event that gpt_init_pas_l1_tables 346 * is called multiple times to place L1 tables in different areas of memory. It 347 * also counts the number of L1 tables needed and returns it on success. 348 * 349 * Parameters 350 * *pas_regions Pointer to array of PAS region structures. 351 * pas_region_cnt Total number of PAS regions in the array. 352 * 353 * Return 354 * Negative Linux error code in the event of a failure, number of L1 regions 355 * required when successful. 356 */ 357 static int validate_pas_mappings(pas_region_t *pas_regions, 358 unsigned int pas_region_cnt) 359 { 360 unsigned int idx; 361 unsigned int l1_cnt = 0U; 362 unsigned int pas_l1_cnt; 363 uint64_t *l0_desc = (uint64_t *)gpt_config.plat_gpt_l0_base; 364 365 assert(pas_regions != NULL); 366 assert(pas_region_cnt != 0U); 367 368 for (idx = 0U; idx < pas_region_cnt; idx++) { 369 /* Check for arithmetic overflow in region */ 370 if ((ULONG_MAX - pas_regions[idx].base_pa) < 371 pas_regions[idx].size) { 372 ERROR("GPT: Address overflow in PAS[%u]!\n", idx); 373 return -EOVERFLOW; 374 } 375 376 /* Initial checks for PAS validity */ 377 if (((pas_regions[idx].base_pa + pas_regions[idx].size) > 378 GPT_PPS_ACTUAL_SIZE(gpt_config.t)) || 379 !is_gpi_valid(GPT_PAS_ATTR_GPI(pas_regions[idx].attrs))) { 380 ERROR("GPT: PAS[%u] is invalid!\n", idx); 381 return -EFAULT; 382 } 383 384 /* 385 * Make sure this PAS does not overlap with another one. We 386 * start from idx + 1 instead of 0 since prior PAS mappings will 387 * have already checked themselves against this one. 388 */ 389 for (unsigned int i = idx + 1U; i < pas_region_cnt; i++) { 390 if (check_pas_overlap(pas_regions[idx].base_pa, 391 pas_regions[idx].size, 392 pas_regions[i].base_pa, 393 pas_regions[i].size)) { 394 ERROR("GPT: PAS[%u] overlaps with PAS[%u]\n", 395 i, idx); 396 return -EFAULT; 397 } 398 } 399 400 /* 401 * Since this function can be called multiple times with 402 * separate L1 tables we need to check the existing L0 mapping 403 * to see if this PAS would fall into one that has already been 404 * initialized. 405 */ 406 for (unsigned int i = 407 (unsigned int)GPT_L0_IDX(pas_regions[idx].base_pa); 408 i <= GPT_L0_IDX(pas_regions[idx].base_pa + 409 pas_regions[idx].size - 1UL); 410 i++) { 411 if ((GPT_L0_TYPE(l0_desc[i]) == GPT_L0_TYPE_BLK_DESC) && 412 (GPT_L0_BLKD_GPI(l0_desc[i]) == GPT_GPI_ANY)) { 413 /* This descriptor is unused so continue */ 414 continue; 415 } 416 417 /* 418 * This descriptor has been initialized in a previous 419 * call to this function so cannot be initialized again. 420 */ 421 ERROR("GPT: PAS[%u] overlaps with previous L0[%u]!\n", 422 idx, i); 423 return -EFAULT; 424 } 425 426 /* Check for block mapping (L0) type */ 427 if (GPT_PAS_ATTR_MAP_TYPE(pas_regions[idx].attrs) == 428 GPT_PAS_ATTR_MAP_TYPE_BLOCK) { 429 /* Make sure base and size are block-aligned */ 430 if (!GPT_IS_L0_ALIGNED(pas_regions[idx].base_pa) || 431 !GPT_IS_L0_ALIGNED(pas_regions[idx].size)) { 432 ERROR("GPT: PAS[%u] is not block-aligned!\n", 433 idx); 434 return -EFAULT; 435 } 436 437 continue; 438 } 439 440 /* Check for granule mapping (L1) type */ 441 if (GPT_PAS_ATTR_MAP_TYPE(pas_regions[idx].attrs) == 442 GPT_PAS_ATTR_MAP_TYPE_GRANULE) { 443 /* Make sure base and size are granule-aligned */ 444 if (!GPT_IS_L1_ALIGNED(gpt_config.p, pas_regions[idx].base_pa) || 445 !GPT_IS_L1_ALIGNED(gpt_config.p, pas_regions[idx].size)) { 446 ERROR("GPT: PAS[%u] is not granule-aligned!\n", 447 idx); 448 return -EFAULT; 449 } 450 451 /* Find how many L1 tables this PAS occupies */ 452 pas_l1_cnt = (GPT_L0_IDX(pas_regions[idx].base_pa + 453 pas_regions[idx].size - 1UL) - 454 GPT_L0_IDX(pas_regions[idx].base_pa) + 1U); 455 456 /* 457 * This creates a situation where, if multiple PAS 458 * regions occupy the same table descriptor, we can get 459 * an artificially high total L1 table count. The way we 460 * handle this is by checking each PAS against those 461 * before it in the array, and if they both occupy the 462 * same PAS we subtract from pas_l1_cnt and only the 463 * first PAS in the array gets to count it. 464 */ 465 466 /* 467 * If L1 count is greater than 1 we know the start and 468 * end PAs are in different L0 regions so we must check 469 * both for overlap against other PAS. 470 */ 471 if (pas_l1_cnt > 1) { 472 if (does_previous_pas_exist_here( 473 GPT_L0_IDX(pas_regions[idx].base_pa + 474 pas_regions[idx].size - 1UL), 475 pas_regions, idx)) { 476 pas_l1_cnt--; 477 } 478 } 479 480 if (does_previous_pas_exist_here( 481 GPT_L0_IDX(pas_regions[idx].base_pa), 482 pas_regions, idx)) { 483 pas_l1_cnt--; 484 } 485 486 l1_cnt += pas_l1_cnt; 487 continue; 488 } 489 490 /* If execution reaches this point, mapping type is invalid */ 491 ERROR("GPT: PAS[%u] has invalid mapping type 0x%x.\n", idx, 492 GPT_PAS_ATTR_MAP_TYPE(pas_regions[idx].attrs)); 493 return -EINVAL; 494 } 495 496 return l1_cnt; 497 } 498 499 /* 500 * This function validates L0 initialization parameters. 501 * 502 * Parameters 503 * l0_mem_base Base address of memory used for L0 table. 504 * l0_mem_size Size of memory available for L0 table. 505 * 506 * Return 507 * Negative Linux error code in the event of a failure, 0 for success. 508 */ 509 static int validate_l0_params(gpccr_pps_e pps, uintptr_t l0_mem_base, 510 size_t l0_mem_size) 511 { 512 size_t l0_alignment; 513 514 /* 515 * Make sure PPS is valid and then store it since macros need this value 516 * to work. 517 */ 518 if (pps > GPT_PPS_MAX) { 519 ERROR("GPT: Invalid PPS: 0x%x\n", pps); 520 return -EINVAL; 521 } 522 gpt_config.pps = pps; 523 gpt_config.t = gpt_t_lookup[pps]; 524 525 /* Alignment must be the greater of 4KB or L0 table size */ 526 l0_alignment = SZ_4K; 527 if (l0_alignment < GPT_L0_TABLE_SIZE(gpt_config.t)) { 528 l0_alignment = GPT_L0_TABLE_SIZE(gpt_config.t); 529 } 530 531 /* Check base address */ 532 if ((l0_mem_base == 0UL) || 533 ((l0_mem_base & (l0_alignment - 1UL)) != 0UL)) { 534 ERROR("GPT: Invalid L0 base address: 0x%lx\n", l0_mem_base); 535 return -EFAULT; 536 } 537 538 /* Check memory size for L0 table */ 539 if (l0_mem_size < GPT_L0_TABLE_SIZE(gpt_config.t)) { 540 ERROR("GPT: Inadequate L0 memory\n"); 541 ERROR(" Expected 0x%lx bytes, got 0x%lx\n", 542 GPT_L0_TABLE_SIZE(gpt_config.t), l0_mem_size); 543 return -ENOMEM; 544 } 545 546 return 0; 547 } 548 549 /* 550 * In the event that L1 tables are needed, this function validates 551 * the L1 table generation parameters. 552 * 553 * Parameters 554 * l1_mem_base Base address of memory used for L1 table allocation. 555 * l1_mem_size Total size of memory available for L1 tables. 556 * l1_gpt_cnt Number of L1 tables needed. 557 * 558 * Return 559 * Negative Linux error code in the event of a failure, 0 for success. 560 */ 561 static int validate_l1_params(uintptr_t l1_mem_base, size_t l1_mem_size, 562 unsigned int l1_gpt_cnt) 563 { 564 size_t l1_gpt_mem_sz; 565 566 /* Check if the granularity is supported */ 567 if (!xlat_arch_is_granule_size_supported( 568 GPT_PGS_ACTUAL_SIZE(gpt_config.p))) { 569 return -EPERM; 570 } 571 572 /* Make sure L1 tables are aligned to their size */ 573 if ((l1_mem_base & (GPT_L1_TABLE_SIZE(gpt_config.p) - 1UL)) != 0UL) { 574 ERROR("GPT: Unaligned L1 GPT base address: 0x%"PRIxPTR"\n", 575 l1_mem_base); 576 return -EFAULT; 577 } 578 579 /* Get total memory needed for L1 tables */ 580 l1_gpt_mem_sz = l1_gpt_cnt * GPT_L1_TABLE_SIZE(gpt_config.p); 581 582 /* Check for overflow */ 583 if ((l1_gpt_mem_sz / GPT_L1_TABLE_SIZE(gpt_config.p)) != l1_gpt_cnt) { 584 ERROR("GPT: Overflow calculating L1 memory size\n"); 585 return -ENOMEM; 586 } 587 588 /* Make sure enough space was supplied */ 589 if (l1_mem_size < l1_gpt_mem_sz) { 590 ERROR("%sL1 GPTs%s", (const char *)"GPT: Inadequate ", 591 (const char *)" memory\n"); 592 ERROR(" Expected 0x%lx bytes, got 0x%lx\n", 593 l1_gpt_mem_sz, l1_mem_size); 594 return -ENOMEM; 595 } 596 597 VERBOSE("GPT: Requested 0x%lx bytes for L1 GPTs\n", l1_gpt_mem_sz); 598 return 0; 599 } 600 601 /* 602 * This function initializes L0 block descriptors (regions that cannot be 603 * transitioned at the granule level) according to the provided PAS. 604 * 605 * Parameters 606 * *pas Pointer to the structure defining the PAS region to 607 * initialize. 608 */ 609 static void generate_l0_blk_desc(pas_region_t *pas) 610 { 611 uint64_t gpt_desc; 612 unsigned long idx, end_idx; 613 uint64_t *l0_gpt_arr; 614 615 assert(gpt_config.plat_gpt_l0_base != 0UL); 616 assert(pas != NULL); 617 618 /* 619 * Checking of PAS parameters has already been done in 620 * validate_pas_mappings so no need to check the same things again. 621 */ 622 623 l0_gpt_arr = (uint64_t *)gpt_config.plat_gpt_l0_base; 624 625 /* Create the GPT Block descriptor for this PAS region */ 626 gpt_desc = GPT_L0_BLK_DESC(GPT_PAS_ATTR_GPI(pas->attrs)); 627 628 /* Start index of this region in L0 GPTs */ 629 idx = GPT_L0_IDX(pas->base_pa); 630 631 /* 632 * Determine number of L0 GPT descriptors covered by 633 * this PAS region and use the count to populate these 634 * descriptors. 635 */ 636 end_idx = GPT_L0_IDX(pas->base_pa + pas->size); 637 638 /* Generate the needed block descriptors */ 639 for (; idx < end_idx; idx++) { 640 l0_gpt_arr[idx] = gpt_desc; 641 VERBOSE("GPT: L0 entry (BLOCK) index %lu [%p]: GPI = 0x%"PRIx64" (0x%"PRIx64")\n", 642 idx, &l0_gpt_arr[idx], 643 (gpt_desc >> GPT_L0_BLK_DESC_GPI_SHIFT) & 644 GPT_L0_BLK_DESC_GPI_MASK, l0_gpt_arr[idx]); 645 } 646 } 647 648 /* 649 * Helper function to determine if the end physical address lies in the same L0 650 * region as the current physical address. If true, the end physical address is 651 * returned else, the start address of the next region is returned. 652 * 653 * Parameters 654 * cur_pa Physical address of the current PA in the loop through 655 * the range. 656 * end_pa Physical address of the end PA in a PAS range. 657 * 658 * Return 659 * The PA of the end of the current range. 660 */ 661 static uintptr_t get_l1_end_pa(uintptr_t cur_pa, uintptr_t end_pa) 662 { 663 uintptr_t cur_idx; 664 uintptr_t end_idx; 665 666 cur_idx = GPT_L0_IDX(cur_pa); 667 end_idx = GPT_L0_IDX(end_pa); 668 669 assert(cur_idx <= end_idx); 670 671 if (cur_idx == end_idx) { 672 return end_pa; 673 } 674 675 return (cur_idx + 1UL) << GPT_L0_IDX_SHIFT; 676 } 677 678 /* 679 * Helper function to fill out GPI entries from 'first' granule address of 680 * the specified 'length' in a single L1 table with 'l1_desc' Contiguous 681 * descriptor. 682 * 683 * Parameters 684 * l1 Pointer to L1 table to fill out 685 * first Address of first granule in range 686 * length Length of the range in bytes 687 * gpi GPI set this range to 688 * 689 * Return 690 * Address of next granule in range. 691 */ 692 __unused static uintptr_t fill_l1_cont_desc(uint64_t *l1, uintptr_t first, 693 size_t length, unsigned int gpi) 694 { 695 /* 696 * Look up table for contiguous blocks and descriptors. 697 * Entries should be defined in descending block sizes: 698 * 512MB, 32MB and 2MB. 699 */ 700 static const gpt_fill_lookup_t gpt_fill_lookup[] = { 701 #if (RME_GPT_MAX_BLOCK == 512) 702 { SZ_512M, GPT_L1_CONT_DESC_512MB }, 703 #endif 704 #if (RME_GPT_MAX_BLOCK >= 32) 705 { SZ_32M, GPT_L1_CONT_DESC_32MB }, 706 #endif 707 #if (RME_GPT_MAX_BLOCK != 0) 708 { SZ_2M, GPT_L1_CONT_DESC_2MB } 709 #endif 710 }; 711 712 /* 713 * Iterate through all block sizes (512MB, 32MB and 2MB) 714 * starting with maximum supported. 715 */ 716 for (unsigned long i = 0UL; i < ARRAY_SIZE(gpt_fill_lookup); i++) { 717 /* Calculate index */ 718 unsigned long idx = GPT_L1_INDEX(first); 719 720 /* Contiguous block size */ 721 size_t cont_size = gpt_fill_lookup[i].size; 722 723 if (GPT_REGION_IS_CONT(length, first, cont_size)) { 724 725 /* Generate Contiguous descriptor */ 726 uint64_t l1_desc = GPT_L1_GPI_CONT_DESC(gpi, 727 gpt_fill_lookup[i].desc); 728 729 /* Number of 128-bit L1 entries in block */ 730 unsigned int cnt; 731 732 switch (cont_size) { 733 case SZ_512M: 734 cnt = L1_QWORDS_512MB; 735 break; 736 case SZ_32M: 737 cnt = L1_QWORDS_32MB; 738 break; 739 default: /* SZ_2MB */ 740 cnt = L1_QWORDS_2MB; 741 } 742 743 VERBOSE("GPT: Contiguous descriptor 0x%"PRIxPTR" %luMB\n", 744 first, cont_size / SZ_1M); 745 746 /* Fill Contiguous descriptors */ 747 fill_desc(&l1[idx], l1_desc, cnt); 748 return (first + cont_size); 749 } 750 } 751 752 return first; 753 } 754 755 /* Build Granules descriptor with the same 'gpi' for every GPI entry */ 756 static uint64_t build_l1_desc(unsigned int gpi) 757 { 758 uint64_t l1_desc = (uint64_t)gpi | ((uint64_t)gpi << 4); 759 760 l1_desc |= (l1_desc << 8); 761 l1_desc |= (l1_desc << 16); 762 return (l1_desc | (l1_desc << 32)); 763 } 764 765 /* 766 * Helper function to fill out GPI entries from 'first' to 'last' granule 767 * address in a single L1 table with 'l1_desc' Granules descriptor. 768 * 769 * Parameters 770 * l1 Pointer to L1 table to fill out 771 * first Address of first granule in range 772 * last Address of last granule in range (inclusive) 773 * gpi GPI set this range to 774 * 775 * Return 776 * Address of next granule in range. 777 */ 778 static uintptr_t fill_l1_gran_desc(uint64_t *l1, uintptr_t first, 779 uintptr_t last, unsigned int gpi) 780 { 781 uint64_t gpi_mask; 782 unsigned long i; 783 784 /* Generate Granules descriptor */ 785 uint64_t l1_desc = build_l1_desc(gpi); 786 787 /* Shift the mask if we're starting in the middle of an L1 entry */ 788 gpi_mask = ULONG_MAX << (GPT_L1_GPI_IDX(gpt_config.p, first) << 2); 789 790 /* Fill out each L1 entry for this region */ 791 for (i = GPT_L1_INDEX(first); i <= GPT_L1_INDEX(last); i++) { 792 793 /* Account for stopping in the middle of an L1 entry */ 794 if (i == GPT_L1_INDEX(last)) { 795 gpi_mask &= (gpi_mask >> ((15U - 796 GPT_L1_GPI_IDX(gpt_config.p, last)) << 2)); 797 } 798 799 assert((l1[i] & gpi_mask) == (GPT_L1_ANY_DESC & gpi_mask)); 800 801 /* Write GPI values */ 802 l1[i] = (l1[i] & ~gpi_mask) | (l1_desc & gpi_mask); 803 804 /* Reset mask */ 805 gpi_mask = ULONG_MAX; 806 } 807 808 return last + GPT_PGS_ACTUAL_SIZE(gpt_config.p); 809 } 810 811 /* 812 * Helper function to fill out GPI entries in a single L1 table. 813 * This function fills out an entire L1 table with either Granules or Contiguous 814 * (RME_GPT_MAX_BLOCK != 0) descriptors depending on region length and alignment. 815 * Note. If RME_GPT_MAX_BLOCK == 0, then the L1 tables are filled with regular 816 * Granules descriptors. 817 * 818 * Parameters 819 * l1 Pointer to L1 table to fill out 820 * first Address of first granule in range 821 * last Address of last granule in range (inclusive) 822 * gpi GPI set this range to 823 */ 824 static void fill_l1_tbl(uint64_t *l1, uintptr_t first, uintptr_t last, 825 unsigned int gpi) 826 { 827 assert(l1 != NULL); 828 assert(first <= last); 829 assert((first & (GPT_PGS_ACTUAL_SIZE(gpt_config.p) - 1UL)) == 0UL); 830 assert((last & (GPT_PGS_ACTUAL_SIZE(gpt_config.p) - 1UL)) == 0UL); 831 assert(GPT_L0_IDX(first) == GPT_L0_IDX(last)); 832 833 #if (RME_GPT_MAX_BLOCK != 0) 834 while (first <= last) { 835 /* Region length */ 836 size_t length = last - first + GPT_PGS_ACTUAL_SIZE(gpt_config.p); 837 838 if (length < SZ_2M) { 839 /* 840 * Fill with Granule descriptors in case of 841 * region length < 2MB. 842 */ 843 first = fill_l1_gran_desc(l1, first, last, gpi); 844 845 } else if ((first & (SZ_2M - UL(1))) == UL(0)) { 846 /* 847 * For region length >= 2MB and at least 2MB aligned 848 * call to fill_l1_cont_desc will iterate through 849 * all block sizes (512MB, 32MB and 2MB) supported and 850 * fill corresponding Contiguous descriptors. 851 */ 852 first = fill_l1_cont_desc(l1, first, length, gpi); 853 } else { 854 /* 855 * For not aligned region >= 2MB fill with Granules 856 * descriptors up to the next 2MB aligned address. 857 */ 858 uintptr_t new_last = ALIGN_2MB(first + SZ_2M) - 859 GPT_PGS_ACTUAL_SIZE(gpt_config.p); 860 861 first = fill_l1_gran_desc(l1, first, new_last, gpi); 862 } 863 } 864 #else 865 /* Fill with Granule descriptors */ 866 first = fill_l1_gran_desc(l1, first, last, gpi); 867 #endif 868 assert(first == (last + GPT_PGS_ACTUAL_SIZE(gpt_config.p))); 869 } 870 871 /* 872 * This function finds the next available unused L1 table and initializes all 873 * granules descriptor entries to GPI_ANY. This ensures that there are no chunks 874 * of GPI_NO_ACCESS (0b0000) memory floating around in the system in the 875 * event that a PAS region stops midway through an L1 table, thus guaranteeing 876 * that all memory not explicitly assigned is GPI_ANY. This function does not 877 * check for overflow conditions, that should be done by the caller. 878 * 879 * Return 880 * Pointer to the next available L1 table. 881 */ 882 static uint64_t *get_new_l1_tbl(void) 883 { 884 /* Retrieve the next L1 table */ 885 uint64_t *l1 = (uint64_t *)gpt_l1_tbl; 886 887 /* Increment L1 GPT address */ 888 gpt_l1_tbl += GPT_L1_TABLE_SIZE(gpt_config.p); 889 890 /* Initialize all GPIs to GPT_GPI_ANY */ 891 for (unsigned int i = 0U; i < GPT_L1_ENTRY_COUNT(gpt_config.p); i++) { 892 l1[i] = GPT_L1_ANY_DESC; 893 } 894 895 return l1; 896 } 897 898 /* 899 * When L1 tables are needed, this function creates the necessary L0 table 900 * descriptors and fills out the L1 table entries according to the supplied 901 * PAS range. 902 * 903 * Parameters 904 * *pas Pointer to the structure defining the PAS region. 905 */ 906 static void generate_l0_tbl_desc(pas_region_t *pas) 907 { 908 uintptr_t end_pa; 909 uintptr_t cur_pa; 910 uintptr_t last_gran_pa; 911 uint64_t *l0_gpt_base; 912 uint64_t *l1_gpt_arr; 913 unsigned int l0_idx, gpi; 914 915 assert(gpt_config.plat_gpt_l0_base != 0UL); 916 assert(pas != NULL); 917 918 /* 919 * Checking of PAS parameters has already been done in 920 * validate_pas_mappings so no need to check the same things again. 921 */ 922 end_pa = pas->base_pa + pas->size; 923 l0_gpt_base = (uint64_t *)gpt_config.plat_gpt_l0_base; 924 925 /* We start working from the granule at base PA */ 926 cur_pa = pas->base_pa; 927 928 /* Get GPI */ 929 gpi = GPT_PAS_ATTR_GPI(pas->attrs); 930 931 /* Iterate over each L0 region in this memory range */ 932 for (l0_idx = (unsigned int)GPT_L0_IDX(pas->base_pa); 933 l0_idx <= (unsigned int)GPT_L0_IDX(end_pa - 1UL); 934 l0_idx++) { 935 /* 936 * See if the L0 entry is already a table descriptor or if we 937 * need to create one. 938 */ 939 if (GPT_L0_TYPE(l0_gpt_base[l0_idx]) == GPT_L0_TYPE_TBL_DESC) { 940 /* Get the L1 array from the L0 entry */ 941 l1_gpt_arr = GPT_L0_TBLD_ADDR(l0_gpt_base[l0_idx]); 942 } else { 943 /* Get a new L1 table from the L1 memory space */ 944 l1_gpt_arr = get_new_l1_tbl(); 945 946 /* Fill out the L0 descriptor and flush it */ 947 l0_gpt_base[l0_idx] = GPT_L0_TBL_DESC(l1_gpt_arr); 948 } 949 950 VERBOSE("GPT: L0 entry (TABLE) index %u [%p] ==> L1 Addr %p (0x%"PRIx64")\n", 951 l0_idx, &l0_gpt_base[l0_idx], l1_gpt_arr, l0_gpt_base[l0_idx]); 952 953 /* 954 * Determine the PA of the last granule in this L0 descriptor. 955 */ 956 last_gran_pa = get_l1_end_pa(cur_pa, end_pa) - 957 GPT_PGS_ACTUAL_SIZE(gpt_config.p); 958 959 /* 960 * Fill up L1 GPT entries between these two addresses. This 961 * function needs the addresses of the first granule and last 962 * granule in the range. 963 */ 964 fill_l1_tbl(l1_gpt_arr, cur_pa, last_gran_pa, gpi); 965 966 /* Advance cur_pa to first granule in next L0 region */ 967 cur_pa = get_l1_end_pa(cur_pa, end_pa); 968 } 969 } 970 971 /* 972 * This function flushes a range of L0 descriptors used by a given PAS region 973 * array. There is a chance that some unmodified L0 descriptors would be flushed 974 * in the case that there are "holes" in an array of PAS regions but overall 975 * this should be faster than individually flushing each modified L0 descriptor 976 * as they are created. 977 * 978 * Parameters 979 * *pas Pointer to an array of PAS regions. 980 * pas_count Number of entries in the PAS array. 981 */ 982 static void flush_l0_for_pas_array(pas_region_t *pas, unsigned int pas_count) 983 { 984 unsigned long idx; 985 unsigned long start_idx; 986 unsigned long end_idx; 987 uint64_t *l0 = (uint64_t *)gpt_config.plat_gpt_l0_base; 988 989 assert(pas != NULL); 990 assert(pas_count != 0U); 991 992 /* Initial start and end values */ 993 start_idx = GPT_L0_IDX(pas[0].base_pa); 994 end_idx = GPT_L0_IDX(pas[0].base_pa + pas[0].size - 1UL); 995 996 /* Find lowest and highest L0 indices used in this PAS array */ 997 for (idx = 1UL; idx < pas_count; idx++) { 998 if (GPT_L0_IDX(pas[idx].base_pa) < start_idx) { 999 start_idx = GPT_L0_IDX(pas[idx].base_pa); 1000 } 1001 if (GPT_L0_IDX(pas[idx].base_pa + pas[idx].size - 1UL) > end_idx) { 1002 end_idx = GPT_L0_IDX(pas[idx].base_pa + pas[idx].size - 1UL); 1003 } 1004 } 1005 1006 /* 1007 * Flush all covered L0 descriptors, add 1 because we need to include 1008 * the end index value. 1009 */ 1010 flush_dcache_range((uintptr_t)&l0[start_idx], 1011 ((end_idx + 1UL) - start_idx) * sizeof(uint64_t)); 1012 } 1013 1014 /* 1015 * Public API to enable granule protection checks once the tables have all been 1016 * initialized. This function is called at first initialization and then again 1017 * later during warm boots of CPU cores. 1018 * 1019 * Return 1020 * Negative Linux error code in the event of a failure, 0 for success. 1021 */ 1022 int gpt_enable(void) 1023 { 1024 u_register_t gpccr_el3; 1025 1026 /* 1027 * Granule tables must be initialised before enabling 1028 * granule protection. 1029 */ 1030 if (gpt_config.plat_gpt_l0_base == 0UL) { 1031 ERROR("GPT: Tables have not been initialized!\n"); 1032 return -EPERM; 1033 } 1034 1035 /* Write the base address of the L0 tables into GPTBR */ 1036 write_gptbr_el3(((gpt_config.plat_gpt_l0_base >> GPTBR_BADDR_VAL_SHIFT) 1037 >> GPTBR_BADDR_SHIFT) & GPTBR_BADDR_MASK); 1038 1039 /* GPCCR_EL3.PPS */ 1040 gpccr_el3 = SET_GPCCR_PPS(gpt_config.pps); 1041 1042 /* GPCCR_EL3.PGS */ 1043 gpccr_el3 |= SET_GPCCR_PGS(gpt_config.pgs); 1044 1045 /* 1046 * Since EL3 maps the L1 region as Inner shareable, use the same 1047 * shareability attribute for GPC as well so that 1048 * GPC fetches are visible to PEs 1049 */ 1050 gpccr_el3 |= SET_GPCCR_SH(GPCCR_SH_IS); 1051 1052 /* Outer and Inner cacheability set to Normal memory, WB, RA, WA */ 1053 gpccr_el3 |= SET_GPCCR_ORGN(GPCCR_ORGN_WB_RA_WA); 1054 gpccr_el3 |= SET_GPCCR_IRGN(GPCCR_IRGN_WB_RA_WA); 1055 1056 /* Prepopulate GPCCR_EL3 but don't enable GPC yet */ 1057 write_gpccr_el3(gpccr_el3); 1058 isb(); 1059 1060 /* Invalidate any stale TLB entries and any cached register fields */ 1061 tlbipaallos(); 1062 dsb(); 1063 isb(); 1064 1065 /* Enable GPT */ 1066 gpccr_el3 |= GPCCR_GPC_BIT; 1067 1068 /* Enable NSO encoding if FEAT_RME_GPC2 is supported. */ 1069 if (is_feat_rme_gpc2_present()) { 1070 gpccr_el3 |= GPCCR_NSO_BIT; 1071 } 1072 1073 /* TODO: Configure GPCCR_EL3_GPCP for Fault control */ 1074 write_gpccr_el3(gpccr_el3); 1075 isb(); 1076 tlbipaallos(); 1077 dsb(); 1078 isb(); 1079 1080 return 0; 1081 } 1082 1083 /* 1084 * Public API to disable granule protection checks. 1085 */ 1086 void gpt_disable(void) 1087 { 1088 u_register_t gpccr_el3 = read_gpccr_el3(); 1089 1090 write_gpccr_el3(gpccr_el3 & ~GPCCR_GPC_BIT); 1091 dsbsy(); 1092 isb(); 1093 } 1094 1095 /* 1096 * Public API that initializes the entire protected space to GPT_GPI_ANY using 1097 * the L0 tables (block descriptors). Ideally, this function is invoked prior 1098 * to DDR discovery and initialization. The MMU must be initialized before 1099 * calling this function. 1100 * 1101 * Parameters 1102 * pps PPS value to use for table generation 1103 * l0_mem_base Base address of L0 tables in memory. 1104 * l0_mem_size Total size of memory available for L0 tables. 1105 * 1106 * Return 1107 * Negative Linux error code in the event of a failure, 0 for success. 1108 */ 1109 int gpt_init_l0_tables(gpccr_pps_e pps, uintptr_t l0_mem_base, 1110 size_t l0_mem_size) 1111 { 1112 uint64_t gpt_desc; 1113 int ret; 1114 1115 /* Ensure that MMU and Data caches are enabled */ 1116 assert((read_sctlr_el3() & SCTLR_C_BIT) != 0UL); 1117 1118 /* Validate other parameters */ 1119 ret = validate_l0_params(pps, l0_mem_base, l0_mem_size); 1120 if (ret != 0) { 1121 return ret; 1122 } 1123 1124 /* Create the descriptor to initialize L0 entries with */ 1125 gpt_desc = GPT_L0_BLK_DESC(GPT_GPI_ANY); 1126 1127 /* Iterate through all L0 entries */ 1128 for (unsigned int i = 0U; i < GPT_L0_REGION_COUNT(gpt_config.t); i++) { 1129 ((uint64_t *)l0_mem_base)[i] = gpt_desc; 1130 } 1131 1132 /* Flush updated L0 table to memory */ 1133 flush_dcache_range((uintptr_t)l0_mem_base, GPT_L0_TABLE_SIZE(gpt_config.t)); 1134 1135 /* Stash the L0 base address once initial setup is complete */ 1136 gpt_config.plat_gpt_l0_base = l0_mem_base; 1137 1138 return 0; 1139 } 1140 1141 /* 1142 * Public API that carves out PAS regions from the L0 tables and builds any L1 1143 * tables that are needed. This function ideally is run after DDR discovery and 1144 * initialization. The L0 tables must have already been initialized to GPI_ANY 1145 * when this function is called. 1146 * 1147 * This function can be called multiple times with different L1 memory ranges 1148 * and PAS regions if it is desirable to place L1 tables in different locations 1149 * in memory. (ex: you have multiple DDR banks and want to place the L1 tables 1150 * in the DDR bank that they control). 1151 * 1152 * Parameters 1153 * pgs PGS value to use for table generation. 1154 * l1_mem_base Base address of memory used for L1 tables. 1155 * l1_mem_size Total size of memory available for L1 tables. 1156 * *pas_regions Pointer to PAS regions structure array. 1157 * pas_count Total number of PAS regions. 1158 * 1159 * Return 1160 * Negative Linux error code in the event of a failure, 0 for success. 1161 */ 1162 int gpt_init_pas_l1_tables(gpccr_pgs_e pgs, uintptr_t l1_mem_base, 1163 size_t l1_mem_size, pas_region_t *pas_regions, 1164 unsigned int pas_count) 1165 { 1166 int l1_gpt_cnt, ret; 1167 1168 /* Ensure that MMU and Data caches are enabled */ 1169 assert((read_sctlr_el3() & SCTLR_C_BIT) != 0UL); 1170 1171 /* PGS is needed for validate_pas_mappings so check it now */ 1172 if (pgs > GPT_PGS_MAX) { 1173 ERROR("GPT: Invalid PGS: 0x%x\n", pgs); 1174 return -EINVAL; 1175 } 1176 gpt_config.pgs = pgs; 1177 gpt_config.p = gpt_p_lookup[pgs]; 1178 1179 /* Make sure L0 tables have been initialized */ 1180 if (gpt_config.plat_gpt_l0_base == 0UL) { 1181 ERROR("GPT: L0 tables must be initialized first!\n"); 1182 return -EPERM; 1183 } 1184 1185 /* Check if L1 GPTs are required and how many */ 1186 l1_gpt_cnt = validate_pas_mappings(pas_regions, pas_count); 1187 if (l1_gpt_cnt < 0) { 1188 return l1_gpt_cnt; 1189 } 1190 1191 VERBOSE("GPT: %i L1 GPTs requested\n", l1_gpt_cnt); 1192 1193 /* If L1 tables are needed then validate the L1 parameters */ 1194 if (l1_gpt_cnt > 0) { 1195 ret = validate_l1_params(l1_mem_base, l1_mem_size, 1196 (unsigned int)l1_gpt_cnt); 1197 if (ret != 0) { 1198 return ret; 1199 } 1200 1201 /* Set up parameters for L1 table generation */ 1202 gpt_l1_tbl = l1_mem_base; 1203 } 1204 1205 /* Number of L1 entries in 2MB depends on GPCCR_EL3.PGS value */ 1206 gpt_l1_cnt_2mb = (unsigned int)GPT_L1_ENTRY_COUNT_2MB(gpt_config.p); 1207 1208 /* Mask for the L1 index field */ 1209 gpt_l1_index_mask = GPT_L1_IDX_MASK(gpt_config.p); 1210 1211 INFO("GPT: Boot Configuration\n"); 1212 INFO(" PPS/T: 0x%x/%u\n", gpt_config.pps, gpt_config.t); 1213 INFO(" PGS/P: 0x%x/%u\n", gpt_config.pgs, gpt_config.p); 1214 INFO(" L0GPTSZ/S: 0x%x/%u\n", GPT_L0GPTSZ, GPT_S_VAL); 1215 INFO(" PAS count: %u\n", pas_count); 1216 INFO(" L0 base: 0x%"PRIxPTR"\n", gpt_config.plat_gpt_l0_base); 1217 1218 /* Generate the tables in memory */ 1219 for (unsigned int idx = 0U; idx < pas_count; idx++) { 1220 VERBOSE("GPT: PAS[%u]: base 0x%"PRIxPTR"\tsize 0x%lx\tGPI 0x%x\ttype 0x%x\n", 1221 idx, pas_regions[idx].base_pa, pas_regions[idx].size, 1222 GPT_PAS_ATTR_GPI(pas_regions[idx].attrs), 1223 GPT_PAS_ATTR_MAP_TYPE(pas_regions[idx].attrs)); 1224 1225 /* Check if a block or table descriptor is required */ 1226 if (GPT_PAS_ATTR_MAP_TYPE(pas_regions[idx].attrs) == 1227 GPT_PAS_ATTR_MAP_TYPE_BLOCK) { 1228 generate_l0_blk_desc(&pas_regions[idx]); 1229 1230 } else { 1231 generate_l0_tbl_desc(&pas_regions[idx]); 1232 } 1233 } 1234 1235 /* Flush modified L0 tables */ 1236 flush_l0_for_pas_array(pas_regions, pas_count); 1237 1238 /* Flush L1 tables if needed */ 1239 if (l1_gpt_cnt > 0) { 1240 flush_dcache_range(l1_mem_base, 1241 GPT_L1_TABLE_SIZE(gpt_config.p) * 1242 (size_t)l1_gpt_cnt); 1243 } 1244 1245 /* Make sure that all the entries are written to the memory */ 1246 dsbishst(); 1247 tlbipaallos(); 1248 dsb(); 1249 isb(); 1250 1251 return 0; 1252 } 1253 1254 /* 1255 * Public API to initialize the runtime gpt_config structure based on the values 1256 * present in the GPTBR_EL3 and GPCCR_EL3 registers. GPT initialization 1257 * typically happens in a bootloader stage prior to setting up the EL3 runtime 1258 * environment for the granule transition service so this function detects the 1259 * initialization from a previous stage. Granule protection checks must be 1260 * enabled already or this function will return an error. 1261 * 1262 * Parameters 1263 * l1_bitlocks_base Base address of memory for L1 tables bitlocks. 1264 * l1_bitlocks_size Total size of memory available for L1 tables bitlocks. 1265 * 1266 * Return 1267 * Negative Linux error code in the event of a failure, 0 for success. 1268 */ 1269 int gpt_runtime_init(uintptr_t l1_bitlocks_base, size_t l1_bitlocks_size) 1270 { 1271 u_register_t reg; 1272 __unused size_t locks_size; 1273 1274 /* Ensure that MMU and Data caches are enabled */ 1275 assert((read_sctlr_el3() & SCTLR_C_BIT) != 0UL); 1276 1277 /* Ensure GPC are already enabled */ 1278 if ((read_gpccr_el3() & GPCCR_GPC_BIT) == 0UL) { 1279 ERROR("GPT: Granule protection checks are not enabled!\n"); 1280 return -EPERM; 1281 } 1282 1283 /* 1284 * Read the L0 table address from GPTBR, we don't need the L1 base 1285 * address since those are included in the L0 tables as needed. 1286 */ 1287 reg = read_gptbr_el3(); 1288 gpt_config.plat_gpt_l0_base = ((reg >> GPTBR_BADDR_SHIFT) & 1289 GPTBR_BADDR_MASK) << 1290 GPTBR_BADDR_VAL_SHIFT; 1291 1292 /* Read GPCCR to get PGS and PPS values */ 1293 reg = read_gpccr_el3(); 1294 gpt_config.pps = (reg >> GPCCR_PPS_SHIFT) & GPCCR_PPS_MASK; 1295 gpt_config.t = gpt_t_lookup[gpt_config.pps]; 1296 gpt_config.pgs = (reg >> GPCCR_PGS_SHIFT) & GPCCR_PGS_MASK; 1297 gpt_config.p = gpt_p_lookup[gpt_config.pgs]; 1298 1299 /* Number of L1 entries in 2MB depends on GPCCR_EL3.PGS value */ 1300 gpt_l1_cnt_2mb = (unsigned int)GPT_L1_ENTRY_COUNT_2MB(gpt_config.p); 1301 1302 /* Mask for the L1 index field */ 1303 gpt_l1_index_mask = GPT_L1_IDX_MASK(gpt_config.p); 1304 1305 #if (RME_GPT_BITLOCK_BLOCK != 0) 1306 /* 1307 * Size of GPT bitlocks in bytes for the protected address space 1308 * with RME_GPT_BITLOCK_BLOCK * 512MB per bitlock. 1309 */ 1310 locks_size = GPT_PPS_ACTUAL_SIZE(gpt_config.t) / 1311 (RME_GPT_BITLOCK_BLOCK * SZ_512M * 8U); 1312 /* 1313 * If protected space size is less than the size covered 1314 * by 'bitlock' structure, check for a single bitlock. 1315 */ 1316 if (locks_size < LOCK_SIZE) { 1317 locks_size = LOCK_SIZE; 1318 /* Check bitlocks array size */ 1319 } else if (locks_size > l1_bitlocks_size) { 1320 ERROR("GPT: Inadequate GPT bitlocks memory\n"); 1321 ERROR(" Expected 0x%lx bytes, got 0x%lx\n", 1322 locks_size, l1_bitlocks_size); 1323 return -ENOMEM; 1324 } 1325 1326 gpt_bitlock = (bitlock_t *)l1_bitlocks_base; 1327 1328 /* Initialise GPT bitlocks */ 1329 (void)memset((void *)gpt_bitlock, 0, locks_size); 1330 1331 /* Flush GPT bitlocks to memory */ 1332 flush_dcache_range((uintptr_t)gpt_bitlock, locks_size); 1333 #endif /* RME_GPT_BITLOCK_BLOCK */ 1334 1335 VERBOSE("GPT: Runtime Configuration\n"); 1336 VERBOSE(" PPS/T: 0x%x/%u\n", gpt_config.pps, gpt_config.t); 1337 VERBOSE(" PGS/P: 0x%x/%u\n", gpt_config.pgs, gpt_config.p); 1338 VERBOSE(" L0GPTSZ/S: 0x%x/%u\n", GPT_L0GPTSZ, GPT_S_VAL); 1339 VERBOSE(" L0 base: 0x%"PRIxPTR"\n", gpt_config.plat_gpt_l0_base); 1340 #if (RME_GPT_BITLOCK_BLOCK != 0) 1341 VERBOSE(" Bitlocks: 0x%"PRIxPTR"/0x%lx\n", (uintptr_t)gpt_bitlock, 1342 locks_size); 1343 #endif 1344 return 0; 1345 } 1346 1347 /* 1348 * A helper to write the value (target_pas << gpi_shift) to the index of 1349 * the gpt_l1_addr. 1350 */ 1351 static inline void write_gpt(uint64_t *gpt_l1_desc, uint64_t *gpt_l1_addr, 1352 unsigned int gpi_shift, unsigned int idx, 1353 unsigned int target_pas) 1354 { 1355 *gpt_l1_desc &= ~(GPT_L1_GRAN_DESC_GPI_MASK << gpi_shift); 1356 *gpt_l1_desc |= ((uint64_t)target_pas << gpi_shift); 1357 gpt_l1_addr[idx] = *gpt_l1_desc; 1358 1359 dsboshst(); 1360 } 1361 1362 /* 1363 * Helper to retrieve the gpt_l1_* information from the base address 1364 * returned in gpi_info. 1365 */ 1366 static int get_gpi_params(uint64_t base, gpi_info_t *gpi_info) 1367 { 1368 uint64_t gpt_l0_desc, *gpt_l0_base; 1369 __unused unsigned int block_idx; 1370 1371 gpt_l0_base = (uint64_t *)gpt_config.plat_gpt_l0_base; 1372 gpt_l0_desc = gpt_l0_base[GPT_L0_IDX(base)]; 1373 if (GPT_L0_TYPE(gpt_l0_desc) != GPT_L0_TYPE_TBL_DESC) { 1374 VERBOSE("GPT: Granule is not covered by a table descriptor!\n"); 1375 VERBOSE(" Base=0x%"PRIx64"\n", base); 1376 return -EINVAL; 1377 } 1378 1379 /* Get the table index and GPI shift from PA */ 1380 gpi_info->gpt_l1_addr = GPT_L0_TBLD_ADDR(gpt_l0_desc); 1381 gpi_info->idx = (unsigned int)GPT_L1_INDEX(base); 1382 gpi_info->gpi_shift = GPT_L1_GPI_IDX(gpt_config.p, base) << 2; 1383 1384 #if (RME_GPT_BITLOCK_BLOCK != 0) 1385 /* Block index */ 1386 block_idx = (unsigned int)(base / (RME_GPT_BITLOCK_BLOCK * SZ_512M)); 1387 1388 /* Bitlock address and mask */ 1389 gpi_info->lock = &gpt_bitlock[block_idx / LOCK_BITS]; 1390 gpi_info->mask = 1U << (block_idx & (LOCK_BITS - 1U)); 1391 #endif 1392 return 0; 1393 } 1394 1395 /* 1396 * Helper to retrieve the gpt_l1_desc and GPI information from gpi_info. 1397 * This function is called with bitlock or spinlock acquired. 1398 */ 1399 static void read_gpi(gpi_info_t *gpi_info) 1400 { 1401 gpi_info->gpt_l1_desc = (gpi_info->gpt_l1_addr)[gpi_info->idx]; 1402 1403 if ((gpi_info->gpt_l1_desc & GPT_L1_TYPE_CONT_DESC_MASK) == 1404 GPT_L1_TYPE_CONT_DESC) { 1405 /* Read GPI from Contiguous descriptor */ 1406 gpi_info->gpi = (unsigned int)GPT_L1_CONT_GPI(gpi_info->gpt_l1_desc); 1407 } else { 1408 /* Read GPI from Granules descriptor */ 1409 gpi_info->gpi = (unsigned int)((gpi_info->gpt_l1_desc >> gpi_info->gpi_shift) & 1410 GPT_L1_GRAN_DESC_GPI_MASK); 1411 } 1412 } 1413 1414 static void flush_page_to_popa(uintptr_t addr) 1415 { 1416 size_t size = GPT_PGS_ACTUAL_SIZE(gpt_config.p); 1417 1418 if (is_feat_mte2_supported()) { 1419 flush_dcache_to_popa_range_mte2(addr, size); 1420 } else { 1421 flush_dcache_to_popa_range(addr, size); 1422 } 1423 } 1424 1425 /* 1426 * Helper function to check if all L1 entries in 2MB block have 1427 * the same Granules descriptor value. 1428 * 1429 * Parameters 1430 * base Base address of the region to be checked 1431 * gpi_info Pointer to 'gpt_config_t' structure 1432 * l1_desc GPT Granules descriptor with all entries 1433 * set to the same GPI. 1434 * 1435 * Return 1436 * true if L1 all entries have the same descriptor value, false otherwise. 1437 */ 1438 __unused static bool check_fuse_2mb(uint64_t base, const gpi_info_t *gpi_info, 1439 uint64_t l1_desc) 1440 { 1441 /* Last L1 entry index in 2MB block */ 1442 unsigned int long idx = GPT_L1_INDEX(ALIGN_2MB(base)) + 1443 gpt_l1_cnt_2mb - 1UL; 1444 1445 /* Number of L1 entries in 2MB block */ 1446 unsigned int cnt = gpt_l1_cnt_2mb; 1447 1448 /* 1449 * Start check from the last L1 entry and continue until the first 1450 * non-matching to the passed Granules descriptor value is found. 1451 */ 1452 while (cnt-- != 0U) { 1453 if (gpi_info->gpt_l1_addr[idx--] != l1_desc) { 1454 /* Non-matching L1 entry found */ 1455 return false; 1456 } 1457 } 1458 1459 return true; 1460 } 1461 1462 __unused static void fuse_2mb(uint64_t base, const gpi_info_t *gpi_info, 1463 uint64_t l1_desc) 1464 { 1465 /* L1 entry index of the start of 2MB block */ 1466 unsigned long idx_2 = GPT_L1_INDEX(ALIGN_2MB(base)); 1467 1468 /* 2MB Contiguous descriptor */ 1469 uint64_t l1_cont_desc = GPT_L1_CONT_DESC(l1_desc, 2MB); 1470 1471 VERBOSE("GPT: %s(0x%"PRIxPTR" 0x%"PRIx64")\n", __func__, base, l1_desc); 1472 1473 fill_desc(&gpi_info->gpt_l1_addr[idx_2], l1_cont_desc, L1_QWORDS_2MB); 1474 } 1475 1476 /* 1477 * Helper function to check if all 1st L1 entries of 2MB blocks 1478 * in 32MB have the same 2MB Contiguous descriptor value. 1479 * 1480 * Parameters 1481 * base Base address of the region to be checked 1482 * gpi_info Pointer to 'gpt_config_t' structure 1483 * l1_desc GPT Granules descriptor. 1484 * 1485 * Return 1486 * true if all L1 entries have the same descriptor value, false otherwise. 1487 */ 1488 __unused static bool check_fuse_32mb(uint64_t base, const gpi_info_t *gpi_info, 1489 uint64_t l1_desc) 1490 { 1491 /* The 1st L1 entry index of the last 2MB block in 32MB */ 1492 unsigned long idx = GPT_L1_INDEX(ALIGN_32MB(base)) + 1493 (15UL * gpt_l1_cnt_2mb); 1494 1495 /* 2MB Contiguous descriptor */ 1496 uint64_t l1_cont_desc = GPT_L1_CONT_DESC(l1_desc, 2MB); 1497 1498 /* Number of 2MB blocks in 32MB */ 1499 unsigned int cnt = 16U; 1500 1501 /* Set the first L1 entry to 2MB Contiguous descriptor */ 1502 gpi_info->gpt_l1_addr[GPT_L1_INDEX(ALIGN_2MB(base))] = l1_cont_desc; 1503 1504 /* 1505 * Start check from the 1st L1 entry of the last 2MB block and 1506 * continue until the first non-matching to 2MB Contiguous descriptor 1507 * value is found. 1508 */ 1509 while (cnt-- != 0U) { 1510 if (gpi_info->gpt_l1_addr[idx] != l1_cont_desc) { 1511 /* Non-matching L1 entry found */ 1512 return false; 1513 } 1514 idx -= gpt_l1_cnt_2mb; 1515 } 1516 1517 return true; 1518 } 1519 1520 __unused static void fuse_32mb(uint64_t base, const gpi_info_t *gpi_info, 1521 uint64_t l1_desc) 1522 { 1523 /* L1 entry index of the start of 32MB block */ 1524 unsigned long idx_32 = GPT_L1_INDEX(ALIGN_32MB(base)); 1525 1526 /* 32MB Contiguous descriptor */ 1527 uint64_t l1_cont_desc = GPT_L1_CONT_DESC(l1_desc, 32MB); 1528 1529 VERBOSE("GPT: %s(0x%"PRIxPTR" 0x%"PRIx64")\n", __func__, base, l1_desc); 1530 1531 fill_desc(&gpi_info->gpt_l1_addr[idx_32], l1_cont_desc, L1_QWORDS_32MB); 1532 } 1533 1534 /* 1535 * Helper function to check if all 1st L1 entries of 32MB blocks 1536 * in 512MB have the same 32MB Contiguous descriptor value. 1537 * 1538 * Parameters 1539 * base Base address of the region to be checked 1540 * gpi_info Pointer to 'gpt_config_t' structure 1541 * l1_desc GPT Granules descriptor. 1542 * 1543 * Return 1544 * true if all L1 entries have the same descriptor value, false otherwise. 1545 */ 1546 __unused static bool check_fuse_512mb(uint64_t base, const gpi_info_t *gpi_info, 1547 uint64_t l1_desc) 1548 { 1549 /* The 1st L1 entry index of the last 32MB block in 512MB */ 1550 unsigned long idx = GPT_L1_INDEX(ALIGN_512MB(base)) + 1551 (15UL * 16UL * gpt_l1_cnt_2mb); 1552 1553 /* 32MB Contiguous descriptor */ 1554 uint64_t l1_cont_desc = GPT_L1_CONT_DESC(l1_desc, 32MB); 1555 1556 /* Number of 32MB blocks in 512MB */ 1557 unsigned int cnt = 16U; 1558 1559 /* Set the first L1 entry to 2MB Contiguous descriptor */ 1560 gpi_info->gpt_l1_addr[GPT_L1_INDEX(ALIGN_32MB(base))] = l1_cont_desc; 1561 1562 /* 1563 * Start check from the 1st L1 entry of the last 32MB block and 1564 * continue until the first non-matching to 32MB Contiguous descriptor 1565 * value is found. 1566 */ 1567 while (cnt-- != 0U) { 1568 if (gpi_info->gpt_l1_addr[idx] != l1_cont_desc) { 1569 /* Non-matching L1 entry found */ 1570 return false; 1571 } 1572 idx -= 16UL * gpt_l1_cnt_2mb; 1573 } 1574 1575 return true; 1576 } 1577 1578 __unused static void fuse_512mb(uint64_t base, const gpi_info_t *gpi_info, 1579 uint64_t l1_desc) 1580 { 1581 /* L1 entry index of the start of 512MB block */ 1582 unsigned long idx_512 = GPT_L1_INDEX(ALIGN_512MB(base)); 1583 1584 /* 512MB Contiguous descriptor */ 1585 uint64_t l1_cont_desc = GPT_L1_CONT_DESC(l1_desc, 512MB); 1586 1587 VERBOSE("GPT: %s(0x%"PRIxPTR" 0x%"PRIx64")\n", __func__, base, l1_desc); 1588 1589 fill_desc(&gpi_info->gpt_l1_addr[idx_512], l1_cont_desc, L1_QWORDS_512MB); 1590 } 1591 1592 /* 1593 * Helper function to convert GPI entries in a single L1 table 1594 * from Granules to Contiguous descriptor. 1595 * 1596 * Parameters 1597 * base Base address of the region to be written 1598 * gpi_info Pointer to 'gpt_config_t' structure 1599 * l1_desc GPT Granules descriptor with all entries 1600 * set to the same GPI. 1601 */ 1602 __unused static void fuse_block(uint64_t base, const gpi_info_t *gpi_info, 1603 uint64_t l1_desc) 1604 { 1605 /* Start with check for 2MB block */ 1606 if (!check_fuse_2mb(base, gpi_info, l1_desc)) { 1607 /* Check for 2MB fusing failed */ 1608 return; 1609 } 1610 1611 #if (RME_GPT_MAX_BLOCK == 2) 1612 fuse_2mb(base, gpi_info, l1_desc); 1613 #else 1614 /* Check for 32MB block */ 1615 if (!check_fuse_32mb(base, gpi_info, l1_desc)) { 1616 /* Check for 32MB fusing failed, fuse to 2MB */ 1617 fuse_2mb(base, gpi_info, l1_desc); 1618 return; 1619 } 1620 1621 #if (RME_GPT_MAX_BLOCK == 32) 1622 fuse_32mb(base, gpi_info, l1_desc); 1623 #else 1624 /* Check for 512MB block */ 1625 if (!check_fuse_512mb(base, gpi_info, l1_desc)) { 1626 /* Check for 512MB fusing failed, fuse to 32MB */ 1627 fuse_32mb(base, gpi_info, l1_desc); 1628 return; 1629 } 1630 1631 /* Fuse to 512MB */ 1632 fuse_512mb(base, gpi_info, l1_desc); 1633 1634 #endif /* RME_GPT_MAX_BLOCK == 32 */ 1635 #endif /* RME_GPT_MAX_BLOCK == 2 */ 1636 } 1637 1638 /* 1639 * Helper function to convert GPI entries in a single L1 table 1640 * from Contiguous to Granules descriptor. This function updates 1641 * descriptor to Granules in passed 'gpt_config_t' structure as 1642 * the result of shuttering. 1643 * 1644 * Parameters 1645 * base Base address of the region to be written 1646 * gpi_info Pointer to 'gpt_config_t' structure 1647 * l1_desc GPT Granules descriptor set this range to. 1648 */ 1649 __unused static void shatter_block(uint64_t base, gpi_info_t *gpi_info, 1650 uint64_t l1_desc) 1651 { 1652 /* Look-up table for 2MB, 32MB and 512MB locks shattering */ 1653 static const gpt_shatter_func gpt_shatter_lookup[] = { 1654 shatter_2mb, 1655 shatter_32mb, 1656 shatter_512mb 1657 }; 1658 1659 /* Look-up table for invalidation TLBs for 2MB, 32MB and 512MB blocks */ 1660 static const gpt_tlbi_lookup_t tlbi_lookup[] = { 1661 { tlbirpalos_2m, ~(SZ_2M - 1UL) }, 1662 { tlbirpalos_32m, ~(SZ_32M - 1UL) }, 1663 { tlbirpalos_512m, ~(SZ_512M - 1UL) } 1664 }; 1665 1666 /* Get shattering level from Contig field of Contiguous descriptor */ 1667 unsigned long level = GPT_L1_CONT_CONTIG(gpi_info->gpt_l1_desc) - 1UL; 1668 1669 /* Shatter contiguous block */ 1670 gpt_shatter_lookup[level](base, gpi_info, l1_desc); 1671 1672 tlbi_lookup[level].function(base & tlbi_lookup[level].mask); 1673 dsbosh(); 1674 1675 /* 1676 * Update 'gpt_config_t' structure's descriptor to Granules to reflect 1677 * the shattered GPI back to caller. 1678 */ 1679 gpi_info->gpt_l1_desc = l1_desc; 1680 } 1681 1682 /* 1683 * This function is the granule transition delegate service. When a granule 1684 * transition request occurs it is routed to this function to have the request, 1685 * if valid, fulfilled following A1.1.1 Delegate of RME supplement. 1686 * 1687 * TODO: implement support for transitioning multiple granules at once. 1688 * 1689 * Parameters 1690 * base Base address of the region to transition, must be 1691 * aligned to granule size. 1692 * size Size of region to transition, must be aligned to granule 1693 * size. 1694 * src_sec_state Security state of the caller. 1695 * 1696 * Return 1697 * Negative Linux error code in the event of a failure, 0 for success. 1698 */ 1699 int gpt_delegate_pas(uint64_t base, size_t size, unsigned int src_sec_state) 1700 { 1701 gpi_info_t gpi_info; 1702 uint64_t nse, __unused l1_desc; 1703 unsigned int target_pas; 1704 int res; 1705 1706 /* Ensure that the tables have been set up before taking requests */ 1707 assert(gpt_config.plat_gpt_l0_base != 0UL); 1708 1709 /* Ensure that caches are enabled */ 1710 assert((read_sctlr_el3() & SCTLR_C_BIT) != 0UL); 1711 1712 /* See if this is a single or a range of granule transition */ 1713 if (size != GPT_PGS_ACTUAL_SIZE(gpt_config.p)) { 1714 return -EINVAL; 1715 } 1716 1717 /* Check that base and size are valid */ 1718 if ((ULONG_MAX - base) < size) { 1719 VERBOSE("GPT: Transition request address overflow!\n"); 1720 VERBOSE(" Base=0x%"PRIx64"\n", base); 1721 VERBOSE(" Size=0x%lx\n", size); 1722 return -EINVAL; 1723 } 1724 1725 /* Make sure base and size are valid */ 1726 if (((base & (GPT_PGS_ACTUAL_SIZE(gpt_config.p) - 1UL)) != 0UL) || 1727 ((size & (GPT_PGS_ACTUAL_SIZE(gpt_config.p) - 1UL)) != 0UL) || 1728 (size == 0UL) || 1729 ((base + size) >= GPT_PPS_ACTUAL_SIZE(gpt_config.t))) { 1730 VERBOSE("GPT: Invalid granule transition address range!\n"); 1731 VERBOSE(" Base=0x%"PRIx64"\n", base); 1732 VERBOSE(" Size=0x%lx\n", size); 1733 return -EINVAL; 1734 } 1735 1736 /* Delegate request can only come from REALM or SECURE */ 1737 if ((src_sec_state != SMC_FROM_REALM) && 1738 (src_sec_state != SMC_FROM_SECURE)) { 1739 VERBOSE("GPT: Invalid caller security state 0x%x\n", 1740 src_sec_state); 1741 return -EINVAL; 1742 } 1743 1744 if (src_sec_state == SMC_FROM_REALM) { 1745 target_pas = GPT_GPI_REALM; 1746 nse = (uint64_t)GPT_NSE_REALM << GPT_NSE_SHIFT; 1747 l1_desc = GPT_L1_REALM_DESC; 1748 } else { 1749 target_pas = GPT_GPI_SECURE; 1750 nse = (uint64_t)GPT_NSE_SECURE << GPT_NSE_SHIFT; 1751 l1_desc = GPT_L1_SECURE_DESC; 1752 } 1753 1754 res = get_gpi_params(base, &gpi_info); 1755 if (res != 0) { 1756 return res; 1757 } 1758 1759 /* 1760 * Access to GPT is controlled by a lock to ensure that no more 1761 * than one CPU is allowed to make changes at any given time. 1762 */ 1763 GPT_LOCK; 1764 read_gpi(&gpi_info); 1765 1766 /* Check that the current address is in NS state */ 1767 if (gpi_info.gpi != GPT_GPI_NS) { 1768 VERBOSE("GPT: Only Granule in NS state can be delegated.\n"); 1769 VERBOSE(" Caller: %u, Current GPI: %u\n", src_sec_state, 1770 gpi_info.gpi); 1771 GPT_UNLOCK; 1772 return -EPERM; 1773 } 1774 1775 #if (RME_GPT_MAX_BLOCK != 0) 1776 /* Check for Contiguous descriptor */ 1777 if ((gpi_info.gpt_l1_desc & GPT_L1_TYPE_CONT_DESC_MASK) == 1778 GPT_L1_TYPE_CONT_DESC) { 1779 shatter_block(base, &gpi_info, GPT_L1_NS_DESC); 1780 } 1781 #endif 1782 /* 1783 * In order to maintain mutual distrust between Realm and Secure 1784 * states, remove any data speculatively fetched into the target 1785 * physical address space. 1786 * Issue DC CIPAPA or DC_CIGDPAPA on implementations with FEAT_MTE2. 1787 */ 1788 flush_page_to_popa(base | nse); 1789 1790 write_gpt(&gpi_info.gpt_l1_desc, gpi_info.gpt_l1_addr, 1791 gpi_info.gpi_shift, gpi_info.idx, target_pas); 1792 1793 /* Ensure that all agents observe the new configuration */ 1794 tlbi_page_dsbosh(base); 1795 1796 nse = (uint64_t)GPT_NSE_NS << GPT_NSE_SHIFT; 1797 1798 /* Ensure that the scrubbed data have made it past the PoPA */ 1799 flush_page_to_popa(base | nse); 1800 1801 #if (RME_GPT_MAX_BLOCK != 0) 1802 if (gpi_info.gpt_l1_desc == l1_desc) { 1803 /* Try to fuse */ 1804 fuse_block(base, &gpi_info, l1_desc); 1805 } 1806 #endif 1807 1808 /* Unlock the lock to GPT */ 1809 GPT_UNLOCK; 1810 1811 /* 1812 * The isb() will be done as part of context 1813 * synchronization when returning to lower EL. 1814 */ 1815 VERBOSE("GPT: Granule 0x%"PRIx64" GPI 0x%x->0x%x\n", 1816 base, gpi_info.gpi, target_pas); 1817 1818 return 0; 1819 } 1820 1821 /* 1822 * This function is the granule transition undelegate service. When a granule 1823 * transition request occurs it is routed to this function where the request is 1824 * validated then fulfilled if possible. 1825 * 1826 * TODO: implement support for transitioning multiple granules at once. 1827 * 1828 * Parameters 1829 * base Base address of the region to transition, must be 1830 * aligned to granule size. 1831 * size Size of region to transition, must be aligned to granule 1832 * size. 1833 * src_sec_state Security state of the caller. 1834 * 1835 * Return 1836 * Negative Linux error code in the event of a failure, 0 for success. 1837 */ 1838 int gpt_undelegate_pas(uint64_t base, size_t size, unsigned int src_sec_state) 1839 { 1840 gpi_info_t gpi_info; 1841 uint64_t nse, __unused l1_desc; 1842 int res; 1843 1844 /* Ensure that the tables have been set up before taking requests */ 1845 assert(gpt_config.plat_gpt_l0_base != 0UL); 1846 1847 /* Ensure that MMU and caches are enabled */ 1848 assert((read_sctlr_el3() & SCTLR_C_BIT) != 0UL); 1849 1850 /* See if this is a single or a range of granule transition */ 1851 if (size != GPT_PGS_ACTUAL_SIZE(gpt_config.p)) { 1852 return -EINVAL; 1853 } 1854 1855 /* Check that base and size are valid */ 1856 if ((ULONG_MAX - base) < size) { 1857 VERBOSE("GPT: Transition request address overflow!\n"); 1858 VERBOSE(" Base=0x%"PRIx64"\n", base); 1859 VERBOSE(" Size=0x%lx\n", size); 1860 return -EINVAL; 1861 } 1862 1863 /* Make sure base and size are valid */ 1864 if (((base & (GPT_PGS_ACTUAL_SIZE(gpt_config.p) - 1UL)) != 0UL) || 1865 ((size & (GPT_PGS_ACTUAL_SIZE(gpt_config.p) - 1UL)) != 0UL) || 1866 (size == 0UL) || 1867 ((base + size) >= GPT_PPS_ACTUAL_SIZE(gpt_config.t))) { 1868 VERBOSE("GPT: Invalid granule transition address range!\n"); 1869 VERBOSE(" Base=0x%"PRIx64"\n", base); 1870 VERBOSE(" Size=0x%lx\n", size); 1871 return -EINVAL; 1872 } 1873 1874 res = get_gpi_params(base, &gpi_info); 1875 if (res != 0) { 1876 return res; 1877 } 1878 1879 /* 1880 * Access to GPT is controlled by a lock to ensure that no more 1881 * than one CPU is allowed to make changes at any given time. 1882 */ 1883 GPT_LOCK; 1884 read_gpi(&gpi_info); 1885 1886 /* Check that the current address is in the delegated state */ 1887 if ((src_sec_state == SMC_FROM_REALM) && 1888 (gpi_info.gpi == GPT_GPI_REALM)) { 1889 l1_desc = GPT_L1_REALM_DESC; 1890 nse = (uint64_t)GPT_NSE_REALM << GPT_NSE_SHIFT; 1891 } else if ((src_sec_state == SMC_FROM_SECURE) && 1892 (gpi_info.gpi == GPT_GPI_SECURE)) { 1893 l1_desc = GPT_L1_SECURE_DESC; 1894 nse = (uint64_t)GPT_NSE_SECURE << GPT_NSE_SHIFT; 1895 } else { 1896 VERBOSE("GPT: Only Granule in REALM or SECURE state can be undelegated\n"); 1897 VERBOSE(" Caller: %u Current GPI: %u\n", src_sec_state, 1898 gpi_info.gpi); 1899 GPT_UNLOCK; 1900 return -EPERM; 1901 } 1902 1903 #if (RME_GPT_MAX_BLOCK != 0) 1904 /* Check for Contiguous descriptor */ 1905 if ((gpi_info.gpt_l1_desc & GPT_L1_TYPE_CONT_DESC_MASK) == 1906 GPT_L1_TYPE_CONT_DESC) { 1907 shatter_block(base, &gpi_info, l1_desc); 1908 } 1909 #endif 1910 /* 1911 * In order to maintain mutual distrust between Realm and Secure 1912 * states, remove access now, in order to guarantee that writes 1913 * to the currently-accessible physical address space will not 1914 * later become observable. 1915 */ 1916 write_gpt(&gpi_info.gpt_l1_desc, gpi_info.gpt_l1_addr, 1917 gpi_info.gpi_shift, gpi_info.idx, GPT_GPI_NO_ACCESS); 1918 1919 /* Ensure that all agents observe the new NO_ACCESS configuration */ 1920 tlbi_page_dsbosh(base); 1921 1922 /* Ensure that the scrubbed data have made it past the PoPA */ 1923 flush_page_to_popa(base | nse); 1924 1925 /* 1926 * Remove any data loaded speculatively in NS space from before 1927 * the scrubbing. 1928 */ 1929 nse = (uint64_t)GPT_NSE_NS << GPT_NSE_SHIFT; 1930 1931 flush_page_to_popa(base | nse); 1932 1933 /* Clear existing GPI encoding and transition granule */ 1934 write_gpt(&gpi_info.gpt_l1_desc, gpi_info.gpt_l1_addr, 1935 gpi_info.gpi_shift, gpi_info.idx, GPT_GPI_NS); 1936 1937 /* Ensure that all agents observe the new NS configuration */ 1938 tlbi_page_dsbosh(base); 1939 1940 #if (RME_GPT_MAX_BLOCK != 0) 1941 if (gpi_info.gpt_l1_desc == GPT_L1_NS_DESC) { 1942 /* Try to fuse */ 1943 fuse_block(base, &gpi_info, GPT_L1_NS_DESC); 1944 } 1945 #endif 1946 /* Unlock the lock to GPT */ 1947 GPT_UNLOCK; 1948 1949 /* 1950 * The isb() will be done as part of context 1951 * synchronization when returning to lower EL. 1952 */ 1953 VERBOSE("GPT: Granule 0x%"PRIx64" GPI 0x%x->0x%x\n", 1954 base, gpi_info.gpi, GPT_GPI_NS); 1955 1956 return 0; 1957 } 1958