1 /* 2 * Copyright (c) 2021, Arm Limited. All rights reserved. 3 * 4 * SPDX-License-Identifier: BSD-3-Clause 5 */ 6 7 #include <assert.h> 8 #include <errno.h> 9 #include <inttypes.h> 10 #include <limits.h> 11 #include <stdint.h> 12 13 #include <arch.h> 14 #include <arch_helpers.h> 15 #include <common/debug.h> 16 #include "gpt_rme_private.h" 17 #include <lib/gpt_rme/gpt_rme.h> 18 #include <lib/smccc.h> 19 #include <lib/spinlock.h> 20 #include <lib/xlat_tables/xlat_tables_v2.h> 21 22 #if !ENABLE_RME 23 #error "ENABLE_RME must be enabled to use the GPT library." 24 #endif 25 26 /* 27 * Lookup T from PPS 28 * 29 * PPS Size T 30 * 0b000 4GB 32 31 * 0b001 64GB 36 32 * 0b010 1TB 40 33 * 0b011 4TB 42 34 * 0b100 16TB 44 35 * 0b101 256TB 48 36 * 0b110 4PB 52 37 * 38 * See section 15.1.27 of the RME specification. 39 */ 40 static const gpt_t_val_e gpt_t_lookup[] = {PPS_4GB_T, PPS_64GB_T, 41 PPS_1TB_T, PPS_4TB_T, 42 PPS_16TB_T, PPS_256TB_T, 43 PPS_4PB_T}; 44 45 /* 46 * Lookup P from PGS 47 * 48 * PGS Size P 49 * 0b00 4KB 12 50 * 0b10 16KB 14 51 * 0b01 64KB 16 52 * 53 * Note that pgs=0b10 is 16KB and pgs=0b01 is 64KB, this is not a typo. 54 * 55 * See section 15.1.27 of the RME specification. 56 */ 57 static const gpt_p_val_e gpt_p_lookup[] = {PGS_4KB_P, PGS_64KB_P, PGS_16KB_P}; 58 59 /* 60 * This structure contains GPT configuration data. 61 */ 62 typedef struct { 63 uintptr_t plat_gpt_l0_base; 64 gpccr_pps_e pps; 65 gpt_t_val_e t; 66 gpccr_pgs_e pgs; 67 gpt_p_val_e p; 68 } gpt_config_t; 69 70 static gpt_config_t gpt_config; 71 72 /* These variables are used during initialization of the L1 tables. */ 73 static unsigned int gpt_next_l1_tbl_idx; 74 static uintptr_t gpt_l1_tbl; 75 76 /* 77 * This function checks to see if a GPI value is valid. 78 * 79 * These are valid GPI values. 80 * GPT_GPI_NO_ACCESS U(0x0) 81 * GPT_GPI_SECURE U(0x8) 82 * GPT_GPI_NS U(0x9) 83 * GPT_GPI_ROOT U(0xA) 84 * GPT_GPI_REALM U(0xB) 85 * GPT_GPI_ANY U(0xF) 86 * 87 * Parameters 88 * gpi GPI to check for validity. 89 * 90 * Return 91 * true for a valid GPI, false for an invalid one. 92 */ 93 static bool gpt_is_gpi_valid(unsigned int gpi) 94 { 95 if ((gpi == GPT_GPI_NO_ACCESS) || (gpi == GPT_GPI_ANY) || 96 ((gpi >= GPT_GPI_SECURE) && (gpi <= GPT_GPI_REALM))) { 97 return true; 98 } else { 99 return false; 100 } 101 } 102 103 /* 104 * This function checks to see if two PAS regions overlap. 105 * 106 * Parameters 107 * base_1: base address of first PAS 108 * size_1: size of first PAS 109 * base_2: base address of second PAS 110 * size_2: size of second PAS 111 * 112 * Return 113 * True if PAS regions overlap, false if they do not. 114 */ 115 static bool gpt_check_pas_overlap(uintptr_t base_1, size_t size_1, 116 uintptr_t base_2, size_t size_2) 117 { 118 if (((base_1 + size_1) > base_2) && ((base_2 + size_2) > base_1)) { 119 return true; 120 } else { 121 return false; 122 } 123 } 124 125 /* 126 * This helper function checks to see if a PAS region from index 0 to 127 * (pas_idx - 1) occupies the L0 region at index l0_idx in the L0 table. 128 * 129 * Parameters 130 * l0_idx: Index of the L0 entry to check 131 * pas_regions: PAS region array 132 * pas_idx: Upper bound of the PAS array index. 133 * 134 * Return 135 * True if a PAS region occupies the L0 region in question, false if not. 136 */ 137 static bool gpt_does_previous_pas_exist_here(unsigned int l0_idx, 138 pas_region_t *pas_regions, 139 unsigned int pas_idx) 140 { 141 /* Iterate over PAS regions up to pas_idx. */ 142 for (unsigned int i = 0U; i < pas_idx; i++) { 143 if (gpt_check_pas_overlap((GPT_L0GPTSZ_ACTUAL_SIZE * l0_idx), 144 GPT_L0GPTSZ_ACTUAL_SIZE, 145 pas_regions[i].base_pa, pas_regions[i].size)) { 146 return true; 147 } 148 } 149 return false; 150 } 151 152 /* 153 * This function iterates over all of the PAS regions and checks them to ensure 154 * proper alignment of base and size, that the GPI is valid, and that no regions 155 * overlap. As a part of the overlap checks, this function checks existing L0 156 * mappings against the new PAS regions in the event that gpt_init_pas_l1_tables 157 * is called multiple times to place L1 tables in different areas of memory. It 158 * also counts the number of L1 tables needed and returns it on success. 159 * 160 * Parameters 161 * *pas_regions Pointer to array of PAS region structures. 162 * pas_region_cnt Total number of PAS regions in the array. 163 * 164 * Return 165 * Negative Linux error code in the event of a failure, number of L1 regions 166 * required when successful. 167 */ 168 static int gpt_validate_pas_mappings(pas_region_t *pas_regions, 169 unsigned int pas_region_cnt) 170 { 171 unsigned int idx; 172 unsigned int l1_cnt = 0U; 173 unsigned int pas_l1_cnt; 174 uint64_t *l0_desc = (uint64_t *)gpt_config.plat_gpt_l0_base; 175 176 assert(pas_regions != NULL); 177 assert(pas_region_cnt != 0U); 178 179 for (idx = 0U; idx < pas_region_cnt; idx++) { 180 /* Check for arithmetic overflow in region. */ 181 if ((ULONG_MAX - pas_regions[idx].base_pa) < 182 pas_regions[idx].size) { 183 ERROR("[GPT] Address overflow in PAS[%u]!\n", idx); 184 return -EOVERFLOW; 185 } 186 187 /* Initial checks for PAS validity. */ 188 if (((pas_regions[idx].base_pa + pas_regions[idx].size) > 189 GPT_PPS_ACTUAL_SIZE(gpt_config.t)) || 190 !gpt_is_gpi_valid(GPT_PAS_ATTR_GPI(pas_regions[idx].attrs))) { 191 ERROR("[GPT] PAS[%u] is invalid!\n", idx); 192 return -EFAULT; 193 } 194 195 /* 196 * Make sure this PAS does not overlap with another one. We 197 * start from idx + 1 instead of 0 since prior PAS mappings will 198 * have already checked themselves against this one. 199 */ 200 for (unsigned int i = idx + 1; i < pas_region_cnt; i++) { 201 if (gpt_check_pas_overlap(pas_regions[idx].base_pa, 202 pas_regions[idx].size, 203 pas_regions[i].base_pa, 204 pas_regions[i].size)) { 205 ERROR("[GPT] PAS[%u] overlaps with PAS[%u]\n", 206 i, idx); 207 return -EFAULT; 208 } 209 } 210 211 /* 212 * Since this function can be called multiple times with 213 * separate L1 tables we need to check the existing L0 mapping 214 * to see if this PAS would fall into one that has already been 215 * initialized. 216 */ 217 for (unsigned int i = GPT_L0_IDX(pas_regions[idx].base_pa); 218 i <= GPT_L0_IDX(pas_regions[idx].base_pa + pas_regions[idx].size - 1); 219 i++) { 220 if ((GPT_L0_TYPE(l0_desc[i]) == GPT_L0_TYPE_BLK_DESC) && 221 (GPT_L0_BLKD_GPI(l0_desc[i]) == GPT_GPI_ANY)) { 222 /* This descriptor is unused so continue. */ 223 continue; 224 } 225 226 /* 227 * This descriptor has been initialized in a previous 228 * call to this function so cannot be initialized again. 229 */ 230 ERROR("[GPT] PAS[%u] overlaps with previous L0[%d]!\n", 231 idx, i); 232 return -EFAULT; 233 } 234 235 /* Check for block mapping (L0) type. */ 236 if (GPT_PAS_ATTR_MAP_TYPE(pas_regions[idx].attrs) == 237 GPT_PAS_ATTR_MAP_TYPE_BLOCK) { 238 /* Make sure base and size are block-aligned. */ 239 if (!GPT_IS_L0_ALIGNED(pas_regions[idx].base_pa) || 240 !GPT_IS_L0_ALIGNED(pas_regions[idx].size)) { 241 ERROR("[GPT] PAS[%u] is not block-aligned!\n", 242 idx); 243 return -EFAULT; 244 } 245 246 continue; 247 } 248 249 /* Check for granule mapping (L1) type. */ 250 if (GPT_PAS_ATTR_MAP_TYPE(pas_regions[idx].attrs) == 251 GPT_PAS_ATTR_MAP_TYPE_GRANULE) { 252 /* Make sure base and size are granule-aligned. */ 253 if (!GPT_IS_L1_ALIGNED(gpt_config.p, pas_regions[idx].base_pa) || 254 !GPT_IS_L1_ALIGNED(gpt_config.p, pas_regions[idx].size)) { 255 ERROR("[GPT] PAS[%u] is not granule-aligned!\n", 256 idx); 257 return -EFAULT; 258 } 259 260 /* Find how many L1 tables this PAS occupies. */ 261 pas_l1_cnt = (GPT_L0_IDX(pas_regions[idx].base_pa + 262 pas_regions[idx].size - 1) - 263 GPT_L0_IDX(pas_regions[idx].base_pa) + 1); 264 265 /* 266 * This creates a situation where, if multiple PAS 267 * regions occupy the same table descriptor, we can get 268 * an artificially high total L1 table count. The way we 269 * handle this is by checking each PAS against those 270 * before it in the array, and if they both occupy the 271 * same PAS we subtract from pas_l1_cnt and only the 272 * first PAS in the array gets to count it. 273 */ 274 275 /* 276 * If L1 count is greater than 1 we know the start and 277 * end PAs are in different L0 regions so we must check 278 * both for overlap against other PAS. 279 */ 280 if (pas_l1_cnt > 1) { 281 if (gpt_does_previous_pas_exist_here( 282 GPT_L0_IDX(pas_regions[idx].base_pa + 283 pas_regions[idx].size - 1), 284 pas_regions, idx)) { 285 pas_l1_cnt = pas_l1_cnt - 1; 286 } 287 } 288 289 if (gpt_does_previous_pas_exist_here( 290 GPT_L0_IDX(pas_regions[idx].base_pa), 291 pas_regions, idx)) { 292 pas_l1_cnt = pas_l1_cnt - 1; 293 } 294 295 l1_cnt += pas_l1_cnt; 296 continue; 297 } 298 299 /* If execution reaches this point, mapping type is invalid. */ 300 ERROR("[GPT] PAS[%u] has invalid mapping type 0x%x.\n", idx, 301 GPT_PAS_ATTR_MAP_TYPE(pas_regions[idx].attrs)); 302 return -EINVAL; 303 } 304 305 return l1_cnt; 306 } 307 308 /* 309 * This function validates L0 initialization parameters. 310 * 311 * Parameters 312 * l0_mem_base Base address of memory used for L0 tables. 313 * l1_mem_size Size of memory available for L0 tables. 314 * 315 * Return 316 * Negative Linux error code in the event of a failure, 0 for success. 317 */ 318 static int gpt_validate_l0_params(gpccr_pps_e pps, uintptr_t l0_mem_base, 319 size_t l0_mem_size) 320 { 321 size_t l0_alignment; 322 323 /* 324 * Make sure PPS is valid and then store it since macros need this value 325 * to work. 326 */ 327 if (pps > GPT_PPS_MAX) { 328 ERROR("[GPT] Invalid PPS: 0x%x\n", pps); 329 return -EINVAL; 330 } 331 gpt_config.pps = pps; 332 gpt_config.t = gpt_t_lookup[pps]; 333 334 /* Alignment must be the greater of 4k or l0 table size. */ 335 l0_alignment = PAGE_SIZE_4KB; 336 if (l0_alignment < GPT_L0_TABLE_SIZE(gpt_config.t)) { 337 l0_alignment = GPT_L0_TABLE_SIZE(gpt_config.t); 338 } 339 340 /* Check base address. */ 341 if ((l0_mem_base == 0U) || ((l0_mem_base & (l0_alignment - 1)) != 0U)) { 342 ERROR("[GPT] Invalid L0 base address: 0x%lx\n", l0_mem_base); 343 return -EFAULT; 344 } 345 346 /* Check size. */ 347 if (l0_mem_size < GPT_L0_TABLE_SIZE(gpt_config.t)) { 348 ERROR("[GPT] Inadequate L0 memory: need 0x%lx, have 0x%lx)\n", 349 GPT_L0_TABLE_SIZE(gpt_config.t), 350 l0_mem_size); 351 return -ENOMEM; 352 } 353 354 return 0; 355 } 356 357 /* 358 * In the event that L1 tables are needed, this function validates 359 * the L1 table generation parameters. 360 * 361 * Parameters 362 * l1_mem_base Base address of memory used for L1 table allocation. 363 * l1_mem_size Total size of memory available for L1 tables. 364 * l1_gpt_cnt Number of L1 tables needed. 365 * 366 * Return 367 * Negative Linux error code in the event of a failure, 0 for success. 368 */ 369 static int gpt_validate_l1_params(uintptr_t l1_mem_base, size_t l1_mem_size, 370 unsigned int l1_gpt_cnt) 371 { 372 size_t l1_gpt_mem_sz; 373 374 /* Check if the granularity is supported */ 375 if (!xlat_arch_is_granule_size_supported( 376 GPT_PGS_ACTUAL_SIZE(gpt_config.p))) { 377 return -EPERM; 378 } 379 380 /* Make sure L1 tables are aligned to their size. */ 381 if ((l1_mem_base & (GPT_L1_TABLE_SIZE(gpt_config.p) - 1)) != 0U) { 382 ERROR("[GPT] Unaligned L1 GPT base address: 0x%lx\n", 383 l1_mem_base); 384 return -EFAULT; 385 } 386 387 /* Get total memory needed for L1 tables. */ 388 l1_gpt_mem_sz = l1_gpt_cnt * GPT_L1_TABLE_SIZE(gpt_config.p); 389 390 /* Check for overflow. */ 391 if ((l1_gpt_mem_sz / GPT_L1_TABLE_SIZE(gpt_config.p)) != l1_gpt_cnt) { 392 ERROR("[GPT] Overflow calculating L1 memory size.\n"); 393 return -ENOMEM; 394 } 395 396 /* Make sure enough space was supplied. */ 397 if (l1_mem_size < l1_gpt_mem_sz) { 398 ERROR("[GPT] Inadequate memory for L1 GPTs. "); 399 ERROR(" Expected 0x%lx bytes. Got 0x%lx bytes\n", 400 l1_gpt_mem_sz, l1_mem_size); 401 return -ENOMEM; 402 } 403 404 VERBOSE("[GPT] Requested 0x%lx bytes for L1 GPTs.\n", l1_gpt_mem_sz); 405 return 0; 406 } 407 408 /* 409 * This function initializes L0 block descriptors (regions that cannot be 410 * transitioned at the granule level) according to the provided PAS. 411 * 412 * Parameters 413 * *pas Pointer to the structure defining the PAS region to 414 * initialize. 415 */ 416 static void gpt_generate_l0_blk_desc(pas_region_t *pas) 417 { 418 uint64_t gpt_desc; 419 unsigned int end_idx; 420 unsigned int idx; 421 uint64_t *l0_gpt_arr; 422 423 assert(gpt_config.plat_gpt_l0_base != 0U); 424 assert(pas != NULL); 425 426 /* 427 * Checking of PAS parameters has already been done in 428 * gpt_validate_pas_mappings so no need to check the same things again. 429 */ 430 431 l0_gpt_arr = (uint64_t *)gpt_config.plat_gpt_l0_base; 432 433 /* Create the GPT Block descriptor for this PAS region */ 434 gpt_desc = GPT_L0_BLK_DESC(GPT_PAS_ATTR_GPI(pas->attrs)); 435 436 /* Start index of this region in L0 GPTs */ 437 idx = pas->base_pa >> GPT_L0_IDX_SHIFT; 438 439 /* 440 * Determine number of L0 GPT descriptors covered by 441 * this PAS region and use the count to populate these 442 * descriptors. 443 */ 444 end_idx = (pas->base_pa + pas->size) >> GPT_L0_IDX_SHIFT; 445 446 /* Generate the needed block descriptors. */ 447 for (; idx < end_idx; idx++) { 448 l0_gpt_arr[idx] = gpt_desc; 449 VERBOSE("[GPT] L0 entry (BLOCK) index %u [%p]: GPI = 0x%" PRIx64 " (0x%" PRIx64 ")\n", 450 idx, &l0_gpt_arr[idx], 451 (gpt_desc >> GPT_L0_BLK_DESC_GPI_SHIFT) & 452 GPT_L0_BLK_DESC_GPI_MASK, l0_gpt_arr[idx]); 453 } 454 } 455 456 /* 457 * Helper function to determine if the end physical address lies in the same L0 458 * region as the current physical address. If true, the end physical address is 459 * returned else, the start address of the next region is returned. 460 * 461 * Parameters 462 * cur_pa Physical address of the current PA in the loop through 463 * the range. 464 * end_pa Physical address of the end PA in a PAS range. 465 * 466 * Return 467 * The PA of the end of the current range. 468 */ 469 static uintptr_t gpt_get_l1_end_pa(uintptr_t cur_pa, uintptr_t end_pa) 470 { 471 uintptr_t cur_idx; 472 uintptr_t end_idx; 473 474 cur_idx = cur_pa >> GPT_L0_IDX_SHIFT; 475 end_idx = end_pa >> GPT_L0_IDX_SHIFT; 476 477 assert(cur_idx <= end_idx); 478 479 if (cur_idx == end_idx) { 480 return end_pa; 481 } 482 483 return (cur_idx + 1U) << GPT_L0_IDX_SHIFT; 484 } 485 486 /* 487 * Helper function to fill out GPI entries in a single L1 table. This function 488 * fills out entire L1 descriptors at a time to save memory writes. 489 * 490 * Parameters 491 * gpi GPI to set this range to 492 * l1 Pointer to L1 table to fill out 493 * first Address of first granule in range. 494 * last Address of last granule in range (inclusive). 495 */ 496 static void gpt_fill_l1_tbl(uint64_t gpi, uint64_t *l1, uintptr_t first, 497 uintptr_t last) 498 { 499 uint64_t gpi_field = GPT_BUILD_L1_DESC(gpi); 500 uint64_t gpi_mask = 0xFFFFFFFFFFFFFFFF; 501 502 assert(first <= last); 503 assert((first & (GPT_PGS_ACTUAL_SIZE(gpt_config.p) - 1)) == 0U); 504 assert((last & (GPT_PGS_ACTUAL_SIZE(gpt_config.p) - 1)) == 0U); 505 assert(GPT_L0_IDX(first) == GPT_L0_IDX(last)); 506 assert(l1 != NULL); 507 508 /* Shift the mask if we're starting in the middle of an L1 entry. */ 509 gpi_mask = gpi_mask << (GPT_L1_GPI_IDX(gpt_config.p, first) << 2); 510 511 /* Fill out each L1 entry for this region. */ 512 for (unsigned int i = GPT_L1_IDX(gpt_config.p, first); 513 i <= GPT_L1_IDX(gpt_config.p, last); i++) { 514 /* Account for stopping in the middle of an L1 entry. */ 515 if (i == GPT_L1_IDX(gpt_config.p, last)) { 516 gpi_mask &= (gpi_mask >> ((15 - 517 GPT_L1_GPI_IDX(gpt_config.p, last)) << 2)); 518 } 519 520 /* Write GPI values. */ 521 assert((l1[i] & gpi_mask) == 522 (GPT_BUILD_L1_DESC(GPT_GPI_ANY) & gpi_mask)); 523 l1[i] = (l1[i] & ~gpi_mask) | (gpi_mask & gpi_field); 524 525 /* Reset mask. */ 526 gpi_mask = 0xFFFFFFFFFFFFFFFF; 527 } 528 } 529 530 /* 531 * This function finds the next available unused L1 table and initializes all 532 * granules descriptor entries to GPI_ANY. This ensures that there are no chunks 533 * of GPI_NO_ACCESS (0b0000) memory floating around in the system in the 534 * event that a PAS region stops midway through an L1 table, thus guaranteeing 535 * that all memory not explicitly assigned is GPI_ANY. This function does not 536 * check for overflow conditions, that should be done by the caller. 537 * 538 * Return 539 * Pointer to the next available L1 table. 540 */ 541 static uint64_t *gpt_get_new_l1_tbl(void) 542 { 543 /* Retrieve the next L1 table. */ 544 uint64_t *l1 = (uint64_t *)((uint64_t)(gpt_l1_tbl) + 545 (GPT_L1_TABLE_SIZE(gpt_config.p) * 546 gpt_next_l1_tbl_idx)); 547 548 /* Increment L1 counter. */ 549 gpt_next_l1_tbl_idx++; 550 551 /* Initialize all GPIs to GPT_GPI_ANY */ 552 for (unsigned int i = 0U; i < GPT_L1_ENTRY_COUNT(gpt_config.p); i++) { 553 l1[i] = GPT_BUILD_L1_DESC(GPT_GPI_ANY); 554 } 555 556 return l1; 557 } 558 559 /* 560 * When L1 tables are needed, this function creates the necessary L0 table 561 * descriptors and fills out the L1 table entries according to the supplied 562 * PAS range. 563 * 564 * Parameters 565 * *pas Pointer to the structure defining the PAS region. 566 */ 567 static void gpt_generate_l0_tbl_desc(pas_region_t *pas) 568 { 569 uintptr_t end_pa; 570 uintptr_t cur_pa; 571 uintptr_t last_gran_pa; 572 uint64_t *l0_gpt_base; 573 uint64_t *l1_gpt_arr; 574 unsigned int l0_idx; 575 576 assert(gpt_config.plat_gpt_l0_base != 0U); 577 assert(pas != NULL); 578 579 /* 580 * Checking of PAS parameters has already been done in 581 * gpt_validate_pas_mappings so no need to check the same things again. 582 */ 583 584 end_pa = pas->base_pa + pas->size; 585 l0_gpt_base = (uint64_t *)gpt_config.plat_gpt_l0_base; 586 587 /* We start working from the granule at base PA */ 588 cur_pa = pas->base_pa; 589 590 /* Iterate over each L0 region in this memory range. */ 591 for (l0_idx = GPT_L0_IDX(pas->base_pa); 592 l0_idx <= GPT_L0_IDX(end_pa - 1U); 593 l0_idx++) { 594 595 /* 596 * See if the L0 entry is already a table descriptor or if we 597 * need to create one. 598 */ 599 if (GPT_L0_TYPE(l0_gpt_base[l0_idx]) == GPT_L0_TYPE_TBL_DESC) { 600 /* Get the L1 array from the L0 entry. */ 601 l1_gpt_arr = GPT_L0_TBLD_ADDR(l0_gpt_base[l0_idx]); 602 } else { 603 /* Get a new L1 table from the L1 memory space. */ 604 l1_gpt_arr = gpt_get_new_l1_tbl(); 605 606 /* Fill out the L0 descriptor and flush it. */ 607 l0_gpt_base[l0_idx] = GPT_L0_TBL_DESC(l1_gpt_arr); 608 } 609 610 VERBOSE("[GPT] L0 entry (TABLE) index %u [%p] ==> L1 Addr 0x%llx (0x%" PRIx64 ")\n", 611 l0_idx, &l0_gpt_base[l0_idx], 612 (unsigned long long)(l1_gpt_arr), 613 l0_gpt_base[l0_idx]); 614 615 /* 616 * Determine the PA of the last granule in this L0 descriptor. 617 */ 618 last_gran_pa = gpt_get_l1_end_pa(cur_pa, end_pa) - 619 GPT_PGS_ACTUAL_SIZE(gpt_config.p); 620 621 /* 622 * Fill up L1 GPT entries between these two addresses. This 623 * function needs the addresses of the first granule and last 624 * granule in the range. 625 */ 626 gpt_fill_l1_tbl(GPT_PAS_ATTR_GPI(pas->attrs), l1_gpt_arr, 627 cur_pa, last_gran_pa); 628 629 /* Advance cur_pa to first granule in next L0 region. */ 630 cur_pa = gpt_get_l1_end_pa(cur_pa, end_pa); 631 } 632 } 633 634 /* 635 * This function flushes a range of L0 descriptors used by a given PAS region 636 * array. There is a chance that some unmodified L0 descriptors would be flushed 637 * in the case that there are "holes" in an array of PAS regions but overall 638 * this should be faster than individually flushing each modified L0 descriptor 639 * as they are created. 640 * 641 * Parameters 642 * *pas Pointer to an array of PAS regions. 643 * pas_count Number of entries in the PAS array. 644 */ 645 static void flush_l0_for_pas_array(pas_region_t *pas, unsigned int pas_count) 646 { 647 unsigned int idx; 648 unsigned int start_idx; 649 unsigned int end_idx; 650 uint64_t *l0 = (uint64_t *)gpt_config.plat_gpt_l0_base; 651 652 assert(pas != NULL); 653 assert(pas_count > 0); 654 655 /* Initial start and end values. */ 656 start_idx = GPT_L0_IDX(pas[0].base_pa); 657 end_idx = GPT_L0_IDX(pas[0].base_pa + pas[0].size - 1); 658 659 /* Find lowest and highest L0 indices used in this PAS array. */ 660 for (idx = 1; idx < pas_count; idx++) { 661 if (GPT_L0_IDX(pas[idx].base_pa) < start_idx) { 662 start_idx = GPT_L0_IDX(pas[idx].base_pa); 663 } 664 if (GPT_L0_IDX(pas[idx].base_pa + pas[idx].size - 1) > end_idx) { 665 end_idx = GPT_L0_IDX(pas[idx].base_pa + pas[idx].size - 1); 666 } 667 } 668 669 /* 670 * Flush all covered L0 descriptors, add 1 because we need to include 671 * the end index value. 672 */ 673 flush_dcache_range((uintptr_t)&l0[start_idx], 674 ((end_idx + 1) - start_idx) * sizeof(uint64_t)); 675 } 676 677 /* 678 * Public API to enable granule protection checks once the tables have all been 679 * initialized. This function is called at first initialization and then again 680 * later during warm boots of CPU cores. 681 * 682 * Return 683 * Negative Linux error code in the event of a failure, 0 for success. 684 */ 685 int gpt_enable(void) 686 { 687 u_register_t gpccr_el3; 688 689 /* 690 * Granule tables must be initialised before enabling 691 * granule protection. 692 */ 693 if (gpt_config.plat_gpt_l0_base == 0U) { 694 ERROR("[GPT] Tables have not been initialized!\n"); 695 return -EPERM; 696 } 697 698 /* Invalidate any stale TLB entries */ 699 tlbipaallos(); 700 dsb(); 701 702 /* Write the base address of the L0 tables into GPTBR */ 703 write_gptbr_el3(((gpt_config.plat_gpt_l0_base >> GPTBR_BADDR_VAL_SHIFT) 704 >> GPTBR_BADDR_SHIFT) & GPTBR_BADDR_MASK); 705 706 /* GPCCR_EL3.PPS */ 707 gpccr_el3 = SET_GPCCR_PPS(gpt_config.pps); 708 709 /* GPCCR_EL3.PGS */ 710 gpccr_el3 |= SET_GPCCR_PGS(gpt_config.pgs); 711 712 /* 713 * Since EL3 maps the L1 region as Inner shareable, use the same 714 * shareability attribute for GPC as well so that 715 * GPC fetches are visible to PEs 716 */ 717 gpccr_el3 |= SET_GPCCR_SH(GPCCR_SH_IS); 718 719 /* Outer and Inner cacheability set to Normal memory, WB, RA, WA. */ 720 gpccr_el3 |= SET_GPCCR_ORGN(GPCCR_ORGN_WB_RA_WA); 721 gpccr_el3 |= SET_GPCCR_IRGN(GPCCR_IRGN_WB_RA_WA); 722 723 /* Enable GPT */ 724 gpccr_el3 |= GPCCR_GPC_BIT; 725 726 /* TODO: Configure GPCCR_EL3_GPCP for Fault control. */ 727 write_gpccr_el3(gpccr_el3); 728 isb(); 729 tlbipaallos(); 730 dsb(); 731 isb(); 732 733 return 0; 734 } 735 736 /* 737 * Public API to disable granule protection checks. 738 */ 739 void gpt_disable(void) 740 { 741 u_register_t gpccr_el3 = read_gpccr_el3(); 742 743 write_gpccr_el3(gpccr_el3 & ~GPCCR_GPC_BIT); 744 dsbsy(); 745 isb(); 746 } 747 748 /* 749 * Public API that initializes the entire protected space to GPT_GPI_ANY using 750 * the L0 tables (block descriptors). Ideally, this function is invoked prior 751 * to DDR discovery and initialization. The MMU must be initialized before 752 * calling this function. 753 * 754 * Parameters 755 * pps PPS value to use for table generation 756 * l0_mem_base Base address of L0 tables in memory. 757 * l0_mem_size Total size of memory available for L0 tables. 758 * 759 * Return 760 * Negative Linux error code in the event of a failure, 0 for success. 761 */ 762 int gpt_init_l0_tables(unsigned int pps, uintptr_t l0_mem_base, 763 size_t l0_mem_size) 764 { 765 int ret; 766 uint64_t gpt_desc; 767 768 /* Ensure that MMU and Data caches are enabled. */ 769 assert((read_sctlr_el3() & SCTLR_C_BIT) != 0U); 770 771 /* Validate other parameters. */ 772 ret = gpt_validate_l0_params(pps, l0_mem_base, l0_mem_size); 773 if (ret < 0) { 774 return ret; 775 } 776 777 /* Create the descriptor to initialize L0 entries with. */ 778 gpt_desc = GPT_L0_BLK_DESC(GPT_GPI_ANY); 779 780 /* Iterate through all L0 entries */ 781 for (unsigned int i = 0U; i < GPT_L0_REGION_COUNT(gpt_config.t); i++) { 782 ((uint64_t *)l0_mem_base)[i] = gpt_desc; 783 } 784 785 /* Flush updated L0 tables to memory. */ 786 flush_dcache_range((uintptr_t)l0_mem_base, 787 (size_t)GPT_L0_TABLE_SIZE(gpt_config.t)); 788 789 /* Stash the L0 base address once initial setup is complete. */ 790 gpt_config.plat_gpt_l0_base = l0_mem_base; 791 792 return 0; 793 } 794 795 /* 796 * Public API that carves out PAS regions from the L0 tables and builds any L1 797 * tables that are needed. This function ideally is run after DDR discovery and 798 * initialization. The L0 tables must have already been initialized to GPI_ANY 799 * when this function is called. 800 * 801 * This function can be called multiple times with different L1 memory ranges 802 * and PAS regions if it is desirable to place L1 tables in different locations 803 * in memory. (ex: you have multiple DDR banks and want to place the L1 tables 804 * in the DDR bank that they control) 805 * 806 * Parameters 807 * pgs PGS value to use for table generation. 808 * l1_mem_base Base address of memory used for L1 tables. 809 * l1_mem_size Total size of memory available for L1 tables. 810 * *pas_regions Pointer to PAS regions structure array. 811 * pas_count Total number of PAS regions. 812 * 813 * Return 814 * Negative Linux error code in the event of a failure, 0 for success. 815 */ 816 int gpt_init_pas_l1_tables(gpccr_pgs_e pgs, uintptr_t l1_mem_base, 817 size_t l1_mem_size, pas_region_t *pas_regions, 818 unsigned int pas_count) 819 { 820 int ret; 821 int l1_gpt_cnt; 822 823 /* Ensure that MMU and Data caches are enabled. */ 824 assert((read_sctlr_el3() & SCTLR_C_BIT) != 0U); 825 826 /* PGS is needed for gpt_validate_pas_mappings so check it now. */ 827 if (pgs > GPT_PGS_MAX) { 828 ERROR("[GPT] Invalid PGS: 0x%x\n", pgs); 829 return -EINVAL; 830 } 831 gpt_config.pgs = pgs; 832 gpt_config.p = gpt_p_lookup[pgs]; 833 834 /* Make sure L0 tables have been initialized. */ 835 if (gpt_config.plat_gpt_l0_base == 0U) { 836 ERROR("[GPT] L0 tables must be initialized first!\n"); 837 return -EPERM; 838 } 839 840 /* Check if L1 GPTs are required and how many. */ 841 l1_gpt_cnt = gpt_validate_pas_mappings(pas_regions, pas_count); 842 if (l1_gpt_cnt < 0) { 843 return l1_gpt_cnt; 844 } 845 846 VERBOSE("[GPT] %u L1 GPTs requested.\n", l1_gpt_cnt); 847 848 /* If L1 tables are needed then validate the L1 parameters. */ 849 if (l1_gpt_cnt > 0) { 850 ret = gpt_validate_l1_params(l1_mem_base, l1_mem_size, 851 l1_gpt_cnt); 852 if (ret < 0) { 853 return ret; 854 } 855 856 /* Set up parameters for L1 table generation. */ 857 gpt_l1_tbl = l1_mem_base; 858 gpt_next_l1_tbl_idx = 0U; 859 } 860 861 INFO("[GPT] Boot Configuration\n"); 862 INFO(" PPS/T: 0x%x/%u\n", gpt_config.pps, gpt_config.t); 863 INFO(" PGS/P: 0x%x/%u\n", gpt_config.pgs, gpt_config.p); 864 INFO(" L0GPTSZ/S: 0x%x/%u\n", GPT_L0GPTSZ, GPT_S_VAL); 865 INFO(" PAS count: 0x%x\n", pas_count); 866 INFO(" L0 base: 0x%lx\n", gpt_config.plat_gpt_l0_base); 867 868 /* Generate the tables in memory. */ 869 for (unsigned int idx = 0U; idx < pas_count; idx++) { 870 INFO("[GPT] PAS[%u]: base 0x%lx, size 0x%lx, GPI 0x%x, type 0x%x\n", 871 idx, pas_regions[idx].base_pa, pas_regions[idx].size, 872 GPT_PAS_ATTR_GPI(pas_regions[idx].attrs), 873 GPT_PAS_ATTR_MAP_TYPE(pas_regions[idx].attrs)); 874 875 /* Check if a block or table descriptor is required */ 876 if (GPT_PAS_ATTR_MAP_TYPE(pas_regions[idx].attrs) == 877 GPT_PAS_ATTR_MAP_TYPE_BLOCK) { 878 gpt_generate_l0_blk_desc(&pas_regions[idx]); 879 880 } else { 881 gpt_generate_l0_tbl_desc(&pas_regions[idx]); 882 } 883 } 884 885 /* Flush modified L0 tables. */ 886 flush_l0_for_pas_array(pas_regions, pas_count); 887 888 /* Flush L1 tables if needed. */ 889 if (l1_gpt_cnt > 0) { 890 flush_dcache_range(l1_mem_base, 891 GPT_L1_TABLE_SIZE(gpt_config.p) * 892 l1_gpt_cnt); 893 } 894 895 /* Make sure that all the entries are written to the memory. */ 896 dsbishst(); 897 tlbipaallos(); 898 dsb(); 899 isb(); 900 901 return 0; 902 } 903 904 /* 905 * Public API to initialize the runtime gpt_config structure based on the values 906 * present in the GPTBR_EL3 and GPCCR_EL3 registers. GPT initialization 907 * typically happens in a bootloader stage prior to setting up the EL3 runtime 908 * environment for the granule transition service so this function detects the 909 * initialization from a previous stage. Granule protection checks must be 910 * enabled already or this function will return an error. 911 * 912 * Return 913 * Negative Linux error code in the event of a failure, 0 for success. 914 */ 915 int gpt_runtime_init(void) 916 { 917 u_register_t reg; 918 919 /* Ensure that MMU and Data caches are enabled. */ 920 assert((read_sctlr_el3() & SCTLR_C_BIT) != 0U); 921 922 /* Ensure GPC are already enabled. */ 923 if ((read_gpccr_el3() & GPCCR_GPC_BIT) == 0U) { 924 ERROR("[GPT] Granule protection checks are not enabled!\n"); 925 return -EPERM; 926 } 927 928 /* 929 * Read the L0 table address from GPTBR, we don't need the L1 base 930 * address since those are included in the L0 tables as needed. 931 */ 932 reg = read_gptbr_el3(); 933 gpt_config.plat_gpt_l0_base = ((reg >> GPTBR_BADDR_SHIFT) & 934 GPTBR_BADDR_MASK) << 935 GPTBR_BADDR_VAL_SHIFT; 936 937 /* Read GPCCR to get PGS and PPS values. */ 938 reg = read_gpccr_el3(); 939 gpt_config.pps = (reg >> GPCCR_PPS_SHIFT) & GPCCR_PPS_MASK; 940 gpt_config.t = gpt_t_lookup[gpt_config.pps]; 941 gpt_config.pgs = (reg >> GPCCR_PGS_SHIFT) & GPCCR_PGS_MASK; 942 gpt_config.p = gpt_p_lookup[gpt_config.pgs]; 943 944 VERBOSE("[GPT] Runtime Configuration\n"); 945 VERBOSE(" PPS/T: 0x%x/%u\n", gpt_config.pps, gpt_config.t); 946 VERBOSE(" PGS/P: 0x%x/%u\n", gpt_config.pgs, gpt_config.p); 947 VERBOSE(" L0GPTSZ/S: 0x%x/%u\n", GPT_L0GPTSZ, GPT_S_VAL); 948 VERBOSE(" L0 base: 0x%lx\n", gpt_config.plat_gpt_l0_base); 949 950 return 0; 951 } 952 953 /* 954 * The L1 descriptors are protected by a spinlock to ensure that multiple 955 * CPUs do not attempt to change the descriptors at once. In the future it 956 * would be better to have separate spinlocks for each L1 descriptor. 957 */ 958 static spinlock_t gpt_lock; 959 960 /* 961 * Check if caller is allowed to transition a PAS. 962 * 963 * - Secure world caller can only request S <-> NS transitions on a 964 * granule that is already in either S or NS PAS. 965 * 966 * - Realm world caller can only request R <-> NS transitions on a 967 * granule that is already in either R or NS PAS. 968 * 969 * Parameters 970 * src_sec_state Security state of the caller. 971 * current_gpi Current GPI of the granule. 972 * target_gpi Requested new GPI for the granule. 973 * 974 * Return 975 * Negative Linux error code in the event of a failure, 0 for success. 976 */ 977 static int gpt_check_transition_gpi(unsigned int src_sec_state, 978 unsigned int current_gpi, 979 unsigned int target_gpi) 980 { 981 unsigned int check_gpi; 982 983 /* Cannot transition a granule to the state it is already in. */ 984 if (current_gpi == target_gpi) { 985 return -EINVAL; 986 } 987 988 /* Check security state, only secure and realm can transition. */ 989 if (src_sec_state == SMC_FROM_REALM) { 990 check_gpi = GPT_GPI_REALM; 991 } else if (src_sec_state == SMC_FROM_SECURE) { 992 check_gpi = GPT_GPI_SECURE; 993 } else { 994 return -EINVAL; 995 } 996 997 /* Make sure security state is allowed to make the transition. */ 998 if ((target_gpi != check_gpi) && (target_gpi != GPT_GPI_NS)) { 999 return -EINVAL; 1000 } 1001 if ((current_gpi != check_gpi) && (current_gpi != GPT_GPI_NS)) { 1002 return -EINVAL; 1003 } 1004 1005 return 0; 1006 } 1007 1008 /* 1009 * This function is the core of the granule transition service. When a granule 1010 * transition request occurs it is routed to this function where the request is 1011 * validated then fulfilled if possible. 1012 * 1013 * TODO: implement support for transitioning multiple granules at once. 1014 * 1015 * Parameters 1016 * base Base address of the region to transition, must be 1017 * aligned to granule size. 1018 * size Size of region to transition, must be aligned to granule 1019 * size. 1020 * src_sec_state Security state of the caller. 1021 * target_pas Target PAS of the specified memory region. 1022 * 1023 * Return 1024 * Negative Linux error code in the event of a failure, 0 for success. 1025 */ 1026 int gpt_transition_pas(uint64_t base, size_t size, unsigned int src_sec_state, 1027 unsigned int target_pas) 1028 { 1029 int idx; 1030 unsigned int gpi_shift; 1031 unsigned int gpi; 1032 uint64_t gpt_l0_desc; 1033 uint64_t gpt_l1_desc; 1034 uint64_t *gpt_l1_addr; 1035 uint64_t *gpt_l0_base; 1036 1037 /* Ensure that the tables have been set up before taking requests. */ 1038 assert(gpt_config.plat_gpt_l0_base != 0U); 1039 1040 /* Ensure that MMU and data caches are enabled. */ 1041 assert((read_sctlr_el3() & SCTLR_C_BIT) != 0U); 1042 1043 /* Check for address range overflow. */ 1044 if ((ULONG_MAX - base) < size) { 1045 VERBOSE("[GPT] Transition request address overflow!\n"); 1046 VERBOSE(" Base=0x%" PRIx64 "\n", base); 1047 VERBOSE(" Size=0x%lx\n", size); 1048 return -EINVAL; 1049 } 1050 1051 /* Make sure base and size are valid. */ 1052 if (((base & (GPT_PGS_ACTUAL_SIZE(gpt_config.p) - 1)) != 0U) || 1053 ((size & (GPT_PGS_ACTUAL_SIZE(gpt_config.p) - 1)) != 0U) || 1054 (size == 0U) || 1055 ((base + size) >= GPT_PPS_ACTUAL_SIZE(gpt_config.t))) { 1056 VERBOSE("[GPT] Invalid granule transition address range!\n"); 1057 VERBOSE(" Base=0x%" PRIx64 "\n", base); 1058 VERBOSE(" Size=0x%lx\n", size); 1059 return -EINVAL; 1060 } 1061 1062 /* See if this is a single granule transition or a range of granules. */ 1063 if (size != GPT_PGS_ACTUAL_SIZE(gpt_config.p)) { 1064 /* 1065 * TODO: Add support for transitioning multiple granules with a 1066 * single call to this function. 1067 */ 1068 panic(); 1069 } 1070 1071 /* Get the L0 descriptor and make sure it is for a table. */ 1072 gpt_l0_base = (uint64_t *)gpt_config.plat_gpt_l0_base; 1073 gpt_l0_desc = gpt_l0_base[GPT_L0_IDX(base)]; 1074 if (GPT_L0_TYPE(gpt_l0_desc) != GPT_L0_TYPE_TBL_DESC) { 1075 VERBOSE("[GPT] Granule is not covered by a table descriptor!\n"); 1076 VERBOSE(" Base=0x%" PRIx64 "\n", base); 1077 return -EINVAL; 1078 } 1079 1080 /* Get the table index and GPI shift from PA. */ 1081 gpt_l1_addr = GPT_L0_TBLD_ADDR(gpt_l0_desc); 1082 idx = GPT_L1_IDX(gpt_config.p, base); 1083 gpi_shift = GPT_L1_GPI_IDX(gpt_config.p, base) << 2; 1084 1085 /* 1086 * Access to L1 tables is controlled by a global lock to ensure 1087 * that no more than one CPU is allowed to make changes at any 1088 * given time. 1089 */ 1090 spin_lock(&gpt_lock); 1091 gpt_l1_desc = gpt_l1_addr[idx]; 1092 gpi = (gpt_l1_desc >> gpi_shift) & GPT_L1_GRAN_DESC_GPI_MASK; 1093 1094 /* Make sure caller state and source/target PAS are allowed. */ 1095 if (gpt_check_transition_gpi(src_sec_state, gpi, target_pas) < 0) { 1096 spin_unlock(&gpt_lock); 1097 VERBOSE("[GPT] Invalid caller state and PAS combo!\n"); 1098 VERBOSE(" Caller: %u, Current GPI: %u, Target GPI: %u\n", 1099 src_sec_state, gpi, target_pas); 1100 return -EPERM; 1101 } 1102 1103 /* Clear existing GPI encoding and transition granule. */ 1104 gpt_l1_desc &= ~(GPT_L1_GRAN_DESC_GPI_MASK << gpi_shift); 1105 gpt_l1_desc |= ((uint64_t)target_pas << gpi_shift); 1106 gpt_l1_addr[idx] = gpt_l1_desc; 1107 1108 /* Ensure that the write operation will be observed by GPC */ 1109 dsbishst(); 1110 1111 /* Unlock access to the L1 tables. */ 1112 spin_unlock(&gpt_lock); 1113 1114 gpt_tlbi_by_pa(base, GPT_PGS_ACTUAL_SIZE(gpt_config.p)); 1115 dsbishst(); 1116 /* 1117 * The isb() will be done as part of context 1118 * synchronization when returning to lower EL 1119 */ 1120 VERBOSE("[GPT] Granule 0x%" PRIx64 ", GPI 0x%x->0x%x\n", base, gpi, 1121 target_pas); 1122 1123 return 0; 1124 } 1125