1 // SPDX-License-Identifier: BSD-2-Clause 2 /* 3 * Copyright (c) 2019, Linaro Limited 4 * Copyright (c) 2020, Arm Limited 5 */ 6 7 #include <assert.h> 8 #include <ctype.h> 9 #include <elf32.h> 10 #include <elf64.h> 11 #include <elf_common.h> 12 #include <ldelf.h> 13 #include <link.h> 14 #include <stdio.h> 15 #include <stdlib.h> 16 #include <string_ext.h> 17 #include <string.h> 18 #include <tee_api_types.h> 19 #include <tee_internal_api_extensions.h> 20 #include <unw/unwind.h> 21 #include <user_ta_header.h> 22 #include <util.h> 23 24 #include "sys.h" 25 #include "ta_elf.h" 26 27 /* 28 * Layout of a 32-bit struct dl_phdr_info for a 64-bit ldelf to access a 32-bit 29 * TA 30 */ 31 struct dl_phdr_info32 { 32 uint32_t dlpi_addr; 33 uint32_t dlpi_name; 34 uint32_t dlpi_phdr; 35 uint16_t dlpi_phnum; 36 uint64_t dlpi_adds; 37 uint64_t dlpi_subs; 38 uint32_t dlpi_tls_modid; 39 uint32_t dlpi_tls_data; 40 }; 41 42 static vaddr_t ta_stack; 43 static vaddr_t ta_stack_size; 44 45 struct ta_elf_queue main_elf_queue = TAILQ_HEAD_INITIALIZER(main_elf_queue); 46 47 /* 48 * Main application is always ID 1, shared libraries with TLS take IDs 2 and 49 * above 50 */ 51 static void assign_tls_mod_id(struct ta_elf *elf) 52 { 53 static size_t last_tls_mod_id = 1; 54 55 if (elf->is_main) 56 assert(last_tls_mod_id == 1); /* Main always comes first */ 57 elf->tls_mod_id = last_tls_mod_id++; 58 } 59 60 static struct ta_elf *queue_elf_helper(const TEE_UUID *uuid) 61 { 62 struct ta_elf *elf = calloc(1, sizeof(*elf)); 63 64 if (!elf) 65 return NULL; 66 67 TAILQ_INIT(&elf->segs); 68 69 elf->uuid = *uuid; 70 TAILQ_INSERT_TAIL(&main_elf_queue, elf, link); 71 return elf; 72 } 73 74 static struct ta_elf *queue_elf(const TEE_UUID *uuid) 75 { 76 struct ta_elf *elf = ta_elf_find_elf(uuid); 77 78 if (elf) 79 return NULL; 80 81 elf = queue_elf_helper(uuid); 82 if (!elf) 83 err(TEE_ERROR_OUT_OF_MEMORY, "queue_elf_helper"); 84 85 return elf; 86 } 87 88 struct ta_elf *ta_elf_find_elf(const TEE_UUID *uuid) 89 { 90 struct ta_elf *elf = NULL; 91 92 TAILQ_FOREACH(elf, &main_elf_queue, link) 93 if (!memcmp(uuid, &elf->uuid, sizeof(*uuid))) 94 return elf; 95 96 return NULL; 97 } 98 99 static TEE_Result e32_parse_ehdr(struct ta_elf *elf, Elf32_Ehdr *ehdr) 100 { 101 if (ehdr->e_ident[EI_VERSION] != EV_CURRENT || 102 ehdr->e_ident[EI_CLASS] != ELFCLASS32 || 103 ehdr->e_ident[EI_DATA] != ELFDATA2LSB || 104 ehdr->e_ident[EI_OSABI] != ELFOSABI_NONE || 105 ehdr->e_type != ET_DYN || ehdr->e_machine != EM_ARM || 106 (ehdr->e_flags & EF_ARM_ABIMASK) != EF_ARM_ABI_VERSION || 107 #ifndef CFG_WITH_VFP 108 (ehdr->e_flags & EF_ARM_ABI_FLOAT_HARD) || 109 #endif 110 ehdr->e_phentsize != sizeof(Elf32_Phdr) || 111 ehdr->e_shentsize != sizeof(Elf32_Shdr)) 112 return TEE_ERROR_BAD_FORMAT; 113 114 elf->is_32bit = true; 115 elf->e_entry = ehdr->e_entry; 116 elf->e_phoff = ehdr->e_phoff; 117 elf->e_shoff = ehdr->e_shoff; 118 elf->e_phnum = ehdr->e_phnum; 119 elf->e_shnum = ehdr->e_shnum; 120 elf->e_phentsize = ehdr->e_phentsize; 121 elf->e_shentsize = ehdr->e_shentsize; 122 123 return TEE_SUCCESS; 124 } 125 126 #ifdef ARM64 127 static TEE_Result e64_parse_ehdr(struct ta_elf *elf, Elf64_Ehdr *ehdr) 128 { 129 if (ehdr->e_ident[EI_VERSION] != EV_CURRENT || 130 ehdr->e_ident[EI_CLASS] != ELFCLASS64 || 131 ehdr->e_ident[EI_DATA] != ELFDATA2LSB || 132 ehdr->e_ident[EI_OSABI] != ELFOSABI_NONE || 133 ehdr->e_type != ET_DYN || ehdr->e_machine != EM_AARCH64 || 134 ehdr->e_flags || ehdr->e_phentsize != sizeof(Elf64_Phdr) || 135 ehdr->e_shentsize != sizeof(Elf64_Shdr)) 136 return TEE_ERROR_BAD_FORMAT; 137 138 139 elf->is_32bit = false; 140 elf->e_entry = ehdr->e_entry; 141 elf->e_phoff = ehdr->e_phoff; 142 elf->e_shoff = ehdr->e_shoff; 143 elf->e_phnum = ehdr->e_phnum; 144 elf->e_shnum = ehdr->e_shnum; 145 elf->e_phentsize = ehdr->e_phentsize; 146 elf->e_shentsize = ehdr->e_shentsize; 147 148 return TEE_SUCCESS; 149 } 150 #else /*ARM64*/ 151 static TEE_Result e64_parse_ehdr(struct ta_elf *elf __unused, 152 Elf64_Ehdr *ehdr __unused) 153 { 154 return TEE_ERROR_NOT_SUPPORTED; 155 } 156 #endif /*ARM64*/ 157 158 static void check_phdr_in_range(struct ta_elf *elf, unsigned int type, 159 vaddr_t addr, size_t memsz) 160 { 161 vaddr_t max_addr = 0; 162 163 if (ADD_OVERFLOW(addr, memsz, &max_addr)) 164 err(TEE_ERROR_BAD_FORMAT, "Program header %#x overflow", type); 165 166 /* 167 * elf->load_addr and elf->max_addr are both using the 168 * final virtual addresses, while this program header is 169 * relative to 0. 170 */ 171 if (max_addr > elf->max_addr - elf->load_addr) 172 err(TEE_ERROR_BAD_FORMAT, "Program header %#x out of bounds", 173 type); 174 } 175 176 static void read_dyn(struct ta_elf *elf, vaddr_t addr, 177 size_t idx, unsigned int *tag, size_t *val) 178 { 179 if (elf->is_32bit) { 180 Elf32_Dyn *dyn = (Elf32_Dyn *)(addr + elf->load_addr); 181 182 *tag = dyn[idx].d_tag; 183 *val = dyn[idx].d_un.d_val; 184 } else { 185 Elf64_Dyn *dyn = (Elf64_Dyn *)(addr + elf->load_addr); 186 187 *tag = dyn[idx].d_tag; 188 *val = dyn[idx].d_un.d_val; 189 } 190 } 191 192 static void save_hashtab_from_segment(struct ta_elf *elf, unsigned int type, 193 vaddr_t addr, size_t memsz) 194 { 195 size_t dyn_entsize = 0; 196 size_t num_dyns = 0; 197 size_t n = 0; 198 unsigned int tag = 0; 199 size_t val = 0; 200 201 if (type != PT_DYNAMIC) 202 return; 203 204 check_phdr_in_range(elf, type, addr, memsz); 205 206 if (elf->is_32bit) 207 dyn_entsize = sizeof(Elf32_Dyn); 208 else 209 dyn_entsize = sizeof(Elf64_Dyn); 210 211 assert(!(memsz % dyn_entsize)); 212 num_dyns = memsz / dyn_entsize; 213 214 for (n = 0; n < num_dyns; n++) { 215 read_dyn(elf, addr, n, &tag, &val); 216 if (tag == DT_HASH) { 217 elf->hashtab = (void *)(val + elf->load_addr); 218 break; 219 } 220 } 221 } 222 223 static void check_range(struct ta_elf *elf, const char *name, const void *ptr, 224 size_t sz) 225 { 226 size_t max_addr = 0; 227 228 if ((vaddr_t)ptr < elf->load_addr) 229 err(TEE_ERROR_BAD_FORMAT, "%s %p out of range", name, ptr); 230 231 if (ADD_OVERFLOW((vaddr_t)ptr, sz, &max_addr)) 232 err(TEE_ERROR_BAD_FORMAT, "%s range overflow", name); 233 234 if (max_addr > elf->max_addr) 235 err(TEE_ERROR_BAD_FORMAT, 236 "%s %p..%#zx out of range", name, ptr, max_addr); 237 } 238 239 static void check_hashtab(struct ta_elf *elf, void *ptr, size_t num_buckets, 240 size_t num_chains) 241 { 242 /* 243 * Starting from 2 as the first two words are mandatory and hold 244 * num_buckets and num_chains. So this function is called twice, 245 * first to see that there's indeed room for num_buckets and 246 * num_chains and then to see that all of it fits. 247 * See http://www.sco.com/developers/gabi/latest/ch5.dynamic.html#hash 248 */ 249 size_t num_words = 2; 250 size_t sz = 0; 251 252 if (!ALIGNMENT_IS_OK(ptr, uint32_t)) 253 err(TEE_ERROR_BAD_FORMAT, "Bad alignment of DT_HASH %p", ptr); 254 255 if (ADD_OVERFLOW(num_words, num_buckets, &num_words) || 256 ADD_OVERFLOW(num_words, num_chains, &num_words) || 257 MUL_OVERFLOW(num_words, sizeof(uint32_t), &sz)) 258 err(TEE_ERROR_BAD_FORMAT, "DT_HASH overflow"); 259 260 check_range(elf, "DT_HASH", ptr, sz); 261 } 262 263 static void save_hashtab(struct ta_elf *elf) 264 { 265 uint32_t *hashtab = NULL; 266 size_t n = 0; 267 268 if (elf->is_32bit) { 269 Elf32_Phdr *phdr = elf->phdr; 270 271 for (n = 0; n < elf->e_phnum; n++) 272 save_hashtab_from_segment(elf, phdr[n].p_type, 273 phdr[n].p_vaddr, 274 phdr[n].p_memsz); 275 } else { 276 Elf64_Phdr *phdr = elf->phdr; 277 278 for (n = 0; n < elf->e_phnum; n++) 279 save_hashtab_from_segment(elf, phdr[n].p_type, 280 phdr[n].p_vaddr, 281 phdr[n].p_memsz); 282 } 283 284 check_hashtab(elf, elf->hashtab, 0, 0); 285 hashtab = elf->hashtab; 286 check_hashtab(elf, elf->hashtab, hashtab[0], hashtab[1]); 287 } 288 289 static void save_soname_from_segment(struct ta_elf *elf, unsigned int type, 290 vaddr_t addr, size_t memsz) 291 { 292 size_t dyn_entsize = 0; 293 size_t num_dyns = 0; 294 size_t n = 0; 295 unsigned int tag = 0; 296 size_t val = 0; 297 char *str_tab = NULL; 298 299 if (type != PT_DYNAMIC) 300 return; 301 302 if (elf->is_32bit) 303 dyn_entsize = sizeof(Elf32_Dyn); 304 else 305 dyn_entsize = sizeof(Elf64_Dyn); 306 307 assert(!(memsz % dyn_entsize)); 308 num_dyns = memsz / dyn_entsize; 309 310 for (n = 0; n < num_dyns; n++) { 311 read_dyn(elf, addr, n, &tag, &val); 312 if (tag == DT_STRTAB) { 313 str_tab = (char *)(val + elf->load_addr); 314 break; 315 } 316 } 317 for (n = 0; n < num_dyns; n++) { 318 read_dyn(elf, addr, n, &tag, &val); 319 if (tag == DT_SONAME) { 320 elf->soname = str_tab + val; 321 break; 322 } 323 } 324 } 325 326 static void save_soname(struct ta_elf *elf) 327 { 328 size_t n = 0; 329 330 if (elf->is_32bit) { 331 Elf32_Phdr *phdr = elf->phdr; 332 333 for (n = 0; n < elf->e_phnum; n++) 334 save_soname_from_segment(elf, phdr[n].p_type, 335 phdr[n].p_vaddr, 336 phdr[n].p_memsz); 337 } else { 338 Elf64_Phdr *phdr = elf->phdr; 339 340 for (n = 0; n < elf->e_phnum; n++) 341 save_soname_from_segment(elf, phdr[n].p_type, 342 phdr[n].p_vaddr, 343 phdr[n].p_memsz); 344 } 345 } 346 347 static void e32_save_symtab(struct ta_elf *elf, size_t tab_idx) 348 { 349 Elf32_Shdr *shdr = elf->shdr; 350 size_t str_idx = shdr[tab_idx].sh_link; 351 352 elf->dynsymtab = (void *)(shdr[tab_idx].sh_addr + elf->load_addr); 353 if (!ALIGNMENT_IS_OK(elf->dynsymtab, Elf32_Sym)) 354 err(TEE_ERROR_BAD_FORMAT, "Bad alignment of dynsymtab %p", 355 elf->dynsymtab); 356 check_range(elf, "Dynsymtab", elf->dynsymtab, shdr[tab_idx].sh_size); 357 358 if (shdr[tab_idx].sh_size % sizeof(Elf32_Sym)) 359 err(TEE_ERROR_BAD_FORMAT, 360 "Size of dynsymtab not an even multiple of Elf32_Sym"); 361 elf->num_dynsyms = shdr[tab_idx].sh_size / sizeof(Elf32_Sym); 362 363 if (str_idx >= elf->e_shnum) 364 err(TEE_ERROR_BAD_FORMAT, "Dynstr section index out of range"); 365 elf->dynstr = (void *)(shdr[str_idx].sh_addr + elf->load_addr); 366 check_range(elf, "Dynstr", elf->dynstr, shdr[str_idx].sh_size); 367 368 elf->dynstr_size = shdr[str_idx].sh_size; 369 } 370 371 static void e64_save_symtab(struct ta_elf *elf, size_t tab_idx) 372 { 373 Elf64_Shdr *shdr = elf->shdr; 374 size_t str_idx = shdr[tab_idx].sh_link; 375 376 elf->dynsymtab = (void *)(vaddr_t)(shdr[tab_idx].sh_addr + 377 elf->load_addr); 378 379 if (!ALIGNMENT_IS_OK(elf->dynsymtab, Elf64_Sym)) 380 err(TEE_ERROR_BAD_FORMAT, "Bad alignment of .dynsym/DYNSYM %p", 381 elf->dynsymtab); 382 check_range(elf, ".dynsym/DYNSYM", elf->dynsymtab, 383 shdr[tab_idx].sh_size); 384 385 if (shdr[tab_idx].sh_size % sizeof(Elf64_Sym)) 386 err(TEE_ERROR_BAD_FORMAT, 387 "Size of .dynsym/DYNSYM not an even multiple of Elf64_Sym"); 388 elf->num_dynsyms = shdr[tab_idx].sh_size / sizeof(Elf64_Sym); 389 390 if (str_idx >= elf->e_shnum) 391 err(TEE_ERROR_BAD_FORMAT, 392 ".dynstr/STRTAB section index out of range"); 393 elf->dynstr = (void *)(vaddr_t)(shdr[str_idx].sh_addr + elf->load_addr); 394 check_range(elf, ".dynstr/STRTAB", elf->dynstr, shdr[str_idx].sh_size); 395 396 elf->dynstr_size = shdr[str_idx].sh_size; 397 } 398 399 static void save_symtab(struct ta_elf *elf) 400 { 401 size_t n = 0; 402 403 if (elf->is_32bit) { 404 Elf32_Shdr *shdr = elf->shdr; 405 406 for (n = 0; n < elf->e_shnum; n++) { 407 if (shdr[n].sh_type == SHT_DYNSYM) { 408 e32_save_symtab(elf, n); 409 break; 410 } 411 } 412 } else { 413 Elf64_Shdr *shdr = elf->shdr; 414 415 for (n = 0; n < elf->e_shnum; n++) { 416 if (shdr[n].sh_type == SHT_DYNSYM) { 417 e64_save_symtab(elf, n); 418 break; 419 } 420 } 421 422 } 423 424 save_hashtab(elf); 425 save_soname(elf); 426 } 427 428 static void init_elf(struct ta_elf *elf) 429 { 430 TEE_Result res = TEE_SUCCESS; 431 vaddr_t va = 0; 432 uint32_t flags = LDELF_MAP_FLAG_SHAREABLE; 433 size_t sz = 0; 434 435 res = sys_open_ta_bin(&elf->uuid, &elf->handle); 436 if (res) 437 err(res, "sys_open_ta_bin(%pUl)", (void *)&elf->uuid); 438 439 /* 440 * Map it read-only executable when we're loading a library where 441 * the ELF header is included in a load segment. 442 */ 443 if (!elf->is_main) 444 flags |= LDELF_MAP_FLAG_EXECUTABLE; 445 res = sys_map_ta_bin(&va, SMALL_PAGE_SIZE, flags, elf->handle, 0, 0, 0); 446 if (res) 447 err(res, "sys_map_ta_bin"); 448 elf->ehdr_addr = va; 449 if (!elf->is_main) { 450 elf->load_addr = va; 451 elf->max_addr = va + SMALL_PAGE_SIZE; 452 elf->max_offs = SMALL_PAGE_SIZE; 453 } 454 455 if (!IS_ELF(*(Elf32_Ehdr *)va)) 456 err(TEE_ERROR_BAD_FORMAT, "TA is not an ELF"); 457 458 res = e32_parse_ehdr(elf, (void *)va); 459 if (res == TEE_ERROR_BAD_FORMAT) 460 res = e64_parse_ehdr(elf, (void *)va); 461 if (res) 462 err(res, "Cannot parse ELF"); 463 464 if (MUL_OVERFLOW(elf->e_phnum, elf->e_phentsize, &sz) || 465 ADD_OVERFLOW(sz, elf->e_phoff, &sz)) 466 err(TEE_ERROR_BAD_FORMAT, "Program headers size overflow"); 467 468 if (sz > SMALL_PAGE_SIZE) 469 err(TEE_ERROR_NOT_SUPPORTED, "Cannot read program headers"); 470 471 elf->phdr = (void *)(va + elf->e_phoff); 472 } 473 474 static size_t roundup(size_t v) 475 { 476 return ROUNDUP(v, SMALL_PAGE_SIZE); 477 } 478 479 static size_t rounddown(size_t v) 480 { 481 return ROUNDDOWN(v, SMALL_PAGE_SIZE); 482 } 483 484 static void add_segment(struct ta_elf *elf, size_t offset, size_t vaddr, 485 size_t filesz, size_t memsz, size_t flags, size_t align) 486 { 487 struct segment *seg = calloc(1, sizeof(*seg)); 488 489 if (!seg) 490 err(TEE_ERROR_OUT_OF_MEMORY, "calloc"); 491 492 if (memsz < filesz) 493 err(TEE_ERROR_BAD_FORMAT, "Memsz smaller than filesz"); 494 495 seg->offset = offset; 496 seg->vaddr = vaddr; 497 seg->filesz = filesz; 498 seg->memsz = memsz; 499 seg->flags = flags; 500 seg->align = align; 501 502 TAILQ_INSERT_TAIL(&elf->segs, seg, link); 503 } 504 505 static void parse_load_segments(struct ta_elf *elf) 506 { 507 size_t n = 0; 508 509 if (elf->is_32bit) { 510 Elf32_Phdr *phdr = elf->phdr; 511 512 for (n = 0; n < elf->e_phnum; n++) 513 if (phdr[n].p_type == PT_LOAD) { 514 add_segment(elf, phdr[n].p_offset, 515 phdr[n].p_vaddr, phdr[n].p_filesz, 516 phdr[n].p_memsz, phdr[n].p_flags, 517 phdr[n].p_align); 518 } else if (phdr[n].p_type == PT_ARM_EXIDX) { 519 elf->exidx_start = phdr[n].p_vaddr; 520 elf->exidx_size = phdr[n].p_filesz; 521 } else if (phdr[n].p_type == PT_TLS) { 522 assign_tls_mod_id(elf); 523 } 524 } else { 525 Elf64_Phdr *phdr = elf->phdr; 526 527 for (n = 0; n < elf->e_phnum; n++) 528 if (phdr[n].p_type == PT_LOAD) { 529 add_segment(elf, phdr[n].p_offset, 530 phdr[n].p_vaddr, phdr[n].p_filesz, 531 phdr[n].p_memsz, phdr[n].p_flags, 532 phdr[n].p_align); 533 } else if (phdr[n].p_type == PT_TLS) { 534 elf->tls_start = phdr[n].p_vaddr; 535 elf->tls_filesz = phdr[n].p_filesz; 536 elf->tls_memsz = phdr[n].p_memsz; 537 } 538 } 539 } 540 541 static void copy_remapped_to(struct ta_elf *elf, const struct segment *seg) 542 { 543 uint8_t *dst = (void *)(seg->vaddr + elf->load_addr); 544 size_t n = 0; 545 size_t offs = seg->offset; 546 size_t num_bytes = seg->filesz; 547 548 if (offs < elf->max_offs) { 549 n = MIN(elf->max_offs - offs, num_bytes); 550 memcpy(dst, (void *)(elf->max_addr + offs - elf->max_offs), n); 551 dst += n; 552 offs += n; 553 num_bytes -= n; 554 } 555 556 if (num_bytes) { 557 TEE_Result res = sys_copy_from_ta_bin(dst, num_bytes, 558 elf->handle, offs); 559 560 if (res) 561 err(res, "sys_copy_from_ta_bin"); 562 elf->max_offs += offs; 563 } 564 } 565 566 static void adjust_segments(struct ta_elf *elf) 567 { 568 struct segment *seg = NULL; 569 struct segment *prev_seg = NULL; 570 size_t prev_end_addr = 0; 571 size_t align = 0; 572 size_t mask = 0; 573 574 /* Sanity check */ 575 TAILQ_FOREACH(seg, &elf->segs, link) { 576 size_t dummy __maybe_unused = 0; 577 578 assert(seg->align >= SMALL_PAGE_SIZE); 579 assert(!ADD_OVERFLOW(seg->vaddr, seg->memsz, &dummy)); 580 assert(seg->filesz <= seg->memsz); 581 assert((seg->offset & SMALL_PAGE_MASK) == 582 (seg->vaddr & SMALL_PAGE_MASK)); 583 584 prev_seg = TAILQ_PREV(seg, segment_head, link); 585 if (prev_seg) { 586 assert(seg->vaddr >= prev_seg->vaddr + prev_seg->memsz); 587 assert(seg->offset >= 588 prev_seg->offset + prev_seg->filesz); 589 } 590 if (!align) 591 align = seg->align; 592 assert(align == seg->align); 593 } 594 595 mask = align - 1; 596 597 seg = TAILQ_FIRST(&elf->segs); 598 if (seg) 599 seg = TAILQ_NEXT(seg, link); 600 while (seg) { 601 prev_seg = TAILQ_PREV(seg, segment_head, link); 602 prev_end_addr = prev_seg->vaddr + prev_seg->memsz; 603 604 /* 605 * This segment may overlap with the last "page" in the 606 * previous segment in two different ways: 607 * 1. Virtual address (and offset) overlaps => 608 * Permissions needs to be merged. The offset must have 609 * the SMALL_PAGE_MASK bits set as vaddr and offset must 610 * add up with prevsion segment. 611 * 612 * 2. Only offset overlaps => 613 * The same page in the ELF is mapped at two different 614 * virtual addresses. As a limitation this segment must 615 * be mapped as writeable. 616 */ 617 618 /* Case 1. */ 619 if (rounddown(seg->vaddr) < prev_end_addr) { 620 assert((seg->vaddr & mask) == (seg->offset & mask)); 621 assert(prev_seg->memsz == prev_seg->filesz); 622 623 /* 624 * Merge the segments and their permissions. 625 * Note that the may be a small hole between the 626 * two sections. 627 */ 628 prev_seg->filesz = seg->vaddr + seg->filesz - 629 prev_seg->vaddr; 630 prev_seg->memsz = seg->vaddr + seg->memsz - 631 prev_seg->vaddr; 632 prev_seg->flags |= seg->flags; 633 634 TAILQ_REMOVE(&elf->segs, seg, link); 635 free(seg); 636 seg = TAILQ_NEXT(prev_seg, link); 637 continue; 638 } 639 640 /* Case 2. */ 641 if ((seg->offset & mask) && 642 rounddown(seg->offset) < 643 (prev_seg->offset + prev_seg->filesz)) { 644 645 assert(seg->flags & PF_W); 646 seg->remapped_writeable = true; 647 } 648 649 /* 650 * No overlap, but we may need to align address, offset and 651 * size. 652 */ 653 seg->filesz += seg->vaddr - rounddown(seg->vaddr); 654 seg->memsz += seg->vaddr - rounddown(seg->vaddr); 655 seg->vaddr = rounddown(seg->vaddr); 656 seg->offset = rounddown(seg->offset); 657 seg = TAILQ_NEXT(seg, link); 658 } 659 660 } 661 662 static void populate_segments_legacy(struct ta_elf *elf) 663 { 664 TEE_Result res = TEE_SUCCESS; 665 struct segment *seg = NULL; 666 vaddr_t va = 0; 667 668 assert(elf->is_legacy); 669 TAILQ_FOREACH(seg, &elf->segs, link) { 670 struct segment *last_seg = TAILQ_LAST(&elf->segs, segment_head); 671 size_t pad_end = roundup(last_seg->vaddr + last_seg->memsz - 672 seg->vaddr - seg->memsz); 673 size_t num_bytes = roundup(seg->memsz); 674 675 if (!elf->load_addr) 676 va = 0; 677 else 678 va = seg->vaddr + elf->load_addr; 679 680 681 if (!(seg->flags & PF_R)) 682 err(TEE_ERROR_NOT_SUPPORTED, 683 "Segment must be readable"); 684 685 res = sys_map_zi(num_bytes, 0, &va, 0, pad_end); 686 if (res) 687 err(res, "sys_map_zi"); 688 res = sys_copy_from_ta_bin((void *)va, seg->filesz, 689 elf->handle, seg->offset); 690 if (res) 691 err(res, "sys_copy_from_ta_bin"); 692 693 if (!elf->load_addr) 694 elf->load_addr = va; 695 elf->max_addr = va + num_bytes; 696 elf->max_offs = seg->offset + seg->filesz; 697 } 698 } 699 700 static size_t get_pad_begin(void) 701 { 702 #ifdef CFG_TA_ASLR 703 size_t min = CFG_TA_ASLR_MIN_OFFSET_PAGES; 704 size_t max = CFG_TA_ASLR_MAX_OFFSET_PAGES; 705 TEE_Result res = TEE_SUCCESS; 706 uint32_t rnd32 = 0; 707 size_t rnd = 0; 708 709 COMPILE_TIME_ASSERT(CFG_TA_ASLR_MIN_OFFSET_PAGES < 710 CFG_TA_ASLR_MAX_OFFSET_PAGES); 711 if (max > min) { 712 res = sys_gen_random_num(&rnd32, sizeof(rnd32)); 713 if (res) { 714 DMSG("Random read failed: %#"PRIx32, res); 715 return min * SMALL_PAGE_SIZE; 716 } 717 rnd = rnd32 % (max - min); 718 } 719 720 return (min + rnd) * SMALL_PAGE_SIZE; 721 #else /*!CFG_TA_ASLR*/ 722 return 0; 723 #endif /*!CFG_TA_ASLR*/ 724 } 725 726 static void populate_segments(struct ta_elf *elf) 727 { 728 TEE_Result res = TEE_SUCCESS; 729 struct segment *seg = NULL; 730 vaddr_t va = 0; 731 size_t pad_begin = 0; 732 733 assert(!elf->is_legacy); 734 TAILQ_FOREACH(seg, &elf->segs, link) { 735 struct segment *last_seg = TAILQ_LAST(&elf->segs, segment_head); 736 size_t pad_end = roundup(last_seg->vaddr + last_seg->memsz - 737 seg->vaddr - seg->memsz); 738 739 if (seg->remapped_writeable) { 740 size_t num_bytes = roundup(seg->vaddr + seg->memsz) - 741 rounddown(seg->vaddr); 742 743 assert(elf->load_addr); 744 va = rounddown(elf->load_addr + seg->vaddr); 745 assert(va >= elf->max_addr); 746 res = sys_map_zi(num_bytes, 0, &va, 0, pad_end); 747 if (res) 748 err(res, "sys_map_zi"); 749 750 copy_remapped_to(elf, seg); 751 elf->max_addr = va + num_bytes; 752 } else { 753 uint32_t flags = 0; 754 size_t filesz = seg->filesz; 755 size_t memsz = seg->memsz; 756 size_t offset = seg->offset; 757 size_t vaddr = seg->vaddr; 758 759 if (offset < elf->max_offs) { 760 /* 761 * We're in a load segment which overlaps 762 * with (or is covered by) the first page 763 * of a shared library. 764 */ 765 if (vaddr + filesz < SMALL_PAGE_SIZE) { 766 size_t num_bytes = 0; 767 768 /* 769 * If this segment is completely 770 * covered, take next. 771 */ 772 if (vaddr + memsz <= SMALL_PAGE_SIZE) 773 continue; 774 775 /* 776 * All data of the segment is 777 * loaded, but we need to zero 778 * extend it. 779 */ 780 va = elf->max_addr; 781 num_bytes = roundup(vaddr + memsz) - 782 roundup(vaddr) - 783 SMALL_PAGE_SIZE; 784 assert(num_bytes); 785 res = sys_map_zi(num_bytes, 0, &va, 0, 786 0); 787 if (res) 788 err(res, "sys_map_zi"); 789 elf->max_addr = roundup(va + num_bytes); 790 continue; 791 } 792 793 /* Partial overlap, remove the first page. */ 794 vaddr += SMALL_PAGE_SIZE; 795 filesz -= SMALL_PAGE_SIZE; 796 memsz -= SMALL_PAGE_SIZE; 797 offset += SMALL_PAGE_SIZE; 798 } 799 800 if (!elf->load_addr) { 801 va = 0; 802 pad_begin = get_pad_begin(); 803 /* 804 * If mapping with pad_begin fails we'll 805 * retry without pad_begin, effectively 806 * disabling ASLR for the current ELF file. 807 */ 808 } else { 809 va = vaddr + elf->load_addr; 810 pad_begin = 0; 811 } 812 813 if (seg->flags & PF_W) 814 flags |= LDELF_MAP_FLAG_WRITEABLE; 815 else 816 flags |= LDELF_MAP_FLAG_SHAREABLE; 817 if (seg->flags & PF_X) 818 flags |= LDELF_MAP_FLAG_EXECUTABLE; 819 if (!(seg->flags & PF_R)) 820 err(TEE_ERROR_NOT_SUPPORTED, 821 "Segment must be readable"); 822 if (flags & LDELF_MAP_FLAG_WRITEABLE) { 823 res = sys_map_zi(memsz, 0, &va, pad_begin, 824 pad_end); 825 if (pad_begin && res == TEE_ERROR_OUT_OF_MEMORY) 826 res = sys_map_zi(memsz, 0, &va, 0, 827 pad_end); 828 if (res) 829 err(res, "sys_map_zi"); 830 res = sys_copy_from_ta_bin((void *)va, filesz, 831 elf->handle, offset); 832 if (res) 833 err(res, "sys_copy_from_ta_bin"); 834 } else { 835 if (filesz != memsz) 836 err(TEE_ERROR_BAD_FORMAT, 837 "Filesz and memsz mismatch"); 838 res = sys_map_ta_bin(&va, filesz, flags, 839 elf->handle, offset, 840 pad_begin, pad_end); 841 if (pad_begin && res == TEE_ERROR_OUT_OF_MEMORY) 842 res = sys_map_ta_bin(&va, filesz, flags, 843 elf->handle, 844 offset, 0, 845 pad_end); 846 if (res) 847 err(res, "sys_map_ta_bin"); 848 } 849 850 if (!elf->load_addr) 851 elf->load_addr = va; 852 elf->max_addr = roundup(va + memsz); 853 elf->max_offs += filesz; 854 } 855 } 856 } 857 858 static void map_segments(struct ta_elf *elf) 859 { 860 TEE_Result res = TEE_SUCCESS; 861 862 parse_load_segments(elf); 863 adjust_segments(elf); 864 if (TAILQ_FIRST(&elf->segs)->offset < SMALL_PAGE_SIZE) { 865 vaddr_t va = 0; 866 size_t sz = elf->max_addr - elf->load_addr; 867 struct segment *seg = TAILQ_LAST(&elf->segs, segment_head); 868 size_t pad_begin = get_pad_begin(); 869 870 /* 871 * We're loading a library, if not other parts of the code 872 * need to be updated too. 873 */ 874 assert(!elf->is_main); 875 876 /* 877 * Now that we know how much virtual memory is needed move 878 * the already mapped part to a location which can 879 * accommodate us. 880 */ 881 res = sys_remap(elf->load_addr, &va, sz, pad_begin, 882 roundup(seg->vaddr + seg->memsz)); 883 if (res == TEE_ERROR_OUT_OF_MEMORY) 884 res = sys_remap(elf->load_addr, &va, sz, 0, 885 roundup(seg->vaddr + seg->memsz)); 886 if (res) 887 err(res, "sys_remap"); 888 elf->ehdr_addr = va; 889 elf->load_addr = va; 890 elf->max_addr = va + sz; 891 elf->phdr = (void *)(va + elf->e_phoff); 892 } 893 } 894 895 static void add_deps_from_segment(struct ta_elf *elf, unsigned int type, 896 vaddr_t addr, size_t memsz) 897 { 898 size_t dyn_entsize = 0; 899 size_t num_dyns = 0; 900 size_t n = 0; 901 unsigned int tag = 0; 902 size_t val = 0; 903 TEE_UUID uuid = { }; 904 char *str_tab = NULL; 905 size_t str_tab_sz = 0; 906 907 if (type != PT_DYNAMIC) 908 return; 909 910 check_phdr_in_range(elf, type, addr, memsz); 911 912 if (elf->is_32bit) 913 dyn_entsize = sizeof(Elf32_Dyn); 914 else 915 dyn_entsize = sizeof(Elf64_Dyn); 916 917 assert(!(memsz % dyn_entsize)); 918 num_dyns = memsz / dyn_entsize; 919 920 for (n = 0; n < num_dyns && !(str_tab && str_tab_sz); n++) { 921 read_dyn(elf, addr, n, &tag, &val); 922 if (tag == DT_STRTAB) 923 str_tab = (char *)(val + elf->load_addr); 924 else if (tag == DT_STRSZ) 925 str_tab_sz = val; 926 } 927 check_range(elf, ".dynstr/STRTAB", str_tab, str_tab_sz); 928 929 for (n = 0; n < num_dyns; n++) { 930 read_dyn(elf, addr, n, &tag, &val); 931 if (tag != DT_NEEDED) 932 continue; 933 if (val >= str_tab_sz) 934 err(TEE_ERROR_BAD_FORMAT, 935 "Offset into .dynstr/STRTAB out of range"); 936 tee_uuid_from_str(&uuid, str_tab + val); 937 queue_elf(&uuid); 938 } 939 } 940 941 static void add_dependencies(struct ta_elf *elf) 942 { 943 size_t n = 0; 944 945 if (elf->is_32bit) { 946 Elf32_Phdr *phdr = elf->phdr; 947 948 for (n = 0; n < elf->e_phnum; n++) 949 add_deps_from_segment(elf, phdr[n].p_type, 950 phdr[n].p_vaddr, phdr[n].p_memsz); 951 } else { 952 Elf64_Phdr *phdr = elf->phdr; 953 954 for (n = 0; n < elf->e_phnum; n++) 955 add_deps_from_segment(elf, phdr[n].p_type, 956 phdr[n].p_vaddr, phdr[n].p_memsz); 957 } 958 } 959 960 static void copy_section_headers(struct ta_elf *elf) 961 { 962 TEE_Result res = TEE_SUCCESS; 963 size_t sz = 0; 964 size_t offs = 0; 965 966 if (MUL_OVERFLOW(elf->e_shnum, elf->e_shentsize, &sz)) 967 err(TEE_ERROR_BAD_FORMAT, "Section headers size overflow"); 968 969 elf->shdr = malloc(sz); 970 if (!elf->shdr) 971 err(TEE_ERROR_OUT_OF_MEMORY, "malloc"); 972 973 /* 974 * We're assuming that section headers comes after the load segments, 975 * but if it's a very small dynamically linked library the section 976 * headers can still end up (partially?) in the first mapped page. 977 */ 978 if (elf->e_shoff < SMALL_PAGE_SIZE) { 979 assert(!elf->is_main); 980 offs = MIN(SMALL_PAGE_SIZE - elf->e_shoff, sz); 981 memcpy(elf->shdr, (void *)(elf->load_addr + elf->e_shoff), 982 offs); 983 } 984 985 if (offs < sz) { 986 res = sys_copy_from_ta_bin((uint8_t *)elf->shdr + offs, 987 sz - offs, elf->handle, 988 elf->e_shoff + offs); 989 if (res) 990 err(res, "sys_copy_from_ta_bin"); 991 } 992 } 993 994 static void close_handle(struct ta_elf *elf) 995 { 996 TEE_Result res = sys_close_ta_bin(elf->handle); 997 998 if (res) 999 err(res, "sys_close_ta_bin"); 1000 elf->handle = -1; 1001 } 1002 1003 static void clean_elf_load_main(struct ta_elf *elf) 1004 { 1005 TEE_Result res = TEE_SUCCESS; 1006 1007 /* 1008 * Clean up from last attempt to load 1009 */ 1010 res = sys_unmap(elf->ehdr_addr, SMALL_PAGE_SIZE); 1011 if (res) 1012 err(res, "sys_unmap"); 1013 1014 while (!TAILQ_EMPTY(&elf->segs)) { 1015 struct segment *seg = TAILQ_FIRST(&elf->segs); 1016 vaddr_t va = 0; 1017 size_t num_bytes = 0; 1018 1019 va = rounddown(elf->load_addr + seg->vaddr); 1020 if (seg->remapped_writeable) 1021 num_bytes = roundup(seg->vaddr + seg->memsz) - 1022 rounddown(seg->vaddr); 1023 else 1024 num_bytes = seg->memsz; 1025 1026 res = sys_unmap(va, num_bytes); 1027 if (res) 1028 err(res, "sys_unmap"); 1029 1030 TAILQ_REMOVE(&elf->segs, seg, link); 1031 free(seg); 1032 } 1033 1034 free(elf->shdr); 1035 memset(&elf->is_32bit, 0, 1036 (vaddr_t)&elf->uuid - (vaddr_t)&elf->is_32bit); 1037 1038 TAILQ_INIT(&elf->segs); 1039 } 1040 1041 #ifdef ARM64 1042 /* 1043 * Allocates an offset in the TA's Thread Control Block for the TLS segment of 1044 * the @elf module. 1045 */ 1046 #define TCB_HEAD_SIZE (2 * sizeof(long)) 1047 static void set_tls_offset(struct ta_elf *elf) 1048 { 1049 static size_t next_offs = TCB_HEAD_SIZE; 1050 1051 if (!elf->tls_start) 1052 return; 1053 1054 /* Module has a TLS segment */ 1055 elf->tls_tcb_offs = next_offs; 1056 next_offs += elf->tls_memsz; 1057 } 1058 #else 1059 static void set_tls_offset(struct ta_elf *elf __unused) {} 1060 #endif 1061 1062 static void load_main(struct ta_elf *elf) 1063 { 1064 init_elf(elf); 1065 map_segments(elf); 1066 populate_segments(elf); 1067 add_dependencies(elf); 1068 copy_section_headers(elf); 1069 save_symtab(elf); 1070 close_handle(elf); 1071 set_tls_offset(elf); 1072 1073 elf->head = (struct ta_head *)elf->load_addr; 1074 if (elf->head->depr_entry != UINT64_MAX) { 1075 /* 1076 * Legacy TAs sets their entry point in ta_head. For 1077 * non-legacy TAs the entry point of the ELF is set instead 1078 * and leaving the ta_head entry point set to UINT64_MAX to 1079 * indicate that it's not used. 1080 * 1081 * NB, everything before the commit a73b5878c89d ("Replace 1082 * ta_head.entry with elf entry") is considered legacy TAs 1083 * for ldelf. 1084 * 1085 * Legacy TAs cannot be mapped with shared memory segments 1086 * so restart the mapping if it turned out we're loading a 1087 * legacy TA. 1088 */ 1089 1090 DMSG("Reloading TA %pUl as legacy TA", (void *)&elf->uuid); 1091 clean_elf_load_main(elf); 1092 elf->is_legacy = true; 1093 init_elf(elf); 1094 map_segments(elf); 1095 populate_segments_legacy(elf); 1096 add_dependencies(elf); 1097 copy_section_headers(elf); 1098 save_symtab(elf); 1099 close_handle(elf); 1100 elf->head = (struct ta_head *)elf->load_addr; 1101 /* 1102 * Check that the TA is still a legacy TA, if it isn't give 1103 * up now since we're likely under attack. 1104 */ 1105 if (elf->head->depr_entry == UINT64_MAX) 1106 err(TEE_ERROR_GENERIC, 1107 "TA %pUl was changed on disk to non-legacy", 1108 (void *)&elf->uuid); 1109 } 1110 1111 } 1112 1113 void ta_elf_load_main(const TEE_UUID *uuid, uint32_t *is_32bit, uint64_t *sp, 1114 uint32_t *ta_flags) 1115 { 1116 struct ta_elf *elf = queue_elf(uuid); 1117 vaddr_t va = 0; 1118 TEE_Result res = TEE_SUCCESS; 1119 1120 assert(elf); 1121 elf->is_main = true; 1122 1123 load_main(elf); 1124 1125 *is_32bit = elf->is_32bit; 1126 res = sys_map_zi(elf->head->stack_size, 0, &va, 0, 0); 1127 if (res) 1128 err(res, "sys_map_zi stack"); 1129 1130 if (elf->head->flags & ~TA_FLAGS_MASK) 1131 err(TEE_ERROR_BAD_FORMAT, "Invalid TA flags(s) %#"PRIx32, 1132 elf->head->flags & ~TA_FLAGS_MASK); 1133 1134 *ta_flags = elf->head->flags; 1135 *sp = va + elf->head->stack_size; 1136 ta_stack = va; 1137 ta_stack_size = elf->head->stack_size; 1138 } 1139 1140 void ta_elf_finalize_load_main(uint64_t *entry) 1141 { 1142 struct ta_elf *elf = TAILQ_FIRST(&main_elf_queue); 1143 TEE_Result res = TEE_SUCCESS; 1144 1145 assert(elf->is_main); 1146 1147 res = ta_elf_set_init_fini_info_compat(elf->is_32bit); 1148 if (res) 1149 err(res, "ta_elf_set_init_fini_info_compat"); 1150 res = ta_elf_set_elf_phdr_info(elf->is_32bit); 1151 if (res) 1152 err(res, "ta_elf_set_elf_phdr_info"); 1153 1154 if (elf->is_legacy) 1155 *entry = elf->head->depr_entry; 1156 else 1157 *entry = elf->e_entry + elf->load_addr; 1158 } 1159 1160 1161 void ta_elf_load_dependency(struct ta_elf *elf, bool is_32bit) 1162 { 1163 if (elf->is_main) 1164 return; 1165 1166 init_elf(elf); 1167 if (elf->is_32bit != is_32bit) 1168 err(TEE_ERROR_BAD_FORMAT, "ELF %pUl is %sbit (expected %sbit)", 1169 (void *)&elf->uuid, elf->is_32bit ? "32" : "64", 1170 is_32bit ? "32" : "64"); 1171 1172 map_segments(elf); 1173 populate_segments(elf); 1174 add_dependencies(elf); 1175 copy_section_headers(elf); 1176 save_symtab(elf); 1177 close_handle(elf); 1178 set_tls_offset(elf); 1179 } 1180 1181 void ta_elf_finalize_mappings(struct ta_elf *elf) 1182 { 1183 TEE_Result res = TEE_SUCCESS; 1184 struct segment *seg = NULL; 1185 1186 if (!elf->is_legacy) 1187 return; 1188 1189 TAILQ_FOREACH(seg, &elf->segs, link) { 1190 vaddr_t va = elf->load_addr + seg->vaddr; 1191 uint32_t flags = 0; 1192 1193 if (seg->flags & PF_W) 1194 flags |= LDELF_MAP_FLAG_WRITEABLE; 1195 if (seg->flags & PF_X) 1196 flags |= LDELF_MAP_FLAG_EXECUTABLE; 1197 1198 res = sys_set_prot(va, seg->memsz, flags); 1199 if (res) 1200 err(res, "sys_set_prot"); 1201 } 1202 } 1203 1204 static void __printf(3, 4) print_wrapper(void *pctx, print_func_t print_func, 1205 const char *fmt, ...) 1206 { 1207 va_list ap; 1208 1209 va_start(ap, fmt); 1210 print_func(pctx, fmt, ap); 1211 va_end(ap); 1212 } 1213 1214 static void print_seg(void *pctx, print_func_t print_func, 1215 size_t idx __maybe_unused, int elf_idx __maybe_unused, 1216 vaddr_t va __maybe_unused, paddr_t pa __maybe_unused, 1217 size_t sz __maybe_unused, uint32_t flags) 1218 { 1219 int rc __maybe_unused = 0; 1220 int width __maybe_unused = 8; 1221 char desc[14] __maybe_unused = ""; 1222 char flags_str[] __maybe_unused = "----"; 1223 1224 if (elf_idx > -1) { 1225 rc = snprintf(desc, sizeof(desc), " [%d]", elf_idx); 1226 assert(rc >= 0); 1227 } else { 1228 if (flags & DUMP_MAP_EPHEM) { 1229 rc = snprintf(desc, sizeof(desc), " (param)"); 1230 assert(rc >= 0); 1231 } 1232 if (flags & DUMP_MAP_LDELF) { 1233 rc = snprintf(desc, sizeof(desc), " (ldelf)"); 1234 assert(rc >= 0); 1235 } 1236 if (va == ta_stack) { 1237 rc = snprintf(desc, sizeof(desc), " (stack)"); 1238 assert(rc >= 0); 1239 } 1240 } 1241 1242 if (flags & DUMP_MAP_READ) 1243 flags_str[0] = 'r'; 1244 if (flags & DUMP_MAP_WRITE) 1245 flags_str[1] = 'w'; 1246 if (flags & DUMP_MAP_EXEC) 1247 flags_str[2] = 'x'; 1248 if (flags & DUMP_MAP_SECURE) 1249 flags_str[3] = 's'; 1250 1251 print_wrapper(pctx, print_func, 1252 "region %2zu: va 0x%0*"PRIxVA" pa 0x%0*"PRIxPA" size 0x%06zx flags %s%s\n", 1253 idx, width, va, width, pa, sz, flags_str, desc); 1254 } 1255 1256 static bool get_next_in_order(struct ta_elf_queue *elf_queue, 1257 struct ta_elf **elf, struct segment **seg, 1258 size_t *elf_idx) 1259 { 1260 struct ta_elf *e = NULL; 1261 struct segment *s = NULL; 1262 size_t idx = 0; 1263 vaddr_t va = 0; 1264 struct ta_elf *e2 = NULL; 1265 size_t i2 = 0; 1266 1267 assert(elf && seg && elf_idx); 1268 e = *elf; 1269 s = *seg; 1270 assert((e == NULL && s == NULL) || (e != NULL && s != NULL)); 1271 1272 if (s) { 1273 s = TAILQ_NEXT(s, link); 1274 if (s) { 1275 *seg = s; 1276 return true; 1277 } 1278 } 1279 1280 if (e) 1281 va = e->load_addr; 1282 1283 /* Find the ELF with next load address */ 1284 e = NULL; 1285 TAILQ_FOREACH(e2, elf_queue, link) { 1286 if (e2->load_addr > va) { 1287 if (!e || e2->load_addr < e->load_addr) { 1288 e = e2; 1289 idx = i2; 1290 } 1291 } 1292 i2++; 1293 } 1294 if (!e) 1295 return false; 1296 1297 *elf = e; 1298 *seg = TAILQ_FIRST(&e->segs); 1299 *elf_idx = idx; 1300 return true; 1301 } 1302 1303 void ta_elf_print_mappings(void *pctx, print_func_t print_func, 1304 struct ta_elf_queue *elf_queue, size_t num_maps, 1305 struct dump_map *maps, vaddr_t mpool_base) 1306 { 1307 struct segment *seg = NULL; 1308 struct ta_elf *elf = NULL; 1309 size_t elf_idx = 0; 1310 size_t idx = 0; 1311 size_t map_idx = 0; 1312 1313 /* 1314 * Loop over all segments and maps, printing virtual address in 1315 * order. Segment has priority if the virtual address is present 1316 * in both map and segment. 1317 */ 1318 get_next_in_order(elf_queue, &elf, &seg, &elf_idx); 1319 while (true) { 1320 vaddr_t va = -1; 1321 size_t sz = 0; 1322 uint32_t flags = DUMP_MAP_SECURE; 1323 size_t offs = 0; 1324 1325 if (seg) { 1326 va = rounddown(seg->vaddr + elf->load_addr); 1327 sz = roundup(seg->vaddr + seg->memsz) - 1328 rounddown(seg->vaddr); 1329 } 1330 1331 while (map_idx < num_maps && maps[map_idx].va <= va) { 1332 uint32_t f = 0; 1333 1334 /* If there's a match, it should be the same map */ 1335 if (maps[map_idx].va == va) { 1336 /* 1337 * In shared libraries the first page is 1338 * mapped separately with the rest of that 1339 * segment following back to back in a 1340 * separate entry. 1341 */ 1342 if (map_idx + 1 < num_maps && 1343 maps[map_idx].sz == SMALL_PAGE_SIZE) { 1344 vaddr_t next_va = maps[map_idx].va + 1345 maps[map_idx].sz; 1346 size_t comb_sz = maps[map_idx].sz + 1347 maps[map_idx + 1].sz; 1348 1349 if (next_va == maps[map_idx + 1].va && 1350 comb_sz == sz && 1351 maps[map_idx].flags == 1352 maps[map_idx + 1].flags) { 1353 /* Skip this and next entry */ 1354 map_idx += 2; 1355 continue; 1356 } 1357 } 1358 assert(maps[map_idx].sz == sz); 1359 } else if (maps[map_idx].va < va) { 1360 if (maps[map_idx].va == mpool_base) 1361 f |= DUMP_MAP_LDELF; 1362 print_seg(pctx, print_func, idx, -1, 1363 maps[map_idx].va, maps[map_idx].pa, 1364 maps[map_idx].sz, 1365 maps[map_idx].flags | f); 1366 idx++; 1367 } 1368 map_idx++; 1369 } 1370 1371 if (!seg) 1372 break; 1373 1374 offs = rounddown(seg->offset); 1375 if (seg->flags & PF_R) 1376 flags |= DUMP_MAP_READ; 1377 if (seg->flags & PF_W) 1378 flags |= DUMP_MAP_WRITE; 1379 if (seg->flags & PF_X) 1380 flags |= DUMP_MAP_EXEC; 1381 1382 print_seg(pctx, print_func, idx, elf_idx, va, offs, sz, flags); 1383 idx++; 1384 1385 if (!get_next_in_order(elf_queue, &elf, &seg, &elf_idx)) 1386 seg = NULL; 1387 } 1388 1389 elf_idx = 0; 1390 TAILQ_FOREACH(elf, elf_queue, link) { 1391 print_wrapper(pctx, print_func, 1392 " [%zu] %pUl @ 0x%0*"PRIxVA"\n", 1393 elf_idx, (void *)&elf->uuid, 8, elf->load_addr); 1394 elf_idx++; 1395 } 1396 } 1397 1398 #ifdef CFG_UNWIND 1399 /* Called by libunw */ 1400 bool find_exidx(vaddr_t addr, vaddr_t *idx_start, vaddr_t *idx_end) 1401 { 1402 struct segment *seg = NULL; 1403 struct ta_elf *elf = NULL; 1404 vaddr_t a = 0; 1405 1406 TAILQ_FOREACH(elf, &main_elf_queue, link) { 1407 if (addr < elf->load_addr) 1408 continue; 1409 a = addr - elf->load_addr; 1410 TAILQ_FOREACH(seg, &elf->segs, link) { 1411 if (a < seg->vaddr) 1412 continue; 1413 if (a - seg->vaddr < seg->filesz) { 1414 *idx_start = elf->exidx_start + elf->load_addr; 1415 *idx_end = elf->exidx_start + elf->load_addr + 1416 elf->exidx_size; 1417 return true; 1418 } 1419 } 1420 } 1421 1422 return false; 1423 } 1424 1425 void ta_elf_stack_trace_a32(uint32_t regs[16]) 1426 { 1427 struct unwind_state_arm32 state = { }; 1428 1429 memcpy(state.registers, regs, sizeof(state.registers)); 1430 print_stack_arm32(&state, ta_stack, ta_stack_size); 1431 } 1432 1433 void ta_elf_stack_trace_a64(uint64_t fp, uint64_t sp, uint64_t pc) 1434 { 1435 struct unwind_state_arm64 state = { .fp = fp, .sp = sp, .pc = pc }; 1436 1437 print_stack_arm64(&state, ta_stack, ta_stack_size); 1438 } 1439 #endif 1440 1441 TEE_Result ta_elf_add_library(const TEE_UUID *uuid) 1442 { 1443 TEE_Result res = TEE_ERROR_GENERIC; 1444 struct ta_elf *ta = TAILQ_FIRST(&main_elf_queue); 1445 struct ta_elf *lib = ta_elf_find_elf(uuid); 1446 struct ta_elf *elf = NULL; 1447 1448 if (lib) 1449 return TEE_SUCCESS; /* Already mapped */ 1450 1451 lib = queue_elf_helper(uuid); 1452 if (!lib) 1453 return TEE_ERROR_OUT_OF_MEMORY; 1454 1455 for (elf = lib; elf; elf = TAILQ_NEXT(elf, link)) 1456 ta_elf_load_dependency(elf, ta->is_32bit); 1457 1458 for (elf = lib; elf; elf = TAILQ_NEXT(elf, link)) { 1459 ta_elf_relocate(elf); 1460 ta_elf_finalize_mappings(elf); 1461 } 1462 1463 for (elf = lib; elf; elf = TAILQ_NEXT(elf, link)) 1464 DMSG("ELF (%pUl) at %#"PRIxVA, 1465 (void *)&elf->uuid, elf->load_addr); 1466 1467 res = ta_elf_set_init_fini_info_compat(ta->is_32bit); 1468 if (res) 1469 return res; 1470 1471 return ta_elf_set_elf_phdr_info(ta->is_32bit); 1472 } 1473 1474 /* Get address/size of .init_array and .fini_array from the dynamic segment */ 1475 static void get_init_fini_array(struct ta_elf *elf, unsigned int type, 1476 vaddr_t addr, size_t memsz, vaddr_t *init, 1477 size_t *init_cnt, vaddr_t *fini, 1478 size_t *fini_cnt) 1479 { 1480 size_t addrsz = 0; 1481 size_t dyn_entsize = 0; 1482 size_t num_dyns = 0; 1483 size_t n = 0; 1484 unsigned int tag = 0; 1485 size_t val = 0; 1486 1487 assert(type == PT_DYNAMIC); 1488 1489 check_phdr_in_range(elf, type, addr, memsz); 1490 1491 if (elf->is_32bit) { 1492 dyn_entsize = sizeof(Elf32_Dyn); 1493 addrsz = 4; 1494 } else { 1495 dyn_entsize = sizeof(Elf64_Dyn); 1496 addrsz = 8; 1497 } 1498 1499 assert(!(memsz % dyn_entsize)); 1500 num_dyns = memsz / dyn_entsize; 1501 1502 for (n = 0; n < num_dyns; n++) { 1503 read_dyn(elf, addr, n, &tag, &val); 1504 if (tag == DT_INIT_ARRAY) 1505 *init = val + elf->load_addr; 1506 else if (tag == DT_FINI_ARRAY) 1507 *fini = val + elf->load_addr; 1508 else if (tag == DT_INIT_ARRAYSZ) 1509 *init_cnt = val / addrsz; 1510 else if (tag == DT_FINI_ARRAYSZ) 1511 *fini_cnt = val / addrsz; 1512 } 1513 } 1514 1515 /* Get address/size of .init_array and .fini_array in @elf (if present) */ 1516 static void elf_get_init_fini_array(struct ta_elf *elf, vaddr_t *init, 1517 size_t *init_cnt, vaddr_t *fini, 1518 size_t *fini_cnt) 1519 { 1520 size_t n = 0; 1521 1522 if (elf->is_32bit) { 1523 Elf32_Phdr *phdr = elf->phdr; 1524 1525 for (n = 0; n < elf->e_phnum; n++) { 1526 if (phdr[n].p_type == PT_DYNAMIC) { 1527 get_init_fini_array(elf, phdr[n].p_type, 1528 phdr[n].p_vaddr, 1529 phdr[n].p_memsz, 1530 init, init_cnt, fini, 1531 fini_cnt); 1532 return; 1533 } 1534 } 1535 } else { 1536 Elf64_Phdr *phdr = elf->phdr; 1537 1538 for (n = 0; n < elf->e_phnum; n++) { 1539 if (phdr[n].p_type == PT_DYNAMIC) { 1540 get_init_fini_array(elf, phdr[n].p_type, 1541 phdr[n].p_vaddr, 1542 phdr[n].p_memsz, 1543 init, init_cnt, fini, 1544 fini_cnt); 1545 return; 1546 } 1547 } 1548 } 1549 } 1550 1551 /* 1552 * Deprecated by __elf_phdr_info below. Kept for compatibility. 1553 * 1554 * Pointers to ELF initialization and finalization functions are extracted by 1555 * ldelf and stored on the TA heap, then exported to the TA via the global 1556 * symbol __init_fini_info. libutee in OP-TEE 3.9.0 uses this mechanism. 1557 */ 1558 1559 struct __init_fini { 1560 uint32_t flags; 1561 uint16_t init_size; 1562 uint16_t fini_size; 1563 1564 void (**init)(void); /* @init_size entries */ 1565 void (**fini)(void); /* @fini_size entries */ 1566 }; 1567 1568 #define __IFS_VALID BIT(0) 1569 #define __IFS_INIT_HAS_RUN BIT(1) 1570 #define __IFS_FINI_HAS_RUN BIT(2) 1571 1572 struct __init_fini_info { 1573 uint32_t reserved; 1574 uint16_t size; 1575 uint16_t pad; 1576 struct __init_fini *ifs; /* @size entries */ 1577 }; 1578 1579 /* 32-bit variants for a 64-bit ldelf to access a 32-bit TA */ 1580 1581 struct __init_fini32 { 1582 uint32_t flags; 1583 uint16_t init_size; 1584 uint16_t fini_size; 1585 uint32_t init; 1586 uint32_t fini; 1587 }; 1588 1589 struct __init_fini_info32 { 1590 uint32_t reserved; 1591 uint16_t size; 1592 uint16_t pad; 1593 uint32_t ifs; 1594 }; 1595 1596 static TEE_Result realloc_ifs(vaddr_t va, size_t cnt, bool is_32bit) 1597 { 1598 struct __init_fini_info32 *info32 = (struct __init_fini_info32 *)va; 1599 struct __init_fini_info *info = (struct __init_fini_info *)va; 1600 struct __init_fini32 *ifs32 = NULL; 1601 struct __init_fini *ifs = NULL; 1602 size_t prev_cnt = 0; 1603 void *ptr = NULL; 1604 1605 if (is_32bit) { 1606 ptr = (void *)(vaddr_t)info32->ifs; 1607 ptr = realloc(ptr, cnt * sizeof(struct __init_fini32)); 1608 if (!ptr) 1609 return TEE_ERROR_OUT_OF_MEMORY; 1610 ifs32 = ptr; 1611 prev_cnt = info32->size; 1612 if (cnt > prev_cnt) 1613 memset(ifs32 + prev_cnt, 0, 1614 (cnt - prev_cnt) * sizeof(*ifs32)); 1615 info32->ifs = (uint32_t)(vaddr_t)ifs32; 1616 info32->size = cnt; 1617 } else { 1618 ptr = realloc(info->ifs, cnt * sizeof(struct __init_fini)); 1619 if (!ptr) 1620 return TEE_ERROR_OUT_OF_MEMORY; 1621 ifs = ptr; 1622 prev_cnt = info->size; 1623 if (cnt > prev_cnt) 1624 memset(ifs + prev_cnt, 0, 1625 (cnt - prev_cnt) * sizeof(*ifs)); 1626 info->ifs = ifs; 1627 info->size = cnt; 1628 } 1629 1630 return TEE_SUCCESS; 1631 } 1632 1633 static void fill_ifs(vaddr_t va, size_t idx, struct ta_elf *elf, bool is_32bit) 1634 { 1635 struct __init_fini_info32 *info32 = (struct __init_fini_info32 *)va; 1636 struct __init_fini_info *info = (struct __init_fini_info *)va; 1637 struct __init_fini32 *ifs32 = NULL; 1638 struct __init_fini *ifs = NULL; 1639 size_t init_cnt = 0; 1640 size_t fini_cnt = 0; 1641 vaddr_t init = 0; 1642 vaddr_t fini = 0; 1643 1644 if (is_32bit) { 1645 assert(idx < info32->size); 1646 ifs32 = &((struct __init_fini32 *)(vaddr_t)info32->ifs)[idx]; 1647 1648 if (ifs32->flags & __IFS_VALID) 1649 return; 1650 1651 elf_get_init_fini_array(elf, &init, &init_cnt, &fini, 1652 &fini_cnt); 1653 1654 ifs32->init = (uint32_t)init; 1655 ifs32->init_size = init_cnt; 1656 1657 ifs32->fini = (uint32_t)fini; 1658 ifs32->fini_size = fini_cnt; 1659 1660 ifs32->flags |= __IFS_VALID; 1661 } else { 1662 assert(idx < info->size); 1663 ifs = &info->ifs[idx]; 1664 1665 if (ifs->flags & __IFS_VALID) 1666 return; 1667 1668 elf_get_init_fini_array(elf, &init, &init_cnt, &fini, 1669 &fini_cnt); 1670 1671 ifs->init = (void (**)(void))init; 1672 ifs->init_size = init_cnt; 1673 1674 ifs->fini = (void (**)(void))fini; 1675 ifs->fini_size = fini_cnt; 1676 1677 ifs->flags |= __IFS_VALID; 1678 } 1679 } 1680 1681 /* 1682 * Set or update __init_fini_info in the TA with information from the ELF 1683 * queue 1684 */ 1685 TEE_Result ta_elf_set_init_fini_info_compat(bool is_32bit) 1686 { 1687 struct __init_fini_info *info = NULL; 1688 TEE_Result res = TEE_SUCCESS; 1689 struct ta_elf *elf = NULL; 1690 vaddr_t info_va = 0; 1691 size_t cnt = 0; 1692 1693 res = ta_elf_resolve_sym("__init_fini_info", &info_va, NULL, NULL); 1694 if (res) { 1695 if (res == TEE_ERROR_ITEM_NOT_FOUND) { 1696 /* 1697 * Not an error, only TAs linked against libutee from 1698 * OP-TEE 3.9.0 have this symbol. 1699 */ 1700 return TEE_SUCCESS; 1701 } 1702 return res; 1703 } 1704 assert(info_va); 1705 1706 info = (struct __init_fini_info *)info_va; 1707 if (info->reserved) 1708 return TEE_ERROR_NOT_SUPPORTED; 1709 1710 TAILQ_FOREACH(elf, &main_elf_queue, link) 1711 cnt++; 1712 1713 /* Queue has at least one file (main) */ 1714 assert(cnt); 1715 1716 res = realloc_ifs(info_va, cnt, is_32bit); 1717 if (res) 1718 goto err; 1719 1720 cnt = 0; 1721 TAILQ_FOREACH(elf, &main_elf_queue, link) { 1722 fill_ifs(info_va, cnt, elf, is_32bit); 1723 cnt++; 1724 } 1725 1726 return TEE_SUCCESS; 1727 err: 1728 free(info); 1729 return res; 1730 } 1731 1732 static TEE_Result realloc_elf_phdr_info(vaddr_t va, size_t cnt, bool is_32bit) 1733 { 1734 struct __elf_phdr_info32 *info32 = (struct __elf_phdr_info32 *)va; 1735 struct __elf_phdr_info *info = (struct __elf_phdr_info *)va; 1736 struct dl_phdr_info32 *dlpi32 = NULL; 1737 struct dl_phdr_info *dlpi = NULL; 1738 size_t prev_cnt = 0; 1739 void *ptr = NULL; 1740 1741 if (is_32bit) { 1742 ptr = (void *)(vaddr_t)info32->dlpi; 1743 ptr = realloc(ptr, cnt * sizeof(*dlpi32)); 1744 if (!ptr) 1745 return TEE_ERROR_OUT_OF_MEMORY; 1746 dlpi32 = ptr; 1747 prev_cnt = info32->count; 1748 if (cnt > prev_cnt) 1749 memset(dlpi32 + prev_cnt, 0, 1750 (cnt - prev_cnt) * sizeof(*dlpi32)); 1751 info32->dlpi = (uint32_t)(vaddr_t)dlpi32; 1752 info32->count = cnt; 1753 } else { 1754 ptr = realloc(info->dlpi, cnt * sizeof(*dlpi)); 1755 if (!ptr) 1756 return TEE_ERROR_OUT_OF_MEMORY; 1757 dlpi = ptr; 1758 prev_cnt = info->count; 1759 if (cnt > prev_cnt) 1760 memset(dlpi + prev_cnt, 0, 1761 (cnt - prev_cnt) * sizeof(*dlpi)); 1762 info->dlpi = dlpi; 1763 info->count = cnt; 1764 } 1765 1766 return TEE_SUCCESS; 1767 } 1768 1769 static void fill_elf_phdr_info(vaddr_t va, size_t idx, struct ta_elf *elf, 1770 bool is_32bit) 1771 { 1772 struct __elf_phdr_info32 *info32 = (struct __elf_phdr_info32 *)va; 1773 struct __elf_phdr_info *info = (struct __elf_phdr_info *)va; 1774 struct dl_phdr_info32 *dlpi32 = NULL; 1775 struct dl_phdr_info *dlpi = NULL; 1776 1777 if (is_32bit) { 1778 assert(idx < info32->count); 1779 dlpi32 = (struct dl_phdr_info32 *)(vaddr_t)info32->dlpi + idx; 1780 1781 dlpi32->dlpi_addr = elf->load_addr; 1782 if (elf->soname) 1783 dlpi32->dlpi_name = (vaddr_t)elf->soname; 1784 else 1785 dlpi32->dlpi_name = (vaddr_t)&info32->zero; 1786 dlpi32->dlpi_phdr = (vaddr_t)elf->phdr; 1787 dlpi32->dlpi_phnum = elf->e_phnum; 1788 dlpi32->dlpi_adds = 1; /* No unloading on dlclose() currently */ 1789 dlpi32->dlpi_subs = 0; /* No unloading on dlclose() currently */ 1790 dlpi32->dlpi_tls_modid = elf->tls_mod_id; 1791 dlpi32->dlpi_tls_data = elf->tls_start; 1792 } else { 1793 assert(idx < info->count); 1794 dlpi = info->dlpi + idx; 1795 1796 dlpi->dlpi_addr = elf->load_addr; 1797 if (elf->soname) 1798 dlpi->dlpi_name = elf->soname; 1799 else 1800 dlpi->dlpi_name = &info32->zero; 1801 dlpi->dlpi_phdr = elf->phdr; 1802 dlpi->dlpi_phnum = elf->e_phnum; 1803 dlpi->dlpi_adds = 1; /* No unloading on dlclose() currently */ 1804 dlpi->dlpi_subs = 0; /* No unloading on dlclose() currently */ 1805 dlpi->dlpi_tls_modid = elf->tls_mod_id; 1806 dlpi->dlpi_tls_data = (void *)elf->tls_start; 1807 } 1808 } 1809 1810 /* Set or update __elf_hdr_info in the TA with information from the ELF queue */ 1811 TEE_Result ta_elf_set_elf_phdr_info(bool is_32bit) 1812 { 1813 struct __elf_phdr_info *info = NULL; 1814 TEE_Result res = TEE_SUCCESS; 1815 struct ta_elf *elf = NULL; 1816 vaddr_t info_va = 0; 1817 size_t cnt = 0; 1818 1819 res = ta_elf_resolve_sym("__elf_phdr_info", &info_va, NULL, NULL); 1820 if (res) { 1821 if (res == TEE_ERROR_ITEM_NOT_FOUND) { 1822 /* Older TA */ 1823 return TEE_SUCCESS; 1824 } 1825 return res; 1826 } 1827 assert(info_va); 1828 1829 info = (struct __elf_phdr_info *)info_va; 1830 if (info->reserved) 1831 return TEE_ERROR_NOT_SUPPORTED; 1832 1833 TAILQ_FOREACH(elf, &main_elf_queue, link) 1834 cnt++; 1835 1836 res = realloc_elf_phdr_info(info_va, cnt, is_32bit); 1837 if (res) 1838 return res; 1839 1840 cnt = 0; 1841 TAILQ_FOREACH(elf, &main_elf_queue, link) { 1842 fill_elf_phdr_info(info_va, cnt, elf, is_32bit); 1843 cnt++; 1844 } 1845 1846 return TEE_SUCCESS; 1847 } 1848