52 static void assign_tls_mod_id(struct ta_elf *elf)  in assign_tls_mod_id()  argument
56 	if (elf->is_main)  in assign_tls_mod_id()
58 	elf->tls_mod_id = last_tls_mod_id++;  in assign_tls_mod_id()
63 	struct ta_elf *elf = calloc(1, sizeof(*elf));  in queue_elf_helper()  local
65 	if (!elf)  in queue_elf_helper()
68 	TAILQ_INIT(&elf->segs);  in queue_elf_helper()
70 	elf->uuid = *uuid;  in queue_elf_helper()
71 	TAILQ_INSERT_TAIL(&main_elf_queue, elf, link);  in queue_elf_helper()
72 	return elf;  in queue_elf_helper()
77 	struct ta_elf *elf = ta_elf_find_elf(uuid);  in queue_elf()  local
79 	if (elf)  in queue_elf()
82 	elf = queue_elf_helper(uuid);  in queue_elf()
83 	if (!elf)  in queue_elf()
86 	return elf;  in queue_elf()
91 	struct ta_elf *elf = NULL;  in ta_elf_find_elf()  local
93 	TAILQ_FOREACH(elf, &main_elf_queue, link)  in ta_elf_find_elf()
94 		if (!memcmp(uuid, &elf->uuid, sizeof(*uuid)))  in ta_elf_find_elf()
95 			return elf;  in ta_elf_find_elf()
101 static TEE_Result e32_parse_ehdr(struct ta_elf *elf, Elf32_Ehdr *ehdr)  in e32_parse_ehdr()  argument
124 	elf->is_32bit = true;  in e32_parse_ehdr()
125 	elf->e_entry = ehdr->e_entry;  in e32_parse_ehdr()
126 	elf->e_phoff = ehdr->e_phoff;  in e32_parse_ehdr()
127 	elf->e_shoff = ehdr->e_shoff;  in e32_parse_ehdr()
128 	elf->e_phnum = ehdr->e_phnum;  in e32_parse_ehdr()
129 	elf->e_shnum = ehdr->e_shnum;  in e32_parse_ehdr()
130 	elf->e_phentsize = ehdr->e_phentsize;  in e32_parse_ehdr()
131 	elf->e_shentsize = ehdr->e_shentsize;  in e32_parse_ehdr()
137 static TEE_Result e64_parse_ehdr(struct ta_elf *elf, Elf64_Ehdr *ehdr)  in e64_parse_ehdr()  argument
149 	elf->is_32bit = false;  in e64_parse_ehdr()
150 	elf->e_entry = ehdr->e_entry;  in e64_parse_ehdr()
151 	elf->e_phoff = ehdr->e_phoff;  in e64_parse_ehdr()
152 	elf->e_shoff = ehdr->e_shoff;  in e64_parse_ehdr()
153 	elf->e_phnum = ehdr->e_phnum;  in e64_parse_ehdr()
154 	elf->e_shnum = ehdr->e_shnum;  in e64_parse_ehdr()
155 	elf->e_phentsize = ehdr->e_phentsize;  in e64_parse_ehdr()
156 	elf->e_shentsize = ehdr->e_shentsize;  in e64_parse_ehdr()
161 static TEE_Result e64_parse_ehdr(struct ta_elf *elf __unused,  in e64_parse_ehdr()
170 static TEE_Result e32_parse_ehdr(struct ta_elf *elf __unused,  in e32_parse_ehdr()
176 static TEE_Result e64_parse_ehdr(struct ta_elf *elf, Elf64_Ehdr *ehdr)  in e64_parse_ehdr()  argument
187 	elf->is_32bit = false;  in e64_parse_ehdr()
188 	elf->e_entry = ehdr->e_entry;  in e64_parse_ehdr()
189 	elf->e_phoff = ehdr->e_phoff;  in e64_parse_ehdr()
190 	elf->e_shoff = ehdr->e_shoff;  in e64_parse_ehdr()
191 	elf->e_phnum = ehdr->e_phnum;  in e64_parse_ehdr()
192 	elf->e_shnum = ehdr->e_shnum;  in e64_parse_ehdr()
193 	elf->e_phentsize = ehdr->e_phentsize;  in e64_parse_ehdr()
194 	elf->e_shentsize = ehdr->e_shentsize;  in e64_parse_ehdr()
200 static void check_phdr_in_range(struct ta_elf *elf, unsigned int type,  in check_phdr_in_range()  argument
213 	if (max_addr > elf->max_addr - elf->load_addr)  in check_phdr_in_range()
218 static void read_dyn(struct ta_elf *elf, vaddr_t addr,  in read_dyn()  argument
221 	if (elf->is_32bit) {  in read_dyn()
222 		Elf32_Dyn *dyn = (Elf32_Dyn *)(addr + elf->load_addr);  in read_dyn()
227 		Elf64_Dyn *dyn = (Elf64_Dyn *)(addr + elf->load_addr);  in read_dyn()
234 static void check_range(struct ta_elf *elf, const char *name, const void *ptr,  in check_range()  argument
239 	if ((vaddr_t)ptr < elf->load_addr)  in check_range()
245 	if (max_addr > elf->max_addr)  in check_range()
250 static void check_hashtab(struct ta_elf *elf, void *ptr, size_t num_buckets,  in check_hashtab()  argument
271 	check_range(elf, "DT_HASH", ptr, sz);  in check_hashtab()
274 static void check_gnu_hashtab(struct ta_elf *elf, void *ptr)  in check_gnu_hashtab()  argument
285 	if (elf->gnu_hashtab_size < sizeof(*h))  in check_gnu_hashtab()
290 	if (elf->is_32bit)  in check_gnu_hashtab()
297 	    sz > elf->gnu_hashtab_size)  in check_gnu_hashtab()
301 static void save_hashtab(struct ta_elf *elf)  in save_hashtab()  argument
306 	if (elf->is_32bit) {  in save_hashtab()
307 		Elf32_Shdr *shdr = elf->shdr;  in save_hashtab()
309 		for (n = 0; n < elf->e_shnum; n++) {  in save_hashtab()
311 						       elf->load_addr);  in save_hashtab()
314 				elf->hashtab = addr;  in save_hashtab()
316 				elf->gnu_hashtab = addr;  in save_hashtab()
317 				elf->gnu_hashtab_size = shdr[n].sh_size;  in save_hashtab()
321 		Elf64_Shdr *shdr = elf->shdr;  in save_hashtab()
323 		for (n = 0; n < elf->e_shnum; n++) {  in save_hashtab()
325 						       elf->load_addr);  in save_hashtab()
328 				elf->hashtab = addr;  in save_hashtab()
330 				elf->gnu_hashtab = addr;  in save_hashtab()
331 				elf->gnu_hashtab_size = shdr[n].sh_size;  in save_hashtab()
336 	if (elf->hashtab) {  in save_hashtab()
337 		check_hashtab(elf, elf->hashtab, 0, 0);  in save_hashtab()
338 		hashtab = elf->hashtab;  in save_hashtab()
339 		check_hashtab(elf, elf->hashtab, hashtab[0], hashtab[1]);  in save_hashtab()
341 	if (elf->gnu_hashtab)  in save_hashtab()
342 		check_gnu_hashtab(elf, elf->gnu_hashtab);  in save_hashtab()
345 static void save_soname_from_segment(struct ta_elf *elf, unsigned int type,  in save_soname_from_segment()  argument
358 	if (elf->is_32bit)  in save_soname_from_segment()
367 		read_dyn(elf, addr, n, &tag, &val);  in save_soname_from_segment()
369 			str_tab = (char *)(val + elf->load_addr);  in save_soname_from_segment()
374 		read_dyn(elf, addr, n, &tag, &val);  in save_soname_from_segment()
376 			elf->soname = str_tab + val;  in save_soname_from_segment()
382 static void save_soname(struct ta_elf *elf)  in save_soname()  argument
386 	if (elf->is_32bit) {  in save_soname()
387 		Elf32_Phdr *phdr = elf->phdr;  in save_soname()
389 		for (n = 0; n < elf->e_phnum; n++)  in save_soname()
390 			save_soname_from_segment(elf, phdr[n].p_type,  in save_soname()
394 		Elf64_Phdr *phdr = elf->phdr;  in save_soname()
396 		for (n = 0; n < elf->e_phnum; n++)  in save_soname()
397 			save_soname_from_segment(elf, phdr[n].p_type,  in save_soname()
403 static void e32_save_symtab(struct ta_elf *elf, size_t tab_idx)  in e32_save_symtab()  argument
405 	Elf32_Shdr *shdr = elf->shdr;  in e32_save_symtab()
408 	elf->dynsymtab = (void *)(shdr[tab_idx].sh_addr + elf->load_addr);  in e32_save_symtab()
409 	if (!IS_ALIGNED_WITH_TYPE(elf->dynsymtab, Elf32_Sym))  in e32_save_symtab()
411 		    elf->dynsymtab);  in e32_save_symtab()
412 	check_range(elf, "Dynsymtab", elf->dynsymtab, shdr[tab_idx].sh_size);  in e32_save_symtab()
417 	elf->num_dynsyms = shdr[tab_idx].sh_size / sizeof(Elf32_Sym);  in e32_save_symtab()
419 	if (str_idx >= elf->e_shnum)  in e32_save_symtab()
421 	elf->dynstr = (void *)(shdr[str_idx].sh_addr + elf->load_addr);  in e32_save_symtab()
422 	check_range(elf, "Dynstr", elf->dynstr, shdr[str_idx].sh_size);  in e32_save_symtab()
424 	elf->dynstr_size = shdr[str_idx].sh_size;  in e32_save_symtab()
427 static void e64_save_symtab(struct ta_elf *elf, size_t tab_idx)  in e64_save_symtab()  argument
429 	Elf64_Shdr *shdr = elf->shdr;  in e64_save_symtab()
432 	elf->dynsymtab = (void *)(vaddr_t)(shdr[tab_idx].sh_addr +  in e64_save_symtab()
433 					   elf->load_addr);  in e64_save_symtab()
435 	if (!IS_ALIGNED_WITH_TYPE(elf->dynsymtab, Elf64_Sym))  in e64_save_symtab()
437 		    elf->dynsymtab);  in e64_save_symtab()
438 	check_range(elf, ".dynsym/DYNSYM", elf->dynsymtab,  in e64_save_symtab()
444 	elf->num_dynsyms = shdr[tab_idx].sh_size / sizeof(Elf64_Sym);  in e64_save_symtab()
446 	if (str_idx >= elf->e_shnum)  in e64_save_symtab()
449 	elf->dynstr = (void *)(vaddr_t)(shdr[str_idx].sh_addr + elf->load_addr);  in e64_save_symtab()
450 	check_range(elf, ".dynstr/STRTAB", elf->dynstr, shdr[str_idx].sh_size);  in e64_save_symtab()
452 	elf->dynstr_size = shdr[str_idx].sh_size;  in e64_save_symtab()
455 static void save_symtab(struct ta_elf *elf)  in save_symtab()  argument
459 	if (elf->is_32bit) {  in save_symtab()
460 		Elf32_Shdr *shdr = elf->shdr;  in save_symtab()
462 		for (n = 0; n < elf->e_shnum; n++) {  in save_symtab()
464 				e32_save_symtab(elf, n);  in save_symtab()
469 		Elf64_Shdr *shdr = elf->shdr;  in save_symtab()
471 		for (n = 0; n < elf->e_shnum; n++) {  in save_symtab()
473 				e64_save_symtab(elf, n);  in save_symtab()
480 	save_hashtab(elf);  in save_symtab()
481 	save_soname(elf);  in save_symtab()
484 static void init_elf(struct ta_elf *elf)  in init_elf()  argument
491 	res = sys_open_ta_bin(&elf->uuid, &elf->handle);  in init_elf()
493 		err(res, "sys_open_ta_bin(%pUl)", (void *)&elf->uuid);  in init_elf()
499 	if (!elf->is_main)  in init_elf()
501 	res = sys_map_ta_bin(&va, SMALL_PAGE_SIZE, flags, elf->handle, 0, 0, 0);  in init_elf()
504 	elf->ehdr_addr = va;  in init_elf()
505 	if (!elf->is_main) {  in init_elf()
506 		elf->load_addr = va;  in init_elf()
507 		elf->max_addr = va + SMALL_PAGE_SIZE;  in init_elf()
508 		elf->max_offs = SMALL_PAGE_SIZE;  in init_elf()
514 	res = e32_parse_ehdr(elf, (void *)va);  in init_elf()
516 		res = e64_parse_ehdr(elf, (void *)va);  in init_elf()
520 	if (MUL_OVERFLOW(elf->e_phnum, elf->e_phentsize, &sz) ||  in init_elf()
521 	    ADD_OVERFLOW(sz, elf->e_phoff, &sz))  in init_elf()
527 	elf->phdr = (void *)(va + elf->e_phoff);  in init_elf()
540 static void add_segment(struct ta_elf *elf, size_t offset, size_t vaddr,  in add_segment()  argument
558 	TAILQ_INSERT_TAIL(&elf->segs, seg, link);  in add_segment()
561 static void parse_load_segments(struct ta_elf *elf)  in parse_load_segments()  argument
565 	if (elf->is_32bit) {  in parse_load_segments()
566 		Elf32_Phdr *phdr = elf->phdr;  in parse_load_segments()
568 		for (n = 0; n < elf->e_phnum; n++)  in parse_load_segments()
570 				add_segment(elf, phdr[n].p_offset,  in parse_load_segments()
575 				elf->exidx_start = phdr[n].p_vaddr;  in parse_load_segments()
576 				elf->exidx_size = phdr[n].p_filesz;  in parse_load_segments()
578 				assign_tls_mod_id(elf);  in parse_load_segments()
581 		Elf64_Phdr *phdr = elf->phdr;  in parse_load_segments()
583 		for (n = 0; n < elf->e_phnum; n++)  in parse_load_segments()
585 				add_segment(elf, phdr[n].p_offset,  in parse_load_segments()
590 				elf->tls_start = phdr[n].p_vaddr;  in parse_load_segments()
591 				elf->tls_filesz = phdr[n].p_filesz;  in parse_load_segments()
592 				elf->tls_memsz = phdr[n].p_memsz;  in parse_load_segments()
595 				elf->prop_start = phdr[n].p_vaddr;  in parse_load_segments()
596 				elf->prop_align = phdr[n].p_align;  in parse_load_segments()
597 				elf->prop_memsz = phdr[n].p_memsz;  in parse_load_segments()
602 static void copy_remapped_to(struct ta_elf *elf, const struct segment *seg)  in copy_remapped_to()  argument
604 	uint8_t *dst = (void *)(seg->vaddr + elf->load_addr);  in copy_remapped_to()
609 	if (offs < elf->max_offs) {  in copy_remapped_to()
610 		n = MIN(elf->max_offs - offs, num_bytes);  in copy_remapped_to()
611 		memcpy(dst, (void *)(elf->max_addr + offs - elf->max_offs), n);  in copy_remapped_to()
619 						      elf->handle, offs);  in copy_remapped_to()
623 		elf->max_offs += offs;  in copy_remapped_to()
627 static void adjust_segments(struct ta_elf *elf)  in adjust_segments()  argument
636 	TAILQ_FOREACH(seg, &elf->segs, link) {  in adjust_segments()
658 	seg = TAILQ_FIRST(&elf->segs);  in adjust_segments()
695 			TAILQ_REMOVE(&elf->segs, seg, link);  in adjust_segments()
723 static void populate_segments_legacy(struct ta_elf *elf)  in populate_segments_legacy()  argument
729 	assert(elf->is_legacy);  in populate_segments_legacy()
730 	TAILQ_FOREACH(seg, &elf->segs, link) {  in populate_segments_legacy()
731 		struct segment *last_seg = TAILQ_LAST(&elf->segs, segment_head);  in populate_segments_legacy()
736 		if (!elf->load_addr)  in populate_segments_legacy()
739 			va = seg->vaddr + elf->load_addr;  in populate_segments_legacy()
750 					   elf->handle, seg->offset);  in populate_segments_legacy()
754 		if (!elf->load_addr)  in populate_segments_legacy()
755 			elf->load_addr = va;  in populate_segments_legacy()
756 		elf->max_addr = va + num_bytes;  in populate_segments_legacy()
757 		elf->max_offs = seg->offset + seg->filesz;  in populate_segments_legacy()
787 static void populate_segments(struct ta_elf *elf)  in populate_segments()  argument
794 	assert(!elf->is_legacy);  in populate_segments()
795 	TAILQ_FOREACH(seg, &elf->segs, link) {  in populate_segments()
796 		struct segment *last_seg = TAILQ_LAST(&elf->segs, segment_head);  in populate_segments()
804 			assert(elf->load_addr);  in populate_segments()
805 			va = rounddown(elf->load_addr + seg->vaddr);  in populate_segments()
806 			assert(va >= elf->max_addr);  in populate_segments()
811 			copy_remapped_to(elf, seg);  in populate_segments()
812 			elf->max_addr = va + num_bytes;  in populate_segments()
820 			if (offset < elf->max_offs) {  in populate_segments()
841 					va = elf->max_addr;  in populate_segments()
850 					elf->max_addr = roundup(va + num_bytes);  in populate_segments()
861 			if (!elf->load_addr) {  in populate_segments()
870 				va = vaddr + elf->load_addr;  in populate_segments()
892 							   elf->handle, offset);  in populate_segments()
900 						     elf->handle, offset,  in populate_segments()
904 							     elf->handle,  in populate_segments()
911 			if (!elf->load_addr)  in populate_segments()
912 				elf->load_addr = va;  in populate_segments()
913 			elf->max_addr = roundup(va + memsz);  in populate_segments()
914 			elf->max_offs += filesz;  in populate_segments()
919 static void ta_elf_add_bti(struct ta_elf *elf)  in ta_elf_add_bti()  argument
925 	TAILQ_FOREACH(seg, &elf->segs, link) {  in ta_elf_add_bti()
926 		vaddr_t va = elf->load_addr + seg->vaddr;  in ta_elf_add_bti()
936 static void parse_property_segment(struct ta_elf *elf)  in parse_property_segment()  argument
939 	size_t align = elf->prop_align;  in parse_property_segment()
946 	if (!IS_ENABLED(CFG_TA_BTI) || !elf->prop_start)  in parse_property_segment()
949 	check_phdr_in_range(elf, PT_GNU_PROPERTY, elf->prop_start,  in parse_property_segment()
950 			    elf->prop_memsz);  in parse_property_segment()
952 	va = elf->load_addr + elf->prop_start;  in parse_property_segment()
956 	if (elf->prop_memsz < sizeof(*note) + sizeof(ELF_NOTE_GNU))  in parse_property_segment()
967 	if (desc_offset > elf->prop_memsz ||  in parse_property_segment()
968 	    ROUNDUP2(desc_offset + note->n_descsz, align) > elf->prop_memsz)  in parse_property_segment()
991 				elf->bti_enabled = true;  in parse_property_segment()
999 static void map_segments(struct ta_elf *elf)  in map_segments()  argument
1003 	parse_load_segments(elf);  in map_segments()
1004 	adjust_segments(elf);  in map_segments()
1005 	if (TAILQ_FIRST(&elf->segs)->offset < SMALL_PAGE_SIZE) {  in map_segments()
1007 		size_t sz = elf->max_addr - elf->load_addr;  in map_segments()
1008 		struct segment *seg = TAILQ_LAST(&elf->segs, segment_head);  in map_segments()
1015 		assert(!elf->is_main);  in map_segments()
1022 		res = sys_remap(elf->load_addr, &va, sz, pad_begin,  in map_segments()
1025 			res = sys_remap(elf->load_addr, &va, sz, 0,  in map_segments()
1029 		elf->ehdr_addr = va;  in map_segments()
1030 		elf->load_addr = va;  in map_segments()
1031 		elf->max_addr = va + sz;  in map_segments()
1032 		elf->phdr = (void *)(va + elf->e_phoff);  in map_segments()
1036 static void add_deps_from_segment(struct ta_elf *elf, unsigned int type,  in add_deps_from_segment()  argument
1051 	check_phdr_in_range(elf, type, addr, memsz);  in add_deps_from_segment()
1053 	if (elf->is_32bit)  in add_deps_from_segment()
1062 		read_dyn(elf, addr, n, &tag, &val);  in add_deps_from_segment()
1064 			str_tab = (char *)(val + elf->load_addr);  in add_deps_from_segment()
1068 	check_range(elf, ".dynstr/STRTAB", str_tab, str_tab_sz);  in add_deps_from_segment()
1073 		read_dyn(elf, addr, n, &tag, &val);  in add_deps_from_segment()
1086 static void add_dependencies(struct ta_elf *elf)  in add_dependencies()  argument
1090 	if (elf->is_32bit) {  in add_dependencies()
1091 		Elf32_Phdr *phdr = elf->phdr;  in add_dependencies()
1093 		for (n = 0; n < elf->e_phnum; n++)  in add_dependencies()
1094 			add_deps_from_segment(elf, phdr[n].p_type,  in add_dependencies()
1097 		Elf64_Phdr *phdr = elf->phdr;  in add_dependencies()
1099 		for (n = 0; n < elf->e_phnum; n++)  in add_dependencies()
1100 			add_deps_from_segment(elf, phdr[n].p_type,  in add_dependencies()
1105 static void copy_section_headers(struct ta_elf *elf)  in copy_section_headers()  argument
1111 	if (MUL_OVERFLOW(elf->e_shnum, elf->e_shentsize, &sz))  in copy_section_headers()
1114 	elf->shdr = malloc(sz);  in copy_section_headers()
1115 	if (!elf->shdr)  in copy_section_headers()
1123 	if (elf->e_shoff < SMALL_PAGE_SIZE) {  in copy_section_headers()
1124 		assert(!elf->is_main);  in copy_section_headers()
1125 		offs = MIN(SMALL_PAGE_SIZE - elf->e_shoff, sz);  in copy_section_headers()
1126 		memcpy(elf->shdr, (void *)(elf->load_addr + elf->e_shoff),  in copy_section_headers()
1131 		res = sys_copy_from_ta_bin((uint8_t *)elf->shdr + offs,  in copy_section_headers()
1132 					   sz - offs, elf->handle,  in copy_section_headers()
1133 					   elf->e_shoff + offs);  in copy_section_headers()
1139 static void close_handle(struct ta_elf *elf)  in close_handle()  argument
1141 	TEE_Result res = sys_close_ta_bin(elf->handle);  in close_handle()
1145 	elf->handle = -1;  in close_handle()
1148 static void clean_elf_load_main(struct ta_elf *elf)  in clean_elf_load_main()  argument
1155 	res = sys_unmap(elf->ehdr_addr, SMALL_PAGE_SIZE);  in clean_elf_load_main()
1159 	while (!TAILQ_EMPTY(&elf->segs)) {  in clean_elf_load_main()
1160 		struct segment *seg = TAILQ_FIRST(&elf->segs);  in clean_elf_load_main()
1164 		va = rounddown(elf->load_addr + seg->vaddr);  in clean_elf_load_main()
1175 		TAILQ_REMOVE(&elf->segs, seg, link);  in clean_elf_load_main()
1179 	free(elf->shdr);  in clean_elf_load_main()
1180 	memset(&elf->is_32bit, 0,  in clean_elf_load_main()
1181 	       (vaddr_t)&elf->uuid - (vaddr_t)&elf->is_32bit);  in clean_elf_load_main()
1183 	TAILQ_INIT(&elf->segs);  in clean_elf_load_main()
1192 static void set_tls_offset(struct ta_elf *elf)  in set_tls_offset()  argument
1196 	if (!elf->tls_start)  in set_tls_offset()
1200 	elf->tls_tcb_offs = next_offs;  in set_tls_offset()
1201 	next_offs += elf->tls_memsz;  in set_tls_offset()
1204 static void set_tls_offset(struct ta_elf *elf __unused) {}  in set_tls_offset()
1207 static void load_main(struct ta_elf *elf)  in load_main()  argument
1211 	init_elf(elf);  in load_main()
1212 	map_segments(elf);  in load_main()
1213 	populate_segments(elf);  in load_main()
1214 	add_dependencies(elf);  in load_main()
1215 	copy_section_headers(elf);  in load_main()
1216 	save_symtab(elf);  in load_main()
1217 	close_handle(elf);  in load_main()
1218 	set_tls_offset(elf);  in load_main()
1219 	parse_property_segment(elf);  in load_main()
1220 	if (elf->bti_enabled)  in load_main()
1221 		ta_elf_add_bti(elf);  in load_main()
1223 	if (!ta_elf_resolve_sym("ta_head", &va, NULL, elf))  in load_main()
1224 		elf->head = (struct ta_head *)va;  in load_main()
1226 		elf->head = (struct ta_head *)elf->load_addr;  in load_main()
1227 	if (elf->head->depr_entry != UINT64_MAX) {  in load_main()
1243 		DMSG("Reloading TA %pUl as legacy TA", (void *)&elf->uuid);  in load_main()
1244 		clean_elf_load_main(elf);  in load_main()
1245 		elf->is_legacy = true;  in load_main()
1246 		init_elf(elf);  in load_main()
1247 		map_segments(elf);  in load_main()
1248 		populate_segments_legacy(elf);  in load_main()
1249 		add_dependencies(elf);  in load_main()
1250 		copy_section_headers(elf);  in load_main()
1251 		save_symtab(elf);  in load_main()
1252 		close_handle(elf);  in load_main()
1253 		elf->head = (struct ta_head *)elf->load_addr;  in load_main()
1258 		if (elf->head->depr_entry == UINT64_MAX)  in load_main()
1261 			    (void *)&elf->uuid);  in load_main()
1269 	struct ta_elf *elf = queue_elf(uuid);  in ta_elf_load_main()  local
1273 	assert(elf);  in ta_elf_load_main()
1274 	elf->is_main = true;  in ta_elf_load_main()
1276 	load_main(elf);  in ta_elf_load_main()
1278 	*is_32bit = elf->is_32bit;  in ta_elf_load_main()
1279 	res = sys_map_zi(elf->head->stack_size, 0, &va, 0, 0);  in ta_elf_load_main()
1283 	if (elf->head->flags & ~TA_FLAGS_MASK)  in ta_elf_load_main()
1285 		    elf->head->flags & ~TA_FLAGS_MASK);  in ta_elf_load_main()
1287 	*ta_flags = elf->head->flags;  in ta_elf_load_main()
1288 	*sp = va + elf->head->stack_size;  in ta_elf_load_main()
1290 	ta_stack_size = elf->head->stack_size;  in ta_elf_load_main()
1295 	struct ta_elf *elf = TAILQ_FIRST(&main_elf_queue);  in ta_elf_finalize_load_main()  local
1298 	assert(elf->is_main);  in ta_elf_finalize_load_main()
1300 	res = ta_elf_set_init_fini_info_compat(elf->is_32bit);  in ta_elf_finalize_load_main()
1303 	res = ta_elf_set_elf_phdr_info(elf->is_32bit);  in ta_elf_finalize_load_main()
1307 	if (elf->is_legacy)  in ta_elf_finalize_load_main()
1308 		*entry = elf->head->depr_entry;  in ta_elf_finalize_load_main()
1310 		*entry = elf->e_entry + elf->load_addr;  in ta_elf_finalize_load_main()
1312 	*load_addr = elf->load_addr;  in ta_elf_finalize_load_main()
1316 void ta_elf_load_dependency(struct ta_elf *elf, bool is_32bit)  in ta_elf_load_dependency()  argument
1318 	if (elf->is_main)  in ta_elf_load_dependency()
1321 	init_elf(elf);  in ta_elf_load_dependency()
1322 	if (elf->is_32bit != is_32bit)  in ta_elf_load_dependency()
1324 		    (void *)&elf->uuid, elf->is_32bit ? "32" : "64",  in ta_elf_load_dependency()
1327 	map_segments(elf);  in ta_elf_load_dependency()
1328 	populate_segments(elf);  in ta_elf_load_dependency()
1329 	add_dependencies(elf);  in ta_elf_load_dependency()
1330 	copy_section_headers(elf);  in ta_elf_load_dependency()
1331 	save_symtab(elf);  in ta_elf_load_dependency()
1332 	close_handle(elf);  in ta_elf_load_dependency()
1333 	set_tls_offset(elf);  in ta_elf_load_dependency()
1334 	parse_property_segment(elf);  in ta_elf_load_dependency()
1335 	if (elf->bti_enabled)  in ta_elf_load_dependency()
1336 		ta_elf_add_bti(elf);  in ta_elf_load_dependency()
1339 void ta_elf_finalize_mappings(struct ta_elf *elf)  in ta_elf_finalize_mappings()  argument
1344 	if (!elf->is_legacy)  in ta_elf_finalize_mappings()
1347 	TAILQ_FOREACH(seg, &elf->segs, link) {  in ta_elf_finalize_mappings()
1348 		vaddr_t va = elf->load_addr + seg->vaddr;  in ta_elf_finalize_mappings()
1415 			      struct ta_elf **elf, struct segment **seg,  in get_next_in_order()  argument
1425 	assert(elf && seg && elf_idx);  in get_next_in_order()
1426 	e = *elf;  in get_next_in_order()
1455 	*elf = e;  in get_next_in_order()
1466 	struct ta_elf *elf = NULL;  in ta_elf_print_mappings()  local
1476 	get_next_in_order(elf_queue, &elf, &seg, &elf_idx);  in ta_elf_print_mappings()
1484 			va = rounddown(seg->vaddr + elf->load_addr);  in ta_elf_print_mappings()
1543 		if (!get_next_in_order(elf_queue, &elf, &seg, &elf_idx))  in ta_elf_print_mappings()
1548 	TAILQ_FOREACH(elf, elf_queue, link) {  in ta_elf_print_mappings()
1551 			      elf_idx, (void *)&elf->uuid, 8, elf->load_addr);  in ta_elf_print_mappings()
1563 	struct ta_elf *elf = NULL;  in find_exidx()  local
1566 	TAILQ_FOREACH(elf, &main_elf_queue, link) {  in find_exidx()
1567 		if (addr < elf->load_addr)  in find_exidx()
1569 		a = addr - elf->load_addr;  in find_exidx()
1570 		TAILQ_FOREACH(seg, &elf->segs, link) {  in find_exidx()
1574 				*idx_start = elf->exidx_start + elf->load_addr;  in find_exidx()
1575 				*idx_end = elf->exidx_start + elf->load_addr +  in find_exidx()
1576 					   elf->exidx_size;  in find_exidx()
1615 	struct ta_elf *elf = NULL;  in ta_elf_add_library()  local
1624 	for (elf = lib; elf; elf = TAILQ_NEXT(elf, link))  in ta_elf_add_library()
1625 		ta_elf_load_dependency(elf, ta->is_32bit);  in ta_elf_add_library()
1627 	for (elf = lib; elf; elf = TAILQ_NEXT(elf, link)) {  in ta_elf_add_library()
1628 		ta_elf_relocate(elf);  in ta_elf_add_library()
1629 		ta_elf_finalize_mappings(elf);  in ta_elf_add_library()
1632 	for (elf = lib; elf; elf = TAILQ_NEXT(elf, link))  in ta_elf_add_library()
1634 		     (void *)&elf->uuid, elf->load_addr);  in ta_elf_add_library()
1644 static void get_init_fini_array(struct ta_elf *elf, unsigned int type,  in get_init_fini_array()  argument
1658 	check_phdr_in_range(elf, type, addr, memsz);  in get_init_fini_array()
1660 	if (elf->is_32bit) {  in get_init_fini_array()
1672 		read_dyn(elf, addr, n, &tag, &val);  in get_init_fini_array()
1674 			*init = val + elf->load_addr;  in get_init_fini_array()
1676 			*fini = val + elf->load_addr;  in get_init_fini_array()
1685 static void elf_get_init_fini_array(struct ta_elf *elf, vaddr_t *init,  in elf_get_init_fini_array()  argument
1691 	if (elf->is_32bit) {  in elf_get_init_fini_array()
1692 		Elf32_Phdr *phdr = elf->phdr;  in elf_get_init_fini_array()
1694 		for (n = 0; n < elf->e_phnum; n++) {  in elf_get_init_fini_array()
1696 				get_init_fini_array(elf, phdr[n].p_type,  in elf_get_init_fini_array()
1705 		Elf64_Phdr *phdr = elf->phdr;  in elf_get_init_fini_array()
1707 		for (n = 0; n < elf->e_phnum; n++) {  in elf_get_init_fini_array()
1709 				get_init_fini_array(elf, phdr[n].p_type,  in elf_get_init_fini_array()
1802 static void fill_ifs(vaddr_t va, size_t idx, struct ta_elf *elf, bool is_32bit)  in fill_ifs()  argument
1820 		elf_get_init_fini_array(elf, &init, &init_cnt, &fini,  in fill_ifs()
1837 		elf_get_init_fini_array(elf, &init, &init_cnt, &fini,  in fill_ifs()
1858 	struct ta_elf *elf = NULL;  in ta_elf_set_init_fini_info_compat()  local
1879 	TAILQ_FOREACH(elf, &main_elf_queue, link)  in ta_elf_set_init_fini_info_compat()
1890 	TAILQ_FOREACH(elf, &main_elf_queue, link) {  in ta_elf_set_init_fini_info_compat()
1891 		fill_ifs(info_va, cnt, elf, is_32bit);  in ta_elf_set_init_fini_info_compat()
1938 static void fill_elf_phdr_info(vaddr_t va, size_t idx, struct ta_elf *elf,  in fill_elf_phdr_info()  argument
1950 		dlpi32->dlpi_addr = elf->load_addr;  in fill_elf_phdr_info()
1951 		if (elf->soname)  in fill_elf_phdr_info()
1952 			dlpi32->dlpi_name = (vaddr_t)elf->soname;  in fill_elf_phdr_info()
1955 		dlpi32->dlpi_phdr = (vaddr_t)elf->phdr;  in fill_elf_phdr_info()
1956 		dlpi32->dlpi_phnum = elf->e_phnum;  in fill_elf_phdr_info()
1959 		dlpi32->dlpi_tls_modid = elf->tls_mod_id;  in fill_elf_phdr_info()
1960 		dlpi32->dlpi_tls_data = elf->tls_start;  in fill_elf_phdr_info()
1965 		dlpi->dlpi_addr = elf->load_addr;  in fill_elf_phdr_info()
1966 		if (elf->soname)  in fill_elf_phdr_info()
1967 			dlpi->dlpi_name = elf->soname;  in fill_elf_phdr_info()
1970 		dlpi->dlpi_phdr = elf->phdr;  in fill_elf_phdr_info()
1971 		dlpi->dlpi_phnum = elf->e_phnum;  in fill_elf_phdr_info()
1974 		dlpi->dlpi_tls_modid = elf->tls_mod_id;  in fill_elf_phdr_info()
1975 		dlpi->dlpi_tls_data = (void *)elf->tls_start;  in fill_elf_phdr_info()
1984 	struct ta_elf *elf = NULL;  in ta_elf_set_elf_phdr_info()  local
2002 	TAILQ_FOREACH(elf, &main_elf_queue, link)  in ta_elf_set_elf_phdr_info()
2010 	TAILQ_FOREACH(elf, &main_elf_queue, link) {  in ta_elf_set_elf_phdr_info()
2011 		fill_elf_phdr_info(info_va, cnt, elf, is_32bit);  in ta_elf_set_elf_phdr_info()