1 // SPDX-License-Identifier: BSD-2-Clause 2 /* 3 * Copyright (c) 2014, STMicroelectronics International N.V. 4 * Copyright (c) 2015-2025, Linaro Limited. 5 */ 6 7 #define PROTOTYPES 8 9 /* 10 * BGET CONFIGURATION 11 * ================== 12 */ 13 /* #define BGET_ENABLE_ALL_OPTIONS */ 14 #ifdef BGET_ENABLE_OPTION 15 #define TestProg 20000 /* Generate built-in test program 16 if defined. The value specifies 17 how many buffer allocation attempts 18 the test program should make. */ 19 #endif 20 21 22 #ifdef __LP64__ 23 #define SizeQuant 16 24 #endif 25 #ifdef __ILP32__ 26 #define SizeQuant 8 27 #endif 28 /* Buffer allocation size quantum: 29 all buffers allocated are a 30 multiple of this size. This 31 MUST be a power of two. */ 32 33 #ifdef BGET_ENABLE_OPTION 34 #define BufDump 1 /* Define this symbol to enable the 35 bpoold() function which dumps the 36 buffers in a buffer pool. */ 37 38 #define BufValid 1 /* Define this symbol to enable the 39 bpoolv() function for validating 40 a buffer pool. */ 41 42 #define DumpData 1 /* Define this symbol to enable the 43 bufdump() function which allows 44 dumping the contents of an allocated 45 or free buffer. */ 46 47 #define BufStats 1 /* Define this symbol to enable the 48 bstats() function which calculates 49 the total free space in the buffer 50 pool, the largest available 51 buffer, and the total space 52 currently allocated. */ 53 54 #define FreeWipe 1 /* Wipe free buffers to a guaranteed 55 pattern of garbage to trip up 56 miscreants who attempt to use 57 pointers into released buffers. */ 58 59 #define BestFit 1 /* Use a best fit algorithm when 60 searching for space for an 61 allocation request. This uses 62 memory more efficiently, but 63 allocation will be much slower. */ 64 65 #define BECtl 1 /* Define this symbol to enable the 66 bectl() function for automatic 67 pool space control. */ 68 #endif 69 70 #ifdef MEM_DEBUG 71 #undef NDEBUG 72 #define DumpData 1 73 #define BufValid 1 74 #define FreeWipe 1 75 #endif 76 77 #ifdef CFG_WITH_STATS 78 #define BufStats 1 79 #endif 80 81 #include <compiler.h> 82 #include <config.h> 83 #include <malloc.h> 84 #include <memtag.h> 85 #include <pta_stats.h> 86 #include <stdbool.h> 87 #include <stdint.h> 88 #include <stdlib_ext.h> 89 #include <stdlib.h> 90 #include <string.h> 91 #include <trace.h> 92 #include <util.h> 93 94 #if defined(__KERNEL__) 95 /* Compiling for TEE Core */ 96 #include <kernel/asan.h> 97 #include <kernel/spinlock.h> 98 #include <kernel/unwind.h> 99 100 static void *memset_unchecked(void *s, int c, size_t n) 101 { 102 return asan_memset_unchecked(s, c, n); 103 } 104 105 static __maybe_unused void *memcpy_unchecked(void *dst, const void *src, 106 size_t n) 107 { 108 return asan_memcpy_unchecked(dst, src, n); 109 } 110 111 #else /*__KERNEL__*/ 112 /* Compiling for TA */ 113 114 static void *memset_unchecked(void *s, int c, size_t n) 115 { 116 return memset(s, c, n); 117 } 118 119 static __maybe_unused void *memcpy_unchecked(void *dst, const void *src, 120 size_t n) 121 { 122 return memcpy(dst, src, n); 123 } 124 125 #endif /*__KERNEL__*/ 126 127 #include "bget.c" /* this is ugly, but this is bget */ 128 129 struct malloc_pool { 130 void *buf; 131 size_t len; 132 }; 133 134 struct malloc_ctx { 135 struct bpoolset poolset; 136 struct malloc_pool *pool; 137 size_t pool_len; 138 #ifdef BufStats 139 struct pta_stats_alloc mstats; 140 #endif 141 #ifdef __KERNEL__ 142 unsigned int spinlock; 143 #endif 144 }; 145 146 #ifdef __KERNEL__ 147 148 static uint32_t malloc_lock(struct malloc_ctx *ctx) 149 { 150 return cpu_spin_lock_xsave(&ctx->spinlock); 151 } 152 153 static void malloc_unlock(struct malloc_ctx *ctx, uint32_t exceptions) 154 { 155 cpu_spin_unlock_xrestore(&ctx->spinlock, exceptions); 156 } 157 158 #else /* __KERNEL__ */ 159 160 static uint32_t malloc_lock(struct malloc_ctx *ctx __unused) 161 { 162 return 0; 163 } 164 165 static void malloc_unlock(struct malloc_ctx *ctx __unused, 166 uint32_t exceptions __unused) 167 { 168 } 169 170 #endif /* __KERNEL__ */ 171 172 #define DEFINE_CTX(name) struct malloc_ctx name = \ 173 { .poolset = { .freelist = { {0, 0}, \ 174 {&name.poolset.freelist, \ 175 &name.poolset.freelist}}}} 176 177 static DEFINE_CTX(malloc_ctx); 178 179 #ifdef CFG_NS_VIRTUALIZATION 180 static __nex_data DEFINE_CTX(nex_malloc_ctx); 181 #endif 182 183 static void print_oom(size_t req_size __maybe_unused, void *ctx __maybe_unused) 184 { 185 #if defined(__KERNEL__) && defined(CFG_CORE_DUMP_OOM) 186 EMSG("Memory allocation failed: size %zu context %p", req_size, ctx); 187 print_kernel_stack(); 188 #endif 189 } 190 191 /* Most of the stuff in this function is copied from bgetr() in bget.c */ 192 static __maybe_unused bufsize bget_buf_size(void *buf) 193 { 194 bufsize osize; /* Old size of buffer */ 195 struct bhead *b; 196 197 b = BH(((char *)buf) - sizeof(struct bhead)); 198 osize = -b->bsize; 199 #ifdef BECtl 200 if (osize == 0) { 201 /* Buffer acquired directly through acqfcn. */ 202 struct bdhead *bd; 203 204 bd = BDH(((char *)buf) - sizeof(struct bdhead)); 205 osize = bd->tsize - sizeof(struct bdhead) - bd->offs; 206 } else 207 #endif 208 osize -= sizeof(struct bhead); 209 assert(osize > 0); 210 return osize; 211 } 212 213 static void *maybe_tag_buf(uint8_t *buf, size_t hdr_size, size_t requested_size) 214 { 215 if (!buf) 216 return NULL; 217 218 COMPILE_TIME_ASSERT(MEMTAG_GRANULE_SIZE <= SizeQuant); 219 220 if (MEMTAG_IS_ENABLED) { 221 size_t sz = ROUNDUP(requested_size, MEMTAG_GRANULE_SIZE); 222 223 /* 224 * Allocated buffer can be larger than requested when 225 * allocating with memalign(), but we should never tag more 226 * than allocated. 227 */ 228 assert(bget_buf_size(buf) >= sz + hdr_size); 229 return memtag_set_random_tags(buf, sz + hdr_size); 230 } 231 232 #if defined(__KERNEL__) 233 if (IS_ENABLED(CFG_CORE_SANITIZE_KADDRESS)) 234 asan_tag_access(buf, buf + hdr_size + requested_size); 235 #endif 236 return buf; 237 } 238 239 static void *maybe_untag_buf(void *buf) 240 { 241 if (!buf) 242 return NULL; 243 244 if (MEMTAG_IS_ENABLED) { 245 size_t sz = 0; 246 247 memtag_assert_tag(buf); /* Trying to catch double free early */ 248 sz = bget_buf_size(memtag_strip_tag(buf)); 249 return memtag_set_tags(buf, sz, 0); 250 } 251 252 #if defined(__KERNEL__) 253 if (IS_ENABLED(CFG_CORE_SANITIZE_KADDRESS)) 254 asan_tag_heap_free(buf, (uint8_t *)buf + bget_buf_size(buf)); 255 #endif 256 return buf; 257 } 258 259 static void *strip_tag(void *buf) 260 { 261 if (MEMTAG_IS_ENABLED) 262 return memtag_strip_tag(buf); 263 return buf; 264 } 265 266 static void tag_asan_free(void *buf __maybe_unused, size_t len __maybe_unused) 267 { 268 #if defined(__KERNEL__) 269 asan_tag_heap_free(buf, (uint8_t *)buf + len); 270 #endif 271 } 272 273 #ifdef BufStats 274 275 static void *raw_malloc_return_hook(void *p, size_t hdr_size, 276 size_t requested_size, 277 struct malloc_ctx *ctx) 278 { 279 if (ctx->poolset.totalloc > ctx->mstats.max_allocated) 280 ctx->mstats.max_allocated = ctx->poolset.totalloc; 281 282 if (!p) { 283 ctx->mstats.num_alloc_fail++; 284 print_oom(requested_size, ctx); 285 if (requested_size > ctx->mstats.biggest_alloc_fail) { 286 ctx->mstats.biggest_alloc_fail = requested_size; 287 ctx->mstats.biggest_alloc_fail_used = 288 ctx->poolset.totalloc; 289 } 290 } 291 292 return maybe_tag_buf(p, hdr_size, MAX(SizeQuant, requested_size)); 293 } 294 295 static void gen_malloc_reset_stats(struct malloc_ctx *ctx) 296 { 297 uint32_t exceptions = malloc_lock(ctx); 298 299 ctx->mstats.max_allocated = 0; 300 ctx->mstats.num_alloc_fail = 0; 301 ctx->mstats.biggest_alloc_fail = 0; 302 ctx->mstats.biggest_alloc_fail_used = 0; 303 malloc_unlock(ctx, exceptions); 304 } 305 306 void malloc_reset_stats(void) 307 { 308 gen_malloc_reset_stats(&malloc_ctx); 309 } 310 311 static void gen_malloc_get_stats(struct malloc_ctx *ctx, 312 struct pta_stats_alloc *stats) 313 { 314 uint32_t exceptions = malloc_lock(ctx); 315 316 raw_malloc_get_stats(ctx, stats); 317 malloc_unlock(ctx, exceptions); 318 } 319 320 void malloc_get_stats(struct pta_stats_alloc *stats) 321 { 322 gen_malloc_get_stats(&malloc_ctx, stats); 323 } 324 325 #else /* BufStats */ 326 327 static void *raw_malloc_return_hook(void *p, size_t hdr_size, 328 size_t requested_size, 329 struct malloc_ctx *ctx ) 330 { 331 if (!p) 332 print_oom(requested_size, ctx); 333 334 return maybe_tag_buf(p, hdr_size, MAX(SizeQuant, requested_size)); 335 } 336 337 #endif /* BufStats */ 338 339 #ifdef BufValid 340 static void raw_malloc_validate_pools(struct malloc_ctx *ctx) 341 { 342 size_t n; 343 344 for (n = 0; n < ctx->pool_len; n++) 345 bpoolv(ctx->pool[n].buf); 346 } 347 #else 348 static void raw_malloc_validate_pools(struct malloc_ctx *ctx __unused) 349 { 350 } 351 #endif 352 353 struct bpool_iterator { 354 struct bfhead *next_buf; 355 size_t pool_idx; 356 }; 357 358 static void bpool_foreach_iterator_init(struct malloc_ctx *ctx, 359 struct bpool_iterator *iterator) 360 { 361 iterator->pool_idx = 0; 362 iterator->next_buf = BFH(ctx->pool[0].buf); 363 } 364 365 static bool bpool_foreach_pool(struct bpool_iterator *iterator, void **buf, 366 size_t *len, bool *isfree) 367 { 368 struct bfhead *b = iterator->next_buf; 369 bufsize bs = b->bh.bsize; 370 371 if (bs == ESent) 372 return false; 373 374 if (bs < 0) { 375 /* Allocated buffer */ 376 bs = -bs; 377 378 *isfree = false; 379 } else { 380 /* Free Buffer */ 381 *isfree = true; 382 383 /* Assert that the free list links are intact */ 384 assert(b->ql.blink->ql.flink == b); 385 assert(b->ql.flink->ql.blink == b); 386 } 387 388 *buf = (uint8_t *)b + sizeof(struct bhead); 389 *len = bs - sizeof(struct bhead); 390 391 iterator->next_buf = BFH((uint8_t *)b + bs); 392 return true; 393 } 394 395 static bool bpool_foreach(struct malloc_ctx *ctx, 396 struct bpool_iterator *iterator, void **buf) 397 { 398 while (true) { 399 size_t len; 400 bool isfree; 401 402 if (bpool_foreach_pool(iterator, buf, &len, &isfree)) { 403 if (isfree) 404 continue; 405 return true; 406 } 407 408 if ((iterator->pool_idx + 1) >= ctx->pool_len) 409 return false; 410 411 iterator->pool_idx++; 412 iterator->next_buf = BFH(ctx->pool[iterator->pool_idx].buf); 413 } 414 } 415 416 /* Convenience macro for looping over all allocated buffers */ 417 #define BPOOL_FOREACH(ctx, iterator, bp) \ 418 for (bpool_foreach_iterator_init((ctx),(iterator)); \ 419 bpool_foreach((ctx),(iterator), (bp));) 420 421 static void *raw_mem_alloc(uint32_t flags, void *ptr, size_t hdr_size, 422 size_t ftr_size, size_t alignment, size_t pl_nmemb, 423 size_t pl_size, struct malloc_ctx *ctx) 424 { 425 void *p = NULL; 426 bufsize s = 0; 427 428 raw_malloc_validate_pools(ctx); 429 430 if (!alignment || !IS_POWER_OF_TWO(alignment)) 431 return NULL; 432 433 /* Compute total size, excluding hdr_size */ 434 if (MUL_OVERFLOW(pl_nmemb, pl_size, &s)) 435 goto out; 436 if (ADD_OVERFLOW(s, ftr_size, &s)) 437 goto out; 438 439 /* BGET doesn't like 0 sized allocations */ 440 if (!s) 441 s++; 442 443 if ((flags & MAF_ZERO_INIT) && !ptr) 444 p = bgetz(alignment, hdr_size, s, &ctx->poolset); 445 else 446 p = bget(alignment, hdr_size, s, &ctx->poolset); 447 448 if (p && ptr) { 449 void *old_ptr = maybe_untag_buf(ptr); 450 bufsize old_sz = bget_buf_size(old_ptr); 451 bufsize new_sz = s + hdr_size; 452 453 if (old_sz < new_sz) { 454 memcpy_unchecked(p, old_ptr, old_sz); 455 /* User space reallocations are always zeroed */ 456 if (!IS_ENABLED2(__KERNEL__) || (flags & MAF_ZERO_INIT)) 457 memset_unchecked((uint8_t *)p + old_sz, 0, 458 new_sz - old_sz); 459 } else { 460 memcpy_unchecked(p, old_ptr, new_sz); 461 } 462 463 brel(old_ptr, &ctx->poolset, false /*!wipe*/); 464 } 465 out: 466 return raw_malloc_return_hook(p, hdr_size, pl_nmemb * pl_size, ctx); 467 } 468 469 void *raw_memalign(size_t hdr_size, size_t ftr_size, size_t alignment, 470 size_t pl_size, struct malloc_ctx *ctx) 471 { 472 return raw_mem_alloc(MAF_NULL, NULL, hdr_size, ftr_size, alignment, 1, 473 pl_size, ctx); 474 } 475 476 void *raw_malloc(size_t hdr_size, size_t ftr_size, size_t pl_size, 477 struct malloc_ctx *ctx) 478 { 479 return raw_mem_alloc(MAF_NULL, NULL, hdr_size, ftr_size, 1, 1, 480 pl_size, ctx); 481 } 482 483 void raw_free(void *ptr, struct malloc_ctx *ctx, bool wipe) 484 { 485 raw_malloc_validate_pools(ctx); 486 487 if (ptr) 488 brel(maybe_untag_buf(ptr), &ctx->poolset, wipe); 489 } 490 491 void *raw_calloc(size_t hdr_size, size_t ftr_size, size_t pl_nmemb, 492 size_t pl_size, struct malloc_ctx *ctx) 493 { 494 return raw_mem_alloc(MAF_ZERO_INIT, NULL, hdr_size, ftr_size, 1, 495 pl_nmemb, pl_size, ctx); 496 } 497 498 void *raw_realloc(void *ptr, size_t hdr_size, size_t ftr_size, 499 size_t pl_size, struct malloc_ctx *ctx) 500 { 501 return raw_mem_alloc(MAF_NULL, ptr, hdr_size, ftr_size, 1, 1, 502 pl_size, ctx); 503 } 504 505 struct mdbg_hdr { 506 const char *fname; 507 uint16_t line; 508 #ifdef __LP64__ 509 uint64_t pad; 510 #endif 511 uint32_t pl_size; 512 uint32_t magic; 513 }; 514 515 #define MDBG_HEADER_MAGIC 0xadadadad 516 #define MDBG_FOOTER_MAGIC 0xecececec 517 518 static size_t mdbg_get_ftr_size(size_t pl_size) 519 { 520 size_t ftr_pad = ROUNDUP(pl_size, sizeof(uint32_t)) - pl_size; 521 522 return ftr_pad + sizeof(uint32_t); 523 } 524 525 static uint32_t *mdbg_get_footer(struct mdbg_hdr *hdr) 526 { 527 uint32_t *footer; 528 529 footer = (uint32_t *)((uint8_t *)(hdr + 1) + hdr->pl_size + 530 mdbg_get_ftr_size(hdr->pl_size)); 531 footer--; 532 return strip_tag(footer); 533 } 534 535 static void mdbg_update_hdr(struct mdbg_hdr *hdr, const char *fname, 536 int lineno, size_t pl_size) 537 { 538 uint32_t *footer; 539 540 hdr->fname = fname; 541 hdr->line = lineno; 542 hdr->pl_size = pl_size; 543 hdr->magic = MDBG_HEADER_MAGIC; 544 545 footer = mdbg_get_footer(hdr); 546 *footer = MDBG_FOOTER_MAGIC; 547 } 548 549 static void assert_header(struct mdbg_hdr *hdr __maybe_unused) 550 { 551 assert(hdr->magic == MDBG_HEADER_MAGIC); 552 assert(*mdbg_get_footer(hdr) == MDBG_FOOTER_MAGIC); 553 } 554 555 static void *mem_alloc_unlocked(uint32_t flags, void *ptr, size_t alignment, 556 size_t nmemb, size_t size, const char *fname, 557 int lineno, struct malloc_ctx *ctx) 558 { 559 struct mdbg_hdr *hdr = NULL; 560 size_t ftr_size = 0; 561 size_t hdr_size = 0; 562 563 /* 564 * Check struct mdbg_hdr works with BGET_HDR_QUANTUM. 565 */ 566 static_assert((sizeof(struct mdbg_hdr) % BGET_HDR_QUANTUM) == 0); 567 568 if (IS_ENABLED2(ENABLE_MDBG)) { 569 if (ptr) { 570 hdr = ptr; 571 hdr--; 572 assert_header(hdr); 573 } 574 ftr_size = mdbg_get_ftr_size(nmemb * size); 575 hdr_size = sizeof(struct mdbg_hdr); 576 ptr = hdr; 577 } 578 579 ptr = raw_mem_alloc(flags, ptr, hdr_size, ftr_size, alignment, nmemb, 580 size, ctx); 581 582 if (IS_ENABLED2(ENABLE_MDBG) && ptr) { 583 hdr = ptr; 584 mdbg_update_hdr(hdr, fname, lineno, nmemb * size); 585 hdr++; 586 ptr = hdr; 587 } 588 589 return ptr; 590 } 591 592 static struct malloc_ctx *get_ctx(uint32_t flags __maybe_unused) 593 { 594 #ifdef CFG_NS_VIRTUALIZATION 595 if (flags & MAF_NEX) 596 return &nex_malloc_ctx; 597 #endif 598 return &malloc_ctx; 599 } 600 601 static void *mem_alloc(uint32_t flags, void *ptr, size_t alignment, 602 size_t nmemb, size_t size, const char *fname, int lineno) 603 { 604 struct malloc_ctx *ctx = get_ctx(flags); 605 uint32_t exceptions = 0; 606 void *p = NULL; 607 608 exceptions = malloc_lock(ctx); 609 p = mem_alloc_unlocked(flags, ptr, alignment, nmemb, size, fname, 610 lineno, ctx); 611 malloc_unlock(ctx, exceptions); 612 613 return p; 614 } 615 616 static void mem_free(void *ptr, bool wipe, uint32_t flags) 617 { 618 struct malloc_ctx *ctx = get_ctx(flags); 619 uint32_t exceptions = 0; 620 621 exceptions = malloc_lock(ctx); 622 623 if (IS_ENABLED2(ENABLE_MDBG) && ptr) { 624 struct mdbg_hdr *hdr = ptr; 625 626 hdr--; 627 assert_header(hdr); 628 hdr->magic = 0; 629 *mdbg_get_footer(hdr) = 0; 630 ptr = hdr; 631 } 632 633 raw_free(ptr, ctx, wipe); 634 635 malloc_unlock(ctx, exceptions); 636 } 637 638 static void *get_payload_start_size(void *raw_buf, size_t *size) 639 { 640 if (IS_ENABLED2(ENABLE_MDBG)) { 641 struct mdbg_hdr *hdr = raw_buf; 642 643 assert(bget_buf_size(hdr) >= hdr->pl_size); 644 *size = hdr->pl_size; 645 return hdr + 1; 646 } 647 648 *size = bget_buf_size(raw_buf); 649 return raw_buf; 650 } 651 652 /* For use in raw_malloc_add_pool() below */ 653 #define realloc_unlocked(ctx, ptr, size) \ 654 mem_alloc_unlocked(MAF_NULL, (ptr), 1, 1, (size), __FILE__, __LINE__, \ 655 (ctx)) 656 657 #ifdef ENABLE_MDBG 658 void *__mdbg_alloc(uint32_t flags, void *ptr, size_t alignment, size_t nmemb, 659 size_t size, const char *fname, int lineno) 660 { 661 return mem_alloc(flags, ptr, alignment, nmemb, size, fname, lineno); 662 } 663 664 static void gen_mdbg_check(struct malloc_ctx *ctx, int bufdump) 665 { 666 struct bpool_iterator itr; 667 void *b; 668 uint32_t exceptions = malloc_lock(ctx); 669 670 raw_malloc_validate_pools(ctx); 671 672 BPOOL_FOREACH(ctx, &itr, &b) { 673 struct mdbg_hdr *hdr = (struct mdbg_hdr *)b; 674 675 assert_header(hdr); 676 677 if (bufdump > 0) { 678 const char *fname = hdr->fname; 679 680 if (!fname) 681 fname = "unknown"; 682 683 IMSG("buffer: %d bytes %s:%d", 684 hdr->pl_size, fname, hdr->line); 685 } 686 } 687 688 malloc_unlock(ctx, exceptions); 689 } 690 691 void mdbg_check(int bufdump) 692 { 693 gen_mdbg_check(&malloc_ctx, bufdump); 694 } 695 #endif 696 697 /* 698 * If malloc debug is enabled, malloc() and friends are redirected by macros 699 * to __mdbg_alloc() etc. 700 * We still want to export the standard entry points in case they are referenced 701 * by the application, either directly or via external libraries. 702 */ 703 704 #undef malloc 705 void *malloc(size_t size) 706 { 707 return mem_alloc(MAF_NULL, NULL, 1, 1, size, __FILE__, __LINE__); 708 } 709 710 #undef calloc 711 void *calloc(size_t nmemb, size_t size) 712 { 713 return mem_alloc(MAF_ZERO_INIT, NULL, 1, nmemb, size, __FILE__, 714 __LINE__); 715 } 716 717 #undef realloc 718 void *realloc(void *ptr, size_t size) 719 { 720 return mem_alloc(MAF_NULL, ptr, 1, 1, size, __FILE__, __LINE__); 721 } 722 723 #undef memalign 724 void *memalign(size_t alignment, size_t size) 725 { 726 return mem_alloc(MAF_NULL, NULL, alignment, 1, size, __FILE__, 727 __LINE__); 728 } 729 730 #if __STDC_VERSION__ >= 201112L 731 #undef aligned_alloc 732 void *aligned_alloc(size_t alignment, size_t size) 733 { 734 if (size % alignment) 735 return NULL; 736 737 return mem_alloc(MAF_NULL, NULL, alignment, 1, size, __FILE__, 738 __LINE__); 739 } 740 #endif /* __STDC_VERSION__ */ 741 742 void free(void *ptr) 743 { 744 mem_free(ptr, false, MAF_NULL); 745 } 746 747 void free_wipe(void *ptr) 748 { 749 mem_free(ptr, true, MAF_NULL); 750 } 751 752 static void gen_malloc_add_pool(struct malloc_ctx *ctx, void *buf, size_t len) 753 { 754 uint32_t exceptions = malloc_lock(ctx); 755 756 raw_malloc_add_pool(ctx, buf, len); 757 malloc_unlock(ctx, exceptions); 758 } 759 760 static bool gen_malloc_buffer_is_within_alloced(struct malloc_ctx *ctx, 761 void *buf, size_t len) 762 { 763 uint32_t exceptions = malloc_lock(ctx); 764 bool ret = false; 765 766 ret = raw_malloc_buffer_is_within_alloced(ctx, buf, len); 767 malloc_unlock(ctx, exceptions); 768 769 return ret; 770 } 771 772 static bool gen_malloc_buffer_overlaps_heap(struct malloc_ctx *ctx, 773 void *buf, size_t len) 774 { 775 bool ret = false; 776 uint32_t exceptions = malloc_lock(ctx); 777 778 ret = raw_malloc_buffer_overlaps_heap(ctx, buf, len); 779 malloc_unlock(ctx, exceptions); 780 return ret; 781 } 782 783 size_t raw_malloc_get_ctx_size(void) 784 { 785 return sizeof(struct malloc_ctx); 786 } 787 788 void raw_malloc_init_ctx(struct malloc_ctx *ctx) 789 { 790 memset(ctx, 0, sizeof(*ctx)); 791 ctx->poolset.freelist.ql.flink = &ctx->poolset.freelist; 792 ctx->poolset.freelist.ql.blink = &ctx->poolset.freelist; 793 } 794 795 void raw_malloc_add_pool(struct malloc_ctx *ctx, void *buf, size_t len) 796 { 797 const size_t min_len = sizeof(struct bhead) + sizeof(struct bfhead); 798 uintptr_t start = (uintptr_t)buf; 799 uintptr_t end = start + len; 800 void *p = NULL; 801 size_t l = 0; 802 803 start = ROUNDUP(start, SizeQuant); 804 end = ROUNDDOWN(end, SizeQuant); 805 806 if (start > end || (end - start) < min_len) { 807 DMSG("Skipping too small pool"); 808 return; 809 } 810 811 /* First pool requires a bigger size */ 812 if (!ctx->pool_len && (end - start) < MALLOC_INITIAL_POOL_MIN_SIZE) { 813 DMSG("Skipping too small initial pool"); 814 return; 815 } 816 817 tag_asan_free((void *)start, end - start); 818 bpool((void *)start, end - start, &ctx->poolset); 819 l = ctx->pool_len + 1; 820 p = realloc_unlocked(ctx, ctx->pool, sizeof(struct malloc_pool) * l); 821 assert(p); 822 ctx->pool = p; 823 ctx->pool[ctx->pool_len].buf = (void *)start; 824 ctx->pool[ctx->pool_len].len = end - start; 825 #ifdef BufStats 826 ctx->mstats.size += ctx->pool[ctx->pool_len].len; 827 #endif 828 ctx->pool_len = l; 829 } 830 831 bool raw_malloc_buffer_overlaps_heap(struct malloc_ctx *ctx, 832 void *buf, size_t len) 833 { 834 uintptr_t buf_start = (uintptr_t)strip_tag(buf); 835 uintptr_t buf_end = buf_start + len; 836 size_t n = 0; 837 838 raw_malloc_validate_pools(ctx); 839 840 for (n = 0; n < ctx->pool_len; n++) { 841 uintptr_t pool_start = (uintptr_t)strip_tag(ctx->pool[n].buf); 842 uintptr_t pool_end = pool_start + ctx->pool[n].len; 843 844 if (buf_start > buf_end || pool_start > pool_end) 845 return true; /* Wrapping buffers, shouldn't happen */ 846 847 if ((buf_start >= pool_start && buf_start < pool_end) || 848 (buf_end > pool_start && buf_end < pool_end)) 849 return true; 850 } 851 852 return false; 853 } 854 855 bool raw_malloc_buffer_is_within_alloced(struct malloc_ctx *ctx, 856 void *buf, size_t len) 857 { 858 struct bpool_iterator itr = { }; 859 void *b = NULL; 860 uint8_t *start_buf = strip_tag(buf); 861 uint8_t *end_buf = start_buf + len; 862 863 raw_malloc_validate_pools(ctx); 864 865 /* Check for wrapping */ 866 if (start_buf > end_buf) 867 return false; 868 869 BPOOL_FOREACH(ctx, &itr, &b) { 870 uint8_t *start_b = NULL; 871 uint8_t *end_b = NULL; 872 size_t s = 0; 873 874 start_b = strip_tag(get_payload_start_size(b, &s)); 875 end_b = start_b + s; 876 if (start_buf >= start_b && end_buf <= end_b) 877 return true; 878 } 879 880 return false; 881 } 882 883 #ifdef CFG_WITH_STATS 884 void raw_malloc_get_stats(struct malloc_ctx *ctx, struct pta_stats_alloc *stats) 885 { 886 memcpy_unchecked(stats, &ctx->mstats, sizeof(*stats)); 887 stats->allocated = ctx->poolset.totalloc; 888 } 889 #endif 890 891 void malloc_add_pool(void *buf, size_t len) 892 { 893 gen_malloc_add_pool(&malloc_ctx, buf, len); 894 } 895 896 bool malloc_buffer_is_within_alloced(void *buf, size_t len) 897 { 898 return gen_malloc_buffer_is_within_alloced(&malloc_ctx, buf, len); 899 } 900 901 bool malloc_buffer_overlaps_heap(void *buf, size_t len) 902 { 903 return gen_malloc_buffer_overlaps_heap(&malloc_ctx, buf, len); 904 } 905 906 #ifdef CFG_NS_VIRTUALIZATION 907 908 #ifndef ENABLE_MDBG 909 910 void *nex_malloc(size_t size) 911 { 912 return mem_alloc(MAF_NEX, NULL, 1, 1, size, __FILE__, __LINE__); 913 } 914 915 void *nex_calloc(size_t nmemb, size_t size) 916 { 917 return mem_alloc(MAF_NEX | MAF_ZERO_INIT, NULL, 1, nmemb, size, 918 __FILE__, __LINE__); 919 } 920 921 void *nex_realloc(void *ptr, size_t size) 922 { 923 return mem_alloc(MAF_NEX, ptr, 1, 1, size, __FILE__, __LINE__); 924 } 925 926 void *nex_memalign(size_t alignment, size_t size) 927 { 928 return mem_alloc(MAF_NEX, NULL, alignment, 1, size, __FILE__, __LINE__); 929 } 930 931 #else /* ENABLE_MDBG */ 932 933 void nex_mdbg_check(int bufdump) 934 { 935 gen_mdbg_check(&nex_malloc_ctx, bufdump); 936 } 937 938 #endif /* ENABLE_MDBG */ 939 940 void nex_free(void *ptr) 941 { 942 mem_free(ptr, false, MAF_NEX); 943 } 944 945 void nex_malloc_add_pool(void *buf, size_t len) 946 { 947 gen_malloc_add_pool(&nex_malloc_ctx, buf, len); 948 } 949 950 bool nex_malloc_buffer_is_within_alloced(void *buf, size_t len) 951 { 952 return gen_malloc_buffer_is_within_alloced(&nex_malloc_ctx, buf, len); 953 } 954 955 bool nex_malloc_buffer_overlaps_heap(void *buf, size_t len) 956 { 957 return gen_malloc_buffer_overlaps_heap(&nex_malloc_ctx, buf, len); 958 } 959 960 #ifdef BufStats 961 962 void nex_malloc_reset_stats(void) 963 { 964 gen_malloc_reset_stats(&nex_malloc_ctx); 965 } 966 967 void nex_malloc_get_stats(struct pta_stats_alloc *stats) 968 { 969 gen_malloc_get_stats(&nex_malloc_ctx, stats); 970 } 971 972 #endif 973 974 #endif 975