Lines Matching refs:objp
333 static unsigned long long *dbg_redzone1(struct kmem_cache *cachep, void *objp) in dbg_redzone1() argument
336 return (unsigned long long*) (objp + obj_offset(cachep) - in dbg_redzone1()
340 static unsigned long long *dbg_redzone2(struct kmem_cache *cachep, void *objp) in dbg_redzone2() argument
344 return (unsigned long long *)(objp + cachep->size - in dbg_redzone2()
347 return (unsigned long long *) (objp + cachep->size - in dbg_redzone2()
351 static void **dbg_userword(struct kmem_cache *cachep, void *objp) in dbg_userword() argument
354 return (void **)(objp + cachep->size - BYTES_PER_WORD); in dbg_userword()
360 #define dbg_redzone1(cachep, objp) ({BUG(); (unsigned long long *)NULL;}) argument
361 #define dbg_redzone2(cachep, objp) ({BUG(); (unsigned long long *)NULL;}) argument
362 #define dbg_userword(cachep, objp) ({BUG(); (void **)NULL;}) argument
553 struct page *page, void *objp) in cache_free_pfmemalloc() argument
563 free_block(cachep, &objp, 1, page_node, &list); in cache_free_pfmemalloc()
593 static __always_inline void __free_one(struct array_cache *ac, void *objp) in __free_one() argument
597 WARN_ON_ONCE(ac->avail > 0 && ac->entry[ac->avail - 1] == objp)) in __free_one()
599 ac->entry[ac->avail++] = objp; in __free_one()
617 static inline int cache_free_alien(struct kmem_cache *cachep, void *objp) in cache_free_alien() argument
763 static int __cache_free_alien(struct kmem_cache *cachep, void *objp, in __cache_free_alien() argument
781 __free_one(ac, objp); in __cache_free_alien()
787 free_block(cachep, &objp, 1, page_node, &list); in __cache_free_alien()
794 static inline int cache_free_alien(struct kmem_cache *cachep, void *objp) in cache_free_alien() argument
796 int page_node = page_to_nid(virt_to_page(objp)); in cache_free_alien()
805 return __cache_free_alien(cachep, objp, node, page_node); in cache_free_alien()
1433 static void slab_kernel_map(struct kmem_cache *cachep, void *objp, int map) in slab_kernel_map() argument
1438 __kernel_map_pages(virt_to_page(objp), cachep->size / PAGE_SIZE, map); in slab_kernel_map()
1442 static inline void slab_kernel_map(struct kmem_cache *cachep, void *objp, in slab_kernel_map() argument
1488 static void print_objinfo(struct kmem_cache *cachep, void *objp, int lines) in print_objinfo() argument
1495 *dbg_redzone1(cachep, objp), in print_objinfo()
1496 *dbg_redzone2(cachep, objp)); in print_objinfo()
1500 pr_err("Last user: (%pSR)\n", *dbg_userword(cachep, objp)); in print_objinfo()
1501 realobj = (char *)objp + obj_offset(cachep); in print_objinfo()
1512 static void check_poison_obj(struct kmem_cache *cachep, void *objp) in check_poison_obj() argument
1521 realobj = (char *)objp + obj_offset(cachep); in check_poison_obj()
1536 print_objinfo(cachep, objp, 0); in check_poison_obj()
1555 struct page *page = virt_to_head_page(objp); in check_poison_obj()
1558 objnr = obj_to_index(cachep, page, objp); in check_poison_obj()
1560 objp = index_to_obj(cachep, page, objnr - 1); in check_poison_obj()
1561 realobj = (char *)objp + obj_offset(cachep); in check_poison_obj()
1563 print_objinfo(cachep, objp, 2); in check_poison_obj()
1566 objp = index_to_obj(cachep, page, objnr + 1); in check_poison_obj()
1567 realobj = (char *)objp + obj_offset(cachep); in check_poison_obj()
1569 print_objinfo(cachep, objp, 2); in check_poison_obj()
1587 void *objp = index_to_obj(cachep, page, i); in slab_destroy_debugcheck() local
1590 check_poison_obj(cachep, objp); in slab_destroy_debugcheck()
1591 slab_kernel_map(cachep, objp, 1); in slab_destroy_debugcheck()
1594 if (*dbg_redzone1(cachep, objp) != RED_INACTIVE) in slab_destroy_debugcheck()
1596 if (*dbg_redzone2(cachep, objp) != RED_INACTIVE) in slab_destroy_debugcheck()
2334 void *objp = index_to_obj(cachep, page, i); in cache_init_objs_debug() local
2337 *dbg_userword(cachep, objp) = NULL; in cache_init_objs_debug()
2340 *dbg_redzone1(cachep, objp) = RED_INACTIVE; in cache_init_objs_debug()
2341 *dbg_redzone2(cachep, objp) = RED_INACTIVE; in cache_init_objs_debug()
2350 objp + obj_offset(cachep)); in cache_init_objs_debug()
2351 cachep->ctor(objp + obj_offset(cachep)); in cache_init_objs_debug()
2353 cachep, objp + obj_offset(cachep)); in cache_init_objs_debug()
2357 if (*dbg_redzone2(cachep, objp) != RED_INACTIVE) in cache_init_objs_debug()
2359 if (*dbg_redzone1(cachep, objp) != RED_INACTIVE) in cache_init_objs_debug()
2364 poison_obj(cachep, objp, POISON_FREE); in cache_init_objs_debug()
2365 slab_kernel_map(cachep, objp, 0); in cache_init_objs_debug()
2486 void *objp; in cache_init_objs() local
2500 objp = index_to_obj(cachep, page, i); in cache_init_objs()
2501 objp = kasan_init_slab_obj(cachep, objp); in cache_init_objs()
2505 kasan_unpoison_object_data(cachep, objp); in cache_init_objs()
2506 cachep->ctor(objp); in cache_init_objs()
2507 kasan_poison_object_data(cachep, objp); in cache_init_objs()
2517 void *objp; in slab_get_obj() local
2519 objp = index_to_obj(cachep, page, get_free_obj(page, page->active)); in slab_get_obj()
2522 return objp; in slab_get_obj()
2526 struct page *page, void *objp) in slab_put_obj() argument
2528 unsigned int objnr = obj_to_index(cachep, page, objp); in slab_put_obj()
2536 cachep->name, objp); in slab_put_obj()
2543 page->freelist = objp + obj_offset(cachep); in slab_put_obj()
2675 static void kfree_debugcheck(const void *objp) in kfree_debugcheck() argument
2677 if (!virt_addr_valid(objp)) { in kfree_debugcheck()
2679 (unsigned long)objp); in kfree_debugcheck()
2706 static void *cache_free_debugcheck(struct kmem_cache *cachep, void *objp, in cache_free_debugcheck() argument
2712 BUG_ON(virt_to_cache(objp) != cachep); in cache_free_debugcheck()
2714 objp -= obj_offset(cachep); in cache_free_debugcheck()
2715 kfree_debugcheck(objp); in cache_free_debugcheck()
2716 page = virt_to_head_page(objp); in cache_free_debugcheck()
2719 verify_redzone_free(cachep, objp); in cache_free_debugcheck()
2720 *dbg_redzone1(cachep, objp) = RED_INACTIVE; in cache_free_debugcheck()
2721 *dbg_redzone2(cachep, objp) = RED_INACTIVE; in cache_free_debugcheck()
2724 *dbg_userword(cachep, objp) = (void *)caller; in cache_free_debugcheck()
2726 objnr = obj_to_index(cachep, page, objp); in cache_free_debugcheck()
2729 BUG_ON(objp != index_to_obj(cachep, page, objnr)); in cache_free_debugcheck()
2732 poison_obj(cachep, objp, POISON_FREE); in cache_free_debugcheck()
2733 slab_kernel_map(cachep, objp, 0); in cache_free_debugcheck()
2735 return objp; in cache_free_debugcheck()
2740 #define cache_free_debugcheck(x,objp,z) (objp) argument
2748 void *objp; in fixup_objfreelist_debug() local
2751 objp = next - obj_offset(cachep); in fixup_objfreelist_debug()
2753 poison_obj(cachep, objp, POISON_FREE); in fixup_objfreelist_debug()
2770 void **objp = page->freelist; in fixup_slab_list() local
2772 *objp = *list; in fixup_slab_list()
2773 *list = objp; in fixup_slab_list()
2991 gfp_t flags, void *objp, unsigned long caller) in cache_alloc_debugcheck_after() argument
2994 if (!objp) in cache_alloc_debugcheck_after()
2995 return objp; in cache_alloc_debugcheck_after()
2997 check_poison_obj(cachep, objp); in cache_alloc_debugcheck_after()
2998 slab_kernel_map(cachep, objp, 1); in cache_alloc_debugcheck_after()
2999 poison_obj(cachep, objp, POISON_INUSE); in cache_alloc_debugcheck_after()
3002 *dbg_userword(cachep, objp) = (void *)caller; in cache_alloc_debugcheck_after()
3005 if (*dbg_redzone1(cachep, objp) != RED_INACTIVE || in cache_alloc_debugcheck_after()
3006 *dbg_redzone2(cachep, objp) != RED_INACTIVE) { in cache_alloc_debugcheck_after()
3009 objp, *dbg_redzone1(cachep, objp), in cache_alloc_debugcheck_after()
3010 *dbg_redzone2(cachep, objp)); in cache_alloc_debugcheck_after()
3012 *dbg_redzone1(cachep, objp) = RED_ACTIVE; in cache_alloc_debugcheck_after()
3013 *dbg_redzone2(cachep, objp) = RED_ACTIVE; in cache_alloc_debugcheck_after()
3016 objp += obj_offset(cachep); in cache_alloc_debugcheck_after()
3018 cachep->ctor(objp); in cache_alloc_debugcheck_after()
3019 if ((unsigned long)objp & (arch_slab_minalign() - 1)) { in cache_alloc_debugcheck_after()
3020 pr_err("0x%px: not aligned to arch_slab_minalign()=%u\n", objp, in cache_alloc_debugcheck_after()
3023 return objp; in cache_alloc_debugcheck_after()
3026 #define cache_alloc_debugcheck_after(a,b,objp,d) (objp) argument
3031 void *objp; in ____cache_alloc() local
3039 objp = ac->entry[--ac->avail]; in ____cache_alloc()
3046 objp = cache_alloc_refill(cachep, flags); in ____cache_alloc()
3059 if (objp) in ____cache_alloc()
3061 return objp; in ____cache_alloc()
3266 void *objp; in __do_cache_alloc() local
3269 objp = alternate_node_alloc(cache, flags); in __do_cache_alloc()
3270 if (objp) in __do_cache_alloc()
3273 objp = ____cache_alloc(cache, flags); in __do_cache_alloc()
3279 if (!objp) in __do_cache_alloc()
3280 objp = ____cache_alloc_node(cache, flags, numa_mem_id()); in __do_cache_alloc()
3283 return objp; in __do_cache_alloc()
3299 void *objp; in slab_alloc() local
3308 objp = kfence_alloc(cachep, orig_size, flags); in slab_alloc()
3309 if (unlikely(objp)) in slab_alloc()
3314 objp = __do_cache_alloc(cachep, flags); in slab_alloc()
3316 objp = cache_alloc_debugcheck_after(cachep, flags, objp, caller); in slab_alloc()
3317 prefetchw(objp); in slab_alloc()
3321 slab_post_alloc_hook(cachep, objcg, flags, 1, &objp, init); in slab_alloc()
3322 return objp; in slab_alloc()
3339 void *objp; in free_block() local
3342 objp = objpp[i]; in free_block()
3344 page = virt_to_head_page(objp); in free_block()
3347 slab_put_obj(cachep, page, objp); in free_block()
3423 static __always_inline void __cache_free(struct kmem_cache *cachep, void *objp, in __cache_free() argument
3428 if (is_kfence_address(objp)) { in __cache_free()
3429 kmemleak_free_recursive(objp, cachep->flags); in __cache_free()
3430 memcg_slab_free_hook(cachep, &objp, 1); in __cache_free()
3431 __kfence_free(objp); in __cache_free()
3442 memset(objp, 0, cachep->object_size); in __cache_free()
3444 if (kasan_slab_free(cachep, objp, init)) in __cache_free()
3449 __kcsan_check_access(objp, cachep->object_size, in __cache_free()
3452 ___cache_free(cachep, objp, caller); in __cache_free()
3455 void ___cache_free(struct kmem_cache *cachep, void *objp, in ___cache_free() argument
3461 kmemleak_free_recursive(objp, cachep->flags); in ___cache_free()
3462 objp = cache_free_debugcheck(cachep, objp, caller); in ___cache_free()
3463 memcg_slab_free_hook(cachep, &objp, 1); in ___cache_free()
3472 if (nr_online_nodes > 1 && cache_free_alien(cachep, objp)) in ___cache_free()
3483 struct page *page = virt_to_head_page(objp); in ___cache_free()
3486 cache_free_pfmemalloc(cachep, page, objp); in ___cache_free()
3491 __free_one(ac, objp); in ___cache_free()
3539 void *objp = kfence_alloc(s, s->object_size, flags) ?: __do_cache_alloc(s, flags); in kmem_cache_alloc_bulk() local
3541 if (unlikely(!objp)) in kmem_cache_alloc_bulk()
3543 p[i] = objp; in kmem_cache_alloc_bulk()
3706 void kmem_cache_free(struct kmem_cache *cachep, void *objp) in kmem_cache_free() argument
3709 cachep = cache_from_obj(cachep, objp); in kmem_cache_free()
3714 debug_check_no_locks_freed(objp, cachep->object_size); in kmem_cache_free()
3716 debug_check_no_obj_freed(objp, cachep->object_size); in kmem_cache_free()
3717 __cache_free(cachep, objp, _RET_IP_); in kmem_cache_free()
3720 trace_kmem_cache_free(_RET_IP_, objp); in kmem_cache_free()
3731 void *objp = p[i]; in kmem_cache_free_bulk() local
3734 s = virt_to_cache(objp); in kmem_cache_free_bulk()
3736 s = cache_from_obj(orig_s, objp); in kmem_cache_free_bulk()
3740 debug_check_no_locks_freed(objp, s->object_size); in kmem_cache_free_bulk()
3742 debug_check_no_obj_freed(objp, s->object_size); in kmem_cache_free_bulk()
3744 __cache_free(s, objp, _RET_IP_); in kmem_cache_free_bulk()
3761 void kfree(const void *objp) in kfree() argument
3766 trace_kfree(_RET_IP_, objp); in kfree()
3768 if (unlikely(ZERO_OR_NULL_PTR(objp))) in kfree()
3771 kfree_debugcheck(objp); in kfree()
3772 c = virt_to_cache(objp); in kfree()
3777 debug_check_no_locks_freed(objp, c->object_size); in kfree()
3779 debug_check_no_obj_freed(objp, c->object_size); in kfree()
3780 __cache_free(c, (void *)objp, _RET_IP_); in kfree()
4214 size_t __ksize(const void *objp) in __ksize() argument
4219 BUG_ON(!objp); in __ksize()
4220 if (unlikely(objp == ZERO_SIZE_PTR)) in __ksize()
4223 c = virt_to_cache(objp); in __ksize()