Lines Matching +full:page +full:- +full:size
1 // SPDX-License-Identifier: GPL-2.0
34 #include <linux/fault-inject.h>
49 * 2. node->list_lock
50 * 3. slab_lock(page) (Only on some arches and for debugging)
59 * A. page->freelist -> List of object free in a page
60 * B. page->inuse -> Number of objects in use
61 * C. page->objects -> Number of objects in page
62 * D. page->frozen -> frozen state
66 * slab is the one who can perform list operations on the page. Other
69 * page's freelist.
97 * minimal so we rely on the page allocators per cpu caches for
100 * page->frozen The slab is frozen and exempt from list processing.
137 p += s->red_left_pad; in fixup_red_left()
154 * - Support PAGE_ALLOC_DEBUG. Should be easy to do.
156 * - Variable sizing of the per node arrays
197 #define OO_MASK ((1 << OO_SHIFT) - 1)
198 #define MAX_OBJS_PER_PAGE 32767 /* since page.objects is u15 */
226 * avoid this_cpu_add()'s irq-disable overhead. in stat()
228 raw_cpu_inc(s->cpu_slab->stat[si]); in stat()
238 * with an XOR of the address where the pointer is held and a per-cache
255 return (void *)((unsigned long)ptr ^ s->random ^ in freelist_ptr()
273 return freelist_dereference(s, object + s->offset); in get_freepointer()
278 prefetch(object + s->offset); in prefetch_freepointer()
290 freepointer_addr = (unsigned long)object + s->offset; in get_freepointer_safe()
297 unsigned long freeptr_addr = (unsigned long)object + s->offset; in set_freepointer()
310 __p < (__addr) + (__objects) * (__s)->size; \
311 __p += (__s)->size)
313 static inline unsigned int order_objects(unsigned int order, unsigned int size) in order_objects() argument
315 return ((unsigned int)PAGE_SIZE << order) / size; in order_objects()
319 unsigned int size) in oo_make() argument
322 (order << OO_SHIFT) + order_objects(order, size) in oo_make()
341 static __always_inline void slab_lock(struct page *page) in slab_lock() argument
343 VM_BUG_ON_PAGE(PageTail(page), page); in slab_lock()
344 bit_spin_lock(PG_locked, &page->flags); in slab_lock()
347 static __always_inline void slab_unlock(struct page *page) in slab_unlock() argument
349 VM_BUG_ON_PAGE(PageTail(page), page); in slab_unlock()
350 __bit_spin_unlock(PG_locked, &page->flags); in slab_unlock()
354 static inline bool __cmpxchg_double_slab(struct kmem_cache *s, struct page *page, in __cmpxchg_double_slab() argument
362 if (s->flags & __CMPXCHG_DOUBLE) { in __cmpxchg_double_slab()
363 if (cmpxchg_double(&page->freelist, &page->counters, in __cmpxchg_double_slab()
370 slab_lock(page); in __cmpxchg_double_slab()
371 if (page->freelist == freelist_old && in __cmpxchg_double_slab()
372 page->counters == counters_old) { in __cmpxchg_double_slab()
373 page->freelist = freelist_new; in __cmpxchg_double_slab()
374 page->counters = counters_new; in __cmpxchg_double_slab()
375 slab_unlock(page); in __cmpxchg_double_slab()
378 slab_unlock(page); in __cmpxchg_double_slab()
385 pr_info("%s %s: cmpxchg double redo ", n, s->name); in __cmpxchg_double_slab()
391 static inline bool cmpxchg_double_slab(struct kmem_cache *s, struct page *page, in cmpxchg_double_slab() argument
398 if (s->flags & __CMPXCHG_DOUBLE) { in cmpxchg_double_slab()
399 if (cmpxchg_double(&page->freelist, &page->counters, in cmpxchg_double_slab()
409 slab_lock(page); in cmpxchg_double_slab()
410 if (page->freelist == freelist_old && in cmpxchg_double_slab()
411 page->counters == counters_old) { in cmpxchg_double_slab()
412 page->freelist = freelist_new; in cmpxchg_double_slab()
413 page->counters = counters_new; in cmpxchg_double_slab()
414 slab_unlock(page); in cmpxchg_double_slab()
418 slab_unlock(page); in cmpxchg_double_slab()
426 pr_info("%s %s: cmpxchg double redo ", n, s->name); in cmpxchg_double_slab()
437 struct page *page) in __fill_map() argument
439 void *addr = page_address(page); in __fill_map()
442 bitmap_zero(obj_map, page->objects); in __fill_map()
444 for (p = page->freelist; p; p = get_freepointer(s, p)) in __fill_map()
449 * Determine a map of object in use on a page.
451 * Node listlock must be held to guarantee that the page does
454 static unsigned long *get_map(struct kmem_cache *s, struct page *page) in get_map() argument
461 __fill_map(object_map, s, page); in get_map()
474 if (s->flags & SLAB_RED_ZONE) in size_from_object()
475 return s->size - s->red_left_pad; in size_from_object()
477 return s->size; in size_from_object()
482 if (s->flags & SLAB_RED_ZONE) in restore_red_left()
483 p -= s->red_left_pad; in restore_red_left()
520 /* Verify that a pointer has an address that is valid within a slab page */
522 struct page *page, void *object) in check_valid_pointer() argument
529 base = page_address(page); in check_valid_pointer()
532 if (object < base || object >= base + page->objects * s->size || in check_valid_pointer()
533 (object - base) % s->size) { in check_valid_pointer()
554 return s->offset >= s->inuse; in freeptr_outside_object()
564 return s->inuse + sizeof(void *); in get_info_end()
566 return s->inuse; in get_info_end()
581 * a page to give track structure for each object, the function fn will
587 struct page *page, enum track_item alloc, in get_each_object_track() argument
596 if (!slub_debug || !(s->flags & SLAB_STORE_USER)) in get_each_object_track()
599 slab_lock(page); in get_each_object_track()
600 for_each_object(p, s, page_address(page), page->objects) { in get_each_object_track()
609 slab_unlock(page); in get_each_object_track()
624 nr_entries = stack_trace_save(kasan_reset_tag(p->addrs), in set_track()
629 p->addrs[nr_entries] = 0; in set_track()
633 p->addr = addr; in set_track()
634 p->cpu = smp_processor_id(); in set_track()
635 p->pid = current->pid; in set_track()
636 p->when = jiffies; in set_track()
644 if (!(s->flags & SLAB_STORE_USER)) in init_tracking()
653 if (!t->addr) in print_track()
657 s, (void *)t->addr, pr_time - t->when, t->cpu, t->pid); in print_track()
662 if (t->addrs[i]) in print_track()
663 pr_err("\t%pS\n", (void *)t->addrs[i]); in print_track()
673 if (!(s->flags & SLAB_STORE_USER)) in print_tracking()
680 static void print_page_info(struct page *page) in print_page_info() argument
683 page, page->objects, page->inuse, page->freelist, page->flags); in print_page_info()
696 pr_err("BUG %s (%s): %pV\n", s->name, print_tainted(), &vaf); in slab_bug()
697 pr_err("-----------------------------------------------------------------------------\n\n"); in slab_bug()
709 pr_err("FIX %s: %pV\n", s->name, &vaf); in slab_fix()
713 static bool freelist_corrupted(struct kmem_cache *s, struct page *page, in freelist_corrupted() argument
716 if ((s->flags & SLAB_CONSISTENCY_CHECKS) && in freelist_corrupted()
717 !check_valid_pointer(s, page, nextfree) && freelist) { in freelist_corrupted()
718 object_err(s, page, *freelist, "Freechain corrupt"); in freelist_corrupted()
727 static void print_trailer(struct kmem_cache *s, struct page *page, u8 *p) in print_trailer() argument
730 u8 *addr = page_address(page); in print_trailer()
734 print_page_info(page); in print_trailer()
737 p, p - addr, get_freepointer(s, p)); in print_trailer()
739 if (s->flags & SLAB_RED_ZONE) in print_trailer()
740 print_section(KERN_ERR, "Redzone ", p - s->red_left_pad, in print_trailer()
741 s->red_left_pad); in print_trailer()
743 print_section(KERN_ERR, "Bytes b4 ", p - 16, 16); in print_trailer()
746 min_t(unsigned int, s->object_size, PAGE_SIZE)); in print_trailer()
747 if (s->flags & SLAB_RED_ZONE) in print_trailer()
748 print_section(KERN_ERR, "Redzone ", p + s->object_size, in print_trailer()
749 s->inuse - s->object_size); in print_trailer()
753 if (s->flags & SLAB_STORE_USER) in print_trailer()
761 size_from_object(s) - off); in print_trailer()
766 void object_err(struct kmem_cache *s, struct page *page, in object_err() argument
770 print_trailer(s, page, object); in object_err()
774 static __printf(3, 4) void slab_err(struct kmem_cache *s, struct page *page, in slab_err() argument
784 print_page_info(page); in slab_err()
793 if (s->flags & SLAB_RED_ZONE) in init_object()
794 memset(p - s->red_left_pad, val, s->red_left_pad); in init_object()
796 if (s->flags & __OBJECT_POISON) { in init_object()
797 memset(p, POISON_FREE, s->object_size - 1); in init_object()
798 p[s->object_size - 1] = POISON_END; in init_object()
801 if (s->flags & SLAB_RED_ZONE) in init_object()
802 memset(p + s->object_size, val, s->inuse - s->object_size); in init_object()
808 slab_fix(s, "Restoring 0x%p-0x%p=0x%x\n", from, to - 1, data); in restore_bytes()
809 memset(from, data, to - from); in restore_bytes()
812 static int check_bytes_and_report(struct kmem_cache *s, struct page *page, in check_bytes_and_report() argument
818 u8 *addr = page_address(page); in check_bytes_and_report()
827 while (end > fault && end[-1] == value) in check_bytes_and_report()
828 end--; in check_bytes_and_report()
831 pr_err("INFO: 0x%p-0x%p @offset=%tu. First byte 0x%x instead of 0x%x\n", in check_bytes_and_report()
832 fault, end - 1, fault - addr, in check_bytes_and_report()
834 print_trailer(s, page, object); in check_bytes_and_report()
852 * object + s->object_size
860 * object + s->inuse
871 * object + s->size
872 * Nothing is used beyond s->size.
879 static int check_pad_bytes(struct kmem_cache *s, struct page *page, u8 *p) in check_pad_bytes() argument
883 if (s->flags & SLAB_STORE_USER) in check_pad_bytes()
892 return check_bytes_and_report(s, page, p, "Object padding", in check_pad_bytes()
893 p + off, POISON_INUSE, size_from_object(s) - off); in check_pad_bytes()
896 /* Check the pad bytes at the end of a slab page */
897 static int slab_pad_check(struct kmem_cache *s, struct page *page) in slab_pad_check() argument
906 if (!(s->flags & SLAB_POISON)) in slab_pad_check()
909 start = page_address(page); in slab_pad_check()
910 length = page_size(page); in slab_pad_check()
912 remainder = length % s->size; in slab_pad_check()
916 pad = end - remainder; in slab_pad_check()
922 while (end > fault && end[-1] == POISON_INUSE) in slab_pad_check()
923 end--; in slab_pad_check()
925 slab_err(s, page, "Padding overwritten. 0x%p-0x%p @offset=%tu", in slab_pad_check()
926 fault, end - 1, fault - start); in slab_pad_check()
933 static int check_object(struct kmem_cache *s, struct page *page, in check_object() argument
937 u8 *endobject = object + s->object_size; in check_object()
939 if (s->flags & SLAB_RED_ZONE) { in check_object()
940 if (!check_bytes_and_report(s, page, object, "Left Redzone", in check_object()
941 object - s->red_left_pad, val, s->red_left_pad)) in check_object()
944 if (!check_bytes_and_report(s, page, object, "Right Redzone", in check_object()
945 endobject, val, s->inuse - s->object_size)) in check_object()
948 if ((s->flags & SLAB_POISON) && s->object_size < s->inuse) { in check_object()
949 check_bytes_and_report(s, page, p, "Alignment padding", in check_object()
951 s->inuse - s->object_size); in check_object()
955 if (s->flags & SLAB_POISON) { in check_object()
956 if (val != SLUB_RED_ACTIVE && (s->flags & __OBJECT_POISON) && in check_object()
957 (!check_bytes_and_report(s, page, p, "Poison", p, in check_object()
958 POISON_FREE, s->object_size - 1) || in check_object()
959 !check_bytes_and_report(s, page, p, "End Poison", in check_object()
960 p + s->object_size - 1, POISON_END, 1))) in check_object()
965 check_pad_bytes(s, page, p); in check_object()
976 if (!check_valid_pointer(s, page, get_freepointer(s, p))) { in check_object()
977 object_err(s, page, p, "Freepointer corrupt"); in check_object()
989 static int check_slab(struct kmem_cache *s, struct page *page) in check_slab() argument
995 if (!PageSlab(page)) { in check_slab()
996 slab_err(s, page, "Not a valid slab page"); in check_slab()
1000 maxobj = order_objects(compound_order(page), s->size); in check_slab()
1001 if (page->objects > maxobj) { in check_slab()
1002 slab_err(s, page, "objects %u > max %u", in check_slab()
1003 page->objects, maxobj); in check_slab()
1006 if (page->inuse > page->objects) { in check_slab()
1007 slab_err(s, page, "inuse %u > max %u", in check_slab()
1008 page->inuse, page->objects); in check_slab()
1012 slab_pad_check(s, page); in check_slab()
1017 * Determine if a certain object on a page is on the freelist. Must hold the
1020 static int on_freelist(struct kmem_cache *s, struct page *page, void *search) in on_freelist() argument
1027 fp = page->freelist; in on_freelist()
1028 while (fp && nr <= page->objects) { in on_freelist()
1031 if (!check_valid_pointer(s, page, fp)) { in on_freelist()
1033 object_err(s, page, object, in on_freelist()
1037 slab_err(s, page, "Freepointer corrupt"); in on_freelist()
1038 page->freelist = NULL; in on_freelist()
1039 page->inuse = page->objects; in on_freelist()
1050 max_objects = order_objects(compound_order(page), s->size); in on_freelist()
1054 if (page->objects != max_objects) { in on_freelist()
1055 slab_err(s, page, "Wrong number of objects. Found %d but should be %d", in on_freelist()
1056 page->objects, max_objects); in on_freelist()
1057 page->objects = max_objects; in on_freelist()
1060 if (page->inuse != page->objects - nr) { in on_freelist()
1061 slab_err(s, page, "Wrong object count. Counter is %d but counted were %d", in on_freelist()
1062 page->inuse, page->objects - nr); in on_freelist()
1063 page->inuse = page->objects - nr; in on_freelist()
1069 static void trace(struct kmem_cache *s, struct page *page, void *object, in trace() argument
1072 if (s->flags & SLAB_TRACE) { in trace()
1074 s->name, in trace()
1076 object, page->inuse, in trace()
1077 page->freelist); in trace()
1081 s->object_size); in trace()
1091 struct kmem_cache_node *n, struct page *page) in add_full() argument
1093 if (!(s->flags & SLAB_STORE_USER)) in add_full()
1096 lockdep_assert_held(&n->list_lock); in add_full()
1097 list_add(&page->slab_list, &n->full); in add_full()
1100 static void remove_full(struct kmem_cache *s, struct kmem_cache_node *n, struct page *page) in remove_full() argument
1102 if (!(s->flags & SLAB_STORE_USER)) in remove_full()
1105 lockdep_assert_held(&n->list_lock); in remove_full()
1106 list_del(&page->slab_list); in remove_full()
1114 return atomic_long_read(&n->nr_slabs); in slabs_node()
1119 return atomic_long_read(&n->nr_slabs); in node_nr_slabs()
1128 * kmem_cache_node structure. Solve the chicken-egg in inc_slabs_node()
1133 atomic_long_inc(&n->nr_slabs); in inc_slabs_node()
1134 atomic_long_add(objects, &n->total_objects); in inc_slabs_node()
1141 atomic_long_dec(&n->nr_slabs); in dec_slabs_node()
1142 atomic_long_sub(objects, &n->total_objects); in dec_slabs_node()
1146 static void setup_object_debug(struct kmem_cache *s, struct page *page, in setup_object_debug() argument
1157 void setup_page_debug(struct kmem_cache *s, struct page *page, void *addr) in setup_page_debug() argument
1163 memset(kasan_reset_tag(addr), POISON_INUSE, page_size(page)); in setup_page_debug()
1168 struct page *page, void *object) in alloc_consistency_checks() argument
1170 if (!check_slab(s, page)) in alloc_consistency_checks()
1173 if (!check_valid_pointer(s, page, object)) { in alloc_consistency_checks()
1174 object_err(s, page, object, "Freelist Pointer check fails"); in alloc_consistency_checks()
1178 if (!check_object(s, page, object, SLUB_RED_INACTIVE)) in alloc_consistency_checks()
1185 struct page *page, in alloc_debug_processing() argument
1188 if (s->flags & SLAB_CONSISTENCY_CHECKS) { in alloc_debug_processing()
1189 if (!alloc_consistency_checks(s, page, object)) in alloc_debug_processing()
1194 if (s->flags & SLAB_STORE_USER) in alloc_debug_processing()
1196 trace(s, page, object, 1); in alloc_debug_processing()
1201 if (PageSlab(page)) { in alloc_debug_processing()
1203 * If this is a slab page then lets do the best we can in alloc_debug_processing()
1208 page->inuse = page->objects; in alloc_debug_processing()
1209 page->freelist = NULL; in alloc_debug_processing()
1215 struct page *page, void *object, unsigned long addr) in free_consistency_checks() argument
1217 if (!check_valid_pointer(s, page, object)) { in free_consistency_checks()
1218 slab_err(s, page, "Invalid object pointer 0x%p", object); in free_consistency_checks()
1222 if (on_freelist(s, page, object)) { in free_consistency_checks()
1223 object_err(s, page, object, "Object already free"); in free_consistency_checks()
1227 if (!check_object(s, page, object, SLUB_RED_ACTIVE)) in free_consistency_checks()
1230 if (unlikely(s != page->slab_cache)) { in free_consistency_checks()
1231 if (!PageSlab(page)) { in free_consistency_checks()
1232 slab_err(s, page, "Attempt to free object(0x%p) outside of slab", in free_consistency_checks()
1234 } else if (!page->slab_cache) { in free_consistency_checks()
1239 object_err(s, page, object, in free_consistency_checks()
1240 "page slab pointer corrupt."); in free_consistency_checks()
1248 struct kmem_cache *s, struct page *page, in free_debug_processing() argument
1252 struct kmem_cache_node *n = get_node(s, page_to_nid(page)); in free_debug_processing()
1258 spin_lock_irqsave(&n->list_lock, flags); in free_debug_processing()
1259 slab_lock(page); in free_debug_processing()
1261 if (s->flags & SLAB_CONSISTENCY_CHECKS) { in free_debug_processing()
1262 if (!check_slab(s, page)) in free_debug_processing()
1269 if (s->flags & SLAB_CONSISTENCY_CHECKS) { in free_debug_processing()
1270 if (!free_consistency_checks(s, page, object, addr)) in free_debug_processing()
1274 if (s->flags & SLAB_STORE_USER) in free_debug_processing()
1276 trace(s, page, object, 0); in free_debug_processing()
1289 slab_err(s, page, "Bulk freelist count(%d) invalid(%d)\n", in free_debug_processing()
1292 slab_unlock(page); in free_debug_processing()
1293 spin_unlock_irqrestore(&n->list_lock, flags); in free_debug_processing()
1305 * @init: assume this is initial parsing and not per-kmem-create parsing
1331 case '-': in parse_slub_debug_flags()
1434 pr_info("mem auto-init: SLAB_POISON will take precedence over init_on_alloc/init_on_free\n"); in setup_slub_debug()
1441 * kmem_cache_flags - apply debugging options to the cache
1442 * @object_size: the size of an object without meta data
1448 * slub_debug=<Debug-Options>,<slab name1>,<slab name2> ...
1473 end = next_block - 1; in kmem_cache_flags()
1475 glob = strnchr(iter, end - iter, '*'); in kmem_cache_flags()
1477 cmplen = glob - iter; in kmem_cache_flags()
1479 cmplen = max_t(size_t, len, (end - iter)); in kmem_cache_flags()
1496 struct page *page, void *object) {} in setup_object_debug() argument
1498 void setup_page_debug(struct kmem_cache *s, struct page *page, void *addr) {} in setup_page_debug() argument
1501 struct page *page, void *object, unsigned long addr) { return 0; } in alloc_debug_processing() argument
1504 struct kmem_cache *s, struct page *page, in free_debug_processing() argument
1508 static inline int slab_pad_check(struct kmem_cache *s, struct page *page) in slab_pad_check() argument
1510 static inline int check_object(struct kmem_cache *s, struct page *page, in check_object() argument
1513 struct page *page) {} in add_full() argument
1515 struct page *page) {} in remove_full() argument
1534 static bool freelist_corrupted(struct kmem_cache *s, struct page *page, in freelist_corrupted() argument
1545 static inline void *kmalloc_large_node_hook(void *ptr, size_t size, gfp_t flags) in kmalloc_large_node_hook() argument
1547 ptr = kasan_kmalloc_large(ptr, size, flags); in kmalloc_large_node_hook()
1549 kmemleak_alloc(ptr, size, 1, flags); in kmalloc_large_node_hook()
1562 kmemleak_free_recursive(x, s->flags); in slab_free_hook()
1574 debug_check_no_locks_freed(x, s->object_size); in slab_free_hook()
1578 if (!(s->flags & SLAB_DEBUG_OBJECTS)) in slab_free_hook()
1579 debug_check_no_obj_freed(x, s->object_size); in slab_free_hook()
1581 /* Use KCSAN to help debug racy use-after-free. */ in slab_free_hook()
1582 if (!(s->flags & SLAB_TYPESAFE_BY_RCU)) in slab_free_hook()
1583 __kcsan_check_access(x, s->object_size, in slab_free_hook()
1598 memset(kasan_reset_tag(x), 0, s->object_size); in slab_free_hook()
1599 rsize = (s->flags & SLAB_RED_ZONE) ? s->red_left_pad : 0; in slab_free_hook()
1600 memset((char *)kasan_reset_tag(x) + s->inuse, 0, in slab_free_hook()
1601 s->size - s->inuse - rsize); in slab_free_hook()
1641 --(*cnt); in slab_free_freelist_hook()
1651 static void *setup_object(struct kmem_cache *s, struct page *page, in setup_object() argument
1654 setup_object_debug(s, page, object); in setup_object()
1656 if (unlikely(s->ctor)) { in setup_object()
1658 s->ctor(object); in setup_object()
1667 static inline struct page *alloc_slab_page(struct kmem_cache *s, in alloc_slab_page()
1670 struct page *page; in alloc_slab_page() local
1674 page = alloc_pages(flags, order); in alloc_slab_page()
1676 page = __alloc_pages_node(node, flags, order); in alloc_slab_page()
1678 if (page) in alloc_slab_page()
1679 account_slab_page(page, order, s); in alloc_slab_page()
1681 return page; in alloc_slab_page()
1685 /* Pre-initialize the random sequence cache */
1688 unsigned int count = oo_objects(s->oo); in init_cache_random_seq()
1692 if (s->random_seq) in init_cache_random_seq()
1698 s->name); in init_cache_random_seq()
1703 if (s->random_seq) { in init_cache_random_seq()
1707 s->random_seq[i] *= s->size; in init_cache_random_seq()
1725 /* Get the next entry on the pre-computed freelist randomized */
1726 static void *next_freelist_entry(struct kmem_cache *s, struct page *page, in next_freelist_entry() argument
1734 * If the target page allocation failed, the number of objects on the in next_freelist_entry()
1735 * page might be smaller than the usual size defined by the cache. in next_freelist_entry()
1738 idx = s->random_seq[*pos]; in next_freelist_entry()
1747 /* Shuffle the single linked freelist based on a random pre-computed sequence */
1748 static bool shuffle_freelist(struct kmem_cache *s, struct page *page) in shuffle_freelist() argument
1755 if (page->objects < 2 || !s->random_seq) in shuffle_freelist()
1758 freelist_count = oo_objects(s->oo); in shuffle_freelist()
1761 page_limit = page->objects * s->size; in shuffle_freelist()
1762 start = fixup_red_left(s, page_address(page)); in shuffle_freelist()
1765 cur = next_freelist_entry(s, page, &pos, start, page_limit, in shuffle_freelist()
1767 cur = setup_object(s, page, cur); in shuffle_freelist()
1768 page->freelist = cur; in shuffle_freelist()
1770 for (idx = 1; idx < page->objects; idx++) { in shuffle_freelist()
1771 next = next_freelist_entry(s, page, &pos, start, page_limit, in shuffle_freelist()
1773 next = setup_object(s, page, next); in shuffle_freelist()
1787 static inline bool shuffle_freelist(struct kmem_cache *s, struct page *page) in shuffle_freelist() argument
1793 static struct page *allocate_slab(struct kmem_cache *s, gfp_t flags, int node) in allocate_slab()
1795 struct page *page; in allocate_slab() local
1796 struct kmem_cache_order_objects oo = s->oo; in allocate_slab()
1807 flags |= s->allocflags; in allocate_slab()
1810 * Let the initial higher-order allocation fail under memory pressure in allocate_slab()
1811 * so we fall-back to the minimum order allocation. in allocate_slab()
1814 if ((alloc_gfp & __GFP_DIRECT_RECLAIM) && oo_order(oo) > oo_order(s->min)) in allocate_slab()
1817 page = alloc_slab_page(s, alloc_gfp, node, oo); in allocate_slab()
1818 if (unlikely(!page)) { in allocate_slab()
1819 oo = s->min; in allocate_slab()
1825 page = alloc_slab_page(s, alloc_gfp, node, oo); in allocate_slab()
1826 if (unlikely(!page)) in allocate_slab()
1831 page->objects = oo_objects(oo); in allocate_slab()
1833 page->slab_cache = s; in allocate_slab()
1834 __SetPageSlab(page); in allocate_slab()
1835 if (page_is_pfmemalloc(page)) in allocate_slab()
1836 SetPageSlabPfmemalloc(page); in allocate_slab()
1838 kasan_poison_slab(page); in allocate_slab()
1840 start = page_address(page); in allocate_slab()
1842 setup_page_debug(s, page, start); in allocate_slab()
1844 shuffle = shuffle_freelist(s, page); in allocate_slab()
1848 start = setup_object(s, page, start); in allocate_slab()
1849 page->freelist = start; in allocate_slab()
1850 for (idx = 0, p = start; idx < page->objects - 1; idx++) { in allocate_slab()
1851 next = p + s->size; in allocate_slab()
1852 next = setup_object(s, page, next); in allocate_slab()
1859 page->inuse = page->objects; in allocate_slab()
1860 page->frozen = 1; in allocate_slab()
1865 if (!page) in allocate_slab()
1868 inc_slabs_node(s, page_to_nid(page), page->objects); in allocate_slab()
1870 return page; in allocate_slab()
1873 static struct page *new_slab(struct kmem_cache *s, gfp_t flags, int node) in new_slab()
1882 static void __free_slab(struct kmem_cache *s, struct page *page) in __free_slab() argument
1884 int order = compound_order(page); in __free_slab()
1890 slab_pad_check(s, page); in __free_slab()
1891 for_each_object(p, s, page_address(page), in __free_slab()
1892 page->objects) in __free_slab()
1893 check_object(s, page, p, SLUB_RED_INACTIVE); in __free_slab()
1896 __ClearPageSlabPfmemalloc(page); in __free_slab()
1897 __ClearPageSlab(page); in __free_slab()
1899 page->mapping = NULL; in __free_slab()
1900 if (current->reclaim_state) in __free_slab()
1901 current->reclaim_state->reclaimed_slab += pages; in __free_slab()
1902 unaccount_slab_page(page, order, s); in __free_slab()
1903 __free_pages(page, order); in __free_slab()
1908 struct page *page = container_of(h, struct page, rcu_head); in rcu_free_slab() local
1910 __free_slab(page->slab_cache, page); in rcu_free_slab()
1913 static void free_slab(struct kmem_cache *s, struct page *page) in free_slab() argument
1915 if (unlikely(s->flags & SLAB_TYPESAFE_BY_RCU)) { in free_slab()
1916 call_rcu(&page->rcu_head, rcu_free_slab); in free_slab()
1918 __free_slab(s, page); in free_slab()
1921 static void discard_slab(struct kmem_cache *s, struct page *page) in discard_slab() argument
1923 dec_slabs_node(s, page_to_nid(page), page->objects); in discard_slab()
1924 free_slab(s, page); in discard_slab()
1931 __add_partial(struct kmem_cache_node *n, struct page *page, int tail) in __add_partial() argument
1933 n->nr_partial++; in __add_partial()
1935 list_add_tail(&page->slab_list, &n->partial); in __add_partial()
1937 list_add(&page->slab_list, &n->partial); in __add_partial()
1941 struct page *page, int tail) in add_partial() argument
1943 lockdep_assert_held(&n->list_lock); in add_partial()
1944 __add_partial(n, page, tail); in add_partial()
1948 struct page *page) in remove_partial() argument
1950 lockdep_assert_held(&n->list_lock); in remove_partial()
1951 list_del(&page->slab_list); in remove_partial()
1952 n->nr_partial--; in remove_partial()
1962 struct kmem_cache_node *n, struct page *page, in acquire_slab() argument
1967 struct page new; in acquire_slab()
1969 lockdep_assert_held(&n->list_lock); in acquire_slab()
1976 freelist = page->freelist; in acquire_slab()
1977 counters = page->counters; in acquire_slab()
1979 *objects = new.objects - new.inuse; in acquire_slab()
1981 new.inuse = page->objects; in acquire_slab()
1990 if (!__cmpxchg_double_slab(s, page, in acquire_slab()
1996 remove_partial(n, page); in acquire_slab()
2001 static void put_cpu_partial(struct kmem_cache *s, struct page *page, int drain);
2002 static inline bool pfmemalloc_match(struct page *page, gfp_t gfpflags);
2010 struct page *page, *page2; in get_partial_node() local
2021 if (!n || !n->nr_partial) in get_partial_node()
2024 spin_lock(&n->list_lock); in get_partial_node()
2025 list_for_each_entry_safe(page, page2, &n->partial, slab_list) { in get_partial_node()
2028 if (!pfmemalloc_match(page, flags)) in get_partial_node()
2031 t = acquire_slab(s, n, page, object == NULL, &objects); in get_partial_node()
2037 c->page = page; in get_partial_node()
2041 put_cpu_partial(s, page, 0); in get_partial_node()
2049 spin_unlock(&n->list_lock); in get_partial_node()
2054 * Get a page from somewhere. Search in increasing NUMA distances.
2085 if (!s->remote_node_defrag_ratio || in get_any_partial()
2086 get_cycles() % 1024 > s->remote_node_defrag_ratio) in get_any_partial()
2098 n->nr_partial > s->min_partial) { in get_any_partial()
2103 * here - if mems_allowed was updated in in get_any_partial()
2118 * Get a partial page, lock it and return it.
2177 unsigned long actual_tid = __this_cpu_read(s->cpu_slab->tid); in note_cmpxchg_failure()
2179 pr_info("%s %s: cmpxchg redo ", n, s->name); in note_cmpxchg_failure()
2183 pr_warn("due to cpu change %d -> %d\n", in note_cmpxchg_failure()
2188 pr_warn("due to cpu running other code. Event %ld->%ld\n", in note_cmpxchg_failure()
2202 per_cpu_ptr(s->cpu_slab, cpu)->tid = init_tid(cpu); in init_kmem_cache_cpus()
2208 static void deactivate_slab(struct kmem_cache *s, struct page *page, in deactivate_slab() argument
2212 struct kmem_cache_node *n = get_node(s, page_to_nid(page)); in deactivate_slab()
2217 struct page new; in deactivate_slab()
2218 struct page old; in deactivate_slab()
2220 if (page->freelist) { in deactivate_slab()
2227 * to the page freelist while it is still frozen. Leave the in deactivate_slab()
2230 * There is no need to take the list->lock because the page in deactivate_slab()
2242 if (freelist_corrupted(s, page, &freelist, nextfree)) in deactivate_slab()
2246 prior = page->freelist; in deactivate_slab()
2247 counters = page->counters; in deactivate_slab()
2250 new.inuse--; in deactivate_slab()
2253 } while (!__cmpxchg_double_slab(s, page, in deactivate_slab()
2262 * Stage two: Ensure that the page is unfrozen while the in deactivate_slab()
2267 * with the count. If there is a mismatch then the page in deactivate_slab()
2268 * is not unfrozen but the page is on the wrong list. in deactivate_slab()
2271 * the page from the list that we just put it on again in deactivate_slab()
2277 old.freelist = page->freelist; in deactivate_slab()
2278 old.counters = page->counters; in deactivate_slab()
2284 new.inuse--; in deactivate_slab()
2292 if (!new.inuse && n->nr_partial >= s->min_partial) in deactivate_slab()
2300 * that acquire_slab() will see a slab page that in deactivate_slab()
2303 spin_lock(&n->list_lock); in deactivate_slab()
2308 if ((s->flags & SLAB_STORE_USER) && !lock) { in deactivate_slab()
2315 spin_lock(&n->list_lock); in deactivate_slab()
2322 remove_partial(n, page); in deactivate_slab()
2324 remove_full(s, n, page); in deactivate_slab()
2327 add_partial(n, page, tail); in deactivate_slab()
2329 add_full(s, n, page); in deactivate_slab()
2333 if (!__cmpxchg_double_slab(s, page, in deactivate_slab()
2340 spin_unlock(&n->list_lock); in deactivate_slab()
2348 discard_slab(s, page); in deactivate_slab()
2352 c->page = NULL; in deactivate_slab()
2353 c->freelist = NULL; in deactivate_slab()
2354 c->tid = next_tid(c->tid); in deactivate_slab()
2369 struct page *page, *discard_page = NULL; in unfreeze_partials() local
2371 while ((page = slub_percpu_partial(c))) { in unfreeze_partials()
2372 struct page new; in unfreeze_partials()
2373 struct page old; in unfreeze_partials()
2375 slub_set_percpu_partial(c, page); in unfreeze_partials()
2377 n2 = get_node(s, page_to_nid(page)); in unfreeze_partials()
2380 spin_unlock(&n->list_lock); in unfreeze_partials()
2383 spin_lock(&n->list_lock); in unfreeze_partials()
2388 old.freelist = page->freelist; in unfreeze_partials()
2389 old.counters = page->counters; in unfreeze_partials()
2397 } while (!__cmpxchg_double_slab(s, page, in unfreeze_partials()
2402 if (unlikely(!new.inuse && n->nr_partial >= s->min_partial)) { in unfreeze_partials()
2403 page->next = discard_page; in unfreeze_partials()
2404 discard_page = page; in unfreeze_partials()
2406 add_partial(n, page, DEACTIVATE_TO_TAIL); in unfreeze_partials()
2412 spin_unlock(&n->list_lock); in unfreeze_partials()
2415 page = discard_page; in unfreeze_partials()
2416 discard_page = discard_page->next; in unfreeze_partials()
2419 discard_slab(s, page); in unfreeze_partials()
2426 * Put a page that was just frozen (in __slab_free|get_partial_node) into a
2427 * partial page slot if available.
2432 static void put_cpu_partial(struct kmem_cache *s, struct page *page, int drain) in put_cpu_partial() argument
2435 struct page *oldpage; in put_cpu_partial()
2443 oldpage = this_cpu_read(s->cpu_slab->partial); in put_cpu_partial()
2446 pobjects = oldpage->pobjects; in put_cpu_partial()
2447 pages = oldpage->pages; in put_cpu_partial()
2455 unfreeze_partials(s, this_cpu_ptr(s->cpu_slab)); in put_cpu_partial()
2465 pobjects += page->objects - page->inuse; in put_cpu_partial()
2467 page->pages = pages; in put_cpu_partial()
2468 page->pobjects = pobjects; in put_cpu_partial()
2469 page->next = oldpage; in put_cpu_partial()
2471 } while (this_cpu_cmpxchg(s->cpu_slab->partial, oldpage, page) in put_cpu_partial()
2477 unfreeze_partials(s, this_cpu_ptr(s->cpu_slab)); in put_cpu_partial()
2487 deactivate_slab(s, c->page, c->freelist, c); in flush_slab()
2497 struct kmem_cache_cpu *c = per_cpu_ptr(s->cpu_slab, cpu); in __flush_cpu_slab()
2499 if (c->page) in __flush_cpu_slab()
2515 struct kmem_cache_cpu *c = per_cpu_ptr(s->cpu_slab, cpu); in has_cpu_slab()
2517 return c->page || slub_percpu_partial(c); in has_cpu_slab()
2548 static inline int node_match(struct page *page, int node) in node_match() argument
2551 if (node != NUMA_NO_NODE && page_to_nid(page) != node) in node_match()
2558 static int count_free(struct page *page) in count_free() argument
2560 return page->objects - page->inuse; in count_free()
2565 return atomic_long_read(&n->total_objects); in node_nr_objs()
2571 int (*get_count)(struct page *)) in count_partial() argument
2575 struct page *page; in count_partial() local
2577 spin_lock_irqsave(&n->list_lock, flags); in count_partial()
2578 list_for_each_entry(page, &n->partial, slab_list) in count_partial()
2579 x += get_count(page); in count_partial()
2580 spin_unlock_irqrestore(&n->list_lock, flags); in count_partial()
2599 pr_warn(" cache: %s, object size: %u, buffer size: %u, default order: %u, min order: %u\n", in slab_out_of_memory()
2600 s->name, s->object_size, s->size, oo_order(s->oo), in slab_out_of_memory()
2601 oo_order(s->min)); in slab_out_of_memory()
2603 if (oo_order(s->min) > get_order(s->object_size)) in slab_out_of_memory()
2605 s->name); in slab_out_of_memory()
2627 struct page *page; in new_slab_objects() local
2629 WARN_ON_ONCE(s->ctor && (flags & __GFP_ZERO)); in new_slab_objects()
2636 page = new_slab(s, flags, node); in new_slab_objects()
2637 if (page) { in new_slab_objects()
2638 c = raw_cpu_ptr(s->cpu_slab); in new_slab_objects()
2639 if (c->page) in new_slab_objects()
2643 * No other reference to the page yet so we can in new_slab_objects()
2646 freelist = page->freelist; in new_slab_objects()
2647 page->freelist = NULL; in new_slab_objects()
2650 c->page = page; in new_slab_objects()
2657 static inline bool pfmemalloc_match(struct page *page, gfp_t gfpflags) in pfmemalloc_match() argument
2659 if (unlikely(PageSlabPfmemalloc(page))) in pfmemalloc_match()
2666 * Check the page->freelist of a page and either transfer the freelist to the
2667 * per cpu freelist or deactivate the page.
2669 * The page is still frozen if the return value is not NULL.
2671 * If this function returns NULL then the page has been unfrozen.
2675 static inline void *get_freelist(struct kmem_cache *s, struct page *page) in get_freelist() argument
2677 struct page new; in get_freelist()
2682 freelist = page->freelist; in get_freelist()
2683 counters = page->counters; in get_freelist()
2688 new.inuse = page->objects; in get_freelist()
2691 } while (!__cmpxchg_double_slab(s, page, in get_freelist()
2713 * a call to the page allocator and the setup of a new slab.
2722 struct page *page; in ___slab_alloc() local
2726 page = c->page; in ___slab_alloc()
2727 if (!page) { in ___slab_alloc()
2739 if (unlikely(!node_match(page, node))) { in ___slab_alloc()
2749 deactivate_slab(s, page, c->freelist, c); in ___slab_alloc()
2755 * By rights, we should be searching for a slab page that was in ___slab_alloc()
2757 * information when the page leaves the per-cpu allocator in ___slab_alloc()
2759 if (unlikely(!pfmemalloc_match(page, gfpflags))) { in ___slab_alloc()
2760 deactivate_slab(s, page, c->freelist, c); in ___slab_alloc()
2764 /* must check again c->freelist in case of cpu migration or IRQ */ in ___slab_alloc()
2765 freelist = c->freelist; in ___slab_alloc()
2769 freelist = get_freelist(s, page); in ___slab_alloc()
2772 c->page = NULL; in ___slab_alloc()
2773 c->tid = next_tid(c->tid); in ___slab_alloc()
2783 * page is pointing to the page from which the objects are obtained. in ___slab_alloc()
2784 * That page must be frozen for per cpu allocations to work. in ___slab_alloc()
2786 VM_BUG_ON(!c->page->frozen); in ___slab_alloc()
2787 c->freelist = get_freepointer(s, freelist); in ___slab_alloc()
2788 c->tid = next_tid(c->tid); in ___slab_alloc()
2794 page = c->page = slub_percpu_partial(c); in ___slab_alloc()
2795 slub_set_percpu_partial(c, page); in ___slab_alloc()
2807 page = c->page; in ___slab_alloc()
2808 if (likely(!kmem_cache_debug(s) && pfmemalloc_match(page, gfpflags))) in ___slab_alloc()
2813 !alloc_debug_processing(s, page, freelist, addr)) in ___slab_alloc()
2816 deactivate_slab(s, page, get_freepointer(s, freelist), c); in ___slab_alloc()
2837 c = this_cpu_ptr(s->cpu_slab); in __slab_alloc()
2853 memset((void *)((char *)kasan_reset_tag(obj) + s->offset), in maybe_wipe_obj_freeptr()
2872 struct page *page; in slab_alloc_node() local
2897 tid = this_cpu_read(s->cpu_slab->tid); in slab_alloc_node()
2898 c = raw_cpu_ptr(s->cpu_slab); in slab_alloc_node()
2900 unlikely(tid != READ_ONCE(c->tid))); in slab_alloc_node()
2905 * on c to guarantee that object and page associated with previous tid in slab_alloc_node()
2907 * page could be one associated with next tid and our alloc/free in slab_alloc_node()
2919 object = c->freelist; in slab_alloc_node()
2920 page = c->page; in slab_alloc_node()
2921 if (unlikely(!object || !page || !node_match(page, node))) { in slab_alloc_node()
2941 s->cpu_slab->freelist, s->cpu_slab->tid, in slab_alloc_node()
2969 void *ret = slab_alloc(s, gfpflags, _RET_IP_, s->object_size); in kmem_cache_alloc()
2971 trace_kmem_cache_alloc(_RET_IP_, ret, s->object_size, in kmem_cache_alloc()
2972 s->size, gfpflags); in kmem_cache_alloc()
2979 void *kmem_cache_alloc_trace(struct kmem_cache *s, gfp_t gfpflags, size_t size) in kmem_cache_alloc_trace() argument
2981 void *ret = slab_alloc(s, gfpflags, _RET_IP_, size); in kmem_cache_alloc_trace()
2982 trace_kmalloc(_RET_IP_, ret, size, s->size, gfpflags); in kmem_cache_alloc_trace()
2983 ret = kasan_kmalloc(s, ret, size, gfpflags); in kmem_cache_alloc_trace()
2992 void *ret = slab_alloc_node(s, gfpflags, node, _RET_IP_, s->object_size); in kmem_cache_alloc_node()
2995 s->object_size, s->size, gfpflags, node); in kmem_cache_alloc_node()
3004 int node, size_t size) in kmem_cache_alloc_node_trace() argument
3006 void *ret = slab_alloc_node(s, gfpflags, node, _RET_IP_, size); in kmem_cache_alloc_node_trace()
3009 size, s->size, gfpflags, node); in kmem_cache_alloc_node_trace()
3011 ret = kasan_kmalloc(s, ret, size, gfpflags); in kmem_cache_alloc_node_trace()
3023 * lock and free the item. If there is no additional partial page
3026 static void __slab_free(struct kmem_cache *s, struct page *page, in __slab_free() argument
3033 struct page new; in __slab_free()
3044 !free_debug_processing(s, page, head, tail, cnt, addr)) in __slab_free()
3049 spin_unlock_irqrestore(&n->list_lock, flags); in __slab_free()
3052 prior = page->freelist; in __slab_free()
3053 counters = page->counters; in __slab_free()
3057 new.inuse -= cnt; in __slab_free()
3072 n = get_node(s, page_to_nid(page)); in __slab_free()
3081 spin_lock_irqsave(&n->list_lock, flags); in __slab_free()
3086 } while (!cmpxchg_double_slab(s, page, in __slab_free()
3101 * If we just froze the page then put it onto the in __slab_free()
3104 put_cpu_partial(s, page, 1); in __slab_free()
3111 if (unlikely(!new.inuse && n->nr_partial >= s->min_partial)) in __slab_free()
3119 remove_full(s, n, page); in __slab_free()
3120 add_partial(n, page, DEACTIVATE_TO_TAIL); in __slab_free()
3123 spin_unlock_irqrestore(&n->list_lock, flags); in __slab_free()
3131 remove_partial(n, page); in __slab_free()
3135 remove_full(s, n, page); in __slab_free()
3138 spin_unlock_irqrestore(&n->list_lock, flags); in __slab_free()
3140 discard_slab(s, page); in __slab_free()
3155 * same page) possible by specifying head and tail ptr, plus objects
3159 struct page *page, void *head, void *tail, in do_slab_free() argument
3177 tid = this_cpu_read(s->cpu_slab->tid); in do_slab_free()
3178 c = raw_cpu_ptr(s->cpu_slab); in do_slab_free()
3180 unlikely(tid != READ_ONCE(c->tid))); in do_slab_free()
3185 if (likely(page == c->page)) { in do_slab_free()
3186 void **freelist = READ_ONCE(c->freelist); in do_slab_free()
3191 s->cpu_slab->freelist, s->cpu_slab->tid, in do_slab_free()
3200 __slab_free(s, page, head, tail_obj, cnt, addr); in do_slab_free()
3204 static __always_inline void slab_free(struct kmem_cache *s, struct page *page, in slab_free() argument
3213 do_slab_free(s, page, head, tail, cnt, addr); in slab_free()
3234 struct page *page; member
3244 * page. It builds a detached freelist directly within the given
3245 * page/objects. This can happen without any need for
3254 int build_detached_freelist(struct kmem_cache *s, size_t size, in build_detached_freelist() argument
3260 struct page *page; in build_detached_freelist() local
3262 /* Always re-init detached_freelist */ in build_detached_freelist()
3263 df->page = NULL; in build_detached_freelist()
3266 object = p[--size]; in build_detached_freelist()
3268 } while (!object && size); in build_detached_freelist()
3273 page = virt_to_head_page(object); in build_detached_freelist()
3276 if (unlikely(!PageSlab(page))) { in build_detached_freelist()
3277 BUG_ON(!PageCompound(page)); in build_detached_freelist()
3279 __free_pages(page, compound_order(page)); in build_detached_freelist()
3280 p[size] = NULL; /* mark object processed */ in build_detached_freelist()
3281 return size; in build_detached_freelist()
3284 df->s = page->slab_cache; in build_detached_freelist()
3286 df->s = cache_from_obj(s, object); /* Support for memcg */ in build_detached_freelist()
3290 slab_free_hook(df->s, object, false); in build_detached_freelist()
3292 p[size] = NULL; /* mark object processed */ in build_detached_freelist()
3293 return size; in build_detached_freelist()
3297 df->page = page; in build_detached_freelist()
3298 set_freepointer(df->s, object, NULL); in build_detached_freelist()
3299 df->tail = object; in build_detached_freelist()
3300 df->freelist = object; in build_detached_freelist()
3301 p[size] = NULL; /* mark object processed */ in build_detached_freelist()
3302 df->cnt = 1; in build_detached_freelist()
3304 while (size) { in build_detached_freelist()
3305 object = p[--size]; in build_detached_freelist()
3309 /* df->page is always set at this point */ in build_detached_freelist()
3310 if (df->page == virt_to_head_page(object)) { in build_detached_freelist()
3312 set_freepointer(df->s, object, df->freelist); in build_detached_freelist()
3313 df->freelist = object; in build_detached_freelist()
3314 df->cnt++; in build_detached_freelist()
3315 p[size] = NULL; /* mark object processed */ in build_detached_freelist()
3321 if (!--lookahead) in build_detached_freelist()
3325 first_skipped_index = size + 1; in build_detached_freelist()
3332 void kmem_cache_free_bulk(struct kmem_cache *s, size_t size, void **p) in kmem_cache_free_bulk() argument
3334 if (WARN_ON(!size)) in kmem_cache_free_bulk()
3337 memcg_slab_free_hook(s, p, size); in kmem_cache_free_bulk()
3341 size = build_detached_freelist(s, size, p, &df); in kmem_cache_free_bulk()
3342 if (!df.page) in kmem_cache_free_bulk()
3345 slab_free(df.s, df.page, df.freelist, df.tail, df.cnt,_RET_IP_); in kmem_cache_free_bulk()
3346 } while (likely(size)); in kmem_cache_free_bulk()
3351 int kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags, size_t size, in kmem_cache_alloc_bulk() argument
3359 s = slab_pre_alloc_hook(s, &objcg, size, flags); in kmem_cache_alloc_bulk()
3368 c = this_cpu_ptr(s->cpu_slab); in kmem_cache_alloc_bulk()
3370 for (i = 0; i < size; i++) { in kmem_cache_alloc_bulk()
3371 void *object = kfence_alloc(s, s->object_size, flags); in kmem_cache_alloc_bulk()
3378 object = c->freelist; in kmem_cache_alloc_bulk()
3381 * We may have removed an object from c->freelist using in kmem_cache_alloc_bulk()
3383 * c->tid has not been bumped yet. in kmem_cache_alloc_bulk()
3385 * allocating memory, we should bump c->tid now. in kmem_cache_alloc_bulk()
3387 c->tid = next_tid(c->tid); in kmem_cache_alloc_bulk()
3390 * Invoking slow path likely have side-effect in kmem_cache_alloc_bulk()
3391 * of re-populating per CPU c->freelist in kmem_cache_alloc_bulk()
3398 c = this_cpu_ptr(s->cpu_slab); in kmem_cache_alloc_bulk()
3401 continue; /* goto for-loop */ in kmem_cache_alloc_bulk()
3403 c->freelist = get_freepointer(s, object); in kmem_cache_alloc_bulk()
3407 c->tid = next_tid(c->tid); in kmem_cache_alloc_bulk()
3414 slab_post_alloc_hook(s, objcg, flags, size, p, in kmem_cache_alloc_bulk()
3428 * offset 0. If we tune the size of the object to the alignment then we can
3450 * Calculate the order of allocation given an slab object size.
3454 * order 0 does not cause fragmentation in the page allocator. Larger objects
3466 * we try to keep the page order as low as possible. So we accept more waste
3467 * of space in favor of a small page order.
3474 static inline unsigned int slab_order(unsigned int size, in slab_order() argument
3481 if (order_objects(min_order, size) > MAX_OBJS_PER_PAGE) in slab_order()
3482 return get_order(size * MAX_OBJS_PER_PAGE) - 1; in slab_order()
3484 for (order = max(min_order, (unsigned int)get_order(min_objects * size)); in slab_order()
3490 rem = slab_size % size; in slab_order()
3499 static inline int calculate_order(unsigned int size) in calculate_order() argument
3516 max_objects = order_objects(slub_max_order, size); in calculate_order()
3524 order = slab_order(size, min_objects, in calculate_order()
3530 min_objects--; in calculate_order()
3537 order = slab_order(size, 1, slub_max_order, 1); in calculate_order()
3544 order = slab_order(size, 1, MAX_ORDER, 1); in calculate_order()
3547 return -ENOSYS; in calculate_order()
3553 n->nr_partial = 0; in init_kmem_cache_node()
3554 spin_lock_init(&n->list_lock); in init_kmem_cache_node()
3555 INIT_LIST_HEAD(&n->partial); in init_kmem_cache_node()
3557 atomic_long_set(&n->nr_slabs, 0); in init_kmem_cache_node()
3558 atomic_long_set(&n->total_objects, 0); in init_kmem_cache_node()
3559 INIT_LIST_HEAD(&n->full); in init_kmem_cache_node()
3572 s->cpu_slab = __alloc_percpu(sizeof(struct kmem_cache_cpu), in alloc_kmem_cache_cpus()
3575 if (!s->cpu_slab) in alloc_kmem_cache_cpus()
3596 struct page *page; in early_kmem_cache_node_alloc() local
3599 BUG_ON(kmem_cache_node->size < sizeof(struct kmem_cache_node)); in early_kmem_cache_node_alloc()
3601 page = new_slab(kmem_cache_node, GFP_NOWAIT, node); in early_kmem_cache_node_alloc()
3603 BUG_ON(!page); in early_kmem_cache_node_alloc()
3604 if (page_to_nid(page) != node) { in early_kmem_cache_node_alloc()
3609 n = page->freelist; in early_kmem_cache_node_alloc()
3616 page->freelist = get_freepointer(kmem_cache_node, n); in early_kmem_cache_node_alloc()
3617 page->inuse = 1; in early_kmem_cache_node_alloc()
3618 page->frozen = 0; in early_kmem_cache_node_alloc()
3619 kmem_cache_node->node[node] = n; in early_kmem_cache_node_alloc()
3621 inc_slabs_node(kmem_cache_node, node, page->objects); in early_kmem_cache_node_alloc()
3627 __add_partial(n, page, DEACTIVATE_TO_HEAD); in early_kmem_cache_node_alloc()
3636 s->node[node] = NULL; in free_kmem_cache_nodes()
3644 free_percpu(s->cpu_slab); in __kmem_cache_release()
3668 s->node[node] = n; in init_kmem_cache_nodes()
3679 s->min_partial = min; in set_min_partial()
3704 else if (s->size >= PAGE_SIZE) in set_cpu_partial()
3706 else if (s->size >= 1024) in set_cpu_partial()
3708 else if (s->size >= 256) in set_cpu_partial()
3721 slab_flags_t flags = s->flags; in calculate_sizes()
3722 unsigned int size = s->object_size; in calculate_sizes() local
3726 * Round up object size to the next word boundary. We can only in calculate_sizes()
3730 size = ALIGN(size, sizeof(void *)); in calculate_sizes()
3739 !s->ctor) in calculate_sizes()
3740 s->flags |= __OBJECT_POISON; in calculate_sizes()
3742 s->flags &= ~__OBJECT_POISON; in calculate_sizes()
3750 if ((flags & SLAB_RED_ZONE) && size == s->object_size) in calculate_sizes()
3751 size += sizeof(void *); in calculate_sizes()
3758 s->inuse = size; in calculate_sizes()
3761 ((flags & SLAB_RED_ZONE) && s->object_size < sizeof(void *)) || in calculate_sizes()
3762 s->ctor) { in calculate_sizes()
3772 * The assumption that s->offset >= s->inuse means free in calculate_sizes()
3777 s->offset = size; in calculate_sizes()
3778 size += sizeof(void *); in calculate_sizes()
3785 s->offset = ALIGN_DOWN(s->object_size / 2, sizeof(void *)); in calculate_sizes()
3794 size += 2 * sizeof(struct track); in calculate_sizes()
3797 kasan_cache_create(s, &size, &s->flags); in calculate_sizes()
3807 size += sizeof(void *); in calculate_sizes()
3809 s->red_left_pad = sizeof(void *); in calculate_sizes()
3810 s->red_left_pad = ALIGN(s->red_left_pad, s->align); in calculate_sizes()
3811 size += s->red_left_pad; in calculate_sizes()
3817 * offset 0. In order to align the objects we have to simply size in calculate_sizes()
3820 size = ALIGN(size, s->align); in calculate_sizes()
3821 s->size = size; in calculate_sizes()
3822 s->reciprocal_size = reciprocal_value(size); in calculate_sizes()
3826 order = calculate_order(size); in calculate_sizes()
3831 s->allocflags = 0; in calculate_sizes()
3833 s->allocflags |= __GFP_COMP; in calculate_sizes()
3835 if (s->flags & SLAB_CACHE_DMA) in calculate_sizes()
3836 s->allocflags |= GFP_DMA; in calculate_sizes()
3838 if (s->flags & SLAB_CACHE_DMA32) in calculate_sizes()
3839 s->allocflags |= GFP_DMA32; in calculate_sizes()
3841 if (s->flags & SLAB_RECLAIM_ACCOUNT) in calculate_sizes()
3842 s->allocflags |= __GFP_RECLAIMABLE; in calculate_sizes()
3847 s->oo = oo_make(order, size); in calculate_sizes()
3848 s->min = oo_make(get_order(size), size); in calculate_sizes()
3849 if (oo_objects(s->oo) > oo_objects(s->max)) in calculate_sizes()
3850 s->max = s->oo; in calculate_sizes()
3852 return !!oo_objects(s->oo); in calculate_sizes()
3857 s->flags = kmem_cache_flags(s->size, flags, s->name); in kmem_cache_open()
3859 s->random = get_random_long(); in kmem_cache_open()
3862 if (!calculate_sizes(s, -1)) in kmem_cache_open()
3869 if (get_order(s->size) > get_order(s->object_size)) { in kmem_cache_open()
3870 s->flags &= ~DEBUG_METADATA_FLAGS; in kmem_cache_open()
3871 s->offset = 0; in kmem_cache_open()
3872 if (!calculate_sizes(s, -1)) in kmem_cache_open()
3879 if (system_has_cmpxchg_double() && (s->flags & SLAB_NO_CMPXCHG) == 0) in kmem_cache_open()
3881 s->flags |= __CMPXCHG_DOUBLE; in kmem_cache_open()
3885 * The larger the object size is, the more pages we want on the partial in kmem_cache_open()
3886 * list to avoid pounding the page allocator excessively. in kmem_cache_open()
3888 set_min_partial(s, ilog2(s->size) / 2); in kmem_cache_open()
3893 s->remote_node_defrag_ratio = 1000; in kmem_cache_open()
3896 /* Initialize the pre-computed randomized freelist if slab is up */ in kmem_cache_open()
3910 return -EINVAL; in kmem_cache_open()
3913 static void list_slab_objects(struct kmem_cache *s, struct page *page, in list_slab_objects() argument
3917 void *addr = page_address(page); in list_slab_objects()
3921 slab_err(s, page, text, s->name); in list_slab_objects()
3922 slab_lock(page); in list_slab_objects()
3924 map = get_map(s, page); in list_slab_objects()
3925 for_each_object(p, s, addr, page->objects) { in list_slab_objects()
3928 pr_err("INFO: Object 0x%p @offset=%tu\n", p, p - addr); in list_slab_objects()
3933 slab_unlock(page); in list_slab_objects()
3945 struct page *page, *h; in free_partial() local
3948 spin_lock_irq(&n->list_lock); in free_partial()
3949 list_for_each_entry_safe(page, h, &n->partial, slab_list) { in free_partial()
3950 if (!page->inuse) { in free_partial()
3951 remove_partial(n, page); in free_partial()
3952 list_add(&page->slab_list, &discard); in free_partial()
3954 list_slab_objects(s, page, in free_partial()
3958 spin_unlock_irq(&n->list_lock); in free_partial()
3960 list_for_each_entry_safe(page, h, &discard, slab_list) in free_partial()
3961 discard_slab(s, page); in free_partial()
3970 if (n->nr_partial || slabs_node(s, node)) in __kmem_cache_empty()
3987 if (n->nr_partial || slabs_node(s, node)) in __kmem_cache_shutdown()
4009 slub_max_order = min(slub_max_order, (unsigned int)MAX_ORDER - 1); in setup_slub_max_order()
4025 void *__kmalloc(size_t size, gfp_t flags) in __kmalloc() argument
4030 if (unlikely(size > KMALLOC_MAX_CACHE_SIZE)) in __kmalloc()
4031 return kmalloc_large(size, flags); in __kmalloc()
4033 s = kmalloc_slab(size, flags); in __kmalloc()
4038 ret = slab_alloc(s, flags, _RET_IP_, size); in __kmalloc()
4040 trace_kmalloc(_RET_IP_, ret, size, s->size, flags); in __kmalloc()
4042 ret = kasan_kmalloc(s, ret, size, flags); in __kmalloc()
4049 static void *kmalloc_large_node(size_t size, gfp_t flags, int node) in kmalloc_large_node() argument
4051 struct page *page; in kmalloc_large_node() local
4053 unsigned int order = get_order(size); in kmalloc_large_node()
4056 page = alloc_pages_node(node, flags, order); in kmalloc_large_node()
4057 if (page) { in kmalloc_large_node()
4058 ptr = page_address(page); in kmalloc_large_node()
4059 mod_lruvec_page_state(page, NR_SLAB_UNRECLAIMABLE_B, in kmalloc_large_node()
4063 return kmalloc_large_node_hook(ptr, size, flags); in kmalloc_large_node()
4066 void *__kmalloc_node(size_t size, gfp_t flags, int node) in __kmalloc_node() argument
4071 if (unlikely(size > KMALLOC_MAX_CACHE_SIZE)) { in __kmalloc_node()
4072 ret = kmalloc_large_node(size, flags, node); in __kmalloc_node()
4075 size, PAGE_SIZE << get_order(size), in __kmalloc_node()
4081 s = kmalloc_slab(size, flags); in __kmalloc_node()
4086 ret = slab_alloc_node(s, flags, node, _RET_IP_, size); in __kmalloc_node()
4088 trace_kmalloc_node(_RET_IP_, ret, size, s->size, flags, node); in __kmalloc_node()
4090 ret = kasan_kmalloc(s, ret, size, flags); in __kmalloc_node()
4106 void __check_heap_object(const void *ptr, unsigned long n, struct page *page, in __check_heap_object() argument
4116 /* Find object and usable object size. */ in __check_heap_object()
4117 s = page->slab_cache; in __check_heap_object()
4120 if (ptr < page_address(page)) in __check_heap_object()
4121 usercopy_abort("SLUB object not in SLUB page?!", NULL, in __check_heap_object()
4126 offset = ptr - kfence_object_start(ptr); in __check_heap_object()
4128 offset = (ptr - page_address(page)) % s->size; in __check_heap_object()
4132 if (offset < s->red_left_pad) in __check_heap_object()
4134 s->name, to_user, offset, n); in __check_heap_object()
4135 offset -= s->red_left_pad; in __check_heap_object()
4139 if (offset >= s->useroffset && in __check_heap_object()
4140 offset - s->useroffset <= s->usersize && in __check_heap_object()
4141 n <= s->useroffset - offset + s->usersize) in __check_heap_object()
4152 offset <= object_size && n <= object_size - offset) { in __check_heap_object()
4153 usercopy_warn("SLUB object", s->name, to_user, offset, n); in __check_heap_object()
4157 usercopy_abort("SLUB object", s->name, to_user, offset, n); in __check_heap_object()
4163 struct page *page; in __ksize() local
4168 page = virt_to_head_page(object); in __ksize()
4170 if (unlikely(!PageSlab(page))) { in __ksize()
4171 WARN_ON(!PageCompound(page)); in __ksize()
4172 return page_size(page); in __ksize()
4175 return slab_ksize(page->slab_cache); in __ksize()
4181 struct page *page; in kfree() local
4189 page = virt_to_head_page(x); in kfree()
4190 if (unlikely(!PageSlab(page))) { in kfree()
4191 unsigned int order = compound_order(page); in kfree()
4193 BUG_ON(!PageCompound(page)); in kfree()
4195 mod_lruvec_page_state(page, NR_SLAB_UNRECLAIMABLE_B, in kfree()
4196 -(PAGE_SIZE << order)); in kfree()
4197 __free_pages(page, order); in kfree()
4200 slab_free(page->slab_cache, page, object, NULL, 1, _RET_IP_); in kfree()
4220 struct page *page; in __kmem_cache_shrink() local
4221 struct page *t; in __kmem_cache_shrink()
4233 spin_lock_irqsave(&n->list_lock, flags); in __kmem_cache_shrink()
4239 * list_lock. page->inuse here is the upper limit. in __kmem_cache_shrink()
4241 list_for_each_entry_safe(page, t, &n->partial, slab_list) { in __kmem_cache_shrink()
4242 int free = page->objects - page->inuse; in __kmem_cache_shrink()
4244 /* Do not reread page->inuse */ in __kmem_cache_shrink()
4250 if (free == page->objects) { in __kmem_cache_shrink()
4251 list_move(&page->slab_list, &discard); in __kmem_cache_shrink()
4252 n->nr_partial--; in __kmem_cache_shrink()
4254 list_move(&page->slab_list, promote + free - 1); in __kmem_cache_shrink()
4261 for (i = SHRINK_PROMOTE_MAX - 1; i >= 0; i--) in __kmem_cache_shrink()
4262 list_splice(promote + i, &n->partial); in __kmem_cache_shrink()
4264 spin_unlock_irqrestore(&n->list_lock, flags); in __kmem_cache_shrink()
4267 list_for_each_entry_safe(page, t, &discard, slab_list) in __kmem_cache_shrink()
4268 discard_slab(s, page); in __kmem_cache_shrink()
4296 offline_node = marg->status_change_nid_normal; in slab_mem_offline_callback()
4310 * if n->nr_slabs > 0, slabs still exist on the node in slab_mem_offline_callback()
4317 s->node[offline_node] = NULL; in slab_mem_offline_callback()
4329 int nid = marg->status_change_nid_normal; in slab_mem_going_online_callback()
4353 ret = -ENOMEM; in slab_mem_going_online_callback()
4357 s->node[nid] = n; in slab_mem_going_online_callback()
4402 * the page allocator. Allocate them properly then fix up the pointers
4412 memcpy(s, static_cache, kmem_cache->object_size); in bootstrap()
4421 struct page *p; in bootstrap()
4423 list_for_each_entry(p, &n->partial, slab_list) in bootstrap()
4424 p->slab_cache = s; in bootstrap()
4427 list_for_each_entry(p, &n->full, slab_list) in bootstrap()
4428 p->slab_cache = s; in bootstrap()
4431 list_add(&s->list, &slab_caches); in bootstrap()
4472 pr_info("SLUB: HWalign=%d, Order=%u-%u, MinObjects=%u, CPUs=%u, Nodes=%u\n", in kmem_cache_init()
4483 __kmem_cache_alias(const char *name, unsigned int size, unsigned int align, in __kmem_cache_alias() argument
4488 s = find_mergeable(size, align, flags, name, ctor); in __kmem_cache_alias()
4490 s->refcount++; in __kmem_cache_alias()
4496 s->object_size = max(s->object_size, size); in __kmem_cache_alias()
4497 s->inuse = max(s->inuse, ALIGN(size, sizeof(void *))); in __kmem_cache_alias()
4500 s->refcount--; in __kmem_cache_alias()
4526 if (s->flags & SLAB_STORE_USER) in __kmem_cache_create()
4532 void *__kmalloc_track_caller(size_t size, gfp_t gfpflags, unsigned long caller) in __kmalloc_track_caller() argument
4537 if (unlikely(size > KMALLOC_MAX_CACHE_SIZE)) in __kmalloc_track_caller()
4538 return kmalloc_large(size, gfpflags); in __kmalloc_track_caller()
4540 s = kmalloc_slab(size, gfpflags); in __kmalloc_track_caller()
4545 ret = slab_alloc(s, gfpflags, caller, size); in __kmalloc_track_caller()
4548 trace_kmalloc(caller, ret, size, s->size, gfpflags); in __kmalloc_track_caller()
4555 void *__kmalloc_node_track_caller(size_t size, gfp_t gfpflags, in __kmalloc_node_track_caller() argument
4561 if (unlikely(size > KMALLOC_MAX_CACHE_SIZE)) { in __kmalloc_node_track_caller()
4562 ret = kmalloc_large_node(size, gfpflags, node); in __kmalloc_node_track_caller()
4565 size, PAGE_SIZE << get_order(size), in __kmalloc_node_track_caller()
4571 s = kmalloc_slab(size, gfpflags); in __kmalloc_node_track_caller()
4576 ret = slab_alloc_node(s, gfpflags, node, caller, size); in __kmalloc_node_track_caller()
4579 trace_kmalloc_node(caller, ret, size, s->size, gfpflags, node); in __kmalloc_node_track_caller()
4587 static int count_inuse(struct page *page) in count_inuse() argument
4589 return page->inuse; in count_inuse()
4592 static int count_total(struct page *page) in count_total() argument
4594 return page->objects; in count_total()
4599 static void validate_slab(struct kmem_cache *s, struct page *page) in validate_slab() argument
4602 void *addr = page_address(page); in validate_slab()
4605 slab_lock(page); in validate_slab()
4607 if (!check_slab(s, page) || !on_freelist(s, page, NULL)) in validate_slab()
4611 map = get_map(s, page); in validate_slab()
4612 for_each_object(p, s, addr, page->objects) { in validate_slab()
4616 if (!check_object(s, page, p, val)) in validate_slab()
4621 slab_unlock(page); in validate_slab()
4628 struct page *page; in validate_slab_node() local
4631 spin_lock_irqsave(&n->list_lock, flags); in validate_slab_node()
4633 list_for_each_entry(page, &n->partial, slab_list) { in validate_slab_node()
4634 validate_slab(s, page); in validate_slab_node()
4637 if (count != n->nr_partial) in validate_slab_node()
4639 s->name, count, n->nr_partial); in validate_slab_node()
4641 if (!(s->flags & SLAB_STORE_USER)) in validate_slab_node()
4644 list_for_each_entry(page, &n->full, slab_list) { in validate_slab_node()
4645 validate_slab(s, page); in validate_slab_node()
4648 if (count != atomic_long_read(&n->nr_slabs)) in validate_slab_node()
4650 s->name, count, atomic_long_read(&n->nr_slabs)); in validate_slab_node()
4653 spin_unlock_irqrestore(&n->list_lock, flags); in validate_slab_node()
4699 if (t->max) in free_loc_track()
4700 free_pages((unsigned long)t->loc, in free_loc_track()
4701 get_order(sizeof(struct location) * t->max)); in free_loc_track()
4715 if (t->count) { in alloc_loc_track()
4716 memcpy(l, t->loc, sizeof(struct location) * t->count); in alloc_loc_track()
4719 t->max = max; in alloc_loc_track()
4720 t->loc = l; in alloc_loc_track()
4730 unsigned long age = jiffies - track->when; in add_location()
4732 start = -1; in add_location()
4733 end = t->count; in add_location()
4736 pos = start + (end - start + 1) / 2; in add_location()
4745 caddr = t->loc[pos].addr; in add_location()
4746 if (track->addr == caddr) { in add_location()
4748 l = &t->loc[pos]; in add_location()
4749 l->count++; in add_location()
4750 if (track->when) { in add_location()
4751 l->sum_time += age; in add_location()
4752 if (age < l->min_time) in add_location()
4753 l->min_time = age; in add_location()
4754 if (age > l->max_time) in add_location()
4755 l->max_time = age; in add_location()
4757 if (track->pid < l->min_pid) in add_location()
4758 l->min_pid = track->pid; in add_location()
4759 if (track->pid > l->max_pid) in add_location()
4760 l->max_pid = track->pid; in add_location()
4762 cpumask_set_cpu(track->cpu, in add_location()
4763 to_cpumask(l->cpus)); in add_location()
4765 node_set(page_to_nid(virt_to_page(track)), l->nodes); in add_location()
4769 if (track->addr < caddr) in add_location()
4778 if (t->count >= t->max && !alloc_loc_track(t, 2 * t->max, GFP_ATOMIC)) in add_location()
4781 l = t->loc + pos; in add_location()
4782 if (pos < t->count) in add_location()
4784 (t->count - pos) * sizeof(struct location)); in add_location()
4785 t->count++; in add_location()
4786 l->count = 1; in add_location()
4787 l->addr = track->addr; in add_location()
4788 l->sum_time = age; in add_location()
4789 l->min_time = age; in add_location()
4790 l->max_time = age; in add_location()
4791 l->min_pid = track->pid; in add_location()
4792 l->max_pid = track->pid; in add_location()
4793 cpumask_clear(to_cpumask(l->cpus)); in add_location()
4794 cpumask_set_cpu(track->cpu, to_cpumask(l->cpus)); in add_location()
4795 nodes_clear(l->nodes); in add_location()
4796 node_set(page_to_nid(virt_to_page(track)), l->nodes); in add_location()
4801 struct page *page, enum track_item alloc, in process_slab() argument
4804 void *addr = page_address(page); in process_slab()
4807 __fill_map(obj_map, s, page); in process_slab()
4809 for_each_object(p, s, addr, page->objects) in process_slab()
4825 pr_err("-----------------------\n"); in resiliency_test()
4830 pr_err("\n1. kmalloc-16: Clobber Redzone/next pointer 0x12->0x%p\n\n", in resiliency_test()
4838 pr_err("\n2. kmalloc-32: Clobber next pointer/next slab 0x34 -> -0x%p\n", in resiliency_test()
4846 pr_err("\n3. kmalloc-64: corrupting random byte 0x56->0x%p\n", in resiliency_test()
4855 pr_err("1. kmalloc-128: Clobber first word 0x78->0x%p\n\n", p); in resiliency_test()
4861 pr_err("\n2. kmalloc-256: Clobber 50th byte 0x9a->0x%p\n\n", p); in resiliency_test()
4867 pr_err("\n3. kmalloc-512: Clobber redzone 0xab->0x%p\n\n", p); in resiliency_test()
4917 return -ENOMEM; in show_slab_objects()
4923 struct kmem_cache_cpu *c = per_cpu_ptr(s->cpu_slab, in show_slab_objects()
4926 struct page *page; in show_slab_objects() local
4928 page = READ_ONCE(c->page); in show_slab_objects()
4929 if (!page) in show_slab_objects()
4932 node = page_to_nid(page); in show_slab_objects()
4934 x = page->objects; in show_slab_objects()
4936 x = page->inuse; in show_slab_objects()
4943 page = slub_percpu_partial_read_once(c); in show_slab_objects()
4944 if (page) { in show_slab_objects()
4945 node = page_to_nid(page); in show_slab_objects()
4951 x = page->pages; in show_slab_objects()
4962 * mem_hotplug_lock->slab_mutex->kernfs_mutex in show_slab_objects()
4966 * unplug code doesn't destroy the kmem_cache->node[] data. in show_slab_objects()
4976 x = atomic_long_read(&n->total_objects); in show_slab_objects()
4978 x = atomic_long_read(&n->total_objects) - in show_slab_objects()
4981 x = atomic_long_read(&n->nr_slabs); in show_slab_objects()
4997 x = n->nr_partial; in show_slab_objects()
5032 return sprintf(buf, "%u\n", s->size); in slab_size_show()
5038 return sprintf(buf, "%u\n", s->align); in align_show()
5044 return sprintf(buf, "%u\n", s->object_size); in object_size_show()
5050 return sprintf(buf, "%u\n", oo_objects(s->oo)); in objs_per_slab_show()
5056 return sprintf(buf, "%u\n", oo_order(s->oo)); in order_show()
5062 return sprintf(buf, "%lu\n", s->min_partial); in min_partial_show()
5095 return -EINVAL; in cpu_partial_store()
5105 if (!s->ctor) in ctor_show()
5107 return sprintf(buf, "%pS\n", s->ctor); in ctor_show()
5113 return sprintf(buf, "%d\n", s->refcount < 0 ? 0 : s->refcount - 1); in aliases_show()
5149 struct page *page; in slabs_cpu_partial_show() local
5151 page = slub_percpu_partial(per_cpu_ptr(s->cpu_slab, cpu)); in slabs_cpu_partial_show()
5153 if (page) { in slabs_cpu_partial_show()
5154 pages += page->pages; in slabs_cpu_partial_show()
5155 objects += page->pobjects; in slabs_cpu_partial_show()
5163 struct page *page; in slabs_cpu_partial_show() local
5165 page = slub_percpu_partial(per_cpu_ptr(s->cpu_slab, cpu)); in slabs_cpu_partial_show()
5167 if (page && len < PAGE_SIZE - 20) in slabs_cpu_partial_show()
5169 page->pobjects, page->pages); in slabs_cpu_partial_show()
5178 return sprintf(buf, "%d\n", !!(s->flags & SLAB_RECLAIM_ACCOUNT)); in reclaim_account_show()
5184 return sprintf(buf, "%d\n", !!(s->flags & SLAB_HWCACHE_ALIGN)); in hwcache_align_show()
5191 return sprintf(buf, "%d\n", !!(s->flags & SLAB_CACHE_DMA)); in cache_dma_show()
5198 return sprintf(buf, "%u\n", s->usersize); in usersize_show()
5204 return sprintf(buf, "%d\n", !!(s->flags & SLAB_TYPESAFE_BY_RCU)); in destroy_by_rcu_show()
5223 return sprintf(buf, "%d\n", !!(s->flags & SLAB_CONSISTENCY_CHECKS)); in sanity_checks_show()
5229 return sprintf(buf, "%d\n", !!(s->flags & SLAB_TRACE)); in trace_show()
5235 return sprintf(buf, "%d\n", !!(s->flags & SLAB_RED_ZONE)); in red_zone_show()
5242 return sprintf(buf, "%d\n", !!(s->flags & SLAB_POISON)); in poison_show()
5249 return sprintf(buf, "%d\n", !!(s->flags & SLAB_STORE_USER)); in store_user_show()
5262 int ret = -EINVAL; in validate_store()
5278 return sprintf(buf, "%d\n", !!(s->flags & SLAB_FAILSLAB)); in failslab_show()
5294 return -EINVAL; in shrink_store()
5302 return sprintf(buf, "%u\n", s->remote_node_defrag_ratio / 10); in remote_node_defrag_ratio_show()
5315 return -ERANGE; in remote_node_defrag_ratio_store()
5317 s->remote_node_defrag_ratio = ratio * 10; in remote_node_defrag_ratio_store()
5333 return -ENOMEM; in show_stat()
5336 unsigned x = per_cpu_ptr(s->cpu_slab, cpu)->stat[si]; in show_stat()
5346 if (data[cpu] && len < PAGE_SIZE - 20) in show_stat()
5359 per_cpu_ptr(s->cpu_slab, cpu)->stat[si] = 0; in clear_stat()
5371 return -EINVAL; \
5491 if (!attribute->show) in slab_attr_show()
5492 return -EIO; in slab_attr_show()
5494 err = attribute->show(s, buf); in slab_attr_show()
5510 if (!attribute->store) in slab_attr_store()
5511 return -EIO; in slab_attr_store()
5513 err = attribute->store(s, buf, len); in slab_attr_store()
5543 * Format :[flags-]size
5551 return ERR_PTR(-ENOMEM); in create_unique_id()
5561 if (s->flags & SLAB_CACHE_DMA) in create_unique_id()
5563 if (s->flags & SLAB_CACHE_DMA32) in create_unique_id()
5565 if (s->flags & SLAB_RECLAIM_ACCOUNT) in create_unique_id()
5567 if (s->flags & SLAB_CONSISTENCY_CHECKS) in create_unique_id()
5569 if (s->flags & SLAB_ACCOUNT) in create_unique_id()
5572 *p++ = '-'; in create_unique_id()
5573 p += sprintf(p, "%07u", s->size); in create_unique_id()
5575 BUG_ON(p > name + ID_STR_LENGTH - 1); in create_unique_id()
5587 kobject_init(&s->kobj, &slab_ktype); in sysfs_slab_add()
5601 sysfs_remove_link(&slab_kset->kobj, s->name); in sysfs_slab_add()
5602 name = s->name; in sysfs_slab_add()
5613 s->kobj.kset = kset; in sysfs_slab_add()
5614 err = kobject_init_and_add(&s->kobj, &slab_ktype, NULL, "%s", name); in sysfs_slab_add()
5618 err = sysfs_create_group(&s->kobj, &slab_attr_group); in sysfs_slab_add()
5624 sysfs_slab_alias(s, s->name); in sysfs_slab_add()
5631 kobject_del(&s->kobj); in sysfs_slab_add()
5638 kobject_del(&s->kobj); in sysfs_slab_unlink()
5644 kobject_put(&s->kobj); in sysfs_slab_release()
5667 sysfs_remove_link(&slab_kset->kobj, name); in sysfs_slab_alias()
5668 return sysfs_create_link(&slab_kset->kobj, &s->kobj, name); in sysfs_slab_alias()
5673 return -ENOMEM; in sysfs_slab_alias()
5675 al->s = s; in sysfs_slab_alias()
5676 al->name = name; in sysfs_slab_alias()
5677 al->next = alias_list; in sysfs_slab_alias()
5693 return -ENOSYS; in slab_sysfs_init()
5702 s->name); in slab_sysfs_init()
5708 alias_list = alias_list->next; in slab_sysfs_init()
5709 err = sysfs_slab_alias(al->s, al->name); in slab_sysfs_init()
5712 al->name); in slab_sysfs_init()
5727 struct loc_track *t = seq->private; in slab_debugfs_show()
5731 idx = (unsigned long) t->idx; in slab_debugfs_show()
5732 if (idx < t->count) { in slab_debugfs_show()
5733 l = &t->loc[idx]; in slab_debugfs_show()
5735 seq_printf(seq, "%7ld ", l->count); in slab_debugfs_show()
5737 if (l->addr) in slab_debugfs_show()
5738 seq_printf(seq, "%pS", (void *)l->addr); in slab_debugfs_show()
5740 seq_puts(seq, "<not-available>"); in slab_debugfs_show()
5742 if (l->sum_time != l->min_time) { in slab_debugfs_show()
5744 l->min_time, div_u64(l->sum_time, l->count), in slab_debugfs_show()
5745 l->max_time); in slab_debugfs_show()
5747 seq_printf(seq, " age=%ld", l->min_time); in slab_debugfs_show()
5749 if (l->min_pid != l->max_pid) in slab_debugfs_show()
5750 seq_printf(seq, " pid=%ld-%ld", l->min_pid, l->max_pid); in slab_debugfs_show()
5753 l->min_pid); in slab_debugfs_show()
5755 if (num_online_cpus() > 1 && !cpumask_empty(to_cpumask(l->cpus))) in slab_debugfs_show()
5757 cpumask_pr_args(to_cpumask(l->cpus))); in slab_debugfs_show()
5759 if (nr_online_nodes > 1 && !nodes_empty(l->nodes)) in slab_debugfs_show()
5761 nodemask_pr_args(&l->nodes)); in slab_debugfs_show()
5766 if (!idx && !t->count) in slab_debugfs_show()
5778 struct loc_track *t = seq->private; in slab_debugfs_next()
5780 t->idx = ++(*ppos); in slab_debugfs_next()
5781 if (*ppos <= t->count) in slab_debugfs_next()
5789 struct loc_track *t = seq->private; in slab_debugfs_start()
5791 t->idx = *ppos; in slab_debugfs_start()
5810 struct kmem_cache *s = file_inode(filep)->i_private; in slab_debug_trace_open()
5814 return -ENOMEM; in slab_debug_trace_open()
5816 obj_map = bitmap_alloc(oo_objects(s->oo), GFP_KERNEL); in slab_debug_trace_open()
5819 return -ENOMEM; in slab_debug_trace_open()
5822 if (strcmp(filep->f_path.dentry->d_name.name, "alloc_traces") == 0) in slab_debug_trace_open()
5830 return -ENOMEM; in slab_debug_trace_open()
5838 struct page *page; in slab_debug_trace_open() local
5840 if (!atomic_long_read(&n->nr_slabs)) in slab_debug_trace_open()
5843 spin_lock_irqsave(&n->list_lock, flags); in slab_debug_trace_open()
5844 list_for_each_entry(page, &n->partial, slab_list) in slab_debug_trace_open()
5845 process_slab(t, s, page, alloc, obj_map); in slab_debug_trace_open()
5846 list_for_each_entry(page, &n->full, slab_list) in slab_debug_trace_open()
5847 process_slab(t, s, page, alloc, obj_map); in slab_debug_trace_open()
5848 spin_unlock_irqrestore(&n->list_lock, flags); in slab_debug_trace_open()
5857 struct seq_file *seq = file->private_data; in slab_debug_trace_release()
5858 struct loc_track *t = seq->private; in slab_debug_trace_release()
5878 slab_cache_dir = debugfs_create_dir(s->name, slab_debugfs_root); in debugfs_slab_add()
5889 debugfs_remove_recursive(debugfs_lookup(s->name, slab_debugfs_root)); in debugfs_slab_release()
5899 if (s->flags & SLAB_STORE_USER) in slab_debugfs_init()
5925 sinfo->active_objs = nr_objs - nr_free; in get_slabinfo()
5926 sinfo->num_objs = nr_objs; in get_slabinfo()
5927 sinfo->active_slabs = nr_slabs; in get_slabinfo()
5928 sinfo->num_slabs = nr_slabs; in get_slabinfo()
5929 sinfo->objects_per_slab = oo_objects(s->oo); in get_slabinfo()
5930 sinfo->cache_order = oo_order(s->oo); in get_slabinfo()
5941 return -EIO; in slabinfo_write()