227 #define SHRINKER_REGISTERING ((struct shrinker *)~0UL)
232 static int prealloc_memcg_shrinker(struct shrinker *shrinker)  in prealloc_memcg_shrinker()  argument
250 	shrinker->id = id;  in prealloc_memcg_shrinker()
257 static void unregister_memcg_shrinker(struct shrinker *shrinker)  in unregister_memcg_shrinker()  argument
259 	int id = shrinker->id;  in unregister_memcg_shrinker()
297 static int prealloc_memcg_shrinker(struct shrinker *shrinker)  in prealloc_memcg_shrinker()  argument
302 static void unregister_memcg_shrinker(struct shrinker *shrinker)  in unregister_memcg_shrinker()  argument
363 int prealloc_shrinker(struct shrinker *shrinker)  in prealloc_shrinker()  argument
365 	unsigned int size = sizeof(*shrinker->nr_deferred);  in prealloc_shrinker()
367 	if (shrinker->flags & SHRINKER_NUMA_AWARE)  in prealloc_shrinker()
370 	shrinker->nr_deferred = kzalloc(size, GFP_KERNEL);  in prealloc_shrinker()
371 	if (!shrinker->nr_deferred)  in prealloc_shrinker()
374 	if (shrinker->flags & SHRINKER_MEMCG_AWARE) {  in prealloc_shrinker()
375 		if (prealloc_memcg_shrinker(shrinker))  in prealloc_shrinker()
382 	kfree(shrinker->nr_deferred);  in prealloc_shrinker()
383 	shrinker->nr_deferred = NULL;  in prealloc_shrinker()
387 void free_prealloced_shrinker(struct shrinker *shrinker)  in free_prealloced_shrinker()  argument
389 	if (!shrinker->nr_deferred)  in free_prealloced_shrinker()
392 	if (shrinker->flags & SHRINKER_MEMCG_AWARE)  in free_prealloced_shrinker()
393 		unregister_memcg_shrinker(shrinker);  in free_prealloced_shrinker()
395 	kfree(shrinker->nr_deferred);  in free_prealloced_shrinker()
396 	shrinker->nr_deferred = NULL;  in free_prealloced_shrinker()
399 void register_shrinker_prepared(struct shrinker *shrinker)  in register_shrinker_prepared()  argument
402 	list_add_tail(&shrinker->list, &shrinker_list);  in register_shrinker_prepared()
404 	if (shrinker->flags & SHRINKER_MEMCG_AWARE)  in register_shrinker_prepared()
405 		idr_replace(&shrinker_idr, shrinker, shrinker->id);  in register_shrinker_prepared()
410 int register_shrinker(struct shrinker *shrinker)  in register_shrinker()  argument
412 	int err = prealloc_shrinker(shrinker);  in register_shrinker()
416 	register_shrinker_prepared(shrinker);  in register_shrinker()
424 void unregister_shrinker(struct shrinker *shrinker)  in unregister_shrinker()  argument
426 	if (!shrinker->nr_deferred)  in unregister_shrinker()
428 	if (shrinker->flags & SHRINKER_MEMCG_AWARE)  in unregister_shrinker()
429 		unregister_memcg_shrinker(shrinker);  in unregister_shrinker()
431 	list_del(&shrinker->list);  in unregister_shrinker()
433 	kfree(shrinker->nr_deferred);  in unregister_shrinker()
434 	shrinker->nr_deferred = NULL;  in unregister_shrinker()
441 				    struct shrinker *shrinker, int priority)  in do_shrink_slab()  argument
450 	long batch_size = shrinker->batch ? shrinker->batch  in do_shrink_slab()
454 	if (!(shrinker->flags & SHRINKER_NUMA_AWARE))  in do_shrink_slab()
457 	freeable = shrinker->count_objects(shrinker, shrinkctl);  in do_shrink_slab()
466 	nr = atomic_long_xchg(&shrinker->nr_deferred[nid], 0);  in do_shrink_slab()
469 	if (shrinker->seeks) {  in do_shrink_slab()
472 		do_div(delta, shrinker->seeks);  in do_shrink_slab()
485 		       shrinker->scan_objects, total_scan);  in do_shrink_slab()
514 	trace_mm_shrink_slab_start(shrinker, shrinkctl, nr,  in do_shrink_slab()
539 		ret = shrinker->scan_objects(shrinker, shrinkctl);  in do_shrink_slab()
562 						&shrinker->nr_deferred[nid]);  in do_shrink_slab()
564 		new_nr = atomic_long_read(&shrinker->nr_deferred[nid]);  in do_shrink_slab()
566 	trace_mm_shrink_slab_end(shrinker, nid, freed, nr, new_nr, total_scan);  in do_shrink_slab()
595 		struct shrinker *shrinker;  in shrink_slab_memcg()  local
597 		shrinker = idr_find(&shrinker_idr, i);  in shrink_slab_memcg()
598 		if (unlikely(!shrinker || shrinker == SHRINKER_REGISTERING)) {  in shrink_slab_memcg()
599 			if (!shrinker)  in shrink_slab_memcg()
606 		    !(shrinker->flags & SHRINKER_NONSLAB))  in shrink_slab_memcg()
609 		ret = do_shrink_slab(&sc, shrinker, priority);  in shrink_slab_memcg()
628 			ret = do_shrink_slab(&sc, shrinker, priority);  in shrink_slab_memcg()
678 	struct shrinker *shrinker;  in shrink_slab()  local
698 	list_for_each_entry(shrinker, &shrinker_list, list) {  in shrink_slab()
705 		ret = do_shrink_slab(&sc, shrinker, priority);  in shrink_slab()