2252 static void drain_obj_stock(struct memcg_stock_pcp *stock);
2253 static bool obj_stock_flush_required(struct memcg_stock_pcp *stock,
2257 static inline void drain_obj_stock(struct memcg_stock_pcp *stock)  in drain_obj_stock()  argument
2260 static bool obj_stock_flush_required(struct memcg_stock_pcp *stock,  in obj_stock_flush_required()  argument
2280 	struct memcg_stock_pcp *stock;  in consume_stock()  local
2289 	stock = this_cpu_ptr(&memcg_stock);  in consume_stock()
2290 	if (memcg == stock->cached && stock->nr_pages >= nr_pages) {  in consume_stock()
2291 		stock->nr_pages -= nr_pages;  in consume_stock()
2303 static void drain_stock(struct memcg_stock_pcp *stock)  in drain_stock()  argument
2305 	struct mem_cgroup *old = stock->cached;  in drain_stock()
2310 	if (stock->nr_pages) {  in drain_stock()
2311 		page_counter_uncharge(&old->memory, stock->nr_pages);  in drain_stock()
2313 			page_counter_uncharge(&old->memsw, stock->nr_pages);  in drain_stock()
2314 		stock->nr_pages = 0;  in drain_stock()
2318 	stock->cached = NULL;  in drain_stock()
2323 	struct memcg_stock_pcp *stock;  in drain_local_stock()  local
2332 	stock = this_cpu_ptr(&memcg_stock);  in drain_local_stock()
2333 	drain_obj_stock(stock);  in drain_local_stock()
2334 	drain_stock(stock);  in drain_local_stock()
2335 	clear_bit(FLUSHING_CACHED_CHARGE, &stock->flags);  in drain_local_stock()
2346 	struct memcg_stock_pcp *stock;  in refill_stock()  local
2351 	stock = this_cpu_ptr(&memcg_stock);  in refill_stock()
2352 	if (stock->cached != memcg) { /* reset if necessary */  in refill_stock()
2353 		drain_stock(stock);  in refill_stock()
2355 		stock->cached = memcg;  in refill_stock()
2357 	stock->nr_pages += nr_pages;  in refill_stock()
2359 	if (stock->nr_pages > MEMCG_CHARGE_BATCH)  in refill_stock()
2360 		drain_stock(stock);  in refill_stock()
2384 		struct memcg_stock_pcp *stock = &per_cpu(memcg_stock, cpu);  in drain_all_stock()  local
2389 		memcg = stock->cached;  in drain_all_stock()
2390 		if (memcg && stock->nr_pages &&  in drain_all_stock()
2393 		if (obj_stock_flush_required(stock, root_memcg))  in drain_all_stock()
2398 		    !test_and_set_bit(FLUSHING_CACHED_CHARGE, &stock->flags)) {  in drain_all_stock()
2400 				drain_local_stock(&stock->work);  in drain_all_stock()
2402 				schedule_work_on(cpu, &stock->work);  in drain_all_stock()
2411 	struct memcg_stock_pcp *stock;  in memcg_hotplug_cpu_dead()  local
2414 	stock = &per_cpu(memcg_stock, cpu);  in memcg_hotplug_cpu_dead()
2415 	drain_stock(stock);  in memcg_hotplug_cpu_dead()
3166 	struct memcg_stock_pcp *stock;  in consume_obj_stock()  local
3172 	stock = this_cpu_ptr(&memcg_stock);  in consume_obj_stock()
3173 	if (objcg == stock->cached_objcg && stock->nr_bytes >= nr_bytes) {  in consume_obj_stock()
3174 		stock->nr_bytes -= nr_bytes;  in consume_obj_stock()
3183 static void drain_obj_stock(struct memcg_stock_pcp *stock)  in drain_obj_stock()  argument
3185 	struct obj_cgroup *old = stock->cached_objcg;  in drain_obj_stock()
3190 	if (stock->nr_bytes) {  in drain_obj_stock()
3191 		unsigned int nr_pages = stock->nr_bytes >> PAGE_SHIFT;  in drain_obj_stock()
3192 		unsigned int nr_bytes = stock->nr_bytes & (PAGE_SIZE - 1);  in drain_obj_stock()
3219 		stock->nr_bytes = 0;  in drain_obj_stock()
3223 	stock->cached_objcg = NULL;  in drain_obj_stock()
3226 static bool obj_stock_flush_required(struct memcg_stock_pcp *stock,  in obj_stock_flush_required()  argument
3231 	if (stock->cached_objcg) {  in obj_stock_flush_required()
3232 		memcg = obj_cgroup_memcg(stock->cached_objcg);  in obj_stock_flush_required()
3242 	struct memcg_stock_pcp *stock;  in refill_obj_stock()  local
3247 	stock = this_cpu_ptr(&memcg_stock);  in refill_obj_stock()
3248 	if (stock->cached_objcg != objcg) { /* reset if necessary */  in refill_obj_stock()
3249 		drain_obj_stock(stock);  in refill_obj_stock()
3251 		stock->cached_objcg = objcg;  in refill_obj_stock()
3252 		stock->nr_bytes = atomic_xchg(&objcg->nr_charged_bytes, 0);  in refill_obj_stock()
3254 	stock->nr_bytes += nr_bytes;  in refill_obj_stock()
3256 	if (stock->nr_bytes > PAGE_SIZE)  in refill_obj_stock()
3257 		drain_obj_stock(stock);  in refill_obj_stock()