3671 static bool __should_fail_alloc_page(gfp_t gfp_mask, unsigned int order)  in __should_fail_alloc_page()  argument
3675 	if (gfp_mask & __GFP_NOFAIL)  in __should_fail_alloc_page()
3677 	if (fail_page_alloc.ignore_gfp_highmem && (gfp_mask & __GFP_HIGHMEM))  in __should_fail_alloc_page()
3680 			(gfp_mask & __GFP_DIRECT_RECLAIM))  in __should_fail_alloc_page()
3711 static inline bool __should_fail_alloc_page(gfp_t gfp_mask, unsigned int order)  in __should_fail_alloc_page()  argument
3718 noinline bool should_fail_alloc_page(gfp_t gfp_mask, unsigned int order)  in should_fail_alloc_page()  argument
3720 	return __should_fail_alloc_page(gfp_mask, order);  in should_fail_alloc_page()
3835 				unsigned int alloc_flags, gfp_t gfp_mask)  in zone_watermark_fast()  argument
3867 	if (unlikely(!order && (gfp_mask & __GFP_ATOMIC) && z->watermark_boost  in zone_watermark_fast()
3912 alloc_flags_nofragment(struct zone *zone, gfp_t gfp_mask)  in alloc_flags_nofragment()  argument
3920 	alloc_flags = (__force int) (gfp_mask & __GFP_KSWAPD_RECLAIM);  in alloc_flags_nofragment()
3943 static inline unsigned int current_alloc_flags(gfp_t gfp_mask,  in current_alloc_flags()  argument
3950 			gfp_migratetype(gfp_mask) == MIGRATE_MOVABLE &&  in current_alloc_flags()
3951 			gfp_mask & __GFP_CMA)  in current_alloc_flags()
3963 get_page_from_freelist(gfp_t gfp_mask, unsigned int order, int alloc_flags,  in get_page_from_freelist()  argument
3985 			!__cpuset_zone_allowed(zone, gfp_mask))  in get_page_from_freelist()
4035 				       gfp_mask)) {  in get_page_from_freelist()
4057 			ret = node_reclaim(zone->zone_pgdat, gfp_mask, order);  in get_page_from_freelist()
4077 				gfp_mask, alloc_flags, ac->migratetype);  in get_page_from_freelist()
4079 			prep_new_page(page, order, gfp_mask, alloc_flags);  in get_page_from_freelist()
4112 static void warn_alloc_show_mem(gfp_t gfp_mask, nodemask_t *nodemask)  in warn_alloc_show_mem()  argument
4121 	if (!(gfp_mask & __GFP_NOMEMALLOC))  in warn_alloc_show_mem()
4125 	if (in_interrupt() || !(gfp_mask & __GFP_DIRECT_RECLAIM))  in warn_alloc_show_mem()
4131 void warn_alloc(gfp_t gfp_mask, nodemask_t *nodemask, const char *fmt, ...)  in warn_alloc()  argument
4137 	if ((gfp_mask & __GFP_NOWARN) ||  in warn_alloc()
4139 	     ((gfp_mask & __GFP_DMA) && !has_managed_dma()))  in warn_alloc()
4146 			current->comm, &vaf, gfp_mask, &gfp_mask,  in warn_alloc()
4153 	warn_alloc_show_mem(gfp_mask, nodemask);  in warn_alloc()
4157 __alloc_pages_cpuset_fallback(gfp_t gfp_mask, unsigned int order,  in __alloc_pages_cpuset_fallback()  argument
4163 	page = get_page_from_freelist(gfp_mask, order,  in __alloc_pages_cpuset_fallback()
4170 		page = get_page_from_freelist(gfp_mask, order,  in __alloc_pages_cpuset_fallback()
4177 __alloc_pages_may_oom(gfp_t gfp_mask, unsigned int order,  in __alloc_pages_may_oom()  argument
4184 		.gfp_mask = gfp_mask,  in __alloc_pages_may_oom()
4208 	page = get_page_from_freelist((gfp_mask | __GFP_HARDWALL) &  in __alloc_pages_may_oom()
4228 	if (gfp_mask & (__GFP_RETRY_MAYFAIL | __GFP_THISNODE))  in __alloc_pages_may_oom()
4246 	if (out_of_memory(&oc) || WARN_ON_ONCE(gfp_mask & __GFP_NOFAIL)) {  in __alloc_pages_may_oom()
4253 		if (gfp_mask & __GFP_NOFAIL)  in __alloc_pages_may_oom()
4254 			page = __alloc_pages_cpuset_fallback(gfp_mask, order,  in __alloc_pages_may_oom()
4271 __alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order,  in __alloc_pages_direct_compact()  argument
4285 	*compact_result = try_to_compact_pages(gfp_mask, order, alloc_flags, ac,  in __alloc_pages_direct_compact()
4299 		prep_new_page(page, order, gfp_mask, alloc_flags);  in __alloc_pages_direct_compact()
4303 		page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac);  in __alloc_pages_direct_compact()
4404 __alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order,  in __alloc_pages_direct_compact()  argument
4444 static bool __need_fs_reclaim(gfp_t gfp_mask)  in __need_fs_reclaim()  argument
4446 	gfp_mask = current_gfp_context(gfp_mask);  in __need_fs_reclaim()
4449 	if (!(gfp_mask & __GFP_DIRECT_RECLAIM))  in __need_fs_reclaim()
4457 	if (!(gfp_mask & __GFP_FS))  in __need_fs_reclaim()
4460 	if (gfp_mask & __GFP_NOLOCKDEP)  in __need_fs_reclaim()
4476 void fs_reclaim_acquire(gfp_t gfp_mask)  in fs_reclaim_acquire()  argument
4478 	if (__need_fs_reclaim(gfp_mask))  in fs_reclaim_acquire()
4483 void fs_reclaim_release(gfp_t gfp_mask)  in fs_reclaim_release()  argument
4485 	if (__need_fs_reclaim(gfp_mask))  in fs_reclaim_release()
4517 __perform_reclaim(gfp_t gfp_mask, unsigned int order,  in __perform_reclaim()  argument
4527 	fs_reclaim_acquire(gfp_mask);  in __perform_reclaim()
4530 	progress = try_to_free_pages(ac->zonelist, order, gfp_mask,  in __perform_reclaim()
4534 	fs_reclaim_release(gfp_mask);  in __perform_reclaim()
4543 __alloc_pages_direct_reclaim(gfp_t gfp_mask, unsigned int order,  in __alloc_pages_direct_reclaim()  argument
4553 	*did_some_progress = __perform_reclaim(gfp_mask, order, ac);  in __alloc_pages_direct_reclaim()
4558 	page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac);  in __alloc_pages_direct_reclaim()
4567 		trace_android_vh_drain_all_pages_bypass(gfp_mask, order,  in __alloc_pages_direct_reclaim()
4580 static void wake_all_kswapds(unsigned int order, gfp_t gfp_mask,  in wake_all_kswapds()  argument
4591 			wakeup_kswapd(zone, gfp_mask, order, highest_zoneidx);  in wake_all_kswapds()
4597 gfp_to_alloc_flags(gfp_t gfp_mask)  in gfp_to_alloc_flags()  argument
4616 		(gfp_mask & (__GFP_HIGH | __GFP_KSWAPD_RECLAIM));  in gfp_to_alloc_flags()
4618 	if (gfp_mask & __GFP_ATOMIC) {  in gfp_to_alloc_flags()
4623 		if (!(gfp_mask & __GFP_NOMEMALLOC))  in gfp_to_alloc_flags()
4633 	alloc_flags = current_alloc_flags(gfp_mask, alloc_flags);  in gfp_to_alloc_flags()
4657 static inline int __gfp_pfmemalloc_flags(gfp_t gfp_mask)  in __gfp_pfmemalloc_flags()  argument
4659 	if (unlikely(gfp_mask & __GFP_NOMEMALLOC))  in __gfp_pfmemalloc_flags()
4661 	if (gfp_mask & __GFP_MEMALLOC)  in __gfp_pfmemalloc_flags()
4675 bool gfp_pfmemalloc_allowed(gfp_t gfp_mask)  in gfp_pfmemalloc_allowed()  argument
4677 	return !!__gfp_pfmemalloc_flags(gfp_mask);  in gfp_pfmemalloc_allowed()
4691 should_reclaim_retry(gfp_t gfp_mask, unsigned order,  in should_reclaim_retry()  argument
4815 __alloc_pages_slowpath(gfp_t gfp_mask, unsigned int order,  in __alloc_pages_slowpath()  argument
4818 	bool can_direct_reclaim = gfp_mask & __GFP_DIRECT_RECLAIM;  in __alloc_pages_slowpath()
4832 	trace_android_vh_alloc_pages_slowpath_begin(gfp_mask, order, &vh_record);  in __alloc_pages_slowpath()
4837 	if (WARN_ON_ONCE((gfp_mask & (__GFP_ATOMIC|__GFP_DIRECT_RECLAIM)) ==  in __alloc_pages_slowpath()
4839 		gfp_mask &= ~__GFP_ATOMIC;  in __alloc_pages_slowpath()
4853 	alloc_flags = gfp_to_alloc_flags(gfp_mask);  in __alloc_pages_slowpath()
4867 		wake_all_kswapds(order, gfp_mask, ac);  in __alloc_pages_slowpath()
4873 	page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac);  in __alloc_pages_slowpath()
4889 			&& !gfp_pfmemalloc_allowed(gfp_mask)) {  in __alloc_pages_slowpath()
4890 		page = __alloc_pages_direct_compact(gfp_mask, order,  in __alloc_pages_slowpath()
4901 		if (costly_order && (gfp_mask & __GFP_NORETRY)) {  in __alloc_pages_slowpath()
4935 		wake_all_kswapds(order, gfp_mask, ac);  in __alloc_pages_slowpath()
4937 	reserve_flags = __gfp_pfmemalloc_flags(gfp_mask);  in __alloc_pages_slowpath()
4939 		alloc_flags = current_alloc_flags(gfp_mask, reserve_flags);  in __alloc_pages_slowpath()
4953 	page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac);  in __alloc_pages_slowpath()
4965 	trace_android_vh_alloc_pages_reclaim_bypass(gfp_mask, order,  in __alloc_pages_slowpath()
4972 	page = __alloc_pages_direct_reclaim(gfp_mask, order, alloc_flags, ac,  in __alloc_pages_slowpath()
4978 	page = __alloc_pages_direct_compact(gfp_mask, order, alloc_flags, ac,  in __alloc_pages_slowpath()
4984 	if (gfp_mask & __GFP_NORETRY)  in __alloc_pages_slowpath()
4991 	if (costly_order && !(gfp_mask & __GFP_RETRY_MAYFAIL))  in __alloc_pages_slowpath()
4994 	if (should_reclaim_retry(gfp_mask, order, ac, alloc_flags,  in __alloc_pages_slowpath()
5020 	page = __alloc_pages_may_oom(gfp_mask, order, ac, &did_some_progress);  in __alloc_pages_slowpath()
5027 	     (gfp_mask & __GFP_NOMEMALLOC)))  in __alloc_pages_slowpath()
5049 	if (gfp_mask & __GFP_NOFAIL) {  in __alloc_pages_slowpath()
5078 		page = __alloc_pages_cpuset_fallback(gfp_mask, order, ALLOC_HARDER, ac);  in __alloc_pages_slowpath()
5086 	trace_android_vh_alloc_pages_failure_bypass(gfp_mask, order,  in __alloc_pages_slowpath()
5091 	warn_alloc(gfp_mask, ac->nodemask,  in __alloc_pages_slowpath()
5094 	trace_android_vh_alloc_pages_slowpath_end(gfp_mask, order, vh_record);  in __alloc_pages_slowpath()
5098 static inline bool prepare_alloc_pages(gfp_t gfp_mask, unsigned int order,  in prepare_alloc_pages()  argument
5103 	ac->highest_zoneidx = gfp_zone(gfp_mask);  in prepare_alloc_pages()
5104 	ac->zonelist = node_zonelist(preferred_nid, gfp_mask);  in prepare_alloc_pages()
5106 	ac->migratetype = gfp_migratetype(gfp_mask);  in prepare_alloc_pages()
5120 	fs_reclaim_acquire(gfp_mask);  in prepare_alloc_pages()
5121 	fs_reclaim_release(gfp_mask);  in prepare_alloc_pages()
5123 	might_sleep_if(gfp_mask & __GFP_DIRECT_RECLAIM);  in prepare_alloc_pages()
5125 	if (should_fail_alloc_page(gfp_mask, order))  in prepare_alloc_pages()
5128 	*alloc_flags = current_alloc_flags(gfp_mask, *alloc_flags);  in prepare_alloc_pages()
5131 	ac->spread_dirty_pages = (gfp_mask & __GFP_WRITE);  in prepare_alloc_pages()
5148 __alloc_pages_nodemask(gfp_t gfp_mask, unsigned int order, int preferred_nid,  in __alloc_pages_nodemask()  argument
5161 		WARN_ON_ONCE(!(gfp_mask & __GFP_NOWARN));  in __alloc_pages_nodemask()
5165 	gfp_mask &= gfp_allowed_mask;  in __alloc_pages_nodemask()
5166 	alloc_mask = gfp_mask;  in __alloc_pages_nodemask()
5167 	if (!prepare_alloc_pages(gfp_mask, order, preferred_nid, nodemask, &ac, &alloc_mask, &alloc_flags))  in __alloc_pages_nodemask()
5174 	alloc_flags |= alloc_flags_nofragment(ac.preferred_zoneref->zone, gfp_mask);  in __alloc_pages_nodemask()
5187 	alloc_mask = current_gfp_context(gfp_mask);  in __alloc_pages_nodemask()
5199 	if (memcg_kmem_enabled() && (gfp_mask & __GFP_ACCOUNT) && page &&  in __alloc_pages_nodemask()
5200 	    unlikely(__memcg_kmem_charge_page(page, gfp_mask, order) != 0)) {  in __alloc_pages_nodemask()
5216 unsigned long __get_free_pages(gfp_t gfp_mask, unsigned int order)  in __get_free_pages()  argument
5220 	page = alloc_pages(gfp_mask & ~__GFP_HIGHMEM, order);  in __get_free_pages()
5227 unsigned long get_zeroed_page(gfp_t gfp_mask)  in get_zeroed_page()  argument
5229 	return __get_free_pages(gfp_mask | __GFP_ZERO, 0);  in get_zeroed_page()
5274 					     gfp_t gfp_mask)  in __page_frag_cache_refill()  argument
5277 	gfp_t gfp = gfp_mask;  in __page_frag_cache_refill()
5280 	gfp_mask |= __GFP_COMP | __GFP_NOWARN | __GFP_NORETRY |  in __page_frag_cache_refill()
5282 	page = alloc_pages_node(NUMA_NO_NODE, gfp_mask,  in __page_frag_cache_refill()
5304 		      unsigned int fragsz, gfp_t gfp_mask)  in page_frag_alloc()  argument
5312 		page = __page_frag_cache_refill(nc, gfp_mask);  in page_frag_alloc()
5417 void *alloc_pages_exact(size_t size, gfp_t gfp_mask)  in alloc_pages_exact()  argument
5422 	if (WARN_ON_ONCE(gfp_mask & __GFP_COMP))  in alloc_pages_exact()
5423 		gfp_mask &= ~__GFP_COMP;  in alloc_pages_exact()
5425 	addr = __get_free_pages(gfp_mask, order);  in alloc_pages_exact()
5442 void * __meminit alloc_pages_exact_nid(int nid, size_t size, gfp_t gfp_mask)  in alloc_pages_exact_nid()  argument
5447 	if (WARN_ON_ONCE(gfp_mask & __GFP_COMP))  in alloc_pages_exact_nid()
5448 		gfp_mask &= ~__GFP_COMP;  in alloc_pages_exact_nid()
5450 	p = alloc_pages_node(nid, gfp_mask, order);  in alloc_pages_exact_nid()
8716 		.gfp_mask = GFP_USER | __GFP_MOVABLE | __GFP_RETRY_MAYFAIL,  in __alloc_contig_migrate_range()
8798 		       unsigned migratetype, gfp_t gfp_mask,  in alloc_contig_range()  argument
8810 		.mode = gfp_mask & __GFP_NORETRY ? MIGRATE_ASYNC : MIGRATE_SYNC,  in alloc_contig_range()
8813 		.gfp_mask = current_gfp_context(gfp_mask),  in alloc_contig_range()
8866 	if (ret && (ret != -EBUSY || (gfp_mask & __GFP_NORETRY)))  in alloc_contig_range()
8940 				unsigned long nr_pages, gfp_t gfp_mask)  in __alloc_contig_pages()  argument
8946 				  gfp_mask, &dummy);  in __alloc_contig_pages()
9004 struct page *alloc_contig_pages(unsigned long nr_pages, gfp_t gfp_mask,  in alloc_contig_pages()  argument
9012 	zonelist = node_zonelist(nid, gfp_mask);  in alloc_contig_pages()
9014 					gfp_zone(gfp_mask), nodemask) {  in alloc_contig_pages()
9029 							gfp_mask);  in alloc_contig_pages()