33 static void hpage_pincount_add(struct page *page, int refs)  in hpage_pincount_add()  argument
38 	atomic_add(refs, compound_pincount_ptr(page));  in hpage_pincount_add()
41 static void hpage_pincount_sub(struct page *page, int refs)  in hpage_pincount_sub()  argument
46 	atomic_sub(refs, compound_pincount_ptr(page));  in hpage_pincount_sub()
50 static void put_page_refs(struct page *page, int refs)  in put_page_refs()  argument
53 	if (VM_WARN_ON_ONCE_PAGE(page_ref_count(page) < refs, page))  in put_page_refs()
61 	if (refs > 1)  in put_page_refs()
62 		page_ref_sub(page, refs - 1);  in put_page_refs()
70 static inline struct page *try_get_compound_head(struct page *page, int refs)  in try_get_compound_head()  argument
76 	if (unlikely(!page_cache_add_speculative(head, refs)))  in try_get_compound_head()
89 		put_page_refs(head, refs);  in try_get_compound_head()
116 							  int refs,  in try_grab_compound_head()  argument
120 		struct page *head = try_get_compound_head(page, refs);  in try_grab_compound_head()
125 		int orig_refs = refs;  in try_grab_compound_head()
139 		page = try_get_compound_head(page, refs);  in try_grab_compound_head()
152 			hpage_pincount_add(page, refs);  in try_grab_compound_head()
154 			page_ref_add(page, refs * (GUP_PIN_COUNTING_BIAS - 1));  in try_grab_compound_head()
166 static void put_compound_head(struct page *page, int refs, unsigned int flags)  in put_compound_head()  argument
170 				    refs);  in put_compound_head()
173 			hpage_pincount_sub(page, refs);  in put_compound_head()
175 			refs *= GUP_PIN_COUNTING_BIAS;  in put_compound_head()
180 	put_page_refs(page, refs);  in put_compound_head()
217 		int refs = 1;  in try_grab_page()  local
227 			refs = GUP_PIN_COUNTING_BIAS;  in try_grab_page()
235 		page_ref_add(page, refs);  in try_grab_page()
2346 	int refs;  in gup_hugepte()  local
2362 	refs = record_subpages(page, addr, end, pages + *nr);  in gup_hugepte()
2364 	head = try_grab_compound_head(head, refs, flags);  in gup_hugepte()
2369 		put_compound_head(head, refs, flags);  in gup_hugepte()
2373 	*nr += refs;  in gup_hugepte()
2409 	int refs;  in gup_huge_pmd()  local
2422 	refs = record_subpages(page, addr, end, pages + *nr);  in gup_huge_pmd()
2424 	head = try_grab_compound_head(pmd_page(orig), refs, flags);  in gup_huge_pmd()
2429 		put_compound_head(head, refs, flags);  in gup_huge_pmd()
2433 	*nr += refs;  in gup_huge_pmd()
2443 	int refs;  in gup_huge_pud()  local
2456 	refs = record_subpages(page, addr, end, pages + *nr);  in gup_huge_pud()
2458 	head = try_grab_compound_head(pud_page(orig), refs, flags);  in gup_huge_pud()
2463 		put_compound_head(head, refs, flags);  in gup_huge_pud()
2467 	*nr += refs;  in gup_huge_pud()
2476 	int refs;  in gup_huge_pgd()  local
2485 	refs = record_subpages(page, addr, end, pages + *nr);  in gup_huge_pgd()
2487 	head = try_grab_compound_head(pgd_page(orig), refs, flags);  in gup_huge_pgd()
2492 		put_compound_head(head, refs, flags);  in gup_huge_pgd()
2496 	*nr += refs;  in gup_huge_pgd()