72 DEFINE_POPULATE(p4d_populate, p4d, pud, init)
74 DEFINE_POPULATE(pud_populate, pud, pmd, init)
88 DEFINE_ENTRY(pud, pud, init)
258 		pud_t *pud = (pud_t *)spp_getpage();  in fill_pud()  local
259 		p4d_populate(&init_mm, p4d, pud);  in fill_pud()
260 		if (pud != pud_offset(p4d, 0))  in fill_pud()
262 			       pud, pud_offset(p4d, 0));  in fill_pud()
267 static pmd_t *fill_pmd(pud_t *pud, unsigned long vaddr)  in fill_pmd()  argument
269 	if (pud_none(*pud)) {  in fill_pmd()
271 		pud_populate(&init_mm, pud, pmd);  in fill_pmd()
272 		if (pmd != pmd_offset(pud, 0))  in fill_pmd()
274 			       pmd, pmd_offset(pud, 0));  in fill_pmd()
276 	return pmd_offset(pud, vaddr);  in fill_pmd()
290 static void __set_pte_vaddr(pud_t *pud, unsigned long vaddr, pte_t new_pte)  in __set_pte_vaddr()  argument
292 	pmd_t *pmd = fill_pmd(pud, vaddr);  in __set_pte_vaddr()
307 	pud_t *pud = fill_pud(p4d, vaddr);  in set_pte_vaddr_p4d()  local
309 	__set_pte_vaddr(pud, vaddr, new_pte);  in set_pte_vaddr_p4d()
314 	pud_t *pud = pud_page + pud_index(vaddr);  in set_pte_vaddr_pud()  local
316 	__set_pte_vaddr(pud, vaddr, new_pte);  in set_pte_vaddr_pud()
341 	pud_t *pud;  in populate_extra_pmd()  local
345 	pud = fill_pud(p4d, vaddr);  in populate_extra_pmd()
346 	return fill_pmd(pud, vaddr);  in populate_extra_pmd()
365 	pud_t *pud;  in __init_extra_mapping()  local
381 			pud = (pud_t *) spp_getpage();  in __init_extra_mapping()
382 			set_p4d(p4d, __p4d(__pa(pud) | _KERNPG_TABLE |  in __init_extra_mapping()
385 		pud = pud_offset(p4d, (unsigned long)__va(phys));  in __init_extra_mapping()
386 		if (pud_none(*pud)) {  in __init_extra_mapping()
388 			set_pud(pud, __pud(__pa(pmd) | _KERNPG_TABLE |  in __init_extra_mapping()
391 		pmd = pmd_offset(pud, phys);  in __init_extra_mapping()
596 		pud_t *pud;  in phys_pud_init()  local
601 		pud = pud_page + pud_index(vaddr);  in phys_pud_init()
610 				set_pud_init(pud, __pud(0), init);  in phys_pud_init()
614 		if (!pud_none(*pud)) {  in phys_pud_init()
615 			if (!pud_large(*pud)) {  in phys_pud_init()
616 				pmd = pmd_offset(pud, 0);  in phys_pud_init()
641 			prot = pte_pgprot(pte_clrhuge(*(pte_t *)pud));  in phys_pud_init()
650 			set_pte_init((pte_t *)pud,  in phys_pud_init()
664 		pud_populate_init(&init_mm, pud, pmd, init);  in phys_pud_init()
689 		pud_t *pud;  in phys_p4d_init()  local
706 			pud = pud_offset(p4d, 0);  in phys_p4d_init()
707 			paddr_last = phys_pud_init(pud, paddr, __pa(vaddr_end),  in phys_p4d_init()
712 		pud = alloc_low_page();  in phys_p4d_init()
713 		paddr_last = phys_pud_init(pud, paddr, __pa(vaddr_end),  in phys_p4d_init()
717 		p4d_populate_init(&init_mm, p4d, pud, init);  in phys_p4d_init()
923 static void __meminit free_pmd_table(pmd_t *pmd_start, pud_t *pud)  in free_pmd_table()  argument
935 	free_pagetable(pud_page(*pud), 0);  in free_pmd_table()
937 	pud_clear(pud);  in free_pmd_table()
943 	pud_t *pud;  in free_pud_table()  local
947 		pud = pud_start + i;  in free_pud_table()
948 		if (!pud_none(*pud))  in free_pud_table()
1092 	pud_t *pud;  in remove_pud_table()  local
1095 	pud = pud_start + pud_index(addr);  in remove_pud_table()
1096 	for (; addr < end; addr = next, pud++) {  in remove_pud_table()
1099 		if (!pud_present(*pud))  in remove_pud_table()
1102 		if (pud_large(*pud)) {  in remove_pud_table()
1106 					free_pagetable(pud_page(*pud),  in remove_pud_table()
1110 				pud_clear(pud);  in remove_pud_table()
1117 				page_addr = page_address(pud_page(*pud));  in remove_pud_table()
1120 					free_pagetable(pud_page(*pud),  in remove_pud_table()
1124 					pud_clear(pud);  in remove_pud_table()
1132 		pmd_base = pmd_offset(pud, 0);  in remove_pud_table()
1134 		free_pmd_table(pmd_base, pud);  in remove_pud_table()
1248 		pud_t *pud;  in preallocate_vmalloc_pages()  local
1269 		pud = pud_alloc(&init_mm, p4d, addr);  in preallocate_vmalloc_pages()
1270 		if (!pud)  in preallocate_vmalloc_pages()
1380 	pud_t *pud;  in kern_addr_valid()  local
1395 	pud = pud_offset(p4d, addr);  in kern_addr_valid()
1396 	if (!pud_present(*pud))  in kern_addr_valid()
1399 	if (pud_large(*pud))  in kern_addr_valid()
1400 		return pfn_valid(pud_pfn(*pud));  in kern_addr_valid()
1402 	pmd = pmd_offset(pud, addr);  in kern_addr_valid()
1499 	pud_t *pud;  in vmemmap_populate_hugepages()  local
1513 		pud = vmemmap_pud_populate(p4d, addr, node);  in vmemmap_populate_hugepages()
1514 		if (!pud)  in vmemmap_populate_hugepages()
1517 		pmd = pmd_offset(pud, addr);  in vmemmap_populate_hugepages()
1583 	pud_t *pud;  in register_page_bootmem_memmap()  local
1605 		pud = pud_offset(p4d, addr);  in register_page_bootmem_memmap()
1606 		if (pud_none(*pud)) {  in register_page_bootmem_memmap()
1610 		get_page_bootmem(section_nr, pud_page(*pud), MIX_SECTION_INFO);  in register_page_bootmem_memmap()
1614 			pmd = pmd_offset(pud, addr);  in register_page_bootmem_memmap()
1628 			pmd = pmd_offset(pud, addr);  in register_page_bootmem_memmap()