1*4882a593Smuzhiyun // SPDX-License-Identifier: GPL-2.0-or-later
2*4882a593Smuzhiyun /*
3*4882a593Smuzhiyun * Copyright 2005, Paul Mackerras, IBM Corporation.
4*4882a593Smuzhiyun * Copyright 2009, Benjamin Herrenschmidt, IBM Corporation.
5*4882a593Smuzhiyun * Copyright 2015-2016, Aneesh Kumar K.V, IBM Corporation.
6*4882a593Smuzhiyun */
7*4882a593Smuzhiyun
8*4882a593Smuzhiyun #include <linux/sched.h>
9*4882a593Smuzhiyun #include <linux/memblock.h>
10*4882a593Smuzhiyun #include <asm/pgalloc.h>
11*4882a593Smuzhiyun #include <asm/tlb.h>
12*4882a593Smuzhiyun #include <asm/dma.h>
13*4882a593Smuzhiyun
14*4882a593Smuzhiyun #include <mm/mmu_decl.h>
15*4882a593Smuzhiyun
16*4882a593Smuzhiyun #ifdef CONFIG_SPARSEMEM_VMEMMAP
17*4882a593Smuzhiyun /*
18*4882a593Smuzhiyun * On Book3E CPUs, the vmemmap is currently mapped in the top half of
19*4882a593Smuzhiyun * the vmalloc space using normal page tables, though the size of
20*4882a593Smuzhiyun * pages encoded in the PTEs can be different
21*4882a593Smuzhiyun */
vmemmap_create_mapping(unsigned long start,unsigned long page_size,unsigned long phys)22*4882a593Smuzhiyun int __meminit vmemmap_create_mapping(unsigned long start,
23*4882a593Smuzhiyun unsigned long page_size,
24*4882a593Smuzhiyun unsigned long phys)
25*4882a593Smuzhiyun {
26*4882a593Smuzhiyun /* Create a PTE encoding without page size */
27*4882a593Smuzhiyun unsigned long i, flags = _PAGE_PRESENT | _PAGE_ACCESSED |
28*4882a593Smuzhiyun _PAGE_KERNEL_RW;
29*4882a593Smuzhiyun
30*4882a593Smuzhiyun /* PTEs only contain page size encodings up to 32M */
31*4882a593Smuzhiyun BUG_ON(mmu_psize_defs[mmu_vmemmap_psize].enc > 0xf);
32*4882a593Smuzhiyun
33*4882a593Smuzhiyun /* Encode the size in the PTE */
34*4882a593Smuzhiyun flags |= mmu_psize_defs[mmu_vmemmap_psize].enc << 8;
35*4882a593Smuzhiyun
36*4882a593Smuzhiyun /* For each PTE for that area, map things. Note that we don't
37*4882a593Smuzhiyun * increment phys because all PTEs are of the large size and
38*4882a593Smuzhiyun * thus must have the low bits clear
39*4882a593Smuzhiyun */
40*4882a593Smuzhiyun for (i = 0; i < page_size; i += PAGE_SIZE)
41*4882a593Smuzhiyun BUG_ON(map_kernel_page(start + i, phys, __pgprot(flags)));
42*4882a593Smuzhiyun
43*4882a593Smuzhiyun return 0;
44*4882a593Smuzhiyun }
45*4882a593Smuzhiyun
46*4882a593Smuzhiyun #ifdef CONFIG_MEMORY_HOTPLUG
vmemmap_remove_mapping(unsigned long start,unsigned long page_size)47*4882a593Smuzhiyun void vmemmap_remove_mapping(unsigned long start,
48*4882a593Smuzhiyun unsigned long page_size)
49*4882a593Smuzhiyun {
50*4882a593Smuzhiyun }
51*4882a593Smuzhiyun #endif
52*4882a593Smuzhiyun #endif /* CONFIG_SPARSEMEM_VMEMMAP */
53*4882a593Smuzhiyun
early_alloc_pgtable(unsigned long size)54*4882a593Smuzhiyun static void __init *early_alloc_pgtable(unsigned long size)
55*4882a593Smuzhiyun {
56*4882a593Smuzhiyun void *ptr;
57*4882a593Smuzhiyun
58*4882a593Smuzhiyun ptr = memblock_alloc_try_nid(size, size, MEMBLOCK_LOW_LIMIT,
59*4882a593Smuzhiyun __pa(MAX_DMA_ADDRESS), NUMA_NO_NODE);
60*4882a593Smuzhiyun
61*4882a593Smuzhiyun if (!ptr)
62*4882a593Smuzhiyun panic("%s: Failed to allocate %lu bytes align=0x%lx max_addr=%lx\n",
63*4882a593Smuzhiyun __func__, size, size, __pa(MAX_DMA_ADDRESS));
64*4882a593Smuzhiyun
65*4882a593Smuzhiyun return ptr;
66*4882a593Smuzhiyun }
67*4882a593Smuzhiyun
68*4882a593Smuzhiyun /*
69*4882a593Smuzhiyun * map_kernel_page currently only called by __ioremap
70*4882a593Smuzhiyun * map_kernel_page adds an entry to the ioremap page table
71*4882a593Smuzhiyun * and adds an entry to the HPT, possibly bolting it
72*4882a593Smuzhiyun */
map_kernel_page(unsigned long ea,unsigned long pa,pgprot_t prot)73*4882a593Smuzhiyun int __ref map_kernel_page(unsigned long ea, unsigned long pa, pgprot_t prot)
74*4882a593Smuzhiyun {
75*4882a593Smuzhiyun pgd_t *pgdp;
76*4882a593Smuzhiyun p4d_t *p4dp;
77*4882a593Smuzhiyun pud_t *pudp;
78*4882a593Smuzhiyun pmd_t *pmdp;
79*4882a593Smuzhiyun pte_t *ptep;
80*4882a593Smuzhiyun
81*4882a593Smuzhiyun BUILD_BUG_ON(TASK_SIZE_USER64 > PGTABLE_RANGE);
82*4882a593Smuzhiyun if (slab_is_available()) {
83*4882a593Smuzhiyun pgdp = pgd_offset_k(ea);
84*4882a593Smuzhiyun p4dp = p4d_offset(pgdp, ea);
85*4882a593Smuzhiyun pudp = pud_alloc(&init_mm, p4dp, ea);
86*4882a593Smuzhiyun if (!pudp)
87*4882a593Smuzhiyun return -ENOMEM;
88*4882a593Smuzhiyun pmdp = pmd_alloc(&init_mm, pudp, ea);
89*4882a593Smuzhiyun if (!pmdp)
90*4882a593Smuzhiyun return -ENOMEM;
91*4882a593Smuzhiyun ptep = pte_alloc_kernel(pmdp, ea);
92*4882a593Smuzhiyun if (!ptep)
93*4882a593Smuzhiyun return -ENOMEM;
94*4882a593Smuzhiyun } else {
95*4882a593Smuzhiyun pgdp = pgd_offset_k(ea);
96*4882a593Smuzhiyun p4dp = p4d_offset(pgdp, ea);
97*4882a593Smuzhiyun if (p4d_none(*p4dp)) {
98*4882a593Smuzhiyun pudp = early_alloc_pgtable(PUD_TABLE_SIZE);
99*4882a593Smuzhiyun p4d_populate(&init_mm, p4dp, pudp);
100*4882a593Smuzhiyun }
101*4882a593Smuzhiyun pudp = pud_offset(p4dp, ea);
102*4882a593Smuzhiyun if (pud_none(*pudp)) {
103*4882a593Smuzhiyun pmdp = early_alloc_pgtable(PMD_TABLE_SIZE);
104*4882a593Smuzhiyun pud_populate(&init_mm, pudp, pmdp);
105*4882a593Smuzhiyun }
106*4882a593Smuzhiyun pmdp = pmd_offset(pudp, ea);
107*4882a593Smuzhiyun if (!pmd_present(*pmdp)) {
108*4882a593Smuzhiyun ptep = early_alloc_pgtable(PTE_TABLE_SIZE);
109*4882a593Smuzhiyun pmd_populate_kernel(&init_mm, pmdp, ptep);
110*4882a593Smuzhiyun }
111*4882a593Smuzhiyun ptep = pte_offset_kernel(pmdp, ea);
112*4882a593Smuzhiyun }
113*4882a593Smuzhiyun set_pte_at(&init_mm, ea, ptep, pfn_pte(pa >> PAGE_SHIFT, prot));
114*4882a593Smuzhiyun
115*4882a593Smuzhiyun smp_wmb();
116*4882a593Smuzhiyun return 0;
117*4882a593Smuzhiyun }
118