1*4882a593Smuzhiyun // SPDX-License-Identifier: GPL-2.0
2*4882a593Smuzhiyun
3*4882a593Smuzhiyun /*
4*4882a593Smuzhiyun * Handling Page Tables through page fragments
5*4882a593Smuzhiyun *
6*4882a593Smuzhiyun */
7*4882a593Smuzhiyun
8*4882a593Smuzhiyun #include <linux/kernel.h>
9*4882a593Smuzhiyun #include <linux/gfp.h>
10*4882a593Smuzhiyun #include <linux/mm.h>
11*4882a593Smuzhiyun #include <linux/percpu.h>
12*4882a593Smuzhiyun #include <linux/hardirq.h>
13*4882a593Smuzhiyun #include <linux/hugetlb.h>
14*4882a593Smuzhiyun #include <asm/pgalloc.h>
15*4882a593Smuzhiyun #include <asm/tlbflush.h>
16*4882a593Smuzhiyun #include <asm/tlb.h>
17*4882a593Smuzhiyun
pte_frag_destroy(void * pte_frag)18*4882a593Smuzhiyun void pte_frag_destroy(void *pte_frag)
19*4882a593Smuzhiyun {
20*4882a593Smuzhiyun int count;
21*4882a593Smuzhiyun struct page *page;
22*4882a593Smuzhiyun
23*4882a593Smuzhiyun page = virt_to_page(pte_frag);
24*4882a593Smuzhiyun /* drop all the pending references */
25*4882a593Smuzhiyun count = ((unsigned long)pte_frag & ~PAGE_MASK) >> PTE_FRAG_SIZE_SHIFT;
26*4882a593Smuzhiyun /* We allow PTE_FRAG_NR fragments from a PTE page */
27*4882a593Smuzhiyun if (atomic_sub_and_test(PTE_FRAG_NR - count, &page->pt_frag_refcount)) {
28*4882a593Smuzhiyun pgtable_pte_page_dtor(page);
29*4882a593Smuzhiyun __free_page(page);
30*4882a593Smuzhiyun }
31*4882a593Smuzhiyun }
32*4882a593Smuzhiyun
get_pte_from_cache(struct mm_struct * mm)33*4882a593Smuzhiyun static pte_t *get_pte_from_cache(struct mm_struct *mm)
34*4882a593Smuzhiyun {
35*4882a593Smuzhiyun void *pte_frag, *ret;
36*4882a593Smuzhiyun
37*4882a593Smuzhiyun if (PTE_FRAG_NR == 1)
38*4882a593Smuzhiyun return NULL;
39*4882a593Smuzhiyun
40*4882a593Smuzhiyun spin_lock(&mm->page_table_lock);
41*4882a593Smuzhiyun ret = pte_frag_get(&mm->context);
42*4882a593Smuzhiyun if (ret) {
43*4882a593Smuzhiyun pte_frag = ret + PTE_FRAG_SIZE;
44*4882a593Smuzhiyun /*
45*4882a593Smuzhiyun * If we have taken up all the fragments mark PTE page NULL
46*4882a593Smuzhiyun */
47*4882a593Smuzhiyun if (((unsigned long)pte_frag & ~PAGE_MASK) == 0)
48*4882a593Smuzhiyun pte_frag = NULL;
49*4882a593Smuzhiyun pte_frag_set(&mm->context, pte_frag);
50*4882a593Smuzhiyun }
51*4882a593Smuzhiyun spin_unlock(&mm->page_table_lock);
52*4882a593Smuzhiyun return (pte_t *)ret;
53*4882a593Smuzhiyun }
54*4882a593Smuzhiyun
__alloc_for_ptecache(struct mm_struct * mm,int kernel)55*4882a593Smuzhiyun static pte_t *__alloc_for_ptecache(struct mm_struct *mm, int kernel)
56*4882a593Smuzhiyun {
57*4882a593Smuzhiyun void *ret = NULL;
58*4882a593Smuzhiyun struct page *page;
59*4882a593Smuzhiyun
60*4882a593Smuzhiyun if (!kernel) {
61*4882a593Smuzhiyun page = alloc_page(PGALLOC_GFP | __GFP_ACCOUNT);
62*4882a593Smuzhiyun if (!page)
63*4882a593Smuzhiyun return NULL;
64*4882a593Smuzhiyun if (!pgtable_pte_page_ctor(page)) {
65*4882a593Smuzhiyun __free_page(page);
66*4882a593Smuzhiyun return NULL;
67*4882a593Smuzhiyun }
68*4882a593Smuzhiyun } else {
69*4882a593Smuzhiyun page = alloc_page(PGALLOC_GFP);
70*4882a593Smuzhiyun if (!page)
71*4882a593Smuzhiyun return NULL;
72*4882a593Smuzhiyun }
73*4882a593Smuzhiyun
74*4882a593Smuzhiyun atomic_set(&page->pt_frag_refcount, 1);
75*4882a593Smuzhiyun
76*4882a593Smuzhiyun ret = page_address(page);
77*4882a593Smuzhiyun /*
78*4882a593Smuzhiyun * if we support only one fragment just return the
79*4882a593Smuzhiyun * allocated page.
80*4882a593Smuzhiyun */
81*4882a593Smuzhiyun if (PTE_FRAG_NR == 1)
82*4882a593Smuzhiyun return ret;
83*4882a593Smuzhiyun spin_lock(&mm->page_table_lock);
84*4882a593Smuzhiyun /*
85*4882a593Smuzhiyun * If we find pgtable_page set, we return
86*4882a593Smuzhiyun * the allocated page with single fragement
87*4882a593Smuzhiyun * count.
88*4882a593Smuzhiyun */
89*4882a593Smuzhiyun if (likely(!pte_frag_get(&mm->context))) {
90*4882a593Smuzhiyun atomic_set(&page->pt_frag_refcount, PTE_FRAG_NR);
91*4882a593Smuzhiyun pte_frag_set(&mm->context, ret + PTE_FRAG_SIZE);
92*4882a593Smuzhiyun }
93*4882a593Smuzhiyun spin_unlock(&mm->page_table_lock);
94*4882a593Smuzhiyun
95*4882a593Smuzhiyun return (pte_t *)ret;
96*4882a593Smuzhiyun }
97*4882a593Smuzhiyun
pte_fragment_alloc(struct mm_struct * mm,int kernel)98*4882a593Smuzhiyun pte_t *pte_fragment_alloc(struct mm_struct *mm, int kernel)
99*4882a593Smuzhiyun {
100*4882a593Smuzhiyun pte_t *pte;
101*4882a593Smuzhiyun
102*4882a593Smuzhiyun pte = get_pte_from_cache(mm);
103*4882a593Smuzhiyun if (pte)
104*4882a593Smuzhiyun return pte;
105*4882a593Smuzhiyun
106*4882a593Smuzhiyun return __alloc_for_ptecache(mm, kernel);
107*4882a593Smuzhiyun }
108*4882a593Smuzhiyun
pte_fragment_free(unsigned long * table,int kernel)109*4882a593Smuzhiyun void pte_fragment_free(unsigned long *table, int kernel)
110*4882a593Smuzhiyun {
111*4882a593Smuzhiyun struct page *page = virt_to_page(table);
112*4882a593Smuzhiyun
113*4882a593Smuzhiyun if (PageReserved(page))
114*4882a593Smuzhiyun return free_reserved_page(page);
115*4882a593Smuzhiyun
116*4882a593Smuzhiyun BUG_ON(atomic_read(&page->pt_frag_refcount) <= 0);
117*4882a593Smuzhiyun if (atomic_dec_and_test(&page->pt_frag_refcount)) {
118*4882a593Smuzhiyun if (!kernel)
119*4882a593Smuzhiyun pgtable_pte_page_dtor(page);
120*4882a593Smuzhiyun __free_page(page);
121*4882a593Smuzhiyun }
122*4882a593Smuzhiyun }
123