1*4882a593Smuzhiyun /* SPDX-License-Identifier: GPL-2.0-only */
2*4882a593Smuzhiyun /*
3*4882a593Smuzhiyun * Based on arch/arm/include/asm/tlb.h
4*4882a593Smuzhiyun *
5*4882a593Smuzhiyun * Copyright (C) 2002 Russell King
6*4882a593Smuzhiyun * Copyright (C) 2012 ARM Ltd.
7*4882a593Smuzhiyun */
8*4882a593Smuzhiyun #ifndef __ASM_TLB_H
9*4882a593Smuzhiyun #define __ASM_TLB_H
10*4882a593Smuzhiyun
11*4882a593Smuzhiyun #include <linux/pagemap.h>
12*4882a593Smuzhiyun #include <linux/swap.h>
13*4882a593Smuzhiyun
__tlb_remove_table(void * _table)14*4882a593Smuzhiyun static inline void __tlb_remove_table(void *_table)
15*4882a593Smuzhiyun {
16*4882a593Smuzhiyun free_page_and_swap_cache((struct page *)_table);
17*4882a593Smuzhiyun }
18*4882a593Smuzhiyun
19*4882a593Smuzhiyun #define tlb_flush tlb_flush
20*4882a593Smuzhiyun static void tlb_flush(struct mmu_gather *tlb);
21*4882a593Smuzhiyun
22*4882a593Smuzhiyun #include <asm-generic/tlb.h>
23*4882a593Smuzhiyun
24*4882a593Smuzhiyun /*
25*4882a593Smuzhiyun * get the tlbi levels in arm64. Default value is 0 if more than one
26*4882a593Smuzhiyun * of cleared_* is set or neither is set.
27*4882a593Smuzhiyun * Arm64 doesn't support p4ds now.
28*4882a593Smuzhiyun */
tlb_get_level(struct mmu_gather * tlb)29*4882a593Smuzhiyun static inline int tlb_get_level(struct mmu_gather *tlb)
30*4882a593Smuzhiyun {
31*4882a593Smuzhiyun /* The TTL field is only valid for the leaf entry. */
32*4882a593Smuzhiyun if (tlb->freed_tables)
33*4882a593Smuzhiyun return 0;
34*4882a593Smuzhiyun
35*4882a593Smuzhiyun if (tlb->cleared_ptes && !(tlb->cleared_pmds ||
36*4882a593Smuzhiyun tlb->cleared_puds ||
37*4882a593Smuzhiyun tlb->cleared_p4ds))
38*4882a593Smuzhiyun return 3;
39*4882a593Smuzhiyun
40*4882a593Smuzhiyun if (tlb->cleared_pmds && !(tlb->cleared_ptes ||
41*4882a593Smuzhiyun tlb->cleared_puds ||
42*4882a593Smuzhiyun tlb->cleared_p4ds))
43*4882a593Smuzhiyun return 2;
44*4882a593Smuzhiyun
45*4882a593Smuzhiyun if (tlb->cleared_puds && !(tlb->cleared_ptes ||
46*4882a593Smuzhiyun tlb->cleared_pmds ||
47*4882a593Smuzhiyun tlb->cleared_p4ds))
48*4882a593Smuzhiyun return 1;
49*4882a593Smuzhiyun
50*4882a593Smuzhiyun return 0;
51*4882a593Smuzhiyun }
52*4882a593Smuzhiyun
tlb_flush(struct mmu_gather * tlb)53*4882a593Smuzhiyun static inline void tlb_flush(struct mmu_gather *tlb)
54*4882a593Smuzhiyun {
55*4882a593Smuzhiyun struct vm_area_struct vma = TLB_FLUSH_VMA(tlb->mm, 0);
56*4882a593Smuzhiyun bool last_level = !tlb->freed_tables;
57*4882a593Smuzhiyun unsigned long stride = tlb_get_unmap_size(tlb);
58*4882a593Smuzhiyun int tlb_level = tlb_get_level(tlb);
59*4882a593Smuzhiyun
60*4882a593Smuzhiyun /*
61*4882a593Smuzhiyun * If we're tearing down the address space then we only care about
62*4882a593Smuzhiyun * invalidating the walk-cache, since the ASID allocator won't
63*4882a593Smuzhiyun * reallocate our ASID without invalidating the entire TLB.
64*4882a593Smuzhiyun */
65*4882a593Smuzhiyun if (tlb->fullmm) {
66*4882a593Smuzhiyun if (!last_level)
67*4882a593Smuzhiyun flush_tlb_mm(tlb->mm);
68*4882a593Smuzhiyun return;
69*4882a593Smuzhiyun }
70*4882a593Smuzhiyun
71*4882a593Smuzhiyun __flush_tlb_range(&vma, tlb->start, tlb->end, stride,
72*4882a593Smuzhiyun last_level, tlb_level);
73*4882a593Smuzhiyun }
74*4882a593Smuzhiyun
__pte_free_tlb(struct mmu_gather * tlb,pgtable_t pte,unsigned long addr)75*4882a593Smuzhiyun static inline void __pte_free_tlb(struct mmu_gather *tlb, pgtable_t pte,
76*4882a593Smuzhiyun unsigned long addr)
77*4882a593Smuzhiyun {
78*4882a593Smuzhiyun pgtable_pte_page_dtor(pte);
79*4882a593Smuzhiyun tlb_remove_table(tlb, pte);
80*4882a593Smuzhiyun }
81*4882a593Smuzhiyun
82*4882a593Smuzhiyun #if CONFIG_PGTABLE_LEVELS > 2
__pmd_free_tlb(struct mmu_gather * tlb,pmd_t * pmdp,unsigned long addr)83*4882a593Smuzhiyun static inline void __pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmdp,
84*4882a593Smuzhiyun unsigned long addr)
85*4882a593Smuzhiyun {
86*4882a593Smuzhiyun struct page *page = virt_to_page(pmdp);
87*4882a593Smuzhiyun
88*4882a593Smuzhiyun pgtable_pmd_page_dtor(page);
89*4882a593Smuzhiyun tlb_remove_table(tlb, page);
90*4882a593Smuzhiyun }
91*4882a593Smuzhiyun #endif
92*4882a593Smuzhiyun
93*4882a593Smuzhiyun #if CONFIG_PGTABLE_LEVELS > 3
__pud_free_tlb(struct mmu_gather * tlb,pud_t * pudp,unsigned long addr)94*4882a593Smuzhiyun static inline void __pud_free_tlb(struct mmu_gather *tlb, pud_t *pudp,
95*4882a593Smuzhiyun unsigned long addr)
96*4882a593Smuzhiyun {
97*4882a593Smuzhiyun tlb_remove_table(tlb, virt_to_page(pudp));
98*4882a593Smuzhiyun }
99*4882a593Smuzhiyun #endif
100*4882a593Smuzhiyun
101*4882a593Smuzhiyun #endif
102