1 /*
2 *
3 * (C) COPYRIGHT 2010-2017 ARM Limited. All rights reserved.
4 *
5 * This program is free software and is provided to you under the terms of the
6 * GNU General Public License version 2 as published by the Free Software
7 * Foundation, and any use by you of this program is subject to the terms
8 * of such GNU licence.
9 *
10 * A copy of the licence is included with the program, and can also be obtained
11 * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
12 * Boston, MA 02110-1301, USA.
13 *
14 */
15
16
17
18
19
20 #include "mali_kbase_mmu_mode.h"
21
22 #include "mali_kbase.h"
23 #include "mali_midg_regmap.h"
24
25 #define ENTRY_TYPE_MASK 3ULL
26 #define ENTRY_IS_ATE 1ULL
27 #define ENTRY_IS_INVAL 2ULL
28 #define ENTRY_IS_PTE 3ULL
29
30 #define ENTRY_ATTR_BITS (7ULL << 2) /* bits 4:2 */
31 #define ENTRY_RD_BIT (1ULL << 6)
32 #define ENTRY_WR_BIT (1ULL << 7)
33 #define ENTRY_SHARE_BITS (3ULL << 8) /* bits 9:8 */
34 #define ENTRY_ACCESS_BIT (1ULL << 10)
35 #define ENTRY_NX_BIT (1ULL << 54)
36
37 #define ENTRY_FLAGS_MASK (ENTRY_ATTR_BITS | ENTRY_RD_BIT | ENTRY_WR_BIT | \
38 ENTRY_SHARE_BITS | ENTRY_ACCESS_BIT | ENTRY_NX_BIT)
39
40 /* Helper Function to perform assignment of page table entries, to
41 * ensure the use of strd, which is required on LPAE systems.
42 */
page_table_entry_set(u64 * pte,u64 phy)43 static inline void page_table_entry_set(u64 *pte, u64 phy)
44 {
45 #ifdef CONFIG_64BIT
46 *pte = phy;
47 #elif defined(CONFIG_ARM)
48 /*
49 * In order to prevent the compiler keeping cached copies of
50 * memory, we have to explicitly say that we have updated
51 * memory.
52 *
53 * Note: We could manually move the data ourselves into R0 and
54 * R1 by specifying register variables that are explicitly
55 * given registers assignments, the down side of this is that
56 * we have to assume cpu endianness. To avoid this we can use
57 * the ldrd to read the data from memory into R0 and R1 which
58 * will respect the cpu endianness, we then use strd to make
59 * the 64 bit assignment to the page table entry.
60 */
61 asm volatile("ldrd r0, r1, [%[ptemp]]\n\t"
62 "strd r0, r1, [%[pte]]\n\t"
63 : "=m" (*pte)
64 : [ptemp] "r" (&phy), [pte] "r" (pte), "m" (phy)
65 : "r0", "r1");
66 #else
67 #error "64-bit atomic write must be implemented for your architecture"
68 #endif
69 }
70
mmu_get_as_setup(struct kbase_context * kctx,struct kbase_mmu_setup * const setup)71 static void mmu_get_as_setup(struct kbase_context *kctx,
72 struct kbase_mmu_setup * const setup)
73 {
74 /* Set up the required caching policies at the correct indices
75 * in the memattr register. */
76 setup->memattr =
77 (AS_MEMATTR_LPAE_IMPL_DEF_CACHE_POLICY <<
78 (AS_MEMATTR_INDEX_IMPL_DEF_CACHE_POLICY * 8)) |
79 (AS_MEMATTR_LPAE_FORCE_TO_CACHE_ALL <<
80 (AS_MEMATTR_INDEX_FORCE_TO_CACHE_ALL * 8)) |
81 (AS_MEMATTR_LPAE_WRITE_ALLOC <<
82 (AS_MEMATTR_INDEX_WRITE_ALLOC * 8)) |
83 (AS_MEMATTR_LPAE_OUTER_IMPL_DEF <<
84 (AS_MEMATTR_INDEX_OUTER_IMPL_DEF * 8)) |
85 (AS_MEMATTR_LPAE_OUTER_WA <<
86 (AS_MEMATTR_INDEX_OUTER_WA * 8)) |
87 0; /* The other indices are unused for now */
88
89 setup->transtab = ((u64)kctx->pgd &
90 ((0xFFFFFFFFULL << 32) | AS_TRANSTAB_LPAE_ADDR_SPACE_MASK)) |
91 AS_TRANSTAB_LPAE_ADRMODE_TABLE |
92 AS_TRANSTAB_LPAE_READ_INNER;
93
94 setup->transcfg = 0;
95 }
96
mmu_update(struct kbase_context * kctx)97 static void mmu_update(struct kbase_context *kctx)
98 {
99 struct kbase_device * const kbdev = kctx->kbdev;
100 struct kbase_as * const as = &kbdev->as[kctx->as_nr];
101 struct kbase_mmu_setup * const current_setup = &as->current_setup;
102
103 mmu_get_as_setup(kctx, current_setup);
104
105 /* Apply the address space setting */
106 kbase_mmu_hw_configure(kbdev, as, kctx);
107 }
108
mmu_disable_as(struct kbase_device * kbdev,int as_nr)109 static void mmu_disable_as(struct kbase_device *kbdev, int as_nr)
110 {
111 struct kbase_as * const as = &kbdev->as[as_nr];
112 struct kbase_mmu_setup * const current_setup = &as->current_setup;
113
114 current_setup->transtab = AS_TRANSTAB_LPAE_ADRMODE_UNMAPPED;
115
116 /* Apply the address space setting */
117 kbase_mmu_hw_configure(kbdev, as, NULL);
118 }
119
pte_to_phy_addr(u64 entry)120 static phys_addr_t pte_to_phy_addr(u64 entry)
121 {
122 if (!(entry & 1))
123 return 0;
124
125 return entry & ~0xFFF;
126 }
127
ate_is_valid(u64 ate)128 static int ate_is_valid(u64 ate)
129 {
130 return ((ate & ENTRY_TYPE_MASK) == ENTRY_IS_ATE);
131 }
132
pte_is_valid(u64 pte)133 static int pte_is_valid(u64 pte)
134 {
135 return ((pte & ENTRY_TYPE_MASK) == ENTRY_IS_PTE);
136 }
137
138 /*
139 * Map KBASE_REG flags to MMU flags
140 */
get_mmu_flags(unsigned long flags)141 static u64 get_mmu_flags(unsigned long flags)
142 {
143 u64 mmu_flags;
144
145 /* store mem_attr index as 4:2 (macro called ensures 3 bits already) */
146 mmu_flags = KBASE_REG_MEMATTR_VALUE(flags) << 2;
147
148 /* write perm if requested */
149 mmu_flags |= (flags & KBASE_REG_GPU_WR) ? ENTRY_WR_BIT : 0;
150 /* read perm if requested */
151 mmu_flags |= (flags & KBASE_REG_GPU_RD) ? ENTRY_RD_BIT : 0;
152 /* nx if requested */
153 mmu_flags |= (flags & KBASE_REG_GPU_NX) ? ENTRY_NX_BIT : 0;
154
155 if (flags & KBASE_REG_SHARE_BOTH) {
156 /* inner and outer shareable */
157 mmu_flags |= SHARE_BOTH_BITS;
158 } else if (flags & KBASE_REG_SHARE_IN) {
159 /* inner shareable coherency */
160 mmu_flags |= SHARE_INNER_BITS;
161 }
162
163 return mmu_flags;
164 }
165
entry_set_ate(u64 * entry,phys_addr_t phy,unsigned long flags)166 static void entry_set_ate(u64 *entry, phys_addr_t phy, unsigned long flags)
167 {
168 page_table_entry_set(entry, (phy & ~0xFFF) |
169 get_mmu_flags(flags) |
170 ENTRY_IS_ATE);
171 }
172
entry_set_pte(u64 * entry,phys_addr_t phy)173 static void entry_set_pte(u64 *entry, phys_addr_t phy)
174 {
175 page_table_entry_set(entry, (phy & ~0xFFF) | ENTRY_IS_PTE);
176 }
177
entry_invalidate(u64 * entry)178 static void entry_invalidate(u64 *entry)
179 {
180 page_table_entry_set(entry, ENTRY_IS_INVAL);
181 }
182
183 static struct kbase_mmu_mode const lpae_mode = {
184 .update = mmu_update,
185 .get_as_setup = mmu_get_as_setup,
186 .disable_as = mmu_disable_as,
187 .pte_to_phy_addr = pte_to_phy_addr,
188 .ate_is_valid = ate_is_valid,
189 .pte_is_valid = pte_is_valid,
190 .entry_set_ate = entry_set_ate,
191 .entry_set_pte = entry_set_pte,
192 .entry_invalidate = entry_invalidate
193 };
194
kbase_mmu_mode_get_lpae(void)195 struct kbase_mmu_mode const *kbase_mmu_mode_get_lpae(void)
196 {
197 return &lpae_mode;
198 }
199