1 // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
2 /*
3 *
4 * (C) COPYRIGHT 2014-2016, 2018-2022 ARM Limited. All rights reserved.
5 *
6 * This program is free software and is provided to you under the terms of the
7 * GNU General Public License version 2 as published by the Free Software
8 * Foundation, and any use by you of this program is subject to the terms
9 * of such GNU license.
10 *
11 * This program is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
15 *
16 * You should have received a copy of the GNU General Public License
17 * along with this program; if not, you can access it online at
18 * http://www.gnu.org/licenses/gpl-2.0.html.
19 *
20 */
21
22 #include <mali_kbase.h>
23 #include <gpu/mali_kbase_gpu_fault.h>
24 #include <backend/gpu/mali_kbase_instr_internal.h>
25 #include <backend/gpu/mali_kbase_pm_internal.h>
26 #include <device/mali_kbase_device.h>
27 #include <mali_kbase_reset_gpu.h>
28 #include <mmu/mali_kbase_mmu.h>
29
30 #if !IS_ENABLED(CONFIG_MALI_BIFROST_NO_MALI)
kbase_is_gpu_removed(struct kbase_device * kbdev)31 bool kbase_is_gpu_removed(struct kbase_device *kbdev)
32 {
33 u32 val;
34
35 val = kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_ID));
36
37 return val == 0;
38 }
39 #endif /* !IS_ENABLED(CONFIG_MALI_BIFROST_NO_MALI) */
40
busy_wait_on_irq(struct kbase_device * kbdev,u32 irq_bit)41 static int busy_wait_on_irq(struct kbase_device *kbdev, u32 irq_bit)
42 {
43 char *irq_flag_name;
44 /* Previously MMU-AS command was used for L2 cache flush on page-table update.
45 * And we're using the same max-loops count for GPU command, because amount of
46 * L2 cache flush overhead are same between them.
47 */
48 unsigned int max_loops = KBASE_AS_INACTIVE_MAX_LOOPS;
49
50 /* Wait for the GPU cache clean operation to complete */
51 while (--max_loops &&
52 !(kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_RAWSTAT)) & irq_bit)) {
53 ;
54 }
55
56 /* reset gpu if time-out occurred */
57 if (max_loops == 0) {
58 switch (irq_bit) {
59 case CLEAN_CACHES_COMPLETED:
60 irq_flag_name = "CLEAN_CACHES_COMPLETED";
61 break;
62 case FLUSH_PA_RANGE_COMPLETED:
63 irq_flag_name = "FLUSH_PA_RANGE_COMPLETED";
64 break;
65 default:
66 irq_flag_name = "UNKNOWN";
67 break;
68 }
69
70 dev_err(kbdev->dev,
71 "Stuck waiting on %s bit, might be caused by slow/unstable GPU clock or possible faulty FPGA connector\n",
72 irq_flag_name);
73
74 if (kbase_prepare_to_reset_gpu_locked(kbdev, RESET_FLAGS_NONE))
75 kbase_reset_gpu_locked(kbdev);
76 return -EBUSY;
77 }
78
79 /* Clear the interrupt bit. */
80 KBASE_KTRACE_ADD(kbdev, CORE_GPU_IRQ_CLEAR, NULL, irq_bit);
81 kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_CLEAR), irq_bit);
82
83 return 0;
84 }
85
86 #if MALI_USE_CSF
87 #define U64_LO_MASK ((1ULL << 32) - 1)
88 #define U64_HI_MASK (~U64_LO_MASK)
89
kbase_gpu_cache_flush_pa_range_and_busy_wait(struct kbase_device * kbdev,phys_addr_t phys,size_t nr_bytes,u32 flush_op)90 int kbase_gpu_cache_flush_pa_range_and_busy_wait(struct kbase_device *kbdev, phys_addr_t phys,
91 size_t nr_bytes, u32 flush_op)
92 {
93 u64 start_pa, end_pa;
94 int ret = 0;
95
96 lockdep_assert_held(&kbdev->hwaccess_lock);
97
98 /* 1. Clear the interrupt FLUSH_PA_RANGE_COMPLETED bit. */
99 kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_CLEAR), FLUSH_PA_RANGE_COMPLETED);
100
101 /* 2. Issue GPU_CONTROL.COMMAND.FLUSH_PA_RANGE operation. */
102 start_pa = phys;
103 end_pa = start_pa + nr_bytes - 1;
104
105 kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND_ARG0_LO), start_pa & U64_LO_MASK);
106 kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND_ARG0_HI),
107 (start_pa & U64_HI_MASK) >> 32);
108 kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND_ARG1_LO), end_pa & U64_LO_MASK);
109 kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND_ARG1_HI), (end_pa & U64_HI_MASK) >> 32);
110 kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), flush_op);
111
112 /* 3. Busy-wait irq status to be enabled. */
113 ret = busy_wait_on_irq(kbdev, (u32)FLUSH_PA_RANGE_COMPLETED);
114
115 return ret;
116 }
117 #endif /* MALI_USE_CSF */
118
kbase_gpu_cache_flush_and_busy_wait(struct kbase_device * kbdev,u32 flush_op)119 int kbase_gpu_cache_flush_and_busy_wait(struct kbase_device *kbdev,
120 u32 flush_op)
121 {
122 int need_to_wake_up = 0;
123 int ret = 0;
124
125 /* hwaccess_lock must be held to avoid any sync issue with
126 * kbase_gpu_start_cache_clean() / kbase_clean_caches_done()
127 */
128 lockdep_assert_held(&kbdev->hwaccess_lock);
129
130 /* 1. Check if kbdev->cache_clean_in_progress is set.
131 * If it is set, it means there are threads waiting for
132 * CLEAN_CACHES_COMPLETED irq to be raised and that the
133 * corresponding irq mask bit is set.
134 * We'll clear the irq mask bit and busy-wait for the cache
135 * clean operation to complete before submitting the cache
136 * clean command required after the GPU page table update.
137 * Pended flush commands will be merged to requested command.
138 */
139 if (kbdev->cache_clean_in_progress) {
140 /* disable irq first */
141 u32 irq_mask = kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK));
142 kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK),
143 irq_mask & ~CLEAN_CACHES_COMPLETED);
144
145 /* busy wait irq status to be enabled */
146 ret = busy_wait_on_irq(kbdev, (u32)CLEAN_CACHES_COMPLETED);
147 if (ret)
148 return ret;
149
150 /* merge pended command if there's any */
151 flush_op = GPU_COMMAND_FLUSH_CACHE_MERGE(
152 kbdev->cache_clean_queued, flush_op);
153
154 /* enable wake up notify flag */
155 need_to_wake_up = 1;
156 } else {
157 /* Clear the interrupt CLEAN_CACHES_COMPLETED bit. */
158 kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_CLEAR),
159 CLEAN_CACHES_COMPLETED);
160 }
161
162 /* 2. Issue GPU_CONTROL.COMMAND.FLUSH_CACHE operation. */
163 KBASE_KTRACE_ADD(kbdev, CORE_GPU_CLEAN_INV_CACHES, NULL, flush_op);
164 kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), flush_op);
165
166 /* 3. Busy-wait irq status to be enabled. */
167 ret = busy_wait_on_irq(kbdev, (u32)CLEAN_CACHES_COMPLETED);
168 if (ret)
169 return ret;
170
171 /* 4. Wake-up blocked threads when there is any. */
172 if (need_to_wake_up)
173 kbase_gpu_cache_clean_wait_complete(kbdev);
174
175 return ret;
176 }
177
kbase_gpu_start_cache_clean_nolock(struct kbase_device * kbdev,u32 flush_op)178 void kbase_gpu_start_cache_clean_nolock(struct kbase_device *kbdev,
179 u32 flush_op)
180 {
181 u32 irq_mask;
182
183 lockdep_assert_held(&kbdev->hwaccess_lock);
184
185 if (kbdev->cache_clean_in_progress) {
186 /* If this is called while another clean is in progress, we
187 * can't rely on the current one to flush any new changes in
188 * the cache. Instead, accumulate all cache clean operations
189 * and trigger that immediately after this one finishes.
190 */
191 kbdev->cache_clean_queued = GPU_COMMAND_FLUSH_CACHE_MERGE(
192 kbdev->cache_clean_queued, flush_op);
193 return;
194 }
195
196 /* Enable interrupt */
197 irq_mask = kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK));
198 kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK),
199 irq_mask | CLEAN_CACHES_COMPLETED);
200
201 KBASE_KTRACE_ADD(kbdev, CORE_GPU_CLEAN_INV_CACHES, NULL, flush_op);
202 kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), flush_op);
203
204 kbdev->cache_clean_in_progress = true;
205 }
206
kbase_gpu_start_cache_clean(struct kbase_device * kbdev,u32 flush_op)207 void kbase_gpu_start_cache_clean(struct kbase_device *kbdev, u32 flush_op)
208 {
209 unsigned long flags;
210
211 spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
212 kbase_gpu_start_cache_clean_nolock(kbdev, flush_op);
213 spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
214 }
215
kbase_gpu_cache_clean_wait_complete(struct kbase_device * kbdev)216 void kbase_gpu_cache_clean_wait_complete(struct kbase_device *kbdev)
217 {
218 lockdep_assert_held(&kbdev->hwaccess_lock);
219
220 kbdev->cache_clean_queued = 0;
221 kbdev->cache_clean_in_progress = false;
222 wake_up(&kbdev->cache_clean_wait);
223 }
224
kbase_clean_caches_done(struct kbase_device * kbdev)225 void kbase_clean_caches_done(struct kbase_device *kbdev)
226 {
227 u32 irq_mask;
228 unsigned long flags;
229
230 spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
231
232 if (kbdev->cache_clean_in_progress) {
233 /* Clear the interrupt CLEAN_CACHES_COMPLETED bit if set.
234 * It might have already been done by kbase_gpu_cache_flush_and_busy_wait.
235 */
236 KBASE_KTRACE_ADD(kbdev, CORE_GPU_IRQ_CLEAR, NULL, CLEAN_CACHES_COMPLETED);
237 kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_CLEAR), CLEAN_CACHES_COMPLETED);
238
239 if (kbdev->cache_clean_queued) {
240 u32 pended_flush_op = kbdev->cache_clean_queued;
241
242 kbdev->cache_clean_queued = 0;
243
244 KBASE_KTRACE_ADD(kbdev, CORE_GPU_CLEAN_INV_CACHES, NULL, pended_flush_op);
245 kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), pended_flush_op);
246 } else {
247 /* Disable interrupt */
248 irq_mask = kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK));
249 kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK),
250 irq_mask & ~CLEAN_CACHES_COMPLETED);
251
252 kbase_gpu_cache_clean_wait_complete(kbdev);
253 }
254 }
255
256 spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
257 }
258
get_cache_clean_flag(struct kbase_device * kbdev)259 static inline bool get_cache_clean_flag(struct kbase_device *kbdev)
260 {
261 bool cache_clean_in_progress;
262 unsigned long flags;
263
264 spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
265 cache_clean_in_progress = kbdev->cache_clean_in_progress;
266 spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
267
268 return cache_clean_in_progress;
269 }
270
kbase_gpu_wait_cache_clean(struct kbase_device * kbdev)271 void kbase_gpu_wait_cache_clean(struct kbase_device *kbdev)
272 {
273 while (get_cache_clean_flag(kbdev)) {
274 wait_event_interruptible(kbdev->cache_clean_wait,
275 !kbdev->cache_clean_in_progress);
276 }
277 }
278
kbase_gpu_wait_cache_clean_timeout(struct kbase_device * kbdev,unsigned int wait_timeout_ms)279 int kbase_gpu_wait_cache_clean_timeout(struct kbase_device *kbdev,
280 unsigned int wait_timeout_ms)
281 {
282 long remaining = msecs_to_jiffies(wait_timeout_ms);
283
284 while (remaining && get_cache_clean_flag(kbdev)) {
285 remaining = wait_event_timeout(kbdev->cache_clean_wait,
286 !kbdev->cache_clean_in_progress,
287 remaining);
288 }
289
290 return (remaining ? 0 : -ETIMEDOUT);
291 }
292