1 // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
2 /*
3 *
4 * (C) COPYRIGHT 2020-2022 ARM Limited. All rights reserved.
5 *
6 * This program is free software and is provided to you under the terms of the
7 * GNU General Public License version 2 as published by the Free Software
8 * Foundation, and any use by you of this program is subject to the terms
9 * of such GNU license.
10 *
11 * This program is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
15 *
16 * You should have received a copy of the GNU General Public License
17 * along with this program; if not, you can access it online at
18 * http://www.gnu.org/licenses/gpl-2.0.html.
19 *
20 */
21
22 #include <mali_kbase.h>
23 #include <gpu/mali_kbase_gpu_fault.h>
24 #include <backend/gpu/mali_kbase_instr_internal.h>
25 #include <backend/gpu/mali_kbase_pm_internal.h>
26 #include <device/mali_kbase_device.h>
27 #include <device/mali_kbase_device_internal.h>
28 #include <mali_kbase_reset_gpu.h>
29 #include <mmu/mali_kbase_mmu.h>
30 #include <mali_kbase_ctx_sched.h>
31
32 /**
33 * kbase_report_gpu_fault - Report a GPU fault of the device.
34 *
35 * @kbdev: Kbase device pointer
36 * @status: Fault status
37 * @as_nr: Faulty address space
38 * @as_valid: true if address space is valid
39 *
40 * This function is called from the interrupt handler when a GPU fault occurs.
41 */
kbase_report_gpu_fault(struct kbase_device * kbdev,u32 status,u32 as_nr,bool as_valid)42 static void kbase_report_gpu_fault(struct kbase_device *kbdev, u32 status,
43 u32 as_nr, bool as_valid)
44 {
45 u64 address = (u64) kbase_reg_read(kbdev,
46 GPU_CONTROL_REG(GPU_FAULTADDRESS_HI)) << 32;
47
48 address |= kbase_reg_read(kbdev,
49 GPU_CONTROL_REG(GPU_FAULTADDRESS_LO));
50
51 /* Report GPU fault for all contexts in case either
52 * the address space is invalid or it's MCU address space.
53 */
54 kbase_mmu_gpu_fault_interrupt(kbdev, status, as_nr, address, as_valid);
55 }
56
kbase_gpu_fault_interrupt(struct kbase_device * kbdev)57 static void kbase_gpu_fault_interrupt(struct kbase_device *kbdev)
58 {
59 const u32 status = kbase_reg_read(kbdev,
60 GPU_CONTROL_REG(GPU_FAULTSTATUS));
61 const bool as_valid = status & GPU_FAULTSTATUS_JASID_VALID_FLAG;
62 const u32 as_nr = (status & GPU_FAULTSTATUS_JASID_MASK) >>
63 GPU_FAULTSTATUS_JASID_SHIFT;
64 bool bus_fault = (status & GPU_FAULTSTATUS_EXCEPTION_TYPE_MASK) ==
65 GPU_FAULTSTATUS_EXCEPTION_TYPE_GPU_BUS_FAULT;
66
67 if (bus_fault) {
68 /* If as_valid, reset gpu when ASID is for MCU. */
69 if (!as_valid || (as_nr == MCU_AS_NR)) {
70 kbase_report_gpu_fault(kbdev, status, as_nr, as_valid);
71
72 dev_err(kbdev->dev, "GPU bus fault triggering gpu-reset ...\n");
73 if (kbase_prepare_to_reset_gpu(
74 kbdev, RESET_FLAGS_HWC_UNRECOVERABLE_ERROR))
75 kbase_reset_gpu(kbdev);
76 } else {
77 /* Handle Bus fault */
78 if (kbase_mmu_bus_fault_interrupt(kbdev, status, as_nr))
79 dev_warn(kbdev->dev,
80 "fail to handle GPU bus fault ...\n");
81 }
82 } else
83 kbase_report_gpu_fault(kbdev, status, as_nr, as_valid);
84
85 }
86
kbase_gpu_interrupt(struct kbase_device * kbdev,u32 val)87 void kbase_gpu_interrupt(struct kbase_device *kbdev, u32 val)
88 {
89 KBASE_KTRACE_ADD(kbdev, CORE_GPU_IRQ, NULL, val);
90 if (val & GPU_FAULT)
91 kbase_gpu_fault_interrupt(kbdev);
92
93 if (val & GPU_PROTECTED_FAULT) {
94 struct kbase_csf_scheduler *scheduler = &kbdev->csf.scheduler;
95 unsigned long flags;
96
97 dev_err_ratelimited(kbdev->dev, "GPU fault in protected mode");
98
99 /* Mask the protected fault interrupt to avoid the potential
100 * deluge of such interrupts. It will be unmasked on GPU reset.
101 */
102 spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
103 kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK),
104 GPU_IRQ_REG_ALL & ~GPU_PROTECTED_FAULT);
105 spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
106
107 kbase_csf_scheduler_spin_lock(kbdev, &flags);
108 if (!WARN_ON(!kbase_csf_scheduler_protected_mode_in_use(
109 kbdev))) {
110 struct base_gpu_queue_group_error const
111 err_payload = { .error_type =
112 BASE_GPU_QUEUE_GROUP_ERROR_FATAL,
113 .payload = {
114 .fatal_group = {
115 .status =
116 GPU_EXCEPTION_TYPE_SW_FAULT_0,
117 } } };
118
119 kbase_debug_csf_fault_notify(kbdev, scheduler->active_protm_grp->kctx,
120 DF_GPU_PROTECTED_FAULT);
121
122 scheduler->active_protm_grp->faulted = true;
123 kbase_csf_add_group_fatal_error(
124 scheduler->active_protm_grp, &err_payload);
125 kbase_event_wakeup(scheduler->active_protm_grp->kctx);
126 }
127 kbase_csf_scheduler_spin_unlock(kbdev, flags);
128
129 if (kbase_prepare_to_reset_gpu(
130 kbdev, RESET_FLAGS_HWC_UNRECOVERABLE_ERROR))
131 kbase_reset_gpu(kbdev);
132
133 /* Defer the clearing to the GPU reset sequence */
134 val &= ~GPU_PROTECTED_FAULT;
135 }
136
137 if (val & RESET_COMPLETED)
138 kbase_pm_reset_done(kbdev);
139
140 /* Defer clearing CLEAN_CACHES_COMPLETED to kbase_clean_caches_done.
141 * We need to acquire hwaccess_lock to avoid a race condition with
142 * kbase_gpu_cache_flush_and_busy_wait
143 */
144 KBASE_KTRACE_ADD(kbdev, CORE_GPU_IRQ_CLEAR, NULL, val & ~CLEAN_CACHES_COMPLETED);
145 kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_CLEAR), val & ~CLEAN_CACHES_COMPLETED);
146
147 #ifdef KBASE_PM_RUNTIME
148 if (val & DOORBELL_MIRROR) {
149 unsigned long flags;
150
151 dev_dbg(kbdev->dev, "Doorbell mirror interrupt received");
152 spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
153 kbase_pm_disable_db_mirror_interrupt(kbdev);
154 kbdev->pm.backend.exit_gpu_sleep_mode = true;
155 kbase_csf_scheduler_invoke_tick(kbdev);
156 spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
157 }
158 #endif
159
160 /* kbase_pm_check_transitions (called by kbase_pm_power_changed) must
161 * be called after the IRQ has been cleared. This is because it might
162 * trigger further power transitions and we don't want to miss the
163 * interrupt raised to notify us that these further transitions have
164 * finished. The same applies to kbase_clean_caches_done() - if another
165 * clean was queued, it might trigger another clean, which might
166 * generate another interrupt which shouldn't be missed.
167 */
168
169 if (val & CLEAN_CACHES_COMPLETED)
170 kbase_clean_caches_done(kbdev);
171
172 if (val & (POWER_CHANGED_ALL | MCU_STATUS_GPU_IRQ)) {
173 kbase_pm_power_changed(kbdev);
174 } else if (val & CLEAN_CACHES_COMPLETED) {
175 /* If cache line evict messages can be lost when shader cores
176 * power down then we need to flush the L2 cache before powering
177 * down cores. When the flush completes, the shaders' state
178 * machine needs to be re-invoked to proceed with powering down
179 * cores.
180 */
181 if (kbdev->pm.backend.l2_always_on ||
182 kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_TTRX_921))
183 kbase_pm_power_changed(kbdev);
184 }
185
186 KBASE_KTRACE_ADD(kbdev, CORE_GPU_IRQ_DONE, NULL, val);
187 }
188
189 #if !IS_ENABLED(CONFIG_MALI_BIFROST_NO_MALI)
kbase_is_register_accessible(u32 offset)190 bool kbase_is_register_accessible(u32 offset)
191 {
192 #ifdef CONFIG_MALI_BIFROST_DEBUG
193 if (((offset >= MCU_SUBSYSTEM_BASE) && (offset < IPA_CONTROL_BASE)) ||
194 ((offset >= GPU_CONTROL_MCU_BASE) && (offset < USER_BASE))) {
195 WARN(1, "Invalid register offset 0x%x", offset);
196 return false;
197 }
198 #endif
199
200 return true;
201 }
202 #endif /* !IS_ENABLED(CONFIG_MALI_BIFROST_NO_MALI) */
203
204 #if IS_ENABLED(CONFIG_MALI_REAL_HW)
kbase_reg_write(struct kbase_device * kbdev,u32 offset,u32 value)205 void kbase_reg_write(struct kbase_device *kbdev, u32 offset, u32 value)
206 {
207 if (WARN_ON(!kbdev->pm.backend.gpu_powered))
208 return;
209
210 if (WARN_ON(kbdev->dev == NULL))
211 return;
212
213 if (!kbase_is_register_accessible(offset))
214 return;
215
216 writel(value, kbdev->reg + offset);
217
218 #if IS_ENABLED(CONFIG_DEBUG_FS)
219 if (unlikely(kbdev->io_history.enabled))
220 kbase_io_history_add(&kbdev->io_history, kbdev->reg + offset,
221 value, 1);
222 #endif /* CONFIG_DEBUG_FS */
223 dev_dbg(kbdev->dev, "w: reg %08x val %08x", offset, value);
224 }
225 KBASE_EXPORT_TEST_API(kbase_reg_write);
226
kbase_reg_read(struct kbase_device * kbdev,u32 offset)227 u32 kbase_reg_read(struct kbase_device *kbdev, u32 offset)
228 {
229 u32 val;
230
231 if (WARN_ON(!kbdev->pm.backend.gpu_powered))
232 return 0;
233
234 if (WARN_ON(kbdev->dev == NULL))
235 return 0;
236
237 if (!kbase_is_register_accessible(offset))
238 return 0;
239
240 val = readl(kbdev->reg + offset);
241
242 #if IS_ENABLED(CONFIG_DEBUG_FS)
243 if (unlikely(kbdev->io_history.enabled))
244 kbase_io_history_add(&kbdev->io_history, kbdev->reg + offset,
245 val, 0);
246 #endif /* CONFIG_DEBUG_FS */
247 dev_dbg(kbdev->dev, "r: reg %08x val %08x", offset, val);
248
249 return val;
250 }
251 KBASE_EXPORT_TEST_API(kbase_reg_read);
252 #endif /* !IS_ENABLED(CONFIG_MALI_BIFROST_NO_MALI) */
253