1 // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
2 /*
3 *
4 * (C) COPYRIGHT 2020-2022 ARM Limited. All rights reserved.
5 *
6 * This program is free software and is provided to you under the terms of the
7 * GNU General Public License version 2 as published by the Free Software
8 * Foundation, and any use by you of this program is subject to the terms
9 * of such GNU license.
10 *
11 * This program is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
15 *
16 * You should have received a copy of the GNU General Public License
17 * along with this program; if not, you can access it online at
18 * http://www.gnu.org/licenses/gpl-2.0.html.
19 *
20 */
21
22 /*
23 * Implementation of the GPU clock rate trace manager.
24 */
25
26 #include <mali_kbase.h>
27 #include <mali_kbase_config_defaults.h>
28 #include <linux/clk.h>
29 #include <linux/pm_opp.h>
30 #include <asm/div64.h>
31 #include "backend/gpu/mali_kbase_clk_rate_trace_mgr.h"
32
33 #ifdef CONFIG_TRACE_POWER_GPU_FREQUENCY
34 #include <trace/events/power_gpu_frequency.h>
35 #else
36 #include "mali_power_gpu_frequency_trace.h"
37 #endif
38
39 #ifndef CLK_RATE_TRACE_OPS
40 #define CLK_RATE_TRACE_OPS (NULL)
41 #endif
42
43 /**
44 * get_clk_rate_trace_callbacks() - Returns pointer to clk trace ops.
45 * @kbdev: Pointer to kbase device, used to check if arbitration is enabled
46 * when compiled with arbiter support.
47 * Return: Pointer to clk trace ops if supported or NULL.
48 */
49 static struct kbase_clk_rate_trace_op_conf *
get_clk_rate_trace_callbacks(__maybe_unused struct kbase_device * kbdev)50 get_clk_rate_trace_callbacks(__maybe_unused struct kbase_device *kbdev)
51 {
52 /* base case */
53 struct kbase_clk_rate_trace_op_conf *callbacks =
54 (struct kbase_clk_rate_trace_op_conf *)CLK_RATE_TRACE_OPS;
55 #if defined(CONFIG_MALI_ARBITER_SUPPORT) && defined(CONFIG_OF)
56 const void *arbiter_if_node;
57
58 if (WARN_ON(!kbdev) || WARN_ON(!kbdev->dev))
59 return callbacks;
60
61 arbiter_if_node =
62 of_get_property(kbdev->dev->of_node, "arbiter_if", NULL);
63 /* Arbitration enabled, override the callback pointer.*/
64 if (arbiter_if_node)
65 callbacks = &arb_clk_rate_trace_ops;
66 else
67 dev_dbg(kbdev->dev,
68 "Arbitration supported but disabled by platform. Leaving clk rate callbacks as default.\n");
69
70 #endif
71
72 return callbacks;
73 }
74
gpu_clk_rate_change_notifier(struct notifier_block * nb,unsigned long event,void * data)75 static int gpu_clk_rate_change_notifier(struct notifier_block *nb,
76 unsigned long event, void *data)
77 {
78 struct kbase_gpu_clk_notifier_data *ndata = data;
79 struct kbase_clk_data *clk_data =
80 container_of(nb, struct kbase_clk_data, clk_rate_change_nb);
81 struct kbase_clk_rate_trace_manager *clk_rtm = clk_data->clk_rtm;
82 unsigned long flags;
83
84 if (WARN_ON_ONCE(clk_data->gpu_clk_handle != ndata->gpu_clk_handle))
85 return NOTIFY_BAD;
86
87 spin_lock_irqsave(&clk_rtm->lock, flags);
88 if (event == POST_RATE_CHANGE) {
89 if (!clk_rtm->gpu_idle &&
90 (clk_data->clock_val != ndata->new_rate)) {
91 kbase_clk_rate_trace_manager_notify_all(
92 clk_rtm, clk_data->index, ndata->new_rate);
93 }
94
95 clk_data->clock_val = ndata->new_rate;
96 }
97 spin_unlock_irqrestore(&clk_rtm->lock, flags);
98
99 return NOTIFY_DONE;
100 }
101
gpu_clk_data_init(struct kbase_device * kbdev,void * gpu_clk_handle,unsigned int index)102 static int gpu_clk_data_init(struct kbase_device *kbdev,
103 void *gpu_clk_handle, unsigned int index)
104 {
105 struct kbase_clk_rate_trace_op_conf *callbacks;
106 struct kbase_clk_data *clk_data;
107 struct kbase_clk_rate_trace_manager *clk_rtm = &kbdev->pm.clk_rtm;
108 int ret = 0;
109
110 callbacks = get_clk_rate_trace_callbacks(kbdev);
111
112 if (WARN_ON(!callbacks) ||
113 WARN_ON(!gpu_clk_handle) ||
114 WARN_ON(index >= BASE_MAX_NR_CLOCKS_REGULATORS))
115 return -EINVAL;
116
117 clk_data = kzalloc(sizeof(*clk_data), GFP_KERNEL);
118 if (!clk_data) {
119 dev_err(kbdev->dev, "Failed to allocate data for clock enumerated at index %u", index);
120 return -ENOMEM;
121 }
122
123 clk_data->index = (u8)index;
124 clk_data->gpu_clk_handle = gpu_clk_handle;
125 /* Store the initial value of clock */
126 clk_data->clock_val =
127 callbacks->get_gpu_clk_rate(kbdev, gpu_clk_handle);
128
129 {
130 /* At the initialization time, GPU is powered off. */
131 unsigned long flags;
132
133 spin_lock_irqsave(&clk_rtm->lock, flags);
134 kbase_clk_rate_trace_manager_notify_all(
135 clk_rtm, clk_data->index, 0);
136 spin_unlock_irqrestore(&clk_rtm->lock, flags);
137 }
138
139 clk_data->clk_rtm = clk_rtm;
140 clk_rtm->clks[index] = clk_data;
141
142 clk_data->clk_rate_change_nb.notifier_call =
143 gpu_clk_rate_change_notifier;
144
145 if (callbacks->gpu_clk_notifier_register)
146 ret = callbacks->gpu_clk_notifier_register(kbdev,
147 gpu_clk_handle, &clk_data->clk_rate_change_nb);
148 if (ret) {
149 dev_err(kbdev->dev, "Failed to register notifier for clock enumerated at index %u", index);
150 kfree(clk_data);
151 }
152
153 return ret;
154 }
155
kbase_clk_rate_trace_manager_init(struct kbase_device * kbdev)156 int kbase_clk_rate_trace_manager_init(struct kbase_device *kbdev)
157 {
158 struct kbase_clk_rate_trace_op_conf *callbacks;
159 struct kbase_clk_rate_trace_manager *clk_rtm = &kbdev->pm.clk_rtm;
160 unsigned int i;
161 int ret = 0;
162
163 callbacks = get_clk_rate_trace_callbacks(kbdev);
164
165 spin_lock_init(&clk_rtm->lock);
166 INIT_LIST_HEAD(&clk_rtm->listeners);
167
168 /* Return early if no callbacks provided for clock rate tracing */
169 if (!callbacks) {
170 WRITE_ONCE(clk_rtm->clk_rate_trace_ops, NULL);
171 return 0;
172 }
173
174 clk_rtm->gpu_idle = true;
175
176 for (i = 0; i < BASE_MAX_NR_CLOCKS_REGULATORS; i++) {
177 void *gpu_clk_handle =
178 callbacks->enumerate_gpu_clk(kbdev, i);
179
180 if (!gpu_clk_handle)
181 break;
182
183 ret = gpu_clk_data_init(kbdev, gpu_clk_handle, i);
184 if (ret)
185 goto error;
186 }
187
188 /* Activate clock rate trace manager if at least one GPU clock was
189 * enumerated.
190 */
191 if (i) {
192 WRITE_ONCE(clk_rtm->clk_rate_trace_ops, callbacks);
193 } else {
194 dev_info(kbdev->dev, "No clock(s) available for rate tracing");
195 WRITE_ONCE(clk_rtm->clk_rate_trace_ops, NULL);
196 }
197
198 return 0;
199
200 error:
201 while (i--) {
202 clk_rtm->clk_rate_trace_ops->gpu_clk_notifier_unregister(
203 kbdev, clk_rtm->clks[i]->gpu_clk_handle,
204 &clk_rtm->clks[i]->clk_rate_change_nb);
205 kfree(clk_rtm->clks[i]);
206 }
207
208 return ret;
209 }
210
kbase_clk_rate_trace_manager_term(struct kbase_device * kbdev)211 void kbase_clk_rate_trace_manager_term(struct kbase_device *kbdev)
212 {
213 struct kbase_clk_rate_trace_manager *clk_rtm = &kbdev->pm.clk_rtm;
214 unsigned int i;
215
216 WARN_ON(!list_empty(&clk_rtm->listeners));
217
218 if (!clk_rtm->clk_rate_trace_ops)
219 return;
220
221 for (i = 0; i < BASE_MAX_NR_CLOCKS_REGULATORS; i++) {
222 if (!clk_rtm->clks[i])
223 break;
224
225 if (clk_rtm->clk_rate_trace_ops->gpu_clk_notifier_unregister)
226 clk_rtm->clk_rate_trace_ops->gpu_clk_notifier_unregister
227 (kbdev, clk_rtm->clks[i]->gpu_clk_handle,
228 &clk_rtm->clks[i]->clk_rate_change_nb);
229 kfree(clk_rtm->clks[i]);
230 }
231
232 WRITE_ONCE(clk_rtm->clk_rate_trace_ops, NULL);
233 }
234
kbase_clk_rate_trace_manager_gpu_active(struct kbase_device * kbdev)235 void kbase_clk_rate_trace_manager_gpu_active(struct kbase_device *kbdev)
236 {
237 struct kbase_clk_rate_trace_manager *clk_rtm = &kbdev->pm.clk_rtm;
238 unsigned int i;
239 unsigned long flags;
240
241 if (!clk_rtm->clk_rate_trace_ops)
242 return;
243
244 spin_lock_irqsave(&clk_rtm->lock, flags);
245
246 for (i = 0; i < BASE_MAX_NR_CLOCKS_REGULATORS; i++) {
247 struct kbase_clk_data *clk_data = clk_rtm->clks[i];
248
249 if (!clk_data)
250 break;
251
252 if (unlikely(!clk_data->clock_val))
253 continue;
254
255 kbase_clk_rate_trace_manager_notify_all(
256 clk_rtm, clk_data->index, clk_data->clock_val);
257 }
258
259 clk_rtm->gpu_idle = false;
260 spin_unlock_irqrestore(&clk_rtm->lock, flags);
261 }
262
kbase_clk_rate_trace_manager_gpu_idle(struct kbase_device * kbdev)263 void kbase_clk_rate_trace_manager_gpu_idle(struct kbase_device *kbdev)
264 {
265 struct kbase_clk_rate_trace_manager *clk_rtm = &kbdev->pm.clk_rtm;
266 unsigned int i;
267 unsigned long flags;
268
269 if (!clk_rtm->clk_rate_trace_ops)
270 return;
271
272 spin_lock_irqsave(&clk_rtm->lock, flags);
273
274 for (i = 0; i < BASE_MAX_NR_CLOCKS_REGULATORS; i++) {
275 struct kbase_clk_data *clk_data = clk_rtm->clks[i];
276
277 if (!clk_data)
278 break;
279
280 if (unlikely(!clk_data->clock_val))
281 continue;
282
283 kbase_clk_rate_trace_manager_notify_all(
284 clk_rtm, clk_data->index, 0);
285 }
286
287 clk_rtm->gpu_idle = true;
288 spin_unlock_irqrestore(&clk_rtm->lock, flags);
289 }
290
kbase_clk_rate_trace_manager_notify_all(struct kbase_clk_rate_trace_manager * clk_rtm,u32 clk_index,unsigned long new_rate)291 void kbase_clk_rate_trace_manager_notify_all(
292 struct kbase_clk_rate_trace_manager *clk_rtm,
293 u32 clk_index,
294 unsigned long new_rate)
295 {
296 struct kbase_clk_rate_listener *pos;
297 struct kbase_device *kbdev;
298
299 lockdep_assert_held(&clk_rtm->lock);
300
301 kbdev = container_of(clk_rtm, struct kbase_device, pm.clk_rtm);
302
303 dev_dbg(kbdev->dev, "%s - GPU clock %u rate changed to %lu, pid: %d",
304 __func__, clk_index, new_rate, current->pid);
305
306 /* Raise standard `power/gpu_frequency` ftrace event */
307 {
308 unsigned long new_rate_khz = new_rate;
309
310 #if BITS_PER_LONG == 64
311 do_div(new_rate_khz, 1000);
312 #elif BITS_PER_LONG == 32
313 new_rate_khz /= 1000;
314 #else
315 #error "unsigned long division is not supported for this architecture"
316 #endif
317
318 trace_gpu_frequency(new_rate_khz, clk_index);
319 }
320
321 /* Notify the listeners. */
322 list_for_each_entry(pos, &clk_rtm->listeners, node) {
323 pos->notify(pos, clk_index, new_rate);
324 }
325 }
326 KBASE_EXPORT_TEST_API(kbase_clk_rate_trace_manager_notify_all);
327