1 /*
2 *
3 * (C) COPYRIGHT 2014-2017 ARM Limited. All rights reserved.
4 *
5 * This program is free software and is provided to you under the terms of the
6 * GNU General Public License version 2 as published by the Free Software
7 * Foundation, and any use by you of this program is subject to the terms
8 * of such GNU licence.
9 *
10 * A copy of the licence is included with the program, and can also be obtained
11 * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
12 * Boston, MA 02110-1301, USA.
13 *
14 */
15
16
17 #define ENABLE_DEBUG_LOG
18 #include "../../platform/rk/custom_log.h"
19
20
21 #include <mali_kbase.h>
22 #include <mali_kbase_tlstream.h>
23 #include <mali_kbase_config_defaults.h>
24 #include <backend/gpu/mali_kbase_pm_internal.h>
25
26 #include <linux/of.h>
27 #include <linux/clk.h>
28 #include <linux/devfreq.h>
29 #ifdef CONFIG_DEVFREQ_THERMAL
30 #include <linux/devfreq_cooling.h>
31 #endif
32
33 #include <linux/version.h>
34 #if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 13, 0)
35 #include <linux/pm_opp.h>
36 #else /* Linux >= 3.13 */
37 /* In 3.13 the OPP include header file, types, and functions were all
38 * renamed. Use the old filename for the include, and define the new names to
39 * the old, when an old kernel is detected.
40 */
41 #include <linux/opp.h>
42 #define dev_pm_opp opp
43 #define dev_pm_opp_get_voltage opp_get_voltage
44 #define dev_pm_opp_get_opp_count opp_get_opp_count
45 #define dev_pm_opp_find_freq_ceil opp_find_freq_ceil
46 #define dev_pm_opp_find_freq_floor opp_find_freq_floor
47 #endif /* Linux >= 3.13 */
48 #include <soc/rockchip/rockchip_opp_select.h>
49 #include <soc/rockchip/rockchip_system_monitor.h>
50
51 static struct devfreq_simple_ondemand_data ondemand_data;
52
53 static struct monitor_dev_profile mali_mdevp = {
54 .type = MONITOR_TYPE_DEV,
55 .low_temp_adjust = rockchip_monitor_dev_low_temp_adjust,
56 .high_temp_adjust = rockchip_monitor_dev_high_temp_adjust,
57 };
58
59 /**
60 * opp_translate - Translate nominal OPP frequency from devicetree into real
61 * frequency and core mask
62 * @kbdev: Device pointer
63 * @freq: Nominal frequency
64 * @core_mask: Pointer to u64 to store core mask to
65 *
66 * Return: Real target frequency
67 *
68 * This function will only perform translation if an operating-points-v2-mali
69 * table is present in devicetree. If one is not present then it will return an
70 * untranslated frequency and all cores enabled.
71 */
opp_translate(struct kbase_device * kbdev,unsigned long freq,u64 * core_mask)72 static unsigned long opp_translate(struct kbase_device *kbdev,
73 unsigned long freq, u64 *core_mask)
74 {
75 int i;
76
77 for (i = 0; i < kbdev->num_opps; i++) {
78 if (kbdev->opp_table[i].opp_freq == freq) {
79 *core_mask = kbdev->opp_table[i].core_mask;
80 return kbdev->opp_table[i].real_freq;
81 }
82 }
83
84 /* Failed to find OPP - return all cores enabled & nominal frequency */
85 *core_mask = kbdev->gpu_props.props.raw_props.shader_present;
86
87 return freq;
88 }
89
90 static int
kbase_devfreq_target(struct device * dev,unsigned long * target_freq,u32 flags)91 kbase_devfreq_target(struct device *dev, unsigned long *target_freq, u32 flags)
92 {
93 struct kbase_device *kbdev = dev_get_drvdata(dev);
94 struct dev_pm_opp *opp;
95 unsigned long nominal_freq;
96 unsigned long freq = 0;
97 unsigned long old_freq = kbdev->current_freq;
98 unsigned long voltage;
99 int err;
100 u64 core_mask;
101
102 freq = *target_freq;
103
104 opp = devfreq_recommended_opp(dev, &freq, flags);
105 if (IS_ERR(opp)) {
106 dev_err(dev, "Failed to get opp (%ld)\n", PTR_ERR(opp));
107 return PTR_ERR(opp);
108 }
109 voltage = dev_pm_opp_get_voltage(opp);
110
111 nominal_freq = freq;
112
113 /*
114 * Only update if there is a change of frequency
115 */
116 if (kbdev->current_nominal_freq == nominal_freq) {
117 *target_freq = nominal_freq;
118 #ifdef CONFIG_REGULATOR
119 if (kbdev->current_voltage == voltage)
120 return 0;
121 err = regulator_set_voltage(kbdev->regulator, voltage, INT_MAX);
122 if (err) {
123 dev_err(dev, "Failed to set voltage (%d)\n", err);
124 return err;
125 }
126 kbdev->current_voltage = voltage;
127 #endif
128 return 0;
129 }
130
131 freq = opp_translate(kbdev, nominal_freq, &core_mask);
132 #ifdef CONFIG_REGULATOR
133 if (kbdev->regulator && kbdev->current_voltage != voltage &&
134 old_freq < freq) {
135 err = regulator_set_voltage(kbdev->regulator, voltage, INT_MAX);
136 if (err) {
137 dev_err(dev, "Failed to increase voltage (%d)\n", err);
138 return err;
139 }
140 }
141 #endif
142
143 err = clk_set_rate(kbdev->clock, freq);
144 if (err) {
145 dev_err(dev, "Failed to set clock %lu (target %lu)\n",
146 freq, *target_freq);
147 return err;
148 }
149 *target_freq = freq;
150 kbdev->current_freq = freq;
151 if (kbdev->devfreq)
152 kbdev->devfreq->last_status.current_frequency = freq;
153 #ifdef CONFIG_REGULATOR
154 if (kbdev->regulator && kbdev->current_voltage != voltage &&
155 old_freq > freq) {
156 err = regulator_set_voltage(kbdev->regulator, voltage, INT_MAX);
157 if (err) {
158 dev_err(dev, "Failed to decrease voltage (%d)\n", err);
159 return err;
160 }
161 }
162 #endif
163
164 if (kbdev->pm.backend.ca_current_policy->id ==
165 KBASE_PM_CA_POLICY_ID_DEVFREQ)
166 kbase_devfreq_set_core_mask(kbdev, core_mask);
167
168 *target_freq = nominal_freq;
169 kbdev->current_voltage = voltage;
170 kbdev->current_nominal_freq = nominal_freq;
171 kbdev->current_freq = freq;
172 kbdev->current_core_mask = core_mask;
173
174 KBASE_TLSTREAM_AUX_DEVFREQ_TARGET((u64)nominal_freq);
175
176 kbase_pm_reset_dvfs_utilisation(kbdev);
177
178 return err;
179 }
180
181 static int
kbase_devfreq_cur_freq(struct device * dev,unsigned long * freq)182 kbase_devfreq_cur_freq(struct device *dev, unsigned long *freq)
183 {
184 struct kbase_device *kbdev = dev_get_drvdata(dev);
185
186 *freq = kbdev->current_nominal_freq;
187
188 return 0;
189 }
190
191 static int
kbase_devfreq_status(struct device * dev,struct devfreq_dev_status * stat)192 kbase_devfreq_status(struct device *dev, struct devfreq_dev_status *stat)
193 {
194 struct kbase_device *kbdev = dev_get_drvdata(dev);
195
196 stat->current_frequency = kbdev->current_nominal_freq;
197
198 kbase_pm_get_dvfs_utilisation(kbdev,
199 &stat->total_time, &stat->busy_time);
200
201 stat->private_data = NULL;
202
203 return 0;
204 }
205
kbase_devfreq_init_freq_table(struct kbase_device * kbdev,struct devfreq_dev_profile * dp)206 static int kbase_devfreq_init_freq_table(struct kbase_device *kbdev,
207 struct devfreq_dev_profile *dp)
208 {
209 int count;
210 int i = 0;
211 unsigned long freq;
212 struct dev_pm_opp *opp;
213
214 count = dev_pm_opp_get_opp_count(kbdev->dev);
215 if (count < 0) {
216 return count;
217 }
218
219 dp->freq_table = kmalloc_array(count, sizeof(dp->freq_table[0]),
220 GFP_KERNEL);
221 if (!dp->freq_table)
222 return -ENOMEM;
223
224 for (i = 0, freq = ULONG_MAX; i < count; i++, freq--) {
225 opp = dev_pm_opp_find_freq_floor(kbdev->dev, &freq);
226 if (IS_ERR(opp))
227 break;
228 dev_pm_opp_put(opp);
229
230 dp->freq_table[i] = freq;
231 }
232
233 if (count != i)
234 dev_warn(kbdev->dev, "Unable to enumerate all OPPs (%d!=%d\n",
235 count, i);
236
237 dp->max_state = i;
238
239 return 0;
240 }
241
kbase_devfreq_term_freq_table(struct kbase_device * kbdev)242 static void kbase_devfreq_term_freq_table(struct kbase_device *kbdev)
243 {
244 struct devfreq_dev_profile *dp = &kbdev->devfreq_profile;
245
246 kfree(dp->freq_table);
247 dp->freq_table = NULL;
248 }
249
kbase_devfreq_term_core_mask_table(struct kbase_device * kbdev)250 static void kbase_devfreq_term_core_mask_table(struct kbase_device *kbdev)
251 {
252 kfree(kbdev->opp_table);
253 kbdev->opp_table = NULL;
254 }
255
kbase_devfreq_exit(struct device * dev)256 static void kbase_devfreq_exit(struct device *dev)
257 {
258 struct kbase_device *kbdev = dev_get_drvdata(dev);
259
260 if (kbdev)
261 kbase_devfreq_term_freq_table(kbdev);
262 }
263
kbase_devfreq_init_core_mask_table(struct kbase_device * kbdev)264 static int kbase_devfreq_init_core_mask_table(struct kbase_device *kbdev)
265 {
266 struct device_node *opp_node = of_parse_phandle(kbdev->dev->of_node,
267 "operating-points-v2", 0);
268 struct device_node *node;
269 int i = 0;
270 int count;
271
272 if (!opp_node)
273 return 0;
274 if (!of_device_is_compatible(opp_node, "operating-points-v2-mali"))
275 return 0;
276
277 count = dev_pm_opp_get_opp_count(kbdev->dev);
278 kbdev->opp_table = kmalloc_array(count,
279 sizeof(struct kbase_devfreq_opp), GFP_KERNEL);
280 if (!kbdev->opp_table)
281 return -ENOMEM;
282
283 for_each_available_child_of_node(opp_node, node) {
284 u64 core_mask;
285 u64 opp_freq, real_freq;
286 const void *core_count_p;
287
288 if (of_property_read_u64(node, "opp-hz", &opp_freq)) {
289 dev_warn(kbdev->dev, "OPP is missing required opp-hz property\n");
290 continue;
291 }
292 if (of_property_read_u64(node, "opp-hz-real", &real_freq))
293 real_freq = opp_freq;
294 if (of_property_read_u64(node, "opp-core-mask", &core_mask))
295 core_mask =
296 kbdev->gpu_props.props.raw_props.shader_present;
297 core_count_p = of_get_property(node, "opp-core-count", NULL);
298 if (core_count_p) {
299 u64 remaining_core_mask =
300 kbdev->gpu_props.props.raw_props.shader_present;
301 int core_count = be32_to_cpup(core_count_p);
302
303 core_mask = 0;
304
305 for (; core_count > 0; core_count--) {
306 int core = ffs(remaining_core_mask);
307
308 if (!core) {
309 dev_err(kbdev->dev, "OPP has more cores than GPU\n");
310 return -ENODEV;
311 }
312
313 core_mask |= (1ull << (core-1));
314 remaining_core_mask &= ~(1ull << (core-1));
315 }
316 }
317
318 if (!core_mask) {
319 dev_err(kbdev->dev, "OPP has invalid core mask of 0\n");
320 return -ENODEV;
321 }
322
323 kbdev->opp_table[i].opp_freq = opp_freq;
324 kbdev->opp_table[i].real_freq = real_freq;
325 kbdev->opp_table[i].core_mask = core_mask;
326
327 dev_info(kbdev->dev, "OPP %d : opp_freq=%llu real_freq=%llu core_mask=%llx\n",
328 i, opp_freq, real_freq, core_mask);
329
330 i++;
331 }
332
333 kbdev->num_opps = i;
334
335 return 0;
336 }
337
kbase_devfreq_init(struct kbase_device * kbdev)338 int kbase_devfreq_init(struct kbase_device *kbdev)
339 {
340 struct device_node *np = kbdev->dev->of_node;
341 struct devfreq_dev_profile *dp;
342 struct dev_pm_opp *opp;
343 unsigned long opp_rate;
344 int err;
345
346 if (!kbdev->clock) {
347 dev_err(kbdev->dev, "Clock not available for devfreq\n");
348 return -ENODEV;
349 }
350
351 kbdev->current_freq = clk_get_rate(kbdev->clock);
352 kbdev->current_nominal_freq = kbdev->current_freq;
353
354 dp = &kbdev->devfreq_profile;
355
356 dp->initial_freq = kbdev->current_freq;
357 /* .KP : set devfreq_dvfs_interval_in_ms */
358 dp->polling_ms = 20;
359 dp->target = kbase_devfreq_target;
360 dp->get_dev_status = kbase_devfreq_status;
361 dp->get_cur_freq = kbase_devfreq_cur_freq;
362 dp->exit = kbase_devfreq_exit;
363
364 if (kbase_devfreq_init_freq_table(kbdev, dp))
365 return -EFAULT;
366
367 err = kbase_devfreq_init_core_mask_table(kbdev);
368 if (err)
369 goto init_core_mask_table_failed;
370
371 of_property_read_u32(np, "upthreshold",
372 &ondemand_data.upthreshold);
373 of_property_read_u32(np, "downdifferential",
374 &ondemand_data.downdifferential);
375
376 kbdev->devfreq = devfreq_add_device(kbdev->dev, dp,
377 "simple_ondemand", &ondemand_data);
378 if (IS_ERR(kbdev->devfreq)) {
379 err = PTR_ERR(kbdev->devfreq);
380 kbdev->devfreq = NULL;
381 dev_err(kbdev->dev, "Fail to add devfreq device(%d)", err);
382 goto devfreq_add_dev_failed;
383 }
384
385 /* devfreq_add_device only copies a few of kbdev->dev's fields, so
386 * set drvdata explicitly so IPA models can access kbdev. */
387 dev_set_drvdata(&kbdev->devfreq->dev, kbdev);
388
389 err = devfreq_register_opp_notifier(kbdev->dev, kbdev->devfreq);
390 if (err) {
391 dev_err(kbdev->dev,
392 "Failed to register OPP notifier (%d)\n", err);
393 goto opp_notifier_failed;
394 }
395
396 opp_rate = kbdev->current_freq;
397 opp = devfreq_recommended_opp(kbdev->dev, &opp_rate, 0);
398 if (!IS_ERR(opp))
399 dev_pm_opp_put(opp);
400 kbdev->devfreq->last_status.current_frequency = opp_rate;
401
402 mali_mdevp.data = kbdev->devfreq;
403 kbdev->mdev_info = rockchip_system_monitor_register(kbdev->dev,
404 &mali_mdevp);
405 if (IS_ERR(kbdev->mdev_info)) {
406 dev_dbg(kbdev->dev, "without system monitor\n");
407 kbdev->mdev_info = NULL;
408 }
409 #ifdef CONFIG_DEVFREQ_THERMAL
410 err = kbase_ipa_init(kbdev);
411 if (err) {
412 dev_err(kbdev->dev, "IPA initialization failed\n");
413 goto cooling_failed;
414 }
415
416 kbdev->devfreq_cooling = of_devfreq_cooling_register_power(
417 kbdev->dev->of_node,
418 kbdev->devfreq,
419 &kbase_ipa_power_model_ops);
420 if (IS_ERR_OR_NULL(kbdev->devfreq_cooling)) {
421 err = PTR_ERR(kbdev->devfreq_cooling);
422 dev_err(kbdev->dev,
423 "Failed to register cooling device (%d)\n",
424 err);
425 goto cooling_failed;
426 }
427 I("success initing power_model_simple.");
428 #endif
429
430 return 0;
431
432 #ifdef CONFIG_DEVFREQ_THERMAL
433 cooling_failed:
434 devfreq_unregister_opp_notifier(kbdev->dev, kbdev->devfreq);
435 #endif /* CONFIG_DEVFREQ_THERMAL */
436 opp_notifier_failed:
437 if (devfreq_remove_device(kbdev->devfreq))
438 dev_err(kbdev->dev, "Failed to terminate devfreq (%d)\n", err);
439 else
440 kbdev->devfreq = NULL;
441
442 devfreq_add_dev_failed:
443 kbase_devfreq_term_core_mask_table(kbdev);
444
445 init_core_mask_table_failed:
446 kbase_devfreq_term_freq_table(kbdev);
447
448 return err;
449 }
450
kbase_devfreq_term(struct kbase_device * kbdev)451 void kbase_devfreq_term(struct kbase_device *kbdev)
452 {
453 int err;
454
455 dev_dbg(kbdev->dev, "Term Mali devfreq\n");
456
457 rockchip_system_monitor_unregister(kbdev->mdev_info);
458 #ifdef CONFIG_DEVFREQ_THERMAL
459 if (kbdev->devfreq_cooling)
460 devfreq_cooling_unregister(kbdev->devfreq_cooling);
461
462 kbase_ipa_term(kbdev);
463 #endif
464
465 devfreq_unregister_opp_notifier(kbdev->dev, kbdev->devfreq);
466
467 err = devfreq_remove_device(kbdev->devfreq);
468 if (err)
469 dev_err(kbdev->dev, "Failed to terminate devfreq (%d)\n", err);
470 else
471 kbdev->devfreq = NULL;
472
473 kbase_devfreq_term_core_mask_table(kbdev);
474 }
475