1 /*
2 * Copyright (C) 2010-2017 ARM Limited. All rights reserved.
3 *
4 * This program is free software and is provided to you under the terms of the GNU General Public License version 2
5 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence.
6 *
7 * A copy of the licence is included with the program, and can also be obtained from Free Software
8 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
9 */
10 #include "mali_kernel_common.h"
11 #include "mali_osk.h"
12 #include "mali_l2_cache.h"
13 #include "mali_hw_core.h"
14 #include "mali_scheduler.h"
15 #include "mali_pm.h"
16 #include "mali_pm_domain.h"
17
18 /**
19 * Size of the Mali L2 cache registers in bytes
20 */
21 #define MALI400_L2_CACHE_REGISTERS_SIZE 0x30
22
23 /**
24 * Mali L2 cache register numbers
25 * Used in the register read/write routines.
26 * See the hardware documentation for more information about each register
27 */
28 typedef enum mali_l2_cache_register {
29 MALI400_L2_CACHE_REGISTER_SIZE = 0x0004,
30 MALI400_L2_CACHE_REGISTER_STATUS = 0x0008,
31 /*unused = 0x000C */
32 MALI400_L2_CACHE_REGISTER_COMMAND = 0x0010,
33 MALI400_L2_CACHE_REGISTER_CLEAR_PAGE = 0x0014,
34 MALI400_L2_CACHE_REGISTER_MAX_READS = 0x0018,
35 MALI400_L2_CACHE_REGISTER_ENABLE = 0x001C,
36 MALI400_L2_CACHE_REGISTER_PERFCNT_SRC0 = 0x0020,
37 MALI400_L2_CACHE_REGISTER_PERFCNT_VAL0 = 0x0024,
38 MALI400_L2_CACHE_REGISTER_PERFCNT_SRC1 = 0x0028,
39 MALI400_L2_CACHE_REGISTER_PERFCNT_VAL1 = 0x002C,
40 } mali_l2_cache_register;
41
42 /**
43 * Mali L2 cache commands
44 * These are the commands that can be sent to the Mali L2 cache unit
45 */
46 typedef enum mali_l2_cache_command {
47 MALI400_L2_CACHE_COMMAND_CLEAR_ALL = 0x01,
48 } mali_l2_cache_command;
49
50 /**
51 * Mali L2 cache commands
52 * These are the commands that can be sent to the Mali L2 cache unit
53 */
54 typedef enum mali_l2_cache_enable {
55 MALI400_L2_CACHE_ENABLE_DEFAULT = 0x0, /* Default */
56 MALI400_L2_CACHE_ENABLE_ACCESS = 0x01,
57 MALI400_L2_CACHE_ENABLE_READ_ALLOCATE = 0x02,
58 } mali_l2_cache_enable;
59
60 /**
61 * Mali L2 cache status bits
62 */
63 typedef enum mali_l2_cache_status {
64 MALI400_L2_CACHE_STATUS_COMMAND_BUSY = 0x01,
65 MALI400_L2_CACHE_STATUS_DATA_BUSY = 0x02,
66 } mali_l2_cache_status;
67
68 #define MALI400_L2_MAX_READS_NOT_SET -1
69
70 static struct mali_l2_cache_core *
71 mali_global_l2s[MALI_MAX_NUMBER_OF_L2_CACHE_CORES] = { NULL, };
72 static u32 mali_global_num_l2s = 0;
73
74 int mali_l2_max_reads = MALI400_L2_MAX_READS_NOT_SET;
75
76
77 /* Local helper functions */
78
79 static void mali_l2_cache_reset(struct mali_l2_cache_core *cache);
80
81 static _mali_osk_errcode_t mali_l2_cache_send_command(
82 struct mali_l2_cache_core *cache, u32 reg, u32 val);
83
mali_l2_cache_lock(struct mali_l2_cache_core * cache)84 static void mali_l2_cache_lock(struct mali_l2_cache_core *cache)
85 {
86 MALI_DEBUG_ASSERT_POINTER(cache);
87 _mali_osk_spinlock_irq_lock(cache->lock);
88 }
89
mali_l2_cache_unlock(struct mali_l2_cache_core * cache)90 static void mali_l2_cache_unlock(struct mali_l2_cache_core *cache)
91 {
92 MALI_DEBUG_ASSERT_POINTER(cache);
93 _mali_osk_spinlock_irq_unlock(cache->lock);
94 }
95
96 /* Implementation of the L2 cache interface */
97
mali_l2_cache_create(_mali_osk_resource_t * resource,u32 domain_index)98 struct mali_l2_cache_core *mali_l2_cache_create(
99 _mali_osk_resource_t *resource, u32 domain_index)
100 {
101 struct mali_l2_cache_core *cache = NULL;
102 #if defined(DEBUG)
103 u32 cache_size;
104 #endif
105
106 MALI_DEBUG_PRINT(4, ("Mali L2 cache: Creating Mali L2 cache: %s\n",
107 resource->description));
108
109 if (mali_global_num_l2s >= MALI_MAX_NUMBER_OF_L2_CACHE_CORES) {
110 MALI_PRINT_ERROR(("Mali L2 cache: Too many L2 caches\n"));
111 return NULL;
112 }
113
114 cache = _mali_osk_malloc(sizeof(struct mali_l2_cache_core));
115 if (NULL == cache) {
116 MALI_PRINT_ERROR(("Mali L2 cache: Failed to allocate memory for L2 cache core\n"));
117 return NULL;
118 }
119
120 cache->core_id = mali_global_num_l2s;
121 cache->counter_src0 = MALI_HW_CORE_NO_COUNTER;
122 cache->counter_src1 = MALI_HW_CORE_NO_COUNTER;
123 cache->counter_value0_base = 0;
124 cache->counter_value1_base = 0;
125 cache->pm_domain = NULL;
126 cache->power_is_on = MALI_FALSE;
127 cache->last_invalidated_id = 0;
128
129 if (_MALI_OSK_ERR_OK != mali_hw_core_create(&cache->hw_core,
130 resource, MALI400_L2_CACHE_REGISTERS_SIZE)) {
131 _mali_osk_free(cache);
132 return NULL;
133 }
134
135 #if defined(DEBUG)
136 cache_size = mali_hw_core_register_read(&cache->hw_core,
137 MALI400_L2_CACHE_REGISTER_SIZE);
138 MALI_DEBUG_PRINT(2, ("Mali L2 cache: Created %s: % 3uK, %u-way, % 2ubyte cache line, % 3ubit external bus\n",
139 resource->description,
140 1 << (((cache_size >> 16) & 0xff) - 10),
141 1 << ((cache_size >> 8) & 0xff),
142 1 << (cache_size & 0xff),
143 1 << ((cache_size >> 24) & 0xff)));
144 #endif
145
146 cache->lock = _mali_osk_spinlock_irq_init(_MALI_OSK_LOCKFLAG_ORDERED,
147 _MALI_OSK_LOCK_ORDER_L2);
148 if (NULL == cache->lock) {
149 MALI_PRINT_ERROR(("Mali L2 cache: Failed to create counter lock for L2 cache core %s\n",
150 cache->hw_core.description));
151 mali_hw_core_delete(&cache->hw_core);
152 _mali_osk_free(cache);
153 return NULL;
154 }
155
156 /* register with correct power domain */
157 cache->pm_domain = mali_pm_register_l2_cache(
158 domain_index, cache);
159
160 mali_global_l2s[mali_global_num_l2s] = cache;
161 mali_global_num_l2s++;
162
163 return cache;
164 }
165
mali_l2_cache_delete(struct mali_l2_cache_core * cache)166 void mali_l2_cache_delete(struct mali_l2_cache_core *cache)
167 {
168 u32 i;
169 for (i = 0; i < mali_global_num_l2s; i++) {
170 if (mali_global_l2s[i] != cache) {
171 continue;
172 }
173
174 mali_global_l2s[i] = NULL;
175 mali_global_num_l2s--;
176
177 if (i == mali_global_num_l2s) {
178 /* Removed last element, nothing more to do */
179 break;
180 }
181
182 /*
183 * We removed a l2 cache from the middle of the array,
184 * so move the last l2 cache to current position
185 */
186 mali_global_l2s[i] = mali_global_l2s[mali_global_num_l2s];
187 mali_global_l2s[mali_global_num_l2s] = NULL;
188
189 /* All good */
190 break;
191 }
192
193 _mali_osk_spinlock_irq_term(cache->lock);
194 mali_hw_core_delete(&cache->hw_core);
195 _mali_osk_free(cache);
196 }
197
mali_l2_cache_power_up(struct mali_l2_cache_core * cache)198 void mali_l2_cache_power_up(struct mali_l2_cache_core *cache)
199 {
200 MALI_DEBUG_ASSERT_POINTER(cache);
201
202 mali_l2_cache_lock(cache);
203
204 mali_l2_cache_reset(cache);
205
206 if ((1 << MALI_DOMAIN_INDEX_DUMMY) != cache->pm_domain->pmu_mask)
207 MALI_DEBUG_ASSERT(MALI_FALSE == cache->power_is_on);
208 cache->power_is_on = MALI_TRUE;
209
210 mali_l2_cache_unlock(cache);
211 }
212
mali_l2_cache_power_down(struct mali_l2_cache_core * cache)213 void mali_l2_cache_power_down(struct mali_l2_cache_core *cache)
214 {
215 MALI_DEBUG_ASSERT_POINTER(cache);
216
217 mali_l2_cache_lock(cache);
218
219 MALI_DEBUG_ASSERT(MALI_TRUE == cache->power_is_on);
220
221 /*
222 * The HW counters will start from zero again when we resume,
223 * but we should report counters as always increasing.
224 * Take a copy of the HW values now in order to add this to
225 * the values we report after being powered up.
226 *
227 * The physical power off of the L2 cache might be outside our
228 * own control (e.g. runtime PM). That is why we must manually
229 * set set the counter value to zero as well.
230 */
231
232 if (cache->counter_src0 != MALI_HW_CORE_NO_COUNTER) {
233 cache->counter_value0_base += mali_hw_core_register_read(
234 &cache->hw_core,
235 MALI400_L2_CACHE_REGISTER_PERFCNT_VAL0);
236 mali_hw_core_register_write(&cache->hw_core,
237 MALI400_L2_CACHE_REGISTER_PERFCNT_VAL0, 0);
238 }
239
240 if (cache->counter_src1 != MALI_HW_CORE_NO_COUNTER) {
241 cache->counter_value1_base += mali_hw_core_register_read(
242 &cache->hw_core,
243 MALI400_L2_CACHE_REGISTER_PERFCNT_VAL1);
244 mali_hw_core_register_write(&cache->hw_core,
245 MALI400_L2_CACHE_REGISTER_PERFCNT_VAL1, 0);
246 }
247
248
249 cache->power_is_on = MALI_FALSE;
250
251 mali_l2_cache_unlock(cache);
252 }
253
mali_l2_cache_core_set_counter_src(struct mali_l2_cache_core * cache,u32 source_id,u32 counter)254 void mali_l2_cache_core_set_counter_src(
255 struct mali_l2_cache_core *cache, u32 source_id, u32 counter)
256 {
257 u32 reg_offset_src;
258 u32 reg_offset_val;
259
260 MALI_DEBUG_ASSERT_POINTER(cache);
261 MALI_DEBUG_ASSERT(source_id >= 0 && source_id <= 1);
262
263 mali_l2_cache_lock(cache);
264
265 if (0 == source_id) {
266 /* start counting from 0 */
267 cache->counter_value0_base = 0;
268 cache->counter_src0 = counter;
269 reg_offset_src = MALI400_L2_CACHE_REGISTER_PERFCNT_SRC0;
270 reg_offset_val = MALI400_L2_CACHE_REGISTER_PERFCNT_VAL0;
271 } else {
272 /* start counting from 0 */
273 cache->counter_value1_base = 0;
274 cache->counter_src1 = counter;
275 reg_offset_src = MALI400_L2_CACHE_REGISTER_PERFCNT_SRC1;
276 reg_offset_val = MALI400_L2_CACHE_REGISTER_PERFCNT_VAL1;
277 }
278
279 if (cache->power_is_on) {
280 u32 hw_src;
281
282 if (MALI_HW_CORE_NO_COUNTER != counter) {
283 hw_src = counter;
284 } else {
285 hw_src = 0; /* disable value for HW */
286 }
287
288 /* Set counter src */
289 mali_hw_core_register_write(&cache->hw_core,
290 reg_offset_src, hw_src);
291
292 /* Make sure the HW starts counting from 0 again */
293 mali_hw_core_register_write(&cache->hw_core,
294 reg_offset_val, 0);
295 }
296
297 mali_l2_cache_unlock(cache);
298 }
299
mali_l2_cache_core_get_counter_values(struct mali_l2_cache_core * cache,u32 * src0,u32 * value0,u32 * src1,u32 * value1)300 void mali_l2_cache_core_get_counter_values(
301 struct mali_l2_cache_core *cache,
302 u32 *src0, u32 *value0, u32 *src1, u32 *value1)
303 {
304 MALI_DEBUG_ASSERT_POINTER(cache);
305 MALI_DEBUG_ASSERT(NULL != src0);
306 MALI_DEBUG_ASSERT(NULL != value0);
307 MALI_DEBUG_ASSERT(NULL != src1);
308 MALI_DEBUG_ASSERT(NULL != value1);
309
310 mali_l2_cache_lock(cache);
311
312 *src0 = cache->counter_src0;
313 *src1 = cache->counter_src1;
314
315 if (cache->counter_src0 != MALI_HW_CORE_NO_COUNTER) {
316 if (MALI_TRUE == cache->power_is_on) {
317 *value0 = mali_hw_core_register_read(&cache->hw_core,
318 MALI400_L2_CACHE_REGISTER_PERFCNT_VAL0);
319 } else {
320 *value0 = 0;
321 }
322
323 /* Add base offset value (in case we have been power off) */
324 *value0 += cache->counter_value0_base;
325 }
326
327 if (cache->counter_src1 != MALI_HW_CORE_NO_COUNTER) {
328 if (MALI_TRUE == cache->power_is_on) {
329 *value1 = mali_hw_core_register_read(&cache->hw_core,
330 MALI400_L2_CACHE_REGISTER_PERFCNT_VAL1);
331 } else {
332 *value1 = 0;
333 }
334
335 /* Add base offset value (in case we have been power off) */
336 *value1 += cache->counter_value1_base;
337 }
338
339 mali_l2_cache_unlock(cache);
340 }
341
mali_l2_cache_core_get_glob_l2_core(u32 index)342 struct mali_l2_cache_core *mali_l2_cache_core_get_glob_l2_core(u32 index)
343 {
344 if (mali_global_num_l2s > index) {
345 return mali_global_l2s[index];
346 }
347
348 return NULL;
349 }
350
mali_l2_cache_core_get_glob_num_l2_cores(void)351 u32 mali_l2_cache_core_get_glob_num_l2_cores(void)
352 {
353 return mali_global_num_l2s;
354 }
355
mali_l2_cache_invalidate(struct mali_l2_cache_core * cache)356 void mali_l2_cache_invalidate(struct mali_l2_cache_core *cache)
357 {
358 MALI_DEBUG_ASSERT_POINTER(cache);
359
360 if (NULL == cache) {
361 return;
362 }
363
364 mali_l2_cache_lock(cache);
365
366 cache->last_invalidated_id = mali_scheduler_get_new_cache_order();
367 mali_l2_cache_send_command(cache, MALI400_L2_CACHE_REGISTER_COMMAND,
368 MALI400_L2_CACHE_COMMAND_CLEAR_ALL);
369
370 mali_l2_cache_unlock(cache);
371 }
372
mali_l2_cache_invalidate_conditional(struct mali_l2_cache_core * cache,u32 id)373 void mali_l2_cache_invalidate_conditional(
374 struct mali_l2_cache_core *cache, u32 id)
375 {
376 MALI_DEBUG_ASSERT_POINTER(cache);
377
378 if (NULL == cache) {
379 return;
380 }
381
382 /*
383 * If the last cache invalidation was done by a job with a higher id we
384 * don't have to flush. Since user space will store jobs w/ their
385 * corresponding memory in sequence (first job #0, then job #1, ...),
386 * we don't have to flush for job n-1 if job n has already invalidated
387 * the cache since we know for sure that job n-1's memory was already
388 * written when job n was started.
389 */
390
391 mali_l2_cache_lock(cache);
392
393 if (((s32)id) > ((s32)cache->last_invalidated_id)) {
394 /* Set latest invalidated id to current "point in time" */
395 cache->last_invalidated_id =
396 mali_scheduler_get_new_cache_order();
397 mali_l2_cache_send_command(cache,
398 MALI400_L2_CACHE_REGISTER_COMMAND,
399 MALI400_L2_CACHE_COMMAND_CLEAR_ALL);
400 }
401
402 mali_l2_cache_unlock(cache);
403 }
404
mali_l2_cache_invalidate_all(void)405 void mali_l2_cache_invalidate_all(void)
406 {
407 u32 i;
408 for (i = 0; i < mali_global_num_l2s; i++) {
409 struct mali_l2_cache_core *cache = mali_global_l2s[i];
410 _mali_osk_errcode_t ret;
411
412 MALI_DEBUG_ASSERT_POINTER(cache);
413
414 mali_l2_cache_lock(cache);
415
416 if (MALI_TRUE != cache->power_is_on) {
417 mali_l2_cache_unlock(cache);
418 continue;
419 }
420
421 cache->last_invalidated_id =
422 mali_scheduler_get_new_cache_order();
423
424 ret = mali_l2_cache_send_command(cache,
425 MALI400_L2_CACHE_REGISTER_COMMAND,
426 MALI400_L2_CACHE_COMMAND_CLEAR_ALL);
427 if (_MALI_OSK_ERR_OK != ret) {
428 MALI_PRINT_ERROR(("Failed to invalidate cache\n"));
429 }
430
431 mali_l2_cache_unlock(cache);
432 }
433 }
434
mali_l2_cache_invalidate_all_pages(u32 * pages,u32 num_pages)435 void mali_l2_cache_invalidate_all_pages(u32 *pages, u32 num_pages)
436 {
437 u32 i;
438 for (i = 0; i < mali_global_num_l2s; i++) {
439 struct mali_l2_cache_core *cache = mali_global_l2s[i];
440 u32 j;
441
442 MALI_DEBUG_ASSERT_POINTER(cache);
443
444 mali_l2_cache_lock(cache);
445
446 if (MALI_TRUE != cache->power_is_on) {
447 mali_l2_cache_unlock(cache);
448 continue;
449 }
450
451 for (j = 0; j < num_pages; j++) {
452 _mali_osk_errcode_t ret;
453
454 ret = mali_l2_cache_send_command(cache,
455 MALI400_L2_CACHE_REGISTER_CLEAR_PAGE,
456 pages[j]);
457 if (_MALI_OSK_ERR_OK != ret) {
458 MALI_PRINT_ERROR(("Failed to invalidate cache (page)\n"));
459 }
460 }
461
462 mali_l2_cache_unlock(cache);
463 }
464 }
465
466 /* -------- local helper functions below -------- */
467
mali_l2_cache_reset(struct mali_l2_cache_core * cache)468 static void mali_l2_cache_reset(struct mali_l2_cache_core *cache)
469 {
470 MALI_DEBUG_ASSERT_POINTER(cache);
471 MALI_DEBUG_ASSERT_LOCK_HELD(cache->lock);
472
473 /* Invalidate cache (just to keep it in a known state at startup) */
474 mali_l2_cache_send_command(cache, MALI400_L2_CACHE_REGISTER_COMMAND,
475 MALI400_L2_CACHE_COMMAND_CLEAR_ALL);
476
477 /* Enable cache */
478 mali_hw_core_register_write(&cache->hw_core,
479 MALI400_L2_CACHE_REGISTER_ENABLE,
480 (u32)MALI400_L2_CACHE_ENABLE_ACCESS |
481 (u32)MALI400_L2_CACHE_ENABLE_READ_ALLOCATE);
482
483 if (MALI400_L2_MAX_READS_NOT_SET != mali_l2_max_reads) {
484 mali_hw_core_register_write(&cache->hw_core,
485 MALI400_L2_CACHE_REGISTER_MAX_READS,
486 (u32)mali_l2_max_reads);
487 }
488
489 /* Restart any performance counters (if enabled) */
490 if (cache->counter_src0 != MALI_HW_CORE_NO_COUNTER) {
491
492 mali_hw_core_register_write(&cache->hw_core,
493 MALI400_L2_CACHE_REGISTER_PERFCNT_SRC0,
494 cache->counter_src0);
495 }
496
497 if (cache->counter_src1 != MALI_HW_CORE_NO_COUNTER) {
498 mali_hw_core_register_write(&cache->hw_core,
499 MALI400_L2_CACHE_REGISTER_PERFCNT_SRC1,
500 cache->counter_src1);
501 }
502 }
503
mali_l2_cache_send_command(struct mali_l2_cache_core * cache,u32 reg,u32 val)504 static _mali_osk_errcode_t mali_l2_cache_send_command(
505 struct mali_l2_cache_core *cache, u32 reg, u32 val)
506 {
507 int i = 0;
508 const int loop_count = 100000;
509
510 MALI_DEBUG_ASSERT_POINTER(cache);
511 MALI_DEBUG_ASSERT_LOCK_HELD(cache->lock);
512
513 /*
514 * First, wait for L2 cache command handler to go idle.
515 * (Commands received while processing another command will be ignored)
516 */
517 for (i = 0; i < loop_count; i++) {
518 if (!(mali_hw_core_register_read(&cache->hw_core,
519 MALI400_L2_CACHE_REGISTER_STATUS) &
520 (u32)MALI400_L2_CACHE_STATUS_COMMAND_BUSY)) {
521 break;
522 }
523 }
524
525 if (i == loop_count) {
526 MALI_DEBUG_PRINT(1, ("Mali L2 cache: aborting wait for command interface to go idle\n"));
527 return _MALI_OSK_ERR_FAULT;
528 }
529
530 /* then issue the command */
531 mali_hw_core_register_write(&cache->hw_core, reg, val);
532
533 return _MALI_OSK_ERR_OK;
534 }
535