1 /*
2 * Copyright (C) 2013-2017 ARM Limited. All rights reserved.
3 *
4 * This program is free software and is provided to you under the terms of the GNU General Public License version 2
5 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence.
6 *
7 * A copy of the licence is included with the program, and can also be obtained from Free Software
8 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
9 */
10
11 #include "../platform/rk/custom_log.h"
12
13 #include <linux/list.h>
14 #include <linux/mm.h>
15 #include <linux/mm_types.h>
16 #include <linux/fs.h>
17 #include <linux/dma-mapping.h>
18 #include <linux/version.h>
19 #include <linux/platform_device.h>
20 #include <linux/workqueue.h>
21
22 #include "mali_osk.h"
23 #include "mali_memory.h"
24 #include "mali_memory_os_alloc.h"
25 #include "mali_kernel_linux.h"
26
27 /* Minimum size of allocator page pool */
28 #define MALI_OS_MEMORY_KERNEL_BUFFER_SIZE_IN_PAGES (MALI_OS_MEMORY_KERNEL_BUFFER_SIZE_IN_MB * 256)
29 #define MALI_OS_MEMORY_POOL_TRIM_JIFFIES (10 * CONFIG_HZ) /* Default to 10s */
30
31 #if LINUX_VERSION_CODE < KERNEL_VERSION(3, 0, 0)
32 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 35)
33 static int mali_mem_os_shrink(int nr_to_scan, gfp_t gfp_mask);
34 #else
35 static int mali_mem_os_shrink(struct shrinker *shrinker, int nr_to_scan, gfp_t gfp_mask);
36 #endif
37 #else
38 #if LINUX_VERSION_CODE < KERNEL_VERSION(3, 12, 0)
39 static int mali_mem_os_shrink(struct shrinker *shrinker, struct shrink_control *sc);
40 #else
41 static unsigned long mali_mem_os_shrink(struct shrinker *shrinker, struct shrink_control *sc);
42 static unsigned long mali_mem_os_shrink_count(struct shrinker *shrinker, struct shrink_control *sc);
43 #endif
44 #endif
45 static void mali_mem_os_trim_pool(struct work_struct *work);
46
47 struct mali_mem_os_allocator mali_mem_os_allocator = {
48 .pool_lock = __SPIN_LOCK_UNLOCKED(pool_lock),
49 .pool_pages = LIST_HEAD_INIT(mali_mem_os_allocator.pool_pages),
50 .pool_count = 0,
51
52 .allocated_pages = ATOMIC_INIT(0),
53 .allocation_limit = 0,
54
55 #if LINUX_VERSION_CODE < KERNEL_VERSION(3, 12, 0)
56 .shrinker.shrink = mali_mem_os_shrink,
57 #else
58 .shrinker.count_objects = mali_mem_os_shrink_count,
59 .shrinker.scan_objects = mali_mem_os_shrink,
60 #endif
61 .shrinker.seeks = DEFAULT_SEEKS,
62 #if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 7, 0)
63 .timed_shrinker = __DELAYED_WORK_INITIALIZER(mali_mem_os_allocator.timed_shrinker, mali_mem_os_trim_pool, TIMER_DEFERRABLE),
64 #elif LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 38)
65 .timed_shrinker = __DEFERRED_WORK_INITIALIZER(mali_mem_os_allocator.timed_shrinker, mali_mem_os_trim_pool),
66 #else
67 .timed_shrinker = __DELAYED_WORK_INITIALIZER(mali_mem_os_allocator.timed_shrinker, mali_mem_os_trim_pool),
68 #endif
69 };
70
mali_mem_os_free(struct list_head * os_pages,u32 pages_count,mali_bool cow_flag)71 u32 mali_mem_os_free(struct list_head *os_pages, u32 pages_count, mali_bool cow_flag)
72 {
73 LIST_HEAD(pages);
74 struct mali_page_node *m_page, *m_tmp;
75 u32 free_pages_nr = 0;
76
77 if (MALI_TRUE == cow_flag) {
78 list_for_each_entry_safe(m_page, m_tmp, os_pages, list) {
79 /*only handle OS node here */
80 if (m_page->type == MALI_PAGE_NODE_OS) {
81 if (1 == _mali_page_node_get_ref_count(m_page)) {
82 list_move(&m_page->list, &pages);
83 atomic_sub(1, &mali_mem_os_allocator.allocated_pages);
84 free_pages_nr ++;
85 } else {
86 _mali_page_node_unref(m_page);
87 m_page->page = NULL;
88 list_del(&m_page->list);
89 kfree(m_page);
90 }
91 }
92 }
93 } else {
94 list_cut_position(&pages, os_pages, os_pages->prev);
95 atomic_sub(pages_count, &mali_mem_os_allocator.allocated_pages);
96 free_pages_nr = pages_count;
97 }
98
99 /* Put pages on pool. */
100 spin_lock(&mali_mem_os_allocator.pool_lock);
101 list_splice(&pages, &mali_mem_os_allocator.pool_pages);
102 mali_mem_os_allocator.pool_count += free_pages_nr;
103 spin_unlock(&mali_mem_os_allocator.pool_lock);
104
105 if (MALI_OS_MEMORY_KERNEL_BUFFER_SIZE_IN_PAGES < mali_mem_os_allocator.pool_count) {
106 MALI_DEBUG_PRINT(5, ("OS Mem: Starting pool trim timer %u\n", mali_mem_os_allocator.pool_count));
107 queue_delayed_work(mali_mem_os_allocator.wq, &mali_mem_os_allocator.timed_shrinker, MALI_OS_MEMORY_POOL_TRIM_JIFFIES);
108 }
109 return free_pages_nr;
110 }
111
112 /**
113 * put page without put it into page pool
114 */
mali_mem_os_put_page(struct page * page)115 _mali_osk_errcode_t mali_mem_os_put_page(struct page *page)
116 {
117 MALI_DEBUG_ASSERT_POINTER(page);
118 if (1 == page_count(page)) {
119 atomic_sub(1, &mali_mem_os_allocator.allocated_pages);
120 dma_unmap_page(&mali_platform_device->dev, page_private(page),
121 _MALI_OSK_MALI_PAGE_SIZE, DMA_BIDIRECTIONAL);
122 ClearPagePrivate(page);
123 }
124 put_page(page);
125 return _MALI_OSK_ERR_OK;
126 }
127
mali_mem_os_resize_pages(mali_mem_os_mem * mem_from,mali_mem_os_mem * mem_to,u32 start_page,u32 page_count)128 _mali_osk_errcode_t mali_mem_os_resize_pages(mali_mem_os_mem *mem_from, mali_mem_os_mem *mem_to, u32 start_page, u32 page_count)
129 {
130 struct mali_page_node *m_page, *m_tmp;
131 u32 i = 0;
132
133 MALI_DEBUG_ASSERT_POINTER(mem_from);
134 MALI_DEBUG_ASSERT_POINTER(mem_to);
135
136 if (mem_from->count < start_page + page_count) {
137 return _MALI_OSK_ERR_INVALID_ARGS;
138 }
139
140 list_for_each_entry_safe(m_page, m_tmp, &mem_from->pages, list) {
141 if (i >= start_page && i < start_page + page_count) {
142 list_move_tail(&m_page->list, &mem_to->pages);
143 mem_from->count--;
144 mem_to->count++;
145 }
146 i++;
147 }
148
149 return _MALI_OSK_ERR_OK;
150 }
151
152
mali_mem_os_alloc_pages(mali_mem_os_mem * os_mem,u32 size)153 int mali_mem_os_alloc_pages(mali_mem_os_mem *os_mem, u32 size)
154 {
155 struct page *new_page;
156 LIST_HEAD(pages_list);
157 size_t page_count = PAGE_ALIGN(size) / _MALI_OSK_MALI_PAGE_SIZE;
158 size_t remaining = page_count;
159 struct mali_page_node *m_page, *m_tmp;
160 u32 i;
161
162 MALI_DEBUG_ASSERT_POINTER(os_mem);
163
164 if (atomic_read(&mali_mem_os_allocator.allocated_pages) * _MALI_OSK_MALI_PAGE_SIZE + size > mali_mem_os_allocator.allocation_limit) {
165 MALI_DEBUG_PRINT(2, ("Mali Mem: Unable to allocate %u bytes. Currently allocated: %lu, max limit %lu\n",
166 size,
167 atomic_read(&mali_mem_os_allocator.allocated_pages) * _MALI_OSK_MALI_PAGE_SIZE,
168 mali_mem_os_allocator.allocation_limit));
169 return -ENOMEM;
170 }
171
172 INIT_LIST_HEAD(&os_mem->pages);
173 os_mem->count = page_count;
174
175 /* Grab pages from pool. */
176 {
177 size_t pool_pages;
178 spin_lock(&mali_mem_os_allocator.pool_lock);
179 pool_pages = min(remaining, mali_mem_os_allocator.pool_count);
180 for (i = pool_pages; i > 0; i--) {
181 BUG_ON(list_empty(&mali_mem_os_allocator.pool_pages));
182 list_move(mali_mem_os_allocator.pool_pages.next, &pages_list);
183 }
184 mali_mem_os_allocator.pool_count -= pool_pages;
185 remaining -= pool_pages;
186 spin_unlock(&mali_mem_os_allocator.pool_lock);
187 }
188
189 /* Process pages from pool. */
190 i = 0;
191 list_for_each_entry_safe(m_page, m_tmp, &pages_list, list) {
192 BUG_ON(NULL == m_page);
193
194 list_move_tail(&m_page->list, &os_mem->pages);
195 }
196
197 /* Allocate new pages, if needed. */
198 for (i = 0; i < remaining; i++) {
199 dma_addr_t dma_addr;
200 gfp_t flags = __GFP_ZERO | GFP_HIGHUSER;
201 int err;
202
203 #if defined(CONFIG_ARM) && !defined(CONFIG_ARM_LPAE)
204 flags |= GFP_HIGHUSER;
205 #else
206 #ifdef CONFIG_ZONE_DMA32
207 flags |= GFP_DMA32;
208 #else
209 #ifdef CONFIG_ZONE_DMA
210 #else
211 /* arm64 utgard only work on < 4G, but the kernel
212 * didn't provide method to allocte memory < 4G
213 */
214 MALI_DEBUG_ASSERT(0);
215 #endif
216 #endif
217 #endif
218
219 new_page = alloc_page(flags);
220
221 if (unlikely(NULL == new_page)) {
222 E("err.");
223 /* Calculate the number of pages actually allocated, and free them. */
224 os_mem->count = (page_count - remaining) + i;
225 atomic_add(os_mem->count, &mali_mem_os_allocator.allocated_pages);
226 mali_mem_os_free(&os_mem->pages, os_mem->count, MALI_FALSE);
227 return -ENOMEM;
228 }
229
230 /* Ensure page is flushed from CPU caches. */
231 dma_addr = dma_map_page(&mali_platform_device->dev, new_page,
232 0, _MALI_OSK_MALI_PAGE_SIZE, DMA_BIDIRECTIONAL);
233 dma_unmap_page(&mali_platform_device->dev, dma_addr,
234 _MALI_OSK_MALI_PAGE_SIZE, DMA_BIDIRECTIONAL);
235 dma_addr = dma_map_page(&mali_platform_device->dev, new_page,
236 0, _MALI_OSK_MALI_PAGE_SIZE, DMA_BIDIRECTIONAL);
237
238 err = dma_mapping_error(&mali_platform_device->dev, dma_addr);
239 if (unlikely(err)) {
240 MALI_DEBUG_PRINT_ERROR(("OS Mem: Failed to DMA map page %p: %u",
241 new_page, err));
242 __free_page(new_page);
243 os_mem->count = (page_count - remaining) + i;
244 atomic_add(os_mem->count, &mali_mem_os_allocator.allocated_pages);
245 mali_mem_os_free(&os_mem->pages, os_mem->count, MALI_FALSE);
246 return -EFAULT;
247 }
248
249 /* Store page phys addr */
250 SetPagePrivate(new_page);
251 set_page_private(new_page, dma_addr);
252
253 m_page = _mali_page_node_allocate(MALI_PAGE_NODE_OS);
254 if (unlikely(NULL == m_page)) {
255 MALI_PRINT_ERROR(("OS Mem: Can't allocate mali_page node! \n"));
256 dma_unmap_page(&mali_platform_device->dev, page_private(new_page),
257 _MALI_OSK_MALI_PAGE_SIZE, DMA_BIDIRECTIONAL);
258 ClearPagePrivate(new_page);
259 __free_page(new_page);
260 os_mem->count = (page_count - remaining) + i;
261 atomic_add(os_mem->count, &mali_mem_os_allocator.allocated_pages);
262 mali_mem_os_free(&os_mem->pages, os_mem->count, MALI_FALSE);
263 return -EFAULT;
264 }
265 m_page->page = new_page;
266
267 list_add_tail(&m_page->list, &os_mem->pages);
268 }
269
270 atomic_add(page_count, &mali_mem_os_allocator.allocated_pages);
271
272 if (MALI_OS_MEMORY_KERNEL_BUFFER_SIZE_IN_PAGES > mali_mem_os_allocator.pool_count) {
273 MALI_DEBUG_PRINT(4, ("OS Mem: Stopping pool trim timer, only %u pages on pool\n", mali_mem_os_allocator.pool_count));
274 cancel_delayed_work(&mali_mem_os_allocator.timed_shrinker);
275 }
276
277 return 0;
278 }
279
280
mali_mem_os_mali_map(mali_mem_os_mem * os_mem,struct mali_session_data * session,u32 vaddr,u32 start_page,u32 mapping_pgae_num,u32 props)281 _mali_osk_errcode_t mali_mem_os_mali_map(mali_mem_os_mem *os_mem, struct mali_session_data *session, u32 vaddr, u32 start_page, u32 mapping_pgae_num, u32 props)
282 {
283 struct mali_page_directory *pagedir = session->page_directory;
284 struct mali_page_node *m_page;
285 u32 virt;
286 u32 prop = props;
287
288 MALI_DEBUG_ASSERT_POINTER(session);
289 MALI_DEBUG_ASSERT_POINTER(os_mem);
290
291 MALI_DEBUG_ASSERT(start_page <= os_mem->count);
292 MALI_DEBUG_ASSERT((start_page + mapping_pgae_num) <= os_mem->count);
293
294 if ((start_page + mapping_pgae_num) == os_mem->count) {
295
296 virt = vaddr + MALI_MMU_PAGE_SIZE * (start_page + mapping_pgae_num);
297
298 list_for_each_entry_reverse(m_page, &os_mem->pages, list) {
299
300 virt -= MALI_MMU_PAGE_SIZE;
301 if (mapping_pgae_num > 0) {
302 dma_addr_t phys = page_private(m_page->page);
303 #if defined(CONFIG_ARCH_DMA_ADDR_T_64BIT)
304 /* Verify that the "physical" address is 32-bit and
305 * usable for Mali, when on a system with bus addresses
306 * wider than 32-bit. */
307 MALI_DEBUG_ASSERT(0 == (phys >> 32));
308 #endif
309 mali_mmu_pagedir_update(pagedir, virt, (mali_dma_addr)phys, MALI_MMU_PAGE_SIZE, prop);
310 } else {
311 break;
312 }
313 mapping_pgae_num--;
314 }
315
316 } else {
317 u32 i = 0;
318 virt = vaddr;
319 list_for_each_entry(m_page, &os_mem->pages, list) {
320
321 if (i >= start_page) {
322 dma_addr_t phys = page_private(m_page->page);
323
324 #if defined(CONFIG_ARCH_DMA_ADDR_T_64BIT)
325 /* Verify that the "physical" address is 32-bit and
326 * usable for Mali, when on a system with bus addresses
327 * wider than 32-bit. */
328 MALI_DEBUG_ASSERT(0 == (phys >> 32));
329 #endif
330 mali_mmu_pagedir_update(pagedir, virt, (mali_dma_addr)phys, MALI_MMU_PAGE_SIZE, prop);
331 }
332 i++;
333 virt += MALI_MMU_PAGE_SIZE;
334 }
335 }
336 return _MALI_OSK_ERR_OK;
337 }
338
339
mali_mem_os_mali_unmap(mali_mem_allocation * alloc)340 void mali_mem_os_mali_unmap(mali_mem_allocation *alloc)
341 {
342 struct mali_session_data *session;
343 MALI_DEBUG_ASSERT_POINTER(alloc);
344 session = alloc->session;
345 MALI_DEBUG_ASSERT_POINTER(session);
346
347 mali_session_memory_lock(session);
348 mali_mem_mali_map_free(session, alloc->psize, alloc->mali_vma_node.vm_node.start,
349 alloc->flags);
350 mali_session_memory_unlock(session);
351 }
352
mali_mem_os_cpu_map(mali_mem_backend * mem_bkend,struct vm_area_struct * vma)353 int mali_mem_os_cpu_map(mali_mem_backend *mem_bkend, struct vm_area_struct *vma)
354 {
355 mali_mem_os_mem *os_mem = &mem_bkend->os_mem;
356 struct mali_page_node *m_page;
357 struct page *page;
358 int ret;
359 unsigned long addr = vma->vm_start;
360 MALI_DEBUG_ASSERT(MALI_MEM_OS == mem_bkend->type);
361
362 list_for_each_entry(m_page, &os_mem->pages, list) {
363 /* We should use vm_insert_page, but it does a dcache
364 * flush which makes it way slower than remap_pfn_range or vmf_insert_pfn.
365 ret = vm_insert_page(vma, addr, page);
366 */
367 page = m_page->page;
368 ret = vmf_insert_pfn(vma, addr, page_to_pfn(page));
369
370 if (unlikely(VM_FAULT_NOPAGE != ret)) {
371 return -EFAULT;
372 }
373 addr += _MALI_OSK_MALI_PAGE_SIZE;
374 }
375
376 return 0;
377 }
378
mali_mem_os_resize_cpu_map_locked(mali_mem_backend * mem_bkend,struct vm_area_struct * vma,unsigned long start_vaddr,u32 mappig_size)379 _mali_osk_errcode_t mali_mem_os_resize_cpu_map_locked(mali_mem_backend *mem_bkend, struct vm_area_struct *vma, unsigned long start_vaddr, u32 mappig_size)
380 {
381 mali_mem_os_mem *os_mem = &mem_bkend->os_mem;
382 struct mali_page_node *m_page;
383 int ret;
384 int offset;
385 int mapping_page_num;
386 int count ;
387
388 unsigned long vstart = vma->vm_start;
389 count = 0;
390 MALI_DEBUG_ASSERT(mem_bkend->type == MALI_MEM_OS);
391 MALI_DEBUG_ASSERT(0 == start_vaddr % _MALI_OSK_MALI_PAGE_SIZE);
392 MALI_DEBUG_ASSERT(0 == vstart % _MALI_OSK_MALI_PAGE_SIZE);
393 offset = (start_vaddr - vstart) / _MALI_OSK_MALI_PAGE_SIZE;
394 MALI_DEBUG_ASSERT(offset <= os_mem->count);
395 mapping_page_num = mappig_size / _MALI_OSK_MALI_PAGE_SIZE;
396 MALI_DEBUG_ASSERT((offset + mapping_page_num) <= os_mem->count);
397
398 if ((offset + mapping_page_num) == os_mem->count) {
399
400 unsigned long vm_end = start_vaddr + mappig_size;
401
402 list_for_each_entry_reverse(m_page, &os_mem->pages, list) {
403
404 vm_end -= _MALI_OSK_MALI_PAGE_SIZE;
405 if (mapping_page_num > 0) {
406 ret = vmf_insert_pfn(vma, vm_end, page_to_pfn(m_page->page));
407
408 if (unlikely(VM_FAULT_NOPAGE != ret)) {
409 /*will return -EBUSY If the page has already been mapped into table, but it's OK*/
410 if (-EBUSY == ret) {
411 break;
412 } else {
413 MALI_DEBUG_PRINT(1, ("OS Mem: mali_mem_os_resize_cpu_map_locked failed, ret = %d, offset is %d,page_count is %d\n",
414 ret, offset + mapping_page_num, os_mem->count));
415 }
416 return _MALI_OSK_ERR_FAULT;
417 }
418 } else {
419 break;
420 }
421 mapping_page_num--;
422
423 }
424 } else {
425
426 list_for_each_entry(m_page, &os_mem->pages, list) {
427 if (count >= offset) {
428
429 ret = vmf_insert_pfn(vma, vstart, page_to_pfn(m_page->page));
430
431 if (unlikely(VM_FAULT_NOPAGE != ret)) {
432 /*will return -EBUSY If the page has already been mapped into table, but it's OK*/
433 if (-EBUSY == ret) {
434 break;
435 } else {
436 MALI_DEBUG_PRINT(1, ("OS Mem: mali_mem_os_resize_cpu_map_locked failed, ret = %d, count is %d, offset is %d,page_count is %d\n",
437 ret, count, offset, os_mem->count));
438 }
439 return _MALI_OSK_ERR_FAULT;
440 }
441 }
442 count++;
443 vstart += _MALI_OSK_MALI_PAGE_SIZE;
444 }
445 }
446 return _MALI_OSK_ERR_OK;
447 }
448
mali_mem_os_release(mali_mem_backend * mem_bkend)449 u32 mali_mem_os_release(mali_mem_backend *mem_bkend)
450 {
451
452 mali_mem_allocation *alloc;
453 struct mali_session_data *session;
454 u32 free_pages_nr = 0;
455 MALI_DEBUG_ASSERT_POINTER(mem_bkend);
456 MALI_DEBUG_ASSERT(MALI_MEM_OS == mem_bkend->type);
457
458 alloc = mem_bkend->mali_allocation;
459 MALI_DEBUG_ASSERT_POINTER(alloc);
460
461 session = alloc->session;
462 MALI_DEBUG_ASSERT_POINTER(session);
463
464 /* Unmap the memory from the mali virtual address space. */
465 mali_mem_os_mali_unmap(alloc);
466 mutex_lock(&mem_bkend->mutex);
467 /* Free pages */
468 if (MALI_MEM_BACKEND_FLAG_COWED & mem_bkend->flags) {
469 /* Lock to avoid the free race condition for the cow shared memory page node. */
470 _mali_osk_mutex_wait(session->cow_lock);
471 free_pages_nr = mali_mem_os_free(&mem_bkend->os_mem.pages, mem_bkend->os_mem.count, MALI_TRUE);
472 _mali_osk_mutex_signal(session->cow_lock);
473 } else {
474 free_pages_nr = mali_mem_os_free(&mem_bkend->os_mem.pages, mem_bkend->os_mem.count, MALI_FALSE);
475 }
476 mutex_unlock(&mem_bkend->mutex);
477
478 MALI_DEBUG_PRINT(4, ("OS Mem free : allocated size = 0x%x, free size = 0x%x\n", mem_bkend->os_mem.count * _MALI_OSK_MALI_PAGE_SIZE,
479 free_pages_nr * _MALI_OSK_MALI_PAGE_SIZE));
480
481 mem_bkend->os_mem.count = 0;
482 return free_pages_nr;
483 }
484
485
486 #define MALI_MEM_OS_PAGE_TABLE_PAGE_POOL_SIZE 128
487 static struct {
488 struct {
489 mali_dma_addr phys;
490 mali_io_address mapping;
491 } page[MALI_MEM_OS_PAGE_TABLE_PAGE_POOL_SIZE];
492 size_t count;
493 spinlock_t lock;
494 } mali_mem_page_table_page_pool = {
495 .count = 0,
496 .lock = __SPIN_LOCK_UNLOCKED(pool_lock),
497 };
498
mali_mem_os_get_table_page(mali_dma_addr * phys,mali_io_address * mapping)499 _mali_osk_errcode_t mali_mem_os_get_table_page(mali_dma_addr *phys, mali_io_address *mapping)
500 {
501 _mali_osk_errcode_t ret = _MALI_OSK_ERR_NOMEM;
502 dma_addr_t tmp_phys;
503
504 spin_lock(&mali_mem_page_table_page_pool.lock);
505 if (0 < mali_mem_page_table_page_pool.count) {
506 u32 i = --mali_mem_page_table_page_pool.count;
507 *phys = mali_mem_page_table_page_pool.page[i].phys;
508 *mapping = mali_mem_page_table_page_pool.page[i].mapping;
509
510 ret = _MALI_OSK_ERR_OK;
511 }
512 spin_unlock(&mali_mem_page_table_page_pool.lock);
513
514 if (_MALI_OSK_ERR_OK != ret) {
515 #if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 8, 0)
516 *mapping = dma_alloc_attrs(&mali_platform_device->dev,
517 _MALI_OSK_MALI_PAGE_SIZE, &tmp_phys,
518 GFP_KERNEL, DMA_ATTR_WRITE_COMBINE);
519 #else
520 *mapping = dma_alloc_writecombine(&mali_platform_device->dev,
521 _MALI_OSK_MALI_PAGE_SIZE, &tmp_phys, GFP_KERNEL);
522 #endif
523 if (NULL != *mapping) {
524 ret = _MALI_OSK_ERR_OK;
525
526 #if defined(CONFIG_ARCH_DMA_ADDR_T_64BIT)
527 /* Verify that the "physical" address is 32-bit and
528 * usable for Mali, when on a system with bus addresses
529 * wider than 32-bit. */
530 MALI_DEBUG_ASSERT(0 == (tmp_phys >> 32));
531 #endif
532
533 *phys = (mali_dma_addr)tmp_phys;
534 }
535 }
536
537 return ret;
538 }
539
mali_mem_os_release_table_page(mali_dma_addr phys,void * virt)540 void mali_mem_os_release_table_page(mali_dma_addr phys, void *virt)
541 {
542 spin_lock(&mali_mem_page_table_page_pool.lock);
543 if (MALI_MEM_OS_PAGE_TABLE_PAGE_POOL_SIZE > mali_mem_page_table_page_pool.count) {
544 u32 i = mali_mem_page_table_page_pool.count;
545 mali_mem_page_table_page_pool.page[i].phys = phys;
546 mali_mem_page_table_page_pool.page[i].mapping = virt;
547
548 ++mali_mem_page_table_page_pool.count;
549
550 spin_unlock(&mali_mem_page_table_page_pool.lock);
551 } else {
552 spin_unlock(&mali_mem_page_table_page_pool.lock);
553
554 #if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 8, 0)
555 dma_free_attrs(&mali_platform_device->dev,
556 _MALI_OSK_MALI_PAGE_SIZE, virt, phys,
557 DMA_ATTR_WRITE_COMBINE);
558 #else
559 dma_free_writecombine(&mali_platform_device->dev,
560 _MALI_OSK_MALI_PAGE_SIZE, virt, phys);
561 #endif
562 }
563 }
564
mali_mem_os_free_page_node(struct mali_page_node * m_page)565 void mali_mem_os_free_page_node(struct mali_page_node *m_page)
566 {
567 struct page *page = m_page->page;
568 MALI_DEBUG_ASSERT(m_page->type == MALI_PAGE_NODE_OS);
569
570 if (1 == page_count(page)) {
571 dma_unmap_page(&mali_platform_device->dev, page_private(page),
572 _MALI_OSK_MALI_PAGE_SIZE, DMA_BIDIRECTIONAL);
573 ClearPagePrivate(page);
574 }
575 __free_page(page);
576 m_page->page = NULL;
577 list_del(&m_page->list);
578 kfree(m_page);
579 }
580
581 /* The maximum number of page table pool pages to free in one go. */
582 #define MALI_MEM_OS_CHUNK_TO_FREE 64UL
583
584 /* Free a certain number of pages from the page table page pool.
585 * The pool lock must be held when calling the function, and the lock will be
586 * released before returning.
587 */
mali_mem_os_page_table_pool_free(size_t nr_to_free)588 static void mali_mem_os_page_table_pool_free(size_t nr_to_free)
589 {
590 mali_dma_addr phys_arr[MALI_MEM_OS_CHUNK_TO_FREE];
591 void *virt_arr[MALI_MEM_OS_CHUNK_TO_FREE];
592 u32 i;
593
594 MALI_DEBUG_ASSERT(nr_to_free <= MALI_MEM_OS_CHUNK_TO_FREE);
595
596 /* Remove nr_to_free pages from the pool and store them locally on stack. */
597 for (i = 0; i < nr_to_free; i++) {
598 u32 pool_index = mali_mem_page_table_page_pool.count - i - 1;
599
600 phys_arr[i] = mali_mem_page_table_page_pool.page[pool_index].phys;
601 virt_arr[i] = mali_mem_page_table_page_pool.page[pool_index].mapping;
602 }
603
604 mali_mem_page_table_page_pool.count -= nr_to_free;
605
606 spin_unlock(&mali_mem_page_table_page_pool.lock);
607
608 /* After releasing the spinlock: free the pages we removed from the pool. */
609 for (i = 0; i < nr_to_free; i++) {
610 #if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 8, 0)
611 dma_free_attrs(&mali_platform_device->dev, _MALI_OSK_MALI_PAGE_SIZE,
612 virt_arr[i], (dma_addr_t)phys_arr[i],
613 DMA_ATTR_WRITE_COMBINE);
614 #else
615 dma_free_writecombine(&mali_platform_device->dev,
616 _MALI_OSK_MALI_PAGE_SIZE,
617 virt_arr[i], (dma_addr_t)phys_arr[i]);
618 #endif
619 }
620 }
621
mali_mem_os_trim_page_table_page_pool(void)622 static void mali_mem_os_trim_page_table_page_pool(void)
623 {
624 size_t nr_to_free = 0;
625 size_t nr_to_keep;
626
627 /* Keep 2 page table pages for each 1024 pages in the page cache. */
628 nr_to_keep = mali_mem_os_allocator.pool_count / 512;
629 /* And a minimum of eight pages, to accomodate new sessions. */
630 nr_to_keep += 8;
631
632 if (0 == spin_trylock(&mali_mem_page_table_page_pool.lock)) return;
633
634 if (nr_to_keep < mali_mem_page_table_page_pool.count) {
635 nr_to_free = mali_mem_page_table_page_pool.count - nr_to_keep;
636 nr_to_free = min((size_t)MALI_MEM_OS_CHUNK_TO_FREE, nr_to_free);
637 }
638
639 /* Pool lock will be released by the callee. */
640 mali_mem_os_page_table_pool_free(nr_to_free);
641 }
642
mali_mem_os_shrink_count(struct shrinker * shrinker,struct shrink_control * sc)643 static unsigned long mali_mem_os_shrink_count(struct shrinker *shrinker, struct shrink_control *sc)
644 {
645 return mali_mem_os_allocator.pool_count;
646 }
647
648 #if LINUX_VERSION_CODE < KERNEL_VERSION(3, 0, 0)
649 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 35)
mali_mem_os_shrink(int nr_to_scan,gfp_t gfp_mask)650 static int mali_mem_os_shrink(int nr_to_scan, gfp_t gfp_mask)
651 #else
652 static int mali_mem_os_shrink(struct shrinker *shrinker, int nr_to_scan, gfp_t gfp_mask)
653 #endif /* Linux < 2.6.35 */
654 #else
655 #if LINUX_VERSION_CODE < KERNEL_VERSION(3, 12, 0)
656 static int mali_mem_os_shrink(struct shrinker *shrinker, struct shrink_control *sc)
657 #else
658 static unsigned long mali_mem_os_shrink(struct shrinker *shrinker, struct shrink_control *sc)
659 #endif /* Linux < 3.12.0 */
660 #endif /* Linux < 3.0.0 */
661 {
662 struct mali_page_node *m_page, *m_tmp;
663 unsigned long flags;
664 struct list_head *le, pages;
665 #if LINUX_VERSION_CODE < KERNEL_VERSION(3, 0, 0)
666 int nr = nr_to_scan;
667 #else
668 int nr = sc->nr_to_scan;
669 #endif
670
671 if (0 == nr) {
672 return mali_mem_os_shrink_count(shrinker, sc);
673 }
674
675 if (0 == spin_trylock_irqsave(&mali_mem_os_allocator.pool_lock, flags)) {
676 /* Not able to lock. */
677 return -1;
678 }
679
680 if (0 == mali_mem_os_allocator.pool_count) {
681 /* No pages availble */
682 spin_unlock_irqrestore(&mali_mem_os_allocator.pool_lock, flags);
683 return 0;
684 }
685
686 /* Release from general page pool */
687 nr = min((size_t)nr, mali_mem_os_allocator.pool_count);
688 mali_mem_os_allocator.pool_count -= nr;
689 list_for_each(le, &mali_mem_os_allocator.pool_pages) {
690 --nr;
691 if (0 == nr) break;
692 }
693 list_cut_position(&pages, &mali_mem_os_allocator.pool_pages, le);
694 spin_unlock_irqrestore(&mali_mem_os_allocator.pool_lock, flags);
695
696 list_for_each_entry_safe(m_page, m_tmp, &pages, list) {
697 mali_mem_os_free_page_node(m_page);
698 }
699
700 if (MALI_OS_MEMORY_KERNEL_BUFFER_SIZE_IN_PAGES > mali_mem_os_allocator.pool_count) {
701 /* Pools are empty, stop timer */
702 MALI_DEBUG_PRINT(5, ("Stopping timer, only %u pages on pool\n", mali_mem_os_allocator.pool_count));
703 cancel_delayed_work(&mali_mem_os_allocator.timed_shrinker);
704 }
705
706 #if LINUX_VERSION_CODE < KERNEL_VERSION(3, 12, 0)
707 return mali_mem_os_shrink_count(shrinker, sc);
708 #else
709 return nr;
710 #endif
711 }
712
mali_mem_os_trim_pool(struct work_struct * data)713 static void mali_mem_os_trim_pool(struct work_struct *data)
714 {
715 struct mali_page_node *m_page, *m_tmp;
716 struct list_head *le;
717 LIST_HEAD(pages);
718 size_t nr_to_free;
719
720 MALI_IGNORE(data);
721
722 MALI_DEBUG_PRINT(3, ("OS Mem: Trimming pool %u\n", mali_mem_os_allocator.pool_count));
723
724 /* Release from general page pool */
725 spin_lock(&mali_mem_os_allocator.pool_lock);
726 if (MALI_OS_MEMORY_KERNEL_BUFFER_SIZE_IN_PAGES < mali_mem_os_allocator.pool_count) {
727 size_t count = mali_mem_os_allocator.pool_count - MALI_OS_MEMORY_KERNEL_BUFFER_SIZE_IN_PAGES;
728 const size_t min_to_free = min(64, MALI_OS_MEMORY_KERNEL_BUFFER_SIZE_IN_PAGES);
729
730 /* Free half the pages on the pool above the static limit. Or 64 pages, 256KB. */
731 nr_to_free = max(count / 2, min_to_free);
732
733 mali_mem_os_allocator.pool_count -= nr_to_free;
734 list_for_each(le, &mali_mem_os_allocator.pool_pages) {
735 --nr_to_free;
736 if (0 == nr_to_free) break;
737 }
738 list_cut_position(&pages, &mali_mem_os_allocator.pool_pages, le);
739 }
740 spin_unlock(&mali_mem_os_allocator.pool_lock);
741
742 list_for_each_entry_safe(m_page, m_tmp, &pages, list) {
743 mali_mem_os_free_page_node(m_page);
744 }
745
746 /* Release some pages from page table page pool */
747 mali_mem_os_trim_page_table_page_pool();
748
749 if (MALI_OS_MEMORY_KERNEL_BUFFER_SIZE_IN_PAGES < mali_mem_os_allocator.pool_count) {
750 MALI_DEBUG_PRINT(4, ("OS Mem: Starting pool trim timer %u\n", mali_mem_os_allocator.pool_count));
751 queue_delayed_work(mali_mem_os_allocator.wq, &mali_mem_os_allocator.timed_shrinker, MALI_OS_MEMORY_POOL_TRIM_JIFFIES);
752 }
753 }
754
mali_mem_os_init(void)755 _mali_osk_errcode_t mali_mem_os_init(void)
756 {
757 mali_mem_os_allocator.wq = alloc_workqueue("mali-mem", WQ_UNBOUND, 1);
758 if (NULL == mali_mem_os_allocator.wq) {
759 return _MALI_OSK_ERR_NOMEM;
760 }
761
762 register_shrinker(&mali_mem_os_allocator.shrinker);
763
764 return _MALI_OSK_ERR_OK;
765 }
766
mali_mem_os_term(void)767 void mali_mem_os_term(void)
768 {
769 struct mali_page_node *m_page, *m_tmp;
770 unregister_shrinker(&mali_mem_os_allocator.shrinker);
771 cancel_delayed_work_sync(&mali_mem_os_allocator.timed_shrinker);
772
773 if (NULL != mali_mem_os_allocator.wq) {
774 destroy_workqueue(mali_mem_os_allocator.wq);
775 mali_mem_os_allocator.wq = NULL;
776 }
777
778 spin_lock(&mali_mem_os_allocator.pool_lock);
779 list_for_each_entry_safe(m_page, m_tmp, &mali_mem_os_allocator.pool_pages, list) {
780 mali_mem_os_free_page_node(m_page);
781
782 --mali_mem_os_allocator.pool_count;
783 }
784 BUG_ON(mali_mem_os_allocator.pool_count);
785 spin_unlock(&mali_mem_os_allocator.pool_lock);
786
787 /* Release from page table page pool */
788 do {
789 u32 nr_to_free;
790
791 spin_lock(&mali_mem_page_table_page_pool.lock);
792
793 nr_to_free = min((size_t)MALI_MEM_OS_CHUNK_TO_FREE, mali_mem_page_table_page_pool.count);
794
795 /* Pool lock will be released by the callee. */
796 mali_mem_os_page_table_pool_free(nr_to_free);
797 } while (0 != mali_mem_page_table_page_pool.count);
798 }
799
mali_memory_core_resource_os_memory(u32 size)800 _mali_osk_errcode_t mali_memory_core_resource_os_memory(u32 size)
801 {
802 mali_mem_os_allocator.allocation_limit = size;
803
804 MALI_SUCCESS;
805 }
806
mali_mem_os_stat(void)807 u32 mali_mem_os_stat(void)
808 {
809 return atomic_read(&mali_mem_os_allocator.allocated_pages) * _MALI_OSK_MALI_PAGE_SIZE;
810 }
811