1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * DMA BUF page pool system
4 *
5 * Copyright (C) 2020 Linaro Ltd.
6 *
7 * Based on the ION page pool code
8 * Copyright (C) 2011 Google, Inc.
9 */
10
11 #include <linux/freezer.h>
12 #include <linux/list.h>
13 #include <linux/slab.h>
14 #include <linux/spinlock.h>
15 #include <linux/swap.h>
16 #include <linux/sched/signal.h>
17 #include "page_pool.h"
18
19 struct dmabuf_page_pool_with_spinlock {
20 struct dmabuf_page_pool pool;
21 struct spinlock spinlock;
22 };
23
24 static LIST_HEAD(pool_list);
25 static DEFINE_MUTEX(pool_list_lock);
26
27 static inline
dmabuf_page_pool_alloc_pages(struct dmabuf_page_pool * pool)28 struct page *dmabuf_page_pool_alloc_pages(struct dmabuf_page_pool *pool)
29 {
30 if (fatal_signal_pending(current))
31 return NULL;
32 return alloc_pages(pool->gfp_mask, pool->order);
33 }
34
dmabuf_page_pool_free_pages(struct dmabuf_page_pool * pool,struct page * page)35 static inline void dmabuf_page_pool_free_pages(struct dmabuf_page_pool *pool,
36 struct page *page)
37 {
38 __free_pages(page, pool->order);
39 }
40
dmabuf_page_pool_add(struct dmabuf_page_pool * pool,struct page * page)41 static void dmabuf_page_pool_add(struct dmabuf_page_pool *pool, struct page *page)
42 {
43 int index;
44 struct dmabuf_page_pool_with_spinlock *container_pool =
45 container_of(pool, struct dmabuf_page_pool_with_spinlock, pool);
46
47 if (PageHighMem(page))
48 index = POOL_HIGHPAGE;
49 else
50 index = POOL_LOWPAGE;
51
52 spin_lock(&container_pool->spinlock);
53 list_add_tail(&page->lru, &pool->items[index]);
54 pool->count[index]++;
55 spin_unlock(&container_pool->spinlock);
56 mod_node_page_state(page_pgdat(page), NR_KERNEL_MISC_RECLAIMABLE,
57 1 << pool->order);
58 }
59
dmabuf_page_pool_remove(struct dmabuf_page_pool * pool,int index)60 static struct page *dmabuf_page_pool_remove(struct dmabuf_page_pool *pool, int index)
61 {
62 struct page *page;
63 struct dmabuf_page_pool_with_spinlock *container_pool =
64 container_of(pool, struct dmabuf_page_pool_with_spinlock, pool);
65
66 spin_lock(&container_pool->spinlock);
67 page = list_first_entry_or_null(&pool->items[index], struct page, lru);
68 if (page) {
69 pool->count[index]--;
70 list_del(&page->lru);
71 spin_unlock(&container_pool->spinlock);
72 mod_node_page_state(page_pgdat(page), NR_KERNEL_MISC_RECLAIMABLE,
73 -(1 << pool->order));
74 goto out;
75 }
76 spin_unlock(&container_pool->spinlock);
77
78 out:
79 return page;
80 }
81
dmabuf_page_pool_fetch(struct dmabuf_page_pool * pool)82 static struct page *dmabuf_page_pool_fetch(struct dmabuf_page_pool *pool)
83 {
84 struct page *page = NULL;
85
86 page = dmabuf_page_pool_remove(pool, POOL_HIGHPAGE);
87 if (!page)
88 page = dmabuf_page_pool_remove(pool, POOL_LOWPAGE);
89
90 return page;
91 }
92
dmabuf_page_pool_alloc(struct dmabuf_page_pool * pool)93 struct page *dmabuf_page_pool_alloc(struct dmabuf_page_pool *pool)
94 {
95 struct page *page = NULL;
96
97 if (WARN_ON(!pool))
98 return NULL;
99
100 page = dmabuf_page_pool_fetch(pool);
101
102 if (!page)
103 page = dmabuf_page_pool_alloc_pages(pool);
104 return page;
105 }
106 EXPORT_SYMBOL_GPL(dmabuf_page_pool_alloc);
107
dmabuf_page_pool_free(struct dmabuf_page_pool * pool,struct page * page)108 void dmabuf_page_pool_free(struct dmabuf_page_pool *pool, struct page *page)
109 {
110 if (WARN_ON(pool->order != compound_order(page)))
111 return;
112
113 dmabuf_page_pool_add(pool, page);
114 }
115 EXPORT_SYMBOL_GPL(dmabuf_page_pool_free);
116
dmabuf_page_pool_total(struct dmabuf_page_pool * pool,bool high)117 static int dmabuf_page_pool_total(struct dmabuf_page_pool *pool, bool high)
118 {
119 int count = pool->count[POOL_LOWPAGE];
120
121 if (high)
122 count += pool->count[POOL_HIGHPAGE];
123
124 return count << pool->order;
125 }
126
dmabuf_page_pool_create(gfp_t gfp_mask,unsigned int order)127 struct dmabuf_page_pool *dmabuf_page_pool_create(gfp_t gfp_mask, unsigned int order)
128 {
129 struct dmabuf_page_pool *pool;
130 struct dmabuf_page_pool_with_spinlock *container_pool =
131 kmalloc(sizeof(*container_pool), GFP_KERNEL);
132 int i;
133
134 if (!container_pool)
135 return NULL;
136
137 spin_lock_init(&container_pool->spinlock);
138 pool = &container_pool->pool;
139
140 for (i = 0; i < POOL_TYPE_SIZE; i++) {
141 pool->count[i] = 0;
142 INIT_LIST_HEAD(&pool->items[i]);
143 }
144 pool->gfp_mask = gfp_mask | __GFP_COMP;
145 pool->order = order;
146 mutex_init(&pool->mutex); /* No longer used! */
147 mutex_lock(&pool->mutex); /* Make sure anyone who attempts to acquire this hangs */
148
149 mutex_lock(&pool_list_lock);
150 list_add(&pool->list, &pool_list);
151 mutex_unlock(&pool_list_lock);
152
153 return pool;
154 }
155 EXPORT_SYMBOL_GPL(dmabuf_page_pool_create);
156
dmabuf_page_pool_destroy(struct dmabuf_page_pool * pool)157 void dmabuf_page_pool_destroy(struct dmabuf_page_pool *pool)
158 {
159 struct page *page;
160 struct dmabuf_page_pool_with_spinlock *container_pool;
161 int i;
162
163 /* Remove us from the pool list */
164 mutex_lock(&pool_list_lock);
165 list_del(&pool->list);
166 mutex_unlock(&pool_list_lock);
167
168 /* Free any remaining pages in the pool */
169 for (i = 0; i < POOL_TYPE_SIZE; i++) {
170 while ((page = dmabuf_page_pool_remove(pool, i)))
171 dmabuf_page_pool_free_pages(pool, page);
172 }
173
174 container_pool = container_of(pool, struct dmabuf_page_pool_with_spinlock, pool);
175 kfree(container_pool);
176 }
177 EXPORT_SYMBOL_GPL(dmabuf_page_pool_destroy);
178
dmabuf_page_pool_do_shrink(struct dmabuf_page_pool * pool,gfp_t gfp_mask,int nr_to_scan)179 static int dmabuf_page_pool_do_shrink(struct dmabuf_page_pool *pool, gfp_t gfp_mask,
180 int nr_to_scan)
181 {
182 int freed = 0;
183 bool high;
184
185 if (current_is_kswapd())
186 high = true;
187 else
188 high = !!(gfp_mask & __GFP_HIGHMEM);
189
190 if (nr_to_scan == 0)
191 return dmabuf_page_pool_total(pool, high);
192
193 while (freed < nr_to_scan) {
194 struct page *page;
195
196 /* Try to free low pages first */
197 page = dmabuf_page_pool_remove(pool, POOL_LOWPAGE);
198 if (!page)
199 page = dmabuf_page_pool_remove(pool, POOL_HIGHPAGE);
200
201 if (!page)
202 break;
203
204 dmabuf_page_pool_free_pages(pool, page);
205 freed += (1 << pool->order);
206 }
207
208 return freed;
209 }
210
dmabuf_page_pool_shrink(gfp_t gfp_mask,int nr_to_scan)211 static int dmabuf_page_pool_shrink(gfp_t gfp_mask, int nr_to_scan)
212 {
213 struct dmabuf_page_pool *pool;
214 int nr_total = 0;
215 int nr_freed;
216 int only_scan = 0;
217
218 if (!nr_to_scan)
219 only_scan = 1;
220
221 mutex_lock(&pool_list_lock);
222 list_for_each_entry(pool, &pool_list, list) {
223 if (only_scan) {
224 nr_total += dmabuf_page_pool_do_shrink(pool,
225 gfp_mask,
226 nr_to_scan);
227 } else {
228 nr_freed = dmabuf_page_pool_do_shrink(pool,
229 gfp_mask,
230 nr_to_scan);
231 nr_to_scan -= nr_freed;
232 nr_total += nr_freed;
233 if (nr_to_scan <= 0)
234 break;
235 }
236 }
237 mutex_unlock(&pool_list_lock);
238
239 return nr_total;
240 }
241
dmabuf_page_pool_shrink_count(struct shrinker * shrinker,struct shrink_control * sc)242 static unsigned long dmabuf_page_pool_shrink_count(struct shrinker *shrinker,
243 struct shrink_control *sc)
244 {
245 return dmabuf_page_pool_shrink(sc->gfp_mask, 0);
246 }
247
dmabuf_page_pool_shrink_scan(struct shrinker * shrinker,struct shrink_control * sc)248 static unsigned long dmabuf_page_pool_shrink_scan(struct shrinker *shrinker,
249 struct shrink_control *sc)
250 {
251 if (sc->nr_to_scan == 0)
252 return 0;
253 return dmabuf_page_pool_shrink(sc->gfp_mask, sc->nr_to_scan);
254 }
255
256 struct shrinker pool_shrinker = {
257 .count_objects = dmabuf_page_pool_shrink_count,
258 .scan_objects = dmabuf_page_pool_shrink_scan,
259 .seeks = DEFAULT_SEEKS,
260 .batch = 0,
261 };
262
dmabuf_page_pool_init_shrinker(void)263 static int dmabuf_page_pool_init_shrinker(void)
264 {
265 return register_shrinker(&pool_shrinker);
266 }
267 module_init(dmabuf_page_pool_init_shrinker);
268 MODULE_LICENSE("GPL v2");
269