1 // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
2 /*
3 *
4 * (C) COPYRIGHT 2022 ARM Limited. All rights reserved.
5 *
6 * This program is free software and is provided to you under the terms of the
7 * GNU General Public License version 2 as published by the Free Software
8 * Foundation, and any use by you of this program is subject to the terms
9 * of such GNU license.
10 *
11 * This program is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
15 *
16 * You should have received a copy of the GNU General Public License
17 * along with this program; if not, you can access it online at
18 * http://www.gnu.org/licenses/gpl-2.0.html.
19 *
20 */
21
22 #include <mali_kbase.h>
23 #include "mali_kbase_csf.h"
24 #include "mali_kbase_csf_tiler_heap.h"
25 #include "mali_kbase_csf_tiler_heap_reclaim.h"
26
27 /* Tiler heap shrinker seek value, needs to be higher than jit and memory pools */
28 #define HEAP_SHRINKER_SEEKS (DEFAULT_SEEKS + 2)
29
30 /* Tiler heap shrinker batch value */
31 #define HEAP_SHRINKER_BATCH (512)
32
33 /* Tiler heap reclaim scan (free) method size for limiting a scan run length */
34 #define HEAP_RECLAIM_SCAN_BATCH_SIZE (HEAP_SHRINKER_BATCH << 7)
35
get_kctx_highest_csg_priority(struct kbase_context * kctx)36 static u8 get_kctx_highest_csg_priority(struct kbase_context *kctx)
37 {
38 u8 prio;
39
40 for (prio = KBASE_QUEUE_GROUP_PRIORITY_REALTIME; prio < KBASE_QUEUE_GROUP_PRIORITY_LOW;
41 prio++)
42 if (!list_empty(&kctx->csf.sched.runnable_groups[prio]))
43 break;
44
45 if (prio != KBASE_QUEUE_GROUP_PRIORITY_REALTIME && kctx->csf.sched.num_idle_wait_grps) {
46 struct kbase_queue_group *group;
47
48 list_for_each_entry(group, &kctx->csf.sched.idle_wait_groups, link) {
49 if (group->priority < prio)
50 prio = group->priority;
51 }
52 }
53
54 return prio;
55 }
56
detach_ctx_from_heap_reclaim_mgr(struct kbase_context * kctx)57 static void detach_ctx_from_heap_reclaim_mgr(struct kbase_context *kctx)
58 {
59 struct kbase_csf_scheduler *const scheduler = &kctx->kbdev->csf.scheduler;
60 struct kbase_csf_ctx_heap_reclaim_info *info = &kctx->csf.sched.heap_info;
61
62 lockdep_assert_held(&scheduler->lock);
63
64 if (!list_empty(&info->mgr_link)) {
65 u32 remaining = (info->nr_est_unused_pages > info->nr_freed_pages) ?
66 info->nr_est_unused_pages - info->nr_freed_pages :
67 0;
68
69 list_del_init(&info->mgr_link);
70 if (remaining)
71 WARN_ON(atomic_sub_return(remaining, &scheduler->reclaim_mgr.unused_pages) <
72 0);
73
74 dev_dbg(kctx->kbdev->dev,
75 "Reclaim_mgr_detach: ctx_%d_%d, est_pages=0%u, freed_pages=%u", kctx->tgid,
76 kctx->id, info->nr_est_unused_pages, info->nr_freed_pages);
77 }
78 }
79
attach_ctx_to_heap_reclaim_mgr(struct kbase_context * kctx)80 static void attach_ctx_to_heap_reclaim_mgr(struct kbase_context *kctx)
81 {
82 struct kbase_csf_ctx_heap_reclaim_info *const info = &kctx->csf.sched.heap_info;
83 struct kbase_csf_scheduler *const scheduler = &kctx->kbdev->csf.scheduler;
84 u8 const prio = get_kctx_highest_csg_priority(kctx);
85
86 lockdep_assert_held(&scheduler->lock);
87
88 if (WARN_ON(!list_empty(&info->mgr_link)))
89 list_del_init(&info->mgr_link);
90
91 /* Count the pages that could be freed */
92 info->nr_est_unused_pages = kbase_csf_tiler_heap_count_kctx_unused_pages(kctx);
93 /* Initialize the scan operation tracking pages */
94 info->nr_freed_pages = 0;
95
96 list_add_tail(&info->mgr_link, &scheduler->reclaim_mgr.ctx_lists[prio]);
97 /* Accumulate the estimated pages to the manager total field */
98 atomic_add(info->nr_est_unused_pages, &scheduler->reclaim_mgr.unused_pages);
99
100 dev_dbg(kctx->kbdev->dev, "Reclaim_mgr_attach: ctx_%d_%d, est_count_pages=%u", kctx->tgid,
101 kctx->id, info->nr_est_unused_pages);
102 }
103
kbase_csf_tiler_heap_reclaim_sched_notify_grp_active(struct kbase_queue_group * group)104 void kbase_csf_tiler_heap_reclaim_sched_notify_grp_active(struct kbase_queue_group *group)
105 {
106 struct kbase_context *kctx = group->kctx;
107 struct kbase_csf_ctx_heap_reclaim_info *info = &kctx->csf.sched.heap_info;
108
109 lockdep_assert_held(&kctx->kbdev->csf.scheduler.lock);
110
111 info->on_slot_grps++;
112 /* If the kctx has an on-slot change from 0 => 1, detach it from reclaim_mgr */
113 if (info->on_slot_grps == 1) {
114 dev_dbg(kctx->kbdev->dev, "CSG_%d_%d_%d on-slot, remove kctx from reclaim manager",
115 group->kctx->tgid, group->kctx->id, group->handle);
116
117 detach_ctx_from_heap_reclaim_mgr(kctx);
118 }
119 }
120
kbase_csf_tiler_heap_reclaim_sched_notify_grp_evict(struct kbase_queue_group * group)121 void kbase_csf_tiler_heap_reclaim_sched_notify_grp_evict(struct kbase_queue_group *group)
122 {
123 struct kbase_context *kctx = group->kctx;
124 struct kbase_csf_ctx_heap_reclaim_info *const info = &kctx->csf.sched.heap_info;
125 struct kbase_csf_scheduler *const scheduler = &kctx->kbdev->csf.scheduler;
126 const u32 num_groups = kctx->kbdev->csf.global_iface.group_num;
127 u32 on_slot_grps = 0;
128 u32 i;
129
130 lockdep_assert_held(&scheduler->lock);
131
132 /* Group eviction from the scheduler is a bit more complex, but fairly less
133 * frequent in operations. Taking the opportunity to actually count the
134 * on-slot CSGs from the given kctx, for robustness and clearer code logic.
135 */
136 for_each_set_bit(i, scheduler->csg_inuse_bitmap, num_groups) {
137 struct kbase_csf_csg_slot *csg_slot = &scheduler->csg_slots[i];
138 struct kbase_queue_group *grp = csg_slot->resident_group;
139
140 if (unlikely(!grp))
141 continue;
142
143 if (grp->kctx == kctx)
144 on_slot_grps++;
145 }
146
147 info->on_slot_grps = on_slot_grps;
148
149 /* If the kctx has no other CSGs on-slot, handle the heap reclaim related actions */
150 if (!info->on_slot_grps) {
151 if (kctx->csf.sched.num_runnable_grps || kctx->csf.sched.num_idle_wait_grps) {
152 /* The kctx has other operational CSGs, attach it if not yet done */
153 if (list_empty(&info->mgr_link)) {
154 dev_dbg(kctx->kbdev->dev,
155 "CSG_%d_%d_%d evict, add kctx to reclaim manager",
156 group->kctx->tgid, group->kctx->id, group->handle);
157
158 attach_ctx_to_heap_reclaim_mgr(kctx);
159 }
160 } else {
161 /* The kctx is a zombie after the group eviction, drop it out */
162 dev_dbg(kctx->kbdev->dev,
163 "CSG_%d_%d_%d evict leading to zombie kctx, dettach from reclaim manager",
164 group->kctx->tgid, group->kctx->id, group->handle);
165
166 detach_ctx_from_heap_reclaim_mgr(kctx);
167 }
168 }
169 }
170
kbase_csf_tiler_heap_reclaim_sched_notify_grp_suspend(struct kbase_queue_group * group)171 void kbase_csf_tiler_heap_reclaim_sched_notify_grp_suspend(struct kbase_queue_group *group)
172 {
173 struct kbase_context *kctx = group->kctx;
174 struct kbase_csf_ctx_heap_reclaim_info *info = &kctx->csf.sched.heap_info;
175
176 lockdep_assert_held(&kctx->kbdev->csf.scheduler.lock);
177
178 if (!WARN_ON(info->on_slot_grps == 0))
179 info->on_slot_grps--;
180 /* If the kctx has no CSGs on-slot, attach it to scheduler's reclaim manager */
181 if (info->on_slot_grps == 0) {
182 dev_dbg(kctx->kbdev->dev, "CSG_%d_%d_%d off-slot, add kctx to reclaim manager",
183 group->kctx->tgid, group->kctx->id, group->handle);
184
185 attach_ctx_to_heap_reclaim_mgr(kctx);
186 }
187 }
188
reclaim_unused_heap_pages(struct kbase_device * kbdev)189 static unsigned long reclaim_unused_heap_pages(struct kbase_device *kbdev)
190 {
191 struct kbase_csf_scheduler *const scheduler = &kbdev->csf.scheduler;
192 struct kbase_csf_sched_heap_reclaim_mgr *const mgr = &scheduler->reclaim_mgr;
193 unsigned long total_freed_pages = 0;
194 int prio;
195
196 lockdep_assert_held(&kbdev->csf.scheduler.lock);
197
198 for (prio = KBASE_QUEUE_GROUP_PRIORITY_LOW;
199 total_freed_pages < HEAP_RECLAIM_SCAN_BATCH_SIZE &&
200 prio >= KBASE_QUEUE_GROUP_PRIORITY_REALTIME;
201 prio--) {
202 struct kbase_csf_ctx_heap_reclaim_info *info, *tmp;
203 u32 cnt_ctxs = 0;
204
205 list_for_each_entry_safe(info, tmp, &scheduler->reclaim_mgr.ctx_lists[prio],
206 mgr_link) {
207 struct kbase_context *kctx =
208 container_of(info, struct kbase_context, csf.sched.heap_info);
209 u32 freed_pages = kbase_csf_tiler_heap_scan_kctx_unused_pages(
210 kctx, info->nr_est_unused_pages);
211
212 if (freed_pages) {
213 /* Remove the freed pages from the manager retained estimate. The
214 * accumulated removals from the kctx should not exceed the kctx
215 * initially notified contribution amount:
216 * info->nr_est_unused_pages.
217 */
218 u32 rm_cnt = MIN(info->nr_est_unused_pages - info->nr_freed_pages,
219 freed_pages);
220
221 WARN_ON(atomic_sub_return(rm_cnt, &mgr->unused_pages) < 0);
222
223 /* tracking the freed pages, before a potential detach call */
224 info->nr_freed_pages += freed_pages;
225 total_freed_pages += freed_pages;
226
227 schedule_work(&kctx->jit_work);
228 }
229
230 /* If the kctx can't offer anymore, drop it from the reclaim manger,
231 * otherwise leave it remaining in. If the kctx changes its state (i.e.
232 * some CSGs becoming on-slot), the scheduler will pull it out.
233 */
234 if (info->nr_freed_pages >= info->nr_est_unused_pages || freed_pages == 0)
235 detach_ctx_from_heap_reclaim_mgr(kctx);
236
237 cnt_ctxs++;
238
239 /* Enough has been freed, break to avoid holding the lock too long */
240 if (total_freed_pages >= HEAP_RECLAIM_SCAN_BATCH_SIZE)
241 break;
242 }
243
244 dev_dbg(kbdev->dev, "Reclaim free heap pages: %lu (cnt_ctxs: %u, prio: %d)",
245 total_freed_pages, cnt_ctxs, prio);
246 }
247
248 dev_dbg(kbdev->dev, "Reclaim free total heap pages: %lu (across all CSG priority)",
249 total_freed_pages);
250
251 return total_freed_pages;
252 }
253
kbase_csf_tiler_heap_reclaim_count_free_pages(struct kbase_device * kbdev,struct shrink_control * sc)254 static unsigned long kbase_csf_tiler_heap_reclaim_count_free_pages(struct kbase_device *kbdev,
255 struct shrink_control *sc)
256 {
257 struct kbase_csf_sched_heap_reclaim_mgr *mgr = &kbdev->csf.scheduler.reclaim_mgr;
258 unsigned long page_cnt = atomic_read(&mgr->unused_pages);
259
260 dev_dbg(kbdev->dev, "Reclaim count unused pages (estimate): %lu", page_cnt);
261
262 return page_cnt;
263 }
264
kbase_csf_tiler_heap_reclaim_scan_free_pages(struct kbase_device * kbdev,struct shrink_control * sc)265 static unsigned long kbase_csf_tiler_heap_reclaim_scan_free_pages(struct kbase_device *kbdev,
266 struct shrink_control *sc)
267 {
268 struct kbase_csf_sched_heap_reclaim_mgr *mgr = &kbdev->csf.scheduler.reclaim_mgr;
269 unsigned long freed = 0;
270 unsigned long avail = 0;
271
272 /* If Scheduler is busy in action, return 0 */
273 if (!mutex_trylock(&kbdev->csf.scheduler.lock)) {
274 struct kbase_csf_scheduler *const scheduler = &kbdev->csf.scheduler;
275
276 /* Wait for roughly 2-ms */
277 wait_event_timeout(kbdev->csf.event_wait, (scheduler->state != SCHED_BUSY),
278 msecs_to_jiffies(2));
279 if (!mutex_trylock(&kbdev->csf.scheduler.lock)) {
280 dev_dbg(kbdev->dev, "Tiler heap reclaim scan see device busy (freed: 0)");
281 return 0;
282 }
283 }
284
285 avail = atomic_read(&mgr->unused_pages);
286 if (avail)
287 freed = reclaim_unused_heap_pages(kbdev);
288
289 mutex_unlock(&kbdev->csf.scheduler.lock);
290
291 #if (KERNEL_VERSION(4, 14, 0) <= LINUX_VERSION_CODE)
292 if (freed > sc->nr_to_scan)
293 sc->nr_scanned = freed;
294 #endif /* (KERNEL_VERSION(4, 14, 0) <= LINUX_VERSION_CODE) */
295
296 dev_info(kbdev->dev, "Tiler heap reclaim scan freed pages: %lu (unused: %lu)", freed,
297 avail);
298
299 /* On estimate suggesting available, yet actual free failed, return STOP */
300 if (avail && !freed)
301 return SHRINK_STOP;
302 else
303 return freed;
304 }
305
kbase_csf_tiler_heap_reclaim_count_objects(struct shrinker * s,struct shrink_control * sc)306 static unsigned long kbase_csf_tiler_heap_reclaim_count_objects(struct shrinker *s,
307 struct shrink_control *sc)
308 {
309 struct kbase_device *kbdev =
310 container_of(s, struct kbase_device, csf.scheduler.reclaim_mgr.heap_reclaim);
311
312 return kbase_csf_tiler_heap_reclaim_count_free_pages(kbdev, sc);
313 }
314
kbase_csf_tiler_heap_reclaim_scan_objects(struct shrinker * s,struct shrink_control * sc)315 static unsigned long kbase_csf_tiler_heap_reclaim_scan_objects(struct shrinker *s,
316 struct shrink_control *sc)
317 {
318 struct kbase_device *kbdev =
319 container_of(s, struct kbase_device, csf.scheduler.reclaim_mgr.heap_reclaim);
320
321 return kbase_csf_tiler_heap_reclaim_scan_free_pages(kbdev, sc);
322 }
323
kbase_csf_tiler_heap_reclaim_ctx_init(struct kbase_context * kctx)324 void kbase_csf_tiler_heap_reclaim_ctx_init(struct kbase_context *kctx)
325 {
326 /* Per-kctx heap_info object initialization */
327 memset(&kctx->csf.sched.heap_info, 0, sizeof(struct kbase_csf_ctx_heap_reclaim_info));
328 INIT_LIST_HEAD(&kctx->csf.sched.heap_info.mgr_link);
329 }
330
kbase_csf_tiler_heap_reclaim_mgr_init(struct kbase_device * kbdev)331 void kbase_csf_tiler_heap_reclaim_mgr_init(struct kbase_device *kbdev)
332 {
333 struct kbase_csf_scheduler *scheduler = &kbdev->csf.scheduler;
334 struct shrinker *reclaim = &scheduler->reclaim_mgr.heap_reclaim;
335 u8 prio;
336
337 for (prio = KBASE_QUEUE_GROUP_PRIORITY_REALTIME; prio < KBASE_QUEUE_GROUP_PRIORITY_COUNT;
338 prio++)
339 INIT_LIST_HEAD(&scheduler->reclaim_mgr.ctx_lists[prio]);
340
341 atomic_set(&scheduler->reclaim_mgr.unused_pages, 0);
342
343 reclaim->count_objects = kbase_csf_tiler_heap_reclaim_count_objects;
344 reclaim->scan_objects = kbase_csf_tiler_heap_reclaim_scan_objects;
345 reclaim->seeks = HEAP_SHRINKER_SEEKS;
346 reclaim->batch = HEAP_SHRINKER_BATCH;
347 }
348
kbase_csf_tiler_heap_reclaim_mgr_term(struct kbase_device * kbdev)349 void kbase_csf_tiler_heap_reclaim_mgr_term(struct kbase_device *kbdev)
350 {
351 struct kbase_csf_scheduler *scheduler = &kbdev->csf.scheduler;
352 u8 prio;
353
354 for (prio = KBASE_QUEUE_GROUP_PRIORITY_REALTIME; prio < KBASE_QUEUE_GROUP_PRIORITY_COUNT;
355 prio++)
356 WARN_ON(!list_empty(&scheduler->reclaim_mgr.ctx_lists[prio]));
357
358 WARN_ON(atomic_read(&scheduler->reclaim_mgr.unused_pages));
359 }
360