1 /*
2 *
3 * (C) COPYRIGHT 2012-2016 ARM Limited. All rights reserved.
4 *
5 * This program is free software and is provided to you under the terms of the
6 * GNU General Public License version 2 as published by the Free Software
7 * Foundation, and any use by you of this program is subject to the terms
8 * of such GNU licence.
9 *
10 * A copy of the licence is included with the program, and can also be obtained
11 * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
12 * Boston, MA 02110-1301, USA.
13 *
14 */
15
16
17
18
19 #include <mali_kbase.h>
20 #include <mali_kbase_config.h>
21
22 /*
23 * Private functions follow
24 */
25
26 /**
27 * @brief Check whether a ctx has a certain attribute, and if so, retain that
28 * attribute on the runpool.
29 *
30 * Requires:
31 * - jsctx mutex
32 * - runpool_irq spinlock
33 * - ctx is scheduled on the runpool
34 *
35 * @return true indicates a change in ctx attributes state of the runpool.
36 * In this state, the scheduler might be able to submit more jobs than
37 * previously, and so the caller should ensure kbasep_js_try_run_next_job_nolock()
38 * or similar is called sometime later.
39 * @return false indicates no change in ctx attributes state of the runpool.
40 */
kbasep_js_ctx_attr_runpool_retain_attr(struct kbase_device * kbdev,struct kbase_context * kctx,enum kbasep_js_ctx_attr attribute)41 static bool kbasep_js_ctx_attr_runpool_retain_attr(struct kbase_device *kbdev, struct kbase_context *kctx, enum kbasep_js_ctx_attr attribute)
42 {
43 struct kbasep_js_device_data *js_devdata;
44 struct kbasep_js_kctx_info *js_kctx_info;
45 bool runpool_state_changed = false;
46
47 KBASE_DEBUG_ASSERT(kbdev != NULL);
48 KBASE_DEBUG_ASSERT(kctx != NULL);
49 KBASE_DEBUG_ASSERT(attribute < KBASEP_JS_CTX_ATTR_COUNT);
50 js_devdata = &kbdev->js_data;
51 js_kctx_info = &kctx->jctx.sched_info;
52
53 lockdep_assert_held(&js_kctx_info->ctx.jsctx_mutex);
54 lockdep_assert_held(&kbdev->hwaccess_lock);
55
56 KBASE_DEBUG_ASSERT(kbase_ctx_flag(kctx, KCTX_SCHEDULED));
57
58 if (kbasep_js_ctx_attr_is_attr_on_ctx(kctx, attribute) != false) {
59 KBASE_DEBUG_ASSERT(js_devdata->runpool_irq.ctx_attr_ref_count[attribute] < S8_MAX);
60 ++(js_devdata->runpool_irq.ctx_attr_ref_count[attribute]);
61
62 if (js_devdata->runpool_irq.ctx_attr_ref_count[attribute] == 1) {
63 /* First refcount indicates a state change */
64 runpool_state_changed = true;
65 KBASE_TRACE_ADD(kbdev, JS_CTX_ATTR_NOW_ON_RUNPOOL, kctx, NULL, 0u, attribute);
66 }
67 }
68
69 return runpool_state_changed;
70 }
71
72 /**
73 * @brief Check whether a ctx has a certain attribute, and if so, release that
74 * attribute on the runpool.
75 *
76 * Requires:
77 * - jsctx mutex
78 * - runpool_irq spinlock
79 * - ctx is scheduled on the runpool
80 *
81 * @return true indicates a change in ctx attributes state of the runpool.
82 * In this state, the scheduler might be able to submit more jobs than
83 * previously, and so the caller should ensure kbasep_js_try_run_next_job_nolock()
84 * or similar is called sometime later.
85 * @return false indicates no change in ctx attributes state of the runpool.
86 */
kbasep_js_ctx_attr_runpool_release_attr(struct kbase_device * kbdev,struct kbase_context * kctx,enum kbasep_js_ctx_attr attribute)87 static bool kbasep_js_ctx_attr_runpool_release_attr(struct kbase_device *kbdev, struct kbase_context *kctx, enum kbasep_js_ctx_attr attribute)
88 {
89 struct kbasep_js_device_data *js_devdata;
90 struct kbasep_js_kctx_info *js_kctx_info;
91 bool runpool_state_changed = false;
92
93 KBASE_DEBUG_ASSERT(kbdev != NULL);
94 KBASE_DEBUG_ASSERT(kctx != NULL);
95 KBASE_DEBUG_ASSERT(attribute < KBASEP_JS_CTX_ATTR_COUNT);
96 js_devdata = &kbdev->js_data;
97 js_kctx_info = &kctx->jctx.sched_info;
98
99 lockdep_assert_held(&js_kctx_info->ctx.jsctx_mutex);
100 lockdep_assert_held(&kbdev->hwaccess_lock);
101 KBASE_DEBUG_ASSERT(kbase_ctx_flag(kctx, KCTX_SCHEDULED));
102
103 if (kbasep_js_ctx_attr_is_attr_on_ctx(kctx, attribute) != false) {
104 KBASE_DEBUG_ASSERT(js_devdata->runpool_irq.ctx_attr_ref_count[attribute] > 0);
105 --(js_devdata->runpool_irq.ctx_attr_ref_count[attribute]);
106
107 if (js_devdata->runpool_irq.ctx_attr_ref_count[attribute] == 0) {
108 /* Last de-refcount indicates a state change */
109 runpool_state_changed = true;
110 KBASE_TRACE_ADD(kbdev, JS_CTX_ATTR_NOW_OFF_RUNPOOL, kctx, NULL, 0u, attribute);
111 }
112 }
113
114 return runpool_state_changed;
115 }
116
117 /**
118 * @brief Retain a certain attribute on a ctx, also retaining it on the runpool
119 * if the context is scheduled.
120 *
121 * Requires:
122 * - jsctx mutex
123 * - If the context is scheduled, then runpool_irq spinlock must also be held
124 *
125 * @return true indicates a change in ctx attributes state of the runpool.
126 * This may allow the scheduler to submit more jobs than previously.
127 * @return false indicates no change in ctx attributes state of the runpool.
128 */
kbasep_js_ctx_attr_ctx_retain_attr(struct kbase_device * kbdev,struct kbase_context * kctx,enum kbasep_js_ctx_attr attribute)129 static bool kbasep_js_ctx_attr_ctx_retain_attr(struct kbase_device *kbdev, struct kbase_context *kctx, enum kbasep_js_ctx_attr attribute)
130 {
131 struct kbasep_js_kctx_info *js_kctx_info;
132 bool runpool_state_changed = false;
133
134 KBASE_DEBUG_ASSERT(kbdev != NULL);
135 KBASE_DEBUG_ASSERT(kctx != NULL);
136 KBASE_DEBUG_ASSERT(attribute < KBASEP_JS_CTX_ATTR_COUNT);
137 js_kctx_info = &kctx->jctx.sched_info;
138
139 lockdep_assert_held(&kbdev->hwaccess_lock);
140 lockdep_assert_held(&js_kctx_info->ctx.jsctx_mutex);
141 KBASE_DEBUG_ASSERT(js_kctx_info->ctx.ctx_attr_ref_count[attribute] < U32_MAX);
142
143 ++(js_kctx_info->ctx.ctx_attr_ref_count[attribute]);
144
145 if (kbase_ctx_flag(kctx, KCTX_SCHEDULED) && js_kctx_info->ctx.ctx_attr_ref_count[attribute] == 1) {
146 /* Only ref-count the attribute on the runpool for the first time this contexts sees this attribute */
147 KBASE_TRACE_ADD(kbdev, JS_CTX_ATTR_NOW_ON_CTX, kctx, NULL, 0u, attribute);
148 runpool_state_changed = kbasep_js_ctx_attr_runpool_retain_attr(kbdev, kctx, attribute);
149 }
150
151 return runpool_state_changed;
152 }
153
154 /*
155 * @brief Release a certain attribute on a ctx, also releasing it from the runpool
156 * if the context is scheduled.
157 *
158 * Requires:
159 * - jsctx mutex
160 * - If the context is scheduled, then runpool_irq spinlock must also be held
161 *
162 * @return true indicates a change in ctx attributes state of the runpool.
163 * This may allow the scheduler to submit more jobs than previously.
164 * @return false indicates no change in ctx attributes state of the runpool.
165 */
kbasep_js_ctx_attr_ctx_release_attr(struct kbase_device * kbdev,struct kbase_context * kctx,enum kbasep_js_ctx_attr attribute)166 static bool kbasep_js_ctx_attr_ctx_release_attr(struct kbase_device *kbdev, struct kbase_context *kctx, enum kbasep_js_ctx_attr attribute)
167 {
168 struct kbasep_js_kctx_info *js_kctx_info;
169 bool runpool_state_changed = false;
170
171 KBASE_DEBUG_ASSERT(kbdev != NULL);
172 KBASE_DEBUG_ASSERT(kctx != NULL);
173 KBASE_DEBUG_ASSERT(attribute < KBASEP_JS_CTX_ATTR_COUNT);
174 js_kctx_info = &kctx->jctx.sched_info;
175
176 lockdep_assert_held(&js_kctx_info->ctx.jsctx_mutex);
177 KBASE_DEBUG_ASSERT(js_kctx_info->ctx.ctx_attr_ref_count[attribute] > 0);
178
179 if (kbase_ctx_flag(kctx, KCTX_SCHEDULED) && js_kctx_info->ctx.ctx_attr_ref_count[attribute] == 1) {
180 lockdep_assert_held(&kbdev->hwaccess_lock);
181 /* Only de-ref-count the attribute on the runpool when this is the last ctx-reference to it */
182 runpool_state_changed = kbasep_js_ctx_attr_runpool_release_attr(kbdev, kctx, attribute);
183 KBASE_TRACE_ADD(kbdev, JS_CTX_ATTR_NOW_OFF_CTX, kctx, NULL, 0u, attribute);
184 }
185
186 /* De-ref must happen afterwards, because kbasep_js_ctx_attr_runpool_release() needs to check it too */
187 --(js_kctx_info->ctx.ctx_attr_ref_count[attribute]);
188
189 return runpool_state_changed;
190 }
191
192 /*
193 * More commonly used public functions
194 */
195
kbasep_js_ctx_attr_set_initial_attrs(struct kbase_device * kbdev,struct kbase_context * kctx)196 void kbasep_js_ctx_attr_set_initial_attrs(struct kbase_device *kbdev, struct kbase_context *kctx)
197 {
198 bool runpool_state_changed = false;
199
200 KBASE_DEBUG_ASSERT(kbdev != NULL);
201 KBASE_DEBUG_ASSERT(kctx != NULL);
202
203 if (kbase_ctx_flag(kctx, KCTX_SUBMIT_DISABLED)) {
204 /* This context never submits, so don't track any scheduling attributes */
205 return;
206 }
207
208 /* Transfer attributes held in the context flags for contexts that have submit enabled */
209
210 /* ... More attributes can be added here ... */
211
212 /* The context should not have been scheduled yet, so ASSERT if this caused
213 * runpool state changes (note that other threads *can't* affect the value
214 * of runpool_state_changed, due to how it's calculated) */
215 KBASE_DEBUG_ASSERT(runpool_state_changed == false);
216 CSTD_UNUSED(runpool_state_changed);
217 }
218
kbasep_js_ctx_attr_runpool_retain_ctx(struct kbase_device * kbdev,struct kbase_context * kctx)219 void kbasep_js_ctx_attr_runpool_retain_ctx(struct kbase_device *kbdev, struct kbase_context *kctx)
220 {
221 bool runpool_state_changed;
222 int i;
223
224 /* Retain any existing attributes */
225 for (i = 0; i < KBASEP_JS_CTX_ATTR_COUNT; ++i) {
226 if (kbasep_js_ctx_attr_is_attr_on_ctx(kctx, (enum kbasep_js_ctx_attr) i) != false) {
227 /* The context is being scheduled in, so update the runpool with the new attributes */
228 runpool_state_changed = kbasep_js_ctx_attr_runpool_retain_attr(kbdev, kctx, (enum kbasep_js_ctx_attr) i);
229
230 /* We don't need to know about state changed, because retaining a
231 * context occurs on scheduling it, and that itself will also try
232 * to run new atoms */
233 CSTD_UNUSED(runpool_state_changed);
234 }
235 }
236 }
237
kbasep_js_ctx_attr_runpool_release_ctx(struct kbase_device * kbdev,struct kbase_context * kctx)238 bool kbasep_js_ctx_attr_runpool_release_ctx(struct kbase_device *kbdev, struct kbase_context *kctx)
239 {
240 bool runpool_state_changed = false;
241 int i;
242
243 /* Release any existing attributes */
244 for (i = 0; i < KBASEP_JS_CTX_ATTR_COUNT; ++i) {
245 if (kbasep_js_ctx_attr_is_attr_on_ctx(kctx, (enum kbasep_js_ctx_attr) i) != false) {
246 /* The context is being scheduled out, so update the runpool on the removed attributes */
247 runpool_state_changed |= kbasep_js_ctx_attr_runpool_release_attr(kbdev, kctx, (enum kbasep_js_ctx_attr) i);
248 }
249 }
250
251 return runpool_state_changed;
252 }
253
kbasep_js_ctx_attr_ctx_retain_atom(struct kbase_device * kbdev,struct kbase_context * kctx,struct kbase_jd_atom * katom)254 void kbasep_js_ctx_attr_ctx_retain_atom(struct kbase_device *kbdev, struct kbase_context *kctx, struct kbase_jd_atom *katom)
255 {
256 bool runpool_state_changed = false;
257 base_jd_core_req core_req;
258
259 KBASE_DEBUG_ASSERT(katom);
260 core_req = katom->core_req;
261
262 if (core_req & BASE_JD_REQ_ONLY_COMPUTE)
263 runpool_state_changed |= kbasep_js_ctx_attr_ctx_retain_attr(kbdev, kctx, KBASEP_JS_CTX_ATTR_COMPUTE);
264 else
265 runpool_state_changed |= kbasep_js_ctx_attr_ctx_retain_attr(kbdev, kctx, KBASEP_JS_CTX_ATTR_NON_COMPUTE);
266
267 if ((core_req & (BASE_JD_REQ_CS | BASE_JD_REQ_ONLY_COMPUTE | BASE_JD_REQ_T)) != 0 && (core_req & (BASE_JD_REQ_COHERENT_GROUP | BASE_JD_REQ_SPECIFIC_COHERENT_GROUP)) == 0) {
268 /* Atom that can run on slot1 or slot2, and can use all cores */
269 runpool_state_changed |= kbasep_js_ctx_attr_ctx_retain_attr(kbdev, kctx, KBASEP_JS_CTX_ATTR_COMPUTE_ALL_CORES);
270 }
271
272 /* We don't need to know about state changed, because retaining an
273 * atom occurs on adding it, and that itself will also try to run
274 * new atoms */
275 CSTD_UNUSED(runpool_state_changed);
276 }
277
kbasep_js_ctx_attr_ctx_release_atom(struct kbase_device * kbdev,struct kbase_context * kctx,struct kbasep_js_atom_retained_state * katom_retained_state)278 bool kbasep_js_ctx_attr_ctx_release_atom(struct kbase_device *kbdev, struct kbase_context *kctx, struct kbasep_js_atom_retained_state *katom_retained_state)
279 {
280 bool runpool_state_changed = false;
281 base_jd_core_req core_req;
282
283 KBASE_DEBUG_ASSERT(katom_retained_state);
284 core_req = katom_retained_state->core_req;
285
286 /* No-op for invalid atoms */
287 if (kbasep_js_atom_retained_state_is_valid(katom_retained_state) == false)
288 return false;
289
290 if (core_req & BASE_JD_REQ_ONLY_COMPUTE)
291 runpool_state_changed |= kbasep_js_ctx_attr_ctx_release_attr(kbdev, kctx, KBASEP_JS_CTX_ATTR_COMPUTE);
292 else
293 runpool_state_changed |= kbasep_js_ctx_attr_ctx_release_attr(kbdev, kctx, KBASEP_JS_CTX_ATTR_NON_COMPUTE);
294
295 if ((core_req & (BASE_JD_REQ_CS | BASE_JD_REQ_ONLY_COMPUTE | BASE_JD_REQ_T)) != 0 && (core_req & (BASE_JD_REQ_COHERENT_GROUP | BASE_JD_REQ_SPECIFIC_COHERENT_GROUP)) == 0) {
296 /* Atom that can run on slot1 or slot2, and can use all cores */
297 runpool_state_changed |= kbasep_js_ctx_attr_ctx_release_attr(kbdev, kctx, KBASEP_JS_CTX_ATTR_COMPUTE_ALL_CORES);
298 }
299
300 return runpool_state_changed;
301 }
302