xref: /OK3568_Linux_fs/kernel/drivers/gpu/arm/midgard/mali_kbase_jd.c (revision 4882a59341e53eb6f0b4789bf948001014eff981)

/*
 *
 * (C) COPYRIGHT 2010-2017 ARM Limited. All rights reserved.
 *
 * This program is free software and is provided to you under the terms of the
 * GNU General Public License version 2 as published by the Free Software
 * Foundation, and any use by you of this program is subject to the terms
 * of such GNU licence.
 *
 * A copy of the licence is included with the program, and can also be obtained
 * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
 * Boston, MA  02110-1301, USA.
 *
 */





#if defined(CONFIG_DMA_SHARED_BUFFER)
#include <linux/dma-buf.h>
#endif				/* defined(CONFIG_DMA_SHARED_BUFFER) */
#ifdef CONFIG_COMPAT
#include <linux/compat.h>
#endif
#include <mali_kbase.h>
#include <mali_kbase_uku.h>
#include <linux/random.h>
#include <linux/version.h>
#include <linux/ratelimit.h>
#include <linux/nospec.h>

#include <mali_kbase_jm.h>
#include <mali_kbase_hwaccess_jm.h>
#include <mali_kbase_tlstream.h>

#include "mali_kbase_dma_fence.h"

#define beenthere(kctx, f, a...)  dev_dbg(kctx->kbdev->dev, "%s:" f, __func__, ##a)

#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 8, 0)
/* random32 was renamed to prandom_u32 in 3.8 */
#define prandom_u32 random32
#endif

/* Return whether katom will run on the GPU or not. Currently only soft jobs and
 * dependency-only atoms do not run on the GPU */
#define IS_GPU_ATOM(katom) (!((katom->core_req & BASE_JD_REQ_SOFT_JOB) ||  \
			((katom->core_req & BASE_JD_REQ_ATOM_TYPE) ==    \
							BASE_JD_REQ_DEP)))
/*
 * This is the kernel side of the API. Only entry points are:
 * - kbase_jd_submit(): Called from userspace to submit a single bag
 * - kbase_jd_done(): Called from interrupt context to track the
 *   completion of a job.
 * Callouts:
 * - to the job manager (enqueue a job)
 * - to the event subsystem (signals the completion/failure of bag/job-chains).
 */

static void __user *
get_compat_pointer(struct kbase_context *kctx, const union kbase_pointer *p)
{
#ifdef CONFIG_COMPAT
	if (kbase_ctx_flag(kctx, KCTX_COMPAT))
		return compat_ptr(p->compat_value);
#endif
	return p->value;
}

/* Runs an atom, either by handing to the JS or by immediately running it in the case of soft-jobs
 *
 * Returns whether the JS needs a reschedule.
 *
 * Note that the caller must also check the atom status and
 * if it is KBASE_JD_ATOM_STATE_COMPLETED must call jd_done_nolock
 */
static int jd_run_atom(struct kbase_jd_atom *katom)
{
	struct kbase_context *kctx = katom->kctx;

	KBASE_DEBUG_ASSERT(katom->status != KBASE_JD_ATOM_STATE_UNUSED);

	if ((katom->core_req & BASE_JD_REQ_ATOM_TYPE) == BASE_JD_REQ_DEP) {
		/* Dependency only atom */
		katom->status = KBASE_JD_ATOM_STATE_COMPLETED;
		return 0;
	} else if (katom->core_req & BASE_JD_REQ_SOFT_JOB) {
		/* Soft-job */
		if (katom->will_fail_event_code) {
			katom->status = KBASE_JD_ATOM_STATE_COMPLETED;
			return 0;
		}
		if ((katom->core_req & BASE_JD_REQ_SOFT_JOB_TYPE)
						  == BASE_JD_REQ_SOFT_REPLAY) {
			if (!kbase_replay_process(katom))
				katom->status = KBASE_JD_ATOM_STATE_COMPLETED;
		} else if (kbase_process_soft_job(katom) == 0) {
			kbase_finish_soft_job(katom);
			katom->status = KBASE_JD_ATOM_STATE_COMPLETED;
		}
		return 0;
	}

	katom->status = KBASE_JD_ATOM_STATE_IN_JS;
	/* Queue an action about whether we should try scheduling a context */
	return kbasep_js_add_job(kctx, katom);
}

#if defined(CONFIG_KDS) || defined(CONFIG_MALI_DMA_FENCE)
void kbase_jd_dep_clear_locked(struct kbase_jd_atom *katom)
{
	struct kbase_device *kbdev;

	KBASE_DEBUG_ASSERT(katom);
	kbdev = katom->kctx->kbdev;
	KBASE_DEBUG_ASSERT(kbdev);

	/* Check whether the atom's other dependencies were already met. If
	 * katom is a GPU atom then the job scheduler may be able to represent
	 * the dependencies, hence we may attempt to submit it before they are
	 * met. Other atoms must have had both dependencies resolved.
	 */
	if (IS_GPU_ATOM(katom) ||
			(!kbase_jd_katom_dep_atom(&katom->dep[0]) &&
			!kbase_jd_katom_dep_atom(&katom->dep[1]))) {
		/* katom dep complete, attempt to run it */
		bool resched = false;

		resched = jd_run_atom(katom);

		if (katom->status == KBASE_JD_ATOM_STATE_COMPLETED) {
			/* The atom has already finished */
			resched |= jd_done_nolock(katom, NULL);
		}

		if (resched)
			kbase_js_sched_all(kbdev);
	}
}
#endif

#ifdef CONFIG_KDS

/* Add the katom to the kds waiting list.
 * Atoms must be added to the waiting list after a successful call to kds_async_waitall.
 * The caller must hold the kbase_jd_context.lock */

static void kbase_jd_kds_waiters_add(struct kbase_jd_atom *katom)
{
	struct kbase_context *kctx;

	KBASE_DEBUG_ASSERT(katom);

	kctx = katom->kctx;

	list_add_tail(&katom->node, &kctx->waiting_kds_resource);
}

/* Remove the katom from the kds waiting list.
 * Atoms must be removed from the waiting list before a call to kds_resource_set_release_sync.
 * The supplied katom must first have been added to the list with a call to kbase_jd_kds_waiters_add.
 * The caller must hold the kbase_jd_context.lock */

static void kbase_jd_kds_waiters_remove(struct kbase_jd_atom *katom)
{
	KBASE_DEBUG_ASSERT(katom);
	list_del(&katom->node);
}

static void kds_dep_clear(void *callback_parameter, void *callback_extra_parameter)
{
	struct kbase_jd_atom *katom;
	struct kbase_jd_context *ctx;

	katom = (struct kbase_jd_atom *)callback_parameter;
	KBASE_DEBUG_ASSERT(katom);

	ctx = &katom->kctx->jctx;

	/* If KDS resource has already been satisfied (e.g. due to zapping)
	 * do nothing.
	 */
	mutex_lock(&ctx->lock);
	if (!katom->kds_dep_satisfied) {
		katom->kds_dep_satisfied = true;
		kbase_jd_dep_clear_locked(katom);
	}
	mutex_unlock(&ctx->lock);
}

static void kbase_cancel_kds_wait_job(struct kbase_jd_atom *katom)
{
	KBASE_DEBUG_ASSERT(katom);

	/* Prevent job_done_nolock from being called twice on an atom when
	 *  there is a race between job completion and cancellation */

	if (katom->status == KBASE_JD_ATOM_STATE_QUEUED) {
		/* Wait was cancelled - zap the atom */
		katom->event_code = BASE_JD_EVENT_JOB_CANCELLED;
		if (jd_done_nolock(katom, NULL))
			kbase_js_sched_all(katom->kctx->kbdev);
	}
}
#endif				/* CONFIG_KDS */

void kbase_jd_free_external_resources(struct kbase_jd_atom *katom)
{
#ifdef CONFIG_KDS
	if (katom->kds_rset) {
		struct kbase_jd_context *jctx = &katom->kctx->jctx;

		/*
		 * As the atom is no longer waiting, remove it from
		 * the waiting list.
		 */

		mutex_lock(&jctx->lock);
		kbase_jd_kds_waiters_remove(katom);
		mutex_unlock(&jctx->lock);

		/* Release the kds resource or cancel if zapping */
		kds_resource_set_release_sync(&katom->kds_rset);
	}
#endif				/* CONFIG_KDS */

#ifdef CONFIG_MALI_DMA_FENCE
	/* Flush dma-fence workqueue to ensure that any callbacks that may have
	 * been queued are done before continuing.
	 * Any successfully completed atom would have had all it's callbacks
	 * completed before the atom was run, so only flush for failed atoms.
	 */
	if (katom->event_code != BASE_JD_EVENT_DONE)
		flush_workqueue(katom->kctx->dma_fence.wq);
#endif /* CONFIG_MALI_DMA_FENCE */
}

static void kbase_jd_post_external_resources(struct kbase_jd_atom *katom)
{
	KBASE_DEBUG_ASSERT(katom);
	KBASE_DEBUG_ASSERT(katom->core_req & BASE_JD_REQ_EXTERNAL_RESOURCES);

#ifdef CONFIG_KDS
	/* Prevent the KDS resource from triggering the atom in case of zapping */
	if (katom->kds_rset)
		katom->kds_dep_satisfied = true;
#endif				/* CONFIG_KDS */

#ifdef CONFIG_MALI_DMA_FENCE
	kbase_dma_fence_signal(katom);
#endif /* CONFIG_MALI_DMA_FENCE */

	kbase_gpu_vm_lock(katom->kctx);
	/* only roll back if extres is non-NULL */
	if (katom->extres) {
		u32 res_no;

		res_no = katom->nr_extres;
		while (res_no-- > 0) {
			struct kbase_mem_phy_alloc *alloc = katom->extres[res_no].alloc;
			struct kbase_va_region *reg;

			reg = kbase_region_tracker_find_region_base_address(
					katom->kctx,
					katom->extres[res_no].gpu_address);
			kbase_unmap_external_resource(katom->kctx, reg, alloc);
		}
		kfree(katom->extres);
		katom->extres = NULL;
	}
	kbase_gpu_vm_unlock(katom->kctx);
}

/*
 * Set up external resources needed by this job.
 *
 * jctx.lock must be held when this is called.
 */

static int kbase_jd_pre_external_resources(struct kbase_jd_atom *katom, const struct base_jd_atom_v2 *user_atom)
{
	int err_ret_val = -EINVAL;
	u32 res_no;
#ifdef CONFIG_KDS
	u32 kds_res_count = 0;
	struct kds_resource **kds_resources = NULL;
	unsigned long *kds_access_bitmap = NULL;
#endif				/* CONFIG_KDS */
#ifdef CONFIG_MALI_DMA_FENCE
	struct kbase_dma_fence_resv_info info = {
		.dma_fence_resv_count = 0,
	};
#ifdef CONFIG_SYNC
	/*
	 * When both dma-buf fence and Android native sync is enabled, we
	 * disable dma-buf fence for contexts that are using Android native
	 * fences.
	 */
	const bool implicit_sync = !kbase_ctx_flag(katom->kctx,
						   KCTX_NO_IMPLICIT_SYNC);
#else /* CONFIG_SYNC */
	const bool implicit_sync = true;
#endif /* CONFIG_SYNC */
#endif /* CONFIG_MALI_DMA_FENCE */
	struct base_external_resource *input_extres;

	KBASE_DEBUG_ASSERT(katom);
	KBASE_DEBUG_ASSERT(katom->core_req & BASE_JD_REQ_EXTERNAL_RESOURCES);

	/* no resources encoded, early out */
	if (!katom->nr_extres)
		return -EINVAL;

	katom->extres = kmalloc_array(katom->nr_extres, sizeof(*katom->extres), GFP_KERNEL);
	if (NULL == katom->extres) {
		err_ret_val = -ENOMEM;
		goto early_err_out;
	}

	/* copy user buffer to the end of our real buffer.
	 * Make sure the struct sizes haven't changed in a way
	 * we don't support */
	BUILD_BUG_ON(sizeof(*input_extres) > sizeof(*katom->extres));
	input_extres = (struct base_external_resource *)
			(((unsigned char *)katom->extres) +
			(sizeof(*katom->extres) - sizeof(*input_extres)) *
			katom->nr_extres);

	if (copy_from_user(input_extres,
			get_compat_pointer(katom->kctx, &user_atom->extres_list),
			sizeof(*input_extres) * katom->nr_extres) != 0) {
		err_ret_val = -EINVAL;
		goto early_err_out;
	}
#ifdef CONFIG_KDS
	/* assume we have to wait for all */
	KBASE_DEBUG_ASSERT(0 != katom->nr_extres);
	kds_resources = kmalloc_array(katom->nr_extres, sizeof(struct kds_resource *), GFP_KERNEL);

	if (!kds_resources) {
		err_ret_val = -ENOMEM;
		goto early_err_out;
	}

	KBASE_DEBUG_ASSERT(0 != katom->nr_extres);
	kds_access_bitmap = kcalloc(BITS_TO_LONGS(katom->nr_extres),
				    sizeof(unsigned long),
				    GFP_KERNEL);
	if (!kds_access_bitmap) {
		err_ret_val = -ENOMEM;
		goto early_err_out;
	}
#endif				/* CONFIG_KDS */

#ifdef CONFIG_MALI_DMA_FENCE
	if (implicit_sync) {
		info.resv_objs = kmalloc_array(katom->nr_extres,
					sizeof(struct reservation_object *),
					GFP_KERNEL);
		if (!info.resv_objs) {
			err_ret_val = -ENOMEM;
			goto early_err_out;
		}

		info.dma_fence_excl_bitmap =
				kcalloc(BITS_TO_LONGS(katom->nr_extres),
					sizeof(unsigned long), GFP_KERNEL);
		if (!info.dma_fence_excl_bitmap) {
			err_ret_val = -ENOMEM;
			goto early_err_out;
		}
	}
#endif /* CONFIG_MALI_DMA_FENCE */

	/* Take the processes mmap lock */
	down_read(&current->mm->mmap_lock);

	/* need to keep the GPU VM locked while we set up UMM buffers */
	kbase_gpu_vm_lock(katom->kctx);
	for (res_no = 0; res_no < katom->nr_extres; res_no++) {
		struct base_external_resource *res;
		struct kbase_va_region *reg;
		struct kbase_mem_phy_alloc *alloc;
		bool exclusive;

		res = &input_extres[res_no];
		exclusive = (res->ext_resource & BASE_EXT_RES_ACCESS_EXCLUSIVE)
				? true : false;
		reg = kbase_region_tracker_find_region_enclosing_address(
				katom->kctx,
				res->ext_resource & ~BASE_EXT_RES_ACCESS_EXCLUSIVE);
		/* did we find a matching region object? */
		if (NULL == reg || (reg->flags & KBASE_REG_FREE)) {
			/* roll back */
			goto failed_loop;
		}

		if (!(katom->core_req & BASE_JD_REQ_SOFT_JOB) &&
				(reg->flags & KBASE_REG_SECURE)) {
			katom->atom_flags |= KBASE_KATOM_FLAG_PROTECTED;
		}

		alloc = kbase_map_external_resource(katom->kctx, reg,
				current->mm
#ifdef CONFIG_KDS
				, &kds_res_count, kds_resources,
				kds_access_bitmap, exclusive
#endif
				);
		if (!alloc) {
			err_ret_val = -EINVAL;
			goto failed_loop;
		}

#ifdef CONFIG_MALI_DMA_FENCE
		if (implicit_sync &&
		    reg->gpu_alloc->type == KBASE_MEM_TYPE_IMPORTED_UMM) {
			struct reservation_object *resv;

			resv = reg->gpu_alloc->imported.umm.dma_buf->resv;
			if (resv)
				kbase_dma_fence_add_reservation(resv, &info,
								exclusive);
		}
#endif /* CONFIG_MALI_DMA_FENCE */

		/* finish with updating out array with the data we found */
		/* NOTE: It is important that this is the last thing we do (or
		 * at least not before the first write) as we overwrite elements
		 * as we loop and could be overwriting ourself, so no writes
		 * until the last read for an element.
		 * */
		katom->extres[res_no].gpu_address = reg->start_pfn << PAGE_SHIFT; /* save the start_pfn (as an address, not pfn) to use fast lookup later */
		katom->extres[res_no].alloc = alloc;
	}
	/* successfully parsed the extres array */
	/* drop the vm lock before we call into kds */
	kbase_gpu_vm_unlock(katom->kctx);

	/* Release the processes mmap lock */
	up_read(&current->mm->mmap_lock);

#ifdef CONFIG_KDS
	if (kds_res_count) {
		int wait_failed;

		/* We have resources to wait for with kds */
		katom->kds_dep_satisfied = false;

		wait_failed = kds_async_waitall(&katom->kds_rset,
				&katom->kctx->jctx.kds_cb, katom, NULL,
				kds_res_count, kds_access_bitmap,
				kds_resources);

		if (wait_failed)
			goto failed_kds_setup;
		else
			kbase_jd_kds_waiters_add(katom);
	} else {
		/* Nothing to wait for, so kds dep met */
		katom->kds_dep_satisfied = true;
	}
	kfree(kds_resources);
	kfree(kds_access_bitmap);
#endif				/* CONFIG_KDS */

#ifdef CONFIG_MALI_DMA_FENCE
	if (implicit_sync) {
		if (info.dma_fence_resv_count) {
			int ret;

			ret = kbase_dma_fence_wait(katom, &info);
			if (ret < 0)
				goto failed_dma_fence_setup;
		}

		kfree(info.resv_objs);
		kfree(info.dma_fence_excl_bitmap);
	}
#endif /* CONFIG_MALI_DMA_FENCE */

	/* all done OK */
	return 0;

/* error handling section */

#ifdef CONFIG_MALI_DMA_FENCE
failed_dma_fence_setup:
#ifdef CONFIG_KDS
	/* If we are here, dma_fence setup failed but KDS didn't.
	 * Revert KDS setup if any.
	 */
	if (kds_res_count) {
		mutex_unlock(&katom->kctx->jctx.lock);
		kds_resource_set_release_sync(&katom->kds_rset);
		mutex_lock(&katom->kctx->jctx.lock);

		kbase_jd_kds_waiters_remove(katom);
		katom->kds_dep_satisfied = true;
	}
#endif /* CONFIG_KDS */
#endif /* CONFIG_MALI_DMA_FENCE */
#ifdef CONFIG_KDS
failed_kds_setup:
#endif
#if defined(CONFIG_KDS) || defined(CONFIG_MALI_DMA_FENCE)
	/* Lock the processes mmap lock */
	down_read(&current->mm->mmap_lock);

	/* lock before we unmap */
	kbase_gpu_vm_lock(katom->kctx);
#endif

 failed_loop:
	/* undo the loop work */
	while (res_no-- > 0) {
		struct kbase_mem_phy_alloc *alloc = katom->extres[res_no].alloc;

		kbase_unmap_external_resource(katom->kctx, NULL, alloc);
	}
	kbase_gpu_vm_unlock(katom->kctx);

	/* Release the processes mmap lock */
	up_read(&current->mm->mmap_lock);

 early_err_out:
	kfree(katom->extres);
	katom->extres = NULL;
#ifdef CONFIG_KDS
	kfree(kds_resources);
	kfree(kds_access_bitmap);
#endif				/* CONFIG_KDS */
#ifdef CONFIG_MALI_DMA_FENCE
	if (implicit_sync) {
		kfree(info.resv_objs);
		kfree(info.dma_fence_excl_bitmap);
	}
#endif
	return err_ret_val;
}

static inline void jd_resolve_dep(struct list_head *out_list,
					struct kbase_jd_atom *katom,
					u8 d, bool ctx_is_dying)
{
	u8 other_d = !d;

	while (!list_empty(&katom->dep_head[d])) {
		struct kbase_jd_atom *dep_atom;
		struct kbase_jd_atom *other_dep_atom;
		u8 dep_type;

		dep_atom = list_entry(katom->dep_head[d].next,
				struct kbase_jd_atom, dep_item[d]);
		list_del(katom->dep_head[d].next);

		dep_type = kbase_jd_katom_dep_type(&dep_atom->dep[d]);
		kbase_jd_katom_dep_clear(&dep_atom->dep[d]);

		if (katom->event_code != BASE_JD_EVENT_DONE &&
			(dep_type != BASE_JD_DEP_TYPE_ORDER)) {
#ifdef CONFIG_KDS
			if (!dep_atom->kds_dep_satisfied) {
				/* Just set kds_dep_satisfied to true. If the callback happens after this then it will early out and
				 * do nothing. If the callback doesn't happen then kbase_jd_post_external_resources will clean up
				 */
				dep_atom->kds_dep_satisfied = true;
			}
#endif

#ifdef CONFIG_MALI_DMA_FENCE
			kbase_dma_fence_cancel_callbacks(dep_atom);
#endif

			dep_atom->event_code = katom->event_code;
			KBASE_DEBUG_ASSERT(dep_atom->status !=
						KBASE_JD_ATOM_STATE_UNUSED);

			if ((dep_atom->core_req & BASE_JD_REQ_SOFT_REPLAY)
					!= BASE_JD_REQ_SOFT_REPLAY) {
				dep_atom->will_fail_event_code =
					dep_atom->event_code;
			} else {
				dep_atom->status =
					KBASE_JD_ATOM_STATE_COMPLETED;
			}
		}
		other_dep_atom = (struct kbase_jd_atom *)
			kbase_jd_katom_dep_atom(&dep_atom->dep[other_d]);

		if (!dep_atom->in_jd_list && (!other_dep_atom ||
				(IS_GPU_ATOM(dep_atom) && !ctx_is_dying &&
				!dep_atom->will_fail_event_code &&
				!other_dep_atom->will_fail_event_code))) {
			bool dep_satisfied = true;
#ifdef CONFIG_MALI_DMA_FENCE
			int dep_count;

			dep_count = kbase_fence_dep_count_read(dep_atom);
			if (likely(dep_count == -1)) {
				dep_satisfied = true;
			} else {
				/*
				 * There are either still active callbacks, or
				 * all fences for this @dep_atom has signaled,
				 * but the worker that will queue the atom has
				 * not yet run.
				 *
				 * Wait for the fences to signal and the fence
				 * worker to run and handle @dep_atom. If
				 * @dep_atom was completed due to error on
				 * @katom, then the fence worker will pick up
				 * the complete status and error code set on
				 * @dep_atom above.
				 */
				dep_satisfied = false;
			}
#endif /* CONFIG_MALI_DMA_FENCE */

#ifdef CONFIG_KDS
			dep_satisfied = dep_satisfied && dep_atom->kds_dep_satisfied;
#endif

			if (dep_satisfied) {
				dep_atom->in_jd_list = true;
				list_add_tail(&dep_atom->jd_item, out_list);
			}
		}
	}
}

KBASE_EXPORT_TEST_API(jd_resolve_dep);

#if MALI_CUSTOMER_RELEASE == 0
static void jd_force_failure(struct kbase_device *kbdev, struct kbase_jd_atom *katom)
{
	kbdev->force_replay_count++;

	if (kbdev->force_replay_count >= kbdev->force_replay_limit) {
		kbdev->force_replay_count = 0;
		katom->event_code = BASE_JD_EVENT_FORCE_REPLAY;

		if (kbdev->force_replay_random)
			kbdev->force_replay_limit =
			   (prandom_u32() % KBASEP_FORCE_REPLAY_RANDOM_LIMIT) + 1;

		dev_info(kbdev->dev, "force_replay : promoting to error\n");
	}
}

/** Test to see if atom should be forced to fail.
 *
 * This function will check if an atom has a replay job as a dependent. If so
 * then it will be considered for forced failure. */
static void jd_check_force_failure(struct kbase_jd_atom *katom)
{
	struct kbase_context *kctx = katom->kctx;
	struct kbase_device *kbdev = kctx->kbdev;
	int i;

	if ((kbdev->force_replay_limit == KBASEP_FORCE_REPLAY_DISABLED) ||
	    (katom->core_req & BASEP_JD_REQ_EVENT_NEVER))
		return;

	for (i = 1; i < BASE_JD_ATOM_COUNT; i++) {
		if (kbase_jd_katom_dep_atom(&kctx->jctx.atoms[i].dep[0]) == katom ||
		    kbase_jd_katom_dep_atom(&kctx->jctx.atoms[i].dep[1]) == katom) {
			struct kbase_jd_atom *dep_atom = &kctx->jctx.atoms[i];

			if ((dep_atom->core_req & BASE_JD_REQ_SOFT_JOB_TYPE) ==
						     BASE_JD_REQ_SOFT_REPLAY &&
			    (dep_atom->core_req & kbdev->force_replay_core_req)
					     == kbdev->force_replay_core_req) {
				jd_force_failure(kbdev, katom);
				return;
			}
		}
	}
}
#endif

/**
 * is_dep_valid - Validate that a dependency is valid for early dependency
 *                submission
 * @katom: Dependency atom to validate
 *
 * A dependency is valid if any of the following are true :
 * - It does not exist (a non-existent dependency does not block submission)
 * - It is in the job scheduler
 * - It has completed, does not have a failure event code, and has not been
 *   marked to fail in the future
 *
 * Return: true if valid, false otherwise
 */
static bool is_dep_valid(struct kbase_jd_atom *katom)
{
	/* If there's no dependency then this is 'valid' from the perspective of
	 * early dependency submission */
	if (!katom)
		return true;

	/* Dependency must have reached the job scheduler */
	if (katom->status < KBASE_JD_ATOM_STATE_IN_JS)
		return false;

	/* If dependency has completed and has failed or will fail then it is
	 * not valid */
	if (katom->status >= KBASE_JD_ATOM_STATE_HW_COMPLETED &&
			(katom->event_code != BASE_JD_EVENT_DONE ||
			katom->will_fail_event_code))
		return false;

	return true;
}

static void jd_try_submitting_deps(struct list_head *out_list,
		struct kbase_jd_atom *node)
{
	int i;

	for (i = 0; i < 2; i++) {
		struct list_head *pos;

		list_for_each(pos, &node->dep_head[i]) {
			struct kbase_jd_atom *dep_atom = list_entry(pos,
					struct kbase_jd_atom, dep_item[i]);

			if (IS_GPU_ATOM(dep_atom) && !dep_atom->in_jd_list) {
				/*Check if atom deps look sane*/
				bool dep0_valid = is_dep_valid(
						dep_atom->dep[0].atom);
				bool dep1_valid = is_dep_valid(
						dep_atom->dep[1].atom);
				bool dep_satisfied = true;
#ifdef CONFIG_MALI_DMA_FENCE
				int dep_count;

				dep_count = kbase_fence_dep_count_read(
								dep_atom);
				if (likely(dep_count == -1)) {
					dep_satisfied = true;
				} else {
				/*
				 * There are either still active callbacks, or
				 * all fences for this @dep_atom has signaled,
				 * but the worker that will queue the atom has
				 * not yet run.
				 *
				 * Wait for the fences to signal and the fence
				 * worker to run and handle @dep_atom. If
				 * @dep_atom was completed due to error on
				 * @katom, then the fence worker will pick up
				 * the complete status and error code set on
				 * @dep_atom above.
				 */
					dep_satisfied = false;
				}
#endif /* CONFIG_MALI_DMA_FENCE */
#ifdef CONFIG_KDS
				dep_satisfied = dep_satisfied &&
						dep_atom->kds_dep_satisfied;
#endif

				if (dep0_valid && dep1_valid && dep_satisfied) {
					dep_atom->in_jd_list = true;
					list_add(&dep_atom->jd_item, out_list);
				}
			}
		}
	}
}

/*
 * Perform the necessary handling of an atom that has finished running
 * on the GPU.
 *
 * Note that if this is a soft-job that has had kbase_prepare_soft_job called on it then the caller
 * is responsible for calling kbase_finish_soft_job *before* calling this function.
 *
 * The caller must hold the kbase_jd_context.lock.
 */
bool jd_done_nolock(struct kbase_jd_atom *katom,
		struct list_head *completed_jobs_ctx)
{
	struct kbase_context *kctx = katom->kctx;
	struct kbase_device *kbdev = kctx->kbdev;
	struct list_head completed_jobs;
	struct list_head runnable_jobs;
	bool need_to_try_schedule_context = false;
	int i;

	INIT_LIST_HEAD(&completed_jobs);
	INIT_LIST_HEAD(&runnable_jobs);

	KBASE_DEBUG_ASSERT(katom->status != KBASE_JD_ATOM_STATE_UNUSED);

#if MALI_CUSTOMER_RELEASE == 0
	jd_check_force_failure(katom);
#endif

	/* This is needed in case an atom is failed due to being invalid, this
	 * can happen *before* the jobs that the atom depends on have completed */
	for (i = 0; i < 2; i++) {
		if (kbase_jd_katom_dep_atom(&katom->dep[i])) {
			list_del(&katom->dep_item[i]);
			kbase_jd_katom_dep_clear(&katom->dep[i]);
		}
	}

	/* With PRLAM-10817 or PRLAM-10959 the last tile of a fragment job being soft-stopped can fail with
	 * BASE_JD_EVENT_TILE_RANGE_FAULT.
	 *
	 * So here if the fragment job failed with TILE_RANGE_FAULT and it has been soft-stopped, then we promote the
	 * error code to BASE_JD_EVENT_DONE
	 */

	if ((kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_10817) || kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_10959)) &&
		  katom->event_code == BASE_JD_EVENT_TILE_RANGE_FAULT) {
		if ((katom->core_req & BASE_JD_REQ_FS) && (katom->atom_flags & KBASE_KATOM_FLAG_BEEN_SOFT_STOPPPED)) {
			/* Promote the failure to job done */
			katom->event_code = BASE_JD_EVENT_DONE;
			katom->atom_flags = katom->atom_flags & (~KBASE_KATOM_FLAG_BEEN_SOFT_STOPPPED);
		}
	}

	katom->status = KBASE_JD_ATOM_STATE_COMPLETED;
	list_add_tail(&katom->jd_item, &completed_jobs);

	while (!list_empty(&completed_jobs)) {
		katom = list_entry(completed_jobs.prev, struct kbase_jd_atom, jd_item);
		list_del(completed_jobs.prev);
		KBASE_DEBUG_ASSERT(katom->status == KBASE_JD_ATOM_STATE_COMPLETED);

		for (i = 0; i < 2; i++)
			jd_resolve_dep(&runnable_jobs, katom, i,
					kbase_ctx_flag(kctx, KCTX_DYING));

		if (katom->core_req & BASE_JD_REQ_EXTERNAL_RESOURCES)
			kbase_jd_post_external_resources(katom);

		while (!list_empty(&runnable_jobs)) {
			struct kbase_jd_atom *node;

			node = list_entry(runnable_jobs.next,
					struct kbase_jd_atom, jd_item);
			list_del(runnable_jobs.next);
			node->in_jd_list = false;

			KBASE_DEBUG_ASSERT(node->status != KBASE_JD_ATOM_STATE_UNUSED);

			if (node->status != KBASE_JD_ATOM_STATE_COMPLETED &&
					!kbase_ctx_flag(kctx, KCTX_DYING)) {
				need_to_try_schedule_context |= jd_run_atom(node);
			} else {
				node->event_code = katom->event_code;

				if ((node->core_req &
					BASE_JD_REQ_SOFT_JOB_TYPE) ==
					BASE_JD_REQ_SOFT_REPLAY) {
					if (kbase_replay_process(node))
						/* Don't complete this atom */
						continue;
				} else if (node->core_req &
							BASE_JD_REQ_SOFT_JOB) {
					/* If this is a fence wait soft job
					 * then remove it from the list of sync
					 * waiters.
					 */
					if (BASE_JD_REQ_SOFT_FENCE_WAIT == node->core_req)
						kbasep_remove_waiting_soft_job(node);

					kbase_finish_soft_job(node);
				}
				node->status = KBASE_JD_ATOM_STATE_COMPLETED;
			}

			if (node->status == KBASE_JD_ATOM_STATE_COMPLETED) {
				list_add_tail(&node->jd_item, &completed_jobs);
			} else if (node->status == KBASE_JD_ATOM_STATE_IN_JS &&
					!node->will_fail_event_code) {
				/* Node successfully submitted, try submitting
				 * dependencies as they may now be representable
				 * in JS */
				jd_try_submitting_deps(&runnable_jobs, node);
			}
		}

		/* Register a completed job as a disjoint event when the GPU
		 * is in a disjoint state (ie. being reset or replaying jobs).
		 */
		kbase_disjoint_event_potential(kctx->kbdev);
		if (completed_jobs_ctx)
			list_add_tail(&katom->jd_item, completed_jobs_ctx);
		else
			kbase_event_post(kctx, katom);

		/* Decrement and check the TOTAL number of jobs. This includes
		 * those not tracked by the scheduler: 'not ready to run' and
		 * 'dependency-only' jobs. */
		if (--kctx->jctx.job_nr == 0)
			wake_up(&kctx->jctx.zero_jobs_wait);	/* All events are safely queued now, and we can signal any waiter
								 * that we've got no more jobs (so we can be safely terminated) */
	}

	return need_to_try_schedule_context;
}

KBASE_EXPORT_TEST_API(jd_done_nolock);

#ifdef CONFIG_GPU_TRACEPOINTS
enum {
	CORE_REQ_DEP_ONLY,
	CORE_REQ_SOFT,
	CORE_REQ_COMPUTE,
	CORE_REQ_FRAGMENT,
	CORE_REQ_VERTEX,
	CORE_REQ_TILER,
	CORE_REQ_FRAGMENT_VERTEX,
	CORE_REQ_FRAGMENT_VERTEX_TILER,
	CORE_REQ_FRAGMENT_TILER,
	CORE_REQ_VERTEX_TILER,
	CORE_REQ_UNKNOWN
};
static const char * const core_req_strings[] = {
	"Dependency Only Job",
	"Soft Job",
	"Compute Shader Job",
	"Fragment Shader Job",
	"Vertex/Geometry Shader Job",
	"Tiler Job",
	"Fragment Shader + Vertex/Geometry Shader Job",
	"Fragment Shader + Vertex/Geometry Shader Job + Tiler Job",
	"Fragment Shader + Tiler Job",
	"Vertex/Geometry Shader Job + Tiler Job",
	"Unknown Job"
};
static const char *kbasep_map_core_reqs_to_string(base_jd_core_req core_req)
{
	if (core_req & BASE_JD_REQ_SOFT_JOB)
		return core_req_strings[CORE_REQ_SOFT];
	if (core_req & BASE_JD_REQ_ONLY_COMPUTE)
		return core_req_strings[CORE_REQ_COMPUTE];
	switch (core_req & (BASE_JD_REQ_FS | BASE_JD_REQ_CS | BASE_JD_REQ_T)) {
	case BASE_JD_REQ_DEP:
		return core_req_strings[CORE_REQ_DEP_ONLY];
	case BASE_JD_REQ_FS:
		return core_req_strings[CORE_REQ_FRAGMENT];
	case BASE_JD_REQ_CS:
		return core_req_strings[CORE_REQ_VERTEX];
	case BASE_JD_REQ_T:
		return core_req_strings[CORE_REQ_TILER];
	case (BASE_JD_REQ_FS | BASE_JD_REQ_CS):
		return core_req_strings[CORE_REQ_FRAGMENT_VERTEX];
	case (BASE_JD_REQ_FS | BASE_JD_REQ_T):
		return core_req_strings[CORE_REQ_FRAGMENT_TILER];
	case (BASE_JD_REQ_CS | BASE_JD_REQ_T):
		return core_req_strings[CORE_REQ_VERTEX_TILER];
	case (BASE_JD_REQ_FS | BASE_JD_REQ_CS | BASE_JD_REQ_T):
		return core_req_strings[CORE_REQ_FRAGMENT_VERTEX_TILER];
	}
	return core_req_strings[CORE_REQ_UNKNOWN];
}
#endif

bool jd_submit_atom(struct kbase_context *kctx, const struct base_jd_atom_v2 *user_atom, struct kbase_jd_atom *katom)
{
	struct kbase_jd_context *jctx = &kctx->jctx;
	int queued = 0;
	int i;
	int sched_prio;
	bool ret;
	bool will_fail = false;

	/* Update the TOTAL number of jobs. This includes those not tracked by
	 * the scheduler: 'not ready to run' and 'dependency-only' jobs. */
	jctx->job_nr++;

#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)
	katom->start_timestamp.tv64 = 0;
#else
	katom->start_timestamp = 0;
#endif
	katom->udata = user_atom->udata;
	katom->kctx = kctx;
	katom->nr_extres = user_atom->nr_extres;
	katom->extres = NULL;
	katom->device_nr = user_atom->device_nr;
	katom->affinity = 0;
	katom->jc = user_atom->jc;
	katom->coreref_state = KBASE_ATOM_COREREF_STATE_NO_CORES_REQUESTED;
	katom->core_req = user_atom->core_req;
	katom->atom_flags = 0;
	katom->retry_count = 0;
	katom->need_cache_flush_cores_retained = 0;
	katom->pre_dep = NULL;
	katom->post_dep = NULL;
	katom->x_pre_dep = NULL;
	katom->x_post_dep = NULL;
	katom->will_fail_event_code = BASE_JD_EVENT_NOT_STARTED;

	/* Implicitly sets katom->protected_state.enter as well. */
	katom->protected_state.exit = KBASE_ATOM_EXIT_PROTECTED_CHECK;

	katom->age = kctx->age_count++;

	INIT_LIST_HEAD(&katom->jd_item);
#ifdef CONFIG_KDS
	/* Start by assuming that the KDS dependencies are satisfied,
	 * kbase_jd_pre_external_resources will correct this if there are dependencies */
	katom->kds_dep_satisfied = true;
	katom->kds_rset = NULL;
#endif				/* CONFIG_KDS */
#ifdef CONFIG_MALI_DMA_FENCE
	kbase_fence_dep_count_set(katom, -1);
#endif

	/* Don't do anything if there is a mess up with dependencies.
	   This is done in a separate cycle to check both the dependencies at ones, otherwise
	   it will be extra complexity to deal with 1st dependency ( just added to the list )
	   if only the 2nd one has invalid config.
	 */
	for (i = 0; i < 2; i++) {
		int dep_atom_number = user_atom->pre_dep[i].atom_id;
		base_jd_dep_type dep_atom_type = user_atom->pre_dep[i].dependency_type;

		if (dep_atom_number) {
			if (dep_atom_type != BASE_JD_DEP_TYPE_ORDER &&
					dep_atom_type != BASE_JD_DEP_TYPE_DATA) {
				katom->event_code = BASE_JD_EVENT_JOB_CONFIG_FAULT;
				katom->status = KBASE_JD_ATOM_STATE_COMPLETED;

				/* Wrong dependency setup. Atom will be sent
				 * back to user space. Do not record any
				 * dependencies. */
				KBASE_TLSTREAM_TL_NEW_ATOM(
						katom,
						kbase_jd_atom_id(kctx, katom));
				KBASE_TLSTREAM_TL_RET_ATOM_CTX(
						katom, kctx);
				KBASE_TLSTREAM_TL_ATTRIB_ATOM_STATE(katom,
						TL_ATOM_STATE_IDLE);

				ret = jd_done_nolock(katom, NULL);
				goto out;
			}
		}
	}

	/* Add dependencies */
	for (i = 0; i < 2; i++) {
		int dep_atom_number = user_atom->pre_dep[i].atom_id;
		base_jd_dep_type dep_atom_type;
		struct kbase_jd_atom *dep_atom = &jctx->atoms[dep_atom_number];

		dep_atom_type = user_atom->pre_dep[i].dependency_type;
		kbase_jd_katom_dep_clear(&katom->dep[i]);

		if (!dep_atom_number)
			continue;

		if (dep_atom->status == KBASE_JD_ATOM_STATE_UNUSED ||
				dep_atom->status == KBASE_JD_ATOM_STATE_COMPLETED) {

			if (dep_atom->event_code == BASE_JD_EVENT_DONE)
				continue;
			/* don't stop this atom if it has an order dependency
			 * only to the failed one, try to submit it through
			 * the normal path
			 */
			if (dep_atom_type == BASE_JD_DEP_TYPE_ORDER &&
					dep_atom->event_code > BASE_JD_EVENT_ACTIVE) {
				continue;
			}

			/* Atom has completed, propagate the error code if any */
			katom->event_code = dep_atom->event_code;
			katom->status = KBASE_JD_ATOM_STATE_QUEUED;

			/* This atom is going through soft replay or
			 * will be sent back to user space. Do not record any
			 * dependencies. */
			KBASE_TLSTREAM_TL_NEW_ATOM(
					katom,
					kbase_jd_atom_id(kctx, katom));
			KBASE_TLSTREAM_TL_RET_ATOM_CTX(katom, kctx);
			KBASE_TLSTREAM_TL_ATTRIB_ATOM_STATE(katom,
					TL_ATOM_STATE_IDLE);

			if ((katom->core_req & BASE_JD_REQ_SOFT_JOB_TYPE)
					 == BASE_JD_REQ_SOFT_REPLAY) {
				if (kbase_replay_process(katom)) {
					ret = false;
					goto out;
				}
			}
			will_fail = true;

		} else {
			/* Atom is in progress, add this atom to the list */
			list_add_tail(&katom->dep_item[i], &dep_atom->dep_head[i]);
			kbase_jd_katom_dep_set(&katom->dep[i], dep_atom, dep_atom_type);
			queued = 1;
		}
	}

	if (will_fail) {
		if (!queued) {
			ret = jd_done_nolock(katom, NULL);

			goto out;
		} else {
			katom->will_fail_event_code = katom->event_code;
			ret = false;

			goto out;
		}
	} else {
		/* These must occur after the above loop to ensure that an atom
		 * that depends on a previous atom with the same number behaves
		 * as expected */
		katom->event_code = BASE_JD_EVENT_DONE;
		katom->status = KBASE_JD_ATOM_STATE_QUEUED;
	}

	/* For invalid priority, be most lenient and choose the default */
	sched_prio = kbasep_js_atom_prio_to_sched_prio(user_atom->prio);
	if (sched_prio == KBASE_JS_ATOM_SCHED_PRIO_INVALID)
		sched_prio = KBASE_JS_ATOM_SCHED_PRIO_DEFAULT;
	katom->sched_priority = sched_prio;

	/* Create a new atom recording all dependencies it was set up with. */
	KBASE_TLSTREAM_TL_NEW_ATOM(
			katom,
			kbase_jd_atom_id(kctx, katom));
	KBASE_TLSTREAM_TL_ATTRIB_ATOM_STATE(katom, TL_ATOM_STATE_IDLE);
	KBASE_TLSTREAM_TL_ATTRIB_ATOM_PRIORITY(katom, katom->sched_priority);
	KBASE_TLSTREAM_TL_RET_ATOM_CTX(katom, kctx);
	for (i = 0; i < 2; i++)
		if (BASE_JD_DEP_TYPE_INVALID != kbase_jd_katom_dep_type(
					&katom->dep[i])) {
			KBASE_TLSTREAM_TL_DEP_ATOM_ATOM(
					(void *)kbase_jd_katom_dep_atom(
						&katom->dep[i]),
					(void *)katom);
		} else if (BASE_JD_DEP_TYPE_INVALID !=
				user_atom->pre_dep[i].dependency_type) {
			/* Resolved dependency. */
			int dep_atom_number =
				user_atom->pre_dep[i].atom_id;
			struct kbase_jd_atom *dep_atom =
				&jctx->atoms[dep_atom_number];

			KBASE_TLSTREAM_TL_RDEP_ATOM_ATOM(
					(void *)dep_atom,
					(void *)katom);
		}

	/* Reject atoms with job chain = NULL, as these cause issues with soft-stop */
	if (!katom->jc && (katom->core_req & BASE_JD_REQ_ATOM_TYPE) != BASE_JD_REQ_DEP) {
		dev_warn(kctx->kbdev->dev, "Rejecting atom with jc = NULL");
		katom->event_code = BASE_JD_EVENT_JOB_INVALID;
		ret = jd_done_nolock(katom, NULL);
		goto out;
	}

	/* Reject atoms with an invalid device_nr */
	if ((katom->core_req & BASE_JD_REQ_SPECIFIC_COHERENT_GROUP) &&
	    (katom->device_nr >= kctx->kbdev->gpu_props.num_core_groups)) {
		dev_warn(kctx->kbdev->dev,
				"Rejecting atom with invalid device_nr %d",
				katom->device_nr);
		katom->event_code = BASE_JD_EVENT_JOB_INVALID;
		ret = jd_done_nolock(katom, NULL);
		goto out;
	}

	/* Reject atoms with invalid core requirements */
	if ((katom->core_req & BASE_JD_REQ_EXTERNAL_RESOURCES) &&
			(katom->core_req & BASE_JD_REQ_EVENT_COALESCE)) {
		dev_warn(kctx->kbdev->dev,
				"Rejecting atom with invalid core requirements");
		katom->event_code = BASE_JD_EVENT_JOB_INVALID;
		katom->core_req &= ~BASE_JD_REQ_EVENT_COALESCE;
		ret = jd_done_nolock(katom, NULL);
		goto out;
	}

	if (katom->core_req & BASE_JD_REQ_EXTERNAL_RESOURCES) {
		/* handle what we need to do to access the external resources */
		if (kbase_jd_pre_external_resources(katom, user_atom) != 0) {
			/* setup failed (no access, bad resource, unknown resource types, etc.) */
			katom->event_code = BASE_JD_EVENT_JOB_INVALID;
			ret = jd_done_nolock(katom, NULL);
			goto out;
		}
	}

	/* Validate the atom. Function will return error if the atom is
	 * malformed.
	 *
	 * Soft-jobs never enter the job scheduler but have their own initialize method.
	 *
	 * If either fail then we immediately complete the atom with an error.
	 */
	if ((katom->core_req & BASE_JD_REQ_SOFT_JOB) == 0) {
		if (!kbase_js_is_atom_valid(kctx->kbdev, katom)) {
			katom->event_code = BASE_JD_EVENT_JOB_INVALID;
			ret = jd_done_nolock(katom, NULL);
			goto out;
		}
	} else {
		/* Soft-job */
		if (kbase_prepare_soft_job(katom) != 0) {
			katom->event_code = BASE_JD_EVENT_JOB_INVALID;
			ret = jd_done_nolock(katom, NULL);
			goto out;
		}
	}

#ifdef CONFIG_GPU_TRACEPOINTS
	katom->work_id = atomic_inc_return(&jctx->work_id);
	trace_gpu_job_enqueue((u32)kctx->id, katom->work_id,
			kbasep_map_core_reqs_to_string(katom->core_req));
#endif

	if (queued && !IS_GPU_ATOM(katom)) {
		ret = false;
		goto out;
	}
#ifdef CONFIG_KDS
	if (!katom->kds_dep_satisfied) {
		/* Queue atom due to KDS dependency */
		ret = false;
		goto out;
	}
#endif				/* CONFIG_KDS */


#ifdef CONFIG_MALI_DMA_FENCE
	if (kbase_fence_dep_count_read(katom) != -1) {
		ret = false;
		goto out;
	}
#endif /* CONFIG_MALI_DMA_FENCE */

	if ((katom->core_req & BASE_JD_REQ_SOFT_JOB_TYPE)
						  == BASE_JD_REQ_SOFT_REPLAY) {
		if (kbase_replay_process(katom))
			ret = false;
		else
			ret = jd_done_nolock(katom, NULL);

		goto out;
	} else if (katom->core_req & BASE_JD_REQ_SOFT_JOB) {
		if (kbase_process_soft_job(katom) == 0) {
			kbase_finish_soft_job(katom);
			ret = jd_done_nolock(katom, NULL);
			goto out;
		}

		ret = false;
	} else if ((katom->core_req & BASE_JD_REQ_ATOM_TYPE) != BASE_JD_REQ_DEP) {
		katom->status = KBASE_JD_ATOM_STATE_IN_JS;
		ret = kbasep_js_add_job(kctx, katom);
		/* If job was cancelled then resolve immediately */
		if (katom->event_code == BASE_JD_EVENT_JOB_CANCELLED)
			ret = jd_done_nolock(katom, NULL);
	} else {
		/* This is a pure dependency. Resolve it immediately */
		ret = jd_done_nolock(katom, NULL);
	}

 out:
	return ret;
}

int kbase_jd_submit(struct kbase_context *kctx,
		void __user *user_addr, u32 nr_atoms, u32 stride,
		bool uk6_atom)
{
	struct kbase_jd_context *jctx = &kctx->jctx;
	int err = 0;
	int i;
	bool need_to_try_schedule_context = false;
	struct kbase_device *kbdev;
	u32 latest_flush;

	/*
	 * kbase_jd_submit isn't expected to fail and so all errors with the
	 * jobs are reported by immediately failing them (through event system)
	 */
	kbdev = kctx->kbdev;

	beenthere(kctx, "%s", "Enter");

	if (kbase_ctx_flag(kctx, KCTX_SUBMIT_DISABLED)) {
		dev_err(kbdev->dev, "Attempt to submit to a context that has SUBMIT_DISABLED set on it");
		return -EINVAL;
	}

	if (stride != sizeof(base_jd_atom_v2)) {
		dev_err(kbdev->dev, "Stride passed to job_submit doesn't match kernel");
		return -EINVAL;
	}

	KBASE_TIMELINE_ATOMS_IN_FLIGHT(kctx, atomic_add_return(nr_atoms,
				&kctx->timeline.jd_atoms_in_flight));

	/* All atoms submitted in this call have the same flush ID */
	latest_flush = kbase_backend_get_current_flush_id(kbdev);

	for (i = 0; i < nr_atoms; i++) {
		struct base_jd_atom_v2 user_atom;
		struct kbase_jd_atom *katom;

#ifdef BASE_LEGACY_UK6_SUPPORT
		BUILD_BUG_ON(sizeof(struct base_jd_atom_v2_uk6) !=
				sizeof(base_jd_atom_v2));

		if (uk6_atom) {
			struct base_jd_atom_v2_uk6 user_atom_v6;
			base_jd_dep_type dep_types[2] = {BASE_JD_DEP_TYPE_DATA, BASE_JD_DEP_TYPE_DATA};

			if (copy_from_user(&user_atom_v6, user_addr,
					sizeof(user_atom_v6))) {
				err = -EINVAL;
				KBASE_TIMELINE_ATOMS_IN_FLIGHT(kctx,
					atomic_sub_return(
					nr_atoms - i,
					&kctx->timeline.jd_atoms_in_flight));
				break;
			}
			/* Convert from UK6 atom format to UK7 format */
			user_atom.jc = user_atom_v6.jc;
			user_atom.udata = user_atom_v6.udata;
			user_atom.extres_list = user_atom_v6.extres_list;
			user_atom.nr_extres = user_atom_v6.nr_extres;
			user_atom.core_req = (u32)(user_atom_v6.core_req & 0x7fff);

			/* atom number 0 is used for no dependency atoms */
			if (!user_atom_v6.pre_dep[0])
				dep_types[0] = BASE_JD_DEP_TYPE_INVALID;

			base_jd_atom_dep_set(&user_atom.pre_dep[0],
					user_atom_v6.pre_dep[0],
					dep_types[0]);

			/* atom number 0 is used for no dependency atoms */
			if (!user_atom_v6.pre_dep[1])
				dep_types[1] = BASE_JD_DEP_TYPE_INVALID;

			base_jd_atom_dep_set(&user_atom.pre_dep[1],
					user_atom_v6.pre_dep[1],
					dep_types[1]);

			user_atom.atom_number = user_atom_v6.atom_number;
			user_atom.prio = user_atom_v6.prio;
			user_atom.device_nr = user_atom_v6.device_nr;
		} else {
#endif /* BASE_LEGACY_UK6_SUPPORT */
			if (copy_from_user(&user_atom, user_addr,
						sizeof(user_atom)) != 0) {
				err = -EINVAL;
				KBASE_TIMELINE_ATOMS_IN_FLIGHT(kctx,
					atomic_sub_return(nr_atoms - i,
					&kctx->timeline.jd_atoms_in_flight));
				break;
			}
#ifdef BASE_LEGACY_UK6_SUPPORT
		}
#endif

#ifdef BASE_LEGACY_UK10_2_SUPPORT
		if (KBASE_API_VERSION(10, 3) > kctx->api_version)
			user_atom.core_req = (u32)(user_atom.compat_core_req
					      & 0x7fff);
#endif /* BASE_LEGACY_UK10_2_SUPPORT */

		user_addr = (void __user *)((uintptr_t) user_addr + stride);

		mutex_lock(&jctx->lock);
#ifndef compiletime_assert
#define compiletime_assert_defined
#define compiletime_assert(x, msg) do { switch (0) { case 0: case (x):; } } \
while (false)
#endif
		compiletime_assert((1 << (8*sizeof(user_atom.atom_number))) >=
					BASE_JD_ATOM_COUNT,
			"BASE_JD_ATOM_COUNT and base_atom_id type out of sync");
		compiletime_assert(sizeof(user_atom.pre_dep[0].atom_id) ==
					sizeof(user_atom.atom_number),
			"BASE_JD_ATOM_COUNT and base_atom_id type out of sync");
#ifdef compiletime_assert_defined
#undef compiletime_assert
#undef compiletime_assert_defined
#endif
		if (user_atom.atom_number >= BASE_JD_ATOM_COUNT) {
			err = -EINVAL;
			break;
		}
		user_atom.atom_number =
			array_index_nospec(user_atom.atom_number,
					   BASE_JD_ATOM_COUNT);
		katom = &jctx->atoms[user_atom.atom_number];

		/* Record the flush ID for the cache flush optimisation */
		katom->flush_id = latest_flush;

		while (katom->status != KBASE_JD_ATOM_STATE_UNUSED) {
			/* Atom number is already in use, wait for the atom to
			 * complete
			 */
			mutex_unlock(&jctx->lock);

			/* This thread will wait for the atom to complete. Due
			 * to thread scheduling we are not sure that the other
			 * thread that owns the atom will also schedule the
			 * context, so we force the scheduler to be active and
			 * hence eventually schedule this context at some point
			 * later.
			 */
			kbase_js_sched_all(kbdev);

			if (wait_event_killable(katom->completed,
					katom->status ==
					KBASE_JD_ATOM_STATE_UNUSED) != 0) {
				/* We're being killed so the result code
				 * doesn't really matter
				 */
				return 0;
			}
			mutex_lock(&jctx->lock);
		}

		need_to_try_schedule_context |=
				       jd_submit_atom(kctx, &user_atom, katom);

		/* Register a completed job as a disjoint event when the GPU is in a disjoint state
		 * (ie. being reset or replaying jobs).
		 */
		kbase_disjoint_event_potential(kbdev);

		mutex_unlock(&jctx->lock);
	}

	if (need_to_try_schedule_context)
		kbase_js_sched_all(kbdev);

	return err;
}

KBASE_EXPORT_TEST_API(kbase_jd_submit);

void kbase_jd_done_worker(struct work_struct *data)
{
	struct kbase_jd_atom *katom = container_of(data, struct kbase_jd_atom, work);
	struct kbase_jd_context *jctx;
	struct kbase_context *kctx;
	struct kbasep_js_kctx_info *js_kctx_info;
	struct kbase_device *kbdev;
	struct kbasep_js_device_data *js_devdata;
	u64 cache_jc = katom->jc;
	struct kbasep_js_atom_retained_state katom_retained_state;
	bool context_idle;
	base_jd_core_req core_req = katom->core_req;
	u64 affinity = katom->affinity;
	enum kbase_atom_coreref_state coreref_state = katom->coreref_state;

	/* Soft jobs should never reach this function */
	KBASE_DEBUG_ASSERT((katom->core_req & BASE_JD_REQ_SOFT_JOB) == 0);

	kctx = katom->kctx;
	jctx = &kctx->jctx;
	kbdev = kctx->kbdev;
	js_kctx_info = &kctx->jctx.sched_info;
	js_devdata = &kbdev->js_data;

	KBASE_TRACE_ADD(kbdev, JD_DONE_WORKER, kctx, katom, katom->jc, 0);

	kbase_backend_complete_wq(kbdev, katom);

	/*
	 * Begin transaction on JD context and JS context
	 */
	mutex_lock(&jctx->lock);
	KBASE_TLSTREAM_TL_ATTRIB_ATOM_STATE(katom, TL_ATOM_STATE_DONE);
	mutex_lock(&js_devdata->queue_mutex);
	mutex_lock(&js_kctx_info->ctx.jsctx_mutex);

	/* This worker only gets called on contexts that are scheduled *in*. This is
	 * because it only happens in response to an IRQ from a job that was
	 * running.
	 */
	KBASE_DEBUG_ASSERT(kbase_ctx_flag(kctx, KCTX_SCHEDULED));

	if (katom->event_code == BASE_JD_EVENT_STOPPED) {
		/* Atom has been promoted to stopped */
		unsigned long flags;

		mutex_unlock(&js_kctx_info->ctx.jsctx_mutex);
		mutex_unlock(&js_devdata->queue_mutex);

		spin_lock_irqsave(&kbdev->hwaccess_lock, flags);

		katom->status = KBASE_JD_ATOM_STATE_IN_JS;
		kbase_js_unpull(kctx, katom);

		spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
		mutex_unlock(&jctx->lock);

		return;
	}

	if (katom->event_code != BASE_JD_EVENT_DONE)
		dev_err(kbdev->dev,
			"t6xx: GPU fault 0x%02lx from job slot %d\n",
					(unsigned long)katom->event_code,
								katom->slot_nr);

	if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8316))
		kbase_as_poking_timer_release_atom(kbdev, kctx, katom);

	/* Retain state before the katom disappears */
	kbasep_js_atom_retained_state_copy(&katom_retained_state, katom);

	context_idle = kbase_js_complete_atom_wq(kctx, katom);

	KBASE_DEBUG_ASSERT(kbasep_js_has_atom_finished(&katom_retained_state));

	kbasep_js_remove_job(kbdev, kctx, katom);
	mutex_unlock(&js_kctx_info->ctx.jsctx_mutex);
	mutex_unlock(&js_devdata->queue_mutex);
	katom->atom_flags &= ~KBASE_KATOM_FLAG_HOLDING_CTX_REF;
	/* jd_done_nolock() requires the jsctx_mutex lock to be dropped */
	jd_done_nolock(katom, &kctx->completed_jobs);

	/* katom may have been freed now, do not use! */

	if (context_idle) {
		unsigned long flags;

		context_idle = false;
		mutex_lock(&js_devdata->queue_mutex);
		spin_lock_irqsave(&kbdev->hwaccess_lock, flags);

		/* If kbase_sched() has scheduled this context back in then
		 * KCTX_ACTIVE will have been set after we marked it as
		 * inactive, and another pm reference will have been taken, so
		 * drop our reference. But do not call kbase_jm_idle_ctx(), as
		 * the context is active and fast-starting is allowed.
		 *
		 * If an atom has been fast-started then kctx->atoms_pulled will
		 * be non-zero but KCTX_ACTIVE will still be false (as the
		 * previous pm reference has been inherited). Do NOT drop our
		 * reference, as it has been re-used, and leave the context as
		 * active.
		 *
		 * If no new atoms have been started then KCTX_ACTIVE will still
		 * be false and atoms_pulled will be zero, so drop the reference
		 * and call kbase_jm_idle_ctx().
		 *
		 * As the checks are done under both the queue_mutex and
		 * hwaccess_lock is should be impossible for this to race
		 * with the scheduler code.
		 */
		if (kbase_ctx_flag(kctx, KCTX_ACTIVE) ||
		    !atomic_read(&kctx->atoms_pulled)) {
			/* Calling kbase_jm_idle_ctx() here will ensure that
			 * atoms are not fast-started when we drop the
			 * hwaccess_lock. This is not performed if
			 * KCTX_ACTIVE is set as in that case another pm
			 * reference has been taken and a fast-start would be
			 * valid.
			 */
			if (!kbase_ctx_flag(kctx, KCTX_ACTIVE))
				kbase_jm_idle_ctx(kbdev, kctx);
			context_idle = true;
		} else {
			kbase_ctx_flag_set(kctx, KCTX_ACTIVE);
		}
		spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
		mutex_unlock(&js_devdata->queue_mutex);
	}

	/*
	 * Transaction complete
	 */
	mutex_unlock(&jctx->lock);

	/* Job is now no longer running, so can now safely release the context
	 * reference, and handle any actions that were logged against the atom's retained state */

	kbasep_js_runpool_release_ctx_and_katom_retained_state(kbdev, kctx, &katom_retained_state);

	kbase_js_sched_all(kbdev);

	if (!atomic_dec_return(&kctx->work_count)) {
		/* If worker now idle then post all events that jd_done_nolock()
		 * has queued */
		mutex_lock(&jctx->lock);
		while (!list_empty(&kctx->completed_jobs)) {
			struct kbase_jd_atom *atom = list_entry(
					kctx->completed_jobs.next,
					struct kbase_jd_atom, jd_item);
			list_del(kctx->completed_jobs.next);

			kbase_event_post(kctx, atom);
		}
		mutex_unlock(&jctx->lock);
	}

	kbase_backend_complete_wq_post_sched(kbdev, core_req, affinity,
			coreref_state);

	if (context_idle)
		kbase_pm_context_idle(kbdev);

	KBASE_TRACE_ADD(kbdev, JD_DONE_WORKER_END, kctx, NULL, cache_jc, 0);
}

/**
 * jd_cancel_worker - Work queue job cancel function.
 * @data: a &struct work_struct
 *
 * Only called as part of 'Zapping' a context (which occurs on termination).
 * Operates serially with the kbase_jd_done_worker() on the work queue.
 *
 * This can only be called on contexts that aren't scheduled.
 *
 * We don't need to release most of the resources that would occur on
 * kbase_jd_done() or kbase_jd_done_worker(), because the atoms here must not be
 * running (by virtue of only being called on contexts that aren't
 * scheduled).
 */
static void jd_cancel_worker(struct work_struct *data)
{
	struct kbase_jd_atom *katom = container_of(data, struct kbase_jd_atom, work);
	struct kbase_jd_context *jctx;
	struct kbase_context *kctx;
	struct kbasep_js_kctx_info *js_kctx_info;
	bool need_to_try_schedule_context;
	bool attr_state_changed;
	struct kbase_device *kbdev;

	/* Soft jobs should never reach this function */
	KBASE_DEBUG_ASSERT((katom->core_req & BASE_JD_REQ_SOFT_JOB) == 0);

	kctx = katom->kctx;
	kbdev = kctx->kbdev;
	jctx = &kctx->jctx;
	js_kctx_info = &kctx->jctx.sched_info;

	KBASE_TRACE_ADD(kbdev, JD_CANCEL_WORKER, kctx, katom, katom->jc, 0);

	/* This only gets called on contexts that are scheduled out. Hence, we must
	 * make sure we don't de-ref the number of running jobs (there aren't
	 * any), nor must we try to schedule out the context (it's already
	 * scheduled out).
	 */
	KBASE_DEBUG_ASSERT(!kbase_ctx_flag(kctx, KCTX_SCHEDULED));

	/* Scheduler: Remove the job from the system */
	mutex_lock(&js_kctx_info->ctx.jsctx_mutex);
	attr_state_changed = kbasep_js_remove_cancelled_job(kbdev, kctx, katom);
	mutex_unlock(&js_kctx_info->ctx.jsctx_mutex);

	mutex_lock(&jctx->lock);

	need_to_try_schedule_context = jd_done_nolock(katom, NULL);
	/* Because we're zapping, we're not adding any more jobs to this ctx, so no need to
	 * schedule the context. There's also no need for the jsctx_mutex to have been taken
	 * around this too. */
	KBASE_DEBUG_ASSERT(!need_to_try_schedule_context);

	/* katom may have been freed now, do not use! */
	mutex_unlock(&jctx->lock);

	if (attr_state_changed)
		kbase_js_sched_all(kbdev);
}

/**
 * kbase_jd_done - Complete a job that has been removed from the Hardware
 * @katom: atom which has been completed
 * @slot_nr: slot the atom was on
 * @end_timestamp: completion time
 * @done_code: completion code
 *
 * This must be used whenever a job has been removed from the Hardware, e.g.:
 * An IRQ indicates that the job finished (for both error and 'done' codes), or
 * the job was evicted from the JS_HEAD_NEXT registers during a Soft/Hard stop.
 *
 * Some work is carried out immediately, and the rest is deferred onto a
 * workqueue
 *
 * Context:
 *   This can be called safely from atomic context.
 *   The caller must hold kbdev->hwaccess_lock
 */
void kbase_jd_done(struct kbase_jd_atom *katom, int slot_nr,
		ktime_t *end_timestamp, kbasep_js_atom_done_code done_code)
{
	struct kbase_context *kctx;
	struct kbase_device *kbdev;

	KBASE_DEBUG_ASSERT(katom);
	kctx = katom->kctx;
	KBASE_DEBUG_ASSERT(kctx);
	kbdev = kctx->kbdev;
	KBASE_DEBUG_ASSERT(kbdev);

	if (done_code & KBASE_JS_ATOM_DONE_EVICTED_FROM_NEXT)
		katom->event_code = BASE_JD_EVENT_REMOVED_FROM_NEXT;

	KBASE_TRACE_ADD(kbdev, JD_DONE, kctx, katom, katom->jc, 0);

	kbase_job_check_leave_disjoint(kbdev, katom);

	katom->slot_nr = slot_nr;

	atomic_inc(&kctx->work_count);

#ifdef CONFIG_DEBUG_FS
	/* a failed job happened and is waiting for dumping*/
	if (!katom->will_fail_event_code &&
			kbase_debug_job_fault_process(katom, katom->event_code))
		return;
#endif

	WARN_ON(work_pending(&katom->work));
	KBASE_DEBUG_ASSERT(0 == object_is_on_stack(&katom->work));
	INIT_WORK(&katom->work, kbase_jd_done_worker);
	queue_work(kctx->jctx.job_done_wq, &katom->work);
}

KBASE_EXPORT_TEST_API(kbase_jd_done);

void kbase_jd_cancel(struct kbase_device *kbdev, struct kbase_jd_atom *katom)
{
	struct kbase_context *kctx;

	KBASE_DEBUG_ASSERT(NULL != kbdev);
	KBASE_DEBUG_ASSERT(NULL != katom);
	kctx = katom->kctx;
	KBASE_DEBUG_ASSERT(NULL != kctx);

	KBASE_TRACE_ADD(kbdev, JD_CANCEL, kctx, katom, katom->jc, 0);

	/* This should only be done from a context that is not scheduled */
	KBASE_DEBUG_ASSERT(!kbase_ctx_flag(kctx, KCTX_SCHEDULED));

	WARN_ON(work_pending(&katom->work));

	katom->event_code = BASE_JD_EVENT_JOB_CANCELLED;

	KBASE_DEBUG_ASSERT(0 == object_is_on_stack(&katom->work));
	INIT_WORK(&katom->work, jd_cancel_worker);
	queue_work(kctx->jctx.job_done_wq, &katom->work);
}


void kbase_jd_zap_context(struct kbase_context *kctx)
{
	struct kbase_jd_atom *katom;
	struct list_head *entry, *tmp;
	struct kbase_device *kbdev;

	KBASE_DEBUG_ASSERT(kctx);

	kbdev = kctx->kbdev;

	KBASE_TRACE_ADD(kbdev, JD_ZAP_CONTEXT, kctx, NULL, 0u, 0u);

	kbase_js_zap_context(kctx);

	mutex_lock(&kctx->jctx.lock);

	/*
	 * While holding the struct kbase_jd_context lock clean up jobs which are known to kbase but are
	 * queued outside the job scheduler.
	 */

	del_timer_sync(&kctx->soft_job_timeout);
	list_for_each_safe(entry, tmp, &kctx->waiting_soft_jobs) {
		katom = list_entry(entry, struct kbase_jd_atom, queue);
		kbase_cancel_soft_job(katom);
	}


#ifdef CONFIG_KDS

	/* For each job waiting on a kds resource, cancel the wait and force the job to
	 * complete early, this is done so that we don't leave jobs outstanding waiting
	 * on kds resources which may never be released when contexts are zapped, resulting
	 * in a hang.
	 *
	 * Note that we can safely iterate over the list as the struct kbase_jd_context lock is held,
	 * this prevents items being removed when calling job_done_nolock in kbase_cancel_kds_wait_job.
	 */

	list_for_each(entry, &kctx->waiting_kds_resource) {
		katom = list_entry(entry, struct kbase_jd_atom, node);

		kbase_cancel_kds_wait_job(katom);
	}
#endif

#ifdef CONFIG_MALI_DMA_FENCE
	kbase_dma_fence_cancel_all_atoms(kctx);
#endif

	mutex_unlock(&kctx->jctx.lock);

#ifdef CONFIG_MALI_DMA_FENCE
	/* Flush dma-fence workqueue to ensure that any callbacks that may have
	 * been queued are done before continuing.
	 */
	flush_workqueue(kctx->dma_fence.wq);
#endif

	kbase_jm_wait_for_zero_jobs(kctx);
}

KBASE_EXPORT_TEST_API(kbase_jd_zap_context);

int kbase_jd_init(struct kbase_context *kctx)
{
	int i;
	int mali_err = 0;
#ifdef CONFIG_KDS
	int err;
#endif				/* CONFIG_KDS */

	KBASE_DEBUG_ASSERT(kctx);

	kctx->jctx.job_done_wq = alloc_workqueue("mali_jd",
			WQ_HIGHPRI | WQ_UNBOUND, 1);
	if (NULL == kctx->jctx.job_done_wq) {
		mali_err = -ENOMEM;
		goto out1;
	}

	for (i = 0; i < BASE_JD_ATOM_COUNT; i++) {
		init_waitqueue_head(&kctx->jctx.atoms[i].completed);

		INIT_LIST_HEAD(&kctx->jctx.atoms[i].dep_head[0]);
		INIT_LIST_HEAD(&kctx->jctx.atoms[i].dep_head[1]);

		/* Catch userspace attempting to use an atom which doesn't exist as a pre-dependency */
		kctx->jctx.atoms[i].event_code = BASE_JD_EVENT_JOB_INVALID;
		kctx->jctx.atoms[i].status = KBASE_JD_ATOM_STATE_UNUSED;

#if defined(CONFIG_MALI_DMA_FENCE) || defined(CONFIG_SYNC_FILE)
		kctx->jctx.atoms[i].dma_fence.context =
						dma_fence_context_alloc(1);
		atomic_set(&kctx->jctx.atoms[i].dma_fence.seqno, 0);
		INIT_LIST_HEAD(&kctx->jctx.atoms[i].dma_fence.callbacks);
#endif
	}

	mutex_init(&kctx->jctx.lock);

	init_waitqueue_head(&kctx->jctx.zero_jobs_wait);

	spin_lock_init(&kctx->jctx.tb_lock);

#ifdef CONFIG_KDS
	err = kds_callback_init(&kctx->jctx.kds_cb, 0, kds_dep_clear);
	if (0 != err) {
		mali_err = -EINVAL;
		goto out2;
	}
#endif				/* CONFIG_KDS */

	kctx->jctx.job_nr = 0;
	INIT_LIST_HEAD(&kctx->completed_jobs);
	atomic_set(&kctx->work_count, 0);

	return 0;

#ifdef CONFIG_KDS
 out2:
	destroy_workqueue(kctx->jctx.job_done_wq);
#endif				/* CONFIG_KDS */
 out1:
	return mali_err;
}

KBASE_EXPORT_TEST_API(kbase_jd_init);

void kbase_jd_exit(struct kbase_context *kctx)
{
	KBASE_DEBUG_ASSERT(kctx);

#ifdef CONFIG_KDS
	kds_callback_term(&kctx->jctx.kds_cb);
#endif				/* CONFIG_KDS */
	/* Work queue is emptied by this */
	destroy_workqueue(kctx->jctx.job_done_wq);
}

KBASE_EXPORT_TEST_API(kbase_jd_exit);