arm/bifrost/mali_kbase_debug_job_fault.c

// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
/*
 *
 * (C) COPYRIGHT 2012-2016, 2018-2022 ARM Limited. All rights reserved.
 *
 * This program is free software and is provided to you under the terms of the
 * GNU General Public License version 2 as published by the Free Software
 * Foundation, and any use by you of this program is subject to the terms
 * of such GNU license.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, you can access it online at
 * http://www.gnu.org/licenses/gpl-2.0.html.
 *
 */

#include <mali_kbase.h>
#include <linux/spinlock.h>
#include <mali_kbase_hwaccess_jm.h>

#if IS_ENABLED(CONFIG_DEBUG_FS)

static bool kbase_is_job_fault_event_pending(struct kbase_device *kbdev)
{
	struct list_head *event_list = &kbdev->job_fault_event_list;
	unsigned long    flags;
	bool             ret;

	spin_lock_irqsave(&kbdev->job_fault_event_lock, flags);
	ret = !list_empty(event_list);
	spin_unlock_irqrestore(&kbdev->job_fault_event_lock, flags);

	return ret;
}

static void kbase_ctx_remove_pending_event(struct kbase_context *kctx)
{
	struct list_head *event_list = &kctx->kbdev->job_fault_event_list;
	struct base_job_fault_event *event;
	unsigned long flags;

	spin_lock_irqsave(&kctx->kbdev->job_fault_event_lock, flags);
	list_for_each_entry(event, event_list, head) {
		if (event->katom->kctx == kctx) {
			list_del(&event->head);
			spin_unlock_irqrestore(&kctx->kbdev->job_fault_event_lock, flags);

			wake_up(&kctx->kbdev->job_fault_resume_wq);
			flush_work(&event->job_fault_work);

			/* job_fault_event_list can only have a single atom for
			 * each context.
			 */
			return;
		}
	}
	spin_unlock_irqrestore(&kctx->kbdev->job_fault_event_lock, flags);
}

static bool kbase_ctx_has_no_event_pending(struct kbase_context *kctx)
{
	struct kbase_device *kbdev = kctx->kbdev;
	struct list_head *event_list = &kctx->kbdev->job_fault_event_list;
	struct base_job_fault_event *event;
	unsigned long               flags;

	spin_lock_irqsave(&kbdev->job_fault_event_lock, flags);
	if (list_empty(event_list)) {
		spin_unlock_irqrestore(&kbdev->job_fault_event_lock, flags);
		return true;
	}
	list_for_each_entry(event, event_list, head) {
		if (event->katom->kctx == kctx) {
			spin_unlock_irqrestore(&kbdev->job_fault_event_lock,
					flags);
			return false;
		}
	}
	spin_unlock_irqrestore(&kbdev->job_fault_event_lock, flags);
	return true;
}

static int wait_for_job_fault(struct kbase_device *kbdev)
{
#if KERNEL_VERSION(4, 15, 0) > LINUX_VERSION_CODE
	int ret = wait_event_interruptible_timeout(kbdev->job_fault_wq,
			kbase_is_job_fault_event_pending(kbdev),
			msecs_to_jiffies(2000));
	if (ret == 0)
		return -EAGAIN;
	else if (ret > 0)
		return 0;
	else
		return ret;
#else
	return wait_event_interruptible(kbdev->job_fault_wq,
			kbase_is_job_fault_event_pending(kbdev));
#endif
}

/* wait until the fault happen and copy the event */
static int kbase_job_fault_event_wait(struct kbase_device *kbdev,
		struct base_job_fault_event *event)
{
	struct list_head            *event_list = &kbdev->job_fault_event_list;
	struct base_job_fault_event *event_in;
	unsigned long               flags;

	spin_lock_irqsave(&kbdev->job_fault_event_lock, flags);
	while (list_empty(event_list)) {
		int err;

		spin_unlock_irqrestore(&kbdev->job_fault_event_lock, flags);

		err = wait_for_job_fault(kbdev);
		if (err)
			return err;

		spin_lock_irqsave(&kbdev->job_fault_event_lock, flags);
	}

	event_in = list_entry(event_list->next,
			struct base_job_fault_event, head);
	event->event_code = event_in->event_code;
	event->katom = event_in->katom;

	spin_unlock_irqrestore(&kbdev->job_fault_event_lock, flags);

	return 0;

}

/* remove the event from the queue */
static struct base_job_fault_event *kbase_job_fault_event_dequeue(
		struct kbase_device *kbdev, struct list_head *event_list)
{
	struct base_job_fault_event *event;

	event = list_entry(event_list->next,
			struct base_job_fault_event, head);
	list_del(event_list->next);

	return event;

}

/* Remove all the following atoms after the failed atom in the same context
 * Call the postponed bottom half of job done.
 * Then, this context could be rescheduled.
 */
static void kbase_job_fault_resume_event_cleanup(struct kbase_context *kctx)
{
	struct list_head *event_list = &kctx->job_fault_resume_event_list;

	while (!list_empty(event_list)) {
		struct base_job_fault_event *event;

		event = kbase_job_fault_event_dequeue(kctx->kbdev,
				&kctx->job_fault_resume_event_list);
		kbase_jd_done_worker(&event->katom->work);
	}

}

static void kbase_job_fault_resume_worker(struct work_struct *data)
{
	struct base_job_fault_event *event = container_of(data,
			struct base_job_fault_event, job_fault_work);
	struct kbase_context *kctx;
	struct kbase_jd_atom *katom;

	katom = event->katom;
	kctx = katom->kctx;

	dev_info(kctx->kbdev->dev, "Job dumping wait\n");

	/* When it was waked up, it need to check if queue is empty or the
	 * failed atom belongs to different context. If yes, wake up. Both
	 * of them mean the failed job has been dumped. Please note, it
	 * should never happen that the job_fault_event_list has the two
	 * atoms belong to the same context.
	 */
	wait_event(kctx->kbdev->job_fault_resume_wq,
			 kbase_ctx_has_no_event_pending(kctx));

	atomic_set(&kctx->job_fault_count, 0);
	kbase_jd_done_worker(&katom->work);

	/* In case the following atoms were scheduled during failed job dump
	 * the job_done_worker was held. We need to rerun it after the dump
	 * was finished
	 */
	kbase_job_fault_resume_event_cleanup(kctx);

	dev_info(kctx->kbdev->dev, "Job dumping finish, resume scheduler\n");
}

static struct base_job_fault_event *kbase_job_fault_event_queue(
		struct list_head *event_list,
		struct kbase_jd_atom *atom,
		u32 completion_code)
{
	struct base_job_fault_event *event;

	event = &atom->fault_event;

	event->katom = atom;
	event->event_code = completion_code;

	list_add_tail(&event->head, event_list);

	return event;

}

static void kbase_job_fault_event_post(struct kbase_device *kbdev,
		struct kbase_jd_atom *katom, u32 completion_code)
{
	struct base_job_fault_event *event;
	unsigned long flags;

	spin_lock_irqsave(&kbdev->job_fault_event_lock, flags);
	event = kbase_job_fault_event_queue(&kbdev->job_fault_event_list,
				katom, completion_code);
	spin_unlock_irqrestore(&kbdev->job_fault_event_lock, flags);

	wake_up_interruptible(&kbdev->job_fault_wq);

	INIT_WORK(&event->job_fault_work, kbase_job_fault_resume_worker);
	queue_work(kbdev->job_fault_resume_workq, &event->job_fault_work);

	dev_info(katom->kctx->kbdev->dev, "Job fault happen, start dump: %d_%d",
			katom->kctx->tgid, katom->kctx->id);

}

/*
 * This function will process the job fault
 * Get the register copy
 * Send the failed job dump event
 * Create a Wait queue to wait until the job dump finish
 */

bool kbase_debug_job_fault_process(struct kbase_jd_atom *katom,
		u32 completion_code)
{
	struct kbase_context *kctx = katom->kctx;

	/* Check if dumping is in the process
	 * only one atom of each context can be dumped at the same time
	 * If the atom belongs to different context, it can be dumped
	 */
	if (atomic_read(&kctx->job_fault_count) > 0) {
		kbase_job_fault_event_queue(
				&kctx->job_fault_resume_event_list,
				katom, completion_code);
		dev_info(kctx->kbdev->dev, "queue:%d\n",
				kbase_jd_atom_id(kctx, katom));
		return true;
	}

	if (kbase_ctx_flag(kctx, KCTX_DYING))
		return false;

	if (atomic_read(&kctx->kbdev->job_fault_debug) > 0) {

		if (completion_code != BASE_JD_EVENT_DONE) {

			if (kbase_job_fault_get_reg_snapshot(kctx) == false) {
				dev_warn(kctx->kbdev->dev, "get reg dump failed\n");
				return false;
			}

			kbase_job_fault_event_post(kctx->kbdev, katom,
					completion_code);
			atomic_inc(&kctx->job_fault_count);
			dev_info(kctx->kbdev->dev, "post:%d\n",
					kbase_jd_atom_id(kctx, katom));
			return true;

		}
	}
	return false;

}

static int debug_job_fault_show(struct seq_file *m, void *v)
{
	struct kbase_device *kbdev = m->private;
	struct base_job_fault_event *event = (struct base_job_fault_event *)v;
	struct kbase_context *kctx = event->katom->kctx;
	int i;

	dev_info(kbdev->dev, "debug job fault seq show:%d_%d, %d",
			kctx->tgid, kctx->id, event->reg_offset);

	if (kctx->reg_dump == NULL) {
		dev_warn(kbdev->dev, "reg dump is NULL");
		return -1;
	}

	if (kctx->reg_dump[event->reg_offset] ==
			REGISTER_DUMP_TERMINATION_FLAG) {
		/* Return the error here to stop the read. And the
		 * following next() will not be called. The stop can
		 * get the real event resource and release it
		 */
		return -1;
	}

	if (event->reg_offset == 0)
		seq_printf(m, "%d_%d\n", kctx->tgid, kctx->id);

	for (i = 0; i < 50; i++) {
		if (kctx->reg_dump[event->reg_offset] ==
				REGISTER_DUMP_TERMINATION_FLAG) {
			break;
		}
		seq_printf(m, "%08x: %08x\n",
				kctx->reg_dump[event->reg_offset],
				kctx->reg_dump[1+event->reg_offset]);
		event->reg_offset += 2;

	}


	return 0;
}
static void *debug_job_fault_next(struct seq_file *m, void *v, loff_t *pos)
{
	struct kbase_device *kbdev = m->private;
	struct base_job_fault_event *event = (struct base_job_fault_event *)v;

	dev_info(kbdev->dev, "debug job fault seq next:%d, %d",
			event->reg_offset, (int)*pos);

	return event;
}

static void *debug_job_fault_start(struct seq_file *m, loff_t *pos)
{
	struct kbase_device *kbdev = m->private;
	struct base_job_fault_event *event;

	dev_info(kbdev->dev, "fault job seq start:%d", (int)*pos);

	/* The condition is trick here. It needs make sure the
	 * fault hasn't happened and the dumping hasn't been started,
	 * or the dumping has finished
	 */
	if (*pos == 0) {
		event = kmalloc(sizeof(*event), GFP_KERNEL);
		if (!event)
			return NULL;
		event->reg_offset = 0;
		if (kbase_job_fault_event_wait(kbdev, event)) {
			kfree(event);
			return NULL;
		}

		/* The cache flush workaround is called in bottom half of
		 * job done but we delayed it. Now we should clean cache
		 * earlier. Then the GPU memory dump should be correct.
		 */
		kbase_backend_cache_clean(kbdev, event->katom);
	} else
		return NULL;

	return event;
}

static void debug_job_fault_stop(struct seq_file *m, void *v)
{
	struct kbase_device *kbdev = m->private;

	/* here we wake up the kbase_jd_done_worker after stop, it needs
	 * get the memory dump before the register dump in debug daemon,
	 * otherwise, the memory dump may be incorrect.
	 */

	if (v != NULL) {
		kfree(v);
		dev_info(kbdev->dev, "debug job fault seq stop stage 1");

	} else {
		unsigned long flags;

		spin_lock_irqsave(&kbdev->job_fault_event_lock, flags);
		if (!list_empty(&kbdev->job_fault_event_list)) {
			kbase_job_fault_event_dequeue(kbdev,
				&kbdev->job_fault_event_list);
			wake_up(&kbdev->job_fault_resume_wq);
		}
		spin_unlock_irqrestore(&kbdev->job_fault_event_lock, flags);
		dev_info(kbdev->dev, "debug job fault seq stop stage 2");
	}

}

static const struct seq_operations ops = {
	.start = debug_job_fault_start,
	.next = debug_job_fault_next,
	.stop = debug_job_fault_stop,
	.show = debug_job_fault_show,
};

static int debug_job_fault_open(struct inode *in, struct file *file)
{
	struct kbase_device *kbdev = in->i_private;

	if (atomic_cmpxchg(&kbdev->job_fault_debug, 0, 1) == 1) {
		dev_warn(kbdev->dev, "debug job fault is busy, only a single client is allowed");
		return -EBUSY;
	}

	seq_open(file, &ops);

	((struct seq_file *)file->private_data)->private = kbdev;
	dev_info(kbdev->dev, "debug job fault seq open");


	return 0;

}

static int debug_job_fault_release(struct inode *in, struct file *file)
{
	struct kbase_device *kbdev = in->i_private;
	struct list_head *event_list = &kbdev->job_fault_event_list;
	unsigned long    flags;

	seq_release(in, file);

	spin_lock_irqsave(&kbdev->job_fault_event_lock, flags);

	/* Disable job fault dumping. This will let kbase run jobs as normal,
	 * without blocking waiting for a job_fault client to read failed jobs.
	 *
	 * After this a new client may open the file, and may re-enable job
	 * fault dumping, but the job_fault_event_lock we hold here will block
	 * that from interfering until after we've completed the cleanup.
	 */
	atomic_dec(&kbdev->job_fault_debug);

	/* Clean the unprocessed job fault. After that, all the suspended
	 * contexts could be rescheduled. Remove all the failed atoms that
	 * belong to different contexts Resume all the contexts that were
	 * suspend due to failed job.
	 */
	while (!list_empty(event_list)) {
		kbase_job_fault_event_dequeue(kbdev, event_list);
		spin_unlock_irqrestore(&kbdev->job_fault_event_lock, flags);
		wake_up(&kbdev->job_fault_resume_wq);
		spin_lock_irqsave(&kbdev->job_fault_event_lock, flags);
	}

	spin_unlock_irqrestore(&kbdev->job_fault_event_lock, flags);

	dev_info(kbdev->dev, "debug job fault seq close");

	return 0;
}

static const struct file_operations kbasep_debug_job_fault_fops = {
	.owner = THIS_MODULE,
	.open = debug_job_fault_open,
	.read = seq_read,
	.llseek = seq_lseek,
	.release = debug_job_fault_release,
};

/*
 *  Initialize debugfs entry for job fault dump
 */
void kbase_debug_job_fault_debugfs_init(struct kbase_device *kbdev)
{
	debugfs_create_file("job_fault", 0400,
			kbdev->mali_debugfs_directory, kbdev,
			&kbasep_debug_job_fault_fops);
}


int kbase_debug_job_fault_dev_init(struct kbase_device *kbdev)
{

	INIT_LIST_HEAD(&kbdev->job_fault_event_list);

	init_waitqueue_head(&(kbdev->job_fault_wq));
	init_waitqueue_head(&(kbdev->job_fault_resume_wq));
	spin_lock_init(&kbdev->job_fault_event_lock);

	kbdev->job_fault_resume_workq = alloc_workqueue(
			"kbase_job_fault_resume_work_queue", WQ_MEM_RECLAIM, 1);
	if (!kbdev->job_fault_resume_workq)
		return -ENOMEM;

	atomic_set(&kbdev->job_fault_debug, 0);

	return 0;
}

/*
 * Release the relevant resource per device
 */
void kbase_debug_job_fault_dev_term(struct kbase_device *kbdev)
{
	destroy_workqueue(kbdev->job_fault_resume_workq);
}


/*
 *  Initialize the relevant data structure per context
 */
int kbase_debug_job_fault_context_init(struct kbase_context *kctx)
{

	/* We need allocate double size register range
	 * Because this memory will keep the register address and value
	 */
	kctx->reg_dump = vmalloc(0x4000 * 2);
	if (kctx->reg_dump != NULL) {
		if (kbase_debug_job_fault_reg_snapshot_init(kctx, 0x4000) ==
		    false) {
			vfree(kctx->reg_dump);
			kctx->reg_dump = NULL;
		}
		INIT_LIST_HEAD(&kctx->job_fault_resume_event_list);
		atomic_set(&kctx->job_fault_count, 0);
	}

	return 0;
}

/*
 *  release the relevant resource per context
 */
void kbase_debug_job_fault_context_term(struct kbase_context *kctx)
{
	vfree(kctx->reg_dump);
}

void kbase_debug_job_fault_kctx_unblock(struct kbase_context *kctx)
{
	WARN_ON(!kbase_ctx_flag(kctx, KCTX_DYING));

	/* Return early if the job fault part of the kbase_device is not
	 * initialized yet. An error can happen during the device probe after
	 * the privileged Kbase context was created for the HW counter dumping
	 * but before the job fault part is initialized.
	 */
	if (!kctx->kbdev->job_fault_resume_workq)
		return;

	kbase_ctx_remove_pending_event(kctx);
}

#else /* CONFIG_DEBUG_FS */

int kbase_debug_job_fault_dev_init(struct kbase_device *kbdev)
{
	return 0;
}

void kbase_debug_job_fault_dev_term(struct kbase_device *kbdev)
{
}

#endif /* CONFIG_DEBUG_FS */