195 	struct l3cache_pmu *l3pmu = to_l3cache_pmu(event->pmu);  in qcom_l3_cache__64bit_counter_start()  local
201 	gang = readl_relaxed(l3pmu->regs + L3_M_BC_GANG);  in qcom_l3_cache__64bit_counter_start()
203 	writel_relaxed(gang, l3pmu->regs + L3_M_BC_GANG);  in qcom_l3_cache__64bit_counter_start()
207 	writel_relaxed(0, l3pmu->regs + L3_HML3_PM_EVCNTR(idx + 1));  in qcom_l3_cache__64bit_counter_start()
208 	writel_relaxed(0, l3pmu->regs + L3_HML3_PM_EVCNTR(idx));  in qcom_l3_cache__64bit_counter_start()
214 	writel_relaxed(EVSEL(0), l3pmu->regs + L3_HML3_PM_EVTYPE(idx + 1));  in qcom_l3_cache__64bit_counter_start()
215 	writel_relaxed(EVSEL(evsel), l3pmu->regs + L3_HML3_PM_EVTYPE(idx));  in qcom_l3_cache__64bit_counter_start()
218 	writel_relaxed(PMCNT_RESET, l3pmu->regs + L3_HML3_PM_CNTCTL(idx + 1));  in qcom_l3_cache__64bit_counter_start()
219 	writel_relaxed(PMCNTENSET(idx + 1), l3pmu->regs + L3_M_BC_CNTENSET);  in qcom_l3_cache__64bit_counter_start()
220 	writel_relaxed(PMCNT_RESET, l3pmu->regs + L3_HML3_PM_CNTCTL(idx));  in qcom_l3_cache__64bit_counter_start()
221 	writel_relaxed(PMCNTENSET(idx), l3pmu->regs + L3_M_BC_CNTENSET);  in qcom_l3_cache__64bit_counter_start()
227 	struct l3cache_pmu *l3pmu = to_l3cache_pmu(event->pmu);  in qcom_l3_cache__64bit_counter_stop()  local
229 	u32 gang = readl_relaxed(l3pmu->regs + L3_M_BC_GANG);  in qcom_l3_cache__64bit_counter_stop()
232 	writel_relaxed(PMCNTENCLR(idx), l3pmu->regs + L3_M_BC_CNTENCLR);  in qcom_l3_cache__64bit_counter_stop()
233 	writel_relaxed(PMCNTENCLR(idx + 1), l3pmu->regs + L3_M_BC_CNTENCLR);  in qcom_l3_cache__64bit_counter_stop()
236 	writel_relaxed(gang & ~GANG_EN(idx + 1), l3pmu->regs + L3_M_BC_GANG);  in qcom_l3_cache__64bit_counter_stop()
241 	struct l3cache_pmu *l3pmu = to_l3cache_pmu(event->pmu);  in qcom_l3_cache__64bit_counter_update()  local
249 			hi = readl_relaxed(l3pmu->regs + L3_HML3_PM_EVCNTR(idx + 1));  in qcom_l3_cache__64bit_counter_update()
250 			lo = readl_relaxed(l3pmu->regs + L3_HML3_PM_EVCNTR(idx));  in qcom_l3_cache__64bit_counter_update()
251 		} while (hi != readl_relaxed(l3pmu->regs + L3_HML3_PM_EVCNTR(idx + 1)));  in qcom_l3_cache__64bit_counter_update()
276 	struct l3cache_pmu *l3pmu = to_l3cache_pmu(event->pmu);  in qcom_l3_cache__32bit_counter_start()  local
279 	u32 irqctl = readl_relaxed(l3pmu->regs + L3_M_BC_IRQCTL);  in qcom_l3_cache__32bit_counter_start()
282 	writel_relaxed(irqctl | PMIRQONMSBEN(idx), l3pmu->regs + L3_M_BC_IRQCTL);  in qcom_l3_cache__32bit_counter_start()
286 	writel_relaxed(0, l3pmu->regs + L3_HML3_PM_EVCNTR(idx));  in qcom_l3_cache__32bit_counter_start()
289 	writel_relaxed(EVSEL(evsel), l3pmu->regs + L3_HML3_PM_EVTYPE(idx));  in qcom_l3_cache__32bit_counter_start()
292 	writel_relaxed(PMINTENSET(idx), l3pmu->regs + L3_M_BC_INTENSET);  in qcom_l3_cache__32bit_counter_start()
295 	writel_relaxed(PMCNT_RESET, l3pmu->regs + L3_HML3_PM_CNTCTL(idx));  in qcom_l3_cache__32bit_counter_start()
296 	writel_relaxed(PMCNTENSET(idx), l3pmu->regs + L3_M_BC_CNTENSET);  in qcom_l3_cache__32bit_counter_start()
302 	struct l3cache_pmu *l3pmu = to_l3cache_pmu(event->pmu);  in qcom_l3_cache__32bit_counter_stop()  local
304 	u32 irqctl = readl_relaxed(l3pmu->regs + L3_M_BC_IRQCTL);  in qcom_l3_cache__32bit_counter_stop()
307 	writel_relaxed(PMCNTENCLR(idx), l3pmu->regs + L3_M_BC_CNTENCLR);  in qcom_l3_cache__32bit_counter_stop()
310 	writel_relaxed(PMINTENCLR(idx), l3pmu->regs + L3_M_BC_INTENCLR);  in qcom_l3_cache__32bit_counter_stop()
313 	writel_relaxed(irqctl & ~PMIRQONMSBEN(idx), l3pmu->regs + L3_M_BC_IRQCTL);  in qcom_l3_cache__32bit_counter_stop()
318 	struct l3cache_pmu *l3pmu = to_l3cache_pmu(event->pmu);  in qcom_l3_cache__32bit_counter_update()  local
324 		new = readl_relaxed(l3pmu->regs + L3_HML3_PM_EVCNTR(idx));  in qcom_l3_cache__32bit_counter_update()
350 static inline void qcom_l3_cache__init(struct l3cache_pmu *l3pmu)  in qcom_l3_cache__init()  argument
354 	writel_relaxed(BC_RESET, l3pmu->regs + L3_M_BC_CR);  in qcom_l3_cache__init()
360 	writel(BC_SATROLL_CR_RESET, l3pmu->regs + L3_M_BC_SATROLL_CR);  in qcom_l3_cache__init()
362 	writel_relaxed(BC_CNTENCLR_RESET, l3pmu->regs + L3_M_BC_CNTENCLR);  in qcom_l3_cache__init()
363 	writel_relaxed(BC_INTENCLR_RESET, l3pmu->regs + L3_M_BC_INTENCLR);  in qcom_l3_cache__init()
364 	writel_relaxed(PMOVSRCLR_RESET, l3pmu->regs + L3_M_BC_OVSR);  in qcom_l3_cache__init()
365 	writel_relaxed(BC_GANG_RESET, l3pmu->regs + L3_M_BC_GANG);  in qcom_l3_cache__init()
366 	writel_relaxed(BC_IRQCTL_RESET, l3pmu->regs + L3_M_BC_IRQCTL);  in qcom_l3_cache__init()
367 	writel_relaxed(PM_CR_RESET, l3pmu->regs + L3_HML3_PM_CR);  in qcom_l3_cache__init()
370 		writel_relaxed(PMCNT_RESET, l3pmu->regs + L3_HML3_PM_CNTCTL(i));  in qcom_l3_cache__init()
371 		writel_relaxed(EVSEL(0), l3pmu->regs + L3_HML3_PM_EVTYPE(i));  in qcom_l3_cache__init()
374 	writel_relaxed(PM_FLTR_RESET, l3pmu->regs + L3_HML3_PM_FILTRA);  in qcom_l3_cache__init()
375 	writel_relaxed(PM_FLTR_RESET, l3pmu->regs + L3_HML3_PM_FILTRAM);  in qcom_l3_cache__init()
376 	writel_relaxed(PM_FLTR_RESET, l3pmu->regs + L3_HML3_PM_FILTRB);  in qcom_l3_cache__init()
377 	writel_relaxed(PM_FLTR_RESET, l3pmu->regs + L3_HML3_PM_FILTRBM);  in qcom_l3_cache__init()
378 	writel_relaxed(PM_FLTR_RESET, l3pmu->regs + L3_HML3_PM_FILTRC);  in qcom_l3_cache__init()
379 	writel_relaxed(PM_FLTR_RESET, l3pmu->regs + L3_HML3_PM_FILTRCM);  in qcom_l3_cache__init()
385 	writel(BC_ENABLE, l3pmu->regs + L3_M_BC_CR);  in qcom_l3_cache__init()
390 	struct l3cache_pmu *l3pmu = data;  in qcom_l3_cache__handle_irq()  local
392 	long status = readl_relaxed(l3pmu->regs + L3_M_BC_OVSR);  in qcom_l3_cache__handle_irq()
399 	writel_relaxed(status, l3pmu->regs + L3_M_BC_OVSR);  in qcom_l3_cache__handle_irq()
405 		event = l3pmu->events[idx];  in qcom_l3_cache__handle_irq()
429 	struct l3cache_pmu *l3pmu = to_l3cache_pmu(pmu);  in qcom_l3_cache__pmu_enable()  local
434 	writel_relaxed(BC_ENABLE, l3pmu->regs + L3_M_BC_CR);  in qcom_l3_cache__pmu_enable()
439 	struct l3cache_pmu *l3pmu = to_l3cache_pmu(pmu);  in qcom_l3_cache__pmu_disable()  local
441 	writel_relaxed(0, l3pmu->regs + L3_M_BC_CR);  in qcom_l3_cache__pmu_disable()
480 	struct l3cache_pmu *l3pmu = to_l3cache_pmu(event->pmu);  in qcom_l3_cache__event_init()  local
519 	event->cpu = cpumask_first(&l3pmu->cpumask);  in qcom_l3_cache__event_init()
549 	struct l3cache_pmu *l3pmu = to_l3cache_pmu(event->pmu);  in qcom_l3_cache__event_add()  local
557 	idx = bitmap_find_free_region(l3pmu->used_mask, L3_NUM_COUNTERS, order);  in qcom_l3_cache__event_add()
564 	l3pmu->events[idx] = event;  in qcom_l3_cache__event_add()
577 	struct l3cache_pmu *l3pmu = to_l3cache_pmu(event->pmu);  in qcom_l3_cache__event_del()  local
583 	l3pmu->events[hwc->idx] = NULL;  in qcom_l3_cache__event_del()
584 	bitmap_release_region(l3pmu->used_mask, hwc->idx, order);  in qcom_l3_cache__event_del()
676 	struct l3cache_pmu *l3pmu = to_l3cache_pmu(dev_get_drvdata(dev));  in qcom_l3_cache_pmu_cpumask_show()  local
678 	return cpumap_print_to_pagebuf(true, buf, &l3pmu->cpumask);  in qcom_l3_cache_pmu_cpumask_show()
708 	struct l3cache_pmu *l3pmu = hlist_entry_safe(node, struct l3cache_pmu, node);  in qcom_l3_cache_pmu_online_cpu()  local
711 	if (cpumask_empty(&l3pmu->cpumask))  in qcom_l3_cache_pmu_online_cpu()
712 		cpumask_set_cpu(cpu, &l3pmu->cpumask);  in qcom_l3_cache_pmu_online_cpu()
719 	struct l3cache_pmu *l3pmu = hlist_entry_safe(node, struct l3cache_pmu, node);  in qcom_l3_cache_pmu_offline_cpu()  local
722 	if (!cpumask_test_and_clear_cpu(cpu, &l3pmu->cpumask))  in qcom_l3_cache_pmu_offline_cpu()
727 	perf_pmu_migrate_context(&l3pmu->pmu, cpu, target);  in qcom_l3_cache_pmu_offline_cpu()
728 	cpumask_set_cpu(target, &l3pmu->cpumask);  in qcom_l3_cache_pmu_offline_cpu()
734 	struct l3cache_pmu *l3pmu;  in qcom_l3_cache_pmu_probe()  local
746 	l3pmu = devm_kzalloc(&pdev->dev, sizeof(*l3pmu), GFP_KERNEL);  in qcom_l3_cache_pmu_probe()
749 	if (!l3pmu || !name)  in qcom_l3_cache_pmu_probe()
752 	l3pmu->pmu = (struct pmu) {  in qcom_l3_cache_pmu_probe()
769 	l3pmu->regs = devm_ioremap_resource(&pdev->dev, memrc);  in qcom_l3_cache_pmu_probe()
770 	if (IS_ERR(l3pmu->regs)) {  in qcom_l3_cache_pmu_probe()
772 		return PTR_ERR(l3pmu->regs);  in qcom_l3_cache_pmu_probe()
775 	qcom_l3_cache__init(l3pmu);  in qcom_l3_cache_pmu_probe()
782 			       name, l3pmu);  in qcom_l3_cache_pmu_probe()
790 	ret = cpuhp_state_add_instance(CPUHP_AP_PERF_ARM_QCOM_L3_ONLINE, &l3pmu->node);  in qcom_l3_cache_pmu_probe()
796 	ret = perf_pmu_register(&l3pmu->pmu, name, -1);  in qcom_l3_cache_pmu_probe()
802 	dev_info(&pdev->dev, "Registered %s, type: %d\n", name, l3pmu->pmu.type);  in qcom_l3_cache_pmu_probe()