101 struct smmu_pmu {  struct
118 #define to_smmu_pmu(p) (container_of(p, struct smmu_pmu, pmu))  argument
134 	struct smmu_pmu *smmu_pmu = to_smmu_pmu(pmu);  in smmu_pmu_enable()  local
137 	       smmu_pmu->reg_base + SMMU_PMCG_IRQ_CTRL);  in smmu_pmu_enable()
138 	writel(SMMU_PMCG_CR_ENABLE, smmu_pmu->reg_base + SMMU_PMCG_CR);  in smmu_pmu_enable()
143 	struct smmu_pmu *smmu_pmu = to_smmu_pmu(pmu);  in smmu_pmu_disable()  local
145 	writel(0, smmu_pmu->reg_base + SMMU_PMCG_CR);  in smmu_pmu_disable()
146 	writel(0, smmu_pmu->reg_base + SMMU_PMCG_IRQ_CTRL);  in smmu_pmu_disable()
149 static inline void smmu_pmu_counter_set_value(struct smmu_pmu *smmu_pmu,  in smmu_pmu_counter_set_value()  argument
152 	if (smmu_pmu->counter_mask & BIT(32))  in smmu_pmu_counter_set_value()
153 		writeq(value, smmu_pmu->reloc_base + SMMU_PMCG_EVCNTR(idx, 8));  in smmu_pmu_counter_set_value()
155 		writel(value, smmu_pmu->reloc_base + SMMU_PMCG_EVCNTR(idx, 4));  in smmu_pmu_counter_set_value()
158 static inline u64 smmu_pmu_counter_get_value(struct smmu_pmu *smmu_pmu, u32 idx)  in smmu_pmu_counter_get_value()  argument
162 	if (smmu_pmu->counter_mask & BIT(32))  in smmu_pmu_counter_get_value()
163 		value = readq(smmu_pmu->reloc_base + SMMU_PMCG_EVCNTR(idx, 8));  in smmu_pmu_counter_get_value()
165 		value = readl(smmu_pmu->reloc_base + SMMU_PMCG_EVCNTR(idx, 4));  in smmu_pmu_counter_get_value()
170 static inline void smmu_pmu_counter_enable(struct smmu_pmu *smmu_pmu, u32 idx)  in smmu_pmu_counter_enable()  argument
172 	writeq(BIT(idx), smmu_pmu->reg_base + SMMU_PMCG_CNTENSET0);  in smmu_pmu_counter_enable()
175 static inline void smmu_pmu_counter_disable(struct smmu_pmu *smmu_pmu, u32 idx)  in smmu_pmu_counter_disable()  argument
177 	writeq(BIT(idx), smmu_pmu->reg_base + SMMU_PMCG_CNTENCLR0);  in smmu_pmu_counter_disable()
180 static inline void smmu_pmu_interrupt_enable(struct smmu_pmu *smmu_pmu, u32 idx)  in smmu_pmu_interrupt_enable()  argument
182 	writeq(BIT(idx), smmu_pmu->reg_base + SMMU_PMCG_INTENSET0);  in smmu_pmu_interrupt_enable()
185 static inline void smmu_pmu_interrupt_disable(struct smmu_pmu *smmu_pmu,  in smmu_pmu_interrupt_disable()  argument
188 	writeq(BIT(idx), smmu_pmu->reg_base + SMMU_PMCG_INTENCLR0);  in smmu_pmu_interrupt_disable()
191 static inline void smmu_pmu_set_evtyper(struct smmu_pmu *smmu_pmu, u32 idx,  in smmu_pmu_set_evtyper()  argument
194 	writel(val, smmu_pmu->reg_base + SMMU_PMCG_EVTYPER(idx));  in smmu_pmu_set_evtyper()
197 static inline void smmu_pmu_set_smr(struct smmu_pmu *smmu_pmu, u32 idx, u32 val)  in smmu_pmu_set_smr()  argument
199 	writel(val, smmu_pmu->reg_base + SMMU_PMCG_SMR(idx));  in smmu_pmu_set_smr()
205 	struct smmu_pmu *smmu_pmu = to_smmu_pmu(event->pmu);  in smmu_pmu_event_update()  local
211 		now = smmu_pmu_counter_get_value(smmu_pmu, idx);  in smmu_pmu_event_update()
216 	delta &= smmu_pmu->counter_mask;  in smmu_pmu_event_update()
221 static void smmu_pmu_set_period(struct smmu_pmu *smmu_pmu,  in smmu_pmu_set_period()  argument
227 	if (smmu_pmu->options & SMMU_PMCG_EVCNTR_RDONLY) {  in smmu_pmu_set_period()
235 		new = smmu_pmu_counter_get_value(smmu_pmu, idx);  in smmu_pmu_set_period()
243 		new = smmu_pmu->counter_mask >> 1;  in smmu_pmu_set_period()
244 		smmu_pmu_counter_set_value(smmu_pmu, idx, new);  in smmu_pmu_set_period()
253 	struct smmu_pmu *smmu_pmu = to_smmu_pmu(event->pmu);  in smmu_pmu_set_event_filter()  local
257 	smmu_pmu_set_evtyper(smmu_pmu, idx, evtyper);  in smmu_pmu_set_event_filter()
258 	smmu_pmu_set_smr(smmu_pmu, idx, sid);  in smmu_pmu_set_event_filter()
274 static int smmu_pmu_apply_event_filter(struct smmu_pmu *smmu_pmu,  in smmu_pmu_apply_event_filter()  argument
278 	unsigned int cur_idx, num_ctrs = smmu_pmu->num_counters;  in smmu_pmu_apply_event_filter()
286 	cur_idx = find_first_bit(smmu_pmu->used_counters, num_ctrs);  in smmu_pmu_apply_event_filter()
291 	if (!smmu_pmu->global_filter || cur_idx == num_ctrs) {  in smmu_pmu_apply_event_filter()
297 	if (smmu_pmu_check_global_filter(smmu_pmu->events[cur_idx], event)) {  in smmu_pmu_apply_event_filter()
298 		smmu_pmu_set_evtyper(smmu_pmu, idx, get_event(event));  in smmu_pmu_apply_event_filter()
305 static int smmu_pmu_get_event_idx(struct smmu_pmu *smmu_pmu,  in smmu_pmu_get_event_idx()  argument
309 	unsigned int num_ctrs = smmu_pmu->num_counters;  in smmu_pmu_get_event_idx()
311 	idx = find_first_zero_bit(smmu_pmu->used_counters, num_ctrs);  in smmu_pmu_get_event_idx()
316 	err = smmu_pmu_apply_event_filter(smmu_pmu, event, idx);  in smmu_pmu_get_event_idx()
320 	set_bit(idx, smmu_pmu->used_counters);  in smmu_pmu_get_event_idx()
346 	struct smmu_pmu *smmu_pmu = to_smmu_pmu(event->pmu);  in smmu_pmu_event_init()  local
347 	struct device *dev = smmu_pmu->dev;  in smmu_pmu_event_init()
368 	    (!test_bit(event_id, smmu_pmu->supported_events))) {  in smmu_pmu_event_init()
378 		if (++group_num_events > smmu_pmu->num_counters)  in smmu_pmu_event_init()
389 		if (++group_num_events > smmu_pmu->num_counters)  in smmu_pmu_event_init()
399 	event->cpu = smmu_pmu->on_cpu;  in smmu_pmu_event_init()
406 	struct smmu_pmu *smmu_pmu = to_smmu_pmu(event->pmu);  in smmu_pmu_event_start()  local
412 	smmu_pmu_set_period(smmu_pmu, hwc);  in smmu_pmu_event_start()
414 	smmu_pmu_counter_enable(smmu_pmu, idx);  in smmu_pmu_event_start()
419 	struct smmu_pmu *smmu_pmu = to_smmu_pmu(event->pmu);  in smmu_pmu_event_stop()  local
426 	smmu_pmu_counter_disable(smmu_pmu, idx);  in smmu_pmu_event_stop()
436 	struct smmu_pmu *smmu_pmu = to_smmu_pmu(event->pmu);  in smmu_pmu_event_add()  local
438 	idx = smmu_pmu_get_event_idx(smmu_pmu, event);  in smmu_pmu_event_add()
444 	smmu_pmu->events[idx] = event;  in smmu_pmu_event_add()
447 	smmu_pmu_interrupt_enable(smmu_pmu, idx);  in smmu_pmu_event_add()
461 	struct smmu_pmu *smmu_pmu = to_smmu_pmu(event->pmu);  in smmu_pmu_event_del()  local
465 	smmu_pmu_interrupt_disable(smmu_pmu, idx);  in smmu_pmu_event_del()
466 	smmu_pmu->events[idx] = NULL;  in smmu_pmu_event_del()
467 	clear_bit(idx, smmu_pmu->used_counters);  in smmu_pmu_event_del()
483 	struct smmu_pmu *smmu_pmu = to_smmu_pmu(dev_get_drvdata(dev));  in smmu_pmu_cpumask_show()  local
485 	return cpumap_print_to_pagebuf(true, buf, cpumask_of(smmu_pmu->on_cpu));  in smmu_pmu_cpumask_show()
540 	struct smmu_pmu *smmu_pmu = to_smmu_pmu(dev_get_drvdata(dev));  in smmu_pmu_event_is_visible()  local
545 	if (test_bit(pmu_attr->id, smmu_pmu->supported_events))  in smmu_pmu_event_is_visible()
589 	struct smmu_pmu *smmu_pmu;  in smmu_pmu_offline_cpu()  local
592 	smmu_pmu = hlist_entry_safe(node, struct smmu_pmu, node);  in smmu_pmu_offline_cpu()
593 	if (cpu != smmu_pmu->on_cpu)  in smmu_pmu_offline_cpu()
600 	perf_pmu_migrate_context(&smmu_pmu->pmu, cpu, target);  in smmu_pmu_offline_cpu()
601 	smmu_pmu->on_cpu = target;  in smmu_pmu_offline_cpu()
602 	WARN_ON(irq_set_affinity_hint(smmu_pmu->irq, cpumask_of(target)));  in smmu_pmu_offline_cpu()
609 	struct smmu_pmu *smmu_pmu = data;  in smmu_pmu_handle_irq()  local
613 	ovsr = readq(smmu_pmu->reloc_base + SMMU_PMCG_OVSSET0);  in smmu_pmu_handle_irq()
617 	writeq(ovsr, smmu_pmu->reloc_base + SMMU_PMCG_OVSCLR0);  in smmu_pmu_handle_irq()
619 	for_each_set_bit(idx, (unsigned long *)&ovsr, smmu_pmu->num_counters) {  in smmu_pmu_handle_irq()
620 		struct perf_event *event = smmu_pmu->events[idx];  in smmu_pmu_handle_irq()
629 		smmu_pmu_set_period(smmu_pmu, hwc);  in smmu_pmu_handle_irq()
646 	struct smmu_pmu *pmu = dev_get_drvdata(dev);  in smmu_pmu_write_msi_msg()
657 static void smmu_pmu_setup_msi(struct smmu_pmu *pmu)  in smmu_pmu_setup_msi()
684 static int smmu_pmu_setup_irq(struct smmu_pmu *pmu)  in smmu_pmu_setup_irq()
698 static void smmu_pmu_reset(struct smmu_pmu *smmu_pmu)  in smmu_pmu_reset()  argument
700 	u64 counter_present_mask = GENMASK_ULL(smmu_pmu->num_counters - 1, 0);  in smmu_pmu_reset()
702 	smmu_pmu_disable(&smmu_pmu->pmu);  in smmu_pmu_reset()
706 		       smmu_pmu->reg_base + SMMU_PMCG_CNTENCLR0);  in smmu_pmu_reset()
708 		       smmu_pmu->reg_base + SMMU_PMCG_INTENCLR0);  in smmu_pmu_reset()
710 		       smmu_pmu->reloc_base + SMMU_PMCG_OVSCLR0);  in smmu_pmu_reset()
713 static void smmu_pmu_get_acpi_options(struct smmu_pmu *smmu_pmu)  in smmu_pmu_get_acpi_options()  argument
717 	model = *(u32 *)dev_get_platdata(smmu_pmu->dev);  in smmu_pmu_get_acpi_options()
722 		smmu_pmu->options |= SMMU_PMCG_EVCNTR_RDONLY;  in smmu_pmu_get_acpi_options()
726 	dev_notice(smmu_pmu->dev, "option mask 0x%x\n", smmu_pmu->options);  in smmu_pmu_get_acpi_options()
731 	struct smmu_pmu *smmu_pmu;  in smmu_pmu_probe()  local
739 	smmu_pmu = devm_kzalloc(dev, sizeof(*smmu_pmu), GFP_KERNEL);  in smmu_pmu_probe()
740 	if (!smmu_pmu)  in smmu_pmu_probe()
743 	smmu_pmu->dev = dev;  in smmu_pmu_probe()
744 	platform_set_drvdata(pdev, smmu_pmu);  in smmu_pmu_probe()
746 	smmu_pmu->pmu = (struct pmu) {  in smmu_pmu_probe()
761 	smmu_pmu->reg_base = devm_platform_get_and_ioremap_resource(pdev, 0, &res_0);  in smmu_pmu_probe()
762 	if (IS_ERR(smmu_pmu->reg_base))  in smmu_pmu_probe()
763 		return PTR_ERR(smmu_pmu->reg_base);  in smmu_pmu_probe()
765 	cfgr = readl_relaxed(smmu_pmu->reg_base + SMMU_PMCG_CFGR);  in smmu_pmu_probe()
769 		smmu_pmu->reloc_base = devm_platform_ioremap_resource(pdev, 1);  in smmu_pmu_probe()
770 		if (IS_ERR(smmu_pmu->reloc_base))  in smmu_pmu_probe()
771 			return PTR_ERR(smmu_pmu->reloc_base);  in smmu_pmu_probe()
773 		smmu_pmu->reloc_base = smmu_pmu->reg_base;  in smmu_pmu_probe()
778 		smmu_pmu->irq = irq;  in smmu_pmu_probe()
780 	ceid_64[0] = readq_relaxed(smmu_pmu->reg_base + SMMU_PMCG_CEID0);  in smmu_pmu_probe()
781 	ceid_64[1] = readq_relaxed(smmu_pmu->reg_base + SMMU_PMCG_CEID1);  in smmu_pmu_probe()
782 	bitmap_from_arr32(smmu_pmu->supported_events, (u32 *)ceid_64,  in smmu_pmu_probe()
785 	smmu_pmu->num_counters = FIELD_GET(SMMU_PMCG_CFGR_NCTR, cfgr) + 1;  in smmu_pmu_probe()
787 	smmu_pmu->global_filter = !!(cfgr & SMMU_PMCG_CFGR_SID_FILTER_TYPE);  in smmu_pmu_probe()
790 	smmu_pmu->counter_mask = GENMASK_ULL(reg_size, 0);  in smmu_pmu_probe()
792 	smmu_pmu_reset(smmu_pmu);  in smmu_pmu_probe()
794 	err = smmu_pmu_setup_irq(smmu_pmu);  in smmu_pmu_probe()
807 	smmu_pmu_get_acpi_options(smmu_pmu);  in smmu_pmu_probe()
810 	smmu_pmu->on_cpu = raw_smp_processor_id();  in smmu_pmu_probe()
811 	WARN_ON(irq_set_affinity_hint(smmu_pmu->irq,  in smmu_pmu_probe()
812 				      cpumask_of(smmu_pmu->on_cpu)));  in smmu_pmu_probe()
815 					       &smmu_pmu->node);  in smmu_pmu_probe()
822 	err = perf_pmu_register(&smmu_pmu->pmu, name, -1);  in smmu_pmu_probe()
830 		 &res_0->start, smmu_pmu->num_counters,  in smmu_pmu_probe()
831 		 smmu_pmu->global_filter ? "Global(Counter0)" :  in smmu_pmu_probe()
837 	cpuhp_state_remove_instance_nocalls(cpuhp_state_num, &smmu_pmu->node);  in smmu_pmu_probe()
839 	irq_set_affinity_hint(smmu_pmu->irq, NULL);  in smmu_pmu_probe()
845 	struct smmu_pmu *smmu_pmu = platform_get_drvdata(pdev);  in smmu_pmu_remove()  local
847 	perf_pmu_unregister(&smmu_pmu->pmu);  in smmu_pmu_remove()
848 	cpuhp_state_remove_instance_nocalls(cpuhp_state_num, &smmu_pmu->node);  in smmu_pmu_remove()
849 	irq_set_affinity_hint(smmu_pmu->irq, NULL);  in smmu_pmu_remove()
856 	struct smmu_pmu *smmu_pmu = platform_get_drvdata(pdev);  in smmu_pmu_shutdown()  local
858 	smmu_pmu_disable(&smmu_pmu->pmu);  in smmu_pmu_shutdown()