113 struct dsu_pmu {  struct
128 static inline struct dsu_pmu *to_dsu_pmu(struct pmu *pmu)  in to_dsu_pmu()  argument
130 	return container_of(pmu, struct dsu_pmu, pmu);  in to_dsu_pmu()
157 	struct dsu_pmu *dsu_pmu = to_dsu_pmu(pmu);  in dsu_pmu_cpumask_show()  local
165 		cpumask = &dsu_pmu->active_cpu;  in dsu_pmu_cpumask_show()
168 		cpumask = &dsu_pmu->associated_cpus;  in dsu_pmu_cpumask_show()
203 	struct dsu_pmu *dsu_pmu = to_dsu_pmu(pmu);  in dsu_pmu_event_attr_is_visible()  local
208 	return test_bit(evt, dsu_pmu->cpmceid_bitmap) ? attr->mode : 0;  in dsu_pmu_event_attr_is_visible()
234 static int dsu_pmu_get_online_cpu_any_but(struct dsu_pmu *dsu_pmu, int cpu)  in dsu_pmu_get_online_cpu_any_but()  argument
239 			 &dsu_pmu->associated_cpus, cpu_online_mask);  in dsu_pmu_get_online_cpu_any_but()
243 static inline bool dsu_pmu_counter_valid(struct dsu_pmu *dsu_pmu, u32 idx)  in dsu_pmu_counter_valid()  argument
245 	return (idx < dsu_pmu->num_counters) ||  in dsu_pmu_counter_valid()
253 	struct dsu_pmu *dsu_pmu = to_dsu_pmu(event->pmu);  in dsu_pmu_read_counter()  local
257 				 &dsu_pmu->associated_cpus)))  in dsu_pmu_read_counter()
260 	if (!dsu_pmu_counter_valid(dsu_pmu, idx)) {  in dsu_pmu_read_counter()
266 	raw_spin_lock_irqsave(&dsu_pmu->pmu_lock, flags);  in dsu_pmu_read_counter()
271 	raw_spin_unlock_irqrestore(&dsu_pmu->pmu_lock, flags);  in dsu_pmu_read_counter()
279 	struct dsu_pmu *dsu_pmu = to_dsu_pmu(event->pmu);  in dsu_pmu_write_counter()  local
283 			 &dsu_pmu->associated_cpus)))  in dsu_pmu_write_counter()
286 	if (!dsu_pmu_counter_valid(dsu_pmu, idx)) {  in dsu_pmu_write_counter()
292 	raw_spin_lock_irqsave(&dsu_pmu->pmu_lock, flags);  in dsu_pmu_write_counter()
297 	raw_spin_unlock_irqrestore(&dsu_pmu->pmu_lock, flags);  in dsu_pmu_write_counter()
305 	struct dsu_pmu *dsu_pmu = to_dsu_pmu(event->pmu);  in dsu_pmu_get_event_idx()  local
314 	idx = find_first_zero_bit(used_mask, dsu_pmu->num_counters);  in dsu_pmu_get_event_idx()
315 	if (idx >= dsu_pmu->num_counters)  in dsu_pmu_get_event_idx()
321 static void dsu_pmu_enable_counter(struct dsu_pmu *dsu_pmu, int idx)  in dsu_pmu_enable_counter()  argument
327 static void dsu_pmu_disable_counter(struct dsu_pmu *dsu_pmu, int idx)  in dsu_pmu_disable_counter()  argument
333 static inline void dsu_pmu_set_event(struct dsu_pmu *dsu_pmu,  in dsu_pmu_set_event()  argument
339 	if (!dsu_pmu_counter_valid(dsu_pmu, idx)) {  in dsu_pmu_set_event()
345 	raw_spin_lock_irqsave(&dsu_pmu->pmu_lock, flags);  in dsu_pmu_set_event()
347 	raw_spin_unlock_irqrestore(&dsu_pmu->pmu_lock, flags);  in dsu_pmu_set_event()
395 	struct dsu_pmu *dsu_pmu = dev;  in dsu_pmu_handle_irq()  local
396 	struct dsu_hw_events *hw_events = &dsu_pmu->hw_events;  in dsu_pmu_handle_irq()
418 	struct dsu_pmu *dsu_pmu = to_dsu_pmu(event->pmu);  in dsu_pmu_start()  local
425 		dsu_pmu_set_event(dsu_pmu, event);  in dsu_pmu_start()
427 	dsu_pmu_enable_counter(dsu_pmu, event->hw.idx);  in dsu_pmu_start()
432 	struct dsu_pmu *dsu_pmu = to_dsu_pmu(event->pmu);  in dsu_pmu_stop()  local
436 	dsu_pmu_disable_counter(dsu_pmu, event->hw.idx);  in dsu_pmu_stop()
443 	struct dsu_pmu *dsu_pmu = to_dsu_pmu(event->pmu);  in dsu_pmu_add()  local
444 	struct dsu_hw_events *hw_events = &dsu_pmu->hw_events;  in dsu_pmu_add()
449 					   &dsu_pmu->associated_cpus)))  in dsu_pmu_add()
469 	struct dsu_pmu *dsu_pmu = to_dsu_pmu(event->pmu);  in dsu_pmu_del()  local
470 	struct dsu_hw_events *hw_events = &dsu_pmu->hw_events;  in dsu_pmu_del()
484 	struct dsu_pmu *dsu_pmu = to_dsu_pmu(pmu);  in dsu_pmu_enable()  local
487 	if (bitmap_empty(dsu_pmu->hw_events.used_mask, DSU_PMU_MAX_HW_CNTRS))  in dsu_pmu_enable()
490 	raw_spin_lock_irqsave(&dsu_pmu->pmu_lock, flags);  in dsu_pmu_enable()
494 	raw_spin_unlock_irqrestore(&dsu_pmu->pmu_lock, flags);  in dsu_pmu_enable()
501 	struct dsu_pmu *dsu_pmu = to_dsu_pmu(pmu);  in dsu_pmu_disable()  local
503 	raw_spin_lock_irqsave(&dsu_pmu->pmu_lock, flags);  in dsu_pmu_disable()
507 	raw_spin_unlock_irqrestore(&dsu_pmu->pmu_lock, flags);  in dsu_pmu_disable()
546 	struct dsu_pmu *dsu_pmu = to_dsu_pmu(event->pmu);  in dsu_pmu_event_init()  local
553 		dev_dbg(dsu_pmu->pmu.dev, "Can't support sampling events\n");  in dsu_pmu_event_init()
559 		dev_dbg(dsu_pmu->pmu.dev, "Can't support per-task counters\n");  in dsu_pmu_event_init()
564 		dev_dbg(dsu_pmu->pmu.dev, "Can't support filtering\n");  in dsu_pmu_event_init()
568 	if (!cpumask_test_cpu(event->cpu, &dsu_pmu->associated_cpus)) {  in dsu_pmu_event_init()
569 		dev_dbg(dsu_pmu->pmu.dev,  in dsu_pmu_event_init()
579 	event->cpu = cpumask_first(&dsu_pmu->active_cpu);  in dsu_pmu_event_init()
589 static struct dsu_pmu *dsu_pmu_alloc(struct platform_device *pdev)  in dsu_pmu_alloc()
591 	struct dsu_pmu *dsu_pmu;  in dsu_pmu_alloc()  local
593 	dsu_pmu = devm_kzalloc(&pdev->dev, sizeof(*dsu_pmu), GFP_KERNEL);  in dsu_pmu_alloc()
594 	if (!dsu_pmu)  in dsu_pmu_alloc()
597 	raw_spin_lock_init(&dsu_pmu->pmu_lock);  in dsu_pmu_alloc()
602 	dsu_pmu->num_counters = -1;  in dsu_pmu_alloc()
603 	return dsu_pmu;  in dsu_pmu_alloc()
669 static void dsu_pmu_probe_pmu(struct dsu_pmu *dsu_pmu)  in dsu_pmu_probe_pmu()  argument
679 	dsu_pmu->num_counters = num_counters;  in dsu_pmu_probe_pmu()
680 	if (!dsu_pmu->num_counters)  in dsu_pmu_probe_pmu()
684 	bitmap_from_arr32(dsu_pmu->cpmceid_bitmap, cpmceid,  in dsu_pmu_probe_pmu()
688 static void dsu_pmu_set_active_cpu(int cpu, struct dsu_pmu *dsu_pmu)  in dsu_pmu_set_active_cpu()  argument
690 	cpumask_set_cpu(cpu, &dsu_pmu->active_cpu);  in dsu_pmu_set_active_cpu()
691 	if (irq_set_affinity_hint(dsu_pmu->irq, &dsu_pmu->active_cpu))  in dsu_pmu_set_active_cpu()
699 static void dsu_pmu_init_pmu(struct dsu_pmu *dsu_pmu)  in dsu_pmu_init_pmu()  argument
701 	if (dsu_pmu->num_counters == -1)  in dsu_pmu_init_pmu()
702 		dsu_pmu_probe_pmu(dsu_pmu);  in dsu_pmu_init_pmu()
710 	struct dsu_pmu *dsu_pmu;  in dsu_pmu_device_probe()  local
715 	dsu_pmu = dsu_pmu_alloc(pdev);  in dsu_pmu_device_probe()
716 	if (IS_ERR(dsu_pmu))  in dsu_pmu_device_probe()
717 		return PTR_ERR(dsu_pmu);  in dsu_pmu_device_probe()
723 		rc = dsu_pmu_dt_get_cpus(&pdev->dev, &dsu_pmu->associated_cpus);  in dsu_pmu_device_probe()
725 		rc = dsu_pmu_acpi_get_cpus(&pdev->dev, &dsu_pmu->associated_cpus);  in dsu_pmu_device_probe()
743 			      IRQF_NOBALANCING, name, dsu_pmu);  in dsu_pmu_device_probe()
749 	dsu_pmu->irq = irq;  in dsu_pmu_device_probe()
750 	platform_set_drvdata(pdev, dsu_pmu);  in dsu_pmu_device_probe()
752 						&dsu_pmu->cpuhp_node);  in dsu_pmu_device_probe()
756 	dsu_pmu->pmu = (struct pmu) {  in dsu_pmu_device_probe()
772 	rc = perf_pmu_register(&dsu_pmu->pmu, name, -1);  in dsu_pmu_device_probe()
775 						 &dsu_pmu->cpuhp_node);  in dsu_pmu_device_probe()
776 		irq_set_affinity_hint(dsu_pmu->irq, NULL);  in dsu_pmu_device_probe()
784 	struct dsu_pmu *dsu_pmu = platform_get_drvdata(pdev);  in dsu_pmu_device_remove()  local
786 	perf_pmu_unregister(&dsu_pmu->pmu);  in dsu_pmu_device_remove()
787 	cpuhp_state_remove_instance(dsu_pmu_cpuhp_state, &dsu_pmu->cpuhp_node);  in dsu_pmu_device_remove()
788 	irq_set_affinity_hint(dsu_pmu->irq, NULL);  in dsu_pmu_device_remove()
820 	struct dsu_pmu *dsu_pmu = hlist_entry_safe(node, struct dsu_pmu,  in dsu_pmu_cpu_online()  local
823 	if (!cpumask_test_cpu(cpu, &dsu_pmu->associated_cpus))  in dsu_pmu_cpu_online()
827 	if (!cpumask_empty(&dsu_pmu->active_cpu))  in dsu_pmu_cpu_online()
830 	dsu_pmu_init_pmu(dsu_pmu);  in dsu_pmu_cpu_online()
831 	dsu_pmu_set_active_cpu(cpu, dsu_pmu);  in dsu_pmu_cpu_online()
839 	struct dsu_pmu *dsu_pmu = hlist_entry_safe(node, struct dsu_pmu,  in dsu_pmu_cpu_teardown()  local
842 	if (!cpumask_test_and_clear_cpu(cpu, &dsu_pmu->active_cpu))  in dsu_pmu_cpu_teardown()
845 	dst = dsu_pmu_get_online_cpu_any_but(dsu_pmu, cpu);  in dsu_pmu_cpu_teardown()
848 		irq_set_affinity_hint(dsu_pmu->irq, NULL);  in dsu_pmu_cpu_teardown()
852 	perf_pmu_migrate_context(&dsu_pmu->pmu, cpu, dst);  in dsu_pmu_cpu_teardown()
853 	dsu_pmu_set_active_cpu(dst, dsu_pmu);  in dsu_pmu_cpu_teardown()