108 	void (*write_evttype)(struct xgene_pmu_dev *pmu_dev, int idx, u32 val);
109 	void (*write_agentmsk)(struct xgene_pmu_dev *pmu_dev, u32 val);
110 	void (*write_agent1msk)(struct xgene_pmu_dev *pmu_dev, u32 val);
111 	void (*enable_counter)(struct xgene_pmu_dev *pmu_dev, int idx);
112 	void (*disable_counter)(struct xgene_pmu_dev *pmu_dev, int idx);
113 	void (*enable_counter_int)(struct xgene_pmu_dev *pmu_dev, int idx);
114 	void (*disable_counter_int)(struct xgene_pmu_dev *pmu_dev, int idx);
115 	void (*reset_counters)(struct xgene_pmu_dev *pmu_dev);
116 	void (*start_counters)(struct xgene_pmu_dev *pmu_dev);
117 	void (*stop_counters)(struct xgene_pmu_dev *pmu_dev);
141 	struct xgene_pmu_dev *pmu_dev;  member
610 	struct xgene_pmu_dev *pmu_dev = to_pmu_dev(dev_get_drvdata(dev));  in xgene_pmu_cpumask_show()  local
612 	return cpumap_print_to_pagebuf(true, buf, &pmu_dev->parent->cpu);  in xgene_pmu_cpumask_show()
695 static int get_next_avail_cntr(struct xgene_pmu_dev *pmu_dev)  in get_next_avail_cntr()  argument
699 	cntr = find_first_zero_bit(pmu_dev->cntr_assign_mask,  in get_next_avail_cntr()
700 				pmu_dev->max_counters);  in get_next_avail_cntr()
701 	if (cntr == pmu_dev->max_counters)  in get_next_avail_cntr()
703 	set_bit(cntr, pmu_dev->cntr_assign_mask);  in get_next_avail_cntr()
708 static void clear_avail_cntr(struct xgene_pmu_dev *pmu_dev, int cntr)  in clear_avail_cntr()  argument
710 	clear_bit(cntr, pmu_dev->cntr_assign_mask);  in clear_avail_cntr()
734 static inline u64 xgene_pmu_read_counter32(struct xgene_pmu_dev *pmu_dev,  in xgene_pmu_read_counter32()  argument
737 	return readl(pmu_dev->inf->csr + PMU_PMEVCNTR0 + (4 * idx));  in xgene_pmu_read_counter32()
740 static inline u64 xgene_pmu_read_counter64(struct xgene_pmu_dev *pmu_dev,  in xgene_pmu_read_counter64()  argument
752 		hi = xgene_pmu_read_counter32(pmu_dev, 2 * idx + 1);  in xgene_pmu_read_counter64()
753 		lo = xgene_pmu_read_counter32(pmu_dev, 2 * idx);  in xgene_pmu_read_counter64()
754 	} while (hi != xgene_pmu_read_counter32(pmu_dev, 2 * idx + 1));  in xgene_pmu_read_counter64()
760 xgene_pmu_write_counter32(struct xgene_pmu_dev *pmu_dev, int idx, u64 val)  in xgene_pmu_write_counter32()  argument
762 	writel(val, pmu_dev->inf->csr + PMU_PMEVCNTR0 + (4 * idx));  in xgene_pmu_write_counter32()
766 xgene_pmu_write_counter64(struct xgene_pmu_dev *pmu_dev, int idx, u64 val)  in xgene_pmu_write_counter64()  argument
774 	xgene_pmu_write_counter32(pmu_dev, 2 * idx, cnt_lo);  in xgene_pmu_write_counter64()
775 	xgene_pmu_write_counter32(pmu_dev, 2 * idx + 1, cnt_hi);  in xgene_pmu_write_counter64()
779 xgene_pmu_write_evttype(struct xgene_pmu_dev *pmu_dev, int idx, u32 val)  in xgene_pmu_write_evttype()  argument
781 	writel(val, pmu_dev->inf->csr + PMU_PMEVTYPER0 + (4 * idx));  in xgene_pmu_write_evttype()
785 xgene_pmu_write_agentmsk(struct xgene_pmu_dev *pmu_dev, u32 val)  in xgene_pmu_write_agentmsk()  argument
787 	writel(val, pmu_dev->inf->csr + PMU_PMAMR0);  in xgene_pmu_write_agentmsk()
791 xgene_pmu_v3_write_agentmsk(struct xgene_pmu_dev *pmu_dev, u32 val) { }  in xgene_pmu_v3_write_agentmsk()  argument
794 xgene_pmu_write_agent1msk(struct xgene_pmu_dev *pmu_dev, u32 val)  in xgene_pmu_write_agent1msk()  argument
796 	writel(val, pmu_dev->inf->csr + PMU_PMAMR1);  in xgene_pmu_write_agent1msk()
800 xgene_pmu_v3_write_agent1msk(struct xgene_pmu_dev *pmu_dev, u32 val) { }  in xgene_pmu_v3_write_agent1msk()  argument
803 xgene_pmu_enable_counter(struct xgene_pmu_dev *pmu_dev, int idx)  in xgene_pmu_enable_counter()  argument
807 	val = readl(pmu_dev->inf->csr + PMU_PMCNTENSET);  in xgene_pmu_enable_counter()
809 	writel(val, pmu_dev->inf->csr + PMU_PMCNTENSET);  in xgene_pmu_enable_counter()
813 xgene_pmu_disable_counter(struct xgene_pmu_dev *pmu_dev, int idx)  in xgene_pmu_disable_counter()  argument
817 	val = readl(pmu_dev->inf->csr + PMU_PMCNTENCLR);  in xgene_pmu_disable_counter()
819 	writel(val, pmu_dev->inf->csr + PMU_PMCNTENCLR);  in xgene_pmu_disable_counter()
823 xgene_pmu_enable_counter_int(struct xgene_pmu_dev *pmu_dev, int idx)  in xgene_pmu_enable_counter_int()  argument
827 	val = readl(pmu_dev->inf->csr + PMU_PMINTENSET);  in xgene_pmu_enable_counter_int()
829 	writel(val, pmu_dev->inf->csr + PMU_PMINTENSET);  in xgene_pmu_enable_counter_int()
833 xgene_pmu_disable_counter_int(struct xgene_pmu_dev *pmu_dev, int idx)  in xgene_pmu_disable_counter_int()  argument
837 	val = readl(pmu_dev->inf->csr + PMU_PMINTENCLR);  in xgene_pmu_disable_counter_int()
839 	writel(val, pmu_dev->inf->csr + PMU_PMINTENCLR);  in xgene_pmu_disable_counter_int()
842 static inline void xgene_pmu_reset_counters(struct xgene_pmu_dev *pmu_dev)  in xgene_pmu_reset_counters()  argument
846 	val = readl(pmu_dev->inf->csr + PMU_PMCR);  in xgene_pmu_reset_counters()
848 	writel(val, pmu_dev->inf->csr + PMU_PMCR);  in xgene_pmu_reset_counters()
851 static inline void xgene_pmu_start_counters(struct xgene_pmu_dev *pmu_dev)  in xgene_pmu_start_counters()  argument
855 	val = readl(pmu_dev->inf->csr + PMU_PMCR);  in xgene_pmu_start_counters()
857 	writel(val, pmu_dev->inf->csr + PMU_PMCR);  in xgene_pmu_start_counters()
860 static inline void xgene_pmu_stop_counters(struct xgene_pmu_dev *pmu_dev)  in xgene_pmu_stop_counters()  argument
864 	val = readl(pmu_dev->inf->csr + PMU_PMCR);  in xgene_pmu_stop_counters()
866 	writel(val, pmu_dev->inf->csr + PMU_PMCR);  in xgene_pmu_stop_counters()
871 	struct xgene_pmu_dev *pmu_dev = to_pmu_dev(pmu);  in xgene_perf_pmu_enable()  local
872 	struct xgene_pmu *xgene_pmu = pmu_dev->parent;  in xgene_perf_pmu_enable()
873 	int enabled = bitmap_weight(pmu_dev->cntr_assign_mask,  in xgene_perf_pmu_enable()
874 			pmu_dev->max_counters);  in xgene_perf_pmu_enable()
879 	xgene_pmu->ops->start_counters(pmu_dev);  in xgene_perf_pmu_enable()
884 	struct xgene_pmu_dev *pmu_dev = to_pmu_dev(pmu);  in xgene_perf_pmu_disable()  local
885 	struct xgene_pmu *xgene_pmu = pmu_dev->parent;  in xgene_perf_pmu_disable()
887 	xgene_pmu->ops->stop_counters(pmu_dev);  in xgene_perf_pmu_disable()
892 	struct xgene_pmu_dev *pmu_dev = to_pmu_dev(event->pmu);  in xgene_perf_event_init()  local
919 	event->cpu = cpumask_first(&pmu_dev->parent->cpu);  in xgene_perf_event_init()
949 	struct xgene_pmu_dev *pmu_dev = to_pmu_dev(event->pmu);  in xgene_perf_enable_event()  local
950 	struct xgene_pmu *xgene_pmu = pmu_dev->parent;  in xgene_perf_enable_event()
952 	xgene_pmu->ops->write_evttype(pmu_dev, GET_CNTR(event),  in xgene_perf_enable_event()
954 	xgene_pmu->ops->write_agentmsk(pmu_dev, ~((u32)GET_AGENTID(event)));  in xgene_perf_enable_event()
955 	if (pmu_dev->inf->type == PMU_TYPE_IOB)  in xgene_perf_enable_event()
956 		xgene_pmu->ops->write_agent1msk(pmu_dev,  in xgene_perf_enable_event()
959 	xgene_pmu->ops->enable_counter(pmu_dev, GET_CNTR(event));  in xgene_perf_enable_event()
960 	xgene_pmu->ops->enable_counter_int(pmu_dev, GET_CNTR(event));  in xgene_perf_enable_event()
965 	struct xgene_pmu_dev *pmu_dev = to_pmu_dev(event->pmu);  in xgene_perf_disable_event()  local
966 	struct xgene_pmu *xgene_pmu = pmu_dev->parent;  in xgene_perf_disable_event()
968 	xgene_pmu->ops->disable_counter(pmu_dev, GET_CNTR(event));  in xgene_perf_disable_event()
969 	xgene_pmu->ops->disable_counter_int(pmu_dev, GET_CNTR(event));  in xgene_perf_disable_event()
974 	struct xgene_pmu_dev *pmu_dev = to_pmu_dev(event->pmu);  in xgene_perf_event_set_period()  local
975 	struct xgene_pmu *xgene_pmu = pmu_dev->parent;  in xgene_perf_event_set_period()
987 	xgene_pmu->ops->write_counter(pmu_dev, hw->idx, val);  in xgene_perf_event_set_period()
992 	struct xgene_pmu_dev *pmu_dev = to_pmu_dev(event->pmu);  in xgene_perf_event_update()  local
993 	struct xgene_pmu *xgene_pmu = pmu_dev->parent;  in xgene_perf_event_update()
999 	new_raw_count = xgene_pmu->ops->read_counter(pmu_dev, GET_CNTR(event));  in xgene_perf_event_update()
1005 	delta = (new_raw_count - prev_raw_count) & pmu_dev->max_period;  in xgene_perf_event_update()
1017 	struct xgene_pmu_dev *pmu_dev = to_pmu_dev(event->pmu);  in xgene_perf_start()  local
1018 	struct xgene_pmu *xgene_pmu = pmu_dev->parent;  in xgene_perf_start()
1032 		xgene_pmu->ops->write_counter(pmu_dev, GET_CNTR(event),  in xgene_perf_start()
1060 	struct xgene_pmu_dev *pmu_dev = to_pmu_dev(event->pmu);  in xgene_perf_add()  local
1066 	hw->idx = get_next_avail_cntr(pmu_dev);  in xgene_perf_add()
1071 	pmu_dev->pmu_counter_event[hw->idx] = event;  in xgene_perf_add()
1081 	struct xgene_pmu_dev *pmu_dev = to_pmu_dev(event->pmu);  in xgene_perf_del()  local
1087 	clear_avail_cntr(pmu_dev, GET_CNTR(event));  in xgene_perf_del()
1090 	pmu_dev->pmu_counter_event[hw->idx] = NULL;  in xgene_perf_del()
1093 static int xgene_init_perf(struct xgene_pmu_dev *pmu_dev, char *name)  in xgene_init_perf()  argument
1097 	if (pmu_dev->parent->version == PCP_PMU_V3)  in xgene_init_perf()
1098 		pmu_dev->max_period = PMU_V3_CNT_MAX_PERIOD;  in xgene_init_perf()
1100 		pmu_dev->max_period = PMU_CNT_MAX_PERIOD;  in xgene_init_perf()
1102 	xgene_pmu = pmu_dev->parent;  in xgene_init_perf()
1104 		pmu_dev->max_counters = 1;  in xgene_init_perf()
1106 		pmu_dev->max_counters = PMU_MAX_COUNTERS;  in xgene_init_perf()
1109 	pmu_dev->pmu = (struct pmu) {  in xgene_init_perf()
1110 		.attr_groups	= pmu_dev->attr_groups,  in xgene_init_perf()
1124 	xgene_pmu->ops->stop_counters(pmu_dev);  in xgene_init_perf()
1125 	xgene_pmu->ops->reset_counters(pmu_dev);  in xgene_init_perf()
1127 	return perf_pmu_register(&pmu_dev->pmu, name, -1);  in xgene_init_perf()
1141 	ctx->pmu_dev = pmu;  in xgene_pmu_dev_add()
1192 static void _xgene_pmu_isr(int irq, struct xgene_pmu_dev *pmu_dev)  in _xgene_pmu_isr()  argument
1194 	struct xgene_pmu *xgene_pmu = pmu_dev->parent;  in _xgene_pmu_isr()
1195 	void __iomem *csr = pmu_dev->inf->csr;  in _xgene_pmu_isr()
1199 	xgene_pmu->ops->stop_counters(pmu_dev);  in _xgene_pmu_isr()
1218 		struct perf_event *event = pmu_dev->pmu_counter_event[idx];  in _xgene_pmu_isr()
1229 	xgene_pmu->ops->start_counters(pmu_dev);  in _xgene_pmu_isr()
1257 			_xgene_pmu_isr(irq, ctx->pmu_dev);  in xgene_pmu_isr()
1262 			_xgene_pmu_isr(irq, ctx->pmu_dev);  in xgene_pmu_isr()
1267 			_xgene_pmu_isr(irq, ctx->pmu_dev);  in xgene_pmu_isr()
1272 			_xgene_pmu_isr(irq, ctx->pmu_dev);  in xgene_pmu_isr()
1818 		perf_pmu_migrate_context(&ctx->pmu_dev->pmu, cpu, target);  in xgene_pmu_offline_cpu()
1821 		perf_pmu_migrate_context(&ctx->pmu_dev->pmu, cpu, target);  in xgene_pmu_offline_cpu()
1824 		perf_pmu_migrate_context(&ctx->pmu_dev->pmu, cpu, target);  in xgene_pmu_offline_cpu()
1827 		perf_pmu_migrate_context(&ctx->pmu_dev->pmu, cpu, target);  in xgene_pmu_offline_cpu()
1955 		perf_pmu_unregister(&ctx->pmu_dev->pmu);  in xgene_pmu_dev_cleanup()