173 static void intel_pmu_lbr_filter(struct cpu_hw_events *cpuc);
190 	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);  in __intel_pmu_lbr_enable()  local
204 	if (cpuc->lbr_sel)  in __intel_pmu_lbr_enable()
205 		lbr_select = cpuc->lbr_sel->config & x86_pmu.lbr_sel_mask;  in __intel_pmu_lbr_enable()
206 	if (!static_cpu_has(X86_FEATURE_ARCH_LBR) && !pmi && cpuc->lbr_sel)  in __intel_pmu_lbr_enable()
273 	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);  in intel_pmu_lbr_reset()  local
280 	cpuc->last_task_ctx = NULL;  in intel_pmu_lbr_reset()
281 	cpuc->last_log_id = 0;  in intel_pmu_lbr_reset()
443 	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);  in intel_pmu_lbr_restore()  local
465 	if (cpuc->lbr_select)  in intel_pmu_lbr_restore()
507 	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);  in __intel_pmu_lbr_restore()  local
520 	if ((ctx == cpuc->last_task_ctx) &&  in __intel_pmu_lbr_restore()
521 	    (task_context_opt(ctx)->log_id == cpuc->last_log_id) &&  in __intel_pmu_lbr_restore()
535 	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);  in intel_pmu_lbr_save()  local
551 	if (cpuc->lbr_select)  in intel_pmu_lbr_save()
584 	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);  in __intel_pmu_lbr_save()  local
595 	cpuc->last_task_ctx = ctx;  in __intel_pmu_lbr_save()
596 	cpuc->last_log_id = ++task_context_opt(ctx)->log_id;  in __intel_pmu_lbr_save()
624 	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);  in intel_pmu_lbr_sched_task()  local
627 	if (!cpuc->lbr_users)  in intel_pmu_lbr_sched_task()
661 	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);  in intel_pmu_lbr_add()  local
667 		cpuc->lbr_select = 1;  in intel_pmu_lbr_add()
669 	cpuc->br_sel = event->hw.branch_reg.reg;  in intel_pmu_lbr_add()
671 	if (branch_user_callstack(cpuc->br_sel) && event->ctx->task_ctx_data)  in intel_pmu_lbr_add()
694 		cpuc->lbr_pebs_users++;  in intel_pmu_lbr_add()
696 	if (!cpuc->lbr_users++ && !event->total_time_running)  in intel_pmu_lbr_add()
703 	struct cpu_hw_events *cpuc;  in release_lbr_buffers()  local
710 		cpuc = per_cpu_ptr(&cpu_hw_events, cpu);  in release_lbr_buffers()
712 		if (kmem_cache && cpuc->lbr_xsave) {  in release_lbr_buffers()
713 			kmem_cache_free(kmem_cache, cpuc->lbr_xsave);  in release_lbr_buffers()
714 			cpuc->lbr_xsave = NULL;  in release_lbr_buffers()
722 	struct cpu_hw_events *cpuc;  in reserve_lbr_buffers()  local
729 		cpuc = per_cpu_ptr(&cpu_hw_events, cpu);  in reserve_lbr_buffers()
731 		if (!kmem_cache || cpuc->lbr_xsave)  in reserve_lbr_buffers()
734 		cpuc->lbr_xsave = kmem_cache_alloc_node(kmem_cache,  in reserve_lbr_buffers()
742 	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);  in intel_pmu_lbr_del()  local
747 	if (branch_user_callstack(cpuc->br_sel) &&  in intel_pmu_lbr_del()
752 		cpuc->lbr_select = 0;  in intel_pmu_lbr_del()
755 		cpuc->lbr_pebs_users--;  in intel_pmu_lbr_del()
756 	cpuc->lbr_users--;  in intel_pmu_lbr_del()
757 	WARN_ON_ONCE(cpuc->lbr_users < 0);  in intel_pmu_lbr_del()
758 	WARN_ON_ONCE(cpuc->lbr_pebs_users < 0);  in intel_pmu_lbr_del()
764 	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);  in vlbr_exclude_host()  local
767 		(unsigned long *)&cpuc->intel_ctrl_guest_mask);  in vlbr_exclude_host()
772 	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);  in intel_pmu_lbr_enable_all()  local
774 	if (cpuc->lbr_users && !vlbr_exclude_host())  in intel_pmu_lbr_enable_all()
780 	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);  in intel_pmu_lbr_disable_all()  local
782 	if (cpuc->lbr_users && !vlbr_exclude_host())  in intel_pmu_lbr_disable_all()
786 void intel_pmu_lbr_read_32(struct cpu_hw_events *cpuc)  in intel_pmu_lbr_read_32()  argument
804 		cpuc->lbr_entries[i].from	= msr_lastbranch.from;  in intel_pmu_lbr_read_32()
805 		cpuc->lbr_entries[i].to		= msr_lastbranch.to;  in intel_pmu_lbr_read_32()
806 		cpuc->lbr_entries[i].mispred	= 0;  in intel_pmu_lbr_read_32()
807 		cpuc->lbr_entries[i].predicted	= 0;  in intel_pmu_lbr_read_32()
808 		cpuc->lbr_entries[i].in_tx	= 0;  in intel_pmu_lbr_read_32()
809 		cpuc->lbr_entries[i].abort	= 0;  in intel_pmu_lbr_read_32()
810 		cpuc->lbr_entries[i].cycles	= 0;  in intel_pmu_lbr_read_32()
811 		cpuc->lbr_entries[i].type	= 0;  in intel_pmu_lbr_read_32()
812 		cpuc->lbr_entries[i].reserved	= 0;  in intel_pmu_lbr_read_32()
814 	cpuc->lbr_stack.nr = i;  in intel_pmu_lbr_read_32()
815 	cpuc->lbr_stack.hw_idx = tos;  in intel_pmu_lbr_read_32()
823 void intel_pmu_lbr_read_64(struct cpu_hw_events *cpuc)  in intel_pmu_lbr_read_64()  argument
833 	if (cpuc->lbr_sel) {  in intel_pmu_lbr_read_64()
834 		need_info = !(cpuc->lbr_sel->config & LBR_NO_INFO);  in intel_pmu_lbr_read_64()
835 		if (cpuc->lbr_sel->config & LBR_CALL_STACK)  in intel_pmu_lbr_read_64()
899 		cpuc->lbr_entries[out].from	 = from;  in intel_pmu_lbr_read_64()
900 		cpuc->lbr_entries[out].to	 = to;  in intel_pmu_lbr_read_64()
901 		cpuc->lbr_entries[out].mispred	 = mis;  in intel_pmu_lbr_read_64()
902 		cpuc->lbr_entries[out].predicted = pred;  in intel_pmu_lbr_read_64()
903 		cpuc->lbr_entries[out].in_tx	 = in_tx;  in intel_pmu_lbr_read_64()
904 		cpuc->lbr_entries[out].abort	 = abort;  in intel_pmu_lbr_read_64()
905 		cpuc->lbr_entries[out].cycles	 = cycles;  in intel_pmu_lbr_read_64()
906 		cpuc->lbr_entries[out].type	 = 0;  in intel_pmu_lbr_read_64()
907 		cpuc->lbr_entries[out].reserved	 = 0;  in intel_pmu_lbr_read_64()
910 	cpuc->lbr_stack.nr = out;  in intel_pmu_lbr_read_64()
911 	cpuc->lbr_stack.hw_idx = tos;  in intel_pmu_lbr_read_64()
947 static void intel_pmu_store_lbr(struct cpu_hw_events *cpuc,  in intel_pmu_store_lbr()  argument
957 		e = &cpuc->lbr_entries[i];  in intel_pmu_store_lbr()
980 	cpuc->lbr_stack.nr = i;  in intel_pmu_store_lbr()
983 static void intel_pmu_arch_lbr_read(struct cpu_hw_events *cpuc)  in intel_pmu_arch_lbr_read()  argument
985 	intel_pmu_store_lbr(cpuc, NULL);  in intel_pmu_arch_lbr_read()
988 static void intel_pmu_arch_lbr_read_xsave(struct cpu_hw_events *cpuc)  in intel_pmu_arch_lbr_read_xsave()  argument
990 	struct x86_perf_task_context_arch_lbr_xsave *xsave = cpuc->lbr_xsave;  in intel_pmu_arch_lbr_read_xsave()
993 		intel_pmu_store_lbr(cpuc, NULL);  in intel_pmu_arch_lbr_read_xsave()
998 	intel_pmu_store_lbr(cpuc, xsave->lbr.entries);  in intel_pmu_arch_lbr_read_xsave()
1003 	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);  in intel_pmu_lbr_read()  local
1011 	if (!cpuc->lbr_users || vlbr_exclude_host() ||  in intel_pmu_lbr_read()
1012 	    cpuc->lbr_users == cpuc->lbr_pebs_users)  in intel_pmu_lbr_read()
1015 	x86_pmu.lbr_read(cpuc);  in intel_pmu_lbr_read()
1017 	intel_pmu_lbr_filter(cpuc);  in intel_pmu_lbr_read()
1413 intel_pmu_lbr_filter(struct cpu_hw_events *cpuc)  in intel_pmu_lbr_filter()  argument
1416 	int br_sel = cpuc->br_sel;  in intel_pmu_lbr_filter()
1425 	for (i = 0; i < cpuc->lbr_stack.nr; i++) {  in intel_pmu_lbr_filter()
1427 		from = cpuc->lbr_entries[i].from;  in intel_pmu_lbr_filter()
1428 		to = cpuc->lbr_entries[i].to;  in intel_pmu_lbr_filter()
1429 		type = cpuc->lbr_entries[i].type;  in intel_pmu_lbr_filter()
1441 			type = branch_type(from, to, cpuc->lbr_entries[i].abort);  in intel_pmu_lbr_filter()
1443 			if (cpuc->lbr_entries[i].in_tx)  in intel_pmu_lbr_filter()
1451 			cpuc->lbr_entries[i].from = 0;  in intel_pmu_lbr_filter()
1456 			cpuc->lbr_entries[i].type = common_branch_type(type);  in intel_pmu_lbr_filter()
1463 	for (i = 0; i < cpuc->lbr_stack.nr; ) {  in intel_pmu_lbr_filter()
1464 		if (!cpuc->lbr_entries[i].from) {  in intel_pmu_lbr_filter()
1466 			while (++j < cpuc->lbr_stack.nr)  in intel_pmu_lbr_filter()
1467 				cpuc->lbr_entries[j-1] = cpuc->lbr_entries[j];  in intel_pmu_lbr_filter()
1468 			cpuc->lbr_stack.nr--;  in intel_pmu_lbr_filter()
1469 			if (!cpuc->lbr_entries[i].from)  in intel_pmu_lbr_filter()
1478 	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);  in intel_pmu_store_pebs_lbrs()  local
1482 	    (cpuc->n_pebs == cpuc->n_large_pebs))  in intel_pmu_store_pebs_lbrs()
1483 		cpuc->lbr_stack.hw_idx = -1ULL;  in intel_pmu_store_pebs_lbrs()
1485 		cpuc->lbr_stack.hw_idx = intel_pmu_lbr_tos();  in intel_pmu_store_pebs_lbrs()
1487 	intel_pmu_store_lbr(cpuc, lbr);  in intel_pmu_store_pebs_lbrs()
1488 	intel_pmu_lbr_filter(cpuc);  in intel_pmu_store_pebs_lbrs()