287 static int intel_pt_get_buffer(struct intel_pt_queue *ptq,  in intel_pt_get_buffer()  argument
295 		int fd = perf_data__fd(ptq->pt->session->data);  in intel_pt_get_buffer()
302 	might_overlap = ptq->pt->snapshot_mode || ptq->pt->sampling_mode;  in intel_pt_get_buffer()
304 	    intel_pt_do_fix_overlap(ptq->pt, old_buffer, buffer))  in intel_pt_get_buffer()
327 static void intel_pt_lookahead_drop_buffer(struct intel_pt_queue *ptq,  in intel_pt_lookahead_drop_buffer()  argument
330 	if (!buffer || buffer == ptq->buffer || buffer == ptq->old_buffer)  in intel_pt_lookahead_drop_buffer()
340 	struct intel_pt_queue *ptq = data;  in intel_pt_lookahead()  local
341 	struct auxtrace_buffer *buffer = ptq->buffer;  in intel_pt_lookahead()
342 	struct auxtrace_buffer *old_buffer = ptq->old_buffer;  in intel_pt_lookahead()
346 	queue = &ptq->pt->queues.queue_array[ptq->queue_nr];  in intel_pt_lookahead()
355 		err = intel_pt_get_buffer(ptq, buffer, old_buffer, &b);  in intel_pt_lookahead()
360 			intel_pt_lookahead_drop_buffer(ptq, old_buffer);  in intel_pt_lookahead()
363 			intel_pt_lookahead_drop_buffer(ptq, buffer);  in intel_pt_lookahead()
373 		intel_pt_lookahead_drop_buffer(ptq, buffer);  in intel_pt_lookahead()
374 	intel_pt_lookahead_drop_buffer(ptq, old_buffer);  in intel_pt_lookahead()
385 	struct intel_pt_queue *ptq = data;  in intel_pt_get_trace()  local
386 	struct auxtrace_buffer *buffer = ptq->buffer;  in intel_pt_get_trace()
387 	struct auxtrace_buffer *old_buffer = ptq->old_buffer;  in intel_pt_get_trace()
391 	if (ptq->stop) {  in intel_pt_get_trace()
396 	queue = &ptq->pt->queues.queue_array[ptq->queue_nr];  in intel_pt_get_trace()
406 	ptq->buffer = buffer;  in intel_pt_get_trace()
408 	err = intel_pt_get_buffer(ptq, buffer, old_buffer, b);  in intel_pt_get_trace()
412 	if (ptq->step_through_buffers)  in intel_pt_get_trace()
413 		ptq->stop = true;  in intel_pt_get_trace()
418 		ptq->old_buffer = buffer;  in intel_pt_get_trace()
564 	struct intel_pt_queue *ptq = data;  in intel_pt_walk_next_insn()  local
565 	struct machine *machine = ptq->pt->machine;  in intel_pt_walk_next_insn()
581 	cpumode = intel_pt_cpumode(ptq->pt, *ip);  in intel_pt_walk_next_insn()
583 	thread = ptq->thread;  in intel_pt_walk_next_insn()
587 		thread = ptq->pt->unknown_thread;  in intel_pt_walk_next_insn()
731 	struct intel_pt_queue *ptq = data;  in __intel_pt_pgd_ip()  local
737 	if (ip >= ptq->pt->kernel_start)  in __intel_pt_pgd_ip()
738 		return intel_pt_match_pgd_ip(ptq->pt, ip, ip, NULL);  in __intel_pt_pgd_ip()
742 	thread = ptq->thread;  in __intel_pt_pgd_ip()
751 	return intel_pt_match_pgd_ip(ptq->pt, ip, offset,  in __intel_pt_pgd_ip()
1005 	struct intel_pt_queue *ptq;  in intel_pt_alloc_queue()  local
1007 	ptq = zalloc(sizeof(struct intel_pt_queue));  in intel_pt_alloc_queue()
1008 	if (!ptq)  in intel_pt_alloc_queue()
1012 		ptq->chain = intel_pt_alloc_chain(pt);  in intel_pt_alloc_queue()
1013 		if (!ptq->chain)  in intel_pt_alloc_queue()
1020 		ptq->last_branch = intel_pt_alloc_br_stack(entry_cnt);  in intel_pt_alloc_queue()
1021 		if (!ptq->last_branch)  in intel_pt_alloc_queue()
1025 	ptq->event_buf = malloc(PERF_SAMPLE_MAX_SIZE);  in intel_pt_alloc_queue()
1026 	if (!ptq->event_buf)  in intel_pt_alloc_queue()
1029 	ptq->pt = pt;  in intel_pt_alloc_queue()
1030 	ptq->queue_nr = queue_nr;  in intel_pt_alloc_queue()
1031 	ptq->exclude_kernel = intel_pt_exclude_kernel(pt);  in intel_pt_alloc_queue()
1032 	ptq->pid = -1;  in intel_pt_alloc_queue()
1033 	ptq->tid = -1;  in intel_pt_alloc_queue()
1034 	ptq->cpu = -1;  in intel_pt_alloc_queue()
1035 	ptq->next_tid = -1;  in intel_pt_alloc_queue()
1040 	params.data = ptq;  in intel_pt_alloc_queue()
1084 	ptq->decoder = intel_pt_decoder_new(&params);  in intel_pt_alloc_queue()
1085 	if (!ptq->decoder)  in intel_pt_alloc_queue()
1088 	return ptq;  in intel_pt_alloc_queue()
1091 	zfree(&ptq->event_buf);  in intel_pt_alloc_queue()
1092 	zfree(&ptq->last_branch);  in intel_pt_alloc_queue()
1093 	zfree(&ptq->chain);  in intel_pt_alloc_queue()
1094 	free(ptq);  in intel_pt_alloc_queue()
1100 	struct intel_pt_queue *ptq = priv;  in intel_pt_free_queue()  local
1102 	if (!ptq)  in intel_pt_free_queue()
1104 	thread__zput(ptq->thread);  in intel_pt_free_queue()
1105 	intel_pt_decoder_free(ptq->decoder);  in intel_pt_free_queue()
1106 	zfree(&ptq->event_buf);  in intel_pt_free_queue()
1107 	zfree(&ptq->last_branch);  in intel_pt_free_queue()
1108 	zfree(&ptq->chain);  in intel_pt_free_queue()
1109 	free(ptq);  in intel_pt_free_queue()
1115 	struct intel_pt_queue *ptq = queue->priv;  in intel_pt_set_pid_tid_cpu()  local
1118 		ptq->tid = machine__get_current_tid(pt->machine, ptq->cpu);  in intel_pt_set_pid_tid_cpu()
1119 		if (ptq->tid == -1)  in intel_pt_set_pid_tid_cpu()
1120 			ptq->pid = -1;  in intel_pt_set_pid_tid_cpu()
1121 		thread__zput(ptq->thread);  in intel_pt_set_pid_tid_cpu()
1124 	if (!ptq->thread && ptq->tid != -1)  in intel_pt_set_pid_tid_cpu()
1125 		ptq->thread = machine__find_thread(pt->machine, -1, ptq->tid);  in intel_pt_set_pid_tid_cpu()
1127 	if (ptq->thread) {  in intel_pt_set_pid_tid_cpu()
1128 		ptq->pid = ptq->thread->pid_;  in intel_pt_set_pid_tid_cpu()
1130 			ptq->cpu = ptq->thread->cpu;  in intel_pt_set_pid_tid_cpu()
1134 static void intel_pt_sample_flags(struct intel_pt_queue *ptq)  in intel_pt_sample_flags()  argument
1136 	ptq->insn_len = 0;  in intel_pt_sample_flags()
1137 	if (ptq->state->flags & INTEL_PT_ABORT_TX) {  in intel_pt_sample_flags()
1138 		ptq->flags = PERF_IP_FLAG_BRANCH | PERF_IP_FLAG_TX_ABORT;  in intel_pt_sample_flags()
1139 	} else if (ptq->state->flags & INTEL_PT_ASYNC) {  in intel_pt_sample_flags()
1140 		if (ptq->state->to_ip)  in intel_pt_sample_flags()
1141 			ptq->flags = PERF_IP_FLAG_BRANCH | PERF_IP_FLAG_CALL |  in intel_pt_sample_flags()
1145 			ptq->flags = PERF_IP_FLAG_BRANCH |  in intel_pt_sample_flags()
1147 		ptq->insn_len = 0;  in intel_pt_sample_flags()
1149 		if (ptq->state->from_ip)  in intel_pt_sample_flags()
1150 			ptq->flags = intel_pt_insn_type(ptq->state->insn_op);  in intel_pt_sample_flags()
1152 			ptq->flags = PERF_IP_FLAG_BRANCH |  in intel_pt_sample_flags()
1154 		if (ptq->state->flags & INTEL_PT_IN_TX)  in intel_pt_sample_flags()
1155 			ptq->flags |= PERF_IP_FLAG_IN_TX;  in intel_pt_sample_flags()
1156 		ptq->insn_len = ptq->state->insn_len;  in intel_pt_sample_flags()
1157 		memcpy(ptq->insn, ptq->state->insn, INTEL_PT_INSN_BUF_SZ);  in intel_pt_sample_flags()
1160 	if (ptq->state->type & INTEL_PT_TRACE_BEGIN)  in intel_pt_sample_flags()
1161 		ptq->flags |= PERF_IP_FLAG_TRACE_BEGIN;  in intel_pt_sample_flags()
1162 	if (ptq->state->type & INTEL_PT_TRACE_END)  in intel_pt_sample_flags()
1163 		ptq->flags |= PERF_IP_FLAG_TRACE_END;  in intel_pt_sample_flags()
1167 				      struct intel_pt_queue *ptq)  in intel_pt_setup_time_range()  argument
1172 	ptq->sel_timestamp = pt->time_ranges[0].start;  in intel_pt_setup_time_range()
1173 	ptq->sel_idx = 0;  in intel_pt_setup_time_range()
1175 	if (ptq->sel_timestamp) {  in intel_pt_setup_time_range()
1176 		ptq->sel_start = true;  in intel_pt_setup_time_range()
1178 		ptq->sel_timestamp = pt->time_ranges[0].end;  in intel_pt_setup_time_range()
1179 		ptq->sel_start = false;  in intel_pt_setup_time_range()
1187 	struct intel_pt_queue *ptq = queue->priv;  in intel_pt_setup_queue()  local
1192 	if (!ptq) {  in intel_pt_setup_queue()
1193 		ptq = intel_pt_alloc_queue(pt, queue_nr);  in intel_pt_setup_queue()
1194 		if (!ptq)  in intel_pt_setup_queue()
1196 		queue->priv = ptq;  in intel_pt_setup_queue()
1199 			ptq->cpu = queue->cpu;  in intel_pt_setup_queue()
1200 		ptq->tid = queue->tid;  in intel_pt_setup_queue()
1202 		ptq->cbr_seen = UINT_MAX;  in intel_pt_setup_queue()
1206 			ptq->step_through_buffers = true;  in intel_pt_setup_queue()
1208 		ptq->sync_switch = pt->sync_switch;  in intel_pt_setup_queue()
1210 		intel_pt_setup_time_range(pt, ptq);  in intel_pt_setup_queue()
1213 	if (!ptq->on_heap &&  in intel_pt_setup_queue()
1214 	    (!ptq->sync_switch ||  in intel_pt_setup_queue()
1215 	     ptq->switch_state != INTEL_PT_SS_EXPECTING_SWITCH_EVENT)) {  in intel_pt_setup_queue()
1224 			     queue_nr, ptq->cpu, ptq->pid, ptq->tid);  in intel_pt_setup_queue()
1226 		if (ptq->sel_start && ptq->sel_timestamp) {  in intel_pt_setup_queue()
1227 			ret = intel_pt_fast_forward(ptq->decoder,  in intel_pt_setup_queue()
1228 						    ptq->sel_timestamp);  in intel_pt_setup_queue()
1234 			state = intel_pt_decode(ptq->decoder);  in intel_pt_setup_queue()
1247 		ptq->timestamp = state->timestamp;  in intel_pt_setup_queue()
1249 			     queue_nr, ptq->timestamp);  in intel_pt_setup_queue()
1250 		ptq->state = state;  in intel_pt_setup_queue()
1251 		ptq->have_sample = true;  in intel_pt_setup_queue()
1252 		if (ptq->sel_start && ptq->sel_timestamp &&  in intel_pt_setup_queue()
1253 		    ptq->timestamp < ptq->sel_timestamp)  in intel_pt_setup_queue()
1254 			ptq->have_sample = false;  in intel_pt_setup_queue()
1255 		intel_pt_sample_flags(ptq);  in intel_pt_setup_queue()
1256 		ret = auxtrace_heap__add(&pt->heap, queue_nr, ptq->timestamp);  in intel_pt_setup_queue()
1259 		ptq->on_heap = true;  in intel_pt_setup_queue()
1295 static void intel_pt_prep_a_sample(struct intel_pt_queue *ptq,  in intel_pt_prep_a_sample()  argument
1302 	sample->pid = ptq->pid;  in intel_pt_prep_a_sample()
1303 	sample->tid = ptq->tid;  in intel_pt_prep_a_sample()
1304 	sample->cpu = ptq->cpu;  in intel_pt_prep_a_sample()
1305 	sample->insn_len = ptq->insn_len;  in intel_pt_prep_a_sample()
1306 	memcpy(sample->insn, ptq->insn, INTEL_PT_INSN_BUF_SZ);  in intel_pt_prep_a_sample()
1310 				   struct intel_pt_queue *ptq,  in intel_pt_prep_b_sample()  argument
1314 	intel_pt_prep_a_sample(ptq, event, sample);  in intel_pt_prep_b_sample()
1317 		sample->time = tsc_to_perf_time(ptq->timestamp, &pt->tc);  in intel_pt_prep_b_sample()
1319 	sample->ip = ptq->state->from_ip;  in intel_pt_prep_b_sample()
1321 	sample->addr = ptq->state->to_ip;  in intel_pt_prep_b_sample()
1323 	sample->flags = ptq->flags;  in intel_pt_prep_b_sample()
1362 static int intel_pt_synth_branch_sample(struct intel_pt_queue *ptq)  in intel_pt_synth_branch_sample()  argument
1364 	struct intel_pt *pt = ptq->pt;  in intel_pt_synth_branch_sample()
1365 	union perf_event *event = ptq->event_buf;  in intel_pt_synth_branch_sample()
1373 	if (pt->branches_filter && !(pt->branches_filter & ptq->flags))  in intel_pt_synth_branch_sample()
1379 	intel_pt_prep_b_sample(pt, ptq, event, &sample);  in intel_pt_synth_branch_sample()
1381 	sample.id = ptq->pt->branches_id;  in intel_pt_synth_branch_sample()
1382 	sample.stream_id = ptq->pt->branches_id;  in intel_pt_synth_branch_sample()
1400 	if (ptq->state->flags & INTEL_PT_SAMPLE_IPC)  in intel_pt_synth_branch_sample()
1401 		sample.cyc_cnt = ptq->ipc_cyc_cnt - ptq->last_br_cyc_cnt;  in intel_pt_synth_branch_sample()
1403 		sample.insn_cnt = ptq->ipc_insn_cnt - ptq->last_br_insn_cnt;  in intel_pt_synth_branch_sample()
1404 		ptq->last_br_insn_cnt = ptq->ipc_insn_cnt;  in intel_pt_synth_branch_sample()
1405 		ptq->last_br_cyc_cnt = ptq->ipc_cyc_cnt;  in intel_pt_synth_branch_sample()
1413 				 struct intel_pt_queue *ptq,  in intel_pt_prep_sample()  argument
1417 	intel_pt_prep_b_sample(pt, ptq, event, sample);  in intel_pt_prep_sample()
1420 		thread_stack__sample(ptq->thread, ptq->cpu, ptq->chain,  in intel_pt_prep_sample()
1423 		sample->callchain = ptq->chain;  in intel_pt_prep_sample()
1427 		thread_stack__br_sample(ptq->thread, ptq->cpu, ptq->last_branch,  in intel_pt_prep_sample()
1429 		sample->branch_stack = ptq->last_branch;  in intel_pt_prep_sample()
1433 static int intel_pt_synth_instruction_sample(struct intel_pt_queue *ptq)  in intel_pt_synth_instruction_sample()  argument
1435 	struct intel_pt *pt = ptq->pt;  in intel_pt_synth_instruction_sample()
1436 	union perf_event *event = ptq->event_buf;  in intel_pt_synth_instruction_sample()
1442 	intel_pt_prep_sample(pt, ptq, event, &sample);  in intel_pt_synth_instruction_sample()
1444 	sample.id = ptq->pt->instructions_id;  in intel_pt_synth_instruction_sample()
1445 	sample.stream_id = ptq->pt->instructions_id;  in intel_pt_synth_instruction_sample()
1449 		sample.period = ptq->state->tot_insn_cnt - ptq->last_insn_cnt;  in intel_pt_synth_instruction_sample()
1451 	if (ptq->state->flags & INTEL_PT_SAMPLE_IPC)  in intel_pt_synth_instruction_sample()
1452 		sample.cyc_cnt = ptq->ipc_cyc_cnt - ptq->last_in_cyc_cnt;  in intel_pt_synth_instruction_sample()
1454 		sample.insn_cnt = ptq->ipc_insn_cnt - ptq->last_in_insn_cnt;  in intel_pt_synth_instruction_sample()
1455 		ptq->last_in_insn_cnt = ptq->ipc_insn_cnt;  in intel_pt_synth_instruction_sample()
1456 		ptq->last_in_cyc_cnt = ptq->ipc_cyc_cnt;  in intel_pt_synth_instruction_sample()
1459 	ptq->last_insn_cnt = ptq->state->tot_insn_cnt;  in intel_pt_synth_instruction_sample()
1465 static int intel_pt_synth_transaction_sample(struct intel_pt_queue *ptq)  in intel_pt_synth_transaction_sample()  argument
1467 	struct intel_pt *pt = ptq->pt;  in intel_pt_synth_transaction_sample()
1468 	union perf_event *event = ptq->event_buf;  in intel_pt_synth_transaction_sample()
1474 	intel_pt_prep_sample(pt, ptq, event, &sample);  in intel_pt_synth_transaction_sample()
1476 	sample.id = ptq->pt->transactions_id;  in intel_pt_synth_transaction_sample()
1477 	sample.stream_id = ptq->pt->transactions_id;  in intel_pt_synth_transaction_sample()
1484 				   struct intel_pt_queue *ptq,  in intel_pt_prep_p_sample()  argument
1488 	intel_pt_prep_sample(pt, ptq, event, sample);  in intel_pt_prep_p_sample()
1498 static int intel_pt_synth_ptwrite_sample(struct intel_pt_queue *ptq)  in intel_pt_synth_ptwrite_sample()  argument
1500 	struct intel_pt *pt = ptq->pt;  in intel_pt_synth_ptwrite_sample()
1501 	union perf_event *event = ptq->event_buf;  in intel_pt_synth_ptwrite_sample()
1508 	intel_pt_prep_p_sample(pt, ptq, event, &sample);  in intel_pt_synth_ptwrite_sample()
1510 	sample.id = ptq->pt->ptwrites_id;  in intel_pt_synth_ptwrite_sample()
1511 	sample.stream_id = ptq->pt->ptwrites_id;  in intel_pt_synth_ptwrite_sample()
1514 	raw.ip = !!(ptq->state->flags & INTEL_PT_FUP_IP);  in intel_pt_synth_ptwrite_sample()
1515 	raw.payload = cpu_to_le64(ptq->state->ptw_payload);  in intel_pt_synth_ptwrite_sample()
1524 static int intel_pt_synth_cbr_sample(struct intel_pt_queue *ptq)  in intel_pt_synth_cbr_sample()  argument
1526 	struct intel_pt *pt = ptq->pt;  in intel_pt_synth_cbr_sample()
1527 	union perf_event *event = ptq->event_buf;  in intel_pt_synth_cbr_sample()
1535 	ptq->cbr_seen = ptq->state->cbr;  in intel_pt_synth_cbr_sample()
1537 	intel_pt_prep_p_sample(pt, ptq, event, &sample);  in intel_pt_synth_cbr_sample()
1539 	sample.id = ptq->pt->cbr_id;  in intel_pt_synth_cbr_sample()
1540 	sample.stream_id = ptq->pt->cbr_id;  in intel_pt_synth_cbr_sample()
1542 	flags = (u16)ptq->state->cbr_payload | (pt->max_non_turbo_ratio << 16);  in intel_pt_synth_cbr_sample()
1554 static int intel_pt_synth_mwait_sample(struct intel_pt_queue *ptq)  in intel_pt_synth_mwait_sample()  argument
1556 	struct intel_pt *pt = ptq->pt;  in intel_pt_synth_mwait_sample()
1557 	union perf_event *event = ptq->event_buf;  in intel_pt_synth_mwait_sample()
1564 	intel_pt_prep_p_sample(pt, ptq, event, &sample);  in intel_pt_synth_mwait_sample()
1566 	sample.id = ptq->pt->mwait_id;  in intel_pt_synth_mwait_sample()
1567 	sample.stream_id = ptq->pt->mwait_id;  in intel_pt_synth_mwait_sample()
1570 	raw.payload = cpu_to_le64(ptq->state->mwait_payload);  in intel_pt_synth_mwait_sample()
1579 static int intel_pt_synth_pwre_sample(struct intel_pt_queue *ptq)  in intel_pt_synth_pwre_sample()  argument
1581 	struct intel_pt *pt = ptq->pt;  in intel_pt_synth_pwre_sample()
1582 	union perf_event *event = ptq->event_buf;  in intel_pt_synth_pwre_sample()
1589 	intel_pt_prep_p_sample(pt, ptq, event, &sample);  in intel_pt_synth_pwre_sample()
1591 	sample.id = ptq->pt->pwre_id;  in intel_pt_synth_pwre_sample()
1592 	sample.stream_id = ptq->pt->pwre_id;  in intel_pt_synth_pwre_sample()
1595 	raw.payload = cpu_to_le64(ptq->state->pwre_payload);  in intel_pt_synth_pwre_sample()
1604 static int intel_pt_synth_exstop_sample(struct intel_pt_queue *ptq)  in intel_pt_synth_exstop_sample()  argument
1606 	struct intel_pt *pt = ptq->pt;  in intel_pt_synth_exstop_sample()
1607 	union perf_event *event = ptq->event_buf;  in intel_pt_synth_exstop_sample()
1614 	intel_pt_prep_p_sample(pt, ptq, event, &sample);  in intel_pt_synth_exstop_sample()
1616 	sample.id = ptq->pt->exstop_id;  in intel_pt_synth_exstop_sample()
1617 	sample.stream_id = ptq->pt->exstop_id;  in intel_pt_synth_exstop_sample()
1620 	raw.ip = !!(ptq->state->flags & INTEL_PT_FUP_IP);  in intel_pt_synth_exstop_sample()
1629 static int intel_pt_synth_pwrx_sample(struct intel_pt_queue *ptq)  in intel_pt_synth_pwrx_sample()  argument
1631 	struct intel_pt *pt = ptq->pt;  in intel_pt_synth_pwrx_sample()
1632 	union perf_event *event = ptq->event_buf;  in intel_pt_synth_pwrx_sample()
1639 	intel_pt_prep_p_sample(pt, ptq, event, &sample);  in intel_pt_synth_pwrx_sample()
1641 	sample.id = ptq->pt->pwrx_id;  in intel_pt_synth_pwrx_sample()
1642 	sample.stream_id = ptq->pt->pwrx_id;  in intel_pt_synth_pwrx_sample()
1645 	raw.payload = cpu_to_le64(ptq->state->pwrx_payload);  in intel_pt_synth_pwrx_sample()
1781 static int intel_pt_synth_pebs_sample(struct intel_pt_queue *ptq)  in intel_pt_synth_pebs_sample()  argument
1783 	const struct intel_pt_blk_items *items = &ptq->state->items;  in intel_pt_synth_pebs_sample()
1785 	union perf_event *event = ptq->event_buf;  in intel_pt_synth_pebs_sample()
1786 	struct intel_pt *pt = ptq->pt;  in intel_pt_synth_pebs_sample()
1796 	intel_pt_prep_a_sample(ptq, event, &sample);  in intel_pt_synth_pebs_sample()
1810 		sample.ip = ptq->state->from_ip;  in intel_pt_synth_pebs_sample()
1813 	cpumode = sample.ip < ptq->pt->kernel_start ?  in intel_pt_synth_pebs_sample()
1827 			timestamp = ptq->timestamp;  in intel_pt_synth_pebs_sample()
1834 		thread_stack__sample(ptq->thread, ptq->cpu, ptq->chain,  in intel_pt_synth_pebs_sample()
1837 		sample.callchain = ptq->chain;  in intel_pt_synth_pebs_sample()
1860 			intel_pt_add_lbrs(ptq->last_branch, items);  in intel_pt_synth_pebs_sample()
1862 			thread_stack__br_sample(ptq->thread, ptq->cpu,  in intel_pt_synth_pebs_sample()
1863 						ptq->last_branch,  in intel_pt_synth_pebs_sample()
1866 			ptq->last_branch->nr = 0;  in intel_pt_synth_pebs_sample()
1868 		sample.branch_stack = ptq->last_branch;  in intel_pt_synth_pebs_sample()
1930 static int intel_ptq_synth_error(struct intel_pt_queue *ptq,  in intel_ptq_synth_error()  argument
1933 	struct intel_pt *pt = ptq->pt;  in intel_ptq_synth_error()
1934 	u64 tm = ptq->timestamp;  in intel_ptq_synth_error()
1938 	return intel_pt_synth_error(pt, state->err, ptq->cpu, ptq->pid,  in intel_ptq_synth_error()
1939 				    ptq->tid, state->from_ip, tm);  in intel_ptq_synth_error()
1942 static int intel_pt_next_tid(struct intel_pt *pt, struct intel_pt_queue *ptq)  in intel_pt_next_tid()  argument
1945 	pid_t tid = ptq->next_tid;  in intel_pt_next_tid()
1951 	intel_pt_log("switch: cpu %d tid %d\n", ptq->cpu, tid);  in intel_pt_next_tid()
1953 	err = machine__set_current_tid(pt->machine, ptq->cpu, -1, tid);  in intel_pt_next_tid()
1955 	queue = &pt->queues.queue_array[ptq->queue_nr];  in intel_pt_next_tid()
1958 	ptq->next_tid = -1;  in intel_pt_next_tid()
1963 static inline bool intel_pt_is_switch_ip(struct intel_pt_queue *ptq, u64 ip)  in intel_pt_is_switch_ip()  argument
1965 	struct intel_pt *pt = ptq->pt;  in intel_pt_is_switch_ip()
1968 	       (ptq->flags & PERF_IP_FLAG_BRANCH) &&  in intel_pt_is_switch_ip()
1969 	       !(ptq->flags & (PERF_IP_FLAG_CONDITIONAL | PERF_IP_FLAG_ASYNC |  in intel_pt_is_switch_ip()
1976 static int intel_pt_sample(struct intel_pt_queue *ptq)  in intel_pt_sample()  argument
1978 	const struct intel_pt_state *state = ptq->state;  in intel_pt_sample()
1979 	struct intel_pt *pt = ptq->pt;  in intel_pt_sample()
1982 	if (!ptq->have_sample)  in intel_pt_sample()
1985 	ptq->have_sample = false;  in intel_pt_sample()
1987 	ptq->ipc_insn_cnt = ptq->state->tot_insn_cnt;  in intel_pt_sample()
1988 	ptq->ipc_cyc_cnt = ptq->state->tot_cyc_cnt;  in intel_pt_sample()
1995 		err = intel_pt_synth_pebs_sample(ptq);  in intel_pt_sample()
2001 		if (ptq->state->cbr != ptq->cbr_seen) {  in intel_pt_sample()
2002 			err = intel_pt_synth_cbr_sample(ptq);  in intel_pt_sample()
2008 				err = intel_pt_synth_mwait_sample(ptq);  in intel_pt_sample()
2013 				err = intel_pt_synth_pwre_sample(ptq);  in intel_pt_sample()
2018 				err = intel_pt_synth_exstop_sample(ptq);  in intel_pt_sample()
2023 				err = intel_pt_synth_pwrx_sample(ptq);  in intel_pt_sample()
2031 		err = intel_pt_synth_instruction_sample(ptq);  in intel_pt_sample()
2037 		err = intel_pt_synth_transaction_sample(ptq);  in intel_pt_sample()
2043 		err = intel_pt_synth_ptwrite_sample(ptq);  in intel_pt_sample()
2052 		thread_stack__event(ptq->thread, ptq->cpu, ptq->flags,  in intel_pt_sample()
2053 				    state->from_ip, state->to_ip, ptq->insn_len,  in intel_pt_sample()
2058 		thread_stack__set_trace_nr(ptq->thread, ptq->cpu, state->trace_nr);  in intel_pt_sample()
2062 		err = intel_pt_synth_branch_sample(ptq);  in intel_pt_sample()
2067 	if (!ptq->sync_switch)  in intel_pt_sample()
2070 	if (intel_pt_is_switch_ip(ptq, state->to_ip)) {  in intel_pt_sample()
2071 		switch (ptq->switch_state) {  in intel_pt_sample()
2075 			err = intel_pt_next_tid(pt, ptq);  in intel_pt_sample()
2078 			ptq->switch_state = INTEL_PT_SS_TRACING;  in intel_pt_sample()
2081 			ptq->switch_state = INTEL_PT_SS_EXPECTING_SWITCH_EVENT;  in intel_pt_sample()
2085 		ptq->switch_state = INTEL_PT_SS_NOT_TRACING;  in intel_pt_sample()
2086 	} else if (ptq->switch_state == INTEL_PT_SS_NOT_TRACING) {  in intel_pt_sample()
2087 		ptq->switch_state = INTEL_PT_SS_UNKNOWN;  in intel_pt_sample()
2088 	} else if (ptq->switch_state == INTEL_PT_SS_UNKNOWN &&  in intel_pt_sample()
2090 		   (ptq->flags & PERF_IP_FLAG_CALL)) {  in intel_pt_sample()
2091 		ptq->switch_state = INTEL_PT_SS_TRACING;  in intel_pt_sample()
2157 		struct intel_pt_queue *ptq = queue->priv;  in intel_pt_enable_sync_switch()  local
2159 		if (ptq)  in intel_pt_enable_sync_switch()
2160 			ptq->sync_switch = true;  in intel_pt_enable_sync_switch()
2168 static bool intel_pt_next_time(struct intel_pt_queue *ptq)  in intel_pt_next_time()  argument
2170 	struct intel_pt *pt = ptq->pt;  in intel_pt_next_time()
2172 	if (ptq->sel_start) {  in intel_pt_next_time()
2174 		ptq->sel_start = false;  in intel_pt_next_time()
2175 		ptq->sel_timestamp = pt->time_ranges[ptq->sel_idx].end;  in intel_pt_next_time()
2177 	} else if (ptq->sel_idx + 1 < pt->range_cnt) {  in intel_pt_next_time()
2179 		ptq->sel_start = true;  in intel_pt_next_time()
2180 		ptq->sel_idx += 1;  in intel_pt_next_time()
2181 		ptq->sel_timestamp = pt->time_ranges[ptq->sel_idx].start;  in intel_pt_next_time()
2189 static int intel_pt_time_filter(struct intel_pt_queue *ptq, u64 *ff_timestamp)  in intel_pt_time_filter()  argument
2194 		if (ptq->sel_start) {  in intel_pt_time_filter()
2195 			if (ptq->timestamp >= ptq->sel_timestamp) {  in intel_pt_time_filter()
2197 				intel_pt_next_time(ptq);  in intel_pt_time_filter()
2198 				if (!ptq->sel_timestamp) {  in intel_pt_time_filter()
2206 			ptq->have_sample = false;  in intel_pt_time_filter()
2207 			if (ptq->sel_timestamp > *ff_timestamp) {  in intel_pt_time_filter()
2208 				if (ptq->sync_switch) {  in intel_pt_time_filter()
2209 					intel_pt_next_tid(ptq->pt, ptq);  in intel_pt_time_filter()
2210 					ptq->switch_state = INTEL_PT_SS_UNKNOWN;  in intel_pt_time_filter()
2212 				*ff_timestamp = ptq->sel_timestamp;  in intel_pt_time_filter()
2213 				err = intel_pt_fast_forward(ptq->decoder,  in intel_pt_time_filter()
2214 							    ptq->sel_timestamp);  in intel_pt_time_filter()
2219 		} else if (ptq->timestamp > ptq->sel_timestamp) {  in intel_pt_time_filter()
2221 			if (!intel_pt_next_time(ptq)) {  in intel_pt_time_filter()
2223 				ptq->have_sample = false;  in intel_pt_time_filter()
2224 				ptq->switch_state = INTEL_PT_SS_NOT_TRACING;  in intel_pt_time_filter()
2236 static int intel_pt_run_decoder(struct intel_pt_queue *ptq, u64 *timestamp)  in intel_pt_run_decoder()  argument
2238 	const struct intel_pt_state *state = ptq->state;  in intel_pt_run_decoder()
2239 	struct intel_pt *pt = ptq->pt;  in intel_pt_run_decoder()
2259 		     ptq->queue_nr, ptq->cpu, ptq->pid, ptq->tid);  in intel_pt_run_decoder()
2261 		err = intel_pt_sample(ptq);  in intel_pt_run_decoder()
2265 		state = intel_pt_decode(ptq->decoder);  in intel_pt_run_decoder()
2269 			if (ptq->sync_switch &&  in intel_pt_run_decoder()
2271 				ptq->sync_switch = false;  in intel_pt_run_decoder()
2272 				intel_pt_next_tid(pt, ptq);  in intel_pt_run_decoder()
2274 			ptq->timestamp = state->est_timestamp;  in intel_pt_run_decoder()
2276 				err = intel_ptq_synth_error(ptq, state);  in intel_pt_run_decoder()
2283 		ptq->state = state;  in intel_pt_run_decoder()
2284 		ptq->have_sample = true;  in intel_pt_run_decoder()
2285 		intel_pt_sample_flags(ptq);  in intel_pt_run_decoder()
2293 			ptq->timestamp = state->est_timestamp;  in intel_pt_run_decoder()
2295 		} else if (ptq->sync_switch &&  in intel_pt_run_decoder()
2296 			   ptq->switch_state == INTEL_PT_SS_UNKNOWN &&  in intel_pt_run_decoder()
2297 			   intel_pt_is_switch_ip(ptq, state->to_ip) &&  in intel_pt_run_decoder()
2298 			   ptq->next_tid == -1) {  in intel_pt_run_decoder()
2301 			ptq->timestamp = state->est_timestamp;  in intel_pt_run_decoder()
2302 		} else if (state->timestamp > ptq->timestamp) {  in intel_pt_run_decoder()
2303 			ptq->timestamp = state->timestamp;  in intel_pt_run_decoder()
2306 		if (ptq->sel_timestamp) {  in intel_pt_run_decoder()
2307 			err = intel_pt_time_filter(ptq, &ff_timestamp);  in intel_pt_run_decoder()
2312 		if (!pt->timeless_decoding && ptq->timestamp >= *timestamp) {  in intel_pt_run_decoder()
2313 			*timestamp = ptq->timestamp;  in intel_pt_run_decoder()
2337 		struct intel_pt_queue *ptq;  in intel_pt_process_queues()  local
2347 		ptq = queue->priv;  in intel_pt_process_queues()
2365 		ret = intel_pt_run_decoder(ptq, &ts);  in intel_pt_process_queues()
2377 			ptq->on_heap = false;  in intel_pt_process_queues()
2393 		struct intel_pt_queue *ptq = queue->priv;  in intel_pt_process_timeless_queues()  local
2395 		if (ptq && (tid == -1 || ptq->tid == tid)) {  in intel_pt_process_timeless_queues()
2396 			ptq->time = time_;  in intel_pt_process_timeless_queues()
2398 			intel_pt_run_decoder(ptq, &ts);  in intel_pt_process_timeless_queues()
2404 static void intel_pt_sample_set_pid_tid_cpu(struct intel_pt_queue *ptq,  in intel_pt_sample_set_pid_tid_cpu()  argument
2408 	struct machine *m = ptq->pt->machine;  in intel_pt_sample_set_pid_tid_cpu()
2410 	ptq->pid = sample->pid;  in intel_pt_sample_set_pid_tid_cpu()
2411 	ptq->tid = sample->tid;  in intel_pt_sample_set_pid_tid_cpu()
2412 	ptq->cpu = queue->cpu;  in intel_pt_sample_set_pid_tid_cpu()
2415 		     ptq->queue_nr, ptq->cpu, ptq->pid, ptq->tid);  in intel_pt_sample_set_pid_tid_cpu()
2417 	thread__zput(ptq->thread);  in intel_pt_sample_set_pid_tid_cpu()
2419 	if (ptq->tid == -1)  in intel_pt_sample_set_pid_tid_cpu()
2422 	if (ptq->pid == -1) {  in intel_pt_sample_set_pid_tid_cpu()
2423 		ptq->thread = machine__find_thread(m, -1, ptq->tid);  in intel_pt_sample_set_pid_tid_cpu()
2424 		if (ptq->thread)  in intel_pt_sample_set_pid_tid_cpu()
2425 			ptq->pid = ptq->thread->pid_;  in intel_pt_sample_set_pid_tid_cpu()
2429 	ptq->thread = machine__findnew_thread(m, ptq->pid, ptq->tid);  in intel_pt_sample_set_pid_tid_cpu()
2436 	struct intel_pt_queue *ptq;  in intel_pt_process_timeless_sample()  local
2443 	ptq = queue->priv;  in intel_pt_process_timeless_sample()
2444 	if (!ptq)  in intel_pt_process_timeless_sample()
2447 	ptq->stop = false;  in intel_pt_process_timeless_sample()
2448 	ptq->time = sample->time;  in intel_pt_process_timeless_sample()
2449 	intel_pt_sample_set_pid_tid_cpu(ptq, queue, sample);  in intel_pt_process_timeless_sample()
2450 	intel_pt_run_decoder(ptq, &ts);  in intel_pt_process_timeless_sample()
2491 	struct intel_pt_queue *ptq;  in intel_pt_sync_switch()  local
2497 	ptq = intel_pt_cpu_to_ptq(pt, cpu);  in intel_pt_sync_switch()
2498 	if (!ptq || !ptq->sync_switch)  in intel_pt_sync_switch()
2501 	switch (ptq->switch_state) {  in intel_pt_sync_switch()
2506 		ptq->next_tid = tid;  in intel_pt_sync_switch()
2507 		ptq->switch_state = INTEL_PT_SS_EXPECTING_SWITCH_IP;  in intel_pt_sync_switch()
2510 		if (!ptq->on_heap) {  in intel_pt_sync_switch()
2511 			ptq->timestamp = perf_time_to_tsc(timestamp,  in intel_pt_sync_switch()
2513 			err = auxtrace_heap__add(&pt->heap, ptq->queue_nr,  in intel_pt_sync_switch()
2514 						 ptq->timestamp);  in intel_pt_sync_switch()
2517 			ptq->on_heap = true;  in intel_pt_sync_switch()
2519 		ptq->switch_state = INTEL_PT_SS_TRACING;  in intel_pt_sync_switch()
2528 	ptq->next_tid = -1;  in intel_pt_sync_switch()
2566 		struct intel_pt_queue *ptq;  in intel_pt_context_switch_in()  local
2568 		ptq = intel_pt_cpu_to_ptq(pt, cpu);  in intel_pt_context_switch_in()
2569 		if (ptq && ptq->sync_switch) {  in intel_pt_context_switch_in()
2570 			ptq->next_tid = -1;  in intel_pt_context_switch_in()
2571 			switch (ptq->switch_state) {  in intel_pt_context_switch_in()
2578 				ptq->switch_state = INTEL_PT_SS_TRACING;  in intel_pt_context_switch_in()