223 	struct cs_etm_traceid_queue *tidq;  in cs_etm__clear_all_packet_queues()  local
228 		tidq = etmq->traceid_queues[idx];  in cs_etm__clear_all_packet_queues()
229 		cs_etm__clear_packet_queue(&tidq->packet_queue);  in cs_etm__clear_all_packet_queues()
234 				      struct cs_etm_traceid_queue *tidq,  in cs_etm__init_traceid_queue()  argument
241 	cs_etm__clear_packet_queue(&tidq->packet_queue);  in cs_etm__init_traceid_queue()
244 	tidq->tid = queue->tid;  in cs_etm__init_traceid_queue()
245 	tidq->pid = -1;  in cs_etm__init_traceid_queue()
246 	tidq->trace_chan_id = trace_chan_id;  in cs_etm__init_traceid_queue()
248 	tidq->packet = zalloc(sizeof(struct cs_etm_packet));  in cs_etm__init_traceid_queue()
249 	if (!tidq->packet)  in cs_etm__init_traceid_queue()
252 	tidq->prev_packet = zalloc(sizeof(struct cs_etm_packet));  in cs_etm__init_traceid_queue()
253 	if (!tidq->prev_packet)  in cs_etm__init_traceid_queue()
261 		tidq->last_branch = zalloc(sz);  in cs_etm__init_traceid_queue()
262 		if (!tidq->last_branch)  in cs_etm__init_traceid_queue()
264 		tidq->last_branch_rb = zalloc(sz);  in cs_etm__init_traceid_queue()
265 		if (!tidq->last_branch_rb)  in cs_etm__init_traceid_queue()
269 	tidq->event_buf = malloc(PERF_SAMPLE_MAX_SIZE);  in cs_etm__init_traceid_queue()
270 	if (!tidq->event_buf)  in cs_etm__init_traceid_queue()
276 	zfree(&tidq->last_branch_rb);  in cs_etm__init_traceid_queue()
277 	zfree(&tidq->last_branch);  in cs_etm__init_traceid_queue()
278 	zfree(&tidq->prev_packet);  in cs_etm__init_traceid_queue()
279 	zfree(&tidq->packet);  in cs_etm__init_traceid_queue()
290 	struct cs_etm_traceid_queue *tidq, **traceid_queues;  in cs_etm__etmq_get_traceid_queue()  local
309 	tidq = malloc(sizeof(*tidq));  in cs_etm__etmq_get_traceid_queue()
310 	if (!tidq)  in cs_etm__etmq_get_traceid_queue()
313 	memset(tidq, 0, sizeof(*tidq));  in cs_etm__etmq_get_traceid_queue()
325 	if (cs_etm__init_traceid_queue(etmq, tidq, trace_chan_id))  in cs_etm__etmq_get_traceid_queue()
341 	traceid_queues[idx] = tidq;  in cs_etm__etmq_get_traceid_queue()
352 	free(tidq);  in cs_etm__etmq_get_traceid_queue()
360 	struct cs_etm_traceid_queue *tidq;  in cs_etm__etmq_get_packet_queue()  local
362 	tidq = cs_etm__etmq_get_traceid_queue(etmq, trace_chan_id);  in cs_etm__etmq_get_packet_queue()
363 	if (tidq)  in cs_etm__etmq_get_packet_queue()
364 		return &tidq->packet_queue;  in cs_etm__etmq_get_packet_queue()
370 				struct cs_etm_traceid_queue *tidq)  in cs_etm__packet_swap()  argument
380 		tmp = tidq->packet;  in cs_etm__packet_swap()
381 		tidq->packet = tidq->prev_packet;  in cs_etm__packet_swap()
382 		tidq->prev_packet = tmp;  in cs_etm__packet_swap()
553 	struct cs_etm_traceid_queue *tidq;  in cs_etm__free_traceid_queues()  local
561 		tidq = etmq->traceid_queues[idx];  in cs_etm__free_traceid_queues()
562 		thread__zput(tidq->thread);  in cs_etm__free_traceid_queues()
563 		zfree(&tidq->event_buf);  in cs_etm__free_traceid_queues()
564 		zfree(&tidq->last_branch);  in cs_etm__free_traceid_queues()
565 		zfree(&tidq->last_branch_rb);  in cs_etm__free_traceid_queues()
566 		zfree(&tidq->prev_packet);  in cs_etm__free_traceid_queues()
567 		zfree(&tidq->packet);  in cs_etm__free_traceid_queues()
568 		zfree(&tidq);  in cs_etm__free_traceid_queues()
677 	struct cs_etm_traceid_queue *tidq;  in cs_etm__mem_access()  local
684 	tidq = cs_etm__etmq_get_traceid_queue(etmq, trace_chan_id);  in cs_etm__mem_access()
685 	if (!tidq)  in cs_etm__mem_access()
688 	thread = tidq->thread;  in cs_etm__mem_access()
887 				 struct cs_etm_traceid_queue *tidq)  in cs_etm__copy_last_branch_rb()  argument
889 	struct branch_stack *bs_src = tidq->last_branch_rb;  in cs_etm__copy_last_branch_rb()
890 	struct branch_stack *bs_dst = tidq->last_branch;  in cs_etm__copy_last_branch_rb()
910 	nr = etmq->etm->synth_opts.last_branch_sz - tidq->last_branch_pos;  in cs_etm__copy_last_branch_rb()
912 	       &bs_src->entries[tidq->last_branch_pos],  in cs_etm__copy_last_branch_rb()
925 		       sizeof(struct branch_entry) * tidq->last_branch_pos);  in cs_etm__copy_last_branch_rb()
930 void cs_etm__reset_last_branch_rb(struct cs_etm_traceid_queue *tidq)  in cs_etm__reset_last_branch_rb()  argument
932 	tidq->last_branch_pos = 0;  in cs_etm__reset_last_branch_rb()
933 	tidq->last_branch_rb->nr = 0;  in cs_etm__reset_last_branch_rb()
991 					  struct cs_etm_traceid_queue *tidq)  in cs_etm__update_last_branch_rb()  argument
993 	struct branch_stack *bs = tidq->last_branch_rb;  in cs_etm__update_last_branch_rb()
1002 	if (!tidq->last_branch_pos)  in cs_etm__update_last_branch_rb()
1003 		tidq->last_branch_pos = etmq->etm->synth_opts.last_branch_sz;  in cs_etm__update_last_branch_rb()
1005 	tidq->last_branch_pos -= 1;  in cs_etm__update_last_branch_rb()
1007 	be       = &bs->entries[tidq->last_branch_pos];  in cs_etm__update_last_branch_rb()
1008 	be->from = cs_etm__last_executed_instr(tidq->prev_packet);  in cs_etm__update_last_branch_rb()
1009 	be->to	 = cs_etm__first_executed_instr(tidq->packet);  in cs_etm__update_last_branch_rb()
1073 				    struct cs_etm_traceid_queue *tidq)  in cs_etm__set_pid_tid_cpu()  argument
1075 	if ((!tidq->thread) && (tidq->tid != -1))  in cs_etm__set_pid_tid_cpu()
1076 		tidq->thread = machine__find_thread(etm->machine, -1,  in cs_etm__set_pid_tid_cpu()
1077 						    tidq->tid);  in cs_etm__set_pid_tid_cpu()
1079 	if (tidq->thread)  in cs_etm__set_pid_tid_cpu()
1080 		tidq->pid = tidq->thread->pid_;  in cs_etm__set_pid_tid_cpu()
1088 	struct cs_etm_traceid_queue *tidq;  in cs_etm__etmq_set_tid()  local
1090 	tidq = cs_etm__etmq_get_traceid_queue(etmq, trace_chan_id);  in cs_etm__etmq_set_tid()
1091 	if (!tidq)  in cs_etm__etmq_set_tid()
1101 	tidq->tid = tid;  in cs_etm__etmq_set_tid()
1102 	thread__zput(tidq->thread);  in cs_etm__etmq_set_tid()
1104 	cs_etm__set_pid_tid_cpu(etm, tidq);  in cs_etm__etmq_set_tid()
1143 					    struct cs_etm_traceid_queue *tidq,  in cs_etm__synth_instruction_sample()  argument
1148 	union perf_event *event = tidq->event_buf;  in cs_etm__synth_instruction_sample()
1156 	sample.pid = tidq->pid;  in cs_etm__synth_instruction_sample()
1157 	sample.tid = tidq->tid;  in cs_etm__synth_instruction_sample()
1161 	sample.cpu = tidq->packet->cpu;  in cs_etm__synth_instruction_sample()
1162 	sample.flags = tidq->prev_packet->flags;  in cs_etm__synth_instruction_sample()
1165 	cs_etm__copy_insn(etmq, tidq->trace_chan_id, tidq->packet, &sample);  in cs_etm__synth_instruction_sample()
1168 		sample.branch_stack = tidq->last_branch;  in cs_etm__synth_instruction_sample()
1192 				       struct cs_etm_traceid_queue *tidq)  in cs_etm__synth_branch_sample()  argument
1197 	union perf_event *event = tidq->event_buf;  in cs_etm__synth_branch_sample()
1205 	ip = cs_etm__last_executed_instr(tidq->prev_packet);  in cs_etm__synth_branch_sample()
1212 	sample.pid = tidq->pid;  in cs_etm__synth_branch_sample()
1213 	sample.tid = tidq->tid;  in cs_etm__synth_branch_sample()
1214 	sample.addr = cs_etm__first_executed_instr(tidq->packet);  in cs_etm__synth_branch_sample()
1218 	sample.cpu = tidq->packet->cpu;  in cs_etm__synth_branch_sample()
1219 	sample.flags = tidq->prev_packet->flags;  in cs_etm__synth_branch_sample()
1222 	cs_etm__copy_insn(etmq, tidq->trace_chan_id, tidq->prev_packet,  in cs_etm__synth_branch_sample()
1374 			  struct cs_etm_traceid_queue *tidq)  in cs_etm__sample()  argument
1378 	u8 trace_chan_id = tidq->trace_chan_id;  in cs_etm__sample()
1382 	instrs_prev = tidq->period_instructions;  in cs_etm__sample()
1384 	tidq->period_instructions += tidq->packet->instr_count;  in cs_etm__sample()
1391 	    tidq->prev_packet->sample_type == CS_ETM_RANGE &&  in cs_etm__sample()
1392 	    tidq->prev_packet->last_instr_taken_branch)  in cs_etm__sample()
1393 		cs_etm__update_last_branch_rb(etmq, tidq);  in cs_etm__sample()
1396 	    tidq->period_instructions >= etm->instructions_sample_period) {  in cs_etm__sample()
1454 			cs_etm__copy_last_branch_rb(etmq, tidq);  in cs_etm__sample()
1456 		while (tidq->period_instructions >=  in cs_etm__sample()
1465 						  tidq->packet, offset - 1);  in cs_etm__sample()
1467 				etmq, tidq, addr,  in cs_etm__sample()
1473 			tidq->period_instructions -=  in cs_etm__sample()
1482 		if (tidq->prev_packet->sample_type == CS_ETM_DISCONTINUITY)  in cs_etm__sample()
1486 		if (tidq->prev_packet->sample_type == CS_ETM_RANGE &&  in cs_etm__sample()
1487 		    tidq->prev_packet->last_instr_taken_branch)  in cs_etm__sample()
1491 			ret = cs_etm__synth_branch_sample(etmq, tidq);  in cs_etm__sample()
1497 	cs_etm__packet_swap(etm, tidq);  in cs_etm__sample()
1502 static int cs_etm__exception(struct cs_etm_traceid_queue *tidq)  in cs_etm__exception()  argument
1515 	if (tidq->prev_packet->sample_type == CS_ETM_RANGE)  in cs_etm__exception()
1516 		tidq->prev_packet->last_instr_taken_branch = true;  in cs_etm__exception()
1522 			 struct cs_etm_traceid_queue *tidq)  in cs_etm__flush()  argument
1528 	if (tidq->prev_packet->sample_type == CS_ETM_EMPTY)  in cs_etm__flush()
1532 	    tidq->prev_packet->sample_type == CS_ETM_RANGE) {  in cs_etm__flush()
1536 		cs_etm__copy_last_branch_rb(etmq, tidq);  in cs_etm__flush()
1545 		addr = cs_etm__last_executed_instr(tidq->prev_packet);  in cs_etm__flush()
1548 			etmq, tidq, addr,  in cs_etm__flush()
1549 			tidq->period_instructions);  in cs_etm__flush()
1553 		tidq->period_instructions = 0;  in cs_etm__flush()
1558 	    tidq->prev_packet->sample_type == CS_ETM_RANGE) {  in cs_etm__flush()
1559 		err = cs_etm__synth_branch_sample(etmq, tidq);  in cs_etm__flush()
1565 	cs_etm__packet_swap(etm, tidq);  in cs_etm__flush()
1569 		cs_etm__reset_last_branch_rb(tidq);  in cs_etm__flush()
1575 			     struct cs_etm_traceid_queue *tidq)  in cs_etm__end_block()  argument
1589 	    tidq->prev_packet->sample_type == CS_ETM_RANGE) {  in cs_etm__end_block()
1593 		cs_etm__copy_last_branch_rb(etmq, tidq);  in cs_etm__end_block()
1599 		addr = cs_etm__last_executed_instr(tidq->prev_packet);  in cs_etm__end_block()
1602 			etmq, tidq, addr,  in cs_etm__end_block()
1603 			tidq->period_instructions);  in cs_etm__end_block()
1607 		tidq->period_instructions = 0;  in cs_etm__end_block()
1711 			       struct cs_etm_traceid_queue *tidq, u64 magic)  in cs_etm__is_syscall()  argument
1713 	u8 trace_chan_id = tidq->trace_chan_id;  in cs_etm__is_syscall()
1714 	struct cs_etm_packet *packet = tidq->packet;  in cs_etm__is_syscall()
1715 	struct cs_etm_packet *prev_packet = tidq->prev_packet;  in cs_etm__is_syscall()
1736 static bool cs_etm__is_async_exception(struct cs_etm_traceid_queue *tidq,  in cs_etm__is_async_exception()  argument
1739 	struct cs_etm_packet *packet = tidq->packet;  in cs_etm__is_async_exception()
1763 				      struct cs_etm_traceid_queue *tidq,  in cs_etm__is_sync_exception()  argument
1766 	u8 trace_chan_id = tidq->trace_chan_id;  in cs_etm__is_sync_exception()
1767 	struct cs_etm_packet *packet = tidq->packet;  in cs_etm__is_sync_exception()
1768 	struct cs_etm_packet *prev_packet = tidq->prev_packet;  in cs_etm__is_sync_exception()
1812 				    struct cs_etm_traceid_queue *tidq)  in cs_etm__set_sample_flags()  argument
1814 	struct cs_etm_packet *packet = tidq->packet;  in cs_etm__set_sample_flags()
1815 	struct cs_etm_packet *prev_packet = tidq->prev_packet;  in cs_etm__set_sample_flags()
1816 	u8 trace_chan_id = tidq->trace_chan_id;  in cs_etm__set_sample_flags()
1919 		if (cs_etm__is_syscall(etmq, tidq, magic))  in cs_etm__set_sample_flags()
1927 		else if (cs_etm__is_async_exception(tidq, magic))  in cs_etm__set_sample_flags()
1936 		else if (cs_etm__is_sync_exception(etmq, tidq, magic))  in cs_etm__set_sample_flags()
2019 					 struct cs_etm_traceid_queue *tidq)  in cs_etm__process_traceid_queue()  argument
2024 	packet_queue = &tidq->packet_queue;  in cs_etm__process_traceid_queue()
2029 						 tidq->packet);  in cs_etm__process_traceid_queue()
2044 		ret = cs_etm__set_sample_flags(etmq, tidq);  in cs_etm__process_traceid_queue()
2048 		switch (tidq->packet->sample_type) {  in cs_etm__process_traceid_queue()
2055 			cs_etm__sample(etmq, tidq);  in cs_etm__process_traceid_queue()
2064 			cs_etm__exception(tidq);  in cs_etm__process_traceid_queue()
2071 			cs_etm__flush(etmq, tidq);  in cs_etm__process_traceid_queue()
2092 	struct cs_etm_traceid_queue *tidq;  in cs_etm__clear_all_traceid_queues()  local
2097 		tidq = etmq->traceid_queues[idx];  in cs_etm__clear_all_traceid_queues()
2100 		cs_etm__process_traceid_queue(etmq, tidq);  in cs_etm__clear_all_traceid_queues()
2106 		cs_etm__flush(etmq, tidq);  in cs_etm__clear_all_traceid_queues()
2113 	struct cs_etm_traceid_queue *tidq;  in cs_etm__run_decoder()  local
2115 	tidq = cs_etm__etmq_get_traceid_queue(etmq, CS_ETM_PER_THREAD_TRACEID);  in cs_etm__run_decoder()
2116 	if (!tidq)  in cs_etm__run_decoder()
2136 			err = cs_etm__process_traceid_queue(etmq, tidq);  in cs_etm__run_decoder()
2142 			err = cs_etm__end_block(etmq, tidq);  in cs_etm__run_decoder()
2157 		struct cs_etm_traceid_queue *tidq;  in cs_etm__process_timeless_queues()  local
2162 		tidq = cs_etm__etmq_get_traceid_queue(etmq,  in cs_etm__process_timeless_queues()
2165 		if (!tidq)  in cs_etm__process_timeless_queues()
2168 		if ((tid == -1) || (tidq->tid == tid)) {  in cs_etm__process_timeless_queues()
2169 			cs_etm__set_pid_tid_cpu(etm, tidq);  in cs_etm__process_timeless_queues()
2185 	struct cs_etm_traceid_queue *tidq;  in cs_etm__process_queues()  local
2204 		tidq  = cs_etm__etmq_get_traceid_queue(etmq, trace_chan_id);  in cs_etm__process_queues()
2205 		if (!tidq) {  in cs_etm__process_queues()
2219 		ret = cs_etm__process_traceid_queue(etmq, tidq);  in cs_etm__process_queues()