Lines Matching +full:wakeup +full:- +full:config
1 // SPDX-License-Identifier: GPL-2.0-only
4 * Copyright (c) 2013-2014, Intel Corporation.
85 int cpu = event->cpu; in bts_buffer_setup_aux()
86 int node = (cpu == -1) ? cpu : cpu_to_node(cpu); in bts_buffer_setup_aux()
108 buf->nr_pages = nr_pages; in bts_buffer_setup_aux()
109 buf->nr_bufs = nbuf; in bts_buffer_setup_aux()
110 buf->snapshot = overwrite; in bts_buffer_setup_aux()
111 buf->data_pages = pages; in bts_buffer_setup_aux()
112 buf->real_size = size - size % BTS_RECORD_SIZE; in bts_buffer_setup_aux()
114 for (pg = 0, nbuf = 0, offset = 0, pad = 0; nbuf < buf->nr_bufs; nbuf++) { in bts_buffer_setup_aux()
119 buf->buf[nbuf].page = page; in bts_buffer_setup_aux()
120 buf->buf[nbuf].offset = offset; in bts_buffer_setup_aux()
121 buf->buf[nbuf].displacement = (pad ? BTS_RECORD_SIZE - pad : 0); in bts_buffer_setup_aux()
122 buf->buf[nbuf].size = buf_size(page) - buf->buf[nbuf].displacement; in bts_buffer_setup_aux()
123 pad = buf->buf[nbuf].size % BTS_RECORD_SIZE; in bts_buffer_setup_aux()
124 buf->buf[nbuf].size -= pad; in bts_buffer_setup_aux()
140 return buf->buf[idx].offset + buf->buf[idx].displacement; in bts_buffer_offset()
148 struct bts_phys *phys = &buf->buf[buf->cur_buf]; in bts_config_buffer()
149 unsigned long index, thresh = 0, end = phys->size; in bts_config_buffer()
150 struct page *page = phys->page; in bts_config_buffer()
152 index = local_read(&buf->head); in bts_config_buffer()
154 if (!buf->snapshot) { in bts_config_buffer()
155 if (buf->end < phys->offset + buf_size(page)) in bts_config_buffer()
156 end = buf->end - phys->offset - phys->displacement; in bts_config_buffer()
158 index -= phys->offset + phys->displacement; in bts_config_buffer()
160 if (end - index > BTS_SAFETY_MARGIN) in bts_config_buffer()
161 thresh = end - BTS_SAFETY_MARGIN; in bts_config_buffer()
162 else if (end - index > BTS_RECORD_SIZE) in bts_config_buffer()
163 thresh = end - BTS_RECORD_SIZE; in bts_config_buffer()
168 ds->bts_buffer_base = (u64)(long)page_address(page) + phys->displacement; in bts_config_buffer()
169 ds->bts_index = ds->bts_buffer_base + index; in bts_config_buffer()
170 ds->bts_absolute_maximum = ds->bts_buffer_base + end; in bts_config_buffer()
171 ds->bts_interrupt_threshold = !buf->snapshot in bts_config_buffer()
172 ? ds->bts_buffer_base + thresh in bts_config_buffer()
173 : ds->bts_absolute_maximum + BTS_RECORD_SIZE; in bts_config_buffer()
178 unsigned long index = head - phys->offset; in bts_buffer_pad_out()
180 memset(page_address(phys->page) + index, 0, phys->size - index); in bts_buffer_pad_out()
187 struct bts_buffer *buf = perf_get_aux(&bts->handle); in bts_update()
188 unsigned long index = ds->bts_index - ds->bts_buffer_base, old, head; in bts_update()
193 head = index + bts_buffer_offset(buf, buf->cur_buf); in bts_update()
194 old = local_xchg(&buf->head, head); in bts_update()
196 if (!buf->snapshot) { in bts_update()
200 if (ds->bts_index >= ds->bts_absolute_maximum) in bts_update()
201 perf_aux_output_flag(&bts->handle, in bts_update()
208 local_add(head - old, &buf->data_size); in bts_update()
210 local_set(&buf->data_size, head); in bts_update()
220 * - is set when bts::handle::event is valid, that is, between
222 * - is zero otherwise;
223 * - is ordered against bts::handle::event with a compiler barrier.
229 struct bts_buffer *buf = perf_get_aux(&bts->handle); in __bts_event_start()
230 u64 config = 0; in __bts_event_start() local
232 if (!buf->snapshot) in __bts_event_start()
233 config |= ARCH_PERFMON_EVENTSEL_INT; in __bts_event_start()
234 if (!event->attr.exclude_kernel) in __bts_event_start()
235 config |= ARCH_PERFMON_EVENTSEL_OS; in __bts_event_start()
236 if (!event->attr.exclude_user) in __bts_event_start()
237 config |= ARCH_PERFMON_EVENTSEL_USR; in __bts_event_start()
247 /* INACTIVE/STOPPED -> ACTIVE */ in __bts_event_start()
248 WRITE_ONCE(bts->state, BTS_STATE_ACTIVE); in __bts_event_start()
250 intel_pmu_enable_bts(config); in __bts_event_start()
260 buf = perf_aux_output_begin(&bts->handle, event); in bts_event_start()
264 if (bts_buffer_reset(buf, &bts->handle)) in bts_event_start()
267 bts->ds_back.bts_buffer_base = cpuc->ds->bts_buffer_base; in bts_event_start()
268 bts->ds_back.bts_absolute_maximum = cpuc->ds->bts_absolute_maximum; in bts_event_start()
269 bts->ds_back.bts_interrupt_threshold = cpuc->ds->bts_interrupt_threshold; in bts_event_start()
272 event->hw.state = 0; in bts_event_start()
279 perf_aux_output_end(&bts->handle, 0); in bts_event_start()
282 event->hw.state = PERF_HES_STOPPED; in bts_event_start()
289 /* ACTIVE -> INACTIVE(PMI)/STOPPED(->stop()) */ in __bts_event_stop()
290 WRITE_ONCE(bts->state, state); in __bts_event_stop()
304 int state = READ_ONCE(bts->state); in bts_event_stop()
310 buf = perf_get_aux(&bts->handle); in bts_event_stop()
312 event->hw.state |= PERF_HES_STOPPED; in bts_event_stop()
318 if (buf->snapshot) in bts_event_stop()
319 bts->handle.head = in bts_event_stop()
320 local_xchg(&buf->data_size, in bts_event_stop()
321 buf->nr_pages << PAGE_SHIFT); in bts_event_stop()
322 perf_aux_output_end(&bts->handle, in bts_event_stop()
323 local_xchg(&buf->data_size, 0)); in bts_event_stop()
326 cpuc->ds->bts_index = bts->ds_back.bts_buffer_base; in bts_event_stop()
327 cpuc->ds->bts_buffer_base = bts->ds_back.bts_buffer_base; in bts_event_stop()
328 cpuc->ds->bts_absolute_maximum = bts->ds_back.bts_absolute_maximum; in bts_event_stop()
329 cpuc->ds->bts_interrupt_threshold = bts->ds_back.bts_interrupt_threshold; in bts_event_stop()
336 int state = READ_ONCE(bts->state); in intel_bts_enable_local()
349 if (bts->handle.event) in intel_bts_enable_local()
350 __bts_event_start(bts->handle.event); in intel_bts_enable_local()
361 if (READ_ONCE(bts->state) != BTS_STATE_ACTIVE) in intel_bts_disable_local()
364 if (bts->handle.event) in intel_bts_disable_local()
365 __bts_event_stop(bts->handle.event, BTS_STATE_INACTIVE); in intel_bts_disable_local()
371 unsigned long head, space, next_space, pad, gap, skip, wakeup; in bts_buffer_reset() local
376 if (buf->snapshot) in bts_buffer_reset()
379 head = handle->head & ((buf->nr_pages << PAGE_SHIFT) - 1); in bts_buffer_reset()
381 phys = &buf->buf[buf->cur_buf]; in bts_buffer_reset()
382 space = phys->offset + phys->displacement + phys->size - head; in bts_buffer_reset()
384 if (space > handle->size) { in bts_buffer_reset()
385 space = handle->size; in bts_buffer_reset()
386 space -= space % BTS_RECORD_SIZE; in bts_buffer_reset()
390 next_buf = buf->cur_buf + 1; in bts_buffer_reset()
391 if (next_buf >= buf->nr_bufs) in bts_buffer_reset()
393 next_phys = &buf->buf[next_buf]; in bts_buffer_reset()
394 gap = buf_size(phys->page) - phys->displacement - phys->size + in bts_buffer_reset()
395 next_phys->displacement; in bts_buffer_reset()
397 if (handle->size >= skip) { in bts_buffer_reset()
398 next_space = next_phys->size; in bts_buffer_reset()
399 if (next_space + skip > handle->size) { in bts_buffer_reset()
400 next_space = handle->size - skip; in bts_buffer_reset()
401 next_space -= next_space % BTS_RECORD_SIZE; in bts_buffer_reset()
412 head = phys->offset + phys->displacement; in bts_buffer_reset()
418 buf->cur_buf = next_buf; in bts_buffer_reset()
419 local_set(&buf->head, head); in bts_buffer_reset()
424 /* Don't go far beyond wakeup watermark */ in bts_buffer_reset()
425 wakeup = BTS_SAFETY_MARGIN + BTS_RECORD_SIZE + handle->wakeup - in bts_buffer_reset()
426 handle->head; in bts_buffer_reset()
427 if (space > wakeup) { in bts_buffer_reset()
428 space = wakeup; in bts_buffer_reset()
429 space -= space % BTS_RECORD_SIZE; in bts_buffer_reset()
432 buf->end = head + space; in bts_buffer_reset()
436 * we hit absolute_maximum - see bts_update() in bts_buffer_reset()
439 return -ENOSPC; in bts_buffer_reset()
446 struct debug_store *ds = this_cpu_ptr(&cpu_hw_events)->ds; in intel_bts_interrupt()
448 struct perf_event *event = bts->handle.event; in intel_bts_interrupt()
451 int err = -ENOSPC, handled = 0; in intel_bts_interrupt()
457 if (ds && (ds->bts_index >= ds->bts_interrupt_threshold)) in intel_bts_interrupt()
464 if (READ_ONCE(bts->state) == BTS_STATE_STOPPED) in intel_bts_interrupt()
467 buf = perf_get_aux(&bts->handle); in intel_bts_interrupt()
476 if (buf->snapshot) in intel_bts_interrupt()
479 old_head = local_read(&buf->head); in intel_bts_interrupt()
483 if (old_head == local_read(&buf->head)) in intel_bts_interrupt()
486 perf_aux_output_end(&bts->handle, local_xchg(&buf->data_size, 0)); in intel_bts_interrupt()
488 buf = perf_aux_output_begin(&bts->handle, event); in intel_bts_interrupt()
490 err = bts_buffer_reset(buf, &bts->handle); in intel_bts_interrupt()
493 WRITE_ONCE(bts->state, BTS_STATE_STOPPED); in intel_bts_interrupt()
501 perf_aux_output_end(&bts->handle, 0); in intel_bts_interrupt()
517 struct hw_perf_event *hwc = &event->hw; in bts_event_add()
519 event->hw.state = PERF_HES_STOPPED; in bts_event_add()
521 if (test_bit(INTEL_PMC_IDX_FIXED_BTS, cpuc->active_mask)) in bts_event_add()
522 return -EBUSY; in bts_event_add()
524 if (bts->handle.event) in bts_event_add()
525 return -EBUSY; in bts_event_add()
529 if (hwc->state & PERF_HES_STOPPED) in bts_event_add()
530 return -EINVAL; in bts_event_add()
546 if (event->attr.type != bts_pmu.type) in bts_event_init()
547 return -ENOENT; in bts_event_init()
553 * to the user in a zero-copy fashion. in bts_event_init()
558 if (event->attr.exclude_kernel) { in bts_event_init()
559 ret = perf_allow_kernel(&event->attr); in bts_event_init()
565 return -EBUSY; in bts_event_init()
573 event->destroy = bts_event_destroy; in bts_event_init()
585 return -ENODEV; in bts_init()
593 * However, since this driver supports per-CPU and per-task inherit in bts_init()
602 return -ENODEV; in bts_init()
617 return perf_pmu_register(&bts_pmu, "intel_bts", -1); in bts_init()