1 // SPDX-License-Identifier: BSD-2-Clause 2 /* 3 * Copyright (C) 2019, Linaro Limited 4 */ 5 6 /* 7 * APIs defined in this file are required to use __noprof attribute to 8 * avoid any circular dependency during profiling. So this requirement 9 * prohibits these APIs to use standard library APIs as those can be 10 * profiled too. 11 */ 12 13 #include <assert.h> 14 #include <user_ta_header.h> 15 #if defined(__KERNEL__) 16 #if defined(ARM32) || defined(ARM64) 17 #include <arm.h> 18 #elif defined(RV32) || defined(RV64) 19 #include <riscv.h> 20 #endif 21 #include <kernel/panic.h> 22 #include <kernel/tee_ta_manager.h> 23 #include <kernel/thread.h> 24 #include <mm/core_mmu.h> 25 #else 26 #if defined(ARM32) || defined(ARM64) 27 #include <arm_user_sysreg.h> 28 #elif defined(RV32) || defined(RV64) 29 #include <riscv_user_sysreg.h> 30 #endif 31 #include <setjmp.h> 32 #include <utee_syscalls.h> 33 #endif 34 #include "ftrace.h" 35 36 #define DURATION_MAX_LEN 16 37 #define ENTRY_SIZE(idx) (DURATION_MAX_LEN + (idx) + \ 38 (2 * sizeof(unsigned long)) + 8) 39 #define EXIT_SIZE(idx) (DURATION_MAX_LEN + (idx) + 3) 40 41 static const char hex_str[] = "0123456789abcdef"; 42 43 static __noprof struct ftrace_buf *get_fbuf(void) 44 { 45 #if defined(__KERNEL__) 46 short int ct = thread_get_id_may_fail(); 47 struct ts_session *s = NULL; 48 struct thread_specific_data *tsd = NULL; 49 50 if (ct == -1) 51 return NULL; 52 53 if (!(core_mmu_user_va_range_is_defined() && 54 core_mmu_user_mapping_is_active())) 55 return NULL; 56 57 tsd = thread_get_tsd(); 58 s = TAILQ_FIRST(&tsd->sess_stack); 59 60 if (!s || tsd->ctx != s->ctx) 61 return NULL; 62 63 if (s->fbuf && s->fbuf->syscall_trace_enabled && 64 !s->fbuf->syscall_trace_suspended) 65 return s->fbuf; 66 else 67 return NULL; 68 #else 69 return &__ftrace_buf_start; 70 #endif 71 } 72 73 #if defined(_CFG_FTRACE_BUF_WHEN_FULL_shift) 74 75 /* 76 * This API shifts/moves ftrace buffer to create space for new dump 77 * in case the buffer size falls short of actual dump. 78 */ 79 static bool __noprof fbuf_make_room(struct ftrace_buf *fbuf, size_t size) 80 { 81 char *dst = (char *)fbuf + fbuf->buf_off; 82 const char *src = (char *)fbuf + fbuf->buf_off + size; 83 size_t n = 0; 84 85 fbuf->curr_size -= size; 86 87 for (n = 0; n < fbuf->curr_size; n++) 88 dst[n] = src[n]; 89 90 return true; 91 } 92 93 #elif defined(_CFG_FTRACE_BUF_WHEN_FULL_wrap) 94 95 /* Makes room in the trace buffer by discarding the previously recorded data. */ 96 static bool __noprof fbuf_make_room(struct ftrace_buf *fbuf, 97 size_t size) 98 { 99 if (fbuf->buf_off + size > fbuf->max_size) 100 return false; 101 102 fbuf->curr_size = 0; 103 104 return true; 105 } 106 107 #elif defined(_CFG_FTRACE_BUF_WHEN_FULL_stop) 108 109 static bool __noprof fbuf_make_room(struct ftrace_buf *fbuf __unused, 110 size_t size __unused) 111 { 112 return false; 113 } 114 115 #else 116 #error CFG_FTRACE_BUF_WHEN_FULL value not supported 117 #endif 118 119 static size_t __noprof to_func_enter_fmt(char *buf, uint32_t ret_idx, 120 unsigned long pc) 121 { 122 char *str = buf; 123 uint32_t addr_size = 2 * sizeof(unsigned long); 124 uint32_t i = 0; 125 126 for (i = 0; i < (DURATION_MAX_LEN + ret_idx); i++) 127 if (i == (DURATION_MAX_LEN - 2)) 128 *str++ = '|'; 129 else 130 *str++ = ' '; 131 132 *str++ = '0'; 133 *str++ = 'x'; 134 135 for (i = 0; i < addr_size; i++) 136 *str++ = hex_str[(pc >> 4 * (addr_size - i - 1)) & 0xf]; 137 138 *str++ = '('; 139 *str++ = ')'; 140 *str++ = ' '; 141 *str++ = '{'; 142 *str++ = '\n'; 143 *str = '\0'; 144 145 return str - buf; 146 } 147 148 void __noprof ftrace_enter(unsigned long pc, unsigned long *lr) 149 { 150 struct ftrace_buf *fbuf = NULL; 151 size_t line_size = 0; 152 bool full = false; 153 154 fbuf = get_fbuf(); 155 156 if (!fbuf || !fbuf->buf_off || !fbuf->max_size) 157 return; 158 159 line_size = ENTRY_SIZE(fbuf->ret_idx); 160 161 /* 162 * Check if we have enough space in ftrace buffer. If not then try to 163 * make room. 164 */ 165 full = (fbuf->curr_size + line_size) > fbuf->max_size; 166 if (full) 167 full = !fbuf_make_room(fbuf, line_size); 168 169 if (!full) 170 fbuf->curr_size += to_func_enter_fmt((char *)fbuf + 171 fbuf->buf_off + 172 fbuf->curr_size, 173 fbuf->ret_idx, 174 pc); 175 176 if (fbuf->ret_idx < FTRACE_RETFUNC_DEPTH) { 177 fbuf->ret_stack[fbuf->ret_idx] = *lr; 178 fbuf->begin_time[fbuf->ret_idx] = barrier_read_counter_timer(); 179 fbuf->ret_idx++; 180 } else { 181 /* 182 * This scenario isn't expected as function call depth 183 * shouldn't be more than FTRACE_RETFUNC_DEPTH. 184 */ 185 #if defined(__KERNEL__) 186 panic(); 187 #else 188 _utee_panic(0); 189 #endif 190 } 191 192 *lr = (unsigned long)&__ftrace_return; 193 } 194 195 static void __noprof ftrace_duration(char *buf, uint64_t start, uint64_t end) 196 { 197 uint32_t max_us = CFG_FTRACE_US_MS; 198 uint32_t cntfrq = read_cntfrq(); 199 uint64_t ticks = end - start; 200 uint32_t ms = 0; 201 uint32_t us = 0; 202 uint32_t ns = 0; 203 uint32_t frac = 0; 204 uint32_t in = 0; 205 char unit = 'u'; 206 int i = 0; 207 208 ticks = ticks * 1000000000 / cntfrq; 209 us = ticks / 1000; 210 ns = ticks % 1000; 211 212 if (max_us && us >= max_us) { 213 /* Display value in milliseconds */ 214 unit = 'm'; 215 ms = us / 1000; 216 us = us % 1000; 217 frac = us; 218 in = ms; 219 } else { 220 /* Display value in microseconds */ 221 frac = ns; 222 in = us; 223 } 224 225 *buf-- = 's'; 226 *buf-- = unit; 227 *buf-- = ' '; 228 229 COMPILE_TIME_ASSERT(DURATION_MAX_LEN == 16); 230 if (in > 999999) { 231 /* Not enough space to print the value */ 232 for (i = 0; i < 10; i++) 233 *buf-- = '-'; 234 return; 235 } 236 237 for (i = 0; i < 3; i++) { 238 *buf-- = hex_str[frac % 10]; 239 frac /= 10; 240 } 241 242 *buf-- = '.'; 243 244 while (in) { 245 *buf-- = hex_str[in % 10]; 246 in /= 10; 247 } 248 } 249 250 unsigned long __noprof ftrace_return(void) 251 { 252 struct ftrace_buf *fbuf = NULL; 253 char *line = NULL; 254 size_t line_size = 0; 255 char *dur_loc = NULL; 256 uint32_t i = 0; 257 258 fbuf = get_fbuf(); 259 260 /* Check for valid return index */ 261 if (fbuf && fbuf->ret_idx && fbuf->ret_idx <= FTRACE_RETFUNC_DEPTH) 262 fbuf->ret_idx--; 263 else 264 return 0; 265 266 /* 267 * Check if we have a minimum ftrace buffer current size. If we have 268 * somehow corrupted ftrace buffer formatting, the function call chain 269 * shouldn't get broken since we have a valid fbuf->ret_idx at this 270 * point which can retrieve correct return pointer from fbuf->ret_stack. 271 */ 272 line_size = ENTRY_SIZE(fbuf->ret_idx); 273 if (fbuf->curr_size < line_size) 274 goto out; 275 276 line = (char *)fbuf + fbuf->buf_off + fbuf->curr_size - line_size; 277 278 /* 279 * Check for '{' symbol as it represents if it is an exit from current 280 * or nested function. If exit is from current function, than exit dump 281 * via ';' symbol else exit dump via '}' symbol. 282 */ 283 if (line[line_size - 2] == '{') { 284 line[line_size - 3] = ';'; 285 line[line_size - 2] = '\n'; 286 line[line_size - 1] = '\0'; 287 fbuf->curr_size -= 1; 288 289 dur_loc = &line[DURATION_MAX_LEN - 3]; 290 ftrace_duration(dur_loc, fbuf->begin_time[fbuf->ret_idx], 291 barrier_read_counter_timer()); 292 } else { 293 bool full = false; 294 295 line_size = EXIT_SIZE(fbuf->ret_idx); 296 full = (fbuf->curr_size + line_size) > fbuf->max_size; 297 if (full) 298 full = !fbuf_make_room(fbuf, line_size); 299 300 if (!full) { 301 line = (char *)fbuf + fbuf->buf_off + fbuf->curr_size; 302 303 for (i = 0; i < DURATION_MAX_LEN + fbuf->ret_idx; i++) { 304 if (i == (DURATION_MAX_LEN - 2)) 305 line[i] = '|'; 306 else 307 line[i] = ' '; 308 } 309 310 line[i] = '}'; 311 line[i + 1] = '\n'; 312 line[i + 2] = '\0'; 313 314 fbuf->curr_size += line_size - 1; 315 316 dur_loc = &line[DURATION_MAX_LEN - 4]; 317 ftrace_duration(dur_loc, 318 fbuf->begin_time[fbuf->ret_idx], 319 barrier_read_counter_timer()); 320 } 321 } 322 out: 323 return fbuf->ret_stack[fbuf->ret_idx]; 324 } 325 326 #if !defined(__KERNEL__) 327 void __noprof ftrace_longjmp(unsigned int *ret_idx) 328 { 329 while (__ftrace_buf_start.ret_idx > *ret_idx) 330 ftrace_return(); 331 } 332 333 void __noprof ftrace_setjmp(unsigned int *ret_idx) 334 { 335 *ret_idx = __ftrace_buf_start.ret_idx; 336 } 337 #endif 338