1*4882a593Smuzhiyun#!/bin/bash 2*4882a593Smuzhiyun# SPDX-License-Identifier: GPL-2.0 3*4882a593Smuzhiyun# 4*4882a593Smuzhiyun# Here's how to use this: 5*4882a593Smuzhiyun# 6*4882a593Smuzhiyun# This script is used to help find functions that are being traced by function 7*4882a593Smuzhiyun# tracer or function graph tracing that causes the machine to reboot, hang, or 8*4882a593Smuzhiyun# crash. Here's the steps to take. 9*4882a593Smuzhiyun# 10*4882a593Smuzhiyun# First, determine if function tracing is working with a single function: 11*4882a593Smuzhiyun# 12*4882a593Smuzhiyun# (note, if this is a problem with function_graph tracing, then simply 13*4882a593Smuzhiyun# replace "function" with "function_graph" in the following steps). 14*4882a593Smuzhiyun# 15*4882a593Smuzhiyun# # cd /sys/kernel/debug/tracing 16*4882a593Smuzhiyun# # echo schedule > set_ftrace_filter 17*4882a593Smuzhiyun# # echo function > current_tracer 18*4882a593Smuzhiyun# 19*4882a593Smuzhiyun# If this works, then we know that something is being traced that shouldn't be. 20*4882a593Smuzhiyun# 21*4882a593Smuzhiyun# # echo nop > current_tracer 22*4882a593Smuzhiyun# 23*4882a593Smuzhiyun# # cat available_filter_functions > ~/full-file 24*4882a593Smuzhiyun# # ftrace-bisect ~/full-file ~/test-file ~/non-test-file 25*4882a593Smuzhiyun# # cat ~/test-file > set_ftrace_filter 26*4882a593Smuzhiyun# 27*4882a593Smuzhiyun# *** Note *** this will take several minutes. Setting multiple functions is 28*4882a593Smuzhiyun# an O(n^2) operation, and we are dealing with thousands of functions. So go 29*4882a593Smuzhiyun# have coffee, talk with your coworkers, read facebook. And eventually, this 30*4882a593Smuzhiyun# operation will end. 31*4882a593Smuzhiyun# 32*4882a593Smuzhiyun# # echo function > current_tracer 33*4882a593Smuzhiyun# 34*4882a593Smuzhiyun# If it crashes, we know that ~/test-file has a bad function. 35*4882a593Smuzhiyun# 36*4882a593Smuzhiyun# Reboot back to test kernel. 37*4882a593Smuzhiyun# 38*4882a593Smuzhiyun# # cd /sys/kernel/debug/tracing 39*4882a593Smuzhiyun# # mv ~/test-file ~/full-file 40*4882a593Smuzhiyun# 41*4882a593Smuzhiyun# If it didn't crash. 42*4882a593Smuzhiyun# 43*4882a593Smuzhiyun# # echo nop > current_tracer 44*4882a593Smuzhiyun# # mv ~/non-test-file ~/full-file 45*4882a593Smuzhiyun# 46*4882a593Smuzhiyun# Get rid of the other test file from previous run (or save them off somewhere). 47*4882a593Smuzhiyun# # rm -f ~/test-file ~/non-test-file 48*4882a593Smuzhiyun# 49*4882a593Smuzhiyun# And start again: 50*4882a593Smuzhiyun# 51*4882a593Smuzhiyun# # ftrace-bisect ~/full-file ~/test-file ~/non-test-file 52*4882a593Smuzhiyun# 53*4882a593Smuzhiyun# The good thing is, because this cuts the number of functions in ~/test-file 54*4882a593Smuzhiyun# by half, the cat of it into set_ftrace_filter takes half as long each 55*4882a593Smuzhiyun# iteration, so don't talk so much at the water cooler the second time. 56*4882a593Smuzhiyun# 57*4882a593Smuzhiyun# Eventually, if you did this correctly, you will get down to the problem 58*4882a593Smuzhiyun# function, and all we need to do is to notrace it. 59*4882a593Smuzhiyun# 60*4882a593Smuzhiyun# The way to figure out if the problem function is bad, just do: 61*4882a593Smuzhiyun# 62*4882a593Smuzhiyun# # echo <problem-function> > set_ftrace_notrace 63*4882a593Smuzhiyun# # echo > set_ftrace_filter 64*4882a593Smuzhiyun# # echo function > current_tracer 65*4882a593Smuzhiyun# 66*4882a593Smuzhiyun# And if it doesn't crash, we are done. 67*4882a593Smuzhiyun# 68*4882a593Smuzhiyun# If it does crash, do this again (there's more than one problem function) 69*4882a593Smuzhiyun# but you need to echo the problem function(s) into set_ftrace_notrace before 70*4882a593Smuzhiyun# enabling function tracing in the above steps. Or if you can compile the 71*4882a593Smuzhiyun# kernel, annotate the problem functions with "notrace" and start again. 72*4882a593Smuzhiyun# 73*4882a593Smuzhiyun 74*4882a593Smuzhiyun 75*4882a593Smuzhiyunif [ $# -ne 3 ]; then 76*4882a593Smuzhiyun echo 'usage: ftrace-bisect full-file test-file non-test-file' 77*4882a593Smuzhiyun exit 78*4882a593Smuzhiyunfi 79*4882a593Smuzhiyun 80*4882a593Smuzhiyunfull=$1 81*4882a593Smuzhiyuntest=$2 82*4882a593Smuzhiyunnontest=$3 83*4882a593Smuzhiyun 84*4882a593Smuzhiyunx=`cat $full | wc -l` 85*4882a593Smuzhiyunif [ $x -eq 1 ]; then 86*4882a593Smuzhiyun echo "There's only one function left, must be the bad one" 87*4882a593Smuzhiyun cat $full 88*4882a593Smuzhiyun exit 0 89*4882a593Smuzhiyunfi 90*4882a593Smuzhiyun 91*4882a593Smuzhiyunlet x=$x/2 92*4882a593Smuzhiyunlet y=$x+1 93*4882a593Smuzhiyun 94*4882a593Smuzhiyunif [ ! -f $full ]; then 95*4882a593Smuzhiyun echo "$full does not exist" 96*4882a593Smuzhiyun exit 1 97*4882a593Smuzhiyunfi 98*4882a593Smuzhiyun 99*4882a593Smuzhiyunif [ -f $test ]; then 100*4882a593Smuzhiyun echo -n "$test exists, delete it? [y/N]" 101*4882a593Smuzhiyun read a 102*4882a593Smuzhiyun if [ "$a" != "y" -a "$a" != "Y" ]; then 103*4882a593Smuzhiyun exit 1 104*4882a593Smuzhiyun fi 105*4882a593Smuzhiyunfi 106*4882a593Smuzhiyun 107*4882a593Smuzhiyunif [ -f $nontest ]; then 108*4882a593Smuzhiyun echo -n "$nontest exists, delete it? [y/N]" 109*4882a593Smuzhiyun read a 110*4882a593Smuzhiyun if [ "$a" != "y" -a "$a" != "Y" ]; then 111*4882a593Smuzhiyun exit 1 112*4882a593Smuzhiyun fi 113*4882a593Smuzhiyunfi 114*4882a593Smuzhiyun 115*4882a593Smuzhiyunsed -ne "1,${x}p" $full > $test 116*4882a593Smuzhiyunsed -ne "$y,\$p" $full > $nontest 117