1# Copyright (c) 2014 Google, Inc 2# 3# SPDX-License-Identifier: GPL-2.0+ 4# 5 6import errno 7import glob 8import os 9import shutil 10import threading 11 12import command 13import gitutil 14 15RETURN_CODE_RETRY = -1 16 17def Mkdir(dirname, parents = False): 18 """Make a directory if it doesn't already exist. 19 20 Args: 21 dirname: Directory to create 22 """ 23 try: 24 if parents: 25 os.makedirs(dirname) 26 else: 27 os.mkdir(dirname) 28 except OSError as err: 29 if err.errno == errno.EEXIST: 30 pass 31 else: 32 raise 33 34class BuilderJob: 35 """Holds information about a job to be performed by a thread 36 37 Members: 38 board: Board object to build 39 commits: List of commit options to build. 40 """ 41 def __init__(self): 42 self.board = None 43 self.commits = [] 44 45 46class ResultThread(threading.Thread): 47 """This thread processes results from builder threads. 48 49 It simply passes the results on to the builder. There is only one 50 result thread, and this helps to serialise the build output. 51 """ 52 def __init__(self, builder): 53 """Set up a new result thread 54 55 Args: 56 builder: Builder which will be sent each result 57 """ 58 threading.Thread.__init__(self) 59 self.builder = builder 60 61 def run(self): 62 """Called to start up the result thread. 63 64 We collect the next result job and pass it on to the build. 65 """ 66 while True: 67 result = self.builder.out_queue.get() 68 self.builder.ProcessResult(result) 69 self.builder.out_queue.task_done() 70 71 72class BuilderThread(threading.Thread): 73 """This thread builds U-Boot for a particular board. 74 75 An input queue provides each new job. We run 'make' to build U-Boot 76 and then pass the results on to the output queue. 77 78 Members: 79 builder: The builder which contains information we might need 80 thread_num: Our thread number (0-n-1), used to decide on a 81 temporary directory 82 """ 83 def __init__(self, builder, thread_num): 84 """Set up a new builder thread""" 85 threading.Thread.__init__(self) 86 self.builder = builder 87 self.thread_num = thread_num 88 89 def Make(self, commit, brd, stage, cwd, *args, **kwargs): 90 """Run 'make' on a particular commit and board. 91 92 The source code will already be checked out, so the 'commit' 93 argument is only for information. 94 95 Args: 96 commit: Commit object that is being built 97 brd: Board object that is being built 98 stage: Stage of the build. Valid stages are: 99 mrproper - can be called to clean source 100 config - called to configure for a board 101 build - the main make invocation - it does the build 102 args: A list of arguments to pass to 'make' 103 kwargs: A list of keyword arguments to pass to command.RunPipe() 104 105 Returns: 106 CommandResult object 107 """ 108 return self.builder.do_make(commit, brd, stage, cwd, *args, 109 **kwargs) 110 111 def RunCommit(self, commit_upto, brd, work_dir, do_config, force_build, 112 force_build_failures): 113 """Build a particular commit. 114 115 If the build is already done, and we are not forcing a build, we skip 116 the build and just return the previously-saved results. 117 118 Args: 119 commit_upto: Commit number to build (0...n-1) 120 brd: Board object to build 121 work_dir: Directory to which the source will be checked out 122 do_config: True to run a make <board>_defconfig on the source 123 force_build: Force a build even if one was previously done 124 force_build_failures: Force a bulid if the previous result showed 125 failure 126 127 Returns: 128 tuple containing: 129 - CommandResult object containing the results of the build 130 - boolean indicating whether 'make config' is still needed 131 """ 132 # Create a default result - it will be overwritte by the call to 133 # self.Make() below, in the event that we do a build. 134 result = command.CommandResult() 135 result.return_code = 0 136 if self.builder.in_tree: 137 out_dir = work_dir 138 else: 139 out_dir = os.path.join(work_dir, 'build') 140 141 # Check if the job was already completed last time 142 done_file = self.builder.GetDoneFile(commit_upto, brd.target) 143 result.already_done = os.path.exists(done_file) 144 will_build = (force_build or force_build_failures or 145 not result.already_done) 146 if result.already_done: 147 # Get the return code from that build and use it 148 with open(done_file, 'r') as fd: 149 result.return_code = int(fd.readline()) 150 151 # Check the signal that the build needs to be retried 152 if result.return_code == RETURN_CODE_RETRY: 153 will_build = True 154 elif will_build: 155 err_file = self.builder.GetErrFile(commit_upto, brd.target) 156 if os.path.exists(err_file) and os.stat(err_file).st_size: 157 result.stderr = 'bad' 158 elif not force_build: 159 # The build passed, so no need to build it again 160 will_build = False 161 162 if will_build: 163 # We are going to have to build it. First, get a toolchain 164 if not self.toolchain: 165 try: 166 self.toolchain = self.builder.toolchains.Select(brd.arch) 167 except ValueError as err: 168 result.return_code = 10 169 result.stdout = '' 170 result.stderr = str(err) 171 # TODO(sjg@chromium.org): This gets swallowed, but needs 172 # to be reported. 173 174 if self.toolchain: 175 # Checkout the right commit 176 if self.builder.commits: 177 commit = self.builder.commits[commit_upto] 178 if self.builder.checkout: 179 git_dir = os.path.join(work_dir, '.git') 180 gitutil.Checkout(commit.hash, git_dir, work_dir, 181 force=True) 182 else: 183 commit = 'current' 184 185 # Set up the environment and command line 186 env = self.toolchain.MakeEnvironment(self.builder.full_path) 187 Mkdir(out_dir) 188 args = [] 189 cwd = work_dir 190 src_dir = os.path.realpath(work_dir) 191 if not self.builder.in_tree: 192 if commit_upto is None: 193 # In this case we are building in the original source 194 # directory (i.e. the current directory where buildman 195 # is invoked. The output directory is set to this 196 # thread's selected work directory. 197 # 198 # Symlinks can confuse U-Boot's Makefile since 199 # we may use '..' in our path, so remove them. 200 work_dir = os.path.realpath(work_dir) 201 args.append('O=%s/build' % work_dir) 202 cwd = None 203 src_dir = os.getcwd() 204 else: 205 args.append('O=build') 206 if not self.builder.verbose_build: 207 args.append('-s') 208 if self.builder.num_jobs is not None: 209 args.extend(['-j', str(self.builder.num_jobs)]) 210 config_args = ['%s_defconfig' % brd.target] 211 config_out = '' 212 args.extend(self.builder.toolchains.GetMakeArguments(brd)) 213 214 # If we need to reconfigure, do that now 215 if do_config: 216 result = self.Make(commit, brd, 'mrproper', cwd, 217 'mrproper', *args, env=env) 218 config_out = result.combined 219 result = self.Make(commit, brd, 'config', cwd, 220 *(args + config_args), env=env) 221 config_out += result.combined 222 do_config = False # No need to configure next time 223 if result.return_code == 0: 224 result = self.Make(commit, brd, 'build', cwd, *args, 225 env=env) 226 result.stderr = result.stderr.replace(src_dir + '/', '') 227 if self.builder.verbose_build: 228 result.stdout = config_out + result.stdout 229 else: 230 result.return_code = 1 231 result.stderr = 'No tool chain for %s\n' % brd.arch 232 result.already_done = False 233 234 result.toolchain = self.toolchain 235 result.brd = brd 236 result.commit_upto = commit_upto 237 result.out_dir = out_dir 238 return result, do_config 239 240 def _WriteResult(self, result, keep_outputs): 241 """Write a built result to the output directory. 242 243 Args: 244 result: CommandResult object containing result to write 245 keep_outputs: True to store the output binaries, False 246 to delete them 247 """ 248 # Fatal error 249 if result.return_code < 0: 250 return 251 252 # If we think this might have been aborted with Ctrl-C, record the 253 # failure but not that we are 'done' with this board. A retry may fix 254 # it. 255 maybe_aborted = result.stderr and 'No child processes' in result.stderr 256 257 if result.already_done: 258 return 259 260 # Write the output and stderr 261 output_dir = self.builder._GetOutputDir(result.commit_upto) 262 Mkdir(output_dir) 263 build_dir = self.builder.GetBuildDir(result.commit_upto, 264 result.brd.target) 265 Mkdir(build_dir) 266 267 outfile = os.path.join(build_dir, 'log') 268 with open(outfile, 'w') as fd: 269 if result.stdout: 270 fd.write(result.stdout) 271 272 errfile = self.builder.GetErrFile(result.commit_upto, 273 result.brd.target) 274 if result.stderr: 275 with open(errfile, 'w') as fd: 276 fd.write(result.stderr) 277 elif os.path.exists(errfile): 278 os.remove(errfile) 279 280 if result.toolchain: 281 # Write the build result and toolchain information. 282 done_file = self.builder.GetDoneFile(result.commit_upto, 283 result.brd.target) 284 with open(done_file, 'w') as fd: 285 if maybe_aborted: 286 # Special code to indicate we need to retry 287 fd.write('%s' % RETURN_CODE_RETRY) 288 else: 289 fd.write('%s' % result.return_code) 290 with open(os.path.join(build_dir, 'toolchain'), 'w') as fd: 291 print >>fd, 'gcc', result.toolchain.gcc 292 print >>fd, 'path', result.toolchain.path 293 print >>fd, 'cross', result.toolchain.cross 294 print >>fd, 'arch', result.toolchain.arch 295 fd.write('%s' % result.return_code) 296 297 with open(os.path.join(build_dir, 'toolchain'), 'w') as fd: 298 print >>fd, 'gcc', result.toolchain.gcc 299 print >>fd, 'path', result.toolchain.path 300 301 # Write out the image and function size information and an objdump 302 env = result.toolchain.MakeEnvironment(self.builder.full_path) 303 lines = [] 304 for fname in ['u-boot', 'spl/u-boot-spl']: 305 cmd = ['%snm' % self.toolchain.cross, '--size-sort', fname] 306 nm_result = command.RunPipe([cmd], capture=True, 307 capture_stderr=True, cwd=result.out_dir, 308 raise_on_error=False, env=env) 309 if nm_result.stdout: 310 nm = self.builder.GetFuncSizesFile(result.commit_upto, 311 result.brd.target, fname) 312 with open(nm, 'w') as fd: 313 print >>fd, nm_result.stdout, 314 315 cmd = ['%sobjdump' % self.toolchain.cross, '-h', fname] 316 dump_result = command.RunPipe([cmd], capture=True, 317 capture_stderr=True, cwd=result.out_dir, 318 raise_on_error=False, env=env) 319 rodata_size = '' 320 if dump_result.stdout: 321 objdump = self.builder.GetObjdumpFile(result.commit_upto, 322 result.brd.target, fname) 323 with open(objdump, 'w') as fd: 324 print >>fd, dump_result.stdout, 325 for line in dump_result.stdout.splitlines(): 326 fields = line.split() 327 if len(fields) > 5 and fields[1] == '.rodata': 328 rodata_size = fields[2] 329 330 cmd = ['%ssize' % self.toolchain.cross, fname] 331 size_result = command.RunPipe([cmd], capture=True, 332 capture_stderr=True, cwd=result.out_dir, 333 raise_on_error=False, env=env) 334 if size_result.stdout: 335 lines.append(size_result.stdout.splitlines()[1] + ' ' + 336 rodata_size) 337 338 # Write out the image sizes file. This is similar to the output 339 # of binutil's 'size' utility, but it omits the header line and 340 # adds an additional hex value at the end of each line for the 341 # rodata size 342 if len(lines): 343 sizes = self.builder.GetSizesFile(result.commit_upto, 344 result.brd.target) 345 with open(sizes, 'w') as fd: 346 print >>fd, '\n'.join(lines) 347 348 # Now write the actual build output 349 if keep_outputs: 350 patterns = ['u-boot', '*.bin', 'u-boot.dtb', '*.map', '*.img', 351 'include/autoconf.mk', 'spl/u-boot-spl', 352 'spl/u-boot-spl.bin'] 353 for pattern in patterns: 354 file_list = glob.glob(os.path.join(result.out_dir, pattern)) 355 for fname in file_list: 356 shutil.copy(fname, build_dir) 357 358 359 def RunJob(self, job): 360 """Run a single job 361 362 A job consists of a building a list of commits for a particular board. 363 364 Args: 365 job: Job to build 366 """ 367 brd = job.board 368 work_dir = self.builder.GetThreadDir(self.thread_num) 369 self.toolchain = None 370 if job.commits: 371 # Run 'make board_defconfig' on the first commit 372 do_config = True 373 commit_upto = 0 374 force_build = False 375 for commit_upto in range(0, len(job.commits), job.step): 376 result, request_config = self.RunCommit(commit_upto, brd, 377 work_dir, do_config, 378 force_build or self.builder.force_build, 379 self.builder.force_build_failures) 380 failed = result.return_code or result.stderr 381 did_config = do_config 382 if failed and not do_config: 383 # If our incremental build failed, try building again 384 # with a reconfig. 385 if self.builder.force_config_on_failure: 386 result, request_config = self.RunCommit(commit_upto, 387 brd, work_dir, True, True, False) 388 did_config = True 389 if not self.builder.force_reconfig: 390 do_config = request_config 391 392 # If we built that commit, then config is done. But if we got 393 # an warning, reconfig next time to force it to build the same 394 # files that created warnings this time. Otherwise an 395 # incremental build may not build the same file, and we will 396 # think that the warning has gone away. 397 # We could avoid this by using -Werror everywhere... 398 # For errors, the problem doesn't happen, since presumably 399 # the build stopped and didn't generate output, so will retry 400 # that file next time. So we could detect warnings and deal 401 # with them specially here. For now, we just reconfigure if 402 # anything goes work. 403 # Of course this is substantially slower if there are build 404 # errors/warnings (e.g. 2-3x slower even if only 10% of builds 405 # have problems). 406 if (failed and not result.already_done and not did_config and 407 self.builder.force_config_on_failure): 408 # If this build failed, try the next one with a 409 # reconfigure. 410 # Sometimes if the board_config.h file changes it can mess 411 # with dependencies, and we get: 412 # make: *** No rule to make target `include/autoconf.mk', 413 # needed by `depend'. 414 do_config = True 415 force_build = True 416 else: 417 force_build = False 418 if self.builder.force_config_on_failure: 419 if failed: 420 do_config = True 421 result.commit_upto = commit_upto 422 if result.return_code < 0: 423 raise ValueError('Interrupt') 424 425 # We have the build results, so output the result 426 self._WriteResult(result, job.keep_outputs) 427 self.builder.out_queue.put(result) 428 else: 429 # Just build the currently checked-out build 430 result, request_config = self.RunCommit(None, brd, work_dir, True, 431 True, self.builder.force_build_failures) 432 result.commit_upto = 0 433 self._WriteResult(result, job.keep_outputs) 434 self.builder.out_queue.put(result) 435 436 def run(self): 437 """Our thread's run function 438 439 This thread picks a job from the queue, runs it, and then goes to the 440 next job. 441 """ 442 alive = True 443 while True: 444 job = self.builder.queue.get() 445 if self.builder.active and alive: 446 self.RunJob(job) 447 ''' 448 try: 449 if self.builder.active and alive: 450 self.RunJob(job) 451 except Exception as err: 452 alive = False 453 print err 454 ''' 455 self.builder.queue.task_done() 456