1# Copyright (c) 2014 Google, Inc 2# 3# SPDX-License-Identifier: GPL-2.0+ 4# 5 6import errno 7import glob 8import os 9import shutil 10import threading 11 12import command 13import gitutil 14 15RETURN_CODE_RETRY = -1 16 17def Mkdir(dirname, parents = False): 18 """Make a directory if it doesn't already exist. 19 20 Args: 21 dirname: Directory to create 22 """ 23 try: 24 if parents: 25 os.makedirs(dirname) 26 else: 27 os.mkdir(dirname) 28 except OSError as err: 29 if err.errno == errno.EEXIST: 30 pass 31 else: 32 raise 33 34class BuilderJob: 35 """Holds information about a job to be performed by a thread 36 37 Members: 38 board: Board object to build 39 commits: List of commit options to build. 40 """ 41 def __init__(self): 42 self.board = None 43 self.commits = [] 44 45 46class ResultThread(threading.Thread): 47 """This thread processes results from builder threads. 48 49 It simply passes the results on to the builder. There is only one 50 result thread, and this helps to serialise the build output. 51 """ 52 def __init__(self, builder): 53 """Set up a new result thread 54 55 Args: 56 builder: Builder which will be sent each result 57 """ 58 threading.Thread.__init__(self) 59 self.builder = builder 60 61 def run(self): 62 """Called to start up the result thread. 63 64 We collect the next result job and pass it on to the build. 65 """ 66 while True: 67 result = self.builder.out_queue.get() 68 self.builder.ProcessResult(result) 69 self.builder.out_queue.task_done() 70 71 72class BuilderThread(threading.Thread): 73 """This thread builds U-Boot for a particular board. 74 75 An input queue provides each new job. We run 'make' to build U-Boot 76 and then pass the results on to the output queue. 77 78 Members: 79 builder: The builder which contains information we might need 80 thread_num: Our thread number (0-n-1), used to decide on a 81 temporary directory 82 """ 83 def __init__(self, builder, thread_num): 84 """Set up a new builder thread""" 85 threading.Thread.__init__(self) 86 self.builder = builder 87 self.thread_num = thread_num 88 89 def Make(self, commit, brd, stage, cwd, *args, **kwargs): 90 """Run 'make' on a particular commit and board. 91 92 The source code will already be checked out, so the 'commit' 93 argument is only for information. 94 95 Args: 96 commit: Commit object that is being built 97 brd: Board object that is being built 98 stage: Stage of the build. Valid stages are: 99 mrproper - can be called to clean source 100 config - called to configure for a board 101 build - the main make invocation - it does the build 102 args: A list of arguments to pass to 'make' 103 kwargs: A list of keyword arguments to pass to command.RunPipe() 104 105 Returns: 106 CommandResult object 107 """ 108 return self.builder.do_make(commit, brd, stage, cwd, *args, 109 **kwargs) 110 111 def RunCommit(self, commit_upto, brd, work_dir, do_config, force_build, 112 force_build_failures): 113 """Build a particular commit. 114 115 If the build is already done, and we are not forcing a build, we skip 116 the build and just return the previously-saved results. 117 118 Args: 119 commit_upto: Commit number to build (0...n-1) 120 brd: Board object to build 121 work_dir: Directory to which the source will be checked out 122 do_config: True to run a make <board>_defconfig on the source 123 force_build: Force a build even if one was previously done 124 force_build_failures: Force a bulid if the previous result showed 125 failure 126 127 Returns: 128 tuple containing: 129 - CommandResult object containing the results of the build 130 - boolean indicating whether 'make config' is still needed 131 """ 132 # Create a default result - it will be overwritte by the call to 133 # self.Make() below, in the event that we do a build. 134 result = command.CommandResult() 135 result.return_code = 0 136 if self.builder.in_tree: 137 out_dir = work_dir 138 else: 139 out_dir = os.path.join(work_dir, 'build') 140 141 # Check if the job was already completed last time 142 done_file = self.builder.GetDoneFile(commit_upto, brd.target) 143 result.already_done = os.path.exists(done_file) 144 will_build = (force_build or force_build_failures or 145 not result.already_done) 146 if result.already_done: 147 # Get the return code from that build and use it 148 with open(done_file, 'r') as fd: 149 result.return_code = int(fd.readline()) 150 151 # Check the signal that the build needs to be retried 152 if result.return_code == RETURN_CODE_RETRY: 153 will_build = True 154 elif will_build: 155 err_file = self.builder.GetErrFile(commit_upto, brd.target) 156 if os.path.exists(err_file) and os.stat(err_file).st_size: 157 result.stderr = 'bad' 158 elif not force_build: 159 # The build passed, so no need to build it again 160 will_build = False 161 162 if will_build: 163 # We are going to have to build it. First, get a toolchain 164 if not self.toolchain: 165 try: 166 self.toolchain = self.builder.toolchains.Select(brd.arch) 167 except ValueError as err: 168 result.return_code = 10 169 result.stdout = '' 170 result.stderr = str(err) 171 # TODO(sjg@chromium.org): This gets swallowed, but needs 172 # to be reported. 173 174 if self.toolchain: 175 # Checkout the right commit 176 if self.builder.commits: 177 commit = self.builder.commits[commit_upto] 178 if self.builder.checkout: 179 git_dir = os.path.join(work_dir, '.git') 180 gitutil.Checkout(commit.hash, git_dir, work_dir, 181 force=True) 182 else: 183 commit = 'current' 184 185 # Set up the environment and command line 186 env = self.toolchain.MakeEnvironment(self.builder.full_path) 187 Mkdir(out_dir) 188 args = [] 189 cwd = work_dir 190 src_dir = os.path.realpath(work_dir) 191 if not self.builder.in_tree: 192 if commit_upto is None: 193 # In this case we are building in the original source 194 # directory (i.e. the current directory where buildman 195 # is invoked. The output directory is set to this 196 # thread's selected work directory. 197 # 198 # Symlinks can confuse U-Boot's Makefile since 199 # we may use '..' in our path, so remove them. 200 work_dir = os.path.realpath(work_dir) 201 args.append('O=%s/build' % work_dir) 202 cwd = None 203 src_dir = os.getcwd() 204 else: 205 args.append('O=build') 206 if not self.builder.verbose_build: 207 args.append('-s') 208 if self.builder.num_jobs is not None: 209 args.extend(['-j', str(self.builder.num_jobs)]) 210 config_args = ['%s_defconfig' % brd.target] 211 config_out = '' 212 args.extend(self.builder.toolchains.GetMakeArguments(brd)) 213 214 # If we need to reconfigure, do that now 215 if do_config: 216 result = self.Make(commit, brd, 'mrproper', cwd, 217 'mrproper', *args, env=env) 218 config_out = result.combined 219 result = self.Make(commit, brd, 'config', cwd, 220 *(args + config_args), env=env) 221 config_out += result.combined 222 do_config = False # No need to configure next time 223 if result.return_code == 0: 224 result = self.Make(commit, brd, 'build', cwd, *args, 225 env=env) 226 result.stderr = result.stderr.replace(src_dir + '/', '') 227 if self.builder.verbose_build: 228 result.stdout = config_out + result.stdout 229 else: 230 result.return_code = 1 231 result.stderr = 'No tool chain for %s\n' % brd.arch 232 result.already_done = False 233 234 result.toolchain = self.toolchain 235 result.brd = brd 236 result.commit_upto = commit_upto 237 result.out_dir = out_dir 238 return result, do_config 239 240 def _WriteResult(self, result, keep_outputs): 241 """Write a built result to the output directory. 242 243 Args: 244 result: CommandResult object containing result to write 245 keep_outputs: True to store the output binaries, False 246 to delete them 247 """ 248 # Fatal error 249 if result.return_code < 0: 250 return 251 252 # If we think this might have been aborted with Ctrl-C, record the 253 # failure but not that we are 'done' with this board. A retry may fix 254 # it. 255 maybe_aborted = result.stderr and 'No child processes' in result.stderr 256 257 if result.already_done: 258 return 259 260 # Write the output and stderr 261 output_dir = self.builder._GetOutputDir(result.commit_upto) 262 Mkdir(output_dir) 263 build_dir = self.builder.GetBuildDir(result.commit_upto, 264 result.brd.target) 265 Mkdir(build_dir) 266 267 outfile = os.path.join(build_dir, 'log') 268 with open(outfile, 'w') as fd: 269 if result.stdout: 270 fd.write(result.stdout) 271 272 errfile = self.builder.GetErrFile(result.commit_upto, 273 result.brd.target) 274 if result.stderr: 275 with open(errfile, 'w') as fd: 276 fd.write(result.stderr) 277 elif os.path.exists(errfile): 278 os.remove(errfile) 279 280 if result.toolchain: 281 # Write the build result and toolchain information. 282 done_file = self.builder.GetDoneFile(result.commit_upto, 283 result.brd.target) 284 with open(done_file, 'w') as fd: 285 if maybe_aborted: 286 # Special code to indicate we need to retry 287 fd.write('%s' % RETURN_CODE_RETRY) 288 else: 289 fd.write('%s' % result.return_code) 290 with open(os.path.join(build_dir, 'toolchain'), 'w') as fd: 291 print >>fd, 'gcc', result.toolchain.gcc 292 print >>fd, 'path', result.toolchain.path 293 print >>fd, 'cross', result.toolchain.cross 294 print >>fd, 'arch', result.toolchain.arch 295 fd.write('%s' % result.return_code) 296 297 with open(os.path.join(build_dir, 'toolchain'), 'w') as fd: 298 print >>fd, 'gcc', result.toolchain.gcc 299 print >>fd, 'path', result.toolchain.path 300 301 # Write out the image and function size information and an objdump 302 env = result.toolchain.MakeEnvironment(self.builder.full_path) 303 lines = [] 304 for fname in ['u-boot', 'spl/u-boot-spl']: 305 cmd = ['%snm' % self.toolchain.cross, '--size-sort', fname] 306 nm_result = command.RunPipe([cmd], capture=True, 307 capture_stderr=True, cwd=result.out_dir, 308 raise_on_error=False, env=env) 309 if nm_result.stdout: 310 nm = self.builder.GetFuncSizesFile(result.commit_upto, 311 result.brd.target, fname) 312 with open(nm, 'w') as fd: 313 print >>fd, nm_result.stdout, 314 315 cmd = ['%sobjdump' % self.toolchain.cross, '-h', fname] 316 dump_result = command.RunPipe([cmd], capture=True, 317 capture_stderr=True, cwd=result.out_dir, 318 raise_on_error=False, env=env) 319 rodata_size = '' 320 if dump_result.stdout: 321 objdump = self.builder.GetObjdumpFile(result.commit_upto, 322 result.brd.target, fname) 323 with open(objdump, 'w') as fd: 324 print >>fd, dump_result.stdout, 325 for line in dump_result.stdout.splitlines(): 326 fields = line.split() 327 if len(fields) > 5 and fields[1] == '.rodata': 328 rodata_size = fields[2] 329 330 cmd = ['%ssize' % self.toolchain.cross, fname] 331 size_result = command.RunPipe([cmd], capture=True, 332 capture_stderr=True, cwd=result.out_dir, 333 raise_on_error=False, env=env) 334 if size_result.stdout: 335 lines.append(size_result.stdout.splitlines()[1] + ' ' + 336 rodata_size) 337 338 # Write out the image sizes file. This is similar to the output 339 # of binutil's 'size' utility, but it omits the header line and 340 # adds an additional hex value at the end of each line for the 341 # rodata size 342 if len(lines): 343 sizes = self.builder.GetSizesFile(result.commit_upto, 344 result.brd.target) 345 with open(sizes, 'w') as fd: 346 print >>fd, '\n'.join(lines) 347 348 # Write out the configuration files, with a special case for SPL 349 for dirname in ['', 'spl', 'tpl']: 350 self.CopyFiles(result.out_dir, build_dir, dirname, ['u-boot.cfg', 351 'spl/u-boot-spl.cfg', 'tpl/u-boot-tpl.cfg', '.config', 352 'include/autoconf.mk', 'include/generated/autoconf.h']) 353 354 # Now write the actual build output 355 if keep_outputs: 356 self.CopyFiles(result.out_dir, build_dir, '', ['u-boot', '*.bin', 357 'u-boot.dtb', '*.map', '*.img', 358 'spl/u-boot-spl', 'spl/u-boot-spl.bin', 359 'tpl/u-boot-tpl', 'tpl/u-boot-tpl.bin']) 360 361 def CopyFiles(self, out_dir, build_dir, dirname, patterns): 362 """Copy files from the build directory to the output. 363 364 Args: 365 out_dir: Path to output directory containing the files 366 build_dir: Place to copy the files 367 dirname: Source directory, '' for normal U-Boot, 'spl' for SPL 368 patterns: A list of filenames (strings) to copy, each relative 369 to the build directory 370 """ 371 for pattern in patterns: 372 file_list = glob.glob(os.path.join(out_dir, dirname, pattern)) 373 for fname in file_list: 374 target = os.path.basename(fname) 375 if dirname: 376 base, ext = os.path.splitext(target) 377 if ext: 378 target = '%s-%s%s' % (base, dirname, ext) 379 shutil.copy(fname, os.path.join(build_dir, target)) 380 381 def RunJob(self, job): 382 """Run a single job 383 384 A job consists of a building a list of commits for a particular board. 385 386 Args: 387 job: Job to build 388 """ 389 brd = job.board 390 work_dir = self.builder.GetThreadDir(self.thread_num) 391 self.toolchain = None 392 if job.commits: 393 # Run 'make board_defconfig' on the first commit 394 do_config = True 395 commit_upto = 0 396 force_build = False 397 for commit_upto in range(0, len(job.commits), job.step): 398 result, request_config = self.RunCommit(commit_upto, brd, 399 work_dir, do_config, 400 force_build or self.builder.force_build, 401 self.builder.force_build_failures) 402 failed = result.return_code or result.stderr 403 did_config = do_config 404 if failed and not do_config: 405 # If our incremental build failed, try building again 406 # with a reconfig. 407 if self.builder.force_config_on_failure: 408 result, request_config = self.RunCommit(commit_upto, 409 brd, work_dir, True, True, False) 410 did_config = True 411 if not self.builder.force_reconfig: 412 do_config = request_config 413 414 # If we built that commit, then config is done. But if we got 415 # an warning, reconfig next time to force it to build the same 416 # files that created warnings this time. Otherwise an 417 # incremental build may not build the same file, and we will 418 # think that the warning has gone away. 419 # We could avoid this by using -Werror everywhere... 420 # For errors, the problem doesn't happen, since presumably 421 # the build stopped and didn't generate output, so will retry 422 # that file next time. So we could detect warnings and deal 423 # with them specially here. For now, we just reconfigure if 424 # anything goes work. 425 # Of course this is substantially slower if there are build 426 # errors/warnings (e.g. 2-3x slower even if only 10% of builds 427 # have problems). 428 if (failed and not result.already_done and not did_config and 429 self.builder.force_config_on_failure): 430 # If this build failed, try the next one with a 431 # reconfigure. 432 # Sometimes if the board_config.h file changes it can mess 433 # with dependencies, and we get: 434 # make: *** No rule to make target `include/autoconf.mk', 435 # needed by `depend'. 436 do_config = True 437 force_build = True 438 else: 439 force_build = False 440 if self.builder.force_config_on_failure: 441 if failed: 442 do_config = True 443 result.commit_upto = commit_upto 444 if result.return_code < 0: 445 raise ValueError('Interrupt') 446 447 # We have the build results, so output the result 448 self._WriteResult(result, job.keep_outputs) 449 self.builder.out_queue.put(result) 450 else: 451 # Just build the currently checked-out build 452 result, request_config = self.RunCommit(None, brd, work_dir, True, 453 True, self.builder.force_build_failures) 454 result.commit_upto = 0 455 self._WriteResult(result, job.keep_outputs) 456 self.builder.out_queue.put(result) 457 458 def run(self): 459 """Our thread's run function 460 461 This thread picks a job from the queue, runs it, and then goes to the 462 next job. 463 """ 464 alive = True 465 while True: 466 job = self.builder.queue.get() 467 if self.builder.active and alive: 468 self.RunJob(job) 469 ''' 470 try: 471 if self.builder.active and alive: 472 self.RunJob(job) 473 except Exception as err: 474 alive = False 475 print err 476 ''' 477 self.builder.queue.task_done() 478