1# Copyright (c) 2014 Google, Inc 2# 3# SPDX-License-Identifier: GPL-2.0+ 4# 5 6import errno 7import glob 8import os 9import shutil 10import threading 11 12import command 13import gitutil 14 15RETURN_CODE_RETRY = -1 16 17def Mkdir(dirname, parents = False): 18 """Make a directory if it doesn't already exist. 19 20 Args: 21 dirname: Directory to create 22 """ 23 try: 24 if parents: 25 os.makedirs(dirname) 26 else: 27 os.mkdir(dirname) 28 except OSError as err: 29 if err.errno == errno.EEXIST: 30 pass 31 else: 32 raise 33 34class BuilderJob: 35 """Holds information about a job to be performed by a thread 36 37 Members: 38 board: Board object to build 39 commits: List of commit options to build. 40 """ 41 def __init__(self): 42 self.board = None 43 self.commits = [] 44 45 46class ResultThread(threading.Thread): 47 """This thread processes results from builder threads. 48 49 It simply passes the results on to the builder. There is only one 50 result thread, and this helps to serialise the build output. 51 """ 52 def __init__(self, builder): 53 """Set up a new result thread 54 55 Args: 56 builder: Builder which will be sent each result 57 """ 58 threading.Thread.__init__(self) 59 self.builder = builder 60 61 def run(self): 62 """Called to start up the result thread. 63 64 We collect the next result job and pass it on to the build. 65 """ 66 while True: 67 result = self.builder.out_queue.get() 68 self.builder.ProcessResult(result) 69 self.builder.out_queue.task_done() 70 71 72class BuilderThread(threading.Thread): 73 """This thread builds U-Boot for a particular board. 74 75 An input queue provides each new job. We run 'make' to build U-Boot 76 and then pass the results on to the output queue. 77 78 Members: 79 builder: The builder which contains information we might need 80 thread_num: Our thread number (0-n-1), used to decide on a 81 temporary directory 82 """ 83 def __init__(self, builder, thread_num): 84 """Set up a new builder thread""" 85 threading.Thread.__init__(self) 86 self.builder = builder 87 self.thread_num = thread_num 88 89 def Make(self, commit, brd, stage, cwd, *args, **kwargs): 90 """Run 'make' on a particular commit and board. 91 92 The source code will already be checked out, so the 'commit' 93 argument is only for information. 94 95 Args: 96 commit: Commit object that is being built 97 brd: Board object that is being built 98 stage: Stage of the build. Valid stages are: 99 mrproper - can be called to clean source 100 config - called to configure for a board 101 build - the main make invocation - it does the build 102 args: A list of arguments to pass to 'make' 103 kwargs: A list of keyword arguments to pass to command.RunPipe() 104 105 Returns: 106 CommandResult object 107 """ 108 return self.builder.do_make(commit, brd, stage, cwd, *args, 109 **kwargs) 110 111 def RunCommit(self, commit_upto, brd, work_dir, do_config, force_build, 112 force_build_failures): 113 """Build a particular commit. 114 115 If the build is already done, and we are not forcing a build, we skip 116 the build and just return the previously-saved results. 117 118 Args: 119 commit_upto: Commit number to build (0...n-1) 120 brd: Board object to build 121 work_dir: Directory to which the source will be checked out 122 do_config: True to run a make <board>_defconfig on the source 123 force_build: Force a build even if one was previously done 124 force_build_failures: Force a bulid if the previous result showed 125 failure 126 127 Returns: 128 tuple containing: 129 - CommandResult object containing the results of the build 130 - boolean indicating whether 'make config' is still needed 131 """ 132 # Create a default result - it will be overwritte by the call to 133 # self.Make() below, in the event that we do a build. 134 result = command.CommandResult() 135 result.return_code = 0 136 if self.builder.in_tree: 137 out_dir = work_dir 138 else: 139 out_dir = os.path.join(work_dir, 'build') 140 141 # Check if the job was already completed last time 142 done_file = self.builder.GetDoneFile(commit_upto, brd.target) 143 result.already_done = os.path.exists(done_file) 144 will_build = (force_build or force_build_failures or 145 not result.already_done) 146 if result.already_done: 147 # Get the return code from that build and use it 148 with open(done_file, 'r') as fd: 149 result.return_code = int(fd.readline()) 150 151 # Check the signal that the build needs to be retried 152 if result.return_code == RETURN_CODE_RETRY: 153 will_build = True 154 elif will_build: 155 err_file = self.builder.GetErrFile(commit_upto, brd.target) 156 if os.path.exists(err_file) and os.stat(err_file).st_size: 157 result.stderr = 'bad' 158 elif not force_build: 159 # The build passed, so no need to build it again 160 will_build = False 161 162 if will_build: 163 # We are going to have to build it. First, get a toolchain 164 if not self.toolchain: 165 try: 166 self.toolchain = self.builder.toolchains.Select(brd.arch) 167 except ValueError as err: 168 result.return_code = 10 169 result.stdout = '' 170 result.stderr = str(err) 171 # TODO(sjg@chromium.org): This gets swallowed, but needs 172 # to be reported. 173 174 if self.toolchain: 175 # Checkout the right commit 176 if self.builder.commits: 177 commit = self.builder.commits[commit_upto] 178 if self.builder.checkout: 179 git_dir = os.path.join(work_dir, '.git') 180 gitutil.Checkout(commit.hash, git_dir, work_dir, 181 force=True) 182 else: 183 commit = 'current' 184 185 # Set up the environment and command line 186 env = self.toolchain.MakeEnvironment(self.builder.full_path) 187 Mkdir(out_dir) 188 args = [] 189 cwd = work_dir 190 src_dir = os.path.realpath(work_dir) 191 if not self.builder.in_tree: 192 if commit_upto is None: 193 # In this case we are building in the original source 194 # directory (i.e. the current directory where buildman 195 # is invoked. The output directory is set to this 196 # thread's selected work directory. 197 # 198 # Symlinks can confuse U-Boot's Makefile since 199 # we may use '..' in our path, so remove them. 200 work_dir = os.path.realpath(work_dir) 201 args.append('O=%s/build' % work_dir) 202 cwd = None 203 src_dir = os.getcwd() 204 else: 205 args.append('O=build') 206 if not self.builder.verbose_build: 207 args.append('-s') 208 if self.builder.num_jobs is not None: 209 args.extend(['-j', str(self.builder.num_jobs)]) 210 config_args = ['%s_defconfig' % brd.target] 211 config_out = '' 212 args.extend(self.builder.toolchains.GetMakeArguments(brd)) 213 214 # If we need to reconfigure, do that now 215 if do_config: 216 result = self.Make(commit, brd, 'mrproper', cwd, 217 'mrproper', *args, env=env) 218 config_out = result.combined 219 result = self.Make(commit, brd, 'config', cwd, 220 *(args + config_args), env=env) 221 config_out += result.combined 222 do_config = False # No need to configure next time 223 if result.return_code == 0: 224 result = self.Make(commit, brd, 'build', cwd, *args, 225 env=env) 226 result.stderr = result.stderr.replace(src_dir + '/', '') 227 if self.builder.verbose_build: 228 result.stdout = config_out + result.stdout 229 else: 230 result.return_code = 1 231 result.stderr = 'No tool chain for %s\n' % brd.arch 232 result.already_done = False 233 234 result.toolchain = self.toolchain 235 result.brd = brd 236 result.commit_upto = commit_upto 237 result.out_dir = out_dir 238 return result, do_config 239 240 def _WriteResult(self, result, keep_outputs): 241 """Write a built result to the output directory. 242 243 Args: 244 result: CommandResult object containing result to write 245 keep_outputs: True to store the output binaries, False 246 to delete them 247 """ 248 # Fatal error 249 if result.return_code < 0: 250 return 251 252 # If we think this might have been aborted with Ctrl-C, record the 253 # failure but not that we are 'done' with this board. A retry may fix 254 # it. 255 maybe_aborted = result.stderr and 'No child processes' in result.stderr 256 257 if result.already_done: 258 return 259 260 # Write the output and stderr 261 output_dir = self.builder._GetOutputDir(result.commit_upto) 262 Mkdir(output_dir) 263 build_dir = self.builder.GetBuildDir(result.commit_upto, 264 result.brd.target) 265 Mkdir(build_dir) 266 267 outfile = os.path.join(build_dir, 'log') 268 with open(outfile, 'w') as fd: 269 if result.stdout: 270 fd.write(result.stdout) 271 272 errfile = self.builder.GetErrFile(result.commit_upto, 273 result.brd.target) 274 if result.stderr: 275 with open(errfile, 'w') as fd: 276 fd.write(result.stderr) 277 elif os.path.exists(errfile): 278 os.remove(errfile) 279 280 if result.toolchain: 281 # Write the build result and toolchain information. 282 done_file = self.builder.GetDoneFile(result.commit_upto, 283 result.brd.target) 284 with open(done_file, 'w') as fd: 285 if maybe_aborted: 286 # Special code to indicate we need to retry 287 fd.write('%s' % RETURN_CODE_RETRY) 288 else: 289 fd.write('%s' % result.return_code) 290 with open(os.path.join(build_dir, 'toolchain'), 'w') as fd: 291 print >>fd, 'gcc', result.toolchain.gcc 292 print >>fd, 'path', result.toolchain.path 293 print >>fd, 'cross', result.toolchain.cross 294 print >>fd, 'arch', result.toolchain.arch 295 fd.write('%s' % result.return_code) 296 297 with open(os.path.join(build_dir, 'toolchain'), 'w') as fd: 298 print >>fd, 'gcc', result.toolchain.gcc 299 print >>fd, 'path', result.toolchain.path 300 301 # Write out the image and function size information and an objdump 302 env = result.toolchain.MakeEnvironment(self.builder.full_path) 303 lines = [] 304 for fname in ['u-boot', 'spl/u-boot-spl']: 305 cmd = ['%snm' % self.toolchain.cross, '--size-sort', fname] 306 nm_result = command.RunPipe([cmd], capture=True, 307 capture_stderr=True, cwd=result.out_dir, 308 raise_on_error=False, env=env) 309 if nm_result.stdout: 310 nm = self.builder.GetFuncSizesFile(result.commit_upto, 311 result.brd.target, fname) 312 with open(nm, 'w') as fd: 313 print >>fd, nm_result.stdout, 314 315 cmd = ['%sobjdump' % self.toolchain.cross, '-h', fname] 316 dump_result = command.RunPipe([cmd], capture=True, 317 capture_stderr=True, cwd=result.out_dir, 318 raise_on_error=False, env=env) 319 rodata_size = '' 320 if dump_result.stdout: 321 objdump = self.builder.GetObjdumpFile(result.commit_upto, 322 result.brd.target, fname) 323 with open(objdump, 'w') as fd: 324 print >>fd, dump_result.stdout, 325 for line in dump_result.stdout.splitlines(): 326 fields = line.split() 327 if len(fields) > 5 and fields[1] == '.rodata': 328 rodata_size = fields[2] 329 330 cmd = ['%ssize' % self.toolchain.cross, fname] 331 size_result = command.RunPipe([cmd], capture=True, 332 capture_stderr=True, cwd=result.out_dir, 333 raise_on_error=False, env=env) 334 if size_result.stdout: 335 lines.append(size_result.stdout.splitlines()[1] + ' ' + 336 rodata_size) 337 338 # Write out the image sizes file. This is similar to the output 339 # of binutil's 'size' utility, but it omits the header line and 340 # adds an additional hex value at the end of each line for the 341 # rodata size 342 if len(lines): 343 sizes = self.builder.GetSizesFile(result.commit_upto, 344 result.brd.target) 345 with open(sizes, 'w') as fd: 346 print >>fd, '\n'.join(lines) 347 348 # Write out the configuration files, with a special case for SPL 349 for dirname in ['', 'spl', 'tpl']: 350 self.CopyFiles(result.out_dir, build_dir, dirname, ['u-boot.cfg', 351 'spl/u-boot-spl.cfg', 'tpl/u-boot-tpl.cfg', '.config', 352 'include/autoconf.mk', 'include/generated/autoconf.h']) 353 354 # Now write the actual build output 355 if keep_outputs: 356 self.CopyFiles(result.out_dir, build_dir, '', ['u-boot*', '*.bin', 357 '*.map', '*.img', 'MLO', 'include/autoconf.mk', 358 'spl/u-boot-spl*']) 359 360 def CopyFiles(self, out_dir, build_dir, dirname, patterns): 361 """Copy files from the build directory to the output. 362 363 Args: 364 out_dir: Path to output directory containing the files 365 build_dir: Place to copy the files 366 dirname: Source directory, '' for normal U-Boot, 'spl' for SPL 367 patterns: A list of filenames (strings) to copy, each relative 368 to the build directory 369 """ 370 for pattern in patterns: 371 file_list = glob.glob(os.path.join(out_dir, dirname, pattern)) 372 for fname in file_list: 373 target = os.path.basename(fname) 374 if dirname: 375 base, ext = os.path.splitext(target) 376 if ext: 377 target = '%s-%s%s' % (base, dirname, ext) 378 shutil.copy(fname, os.path.join(build_dir, target)) 379 380 def RunJob(self, job): 381 """Run a single job 382 383 A job consists of a building a list of commits for a particular board. 384 385 Args: 386 job: Job to build 387 """ 388 brd = job.board 389 work_dir = self.builder.GetThreadDir(self.thread_num) 390 self.toolchain = None 391 if job.commits: 392 # Run 'make board_defconfig' on the first commit 393 do_config = True 394 commit_upto = 0 395 force_build = False 396 for commit_upto in range(0, len(job.commits), job.step): 397 result, request_config = self.RunCommit(commit_upto, brd, 398 work_dir, do_config, 399 force_build or self.builder.force_build, 400 self.builder.force_build_failures) 401 failed = result.return_code or result.stderr 402 did_config = do_config 403 if failed and not do_config: 404 # If our incremental build failed, try building again 405 # with a reconfig. 406 if self.builder.force_config_on_failure: 407 result, request_config = self.RunCommit(commit_upto, 408 brd, work_dir, True, True, False) 409 did_config = True 410 if not self.builder.force_reconfig: 411 do_config = request_config 412 413 # If we built that commit, then config is done. But if we got 414 # an warning, reconfig next time to force it to build the same 415 # files that created warnings this time. Otherwise an 416 # incremental build may not build the same file, and we will 417 # think that the warning has gone away. 418 # We could avoid this by using -Werror everywhere... 419 # For errors, the problem doesn't happen, since presumably 420 # the build stopped and didn't generate output, so will retry 421 # that file next time. So we could detect warnings and deal 422 # with them specially here. For now, we just reconfigure if 423 # anything goes work. 424 # Of course this is substantially slower if there are build 425 # errors/warnings (e.g. 2-3x slower even if only 10% of builds 426 # have problems). 427 if (failed and not result.already_done and not did_config and 428 self.builder.force_config_on_failure): 429 # If this build failed, try the next one with a 430 # reconfigure. 431 # Sometimes if the board_config.h file changes it can mess 432 # with dependencies, and we get: 433 # make: *** No rule to make target `include/autoconf.mk', 434 # needed by `depend'. 435 do_config = True 436 force_build = True 437 else: 438 force_build = False 439 if self.builder.force_config_on_failure: 440 if failed: 441 do_config = True 442 result.commit_upto = commit_upto 443 if result.return_code < 0: 444 raise ValueError('Interrupt') 445 446 # We have the build results, so output the result 447 self._WriteResult(result, job.keep_outputs) 448 self.builder.out_queue.put(result) 449 else: 450 # Just build the currently checked-out build 451 result, request_config = self.RunCommit(None, brd, work_dir, True, 452 True, self.builder.force_build_failures) 453 result.commit_upto = 0 454 self._WriteResult(result, job.keep_outputs) 455 self.builder.out_queue.put(result) 456 457 def run(self): 458 """Our thread's run function 459 460 This thread picks a job from the queue, runs it, and then goes to the 461 next job. 462 """ 463 alive = True 464 while True: 465 job = self.builder.queue.get() 466 if self.builder.active and alive: 467 self.RunJob(job) 468 ''' 469 try: 470 if self.builder.active and alive: 471 self.RunJob(job) 472 except Exception as err: 473 alive = False 474 print err 475 ''' 476 self.builder.queue.task_done() 477