1# Copyright (c) 2013 The Chromium OS Authors. 2# 3# SPDX-License-Identifier: GPL-2.0+ 4# 5 6import multiprocessing 7import os 8import shutil 9import sys 10 11import board 12import bsettings 13from builder import Builder 14import gitutil 15import patchstream 16import terminal 17from terminal import Print 18import toolchain 19import command 20import subprocess 21 22def GetPlural(count): 23 """Returns a plural 's' if count is not 1""" 24 return 's' if count != 1 else '' 25 26def GetActionSummary(is_summary, commits, selected, options): 27 """Return a string summarising the intended action. 28 29 Returns: 30 Summary string. 31 """ 32 if commits: 33 count = len(commits) 34 count = (count + options.step - 1) / options.step 35 commit_str = '%d commit%s' % (count, GetPlural(count)) 36 else: 37 commit_str = 'current source' 38 str = '%s %s for %d boards' % ( 39 'Summary of' if is_summary else 'Building', commit_str, 40 len(selected)) 41 str += ' (%d thread%s, %d job%s per thread)' % (options.threads, 42 GetPlural(options.threads), options.jobs, GetPlural(options.jobs)) 43 return str 44 45def ShowActions(series, why_selected, boards_selected, builder, options): 46 """Display a list of actions that we would take, if not a dry run. 47 48 Args: 49 series: Series object 50 why_selected: Dictionary where each key is a buildman argument 51 provided by the user, and the value is the boards brought 52 in by that argument. For example, 'arm' might bring in 53 400 boards, so in this case the key would be 'arm' and 54 the value would be a list of board names. 55 boards_selected: Dict of selected boards, key is target name, 56 value is Board object 57 builder: The builder that will be used to build the commits 58 options: Command line options object 59 """ 60 col = terminal.Color() 61 print 'Dry run, so not doing much. But I would do this:' 62 print 63 if series: 64 commits = series.commits 65 else: 66 commits = None 67 print GetActionSummary(False, commits, boards_selected, 68 options) 69 print 'Build directory: %s' % builder.base_dir 70 if commits: 71 for upto in range(0, len(series.commits), options.step): 72 commit = series.commits[upto] 73 print ' ', col.Color(col.YELLOW, commit.hash[:8], bright=False), 74 print commit.subject 75 print 76 for arg in why_selected: 77 if arg != 'all': 78 print arg, ': %d boards' % why_selected[arg] 79 print ('Total boards to build for each commit: %d\n' % 80 why_selected['all']) 81 82def DoBuildman(options, args, toolchains=None, make_func=None, boards=None, 83 clean_dir=False): 84 """The main control code for buildman 85 86 Args: 87 options: Command line options object 88 args: Command line arguments (list of strings) 89 toolchains: Toolchains to use - this should be a Toolchains() 90 object. If None, then it will be created and scanned 91 make_func: Make function to use for the builder. This is called 92 to execute 'make'. If this is None, the normal function 93 will be used, which calls the 'make' tool with suitable 94 arguments. This setting is useful for tests. 95 board: Boards() object to use, containing a list of available 96 boards. If this is None it will be created and scanned. 97 """ 98 global builder 99 100 if options.full_help: 101 pager = os.getenv('PAGER') 102 if not pager: 103 pager = 'more' 104 fname = os.path.join(os.path.dirname(os.path.realpath(sys.argv[0])), 105 'README') 106 command.Run(pager, fname) 107 return 0 108 109 gitutil.Setup() 110 col = terminal.Color() 111 112 options.git_dir = os.path.join(options.git, '.git') 113 114 if not toolchains: 115 toolchains = toolchain.Toolchains() 116 toolchains.GetSettings() 117 toolchains.Scan(options.list_tool_chains) 118 if options.list_tool_chains: 119 toolchains.List() 120 print 121 return 0 122 123 if options.fetch_arch: 124 if options.fetch_arch == 'list': 125 sorted_list = toolchains.ListArchs() 126 print col.Color(col.BLUE, 'Available architectures: %s\n' % 127 ' '.join(sorted_list)) 128 return 0 129 else: 130 fetch_arch = options.fetch_arch 131 if fetch_arch == 'all': 132 fetch_arch = ','.join(toolchains.ListArchs()) 133 print col.Color(col.CYAN, '\nDownloading toolchains: %s' % 134 fetch_arch) 135 for arch in fetch_arch.split(','): 136 print 137 ret = toolchains.FetchAndInstall(arch) 138 if ret: 139 return ret 140 return 0 141 142 # Work out how many commits to build. We want to build everything on the 143 # branch. We also build the upstream commit as a control so we can see 144 # problems introduced by the first commit on the branch. 145 count = options.count 146 has_range = options.branch and '..' in options.branch 147 if count == -1: 148 if not options.branch: 149 count = 1 150 else: 151 if has_range: 152 count, msg = gitutil.CountCommitsInRange(options.git_dir, 153 options.branch) 154 else: 155 count, msg = gitutil.CountCommitsInBranch(options.git_dir, 156 options.branch) 157 if count is None: 158 sys.exit(col.Color(col.RED, msg)) 159 elif count == 0: 160 sys.exit(col.Color(col.RED, "Range '%s' has no commits" % 161 options.branch)) 162 if msg: 163 print col.Color(col.YELLOW, msg) 164 count += 1 # Build upstream commit also 165 166 if not count: 167 str = ("No commits found to process in branch '%s': " 168 "set branch's upstream or use -c flag" % options.branch) 169 sys.exit(col.Color(col.RED, str)) 170 171 # Work out what subset of the boards we are building 172 if not boards: 173 board_file = os.path.join(options.git, 'boards.cfg') 174 status = subprocess.call([os.path.join(options.git, 175 'tools/genboardscfg.py')]) 176 if status != 0: 177 sys.exit("Failed to generate boards.cfg") 178 179 boards = board.Boards() 180 boards.ReadBoards(os.path.join(options.git, 'boards.cfg')) 181 182 exclude = [] 183 if options.exclude: 184 for arg in options.exclude: 185 exclude += arg.split(',') 186 187 why_selected = boards.SelectBoards(args, exclude) 188 selected = boards.GetSelected() 189 if not len(selected): 190 sys.exit(col.Color(col.RED, 'No matching boards found')) 191 192 # Read the metadata from the commits. First look at the upstream commit, 193 # then the ones in the branch. We would like to do something like 194 # upstream/master~..branch but that isn't possible if upstream/master is 195 # a merge commit (it will list all the commits that form part of the 196 # merge) 197 # Conflicting tags are not a problem for buildman, since it does not use 198 # them. For example, Series-version is not useful for buildman. On the 199 # other hand conflicting tags will cause an error. So allow later tags 200 # to overwrite earlier ones by setting allow_overwrite=True 201 if options.branch: 202 if count == -1: 203 if has_range: 204 range_expr = options.branch 205 else: 206 range_expr = gitutil.GetRangeInBranch(options.git_dir, 207 options.branch) 208 upstream_commit = gitutil.GetUpstream(options.git_dir, 209 options.branch) 210 series = patchstream.GetMetaDataForList(upstream_commit, 211 options.git_dir, 1, series=None, allow_overwrite=True) 212 213 series = patchstream.GetMetaDataForList(range_expr, 214 options.git_dir, None, series, allow_overwrite=True) 215 else: 216 # Honour the count 217 series = patchstream.GetMetaDataForList(options.branch, 218 options.git_dir, count, series=None, allow_overwrite=True) 219 else: 220 series = None 221 options.verbose = True 222 if not options.summary: 223 options.show_errors = True 224 225 # By default we have one thread per CPU. But if there are not enough jobs 226 # we can have fewer threads and use a high '-j' value for make. 227 if not options.threads: 228 options.threads = min(multiprocessing.cpu_count(), len(selected)) 229 if not options.jobs: 230 options.jobs = max(1, (multiprocessing.cpu_count() + 231 len(selected) - 1) / len(selected)) 232 233 if not options.step: 234 options.step = len(series.commits) - 1 235 236 gnu_make = command.Output(os.path.join(options.git, 237 'scripts/show-gnu-make')).rstrip() 238 if not gnu_make: 239 sys.exit('GNU Make not found') 240 241 # Create a new builder with the selected options. 242 output_dir = options.output_dir 243 if options.branch: 244 dirname = options.branch.replace('/', '_') 245 # As a special case allow the board directory to be placed in the 246 # output directory itself rather than any subdirectory. 247 if not options.no_subdirs: 248 output_dir = os.path.join(options.output_dir, dirname) 249 if (clean_dir and output_dir != options.output_dir and 250 os.path.exists(output_dir)): 251 shutil.rmtree(output_dir) 252 builder = Builder(toolchains, output_dir, options.git_dir, 253 options.threads, options.jobs, gnu_make=gnu_make, checkout=True, 254 show_unknown=options.show_unknown, step=options.step, 255 no_subdirs=options.no_subdirs, full_path=options.full_path, 256 verbose_build=options.verbose_build, 257 incremental=options.incremental, 258 per_board_out_dir=options.per_board_out_dir,) 259 builder.force_config_on_failure = not options.quick 260 if make_func: 261 builder.do_make = make_func 262 263 # For a dry run, just show our actions as a sanity check 264 if options.dry_run: 265 ShowActions(series, why_selected, selected, builder, options) 266 else: 267 builder.force_build = options.force_build 268 builder.force_build_failures = options.force_build_failures 269 builder.force_reconfig = options.force_reconfig 270 builder.in_tree = options.in_tree 271 272 # Work out which boards to build 273 board_selected = boards.GetSelectedDict() 274 275 if series: 276 commits = series.commits 277 # Number the commits for test purposes 278 for commit in range(len(commits)): 279 commits[commit].sequence = commit 280 else: 281 commits = None 282 283 Print(GetActionSummary(options.summary, commits, board_selected, 284 options)) 285 286 # We can't show function sizes without board details at present 287 if options.show_bloat: 288 options.show_detail = True 289 builder.SetDisplayOptions(options.show_errors, options.show_sizes, 290 options.show_detail, options.show_bloat, 291 options.list_error_boards, 292 options.show_config) 293 if options.summary: 294 builder.ShowSummary(commits, board_selected) 295 else: 296 fail, warned = builder.BuildBoards(commits, board_selected, 297 options.keep_outputs, options.verbose) 298 if fail: 299 return 128 300 elif warned: 301 return 129 302 return 0 303