1# Copyright (c) 2013 The Chromium OS Authors. 2# 3# SPDX-License-Identifier: GPL-2.0+ 4# 5 6import multiprocessing 7import os 8import shutil 9import sys 10 11import board 12import bsettings 13from builder import Builder 14import gitutil 15import patchstream 16import terminal 17from terminal import Print 18import toolchain 19import command 20import subprocess 21 22def GetPlural(count): 23 """Returns a plural 's' if count is not 1""" 24 return 's' if count != 1 else '' 25 26def GetActionSummary(is_summary, commits, selected, options): 27 """Return a string summarising the intended action. 28 29 Returns: 30 Summary string. 31 """ 32 if commits: 33 count = len(commits) 34 count = (count + options.step - 1) / options.step 35 commit_str = '%d commit%s' % (count, GetPlural(count)) 36 else: 37 commit_str = 'current source' 38 str = '%s %s for %d boards' % ( 39 'Summary of' if is_summary else 'Building', commit_str, 40 len(selected)) 41 str += ' (%d thread%s, %d job%s per thread)' % (options.threads, 42 GetPlural(options.threads), options.jobs, GetPlural(options.jobs)) 43 return str 44 45def ShowActions(series, why_selected, boards_selected, builder, options): 46 """Display a list of actions that we would take, if not a dry run. 47 48 Args: 49 series: Series object 50 why_selected: Dictionary where each key is a buildman argument 51 provided by the user, and the value is the boards brought 52 in by that argument. For example, 'arm' might bring in 53 400 boards, so in this case the key would be 'arm' and 54 the value would be a list of board names. 55 boards_selected: Dict of selected boards, key is target name, 56 value is Board object 57 builder: The builder that will be used to build the commits 58 options: Command line options object 59 """ 60 col = terminal.Color() 61 print 'Dry run, so not doing much. But I would do this:' 62 print 63 if series: 64 commits = series.commits 65 else: 66 commits = None 67 print GetActionSummary(False, commits, boards_selected, 68 options) 69 print 'Build directory: %s' % builder.base_dir 70 if commits: 71 for upto in range(0, len(series.commits), options.step): 72 commit = series.commits[upto] 73 print ' ', col.Color(col.YELLOW, commit.hash[:8], bright=False), 74 print commit.subject 75 print 76 for arg in why_selected: 77 if arg != 'all': 78 print arg, ': %d boards' % why_selected[arg] 79 print ('Total boards to build for each commit: %d\n' % 80 why_selected['all']) 81 82def DoBuildman(options, args, toolchains=None, make_func=None, boards=None, 83 clean_dir=False): 84 """The main control code for buildman 85 86 Args: 87 options: Command line options object 88 args: Command line arguments (list of strings) 89 toolchains: Toolchains to use - this should be a Toolchains() 90 object. If None, then it will be created and scanned 91 make_func: Make function to use for the builder. This is called 92 to execute 'make'. If this is None, the normal function 93 will be used, which calls the 'make' tool with suitable 94 arguments. This setting is useful for tests. 95 board: Boards() object to use, containing a list of available 96 boards. If this is None it will be created and scanned. 97 """ 98 global builder 99 100 if options.full_help: 101 pager = os.getenv('PAGER') 102 if not pager: 103 pager = 'more' 104 fname = os.path.join(os.path.dirname(sys.argv[0]), 'README') 105 command.Run(pager, fname) 106 return 0 107 108 gitutil.Setup() 109 110 options.git_dir = os.path.join(options.git, '.git') 111 112 if not toolchains: 113 toolchains = toolchain.Toolchains() 114 toolchains.GetSettings() 115 toolchains.Scan(options.list_tool_chains) 116 if options.list_tool_chains: 117 toolchains.List() 118 print 119 return 0 120 121 # Work out how many commits to build. We want to build everything on the 122 # branch. We also build the upstream commit as a control so we can see 123 # problems introduced by the first commit on the branch. 124 col = terminal.Color() 125 count = options.count 126 has_range = options.branch and '..' in options.branch 127 if count == -1: 128 if not options.branch: 129 count = 1 130 else: 131 if has_range: 132 count, msg = gitutil.CountCommitsInRange(options.git_dir, 133 options.branch) 134 else: 135 count, msg = gitutil.CountCommitsInBranch(options.git_dir, 136 options.branch) 137 if count is None: 138 sys.exit(col.Color(col.RED, msg)) 139 elif count == 0: 140 sys.exit(col.Color(col.RED, "Range '%s' has no commits" % 141 options.branch)) 142 if msg: 143 print col.Color(col.YELLOW, msg) 144 count += 1 # Build upstream commit also 145 146 if not count: 147 str = ("No commits found to process in branch '%s': " 148 "set branch's upstream or use -c flag" % options.branch) 149 sys.exit(col.Color(col.RED, str)) 150 151 # Work out what subset of the boards we are building 152 if not boards: 153 board_file = os.path.join(options.git, 'boards.cfg') 154 status = subprocess.call([os.path.join(options.git, 155 'tools/genboardscfg.py')]) 156 if status != 0: 157 sys.exit("Failed to generate boards.cfg") 158 159 boards = board.Boards() 160 boards.ReadBoards(os.path.join(options.git, 'boards.cfg')) 161 162 exclude = [] 163 if options.exclude: 164 for arg in options.exclude: 165 exclude += arg.split(',') 166 167 why_selected = boards.SelectBoards(args, exclude) 168 selected = boards.GetSelected() 169 if not len(selected): 170 sys.exit(col.Color(col.RED, 'No matching boards found')) 171 172 # Read the metadata from the commits. First look at the upstream commit, 173 # then the ones in the branch. We would like to do something like 174 # upstream/master~..branch but that isn't possible if upstream/master is 175 # a merge commit (it will list all the commits that form part of the 176 # merge) 177 # Conflicting tags are not a problem for buildman, since it does not use 178 # them. For example, Series-version is not useful for buildman. On the 179 # other hand conflicting tags will cause an error. So allow later tags 180 # to overwrite earlier ones by setting allow_overwrite=True 181 if options.branch: 182 if count == -1: 183 if has_range: 184 range_expr = options.branch 185 else: 186 range_expr = gitutil.GetRangeInBranch(options.git_dir, 187 options.branch) 188 upstream_commit = gitutil.GetUpstream(options.git_dir, 189 options.branch) 190 series = patchstream.GetMetaDataForList(upstream_commit, 191 options.git_dir, 1, series=None, allow_overwrite=True) 192 193 series = patchstream.GetMetaDataForList(range_expr, 194 options.git_dir, None, series, allow_overwrite=True) 195 else: 196 # Honour the count 197 series = patchstream.GetMetaDataForList(options.branch, 198 options.git_dir, count, series=None, allow_overwrite=True) 199 else: 200 series = None 201 options.verbose = True 202 if not options.summary: 203 options.show_errors = True 204 205 # By default we have one thread per CPU. But if there are not enough jobs 206 # we can have fewer threads and use a high '-j' value for make. 207 if not options.threads: 208 options.threads = min(multiprocessing.cpu_count(), len(selected)) 209 if not options.jobs: 210 options.jobs = max(1, (multiprocessing.cpu_count() + 211 len(selected) - 1) / len(selected)) 212 213 if not options.step: 214 options.step = len(series.commits) - 1 215 216 gnu_make = command.Output(os.path.join(options.git, 217 'scripts/show-gnu-make')).rstrip() 218 if not gnu_make: 219 sys.exit('GNU Make not found') 220 221 # Create a new builder with the selected options. 222 output_dir = options.output_dir 223 if options.branch: 224 dirname = options.branch.replace('/', '_') 225 # As a special case allow the board directory to be placed in the 226 # output directory itself rather than any subdirectory. 227 if not options.no_subdirs: 228 output_dir = os.path.join(options.output_dir, dirname) 229 if (clean_dir and output_dir != options.output_dir and 230 os.path.exists(output_dir)): 231 shutil.rmtree(output_dir) 232 builder = Builder(toolchains, output_dir, options.git_dir, 233 options.threads, options.jobs, gnu_make=gnu_make, checkout=True, 234 show_unknown=options.show_unknown, step=options.step, 235 no_subdirs=options.no_subdirs) 236 builder.force_config_on_failure = not options.quick 237 if make_func: 238 builder.do_make = make_func 239 240 # For a dry run, just show our actions as a sanity check 241 if options.dry_run: 242 ShowActions(series, why_selected, selected, builder, options) 243 else: 244 builder.force_build = options.force_build 245 builder.force_build_failures = options.force_build_failures 246 builder.force_reconfig = options.force_reconfig 247 builder.in_tree = options.in_tree 248 249 # Work out which boards to build 250 board_selected = boards.GetSelectedDict() 251 252 if series: 253 commits = series.commits 254 # Number the commits for test purposes 255 for commit in range(len(commits)): 256 commits[commit].sequence = commit 257 else: 258 commits = None 259 260 Print(GetActionSummary(options.summary, commits, board_selected, 261 options)) 262 263 # We can't show function sizes without board details at present 264 if options.show_bloat: 265 options.show_detail = True 266 builder.SetDisplayOptions(options.show_errors, options.show_sizes, 267 options.show_detail, options.show_bloat, 268 options.list_error_boards) 269 if options.summary: 270 builder.ShowSummary(commits, board_selected) 271 else: 272 fail, warned = builder.BuildBoards(commits, board_selected, 273 options.keep_outputs, options.verbose) 274 if fail: 275 return 128 276 elif warned: 277 return 129 278 return 0 279