1# Copyright (c) 2013 The Chromium OS Authors. 2# 3# SPDX-License-Identifier: GPL-2.0+ 4# 5 6import multiprocessing 7import os 8import shutil 9import sys 10 11import board 12import bsettings 13from builder import Builder 14import gitutil 15import patchstream 16import terminal 17from terminal import Print 18import toolchain 19import command 20import subprocess 21 22def GetPlural(count): 23 """Returns a plural 's' if count is not 1""" 24 return 's' if count != 1 else '' 25 26def GetActionSummary(is_summary, commits, selected, options): 27 """Return a string summarising the intended action. 28 29 Returns: 30 Summary string. 31 """ 32 if commits: 33 count = len(commits) 34 count = (count + options.step - 1) / options.step 35 commit_str = '%d commit%s' % (count, GetPlural(count)) 36 else: 37 commit_str = 'current source' 38 str = '%s %s for %d boards' % ( 39 'Summary of' if is_summary else 'Building', commit_str, 40 len(selected)) 41 str += ' (%d thread%s, %d job%s per thread)' % (options.threads, 42 GetPlural(options.threads), options.jobs, GetPlural(options.jobs)) 43 return str 44 45def ShowActions(series, why_selected, boards_selected, builder, options): 46 """Display a list of actions that we would take, if not a dry run. 47 48 Args: 49 series: Series object 50 why_selected: Dictionary where each key is a buildman argument 51 provided by the user, and the value is the boards brought 52 in by that argument. For example, 'arm' might bring in 53 400 boards, so in this case the key would be 'arm' and 54 the value would be a list of board names. 55 boards_selected: Dict of selected boards, key is target name, 56 value is Board object 57 builder: The builder that will be used to build the commits 58 options: Command line options object 59 """ 60 col = terminal.Color() 61 print 'Dry run, so not doing much. But I would do this:' 62 print 63 if series: 64 commits = series.commits 65 else: 66 commits = None 67 print GetActionSummary(False, commits, boards_selected, 68 options) 69 print 'Build directory: %s' % builder.base_dir 70 if commits: 71 for upto in range(0, len(series.commits), options.step): 72 commit = series.commits[upto] 73 print ' ', col.Color(col.YELLOW, commit.hash[:8], bright=False), 74 print commit.subject 75 print 76 for arg in why_selected: 77 if arg != 'all': 78 print arg, ': %d boards' % why_selected[arg] 79 print ('Total boards to build for each commit: %d\n' % 80 why_selected['all']) 81 82def DoBuildman(options, args, toolchains=None, make_func=None, boards=None, 83 clean_dir=False): 84 """The main control code for buildman 85 86 Args: 87 options: Command line options object 88 args: Command line arguments (list of strings) 89 toolchains: Toolchains to use - this should be a Toolchains() 90 object. If None, then it will be created and scanned 91 make_func: Make function to use for the builder. This is called 92 to execute 'make'. If this is None, the normal function 93 will be used, which calls the 'make' tool with suitable 94 arguments. This setting is useful for tests. 95 board: Boards() object to use, containing a list of available 96 boards. If this is None it will be created and scanned. 97 """ 98 global builder 99 100 if options.full_help: 101 pager = os.getenv('PAGER') 102 if not pager: 103 pager = 'more' 104 fname = os.path.join(os.path.dirname(sys.argv[0]), 'README') 105 command.Run(pager, fname) 106 return 0 107 108 gitutil.Setup() 109 110 options.git_dir = os.path.join(options.git, '.git') 111 112 if not toolchains: 113 toolchains = toolchain.Toolchains() 114 toolchains.GetSettings() 115 toolchains.Scan(options.list_tool_chains) 116 if options.list_tool_chains: 117 toolchains.List() 118 print 119 return 0 120 121 # Work out how many commits to build. We want to build everything on the 122 # branch. We also build the upstream commit as a control so we can see 123 # problems introduced by the first commit on the branch. 124 col = terminal.Color() 125 count = options.count 126 if count == -1: 127 if not options.branch: 128 count = 1 129 else: 130 count, msg = gitutil.CountCommitsInBranch(options.git_dir, 131 options.branch) 132 if count is None: 133 sys.exit(col.Color(col.RED, msg)) 134 if msg: 135 print col.Color(col.YELLOW, msg) 136 count += 1 # Build upstream commit also 137 138 if not count: 139 str = ("No commits found to process in branch '%s': " 140 "set branch's upstream or use -c flag" % options.branch) 141 sys.exit(col.Color(col.RED, str)) 142 143 # Work out what subset of the boards we are building 144 if not boards: 145 board_file = os.path.join(options.git, 'boards.cfg') 146 status = subprocess.call([os.path.join(options.git, 147 'tools/genboardscfg.py')]) 148 if status != 0: 149 sys.exit("Failed to generate boards.cfg") 150 151 boards = board.Boards() 152 boards.ReadBoards(os.path.join(options.git, 'boards.cfg')) 153 154 exclude = [] 155 if options.exclude: 156 for arg in options.exclude: 157 exclude += arg.split(',') 158 159 why_selected = boards.SelectBoards(args, exclude) 160 selected = boards.GetSelected() 161 if not len(selected): 162 sys.exit(col.Color(col.RED, 'No matching boards found')) 163 164 # Read the metadata from the commits. First look at the upstream commit, 165 # then the ones in the branch. We would like to do something like 166 # upstream/master~..branch but that isn't possible if upstream/master is 167 # a merge commit (it will list all the commits that form part of the 168 # merge) 169 # Conflicting tags are not a problem for buildman, since it does not use 170 # them. For example, Series-version is not useful for buildman. On the 171 # other hand conflicting tags will cause an error. So allow later tags 172 # to overwrite earlier ones by setting allow_overwrite=True 173 if options.branch: 174 if count == -1: 175 range_expr = gitutil.GetRangeInBranch(options.git_dir, 176 options.branch) 177 upstream_commit = gitutil.GetUpstream(options.git_dir, 178 options.branch) 179 series = patchstream.GetMetaDataForList(upstream_commit, 180 options.git_dir, 1, series=None, allow_overwrite=True) 181 182 series = patchstream.GetMetaDataForList(range_expr, 183 options.git_dir, None, series, allow_overwrite=True) 184 else: 185 # Honour the count 186 series = patchstream.GetMetaDataForList(options.branch, 187 options.git_dir, count, series=None, allow_overwrite=True) 188 else: 189 series = None 190 options.verbose = True 191 if not options.summary: 192 options.show_errors = True 193 194 # By default we have one thread per CPU. But if there are not enough jobs 195 # we can have fewer threads and use a high '-j' value for make. 196 if not options.threads: 197 options.threads = min(multiprocessing.cpu_count(), len(selected)) 198 if not options.jobs: 199 options.jobs = max(1, (multiprocessing.cpu_count() + 200 len(selected) - 1) / len(selected)) 201 202 if not options.step: 203 options.step = len(series.commits) - 1 204 205 gnu_make = command.Output(os.path.join(options.git, 206 'scripts/show-gnu-make')).rstrip() 207 if not gnu_make: 208 sys.exit('GNU Make not found') 209 210 # Create a new builder with the selected options. 211 output_dir = options.output_dir 212 if options.branch: 213 dirname = options.branch.replace('/', '_') 214 # As a special case allow the board directory to be placed in the 215 # output directory itself rather than any subdirectory. 216 if not options.no_subdirs: 217 output_dir = os.path.join(options.output_dir, dirname) 218 if (clean_dir and output_dir != options.output_dir and 219 os.path.exists(output_dir)): 220 shutil.rmtree(output_dir) 221 builder = Builder(toolchains, output_dir, options.git_dir, 222 options.threads, options.jobs, gnu_make=gnu_make, checkout=True, 223 show_unknown=options.show_unknown, step=options.step, 224 no_subdirs=options.no_subdirs) 225 builder.force_config_on_failure = not options.quick 226 if make_func: 227 builder.do_make = make_func 228 229 # For a dry run, just show our actions as a sanity check 230 if options.dry_run: 231 ShowActions(series, why_selected, selected, builder, options) 232 else: 233 builder.force_build = options.force_build 234 builder.force_build_failures = options.force_build_failures 235 builder.force_reconfig = options.force_reconfig 236 builder.in_tree = options.in_tree 237 238 # Work out which boards to build 239 board_selected = boards.GetSelectedDict() 240 241 if series: 242 commits = series.commits 243 # Number the commits for test purposes 244 for commit in range(len(commits)): 245 commits[commit].sequence = commit 246 else: 247 commits = None 248 249 Print(GetActionSummary(options.summary, commits, board_selected, 250 options)) 251 252 # We can't show function sizes without board details at present 253 if options.show_bloat: 254 options.show_detail = True 255 builder.SetDisplayOptions(options.show_errors, options.show_sizes, 256 options.show_detail, options.show_bloat, 257 options.list_error_boards) 258 if options.summary: 259 builder.ShowSummary(commits, board_selected) 260 else: 261 fail, warned = builder.BuildBoards(commits, board_selected, 262 options.keep_outputs, options.verbose) 263 if fail: 264 return 128 265 elif warned: 266 return 129 267 return 0 268