1#!/usr/bin/env python3 2# ex:ts=4:sw=4:sts=4:et 3# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- 4 5# Copyright (c) 2013 Wind River Systems, Inc. 6# 7# SPDX-License-Identifier: GPL-2.0-only 8# 9 10import os 11import sys 12import getopt 13import shutil 14import re 15import warnings 16import subprocess 17import argparse 18 19scripts_path = os.path.abspath(os.path.dirname(os.path.abspath(sys.argv[0]))) 20lib_path = scripts_path + '/lib' 21sys.path = sys.path + [lib_path] 22 23import scriptpath 24 25# Figure out where is the bitbake/lib/bb since we need bb.siggen and bb.process 26bitbakepath = scriptpath.add_bitbake_lib_path() 27if not bitbakepath: 28 sys.stderr.write("Unable to find bitbake by searching parent directory of this script or PATH\n") 29 sys.exit(1) 30scriptpath.add_oe_lib_path() 31import argparse_oe 32 33import bb.siggen 34import bb.process 35 36# Match the stamp's filename 37# group(1): PE_PV (may no PE) 38# group(2): PR 39# group(3): TASK 40# group(4): HASH 41stamp_re = re.compile("(?P<pv>.*)-(?P<pr>r\d+)\.(?P<task>do_\w+)\.(?P<hash>[^\.]*)") 42sigdata_re = re.compile(".*\.sigdata\..*") 43 44def gen_dict(stamps): 45 """ 46 Generate the dict from the stamps dir. 47 The output dict format is: 48 {fake_f: {pn: PN, pv: PV, pr: PR, task: TASK, path: PATH}} 49 Where: 50 fake_f: pv + task + hash 51 path: the path to the stamp file 52 """ 53 # The member of the sub dict (A "path" will be appended below) 54 sub_mem = ("pv", "pr", "task") 55 d = {} 56 for dirpath, _, files in os.walk(stamps): 57 for f in files: 58 # The "bitbake -S" would generate ".sigdata", but no "_setscene". 59 fake_f = re.sub('_setscene.', '.', f) 60 fake_f = re.sub('.sigdata', '', fake_f) 61 subdict = {} 62 tmp = stamp_re.match(fake_f) 63 if tmp: 64 for i in sub_mem: 65 subdict[i] = tmp.group(i) 66 if len(subdict) != 0: 67 pn = os.path.basename(dirpath) 68 subdict['pn'] = pn 69 # The path will be used by os.stat() and bb.siggen 70 subdict['path'] = dirpath + "/" + f 71 fake_f = tmp.group('pv') + tmp.group('task') + tmp.group('hash') 72 d[fake_f] = subdict 73 return d 74 75# Re-construct the dict 76def recon_dict(dict_in): 77 """ 78 The output dict format is: 79 {pn_task: {pv: PV, pr: PR, path: PATH}} 80 """ 81 dict_out = {} 82 for k in dict_in.keys(): 83 subdict = {} 84 # The key 85 pn_task = "%s_%s" % (dict_in.get(k).get('pn'), dict_in.get(k).get('task')) 86 # If more than one stamps are found, use the latest one. 87 if pn_task in dict_out: 88 full_path_pre = dict_out.get(pn_task).get('path') 89 full_path_cur = dict_in.get(k).get('path') 90 if os.stat(full_path_pre).st_mtime > os.stat(full_path_cur).st_mtime: 91 continue 92 subdict['pv'] = dict_in.get(k).get('pv') 93 subdict['pr'] = dict_in.get(k).get('pr') 94 subdict['path'] = dict_in.get(k).get('path') 95 dict_out[pn_task] = subdict 96 97 return dict_out 98 99def split_pntask(s): 100 """ 101 Split the pn_task in to (pn, task) and return it 102 """ 103 tmp = re.match("(.*)_(do_.*)", s) 104 return (tmp.group(1), tmp.group(2)) 105 106 107def print_added(d_new = None, d_old = None): 108 """ 109 Print the newly added tasks 110 """ 111 added = {} 112 for k in list(d_new.keys()): 113 if k not in d_old: 114 # Add the new one to added dict, and remove it from 115 # d_new, so the remaining ones are the changed ones 116 added[k] = d_new.get(k) 117 del(d_new[k]) 118 119 if not added: 120 return 0 121 122 # Format the output, the dict format is: 123 # {pn: task1, task2 ...} 124 added_format = {} 125 counter = 0 126 for k in added.keys(): 127 pn, task = split_pntask(k) 128 if pn in added_format: 129 # Append the value 130 added_format[pn] = "%s %s" % (added_format.get(pn), task) 131 else: 132 added_format[pn] = task 133 counter += 1 134 print("=== Newly added tasks: (%s tasks)" % counter) 135 for k in added_format.keys(): 136 print(" %s: %s" % (k, added_format.get(k))) 137 138 return counter 139 140def print_vrchanged(d_new = None, d_old = None, vr = None): 141 """ 142 Print the pv or pr changed tasks. 143 The arg "vr" is "pv" or "pr" 144 """ 145 pvchanged = {} 146 counter = 0 147 for k in list(d_new.keys()): 148 if d_new.get(k).get(vr) != d_old.get(k).get(vr): 149 counter += 1 150 pn, task = split_pntask(k) 151 if pn not in pvchanged: 152 # Format the output, we only print pn (no task) since 153 # all the tasks would be changed when pn or pr changed, 154 # the dict format is: 155 # {pn: pv/pr_old -> pv/pr_new} 156 pvchanged[pn] = "%s -> %s" % (d_old.get(k).get(vr), d_new.get(k).get(vr)) 157 del(d_new[k]) 158 159 if not pvchanged: 160 return 0 161 162 print("\n=== %s changed: (%s tasks)" % (vr.upper(), counter)) 163 for k in pvchanged.keys(): 164 print(" %s: %s" % (k, pvchanged.get(k))) 165 166 return counter 167 168def print_depchanged(d_new = None, d_old = None, verbose = False): 169 """ 170 Print the dependency changes 171 """ 172 depchanged = {} 173 counter = 0 174 for k in d_new.keys(): 175 counter += 1 176 pn, task = split_pntask(k) 177 if (verbose): 178 full_path_old = d_old.get(k).get("path") 179 full_path_new = d_new.get(k).get("path") 180 # No counter since it is not ready here 181 if sigdata_re.match(full_path_old) and sigdata_re.match(full_path_new): 182 output = bb.siggen.compare_sigfiles(full_path_old, full_path_new) 183 if output: 184 print("\n=== The verbose changes of %s.%s:" % (pn, task)) 185 print('\n'.join(output)) 186 else: 187 # Format the output, the format is: 188 # {pn: task1, task2, ...} 189 if pn in depchanged: 190 depchanged[pn] = "%s %s" % (depchanged.get(pn), task) 191 else: 192 depchanged[pn] = task 193 194 if len(depchanged) > 0: 195 print("\n=== Dependencies changed: (%s tasks)" % counter) 196 for k in depchanged.keys(): 197 print(" %s: %s" % (k, depchanged[k])) 198 199 return counter 200 201 202def main(): 203 """ 204 Print what will be done between the current and last builds: 205 1) Run "STAMPS_DIR=<path> bitbake -S recipe" to re-generate the stamps 206 2) Figure out what are newly added and changed, can't figure out 207 what are removed since we can't know the previous stamps 208 clearly, for example, if there are several builds, we can't know 209 which stamps the last build has used exactly. 210 3) Use bb.siggen.compare_sigfiles to diff the old and new stamps 211 """ 212 213 parser = argparse_oe.ArgumentParser(usage = """%(prog)s [options] [package ...] 214print what will be done between the current and last builds, for example: 215 216 $ bitbake core-image-sato 217 # Edit the recipes 218 $ bitbake-whatchanged core-image-sato 219 220The changes will be printed. 221 222Note: 223 The amount of tasks is not accurate when the task is "do_build" since 224 it usually depends on other tasks. 225 The "nostamp" task is not included. 226""" 227) 228 parser.add_argument("recipe", help="recipe to check") 229 parser.add_argument("-v", "--verbose", help = "print the verbose changes", action = "store_true") 230 args = parser.parse_args() 231 232 # Get the STAMPS_DIR 233 print("Figuring out the STAMPS_DIR ...") 234 cmdline = "bitbake -e | sed -ne 's/^STAMPS_DIR=\"\(.*\)\"/\\1/p'" 235 try: 236 stampsdir, err = bb.process.run(cmdline) 237 except: 238 raise 239 if not stampsdir: 240 print("ERROR: No STAMPS_DIR found for '%s'" % args.recipe, file=sys.stderr) 241 return 2 242 stampsdir = stampsdir.rstrip("\n") 243 if not os.path.isdir(stampsdir): 244 print("ERROR: stamps directory \"%s\" not found!" % stampsdir, file=sys.stderr) 245 return 2 246 247 # The new stamps dir 248 new_stampsdir = stampsdir + ".bbs" 249 if os.path.exists(new_stampsdir): 250 print("ERROR: %s already exists!" % new_stampsdir, file=sys.stderr) 251 return 2 252 253 try: 254 # Generate the new stamps dir 255 print("Generating the new stamps ... (need several minutes)") 256 cmdline = "STAMPS_DIR=%s bitbake -S none %s" % (new_stampsdir, args.recipe) 257 # FIXME 258 # The "bitbake -S" may fail, not fatal error, the stamps will still 259 # be generated, this might be a bug of "bitbake -S". 260 try: 261 bb.process.run(cmdline) 262 except Exception as exc: 263 print(exc) 264 265 # The dict for the new and old stamps. 266 old_dict = gen_dict(stampsdir) 267 new_dict = gen_dict(new_stampsdir) 268 269 # Remove the same one from both stamps. 270 cnt_unchanged = 0 271 for k in list(new_dict.keys()): 272 if k in old_dict: 273 cnt_unchanged += 1 274 del(new_dict[k]) 275 del(old_dict[k]) 276 277 # Re-construct the dict to easily find out what is added or changed. 278 # The dict format is: 279 # {pn_task: {pv: PV, pr: PR, path: PATH}} 280 new_recon = recon_dict(new_dict) 281 old_recon = recon_dict(old_dict) 282 283 del new_dict 284 del old_dict 285 286 # Figure out what are changed, the new_recon would be changed 287 # by the print_xxx function. 288 # Newly added 289 cnt_added = print_added(new_recon, old_recon) 290 291 # PV (including PE) and PR changed 292 # Let the bb.siggen handle them if verbose 293 cnt_rv = {} 294 if not args.verbose: 295 for i in ('pv', 'pr'): 296 cnt_rv[i] = print_vrchanged(new_recon, old_recon, i) 297 298 # Dependencies changed (use bitbake-diffsigs) 299 cnt_dep = print_depchanged(new_recon, old_recon, args.verbose) 300 301 total_changed = cnt_added + (cnt_rv.get('pv') or 0) + (cnt_rv.get('pr') or 0) + cnt_dep 302 303 print("\n=== Summary: (%s changed, %s unchanged)" % (total_changed, cnt_unchanged)) 304 if args.verbose: 305 print("Newly added: %s\nDependencies changed: %s\n" % \ 306 (cnt_added, cnt_dep)) 307 else: 308 print("Newly added: %s\nPV changed: %s\nPR changed: %s\nDependencies changed: %s\n" % \ 309 (cnt_added, cnt_rv.get('pv') or 0, cnt_rv.get('pr') or 0, cnt_dep)) 310 except: 311 print("ERROR occurred!") 312 raise 313 finally: 314 # Remove the newly generated stamps dir 315 if os.path.exists(new_stampsdir): 316 print("Removing the newly generated stamps dir ...") 317 shutil.rmtree(new_stampsdir) 318 319if __name__ == "__main__": 320 sys.exit(main()) 321