1# Copyright (c) 2011 The Chromium OS Authors. 2# 3# SPDX-License-Identifier: GPL-2.0+ 4# 5 6import math 7import os 8import re 9import shutil 10import tempfile 11 12import command 13import commit 14import gitutil 15from series import Series 16 17# Tags that we detect and remove 18re_remove = re.compile('^BUG=|^TEST=|^BRANCH=|^Change-Id:|^Review URL:' 19 '|Reviewed-on:|Commit-\w*:') 20 21# Lines which are allowed after a TEST= line 22re_allowed_after_test = re.compile('^Signed-off-by:') 23 24# Signoffs 25re_signoff = re.compile('^Signed-off-by: *(.*)') 26 27# The start of the cover letter 28re_cover = re.compile('^Cover-letter:') 29 30# A cover letter Cc 31re_cover_cc = re.compile('^Cover-letter-cc: *(.*)') 32 33# Patch series tag 34re_series_tag = re.compile('^Series-([a-z-]*): *(.*)') 35 36# Commit series tag 37re_commit_tag = re.compile('^Commit-([a-z-]*): *(.*)') 38 39# Commit tags that we want to collect and keep 40re_tag = re.compile('^(Tested-by|Acked-by|Reviewed-by|Patch-cc): (.*)') 41 42# The start of a new commit in the git log 43re_commit = re.compile('^commit ([0-9a-f]*)$') 44 45# We detect these since checkpatch doesn't always do it 46re_space_before_tab = re.compile('^[+].* \t') 47 48# States we can be in - can we use range() and still have comments? 49STATE_MSG_HEADER = 0 # Still in the message header 50STATE_PATCH_SUBJECT = 1 # In patch subject (first line of log for a commit) 51STATE_PATCH_HEADER = 2 # In patch header (after the subject) 52STATE_DIFFS = 3 # In the diff part (past --- line) 53 54class PatchStream: 55 """Class for detecting/injecting tags in a patch or series of patches 56 57 We support processing the output of 'git log' to read out the tags we 58 are interested in. We can also process a patch file in order to remove 59 unwanted tags or inject additional ones. These correspond to the two 60 phases of processing. 61 """ 62 def __init__(self, series, name=None, is_log=False): 63 self.skip_blank = False # True to skip a single blank line 64 self.found_test = False # Found a TEST= line 65 self.lines_after_test = 0 # MNumber of lines found after TEST= 66 self.warn = [] # List of warnings we have collected 67 self.linenum = 1 # Output line number we are up to 68 self.in_section = None # Name of start...END section we are in 69 self.notes = [] # Series notes 70 self.section = [] # The current section...END section 71 self.series = series # Info about the patch series 72 self.is_log = is_log # True if indent like git log 73 self.in_change = 0 # Non-zero if we are in a change list 74 self.blank_count = 0 # Number of blank lines stored up 75 self.state = STATE_MSG_HEADER # What state are we in? 76 self.signoff = [] # Contents of signoff line 77 self.commit = None # Current commit 78 79 def AddToSeries(self, line, name, value): 80 """Add a new Series-xxx tag. 81 82 When a Series-xxx tag is detected, we come here to record it, if we 83 are scanning a 'git log'. 84 85 Args: 86 line: Source line containing tag (useful for debug/error messages) 87 name: Tag name (part after 'Series-') 88 value: Tag value (part after 'Series-xxx: ') 89 """ 90 if name == 'notes': 91 self.in_section = name 92 self.skip_blank = False 93 if self.is_log: 94 self.series.AddTag(self.commit, line, name, value) 95 96 def AddToCommit(self, line, name, value): 97 """Add a new Commit-xxx tag. 98 99 When a Commit-xxx tag is detected, we come here to record it. 100 101 Args: 102 line: Source line containing tag (useful for debug/error messages) 103 name: Tag name (part after 'Commit-') 104 value: Tag value (part after 'Commit-xxx: ') 105 """ 106 if name == 'notes': 107 self.in_section = 'commit-' + name 108 self.skip_blank = False 109 110 def CloseCommit(self): 111 """Save the current commit into our commit list, and reset our state""" 112 if self.commit and self.is_log: 113 self.series.AddCommit(self.commit) 114 self.commit = None 115 116 def ProcessLine(self, line): 117 """Process a single line of a patch file or commit log 118 119 This process a line and returns a list of lines to output. The list 120 may be empty or may contain multiple output lines. 121 122 This is where all the complicated logic is located. The class's 123 state is used to move between different states and detect things 124 properly. 125 126 We can be in one of two modes: 127 self.is_log == True: This is 'git log' mode, where most output is 128 indented by 4 characters and we are scanning for tags 129 130 self.is_log == False: This is 'patch' mode, where we already have 131 all the tags, and are processing patches to remove junk we 132 don't want, and add things we think are required. 133 134 Args: 135 line: text line to process 136 137 Returns: 138 list of output lines, or [] if nothing should be output 139 """ 140 # Initially we have no output. Prepare the input line string 141 out = [] 142 line = line.rstrip('\n') 143 144 commit_match = re_commit.match(line) if self.is_log else None 145 146 if self.is_log: 147 if line[:4] == ' ': 148 line = line[4:] 149 150 # Handle state transition and skipping blank lines 151 series_tag_match = re_series_tag.match(line) 152 commit_tag_match = re_commit_tag.match(line) 153 cover_match = re_cover.match(line) 154 cover_cc_match = re_cover_cc.match(line) 155 signoff_match = re_signoff.match(line) 156 tag_match = None 157 if self.state == STATE_PATCH_HEADER: 158 tag_match = re_tag.match(line) 159 is_blank = not line.strip() 160 if is_blank: 161 if (self.state == STATE_MSG_HEADER 162 or self.state == STATE_PATCH_SUBJECT): 163 self.state += 1 164 165 # We don't have a subject in the text stream of patch files 166 # It has its own line with a Subject: tag 167 if not self.is_log and self.state == STATE_PATCH_SUBJECT: 168 self.state += 1 169 elif commit_match: 170 self.state = STATE_MSG_HEADER 171 172 # If a tag is detected, but we are already in a section, 173 # this means 'END' is missing for that section, fix it up. 174 if series_tag_match or commit_tag_match or \ 175 cover_match or cover_cc_match or signoff_match: 176 if self.in_section: 177 self.warn.append("Missing 'END' in section '%s'" % self.in_section) 178 if self.in_section == 'cover': 179 self.series.cover = self.section 180 elif self.in_section == 'notes': 181 if self.is_log: 182 self.series.notes += self.section 183 elif self.in_section == 'commit-notes': 184 if self.is_log: 185 self.commit.notes += self.section 186 else: 187 self.warn.append("Unknown section '%s'" % self.in_section) 188 self.in_section = None 189 self.skip_blank = True 190 self.section = [] 191 192 # If we are in a section, keep collecting lines until we see END 193 if self.in_section: 194 if line == 'END': 195 if self.in_section == 'cover': 196 self.series.cover = self.section 197 elif self.in_section == 'notes': 198 if self.is_log: 199 self.series.notes += self.section 200 elif self.in_section == 'commit-notes': 201 if self.is_log: 202 self.commit.notes += self.section 203 else: 204 self.warn.append("Unknown section '%s'" % self.in_section) 205 self.in_section = None 206 self.skip_blank = True 207 self.section = [] 208 else: 209 self.section.append(line) 210 211 # Detect the commit subject 212 elif not is_blank and self.state == STATE_PATCH_SUBJECT: 213 self.commit.subject = line 214 215 # Detect the tags we want to remove, and skip blank lines 216 elif re_remove.match(line) and not commit_tag_match: 217 self.skip_blank = True 218 219 # TEST= should be the last thing in the commit, so remove 220 # everything after it 221 if line.startswith('TEST='): 222 self.found_test = True 223 elif self.skip_blank and is_blank: 224 self.skip_blank = False 225 226 # Detect the start of a cover letter section 227 elif cover_match: 228 self.in_section = 'cover' 229 self.skip_blank = False 230 231 elif cover_cc_match: 232 value = cover_cc_match.group(1) 233 self.AddToSeries(line, 'cover-cc', value) 234 235 # If we are in a change list, key collected lines until a blank one 236 elif self.in_change: 237 if is_blank: 238 # Blank line ends this change list 239 self.in_change = 0 240 elif line == '---': 241 self.in_change = 0 242 out = self.ProcessLine(line) 243 else: 244 if self.is_log: 245 self.series.AddChange(self.in_change, self.commit, line) 246 self.skip_blank = False 247 248 # Detect Series-xxx tags 249 elif series_tag_match: 250 name = series_tag_match.group(1) 251 value = series_tag_match.group(2) 252 if name == 'changes': 253 # value is the version number: e.g. 1, or 2 254 try: 255 value = int(value) 256 except ValueError as str: 257 raise ValueError("%s: Cannot decode version info '%s'" % 258 (self.commit.hash, line)) 259 self.in_change = int(value) 260 else: 261 self.AddToSeries(line, name, value) 262 self.skip_blank = True 263 264 # Detect Commit-xxx tags 265 elif commit_tag_match: 266 name = commit_tag_match.group(1) 267 value = commit_tag_match.group(2) 268 if name == 'notes': 269 self.AddToCommit(line, name, value) 270 self.skip_blank = True 271 272 # Detect the start of a new commit 273 elif commit_match: 274 self.CloseCommit() 275 self.commit = commit.Commit(commit_match.group(1)) 276 277 # Detect tags in the commit message 278 elif tag_match: 279 # Remove Tested-by self, since few will take much notice 280 if (tag_match.group(1) == 'Tested-by' and 281 tag_match.group(2).find(os.getenv('USER') + '@') != -1): 282 self.warn.append("Ignoring %s" % line) 283 elif tag_match.group(1) == 'Patch-cc': 284 self.commit.AddCc(tag_match.group(2).split(',')) 285 else: 286 out = [line] 287 288 # Suppress duplicate signoffs 289 elif signoff_match: 290 if (self.is_log or not self.commit or 291 self.commit.CheckDuplicateSignoff(signoff_match.group(1))): 292 out = [line] 293 294 # Well that means this is an ordinary line 295 else: 296 pos = 1 297 # Look for ugly ASCII characters 298 for ch in line: 299 # TODO: Would be nicer to report source filename and line 300 if ord(ch) > 0x80: 301 self.warn.append("Line %d/%d ('%s') has funny ascii char" % 302 (self.linenum, pos, line)) 303 pos += 1 304 305 # Look for space before tab 306 m = re_space_before_tab.match(line) 307 if m: 308 self.warn.append('Line %d/%d has space before tab' % 309 (self.linenum, m.start())) 310 311 # OK, we have a valid non-blank line 312 out = [line] 313 self.linenum += 1 314 self.skip_blank = False 315 if self.state == STATE_DIFFS: 316 pass 317 318 # If this is the start of the diffs section, emit our tags and 319 # change log 320 elif line == '---': 321 self.state = STATE_DIFFS 322 323 # Output the tags (signeoff first), then change list 324 out = [] 325 log = self.series.MakeChangeLog(self.commit) 326 out += [line] 327 if self.commit: 328 out += self.commit.notes 329 out += [''] + log 330 elif self.found_test: 331 if not re_allowed_after_test.match(line): 332 self.lines_after_test += 1 333 334 return out 335 336 def Finalize(self): 337 """Close out processing of this patch stream""" 338 self.CloseCommit() 339 if self.lines_after_test: 340 self.warn.append('Found %d lines after TEST=' % 341 self.lines_after_test) 342 343 def ProcessStream(self, infd, outfd): 344 """Copy a stream from infd to outfd, filtering out unwanting things. 345 346 This is used to process patch files one at a time. 347 348 Args: 349 infd: Input stream file object 350 outfd: Output stream file object 351 """ 352 # Extract the filename from each diff, for nice warnings 353 fname = None 354 last_fname = None 355 re_fname = re.compile('diff --git a/(.*) b/.*') 356 while True: 357 line = infd.readline() 358 if not line: 359 break 360 out = self.ProcessLine(line) 361 362 # Try to detect blank lines at EOF 363 for line in out: 364 match = re_fname.match(line) 365 if match: 366 last_fname = fname 367 fname = match.group(1) 368 if line == '+': 369 self.blank_count += 1 370 else: 371 if self.blank_count and (line == '-- ' or match): 372 self.warn.append("Found possible blank line(s) at " 373 "end of file '%s'" % last_fname) 374 outfd.write('+\n' * self.blank_count) 375 outfd.write(line + '\n') 376 self.blank_count = 0 377 self.Finalize() 378 379 380def GetMetaDataForList(commit_range, git_dir=None, count=None, 381 series = None, allow_overwrite=False): 382 """Reads out patch series metadata from the commits 383 384 This does a 'git log' on the relevant commits and pulls out the tags we 385 are interested in. 386 387 Args: 388 commit_range: Range of commits to count (e.g. 'HEAD..base') 389 git_dir: Path to git repositiory (None to use default) 390 count: Number of commits to list, or None for no limit 391 series: Series object to add information into. By default a new series 392 is started. 393 allow_overwrite: Allow tags to overwrite an existing tag 394 Returns: 395 A Series object containing information about the commits. 396 """ 397 if not series: 398 series = Series() 399 series.allow_overwrite = allow_overwrite 400 params = gitutil.LogCmd(commit_range, reverse=True, count=count, 401 git_dir=git_dir) 402 stdout = command.RunPipe([params], capture=True).stdout 403 ps = PatchStream(series, is_log=True) 404 for line in stdout.splitlines(): 405 ps.ProcessLine(line) 406 ps.Finalize() 407 return series 408 409def GetMetaData(start, count): 410 """Reads out patch series metadata from the commits 411 412 This does a 'git log' on the relevant commits and pulls out the tags we 413 are interested in. 414 415 Args: 416 start: Commit to start from: 0=HEAD, 1=next one, etc. 417 count: Number of commits to list 418 """ 419 return GetMetaDataForList('HEAD~%d' % start, None, count) 420 421def FixPatch(backup_dir, fname, series, commit): 422 """Fix up a patch file, by adding/removing as required. 423 424 We remove our tags from the patch file, insert changes lists, etc. 425 The patch file is processed in place, and overwritten. 426 427 A backup file is put into backup_dir (if not None). 428 429 Args: 430 fname: Filename to patch file to process 431 series: Series information about this patch set 432 commit: Commit object for this patch file 433 Return: 434 A list of errors, or [] if all ok. 435 """ 436 handle, tmpname = tempfile.mkstemp() 437 outfd = os.fdopen(handle, 'w') 438 infd = open(fname, 'r') 439 ps = PatchStream(series) 440 ps.commit = commit 441 ps.ProcessStream(infd, outfd) 442 infd.close() 443 outfd.close() 444 445 # Create a backup file if required 446 if backup_dir: 447 shutil.copy(fname, os.path.join(backup_dir, os.path.basename(fname))) 448 shutil.move(tmpname, fname) 449 return ps.warn 450 451def FixPatches(series, fnames): 452 """Fix up a list of patches identified by filenames 453 454 The patch files are processed in place, and overwritten. 455 456 Args: 457 series: The series object 458 fnames: List of patch files to process 459 """ 460 # Current workflow creates patches, so we shouldn't need a backup 461 backup_dir = None #tempfile.mkdtemp('clean-patch') 462 count = 0 463 for fname in fnames: 464 commit = series.commits[count] 465 commit.patch = fname 466 result = FixPatch(backup_dir, fname, series, commit) 467 if result: 468 print '%d warnings for %s:' % (len(result), fname) 469 for warn in result: 470 print '\t', warn 471 print 472 count += 1 473 print 'Cleaned %d patches' % count 474 return series 475 476def InsertCoverLetter(fname, series, count): 477 """Inserts a cover letter with the required info into patch 0 478 479 Args: 480 fname: Input / output filename of the cover letter file 481 series: Series object 482 count: Number of patches in the series 483 """ 484 fd = open(fname, 'r') 485 lines = fd.readlines() 486 fd.close() 487 488 fd = open(fname, 'w') 489 text = series.cover 490 prefix = series.GetPatchPrefix() 491 for line in lines: 492 if line.startswith('Subject:'): 493 # if more than 10 or 100 patches, it should say 00/xx, 000/xxx, etc 494 zero_repeat = int(math.log10(count)) + 1 495 zero = '0' * zero_repeat 496 line = 'Subject: [%s %s/%d] %s\n' % (prefix, zero, count, text[0]) 497 498 # Insert our cover letter 499 elif line.startswith('*** BLURB HERE ***'): 500 # First the blurb test 501 line = '\n'.join(text[1:]) + '\n' 502 if series.get('notes'): 503 line += '\n'.join(series.notes) + '\n' 504 505 # Now the change list 506 out = series.MakeChangeLog(None) 507 line += '\n' + '\n'.join(out) 508 fd.write(line) 509 fd.close() 510