1# Copyright (c) 2011 The Chromium OS Authors. 2# 3# SPDX-License-Identifier: GPL-2.0+ 4# 5 6import math 7import os 8import re 9import shutil 10import tempfile 11 12import command 13import commit 14import gitutil 15from series import Series 16 17# Tags that we detect and remove 18re_remove = re.compile('^BUG=|^TEST=|^BRANCH=|^Change-Id:|^Review URL:' 19 '|Reviewed-on:|Commit-\w*:') 20 21# Lines which are allowed after a TEST= line 22re_allowed_after_test = re.compile('^Signed-off-by:') 23 24# Signoffs 25re_signoff = re.compile('^Signed-off-by: *(.*)') 26 27# The start of the cover letter 28re_cover = re.compile('^Cover-letter:') 29 30# A cover letter Cc 31re_cover_cc = re.compile('^Cover-letter-cc: *(.*)') 32 33# Patch series tag 34re_series_tag = re.compile('^Series-([a-z-]*): *(.*)') 35 36# Commit series tag 37re_commit_tag = re.compile('^Commit-([a-z-]*): *(.*)') 38 39# Commit tags that we want to collect and keep 40re_tag = re.compile('^(Tested-by|Acked-by|Reviewed-by|Patch-cc): (.*)') 41 42# The start of a new commit in the git log 43re_commit = re.compile('^commit ([0-9a-f]*)$') 44 45# We detect these since checkpatch doesn't always do it 46re_space_before_tab = re.compile('^[+].* \t') 47 48# States we can be in - can we use range() and still have comments? 49STATE_MSG_HEADER = 0 # Still in the message header 50STATE_PATCH_SUBJECT = 1 # In patch subject (first line of log for a commit) 51STATE_PATCH_HEADER = 2 # In patch header (after the subject) 52STATE_DIFFS = 3 # In the diff part (past --- line) 53 54class PatchStream: 55 """Class for detecting/injecting tags in a patch or series of patches 56 57 We support processing the output of 'git log' to read out the tags we 58 are interested in. We can also process a patch file in order to remove 59 unwanted tags or inject additional ones. These correspond to the two 60 phases of processing. 61 """ 62 def __init__(self, series, name=None, is_log=False): 63 self.skip_blank = False # True to skip a single blank line 64 self.found_test = False # Found a TEST= line 65 self.lines_after_test = 0 # MNumber of lines found after TEST= 66 self.warn = [] # List of warnings we have collected 67 self.linenum = 1 # Output line number we are up to 68 self.in_section = None # Name of start...END section we are in 69 self.notes = [] # Series notes 70 self.section = [] # The current section...END section 71 self.series = series # Info about the patch series 72 self.is_log = is_log # True if indent like git log 73 self.in_change = 0 # Non-zero if we are in a change list 74 self.blank_count = 0 # Number of blank lines stored up 75 self.state = STATE_MSG_HEADER # What state are we in? 76 self.signoff = [] # Contents of signoff line 77 self.commit = None # Current commit 78 79 def AddToSeries(self, line, name, value): 80 """Add a new Series-xxx tag. 81 82 When a Series-xxx tag is detected, we come here to record it, if we 83 are scanning a 'git log'. 84 85 Args: 86 line: Source line containing tag (useful for debug/error messages) 87 name: Tag name (part after 'Series-') 88 value: Tag value (part after 'Series-xxx: ') 89 """ 90 if name == 'notes': 91 self.in_section = name 92 self.skip_blank = False 93 if self.is_log: 94 self.series.AddTag(self.commit, line, name, value) 95 96 def AddToCommit(self, line, name, value): 97 """Add a new Commit-xxx tag. 98 99 When a Commit-xxx tag is detected, we come here to record it. 100 101 Args: 102 line: Source line containing tag (useful for debug/error messages) 103 name: Tag name (part after 'Commit-') 104 value: Tag value (part after 'Commit-xxx: ') 105 """ 106 if name == 'notes': 107 self.in_section = 'commit-' + name 108 self.skip_blank = False 109 110 def CloseCommit(self): 111 """Save the current commit into our commit list, and reset our state""" 112 if self.commit and self.is_log: 113 self.series.AddCommit(self.commit) 114 self.commit = None 115 # If 'END' is missing in a 'Cover-letter' section, and that section 116 # happens to show up at the very end of the commit message, this is 117 # the chance for us to fix it up. 118 if self.in_section == 'cover' and self.is_log: 119 self.series.cover = self.section 120 self.in_section = None 121 self.skip_blank = True 122 self.section = [] 123 124 def ProcessLine(self, line): 125 """Process a single line of a patch file or commit log 126 127 This process a line and returns a list of lines to output. The list 128 may be empty or may contain multiple output lines. 129 130 This is where all the complicated logic is located. The class's 131 state is used to move between different states and detect things 132 properly. 133 134 We can be in one of two modes: 135 self.is_log == True: This is 'git log' mode, where most output is 136 indented by 4 characters and we are scanning for tags 137 138 self.is_log == False: This is 'patch' mode, where we already have 139 all the tags, and are processing patches to remove junk we 140 don't want, and add things we think are required. 141 142 Args: 143 line: text line to process 144 145 Returns: 146 list of output lines, or [] if nothing should be output 147 """ 148 # Initially we have no output. Prepare the input line string 149 out = [] 150 line = line.rstrip('\n') 151 152 commit_match = re_commit.match(line) if self.is_log else None 153 154 if self.is_log: 155 if line[:4] == ' ': 156 line = line[4:] 157 158 # Handle state transition and skipping blank lines 159 series_tag_match = re_series_tag.match(line) 160 commit_tag_match = re_commit_tag.match(line) 161 cover_match = re_cover.match(line) 162 cover_cc_match = re_cover_cc.match(line) 163 signoff_match = re_signoff.match(line) 164 tag_match = None 165 if self.state == STATE_PATCH_HEADER: 166 tag_match = re_tag.match(line) 167 is_blank = not line.strip() 168 if is_blank: 169 if (self.state == STATE_MSG_HEADER 170 or self.state == STATE_PATCH_SUBJECT): 171 self.state += 1 172 173 # We don't have a subject in the text stream of patch files 174 # It has its own line with a Subject: tag 175 if not self.is_log and self.state == STATE_PATCH_SUBJECT: 176 self.state += 1 177 elif commit_match: 178 self.state = STATE_MSG_HEADER 179 180 # If a tag is detected, but we are already in a section, 181 # this means 'END' is missing for that section, fix it up. 182 if series_tag_match or commit_tag_match or \ 183 cover_match or cover_cc_match or signoff_match: 184 if self.in_section: 185 self.warn.append("Missing 'END' in section '%s'" % self.in_section) 186 if self.in_section == 'cover': 187 self.series.cover = self.section 188 elif self.in_section == 'notes': 189 if self.is_log: 190 self.series.notes += self.section 191 elif self.in_section == 'commit-notes': 192 if self.is_log: 193 self.commit.notes += self.section 194 else: 195 self.warn.append("Unknown section '%s'" % self.in_section) 196 self.in_section = None 197 self.skip_blank = True 198 self.section = [] 199 200 # If we are in a section, keep collecting lines until we see END 201 if self.in_section: 202 if line == 'END': 203 if self.in_section == 'cover': 204 self.series.cover = self.section 205 elif self.in_section == 'notes': 206 if self.is_log: 207 self.series.notes += self.section 208 elif self.in_section == 'commit-notes': 209 if self.is_log: 210 self.commit.notes += self.section 211 else: 212 self.warn.append("Unknown section '%s'" % self.in_section) 213 self.in_section = None 214 self.skip_blank = True 215 self.section = [] 216 else: 217 self.section.append(line) 218 219 # Detect the commit subject 220 elif not is_blank and self.state == STATE_PATCH_SUBJECT: 221 self.commit.subject = line 222 223 # Detect the tags we want to remove, and skip blank lines 224 elif re_remove.match(line) and not commit_tag_match: 225 self.skip_blank = True 226 227 # TEST= should be the last thing in the commit, so remove 228 # everything after it 229 if line.startswith('TEST='): 230 self.found_test = True 231 elif self.skip_blank and is_blank: 232 self.skip_blank = False 233 234 # Detect the start of a cover letter section 235 elif cover_match: 236 self.in_section = 'cover' 237 self.skip_blank = False 238 239 elif cover_cc_match: 240 value = cover_cc_match.group(1) 241 self.AddToSeries(line, 'cover-cc', value) 242 243 # If we are in a change list, key collected lines until a blank one 244 elif self.in_change: 245 if is_blank: 246 # Blank line ends this change list 247 self.in_change = 0 248 elif line == '---': 249 self.in_change = 0 250 out = self.ProcessLine(line) 251 else: 252 if self.is_log: 253 self.series.AddChange(self.in_change, self.commit, line) 254 self.skip_blank = False 255 256 # Detect Series-xxx tags 257 elif series_tag_match: 258 name = series_tag_match.group(1) 259 value = series_tag_match.group(2) 260 if name == 'changes': 261 # value is the version number: e.g. 1, or 2 262 try: 263 value = int(value) 264 except ValueError as str: 265 raise ValueError("%s: Cannot decode version info '%s'" % 266 (self.commit.hash, line)) 267 self.in_change = int(value) 268 else: 269 self.AddToSeries(line, name, value) 270 self.skip_blank = True 271 272 # Detect Commit-xxx tags 273 elif commit_tag_match: 274 name = commit_tag_match.group(1) 275 value = commit_tag_match.group(2) 276 if name == 'notes': 277 self.AddToCommit(line, name, value) 278 self.skip_blank = True 279 280 # Detect the start of a new commit 281 elif commit_match: 282 self.CloseCommit() 283 self.commit = commit.Commit(commit_match.group(1)) 284 285 # Detect tags in the commit message 286 elif tag_match: 287 # Remove Tested-by self, since few will take much notice 288 if (tag_match.group(1) == 'Tested-by' and 289 tag_match.group(2).find(os.getenv('USER') + '@') != -1): 290 self.warn.append("Ignoring %s" % line) 291 elif tag_match.group(1) == 'Patch-cc': 292 self.commit.AddCc(tag_match.group(2).split(',')) 293 else: 294 out = [line] 295 296 # Suppress duplicate signoffs 297 elif signoff_match: 298 if (self.is_log or not self.commit or 299 self.commit.CheckDuplicateSignoff(signoff_match.group(1))): 300 out = [line] 301 302 # Well that means this is an ordinary line 303 else: 304 pos = 1 305 # Look for ugly ASCII characters 306 for ch in line: 307 # TODO: Would be nicer to report source filename and line 308 if ord(ch) > 0x80: 309 self.warn.append("Line %d/%d ('%s') has funny ascii char" % 310 (self.linenum, pos, line)) 311 pos += 1 312 313 # Look for space before tab 314 m = re_space_before_tab.match(line) 315 if m: 316 self.warn.append('Line %d/%d has space before tab' % 317 (self.linenum, m.start())) 318 319 # OK, we have a valid non-blank line 320 out = [line] 321 self.linenum += 1 322 self.skip_blank = False 323 if self.state == STATE_DIFFS: 324 pass 325 326 # If this is the start of the diffs section, emit our tags and 327 # change log 328 elif line == '---': 329 self.state = STATE_DIFFS 330 331 # Output the tags (signeoff first), then change list 332 out = [] 333 log = self.series.MakeChangeLog(self.commit) 334 out += [line] 335 if self.commit: 336 out += self.commit.notes 337 out += [''] + log 338 elif self.found_test: 339 if not re_allowed_after_test.match(line): 340 self.lines_after_test += 1 341 342 return out 343 344 def Finalize(self): 345 """Close out processing of this patch stream""" 346 self.CloseCommit() 347 if self.lines_after_test: 348 self.warn.append('Found %d lines after TEST=' % 349 self.lines_after_test) 350 351 def ProcessStream(self, infd, outfd): 352 """Copy a stream from infd to outfd, filtering out unwanting things. 353 354 This is used to process patch files one at a time. 355 356 Args: 357 infd: Input stream file object 358 outfd: Output stream file object 359 """ 360 # Extract the filename from each diff, for nice warnings 361 fname = None 362 last_fname = None 363 re_fname = re.compile('diff --git a/(.*) b/.*') 364 while True: 365 line = infd.readline() 366 if not line: 367 break 368 out = self.ProcessLine(line) 369 370 # Try to detect blank lines at EOF 371 for line in out: 372 match = re_fname.match(line) 373 if match: 374 last_fname = fname 375 fname = match.group(1) 376 if line == '+': 377 self.blank_count += 1 378 else: 379 if self.blank_count and (line == '-- ' or match): 380 self.warn.append("Found possible blank line(s) at " 381 "end of file '%s'" % last_fname) 382 outfd.write('+\n' * self.blank_count) 383 outfd.write(line + '\n') 384 self.blank_count = 0 385 self.Finalize() 386 387 388def GetMetaDataForList(commit_range, git_dir=None, count=None, 389 series = None, allow_overwrite=False): 390 """Reads out patch series metadata from the commits 391 392 This does a 'git log' on the relevant commits and pulls out the tags we 393 are interested in. 394 395 Args: 396 commit_range: Range of commits to count (e.g. 'HEAD..base') 397 git_dir: Path to git repositiory (None to use default) 398 count: Number of commits to list, or None for no limit 399 series: Series object to add information into. By default a new series 400 is started. 401 allow_overwrite: Allow tags to overwrite an existing tag 402 Returns: 403 A Series object containing information about the commits. 404 """ 405 if not series: 406 series = Series() 407 series.allow_overwrite = allow_overwrite 408 params = gitutil.LogCmd(commit_range, reverse=True, count=count, 409 git_dir=git_dir) 410 stdout = command.RunPipe([params], capture=True).stdout 411 ps = PatchStream(series, is_log=True) 412 for line in stdout.splitlines(): 413 ps.ProcessLine(line) 414 ps.Finalize() 415 return series 416 417def GetMetaData(start, count): 418 """Reads out patch series metadata from the commits 419 420 This does a 'git log' on the relevant commits and pulls out the tags we 421 are interested in. 422 423 Args: 424 start: Commit to start from: 0=HEAD, 1=next one, etc. 425 count: Number of commits to list 426 """ 427 return GetMetaDataForList('HEAD~%d' % start, None, count) 428 429def FixPatch(backup_dir, fname, series, commit): 430 """Fix up a patch file, by adding/removing as required. 431 432 We remove our tags from the patch file, insert changes lists, etc. 433 The patch file is processed in place, and overwritten. 434 435 A backup file is put into backup_dir (if not None). 436 437 Args: 438 fname: Filename to patch file to process 439 series: Series information about this patch set 440 commit: Commit object for this patch file 441 Return: 442 A list of errors, or [] if all ok. 443 """ 444 handle, tmpname = tempfile.mkstemp() 445 outfd = os.fdopen(handle, 'w') 446 infd = open(fname, 'r') 447 ps = PatchStream(series) 448 ps.commit = commit 449 ps.ProcessStream(infd, outfd) 450 infd.close() 451 outfd.close() 452 453 # Create a backup file if required 454 if backup_dir: 455 shutil.copy(fname, os.path.join(backup_dir, os.path.basename(fname))) 456 shutil.move(tmpname, fname) 457 return ps.warn 458 459def FixPatches(series, fnames): 460 """Fix up a list of patches identified by filenames 461 462 The patch files are processed in place, and overwritten. 463 464 Args: 465 series: The series object 466 fnames: List of patch files to process 467 """ 468 # Current workflow creates patches, so we shouldn't need a backup 469 backup_dir = None #tempfile.mkdtemp('clean-patch') 470 count = 0 471 for fname in fnames: 472 commit = series.commits[count] 473 commit.patch = fname 474 result = FixPatch(backup_dir, fname, series, commit) 475 if result: 476 print '%d warnings for %s:' % (len(result), fname) 477 for warn in result: 478 print '\t', warn 479 print 480 count += 1 481 print 'Cleaned %d patches' % count 482 return series 483 484def InsertCoverLetter(fname, series, count): 485 """Inserts a cover letter with the required info into patch 0 486 487 Args: 488 fname: Input / output filename of the cover letter file 489 series: Series object 490 count: Number of patches in the series 491 """ 492 fd = open(fname, 'r') 493 lines = fd.readlines() 494 fd.close() 495 496 fd = open(fname, 'w') 497 text = series.cover 498 prefix = series.GetPatchPrefix() 499 for line in lines: 500 if line.startswith('Subject:'): 501 # if more than 10 or 100 patches, it should say 00/xx, 000/xxx, etc 502 zero_repeat = int(math.log10(count)) + 1 503 zero = '0' * zero_repeat 504 line = 'Subject: [%s %s/%d] %s\n' % (prefix, zero, count, text[0]) 505 506 # Insert our cover letter 507 elif line.startswith('*** BLURB HERE ***'): 508 # First the blurb test 509 line = '\n'.join(text[1:]) + '\n' 510 if series.get('notes'): 511 line += '\n'.join(series.notes) + '\n' 512 513 # Now the change list 514 out = series.MakeChangeLog(None) 515 line += '\n' + '\n'.join(out) 516 fd.write(line) 517 fd.close() 518