1# Copyright (c) 2011 The Chromium OS Authors. 2# 3# SPDX-License-Identifier: GPL-2.0+ 4# 5 6import os 7import re 8import shutil 9import tempfile 10 11import command 12import commit 13import gitutil 14from series import Series 15 16# Tags that we detect and remove 17re_remove = re.compile('^BUG=|^TEST=|^BRANCH=|^Change-Id:|^Review URL:' 18 '|Reviewed-on:|Commit-\w*:') 19 20# Lines which are allowed after a TEST= line 21re_allowed_after_test = re.compile('^Signed-off-by:') 22 23# Signoffs 24re_signoff = re.compile('^Signed-off-by: *(.*)') 25 26# The start of the cover letter 27re_cover = re.compile('^Cover-letter:') 28 29# A cover letter Cc 30re_cover_cc = re.compile('^Cover-letter-cc: *(.*)') 31 32# Patch series tag 33re_series_tag = re.compile('^Series-([a-z-]*): *(.*)') 34 35# Commit series tag 36re_commit_tag = re.compile('^Commit-([a-z-]*): *(.*)') 37 38# Commit tags that we want to collect and keep 39re_tag = re.compile('^(Tested-by|Acked-by|Reviewed-by|Patch-cc): (.*)') 40 41# The start of a new commit in the git log 42re_commit = re.compile('^commit ([0-9a-f]*)$') 43 44# We detect these since checkpatch doesn't always do it 45re_space_before_tab = re.compile('^[+].* \t') 46 47# States we can be in - can we use range() and still have comments? 48STATE_MSG_HEADER = 0 # Still in the message header 49STATE_PATCH_SUBJECT = 1 # In patch subject (first line of log for a commit) 50STATE_PATCH_HEADER = 2 # In patch header (after the subject) 51STATE_DIFFS = 3 # In the diff part (past --- line) 52 53class PatchStream: 54 """Class for detecting/injecting tags in a patch or series of patches 55 56 We support processing the output of 'git log' to read out the tags we 57 are interested in. We can also process a patch file in order to remove 58 unwanted tags or inject additional ones. These correspond to the two 59 phases of processing. 60 """ 61 def __init__(self, series, name=None, is_log=False): 62 self.skip_blank = False # True to skip a single blank line 63 self.found_test = False # Found a TEST= line 64 self.lines_after_test = 0 # MNumber of lines found after TEST= 65 self.warn = [] # List of warnings we have collected 66 self.linenum = 1 # Output line number we are up to 67 self.in_section = None # Name of start...END section we are in 68 self.notes = [] # Series notes 69 self.section = [] # The current section...END section 70 self.series = series # Info about the patch series 71 self.is_log = is_log # True if indent like git log 72 self.in_change = 0 # Non-zero if we are in a change list 73 self.blank_count = 0 # Number of blank lines stored up 74 self.state = STATE_MSG_HEADER # What state are we in? 75 self.tags = [] # Tags collected, like Tested-by... 76 self.signoff = [] # Contents of signoff line 77 self.commit = None # Current commit 78 79 def AddToSeries(self, line, name, value): 80 """Add a new Series-xxx tag. 81 82 When a Series-xxx tag is detected, we come here to record it, if we 83 are scanning a 'git log'. 84 85 Args: 86 line: Source line containing tag (useful for debug/error messages) 87 name: Tag name (part after 'Series-') 88 value: Tag value (part after 'Series-xxx: ') 89 """ 90 if name == 'notes': 91 self.in_section = name 92 self.skip_blank = False 93 if self.is_log: 94 self.series.AddTag(self.commit, line, name, value) 95 96 def AddToCommit(self, line, name, value): 97 """Add a new Commit-xxx tag. 98 99 When a Commit-xxx tag is detected, we come here to record it. 100 101 Args: 102 line: Source line containing tag (useful for debug/error messages) 103 name: Tag name (part after 'Commit-') 104 value: Tag value (part after 'Commit-xxx: ') 105 """ 106 if name == 'notes': 107 self.in_section = 'commit-' + name 108 self.skip_blank = False 109 110 def CloseCommit(self): 111 """Save the current commit into our commit list, and reset our state""" 112 if self.commit and self.is_log: 113 self.series.AddCommit(self.commit) 114 self.commit = None 115 116 def FormatTags(self, tags): 117 out_list = [] 118 for tag in sorted(tags): 119 if tag.startswith('Cc:'): 120 tag_list = tag[4:].split(',') 121 out_list += gitutil.BuildEmailList(tag_list, 'Cc:') 122 else: 123 out_list.append(tag) 124 return out_list 125 126 def ProcessLine(self, line): 127 """Process a single line of a patch file or commit log 128 129 This process a line and returns a list of lines to output. The list 130 may be empty or may contain multiple output lines. 131 132 This is where all the complicated logic is located. The class's 133 state is used to move between different states and detect things 134 properly. 135 136 We can be in one of two modes: 137 self.is_log == True: This is 'git log' mode, where most output is 138 indented by 4 characters and we are scanning for tags 139 140 self.is_log == False: This is 'patch' mode, where we already have 141 all the tags, and are processing patches to remove junk we 142 don't want, and add things we think are required. 143 144 Args: 145 line: text line to process 146 147 Returns: 148 list of output lines, or [] if nothing should be output 149 """ 150 # Initially we have no output. Prepare the input line string 151 out = [] 152 line = line.rstrip('\n') 153 if self.is_log: 154 if line[:4] == ' ': 155 line = line[4:] 156 157 # Handle state transition and skipping blank lines 158 series_tag_match = re_series_tag.match(line) 159 commit_tag_match = re_commit_tag.match(line) 160 commit_match = re_commit.match(line) if self.is_log else None 161 cover_cc_match = re_cover_cc.match(line) 162 signoff_match = re_signoff.match(line) 163 tag_match = None 164 if self.state == STATE_PATCH_HEADER: 165 tag_match = re_tag.match(line) 166 is_blank = not line.strip() 167 if is_blank: 168 if (self.state == STATE_MSG_HEADER 169 or self.state == STATE_PATCH_SUBJECT): 170 self.state += 1 171 172 # We don't have a subject in the text stream of patch files 173 # It has its own line with a Subject: tag 174 if not self.is_log and self.state == STATE_PATCH_SUBJECT: 175 self.state += 1 176 elif commit_match: 177 self.state = STATE_MSG_HEADER 178 179 # If we are in a section, keep collecting lines until we see END 180 if self.in_section: 181 if line == 'END': 182 if self.in_section == 'cover': 183 self.series.cover = self.section 184 elif self.in_section == 'notes': 185 if self.is_log: 186 self.series.notes += self.section 187 elif self.in_section == 'commit-notes': 188 if self.is_log: 189 self.commit.notes += self.section 190 else: 191 self.warn.append("Unknown section '%s'" % self.in_section) 192 self.in_section = None 193 self.skip_blank = True 194 self.section = [] 195 else: 196 self.section.append(line) 197 198 # Detect the commit subject 199 elif not is_blank and self.state == STATE_PATCH_SUBJECT: 200 self.commit.subject = line 201 202 # Detect the tags we want to remove, and skip blank lines 203 elif re_remove.match(line) and not commit_tag_match: 204 self.skip_blank = True 205 206 # TEST= should be the last thing in the commit, so remove 207 # everything after it 208 if line.startswith('TEST='): 209 self.found_test = True 210 elif self.skip_blank and is_blank: 211 self.skip_blank = False 212 213 # Detect the start of a cover letter section 214 elif re_cover.match(line): 215 self.in_section = 'cover' 216 self.skip_blank = False 217 218 elif cover_cc_match: 219 value = cover_cc_match.group(1) 220 self.AddToSeries(line, 'cover-cc', value) 221 222 # If we are in a change list, key collected lines until a blank one 223 elif self.in_change: 224 if is_blank: 225 # Blank line ends this change list 226 self.in_change = 0 227 elif line == '---': 228 self.in_change = 0 229 out = self.ProcessLine(line) 230 else: 231 if self.is_log: 232 self.series.AddChange(self.in_change, self.commit, line) 233 self.skip_blank = False 234 235 # Detect Series-xxx tags 236 elif series_tag_match: 237 name = series_tag_match.group(1) 238 value = series_tag_match.group(2) 239 if name == 'changes': 240 # value is the version number: e.g. 1, or 2 241 try: 242 value = int(value) 243 except ValueError as str: 244 raise ValueError("%s: Cannot decode version info '%s'" % 245 (self.commit.hash, line)) 246 self.in_change = int(value) 247 else: 248 self.AddToSeries(line, name, value) 249 self.skip_blank = True 250 251 # Detect Commit-xxx tags 252 elif commit_tag_match: 253 name = commit_tag_match.group(1) 254 value = commit_tag_match.group(2) 255 if name == 'notes': 256 self.AddToCommit(line, name, value) 257 self.skip_blank = True 258 259 # Detect the start of a new commit 260 elif commit_match: 261 self.CloseCommit() 262 # TODO: We should store the whole hash, and just display a subset 263 self.commit = commit.Commit(commit_match.group(1)[:8]) 264 265 # Detect tags in the commit message 266 elif tag_match: 267 # Remove Tested-by self, since few will take much notice 268 if (tag_match.group(1) == 'Tested-by' and 269 tag_match.group(2).find(os.getenv('USER') + '@') != -1): 270 self.warn.append("Ignoring %s" % line) 271 elif tag_match.group(1) == 'Patch-cc': 272 self.commit.AddCc(tag_match.group(2).split(',')) 273 else: 274 self.tags.append(line); 275 276 # Suppress duplicate signoffs 277 elif signoff_match: 278 if (self.is_log or not self.commit or 279 self.commit.CheckDuplicateSignoff(signoff_match.group(1))): 280 out = [line] 281 282 # Well that means this is an ordinary line 283 else: 284 pos = 1 285 # Look for ugly ASCII characters 286 for ch in line: 287 # TODO: Would be nicer to report source filename and line 288 if ord(ch) > 0x80: 289 self.warn.append("Line %d/%d ('%s') has funny ascii char" % 290 (self.linenum, pos, line)) 291 pos += 1 292 293 # Look for space before tab 294 m = re_space_before_tab.match(line) 295 if m: 296 self.warn.append('Line %d/%d has space before tab' % 297 (self.linenum, m.start())) 298 299 # OK, we have a valid non-blank line 300 out = [line] 301 self.linenum += 1 302 self.skip_blank = False 303 if self.state == STATE_DIFFS: 304 pass 305 306 # If this is the start of the diffs section, emit our tags and 307 # change log 308 elif line == '---': 309 self.state = STATE_DIFFS 310 311 # Output the tags (signeoff first), then change list 312 out = [] 313 log = self.series.MakeChangeLog(self.commit) 314 out += self.FormatTags(self.tags) 315 out += [line] 316 if self.commit: 317 out += self.commit.notes 318 out += [''] + log 319 elif self.found_test: 320 if not re_allowed_after_test.match(line): 321 self.lines_after_test += 1 322 323 return out 324 325 def Finalize(self): 326 """Close out processing of this patch stream""" 327 self.CloseCommit() 328 if self.lines_after_test: 329 self.warn.append('Found %d lines after TEST=' % 330 self.lines_after_test) 331 332 def ProcessStream(self, infd, outfd): 333 """Copy a stream from infd to outfd, filtering out unwanting things. 334 335 This is used to process patch files one at a time. 336 337 Args: 338 infd: Input stream file object 339 outfd: Output stream file object 340 """ 341 # Extract the filename from each diff, for nice warnings 342 fname = None 343 last_fname = None 344 re_fname = re.compile('diff --git a/(.*) b/.*') 345 while True: 346 line = infd.readline() 347 if not line: 348 break 349 out = self.ProcessLine(line) 350 351 # Try to detect blank lines at EOF 352 for line in out: 353 match = re_fname.match(line) 354 if match: 355 last_fname = fname 356 fname = match.group(1) 357 if line == '+': 358 self.blank_count += 1 359 else: 360 if self.blank_count and (line == '-- ' or match): 361 self.warn.append("Found possible blank line(s) at " 362 "end of file '%s'" % last_fname) 363 outfd.write('+\n' * self.blank_count) 364 outfd.write(line + '\n') 365 self.blank_count = 0 366 self.Finalize() 367 368 369def GetMetaDataForList(commit_range, git_dir=None, count=None, 370 series = Series()): 371 """Reads out patch series metadata from the commits 372 373 This does a 'git log' on the relevant commits and pulls out the tags we 374 are interested in. 375 376 Args: 377 commit_range: Range of commits to count (e.g. 'HEAD..base') 378 git_dir: Path to git repositiory (None to use default) 379 count: Number of commits to list, or None for no limit 380 series: Series object to add information into. By default a new series 381 is started. 382 Returns: 383 A Series object containing information about the commits. 384 """ 385 params = gitutil.LogCmd(commit_range,reverse=True, count=count, 386 git_dir=git_dir) 387 stdout = command.RunPipe([params], capture=True).stdout 388 ps = PatchStream(series, is_log=True) 389 for line in stdout.splitlines(): 390 ps.ProcessLine(line) 391 ps.Finalize() 392 return series 393 394def GetMetaData(start, count): 395 """Reads out patch series metadata from the commits 396 397 This does a 'git log' on the relevant commits and pulls out the tags we 398 are interested in. 399 400 Args: 401 start: Commit to start from: 0=HEAD, 1=next one, etc. 402 count: Number of commits to list 403 """ 404 return GetMetaDataForList('HEAD~%d' % start, None, count) 405 406def FixPatch(backup_dir, fname, series, commit): 407 """Fix up a patch file, by adding/removing as required. 408 409 We remove our tags from the patch file, insert changes lists, etc. 410 The patch file is processed in place, and overwritten. 411 412 A backup file is put into backup_dir (if not None). 413 414 Args: 415 fname: Filename to patch file to process 416 series: Series information about this patch set 417 commit: Commit object for this patch file 418 Return: 419 A list of errors, or [] if all ok. 420 """ 421 handle, tmpname = tempfile.mkstemp() 422 outfd = os.fdopen(handle, 'w') 423 infd = open(fname, 'r') 424 ps = PatchStream(series) 425 ps.commit = commit 426 ps.ProcessStream(infd, outfd) 427 infd.close() 428 outfd.close() 429 430 # Create a backup file if required 431 if backup_dir: 432 shutil.copy(fname, os.path.join(backup_dir, os.path.basename(fname))) 433 shutil.move(tmpname, fname) 434 return ps.warn 435 436def FixPatches(series, fnames): 437 """Fix up a list of patches identified by filenames 438 439 The patch files are processed in place, and overwritten. 440 441 Args: 442 series: The series object 443 fnames: List of patch files to process 444 """ 445 # Current workflow creates patches, so we shouldn't need a backup 446 backup_dir = None #tempfile.mkdtemp('clean-patch') 447 count = 0 448 for fname in fnames: 449 commit = series.commits[count] 450 commit.patch = fname 451 result = FixPatch(backup_dir, fname, series, commit) 452 if result: 453 print '%d warnings for %s:' % (len(result), fname) 454 for warn in result: 455 print '\t', warn 456 print 457 count += 1 458 print 'Cleaned %d patches' % count 459 return series 460 461def InsertCoverLetter(fname, series, count): 462 """Inserts a cover letter with the required info into patch 0 463 464 Args: 465 fname: Input / output filename of the cover letter file 466 series: Series object 467 count: Number of patches in the series 468 """ 469 fd = open(fname, 'r') 470 lines = fd.readlines() 471 fd.close() 472 473 fd = open(fname, 'w') 474 text = series.cover 475 prefix = series.GetPatchPrefix() 476 for line in lines: 477 if line.startswith('Subject:'): 478 # TODO: if more than 10 patches this should save 00/xx, not 0/xx 479 line = 'Subject: [%s 0/%d] %s\n' % (prefix, count, text[0]) 480 481 # Insert our cover letter 482 elif line.startswith('*** BLURB HERE ***'): 483 # First the blurb test 484 line = '\n'.join(text[1:]) + '\n' 485 if series.get('notes'): 486 line += '\n'.join(series.notes) + '\n' 487 488 # Now the change list 489 out = series.MakeChangeLog(None) 490 line += '\n' + '\n'.join(out) 491 fd.write(line) 492 fd.close() 493