1# Copyright (c) 2011 The Chromium OS Authors. 2# 3# See file CREDITS for list of people who contributed to this 4# project. 5# 6# This program is free software; you can redistribute it and/or 7# modify it under the terms of the GNU General Public License as 8# published by the Free Software Foundation; either version 2 of 9# the License, or (at your option) any later version. 10# 11# This program is distributed in the hope that it will be useful, 12# but WITHOUT ANY WARRANTY; without even the implied warranty of 13# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14# GNU General Public License for more details. 15# 16# You should have received a copy of the GNU General Public License 17# along with this program; if not, write to the Free Software 18# Foundation, Inc., 59 Temple Place, Suite 330, Boston, 19# MA 02111-1307 USA 20# 21 22import os 23import re 24import shutil 25import tempfile 26 27import command 28import commit 29import gitutil 30from series import Series 31 32# Tags that we detect and remove 33re_remove = re.compile('^BUG=|^TEST=|^BRANCH=|^Change-Id:|^Review URL:' 34 '|Reviewed-on:|Reviewed-by:|Commit-Ready:') 35 36# Lines which are allowed after a TEST= line 37re_allowed_after_test = re.compile('^Signed-off-by:') 38 39# Signoffs 40re_signoff = re.compile('^Signed-off-by:') 41 42# The start of the cover letter 43re_cover = re.compile('^Cover-letter:') 44 45# Patch series tag 46re_series = re.compile('^Series-(\w*): *(.*)') 47 48# Commit tags that we want to collect and keep 49re_tag = re.compile('^(Tested-by|Acked-by|Cc): (.*)') 50 51# The start of a new commit in the git log 52re_commit = re.compile('^commit (.*)') 53 54# We detect these since checkpatch doesn't always do it 55re_space_before_tab = re.compile('^[+].* \t') 56 57# States we can be in - can we use range() and still have comments? 58STATE_MSG_HEADER = 0 # Still in the message header 59STATE_PATCH_SUBJECT = 1 # In patch subject (first line of log for a commit) 60STATE_PATCH_HEADER = 2 # In patch header (after the subject) 61STATE_DIFFS = 3 # In the diff part (past --- line) 62 63class PatchStream: 64 """Class for detecting/injecting tags in a patch or series of patches 65 66 We support processing the output of 'git log' to read out the tags we 67 are interested in. We can also process a patch file in order to remove 68 unwanted tags or inject additional ones. These correspond to the two 69 phases of processing. 70 """ 71 def __init__(self, series, name=None, is_log=False): 72 self.skip_blank = False # True to skip a single blank line 73 self.found_test = False # Found a TEST= line 74 self.lines_after_test = 0 # MNumber of lines found after TEST= 75 self.warn = [] # List of warnings we have collected 76 self.linenum = 1 # Output line number we are up to 77 self.in_section = None # Name of start...END section we are in 78 self.notes = [] # Series notes 79 self.section = [] # The current section...END section 80 self.series = series # Info about the patch series 81 self.is_log = is_log # True if indent like git log 82 self.in_change = 0 # Non-zero if we are in a change list 83 self.blank_count = 0 # Number of blank lines stored up 84 self.state = STATE_MSG_HEADER # What state are we in? 85 self.tags = [] # Tags collected, like Tested-by... 86 self.signoff = [] # Contents of signoff line 87 self.commit = None # Current commit 88 89 def AddToSeries(self, line, name, value): 90 """Add a new Series-xxx tag. 91 92 When a Series-xxx tag is detected, we come here to record it, if we 93 are scanning a 'git log'. 94 95 Args: 96 line: Source line containing tag (useful for debug/error messages) 97 name: Tag name (part after 'Series-') 98 value: Tag value (part after 'Series-xxx: ') 99 """ 100 if name == 'notes': 101 self.in_section = name 102 self.skip_blank = False 103 if self.is_log: 104 self.series.AddTag(self.commit, line, name, value) 105 106 def CloseCommit(self): 107 """Save the current commit into our commit list, and reset our state""" 108 if self.commit and self.is_log: 109 self.series.AddCommit(self.commit) 110 self.commit = None 111 112 def FormatTags(self, tags): 113 out_list = [] 114 for tag in sorted(tags): 115 if tag.startswith('Cc:'): 116 tag_list = tag[4:].split(',') 117 out_list += gitutil.BuildEmailList(tag_list, 'Cc:') 118 else: 119 out_list.append(tag) 120 return out_list 121 122 def ProcessLine(self, line): 123 """Process a single line of a patch file or commit log 124 125 This process a line and returns a list of lines to output. The list 126 may be empty or may contain multiple output lines. 127 128 This is where all the complicated logic is located. The class's 129 state is used to move between different states and detect things 130 properly. 131 132 We can be in one of two modes: 133 self.is_log == True: This is 'git log' mode, where most output is 134 indented by 4 characters and we are scanning for tags 135 136 self.is_log == False: This is 'patch' mode, where we already have 137 all the tags, and are processing patches to remove junk we 138 don't want, and add things we think are required. 139 140 Args: 141 line: text line to process 142 143 Returns: 144 list of output lines, or [] if nothing should be output 145 """ 146 # Initially we have no output. Prepare the input line string 147 out = [] 148 line = line.rstrip('\n') 149 if self.is_log: 150 if line[:4] == ' ': 151 line = line[4:] 152 153 # Handle state transition and skipping blank lines 154 series_match = re_series.match(line) 155 commit_match = re_commit.match(line) if self.is_log else None 156 tag_match = None 157 if self.state == STATE_PATCH_HEADER: 158 tag_match = re_tag.match(line) 159 is_blank = not line.strip() 160 if is_blank: 161 if (self.state == STATE_MSG_HEADER 162 or self.state == STATE_PATCH_SUBJECT): 163 self.state += 1 164 165 # We don't have a subject in the text stream of patch files 166 # It has its own line with a Subject: tag 167 if not self.is_log and self.state == STATE_PATCH_SUBJECT: 168 self.state += 1 169 elif commit_match: 170 self.state = STATE_MSG_HEADER 171 172 # If we are in a section, keep collecting lines until we see END 173 if self.in_section: 174 if line == 'END': 175 if self.in_section == 'cover': 176 self.series.cover = self.section 177 elif self.in_section == 'notes': 178 if self.is_log: 179 self.series.notes += self.section 180 else: 181 self.warn.append("Unknown section '%s'" % self.in_section) 182 self.in_section = None 183 self.skip_blank = True 184 self.section = [] 185 else: 186 self.section.append(line) 187 188 # Detect the commit subject 189 elif not is_blank and self.state == STATE_PATCH_SUBJECT: 190 self.commit.subject = line 191 192 # Detect the tags we want to remove, and skip blank lines 193 elif re_remove.match(line): 194 self.skip_blank = True 195 196 # TEST= should be the last thing in the commit, so remove 197 # everything after it 198 if line.startswith('TEST='): 199 self.found_test = True 200 elif self.skip_blank and is_blank: 201 self.skip_blank = False 202 203 # Detect the start of a cover letter section 204 elif re_cover.match(line): 205 self.in_section = 'cover' 206 self.skip_blank = False 207 208 # If we are in a change list, key collected lines until a blank one 209 elif self.in_change: 210 if is_blank: 211 # Blank line ends this change list 212 self.in_change = 0 213 elif line == '---' or re_signoff.match(line): 214 self.in_change = 0 215 out = self.ProcessLine(line) 216 else: 217 if self.is_log: 218 self.series.AddChange(self.in_change, self.commit, line) 219 self.skip_blank = False 220 221 # Detect Series-xxx tags 222 elif series_match: 223 name = series_match.group(1) 224 value = series_match.group(2) 225 if name == 'changes': 226 # value is the version number: e.g. 1, or 2 227 try: 228 value = int(value) 229 except ValueError as str: 230 raise ValueError("%s: Cannot decode version info '%s'" % 231 (self.commit.hash, line)) 232 self.in_change = int(value) 233 else: 234 self.AddToSeries(line, name, value) 235 self.skip_blank = True 236 237 # Detect the start of a new commit 238 elif commit_match: 239 self.CloseCommit() 240 # TODO: We should store the whole hash, and just display a subset 241 self.commit = commit.Commit(commit_match.group(1)[:8]) 242 243 # Detect tags in the commit message 244 elif tag_match: 245 # Remove Tested-by self, since few will take much notice 246 if (tag_match.group(1) == 'Tested-by' and 247 tag_match.group(2).find(os.getenv('USER') + '@') != -1): 248 self.warn.append("Ignoring %s" % line) 249 elif tag_match.group(1) == 'Cc': 250 self.commit.AddCc(tag_match.group(2).split(',')) 251 else: 252 self.tags.append(line); 253 254 # Well that means this is an ordinary line 255 else: 256 pos = 1 257 # Look for ugly ASCII characters 258 for ch in line: 259 # TODO: Would be nicer to report source filename and line 260 if ord(ch) > 0x80: 261 self.warn.append("Line %d/%d ('%s') has funny ascii char" % 262 (self.linenum, pos, line)) 263 pos += 1 264 265 # Look for space before tab 266 m = re_space_before_tab.match(line) 267 if m: 268 self.warn.append('Line %d/%d has space before tab' % 269 (self.linenum, m.start())) 270 271 # OK, we have a valid non-blank line 272 out = [line] 273 self.linenum += 1 274 self.skip_blank = False 275 if self.state == STATE_DIFFS: 276 pass 277 278 # If this is the start of the diffs section, emit our tags and 279 # change log 280 elif line == '---': 281 self.state = STATE_DIFFS 282 283 # Output the tags (signeoff first), then change list 284 out = [] 285 log = self.series.MakeChangeLog(self.commit) 286 out += self.FormatTags(self.tags) 287 out += [line] + log 288 elif self.found_test: 289 if not re_allowed_after_test.match(line): 290 self.lines_after_test += 1 291 292 return out 293 294 def Finalize(self): 295 """Close out processing of this patch stream""" 296 self.CloseCommit() 297 if self.lines_after_test: 298 self.warn.append('Found %d lines after TEST=' % 299 self.lines_after_test) 300 301 def ProcessStream(self, infd, outfd): 302 """Copy a stream from infd to outfd, filtering out unwanting things. 303 304 This is used to process patch files one at a time. 305 306 Args: 307 infd: Input stream file object 308 outfd: Output stream file object 309 """ 310 # Extract the filename from each diff, for nice warnings 311 fname = None 312 last_fname = None 313 re_fname = re.compile('diff --git a/(.*) b/.*') 314 while True: 315 line = infd.readline() 316 if not line: 317 break 318 out = self.ProcessLine(line) 319 320 # Try to detect blank lines at EOF 321 for line in out: 322 match = re_fname.match(line) 323 if match: 324 last_fname = fname 325 fname = match.group(1) 326 if line == '+': 327 self.blank_count += 1 328 else: 329 if self.blank_count and (line == '-- ' or match): 330 self.warn.append("Found possible blank line(s) at " 331 "end of file '%s'" % last_fname) 332 outfd.write('+\n' * self.blank_count) 333 outfd.write(line + '\n') 334 self.blank_count = 0 335 self.Finalize() 336 337 338def GetMetaDataForList(commit_range, git_dir=None, count=None, 339 series = Series()): 340 """Reads out patch series metadata from the commits 341 342 This does a 'git log' on the relevant commits and pulls out the tags we 343 are interested in. 344 345 Args: 346 commit_range: Range of commits to count (e.g. 'HEAD..base') 347 git_dir: Path to git repositiory (None to use default) 348 count: Number of commits to list, or None for no limit 349 series: Series object to add information into. By default a new series 350 is started. 351 Returns: 352 A Series object containing information about the commits. 353 """ 354 params = ['git', 'log', '--no-color', '--reverse', commit_range] 355 if count is not None: 356 params[2:2] = ['-n%d' % count] 357 if git_dir: 358 params[1:1] = ['--git-dir', git_dir] 359 pipe = [params] 360 stdout = command.RunPipe(pipe, capture=True).stdout 361 ps = PatchStream(series, is_log=True) 362 for line in stdout.splitlines(): 363 ps.ProcessLine(line) 364 ps.Finalize() 365 return series 366 367def GetMetaData(start, count): 368 """Reads out patch series metadata from the commits 369 370 This does a 'git log' on the relevant commits and pulls out the tags we 371 are interested in. 372 373 Args: 374 start: Commit to start from: 0=HEAD, 1=next one, etc. 375 count: Number of commits to list 376 """ 377 return GetMetaDataForList('HEAD~%d' % start, None, count) 378 379def FixPatch(backup_dir, fname, series, commit): 380 """Fix up a patch file, by adding/removing as required. 381 382 We remove our tags from the patch file, insert changes lists, etc. 383 The patch file is processed in place, and overwritten. 384 385 A backup file is put into backup_dir (if not None). 386 387 Args: 388 fname: Filename to patch file to process 389 series: Series information about this patch set 390 commit: Commit object for this patch file 391 Return: 392 A list of errors, or [] if all ok. 393 """ 394 handle, tmpname = tempfile.mkstemp() 395 outfd = os.fdopen(handle, 'w') 396 infd = open(fname, 'r') 397 ps = PatchStream(series) 398 ps.commit = commit 399 ps.ProcessStream(infd, outfd) 400 infd.close() 401 outfd.close() 402 403 # Create a backup file if required 404 if backup_dir: 405 shutil.copy(fname, os.path.join(backup_dir, os.path.basename(fname))) 406 shutil.move(tmpname, fname) 407 return ps.warn 408 409def FixPatches(series, fnames): 410 """Fix up a list of patches identified by filenames 411 412 The patch files are processed in place, and overwritten. 413 414 Args: 415 series: The series object 416 fnames: List of patch files to process 417 """ 418 # Current workflow creates patches, so we shouldn't need a backup 419 backup_dir = None #tempfile.mkdtemp('clean-patch') 420 count = 0 421 for fname in fnames: 422 commit = series.commits[count] 423 commit.patch = fname 424 result = FixPatch(backup_dir, fname, series, commit) 425 if result: 426 print '%d warnings for %s:' % (len(result), fname) 427 for warn in result: 428 print '\t', warn 429 print 430 count += 1 431 print 'Cleaned %d patches' % count 432 return series 433 434def InsertCoverLetter(fname, series, count): 435 """Inserts a cover letter with the required info into patch 0 436 437 Args: 438 fname: Input / output filename of the cover letter file 439 series: Series object 440 count: Number of patches in the series 441 """ 442 fd = open(fname, 'r') 443 lines = fd.readlines() 444 fd.close() 445 446 fd = open(fname, 'w') 447 text = series.cover 448 prefix = series.GetPatchPrefix() 449 for line in lines: 450 if line.startswith('Subject:'): 451 # TODO: if more than 10 patches this should save 00/xx, not 0/xx 452 line = 'Subject: [%s 0/%d] %s\n' % (prefix, count, text[0]) 453 454 # Insert our cover letter 455 elif line.startswith('*** BLURB HERE ***'): 456 # First the blurb test 457 line = '\n'.join(text[1:]) + '\n' 458 if series.get('notes'): 459 line += '\n'.join(series.notes) + '\n' 460 461 # Now the change list 462 out = series.MakeChangeLog(None) 463 line += '\n' + '\n'.join(out) 464 fd.write(line) 465 fd.close() 466