1# 2# Helper functions for committing data to git and pushing upstream 3# 4# Copyright (c) 2017, Intel Corporation. 5# Copyright (c) 2019, Linux Foundation 6# 7# SPDX-License-Identifier: GPL-2.0-only 8# 9 10import os 11import re 12import sys 13from operator import attrgetter 14from collections import namedtuple 15from oeqa.utils.git import GitRepo, GitError 16 17class ArchiveError(Exception): 18 """Internal error handling of this script""" 19 20def format_str(string, fields): 21 """Format string using the given fields (dict)""" 22 try: 23 return string.format(**fields) 24 except KeyError as err: 25 raise ArchiveError("Unable to expand string '{}': unknown field {} " 26 "(valid fields are: {})".format( 27 string, err, ', '.join(sorted(fields.keys())))) 28 29 30def init_git_repo(path, no_create, bare, log): 31 """Initialize local Git repository""" 32 path = os.path.abspath(path) 33 if os.path.isfile(path): 34 raise ArchiveError("Invalid Git repo at {}: path exists but is not a " 35 "directory".format(path)) 36 if not os.path.isdir(path) or not os.listdir(path): 37 if no_create: 38 raise ArchiveError("No git repo at {}, refusing to create " 39 "one".format(path)) 40 if not os.path.isdir(path): 41 try: 42 os.mkdir(path) 43 except (FileNotFoundError, PermissionError) as err: 44 raise ArchiveError("Failed to mkdir {}: {}".format(path, err)) 45 if not os.listdir(path): 46 log.info("Initializing a new Git repo at %s", path) 47 repo = GitRepo.init(path, bare) 48 try: 49 repo = GitRepo(path, is_topdir=True) 50 except GitError: 51 raise ArchiveError("Non-empty directory that is not a Git repository " 52 "at {}\nPlease specify an existing Git repository, " 53 "an empty directory or a non-existing directory " 54 "path.".format(path)) 55 return repo 56 57 58def git_commit_data(repo, data_dir, branch, message, exclude, notes, log): 59 """Commit data into a Git repository""" 60 log.info("Committing data into to branch %s", branch) 61 tmp_index = os.path.join(repo.git_dir, 'index.oe-git-archive') 62 try: 63 # Create new tree object from the data 64 env_update = {'GIT_INDEX_FILE': tmp_index, 65 'GIT_WORK_TREE': os.path.abspath(data_dir)} 66 repo.run_cmd('add .', env_update) 67 68 # Remove files that are excluded 69 if exclude: 70 repo.run_cmd(['rm', '--cached'] + [f for f in exclude], env_update) 71 72 tree = repo.run_cmd('write-tree', env_update) 73 74 # Create new commit object from the tree 75 parent = repo.rev_parse(branch) 76 if not parent: 77 parent = repo.rev_parse("origin/" + branch) 78 git_cmd = ['commit-tree', tree, '-m', message] 79 if parent: 80 git_cmd += ['-p', parent] 81 commit = repo.run_cmd(git_cmd, env_update) 82 83 # Create git notes 84 for ref, filename in notes: 85 ref = ref.format(branch_name=branch) 86 repo.run_cmd(['notes', '--ref', ref, 'add', 87 '-F', os.path.abspath(filename), commit]) 88 89 # Update branch head 90 git_cmd = ['update-ref', 'refs/heads/' + branch, commit] 91 repo.run_cmd(git_cmd) 92 93 # Update current HEAD, if we're on branch 'branch' 94 if not repo.bare and repo.get_current_branch() == branch: 95 log.info("Updating %s HEAD to latest commit", repo.top_dir) 96 repo.run_cmd('reset --hard') 97 98 return commit 99 finally: 100 if os.path.exists(tmp_index): 101 os.unlink(tmp_index) 102 103 104def expand_tag_strings(repo, name_pattern, msg_subj_pattern, msg_body_pattern, 105 keywords): 106 """Generate tag name and message, with support for running id number""" 107 keyws = keywords.copy() 108 # Tag number is handled specially: if not defined, we autoincrement it 109 if 'tag_number' not in keyws: 110 # Fill in all other fields than 'tag_number' 111 keyws['tag_number'] = '{tag_number}' 112 tag_re = format_str(name_pattern, keyws) 113 # Replace parentheses for proper regex matching 114 tag_re = tag_re.replace('(', '\(').replace(')', '\)') + '$' 115 # Inject regex group pattern for 'tag_number' 116 tag_re = tag_re.format(tag_number='(?P<tag_number>[0-9]{1,5})') 117 118 keyws['tag_number'] = 0 119 for existing_tag in repo.run_cmd('tag').splitlines(): 120 match = re.match(tag_re, existing_tag) 121 122 if match and int(match.group('tag_number')) >= keyws['tag_number']: 123 keyws['tag_number'] = int(match.group('tag_number')) + 1 124 125 tag_name = format_str(name_pattern, keyws) 126 msg_subj= format_str(msg_subj_pattern.strip(), keyws) 127 msg_body = format_str(msg_body_pattern, keyws) 128 return tag_name, msg_subj + '\n\n' + msg_body 129 130def gitarchive(data_dir, git_dir, no_create, bare, commit_msg_subject, commit_msg_body, branch_name, no_tag, tagname, tag_msg_subject, tag_msg_body, exclude, notes, push, keywords, log): 131 132 if not os.path.isdir(data_dir): 133 raise ArchiveError("Not a directory: {}".format(data_dir)) 134 135 data_repo = init_git_repo(git_dir, no_create, bare, log) 136 137 # Expand strings early in order to avoid getting into inconsistent 138 # state (e.g. no tag even if data was committed) 139 commit_msg = format_str(commit_msg_subject.strip(), keywords) 140 commit_msg += '\n\n' + format_str(commit_msg_body, keywords) 141 branch_name = format_str(branch_name, keywords) 142 tag_name = None 143 if not no_tag and tagname: 144 tag_name, tag_msg = expand_tag_strings(data_repo, tagname, 145 tag_msg_subject, 146 tag_msg_body, keywords) 147 148 # Commit data 149 commit = git_commit_data(data_repo, data_dir, branch_name, 150 commit_msg, exclude, notes, log) 151 152 # Create tag 153 if tag_name: 154 log.info("Creating tag %s", tag_name) 155 data_repo.run_cmd(['tag', '-a', '-m', tag_msg, tag_name, commit]) 156 157 # Push data to remote 158 if push: 159 cmd = ['push', '--tags'] 160 # If no remote is given we push with the default settings from 161 # gitconfig 162 if push is not True: 163 notes_refs = ['refs/notes/' + ref.format(branch_name=branch_name) 164 for ref, _ in notes] 165 cmd.extend([push, branch_name] + notes_refs) 166 log.info("Pushing data to remote") 167 data_repo.run_cmd(cmd) 168 169# Container class for tester revisions 170TestedRev = namedtuple('TestedRev', 'commit commit_number tags') 171 172def get_test_runs(log, repo, tag_name, **kwargs): 173 """Get a sorted list of test runs, matching given pattern""" 174 # First, get field names from the tag name pattern 175 field_names = [m.group(1) for m in re.finditer(r'{(\w+)}', tag_name)] 176 undef_fields = [f for f in field_names if f not in kwargs.keys()] 177 178 # Fields for formatting tag name pattern 179 str_fields = dict([(f, '*') for f in field_names]) 180 str_fields.update(kwargs) 181 182 # Get a list of all matching tags 183 tag_pattern = tag_name.format(**str_fields) 184 tags = repo.run_cmd(['tag', '-l', tag_pattern]).splitlines() 185 log.debug("Found %d tags matching pattern '%s'", len(tags), tag_pattern) 186 187 # Parse undefined fields from tag names 188 str_fields = dict([(f, r'(?P<{}>[\w\-.()]+)'.format(f)) for f in field_names]) 189 str_fields['branch'] = r'(?P<branch>[\w\-.()/]+)' 190 str_fields['commit'] = '(?P<commit>[0-9a-f]{7,40})' 191 str_fields['commit_number'] = '(?P<commit_number>[0-9]{1,7})' 192 str_fields['tag_number'] = '(?P<tag_number>[0-9]{1,5})' 193 # escape parenthesis in fields in order to not messa up the regexp 194 fixed_fields = dict([(k, v.replace('(', r'\(').replace(')', r'\)')) for k, v in kwargs.items()]) 195 str_fields.update(fixed_fields) 196 tag_re = re.compile(tag_name.format(**str_fields)) 197 198 # Parse fields from tags 199 revs = [] 200 for tag in tags: 201 m = tag_re.match(tag) 202 groups = m.groupdict() 203 revs.append([groups[f] for f in undef_fields] + [tag]) 204 205 # Return field names and a sorted list of revs 206 return undef_fields, sorted(revs) 207 208def get_test_revs(log, repo, tag_name, **kwargs): 209 """Get list of all tested revisions""" 210 fields, runs = get_test_runs(log, repo, tag_name, **kwargs) 211 212 revs = {} 213 commit_i = fields.index('commit') 214 commit_num_i = fields.index('commit_number') 215 for run in runs: 216 commit = run[commit_i] 217 commit_num = run[commit_num_i] 218 tag = run[-1] 219 if not commit in revs: 220 revs[commit] = TestedRev(commit, commit_num, [tag]) 221 else: 222 assert commit_num == revs[commit].commit_number, "Commit numbers do not match" 223 revs[commit].tags.append(tag) 224 225 # Return in sorted table 226 revs = sorted(revs.values(), key=attrgetter('commit_number')) 227 log.debug("Found %d tested revisions:\n %s", len(revs), 228 "\n ".join(['{} ({})'.format(rev.commit_number, rev.commit) for rev in revs])) 229 return revs 230 231def rev_find(revs, attr, val): 232 """Search from a list of TestedRev""" 233 for i, rev in enumerate(revs): 234 if getattr(rev, attr) == val: 235 return i 236 raise ValueError("Unable to find '{}' value '{}'".format(attr, val)) 237 238