1#!/usr/bin/env python3 2# 3# Copyright BitBake Contributors 4# 5# SPDX-License-Identifier: GPL-2.0-only 6# 7 8"""git-make-shallow: make the current git repository shallow 9 10Remove the history of the specified revisions, then optionally filter the 11available refs to those specified. 12""" 13 14import argparse 15import collections 16import errno 17import itertools 18import os 19import subprocess 20import sys 21import warnings 22warnings.simplefilter("default") 23 24version = 1.0 25 26 27def main(): 28 if sys.version_info < (3, 4, 0): 29 sys.exit('Python 3.4 or greater is required') 30 31 git_dir = check_output(['git', 'rev-parse', '--git-dir']).rstrip() 32 shallow_file = os.path.join(git_dir, 'shallow') 33 if os.path.exists(shallow_file): 34 try: 35 check_output(['git', 'fetch', '--unshallow']) 36 except subprocess.CalledProcessError: 37 try: 38 os.unlink(shallow_file) 39 except OSError as exc: 40 if exc.errno != errno.ENOENT: 41 raise 42 43 args = process_args() 44 revs = check_output(['git', 'rev-list'] + args.revisions).splitlines() 45 46 make_shallow(shallow_file, args.revisions, args.refs) 47 48 ref_revs = check_output(['git', 'rev-list'] + args.refs).splitlines() 49 remaining_history = set(revs) & set(ref_revs) 50 for rev in remaining_history: 51 if check_output(['git', 'rev-parse', '{}^@'.format(rev)]): 52 sys.exit('Error: %s was not made shallow' % rev) 53 54 filter_refs(args.refs) 55 56 if args.shrink: 57 shrink_repo(git_dir) 58 subprocess.check_call(['git', 'fsck', '--unreachable']) 59 60 61def process_args(): 62 # TODO: add argument to automatically keep local-only refs, since they 63 # can't be easily restored with a git fetch. 64 parser = argparse.ArgumentParser(description='Remove the history of the specified revisions, then optionally filter the available refs to those specified.') 65 parser.add_argument('--ref', '-r', metavar='REF', action='append', dest='refs', help='remove all but the specified refs (cumulative)') 66 parser.add_argument('--shrink', '-s', action='store_true', help='shrink the git repository by repacking and pruning') 67 parser.add_argument('revisions', metavar='REVISION', nargs='+', help='a git revision/commit') 68 if len(sys.argv) < 2: 69 parser.print_help() 70 sys.exit(2) 71 72 args = parser.parse_args() 73 74 if args.refs: 75 args.refs = check_output(['git', 'rev-parse', '--symbolic-full-name'] + args.refs).splitlines() 76 else: 77 args.refs = get_all_refs(lambda r, t, tt: t == 'commit' or tt == 'commit') 78 79 args.refs = list(filter(lambda r: not r.endswith('/HEAD'), args.refs)) 80 args.revisions = check_output(['git', 'rev-parse'] + ['%s^{}' % i for i in args.revisions]).splitlines() 81 return args 82 83 84def check_output(cmd, input=None): 85 return subprocess.check_output(cmd, universal_newlines=True, input=input) 86 87 88def make_shallow(shallow_file, revisions, refs): 89 """Remove the history of the specified revisions.""" 90 for rev in follow_history_intersections(revisions, refs): 91 print("Processing %s" % rev) 92 with open(shallow_file, 'a') as f: 93 f.write(rev + '\n') 94 95 96def get_all_refs(ref_filter=None): 97 """Return all the existing refs in this repository, optionally filtering the refs.""" 98 ref_output = check_output(['git', 'for-each-ref', '--format=%(refname)\t%(objecttype)\t%(*objecttype)']) 99 ref_split = [tuple(iter_extend(l.rsplit('\t'), 3)) for l in ref_output.splitlines()] 100 if ref_filter: 101 ref_split = (e for e in ref_split if ref_filter(*e)) 102 refs = [r[0] for r in ref_split] 103 return refs 104 105 106def iter_extend(iterable, length, obj=None): 107 """Ensure that iterable is the specified length by extending with obj.""" 108 return itertools.islice(itertools.chain(iterable, itertools.repeat(obj)), length) 109 110 111def filter_refs(refs): 112 """Remove all but the specified refs from the git repository.""" 113 all_refs = get_all_refs() 114 to_remove = set(all_refs) - set(refs) 115 if to_remove: 116 check_output(['xargs', '-0', '-n', '1', 'git', 'update-ref', '-d', '--no-deref'], 117 input=''.join(l + '\0' for l in to_remove)) 118 119 120def follow_history_intersections(revisions, refs): 121 """Determine all the points where the history of the specified revisions intersects the specified refs.""" 122 queue = collections.deque(revisions) 123 seen = set() 124 125 for rev in iter_except(queue.popleft, IndexError): 126 if rev in seen: 127 continue 128 129 parents = check_output(['git', 'rev-parse', '%s^@' % rev]).splitlines() 130 131 yield rev 132 seen.add(rev) 133 134 if not parents: 135 continue 136 137 check_refs = check_output(['git', 'merge-base', '--independent'] + sorted(refs)).splitlines() 138 for parent in parents: 139 for ref in check_refs: 140 print("Checking %s vs %s" % (parent, ref)) 141 try: 142 merge_base = check_output(['git', 'merge-base', parent, ref]).rstrip() 143 except subprocess.CalledProcessError: 144 continue 145 else: 146 queue.append(merge_base) 147 148 149def iter_except(func, exception, start=None): 150 """Yield a function repeatedly until it raises an exception.""" 151 try: 152 if start is not None: 153 yield start() 154 while True: 155 yield func() 156 except exception: 157 pass 158 159 160def shrink_repo(git_dir): 161 """Shrink the newly shallow repository, removing the unreachable objects.""" 162 subprocess.check_call(['git', 'reflog', 'expire', '--expire-unreachable=now', '--all']) 163 subprocess.check_call(['git', 'repack', '-ad']) 164 try: 165 os.unlink(os.path.join(git_dir, 'objects', 'info', 'alternates')) 166 except OSError as exc: 167 if exc.errno != errno.ENOENT: 168 raise 169 subprocess.check_call(['git', 'prune', '--expire', 'now']) 170 171 172if __name__ == '__main__': 173 main() 174