1""" 2BitBake 'Fetch' git implementation 3 4git fetcher support the SRC_URI with format of: 5SRC_URI = "git://some.host/somepath;OptionA=xxx;OptionB=xxx;..." 6 7Supported SRC_URI options are: 8 9- branch 10 The git branch to retrieve from. The default is "master" 11 12 This option also supports multiple branch fetching, with branches 13 separated by commas. In multiple branches case, the name option 14 must have the same number of names to match the branches, which is 15 used to specify the SRC_REV for the branch 16 e.g: 17 SRC_URI="git://some.host/somepath;branch=branchX,branchY;name=nameX,nameY" 18 SRCREV_nameX = "xxxxxxxxxxxxxxxxxxxx" 19 SRCREV_nameY = "YYYYYYYYYYYYYYYYYYYY" 20 21- tag 22 The git tag to retrieve. The default is "master" 23 24- protocol 25 The method to use to access the repository. Common options are "git", 26 "http", "https", "file", "ssh" and "rsync". The default is "git". 27 28- rebaseable 29 rebaseable indicates that the upstream git repo may rebase in the future, 30 and current revision may disappear from upstream repo. This option will 31 remind fetcher to preserve local cache carefully for future use. 32 The default value is "0", set rebaseable=1 for rebaseable git repo. 33 34- nocheckout 35 Don't checkout source code when unpacking. set this option for the recipe 36 who has its own routine to checkout code. 37 The default is "0", set nocheckout=1 if needed. 38 39- bareclone 40 Create a bare clone of the source code and don't checkout the source code 41 when unpacking. Set this option for the recipe who has its own routine to 42 checkout code and tracking branch requirements. 43 The default is "0", set bareclone=1 if needed. 44 45- nobranch 46 Don't check the SHA validation for branch. set this option for the recipe 47 referring to commit which is valid in any namespace (branch, tag, ...) 48 instead of branch. 49 The default is "0", set nobranch=1 if needed. 50 51- usehead 52 For local git:// urls to use the current branch HEAD as the revision for use with 53 AUTOREV. Implies nobranch. 54 55""" 56 57# Copyright (C) 2005 Richard Purdie 58# 59# SPDX-License-Identifier: GPL-2.0-only 60# 61 62import collections 63import errno 64import fnmatch 65import os 66import re 67import shlex 68import subprocess 69import tempfile 70import bb 71import bb.progress 72from contextlib import contextmanager 73from bb.fetch2 import FetchMethod 74from bb.fetch2 import runfetchcmd 75from bb.fetch2 import logger 76 77 78class GitProgressHandler(bb.progress.LineFilterProgressHandler): 79 """Extract progress information from git output""" 80 def __init__(self, d): 81 self._buffer = '' 82 self._count = 0 83 super(GitProgressHandler, self).__init__(d) 84 # Send an initial progress event so the bar gets shown 85 self._fire_progress(-1) 86 87 def write(self, string): 88 self._buffer += string 89 stages = ['Counting objects', 'Compressing objects', 'Receiving objects', 'Resolving deltas'] 90 stage_weights = [0.2, 0.05, 0.5, 0.25] 91 stagenum = 0 92 for i, stage in reversed(list(enumerate(stages))): 93 if stage in self._buffer: 94 stagenum = i 95 self._buffer = '' 96 break 97 self._status = stages[stagenum] 98 percs = re.findall(r'(\d+)%', string) 99 if percs: 100 progress = int(round((int(percs[-1]) * stage_weights[stagenum]) + (sum(stage_weights[:stagenum]) * 100))) 101 rates = re.findall(r'([\d.]+ [a-zA-Z]*/s+)', string) 102 if rates: 103 rate = rates[-1] 104 else: 105 rate = None 106 self.update(progress, rate) 107 else: 108 if stagenum == 0: 109 percs = re.findall(r': (\d+)', string) 110 if percs: 111 count = int(percs[-1]) 112 if count > self._count: 113 self._count = count 114 self._fire_progress(-count) 115 super(GitProgressHandler, self).write(string) 116 117 118class Git(FetchMethod): 119 bitbake_dir = os.path.abspath(os.path.join(os.path.dirname(os.path.join(os.path.abspath(__file__))), '..', '..', '..')) 120 make_shallow_path = os.path.join(bitbake_dir, 'bin', 'git-make-shallow') 121 122 """Class to fetch a module or modules from git repositories""" 123 def init(self, d): 124 pass 125 126 def supports(self, ud, d): 127 """ 128 Check to see if a given url can be fetched with git. 129 """ 130 return ud.type in ['git'] 131 132 def supports_checksum(self, urldata): 133 return False 134 135 def urldata_init(self, ud, d): 136 """ 137 init git specific variable within url data 138 so that the git method like latest_revision() can work 139 """ 140 if 'protocol' in ud.parm: 141 ud.proto = ud.parm['protocol'] 142 elif not ud.host: 143 ud.proto = 'file' 144 else: 145 ud.proto = "git" 146 if ud.host == "github.com" and ud.proto == "git": 147 # github stopped supporting git protocol 148 # https://github.blog/2021-09-01-improving-git-protocol-security-github/#no-more-unauthenticated-git 149 ud.proto = "https" 150 bb.warn("URL: %s uses git protocol which is no longer supported by github. Please change to ;protocol=https in the url." % ud.url) 151 152 if not ud.proto in ('git', 'file', 'ssh', 'http', 'https', 'rsync'): 153 raise bb.fetch2.ParameterError("Invalid protocol type", ud.url) 154 155 ud.nocheckout = ud.parm.get("nocheckout","0") == "1" 156 157 ud.rebaseable = ud.parm.get("rebaseable","0") == "1" 158 159 ud.nobranch = ud.parm.get("nobranch","0") == "1" 160 161 # usehead implies nobranch 162 ud.usehead = ud.parm.get("usehead","0") == "1" 163 if ud.usehead: 164 if ud.proto != "file": 165 raise bb.fetch2.ParameterError("The usehead option is only for use with local ('protocol=file') git repositories", ud.url) 166 ud.nobranch = 1 167 168 # bareclone implies nocheckout 169 ud.bareclone = ud.parm.get("bareclone","0") == "1" 170 if ud.bareclone: 171 ud.nocheckout = 1 172 173 ud.unresolvedrev = {} 174 branches = ud.parm.get("branch", "").split(',') 175 if branches == [""] and not ud.nobranch: 176 bb.warn("URL: %s does not set any branch parameter. The future default branch used by tools and repositories is uncertain and we will therefore soon require this is set in all git urls." % ud.url) 177 branches = ["master"] 178 if len(branches) != len(ud.names): 179 raise bb.fetch2.ParameterError("The number of name and branch parameters is not balanced", ud.url) 180 181 ud.noshared = d.getVar("BB_GIT_NOSHARED") == "1" 182 183 ud.cloneflags = "-n" 184 if not ud.noshared: 185 ud.cloneflags += " -s" 186 if ud.bareclone: 187 ud.cloneflags += " --mirror" 188 189 ud.shallow = d.getVar("BB_GIT_SHALLOW") == "1" 190 ud.shallow_extra_refs = (d.getVar("BB_GIT_SHALLOW_EXTRA_REFS") or "").split() 191 192 depth_default = d.getVar("BB_GIT_SHALLOW_DEPTH") 193 if depth_default is not None: 194 try: 195 depth_default = int(depth_default or 0) 196 except ValueError: 197 raise bb.fetch2.FetchError("Invalid depth for BB_GIT_SHALLOW_DEPTH: %s" % depth_default) 198 else: 199 if depth_default < 0: 200 raise bb.fetch2.FetchError("Invalid depth for BB_GIT_SHALLOW_DEPTH: %s" % depth_default) 201 else: 202 depth_default = 1 203 ud.shallow_depths = collections.defaultdict(lambda: depth_default) 204 205 revs_default = d.getVar("BB_GIT_SHALLOW_REVS") 206 ud.shallow_revs = [] 207 ud.branches = {} 208 for pos, name in enumerate(ud.names): 209 branch = branches[pos] 210 ud.branches[name] = branch 211 ud.unresolvedrev[name] = branch 212 213 shallow_depth = d.getVar("BB_GIT_SHALLOW_DEPTH_%s" % name) 214 if shallow_depth is not None: 215 try: 216 shallow_depth = int(shallow_depth or 0) 217 except ValueError: 218 raise bb.fetch2.FetchError("Invalid depth for BB_GIT_SHALLOW_DEPTH_%s: %s" % (name, shallow_depth)) 219 else: 220 if shallow_depth < 0: 221 raise bb.fetch2.FetchError("Invalid depth for BB_GIT_SHALLOW_DEPTH_%s: %s" % (name, shallow_depth)) 222 ud.shallow_depths[name] = shallow_depth 223 224 revs = d.getVar("BB_GIT_SHALLOW_REVS_%s" % name) 225 if revs is not None: 226 ud.shallow_revs.extend(revs.split()) 227 elif revs_default is not None: 228 ud.shallow_revs.extend(revs_default.split()) 229 230 if (ud.shallow and 231 not ud.shallow_revs and 232 all(ud.shallow_depths[n] == 0 for n in ud.names)): 233 # Shallow disabled for this URL 234 ud.shallow = False 235 236 if ud.usehead: 237 # When usehead is set let's associate 'HEAD' with the unresolved 238 # rev of this repository. This will get resolved into a revision 239 # later. If an actual revision happens to have also been provided 240 # then this setting will be overridden. 241 for name in ud.names: 242 ud.unresolvedrev[name] = 'HEAD' 243 244 ud.basecmd = d.getVar("FETCHCMD_git") or "git -c core.fsyncobjectfiles=0 -c gc.autoDetach=false -c core.pager=cat" 245 246 write_tarballs = d.getVar("BB_GENERATE_MIRROR_TARBALLS") or "0" 247 ud.write_tarballs = write_tarballs != "0" or ud.rebaseable 248 ud.write_shallow_tarballs = (d.getVar("BB_GENERATE_SHALLOW_TARBALLS") or write_tarballs) != "0" 249 250 ud.setup_revisions(d) 251 252 for name in ud.names: 253 # Ensure anything that doesn't look like a sha256 checksum/revision is translated into one 254 if not ud.revisions[name] or len(ud.revisions[name]) != 40 or (False in [c in "abcdef0123456789" for c in ud.revisions[name]]): 255 if ud.revisions[name]: 256 ud.unresolvedrev[name] = ud.revisions[name] 257 ud.revisions[name] = self.latest_revision(ud, d, name) 258 259 gitsrcname = '%s%s' % (ud.host.replace(':', '.'), ud.path.replace('/', '.').replace('*', '.').replace(' ','_')) 260 if gitsrcname.startswith('.'): 261 gitsrcname = gitsrcname[1:] 262 263 # for rebaseable git repo, it is necessary to keep mirror tar ball 264 # per revision, so that even the revision disappears from the 265 # upstream repo in the future, the mirror will remain intact and still 266 # contains the revision 267 if ud.rebaseable: 268 for name in ud.names: 269 gitsrcname = gitsrcname + '_' + ud.revisions[name] 270 271 dl_dir = d.getVar("DL_DIR") 272 gitdir = d.getVar("GITDIR") or (dl_dir + "/git2") 273 ud.clonedir = os.path.join(gitdir, gitsrcname) 274 ud.localfile = ud.clonedir 275 276 mirrortarball = 'git2_%s.tar.gz' % gitsrcname 277 ud.fullmirror = os.path.join(dl_dir, mirrortarball) 278 ud.mirrortarballs = [mirrortarball] 279 if ud.shallow: 280 tarballname = gitsrcname 281 if ud.bareclone: 282 tarballname = "%s_bare" % tarballname 283 284 if ud.shallow_revs: 285 tarballname = "%s_%s" % (tarballname, "_".join(sorted(ud.shallow_revs))) 286 287 for name, revision in sorted(ud.revisions.items()): 288 tarballname = "%s_%s" % (tarballname, ud.revisions[name][:7]) 289 depth = ud.shallow_depths[name] 290 if depth: 291 tarballname = "%s-%s" % (tarballname, depth) 292 293 shallow_refs = [] 294 if not ud.nobranch: 295 shallow_refs.extend(ud.branches.values()) 296 if ud.shallow_extra_refs: 297 shallow_refs.extend(r.replace('refs/heads/', '').replace('*', 'ALL') for r in ud.shallow_extra_refs) 298 if shallow_refs: 299 tarballname = "%s_%s" % (tarballname, "_".join(sorted(shallow_refs)).replace('/', '.')) 300 301 fetcher = self.__class__.__name__.lower() 302 ud.shallowtarball = '%sshallow_%s.tar.gz' % (fetcher, tarballname) 303 ud.fullshallow = os.path.join(dl_dir, ud.shallowtarball) 304 ud.mirrortarballs.insert(0, ud.shallowtarball) 305 306 def localpath(self, ud, d): 307 return ud.clonedir 308 309 def need_update(self, ud, d): 310 return self.clonedir_need_update(ud, d) or self.shallow_tarball_need_update(ud) or self.tarball_need_update(ud) 311 312 def clonedir_need_update(self, ud, d): 313 if not os.path.exists(ud.clonedir): 314 return True 315 if ud.shallow and ud.write_shallow_tarballs and self.clonedir_need_shallow_revs(ud, d): 316 return True 317 for name in ud.names: 318 if not self._contains_ref(ud, d, name, ud.clonedir): 319 return True 320 return False 321 322 def clonedir_need_shallow_revs(self, ud, d): 323 for rev in ud.shallow_revs: 324 try: 325 runfetchcmd('%s rev-parse -q --verify %s' % (ud.basecmd, rev), d, quiet=True, workdir=ud.clonedir) 326 except bb.fetch2.FetchError: 327 return rev 328 return None 329 330 def shallow_tarball_need_update(self, ud): 331 return ud.shallow and ud.write_shallow_tarballs and not os.path.exists(ud.fullshallow) 332 333 def tarball_need_update(self, ud): 334 return ud.write_tarballs and not os.path.exists(ud.fullmirror) 335 336 def try_premirror(self, ud, d): 337 # If we don't do this, updating an existing checkout with only premirrors 338 # is not possible 339 if bb.utils.to_boolean(d.getVar("BB_FETCH_PREMIRRORONLY")): 340 return True 341 if os.path.exists(ud.clonedir): 342 return False 343 return True 344 345 def download(self, ud, d): 346 """Fetch url""" 347 348 # A current clone is preferred to either tarball, a shallow tarball is 349 # preferred to an out of date clone, and a missing clone will use 350 # either tarball. 351 if ud.shallow and os.path.exists(ud.fullshallow) and self.need_update(ud, d): 352 ud.localpath = ud.fullshallow 353 return 354 elif os.path.exists(ud.fullmirror) and not os.path.exists(ud.clonedir): 355 bb.utils.mkdirhier(ud.clonedir) 356 runfetchcmd("tar -xzf %s" % ud.fullmirror, d, workdir=ud.clonedir) 357 358 repourl = self._get_repo_url(ud) 359 360 # If the repo still doesn't exist, fallback to cloning it 361 if not os.path.exists(ud.clonedir): 362 # We do this since git will use a "-l" option automatically for local urls where possible, 363 # but it doesn't work when git/objects is a symlink, only works when it is a directory. 364 if repourl.startswith("file://"): 365 repourl_path = repourl[7:] 366 objects = os.path.join(repourl_path, 'objects') 367 if os.path.isdir(objects) and not os.path.islink(objects): 368 repourl = repourl_path 369 clone_cmd = "LANG=C %s clone --bare --mirror %s %s --progress" % (ud.basecmd, shlex.quote(repourl), ud.clonedir) 370 if ud.proto.lower() != 'file': 371 bb.fetch2.check_network_access(d, clone_cmd, ud.url) 372 progresshandler = GitProgressHandler(d) 373 runfetchcmd(clone_cmd, d, log=progresshandler) 374 375 # Update the checkout if needed 376 if self.clonedir_need_update(ud, d): 377 output = runfetchcmd("%s remote" % ud.basecmd, d, quiet=True, workdir=ud.clonedir) 378 if "origin" in output: 379 runfetchcmd("%s remote rm origin" % ud.basecmd, d, workdir=ud.clonedir) 380 381 runfetchcmd("%s remote add --mirror=fetch origin %s" % (ud.basecmd, shlex.quote(repourl)), d, workdir=ud.clonedir) 382 383 if ud.nobranch: 384 fetch_cmd = "LANG=C %s fetch -f --progress %s refs/*:refs/*" % (ud.basecmd, shlex.quote(repourl)) 385 else: 386 fetch_cmd = "LANG=C %s fetch -f --progress %s refs/heads/*:refs/heads/* refs/tags/*:refs/tags/*" % (ud.basecmd, shlex.quote(repourl)) 387 if ud.proto.lower() != 'file': 388 bb.fetch2.check_network_access(d, fetch_cmd, ud.url) 389 progresshandler = GitProgressHandler(d) 390 runfetchcmd(fetch_cmd, d, log=progresshandler, workdir=ud.clonedir) 391 runfetchcmd("%s prune-packed" % ud.basecmd, d, workdir=ud.clonedir) 392 runfetchcmd("%s pack-refs --all" % ud.basecmd, d, workdir=ud.clonedir) 393 runfetchcmd("%s pack-redundant --all | xargs -r rm" % ud.basecmd, d, workdir=ud.clonedir) 394 try: 395 os.unlink(ud.fullmirror) 396 except OSError as exc: 397 if exc.errno != errno.ENOENT: 398 raise 399 400 for name in ud.names: 401 if not self._contains_ref(ud, d, name, ud.clonedir): 402 raise bb.fetch2.FetchError("Unable to find revision %s in branch %s even from upstream" % (ud.revisions[name], ud.branches[name])) 403 404 if ud.shallow and ud.write_shallow_tarballs: 405 missing_rev = self.clonedir_need_shallow_revs(ud, d) 406 if missing_rev: 407 raise bb.fetch2.FetchError("Unable to find revision %s even from upstream" % missing_rev) 408 409 if self._contains_lfs(ud, d, ud.clonedir) and self._need_lfs(ud): 410 # Unpack temporary working copy, use it to run 'git checkout' to force pre-fetching 411 # of all LFS blobs needed at the srcrev. 412 # 413 # It would be nice to just do this inline here by running 'git-lfs fetch' 414 # on the bare clonedir, but that operation requires a working copy on some 415 # releases of Git LFS. 416 tmpdir = tempfile.mkdtemp(dir=d.getVar('DL_DIR')) 417 try: 418 # Do the checkout. This implicitly involves a Git LFS fetch. 419 Git.unpack(self, ud, tmpdir, d) 420 421 # Scoop up a copy of any stuff that Git LFS downloaded. Merge them into 422 # the bare clonedir. 423 # 424 # As this procedure is invoked repeatedly on incremental fetches as 425 # a recipe's SRCREV is bumped throughout its lifetime, this will 426 # result in a gradual accumulation of LFS blobs in <ud.clonedir>/lfs 427 # corresponding to all the blobs reachable from the different revs 428 # fetched across time. 429 # 430 # Only do this if the unpack resulted in a .git/lfs directory being 431 # created; this only happens if at least one blob needed to be 432 # downloaded. 433 if os.path.exists(os.path.join(tmpdir, "git", ".git", "lfs")): 434 runfetchcmd("tar -cf - lfs | tar -xf - -C %s" % ud.clonedir, d, workdir="%s/git/.git" % tmpdir) 435 finally: 436 bb.utils.remove(tmpdir, recurse=True) 437 438 def build_mirror_data(self, ud, d): 439 440 # Create as a temp file and move atomically into position to avoid races 441 @contextmanager 442 def create_atomic(filename): 443 fd, tfile = tempfile.mkstemp(dir=os.path.dirname(filename)) 444 try: 445 yield tfile 446 umask = os.umask(0o666) 447 os.umask(umask) 448 os.chmod(tfile, (0o666 & ~umask)) 449 os.rename(tfile, filename) 450 finally: 451 os.close(fd) 452 453 if ud.shallow and ud.write_shallow_tarballs: 454 if not os.path.exists(ud.fullshallow): 455 if os.path.islink(ud.fullshallow): 456 os.unlink(ud.fullshallow) 457 tempdir = tempfile.mkdtemp(dir=d.getVar('DL_DIR')) 458 shallowclone = os.path.join(tempdir, 'git') 459 try: 460 self.clone_shallow_local(ud, shallowclone, d) 461 462 logger.info("Creating tarball of git repository") 463 with create_atomic(ud.fullshallow) as tfile: 464 runfetchcmd("tar -czf %s ." % tfile, d, workdir=shallowclone) 465 runfetchcmd("touch %s.done" % ud.fullshallow, d) 466 finally: 467 bb.utils.remove(tempdir, recurse=True) 468 elif ud.write_tarballs and not os.path.exists(ud.fullmirror): 469 if os.path.islink(ud.fullmirror): 470 os.unlink(ud.fullmirror) 471 472 logger.info("Creating tarball of git repository") 473 with create_atomic(ud.fullmirror) as tfile: 474 mtime = runfetchcmd("git log --all -1 --format=%cD", d, 475 quiet=True, workdir=ud.clonedir) 476 runfetchcmd("tar -czf %s --owner oe:0 --group oe:0 --mtime \"%s\" ." 477 % (tfile, mtime), d, workdir=ud.clonedir) 478 runfetchcmd("touch %s.done" % ud.fullmirror, d) 479 480 def clone_shallow_local(self, ud, dest, d): 481 """Clone the repo and make it shallow. 482 483 The upstream url of the new clone isn't set at this time, as it'll be 484 set correctly when unpacked.""" 485 runfetchcmd("%s clone %s %s %s" % (ud.basecmd, ud.cloneflags, ud.clonedir, dest), d) 486 487 to_parse, shallow_branches = [], [] 488 for name in ud.names: 489 revision = ud.revisions[name] 490 depth = ud.shallow_depths[name] 491 if depth: 492 to_parse.append('%s~%d^{}' % (revision, depth - 1)) 493 494 # For nobranch, we need a ref, otherwise the commits will be 495 # removed, and for non-nobranch, we truncate the branch to our 496 # srcrev, to avoid keeping unnecessary history beyond that. 497 branch = ud.branches[name] 498 if ud.nobranch: 499 ref = "refs/shallow/%s" % name 500 elif ud.bareclone: 501 ref = "refs/heads/%s" % branch 502 else: 503 ref = "refs/remotes/origin/%s" % branch 504 505 shallow_branches.append(ref) 506 runfetchcmd("%s update-ref %s %s" % (ud.basecmd, ref, revision), d, workdir=dest) 507 508 # Map srcrev+depths to revisions 509 parsed_depths = runfetchcmd("%s rev-parse %s" % (ud.basecmd, " ".join(to_parse)), d, workdir=dest) 510 511 # Resolve specified revisions 512 parsed_revs = runfetchcmd("%s rev-parse %s" % (ud.basecmd, " ".join('"%s^{}"' % r for r in ud.shallow_revs)), d, workdir=dest) 513 shallow_revisions = parsed_depths.splitlines() + parsed_revs.splitlines() 514 515 # Apply extra ref wildcards 516 all_refs = runfetchcmd('%s for-each-ref "--format=%%(refname)"' % ud.basecmd, 517 d, workdir=dest).splitlines() 518 for r in ud.shallow_extra_refs: 519 if not ud.bareclone: 520 r = r.replace('refs/heads/', 'refs/remotes/origin/') 521 522 if '*' in r: 523 matches = filter(lambda a: fnmatch.fnmatchcase(a, r), all_refs) 524 shallow_branches.extend(matches) 525 else: 526 shallow_branches.append(r) 527 528 # Make the repository shallow 529 shallow_cmd = [self.make_shallow_path, '-s'] 530 for b in shallow_branches: 531 shallow_cmd.append('-r') 532 shallow_cmd.append(b) 533 shallow_cmd.extend(shallow_revisions) 534 runfetchcmd(subprocess.list2cmdline(shallow_cmd), d, workdir=dest) 535 536 def unpack(self, ud, destdir, d): 537 """ unpack the downloaded src to destdir""" 538 539 subdir = ud.parm.get("subdir") 540 subpath = ud.parm.get("subpath") 541 readpathspec = "" 542 def_destsuffix = "git/" 543 544 if subpath: 545 readpathspec = ":%s" % subpath 546 def_destsuffix = "%s/" % os.path.basename(subpath.rstrip('/')) 547 548 if subdir: 549 # If 'subdir' param exists, create a dir and use it as destination for unpack cmd 550 if os.path.isabs(subdir): 551 if not os.path.realpath(subdir).startswith(os.path.realpath(destdir)): 552 raise bb.fetch2.UnpackError("subdir argument isn't a subdirectory of unpack root %s" % destdir, ud.url) 553 destdir = subdir 554 else: 555 destdir = os.path.join(destdir, subdir) 556 def_destsuffix = "" 557 558 destsuffix = ud.parm.get("destsuffix", def_destsuffix) 559 destdir = ud.destdir = os.path.join(destdir, destsuffix) 560 if os.path.exists(destdir): 561 bb.utils.prunedir(destdir) 562 563 need_lfs = self._need_lfs(ud) 564 565 if not need_lfs: 566 ud.basecmd = "GIT_LFS_SKIP_SMUDGE=1 " + ud.basecmd 567 568 source_found = False 569 source_error = [] 570 571 if not source_found: 572 clonedir_is_up_to_date = not self.clonedir_need_update(ud, d) 573 if clonedir_is_up_to_date: 574 runfetchcmd("%s clone %s %s/ %s" % (ud.basecmd, ud.cloneflags, ud.clonedir, destdir), d) 575 source_found = True 576 else: 577 source_error.append("clone directory not available or not up to date: " + ud.clonedir) 578 579 if not source_found: 580 if ud.shallow: 581 if os.path.exists(ud.fullshallow): 582 bb.utils.mkdirhier(destdir) 583 runfetchcmd("tar -xzf %s" % ud.fullshallow, d, workdir=destdir) 584 source_found = True 585 else: 586 source_error.append("shallow clone not available: " + ud.fullshallow) 587 else: 588 source_error.append("shallow clone not enabled") 589 590 if not source_found: 591 raise bb.fetch2.UnpackError("No up to date source found: " + "; ".join(source_error), ud.url) 592 593 repourl = self._get_repo_url(ud) 594 runfetchcmd("%s remote set-url origin %s" % (ud.basecmd, shlex.quote(repourl)), d, workdir=destdir) 595 596 if self._contains_lfs(ud, d, destdir): 597 if need_lfs and not self._find_git_lfs(d): 598 raise bb.fetch2.FetchError("Repository %s has LFS content, install git-lfs on host to download (or set lfs=0 to ignore it)" % (repourl)) 599 elif not need_lfs: 600 bb.note("Repository %s has LFS content but it is not being fetched" % (repourl)) 601 602 if not ud.nocheckout: 603 if subpath: 604 runfetchcmd("%s read-tree %s%s" % (ud.basecmd, ud.revisions[ud.names[0]], readpathspec), d, 605 workdir=destdir) 606 runfetchcmd("%s checkout-index -q -f -a" % ud.basecmd, d, workdir=destdir) 607 elif not ud.nobranch: 608 branchname = ud.branches[ud.names[0]] 609 runfetchcmd("%s checkout -B %s %s" % (ud.basecmd, branchname, \ 610 ud.revisions[ud.names[0]]), d, workdir=destdir) 611 runfetchcmd("%s branch %s --set-upstream-to origin/%s" % (ud.basecmd, branchname, \ 612 branchname), d, workdir=destdir) 613 else: 614 runfetchcmd("%s checkout %s" % (ud.basecmd, ud.revisions[ud.names[0]]), d, workdir=destdir) 615 616 return True 617 618 def clean(self, ud, d): 619 """ clean the git directory """ 620 621 to_remove = [ud.localpath, ud.fullmirror, ud.fullmirror + ".done"] 622 # The localpath is a symlink to clonedir when it is cloned from a 623 # mirror, so remove both of them. 624 if os.path.islink(ud.localpath): 625 clonedir = os.path.realpath(ud.localpath) 626 to_remove.append(clonedir) 627 628 for r in to_remove: 629 if os.path.exists(r): 630 bb.note('Removing %s' % r) 631 bb.utils.remove(r, True) 632 633 def supports_srcrev(self): 634 return True 635 636 def _contains_ref(self, ud, d, name, wd): 637 cmd = "" 638 if ud.nobranch: 639 cmd = "%s log --pretty=oneline -n 1 %s -- 2> /dev/null | wc -l" % ( 640 ud.basecmd, ud.revisions[name]) 641 else: 642 cmd = "%s branch --contains %s --list %s 2> /dev/null | wc -l" % ( 643 ud.basecmd, ud.revisions[name], ud.branches[name]) 644 try: 645 output = runfetchcmd(cmd, d, quiet=True, workdir=wd) 646 except bb.fetch2.FetchError: 647 return False 648 if len(output.split()) > 1: 649 raise bb.fetch2.FetchError("The command '%s' gave output with more then 1 line unexpectedly, output: '%s'" % (cmd, output)) 650 return output.split()[0] != "0" 651 652 def _need_lfs(self, ud): 653 return ud.parm.get("lfs", "1") == "1" 654 655 def _contains_lfs(self, ud, d, wd): 656 """ 657 Check if the repository has 'lfs' (large file) content 658 """ 659 660 if not ud.nobranch: 661 branchname = ud.branches[ud.names[0]] 662 else: 663 branchname = "master" 664 665 # The bare clonedir doesn't use the remote names; it has the branch immediately. 666 if wd == ud.clonedir: 667 refname = ud.branches[ud.names[0]] 668 else: 669 refname = "origin/%s" % ud.branches[ud.names[0]] 670 671 cmd = "%s grep lfs %s:.gitattributes | wc -l" % ( 672 ud.basecmd, refname) 673 674 try: 675 output = runfetchcmd(cmd, d, quiet=True, workdir=wd) 676 if int(output) > 0: 677 return True 678 except (bb.fetch2.FetchError,ValueError): 679 pass 680 return False 681 682 def _find_git_lfs(self, d): 683 """ 684 Return True if git-lfs can be found, False otherwise. 685 """ 686 import shutil 687 return shutil.which("git-lfs", path=d.getVar('PATH')) is not None 688 689 def _get_repo_url(self, ud): 690 """ 691 Return the repository URL 692 """ 693 # Note that we do not support passwords directly in the git urls. There are several 694 # reasons. SRC_URI can be written out to things like buildhistory and people don't 695 # want to leak passwords like that. Its also all too easy to share metadata without 696 # removing the password. ssh keys, ~/.netrc and ~/.ssh/config files can be used as 697 # alternatives so we will not take patches adding password support here. 698 if ud.user: 699 username = ud.user + '@' 700 else: 701 username = "" 702 return "%s://%s%s%s" % (ud.proto, username, ud.host, ud.path) 703 704 def _revision_key(self, ud, d, name): 705 """ 706 Return a unique key for the url 707 """ 708 # Collapse adjacent slashes 709 slash_re = re.compile(r"/+") 710 return "git:" + ud.host + slash_re.sub(".", ud.path) + ud.unresolvedrev[name] 711 712 def _lsremote(self, ud, d, search): 713 """ 714 Run git ls-remote with the specified search string 715 """ 716 # Prevent recursion e.g. in OE if SRCPV is in PV, PV is in WORKDIR, 717 # and WORKDIR is in PATH (as a result of RSS), our call to 718 # runfetchcmd() exports PATH so this function will get called again (!) 719 # In this scenario the return call of the function isn't actually 720 # important - WORKDIR isn't needed in PATH to call git ls-remote 721 # anyway. 722 if d.getVar('_BB_GIT_IN_LSREMOTE', False): 723 return '' 724 d.setVar('_BB_GIT_IN_LSREMOTE', '1') 725 try: 726 repourl = self._get_repo_url(ud) 727 cmd = "%s ls-remote %s %s" % \ 728 (ud.basecmd, shlex.quote(repourl), search) 729 if ud.proto.lower() != 'file': 730 bb.fetch2.check_network_access(d, cmd, repourl) 731 output = runfetchcmd(cmd, d, True) 732 if not output: 733 raise bb.fetch2.FetchError("The command %s gave empty output unexpectedly" % cmd, ud.url) 734 finally: 735 d.delVar('_BB_GIT_IN_LSREMOTE') 736 return output 737 738 def _latest_revision(self, ud, d, name): 739 """ 740 Compute the HEAD revision for the url 741 """ 742 if not d.getVar("__BBSEENSRCREV"): 743 raise bb.fetch2.FetchError("Recipe uses a floating tag/branch '%s' for repo '%s' without a fixed SRCREV yet doesn't call bb.fetch2.get_srcrev() (use SRCPV in PV for OE)." % (ud.unresolvedrev[name], ud.host+ud.path)) 744 745 # Ensure we mark as not cached 746 bb.fetch2.get_autorev(d) 747 748 output = self._lsremote(ud, d, "") 749 # Tags of the form ^{} may not work, need to fallback to other form 750 if ud.unresolvedrev[name][:5] == "refs/" or ud.usehead: 751 head = ud.unresolvedrev[name] 752 tag = ud.unresolvedrev[name] 753 else: 754 head = "refs/heads/%s" % ud.unresolvedrev[name] 755 tag = "refs/tags/%s" % ud.unresolvedrev[name] 756 for s in [head, tag + "^{}", tag]: 757 for l in output.strip().split('\n'): 758 sha1, ref = l.split() 759 if s == ref: 760 return sha1 761 raise bb.fetch2.FetchError("Unable to resolve '%s' in upstream git repository in git ls-remote output for %s" % \ 762 (ud.unresolvedrev[name], ud.host+ud.path)) 763 764 def latest_versionstring(self, ud, d): 765 """ 766 Compute the latest release name like "x.y.x" in "x.y.x+gitHASH" 767 by searching through the tags output of ls-remote, comparing 768 versions and returning the highest match. 769 """ 770 pupver = ('', '') 771 772 tagregex = re.compile(d.getVar('UPSTREAM_CHECK_GITTAGREGEX') or r"(?P<pver>([0-9][\.|_]?)+)") 773 try: 774 output = self._lsremote(ud, d, "refs/tags/*") 775 except (bb.fetch2.FetchError, bb.fetch2.NetworkAccess) as e: 776 bb.note("Could not list remote: %s" % str(e)) 777 return pupver 778 779 verstring = "" 780 revision = "" 781 for line in output.split("\n"): 782 if not line: 783 break 784 785 tag_head = line.split("/")[-1] 786 # Ignore non-released branches 787 m = re.search(r"(alpha|beta|rc|final)+", tag_head) 788 if m: 789 continue 790 791 # search for version in the line 792 tag = tagregex.search(tag_head) 793 if tag is None: 794 continue 795 796 tag = tag.group('pver') 797 tag = tag.replace("_", ".") 798 799 if verstring and bb.utils.vercmp(("0", tag, ""), ("0", verstring, "")) < 0: 800 continue 801 802 verstring = tag 803 revision = line.split()[0] 804 pupver = (verstring, revision) 805 806 return pupver 807 808 def _build_revision(self, ud, d, name): 809 return ud.revisions[name] 810 811 def gitpkgv_revision(self, ud, d, name): 812 """ 813 Return a sortable revision number by counting commits in the history 814 Based on gitpkgv.bblass in meta-openembedded 815 """ 816 rev = self._build_revision(ud, d, name) 817 localpath = ud.localpath 818 rev_file = os.path.join(localpath, "oe-gitpkgv_" + rev) 819 if not os.path.exists(localpath): 820 commits = None 821 else: 822 if not os.path.exists(rev_file) or not os.path.getsize(rev_file): 823 from pipes import quote 824 commits = bb.fetch2.runfetchcmd( 825 "git rev-list %s -- | wc -l" % quote(rev), 826 d, quiet=True).strip().lstrip('0') 827 if commits: 828 open(rev_file, "w").write("%d\n" % int(commits)) 829 else: 830 commits = open(rev_file, "r").readline(128).strip() 831 if commits: 832 return False, "%s+%s" % (commits, rev[:7]) 833 else: 834 return True, str(rev) 835 836 def checkstatus(self, fetch, ud, d): 837 try: 838 self._lsremote(ud, d, "") 839 return True 840 except bb.fetch2.FetchError: 841 return False 842