1""" 2BitBake 'Fetch' implementations 3 4Classes for obtaining upstream sources for the 5BitBake build tools. 6""" 7 8# Copyright (C) 2003, 2004 Chris Larson 9# Copyright (C) 2012 Intel Corporation 10# 11# SPDX-License-Identifier: GPL-2.0-only 12# 13# Based on functions from the base bb module, Copyright 2003 Holger Schurig 14 15import os, re 16import signal 17import logging 18import urllib.request, urllib.parse, urllib.error 19if 'git' not in urllib.parse.uses_netloc: 20 urllib.parse.uses_netloc.append('git') 21import operator 22import collections 23import subprocess 24import pickle 25import errno 26import bb.persist_data, bb.utils 27import bb.checksum 28import bb.process 29import bb.event 30 31__version__ = "2" 32_checksum_cache = bb.checksum.FileChecksumCache() 33 34logger = logging.getLogger("BitBake.Fetcher") 35 36CHECKSUM_LIST = [ "md5", "sha256", "sha1", "sha384", "sha512" ] 37SHOWN_CHECKSUM_LIST = ["sha256"] 38 39class BBFetchException(Exception): 40 """Class all fetch exceptions inherit from""" 41 def __init__(self, message): 42 self.msg = message 43 Exception.__init__(self, message) 44 45 def __str__(self): 46 return self.msg 47 48class UntrustedUrl(BBFetchException): 49 """Exception raised when encountering a host not listed in BB_ALLOWED_NETWORKS""" 50 def __init__(self, url, message=''): 51 if message: 52 msg = message 53 else: 54 msg = "The URL: '%s' is not trusted and cannot be used" % url 55 self.url = url 56 BBFetchException.__init__(self, msg) 57 self.args = (url,) 58 59class MalformedUrl(BBFetchException): 60 """Exception raised when encountering an invalid url""" 61 def __init__(self, url, message=''): 62 if message: 63 msg = message 64 else: 65 msg = "The URL: '%s' is invalid and cannot be interpreted" % url 66 self.url = url 67 BBFetchException.__init__(self, msg) 68 self.args = (url,) 69 70class FetchError(BBFetchException): 71 """General fetcher exception when something happens incorrectly""" 72 def __init__(self, message, url = None): 73 if url: 74 msg = "Fetcher failure for URL: '%s'. %s" % (url, message) 75 else: 76 msg = "Fetcher failure: %s" % message 77 self.url = url 78 BBFetchException.__init__(self, msg) 79 self.args = (message, url) 80 81class ChecksumError(FetchError): 82 """Exception when mismatched checksum encountered""" 83 def __init__(self, message, url = None, checksum = None): 84 self.checksum = checksum 85 FetchError.__init__(self, message, url) 86 87class NoChecksumError(FetchError): 88 """Exception when no checksum is specified, but BB_STRICT_CHECKSUM is set""" 89 90class UnpackError(BBFetchException): 91 """General fetcher exception when something happens incorrectly when unpacking""" 92 def __init__(self, message, url): 93 msg = "Unpack failure for URL: '%s'. %s" % (url, message) 94 self.url = url 95 BBFetchException.__init__(self, msg) 96 self.args = (message, url) 97 98class NoMethodError(BBFetchException): 99 """Exception raised when there is no method to obtain a supplied url or set of urls""" 100 def __init__(self, url): 101 msg = "Could not find a fetcher which supports the URL: '%s'" % url 102 self.url = url 103 BBFetchException.__init__(self, msg) 104 self.args = (url,) 105 106class MissingParameterError(BBFetchException): 107 """Exception raised when a fetch method is missing a critical parameter in the url""" 108 def __init__(self, missing, url): 109 msg = "URL: '%s' is missing the required parameter '%s'" % (url, missing) 110 self.url = url 111 self.missing = missing 112 BBFetchException.__init__(self, msg) 113 self.args = (missing, url) 114 115class ParameterError(BBFetchException): 116 """Exception raised when a url cannot be processed due to invalid parameters.""" 117 def __init__(self, message, url): 118 msg = "URL: '%s' has invalid parameters. %s" % (url, message) 119 self.url = url 120 BBFetchException.__init__(self, msg) 121 self.args = (message, url) 122 123class NetworkAccess(BBFetchException): 124 """Exception raised when network access is disabled but it is required.""" 125 def __init__(self, url, cmd): 126 msg = "Network access disabled through BB_NO_NETWORK (or set indirectly due to use of BB_FETCH_PREMIRRORONLY) but access requested with command %s (for url %s)" % (cmd, url) 127 self.url = url 128 self.cmd = cmd 129 BBFetchException.__init__(self, msg) 130 self.args = (url, cmd) 131 132class NonLocalMethod(Exception): 133 def __init__(self): 134 Exception.__init__(self) 135 136class MissingChecksumEvent(bb.event.Event): 137 def __init__(self, url, **checksums): 138 self.url = url 139 self.checksums = checksums 140 bb.event.Event.__init__(self) 141 142 143class URI(object): 144 """ 145 A class representing a generic URI, with methods for 146 accessing the URI components, and stringifies to the 147 URI. 148 149 It is constructed by calling it with a URI, or setting 150 the attributes manually: 151 152 uri = URI("http://example.com/") 153 154 uri = URI() 155 uri.scheme = 'http' 156 uri.hostname = 'example.com' 157 uri.path = '/' 158 159 It has the following attributes: 160 161 * scheme (read/write) 162 * userinfo (authentication information) (read/write) 163 * username (read/write) 164 * password (read/write) 165 166 Note, password is deprecated as of RFC 3986. 167 168 * hostname (read/write) 169 * port (read/write) 170 * hostport (read only) 171 "hostname:port", if both are set, otherwise just "hostname" 172 * path (read/write) 173 * path_quoted (read/write) 174 A URI quoted version of path 175 * params (dict) (read/write) 176 * query (dict) (read/write) 177 * relative (bool) (read only) 178 True if this is a "relative URI", (e.g. file:foo.diff) 179 180 It stringifies to the URI itself. 181 182 Some notes about relative URIs: while it's specified that 183 a URI beginning with <scheme>:// should either be directly 184 followed by a hostname or a /, the old URI handling of the 185 fetch2 library did not conform to this. Therefore, this URI 186 class has some kludges to make sure that URIs are parsed in 187 a way comforming to bitbake's current usage. This URI class 188 supports the following: 189 190 file:relative/path.diff (IETF compliant) 191 git:relative/path.git (IETF compliant) 192 git:///absolute/path.git (IETF compliant) 193 file:///absolute/path.diff (IETF compliant) 194 195 file://relative/path.diff (not IETF compliant) 196 197 But it does not support the following: 198 199 file://hostname/absolute/path.diff (would be IETF compliant) 200 201 Note that the last case only applies to a list of 202 explicitly allowed schemes (currently only file://), that requires 203 its URIs to not have a network location. 204 """ 205 206 _relative_schemes = ['file', 'git'] 207 _netloc_forbidden = ['file'] 208 209 def __init__(self, uri=None): 210 self.scheme = '' 211 self.userinfo = '' 212 self.hostname = '' 213 self.port = None 214 self._path = '' 215 self.params = {} 216 self.query = {} 217 self.relative = False 218 219 if not uri: 220 return 221 222 # We hijack the URL parameters, since the way bitbake uses 223 # them are not quite RFC compliant. 224 uri, param_str = (uri.split(";", 1) + [None])[:2] 225 226 urlp = urllib.parse.urlparse(uri) 227 self.scheme = urlp.scheme 228 229 reparse = 0 230 231 # Coerce urlparse to make URI scheme use netloc 232 if not self.scheme in urllib.parse.uses_netloc: 233 urllib.parse.uses_params.append(self.scheme) 234 reparse = 1 235 236 # Make urlparse happy(/ier) by converting local resources 237 # to RFC compliant URL format. E.g.: 238 # file://foo.diff -> file:foo.diff 239 if urlp.scheme in self._netloc_forbidden: 240 uri = re.sub("(?<=:)//(?!/)", "", uri, 1) 241 reparse = 1 242 243 if reparse: 244 urlp = urllib.parse.urlparse(uri) 245 246 # Identify if the URI is relative or not 247 if urlp.scheme in self._relative_schemes and \ 248 re.compile(r"^\w+:(?!//)").match(uri): 249 self.relative = True 250 251 if not self.relative: 252 self.hostname = urlp.hostname or '' 253 self.port = urlp.port 254 255 self.userinfo += urlp.username or '' 256 257 if urlp.password: 258 self.userinfo += ':%s' % urlp.password 259 260 self.path = urllib.parse.unquote(urlp.path) 261 262 if param_str: 263 self.params = self._param_str_split(param_str, ";") 264 if urlp.query: 265 self.query = self._param_str_split(urlp.query, "&") 266 267 def __str__(self): 268 userinfo = self.userinfo 269 if userinfo: 270 userinfo += '@' 271 272 return "%s:%s%s%s%s%s%s" % ( 273 self.scheme, 274 '' if self.relative else '//', 275 userinfo, 276 self.hostport, 277 self.path_quoted, 278 self._query_str(), 279 self._param_str()) 280 281 def _param_str(self): 282 return ( 283 ''.join([';', self._param_str_join(self.params, ";")]) 284 if self.params else '') 285 286 def _query_str(self): 287 return ( 288 ''.join(['?', self._param_str_join(self.query, "&")]) 289 if self.query else '') 290 291 def _param_str_split(self, string, elmdelim, kvdelim="="): 292 ret = collections.OrderedDict() 293 for k, v in [x.split(kvdelim, 1) for x in string.split(elmdelim) if x]: 294 ret[k] = v 295 return ret 296 297 def _param_str_join(self, dict_, elmdelim, kvdelim="="): 298 return elmdelim.join([kvdelim.join([k, v]) for k, v in dict_.items()]) 299 300 @property 301 def hostport(self): 302 if not self.port: 303 return self.hostname 304 return "%s:%d" % (self.hostname, self.port) 305 306 @property 307 def path_quoted(self): 308 return urllib.parse.quote(self.path) 309 310 @path_quoted.setter 311 def path_quoted(self, path): 312 self.path = urllib.parse.unquote(path) 313 314 @property 315 def path(self): 316 return self._path 317 318 @path.setter 319 def path(self, path): 320 self._path = path 321 322 if not path or re.compile("^/").match(path): 323 self.relative = False 324 else: 325 self.relative = True 326 327 @property 328 def username(self): 329 if self.userinfo: 330 return (self.userinfo.split(":", 1))[0] 331 return '' 332 333 @username.setter 334 def username(self, username): 335 password = self.password 336 self.userinfo = username 337 if password: 338 self.userinfo += ":%s" % password 339 340 @property 341 def password(self): 342 if self.userinfo and ":" in self.userinfo: 343 return (self.userinfo.split(":", 1))[1] 344 return '' 345 346 @password.setter 347 def password(self, password): 348 self.userinfo = "%s:%s" % (self.username, password) 349 350def decodeurl(url): 351 """Decodes an URL into the tokens (scheme, network location, path, 352 user, password, parameters). 353 """ 354 355 m = re.compile('(?P<type>[^:]*)://((?P<user>[^/;]+)@)?(?P<location>[^;]+)(;(?P<parm>.*))?').match(url) 356 if not m: 357 raise MalformedUrl(url) 358 359 type = m.group('type') 360 location = m.group('location') 361 if not location: 362 raise MalformedUrl(url) 363 user = m.group('user') 364 parm = m.group('parm') 365 366 locidx = location.find('/') 367 if locidx != -1 and type.lower() != 'file': 368 host = location[:locidx] 369 path = location[locidx:] 370 elif type.lower() == 'file': 371 host = "" 372 path = location 373 else: 374 host = location 375 path = "/" 376 if user: 377 m = re.compile('(?P<user>[^:]+)(:?(?P<pswd>.*))').match(user) 378 if m: 379 user = m.group('user') 380 pswd = m.group('pswd') 381 else: 382 user = '' 383 pswd = '' 384 385 p = collections.OrderedDict() 386 if parm: 387 for s in parm.split(';'): 388 if s: 389 if not '=' in s: 390 raise MalformedUrl(url, "The URL: '%s' is invalid: parameter %s does not specify a value (missing '=')" % (url, s)) 391 s1, s2 = s.split('=') 392 p[s1] = s2 393 394 return type, host, urllib.parse.unquote(path), user, pswd, p 395 396def encodeurl(decoded): 397 """Encodes a URL from tokens (scheme, network location, path, 398 user, password, parameters). 399 """ 400 401 type, host, path, user, pswd, p = decoded 402 403 if not type: 404 raise MissingParameterError('type', "encoded from the data %s" % str(decoded)) 405 url = ['%s://' % type] 406 if user and type != "file": 407 url.append("%s" % user) 408 if pswd: 409 url.append(":%s" % pswd) 410 url.append("@") 411 if host and type != "file": 412 url.append("%s" % host) 413 if path: 414 # Standardise path to ensure comparisons work 415 while '//' in path: 416 path = path.replace("//", "/") 417 url.append("%s" % urllib.parse.quote(path)) 418 if p: 419 for parm in p: 420 url.append(";%s=%s" % (parm, p[parm])) 421 422 return "".join(url) 423 424def uri_replace(ud, uri_find, uri_replace, replacements, d, mirrortarball=None): 425 if not ud.url or not uri_find or not uri_replace: 426 logger.error("uri_replace: passed an undefined value, not replacing") 427 return None 428 uri_decoded = list(decodeurl(ud.url)) 429 uri_find_decoded = list(decodeurl(uri_find)) 430 uri_replace_decoded = list(decodeurl(uri_replace)) 431 logger.debug2("For url %s comparing %s to %s" % (uri_decoded, uri_find_decoded, uri_replace_decoded)) 432 result_decoded = ['', '', '', '', '', {}] 433 # 0 - type, 1 - host, 2 - path, 3 - user, 4- pswd, 5 - params 434 for loc, i in enumerate(uri_find_decoded): 435 result_decoded[loc] = uri_decoded[loc] 436 regexp = i 437 if loc == 0 and regexp and not regexp.endswith("$"): 438 # Leaving the type unanchored can mean "https" matching "file" can become "files" 439 # which is clearly undesirable. 440 regexp += "$" 441 if loc == 5: 442 # Handle URL parameters 443 if i: 444 # Any specified URL parameters must match 445 for k in uri_find_decoded[loc]: 446 if uri_decoded[loc][k] != uri_find_decoded[loc][k]: 447 return None 448 # Overwrite any specified replacement parameters 449 for k in uri_replace_decoded[loc]: 450 for l in replacements: 451 uri_replace_decoded[loc][k] = uri_replace_decoded[loc][k].replace(l, replacements[l]) 452 result_decoded[loc][k] = uri_replace_decoded[loc][k] 453 elif (loc == 3 or loc == 4) and uri_replace_decoded[loc]: 454 # User/password in the replacement is just a straight replacement 455 result_decoded[loc] = uri_replace_decoded[loc] 456 elif (re.match(regexp, uri_decoded[loc])): 457 if not uri_replace_decoded[loc]: 458 result_decoded[loc] = "" 459 else: 460 for k in replacements: 461 uri_replace_decoded[loc] = uri_replace_decoded[loc].replace(k, replacements[k]) 462 #bb.note("%s %s %s" % (regexp, uri_replace_decoded[loc], uri_decoded[loc])) 463 result_decoded[loc] = re.sub(regexp, uri_replace_decoded[loc], uri_decoded[loc], 1) 464 if loc == 2: 465 # Handle path manipulations 466 basename = None 467 if uri_decoded[0] != uri_replace_decoded[0] and mirrortarball: 468 # If the source and destination url types differ, must be a mirrortarball mapping 469 basename = os.path.basename(mirrortarball) 470 # Kill parameters, they make no sense for mirror tarballs 471 uri_decoded[5] = {} 472 elif ud.localpath and ud.method.supports_checksum(ud): 473 basename = os.path.basename(ud.localpath) 474 if basename: 475 uri_basename = os.path.basename(uri_decoded[loc]) 476 # Prefix with a slash as a sentinel in case 477 # result_decoded[loc] does not contain one. 478 path = "/" + result_decoded[loc] 479 if uri_basename and basename != uri_basename and path.endswith("/" + uri_basename): 480 result_decoded[loc] = path[1:-len(uri_basename)] + basename 481 elif not path.endswith("/" + basename): 482 result_decoded[loc] = os.path.join(path[1:], basename) 483 else: 484 return None 485 result = encodeurl(result_decoded) 486 if result == ud.url: 487 return None 488 logger.debug2("For url %s returning %s" % (ud.url, result)) 489 return result 490 491methods = [] 492urldata_cache = {} 493saved_headrevs = {} 494 495def fetcher_init(d): 496 """ 497 Called to initialize the fetchers once the configuration data is known. 498 Calls before this must not hit the cache. 499 """ 500 501 revs = bb.persist_data.persist('BB_URI_HEADREVS', d) 502 try: 503 # fetcher_init is called multiple times, so make sure we only save the 504 # revs the first time it is called. 505 if not bb.fetch2.saved_headrevs: 506 bb.fetch2.saved_headrevs = dict(revs) 507 except: 508 pass 509 510 # When to drop SCM head revisions controlled by user policy 511 srcrev_policy = d.getVar('BB_SRCREV_POLICY') or "clear" 512 if srcrev_policy == "cache": 513 logger.debug("Keeping SRCREV cache due to cache policy of: %s", srcrev_policy) 514 elif srcrev_policy == "clear": 515 logger.debug("Clearing SRCREV cache due to cache policy of: %s", srcrev_policy) 516 revs.clear() 517 else: 518 raise FetchError("Invalid SRCREV cache policy of: %s" % srcrev_policy) 519 520 _checksum_cache.init_cache(d) 521 522 for m in methods: 523 if hasattr(m, "init"): 524 m.init(d) 525 526def fetcher_parse_save(): 527 _checksum_cache.save_extras() 528 529def fetcher_parse_done(): 530 _checksum_cache.save_merge() 531 532def fetcher_compare_revisions(d): 533 """ 534 Compare the revisions in the persistent cache with the saved values from 535 when bitbake was started and return true if they have changed. 536 """ 537 538 headrevs = dict(bb.persist_data.persist('BB_URI_HEADREVS', d)) 539 return headrevs != bb.fetch2.saved_headrevs 540 541def mirror_from_string(data): 542 mirrors = (data or "").replace('\\n',' ').split() 543 # Split into pairs 544 if len(mirrors) % 2 != 0: 545 bb.warn('Invalid mirror data %s, should have paired members.' % data) 546 return list(zip(*[iter(mirrors)]*2)) 547 548def verify_checksum(ud, d, precomputed={}): 549 """ 550 verify the MD5 and SHA256 checksum for downloaded src 551 552 Raises a FetchError if one or both of the SRC_URI checksums do not match 553 the downloaded file, or if BB_STRICT_CHECKSUM is set and there are no 554 checksums specified. 555 556 Returns a dict of checksums that can be stored in a done stamp file and 557 passed in as precomputed parameter in a later call to avoid re-computing 558 the checksums from the file. This allows verifying the checksums of the 559 file against those in the recipe each time, rather than only after 560 downloading. See https://bugzilla.yoctoproject.org/show_bug.cgi?id=5571. 561 """ 562 563 if ud.ignore_checksums or not ud.method.supports_checksum(ud): 564 return {} 565 566 def compute_checksum_info(checksum_id): 567 checksum_name = getattr(ud, "%s_name" % checksum_id) 568 569 if checksum_id in precomputed: 570 checksum_data = precomputed[checksum_id] 571 else: 572 checksum_data = getattr(bb.utils, "%s_file" % checksum_id)(ud.localpath) 573 574 checksum_expected = getattr(ud, "%s_expected" % checksum_id) 575 576 if checksum_expected == '': 577 checksum_expected = None 578 579 return { 580 "id": checksum_id, 581 "name": checksum_name, 582 "data": checksum_data, 583 "expected": checksum_expected 584 } 585 586 checksum_infos = [] 587 for checksum_id in CHECKSUM_LIST: 588 checksum_infos.append(compute_checksum_info(checksum_id)) 589 590 checksum_dict = {ci["id"] : ci["data"] for ci in checksum_infos} 591 checksum_event = {"%ssum" % ci["id"] : ci["data"] for ci in checksum_infos} 592 593 for ci in checksum_infos: 594 if ci["id"] in SHOWN_CHECKSUM_LIST: 595 checksum_lines = ["SRC_URI[%s] = \"%s\"" % (ci["name"], ci["data"])] 596 597 # If no checksum has been provided 598 if ud.method.recommends_checksum(ud) and all(ci["expected"] is None for ci in checksum_infos): 599 messages = [] 600 strict = d.getVar("BB_STRICT_CHECKSUM") or "0" 601 602 # If strict checking enabled and neither sum defined, raise error 603 if strict == "1": 604 messages.append("No checksum specified for '%s', please add at " \ 605 "least one to the recipe:" % ud.localpath) 606 messages.extend(checksum_lines) 607 logger.error("\n".join(messages)) 608 raise NoChecksumError("Missing SRC_URI checksum", ud.url) 609 610 bb.event.fire(MissingChecksumEvent(ud.url, **checksum_event), d) 611 612 if strict == "ignore": 613 return checksum_dict 614 615 # Log missing sums so user can more easily add them 616 messages.append("Missing checksum for '%s', consider adding at " \ 617 "least one to the recipe:" % ud.localpath) 618 messages.extend(checksum_lines) 619 logger.warning("\n".join(messages)) 620 621 # We want to alert the user if a checksum is defined in the recipe but 622 # it does not match. 623 messages = [] 624 messages.append("Checksum mismatch!") 625 bad_checksum = None 626 627 for ci in checksum_infos: 628 if ci["expected"] and ci["expected"] != ci["data"]: 629 messages.append("File: '%s' has %s checksum '%s' when '%s' was " \ 630 "expected" % (ud.localpath, ci["id"], ci["data"], ci["expected"])) 631 bad_checksum = ci["data"] 632 633 if bad_checksum: 634 messages.append("If this change is expected (e.g. you have upgraded " \ 635 "to a new version without updating the checksums) " \ 636 "then you can use these lines within the recipe:") 637 messages.extend(checksum_lines) 638 messages.append("Otherwise you should retry the download and/or " \ 639 "check with upstream to determine if the file has " \ 640 "become corrupted or otherwise unexpectedly modified.") 641 raise ChecksumError("\n".join(messages), ud.url, bad_checksum) 642 643 return checksum_dict 644 645def verify_donestamp(ud, d, origud=None): 646 """ 647 Check whether the done stamp file has the right checksums (if the fetch 648 method supports them). If it doesn't, delete the done stamp and force 649 a re-download. 650 651 Returns True, if the donestamp exists and is valid, False otherwise. When 652 returning False, any existing done stamps are removed. 653 """ 654 if not ud.needdonestamp or (origud and not origud.needdonestamp): 655 return True 656 657 if not os.path.exists(ud.localpath): 658 # local path does not exist 659 if os.path.exists(ud.donestamp): 660 # done stamp exists, but the downloaded file does not; the done stamp 661 # must be incorrect, re-trigger the download 662 bb.utils.remove(ud.donestamp) 663 return False 664 665 if (not ud.method.supports_checksum(ud) or 666 (origud and not origud.method.supports_checksum(origud))): 667 # if done stamp exists and checksums not supported; assume the local 668 # file is current 669 return os.path.exists(ud.donestamp) 670 671 precomputed_checksums = {} 672 # Only re-use the precomputed checksums if the donestamp is newer than the 673 # file. Do not rely on the mtime of directories, though. If ud.localpath is 674 # a directory, there will probably not be any checksums anyway. 675 if os.path.exists(ud.donestamp) and (os.path.isdir(ud.localpath) or 676 os.path.getmtime(ud.localpath) < os.path.getmtime(ud.donestamp)): 677 try: 678 with open(ud.donestamp, "rb") as cachefile: 679 pickled = pickle.Unpickler(cachefile) 680 precomputed_checksums.update(pickled.load()) 681 except Exception as e: 682 # Avoid the warnings on the upgrade path from emtpy done stamp 683 # files to those containing the checksums. 684 if not isinstance(e, EOFError): 685 # Ignore errors, they aren't fatal 686 logger.warning("Couldn't load checksums from donestamp %s: %s " 687 "(msg: %s)" % (ud.donestamp, type(e).__name__, 688 str(e))) 689 690 try: 691 checksums = verify_checksum(ud, d, precomputed_checksums) 692 # If the cache file did not have the checksums, compute and store them 693 # as an upgrade path from the previous done stamp file format. 694 if checksums != precomputed_checksums: 695 with open(ud.donestamp, "wb") as cachefile: 696 p = pickle.Pickler(cachefile, 2) 697 p.dump(checksums) 698 return True 699 except ChecksumError as e: 700 # Checksums failed to verify, trigger re-download and remove the 701 # incorrect stamp file. 702 logger.warning("Checksum mismatch for local file %s\n" 703 "Cleaning and trying again." % ud.localpath) 704 if os.path.exists(ud.localpath): 705 rename_bad_checksum(ud, e.checksum) 706 bb.utils.remove(ud.donestamp) 707 return False 708 709 710def update_stamp(ud, d): 711 """ 712 donestamp is file stamp indicating the whole fetching is done 713 this function update the stamp after verifying the checksum 714 """ 715 if not ud.needdonestamp: 716 return 717 718 if os.path.exists(ud.donestamp): 719 # Touch the done stamp file to show active use of the download 720 try: 721 os.utime(ud.donestamp, None) 722 except: 723 # Errors aren't fatal here 724 pass 725 else: 726 try: 727 checksums = verify_checksum(ud, d) 728 # Store the checksums for later re-verification against the recipe 729 with open(ud.donestamp, "wb") as cachefile: 730 p = pickle.Pickler(cachefile, 2) 731 p.dump(checksums) 732 except ChecksumError as e: 733 # Checksums failed to verify, trigger re-download and remove the 734 # incorrect stamp file. 735 logger.warning("Checksum mismatch for local file %s\n" 736 "Cleaning and trying again." % ud.localpath) 737 if os.path.exists(ud.localpath): 738 rename_bad_checksum(ud, e.checksum) 739 bb.utils.remove(ud.donestamp) 740 raise 741 742def subprocess_setup(): 743 # Python installs a SIGPIPE handler by default. This is usually not what 744 # non-Python subprocesses expect. 745 # SIGPIPE errors are known issues with gzip/bash 746 signal.signal(signal.SIGPIPE, signal.SIG_DFL) 747 748def get_autorev(d): 749 # only not cache src rev in autorev case 750 if d.getVar('BB_SRCREV_POLICY') != "cache": 751 d.setVar('BB_DONT_CACHE', '1') 752 return "AUTOINC" 753 754def get_srcrev(d, method_name='sortable_revision'): 755 """ 756 Return the revision string, usually for use in the version string (PV) of the current package 757 Most packages usually only have one SCM so we just pass on the call. 758 In the multi SCM case, we build a value based on SRCREV_FORMAT which must 759 have been set. 760 761 The idea here is that we put the string "AUTOINC+" into return value if the revisions are not 762 incremental, other code is then responsible for turning that into an increasing value (if needed) 763 764 A method_name can be supplied to retrieve an alternatively formatted revision from a fetcher, if 765 that fetcher provides a method with the given name and the same signature as sortable_revision. 766 """ 767 768 d.setVar("__BBSEENSRCREV", "1") 769 recursion = d.getVar("__BBINSRCREV") 770 if recursion: 771 raise FetchError("There are recursive references in fetcher variables, likely through SRC_URI") 772 d.setVar("__BBINSRCREV", True) 773 774 scms = [] 775 fetcher = Fetch(d.getVar('SRC_URI').split(), d) 776 urldata = fetcher.ud 777 for u in urldata: 778 if urldata[u].method.supports_srcrev(): 779 scms.append(u) 780 781 if not scms: 782 raise FetchError("SRCREV was used yet no valid SCM was found in SRC_URI") 783 784 if len(scms) == 1 and len(urldata[scms[0]].names) == 1: 785 autoinc, rev = getattr(urldata[scms[0]].method, method_name)(urldata[scms[0]], d, urldata[scms[0]].names[0]) 786 if len(rev) > 10: 787 rev = rev[:10] 788 d.delVar("__BBINSRCREV") 789 if autoinc: 790 return "AUTOINC+" + rev 791 return rev 792 793 # 794 # Mutiple SCMs are in SRC_URI so we resort to SRCREV_FORMAT 795 # 796 format = d.getVar('SRCREV_FORMAT') 797 if not format: 798 raise FetchError("The SRCREV_FORMAT variable must be set when multiple SCMs are used.\n"\ 799 "The SCMs are:\n%s" % '\n'.join(scms)) 800 801 name_to_rev = {} 802 seenautoinc = False 803 for scm in scms: 804 ud = urldata[scm] 805 for name in ud.names: 806 autoinc, rev = getattr(ud.method, method_name)(ud, d, name) 807 seenautoinc = seenautoinc or autoinc 808 if len(rev) > 10: 809 rev = rev[:10] 810 name_to_rev[name] = rev 811 # Replace names by revisions in the SRCREV_FORMAT string. The approach used 812 # here can handle names being prefixes of other names and names appearing 813 # as substrings in revisions (in which case the name should not be 814 # expanded). The '|' regular expression operator tries matches from left to 815 # right, so we need to sort the names with the longest ones first. 816 names_descending_len = sorted(name_to_rev, key=len, reverse=True) 817 name_to_rev_re = "|".join(re.escape(name) for name in names_descending_len) 818 format = re.sub(name_to_rev_re, lambda match: name_to_rev[match.group(0)], format) 819 820 if seenautoinc: 821 format = "AUTOINC+" + format 822 823 d.delVar("__BBINSRCREV") 824 return format 825 826def localpath(url, d): 827 fetcher = bb.fetch2.Fetch([url], d) 828 return fetcher.localpath(url) 829 830# Need to export PATH as binary could be in metadata paths 831# rather than host provided 832# Also include some other variables. 833FETCH_EXPORT_VARS = ['HOME', 'PATH', 834 'HTTP_PROXY', 'http_proxy', 835 'HTTPS_PROXY', 'https_proxy', 836 'FTP_PROXY', 'ftp_proxy', 837 'FTPS_PROXY', 'ftps_proxy', 838 'NO_PROXY', 'no_proxy', 839 'ALL_PROXY', 'all_proxy', 840 'GIT_PROXY_COMMAND', 841 'GIT_SSH', 842 'GIT_SSH_COMMAND', 843 'GIT_SSL_CAINFO', 844 'GIT_SMART_HTTP', 845 'SSH_AUTH_SOCK', 'SSH_AGENT_PID', 846 'SOCKS5_USER', 'SOCKS5_PASSWD', 847 'DBUS_SESSION_BUS_ADDRESS', 848 'P4CONFIG', 849 'SSL_CERT_FILE', 850 'AWS_PROFILE', 851 'AWS_ACCESS_KEY_ID', 852 'AWS_SECRET_ACCESS_KEY', 853 'AWS_DEFAULT_REGION'] 854 855def get_fetcher_environment(d): 856 newenv = {} 857 origenv = d.getVar("BB_ORIGENV") 858 for name in bb.fetch2.FETCH_EXPORT_VARS: 859 value = d.getVar(name) 860 if not value and origenv: 861 value = origenv.getVar(name) 862 if value: 863 newenv[name] = value 864 return newenv 865 866def runfetchcmd(cmd, d, quiet=False, cleanup=None, log=None, workdir=None): 867 """ 868 Run cmd returning the command output 869 Raise an error if interrupted or cmd fails 870 Optionally echo command output to stdout 871 Optionally remove the files/directories listed in cleanup upon failure 872 """ 873 874 exportvars = FETCH_EXPORT_VARS 875 876 if not cleanup: 877 cleanup = [] 878 879 # If PATH contains WORKDIR which contains PV-PR which contains SRCPV we 880 # can end up in circular recursion here so give the option of breaking it 881 # in a data store copy. 882 try: 883 d.getVar("PV") 884 d.getVar("PR") 885 except bb.data_smart.ExpansionError: 886 d = bb.data.createCopy(d) 887 d.setVar("PV", "fetcheravoidrecurse") 888 d.setVar("PR", "fetcheravoidrecurse") 889 890 origenv = d.getVar("BB_ORIGENV", False) 891 for var in exportvars: 892 val = d.getVar(var) or (origenv and origenv.getVar(var)) 893 if val: 894 cmd = 'export ' + var + '=\"%s\"; %s' % (val, cmd) 895 896 # Disable pseudo as it may affect ssh, potentially causing it to hang. 897 cmd = 'export PSEUDO_DISABLED=1; ' + cmd 898 899 if workdir: 900 logger.debug("Running '%s' in %s" % (cmd, workdir)) 901 else: 902 logger.debug("Running %s", cmd) 903 904 success = False 905 error_message = "" 906 907 try: 908 (output, errors) = bb.process.run(cmd, log=log, shell=True, stderr=subprocess.PIPE, cwd=workdir) 909 success = True 910 except bb.process.NotFoundError as e: 911 error_message = "Fetch command %s not found" % (e.command) 912 except bb.process.ExecutionError as e: 913 if e.stdout: 914 output = "output:\n%s\n%s" % (e.stdout, e.stderr) 915 elif e.stderr: 916 output = "output:\n%s" % e.stderr 917 else: 918 output = "no output" 919 error_message = "Fetch command %s failed with exit code %s, %s" % (e.command, e.exitcode, output) 920 except bb.process.CmdError as e: 921 error_message = "Fetch command %s could not be run:\n%s" % (e.command, e.msg) 922 if not success: 923 for f in cleanup: 924 try: 925 bb.utils.remove(f, True) 926 except OSError: 927 pass 928 929 raise FetchError(error_message) 930 931 return output 932 933def check_network_access(d, info, url): 934 """ 935 log remote network access, and error if BB_NO_NETWORK is set or the given 936 URI is untrusted 937 """ 938 if bb.utils.to_boolean(d.getVar("BB_NO_NETWORK")): 939 raise NetworkAccess(url, info) 940 elif not trusted_network(d, url): 941 raise UntrustedUrl(url, info) 942 else: 943 logger.debug("Fetcher accessed the network with the command %s" % info) 944 945def build_mirroruris(origud, mirrors, ld): 946 uris = [] 947 uds = [] 948 949 replacements = {} 950 replacements["TYPE"] = origud.type 951 replacements["HOST"] = origud.host 952 replacements["PATH"] = origud.path 953 replacements["BASENAME"] = origud.path.split("/")[-1] 954 replacements["MIRRORNAME"] = origud.host.replace(':','.') + origud.path.replace('/', '.').replace('*', '.') 955 956 def adduri(ud, uris, uds, mirrors, tarballs): 957 for line in mirrors: 958 try: 959 (find, replace) = line 960 except ValueError: 961 continue 962 963 for tarball in tarballs: 964 newuri = uri_replace(ud, find, replace, replacements, ld, tarball) 965 if not newuri or newuri in uris or newuri == origud.url: 966 continue 967 968 if not trusted_network(ld, newuri): 969 logger.debug("Mirror %s not in the list of trusted networks, skipping" % (newuri)) 970 continue 971 972 # Create a local copy of the mirrors minus the current line 973 # this will prevent us from recursively processing the same line 974 # as well as indirect recursion A -> B -> C -> A 975 localmirrors = list(mirrors) 976 localmirrors.remove(line) 977 978 try: 979 newud = FetchData(newuri, ld) 980 newud.setup_localpath(ld) 981 except bb.fetch2.BBFetchException as e: 982 logger.debug("Mirror fetch failure for url %s (original url: %s)" % (newuri, origud.url)) 983 logger.debug(str(e)) 984 try: 985 # setup_localpath of file:// urls may fail, we should still see 986 # if mirrors of the url exist 987 adduri(newud, uris, uds, localmirrors, tarballs) 988 except UnboundLocalError: 989 pass 990 continue 991 uris.append(newuri) 992 uds.append(newud) 993 994 adduri(newud, uris, uds, localmirrors, tarballs) 995 996 adduri(origud, uris, uds, mirrors, origud.mirrortarballs or [None]) 997 998 return uris, uds 999 1000def rename_bad_checksum(ud, suffix): 1001 """ 1002 Renames files to have suffix from parameter 1003 """ 1004 1005 if ud.localpath is None: 1006 return 1007 1008 new_localpath = "%s_bad-checksum_%s" % (ud.localpath, suffix) 1009 bb.warn("Renaming %s to %s" % (ud.localpath, new_localpath)) 1010 if not bb.utils.movefile(ud.localpath, new_localpath): 1011 bb.warn("Renaming %s to %s failed, grep movefile in log.do_fetch to see why" % (ud.localpath, new_localpath)) 1012 1013 1014def try_mirror_url(fetch, origud, ud, ld, check = False): 1015 # Return of None or a value means we're finished 1016 # False means try another url 1017 1018 if ud.lockfile and ud.lockfile != origud.lockfile: 1019 lf = bb.utils.lockfile(ud.lockfile) 1020 1021 try: 1022 if check: 1023 found = ud.method.checkstatus(fetch, ud, ld) 1024 if found: 1025 return found 1026 return False 1027 1028 if not verify_donestamp(ud, ld, origud) or ud.method.need_update(ud, ld): 1029 ud.method.download(ud, ld) 1030 if hasattr(ud.method,"build_mirror_data"): 1031 ud.method.build_mirror_data(ud, ld) 1032 1033 if not ud.localpath or not os.path.exists(ud.localpath): 1034 return False 1035 1036 if ud.localpath == origud.localpath: 1037 return ud.localpath 1038 1039 # We may be obtaining a mirror tarball which needs further processing by the real fetcher 1040 # If that tarball is a local file:// we need to provide a symlink to it 1041 dldir = ld.getVar("DL_DIR") 1042 1043 if origud.mirrortarballs and os.path.basename(ud.localpath) in origud.mirrortarballs and os.path.basename(ud.localpath) != os.path.basename(origud.localpath): 1044 # Create donestamp in old format to avoid triggering a re-download 1045 if ud.donestamp: 1046 bb.utils.mkdirhier(os.path.dirname(ud.donestamp)) 1047 open(ud.donestamp, 'w').close() 1048 dest = os.path.join(dldir, os.path.basename(ud.localpath)) 1049 if not os.path.exists(dest): 1050 # In case this is executing without any file locks held (as is 1051 # the case for file:// URLs), two tasks may end up here at the 1052 # same time, in which case we do not want the second task to 1053 # fail when the link has already been created by the first task. 1054 try: 1055 os.symlink(ud.localpath, dest) 1056 except FileExistsError: 1057 pass 1058 if not verify_donestamp(origud, ld) or origud.method.need_update(origud, ld): 1059 origud.method.download(origud, ld) 1060 if hasattr(origud.method, "build_mirror_data"): 1061 origud.method.build_mirror_data(origud, ld) 1062 return origud.localpath 1063 # Otherwise the result is a local file:// and we symlink to it 1064 ensure_symlink(ud.localpath, origud.localpath) 1065 update_stamp(origud, ld) 1066 return ud.localpath 1067 1068 except bb.fetch2.NetworkAccess: 1069 raise 1070 1071 except IOError as e: 1072 if e.errno in [errno.ESTALE]: 1073 logger.warning("Stale Error Observed %s." % ud.url) 1074 return False 1075 raise 1076 1077 except bb.fetch2.BBFetchException as e: 1078 if isinstance(e, ChecksumError): 1079 logger.warning("Mirror checksum failure for url %s (original url: %s)\nCleaning and trying again." % (ud.url, origud.url)) 1080 logger.warning(str(e)) 1081 if os.path.exists(ud.localpath): 1082 rename_bad_checksum(ud, e.checksum) 1083 elif isinstance(e, NoChecksumError): 1084 raise 1085 else: 1086 logger.debug("Mirror fetch failure for url %s (original url: %s)" % (ud.url, origud.url)) 1087 logger.debug(str(e)) 1088 try: 1089 ud.method.clean(ud, ld) 1090 except UnboundLocalError: 1091 pass 1092 return False 1093 finally: 1094 if ud.lockfile and ud.lockfile != origud.lockfile: 1095 bb.utils.unlockfile(lf) 1096 1097 1098def ensure_symlink(target, link_name): 1099 if not os.path.exists(link_name): 1100 dirname = os.path.dirname(link_name) 1101 bb.utils.mkdirhier(dirname) 1102 if os.path.islink(link_name): 1103 # Broken symbolic link 1104 os.unlink(link_name) 1105 1106 # In case this is executing without any file locks held (as is 1107 # the case for file:// URLs), two tasks may end up here at the 1108 # same time, in which case we do not want the second task to 1109 # fail when the link has already been created by the first task. 1110 try: 1111 os.symlink(target, link_name) 1112 except FileExistsError: 1113 pass 1114 1115 1116def try_mirrors(fetch, d, origud, mirrors, check = False): 1117 """ 1118 Try to use a mirrored version of the sources. 1119 This method will be automatically called before the fetchers go. 1120 1121 d Is a bb.data instance 1122 uri is the original uri we're trying to download 1123 mirrors is the list of mirrors we're going to try 1124 """ 1125 ld = d.createCopy() 1126 1127 uris, uds = build_mirroruris(origud, mirrors, ld) 1128 1129 for index, uri in enumerate(uris): 1130 ret = try_mirror_url(fetch, origud, uds[index], ld, check) 1131 if ret: 1132 return ret 1133 return None 1134 1135def trusted_network(d, url): 1136 """ 1137 Use a trusted url during download if networking is enabled and 1138 BB_ALLOWED_NETWORKS is set globally or for a specific recipe. 1139 Note: modifies SRC_URI & mirrors. 1140 """ 1141 if bb.utils.to_boolean(d.getVar("BB_NO_NETWORK")): 1142 return True 1143 1144 pkgname = d.expand(d.getVar('PN', False)) 1145 trusted_hosts = None 1146 if pkgname: 1147 trusted_hosts = d.getVarFlag('BB_ALLOWED_NETWORKS', pkgname, False) 1148 1149 if not trusted_hosts: 1150 trusted_hosts = d.getVar('BB_ALLOWED_NETWORKS') 1151 1152 # Not enabled. 1153 if not trusted_hosts: 1154 return True 1155 1156 scheme, network, path, user, passwd, param = decodeurl(url) 1157 1158 if not network: 1159 return True 1160 1161 network = network.split(':')[0] 1162 network = network.lower() 1163 1164 for host in trusted_hosts.split(" "): 1165 host = host.lower() 1166 if host.startswith("*.") and ("." + network).endswith(host[1:]): 1167 return True 1168 if host == network: 1169 return True 1170 1171 return False 1172 1173def srcrev_internal_helper(ud, d, name): 1174 """ 1175 Return: 1176 a) a source revision if specified 1177 b) latest revision if SRCREV="AUTOINC" 1178 c) None if not specified 1179 """ 1180 1181 srcrev = None 1182 pn = d.getVar("PN") 1183 attempts = [] 1184 if name != '' and pn: 1185 attempts.append("SRCREV_%s:pn-%s" % (name, pn)) 1186 if name != '': 1187 attempts.append("SRCREV_%s" % name) 1188 if pn: 1189 attempts.append("SRCREV:pn-%s" % pn) 1190 attempts.append("SRCREV") 1191 1192 for a in attempts: 1193 srcrev = d.getVar(a) 1194 if srcrev and srcrev != "INVALID": 1195 break 1196 1197 if 'rev' in ud.parm and 'tag' in ud.parm: 1198 raise FetchError("Please specify a ;rev= parameter or a ;tag= parameter in the url %s but not both." % (ud.url)) 1199 1200 if 'rev' in ud.parm or 'tag' in ud.parm: 1201 if 'rev' in ud.parm: 1202 parmrev = ud.parm['rev'] 1203 else: 1204 parmrev = ud.parm['tag'] 1205 if srcrev == "INVALID" or not srcrev: 1206 return parmrev 1207 if srcrev != parmrev: 1208 raise FetchError("Conflicting revisions (%s from SRCREV and %s from the url) found, please specify one valid value" % (srcrev, parmrev)) 1209 return parmrev 1210 1211 if srcrev == "INVALID" or not srcrev: 1212 raise FetchError("Please set a valid SRCREV for url %s (possible key names are %s, or use a ;rev=X URL parameter)" % (str(attempts), ud.url), ud.url) 1213 if srcrev == "AUTOINC": 1214 srcrev = ud.method.latest_revision(ud, d, name) 1215 1216 return srcrev 1217 1218def get_checksum_file_list(d): 1219 """ Get a list of files checksum in SRC_URI 1220 1221 Returns the resolved local paths of all local file entries in 1222 SRC_URI as a space-separated string 1223 """ 1224 fetch = Fetch([], d, cache = False, localonly = True) 1225 1226 dl_dir = d.getVar('DL_DIR') 1227 filelist = [] 1228 for u in fetch.urls: 1229 ud = fetch.ud[u] 1230 1231 if ud and isinstance(ud.method, local.Local): 1232 paths = ud.method.localpaths(ud, d) 1233 for f in paths: 1234 pth = ud.decodedurl 1235 if f.startswith(dl_dir): 1236 # The local fetcher's behaviour is to return a path under DL_DIR if it couldn't find the file anywhere else 1237 if os.path.exists(f): 1238 bb.warn("Getting checksum for %s SRC_URI entry %s: file not found except in DL_DIR" % (d.getVar('PN'), os.path.basename(f))) 1239 else: 1240 bb.warn("Unable to get checksum for %s SRC_URI entry %s: file could not be found" % (d.getVar('PN'), os.path.basename(f))) 1241 filelist.append(f + ":" + str(os.path.exists(f))) 1242 1243 return " ".join(filelist) 1244 1245def get_file_checksums(filelist, pn, localdirsexclude): 1246 """Get a list of the checksums for a list of local files 1247 1248 Returns the checksums for a list of local files, caching the results as 1249 it proceeds 1250 1251 """ 1252 return _checksum_cache.get_checksums(filelist, pn, localdirsexclude) 1253 1254 1255class FetchData(object): 1256 """ 1257 A class which represents the fetcher state for a given URI. 1258 """ 1259 def __init__(self, url, d, localonly = False): 1260 # localpath is the location of a downloaded result. If not set, the file is local. 1261 self.donestamp = None 1262 self.needdonestamp = True 1263 self.localfile = "" 1264 self.localpath = None 1265 self.lockfile = None 1266 self.mirrortarballs = [] 1267 self.basename = None 1268 self.basepath = None 1269 (self.type, self.host, self.path, self.user, self.pswd, self.parm) = decodeurl(d.expand(url)) 1270 self.date = self.getSRCDate(d) 1271 self.url = url 1272 if not self.user and "user" in self.parm: 1273 self.user = self.parm["user"] 1274 if not self.pswd and "pswd" in self.parm: 1275 self.pswd = self.parm["pswd"] 1276 self.setup = False 1277 1278 def configure_checksum(checksum_id): 1279 if "name" in self.parm: 1280 checksum_name = "%s.%ssum" % (self.parm["name"], checksum_id) 1281 else: 1282 checksum_name = "%ssum" % checksum_id 1283 1284 setattr(self, "%s_name" % checksum_id, checksum_name) 1285 1286 if checksum_name in self.parm: 1287 checksum_expected = self.parm[checksum_name] 1288 elif self.type not in ["http", "https", "ftp", "ftps", "sftp", "s3", "az"]: 1289 checksum_expected = None 1290 else: 1291 checksum_expected = d.getVarFlag("SRC_URI", checksum_name) 1292 1293 setattr(self, "%s_expected" % checksum_id, checksum_expected) 1294 1295 for checksum_id in CHECKSUM_LIST: 1296 configure_checksum(checksum_id) 1297 1298 self.ignore_checksums = False 1299 1300 self.names = self.parm.get("name",'default').split(',') 1301 1302 self.method = None 1303 for m in methods: 1304 if m.supports(self, d): 1305 self.method = m 1306 break 1307 1308 if not self.method: 1309 raise NoMethodError(url) 1310 1311 if localonly and not isinstance(self.method, local.Local): 1312 raise NonLocalMethod() 1313 1314 if self.parm.get("proto", None) and "protocol" not in self.parm: 1315 logger.warning('Consider updating %s recipe to use "protocol" not "proto" in SRC_URI.', d.getVar('PN')) 1316 self.parm["protocol"] = self.parm.get("proto", None) 1317 1318 if hasattr(self.method, "urldata_init"): 1319 self.method.urldata_init(self, d) 1320 1321 if "localpath" in self.parm: 1322 # if user sets localpath for file, use it instead. 1323 self.localpath = self.parm["localpath"] 1324 self.basename = os.path.basename(self.localpath) 1325 elif self.localfile: 1326 self.localpath = self.method.localpath(self, d) 1327 1328 dldir = d.getVar("DL_DIR") 1329 1330 if not self.needdonestamp: 1331 return 1332 1333 # Note: .done and .lock files should always be in DL_DIR whereas localpath may not be. 1334 if self.localpath and self.localpath.startswith(dldir): 1335 basepath = self.localpath 1336 elif self.localpath: 1337 basepath = dldir + os.sep + os.path.basename(self.localpath) 1338 elif self.basepath or self.basename: 1339 basepath = dldir + os.sep + (self.basepath or self.basename) 1340 else: 1341 bb.fatal("Can't determine lock path for url %s" % url) 1342 1343 self.donestamp = basepath + '.done' 1344 self.lockfile = basepath + '.lock' 1345 1346 def setup_revisions(self, d): 1347 self.revisions = {} 1348 for name in self.names: 1349 self.revisions[name] = srcrev_internal_helper(self, d, name) 1350 1351 # add compatibility code for non name specified case 1352 if len(self.names) == 1: 1353 self.revision = self.revisions[self.names[0]] 1354 1355 def setup_localpath(self, d): 1356 if not self.localpath: 1357 self.localpath = self.method.localpath(self, d) 1358 1359 def getSRCDate(self, d): 1360 """ 1361 Return the SRC Date for the component 1362 1363 d the bb.data module 1364 """ 1365 if "srcdate" in self.parm: 1366 return self.parm['srcdate'] 1367 1368 pn = d.getVar("PN") 1369 1370 if pn: 1371 return d.getVar("SRCDATE_%s" % pn) or d.getVar("SRCDATE") or d.getVar("DATE") 1372 1373 return d.getVar("SRCDATE") or d.getVar("DATE") 1374 1375class FetchMethod(object): 1376 """Base class for 'fetch'ing data""" 1377 1378 def __init__(self, urls=None): 1379 self.urls = [] 1380 1381 def supports(self, urldata, d): 1382 """ 1383 Check to see if this fetch class supports a given url. 1384 """ 1385 return 0 1386 1387 def localpath(self, urldata, d): 1388 """ 1389 Return the local filename of a given url assuming a successful fetch. 1390 Can also setup variables in urldata for use in go (saving code duplication 1391 and duplicate code execution) 1392 """ 1393 return os.path.join(d.getVar("DL_DIR"), urldata.localfile) 1394 1395 def supports_checksum(self, urldata): 1396 """ 1397 Is localpath something that can be represented by a checksum? 1398 """ 1399 1400 # We cannot compute checksums for directories 1401 if os.path.isdir(urldata.localpath): 1402 return False 1403 return True 1404 1405 def recommends_checksum(self, urldata): 1406 """ 1407 Is the backend on where checksumming is recommended (should warnings 1408 be displayed if there is no checksum)? 1409 """ 1410 return False 1411 1412 def verify_donestamp(self, ud, d): 1413 """ 1414 Verify the donestamp file 1415 """ 1416 return verify_donestamp(ud, d) 1417 1418 def update_donestamp(self, ud, d): 1419 """ 1420 Update the donestamp file 1421 """ 1422 update_stamp(ud, d) 1423 1424 def _strip_leading_slashes(self, relpath): 1425 """ 1426 Remove leading slash as os.path.join can't cope 1427 """ 1428 while os.path.isabs(relpath): 1429 relpath = relpath[1:] 1430 return relpath 1431 1432 def setUrls(self, urls): 1433 self.__urls = urls 1434 1435 def getUrls(self): 1436 return self.__urls 1437 1438 urls = property(getUrls, setUrls, None, "Urls property") 1439 1440 def need_update(self, ud, d): 1441 """ 1442 Force a fetch, even if localpath exists? 1443 """ 1444 if os.path.exists(ud.localpath): 1445 return False 1446 return True 1447 1448 def supports_srcrev(self): 1449 """ 1450 The fetcher supports auto source revisions (SRCREV) 1451 """ 1452 return False 1453 1454 def download(self, urldata, d): 1455 """ 1456 Fetch urls 1457 Assumes localpath was called first 1458 """ 1459 raise NoMethodError(urldata.url) 1460 1461 def unpack(self, urldata, rootdir, data): 1462 iterate = False 1463 file = urldata.localpath 1464 1465 try: 1466 unpack = bb.utils.to_boolean(urldata.parm.get('unpack'), True) 1467 except ValueError as exc: 1468 bb.fatal("Invalid value for 'unpack' parameter for %s: %s" % 1469 (file, urldata.parm.get('unpack'))) 1470 1471 base, ext = os.path.splitext(file) 1472 if ext in ['.gz', '.bz2', '.Z', '.xz', '.lz']: 1473 efile = os.path.join(rootdir, os.path.basename(base)) 1474 else: 1475 efile = file 1476 cmd = None 1477 1478 if unpack: 1479 tar_cmd = 'tar --extract --no-same-owner' 1480 if 'striplevel' in urldata.parm: 1481 tar_cmd += ' --strip-components=%s' % urldata.parm['striplevel'] 1482 if file.endswith('.tar'): 1483 cmd = '%s -f %s' % (tar_cmd, file) 1484 elif file.endswith('.tgz') or file.endswith('.tar.gz') or file.endswith('.tar.Z'): 1485 cmd = '%s -z -f %s' % (tar_cmd, file) 1486 elif file.endswith('.tbz') or file.endswith('.tbz2') or file.endswith('.tar.bz2'): 1487 cmd = 'bzip2 -dc %s | %s -f -' % (file, tar_cmd) 1488 elif file.endswith('.gz') or file.endswith('.Z') or file.endswith('.z'): 1489 cmd = 'gzip -dc %s > %s' % (file, efile) 1490 elif file.endswith('.bz2'): 1491 cmd = 'bzip2 -dc %s > %s' % (file, efile) 1492 elif file.endswith('.txz') or file.endswith('.tar.xz'): 1493 cmd = 'xz -dc %s | %s -f -' % (file, tar_cmd) 1494 elif file.endswith('.xz'): 1495 cmd = 'xz -dc %s > %s' % (file, efile) 1496 elif file.endswith('.tar.lz'): 1497 cmd = 'lzip -dc %s | %s -f -' % (file, tar_cmd) 1498 elif file.endswith('.lz'): 1499 cmd = 'lzip -dc %s > %s' % (file, efile) 1500 elif file.endswith('.tar.7z'): 1501 cmd = '7z x -so %s | %s -f -' % (file, tar_cmd) 1502 elif file.endswith('.7z'): 1503 cmd = '7za x -y %s 1>/dev/null' % file 1504 elif file.endswith('.tzst') or file.endswith('.tar.zst'): 1505 cmd = 'zstd --decompress --stdout %s | %s -f -' % (file, tar_cmd) 1506 elif file.endswith('.zst'): 1507 cmd = 'zstd --decompress --stdout %s > %s' % (file, efile) 1508 elif file.endswith('.zip') or file.endswith('.jar'): 1509 try: 1510 dos = bb.utils.to_boolean(urldata.parm.get('dos'), False) 1511 except ValueError as exc: 1512 bb.fatal("Invalid value for 'dos' parameter for %s: %s" % 1513 (file, urldata.parm.get('dos'))) 1514 cmd = 'unzip -q -o' 1515 if dos: 1516 cmd = '%s -a' % cmd 1517 cmd = "%s '%s'" % (cmd, file) 1518 elif file.endswith('.rpm') or file.endswith('.srpm'): 1519 if 'extract' in urldata.parm: 1520 unpack_file = urldata.parm.get('extract') 1521 cmd = 'rpm2cpio.sh %s | cpio -id %s' % (file, unpack_file) 1522 iterate = True 1523 iterate_file = unpack_file 1524 else: 1525 cmd = 'rpm2cpio.sh %s | cpio -id' % (file) 1526 elif file.endswith('.deb') or file.endswith('.ipk'): 1527 output = subprocess.check_output(['ar', '-t', file], preexec_fn=subprocess_setup) 1528 datafile = None 1529 if output: 1530 for line in output.decode().splitlines(): 1531 if line.startswith('data.tar.'): 1532 datafile = line 1533 break 1534 else: 1535 raise UnpackError("Unable to unpack deb/ipk package - does not contain data.tar.* file", urldata.url) 1536 else: 1537 raise UnpackError("Unable to unpack deb/ipk package - could not list contents", urldata.url) 1538 cmd = 'ar x %s %s && %s -p -f %s && rm %s' % (file, datafile, tar_cmd, datafile, datafile) 1539 1540 # If 'subdir' param exists, create a dir and use it as destination for unpack cmd 1541 if 'subdir' in urldata.parm: 1542 subdir = urldata.parm.get('subdir') 1543 if os.path.isabs(subdir): 1544 if not os.path.realpath(subdir).startswith(os.path.realpath(rootdir)): 1545 raise UnpackError("subdir argument isn't a subdirectory of unpack root %s" % rootdir, urldata.url) 1546 unpackdir = subdir 1547 else: 1548 unpackdir = os.path.join(rootdir, subdir) 1549 bb.utils.mkdirhier(unpackdir) 1550 else: 1551 unpackdir = rootdir 1552 1553 if not unpack or not cmd: 1554 # If file == dest, then avoid any copies, as we already put the file into dest! 1555 dest = os.path.join(unpackdir, os.path.basename(file)) 1556 if file != dest and not (os.path.exists(dest) and os.path.samefile(file, dest)): 1557 destdir = '.' 1558 # For file:// entries all intermediate dirs in path must be created at destination 1559 if urldata.type == "file": 1560 # Trailing '/' does a copying to wrong place 1561 urlpath = urldata.path.rstrip('/') 1562 # Want files places relative to cwd so no leading '/' 1563 urlpath = urlpath.lstrip('/') 1564 if urlpath.find("/") != -1: 1565 destdir = urlpath.rsplit("/", 1)[0] + '/' 1566 bb.utils.mkdirhier("%s/%s" % (unpackdir, destdir)) 1567 cmd = 'cp -fpPRH "%s" "%s"' % (file, destdir) 1568 1569 if not cmd: 1570 return 1571 1572 path = data.getVar('PATH') 1573 if path: 1574 cmd = "PATH=\"%s\" %s" % (path, cmd) 1575 bb.note("Unpacking %s to %s/" % (file, unpackdir)) 1576 ret = subprocess.call(cmd, preexec_fn=subprocess_setup, shell=True, cwd=unpackdir) 1577 1578 if ret != 0: 1579 raise UnpackError("Unpack command %s failed with return value %s" % (cmd, ret), urldata.url) 1580 1581 if iterate is True: 1582 iterate_urldata = urldata 1583 iterate_urldata.localpath = "%s/%s" % (rootdir, iterate_file) 1584 self.unpack(urldata, rootdir, data) 1585 1586 return 1587 1588 def clean(self, urldata, d): 1589 """ 1590 Clean any existing full or partial download 1591 """ 1592 bb.utils.remove(urldata.localpath) 1593 1594 def try_premirror(self, urldata, d): 1595 """ 1596 Should premirrors be used? 1597 """ 1598 return True 1599 1600 def try_mirrors(self, fetch, urldata, d, mirrors, check=False): 1601 """ 1602 Try to use a mirror 1603 """ 1604 return bool(try_mirrors(fetch, d, urldata, mirrors, check)) 1605 1606 def checkstatus(self, fetch, urldata, d): 1607 """ 1608 Check the status of a URL 1609 Assumes localpath was called first 1610 """ 1611 logger.info("URL %s could not be checked for status since no method exists.", urldata.url) 1612 return True 1613 1614 def latest_revision(self, ud, d, name): 1615 """ 1616 Look in the cache for the latest revision, if not present ask the SCM. 1617 """ 1618 if not hasattr(self, "_latest_revision"): 1619 raise ParameterError("The fetcher for this URL does not support _latest_revision", ud.url) 1620 1621 revs = bb.persist_data.persist('BB_URI_HEADREVS', d) 1622 key = self.generate_revision_key(ud, d, name) 1623 try: 1624 return revs[key] 1625 except KeyError: 1626 revs[key] = rev = self._latest_revision(ud, d, name) 1627 return rev 1628 1629 def sortable_revision(self, ud, d, name): 1630 latest_rev = self._build_revision(ud, d, name) 1631 return True, str(latest_rev) 1632 1633 def generate_revision_key(self, ud, d, name): 1634 return self._revision_key(ud, d, name) 1635 1636 def latest_versionstring(self, ud, d): 1637 """ 1638 Compute the latest release name like "x.y.x" in "x.y.x+gitHASH" 1639 by searching through the tags output of ls-remote, comparing 1640 versions and returning the highest match as a (version, revision) pair. 1641 """ 1642 return ('', '') 1643 1644 def done(self, ud, d): 1645 """ 1646 Is the download done ? 1647 """ 1648 if os.path.exists(ud.localpath): 1649 return True 1650 return False 1651 1652 def implicit_urldata(self, ud, d): 1653 """ 1654 Get a list of FetchData objects for any implicit URLs that will also 1655 be downloaded when we fetch the given URL. 1656 """ 1657 return [] 1658 1659class Fetch(object): 1660 def __init__(self, urls, d, cache = True, localonly = False, connection_cache = None): 1661 if localonly and cache: 1662 raise Exception("bb.fetch2.Fetch.__init__: cannot set cache and localonly at same time") 1663 1664 if not urls: 1665 urls = d.getVar("SRC_URI").split() 1666 self.urls = urls 1667 self.d = d 1668 self.ud = {} 1669 self.connection_cache = connection_cache 1670 1671 fn = d.getVar('FILE') 1672 mc = d.getVar('__BBMULTICONFIG') or "" 1673 key = None 1674 if cache and fn: 1675 key = mc + fn + str(id(d)) 1676 if key in urldata_cache: 1677 self.ud = urldata_cache[key] 1678 1679 for url in urls: 1680 if url not in self.ud: 1681 try: 1682 self.ud[url] = FetchData(url, d, localonly) 1683 except NonLocalMethod: 1684 if localonly: 1685 self.ud[url] = None 1686 pass 1687 1688 if key: 1689 urldata_cache[key] = self.ud 1690 1691 def localpath(self, url): 1692 if url not in self.urls: 1693 self.ud[url] = FetchData(url, self.d) 1694 1695 self.ud[url].setup_localpath(self.d) 1696 return self.d.expand(self.ud[url].localpath) 1697 1698 def localpaths(self): 1699 """ 1700 Return a list of the local filenames, assuming successful fetch 1701 """ 1702 local = [] 1703 1704 for u in self.urls: 1705 ud = self.ud[u] 1706 ud.setup_localpath(self.d) 1707 local.append(ud.localpath) 1708 1709 return local 1710 1711 def download(self, urls=None): 1712 """ 1713 Fetch all urls 1714 """ 1715 if not urls: 1716 urls = self.urls 1717 1718 network = self.d.getVar("BB_NO_NETWORK") 1719 premirroronly = bb.utils.to_boolean(self.d.getVar("BB_FETCH_PREMIRRORONLY")) 1720 1721 for u in urls: 1722 ud = self.ud[u] 1723 ud.setup_localpath(self.d) 1724 m = ud.method 1725 done = False 1726 1727 if ud.lockfile: 1728 lf = bb.utils.lockfile(ud.lockfile) 1729 1730 try: 1731 self.d.setVar("BB_NO_NETWORK", network) 1732 1733 if m.verify_donestamp(ud, self.d) and not m.need_update(ud, self.d): 1734 done = True 1735 elif m.try_premirror(ud, self.d): 1736 logger.debug("Trying PREMIRRORS") 1737 mirrors = mirror_from_string(self.d.getVar('PREMIRRORS')) 1738 done = m.try_mirrors(self, ud, self.d, mirrors) 1739 if done: 1740 try: 1741 # early checksum verification so that if the checksum of the premirror 1742 # contents mismatch the fetcher can still try upstream and mirrors 1743 m.update_donestamp(ud, self.d) 1744 except ChecksumError as e: 1745 logger.warning("Checksum failure encountered with premirror download of %s - will attempt other sources." % u) 1746 logger.debug(str(e)) 1747 done = False 1748 1749 if premirroronly: 1750 self.d.setVar("BB_NO_NETWORK", "1") 1751 1752 firsterr = None 1753 verified_stamp = False 1754 if done: 1755 verified_stamp = m.verify_donestamp(ud, self.d) 1756 if not done and (not verified_stamp or m.need_update(ud, self.d)): 1757 try: 1758 if not trusted_network(self.d, ud.url): 1759 raise UntrustedUrl(ud.url) 1760 logger.debug("Trying Upstream") 1761 m.download(ud, self.d) 1762 if hasattr(m, "build_mirror_data"): 1763 m.build_mirror_data(ud, self.d) 1764 done = True 1765 # early checksum verify, so that if checksum mismatched, 1766 # fetcher still have chance to fetch from mirror 1767 m.update_donestamp(ud, self.d) 1768 1769 except bb.fetch2.NetworkAccess: 1770 raise 1771 1772 except BBFetchException as e: 1773 if isinstance(e, ChecksumError): 1774 logger.warning("Checksum failure encountered with download of %s - will attempt other sources if available" % u) 1775 logger.debug(str(e)) 1776 if os.path.exists(ud.localpath): 1777 rename_bad_checksum(ud, e.checksum) 1778 elif isinstance(e, NoChecksumError): 1779 raise 1780 else: 1781 logger.warning('Failed to fetch URL %s, attempting MIRRORS if available' % u) 1782 logger.debug(str(e)) 1783 firsterr = e 1784 # Remove any incomplete fetch 1785 if not verified_stamp: 1786 m.clean(ud, self.d) 1787 logger.debug("Trying MIRRORS") 1788 mirrors = mirror_from_string(self.d.getVar('MIRRORS')) 1789 done = m.try_mirrors(self, ud, self.d, mirrors) 1790 1791 if not done or not m.done(ud, self.d): 1792 if firsterr: 1793 logger.error(str(firsterr)) 1794 raise FetchError("Unable to fetch URL from any source.", u) 1795 1796 m.update_donestamp(ud, self.d) 1797 1798 except IOError as e: 1799 if e.errno in [errno.ESTALE]: 1800 logger.error("Stale Error Observed %s." % u) 1801 raise ChecksumError("Stale Error Detected") 1802 1803 except BBFetchException as e: 1804 if isinstance(e, ChecksumError): 1805 logger.error("Checksum failure fetching %s" % u) 1806 raise 1807 1808 finally: 1809 if ud.lockfile: 1810 bb.utils.unlockfile(lf) 1811 1812 def checkstatus(self, urls=None): 1813 """ 1814 Check all URLs exist upstream. 1815 1816 Returns None if the URLs exist, raises FetchError if the check wasn't 1817 successful but there wasn't an error (such as file not found), and 1818 raises other exceptions in error cases. 1819 """ 1820 1821 if not urls: 1822 urls = self.urls 1823 1824 for u in urls: 1825 ud = self.ud[u] 1826 ud.setup_localpath(self.d) 1827 m = ud.method 1828 logger.debug("Testing URL %s", u) 1829 # First try checking uri, u, from PREMIRRORS 1830 mirrors = mirror_from_string(self.d.getVar('PREMIRRORS')) 1831 ret = m.try_mirrors(self, ud, self.d, mirrors, True) 1832 if not ret: 1833 # Next try checking from the original uri, u 1834 ret = m.checkstatus(self, ud, self.d) 1835 if not ret: 1836 # Finally, try checking uri, u, from MIRRORS 1837 mirrors = mirror_from_string(self.d.getVar('MIRRORS')) 1838 ret = m.try_mirrors(self, ud, self.d, mirrors, True) 1839 1840 if not ret: 1841 raise FetchError("URL %s doesn't work" % u, u) 1842 1843 def unpack(self, root, urls=None): 1844 """ 1845 Unpack urls to root 1846 """ 1847 1848 if not urls: 1849 urls = self.urls 1850 1851 for u in urls: 1852 ud = self.ud[u] 1853 ud.setup_localpath(self.d) 1854 1855 if ud.lockfile: 1856 lf = bb.utils.lockfile(ud.lockfile) 1857 1858 ud.method.unpack(ud, root, self.d) 1859 1860 if ud.lockfile: 1861 bb.utils.unlockfile(lf) 1862 1863 def clean(self, urls=None): 1864 """ 1865 Clean files that the fetcher gets or places 1866 """ 1867 1868 if not urls: 1869 urls = self.urls 1870 1871 for url in urls: 1872 if url not in self.ud: 1873 self.ud[url] = FetchData(url, self.d) 1874 ud = self.ud[url] 1875 ud.setup_localpath(self.d) 1876 1877 if not ud.localfile and ud.localpath is None: 1878 continue 1879 1880 if ud.lockfile: 1881 lf = bb.utils.lockfile(ud.lockfile) 1882 1883 ud.method.clean(ud, self.d) 1884 if ud.donestamp: 1885 bb.utils.remove(ud.donestamp) 1886 1887 if ud.lockfile: 1888 bb.utils.unlockfile(lf) 1889 1890 def expanded_urldata(self, urls=None): 1891 """ 1892 Get an expanded list of FetchData objects covering both the given 1893 URLS and any additional implicit URLs that are added automatically by 1894 the appropriate FetchMethod. 1895 """ 1896 1897 if not urls: 1898 urls = self.urls 1899 1900 urldata = [] 1901 for url in urls: 1902 ud = self.ud[url] 1903 urldata.append(ud) 1904 urldata += ud.method.implicit_urldata(ud, self.d) 1905 1906 return urldata 1907 1908class FetchConnectionCache(object): 1909 """ 1910 A class which represents an container for socket connections. 1911 """ 1912 def __init__(self): 1913 self.cache = {} 1914 1915 def get_connection_name(self, host, port): 1916 return host + ':' + str(port) 1917 1918 def add_connection(self, host, port, connection): 1919 cn = self.get_connection_name(host, port) 1920 1921 if cn not in self.cache: 1922 self.cache[cn] = connection 1923 1924 def get_connection(self, host, port): 1925 connection = None 1926 1927 cn = self.get_connection_name(host, port) 1928 if cn in self.cache: 1929 connection = self.cache[cn] 1930 1931 return connection 1932 1933 def remove_connection(self, host, port): 1934 cn = self.get_connection_name(host, port) 1935 if cn in self.cache: 1936 self.cache[cn].close() 1937 del self.cache[cn] 1938 1939 def close_connections(self): 1940 for cn in list(self.cache.keys()): 1941 self.cache[cn].close() 1942 del self.cache[cn] 1943 1944from . import cvs 1945from . import git 1946from . import gitsm 1947from . import gitannex 1948from . import local 1949from . import svn 1950from . import wget 1951from . import ssh 1952from . import sftp 1953from . import s3 1954from . import perforce 1955from . import bzr 1956from . import hg 1957from . import osc 1958from . import repo 1959from . import clearcase 1960from . import npm 1961from . import npmsw 1962from . import az 1963from . import crate 1964 1965methods.append(local.Local()) 1966methods.append(wget.Wget()) 1967methods.append(svn.Svn()) 1968methods.append(git.Git()) 1969methods.append(gitsm.GitSM()) 1970methods.append(gitannex.GitANNEX()) 1971methods.append(cvs.Cvs()) 1972methods.append(ssh.SSH()) 1973methods.append(sftp.SFTP()) 1974methods.append(s3.S3()) 1975methods.append(perforce.Perforce()) 1976methods.append(bzr.Bzr()) 1977methods.append(hg.Hg()) 1978methods.append(osc.Osc()) 1979methods.append(repo.Repo()) 1980methods.append(clearcase.ClearCase()) 1981methods.append(npm.Npm()) 1982methods.append(npmsw.NpmShrinkWrap()) 1983methods.append(az.Az()) 1984methods.append(crate.Crate()) 1985