1*4882a593Smuzhiyun""" 2*4882a593SmuzhiyunBitBake 'Fetch' Azure Storage implementation 3*4882a593Smuzhiyun 4*4882a593Smuzhiyun""" 5*4882a593Smuzhiyun 6*4882a593Smuzhiyun# Copyright (C) 2021 Alejandro Hernandez Samaniego 7*4882a593Smuzhiyun# 8*4882a593Smuzhiyun# Based on bb.fetch2.wget: 9*4882a593Smuzhiyun# Copyright (C) 2003, 2004 Chris Larson 10*4882a593Smuzhiyun# 11*4882a593Smuzhiyun# SPDX-License-Identifier: GPL-2.0-only 12*4882a593Smuzhiyun# 13*4882a593Smuzhiyun# Based on functions from the base bb module, Copyright 2003 Holger Schurig 14*4882a593Smuzhiyun 15*4882a593Smuzhiyunimport shlex 16*4882a593Smuzhiyunimport os 17*4882a593Smuzhiyunimport bb 18*4882a593Smuzhiyunfrom bb.fetch2 import FetchError 19*4882a593Smuzhiyunfrom bb.fetch2 import logger 20*4882a593Smuzhiyunfrom bb.fetch2.wget import Wget 21*4882a593Smuzhiyun 22*4882a593Smuzhiyun 23*4882a593Smuzhiyunclass Az(Wget): 24*4882a593Smuzhiyun 25*4882a593Smuzhiyun def supports(self, ud, d): 26*4882a593Smuzhiyun """ 27*4882a593Smuzhiyun Check to see if a given url can be fetched from Azure Storage 28*4882a593Smuzhiyun """ 29*4882a593Smuzhiyun return ud.type in ['az'] 30*4882a593Smuzhiyun 31*4882a593Smuzhiyun 32*4882a593Smuzhiyun def checkstatus(self, fetch, ud, d, try_again=True): 33*4882a593Smuzhiyun 34*4882a593Smuzhiyun # checkstatus discards parameters either way, we need to do this before adding the SAS 35*4882a593Smuzhiyun ud.url = ud.url.replace('az://','https://').split(';')[0] 36*4882a593Smuzhiyun 37*4882a593Smuzhiyun az_sas = d.getVar('AZ_SAS') 38*4882a593Smuzhiyun if az_sas and az_sas not in ud.url: 39*4882a593Smuzhiyun ud.url += az_sas 40*4882a593Smuzhiyun 41*4882a593Smuzhiyun return Wget.checkstatus(self, fetch, ud, d, try_again) 42*4882a593Smuzhiyun 43*4882a593Smuzhiyun # Override download method, include retries 44*4882a593Smuzhiyun def download(self, ud, d, retries=3): 45*4882a593Smuzhiyun """Fetch urls""" 46*4882a593Smuzhiyun 47*4882a593Smuzhiyun # If were reaching the account transaction limit we might be refused a connection, 48*4882a593Smuzhiyun # retrying allows us to avoid false negatives since the limit changes over time 49*4882a593Smuzhiyun fetchcmd = self.basecmd + ' --retry-connrefused --waitretry=5' 50*4882a593Smuzhiyun 51*4882a593Smuzhiyun # We need to provide a localpath to avoid wget using the SAS 52*4882a593Smuzhiyun # ud.localfile either has the downloadfilename or ud.path 53*4882a593Smuzhiyun localpath = os.path.join(d.getVar("DL_DIR"), ud.localfile) 54*4882a593Smuzhiyun bb.utils.mkdirhier(os.path.dirname(localpath)) 55*4882a593Smuzhiyun fetchcmd += " -O %s" % shlex.quote(localpath) 56*4882a593Smuzhiyun 57*4882a593Smuzhiyun 58*4882a593Smuzhiyun if ud.user and ud.pswd: 59*4882a593Smuzhiyun fetchcmd += " --user=%s --password=%s --auth-no-challenge" % (ud.user, ud.pswd) 60*4882a593Smuzhiyun 61*4882a593Smuzhiyun # Check if a Shared Access Signature was given and use it 62*4882a593Smuzhiyun az_sas = d.getVar('AZ_SAS') 63*4882a593Smuzhiyun 64*4882a593Smuzhiyun if az_sas: 65*4882a593Smuzhiyun azuri = '%s%s%s%s' % ('https://', ud.host, ud.path, az_sas) 66*4882a593Smuzhiyun else: 67*4882a593Smuzhiyun azuri = '%s%s%s' % ('https://', ud.host, ud.path) 68*4882a593Smuzhiyun 69*4882a593Smuzhiyun if os.path.exists(ud.localpath): 70*4882a593Smuzhiyun # file exists, but we didnt complete it.. trying again. 71*4882a593Smuzhiyun fetchcmd += d.expand(" -c -P ${DL_DIR} '%s'" % azuri) 72*4882a593Smuzhiyun else: 73*4882a593Smuzhiyun fetchcmd += d.expand(" -P ${DL_DIR} '%s'" % azuri) 74*4882a593Smuzhiyun 75*4882a593Smuzhiyun try: 76*4882a593Smuzhiyun self._runwget(ud, d, fetchcmd, False) 77*4882a593Smuzhiyun except FetchError as e: 78*4882a593Smuzhiyun # Azure fails on handshake sometimes when using wget after some stress, producing a 79*4882a593Smuzhiyun # FetchError from the fetcher, if the artifact exists retyring should succeed 80*4882a593Smuzhiyun if 'Unable to establish SSL connection' in str(e): 81*4882a593Smuzhiyun logger.debug2('Unable to establish SSL connection: Retries remaining: %s, Retrying...' % retries) 82*4882a593Smuzhiyun self.download(ud, d, retries -1) 83*4882a593Smuzhiyun 84*4882a593Smuzhiyun # Sanity check since wget can pretend it succeed when it didn't 85*4882a593Smuzhiyun # Also, this used to happen if sourceforge sent us to the mirror page 86*4882a593Smuzhiyun if not os.path.exists(ud.localpath): 87*4882a593Smuzhiyun raise FetchError("The fetch command returned success for url %s but %s doesn't exist?!" % (azuri, ud.localpath), azuri) 88*4882a593Smuzhiyun 89*4882a593Smuzhiyun if os.path.getsize(ud.localpath) == 0: 90*4882a593Smuzhiyun os.remove(ud.localpath) 91*4882a593Smuzhiyun raise FetchError("The fetch of %s resulted in a zero size file?! Deleting and failing since this isn't right." % (azuri), azuri) 92*4882a593Smuzhiyun 93*4882a593Smuzhiyun return True 94