1#!/usr/bin/env bash 2 3# This script is a wrapper to the other download backends. 4# Its role is to ensure atomicity when saving downloaded files 5# back to BR2_DL_DIR, and not clutter BR2_DL_DIR with partial, 6# failed downloads. 7 8# To avoid cluttering BR2_DL_DIR, we download to a trashable 9# location, namely in $(BUILD_DIR). 10# Then, we move the downloaded file to a temporary file in the 11# same directory as the final output file. 12# This allows us to finally atomically rename it to its final 13# name. 14# If anything goes wrong, we just remove all the temporaries 15# created so far. 16 17# We want to catch any unexpected failure, and exit immediately. 18set -e 19 20export BR_BACKEND_DL_GETOPTS=":hc:d:o:n:N:H:ru:qf:e" 21 22check_kgithub() { 23 if wget -q --delete-after \ 24 https://raw.kgithub.com/git/git/master/README.md; then 25 if git ls-remote https://kgithub.com/git/git &>/dev/null; then 26 return 0 27 fi 28 fi 29 30 if grep -q kgithub /etc/hosts; then 31 echo "Oops! The kgithub is down!" 32 return 1 33 fi 34 35 echo "Your DNS doesn't support kgithub.com" 36 echo "Please modify it:" 37 echo "sudo sed -i '\$a 43.154.68.204\tkgithub.com' /etc/hosts" 38 echo "sudo sed -i '\$a 43.155.83.75\traw.kgithub.com objects.githubusercontent.kgithub.com' /etc/hosts" 39 return 1 40} 41 42main() { 43 local OPT OPTARG 44 local backend output hfile recurse quiet rc use_kgithub 45 local -a uris 46 47 # Parse our options; anything after '--' is for the backend 48 while getopts ":c:d:D:o:n:N:H:rf:u:q" OPT; do 49 case "${OPT}" in 50 c) cset="${OPTARG}";; 51 d) dl_dir="${OPTARG}";; 52 D) old_dl_dir="${OPTARG}";; 53 o) output="${OPTARG}";; 54 n) raw_base_name="${OPTARG}";; 55 N) base_name="${OPTARG}";; 56 H) hfile="${OPTARG}";; 57 r) recurse="-r";; 58 f) filename="${OPTARG}";; 59 u) uris+=( "${OPTARG}" );; 60 q) quiet="-q";; 61 :) error "option '%s' expects a mandatory argument\n" "${OPTARG}";; 62 \?) error "unknown option '%s'\n" "${OPTARG}";; 63 esac 64 done 65 66 # Forget our options, and keep only those for the backend 67 shift $((OPTIND-1)) 68 69 if [ -z "${output}" ]; then 70 error "no output specified, use -o\n" 71 fi 72 73 # Legacy handling: check if the file already exists in the global 74 # download directory. If it does, hard-link it. If it turns out it 75 # was an incorrect download, we'd still check it below anyway. 76 # If we can neither link nor copy, fallback to doing a download. 77 # NOTE! This is not atomic, is subject to TOCTTOU, but the whole 78 # dl-wrapper runs under an flock, so we're safe. 79 if [ ! -e "${output}" -a -e "${old_dl_dir}/${filename}" ]; then 80 ln "${old_dl_dir}/${filename}" "${output}" || \ 81 cp "${old_dl_dir}/${filename}" "${output}" || \ 82 true 83 fi 84 85 # If the output file already exists and: 86 # - there's no .hash file: do not download it again and exit promptly 87 # - matches all its hashes: do not download it again and exit promptly 88 # - fails at least one of its hashes: force a re-download 89 # - there's no hash (but a .hash file): consider it a hard error 90 if [ -e "${output}" ]; then 91 if support/download/check-hash ${quiet} "${hfile}" "${output}" "${output##*/}"; then 92 exit 0 93 elif [ ${?} -ne 2 ]; then 94 # Do not remove the file, otherwise it might get re-downloaded 95 # from a later location (i.e. primary -> upstream -> mirror). 96 # Do not print a message, check-hash already did. 97 exit 1 98 fi 99 rm -f "${output}" 100 warn "Re-downloading '%s'...\n" "${output##*/}" 101 fi 102 103 if [ -z "$BR2_NO_KGITHUB" ] && echo "${uris[@]}" | grep -wq github.com; then 104 if ! git ls-remote https://github.com/git/git &>/dev/null; then 105 echo -e "\e[35m" 106 107 echo "Unable to access github.com! Trying kgithub now..." 108 if check_kgithub; then 109 echo "Using kgithub instead..." 110 echo "Setup a VPN or export BR2_NO_KGITHUB=1 to disable this." 111 112 use_kgithub=1 113 fi 114 115 echo -e "\e[0m" 116 fi 117 fi 118 119 # Look through all the uris that we were given to download the package 120 # source 121 download_and_check=0 122 rc=1 123 for uri in "${uris[@]}"; do 124 backend_urlencode="${uri%%+*}" 125 backend="${backend_urlencode%|*}" 126 case "${backend}" in 127 git|svn|cvs|bzr|file|scp|hg) ;; 128 *) backend="wget" ;; 129 esac 130 uri=${uri#*+} 131 132 urlencode=${backend_urlencode#*|} 133 # urlencode must be "urlencode" 134 [ "${urlencode}" != "urlencode" ] && urlencode="" 135 136 if [ "$use_kgithub" ]; then 137 uri=${uri/\/\/github.com\//\/\/kgithub.com\/} 138 uri=${uri/\/\/raw.githubusercontent.com\//\/\/raw.kgithub.com\/} 139 fi 140 141 # tmpd is a temporary directory in which backends may store 142 # intermediate by-products of the download. 143 # tmpf is the file in which the backends should put the downloaded 144 # content. 145 # tmpd is located in $(BUILD_DIR), so as not to clutter the (precious) 146 # $(BR2_DL_DIR) 147 # We let the backends create tmpf, so they are able to set whatever 148 # permission bits they want (although we're only really interested in 149 # the executable bit.) 150 tmpd="$(mktemp -d "${BUILD_DIR}/.${output##*/}.XXXXXX")" 151 tmpf="${tmpd}/output" 152 153 # Helpers expect to run in a directory that is *really* trashable, so 154 # they are free to create whatever files and/or sub-dirs they might need. 155 # Doing the 'cd' here rather than in all backends is easier. 156 cd "${tmpd}" 157 158 # If the backend fails, we can just remove the content of the temporary 159 # directory to remove all the cruft it may have left behind, and try 160 # the next URI until it succeeds. Once out of URI to try, we need to 161 # cleanup and exit. 162 if ! "${OLDPWD}/support/download/${backend}" \ 163 $([ -n "${urlencode}" ] && printf %s '-e') \ 164 -c "${cset}" \ 165 -d "${dl_dir}" \ 166 -n "${raw_base_name}" \ 167 -N "${base_name}" \ 168 -f "${filename}" \ 169 -u "${uri}" \ 170 -o "${tmpf}" \ 171 ${quiet} ${recurse} -- "${@}" 172 then 173 # cd back to keep path coherence 174 cd "${OLDPWD}" 175 rm -rf "${tmpd}" 176 continue 177 fi 178 179 # cd back to free the temp-dir, so we can remove it later 180 cd "${OLDPWD}" 181 182 # Check if the downloaded file is sane, and matches the stored hashes 183 # for that file 184 if support/download/check-hash ${quiet} "${hfile}" "${tmpf}" "${output##*/}"; then 185 rc=0 186 else 187 if [ ${?} -ne 3 ]; then 188 rm -rf "${tmpd}" 189 continue 190 fi 191 192 # the hash file exists and there was no hash to check the file 193 # against 194 rc=1 195 fi 196 download_and_check=1 197 break 198 done 199 200 # We tried every URI possible, none seems to work or to check against the 201 # available hash. *ABORT MISSION* 202 if [ "${download_and_check}" -eq 0 ]; then 203 rm -rf "${tmpd}" 204 exit 1 205 fi 206 207 # tmp_output is in the same directory as the final output, so we can 208 # later move it atomically. 209 tmp_output="$(mktemp "${output}.XXXXXX")" 210 211 # 'mktemp' creates files with 'go=-rwx', so the files are not accessible 212 # to users other than the one doing the download (and root, of course). 213 # This can be problematic when a shared BR2_DL_DIR is used by different 214 # users (e.g. on a build server), where all users may write to the shared 215 # location, since other users would not be allowed to read the files 216 # another user downloaded. 217 # So, we restore the 'go' access rights to a more sensible value, while 218 # still abiding by the current user's umask. We must do that before the 219 # final 'mv', so just do it now. 220 # Some backends (cp and scp) may create executable files, so we need to 221 # carry the executable bit if needed. 222 [ -x "${tmpf}" ] && new_mode=755 || new_mode=644 223 new_mode=$(printf "%04o" $((0${new_mode} & ~0$(umask)))) 224 chmod ${new_mode} "${tmp_output}" 225 226 # We must *not* unlink tmp_output, otherwise there is a small window 227 # during which another download process may create the same tmp_output 228 # name (very, very unlikely; but not impossible.) 229 # Using 'cp' is not reliable, since 'cp' may unlink the destination file 230 # if it is unable to open it with O_WRONLY|O_TRUNC; see: 231 # http://pubs.opengroup.org/onlinepubs/9699919799/utilities/cp.html 232 # Since the destination filesystem can be anything, it might not support 233 # O_TRUNC, so 'cp' would unlink it first. 234 # Use 'cat' and append-redirection '>>' to save to the final location, 235 # since that is the only way we can be 100% sure of the behaviour. 236 if ! cat "${tmpf}" >>"${tmp_output}"; then 237 rm -rf "${tmpd}" "${tmp_output}" 238 exit 1 239 fi 240 rm -rf "${tmpd}" 241 242 # tmp_output and output are on the same filesystem, so POSIX guarantees 243 # that 'mv' is atomic, because it then uses rename() that POSIX mandates 244 # to be atomic, see: 245 # http://pubs.opengroup.org/onlinepubs/9699919799/functions/rename.html 246 if ! mv -f "${tmp_output}" "${output}"; then 247 rm -f "${tmp_output}" 248 exit 1 249 fi 250 251 return ${rc} 252} 253 254trace() { local msg="${1}"; shift; printf "%s: ${msg}" "${my_name}" "${@}"; } 255warn() { trace "${@}" >&2; } 256errorN() { local ret="${1}"; shift; warn "${@}"; exit ${ret}; } 257error() { errorN 1 "${@}"; } 258 259my_name="${0##*/}" 260main "${@}" 261