1#!/usr/bin/env bash 2 3# NOTE: if the output of this backend has to change (e.g. we change what gets 4# included in the archive (e.g. LFS), or we change the format of the archive 5# (e.g. tar options, compression ratio or method)), we MUST update the format 6# version in the variable BR_FMT_VERSION_git, in package/pkg-download.mk. 7 8# We want to catch any unexpected failure, and exit immediately 9set -E 10 11# Download helper for git, to be called from the download wrapper script 12# 13# Options: 14# -q Be quiet. 15# -r Clone and archive sub-modules. 16# -o FILE Generate archive in FILE. 17# -u URI Clone from repository at URI. 18# -c CSET Use changeset CSET. 19# -n NAME Use basename NAME. 20# 21# Environment: 22# GIT : the git command to call 23 24. "${0%/*}/helpers" 25 26# Save our path and options in case we need to call ourselves again 27myname="${0}" 28declare -a OPTS=("${@}") 29 30# This function is called when an error occurs. Its job is to attempt a 31# clone from scratch (only once!) in case the git tree is borked, or in 32# case an unexpected and unsupported situation arises with submodules 33# or uncommitted stuff (e.g. if the user manually mucked around in the 34# git cache). 35_on_error() { 36 local ret=${?} 37 38 printf "Detected a corrupted git cache.\n" >&2 39 if ${BR_GIT_BACKEND_FIRST_FAULT:-false}; then 40 printf "This is the second time in a row; bailing out\n" >&2 41 exit ${ret} 42 fi 43 export BR_GIT_BACKEND_FIRST_FAULT=true 44 45 printf "Removing it and starting afresh.\n" >&2 46 47 popd >/dev/null 48 rm -rf "${git_cache}" 49 50 exec "${myname}" "${OPTS[@]}" || exit ${ret} 51} 52 53quiet= 54recurse=0 55while getopts "${BR_BACKEND_DL_GETOPTS}" OPT; do 56 case "${OPT}" in 57 q) quiet=-q; exec >/dev/null;; 58 r) recurse=1;; 59 o) output="${OPTARG}";; 60 u) uri="${OPTARG}";; 61 c) cset="${OPTARG}";; 62 d) dl_dir="${OPTARG}";; 63 n) basename="${OPTARG}";; 64 :) printf "option '%s' expects a mandatory argument\n" "${OPTARG}"; exit 1;; 65 \?) printf "unknown option '%s'\n" "${OPTARG}" >&2; exit 1;; 66 esac 67done 68 69shift $((OPTIND-1)) # Get rid of our options 70 71# Create and cd into the directory that will contain the local git cache 72git_cache="${dl_dir}/git" 73mkdir -p "${git_cache}" 74pushd "${git_cache}" >/dev/null 75 76# Any error now should try to recover 77trap _on_error ERR 78 79# Caller needs to single-quote its arguments to prevent them from 80# being expanded a second time (in case there are spaces in them) 81_git() { 82 if [ -z "${quiet}" ]; then 83 printf '%s ' GIT_DIR="${git_cache}/.git" ${GIT} "${@}"; printf '\n' 84 fi 85 _plain_git "$@" 86} 87# Note: please keep command below aligned with what is printed above 88_plain_git() { 89 eval GIT_DIR="${git_cache}/.git" ${GIT} "${@}" 90} 91 92# Create a warning file, that the user should not use the git cache. 93# It's ours. Our precious. 94cat <<-_EOF_ >"${dl_dir}/git.readme" 95 IMPORTANT NOTE! 96 97 The git tree located in this directory is for the exclusive use 98 by Buildroot, which uses it as a local cache to reduce bandwidth 99 usage. 100 101 Buildroot *will* trash any changes in that tree whenever it needs 102 to use it. Buildroot may even remove it in case it detects the 103 repository may have been damaged or corrupted. 104 105 Do *not* work in that directory; your changes will eventually get 106 lost. Do *not* even use it as a remote, or as the source for new 107 worktrees; your commits will eventually get lost. 108_EOF_ 109 110# Initialise a repository in the git cache. If the repository already 111# existed, this is a noop, unless the repository was broken, in which 112# case this magically restores it to working conditions. In the latter 113# case, we might be missing blobs, but that's not a problem: we'll 114# fetch what we need later anyway. 115# 116# We can still go through the wrapper, because 'init' does not use the 117# path pointed to by GIT_DIR, but really uses the directory passed as 118# argument. 119_git init . 120 121# Ensure the repo has an origin (in case a previous run was killed). 122if ! _plain_git remote |grep -q -E '^origin$'; then 123 _git remote add origin "'${uri}'" 124fi 125 126_git remote set-url origin "'${uri}'" 127 128printf "Fetching all references\n" 129_git fetch origin 130_git fetch origin -t 131 132# Try to get the special refs exposed by some forges (pull-requests for 133# github, changes for gerrit...). There is no easy way to know whether 134# the cset the user passed us is such a special ref or a tag or a sha1 135# or whatever else. We'll eventually fail at checking out that cset, 136# below, if there is an issue anyway. Since most of the cset we're gonna 137# have to clone are not such special refs, consign the output to oblivion 138# so as not to alarm unsuspecting users, but still trace it as a warning. 139if ! _git fetch origin "'${cset}:${cset}'" >/dev/null 2>&1; then 140 printf "Could not fetch special ref '%s'; assuming it is not special.\n" "${cset}" 141fi 142 143# Check that the changeset does exist. If it does not, re-cloning from 144# scratch won't help, so we don't want to trash the repository for a 145# missing commit. We just exit without going through the ERR trap. 146if ! _git rev-parse --quiet --verify "'${cset}^{commit}'" >/dev/null 2>&1; then 147 printf "Commit '%s' does not exist in this repository.\n" "${cset}" 148 exit 1 149fi 150 151# The new cset we want to checkout might have different submodules, or 152# have sub-dirs converted to/from a submodule. So we would need to 153# deregister _current_ submodules before we checkout. 154# 155# Using "git submodule deinit --all" would remove all the files for 156# all submodules, including the corresponding .git files or directories. 157# However, it was only introduced with git-1.8.3, which is too recent 158# for some enterprise-grade distros. 159# 160# So, we fall-back to just removing all submodules directories. We do 161# not need to be recursive, as removing a submodule will de-facto remove 162# its own submodules. 163# 164# For recent git versions, the repository for submodules is stored 165# inside the repository of the super repository, so the following will 166# only remove the working copies of submodules, effectively caching the 167# submodules. 168# 169# For older versions however, the repository is stored in the .git/ of 170# the submodule directory, so the following will effectively remove the 171# the working copy as well as the repository, which means submodules 172# will not be cached for older versions. 173# 174cmd='printf "Deregistering submodule \"%s\"\n" "${path}" && cd .. && rm -rf "${path##*/}"' 175_git submodule --quiet foreach "'${cmd}'" 176 177# Checkout the required changeset, so that we can update the required 178# submodules. 179_git checkout -f -q "'${cset}'" 180 181# Get rid of now-untracked directories (in case a git operation was 182# interrupted in a previous run, or to get rid of empty directories 183# that were parents of submodules removed above). 184_git clean -ffdx 185 186# Get date of commit to generate a reproducible archive. 187# %ci is ISO 8601, so it's fully qualified, with TZ and all. 188date="$( _plain_git log -1 --pretty=format:%ci )" 189 190# There might be submodules, so fetch them. 191if [ ${recurse} -eq 1 ]; then 192 _git submodule update --init --recursive 193 194 # Older versions of git will store the absolute path of the git tree 195 # in the .git of submodules, while newer versions just use relative 196 # paths. Detect and fix the older variants to use relative paths, so 197 # that the archives are reproducible across a wider range of git 198 # versions. However, we can't do that if git is too old and uses 199 # full repositories for submodules. 200 cmd='printf "%s\n" "${path}/"' 201 for module_dir in $( _plain_git submodule --quiet foreach "'${cmd}'" ); do 202 [ -f "${module_dir}/.git" ] || continue 203 relative_dir="$( sed -r -e 's,/+,/,g; s,[^/]+/,../,g' <<<"${module_dir}" )" 204 sed -r -i -e "s:^gitdir\: $(pwd)/:gitdir\: "${relative_dir}":" "${module_dir}/.git" 205 done 206fi 207 208popd >/dev/null 209 210# Generate the archive. 211# We do not want the .git dir; we keep other .git files, in case they are the 212# only files in their directory. 213# The .git dir would generate non reproducible tarballs as it depends on 214# the state of the remote server. It also would generate large tarballs 215# (gigabytes for some linux trees) when a full clone took place. 216mk_tar_gz "${git_cache}" "${basename}" "${date}" "${output}" ".git/*" 217