1#!/bin/bash 2 3# Copyright (c) 2012 Wind River Systems, Inc. 4# 5# SPDX-License-Identifier: GPL-2.0-only 6# 7 8# Global vars 9cache_dir= 10confirm= 11fsym= 12total_deleted=0 13verbose= 14debug=0 15 16usage () { 17 cat << EOF 18Welcome to sstate cache management utilities. 19sstate-cache-management.sh <OPTION> 20 21Options: 22 -h, --help 23 Display this help and exit. 24 25 --cache-dir=<sstate cache dir> 26 Specify sstate cache directory, will use the environment 27 variable SSTATE_CACHE_DIR if it is not specified. 28 29 --extra-archs=<arch1>,<arch2>...<archn> 30 Specify list of architectures which should be tested, this list 31 will be extended with native arch, allarch and empty arch. The 32 script won't be trying to generate list of available archs from 33 AVAILTUNES in tune files. 34 35 --extra-layer=<layer1>,<layer2>...<layern> 36 Specify the layer which will be used for searching the archs, 37 it will search the meta and meta-* layers in the top dir by 38 default, and will search meta, meta-*, <layer1>, <layer2>, 39 ...<layern> when specified. Use "," as the separator. 40 41 This is useless for --stamps-dir or when --extra-archs is used. 42 43 -d, --remove-duplicated 44 Remove the duplicated sstate cache files of one package, only 45 the newest one will be kept. The duplicated sstate cache files 46 of one package must have the same arch, which means sstate cache 47 files with multiple archs are not considered duplicate. 48 49 Conflicts with --stamps-dir. 50 51 --stamps-dir=<dir1>,<dir2>...<dirn> 52 Specify the build directory's stamps directories, the sstate 53 cache file which IS USED by these build diretories will be KEPT, 54 other sstate cache files in cache-dir will be removed. Use "," 55 as the separator. For example: 56 --stamps-dir=build1/tmp/stamps,build2/tmp/stamps 57 58 Conflicts with --remove-duplicated. 59 60 -L, --follow-symlink 61 Remove both the symbol link and the destination file, default: no. 62 63 -y, --yes 64 Automatic yes to prompts; assume "yes" as answer to all prompts 65 and run non-interactively. 66 67 -v, --verbose 68 Explain what is being done. 69 70 -D, --debug 71 Show debug info, repeat for more debug info. 72 73EOF 74} 75 76if [ $# -lt 1 ]; then 77 usage 78 exit 0 79fi 80 81# Echo no files to remove 82no_files () { 83 echo No files to remove 84} 85 86# Echo nothing to do 87do_nothing () { 88 echo Nothing to do 89} 90 91# Read the input "y" 92read_confirm () { 93 echo "$total_deleted out of $total_files files will be removed! " 94 if [ "$confirm" != "y" ]; then 95 echo "Do you want to continue (y/n)? " 96 while read confirm; do 97 [ "$confirm" = "Y" -o "$confirm" = "y" -o "$confirm" = "n" \ 98 -o "$confirm" = "N" ] && break 99 echo "Invalid input \"$confirm\", please input 'y' or 'n': " 100 done 101 else 102 echo 103 fi 104} 105 106# Print error information and exit. 107echo_error () { 108 echo "ERROR: $1" >&2 109 exit 1 110} 111 112# Generate the remove list: 113# 114# * Add .done/.siginfo to the remove list 115# * Add destination of symlink to the remove list 116# 117# $1: output file, others: sstate cache file (.tar.zst) 118gen_rmlist (){ 119 local rmlist_file="$1" 120 shift 121 local files="$@" 122 for i in $files; do 123 echo $i >> $rmlist_file 124 # Add the ".siginfo" 125 if [ -e $i.siginfo ]; then 126 echo $i.siginfo >> $rmlist_file 127 fi 128 # Add the destination of symlink 129 if [ -L "$i" ]; then 130 if [ "$fsym" = "y" ]; then 131 dest="`readlink -e $i`" 132 if [ -n "$dest" ]; then 133 echo $dest >> $rmlist_file 134 # Remove the .siginfo when .tar.zst is removed 135 if [ -f "$dest.siginfo" ]; then 136 echo $dest.siginfo >> $rmlist_file 137 fi 138 fi 139 fi 140 # Add the ".tar.zst.done" and ".siginfo.done" (may exist in the future) 141 base_fn="${i##/*/}" 142 t_fn="$base_fn.done" 143 s_fn="$base_fn.siginfo.done" 144 for d in $t_fn $s_fn; do 145 if [ -f $cache_dir/$d ]; then 146 echo $cache_dir/$d >> $rmlist_file 147 fi 148 done 149 fi 150 done 151} 152 153# Remove the duplicated cache files for the pkg, keep the newest one 154remove_duplicated () { 155 156 local topdir 157 local oe_core_dir 158 local tunedirs 159 local all_archs 160 local all_machines 161 local ava_archs 162 local arch 163 local file_names 164 local sstate_files_list 165 local fn_tmp 166 local list_suffix=`mktemp` || exit 1 167 168 if [ -z "$extra_archs" ] ; then 169 # Find out the archs in all the layers 170 echo "Figuring out the archs in the layers ... " 171 oe_core_dir=$(dirname $(dirname $(readlink -e $0))) 172 topdir=$(dirname $oe_core_dir) 173 tunedirs="`find $topdir/meta* ${oe_core_dir}/meta* $layers -path '*/meta*/conf/machine/include' 2>/dev/null`" 174 [ -n "$tunedirs" ] || echo_error "Can't find the tune directory" 175 all_machines="`find $topdir/meta* ${oe_core_dir}/meta* $layers -path '*/meta*/conf/machine/*' -name '*.conf' 2>/dev/null | sed -e 's/.*\///' -e 's/.conf$//'`" 176 all_archs=`grep -r -h "^AVAILTUNES .*=" $tunedirs | sed -e 's/.*=//' -e 's/\"//g'` 177 fi 178 179 # Use the "_" to substitute "-", e.g., x86-64 to x86_64, but not for extra_archs which can be something like cortexa9t2-vfp-neon 180 # Sort to remove the duplicated ones 181 # Add allarch and builder arch (native) 182 builder_arch=$(uname -m) 183 all_archs="$(echo allarch $all_archs $all_machines $builder_arch \ 184 | sed -e 's/-/_/g' -e 's/ /\n/g' | sort -u) $extra_archs" 185 echo "Done" 186 187 # Total number of files including sstate-, .siginfo and .done files 188 total_files=`find $cache_dir -name 'sstate*' | wc -l` 189 # Save all the sstate files in a file 190 sstate_files_list=`mktemp` || exit 1 191 find $cache_dir -iname 'sstate:*:*:*:*:*:*:*.tar.zst*' >$sstate_files_list 192 193 echo "Figuring out the suffixes in the sstate cache dir ... " 194 sstate_suffixes="`sed 's%.*/sstate:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^_]*_\([^:]*\)\.tar\.zst.*%\1%g' $sstate_files_list | sort -u`" 195 echo "Done" 196 echo "The following suffixes have been found in the cache dir:" 197 echo $sstate_suffixes 198 199 echo "Figuring out the archs in the sstate cache dir ... " 200 # Using this SSTATE_PKGSPEC definition it's 6th colon separated field 201 # SSTATE_PKGSPEC = "sstate:${PN}:${PACKAGE_ARCH}${TARGET_VENDOR}-${TARGET_OS}:${PV}:${PR}:${SSTATE_PKGARCH}:${SSTATE_VERSION}:" 202 for arch in $all_archs; do 203 grep -q ".*/sstate:[^:]*:[^:]*:[^:]*:[^:]*:$arch:[^:]*:[^:]*\.tar\.zst$" $sstate_files_list 204 [ $? -eq 0 ] && ava_archs="$ava_archs $arch" 205 # ${builder_arch}_$arch used by toolchain sstate 206 grep -q ".*/sstate:[^:]*:[^:]*:[^:]*:[^:]*:${builder_arch}_$arch:[^:]*:[^:]*\.tar\.zst$" $sstate_files_list 207 [ $? -eq 0 ] && ava_archs="$ava_archs ${builder_arch}_$arch" 208 done 209 echo "Done" 210 echo "The following archs have been found in the cache dir:" 211 echo $ava_archs 212 echo "" 213 214 # Save the file list which needs to be removed 215 local remove_listdir=`mktemp -d` || exit 1 216 for suffix in $sstate_suffixes; do 217 if [ "$suffix" = "populate_lic" ] ; then 218 echo "Skipping populate_lic, because removing duplicates doesn't work correctly for them (use --stamps-dir instead)" 219 continue 220 fi 221 # Total number of files including .siginfo and .done files 222 total_files_suffix=`grep ".*/sstate:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^:_]*_$suffix\.tar\.zst.*" $sstate_files_list | wc -l 2>/dev/null` 223 total_archive_suffix=`grep ".*/sstate:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^:_]*_$suffix\.tar\.zst$" $sstate_files_list | wc -l 2>/dev/null` 224 # Save the file list to a file, some suffix's file may not exist 225 grep ".*/sstate:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^:_]*_$suffix\.tar\.zst.*" $sstate_files_list >$list_suffix 2>/dev/null 226 local deleted_archives=0 227 local deleted_files=0 228 for ext in tar.zst tar.zst.siginfo tar.zst.done; do 229 echo "Figuring out the sstate:xxx_$suffix.$ext ... " 230 # Uniq BPNs 231 file_names=`for arch in $ava_archs ""; do 232 sed -ne "s%.*/sstate:\([^:]*\):[^:]*:[^:]*:[^:]*:$arch:[^:]*:[^:]*\.${ext}$%\1%p" $list_suffix 233 done | sort -u` 234 235 fn_tmp=`mktemp` || exit 1 236 rm_list="$remove_listdir/sstate:xxx_$suffix" 237 for fn in $file_names; do 238 [ -z "$verbose" ] || echo "Analyzing sstate:$fn-xxx_$suffix.${ext}" 239 for arch in $ava_archs ""; do 240 grep -h ".*/sstate:$fn:[^:]*:[^:]*:[^:]*:$arch:[^:]*:[^:]*\.${ext}$" $list_suffix >$fn_tmp 241 if [ -s $fn_tmp ] ; then 242 [ $debug -gt 1 ] && echo "Available files for $fn-$arch- with suffix $suffix.${ext}:" && cat $fn_tmp 243 # Use the modification time 244 to_del=$(ls -t $(cat $fn_tmp) | sed -n '1!p') 245 [ $debug -gt 2 ] && echo "Considering to delete: $to_del" 246 # The sstate file which is downloaded from the SSTATE_MIRROR is 247 # put in SSTATE_DIR, and there is a symlink in SSTATE_DIR/??/ to 248 # it, so filter it out from the remove list if it should not be 249 # removed. 250 to_keep=$(ls -t $(cat $fn_tmp) | sed -n '1p') 251 [ $debug -gt 2 ] && echo "Considering to keep: $to_keep" 252 for k in $to_keep; do 253 if [ -L "$k" ]; then 254 # The symlink's destination 255 k_dest="`readlink -e $k`" 256 # Maybe it is the one in cache_dir 257 k_maybe="$cache_dir/${k##/*/}" 258 # Remove it from the remove list if they are the same. 259 if [ "$k_dest" = "$k_maybe" ]; then 260 to_del="`echo $to_del | sed 's#'\"$k_maybe\"'##g'`" 261 fi 262 fi 263 done 264 rm -f $fn_tmp 265 [ $debug -gt 2 ] && echo "Decided to delete: $to_del" 266 gen_rmlist $rm_list.$ext "$to_del" 267 fi 268 done 269 done 270 done 271 deleted_archives=`cat $rm_list.* 2>/dev/null | grep "\.tar\.zst$" | wc -l` 272 deleted_files=`cat $rm_list.* 2>/dev/null | wc -l` 273 [ "$deleted_files" -gt 0 -a $debug -gt 0 ] && cat $rm_list.* 274 echo "($deleted_archives out of $total_archives_suffix .tar.zst files for $suffix suffix will be removed or $deleted_files out of $total_files_suffix when counting also .siginfo and .done files)" 275 let total_deleted=$total_deleted+$deleted_files 276 done 277 deleted_archives=0 278 rm_old_list=$remove_listdir/sstate-old-filenames 279 find $cache_dir -name 'sstate-*.tar.zst' >$rm_old_list 280 [ -s "$rm_old_list" ] && deleted_archives=`cat $rm_old_list | grep "\.tar\.zst$" | wc -l` 281 [ -s "$rm_old_list" ] && deleted_files=`cat $rm_old_list | wc -l` 282 [ -s "$rm_old_list" -a $debug -gt 0 ] && cat $rm_old_list 283 echo "($deleted_archives or .tar.zst files with old sstate-* filenames will be removed or $deleted_files when counting also .siginfo and .done files)" 284 let total_deleted=$total_deleted+$deleted_files 285 286 rm -f $list_suffix 287 rm -f $sstate_files_list 288 if [ $total_deleted -gt 0 ]; then 289 read_confirm 290 if [ "$confirm" = "y" -o "$confirm" = "Y" ]; then 291 for list in `ls $remove_listdir/`; do 292 echo "Removing $list.tar.zst archive (`cat $remove_listdir/$list | wc -w` files) ... " 293 # Remove them one by one to avoid the argument list too long error 294 for i in `cat $remove_listdir/$list`; do 295 rm -f $verbose $i 296 done 297 echo "Done" 298 done 299 echo "$total_deleted files have been removed!" 300 else 301 do_nothing 302 fi 303 else 304 no_files 305 fi 306 [ -d $remove_listdir ] && rm -fr $remove_listdir 307} 308 309# Remove the sstate file by stamps dir, the file not used by the stamps dir 310# will be removed. 311rm_by_stamps (){ 312 313 local cache_list=`mktemp` || exit 1 314 local keep_list=`mktemp` || exit 1 315 local rm_list=`mktemp` || exit 1 316 local sums 317 local all_sums 318 319 # Total number of files including sstate-, .siginfo and .done files 320 total_files=`find $cache_dir -type f -name 'sstate*' | wc -l` 321 # Save all the state file list to a file 322 find $cache_dir -type f -name 'sstate*' | sort -u -o $cache_list 323 324 echo "Figuring out the suffixes in the sstate cache dir ... " 325 local sstate_suffixes="`sed 's%.*/sstate:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^_]*_\([^:]*\)\.tar\.zst.*%\1%g' $cache_list | sort -u`" 326 echo "Done" 327 echo "The following suffixes have been found in the cache dir:" 328 echo $sstate_suffixes 329 330 # Figure out all the md5sums in the stamps dir. 331 echo "Figuring out all the md5sums in stamps dir ... " 332 for i in $sstate_suffixes; do 333 # There is no "\.sigdata" but "_setcene" when it is mirrored 334 # from the SSTATE_MIRRORS, use them to figure out the sum. 335 sums=`find $stamps -maxdepth 3 -name "*.do_$i.*" \ 336 -o -name "*.do_${i}_setscene.*" | \ 337 sed -ne 's#.*_setscene\.##p' -e 's#.*\.sigdata\.##p' | \ 338 sed -e 's#\..*##' | sort -u` 339 all_sums="$all_sums $sums" 340 done 341 echo "Done" 342 343 echo "Figuring out the files which will be removed ... " 344 for i in $all_sums; do 345 grep ".*/sstate:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:${i}_.*" $cache_list >>$keep_list 346 done 347 echo "Done" 348 349 if [ -s $keep_list ]; then 350 sort -u $keep_list -o $keep_list 351 to_del=`comm -1 -3 $keep_list $cache_list` 352 gen_rmlist $rm_list "$to_del" 353 let total_deleted=`cat $rm_list | sort -u | wc -w` 354 if [ $total_deleted -gt 0 ]; then 355 [ $debug -gt 0 ] && cat $rm_list | sort -u 356 read_confirm 357 if [ "$confirm" = "y" -o "$confirm" = "Y" ]; then 358 echo "Removing sstate cache files ... ($total_deleted files)" 359 # Remove them one by one to avoid the argument list too long error 360 for i in `cat $rm_list | sort -u`; do 361 rm -f $verbose $i 362 done 363 echo "$total_deleted files have been removed" 364 else 365 do_nothing 366 fi 367 else 368 no_files 369 fi 370 else 371 echo_error "All files in cache dir will be removed! Abort!" 372 fi 373 374 rm -f $cache_list 375 rm -f $keep_list 376 rm -f $rm_list 377} 378 379# Parse arguments 380while [ -n "$1" ]; do 381 case $1 in 382 --cache-dir=*) 383 cache_dir=`echo $1 | sed -e 's#^--cache-dir=##' | xargs readlink -e` 384 [ -d "$cache_dir" ] || echo_error "Invalid argument to --cache-dir" 385 shift 386 ;; 387 --remove-duplicated|-d) 388 rm_duplicated="y" 389 shift 390 ;; 391 --yes|-y) 392 confirm="y" 393 shift 394 ;; 395 --follow-symlink|-L) 396 fsym="y" 397 shift 398 ;; 399 --extra-archs=*) 400 extra_archs=`echo $1 | sed -e 's#^--extra-archs=##' -e 's#,# #g'` 401 [ -n "$extra_archs" ] || echo_error "Invalid extra arch parameter" 402 shift 403 ;; 404 --extra-layer=*) 405 extra_layers=`echo $1 | sed -e 's#^--extra-layer=##' -e 's#,# #g'` 406 [ -n "$extra_layers" ] || echo_error "Invalid extra layer parameter" 407 for i in $extra_layers; do 408 l=`readlink -e $i` 409 if [ -d "$l" ]; then 410 layers="$layers $l" 411 else 412 echo_error "Can't find layer $i" 413 fi 414 done 415 shift 416 ;; 417 --stamps-dir=*) 418 stamps=`echo $1 | sed -e 's#^--stamps-dir=##' -e 's#,# #g'` 419 [ -n "$stamps" ] || echo_error "Invalid stamps dir $i" 420 for i in $stamps; do 421 [ -d "$i" ] || echo_error "Invalid stamps dir $i" 422 done 423 shift 424 ;; 425 --verbose|-v) 426 verbose="-v" 427 shift 428 ;; 429 --debug|-D) 430 debug=`expr $debug + 1` 431 echo "Debug level $debug" 432 shift 433 ;; 434 --help|-h) 435 usage 436 exit 0 437 ;; 438 *) 439 echo "Invalid arguments $*" 440 echo_error "Try 'sstate-cache-management.sh -h' for more information." 441 ;; 442 esac 443done 444 445# sstate cache directory, use environment variable SSTATE_CACHE_DIR 446# if it was not specified, otherwise, error. 447[ -n "$cache_dir" ] || cache_dir=$SSTATE_CACHE_DIR 448[ -n "$cache_dir" ] || echo_error "No cache dir found!" 449[ -d "$cache_dir" ] || echo_error "Invalid cache directory \"$cache_dir\"" 450 451[ -n "$rm_duplicated" -a -n "$stamps" ] && \ 452 echo_error "Can not use both --remove-duplicated and --stamps-dir" 453 454[ "$rm_duplicated" = "y" ] && remove_duplicated 455[ -n "$stamps" ] && rm_by_stamps 456[ -z "$rm_duplicated" -a -z "$stamps" ] && \ 457 echo "What do you want to do?" 458exit 0 459