| 1 |
#!/bin/bash |
| 2 |
|
| 3 |
if [ ! "x$0" = "x${0#-}" ] |
| 4 |
then |
| 5 |
echo "Don't source me, just run!" 1>&2 |
| 6 |
return |
| 7 |
fi |
| 8 |
. ${0%/*}/libdans.sh |
| 9 |
|
| 10 |
# Load global defaults (vars: vo, archive, LFC_*, SRM_*) |
| 11 |
loadDefaults $0 |
| 12 |
|
| 13 |
|
| 14 |
DEBUG=0 |
| 15 |
QUIET=0 |
| 16 |
KEEPGOING=0 |
| 17 |
START_INDEX=1 |
| 18 |
END_INDEX=0 |
| 19 |
USAGE="\ |
| 20 |
$0 - verify the contents of one or multiple tar.gz files |
| 21 |
Usage: ./check-file [-d|--debug] [-k|--keepgoing] [file|lfn|surl] [[file2|lfn2|surl2] ...] |
| 22 |
Where: |
| 23 |
--keepgoing tells $0 to keep going after an error |
| 24 |
file specifies the full path to a local file OR |
| 25 |
lfn specifies a Logical File Name on the LFC OR |
| 26 |
surl specifies a Storage URL (srm:// or gsiftp://) |
| 27 |
" |
| 28 |
|
| 29 |
# Parse commandline parameters |
| 30 |
while [ $# -gt 0 ] |
| 31 |
do |
| 32 |
case "$1" in |
| 33 |
(-h|--help) echo "${USAGE}" |
| 34 |
exit 0 |
| 35 |
;; |
| 36 |
(-d|--debug) let DEBUG+=1 |
| 37 |
;; |
| 38 |
(-k|--keepgoing) KEEPGOING=1 |
| 39 |
;; |
| 40 |
(-*) echo "Invalid option: $1" |
| 41 |
exit 1 |
| 42 |
;; |
| 43 |
(*) break |
| 44 |
;; |
| 45 |
esac |
| 46 |
shift |
| 47 |
done |
| 48 |
|
| 49 |
if [ $# -lt 1 ] |
| 50 |
then |
| 51 |
abort 1 "${USAGE}" |
| 52 |
fi |
| 53 |
|
| 54 |
# Loop over all files|lfns|surls |
| 55 |
for file |
| 56 |
do |
| 57 |
do_get=0 |
| 58 |
if [ "${file}" != "${file#lfn://}" ] |
| 59 |
then |
| 60 |
do_get=1 |
| 61 |
fi |
| 62 |
if [ "${file}" != "${file#srm://}" ] |
| 63 |
then |
| 64 |
do_get=1 |
| 65 |
fi |
| 66 |
if [ "${file}" != "${file#gsiftp://}" ] |
| 67 |
then |
| 68 |
do_get=1 |
| 69 |
fi |
| 70 |
|
| 71 |
if [ ${do_get} -ne 0 ] |
| 72 |
then |
| 73 |
info "Retrieving file ${file}" |
| 74 |
lcg-cp $SRM_TIMEOUTS "${file}" file://$PWD/"${file##*/}" |
| 75 |
if [ $? -ne 0 ] |
| 76 |
then |
| 77 |
warn "First attempt failed, sleeping 30 seconds and retrying..." |
| 78 |
sleep 30 |
| 79 |
lcg-cp $SRM_TIMEOUTS "${file}" file://$PWD/"${file##*/}" |
| 80 |
fi |
| 81 |
if [ $? -ne 0 ] |
| 82 |
then |
| 83 |
warn "Second attempt failed, sleeping 90 seconds and retrying..." |
| 84 |
sleep 90 |
| 85 |
date +"%Y/%m/%d-%H:%M:%S start lcg-cp -v -v $SRM_TIMEOUTS" 1>&2 |
| 86 |
lcg-cp -v -v $SRM_TIMEOUTS "${file}" file://$PWD/"${file##*/}" |
| 87 |
fi |
| 88 |
if [ $? -ne 0 ] |
| 89 |
then |
| 90 |
abort 3 "Error retrieving ${file}" |
| 91 |
fi |
| 92 |
|
| 93 |
file="${file##*/}" |
| 94 |
fi |
| 95 |
|
| 96 |
# the file should be a .tar.gz file, extract the original archive name from it |
| 97 |
base=`file "${file}" | sed -n 's/.*compressed data, was \"\(.*\)\",.*/\1/p'` |
| 98 |
archive="${base%%-[0-9]*}" |
| 99 |
|
| 100 |
debug "file=${file} base=${base} archive=${archive}" |
| 101 |
|
| 102 |
CHECKSUM_DIR="${HOME}/dans/${archive}/checksums" |
| 103 |
|
| 104 |
# fix numbering issue in RACM tar balls |
| 105 |
if [ "x${archive}" = "xRACM" ] |
| 106 |
then |
| 107 |
index=${base#${archive}-} |
| 108 |
index=${index%.tar} |
| 109 |
if [ ${#index} -lt 4 ] |
| 110 |
then |
| 111 |
base="${archive}-0${index}.tar" |
| 112 |
fi |
| 113 |
fi |
| 114 |
|
| 115 |
# extract the tarball in a separate directory, for easy cleanup later |
| 116 |
mkdir -p workspace |
| 117 |
info "Extracting file ${file} (real name: ${base}.gz) ..." |
| 118 |
tar -C workspace -xzf "${file}" |
| 119 |
|
| 120 |
md5file="${base}.md5sum" |
| 121 |
if [ -r "${md5file}" ] |
| 122 |
then |
| 123 |
warn "\"${md5file}\" exists, append \".$$\"" |
| 124 |
md5file="${md5file}.$$" |
| 125 |
fi |
| 126 |
|
| 127 |
info "Calculating checksums for all files in archive (output=${md5file})" |
| 128 |
echo "# ${base} START" > "${md5file}" |
| 129 |
(cd workspace; md5deep -l -r ${archive}) | sort -k 2 >> "${md5file}" |
| 130 |
echo "# ${base} END" >> "${md5file}" |
| 131 |
|
| 132 |
if [ -r "${CHECKSUM_DIR}/${md5file%.$$}" ] |
| 133 |
then |
| 134 |
debug "Comparing ${md5file} to ${CHECKSUM_DIR}/${md5file%.$$}" |
| 135 |
|
| 136 |
diff "${CHECKSUM_DIR}/${md5file%.$$}" "${md5file}" && info "OK" || warn "Mismatch" |
| 137 |
else |
| 138 |
debug "${CHECKSUM_DIR}/${md5file%.$$} not found, skipping" |
| 139 |
fi |
| 140 |
|
| 141 |
if [ "${file##*/}" != "${base}.gz" ] |
| 142 |
then |
| 143 |
grep "${file##*/}" "${md5file}" |
| 144 |
fi |
| 145 |
|
| 146 |
# delete the tar ball (if downloaded!) |
| 147 |
if [ ${do_get} -ne 0 ] |
| 148 |
then |
| 149 |
rm -f "${file}" |
| 150 |
fi |
| 151 |
# clear out the workspace |
| 152 |
rm -rf workspace |
| 153 |
done |
| 154 |
|
Parent Directory
| 
Revision Log