| 1 |
svn |
2581 |
#!/bin/bash |
| 2 |
|
|
|
| 3 |
|
|
if [ ! "x$0" = "x${0#-}" ] |
| 4 |
|
|
then |
| 5 |
|
|
echo "Don't source me, just run!" 1>&2 |
| 6 |
|
|
return |
| 7 |
|
|
fi |
| 8 |
|
|
. ${0%/*}/libdans.sh |
| 9 |
|
|
|
| 10 |
|
|
# Load global defaults (vars: vo, archive, LFC_*, SRM_*) |
| 11 |
|
|
loadDefaults $0 |
| 12 |
|
|
|
| 13 |
|
|
|
| 14 |
|
|
DEBUG=0 |
| 15 |
|
|
QUIET=0 |
| 16 |
|
|
KEEPGOING=0 |
| 17 |
|
|
START_INDEX=1 |
| 18 |
|
|
END_INDEX=0 |
| 19 |
|
|
USAGE="\ |
| 20 |
|
|
$0 - verify the contents of one or multiple tar.gz files |
| 21 |
|
|
Usage: ./check-file [-d|--debug] [-k|--keepgoing] [file|lfn|surl] [[file2|lfn2|surl2] ...] |
| 22 |
|
|
Where: |
| 23 |
|
|
--keepgoing tells $0 to keep going after an error |
| 24 |
|
|
file specifies the full path to a local file OR |
| 25 |
|
|
lfn specifies a Logical File Name on the LFC OR |
| 26 |
|
|
surl specifies a Storage URL (srm:// or gsiftp://) |
| 27 |
|
|
" |
| 28 |
|
|
|
| 29 |
|
|
# Parse commandline parameters |
| 30 |
|
|
while [ $# -gt 0 ] |
| 31 |
|
|
do |
| 32 |
|
|
case "$1" in |
| 33 |
|
|
(-h|--help) echo "${USAGE}" |
| 34 |
|
|
exit 0 |
| 35 |
|
|
;; |
| 36 |
|
|
(-d|--debug) let DEBUG+=1 |
| 37 |
|
|
;; |
| 38 |
|
|
(-k|--keepgoing) KEEPGOING=1 |
| 39 |
|
|
;; |
| 40 |
|
|
(-*) echo "Invalid option: $1" |
| 41 |
|
|
exit 1 |
| 42 |
|
|
;; |
| 43 |
|
|
(*) break |
| 44 |
|
|
;; |
| 45 |
|
|
esac |
| 46 |
|
|
shift |
| 47 |
|
|
done |
| 48 |
|
|
|
| 49 |
|
|
if [ $# -lt 1 ] |
| 50 |
|
|
then |
| 51 |
|
|
abort 1 "${USAGE}" |
| 52 |
|
|
fi |
| 53 |
|
|
|
| 54 |
|
|
# Loop over all files|lfns|surls |
| 55 |
|
|
for file |
| 56 |
|
|
do |
| 57 |
|
|
do_get=0 |
| 58 |
|
|
if [ "${file}" != "${file#lfn://}" ] |
| 59 |
|
|
then |
| 60 |
|
|
do_get=1 |
| 61 |
|
|
fi |
| 62 |
|
|
if [ "${file}" != "${file#srm://}" ] |
| 63 |
|
|
then |
| 64 |
|
|
do_get=1 |
| 65 |
|
|
fi |
| 66 |
|
|
if [ "${file}" != "${file#gsiftp://}" ] |
| 67 |
|
|
then |
| 68 |
|
|
do_get=1 |
| 69 |
|
|
fi |
| 70 |
|
|
|
| 71 |
|
|
if [ ${do_get} -ne 0 ] |
| 72 |
|
|
then |
| 73 |
|
|
info "Retrieving file ${file}" |
| 74 |
|
|
lcg-cp $SRM_TIMEOUTS "${file}" file://$PWD/"${file##*/}" |
| 75 |
|
|
if [ $? -ne 0 ] |
| 76 |
|
|
then |
| 77 |
|
|
warn "First attempt failed, sleeping 30 seconds and retrying..." |
| 78 |
|
|
sleep 30 |
| 79 |
|
|
lcg-cp $SRM_TIMEOUTS "${file}" file://$PWD/"${file##*/}" |
| 80 |
|
|
fi |
| 81 |
|
|
if [ $? -ne 0 ] |
| 82 |
|
|
then |
| 83 |
|
|
warn "Second attempt failed, sleeping 90 seconds and retrying..." |
| 84 |
|
|
sleep 90 |
| 85 |
|
|
date +"%Y/%m/%d-%H:%M:%S start lcg-cp -v -v $SRM_TIMEOUTS" 1>&2 |
| 86 |
|
|
lcg-cp -v -v $SRM_TIMEOUTS "${file}" file://$PWD/"${file##*/}" |
| 87 |
|
|
fi |
| 88 |
|
|
if [ $? -ne 0 ] |
| 89 |
|
|
then |
| 90 |
|
|
abort 3 "Error retrieving ${file}" |
| 91 |
|
|
fi |
| 92 |
|
|
|
| 93 |
|
|
file="${file##*/}" |
| 94 |
|
|
fi |
| 95 |
|
|
|
| 96 |
|
|
# the file should be a .tar.gz file, extract the original archive name from it |
| 97 |
|
|
base=`file "${file}" | sed -n 's/.*compressed data, was \"\(.*\)\",.*/\1/p'` |
| 98 |
|
|
archive="${base%%-[0-9]*}" |
| 99 |
|
|
|
| 100 |
|
|
debug "file=${file} base=${base} archive=${archive}" |
| 101 |
|
|
|
| 102 |
|
|
CHECKSUM_DIR="${HOME}/dans/${archive}/checksums" |
| 103 |
|
|
|
| 104 |
|
|
# fix numbering issue in RACM tar balls |
| 105 |
|
|
if [ "x${archive}" = "xRACM" ] |
| 106 |
|
|
then |
| 107 |
|
|
index=${base#${archive}-} |
| 108 |
|
|
index=${index%.tar} |
| 109 |
|
|
if [ ${#index} -lt 4 ] |
| 110 |
|
|
then |
| 111 |
|
|
base="${archive}-0${index}.tar" |
| 112 |
|
|
fi |
| 113 |
|
|
fi |
| 114 |
|
|
|
| 115 |
|
|
# extract the tarball in a separate directory, for easy cleanup later |
| 116 |
|
|
mkdir -p workspace |
| 117 |
|
|
info "Extracting file ${file} (real name: ${base}.gz) ..." |
| 118 |
|
|
tar -C workspace -xzf "${file}" |
| 119 |
|
|
|
| 120 |
|
|
md5file="${base}.md5sum" |
| 121 |
|
|
if [ -r "${md5file}" ] |
| 122 |
|
|
then |
| 123 |
|
|
warn "\"${md5file}\" exists, append \".$$\"" |
| 124 |
|
|
md5file="${md5file}.$$" |
| 125 |
|
|
fi |
| 126 |
|
|
|
| 127 |
|
|
info "Calculating checksums for all files in archive (output=${md5file})" |
| 128 |
|
|
echo "# ${base} START" > "${md5file}" |
| 129 |
|
|
(cd workspace; md5deep -l -r ${archive}) | sort -k 2 >> "${md5file}" |
| 130 |
|
|
echo "# ${base} END" >> "${md5file}" |
| 131 |
|
|
|
| 132 |
|
|
if [ -r "${CHECKSUM_DIR}/${md5file%.$$}" ] |
| 133 |
|
|
then |
| 134 |
|
|
debug "Comparing ${md5file} to ${CHECKSUM_DIR}/${md5file%.$$}" |
| 135 |
|
|
|
| 136 |
|
|
diff "${CHECKSUM_DIR}/${md5file%.$$}" "${md5file}" && info "OK" || warn "Mismatch" |
| 137 |
|
|
else |
| 138 |
|
|
debug "${CHECKSUM_DIR}/${md5file%.$$} not found, skipping" |
| 139 |
|
|
fi |
| 140 |
|
|
|
| 141 |
|
|
if [ "${file##*/}" != "${base}.gz" ] |
| 142 |
|
|
then |
| 143 |
|
|
grep "${file##*/}" "${md5file}" |
| 144 |
|
|
fi |
| 145 |
|
|
|
| 146 |
|
|
# delete the tar ball (if downloaded!) |
| 147 |
|
|
if [ ${do_get} -ne 0 ] |
| 148 |
|
|
then |
| 149 |
|
|
rm -f "${file}" |
| 150 |
|
|
fi |
| 151 |
|
|
# clear out the workspace |
| 152 |
|
|
rm -rf workspace |
| 153 |
|
|
done |
| 154 |
|
|
|
Parent Directory
| 
Revision Log