/[pdpsoft]/trunk/nl.biggrid.dans/check-file
ViewVC logotype

Contents of /trunk/nl.biggrid.dans/check-file

Parent Directory Parent Directory | Revision Log Revision Log


Revision 2581 - (show annotations) (download)
Tue Nov 20 12:05:55 2012 UTC (9 years, 10 months ago) by svn
File size: 4016 byte(s)
Initial Import

1 #!/bin/bash
2
3 if [ ! "x$0" = "x${0#-}" ]
4 then
5 echo "Don't source me, just run!" 1>&2
6 return
7 fi
8 . ${0%/*}/libdans.sh
9
10 # Load global defaults (vars: vo, archive, LFC_*, SRM_*)
11 loadDefaults $0
12
13
14 DEBUG=0
15 QUIET=0
16 KEEPGOING=0
17 START_INDEX=1
18 END_INDEX=0
19 USAGE="\
20 $0 - verify the contents of one or multiple tar.gz files
21 Usage: ./check-file [-d|--debug] [-k|--keepgoing] [file|lfn|surl] [[file2|lfn2|surl2] ...]
22 Where:
23 --keepgoing tells $0 to keep going after an error
24 file specifies the full path to a local file OR
25 lfn specifies a Logical File Name on the LFC OR
26 surl specifies a Storage URL (srm:// or gsiftp://)
27 "
28
29 # Parse commandline parameters
30 while [ $# -gt 0 ]
31 do
32 case "$1" in
33 (-h|--help) echo "${USAGE}"
34 exit 0
35 ;;
36 (-d|--debug) let DEBUG+=1
37 ;;
38 (-k|--keepgoing) KEEPGOING=1
39 ;;
40 (-*) echo "Invalid option: $1"
41 exit 1
42 ;;
43 (*) break
44 ;;
45 esac
46 shift
47 done
48
49 if [ $# -lt 1 ]
50 then
51 abort 1 "${USAGE}"
52 fi
53
54 # Loop over all files|lfns|surls
55 for file
56 do
57 do_get=0
58 if [ "${file}" != "${file#lfn://}" ]
59 then
60 do_get=1
61 fi
62 if [ "${file}" != "${file#srm://}" ]
63 then
64 do_get=1
65 fi
66 if [ "${file}" != "${file#gsiftp://}" ]
67 then
68 do_get=1
69 fi
70
71 if [ ${do_get} -ne 0 ]
72 then
73 info "Retrieving file ${file}"
74 lcg-cp $SRM_TIMEOUTS "${file}" file://$PWD/"${file##*/}"
75 if [ $? -ne 0 ]
76 then
77 warn "First attempt failed, sleeping 30 seconds and retrying..."
78 sleep 30
79 lcg-cp $SRM_TIMEOUTS "${file}" file://$PWD/"${file##*/}"
80 fi
81 if [ $? -ne 0 ]
82 then
83 warn "Second attempt failed, sleeping 90 seconds and retrying..."
84 sleep 90
85 date +"%Y/%m/%d-%H:%M:%S start lcg-cp -v -v $SRM_TIMEOUTS" 1>&2
86 lcg-cp -v -v $SRM_TIMEOUTS "${file}" file://$PWD/"${file##*/}"
87 fi
88 if [ $? -ne 0 ]
89 then
90 abort 3 "Error retrieving ${file}"
91 fi
92
93 file="${file##*/}"
94 fi
95
96 # the file should be a .tar.gz file, extract the original archive name from it
97 base=`file "${file}" | sed -n 's/.*compressed data, was \"\(.*\)\",.*/\1/p'`
98 archive="${base%%-[0-9]*}"
99
100 debug "file=${file} base=${base} archive=${archive}"
101
102 CHECKSUM_DIR="${HOME}/dans/${archive}/checksums"
103
104 # fix numbering issue in RACM tar balls
105 if [ "x${archive}" = "xRACM" ]
106 then
107 index=${base#${archive}-}
108 index=${index%.tar}
109 if [ ${#index} -lt 4 ]
110 then
111 base="${archive}-0${index}.tar"
112 fi
113 fi
114
115 # extract the tarball in a separate directory, for easy cleanup later
116 mkdir -p workspace
117 info "Extracting file ${file} (real name: ${base}.gz) ..."
118 tar -C workspace -xzf "${file}"
119
120 md5file="${base}.md5sum"
121 if [ -r "${md5file}" ]
122 then
123 warn "\"${md5file}\" exists, append \".$$\""
124 md5file="${md5file}.$$"
125 fi
126
127 info "Calculating checksums for all files in archive (output=${md5file})"
128 echo "# ${base} START" > "${md5file}"
129 (cd workspace; md5deep -l -r ${archive}) | sort -k 2 >> "${md5file}"
130 echo "# ${base} END" >> "${md5file}"
131
132 if [ -r "${CHECKSUM_DIR}/${md5file%.$$}" ]
133 then
134 debug "Comparing ${md5file} to ${CHECKSUM_DIR}/${md5file%.$$}"
135
136 diff "${CHECKSUM_DIR}/${md5file%.$$}" "${md5file}" && info "OK" || warn "Mismatch"
137 else
138 debug "${CHECKSUM_DIR}/${md5file%.$$} not found, skipping"
139 fi
140
141 if [ "${file##*/}" != "${base}.gz" ]
142 then
143 grep "${file##*/}" "${md5file}"
144 fi
145
146 # delete the tar ball (if downloaded!)
147 if [ ${do_get} -ne 0 ]
148 then
149 rm -f "${file}"
150 fi
151 # clear out the workspace
152 rm -rf workspace
153 done
154

Properties

Name Value
svn:executable

grid.support@nikhef.nl
ViewVC Help
Powered by ViewVC 1.1.28