/[pdpsoft]/trunk/nl.biggrid.dans/compare-checksums
ViewVC logotype

Contents of /trunk/nl.biggrid.dans/compare-checksums

Parent Directory Parent Directory | Revision Log Revision Log


Revision 2582 - (show annotations) (download)
Tue Nov 20 12:07:59 2012 UTC (9 years, 1 month ago) by svn
File size: 8721 byte(s)
First released version
1 #!/bin/bash
2
3 if [ ! "x$0" = "x${0#-}" ]
4 then
5 echo "Don't source me, just run!" 1>&2
6 return
7 fi
8 . ${0%/*}/libdans.sh
9
10 # Load global defaults (vars: vo, ARCHIVE, LFC_*, SRM_*)
11 loadDefaults $0
12
13 CHECKSUM_DIR="$PWD/checksums"
14 DEBUG=0
15 QUIET=0
16 KEEPGOING=0
17 JOBID=0
18 START_INDEX=1
19 END_INDEX=0
20 USAGE="\
21 $0 - compare checksums for DANS '${ARCHIVE}' jobs in
22 ${JOBDIR}
23 to output from
24 ${CHECKSUM_DIR}
25 Usage: ./compare-checksums [-q|--quiet] [-d|--debug] [-k|--keepgoing] [--archive=archive]
26 [--jobdir=dir] [--id=N] [--start=N] [--end=N]
27 Where:
28 --keepgoing tells $0 to keep going after an error
29 --archive=archive specifies the name of the archive (default='${ARCHIVE}')
30 --jobdir=dir specifies the directory where the DANS jobs are stored
31 (default='${JOBDIR}')
32 --id=N specifies the DANS job id
33 --start=N specifies the starting DANS job id (default=1)
34 --end=N specifies the end DANS job id (default=ALL)
35 "
36
37 # Parse commandline parameters
38 while [ $# -gt 0 ]
39 do
40 case "$1" in
41 (-h|--help) echo "${USAGE}"
42 exit 0
43 ;;
44 (-d|--debug) let DEBUG+=1
45 ;;
46 (-q|--quiet) QUIET=1
47 ;;
48 (-k|--keepgoing|--keep-going)
49 KEEPGOING=1
50 ;;
51 (--archive) ARCHIVE=$2
52 shift
53 ;;
54 (--archive=*) ARCHIVE=${1##--archive=}
55 ;;
56 (--jobdir|--job-dir)
57 JOBDIR=$2
58 shift
59 ;;
60 (--jobdir=*) JOBDIR=${1##--jobdir=}
61 ;;
62 (--job-dir=*) JOBDIR=${1##--job-dir=}
63 ;;
64 (--id) JOBID=$2
65 shift
66 ;;
67 (--id=*) JOBID=${1##--id=}
68 ;;
69 (--start) START_INDEX=$2
70 shift
71 ;;
72 (--start=*) START_INDEX=${1##--start=}
73 ;;
74 (--end) END_INDEX=$2
75 shift
76 ;;
77 (--end=*) END_INDEX=${1##--end=}
78 ;;
79 (-*) echo "Invalid option: $1"
80 exit 1
81 ;;
82 esac
83 shift
84 done
85
86 # Check that the jobdir actually exists
87 if [ ! -d "${JOBDIR}/." ]
88 then
89 abort 1 "Job directory '${JOBDIR}' does not exist!"
90 fi
91 # and check that we can find the reference checksums
92 CHECKSUM_DIR="${PWD%/*}/${ARCHIVE}/checksums"
93 if [ ! -d "${CHECKSUM_DIR}/." ]
94 then
95 abort 2 "Checksum directory '${CHECKSUM_DIR}' does not exist!"
96 fi
97
98 # create a working directory for temporary files and clean it
99 output_dir=/tmp/compare_checksums.$$
100 mkdir -p "${output_dir}" || abort 3 "Cannot create working dir ${output_dir}"
101
102 # sanitize start and end indices
103 lastdansjobid=`(cd "${JOBDIR}"; ls -1dr [0-9][0-9][0-9][0-9][0-9] 2> /dev/null| head -1)`
104 if [ ${START_INDEX} -lt 1 -o ${START_INDEX} -gt ${lastdansjobid} ]
105 then
106 abort 4 "Invalid start job index specified!"
107 fi
108 if [ ${END_INDEX} -lt 1 -o ${END_INDEX} -gt ${lastdansjobid} ]
109 then
110 if [ ${END_INDEX} -ne 0 ]
111 then
112 warn "Invalid end job index specified, adjusting to ${lastdansjobid}"
113 fi
114 END_INDEX=${lastdansjobid}
115 fi
116 if [ ${JOBID} -gt 0 ]
117 then
118 if [ ${JOBID} -le ${lastdansjobid} ]
119 then
120 START_INDEX=${JOBID}
121 END_INDEX=${JOBID}
122 else
123 abort 5 "Invalid DANS jobid '${JOBID}' specified!"
124 fi
125 else
126 info "Scanning for DANS '${ARCHIVE}' jobs:"
127 fi
128
129 numjobs=0
130 for jobid in `seq ${START_INDEX} ${END_INDEX}`
131 do
132 jobdir=`printf "${JOBDIR}/%05d" ${jobid}`
133
134 debug "${jobdir}:"
135 # Sanity check #1) is there a 'jobid' file?
136 if [ ! -r "${jobdir}/jobid" ]
137 then
138 debug " No jobid file found - not a DANS-job?"
139 continue
140 fi
141
142 cd "${jobdir}"
143 # Sanity check #2) are 'jdl','job-logging-info.log' and 'Status=Cleared' files present?
144 if [ ! -r jdl ]
145 then
146 debug " No jdl file found, skipping"
147 continue
148 fi
149 if [ ! -r job-logging-info.log ]
150 then
151 debug " No job-logging-info.log file found, skipping"
152 continue
153 fi
154 if [ ! -r Status=Done -a ! -r Status=Cleared ]
155 then
156 debug " It does not look like this job's output has been retrieved, skipping"
157 continue
158 fi
159
160 # Extract the executable (script) and arguments from the jdl
161 executable=`sed -n 's/^Executable = "\(.*\)".*/\1/p' jdl`
162 arguments=`sed -n 's/^Arguments = "\(.*\)".*/\1/p' jdl`
163
164 debug " job exec+args = ${executable} ${arguments}"
165
166 # Sanity check #3) only do this for check-archive jobs
167 if [ "x${executable}" != "xcheck-archive.sh" ]
168 then
169 debug " Not a check-archive job, skipping"
170 continue
171 fi
172
173 # The arguments should look something like 'Crome X Y'
174 job_archive=`echo $arguments | awk '{ print $1 }'`
175 start_index=`echo $arguments | awk '{ print $2 }'`
176 end_index=`echo $arguments | awk '{ print $3 }'`
177
178 debug " ARCHIVE=${ARCHIVE} job_archive=${job_archive}"
179
180 # Sanity check #4) only do this if the job's archive matches the archive we want
181 if [ "x${job_archive}" != "x${ARCHIVE}" ]
182 then
183 debug " Wrong archive (${job_archive}), skipping"
184 continue
185 fi
186
187 info -n "${jobdir##*/}: "
188
189 # find the number of digits
190 num_digits=3
191 ref_output=`printf "${CHECKSUM_DIR}/${ARCHIVE}-%0${num_digits}d.tar.md5sum" ${start_index}`
192 if [ ! -r ${ref_output} ]
193 then
194 num_digits=4
195 ref_output=`printf "${CHECKSUM_DIR}/${ARCHIVE}-%0${num_digits}d.tar.md5sum" ${start_index}`
196 if [ ! -r ${ref_output} ]
197 then
198 num_digits=5
199 ref_output=`printf "${CHECKSUM_DIR}/${ARCHIVE}-%0${num_digits}d.tar.md5sum" ${start_index}`
200 if [ ! -r ${ref_output} ]
201 then
202 warn "Cannot find reference checksum file ${ref_output}, skipping to next"
203 continue
204 fi
205 fi
206 fi
207
208 # only do this for jobs for which the output has been retrieved
209 let numjobs++
210 exitcode=-1
211 timestamp=
212 job_log=`awk 'BEGIN { done=0 };
213 /^Event: Done/{ done=1 };
214 /^- Exit code/{ have_exitcode=1 ; exitcode=$5 };
215 /^- Timestamp/{ timestamp=$4 " " $5 " " $6 " " $7 " "$8 " " $9 };
216 /^- Source/{ if ($4 != "LogMonitor" ) done=0 ;
217 if ( done && have_exitcode)
218 {
219 printf "exitcode=\"%s\" ; timestamp=\"%s\"\n", exitcode, timestamp }
220 }' job-logging-info.log`
221
222 debug " Job logging info: $job_log"
223 eval `echo "$job_log"`
224
225 # continue only if the exit code was 0
226 if [ ${exitcode} -ne 0 ]
227 then
228 warn "Job exit code was not 0, skipping"
229 continue
230 fi
231
232 #info -n "${executable} \"$job_archive ${start_index} ${end_index}\" Comparing md5sums: "
233 printf "${executable} \"%s %4d %4d\" Comparing md5sums: " ${job_archive} ${start_index} ${end_index}
234 # now look at the output produced
235 if [ -r output/md5sums.txt ]
236 then
237 tmp_file="${output_dir}/gridchecksums"
238 rm -f "${tmp_file}"
239 i=${start_index}
240
241 while [ $i -le ${end_index} ]
242 do
243 ref_output=`printf "${CHECKSUM_DIR}/${ARCHIVE}-%0${num_digits}d.tar.md5sum" $i`
244 grep -v "^#" "${ref_output}" >> "${tmp_file}"
245
246 let i++
247 done
248
249 diffs=`diff "${tmp_file}" output/md5sums.txt | grep -v '^> #' | grep -v '^[0-9,]*a[0-9,]*'`
250 if [ -z "${diffs}" ]
251 then
252 info "Equal"
253 else
254 info "Different!"
255 info "${diffs}"
256 fi
257 elif [ -r output/md5sums.tar.gz ]
258 then
259 rm -f ${output_dir}/*md5sum
260 tar -C "${output_dir}" -xzf output/md5sums.tar.gz
261
262 are_equal=1
263 for file in ${output_dir}/*md5sum
264 do
265 debug "\n Comparing ${CHECKSUM_DIR}/${file##*/} to ${file}"
266 diffs=`diff "${CHECKSUM_DIR}/${file##*/}" "${file}"`
267
268 if [ -n "${diffs}" ]
269 then
270 are_equal=0
271 info "\n${diffs}"
272 fi
273 done
274 if [ ${are_equal} -eq 1 ]
275 then
276 info "Equal"
277 fi
278 else
279 warn "\nNo md5sums.txt or md5sums.tar.gz file found in job output, skipping"
280 fi
281 done
282
283 rm -rf ${output_dir}
284
285 if [ ${numjobs} -eq 0 ]
286 then
287 info "No jobs found!"
288 elif [ ${numjobs} -gt 1 ]
289 then
290 info "Found ${numjobs} DANS jobs"
291 fi
292

Properties

Name Value
svn:executable

grid.support@nikhef.nl
ViewVC Help
Powered by ViewVC 1.1.28