/[pdpsoft]/trunk/nl.biggrid.dans/compare-checksums
ViewVC logotype

Diff of /trunk/nl.biggrid.dans/compare-checksums

Parent Directory Parent Directory | Revision Log Revision Log | View Patch Patch

revision 2581 by svn, Tue Nov 20 12:05:55 2012 UTC revision 2582 by svn, Tue Nov 20 12:07:59 2012 UTC
# Line 14  CHECKSUM_DIR="$PWD/checksums" Line 14  CHECKSUM_DIR="$PWD/checksums"
14  DEBUG=0  DEBUG=0
15  QUIET=0  QUIET=0
16  KEEPGOING=0  KEEPGOING=0
17    JOBID=0
18  START_INDEX=1  START_INDEX=1
19  END_INDEX=0  END_INDEX=0
20  USAGE="\  USAGE="\
# Line 22  $0 - compare checksums for DANS '${ARCHI Line 23  $0 - compare checksums for DANS '${ARCHI
23       to output from       to output from
24         ${CHECKSUM_DIR}         ${CHECKSUM_DIR}
25  Usage: ./compare-checksums [-q|--quiet] [-d|--debug] [-k|--keepgoing] [--archive=archive]  Usage: ./compare-checksums [-q|--quiet] [-d|--debug] [-k|--keepgoing] [--archive=archive]
26                             [--jobdir=dir] [--start=N] [--end=N]                             [--jobdir=dir] [--id=N] [--start=N] [--end=N]
27  Where:  Where:
28    --keepgoing        tells $0 to keep going after an error    --keepgoing        tells $0 to keep going after an error
29    --archive=archive  specifies the name of the archive (default='${ARCHIVE}')    --archive=archive  specifies the name of the archive (default='${ARCHIVE}')
30    --jobdir=dir       specifies the directory where the DANS jobs are stored    --jobdir=dir       specifies the directory where the DANS jobs are stored
31                       (default='${JOBDIR}')                       (default='${JOBDIR}')
32      --id=N             specifies the DANS job id
33    --start=N          specifies the starting DANS job id (default=1)    --start=N          specifies the starting DANS job id (default=1)
34    --end=N            specifies the end DANS job id (default=ALL)    --end=N            specifies the end DANS job id (default=ALL)
35  "  "
# Line 59  do Line 61  do
61                            ;;                            ;;
62          (--job-dir=*)     JOBDIR=${1##--job-dir=}          (--job-dir=*)     JOBDIR=${1##--job-dir=}
63                            ;;                            ;;
64            (--id)            JOBID=$2
65                              shift
66                              ;;
67            (--id=*)          JOBID=${1##--id=}
68                              ;;
69          (--start)         START_INDEX=$2          (--start)         START_INDEX=$2
70                            shift                            shift
71                            ;;                            ;;
# Line 76  do Line 83  do
83      shift      shift
84  done  done
85    
86    # Check that the jobdir actually exists
87    if [ ! -d "${JOBDIR}/." ]
88    then
89        abort 1 "Job directory '${JOBDIR}' does not exist!"
90    fi
91    # and check that we can find the reference checksums
92    CHECKSUM_DIR="${PWD%/*}/${ARCHIVE}/checksums"
93    if [ ! -d "${CHECKSUM_DIR}/." ]
94    then
95        abort 2 "Checksum directory '${CHECKSUM_DIR}' does not exist!"
96    fi
97    
98  # create a working directory for temporary files and clean it  # create a working directory for temporary files and clean it
99  output_dir=/tmp/compare_checksums.$$  output_dir=/tmp/compare_checksums.$$
100  mkdir -p "${output_dir}" || abort 1 "Cannot create working dir ${output_dir}"  mkdir -p "${output_dir}" || abort 3 "Cannot create working dir ${output_dir}"
101    
102  # sanitize start and end indices  # sanitize start and end indices
103  lastdansjobid=`(cd  "${JOBDIR}"; ls -1dr [0-9][0-9][0-9][0-9][0-9] 2> /dev/null| head -1)`  lastdansjobid=`(cd  "${JOBDIR}"; ls -1dr [0-9][0-9][0-9][0-9][0-9] 2> /dev/null| head -1)`
104  if [ ${START_INDEX} -lt 1 -o ${START_INDEX} -gt ${lastdansjobid} ]  if [ ${START_INDEX} -lt 1 -o ${START_INDEX} -gt ${lastdansjobid} ]
105  then  then
106      START_INDEX=${lastdansjobid}      abort 4 "Invalid start job index specified!"
107  fi  fi
108  if [ ${END_INDEX} -lt 1 -o ${END_INDEX} -gt ${lastdansjobid} ]  if [ ${END_INDEX} -lt 1 -o ${END_INDEX} -gt ${lastdansjobid} ]
109  then  then
110        if [ ${END_INDEX} -ne 0 ]
111        then
112            warn "Invalid end job index specified, adjusting to ${lastdansjobid}"
113        fi
114      END_INDEX=${lastdansjobid}      END_INDEX=${lastdansjobid}
115  fi  fi
116    if [ ${JOBID} -gt 0 ]
117    then
118        if [ ${JOBID} -le ${lastdansjobid} ]
119        then
120            START_INDEX=${JOBID}
121            END_INDEX=${JOBID}
122        else
123            abort 5 "Invalid DANS jobid '${JOBID}' specified!"
124        fi
125    else
126        info "Scanning for DANS '${ARCHIVE}' jobs:"
127    fi
128    
129    numjobs=0
130  for jobid in `seq ${START_INDEX} ${END_INDEX}`  for jobid in `seq ${START_INDEX} ${END_INDEX}`
131  do  do
132      jobdir=`printf "%05d" ${jobid}`      jobdir=`printf "${JOBDIR}/%05d" ${jobid}`
133      cd "${JOBDIR}/${jobdir}"  
134      if [ ! -r jobid ]      debug "${jobdir}:"
135        # Sanity check #1) is there a 'jobid' file?
136        if [ ! -r "${jobdir}/jobid" ]
137      then      then
138          debug "\nNo jobid file found in ${jobdir} - not a DANS-job?"          debug "  No jobid file found - not a DANS-job?"
139          continue          continue
140      fi      fi
141    
142      info -n "${jobdir}: "      cd "${jobdir}"
143      # check for job files      # Sanity check #2) are 'jdl','job-logging-info.log' and 'Status=Cleared' files present?
     files_found=`(ls -d jdl job*log output 2> /dev/null)`  
     debug "Job files found: " $files_found  
   
144      if [ ! -r jdl ]      if [ ! -r jdl ]
145      then      then
146          warn "No jdl file found in ${JOBDIR}/${jobdir}, skipping"          debug "  No jdl file found, skipping"
147            continue
148        fi
149        if [ ! -r job-logging-info.log ]
150        then
151            debug "  No job-logging-info.log file found, skipping"
152            continue
153        fi
154        if [ ! -r Status=Done -a ! -r Status=Cleared ]
155        then
156            debug "  It does not look like this job's output has been retrieved, skipping"
157          continue          continue
158      fi      fi
159    
# Line 116  do Line 161  do
161      executable=`sed -n 's/^Executable = "\(.*\)".*/\1/p' jdl`      executable=`sed -n 's/^Executable = "\(.*\)".*/\1/p' jdl`
162      arguments=`sed -n 's/^Arguments = "\(.*\)".*/\1/p' jdl`      arguments=`sed -n 's/^Arguments = "\(.*\)".*/\1/p' jdl`
163    
164      debug "job exec+args = ${executable} ${arguments}"      debug "  job exec+args = ${executable} ${arguments}"
165    
166      # only do this for check-archive jobs      # Sanity check #3) only do this for check-archive jobs
167      if [ "x${executable}" != "xcheck-archive.sh" ]      if [ "x${executable}" != "xcheck-archive.sh" ]
168      then      then
169          warn "Not a check-archive job, skipping"          debug "  Not a check-archive job, skipping"
170          continue          continue
171      fi      fi
172    
173      # The arguments should look something like 'Crome X Y'      # The arguments should look something like 'Crome X Y'
174      jobARCHIVE=`echo $arguments | awk '{ print $1 }'`      job_archive=`echo $arguments | awk '{ print $1 }'`
175      start_index=`echo $arguments | awk '{ print $2 }'`      start_index=`echo $arguments | awk '{ print $2 }'`
176      end_index=`echo $arguments | awk '{ print $3 }'`      end_index=`echo $arguments | awk '{ print $3 }'`
177    
178      # only do this for jobs for which the output has been retrieved      debug "  ARCHIVE=${ARCHIVE} job_archive=${job_archive}"
     exitcode=-1  
     timestamp=  
179    
180      if [ ! -r job-logging-info.log ]      # Sanity check #4)  only do this if the job's archive matches the archive we want
181        if [ "x${job_archive}" != "x${ARCHIVE}" ]
182      then      then
183          warn "No job-logging-info.log file found in ${JOBDIR}/${jobdir}, skipping"          debug "  Wrong archive (${job_archive}), skipping"
184          continue          continue
185      fi      fi
186    
187        info -n "${jobdir##*/}: "
188    
189      # find the number of digits      # find the number of digits
190      num_digits=3      num_digits=3
191      ref_output=`printf "${CHECKSUM_DIR}/${ARCHIVE}-%0${num_digits}d.tar.md5sum" ${start_index}`      ref_output=`printf "${CHECKSUM_DIR}/${ARCHIVE}-%0${num_digits}d.tar.md5sum" ${start_index}`
# Line 159  do Line 205  do
205          fi          fi
206      fi      fi
207    
208        # only do this for jobs for which the output has been retrieved
209        let numjobs++
210        exitcode=-1
211        timestamp=
212      job_log=`awk 'BEGIN { done=0 };      job_log=`awk 'BEGIN { done=0 };
213  /^Event: Done/{ done=1 };  /^Event: Done/{ done=1 };
214  /^- Exit code/{ have_exitcode=1 ; exitcode=$5 };  /^- Exit code/{ have_exitcode=1 ; exitcode=$5 };
# Line 169  if ( done && have_exitcode) Line 219  if ( done && have_exitcode)
219     printf "exitcode=\"%s\" ; timestamp=\"%s\"\n", exitcode, timestamp }     printf "exitcode=\"%s\" ; timestamp=\"%s\"\n", exitcode, timestamp }
220  }' job-logging-info.log`  }' job-logging-info.log`
221    
222      debug "Job logging info: $job_log"      debug "  Job logging info: $job_log"
223      eval `echo "$job_log"`      eval `echo "$job_log"`
224    
225      # continue only if the exit code was 0      # continue only if the exit code was 0
# Line 179  if ( done && have_exitcode) Line 229  if ( done && have_exitcode)
229          continue          continue
230      fi      fi
231    
232      #info -n "${executable} \"$jobARCHIVE ${start_index} ${end_index}\" Comparing md5sums: "      #info -n "${executable} \"$job_archive ${start_index} ${end_index}\" Comparing md5sums: "
233      printf "${executable} \"%s %4d %4d\" Comparing md5sums: " ${jobARCHIVE} ${start_index} ${end_index}      printf "${executable} \"%s %4d %4d\" Comparing md5sums: " ${job_archive} ${start_index} ${end_index}
234      # now look at the output produced      # now look at the output produced
235      if [ -r output/md5sums.txt ]      if [ -r output/md5sums.txt ]
236      then      then
# Line 193  if ( done && have_exitcode) Line 243  if ( done && have_exitcode)
243              ref_output=`printf "${CHECKSUM_DIR}/${ARCHIVE}-%0${num_digits}d.tar.md5sum" $i`              ref_output=`printf "${CHECKSUM_DIR}/${ARCHIVE}-%0${num_digits}d.tar.md5sum" $i`
244              grep -v "^#" "${ref_output}" >> "${tmp_file}"              grep -v "^#" "${ref_output}" >> "${tmp_file}"
245    
246              let i=i+1              let i++
247          done          done
248                                                    
249          diffs=`diff "${tmp_file}" output/md5sums.txt | grep -v '^> #' | grep -v '^[0-9,]*a[0-9,]*'`          diffs=`diff "${tmp_file}" output/md5sums.txt | grep -v '^> #' | grep -v '^[0-9,]*a[0-9,]*'`
# Line 206  if ( done && have_exitcode) Line 256  if ( done && have_exitcode)
256          fi          fi
257      elif [ -r output/md5sums.tar.gz ]      elif [ -r output/md5sums.tar.gz ]
258      then      then
259          rm -f "${output_dir}/*md5sum"          rm -f ${output_dir}/*md5sum
260          tar -C "${output_dir}" -xzf output/md5sums.tar.gz          tar -C "${output_dir}" -xzf output/md5sums.tar.gz
261    
262          are_equal=1          are_equal=1
# Line 232  done Line 282  done
282    
283  rm -rf ${output_dir}  rm -rf ${output_dir}
284    
285    if [ ${numjobs} -eq 0 ]
286    then
287        info "No jobs found!"
288    elif [ ${numjobs} -gt 1 ]
289    then
290        info "Found ${numjobs} DANS jobs"
291    fi
292    
Legend:
Removed from v.2581  
changed lines
  Added in v.2582
grid.support@nikhef.nl
 ViewVC Help
Powered by ViewVC 1.1.28