/[pdpsoft]/trunk/nl.biggrid.dans/check-file
ViewVC logotype

Annotation of /trunk/nl.biggrid.dans/check-file

Parent Directory Parent Directory | Revision Log Revision Log


Revision 2581 - (hide annotations) (download)
Tue Nov 20 12:05:55 2012 UTC (9 years, 10 months ago) by svn
File size: 4016 byte(s)
Initial Import

1 svn 2581 #!/bin/bash
2    
3     if [ ! "x$0" = "x${0#-}" ]
4     then
5     echo "Don't source me, just run!" 1>&2
6     return
7     fi
8     . ${0%/*}/libdans.sh
9    
10     # Load global defaults (vars: vo, archive, LFC_*, SRM_*)
11     loadDefaults $0
12    
13    
14     DEBUG=0
15     QUIET=0
16     KEEPGOING=0
17     START_INDEX=1
18     END_INDEX=0
19     USAGE="\
20     $0 - verify the contents of one or multiple tar.gz files
21     Usage: ./check-file [-d|--debug] [-k|--keepgoing] [file|lfn|surl] [[file2|lfn2|surl2] ...]
22     Where:
23     --keepgoing tells $0 to keep going after an error
24     file specifies the full path to a local file OR
25     lfn specifies a Logical File Name on the LFC OR
26     surl specifies a Storage URL (srm:// or gsiftp://)
27     "
28    
29     # Parse commandline parameters
30     while [ $# -gt 0 ]
31     do
32     case "$1" in
33     (-h|--help) echo "${USAGE}"
34     exit 0
35     ;;
36     (-d|--debug) let DEBUG+=1
37     ;;
38     (-k|--keepgoing) KEEPGOING=1
39     ;;
40     (-*) echo "Invalid option: $1"
41     exit 1
42     ;;
43     (*) break
44     ;;
45     esac
46     shift
47     done
48    
49     if [ $# -lt 1 ]
50     then
51     abort 1 "${USAGE}"
52     fi
53    
54     # Loop over all files|lfns|surls
55     for file
56     do
57     do_get=0
58     if [ "${file}" != "${file#lfn://}" ]
59     then
60     do_get=1
61     fi
62     if [ "${file}" != "${file#srm://}" ]
63     then
64     do_get=1
65     fi
66     if [ "${file}" != "${file#gsiftp://}" ]
67     then
68     do_get=1
69     fi
70    
71     if [ ${do_get} -ne 0 ]
72     then
73     info "Retrieving file ${file}"
74     lcg-cp $SRM_TIMEOUTS "${file}" file://$PWD/"${file##*/}"
75     if [ $? -ne 0 ]
76     then
77     warn "First attempt failed, sleeping 30 seconds and retrying..."
78     sleep 30
79     lcg-cp $SRM_TIMEOUTS "${file}" file://$PWD/"${file##*/}"
80     fi
81     if [ $? -ne 0 ]
82     then
83     warn "Second attempt failed, sleeping 90 seconds and retrying..."
84     sleep 90
85     date +"%Y/%m/%d-%H:%M:%S start lcg-cp -v -v $SRM_TIMEOUTS" 1>&2
86     lcg-cp -v -v $SRM_TIMEOUTS "${file}" file://$PWD/"${file##*/}"
87     fi
88     if [ $? -ne 0 ]
89     then
90     abort 3 "Error retrieving ${file}"
91     fi
92    
93     file="${file##*/}"
94     fi
95    
96     # the file should be a .tar.gz file, extract the original archive name from it
97     base=`file "${file}" | sed -n 's/.*compressed data, was \"\(.*\)\",.*/\1/p'`
98     archive="${base%%-[0-9]*}"
99    
100     debug "file=${file} base=${base} archive=${archive}"
101    
102     CHECKSUM_DIR="${HOME}/dans/${archive}/checksums"
103    
104     # fix numbering issue in RACM tar balls
105     if [ "x${archive}" = "xRACM" ]
106     then
107     index=${base#${archive}-}
108     index=${index%.tar}
109     if [ ${#index} -lt 4 ]
110     then
111     base="${archive}-0${index}.tar"
112     fi
113     fi
114    
115     # extract the tarball in a separate directory, for easy cleanup later
116     mkdir -p workspace
117     info "Extracting file ${file} (real name: ${base}.gz) ..."
118     tar -C workspace -xzf "${file}"
119    
120     md5file="${base}.md5sum"
121     if [ -r "${md5file}" ]
122     then
123     warn "\"${md5file}\" exists, append \".$$\""
124     md5file="${md5file}.$$"
125     fi
126    
127     info "Calculating checksums for all files in archive (output=${md5file})"
128     echo "# ${base} START" > "${md5file}"
129     (cd workspace; md5deep -l -r ${archive}) | sort -k 2 >> "${md5file}"
130     echo "# ${base} END" >> "${md5file}"
131    
132     if [ -r "${CHECKSUM_DIR}/${md5file%.$$}" ]
133     then
134     debug "Comparing ${md5file} to ${CHECKSUM_DIR}/${md5file%.$$}"
135    
136     diff "${CHECKSUM_DIR}/${md5file%.$$}" "${md5file}" && info "OK" || warn "Mismatch"
137     else
138     debug "${CHECKSUM_DIR}/${md5file%.$$} not found, skipping"
139     fi
140    
141     if [ "${file##*/}" != "${base}.gz" ]
142     then
143     grep "${file##*/}" "${md5file}"
144     fi
145    
146     # delete the tar ball (if downloaded!)
147     if [ ${do_get} -ne 0 ]
148     then
149     rm -f "${file}"
150     fi
151     # clear out the workspace
152     rm -rf workspace
153     done
154    

Properties

Name Value
svn:executable

grid.support@nikhef.nl
ViewVC Help
Powered by ViewVC 1.1.28