/[pdpsoft]/trunk/nl.nikhef.ndpf.tools/cvmfs/cvmfs-rsync-multi.sh
ViewVC logotype

Contents of /trunk/nl.nikhef.ndpf.tools/cvmfs/cvmfs-rsync-multi.sh

Parent Directory Parent Directory | Revision Log Revision Log


Revision 2759 - (show annotations) (download) (as text)
Fri Nov 28 14:27:04 2014 UTC (7 years ago) by dennisvd
File MIME type: application/x-shellscript
File size: 5013 byte(s)
typo: missing ;; in case

1 #!/bin/sh
2
3 # Synchronise the Stratum 0 from an upstream rsync source,
4 # with multiple sync points.
5
6 # The remote source directory is traversed, searching for files named
7 # cvmfs.modified. If such a file exists, its timestamp is compared to
8 # the timestamp of the same file on the target directory (i.e. on the
9 # stratum-0 side). If the remote file is newer, or if the local file
10 # is absent, the directory is added to the list to synchronise.
11 # This method gives the maintainer the option to synchronise directories
12 # selectively, which is useful if a repository is shared among various
13 # users (or user groups), each with their own directory maintained
14 # independently of the others.
15
16 logfile=
17
18 die() {
19 err=${2:-1}
20 if [ -n "$logfile" ]; then
21 log "$1"
22 fi
23 echo -e "$1" >&2
24 exit $2
25 }
26
27 # prefix output with a timestamp
28 log() {
29 if [ -n "$logfile" ]; then
30 d=`LC_TIME=C date +'%b %d %H:%M:%S'`
31 echo $d "$@" >> "$logfile"
32 else
33 echo "$@"
34 fi
35 }
36
37 # These parameters must be set on the command line
38 repo=
39 rsyncsrc=
40
41 # If rsyncsrc requires a password this variable
42 # holds the name of the password file
43 rsyncopts=
44
45 # Exclude certain trees based on an excludes-from file
46 excludeopts=
47
48 # These are defaults for cvmfs, but can be
49 # overridden for debugging purposes
50 rsyncdest=/cvmfs # the destination once a transaction started
51 rdonlylocation= # the compare-dest before transactions start
52
53 while getopts :r:s:d:S:R:i:l: OPT; do
54 case $OPT in
55 r|+r)
56 repo="$OPTARG"
57 ;;
58 s|+s)
59 rsyncsrc="$OPTARG"
60 ;;
61 d|+d)
62 rsyncdest="$OPTARG"
63 ;;
64 S|+S)
65 rsyncopts="--password-file $OPTARG"
66 ;;
67 R|+R)
68 rdonlylocation="$OPTARG"
69 ;;
70 i|+i)
71 inclexclfile="$OPTARG"
72 excludeopts="--exclude-from=$OPTARG"
73 ;;
74 l|+l)
75 logfile="$OPTARG"
76 ;;
77 *)
78 echo "usage: `basename $0` -r repository -s source [ -S secretsfile ] [ -i inclexclfile ]"
79 exit 2
80 esac
81 done
82 shift `expr $OPTIND - 1`
83 OPTIND=1
84
85 if [ -z "$repo" ]; then
86 die "missing argument -r repository"
87 elif [ -z "$rsyncsrc" ]; then
88 die "missing argument -s rsyncsource"
89 fi
90
91 # Set the compare-dest location if it was not set with an option.
92 # Can't do this until $repo is set.
93 if [ -z $rdonlylocation ]; then
94 rdonlylocation=/var/spool/cvmfs/$repo/rdonly
95 fi
96
97
98 # Since the entire process may take a long time, we need a lockfile to signal
99 # that the process is already at work.
100
101 LOCKFILE=/tmp/cvmfs-$repo-update.lock
102 exec 9> $LOCKFILE;
103 flock -xn 9 || die "could not lock $LOCKFILE"
104
105 # create a temporary directory to get the rsync 'cvmfs.modified' files
106 tmp=`mktemp -d --tmpdir cvmfs.$repo.XXXXXXXXXX` || die "cannot create temporary directory. Aborting."
107
108 # make sure everything gets cleaned up at the end
109 trap "rm -rf $LOCKFILE $tmp" EXIT;
110
111 # Sanity check: see if we can rsync at all.
112 # This doesn't actually transfer anything.
113 rsync -q --no-recursive $rsyncopts $rsyncsrc /tmp
114 test $? -eq 0 || die "Can't rsync from $rsyncsrc, aborting."
115
116
117 # PHASE 1: find the cvmfs.modified files and build a list
118
119 # base this on the following rules:
120 # rsync -rt --include='/*/' --include='/*/*/' --include='cvmfs.modified' --exclude=\* tree/ newtree
121 # this includes first and second level directories, cvmfs.modified files, but excludes all the rest.
122
123 # FIXME: compare-dest is not populated until we start a transaction
124 # we should compare with the read-only version
125 errstr=`rsync -q -rt $rsyncopts $excludeopts \
126 --filter 'include /*/' \
127 --filter 'include cvmfs.modified' \
128 --filter 'exclude *' \
129 --compare-dest="$rdonlylocation/" \
130 $rsyncsrc/ $tmp/ 2>&1`
131
132 result=$?
133
134 case $result in
135 0) ;;
136 23) ;; # file missing, this is ok
137 *) # other error
138 log "$errstr"
139 die "rsync failed. Aborting." $result
140 ;;
141 esac
142
143 # Now $tmp contains a copy of the source tree's 'cvmfs.modified', but
144 # only the ones that are newer that the ones we already got in our
145 # older copy in $rsyncdest/$repo.
146 # For each of these, we need to synchronise the corresponding subtree.
147
148 find $tmp -name cvmfs.modified | sed 's,.*/\([^/]*\)/cvmfs\.modified$,\1,' | while read p
149 do
150 # Start a transaction, do the rsync and if anything goes wrong abort.
151 log "Update requested of $p; starting transaction of cvmfs repository $repo"
152 cvmfs_server transaction $repo || die "failed cvmfs transaction on $repo"
153
154 rsync $rsyncopts --filter 'include /cvmfs.modified' \
155 --filter 'include /cvmfs.tree/' \
156 --filter 'include /cvmfs.tree/**' \
157 --filter 'exclude *' \
158 -r -q -l -t --delete $rsyncsrc/$p/ "$rsyncdest/$repo/$p/"
159
160 if [ $? -ne 0 ] ; then
161 log "Rsync from $rsyncsrc/$p failed, aborting transaction."
162 cvmfs_server abort -f $repo || die "failed to abort the transaction, exiting"
163 else
164 # The rsync succeeded
165 cvmfs_server publish $repo || die "failed to publish the new CVMFS repo $repo, exiting"
166 log "cvmfs publishing done for $repo, tree $p"
167 fi
168 done
169
170
171

grid.support@nikhef.nl
ViewVC Help
Powered by ViewVC 1.1.28