/[pdpsoft]/trunk/nl.nikhef.ndpf.tools/cvmfs/cvmfs-rsync-multi.sh
ViewVC logotype

Contents of /trunk/nl.nikhef.ndpf.tools/cvmfs/cvmfs-rsync-multi.sh

Parent Directory Parent Directory | Revision Log Revision Log


Revision 2747 - (show annotations) (download) (as text)
Wed Nov 12 16:24:46 2014 UTC (7 years, 2 months ago) by dennisvd
File MIME type: application/x-shellscript
File size: 4676 byte(s)
defer setting rdonlylocation until $repo is set.

1 #!/bin/sh
2
3 # Synchronise the Stratum 0 from an upstream rsync source,
4 # with multiple sync points.
5
6 # The remote source directory is traversed, searching for files named
7 # cvmfs.modified. If such a file exists, its timestamp is compared to
8 # the timestamp of the same file on the target directory (i.e. on the
9 # stratum-0 side). If the remote file is newer, or if the local file
10 # is absent, the directory is added to the list to synchronise.
11 # This method gives the maintainer the option to synchronise directories
12 # selectively, which is useful if a repository is shared among various
13 # users (or user groups), each with their own directory maintained
14 # independently of the others.
15
16 die() {
17 echo -e "$1" >&2
18 exit 1
19 }
20
21 # These parameters must be set on the command line
22 repo=
23 rsyncsrc=
24
25 # If rsyncsrc requires a password this variable
26 # holds the name of the password file
27 rsyncopts=
28
29 # These are defaults for cvmfs, but can be
30 # overridden for debugging purposes
31 rsyncdest=/cvmfs # the destination once a transaction started
32 rdonlylocation= # the compare-dest before transactions start
33
34 while getopts :r:s:d:S:R:i: OPT; do
35 case $OPT in
36 r|+r)
37 repo="$OPTARG"
38 ;;
39 s|+s)
40 rsyncsrc="$OPTARG"
41 ;;
42 d|+d)
43 rsyncdest="$OPTARG"
44 ;;
45 S|+S)
46 rsyncopts="$rsyncopts --password-file $OPTARG"
47 ;;
48 R|+R)
49 rdonlylocation="$OPTARG"
50 ;;
51 i|+i)
52 inclexclfile="$OPTARG"
53 rsyncopts="$rsyncopts --include-from=$OPTARG"
54 ;;
55 *)
56 echo "usage: `basename $0` -r repository -s source [ -S secretsfile ] [ -i inclexclfile ]"
57 exit 2
58 esac
59 done
60 shift `expr $OPTIND - 1`
61 OPTIND=1
62
63 if [ -z "$repo" ]; then
64 die "missing argument -r repository"
65 elif [ -z "$rsyncsrc" ]; then
66 die "missing argument -s rsyncsource"
67 fi
68
69 # Set the compare-dest location if it was not set with an option.
70 # Can't do this until $repo is set.
71 if [ -z $rdonlylocation ]; then
72 rdonlylocation=/var/spool/cvmfs/$repo/rdonly
73 fi
74
75
76 # Since the entire process may take a long time, we need a lockfile to signal
77 # that the process is already at work.
78
79 LOCKFILE=/tmp/cvmfs-$repo-update.lock
80 exec 9> $LOCKFILE;
81 flock -xn 9 || die "could not lock $LOCKFILE"
82
83 # create a temporary directory to get the rsync 'cvmfs.modified' files
84 tmp=`mktemp -d --tmpdir cvmfs.$repo.XXXXXXXXXX` || die "cannot create temporary directory. Aborting."
85
86 # make sure everything gets cleaned up at the end
87 trap "rm -rf $LOCKFILE $tmp" EXIT;
88
89 # Sanity check: see if we can rsync at all.
90 # This doesn't actually transfer anything.
91 rsync -q --no-recursive $rsyncopts $rsyncsrc /tmp
92 test $? -eq 0 || die "Can't rsync from $rsyncsrc, aborting."
93
94
95 # PHASE 1: find the cvmfs.modified files and build a list
96
97 # base this on the following rules:
98 # rsync -rt --include='/*/' --include='/*/*/' --include='cvmfs.modified' --exclude=\* tree/ newtree
99 # this includes first and second level directories, cvmfs.modified files, but excludes all the rest.
100
101 # FIXME: compare-dest is not populated until we start a transaction
102 # we should compare with the read-only version
103 errstr=`rsync -q -rt $rsyncopts \
104 --filter 'include /*/' \
105 --filter 'include cvmfs.modified' \
106 --filter 'exclude *' \
107 --compare-dest="$rdonlylocation/$repo" \
108 $rsyncsrc/ $tmp/ 2>&1`
109
110 result=$?
111
112 case $result in
113 0) ;;
114 23) ;; # file missing, this is ok
115 *) # other error
116 printf "%s\n" "$errstr"
117 echo "rsync failed. Aborting." >&2
118 exit $result
119 ;;
120 esac
121
122 # Now $tmp contains a copy of the source tree's 'cvmfs.modified', but
123 # only the ones that are newer that the ones we already got in our
124 # older copy in $rsyncdest/$repo.
125 # For each of these, we need to synchronise the corresponding subtree.
126
127 cd $tmp
128 find . -name cvmfs.modified | sed 's,/cvmfs\.modified$,,' | while read p
129 do
130 # Start a transaction, do the rsync and if anything goes wrong abort.
131 echo "Update requested of $p; starting transaction of cvmfs repository $repo"
132 cvmfs_server transaction $repo || die "failed cvmfs transaction on $repo"
133
134 rsync $rsyncopts --filter 'include /cvmfs.modified' \
135 --filter 'include /cvmfs.tree/' \
136 --filter 'include /cvmfs.tree/**' \
137 --filter 'exclude *' \
138 -r -q -t --delete $rsyncsrc/$p/ "$rsyncdest/$repo/$p/"
139
140 if [ $? -ne 0 ] ; then
141 echo "Rsync from $rsyncsrc/$p failed, aborting transaction."
142 cvmfs_server abort -f $repo || die "failed to abort the transaction, exiting"
143 else
144 # The rsync succeeded
145 cvmfs_server publish $repo || die "failed to publish the new CVMFS repo $repo, exiting"
146 fi
147 done
148
149
150
151 echo "cvmfs publishing done for $repo."

grid.support@nikhef.nl
ViewVC Help
Powered by ViewVC 1.1.28