/[pdpsoft]/trunk/nl.nikhef.ndpf.tools/cvmfs/cvmfs-rsync-multi.sh
ViewVC logotype

Contents of /trunk/nl.nikhef.ndpf.tools/cvmfs/cvmfs-rsync-multi.sh

Parent Directory Parent Directory | Revision Log Revision Log


Revision 2744 - (show annotations) (download) (as text)
Wed Nov 12 15:20:48 2014 UTC (7 years, 2 months ago) by dennisvd
File MIME type: application/x-shellscript
File size: 4513 byte(s)
Fix a bug where $OPTARG would not be expanded for the password file due to single quotes; add the option to have an include/exclude file.

1 #!/bin/sh
2
3 # Synchronise the Stratum 0 from an upstream rsync source,
4 # with multiple sync points.
5
6 # The remote source directory is traversed, searching for files named
7 # cvmfs.modified. If such a file exists, its timestamp is compared to
8 # the timestamp of the same file on the target directory (i.e. on the
9 # stratum-0 side). If the remote file is newer, or if the local file
10 # is absent, the directory is added to the list to synchronise.
11 # This method gives the maintainer the option to synchronise directories
12 # selectively, which is useful if a repository is shared among various
13 # users (or user groups), each with their own directory maintained
14 # independently of the others.
15
16 die() {
17 echo -e "$1" >&2
18 exit 1
19 }
20
21 # These parameters must be set on the command line
22 repo=
23 rsyncsrc=
24
25 # If rsyncsrc requires a password this variable
26 # holds the name of the password file
27 rsyncopts=
28
29 # These are defaults for cvmfs, but can be
30 # overridden for debugging purposes
31 rsyncdest=/cvmfs # the destination once a transaction started
32 rdonlylocation=/var/spool/cvmfs # the compare-dest before transactions start
33
34 while getopts :r:s:d:S:R:i: OPT; do
35 case $OPT in
36 r|+r)
37 repo="$OPTARG"
38 ;;
39 s|+s)
40 rsyncsrc="$OPTARG"
41 ;;
42 d|+d)
43 rsyncdest="$OPTARG"
44 ;;
45 S|+S)
46 rsyncopts="$rsyncopts --password-file \"$OPTARG\""
47 ;;
48 R|+R)
49 rdonlylocation="$OPTARG"
50 ;;
51 i|+i)
52 inclexclfile="$OPTARG"
53 rsyncopts="$rsyncopts --include-from=\"$OPTARG\""
54 ;;
55 *)
56 echo "usage: `basename $0` -r repository -s source [ -S secretsfile ] [ -i inclexclfile ]"
57 exit 2
58 esac
59 done
60 shift `expr $OPTIND - 1`
61 OPTIND=1
62
63 if [ -z "$repo" ]; then
64 die "missing argument -r repository"
65 elif [ -z "$rsyncsrc" ]; then
66 die "missing argument -s rsyncsource"
67 fi
68
69
70 # Since the entire process may take a long time, we need a lockfile to signal
71 # that the process is already at work.
72
73 LOCKFILE=/tmp/cvmfs-$repo-update.lock
74 exec 9> $LOCKFILE;
75 flock -xn 9 || die "could not lock $LOCKFILE"
76
77 # create a temporary directory to get the rsync 'cvmfs.modified' files
78 tmp=`mktemp -d --tmpdir cvmfs.$repo.XXXXXXXXXX` || die "cannot create temporary directory. Aborting."
79
80 # make sure everything gets cleaned up at the end
81 trap "rm -rf $LOCKFILE $tmp" EXIT;
82
83 # Sanity check: see if we can rsync at all.
84 # This doesn't actually transfer anything.
85 rsync -q --no-recursive $rsyncopts $rsyncsrc /tmp
86 test $? -eq 0 || die "Can't rsync from $rsyncsrc, aborting."
87
88
89 # PHASE 1: find the cvmfs.modified files and build a list
90
91 # base this on the following rules:
92 # rsync -rt --include='/*/' --include='/*/*/' --include='cvmfs.modified' --exclude=\* tree/ newtree
93 # this includes first and second level directories, cvmfs.modified files, but excludes all the rest.
94
95 # FIXME: compare-dest is not populated until we start a transaction
96 # we should compare with the read-only version
97 errstr=`rsync -q -rt $rsyncopts \
98 --filter 'include /*/' \
99 --filter 'include cvmfs.modified' \
100 --filter 'exclude *' \
101 --compare-dest="$rdonlylocation/$repo" \
102 $rsyncsrc/ $tmp/ 2>&1`
103
104 result=$?
105
106 case $result in
107 0) ;;
108 23) ;; # file missing, this is ok
109 *) # other error
110 printf "%s\n" "$errstr"
111 echo "rsync failed. Aborting." >&2
112 exit $result
113 ;;
114 esac
115
116 # Now $tmp contains a copy of the source tree's 'cvmfs.modified', but
117 # only the ones that are newer that the ones we already got in our
118 # older copy in $rsyncdest/$repo.
119 # For each of these, we need to synchronise the corresponding subtree.
120
121 cd $tmp
122 find . -name cvmfs.modified | sed 's,/cvmfs\.modified$,,' | while read p
123 do
124 # Start a transaction, do the rsync and if anything goes wrong abort.
125 echo "Update requested of $p; starting transaction of cvmfs repository $repo"
126 cvmfs_server transaction $repo || die "failed cvmfs transaction on $repo"
127
128 rsync $rsyncopts --filter 'include /cvmfs.modified' \
129 --filter 'include /cvmfs.tree/' \
130 --filter 'include /cvmfs.tree/**' \
131 --filter 'exclude *' \
132 -r -q -t --delete $rsyncsrc/$p/ "$rsyncdest/$repo/$p/"
133
134 if [ $? -ne 0 ] ; then
135 echo "Rsync from $rsyncsrc/$p failed, aborting transaction."
136 cvmfs_server abort -f $repo || die "failed to abort the transaction, exiting"
137 else
138 # The rsync succeeded
139 cvmfs_server publish $repo || die "failed to publish the new CVMFS repo $repo, exiting"
140 fi
141 done
142
143
144
145 echo "cvmfs publishing done for $repo."

grid.support@nikhef.nl
ViewVC Help
Powered by ViewVC 1.1.28