/[pdpsoft]/trunk/nl.nikhef.ndpf.tools/cvmfs/cvmfs-rsync-multi.sh
ViewVC logotype

Annotation of /trunk/nl.nikhef.ndpf.tools/cvmfs/cvmfs-rsync-multi.sh

Parent Directory Parent Directory | Revision Log Revision Log


Revision 2744 - (hide annotations) (download) (as text)
Wed Nov 12 15:20:48 2014 UTC (7 years, 6 months ago) by dennisvd
File MIME type: application/x-shellscript
File size: 4513 byte(s)
Fix a bug where $OPTARG would not be expanded for the password file due to single quotes; add the option to have an include/exclude file.

1 dennisvd 2743 #!/bin/sh
2    
3     # Synchronise the Stratum 0 from an upstream rsync source,
4     # with multiple sync points.
5    
6     # The remote source directory is traversed, searching for files named
7     # cvmfs.modified. If such a file exists, its timestamp is compared to
8     # the timestamp of the same file on the target directory (i.e. on the
9     # stratum-0 side). If the remote file is newer, or if the local file
10     # is absent, the directory is added to the list to synchronise.
11     # This method gives the maintainer the option to synchronise directories
12     # selectively, which is useful if a repository is shared among various
13     # users (or user groups), each with their own directory maintained
14     # independently of the others.
15    
16     die() {
17     echo -e "$1" >&2
18     exit 1
19     }
20    
21     # These parameters must be set on the command line
22     repo=
23     rsyncsrc=
24    
25     # If rsyncsrc requires a password this variable
26     # holds the name of the password file
27     rsyncopts=
28    
29     # These are defaults for cvmfs, but can be
30     # overridden for debugging purposes
31     rsyncdest=/cvmfs # the destination once a transaction started
32     rdonlylocation=/var/spool/cvmfs # the compare-dest before transactions start
33    
34 dennisvd 2744 while getopts :r:s:d:S:R:i: OPT; do
35 dennisvd 2743 case $OPT in
36     r|+r)
37     repo="$OPTARG"
38     ;;
39     s|+s)
40     rsyncsrc="$OPTARG"
41     ;;
42     d|+d)
43     rsyncdest="$OPTARG"
44     ;;
45     S|+S)
46 dennisvd 2744 rsyncopts="$rsyncopts --password-file \"$OPTARG\""
47 dennisvd 2743 ;;
48     R|+R)
49     rdonlylocation="$OPTARG"
50     ;;
51 dennisvd 2744 i|+i)
52     inclexclfile="$OPTARG"
53     rsyncopts="$rsyncopts --include-from=\"$OPTARG\""
54     ;;
55 dennisvd 2743 *)
56 dennisvd 2744 echo "usage: `basename $0` -r repository -s source [ -S secretsfile ] [ -i inclexclfile ]"
57 dennisvd 2743 exit 2
58     esac
59     done
60     shift `expr $OPTIND - 1`
61     OPTIND=1
62    
63     if [ -z "$repo" ]; then
64     die "missing argument -r repository"
65     elif [ -z "$rsyncsrc" ]; then
66     die "missing argument -s rsyncsource"
67     fi
68    
69    
70     # Since the entire process may take a long time, we need a lockfile to signal
71     # that the process is already at work.
72    
73     LOCKFILE=/tmp/cvmfs-$repo-update.lock
74     exec 9> $LOCKFILE;
75     flock -xn 9 || die "could not lock $LOCKFILE"
76    
77     # create a temporary directory to get the rsync 'cvmfs.modified' files
78     tmp=`mktemp -d --tmpdir cvmfs.$repo.XXXXXXXXXX` || die "cannot create temporary directory. Aborting."
79    
80     # make sure everything gets cleaned up at the end
81     trap "rm -rf $LOCKFILE $tmp" EXIT;
82    
83     # Sanity check: see if we can rsync at all.
84     # This doesn't actually transfer anything.
85     rsync -q --no-recursive $rsyncopts $rsyncsrc /tmp
86     test $? -eq 0 || die "Can't rsync from $rsyncsrc, aborting."
87    
88    
89     # PHASE 1: find the cvmfs.modified files and build a list
90    
91     # base this on the following rules:
92     # rsync -rt --include='/*/' --include='/*/*/' --include='cvmfs.modified' --exclude=\* tree/ newtree
93     # this includes first and second level directories, cvmfs.modified files, but excludes all the rest.
94    
95     # FIXME: compare-dest is not populated until we start a transaction
96     # we should compare with the read-only version
97     errstr=`rsync -q -rt $rsyncopts \
98     --filter 'include /*/' \
99     --filter 'include cvmfs.modified' \
100     --filter 'exclude *' \
101     --compare-dest="$rdonlylocation/$repo" \
102     $rsyncsrc/ $tmp/ 2>&1`
103    
104     result=$?
105    
106     case $result in
107     0) ;;
108     23) ;; # file missing, this is ok
109     *) # other error
110     printf "%s\n" "$errstr"
111     echo "rsync failed. Aborting." >&2
112     exit $result
113     ;;
114     esac
115    
116     # Now $tmp contains a copy of the source tree's 'cvmfs.modified', but
117     # only the ones that are newer that the ones we already got in our
118     # older copy in $rsyncdest/$repo.
119     # For each of these, we need to synchronise the corresponding subtree.
120    
121     cd $tmp
122     find . -name cvmfs.modified | sed 's,/cvmfs\.modified$,,' | while read p
123     do
124     # Start a transaction, do the rsync and if anything goes wrong abort.
125     echo "Update requested of $p; starting transaction of cvmfs repository $repo"
126     cvmfs_server transaction $repo || die "failed cvmfs transaction on $repo"
127    
128     rsync $rsyncopts --filter 'include /cvmfs.modified' \
129     --filter 'include /cvmfs.tree/' \
130     --filter 'include /cvmfs.tree/**' \
131     --filter 'exclude *' \
132     -r -q -t --delete $rsyncsrc/$p/ "$rsyncdest/$repo/$p/"
133    
134     if [ $? -ne 0 ] ; then
135     echo "Rsync from $rsyncsrc/$p failed, aborting transaction."
136     cvmfs_server abort -f $repo || die "failed to abort the transaction, exiting"
137     else
138     # The rsync succeeded
139     cvmfs_server publish $repo || die "failed to publish the new CVMFS repo $repo, exiting"
140     fi
141     done
142    
143    
144    
145     echo "cvmfs publishing done for $repo."

grid.support@nikhef.nl
ViewVC Help
Powered by ViewVC 1.1.28