/[pdpsoft]/trunk/nl.nikhef.ndpf.tools/cvmfs/cvmfs-rsync-multi.sh
ViewVC logotype

Annotation of /trunk/nl.nikhef.ndpf.tools/cvmfs/cvmfs-rsync-multi.sh

Parent Directory Parent Directory | Revision Log Revision Log


Revision 2758 - (hide annotations) (download) (as text)
Fri Nov 28 14:25:22 2014 UTC (7 years, 5 months ago) by dennisvd
File MIME type: application/x-shellscript
File size: 5005 byte(s)
Add logfile option and logging code

1 dennisvd 2743 #!/bin/sh
2    
3     # Synchronise the Stratum 0 from an upstream rsync source,
4     # with multiple sync points.
5    
6     # The remote source directory is traversed, searching for files named
7     # cvmfs.modified. If such a file exists, its timestamp is compared to
8     # the timestamp of the same file on the target directory (i.e. on the
9     # stratum-0 side). If the remote file is newer, or if the local file
10     # is absent, the directory is added to the list to synchronise.
11     # This method gives the maintainer the option to synchronise directories
12     # selectively, which is useful if a repository is shared among various
13     # users (or user groups), each with their own directory maintained
14     # independently of the others.
15    
16 dennisvd 2758 logfile=
17    
18 dennisvd 2743 die() {
19 dennisvd 2758 err=${2:-1}
20     if [ -n "$logfile" ]; then
21     log "$1"
22     fi
23 dennisvd 2743 echo -e "$1" >&2
24 dennisvd 2758 exit $2
25 dennisvd 2743 }
26    
27 dennisvd 2758 # prefix output with a timestamp
28     log() {
29     if [ -n "$logfile" ]; then
30     d=`LC_TIME=C date +'%b %d %H:%M:%S'`
31     echo $d "$@" >> "$logfile"
32     else
33     echo "$@"
34     fi
35     }
36    
37 dennisvd 2743 # These parameters must be set on the command line
38     repo=
39     rsyncsrc=
40    
41     # If rsyncsrc requires a password this variable
42     # holds the name of the password file
43     rsyncopts=
44    
45 dennisvd 2751 # Exclude certain trees based on an excludes-from file
46     excludeopts=
47    
48 dennisvd 2743 # These are defaults for cvmfs, but can be
49     # overridden for debugging purposes
50     rsyncdest=/cvmfs # the destination once a transaction started
51 dennisvd 2747 rdonlylocation= # the compare-dest before transactions start
52 dennisvd 2743
53 dennisvd 2758 while getopts :r:s:d:S:R:i:l: OPT; do
54 dennisvd 2743 case $OPT in
55     r|+r)
56     repo="$OPTARG"
57     ;;
58     s|+s)
59     rsyncsrc="$OPTARG"
60     ;;
61     d|+d)
62     rsyncdest="$OPTARG"
63     ;;
64     S|+S)
65 dennisvd 2751 rsyncopts="--password-file $OPTARG"
66 dennisvd 2743 ;;
67     R|+R)
68     rdonlylocation="$OPTARG"
69     ;;
70 dennisvd 2744 i|+i)
71     inclexclfile="$OPTARG"
72 dennisvd 2751 excludeopts="--exclude-from=$OPTARG"
73 dennisvd 2744 ;;
74 dennisvd 2758 l|+l)
75     logfile="$OPTARG"
76 dennisvd 2743 *)
77 dennisvd 2744 echo "usage: `basename $0` -r repository -s source [ -S secretsfile ] [ -i inclexclfile ]"
78 dennisvd 2743 exit 2
79     esac
80     done
81     shift `expr $OPTIND - 1`
82     OPTIND=1
83    
84     if [ -z "$repo" ]; then
85     die "missing argument -r repository"
86     elif [ -z "$rsyncsrc" ]; then
87     die "missing argument -s rsyncsource"
88     fi
89    
90 dennisvd 2747 # Set the compare-dest location if it was not set with an option.
91     # Can't do this until $repo is set.
92     if [ -z $rdonlylocation ]; then
93     rdonlylocation=/var/spool/cvmfs/$repo/rdonly
94     fi
95 dennisvd 2743
96 dennisvd 2747
97 dennisvd 2743 # Since the entire process may take a long time, we need a lockfile to signal
98     # that the process is already at work.
99    
100     LOCKFILE=/tmp/cvmfs-$repo-update.lock
101     exec 9> $LOCKFILE;
102     flock -xn 9 || die "could not lock $LOCKFILE"
103    
104     # create a temporary directory to get the rsync 'cvmfs.modified' files
105     tmp=`mktemp -d --tmpdir cvmfs.$repo.XXXXXXXXXX` || die "cannot create temporary directory. Aborting."
106    
107     # make sure everything gets cleaned up at the end
108     trap "rm -rf $LOCKFILE $tmp" EXIT;
109    
110     # Sanity check: see if we can rsync at all.
111     # This doesn't actually transfer anything.
112     rsync -q --no-recursive $rsyncopts $rsyncsrc /tmp
113     test $? -eq 0 || die "Can't rsync from $rsyncsrc, aborting."
114    
115    
116     # PHASE 1: find the cvmfs.modified files and build a list
117    
118     # base this on the following rules:
119     # rsync -rt --include='/*/' --include='/*/*/' --include='cvmfs.modified' --exclude=\* tree/ newtree
120     # this includes first and second level directories, cvmfs.modified files, but excludes all the rest.
121    
122     # FIXME: compare-dest is not populated until we start a transaction
123     # we should compare with the read-only version
124 dennisvd 2751 errstr=`rsync -q -rt $rsyncopts $excludeopts \
125 dennisvd 2743 --filter 'include /*/' \
126     --filter 'include cvmfs.modified' \
127     --filter 'exclude *' \
128 dennisvd 2748 --compare-dest="$rdonlylocation/" \
129 dennisvd 2743 $rsyncsrc/ $tmp/ 2>&1`
130    
131     result=$?
132    
133     case $result in
134     0) ;;
135     23) ;; # file missing, this is ok
136     *) # other error
137 dennisvd 2758 log "$errstr"
138     die "rsync failed. Aborting." $result
139 dennisvd 2743 ;;
140     esac
141    
142     # Now $tmp contains a copy of the source tree's 'cvmfs.modified', but
143     # only the ones that are newer that the ones we already got in our
144     # older copy in $rsyncdest/$repo.
145     # For each of these, we need to synchronise the corresponding subtree.
146    
147 dennisvd 2750 find $tmp -name cvmfs.modified | sed 's,.*/\([^/]*\)/cvmfs\.modified$,\1,' | while read p
148 dennisvd 2743 do
149     # Start a transaction, do the rsync and if anything goes wrong abort.
150 dennisvd 2758 log "Update requested of $p; starting transaction of cvmfs repository $repo"
151 dennisvd 2743 cvmfs_server transaction $repo || die "failed cvmfs transaction on $repo"
152    
153     rsync $rsyncopts --filter 'include /cvmfs.modified' \
154     --filter 'include /cvmfs.tree/' \
155     --filter 'include /cvmfs.tree/**' \
156     --filter 'exclude *' \
157 dennisvd 2757 -r -q -l -t --delete $rsyncsrc/$p/ "$rsyncdest/$repo/$p/"
158 dennisvd 2743
159     if [ $? -ne 0 ] ; then
160 dennisvd 2758 log "Rsync from $rsyncsrc/$p failed, aborting transaction."
161 dennisvd 2743 cvmfs_server abort -f $repo || die "failed to abort the transaction, exiting"
162     else
163     # The rsync succeeded
164     cvmfs_server publish $repo || die "failed to publish the new CVMFS repo $repo, exiting"
165 dennisvd 2758 log "cvmfs publishing done for $repo, tree $p"
166 dennisvd 2743 fi
167     done
168    
169    
170    

grid.support@nikhef.nl
ViewVC Help
Powered by ViewVC 1.1.28