nl.nikhef.ndpf.tools/cvmfs/cvmfs-rsync-multi.sh

#!/bin/sh

# Synchronise the Stratum 0 from an upstream rsync source,
# with multiple sync points.

# The remote source directory is traversed, searching for files named
# cvmfs.modified. If such a file exists, its timestamp is compared to
# the timestamp of the same file on the target directory (i.e. on the
# stratum-0 side). If the remote file is newer, or if the local file
# is absent, the directory is added to the list to synchronise.
# This method gives the maintainer the option to synchronise directories
# selectively, which is useful if a repository is shared among various
# users (or user groups), each with their own directory maintained
# independently of the others.

logfile=

die() {
    err=${2:-1}
    if [ -n "$logfile" ]; then
        log "$1"
    fi
    echo -e "$1" >&2
    exit $2
}

# prefix output with a timestamp
log() {
    if [ -n "$logfile" ]; then
        d=`LC_TIME=C date +'%b %d %H:%M:%S'`
        echo $d "$@" >> "$logfile"
    else
        echo "$@"
    fi
}

# These parameters must be set on the command line
repo=
rsyncsrc=

# If rsyncsrc requires a password this variable
# holds the name of the password file
rsyncopts=

# Exclude certain trees based on an excludes-from file
excludeopts=

# These are defaults for cvmfs, but can be
# overridden for debugging purposes
rsyncdest=/cvmfs # the destination once a transaction started
rdonlylocation= # the compare-dest before transactions start

while getopts :r:s:d:S:R:i:l: OPT; do
    case $OPT in
        r|+r)
            repo="$OPTARG"
            ;;
        s|+s)
            rsyncsrc="$OPTARG"
            ;;
        d|+d)
            rsyncdest="$OPTARG"
            ;;
        S|+S)
            rsyncopts="--password-file $OPTARG"
            ;;
        R|+R)
            rdonlylocation="$OPTARG"
            ;;
        i|+i)
            inclexclfile="$OPTARG"
            excludeopts="--exclude-from=$OPTARG"
            ;;
        l|+l)
            logfile="$OPTARG"
        *)
            echo "usage: `basename $0` -r repository -s source [ -S secretsfile ] [ -i inclexclfile ]"
            exit 2
    esac
done
shift `expr $OPTIND - 1`
OPTIND=1

if [ -z "$repo" ]; then
    die "missing argument -r repository"
elif [ -z "$rsyncsrc" ]; then
    die "missing argument -s rsyncsource"
fi

# Set the compare-dest location if it was not set with an option.
# Can't do this until $repo is set.
if [ -z $rdonlylocation ]; then
    rdonlylocation=/var/spool/cvmfs/$repo/rdonly
fi


# Since the entire process may take a long time, we need a lockfile to signal
# that the process is already at work.

LOCKFILE=/tmp/cvmfs-$repo-update.lock
exec 9> $LOCKFILE;
flock -xn 9 || die "could not lock $LOCKFILE"

# create a temporary directory to get the rsync 'cvmfs.modified' files
tmp=`mktemp -d --tmpdir cvmfs.$repo.XXXXXXXXXX` || die "cannot create temporary directory. Aborting."

# make sure everything gets cleaned up at the end
trap "rm -rf $LOCKFILE $tmp" EXIT;

# Sanity check: see if we can rsync at all.
# This doesn't actually transfer anything.
rsync -q --no-recursive $rsyncopts $rsyncsrc /tmp
test $? -eq 0 || die "Can't rsync from $rsyncsrc, aborting."


# PHASE 1: find the cvmfs.modified files and build a list

# base this on the following rules:
#  rsync -rt --include='/*/' --include='/*/*/' --include='cvmfs.modified' --exclude=\* tree/ newtree
# this includes first and second level directories, cvmfs.modified files, but excludes all the rest.

# FIXME: compare-dest is not populated until we start a transaction
# we should compare with the read-only version
errstr=`rsync -q -rt $rsyncopts $excludeopts \
                    --filter 'include /*/' \
                    --filter 'include cvmfs.modified' \
                    --filter 'exclude *' \
                    --compare-dest="$rdonlylocation/" \
        $rsyncsrc/ $tmp/ 2>&1`

result=$?

case $result in
    0)  ;;
    23) ;; # file missing, this is ok
    *) # other error
        log "$errstr"
        die "rsync failed. Aborting." $result
        ;;
esac

# Now $tmp contains a copy of the source tree's 'cvmfs.modified', but
# only the ones that are newer that the ones we already got in our
# older copy in $rsyncdest/$repo.
# For each of these, we need to synchronise the corresponding subtree.

find $tmp -name cvmfs.modified | sed 's,.*/\([^/]*\)/cvmfs\.modified$,\1,' | while read p
do
    # Start a transaction, do the rsync and if anything goes wrong abort.
    log "Update requested of $p; starting transaction of cvmfs repository $repo"
    cvmfs_server transaction $repo || die "failed cvmfs transaction on $repo"
    
    rsync $rsyncopts --filter 'include /cvmfs.modified' \
                     --filter 'include /cvmfs.tree/' \
                     --filter 'include /cvmfs.tree/**' \
                     --filter 'exclude *' \
        -r -q -l -t --delete $rsyncsrc/$p/ "$rsyncdest/$repo/$p/"

    if [ $? -ne 0 ] ; then
        log "Rsync from $rsyncsrc/$p failed, aborting transaction."
        cvmfs_server abort -f $repo || die "failed to abort the transaction, exiting"
    else
        # The rsync succeeded
        cvmfs_server publish $repo || die "failed to publish the new CVMFS repo $repo, exiting"
        log "cvmfs publishing done for $repo, tree $p"
    fi
done


1	dennisvd	2743	#!/bin/sh
2
3			# Synchronise the Stratum 0 from an upstream rsync source,
4			# with multiple sync points.
5
6			# The remote source directory is traversed, searching for files named
7			# cvmfs.modified. If such a file exists, its timestamp is compared to
8			# the timestamp of the same file on the target directory (i.e. on the
9			# stratum-0 side). If the remote file is newer, or if the local file
10			# is absent, the directory is added to the list to synchronise.
11			# This method gives the maintainer the option to synchronise directories
12			# selectively, which is useful if a repository is shared among various
13			# users (or user groups), each with their own directory maintained
14			# independently of the others.
15
16	dennisvd	2758	logfile=
17
18	dennisvd	2743	die() {
19	dennisvd	2758	err=${2:-1}
20			if [ -n "$logfile" ]; then
21			log "$1"
22			fi
23	dennisvd	2743	echo -e "$1" >&2
24	dennisvd	2758	exit $2
25	dennisvd	2743	}
26
27	dennisvd	2758	# prefix output with a timestamp
28			log() {
29			if [ -n "$logfile" ]; then
30			d=`LC_TIME=C date +'%b %d %H:%M:%S'`
31			echo $d "$@" >> "$logfile"
32			else
33			echo "$@"
34			fi
35			}
36
37	dennisvd	2743	# These parameters must be set on the command line
38			repo=
39			rsyncsrc=
40
41			# If rsyncsrc requires a password this variable
42			# holds the name of the password file
43			rsyncopts=
44
45	dennisvd	2751	# Exclude certain trees based on an excludes-from file
46			excludeopts=
47
48	dennisvd	2743	# These are defaults for cvmfs, but can be
49			# overridden for debugging purposes
50			rsyncdest=/cvmfs # the destination once a transaction started
51	dennisvd	2747	rdonlylocation= # the compare-dest before transactions start
52	dennisvd	2743
53	dennisvd	2758	while getopts :r:s:d:S:R:i:l: OPT; do
54	dennisvd	2743	case $OPT in
55			r\|+r)
56			repo="$OPTARG"
57			;;
58			s\|+s)
59			rsyncsrc="$OPTARG"
60			;;
61			d\|+d)
62			rsyncdest="$OPTARG"
63			;;
64			S\|+S)
65	dennisvd	2751	rsyncopts="--password-file $OPTARG"
66	dennisvd	2743	;;
67			R\|+R)
68			rdonlylocation="$OPTARG"
69			;;
70	dennisvd	2744	i\|+i)
71			inclexclfile="$OPTARG"
72	dennisvd	2751	excludeopts="--exclude-from=$OPTARG"
73	dennisvd	2744	;;
74	dennisvd	2758	l\|+l)
75			logfile="$OPTARG"
76	dennisvd	2743	*)
77	dennisvd	2744	echo "usage: `basename $0` -r repository -s source [ -S secretsfile ] [ -i inclexclfile ]"
78	dennisvd	2743	exit 2
79			esac
80			done
81			shift `expr $OPTIND - 1`
82			OPTIND=1
83
84			if [ -z "$repo" ]; then
85			die "missing argument -r repository"
86			elif [ -z "$rsyncsrc" ]; then
87			die "missing argument -s rsyncsource"
88			fi
89
90	dennisvd	2747	# Set the compare-dest location if it was not set with an option.
91			# Can't do this until $repo is set.
92			if [ -z $rdonlylocation ]; then
93			rdonlylocation=/var/spool/cvmfs/$repo/rdonly
94			fi
95	dennisvd	2743
96	dennisvd	2747
97	dennisvd	2743	# Since the entire process may take a long time, we need a lockfile to signal
98			# that the process is already at work.
99
100			LOCKFILE=/tmp/cvmfs-$repo-update.lock
101			exec 9> $LOCKFILE;
102			flock -xn 9 \|\| die "could not lock $LOCKFILE"
103
104			# create a temporary directory to get the rsync 'cvmfs.modified' files
105			tmp=`mktemp -d --tmpdir cvmfs.$repo.XXXXXXXXXX` \|\| die "cannot create temporary directory. Aborting."
106
107			# make sure everything gets cleaned up at the end
108			trap "rm -rf $LOCKFILE $tmp" EXIT;
109
110			# Sanity check: see if we can rsync at all.
111			# This doesn't actually transfer anything.
112			rsync -q --no-recursive $rsyncopts $rsyncsrc /tmp
113			test $? -eq 0 \|\| die "Can't rsync from $rsyncsrc, aborting."
114
115
116			# PHASE 1: find the cvmfs.modified files and build a list
117
118			# base this on the following rules:
119			# rsync -rt --include='//' --include='///' --include='cvmfs.modified' --exclude=\ tree/ newtree
120			# this includes first and second level directories, cvmfs.modified files, but excludes all the rest.
121
122			# FIXME: compare-dest is not populated until we start a transaction
123			# we should compare with the read-only version
124	dennisvd	2751	errstr=`rsync -q -rt $rsyncopts $excludeopts \
125	dennisvd	2743	--filter 'include /*/' \
126			--filter 'include cvmfs.modified' \
127			--filter 'exclude *' \
128	dennisvd	2748	--compare-dest="$rdonlylocation/" \
129	dennisvd	2743	$rsyncsrc/ $tmp/ 2>&1`
130
131			result=$?
132
133			case $result in
134			0) ;;
135			23) ;; # file missing, this is ok
136			*) # other error
137	dennisvd	2758	log "$errstr"
138			die "rsync failed. Aborting." $result
139	dennisvd	2743	;;
140			esac
141
142			# Now $tmp contains a copy of the source tree's 'cvmfs.modified', but
143			# only the ones that are newer that the ones we already got in our
144			# older copy in $rsyncdest/$repo.
145			# For each of these, we need to synchronise the corresponding subtree.
146
147	dennisvd	2750	find $tmp -name cvmfs.modified \| sed 's,./\([^/]\)/cvmfs\.modified$,\1,' \| while read p
148	dennisvd	2743	do
149			# Start a transaction, do the rsync and if anything goes wrong abort.
150	dennisvd	2758	log "Update requested of $p; starting transaction of cvmfs repository $repo"
151	dennisvd	2743	cvmfs_server transaction $repo \|\| die "failed cvmfs transaction on $repo"
152
153			rsync $rsyncopts --filter 'include /cvmfs.modified' \
154			--filter 'include /cvmfs.tree/' \
155			--filter 'include /cvmfs.tree/**' \
156			--filter 'exclude *' \
157	dennisvd	2757	-r -q -l -t --delete $rsyncsrc/$p/ "$rsyncdest/$repo/$p/"
158	dennisvd	2743
159			if [ $? -ne 0 ] ; then
160	dennisvd	2758	log "Rsync from $rsyncsrc/$p failed, aborting transaction."
161	dennisvd	2743	cvmfs_server abort -f $repo \|\| die "failed to abort the transaction, exiting"
162			else
163			# The rsync succeeded
164			cvmfs_server publish $repo \|\| die "failed to publish the new CVMFS repo $repo, exiting"
165	dennisvd	2758	log "cvmfs publishing done for $repo, tree $p"
166	dennisvd	2743	fi
167			done
168
169
170
grid.support@nikhef.nl	ViewVC Help
Powered by ViewVC 1.1.28