From 3b0e38385f16756f8e6e0e13cb59e068c25110aa Mon Sep 17 00:00:00 2001
From: Turbo Fredriksson <turbo@bayour.com>
Date: Thu, 2 Apr 2015 21:50:53 +0200
Subject: [PATCH 1/7] Implement snapshot replication.

---
 src/zfs-auto-snapshot.8  |  14 +++--
 src/zfs-auto-snapshot.sh | 127 ++++++++++++++++++++++++++++++++++++++-
 2 files changed, 135 insertions(+), 6 deletions(-)

diff --git a/src/zfs-auto-snapshot.8 b/src/zfs-auto-snapshot.8
index 51fdc7f..3c6c8bd 100644
--- a/src/zfs-auto-snapshot.8
+++ b/src/zfs-auto-snapshot.8
@@ -46,11 +46,17 @@ PRE is 'zfs\-auto\-snap' by default.
 \fB\-q\fR, \fB\-\-quiet\fR
 Suppress warnings and notices at the console.
 .TP
-\fB\-\-send\-full\fR=\fIF\fR
-Send zfs full backup. Unimplemented.
+\fB\-\-send\-full\fR=[\fIremote host\fR]:[\fIremote pool\fR]
+Send zfs full backup to remote hostname (or IP address) and put it in  remote pool.
 .TP
-\fB\-\-send\-incr\fR=\fIF\fR
-Send zfs incremental backup. Unimplemented.
+\fB\-\-send\-incr\fR=[\fIremote host\fR]:[\fIremote pool\fR]
+Send zfs incremental backup to remote hostname (or IP address) and put it in remote pool.
+.TP
+\fB\-\-send\-opts\fR=\fIOPTS\fR
+Option(s) passed to 'zfs send'.
+.TP
+\fB\-\-recv\-opts\fR=\fIOPTS\fR
+Option(s) passed to 'zfs receive'.
 .TP
 \fB\-\-sep\fR=\fICHAR\fR
 Use CHAR to separate date stamps in snapshot names.
diff --git a/src/zfs-auto-snapshot.sh b/src/zfs-auto-snapshot.sh
index 2a82c9a..e9d9489 100755
--- a/src/zfs-auto-snapshot.sh
+++ b/src/zfs-auto-snapshot.sh
@@ -34,6 +34,11 @@ opt_keep=''
 opt_label=''
 opt_prefix='zfs-auto-snap'
 opt_recursive=''
+opt_send_type=''
+opt_send_host=''
+opt_recv_pool=''
+opt_send_opts=''
+opt_recv_opts=''
 opt_sep='_'
 opt_setauto=''
 opt_syslog=''
@@ -50,6 +55,7 @@ WARNING_COUNT='0'
 
 # Other global variables.
 SNAPSHOTS_OLD=''
+SNAPS_DONE=''
 
 
 print_usage ()
@@ -66,8 +72,10 @@ print_usage ()
   -l, --label=LAB    LAB is usually 'hourly', 'daily', or 'monthly'.
   -p, --prefix=PRE   PRE is 'zfs-auto-snap' by default.
   -q, --quiet        Suppress warnings and notices at the console.
-      --send-full=F  Send zfs full backup. Unimplemented.
-      --send-incr=F  Send zfs incremental backup. Unimplemented.
+      --send-full=F  Send zfs full backup.
+      --send-incr=F  Send zfs incremental backup.
+      --send-opts=F  Option(s) passed to 'zfs send'.
+      --recv-opts=F  Option(s) passed to 'zfs receive'.
       --sep=CHAR     Use CHAR to separate date stamps in snapshot names.
   -g, --syslog       Write messages into the system log.
   -r, --recursive    Snapshot named filesystem and all descendants.
@@ -144,6 +152,64 @@ do_run () # [argv]
 }
 
 
+find_last_snap () # dset, GLOB
+{
+	local snap="$1"
+	local GLOB="$2"
+
+	local dset="${snap%@*}"
+	local last_snap
+	local jj
+
+	# STEP 1: Go through ALL snapshots that exist, look for exact
+	#         match on dataset/volume (with snapshot matching 'GLOB').
+	for jj in $SNAPSHOTS_OLD
+	do
+		# Check whether this is an old snapshot of the filesystem.
+		if [ -z "${jj#$dset@$GLOB}" ]; then
+			# We want the FIRST one (which is the last in time
+			# before the one we just created in do_snapshot()).
+			# Also, here we just need the snapshot name.
+			last_snap="${jj#*@}"
+			break
+		fi
+	done
+
+	# NOTE: If we don't have any previous snapshots (for example, we've
+	#       just created the first one) we can end up with last_snap=''
+	#       here.
+	#       If we're called with '--send-incr' we have to options:
+	#         1: We change from incremental to full.
+	#         2: We accept that the user have said INCR, and stick with
+	#            it.
+	if [ "$opt_send_type" = "incr" -a -z "$last_snap" ]; then
+		if [ -n "$opt_verbose" ]; then
+			echo > /dev/stderr "WARNING: No previous snapshots exist but we where called"
+			echo > /dev/stderr "         with --send-incr. Can not continue."
+			echo > /dev/stderr "         Please rerun with --send-full."
+		fi
+		return 1
+	fi
+
+	if [ -n "$opt_recursive" -a -n "$last_snap" ]; then
+		# STEP 2: Again, go through ALL snapshots that exists, but this
+		#         time only look for the snapshots that 'starts with'
+		#         the dataset/volume in question AND 'ends with'
+		#         the exact snapshot name/date in step 2.
+		for jj in $SNAPSHOTS_OLD
+		do
+			if echo "$jj" | grep -qE "^$dset.*@$last_snap"; then
+				echo $jj
+			fi
+		done
+	else
+		echo "$snap"
+	fi
+
+	return 0
+}
+
+
 do_snapshots () # properties, flags, snapname, oldglob, [targets...]
 {
 	local PROPS="$1"
@@ -170,6 +236,8 @@ do_snapshots () # properties, flags, snapname, oldglob, [targets...]
 			if [ $RUNSNAP -eq 1 ] && do_run "zfs snapshot $PROPS $FLAGS '$ii@$NAME'"
 			then
 				[ "$opt_post_snapshot" != "" ] && do_run "$opt_post_snapshot $ii $NAME"
+				[ -n "$opt_send_host" ] && SNAPS_DONE="$SNAPS_DONE
+$ii@$NAME"
 				SNAPSHOT_COUNT=$(( $SNAPSHOT_COUNT + 1 ))
 			else
 				WARNING_COUNT=$(( $WARNING_COUNT + 1 ))
@@ -203,6 +271,34 @@ do_snapshots () # properties, flags, snapname, oldglob, [targets...]
 	done
 }
 
+do_send () # snapname, oldglob
+{
+	local NAME="$1"
+	local GLOB="$2"
+	local remote="ssh $opt_send_host zfs receive $opt_recv_opts $opt_recv_pool"
+	local ii
+	local jj
+
+	# STEP 1: Go throug all snapshots we've created
+	for ii in $SNAPS_DONE
+	do
+		opts=''
+		SNAPS_SEND=''
+
+		# STEP 2: Find the last snapshot
+		SNAPS_SEND=$(find_last_snap "$ii" "$GLOB")
+
+		# STEP 3: Go through all snapshots that is to be transfered and send them.
+		for jj in $SNAPS_SEND
+		do
+			if [ "$opt_send_type" = "incr" ]; then
+				do_run "zfs send $opt_send_opts -i $jj $ii | $remote"
+			else
+				do_run "zfs send $opt_send_opts -R $jj | $remote"
+			fi
+		done
+	done
+}
 
 # main ()
 # {
@@ -212,6 +308,7 @@ GETOPT=$(getopt \
   --longoptions=event:,keep:,label:,prefix:,sep: \
   --longoptions=debug,help,quiet,syslog,verbose \
   --longoptions=pre-snapshot:,post-snapshot:,destroy-only \
+  --longoptions=send-full:,send-incr:,send-opts:,recv-opts: \
   --options=dnshe:l:k:p:rs:qgv \
   -- "$@" ) \
   || exit 128
@@ -296,6 +393,30 @@ do
 			opt_recursive='1'
 			shift 1
 			;;
+		(--send-full)
+			opt_send_type='full'
+
+			opt_send_host=$(echo "$2" | sed 's,:.*,,')
+			opt_recv_pool=$(echo "$2" | sed 's,.*:,,')
+
+			opt_send_opts="$opt_send_opts -R"
+			shift 2
+			;;
+		(--send-incr)
+			opt_send_type='incr'
+
+			opt_send_host=$(echo "$2" | sed 's,:.*,,')
+			opt_recv_pool=$(echo "$2" | sed 's,.*:,,')
+			shift 2
+			;;
+		(--send-opts)
+			opt_send_opts="$2"
+			shift 2
+			;;
+		(--recv-opts)
+			opt_recv_opts="$2"
+			shift 2
+			;;
 		(--sep)
 			case "$2" in 
 				([[:alnum:]_.:\ -])
@@ -567,6 +688,8 @@ test -n "$opt_dry_run" \
 do_snapshots "$SNAPPROP" ""   "$SNAPNAME" "$SNAPGLOB" "$TARGETS_REGULAR"
 do_snapshots "$SNAPPROP" "-r" "$SNAPNAME" "$SNAPGLOB" "$TARGETS_RECURSIVE"
 
+do_send "$SNAPNAME" "$SNAPGLOB"
+
 print_log notice "@$SNAPNAME," \
   "$SNAPSHOT_COUNT created," \
   "$DESTRUCTION_COUNT destroyed," \

From 239b5faee2b7e02a02fe43bc394707ed345b5e4a Mon Sep 17 00:00:00 2001
From: Turbo Fredriksson <turbo@bayour.com>
Date: Fri, 3 Apr 2015 17:56:53 +0200
Subject: [PATCH 2/7] Implement --{pre,post}-send option for the snapshot
 replication.

---
 src/zfs-auto-snapshot.8  | 10 ++++++++++
 src/zfs-auto-snapshot.sh | 30 ++++++++++++++++++++++++++----
 2 files changed, 36 insertions(+), 4 deletions(-)

diff --git a/src/zfs-auto-snapshot.8 b/src/zfs-auto-snapshot.8
index 3c6c8bd..57631f1 100644
--- a/src/zfs-auto-snapshot.8
+++ b/src/zfs-auto-snapshot.8
@@ -80,6 +80,16 @@ aborted.
 Command to run after each dataset is snapshotted. 
 It is passed the dataset and snapshot name.
 .TP
+\fB\-\-pre-send\fR=\fICOMMAND\fR
+Command to run before each snapshot is sent to
+remote host. It is passed the snapshot name. If
+it returns non-zero, sending this dataset is
+aborted.
+.TP
+\fB\-\-post-send\fR=\fICOMMAND\fR
+Command to run after each snapshot have been sent.
+It is passed the snapshot name.
+.TP
 \fB\-\-destroy-only\fR
 Do not create new snapshots, but do destroy older 
 snapshots. Has no effect unless used with \fB\-k\fR.
diff --git a/src/zfs-auto-snapshot.sh b/src/zfs-auto-snapshot.sh
index e9d9489..b3ca6b5 100755
--- a/src/zfs-auto-snapshot.sh
+++ b/src/zfs-auto-snapshot.sh
@@ -46,6 +46,8 @@ opt_skip_scrub=''
 opt_verbose=''
 opt_pre_snapshot=''
 opt_post_snapshot=''
+opt_pre_send=''
+opt_post_send=''
 opt_do_snapshots=1
 
 # Global summary statistics.
@@ -275,6 +277,7 @@ do_send () # snapname, oldglob
 {
 	local NAME="$1"
 	local GLOB="$2"
+	local RUNSEND=1
 	local remote="ssh $opt_send_host zfs receive $opt_recv_opts $opt_recv_pool"
 	local ii
 	local jj
@@ -291,10 +294,20 @@ do_send () # snapname, oldglob
 		# STEP 3: Go through all snapshots that is to be transfered and send them.
 		for jj in $SNAPS_SEND
 		do
-			if [ "$opt_send_type" = "incr" ]; then
-				do_run "zfs send $opt_send_opts -i $jj $ii | $remote"
-			else
-				do_run "zfs send $opt_send_opts -R $jj | $remote"
+			if [ -n "$opt_pre_send" ]; then
+				do_run "$opt_pre_send $jj" || RUNSEND=0
+			fi
+
+			if [ $RUNSEND -eq 1 ]; then
+				if [ "$opt_send_type" = "incr" ]; then
+					do_run "zfs send $opt_send_opts -i $jj $ii | $remote" || RUNSEND=0
+				else
+					do_run "zfs send $opt_send_opts -R $jj | $remote" || RUNSEND=0
+				fi
+
+				if [ $RUNSEND = 1 -a -n "$opt_post_send" ]; then
+					do_run "$opt_post_send $jj" || RUNSEND=0
+				fi
 			fi
 		done
 	done
@@ -309,6 +322,7 @@ GETOPT=$(getopt \
   --longoptions=debug,help,quiet,syslog,verbose \
   --longoptions=pre-snapshot:,post-snapshot:,destroy-only \
   --longoptions=send-full:,send-incr:,send-opts:,recv-opts: \
+  --longoptions=pre-send:,post-send: \
   --options=dnshe:l:k:p:rs:qgv \
   -- "$@" ) \
   || exit 128
@@ -451,6 +465,14 @@ do
 			opt_post_snapshot="$2"
 			shift 2
 			;;
+		(--pre-send)
+			opt_pre_send="$2"
+			shift 2
+			;;
+		(--post-send)
+			opt_post_send="$2"
+			shift 2
+			;;
 		(--destroy-only)
 			opt_do_snapshots=''
 			shift 1

From 1196bcf2b5bbb3df523616f8040bd941a9ded713 Mon Sep 17 00:00:00 2001
From: Turbo Fredriksson <turbo@bayour.com>
Date: Fri, 3 Apr 2015 18:20:04 +0200
Subject: [PATCH 3/7] Implement --send-{ssh,mbuf}-opts for the snapshot
 replication.

---
 src/zfs-auto-snapshot.8  | 11 +++++++
 src/zfs-auto-snapshot.sh | 63 +++++++++++++++++++++++++---------------
 2 files changed, 51 insertions(+), 23 deletions(-)

diff --git a/src/zfs-auto-snapshot.8 b/src/zfs-auto-snapshot.8
index 57631f1..00737c2 100644
--- a/src/zfs-auto-snapshot.8
+++ b/src/zfs-auto-snapshot.8
@@ -58,6 +58,17 @@ Option(s) passed to 'zfs send'.
 \fB\-\-recv\-opts\fR=\fIOPTS\fR
 Option(s) passed to 'zfs receive'.
 .TP
+\fB\-\-send\-ssh\-opts\fR=\fIOPTS\fR
+Option(s) passed to 'ssh' in the replication command line.
+.sp
+See \fBssh\fR(1) for availible options.
+.TP
+\fB\-\-send\-mbuf\-opts\fR=\fIOPTS\fR
+Use \fBmbuffer\fR (with these options) between 'zfs send' and
+\'ssh <host> zfs receive <pool>'.
+.sp
+See \fBmbuffer\fR(1) for availible options.
+.TP
 \fB\-\-sep\fR=\fICHAR\fR
 Use CHAR to separate date stamps in snapshot names.
 .TP
diff --git a/src/zfs-auto-snapshot.sh b/src/zfs-auto-snapshot.sh
index b3ca6b5..58cd15f 100755
--- a/src/zfs-auto-snapshot.sh
+++ b/src/zfs-auto-snapshot.sh
@@ -39,6 +39,8 @@ opt_send_host=''
 opt_recv_pool=''
 opt_send_opts=''
 opt_recv_opts=''
+opt_send_ssh_opts=''
+opt_send_mbuf_opts=''
 opt_sep='_'
 opt_setauto=''
 opt_syslog=''
@@ -63,27 +65,30 @@ SNAPS_DONE=''
 print_usage ()
 {
 	echo "Usage: $0 [options] [-l label] <'//' | name [name...]>
-  --default-exclude  Exclude datasets if com.sun:auto-snapshot is unset.
-  -d, --debug        Print debugging messages.
-  -e, --event=EVENT  Set the com.sun:auto-snapshot-desc property to EVENT.
-      --fast         Use a faster zfs list invocation.
-  -n, --dry-run      Print actions without actually doing anything.
-  -s, --skip-scrub   Do not snapshot filesystems in scrubbing pools.
-  -h, --help         Print this usage message.
-  -k, --keep=NUM     Keep NUM recent snapshots and destroy older snapshots.
-  -l, --label=LAB    LAB is usually 'hourly', 'daily', or 'monthly'.
-  -p, --prefix=PRE   PRE is 'zfs-auto-snap' by default.
-  -q, --quiet        Suppress warnings and notices at the console.
-      --send-full=F  Send zfs full backup.
-      --send-incr=F  Send zfs incremental backup.
-      --send-opts=F  Option(s) passed to 'zfs send'.
-      --recv-opts=F  Option(s) passed to 'zfs receive'.
-      --sep=CHAR     Use CHAR to separate date stamps in snapshot names.
-  -g, --syslog       Write messages into the system log.
-  -r, --recursive    Snapshot named filesystem and all descendants.
-  -v, --verbose      Print info messages.
-      --destroy-only Only destroy older snapshots, do not create new ones.
-      name           Filesystem and volume names, or '//' for all ZFS datasets.
+  --default-exclude     Exclude datasets if com.sun:auto-snapshot is unset.
+  -d, --debug           Print debugging messages.
+  -e, --event=EVENT     Set the com.sun:auto-snapshot-desc property to EVENT.
+      --fast            Use a faster zfs list invocation.
+  -n, --dry-run         Print actions without actually doing anything.
+  -s, --skip-scrub      Do not snapshot filesystems in scrubbing pools.
+  -h, --help            Print this usage message.
+  -k, --keep=NUM        Keep NUM recent snapshots and destroy older snapshots.
+  -l, --label=LAB       LAB is usually 'hourly', 'daily', or 'monthly'.
+  -p, --prefix=PRE      PRE is 'zfs-auto-snap' by default.
+  -q, --quiet           Suppress warnings and notices at the console.
+      --send-full=F     Send zfs full backup.
+      --send-incr=F     Send zfs incremental backup.
+      --send-opts=F     Option(s) passed to 'zfs send'.
+      --recv-opts=F     Option(s) passed to 'zfs receive'.
+      --send-ssh-opts   Option(s) passed to 'ssh'.
+      --send-mbuf-opts  Use mbuffer (with these options) between 'zfs send'
+                        and 'ssh <host> zfs receive'.
+      --sep=CHAR        Use CHAR to separate date stamps in snapshot names.
+  -g, --syslog          Write messages into the system log.
+  -r, --recursive       Snapshot named filesystem and all descendants.
+  -v, --verbose         Print info messages.
+      --destroy-only    Only destroy older snapshots, do not create new ones.
+      name              Filesystem and volume names, or '//' for all ZFS datasets.
 " 
 }
 
@@ -278,10 +283,14 @@ do_send () # snapname, oldglob
 	local NAME="$1"
 	local GLOB="$2"
 	local RUNSEND=1
-	local remote="ssh $opt_send_host zfs receive $opt_recv_opts $opt_recv_pool"
+	local remote
 	local ii
 	local jj
 
+	[ -n "$opt_send_mbuf_opts" ] && remote="mbuffer $opt_send_mbuf_opts |"
+	remote="$remote ssh $opt_send_ssh_opts $opt_send_host "
+	remote="$remote zfs receive $opt_recv_opts $opt_recv_pool"
+
 	# STEP 1: Go throug all snapshots we've created
 	for ii in $SNAPS_DONE
 	do
@@ -322,7 +331,7 @@ GETOPT=$(getopt \
   --longoptions=debug,help,quiet,syslog,verbose \
   --longoptions=pre-snapshot:,post-snapshot:,destroy-only \
   --longoptions=send-full:,send-incr:,send-opts:,recv-opts: \
-  --longoptions=pre-send:,post-send: \
+  --longoptions=send-ssh-opts:,send-mbuf-opts:,pre-send:,post-send: \
   --options=dnshe:l:k:p:rs:qgv \
   -- "$@" ) \
   || exit 128
@@ -431,6 +440,14 @@ do
 			opt_recv_opts="$2"
 			shift 2
 			;;
+		(--send-ssh-opts)
+			opt_send_ssh_opts="$2"
+			shift 2
+			;;
+		(--send-mbuf-opts)
+			opt_send_mbuf_opts="$2"
+			shift 2
+			;;
 		(--sep)
 			case "$2" in 
 				([[:alnum:]_.:\ -])

From 41054fc7b5f93b671d365c65cf1c583897ccf9b1 Mon Sep 17 00:00:00 2001
From: Turbo Fredriksson <turbo@bayour.com>
Date: Fri, 3 Apr 2015 18:51:05 +0200
Subject: [PATCH 4/7] Implement --send-fallback for the snapshot replication.
 This allows a --send-incr to fallback to --send-full if it can't find an
 earlier snapshot.

---
 src/zfs-auto-snapshot.8  |  5 +++++
 src/zfs-auto-snapshot.sh | 36 ++++++++++++++++++++++++++++++------
 2 files changed, 35 insertions(+), 6 deletions(-)

diff --git a/src/zfs-auto-snapshot.8 b/src/zfs-auto-snapshot.8
index 00737c2..f98c958 100644
--- a/src/zfs-auto-snapshot.8
+++ b/src/zfs-auto-snapshot.8
@@ -52,6 +52,11 @@ Send zfs full backup to remote hostname (or IP address) and put it in  remote po
 \fB\-\-send\-incr\fR=[\fIremote host\fR]:[\fIremote pool\fR]
 Send zfs incremental backup to remote hostname (or IP address) and put it in remote pool.
 .TP
+\fB\-\-send\-fallback\fR
+Allow to fallback from incremental to full snapshot replication.
+.sp
+This implies \fI-R\fR to \fBzfs send\fR and \fI-F\fR to \fBzfs receive\fR.
+.TP
 \fB\-\-send\-opts\fR=\fIOPTS\fR
 Option(s) passed to 'zfs send'.
 .TP
diff --git a/src/zfs-auto-snapshot.sh b/src/zfs-auto-snapshot.sh
index 58cd15f..02e2ee3 100755
--- a/src/zfs-auto-snapshot.sh
+++ b/src/zfs-auto-snapshot.sh
@@ -41,6 +41,7 @@ opt_send_opts=''
 opt_recv_opts=''
 opt_send_ssh_opts=''
 opt_send_mbuf_opts=''
+opt_send_fallback=''
 opt_sep='_'
 opt_setauto=''
 opt_syslog=''
@@ -78,6 +79,7 @@ print_usage ()
   -q, --quiet           Suppress warnings and notices at the console.
       --send-full=F     Send zfs full backup.
       --send-incr=F     Send zfs incremental backup.
+      --send-fallback   Fallback from incremental to full if needed.
       --send-opts=F     Option(s) passed to 'zfs send'.
       --recv-opts=F     Option(s) passed to 'zfs receive'.
       --send-ssh-opts   Option(s) passed to 'ssh'.
@@ -189,22 +191,33 @@ find_last_snap () # dset, GLOB
 	#         1: We change from incremental to full.
 	#         2: We accept that the user have said INCR, and stick with
 	#            it.
-	if [ "$opt_send_type" = "incr" -a -z "$last_snap" ]; then
+	#       Normally we do point 2, but if --send-fallback is specified,
+	#       we allow it and convert to a full send instead.
+	if [ "$opt_send_type" = "incr" -a -z "$last_snap" -a -z "$opt_send_fallback" ]; then
 		if [ -n "$opt_verbose" ]; then
 			echo > /dev/stderr "WARNING: No previous snapshots exist but we where called"
 			echo > /dev/stderr "         with --send-incr. Can not continue."
 			echo > /dev/stderr "         Please rerun with --send-full."
+			echo > /dev/stderr "         Or use --send-fallback."
 		fi
 		return 1
 	fi
 
-	if [ -n "$opt_recursive" -a -n "$last_snap" ]; then
+	if [ -n "$opt_recursive" ]; then
 		# STEP 2: Again, go through ALL snapshots that exists, but this
 		#         time only look for the snapshots that 'starts with'
 		#         the dataset/volume in question AND 'ends with'
 		#         the exact snapshot name/date in step 2.
 		for jj in $SNAPSHOTS_OLD
 		do
+			# When trying to find snapshots recurively, we MUST have a 'last_snap'
+			# value. Othervise, it will match ALL snapshots for dset (if we had
+			# used '"^$dset.*@$GLOB" only).
+			if [ -z "$last_snap" ] && echo "$jj" | grep -qE "^$dset.*@$GLOB"; then
+				# Use this as last snapshot name
+				last_snap="${jj#*@}"
+			fi
+
 			if echo "$jj" | grep -qE "^$dset.*@$last_snap"; then
 				echo $jj
 			fi
@@ -288,8 +301,8 @@ do_send () # snapname, oldglob
 	local jj
 
 	[ -n "$opt_send_mbuf_opts" ] && remote="mbuffer $opt_send_mbuf_opts |"
-	remote="$remote ssh $opt_send_ssh_opts $opt_send_host "
-	remote="$remote zfs receive $opt_recv_opts $opt_recv_pool"
+	remote="$remote ssh $opt_send_ssh_opts $opt_send_host"
+	remote="$remote zfs receive $opt_recv_opts"
 
 	# STEP 1: Go throug all snapshots we've created
 	for ii in $SNAPS_DONE
@@ -309,9 +322,15 @@ do_send () # snapname, oldglob
 
 			if [ $RUNSEND -eq 1 ]; then
 				if [ "$opt_send_type" = "incr" ]; then
-					do_run "zfs send $opt_send_opts -i $jj $ii | $remote" || RUNSEND=0
+					if [ "$jj" = "$ii" -a -n "$opt_send_fallback" ]; then
+						do_run "zfs send $opt_send_opts -R $ii | $remote -F $opt_recv_pool" \
+							|| RUNSEND=0
+					else
+						do_run "zfs send $opt_send_opts -i $jj $ii | $remote $opt_recv_pool" \
+							|| RUNSEND=0
+					fi
 				else
-					do_run "zfs send $opt_send_opts -R $jj | $remote" || RUNSEND=0
+					do_run "zfs send $opt_send_opts -R $jj | $remote $opt_recv_pool" || RUNSEND=0
 				fi
 
 				if [ $RUNSEND = 1 -a -n "$opt_post_send" ]; then
@@ -332,6 +351,7 @@ GETOPT=$(getopt \
   --longoptions=pre-snapshot:,post-snapshot:,destroy-only \
   --longoptions=send-full:,send-incr:,send-opts:,recv-opts: \
   --longoptions=send-ssh-opts:,send-mbuf-opts:,pre-send:,post-send: \
+  --longoptions=send-fallback \
   --options=dnshe:l:k:p:rs:qgv \
   -- "$@" ) \
   || exit 128
@@ -432,6 +452,10 @@ do
 			opt_recv_pool=$(echo "$2" | sed 's,.*:,,')
 			shift 2
 			;;
+		(--send-fallback)
+			opt_send_fallback=1
+			shift 1
+			;;
 		(--send-opts)
 			opt_send_opts="$2"
 			shift 2

From 705ec33aae0382edb129d61bfe34323d6bff1d29 Mon Sep 17 00:00:00 2001
From: Dan VerWeire <dverweire@gmail.com>
Date: Fri, 3 Apr 2015 16:18:34 -0400
Subject: [PATCH 5/7] Implement --send-only for the snapshot replication. Used
 for skipping snapshotting and only do the replication part.

---
 src/zfs-auto-snapshot.sh | 43 ++++++++++++++++++++++++++--------------
 1 file changed, 28 insertions(+), 15 deletions(-)

diff --git a/src/zfs-auto-snapshot.sh b/src/zfs-auto-snapshot.sh
index 02e2ee3..c44aac6 100755
--- a/src/zfs-auto-snapshot.sh
+++ b/src/zfs-auto-snapshot.sh
@@ -38,6 +38,7 @@ opt_send_type=''
 opt_send_host=''
 opt_recv_pool=''
 opt_send_opts=''
+opt_send_only=''
 opt_recv_opts=''
 opt_send_ssh_opts=''
 opt_send_mbuf_opts=''
@@ -85,6 +86,7 @@ print_usage ()
       --send-ssh-opts   Option(s) passed to 'ssh'.
       --send-mbuf-opts  Use mbuffer (with these options) between 'zfs send'
                         and 'ssh <host> zfs receive'.
+      --send-only       Only send the the most recent snapshot
       --sep=CHAR        Use CHAR to separate date stamps in snapshot names.
   -g, --syslog          Write messages into the system log.
   -r, --recursive       Snapshot named filesystem and all descendants.
@@ -265,29 +267,35 @@ $ii@$NAME"
 			fi 
 		fi
 
+		[ -n "$opt_send_only" ] && tmp=$(find_last_snap "$ii@$NAME" "$GLOB")
+		[ -n "$tmp" ] && SNAPS_DONE="$SNAPS_DONE
+$tmp"
+
 		# Retain at most $opt_keep number of old snapshots of this filesystem,
 		# including the one that was just recently created.
 		test -z "$opt_keep" && continue
 		KEEP="$opt_keep"
 
-		# ASSERT: The old snapshot list is sorted by increasing age.
-		for jj in $SNAPSHOTS_OLD
-		do
-			# Check whether this is an old snapshot of the filesystem.
-			if [ -z "${jj#$ii@$GLOB}" ]
-			then
-				KEEP=$(( $KEEP - 1 ))
-				if [ "$KEEP" -le '0' ]
+		if [ -z "$opt_send_only" ]; then
+			# ASSERT: The old snapshot list is sorted by increasing age.
+			for jj in $SNAPSHOTS_OLD
+			do
+				# Check whether this is an old snapshot of the filesystem.
+				if [ -z "${jj#$ii@$GLOB}" ]
 				then
-					if do_run "zfs destroy $FLAGS '$jj'" 
+					KEEP=$(( $KEEP - 1 ))
+					if [ "$KEEP" -le '0' ]
 					then
-						DESTRUCTION_COUNT=$(( $DESTRUCTION_COUNT + 1 ))
-					else
-						WARNING_COUNT=$(( $WARNING_COUNT + 1 ))
+						if do_run "zfs destroy $FLAGS '$jj'" 
+						then
+							DESTRUCTION_COUNT=$(( $DESTRUCTION_COUNT + 1 ))
+						else
+							WARNING_COUNT=$(( $WARNING_COUNT + 1 ))
+						fi
 					fi
 				fi
-			fi
-		done
+			done
+		fi
 	done
 }
 
@@ -351,7 +359,7 @@ GETOPT=$(getopt \
   --longoptions=pre-snapshot:,post-snapshot:,destroy-only \
   --longoptions=send-full:,send-incr:,send-opts:,recv-opts: \
   --longoptions=send-ssh-opts:,send-mbuf-opts:,pre-send:,post-send: \
-  --longoptions=send-fallback \
+  --longoptions=send-fallback,send-only \
   --options=dnshe:l:k:p:rs:qgv \
   -- "$@" ) \
   || exit 128
@@ -456,6 +464,11 @@ do
 			opt_send_fallback=1
 			shift 1
 			;;
+		(--send-only)
+			opt_send_only=1
+			opt_do_snapshots=''
+			shift 1
+			;;
 		(--send-opts)
 			opt_send_opts="$2"
 			shift 2

From 88725d947da59560f06e3a4d58666202e03771dd Mon Sep 17 00:00:00 2001
From: Turbo Fredriksson <turbo@bayour.com>
Date: Fri, 3 Apr 2015 21:07:59 +0200
Subject: [PATCH 6/7] Implement multiple receive hosts for the snapshot
 replication.

---
 src/zfs-auto-snapshot.8  |  4 +--
 src/zfs-auto-snapshot.sh | 55 +++++++++++++++++++++++-----------------
 2 files changed, 34 insertions(+), 25 deletions(-)

diff --git a/src/zfs-auto-snapshot.8 b/src/zfs-auto-snapshot.8
index f98c958..c8ae42b 100644
--- a/src/zfs-auto-snapshot.8
+++ b/src/zfs-auto-snapshot.8
@@ -46,10 +46,10 @@ PRE is 'zfs\-auto\-snap' by default.
 \fB\-q\fR, \fB\-\-quiet\fR
 Suppress warnings and notices at the console.
 .TP
-\fB\-\-send\-full\fR=[\fIremote host\fR]:[\fIremote pool\fR]
+\fB\-\-send\-full\fR=[\fIremote host\fR]:[\fIremote pool\fR][;...]
 Send zfs full backup to remote hostname (or IP address) and put it in  remote pool.
 .TP
-\fB\-\-send\-incr\fR=[\fIremote host\fR]:[\fIremote pool\fR]
+\fB\-\-send\-incr\fR=[\fIremote host\fR]:[\fIremote pool\fR][;...]
 Send zfs incremental backup to remote hostname (or IP address) and put it in remote pool.
 .TP
 \fB\-\-send\-fallback\fR
diff --git a/src/zfs-auto-snapshot.sh b/src/zfs-auto-snapshot.sh
index c44aac6..5c67f75 100755
--- a/src/zfs-auto-snapshot.sh
+++ b/src/zfs-auto-snapshot.sh
@@ -35,8 +35,7 @@ opt_label=''
 opt_prefix='zfs-auto-snap'
 opt_recursive=''
 opt_send_type=''
-opt_send_host=''
-opt_recv_pool=''
+opt_send=''
 opt_send_opts=''
 opt_send_only=''
 opt_recv_opts=''
@@ -258,7 +257,7 @@ do_snapshots () # properties, flags, snapname, oldglob, [targets...]
 			if [ $RUNSNAP -eq 1 ] && do_run "zfs snapshot $PROPS $FLAGS '$ii@$NAME'"
 			then
 				[ "$opt_post_snapshot" != "" ] && do_run "$opt_post_snapshot $ii $NAME"
-				[ -n "$opt_send_host" ] && SNAPS_DONE="$SNAPS_DONE
+				[ -n "$opt_send" ] && SNAPS_DONE="$SNAPS_DONE
 $ii@$NAME"
 				SNAPSHOT_COUNT=$(( $SNAPSHOT_COUNT + 1 ))
 			else
@@ -304,13 +303,13 @@ do_send () # snapname, oldglob
 	local NAME="$1"
 	local GLOB="$2"
 	local RUNSEND=1
-	local remote
+	local remote_ssh="ssh $opt_send_ssh_opts"
+	local remote_recv="zfs receive $opt_recv_opts"
+	local remote_mbuf=""
 	local ii
 	local jj
 
-	[ -n "$opt_send_mbuf_opts" ] && remote="mbuffer $opt_send_mbuf_opts |"
-	remote="$remote ssh $opt_send_ssh_opts $opt_send_host"
-	remote="$remote zfs receive $opt_recv_opts"
+	[ -n "$opt_send_mbuf_opts" ] && remote_mbuf="mbuffer $opt_send_mbuf_opts |"
 
 	# STEP 1: Go throug all snapshots we've created
 	for ii in $SNAPS_DONE
@@ -329,17 +328,30 @@ do_send () # snapname, oldglob
 			fi
 
 			if [ $RUNSEND -eq 1 ]; then
-				if [ "$opt_send_type" = "incr" ]; then
-					if [ "$jj" = "$ii" -a -n "$opt_send_fallback" ]; then
-						do_run "zfs send $opt_send_opts -R $ii | $remote -F $opt_recv_pool" \
-							|| RUNSEND=0
+				# Go through each option to --send-{incr,full}.
+				# rem=<remote_host>:<remote_pool>
+				for rem in $opt_send; do
+					if [ "$opt_send_type" = "incr" ]; then
+						if [ "$jj" = "$ii" -a -n "$opt_send_fallback" ]; then
+							cmd="zfs send $opt_send_opts -R $ii |"
+							cmd="$cmd $remote_mbuf"
+							cmd="$cmd $remote_ssh ${rem%:*}"
+							cmd="$cmd $remote_recv -F ${rem#*:}"
+						else
+							cmd="zfs send $opt_send_opts -i $jj $ii |"
+							cmd="$cmd $remote_mbuf"
+							cmd="$cmd $remote_ssh ${rem%:*}"
+							cmd="$cmd $remote_recv ${rem#*:}"
+						fi
 					else
-						do_run "zfs send $opt_send_opts -i $jj $ii | $remote $opt_recv_pool" \
-							|| RUNSEND=0
+						cmd="zfs send $opt_send_opts -R $jj |"
+						cmd="$cmd $remote_mbuf"
+						cmd="$cmd $remote_ssh ${rem%:*}"
+						cmd="$cmd $remote_recv ${rem#*:}"
 					fi
-				else
-					do_run "zfs send $opt_send_opts -R $jj | $remote $opt_recv_pool" || RUNSEND=0
-				fi
+
+					do_run "$cmd" || RUNSEND=0
+				done
 
 				if [ $RUNSEND = 1 -a -n "$opt_post_send" ]; then
 					do_run "$opt_post_send $jj" || RUNSEND=0
@@ -446,18 +458,15 @@ do
 			;;
 		(--send-full)
 			opt_send_type='full'
-
-			opt_send_host=$(echo "$2" | sed 's,:.*,,')
-			opt_recv_pool=$(echo "$2" | sed 's,.*:,,')
-
 			opt_send_opts="$opt_send_opts -R"
+			opt_send=$(echo "$2" | sed 's,;,\
+,g')
 			shift 2
 			;;
 		(--send-incr)
 			opt_send_type='incr'
-
-			opt_send_host=$(echo "$2" | sed 's,:.*,,')
-			opt_recv_pool=$(echo "$2" | sed 's,.*:,,')
+			opt_send=$(echo "$2" | sed 's,;,\
+,g')
 			shift 2
 			;;
 		(--send-fallback)

From 074b2e79051e56fa75ea56a12451eb6f7d8d659a Mon Sep 17 00:00:00 2001
From: Turbo Fredriksson <turbo@bayour.com>
Date: Sat, 4 Apr 2015 01:37:58 +0200
Subject: [PATCH 7/7] Implement do_destroy_remotes() to destroy the exact same
 snapshots on remote host(s) which is destroyed on localhost.

---
 src/zfs-auto-snapshot.sh | 22 ++++++++++++++++++++++
 1 file changed, 22 insertions(+)

diff --git a/src/zfs-auto-snapshot.sh b/src/zfs-auto-snapshot.sh
index 5c67f75..7195638 100755
--- a/src/zfs-auto-snapshot.sh
+++ b/src/zfs-auto-snapshot.sh
@@ -61,6 +61,7 @@ WARNING_COUNT='0'
 # Other global variables.
 SNAPSHOTS_OLD=''
 SNAPS_DONE=''
+SNAPS_DESTROY=''
 
 
 print_usage ()
@@ -288,6 +289,8 @@ $tmp"
 						if do_run "zfs destroy $FLAGS '$jj'" 
 						then
 							DESTRUCTION_COUNT=$(( $DESTRUCTION_COUNT + 1 ))
+							[ -n "$opt_send" ] && SNAPS_DESTROY="$SNAPS_DESTROY
+$jj"
 						else
 							WARNING_COUNT=$(( $WARNING_COUNT + 1 ))
 						fi
@@ -361,6 +364,24 @@ do_send () # snapname, oldglob
 	done
 }
 
+
+do_destroy_remotes ()
+{
+	local FLAGS="$1"
+	local ii
+	local remote_ssh="ssh $opt_send_ssh_opts"
+
+	# Go through each option to --send-{incr,full}.
+	# rem=<remote_host>:<remote_pool>
+	for rem in $opt_send; do
+		for ii in $SNAPS_DESTROY
+		do
+			do_run "$remote_ssh ${rem%:*} zfs destroy $FLAGS '$ii'"
+		done
+	done
+}
+
+
 # main ()
 # {
 
@@ -774,6 +795,7 @@ do_snapshots "$SNAPPROP" ""   "$SNAPNAME" "$SNAPGLOB" "$TARGETS_REGULAR"
 do_snapshots "$SNAPPROP" "-r" "$SNAPNAME" "$SNAPGLOB" "$TARGETS_RECURSIVE"
 
 do_send "$SNAPNAME" "$SNAPGLOB"
+do_destroy_remotes
 
 print_log notice "@$SNAPNAME," \
   "$SNAPSHOT_COUNT created," \