#!/bin/bash

# program to perform backups
# called by at.

# depends: lockfile-progs


# this program will ostensibly be able to fix situations where a snapshot
# was failed to be sent, maybe due to network problems.  missing snapshots
# should be sent, and that includes all of them if none have ever been sent.

declare -A CONFIG
export CONFIG
ETC=/etc
export PRE=/usr/local
export ETCDIR=$ETC/snapsched
export CFGFILE=$ETCDIR/config
export INTCRONFILE_BASE=snapsched
export CRONTAB=$ETC/cron.d/$INTCRONFILE_BASE
export CRONPROG=$PRE/lib/snapsched/scheduled
export TMPBASE=/tmp/ssched

exec 2>&1

export OSILENT=

# load the library routines
. $PRE/lib/snapsched/snapsched-funcs

# send a single snapshot to the backup host
cp_snap()
{
	local SSH_C="$1"
	local SVOL="$2"
	local RECV_DST="$3"
	local SARGS
	local C
	local ES

	shift;shift;shift

	# bring in the clones
	for C in "${SNAP_SENT_LST[@]}" ; do
		SARGS+=" -c ${C#* }"
	done

	# if this is the first snap EVER to be sent, then
	# SARGS will be the the null string, so the entire snap will be sent.
	# otherwise, previously sent snaps will be mentioned as possible
	# clone sources, and btrfs can supposedly figure out the parent
	# relationship.  however, if it fails to do that because the snapshots
	# have been screwed with, then this will fail.

	omsg "sending $SVOL ..."

	# might need an eval here to get SARGS to work right
	eval btrfs send -q $SARGS "$SVOL" | $SSH_C btrfs rec "$RECV_DST" | omsg
	ES=$?

	return $ES
}

# check to see if the snapshot argument (string: "UUID <snap-path>")
# already exists on the backup host
# uses the global array RSNAPS and RX (the number of elements in RSNAPS)
_already_sent()
{
	local S="$1"
	local -i X

	for ((X = 0; X < RX; X++)) ; do
		# check the UUID of the snap with all the UUIDs of already sent snaps
		if [ "${S% *}" = "${RSNAPS[X]% *}" ] ; then
			# it matches, so yes, it's already been sent
			return 0
		fi
	done

	# no matches, so no, it hasn't been sent
	return 1
}


if $OLDBASH ; then
_rsnaps2sent()
{
	local I
	local -i D
	local LSNAP

	for I in "${!SNAP_SENT_LST[@]}" ; do
		unset SNAP_SENT_LST[$I]
	done
	D=0
	for I in "${!RSNAPS[@]}" ; do
		LSNAP=${RSNAPS[$I]#* }
		LSNAP=${LSNAP/`hostname`/${CONFIG[SNAP_BASE_DIR]}}
		if [ -d "${CONFIG[SNAP_MOUNT_DIR]}/$LSNAP" ] ; then
			SNAP_SENT_LST[$D]="$LSNAP"
				D=`expr $D + 1`
		fi
	done
}
else
# create the initial clone source list
_rsnaps2sent()
{
	local -n DARR=$1 SARR=$2
	local I
	local -i D
	local LSNAP

	for I in "${!DARR[@]}" ; do
		unset DARR[$I]
	done
	D=0
	for I in "${!SARR[@]}" ; do
		LSNAP=${SARR[$I]#* }
		LSNAP=${LSNAP/`hostname`/${CONFIG[SNAP_BASE_DIR]}}
		if [ -d "${CONFIG[SNAP_MOUNT_DIR]}/$LSNAP" ] ; then
			DARR[D++]="$LSNAP"
		fi
	done
}
fi


# first arg is snapshot source

NSRC="$1"
shift

# validate NSRC arg, and read in config file
# no usage message, since this is called by cron.  supposedly.
_ssched_validate_nsrc NSRC "" ||
	exit 1

# is this NSRC configured for backup?
if [ -z "${CONFIG["SSRC_${NSRC}%BACKUPINT"]}" ] ; then
	# omsg "No backups configured for filesystem '$NSRC'."
	exit 1
fi

SSRC=${CONFIG[SNAP_MOUNT_DIR]}/$SSRC
BHOST1=${CONFIG["SSRC_${NSRC}%BACKUPHOST1"]}
BHOST1FS=${CONFIG["SSRC_${NSRC}%BACKUPHOST1FS"]}
BHOST1Z=${CONFIG["SSRC_${NSRC}%BACKUPHOST1_Z"]}
FREQ=${CONFIG["SSRC_${NSRC}%BACKUPINT"]}
MTYPE="${CONFIG[SSRC_${NSRC}%BACKUPMTYPE]}"

# check to see if we can see the backup host
# a laptop might be traveling, or backup host offline for maintenance
ping -c 1 $BHOST1 &> /dev/null || {
	omsg "Unable to contact '$BHOST1' backup host."
	exit 1
}

# iffy check, but technically the user shouldn't specify an interval
# that isn't scheduled for snapping
if [ "${CONFIG[SSRC_$NSRC%`_ssched_int2u $FREQ`_MSC]}" -eq 0 ] ; then
	# there's nothing to backup
	exit 0
fi

_ssched_set_verbosity


# mitigate the network traffic usage and the overhead of multiple backups
# sending at the same time.  possible configs are:
#    N simultaneous to the same backup host
#       N simultaneous from the perspective of the backup host - other
#         systems sending to that host besides this one
#    N simultaneous
#    N simultaneous per network segment

LCK_FILE=$SNAP_VAR/snapback_backups_
SLEEP_S=180

mkdir -p `dirname $LCK_FILE` 2>/dev/null

case "${MTYPE% *}" in
	host)
		LCK_FILE=${LCK_FILE}${BHOST1}_
		;&
	backups)
		while : ; do
			NLOCKS=`\ls $LCK_FILE}*.lck 2>/dev/null`
			if [ -z "$NLOCKS" ] ; then
				GLOCK_FILE=${LCK_FILE}1.lck
				_ssched_lockfile_p $GLOCK_FILE
			else
				# wait 3 minutes and see if there's an open slot
				if [ "$NLOCKS" -ge ${MTYPE#* } ] ; then
					sleep $SLEEP_S
					continue
				fi

				for I in 1 2 3 4 ; do
					if [ "$I" -gt "${MTYPE#* }" ] ; then
						break
					fi
					if [ -e "${LCK_FILE}$I" ] ; then
						continue
					fi
					GLOCK_FILE=${LCK_FILE}$I.lck
					_ssched_lockfile_p $GLOCK_FILE
					break
				done
				if [ -z "$GLOCK_FILE" ] ; then
					omsg "$NSRC: failed to find slot when expecting to.  NLOCKS=$NLOCKS"
					sleep $SLEEP_S
					continue
				fi
			fi
			break
		done
		;;
esac


_ssched_mount_rootvol ${CONFIG[SNAP_MOUNT_DIR]} || {
	ES=$?
	omsg "Unable to mount root volume '${CONFIG[SNAP_MOUNT_DIR]}': $ES"
	if [ ! -d "${CONFIG[SNAP_MOUNT_DIR]}" ] ; then
		omsg "Directory  '${CONFIG[SNAP_MOUNT_DIR]}' does not exist."
	elif [ "$ES" -eq 4 ] ; then
		omsg "Perhaps it needs to be configured in /etc/fstab?  Snapsched requires that."
	fi
	exit 1
}

# don't let the mount point be unmounted in the middle of this
cd ${CONFIG[SNAP_MOUNT_DIR]}

TBASE="${CONFIG["SNAP_MOUNT_DIR"]}/${CONFIG["SNAP_BASE_DIR"]}/$NSRC"

BSDIR=$BHOST1FS/`hostname`/$NSRC

# whether or not to use compression with ssh.  generally makes xfers take
# roughly 2.5 times longer on 1Gb ethernet or faster connections
if $BHOST1Z ; then
	SSHZ_ARG=-C
else
	SSHZ_ARG=
fi

# make directories on bhost
ssh -x $BHOST1 mkdir -p $BSDIR 2>/dev/null

declare -i SX RX
declare -a SSNAPS RSNAPS

# get the list of relevant snaps on this host
_ssched_bsub_list "$NSRC" "" SSNAPS "${CONFIG[SNAP_MOUNT_DIR]}"
SX=${#SSNAPS[*]}

if [ -z "${SSNAPS[0]}" ] ; then
	exit
fi

if [ "$SX" -eq 0 ] ; then
	# nothing to do?
	exit 0
fi

# get the list of snaps on the backup host for this nsrc/intval
_ssched_bsub_list "$NSRC" "ssh -x $BHOST1" RSNAPS $BHOST1FS
RX=${#RSNAPS[*]}

declare -a SNAP_SENT_LST
declare -a SNAP_XFER_LST
declare -a TPDIRS

_rsnaps2sent SNAP_SENT_LST RSNAPS

if $OLDBASH ; then
	TPDIRS=(monthly weekly daily)
fi

# create xfer list and possible target directories
for SNAP in "${SSNAPS[@]}" ; do
	# this func uses RSNAPS and RX as global variables
	if _already_sent "$SNAP" ; then
		continue
	fi
	SNAP_XFER_LST+=(${SNAP#* })
	$OLDBASH ||
	_sort -u TPDIRS `egrep -o "hourly|daily|weekly|monthly" <<<"${SNAP#* }"`
done

if [ "${#SNAP_XFER_LST[*]}" -gt 0 ] ; then

	omsg ""
	omsg "backup job for '$NSRC' on `hostname` at `date '+%F %T'`"
	if [ "${#SNAP_XFER_LST[*]}" -gt 1 ] ; then
		V=are
		S=s
	else
		V=is
		S=
	fi
	omsg "There $V ${#SNAP_XFER_LST[*]} snapshot$S to be sent"
	omsg ""
	omsg ""

	# create target directories on backup host if needed
	# broken...
	# TDIR="${TPDIRS[*]}"
	# eval ssh -x $BHOST1 mkdir -p $BSDIR/{${TDIR%,}} >/dev/null

	# send the snaps
	for SNAP in "${SNAP_XFER_LST[@]}" ; do
		D=`dirname $SNAP`
		D=`basename $D`
		# echo "cp_snap 'ssh -x' '$SSHZ_ARG' '$BHOST1' '$SNAP' '$BSDIR/$D'"
		cp_snap "ssh -x $SSHZ_ARG $BHOST1" "$SNAP" "$BSDIR/$D" &&
			SNAP_SENT_LST+=("$SNAP") || {
				omsg -e "Sending failure for snap '$SNAP' BHOST='$BHOST1:$BSDIR/$D'\n" "SNAP_SENT_LST=(${SNAP_SENT_LST[*]})\n"
				omsg -e "SNAP_XFER_LST=(${SNAP_XFER_LST[*]})\n"
				break
			}
	done
fi

# release the mount point
cd - >/dev/null

_ssched_umount_rootvol ${CONFIG[SNAP_MOUNT_DIR]}

if [ "$MTYPE" ] ; then
	_ssched_lockfile_v $GLOCK_FILE
fi

exit
