zrep_sync

##### File: zrep_sync

# contains meat of the "sync" level operations, which deal with
# data transfer.
# basic snap routines, and init routines, are in zrep_snap



## file-internal routine that gets used a lot in zrep_sync
_gensentprop(){
	typeset timeinsec=`zrep_gettimeinseconds`
	print "${ZREPTAG}:sent=$timeinsec"
}


# This is a RECOVERY ROUTINE ONLY.
# I put lots of sanity checking in here, that doesnt make sense to keep
# with a more general case internal routine
# Certain people say that for some odd reason on their systems, the 
# zfs send completes, but zrep gets killed before zrep updates properties.
# To help people save the time on resyncing hundreds of TB, 
# give them a way to update the sent property.
# This only works with newstyle ZFS that allows property setting on snapshots
# Needs to follow whatever is done in _sync(), after the zfs send
#
# Usage:   zrep_syncsync [-L] fs@snap
#
zrep_sentsync(){
	typeset local=0
	if [[ "$1" == "-L" ]] ; then
		local=1;
		shift
	fi

	typeset srcsnap="$1"
	typeset srcfs=${srcsnap%@*}
	typeset snapname=${srcsnap#*@}

	if (( ! Z_HAS_SNAPPROPS )) ; then
		zrep_errquit This sentsync operation only supported with modern ZFS implementations
	fi

	zfs list -t snapshot "$1" >/dev/null 2>&1
	if [[ $? -ne 0 ]] ; then
		zrep_errquit Expected snapshot for $1. Cannot continue
	fi
	case "$1" in
		*@${ZREPTAG}_[0-9]*)
		:
		;;
		*)
		zrep_errquit $1 does not follow zrep naming standards. Cannot continue
		;;
	esac

	typeset sentcheck=`$ZFSGETLVAL ${ZREPTAG}:sent $srcfs`
	if [[ "$sentcheck" != "" ]] ; then
		zrep_errquit ${ZREPTAG}:sent already present on $srcfs
	fi

	typeset desthost destfs
	desthost=`$ZFSGETVAL ${ZREPTAG}:dest-host $srcfs`
	destfs=`$ZFSGETVAL ${ZREPTAG}:dest-fs $srcfs`

	typeset	senttimeprop="`_gensentprop`"

	if ((local == 0)) ; then
		zrep_ssh $desthost zfs set $senttimeprop $destfs@$snapname
	fi

	zfs set $senttimeprop  ${srcsnap}
	# This will be redundant for recovery, but crucial for when
	# user is trying to convert existing snapshot to
	# zrep snapshot.
	# Make sure it matches  zrep_init
	zfs set ${ZREPTAG}:master=yes	${srcfs}

	
}

####################
# synctosnap: called by zrep_sync, if a specific snapshot is specified.
#
# This LOCAL side, *and*  REMOTE side, match up with local zrep_created 
# snapshot. ...
# 
# Note that it uses zrep_lock_fs
#
# WARNING: if we force other side to roll to snap.... 
#  we should NOT BE SYNCING ANY more.
# At the moment, it is up to the user to ensure that nothing is going on 
# locally, and future zrep syncs wont just effectively roll forward again
#  on the remote side.
# zrep sync jobs  should probably be halted, until it is decided that
# you want to sync again.
#
# In the future, I should support some kind of "pause" option, for 
#   zrep sync all    to ignore a rolled back filesystem
#
#
synctosnap(){
	typeset srcsnap=$1 destfs=$2 desthost=$3
	typeset newsentlist

	typeset srcfs snapname destsnap
	if [[ "$desthost" == "" ]] ; then
		print ERROR: synctosnap did not receive all required args
		zrep_errquit "args=$@"
	fi
	srcfs=${srcsnap%@*}
	snapname=${srcsnap#*@}
	destsnap=${snapname}

	# Have to enforce OUR syntax. otherwise, any future attempt to
	# continue sync will fail.
	#  ( getlastsnap() wont find it! )
	#
	case $snapname in
		zrep_[0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f]*)
		: ;;
		*)
			zrep_errquit $srcsnap is not zrep snapshot. Cannot roll with it.
		;;
	esac
		
		

	print Validating remote snap
	zrep_ssh $desthost zfs list -t snapshot $destfs@$destsnap >/dev/null
	if [[ $? -ne 0 ]] ; then
		zrep_errquit  $destfs@$destsnap does not exist. Cannot roll to snap
	fi
	

	print "WARNING: We will be rolling back  $destfs, on $desthost"
	print -n "   to  $snapname, made at: "
	zfs get -H -o value creation $srcsnap
	print ""
	print "All newer snapshots on remote side will be destroyed"
	print "You should have paused ongoing sync jobs for $destfs before continuing"
	print "Continuing in 20 seconds...."
	sleep 10
	print "Continuing in 10 seconds...."
	sleep 10

	zrep_lock_fs $srcfs || zrep_errquit "Cannot lock $srcfs"
	zrep_ssh $desthost zfs rollback -Rr $destfs@$destsnap || zrep_errquit roll failed

	print $desthost:$destfs rolled back successfully to $destsnap
	print Now cleaning up local snapshots

	# need to undo whatever zrep_sync does 
	newsentlist=`getallsnaps $srcfs|sed "1,/@$snapname/d"`
	for snap in $newsentlist ; do
		zfs inherit ${ZREPTAG}:sent $snap
	done

	zrep_unlock_fs $srcfs
	
}

#
# called by _sync
# Check if there have been changes since specified snap
# If no, then return 0 == true [no changes ]
#
_sync_nochanges(){
	typeset changecheck tmpfile="/tmp/zrep.$$.c"

	rm -f $tmpfile
	zfs diff -H $1  >$tmpfile
	if [[ $? -ne 0 ]] ; then
		_errprint _sync_nochanges: zfs diff command unrecognized
		rm -f $tmpfile
		return 1
	fi
	if test -s $tmpfile ; then
		rm -f $tmpfile
		return 1
	fi

	_debugprint sync_nochanges did not find any changes
	rm -f $tmpfile
	return 0
}

# Usage: _snapandsync  fs desthost destfs
# internal routine called by zrep_sync and zrep_failover,
# to do an incremental send.
# You must hold filesystem lock before calling this
# WE DO NOT DO ANY SAFETY OR LOCK CHECKS HERE.
#
# Wil create a new snap on srcfs, and sync it over to given destination
# Sets our 'synced' marker on it as well.
#
_snapandsync(){
	typeset srcfs=$1 desthost=$2 destfs=$3

	typeset sentsnap newsnap snapname



	#srchost=`$ZFSGETVAL ${ZREPTAG}:src-host $srcfs`
	#if [[ "$srchost" != "$Z_LOCAL_HOST" ]] ; then
	#	zrep_errquit _sync: We are not master for $srcfs
	#fi


	# Find incremental send starting point
	# Do this BEFORE creating new snap, because we should make new snap
	# if we cant do incremental anyway
	sentsnap=`getlastsnapsent $srcfs`
	if [[ "$sentsnap" == "" ]] ; then
		print zrep_sync could not find sent snap for $srcfs. 
		zrep_errquit You must initialize $srcfs for zrep
	fi

	if [[ "$ZREP_CHANGEDONLY" != "" ]] ; then
		if _sync_nochanges $sentsnap ; then
		_debugprint No changes found in $srcfs. Updating timestamp only
		typeset timeinsec=`zrep_gettimeinseconds`
		typeset senttimeprop="${ZREPTAG}:sent=$timeinsec"

		if (( Z_HAS_SNAPPROPS )) ; then
			zfs set $senttimeprop  ${sentsnap}
		else
			#note that this is only for old-ZFS compatibility.
			# We dont really want to use this style if possible!
			zfs set ${ZREPTAG}:lastsenttime=${timeinsec} $srcfs
		fi
		return 0
		fi
	fi

	newsnap=`makesnap $srcfs`
	if [[ "$newsnap" == "" ]] ; then
		zrep_errquit zrep_sync could not create new snapshot for $srcfs
	fi

	_sync $srcfs $desthost $destfs $sentsnap $newsnap

}

# called by _snapandsync, and also zrep_synconly
# Usage: _sync sourcefs destinationhost destinationfs (lastsent (newsnap))
_sync(){
	typeset force
	if [[ "$ZREP_FORCE" == "-f" ]] ; then
		force=-F
	fi

	typeset srcfs=$1 desthost=$2 destfs=$3
	typeset lastsent=$4 newsnap=$5
	typeset snapname

	if [[ "$lastsent" == "" ]] ; then
		lastsent=`getlastsnapsent $srcfs`
		if [[ "$lastsent" == "" ]] ; then
			print zrep_sync could not find sent snap for $srcfs. 
			zrep_errquit You must initialize $srcfs for zrep
		fi
	fi

	if [[ "$newsnap" == "" ]] ; then
		newsnap=`getlastsnap $srcfs`
		if [[ "$newsnap" == "" ]] ; then
			print zrep_sync could not find sent snap for $srcfs. 
			zrep_errquit You must initialize $srcfs for zrep
		fi
	fi

	if [[ "$newsnap" == "$lastsent" ]] ; then
		print $newsnap already sent
		return 0
	fi

	snapname=${newsnap#*@}
	typeset timeinsec=`zrep_gettimeinseconds`
	typeset senttimeprop="${ZREPTAG}:sent=$timeinsec"

	print sending $newsnap to $desthost:$destfs

	# Note: doing "-o $senttimeprop" sets prop on FILESYSTEM, not snap.
	# So we dont do that usually

	# other than zrep_init, this should be the ONLY place we do a send
	#   Sigh. but now we also do in _refreshpull
	if [[ "$BBCP" != "" ]] ; then
	        SENDCMD="zfs send ${ZREP_R} ${ZREP_INC_FLAG} $lastsent $newsnap"
		$BBCP -N io "$SENDCMD" \
		   "$desthost:zfs recv $force $destfs"
	else
		eval zfs send ${ZREP_R} ${ZREP_INC_FLAG} $lastsent $newsnap ${Z_F_OUT} | 
		   zrep_ssh $desthost "${Z_F_IN} zfs recv $force $destfs"
	fi

	# I rename this to _unsent rather than just delete, in case people are using zrep
	# for the DUAL use, of replication,
	# plus convenient user-based "oops" recovery from the automatic .zfs/snapshots directory
	if [[ $? -ne 0 ]] ; then
		zfs rename ${Z_SNAP_R} ${newsnap} ${newsnap}_unsent
		zrep_errquit Problem doing sync for $newsnap. renamed to ${newsnap}_unsent
	fi


	##################################################
	##### Okay. data sync completed. Now register that fact with ZFS properties.
	##### If you modify below here, you also need to update zrep_sentsync

	#Even if we are "old mode", other side may not be.
	# So try newer way first.
	zrep_ssh $desthost zfs set $senttimeprop $destfs@$snapname
	if [[ $? -ne 0 ]] ; then
		print WARNING: setting ${ZREPTAG}:sent failed on $desthost:$destfs@$snapname
		print Using fallback methods. You should go patch $destfs to have newer ZFS version
		zrep_ssh $desthost zfs set ${ZREPTAG}:lastsent=${newsnap} $destfs
		zrep_ssh $desthost zfs set ${ZREPTAG}:lastsenttime=${timeinsec} $destfs
	fi

	if (( Z_HAS_SNAPPROPS )) ; then
		zfs set $senttimeprop  ${newsnap}
	else
		#note that this is only for old-ZFS compatibility.
		# We dont really want to use this style if possible!
		zfs set ${ZREPTAG}:lastsent=${newsnap} $srcfs
		zfs set ${ZREPTAG}:lastsenttime=${timeinsec} $srcfs
	fi
}


#User entrypoint, for synconly, which is the pair of snaponly
zrep_synconly(){
#	annoyingly..need to make this almost identical to our current full
#	zrep_sync. but just skipping first steps :(
#	we can skip retries, though.
	typeset srcfs desthost destfs

	# at one point, accept multiple args. But not for now...?
	srcfs=$1

	[[ "$srcfs" == "" ]] && zrep_errquit No fileystem specified for synconly

	desthost=`$ZFSGETVAL ${ZREPTAG}:dest-host $srcfs`
	destfs=`$ZFSGETVAL ${ZREPTAG}:dest-fs $srcfs`
	if [[ $? -ne 0 ]] || [[ "$desthost" == "-" ]] || [[ "$destfs" == "-" ]];
	then
		zrep_errquit Problem getting zrep properties for fs $srcfs
	fi

	zrep_lock_fs $srcfs
	if [[ $? -ne 0 ]] ; then
		zrep_errquit Failed to acquire zrep lock for $srcfs
	fi

	_sync $srcfs $desthost $destfs || zrep_errquit sync failed for $srcfs

	_expire $srcfs	#dont care so much if this fails

	zrep_unlock_fs $srcfs
}


#zrep_sync
# make a new snapshot and copy it over.
# Usage: zrep_sync [-q quiettime] (all|fs1 .. fsX)
# See workflow.txt 
# Will call synctosnap if a snapshot is given instead of fsname
# Normally, will bail out if another instance of zrep holds lock.
# -q option says to check last update time of locked filesystems.
#   If sync more recent than given quiettime, then quietly ignore
#   
zrep_sync(){
	typeset srcfs destfs desthost sentsnap newsnap
	typeset quiettime=0

	if [[ "$1" == "-c" ]] ; then
		export ZREP_CHANGEDONLY="yes"
		shift
	fi

	if [[ "$1" == "-q" ]] ; then
		quiettime="$2"
		shift
		shift
		if (( quiettime < 30 )) ; then
			zrep_errquit "-q must use value greater than 30"
		fi
	fi

	if [[ "$1" == "all" ]] ; then
		set -- `zrep_list_master`
		if [[ "$1" == "" ]] ; then
			# Stay quiet, so we dont spew if in cron
			#print No zrep mastered filesystems found
			exit
		fi
	fi

	if [[ "$1" == "" ]] ; then
		_errprint Error: no filesystems specified for sync
		return 1
	fi

	while [[ "$1" != "" ]] ; do
	srcfs="$1" 

	[[ "$srcfs" == "" ]] && zrep_errquit No fileystem specified for sync

	desthost=`$ZFSGETVAL ${ZREPTAG}:dest-host $srcfs`
	destfs=`$ZFSGETVAL ${ZREPTAG}:dest-fs $srcfs`
	if [[ $? -ne 0 ]] || [[ "$desthost" == "-" ]] || [[ "$destfs" == "-" ]];
	then
		zrep_errquit Problem getting zrep properties for fs $srcfs
	fi
	
	case $srcfs in
		*@*)
			synctosnap $srcfs $destfs $desthost
			return
			;;
	esac

	zrep_lock_fs $srcfs

	if [[ $? -ne 0 ]] ; then
		# retry for lock for a while, if (quiettime>0 )

		if ((quiettime==0)); then
			zrep_errquit Cannot lock $srcfs. Cannot continue
		fi
		typeset currtime=`zrep_gettimeinseconds`  snaptime elapsed

		snaptime=`getlastsynctime $srcfs`
		if (( snaptime == 0 )) ; then
			zrep_errquit quiet mode set, but no last snap for $srcfs
		fi

		elapsed=$((currtime - snaptime))
		if ((elapsed > quiettime)) ; then
			_debugprint $elapsed seconds have elapsed since last sync of $srcfs
			zrep_errquit quiet time limit of $quiettime seconds exceeded for busy fs $srcfs
		else
			print Quiet mode: skipping busy fs $srcfs at `date`
			return 
		fi
	fi

	_snapandsync $srcfs $desthost $destfs


	# Make this message match what zrep_expire uses..
	print Expiring zrep snaps on $srcfs
	_expire $srcfs

	zrep_unlock_fs $srcfs

	shift

	done

}



# zrep_refresh is a "pull" version of "zrep_sync"
# The concept is a bit of a hack.
# It primarily exists so people can run a secure backup server, that
# has ssh access to all hosts, but not vice versa
#
# Implementation is a bit sketchy. 
# For initial, non-optimal run, perhaps take advantage of
#    ssh host zrep synconly
# to avoid too much duplication of things?
# but will still need to set all the perms n things. Nastyyy..
# The MAIN nastiness, is that all our locks are on the "master" side.
# Which depends on the PID still being there!!
# But if we start now running things on the "slave" side.. 
# There is potential for problems
# Examine critical points and reasons for lock:
#   1. while doing analysis of which snap to send
#   2. to avoid paralel "zfs send"s running.
#   3. for update of timestamp
#
#   We can still wrap #1 and #2 in a single lock call. 
#    (and still on the src side!)
#   The ugly comes when updating zrep:sent. Dont want to update wrong snap!
#   So long as we do some kind of check to see that we're not going
#   backwards when we get lock a second time ... we should be relatively okay.
#   However.. for simplicity... going to just cross fingers and wrap
#   all three in single remote lock call, through _refreshpull
#
zrep_refresh(){
	typeset srcfs destfs desthost newsnap newseq master
	typeset force

	if [[ "$ZREP_FORCE" == "-f" ]] ; then
		force=-F
	fi

	# for now, just handle ONE arg, not multiple fs list

	destfs="$1"
	if [[ "$1" == "" ]] ; then
		_errprint Error: no filesystems specified for refresh
		return 1
	fi

	master=`$ZFSGETLVAL ${ZREPTAG}:master $destfs`
	if [[ "$master" == "yes" ]] ; then
		zrep_errquit Sorry, you cant run refresh on a master mode fs $destfs
	fi

	srchost=`$ZFSGETVAL ${ZREPTAG}:src-host $destfs`
	srcfs=`$ZFSGETVAL ${ZREPTAG}:src-fs $destfs`

	zrep_lock_fs $destfs
	if [[ $? -ne 0 ]] ; then
		zrep_errquit Cannot lock $destfs. Cannot continue
	fi

	_debugprint refresh step 1: Going to $srchost to snapshot $destfs
	newsnap=`zrep_ssh $srchost $ZREP_PATH snaponly $srcfs`
	if [[ $? -ne 0 ]] ; then
		zrep_errquit snap of src $srcfs on $srchost failed
	fi
	# yes, MORE paranoia..
	case $newsnap in
		*@${ZREPTAG}_*)
			newseq=${newsnap#*@}
			;;
		*)
			zrep_errquit Unrecognized output from src snap. Cannot continue
			;;					
	esac
	
	typeset	senttimeprop="`_gensentprop`"

	_debugprint refresh step 2: Pulling $newsnap

	if [[ "$BBCP" != "" ]] ; then
		$BBCP "$srchost:$ZREP_PATH _refreshpull $newsnap" \
		  "zfs recv $force $destfs"
	else
		zrep_ssh $srchost "$ZREP_PATH _refreshpull $newsnap ${Z_F_OUT}" |
		  eval ${Z_F_IN} zfs recv $force $destfs
	fi
	if [[ $? -ne 0 ]] ; then
		zrep_errquit Unforseen error pulling snapshot $newsnap from $srchost
	fi

	zfs set $senttimeprop $destfs@$newseq
	if [[ $? -ne 0 ]] ; then
		_errprint WARNING: expected local copy $destfs@newseq does not exist
	fi

	_debugprint Running local expires on $destfs
	_expire $destfs
	_debugprint Running remote expires on $srcfs
	zrep_ssh $srchost "$ZREP_PATH expire -L $srcfs"

	zrep_unlock_fs $destfs

}

# Hidden command-line option for "zrep refresh"
# This is the "remote call" to support zrep refresh
#    ( aka  zrep_refresh  )
# In principle, its kinda like "zrep expire" being callable by
# both the user, and the program itself. 
# However, this routine is definitely not supposed to be user visible
#   .. eh... maybe someday. but initial design is "private"
_refreshpull(){
	typeset fs snapname lastsent latest

	snapname="$1"
	fs=${snapname%@*}

	# Keep in mind that stdin/out is busy so have to use stderr.
	# Cant use regular debugprint
	if [[ "$DEBUG" != "" ]] ; then
	     _errprint _refreshpull: snapname=$snapname, fs=$fs
	fi

	zrep_lock_fs $fs
	if [[ $? -ne 0  ]] ; then
		zrep_errquit Could not lock $fs
	fi

	#We should now;
	# 1. compare to latest snap. quit if not latest
	# 2. get timestamp
	# 3. trigger a zfs send
	# 4.  set timestamp if no errors.
	#    I think it is reasonable to presume that if the receive failed,
	#    we will see an error by the pipe blowing up.
	#

	lastsent=`getlastsnapsent $fs`
	if [[ "$lastsent" == "" ]] ; then
		zrep_errquit Canthappen: _refreshpull cant findlastsent snap
	fi
	latest=`getlastsnap $fs`

	if [[ "$latest" != "$snapname" ]] ; then
		zrep_errquit Sync error: $snapname is not latest snap for $fs
	fi

	typeset senttimeprop="`_gensentprop`"

	if (( Z_HAS_SNAPPROPS ==0)) ; then
		zrep_errquit Error: we currently only support modern ZFS that allows setting props on snaps
	fi

	zfs send ${ZREP_R} ${ZREP_INC_FLAG} $lastsent $latest

	if [[ $? -ne 0 ]] ; then
		zrep_errquit Some kind of error during sending. Bailing out of _refreshpull
	fi


	zfs set $senttimeprop $latest
	zrep_unlock_fs $fs
}

# _expire: 
#   get rid of "old" snapshots for a specifically named filesystem
#
# Note0: you must hold local(master) fs lock first
#
# Note1: expire BOTH SIDES, if we are master
# Keep in mind that sometimes master and dest are on same system
# 
# Note2: Be sure to NEVER delete most recent sent snapshot!!

# INTERNAL routine. For external-facing routine, see zrep_expire
_expire(){
	typeset savecount currcount lastsent remotehost remotefs sanity
	typeset tmpfile=$ZREP_RUNDIR/zrep_expire.$$
	typeset local=0 master

	if [[ "$1" == "-L" ]] ; then
		local=1;
		shift
	fi

	master=`$ZFSGETLVAL ${ZREPTAG}:master $1`

	zrep_has_fs_lock $1 || zrep_errquit zrep_expire Internal Err caller did not hold fs lock on $1

	# Allow propagated values as well as local, for savecount
	savecount=`$ZFSGETVAL ${ZREPTAG}:savecount $1`
	# do not use (()) in case value unset
	if [[ $savecount < 1 ]] ; then
		zrep_errquit ${ZREPTAG}:savecount on $1 set to improper value $savecount
	fi

	if [[ "$master" == "yes" ]] ; then
		lastsent=`getlastsnapsent $1`
		if [[ "$lastsent" == "" ]] ; then
			zrep_errquit corrupted zrep data: no last sent detected. Stopping expire
		fi

		getallsnaps $1 |egrep -v $lastsent >$tmpfile
		savecount=$((savecount-1))
	else
		getallsnaps $1 >$tmpfile
	fi
	currcount=`wc -l < $tmpfile`

	if ((currcount > savecount )) ; then
		currcount=$((currcount - savecount))

		head -$currcount $tmpfile >$tmpfile.2
		mv $tmpfile.2 $tmpfile
		for snap in `cat $tmpfile` ; do
			_debugprint expiring $snap
			# Paranoia is good.
			case $snap in
			     *@*)
			    zfs destroy -r $snap
			    ;;
			    *)
			    zrep_errquit "Expire was about to destroy NON-snapshot $snap"
			    ;;
			esac
		done
	fi
	rm $tmpfile
	

	if [[ "$master" != "yes" ]] || ((local ==1)) ; then
		#This fs is dest fs. We are done.
		return
		#otherwise, go expire on remote side as well
	fi

	remotehost=`$ZFSGETVAL ${ZREPTAG}:dest-host $1`
	remotefs=`$ZFSGETVAL ${ZREPTAG}:dest-fs $1`
	print Also running expire on $remotehost:$remotefs now...
	sanity=`zrep_ssh $remotehost $ZFSGETLVAL ${ZREPTAG}:master $remotefs`

	# Normally, dont quit on error. But this is super-bad.
	if [[ "$sanity" == "yes" ]] ; then
		zrep_errquit "Remote side also marked as master ** $remotehost:$remotefs"
	fi

	zrep_ssh $remotehost "$ZREP_PATH expire $remotefs" ||print REMOTE expire failed
}


# top-level user-facing routine.
# expire old snaps for some or all zrep filesystems.
# Different ways of calling:
#   zrep expire all	Run expire on all zrep fs
#   zrep expire		Run expire on zrep fs we are master for, plus remote
#   zrep expire	-L	Run expire on zrep fs we are master for. SKIP remote
#   zrep expire	 fs ..	Run expire only on fs, plus remote if it is a master
#   zrep expire	-L fs 	Run expire only on fs. Skip remote
#
# If no arg given, expire only filesystems we are master for
# If "all" given, expire literally all.
#
zrep_expire()
{
	typeset local
	if [[ "$1" == "-L" ]] ; then
		local="-L"
		shift
	fi

	if [[ "$1" == "all" ]] ; then
		set -- `zrep_list`
	elif [[ "$1" == "" ]] ; then
		set -- `zrep_list_master`
	fi

	# Note: we should continue if we hit problems with an individual
	# filesystem. Otherwise we risk letting server selfdestruct fill
	# over one troublesome filesystem
	#
	while [[ "$1" != "" ]] ; do
		zrep_lock_fs $1
		print Expiring zrep snaps on $1
		 _expire $local $1 || print WARNING: expire failed for $1
		zrep_unlock_fs $1
		shift
	done
	
}