Skip to content

Commit

Permalink
implemented --destroy-missing
Browse files Browse the repository at this point in the history
  • Loading branch information
psy0rz committed Jul 17, 2020
1 parent 7b8b536 commit 9fe13a4
Show file tree
Hide file tree
Showing 3 changed files with 198 additions and 13 deletions.
2 changes: 2 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,8 @@ zfs-autobackup tries to be the easiest to use backup tool for zfs.
* Uses **progressive thinning** for older snapshots.
* Uses zfs-holds on important snapshots so they cant be accidentally destroyed.
* Automatic resuming of failed transfers.
* Can continue from existing common snapshots. (e.g. easy migration)
* Gracefully handles destroyed datasets on source.
* Easy installation:
* Just install zfs-autobackup via pip, or download it manually.
* Written in python and uses zfs-commands, no 3rd party dependency's or libraries.
Expand Down
94 changes: 81 additions & 13 deletions bin/zfs-autobackup
Original file line number Diff line number Diff line change
Expand Up @@ -700,10 +700,11 @@ class ZfsDataset():

self.verbose("Destroying")

self.release()
if self.is_snapshot:
self.release()

try:
self.zfs_node.run(["zfs", "destroy", "-d", self.name])
self.zfs_node.run(["zfs", "destroy", self.name])
self.invalidate()
self.force_exists=False
return(True)
Expand Down Expand Up @@ -898,9 +899,9 @@ class ZfsDataset():

@cached_property
def recursive_datasets(self, types="filesystem,volume"):
"""get all datasets recursively under us"""
"""get all (non-snapshot) datasets recursively under us"""

self.debug("Getting all datasets under us")
self.debug("Getting all recursive datasets under us")

names=self.zfs_node.run(tab_split=False, readonly=True, valid_exitcodes=[ 0 ], cmd=[
"zfs", "list", "-r", "-t", types, "-o", "name", "-H", self.name
Expand All @@ -909,6 +910,19 @@ class ZfsDataset():
return(self.from_names(names[1:]))


@cached_property
def datasets(self, types="filesystem,volume"):
"""get all (non-snapshot) datasets directly under us"""

self.debug("Getting all datasets under us")

names=self.zfs_node.run(tab_split=False, readonly=True, valid_exitcodes=[ 0 ], cmd=[
"zfs", "list", "-r", "-t", types, "-o", "name", "-H", "-d", "1", self.name
])

return(self.from_names(names[1:]))


def send_pipe(self, features, prev_snapshot=None, resume_token=None, show_progress=False, raw=False):
"""returns a pipe with zfs send output for this snapshot
Expand Down Expand Up @@ -1534,7 +1548,7 @@ class ZfsNode(ExecuteNode):
selected_filesystems.append(dataset)
dataset.verbose("Selected (inherited selection)")
else:
dataset.verbose("Ignored (already a backup)")
dataset.debug("Ignored (already a backup)")
else:
dataset.verbose("Ignored (only childs)")

Expand Down Expand Up @@ -1572,13 +1586,13 @@ class ZfsAutobackup:
# parser.add_argument('--buffer', default="", help='Use mbuffer with specified size to speedup zfs transfer. (e.g. --buffer 1G) Will also show nice progress output.')


# parser.add_argument('--destroy-stale', action='store_true', help='Destroy stale backups that have no more snapshots. Be sure to verify the output before using this! ')
parser.add_argument('--clear-refreservation', action='store_true', help='Filter "refreservation" property. (recommended, safes space. same as --filter-properties refreservation)')
parser.add_argument('--clear-mountpoint', action='store_true', help='Set property canmount=noauto for new datasets. (recommended, prevents mount conflicts. same as --set-properties canmount=noauto)')
parser.add_argument('--filter-properties', type=str, help='List of properties to "filter" when receiving filesystems. (you can still restore them with zfs inherit -S)')
parser.add_argument('--set-properties', type=str, help='List of propererties to override when receiving filesystems. (you can still restore them with zfs inherit -S)')
parser.add_argument('--rollback', action='store_true', help='Rollback changes to the latest target snapshot before starting. (normally you can prevent changes by setting the readonly property on the target_path to on)')
parser.add_argument('--destroy-incompatible', action='store_true', help='Destroy incompatible snapshots on target. Use with care! (implies --rollback)')
parser.add_argument('--destroy-missing', type=str, default=None, help='Destroy datasets on target that are missing on the source. Specify the time since the last snapshot, e.g: --destroy-missing 30d')
parser.add_argument('--ignore-transfer-errors', action='store_true', help='Ignore transfer errors (still checks if received filesystem exists. useful for acltype errors)')
parser.add_argument('--raw', action='store_true', help='For encrypted datasets, send data exactly as it exists on disk.')

Expand Down Expand Up @@ -1697,17 +1711,71 @@ class ZfsAutobackup:
if self.args.debug:
raise

#also thin target_datasets that are not on the source any more
self.thin_missing_targets(ZfsDataset(target_node, self.args.target_path), target_datasets)


return(fail_count)


def thin_missing_targets(self, target_dataset, used_target_datasets):
"""thin/destroy target datasets that are missing on the source."""

self.debug("Thinning obsolete datasets")
for dataset in ZfsDataset(target_node, self.args.target_path).recursive_datasets:
if dataset not in target_datasets:
dataset.debug("Missing on source")
dataset.thin()

return(fail_count)
for dataset in target_dataset.recursive_datasets:
try:
if dataset not in used_target_datasets:
dataset.debug("Missing on source, thinning")
dataset.thin()

#destroy_missing enabled?
if self.args.destroy_missing!=None:

#cant do anything without our own snapshots
if not dataset.our_snapshots:
if dataset.datasets:
dataset.debug("Destroy missing: ignoring")
else:
dataset.verbose("Destroy missing: has no snapshots made by us. (please destroy manually)")
else:
#past the deadline?
deadline_ttl=ThinnerRule("0s"+self.args.destroy_missing).ttl
now=int(time.time())
if dataset.our_snapshots[-1].timestamp + deadline_ttl > now:
dataset.verbose("Destroy missing: Waiting for deadline.")
else:

dataset.debug("Destroy missing: Removing our snapshots.")

#remove all our snaphots, except last, to safe space in case we fail later on
for snapshot in dataset.our_snapshots[:-1]:
snapshot.destroy(fail_exception=True)

#does it have other snapshots?
has_others=False
for snapshot in dataset.snapshots:
if not snapshot.is_ours():
has_others=True
break

if has_others:
dataset.verbose("Destroy missing: Still in use by other snapshots")
else:
if dataset.datasets:
dataset.verbose("Destroy missing: Still has children here.")
else:
dataset.verbose("Destroy missing.")
dataset.our_snapshots[-1].destroy(fail_exception=True)
dataset.destroy(fail_exception=True)

except Exception as e:
dataset.error("Error during destoy missing ({})".format(str(e)))




def thin_source(self, source_datasets):

self.set_title("Thinning source")

for source_dataset in source_datasets:
Expand Down Expand Up @@ -1752,7 +1820,7 @@ class ZfsAutobackup:
self.set_title("Snapshotting")
source_node.consistent_snapshot(source_datasets, source_node.new_snapshotname(), min_changed_bytes=self.args.min_change)

#if target is specified, we sync the datasets, otherwise we just thin the source.
#if target is specified, we sync the datasets, otherwise we just thin the source. (e.g. snapshot mode)
if self.args.target_path:
fail_count=self.sync_datasets(source_node, source_datasets)
else:
Expand Down
115 changes: 115 additions & 0 deletions test_destroymissing.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,115 @@

from basetest import *


class TestZfsNode(unittest2.TestCase):

def setUp(self):
prepare_zpools()
self.longMessage=True



def test_destroymissing(self):

#initial backup
with patch('time.strftime', return_value="10101111000000"): #1000 years in past
self.assertFalse(ZfsAutobackup("test test_target1 --verbose --no-holds".split(" ")).run())

with patch('time.strftime', return_value="20101111000000"): #far in past
self.assertFalse(ZfsAutobackup("test test_target1 --verbose --no-holds --allow-empty".split(" ")).run())


with self.subTest("Should do nothing yet"):
with OutputIO() as buf:
with redirect_stdout(buf):
self.assertFalse(ZfsAutobackup("test test_target1 --verbose --no-snapshot --destroy-missing 0s".split(" ")).run())

print(buf.getvalue())
self.assertNotIn(": Destroy missing", buf.getvalue())


with self.subTest("Normal destroyed leaf"):
shelltest("zfs destroy -r test_source1/fs1/sub")

#wait for deadline of last snapshot
with OutputIO() as buf:
with redirect_stdout(buf):
#100y: lastest should not be old enough, while second to latest snapshot IS old enough:
self.assertFalse(ZfsAutobackup("test test_target1 --verbose --no-snapshot --destroy-missing 100y".split(" ")).run())

print(buf.getvalue())
self.assertIn(": Waiting for deadline", buf.getvalue())

#past deadline, destroy
with OutputIO() as buf:
with redirect_stdout(buf):
self.assertFalse(ZfsAutobackup("test test_target1 --verbose --no-snapshot --destroy-missing 1y".split(" ")).run())

print(buf.getvalue())
self.assertIn("sub: Destroying", buf.getvalue())


with self.subTest("Leaf with other snapshot still using it"):
shelltest("zfs destroy -r test_source1/fs1")
shelltest("zfs snapshot -r test_target1/test_source1/fs1@other1")


with OutputIO() as buf:
with redirect_stdout(buf):
self.assertFalse(ZfsAutobackup("test test_target1 --verbose --no-snapshot --destroy-missing 0s".split(" ")).run())

print(buf.getvalue())
#should have done the snapshot cleanup for destoy missing:
self.assertIn("fs1@test-10101111000000: Destroying", buf.getvalue())
#but cant finish because still in use:
self.assertIn("fs1: Destroy missing: Still in use", buf.getvalue())

shelltest("zfs destroy test_target1/test_source1/fs1@other1")


with self.subTest("In use by clone"):
shelltest("zfs clone test_target1/test_source1/fs1@test-20101111000000 test_target1/clone1")

with OutputIO() as buf:
with redirect_stdout(buf), redirect_stderr(buf):
self.assertFalse(ZfsAutobackup("test test_target1 --verbose --no-snapshot --destroy-missing 0s".split(" ")).run())

print(buf.getvalue())
#now tries to destroy our own last snapshot (before the final destroy of the dataset)
self.assertIn("fs1@test-20101111000000: Destroying", buf.getvalue())
#but cant finish because still in use:
self.assertIn("fs1: Error during destoy missing", buf.getvalue())

shelltest("zfs destroy test_target1/clone1")


with self.subTest("Should leave test_source1 parent"):

with OutputIO() as buf:
with redirect_stdout(buf), redirect_stderr(buf):
self.assertFalse(ZfsAutobackup("test test_target1 --verbose --no-snapshot --destroy-missing 0s".split(" ")).run())

print(buf.getvalue())
#should have done the snapshot cleanup for destoy missing:
self.assertIn("fs1: Destroying", buf.getvalue())

with OutputIO() as buf:
with redirect_stdout(buf), redirect_stderr(buf):
self.assertFalse(ZfsAutobackup("test test_target1 --verbose --no-snapshot --destroy-missing 0s".split(" ")).run())

print(buf.getvalue())
#on second run it sees the dangling ex-parent but doesnt know what to do with it (since it has no own snapshot)
self.assertIn("test_source1: Destroy missing: has no snapshots made by us.", buf.getvalue())

r=shelltest("zfs list -H -o name -r -t all test_target1")
self.assertMultiLineEqual(r,"""
test_target1
test_target1/test_source1
test_target1/test_source2
test_target1/test_source2/fs2
test_target1/test_source2/fs2/sub
test_target1/test_source2/fs2/sub@test-10101111000000
test_target1/test_source2/fs2/sub@test-20101111000000
""")

0 comments on commit 9fe13a4

Please sign in to comment.