update docs for upcoming change. more tests

README.md

@@ -37,7 +37,7 @@ Since its using ZFS commands, you can see what its actually doing by specifying
 
 An important feature thats missing from other tools is a reliable `--test` option: This allows you to see what zfs-autobackup will do and tune your parameters. It will do everything, except make changes to your zfs datasets.
 
-Another nice thing is progress reporting with `--progress`. Its very useful with HUGE datasets, when you want to know how many hours/days it will take.
+Another nice thing is progress reporting: Its very useful with HUGE datasets, when you want to know how many hours/days it will take.
 
 zfs-autobackup tries to be the easiest to use backup tool for zfs.
 
@@ -61,6 +61,7 @@ zfs-autobackup tries to be the easiest to use backup tool for zfs.
 * Easy to debug and has a test-mode. Actual unix commands are printed.
 * Uses **progressive thinning** for older snapshots.
 * Uses zfs-holds on important snapshots so they cant be accidentally destroyed.
+* Automatic resuming of failed transfers.
 * Easy installation:
   * Just install zfs-autobackup via pip, or download it manually.
   * Written in python and uses zfs-commands, no 3rd party dependency's or libraries.
@@ -296,6 +297,24 @@ The thinner operates "stateless": There is nothing in the name or properties of
 
 Note that the thinner will ONLY destroy snapshots that are matching the naming pattern of zfs-autobackup. If you use `--other-snapshots`, it wont destroy those snapshots after replicating them to the target.
 
+### Destroying missing datasets
+
+When a dataset has been destroyed on the source, but still exists on the target we call it a missing dataset. Missing datasets will be still thinned out according to the schedule.
+
+The final snapshot will never be destroyed, unless you specify a **deadline** with the `--destroy-missing` option:
+
+In that case it will look at the last snapshot we took and determine if is older than the deadline you specified. e.g: `--destroy-missing 30d` will start destroying things 30 days after the last snapshot.
+
+#### After the deadline
+
+When the deadline is passed, all our snapshots, except the last one will be destroyed. Irregardless of the normal thinning schedule.
+
+The dataset has to have the following properties to be finally really destroyed:
+
+* The dataset has no direct child-filesystems or volumes.
+* The only snapshot left is the last one created by zfs-autobackup.
+* The remaining snapshot has no clones.
+
 ### Thinning schedule
 
 The default thinning schedule is: `10,1d1w,1w1m,1m1y`.

test_zfsautobackup.py

@@ -14,6 +14,7 @@ def test_invalidpars(self):
         self.assertEqual(ZfsAutobackup("test test_target1 --keep-source -1".split(" ")).run(), 255)
 
     def  test_snapshotmode(self):
+        """test snapshot tool mode"""
 
         with patch('time.strftime', return_value="20101111000000"):
             self.assertFalse(ZfsAutobackup("test test_target1 --verbose".split(" ")).run())
@@ -24,9 +25,9 @@ def  test_snapshotmode(self):
         with patch('time.strftime', return_value="20101111000002"):
             self.assertFalse(ZfsAutobackup("test --verbose --allow-empty --keep-source 0".split(" ")).run())
 
-        #on source: only has 1 and 2
+        #on source: only has 1 and 2 (1 was hold)
         #on target: has 0 and 1  
-
+        #XXX:
         r=shelltest("zfs list -H -o name -r -t all "+TEST_POOLS)
         self.assertMultiLineEqual(r,"""
 test_source1
@@ -63,8 +64,6 @@ def  test_snapshotmode(self):
     def  test_defaults(self):
 
         with self.subTest("no datasets selected"):
-            #should resume and succeed
-
             with OutputIO() as buf:
                 with redirect_stderr(buf):
                     with patch('time.strftime', return_value="20101111000000"):
@@ -175,7 +174,49 @@ def  test_defaults(self):
 test_target1/test_source2/fs2/sub@test-20101111000001  userrefs  1         -
 """)
 
+        #make sure time handling is correctly. try to make snapshots a year appart and verify that only snapshots mostly 1y old are kept
+        with self.subTest("test time checking"):
+            with patch('time.strftime', return_value="20111111000000"):
+                self.assertFalse(ZfsAutobackup("test test_target1 --allow-empty --verbose".split(" ")).run())
+
+
+            time_str="20111112000000" #month in the "future"
+            future_timestamp=time_secs=time.mktime(time.strptime(time_str,"%Y%m%d%H%M%S"))
+            with patch('time.time', return_value=future_timestamp):
+                with patch('time.strftime', return_value="20111111000001"):
+                    self.assertFalse(ZfsAutobackup("test test_target1 --allow-empty --verbose --keep-source 1y1y --keep-target 1d1y".split(" ")).run())
+
 
+            r=shelltest("zfs list -H -o name -r -t all "+TEST_POOLS)
+            self.assertMultiLineEqual(r,"""
+test_source1
+test_source1/fs1
+test_source1/fs1@test-20111111000000
+test_source1/fs1@test-20111111000001
+test_source1/fs1/sub
+test_source1/fs1/sub@test-20111111000000
+test_source1/fs1/sub@test-20111111000001
+test_source2
+test_source2/fs2
+test_source2/fs2/sub
+test_source2/fs2/sub@test-20111111000000
+test_source2/fs2/sub@test-20111111000001
+test_source2/fs3
+test_source2/fs3/sub
+test_target1
+test_target1/test_source1
+test_target1/test_source1/fs1
+test_target1/test_source1/fs1@test-20111111000000
+test_target1/test_source1/fs1@test-20111111000001
+test_target1/test_source1/fs1/sub
+test_target1/test_source1/fs1/sub@test-20111111000000
+test_target1/test_source1/fs1/sub@test-20111111000001
+test_target1/test_source2
+test_target1/test_source2/fs2
+test_target1/test_source2/fs2/sub
+test_target1/test_source2/fs2/sub@test-20111111000000
+test_target1/test_source2/fs2/sub@test-20111111000001
+""")
 
 
     def  test_ignore_othersnaphots(self):
@@ -757,10 +798,50 @@ def  test_test(self):
 """)
 
 
+    def test_migrate(self):
+        """test migration from other snapshotting systems. zfs-autobackup should be able to continue from any common snapshot, not just its own."""
+
+        shelltest("zfs snapshot test_source1/fs1@migrate1")
+        shelltest("zfs create test_target1/test_source1")
+        shelltest("zfs send  test_source1/fs1@migrate1| zfs recv test_target1/test_source1/fs1")
+
+        with patch('time.strftime', return_value="20101111000000"):
+            self.assertFalse(ZfsAutobackup("test test_target1 --verbose".split(" ")).run())
+
+        r=shelltest("zfs list -H -o name -r -t all "+TEST_POOLS)
+        self.assertMultiLineEqual(r,"""
+test_source1
+test_source1/fs1
+test_source1/fs1@migrate1
+test_source1/fs1@test-20101111000000
+test_source1/fs1/sub
+test_source1/fs1/sub@test-20101111000000
+test_source2
+test_source2/fs2
+test_source2/fs2/sub
+test_source2/fs2/sub@test-20101111000000
+test_source2/fs3
+test_source2/fs3/sub
+test_target1
+test_target1/test_source1
+test_target1/test_source1/fs1
+test_target1/test_source1/fs1@migrate1
+test_target1/test_source1/fs1@test-20101111000000
+test_target1/test_source1/fs1/sub
+test_target1/test_source1/fs1/sub@test-20101111000000
+test_target1/test_source2
+test_target1/test_source2/fs2
+test_target1/test_source2/fs2/sub
+test_target1/test_source2/fs2/sub@test-20101111000000
+""")
+
+
 ###########################
 # TODO:
 
     def  test_raw(self):
 
         self.skipTest("todo: later when travis supports zfs 0.8")
 
+
+