diff --git a/CHANGELOG.md b/CHANGELOG.md index 7a2a12146..031a2b6b9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added * Add support for Python 3.9 +* Support for bucket to bucket sync ### Removed * Drop Python 2 and Python 3.4 support :tada: diff --git a/b2sdk/b2http.py b/b2sdk/b2http.py index 40e8a4dd0..0de3eeffa 100644 --- a/b2sdk/b2http.py +++ b/b2sdk/b2http.py @@ -246,7 +246,7 @@ class B2Http(object): """ # timeout for HTTP GET/POST requests - TIMEOUT = 130 + TIMEOUT = 900 # 15 minutes as server-side copy can take time def __init__(self, requests_module=None, install_clock_skew_hook=True): """ diff --git a/b2sdk/bucket.py b/b2sdk/bucket.py index d0facde52..4530b4a34 100644 --- a/b2sdk/bucket.py +++ b/b2sdk/bucket.py @@ -687,10 +687,11 @@ def copy( """ copy_source = CopySource(file_id, offset=offset, length=length) - if length is None: + if not length: # TODO: it feels like this should be checked on lower level - eg. RawApi validate_b2_file_name(new_file_name) - return self.api.services.upload_manager.copy_file( + progress_listener = progress_listener or DoNothingProgressListener() + return self.api.services.copy_manager.copy_file( copy_source, new_file_name, content_type=content_type, diff --git a/b2sdk/sync/action.py b/b2sdk/sync/action.py index 1efc5c5dc..13124e9cd 100644 --- a/b2sdk/sync/action.py +++ b/b2sdk/sync/action.py @@ -119,11 +119,15 @@ def do_action(self, bucket, reporter): :type bucket: b2sdk.bucket.Bucket :param reporter: a place to report errors """ + if reporter: + progress_listener = SyncFileReporter(reporter) + else: + progress_listener = None bucket.upload( UploadSourceLocalFile(self.local_full_path), self.b2_file_name, file_info={SRC_LAST_MODIFIED_MILLIS: str(self.mod_time_millis)}, - progress_listener=SyncFileReporter(reporter) + progress_listener=progress_listener ) def do_report(self, bucket, reporter): @@ -238,10 +242,15 @@ def do_action(self, bucket, reporter): if not os.path.isdir(parent_dir): raise Exception('could not create directory %s' % (parent_dir,)) + if reporter: + progress_listener = SyncFileReporter(reporter) + else: + progress_listener = None + # Download the file to a .tmp file download_path = self.local_full_path + '.b2.sync.tmp' download_dest = DownloadDestLocalFile(download_path) - bucket.download_file_by_id(self.file_id, download_dest, SyncFileReporter(reporter)) + bucket.download_file_by_id(self.file_id, download_dest, progress_listener) # Move the file into place try: @@ -267,6 +276,74 @@ def __str__(self): ) +class B2CopyAction(AbstractAction): + """ + File copying action. + """ + + def __init__( + self, relative_name, b2_file_name, file_id, dest_b2_file_name, mod_time_millis, size + ): + """ + :param str relative_name: a relative file name + :param str b2_file_name: a name of a remote file + :param str file_id: a file ID + :param str dest_b2_file_name: a name of a destination remote file + :param int mod_time_millis: file modification time in milliseconds + :param int size: a file size + """ + self.relative_name = relative_name + self.b2_file_name = b2_file_name + self.file_id = file_id + self.dest_b2_file_name = dest_b2_file_name + self.mod_time_millis = mod_time_millis + self.size = size + + def get_bytes(self): + """ + Return file size. + + :rtype: int + """ + return self.size + + def do_action(self, bucket, reporter): + """ + Perform the copying action, returning only after the action is completed. + + :param bucket: a Bucket object + :type bucket: b2sdk.bucket.Bucket + :param reporter: a place to report errors + """ + if reporter: + progress_listener = SyncFileReporter(reporter) + else: + progress_listener = None + + bucket.copy( + self.file_id, + self.dest_b2_file_name, + length=self.size, + progress_listener=progress_listener + ) + + def do_report(self, bucket, reporter): + """ + Report the copying action performed. + + :param bucket: a Bucket object + :type bucket: b2sdk.bucket.Bucket + :param reporter: a place to report errors + """ + reporter.print_completion('copy ' + self.relative_name) + + def __str__(self): + return ( + 'b2_copy(%s, %s, %s, %d)' % + (self.b2_file_name, self.file_id, self.dest_b2_file_name, self.mod_time_millis) + ) + + class B2DeleteAction(AbstractAction): def __init__(self, relative_name, b2_file_name, file_id, note): """ diff --git a/b2sdk/sync/folder.py b/b2sdk/sync/folder.py index 714ba0414..a7b53ba39 100644 --- a/b2sdk/sync/folder.py +++ b/b2sdk/sync/folder.py @@ -166,7 +166,7 @@ def ensure_present(self): if not os.path.exists(self.root): try: os.mkdir(self.root) - except: + except OSError: raise Exception('unable to create directory %s' % (self.root,)) elif not os.path.isdir(self.root): raise Exception('%s is not a directory' % (self.root,)) @@ -232,7 +232,8 @@ def _walk_relative_paths(cls, local_dir, b2_dir, reporter, policies_manager): continue if policies_manager.exclude_all_symlinks and os.path.islink(local_path): - reporter.symlink_skipped(local_path) + if reporter is not None: + reporter.symlink_skipped(local_path) continue if os.path.isdir(local_path): diff --git a/b2sdk/sync/policy.py b/b2sdk/sync/policy.py index 4fd4c5d53..149c678b5 100644 --- a/b2sdk/sync/policy.py +++ b/b2sdk/sync/policy.py @@ -14,7 +14,7 @@ import logging from ..exception import DestFileNewer -from .action import LocalDeleteAction, B2DeleteAction, B2DownloadAction, B2HideAction, B2UploadAction +from .action import LocalDeleteAction, B2CopyAction, B2DeleteAction, B2DownloadAction, B2HideAction, B2UploadAction from .exception import InvalidArgument ONE_DAY_IN_MS = 24 * 60 * 60 * 1000 @@ -305,6 +305,60 @@ class DownAndKeepDaysPolicy(DownPolicy): pass +class CopyPolicy(AbstractFileSyncPolicy): + """ + File is copied (server-side). + """ + DESTINATION_PREFIX = 'b2://' + SOURCE_PREFIX = 'b2://' + + def _make_transfer_action(self): + return B2CopyAction( + self._source_file.name, + self._source_folder.make_full_path(self._source_file.name), + self._source_file.latest_version().id_, + self._dest_folder.make_full_path(self._source_file.name), + self._get_source_mod_time(), + self._source_file.latest_version().size, + ) + + +class CopyAndDeletePolicy(CopyPolicy): + """ + File is copied (server-side) and the delete flag is SET. + """ + + def _get_hide_delete_actions(self): + for action in super()._get_hide_delete_actions(): + yield action + for action in make_b2_delete_actions( + self._source_file, + self._dest_file, + self._dest_folder, + self._transferred, + ): + yield action + + +class CopyAndKeepDaysPolicy(CopyPolicy): + """ + File is copied (server-side) and the keepDays flag is SET. + """ + + def _get_hide_delete_actions(self): + for action in super()._get_hide_delete_actions(): + yield action + for action in make_b2_keep_days_actions( + self._source_file, + self._dest_file, + self._dest_folder, + self._transferred, + self._keep_days, + self._now_millis, + ): + yield action + + def make_b2_delete_note(version, index, transferred): """ Create a note message for delete action. diff --git a/b2sdk/sync/policy_manager.py b/b2sdk/sync/policy_manager.py index ff82f4a4e..61e43db96 100644 --- a/b2sdk/sync/policy_manager.py +++ b/b2sdk/sync/policy_manager.py @@ -8,8 +8,9 @@ # ###################################################################### -from .policy import DownAndDeletePolicy, DownAndKeepDaysPolicy, DownPolicy -from .policy import UpAndDeletePolicy, UpAndKeepDaysPolicy, UpPolicy +from .policy import CopyAndDeletePolicy, CopyAndKeepDaysPolicy, CopyPolicy, \ + DownAndDeletePolicy, DownAndKeepDaysPolicy, DownPolicy, UpAndDeletePolicy, \ + UpAndKeepDaysPolicy, UpPolicy class SyncPolicyManager(object): @@ -87,10 +88,19 @@ def get_policy_class(self, sync_type, delete, keep_days): return DownAndKeepDaysPolicy else: return DownPolicy - assert False, 'invalid sync type: %s, keep_days: %s, delete: %s' % ( - sync_type, - keep_days, - delete, + elif sync_type == 'b2-to-b2': + if delete: + return CopyAndDeletePolicy + elif keep_days: + return CopyAndKeepDaysPolicy + else: + return CopyPolicy + raise NotImplementedError( + 'invalid sync type: %s, keep_days: %s, delete: %s' % ( + sync_type, + keep_days, + delete, + ) ) diff --git a/b2sdk/sync/report.py b/b2sdk/sync/report.py index 28947521c..68eebec80 100644 --- a/b2sdk/sync/report.py +++ b/b2sdk/sync/report.py @@ -18,7 +18,7 @@ logger = logging.getLogger(__name__) -class SyncReport(object): +class SyncReport: """ Handle reporting progress for syncing. @@ -45,8 +45,8 @@ def __init__(self, stdout, no_progress): self.stdout = stdout self.no_progress = no_progress self.start_time = time.time() - self.local_file_count = 0 - self.local_done = False + self.total_count = 0 + self.total_done = False self.compare_done = False self.compare_count = 0 self.total_transfer_files = 0 # set in end_compare() @@ -109,9 +109,9 @@ def _update_progress(self): self._last_update_time = now time_delta = time.time() - self.start_time rate = 0 if time_delta == 0 else int(self.transfer_bytes / time_delta) - if not self.local_done: + if not self.total_done: message = ' count: %d files compare: %d files updated: %d files %s %s' % ( - self.local_file_count, + self.total_count, self.compare_count, self.transfer_files, format_and_scale_number(self.transfer_bytes, 'B'), @@ -120,7 +120,7 @@ def _update_progress(self): elif not self.compare_done: message = ' compare: %d/%d files updated: %d files %s %s' % ( self.compare_count, - self.local_file_count, + self.total_count, self.transfer_files, format_and_scale_number(self.transfer_bytes, 'B'), format_and_scale_number(rate, 'B/s') @@ -128,7 +128,7 @@ def _update_progress(self): else: message = ' compare: %d/%d files updated: %d/%d files %s %s' % ( self.compare_count, - self.local_file_count, + self.total_count, self.transfer_files, self.total_transfer_files, format_and_scale_fraction(self.transfer_bytes, self.total_transfer_bytes, 'B'), @@ -169,23 +169,23 @@ def _print_line(self, line, newline): self.current_line = line self.stdout.flush() - def update_local(self, delta): + def update_total(self, delta): """ - Report that more local files have been found. + Report that more files have been found for comparison. :param delta: number of files found since the last check :type delta: int """ with self.lock: - self.local_file_count += delta + self.total_count += delta self._update_progress() - def end_local(self): + def end_total(self): """ - Local file count is done. Can proceed to step 2. + Total files count is done. Can proceed to step 2. """ with self.lock: - self.local_done = True + self.total_done = True self._update_progress() def update_compare(self, delta): @@ -251,6 +251,30 @@ def local_permission_error(self, path): def symlink_skipped(self, path): pass + @property + def local_file_count(self): + # TODO: Deprecated. Should be removed in v2 + return self.total_count + + @local_file_count.setter + def local_file_count(self, value): + # TODO: Deprecated. Should be removed in v2 + self.total_count = value + + @property + def local_done(self): + # TODO: Deprecated. Should be removed in v2 + return self.total_done + + @local_done.setter + def local_done(self, value): + # TODO: Deprecated. Should be removed in v2 + self.total_done = value + + # TODO: Deprecated. Should be removed in v2 + update_local = update_total + end_local = end_total + class SyncFileReporter(AbstractProgressListener): """ @@ -300,13 +324,13 @@ def sample_sync_report_run(): sync_report = SyncReport(sys.stdout, False) for i in range(20): - sync_report.update_local(1) + sync_report.update_total(1) time.sleep(0.2) if i == 10: sync_report.print_completion('transferred: a.txt') if i % 2 == 0: sync_report.update_compare(1) - sync_report.end_local() + sync_report.end_total() for i in range(10): sync_report.update_compare(1) diff --git a/b2sdk/sync/sync.py b/b2sdk/sync/sync.py index 7b2b94f4a..310e64ac5 100644 --- a/b2sdk/sync/sync.py +++ b/b2sdk/sync/sync.py @@ -73,7 +73,7 @@ def zip_folders(folder_a, folder_b, reporter, policies_manager=DEFAULT_SCAN_MANA current_b = next_or_none(iter_b) -def count_files(local_folder, reporter): +def count_files(local_folder, reporter, policies_manager): """ Count all of the files in a local folder. @@ -82,9 +82,9 @@ def count_files(local_folder, reporter): """ # Don't pass in a reporter to all_files. Broken symlinks will be reported # during the next pass when the source and dest files are compared. - for _ in local_folder.all_files(None): - reporter.update_local(1) - reporter.end_local() + for _ in local_folder.all_files(None, policies_manager=policies_manager): + reporter.update_total(1) + reporter.end_total() @unique @@ -171,11 +171,17 @@ def sync_folders(self, source_folder, dest_folder, now_millis, reporter): :param int now_millis: current time in milliseconds :param b2sdk.sync.report.SyncReport,None reporter: progress reporter """ + source_type = source_folder.folder_type() + dest_type = dest_folder.folder_type() + + if source_type != 'b2' and dest_type != 'b2': + raise ValueError('Sync between two local folders is not supported!') + # For downloads, make sure that the target directory is there. - if dest_folder.folder_type() == 'local' and not self.dry_run: + if dest_type == 'local' and not self.dry_run: dest_folder.ensure_present() - if source_folder.folder_type() == 'local' and not self.allow_empty_source: + if source_type == 'local' and not self.allow_empty_source: source_folder.ensure_non_empty() # Make an executor to count files and run all of the actions. This is @@ -189,37 +195,26 @@ def sync_folders(self, source_folder, dest_folder, now_millis, reporter): queue_limit = self.max_workers + 1000 sync_executor = BoundedQueueExecutor(unbounded_executor, queue_limit=queue_limit) - # First, start the thread that counts the local files. That's the operation - # that should be fastest, and it provides scale for the progress reporting. - local_folder = None - if source_folder.folder_type() == 'local': - local_folder = source_folder - if dest_folder.folder_type() == 'local': - local_folder = dest_folder - if local_folder is None: - raise ValueError('neither folder is a local folder') - if reporter: - sync_executor.submit(count_files, local_folder, reporter) - - # Schedule each of the actions - bucket = None - if source_folder.folder_type() == 'b2': - bucket = source_folder.bucket - if dest_folder.folder_type() == 'b2': - bucket = dest_folder.bucket - if bucket is None: - raise ValueError('neither folder is a b2 folder') - total_files = 0 - total_bytes = 0 + if source_type == 'local' and reporter is not None: + # Start the thread that counts the local files. That's the operation + # that should be fastest, and it provides scale for the progress reporting. + sync_executor.submit(count_files, source_folder, reporter, self.policies_manager) + + # Bucket for scheduling actions. + # For bucket-to-bucket sync, the bucket for the API calls should be the destination. + action_bucket = None + if dest_type == 'b2': + action_bucket = dest_folder.bucket + elif source_type == 'b2': + action_bucket = source_folder.bucket + + # Schedule each of the actions. for action in self.make_folder_sync_actions( source_folder, dest_folder, now_millis, reporter, self.policies_manager ): - logging.debug('scheduling action %s on bucket %s', action, bucket) - sync_executor.submit(action.run, bucket, reporter, self.dry_run) - total_files += 1 - total_bytes += action.get_bytes() - if reporter: - reporter.end_compare(total_files, total_bytes) + logging.debug('scheduling action %s on bucket %s', action, action_bucket) + sync_executor.submit(action.run, action_bucket, reporter, self.dry_run) + # Wait for everything to finish sync_executor.shutdown() if sync_executor.get_num_exceptions() != 0: @@ -250,9 +245,11 @@ def make_folder_sync_actions( source_type = source_folder.folder_type() dest_type = dest_folder.folder_type() sync_type = '%s-to-%s' % (source_type, dest_type) - if (source_type, dest_type) not in [('b2', 'local'), ('local', 'b2')]: - raise NotImplementedError("Sync support only local-to-b2 and b2-to-local") + if source_type != 'b2' and dest_type != 'b2': + raise ValueError('Sync between two local folders is not supported!') + total_files = 0 + total_bytes = 0 for source_file, dest_file in zip_folders( source_folder, dest_folder, @@ -264,12 +261,12 @@ def make_folder_sync_actions( elif dest_file is None: logger.debug('determined that %s is not present on destination', source_file) - if source_folder.folder_type() == 'local': - if source_file is not None: - reporter.update_compare(1) - else: - if dest_file is not None: - reporter.update_compare(1) + if source_file is not None: + if source_type == 'b2': + # For buckets we don't want to count files separately as it would require + # more API calls. Instead, we count them when comparing. + reporter.update_total(1) + reporter.update_compare(1) for action in self.make_file_sync_actions( sync_type, @@ -279,8 +276,17 @@ def make_folder_sync_actions( dest_folder, now_millis, ): + total_files += 1 + total_bytes += action.get_bytes() yield action + if reporter is not None: + if source_type == 'b2': + # For buckets we don't want to count files separately as it would require + # more API calls. Instead, we count them when comparing. + reporter.end_total() + reporter.end_compare(total_files, total_bytes) + def make_file_sync_actions( self, sync_type, diff --git a/b2sdk/transfer/emerge/planner/planner.py b/b2sdk/transfer/emerge/planner/planner.py index 14ee49114..ce002376a 100644 --- a/b2sdk/transfer/emerge/planner/planner.py +++ b/b2sdk/transfer/emerge/planner/planner.py @@ -607,7 +607,7 @@ def get_plan_id(self): return None json_id = json.dumps([emerge_part.get_part_id() for emerge_part in self.emerge_parts]) - return hashlib.sha1(json_id).hexdigest() + return hashlib.sha1(json_id.encode()).hexdigest() class StreamingEmergePlan(BaseEmergePlan): diff --git a/b2sdk/transfer/outbound/copy_manager.py b/b2sdk/transfer/outbound/copy_manager.py index 4d91c5a0c..fa39edf1c 100644 --- a/b2sdk/transfer/outbound/copy_manager.py +++ b/b2sdk/transfer/outbound/copy_manager.py @@ -65,10 +65,10 @@ def copy_file( self, copy_source, file_name, - content_type=None, - file_info=None, - destination_bucket_id=None, - progress_listener=None, + content_type, + file_info, + destination_bucket_id, + progress_listener, ): # Run small copies in the same thread pool as large file copies, # so that they share resources during a sync. @@ -146,35 +146,36 @@ def _copy_small_file( self, copy_source, file_name, - content_type=None, - file_info=None, - destination_bucket_id=None, - progress_listener=None, + content_type, + file_info, + destination_bucket_id, + progress_listener, ): - if progress_listener is not None: + with progress_listener: progress_listener.set_total_bytes(copy_source.get_content_length() or 0) - bytes_range = copy_source.get_bytes_range() + bytes_range = copy_source.get_bytes_range() + + if content_type is None: + if file_info is not None: + raise ValueError('File info can be set only when content type is set') + metadata_directive = MetadataDirectiveMode.COPY + else: + if file_info is None: + raise ValueError('File info can be not set only when content type is not set') + metadata_directive = MetadataDirectiveMode.REPLACE + + response = self.services.session.copy_file( + copy_source.file_id, + file_name, + bytes_range=bytes_range, + metadata_directive=metadata_directive, + content_type=content_type, + file_info=file_info, + destination_bucket_id=destination_bucket_id + ) + file_info = FileVersionInfoFactory.from_api_response(response) + if progress_listener is not None: + progress_listener.bytes_completed(file_info.size) - if content_type is None: - if file_info is not None: - raise ValueError('File info can be set only when content type is set') - metadata_directive = MetadataDirectiveMode.COPY - else: - if file_info is None: - raise ValueError('File info can be not set only when content type is not set') - metadata_directive = MetadataDirectiveMode.REPLACE - - response = self.services.session.copy_file( - copy_source.file_id, - file_name, - bytes_range=bytes_range, - metadata_directive=metadata_directive, - content_type=content_type, - file_info=file_info, - destination_bucket_id=destination_bucket_id - ) - file_info = FileVersionInfoFactory.from_api_response(response) - if progress_listener is not None: - progress_listener.bytes_completed(file_info.size) return file_info diff --git a/b2sdk/transfer/outbound/copy_source.py b/b2sdk/transfer/outbound/copy_source.py index f304324bb..ff5ba19f2 100644 --- a/b2sdk/transfer/outbound/copy_source.py +++ b/b2sdk/transfer/outbound/copy_source.py @@ -13,8 +13,8 @@ class CopySource(OutboundTransferSource): def __init__(self, file_id, offset=0, length=None): - if length is None and offset > 0: - raise ValueError('Cannot copy with non zero offset and unknown length') + if not length and offset > 0: + raise ValueError('Cannot copy with non zero offset and unknown or zero length') self.file_id = file_id self.length = length self.offset = offset @@ -38,13 +38,15 @@ def is_copy(self): return True def get_bytes_range(self): - if self.length is None: + if not self.length: if self.offset > 0: # auto mode should get file info and create correct copy source (with length) - raise ValueError('cannot return bytes range for non zero offset and unknown length') + raise ValueError( + 'cannot return bytes range for non zero offset and unknown or zero length' + ) return None - return (self.offset, self.offset + self.length - 1) + return self.offset, self.offset + self.length - 1 def get_copy_source_range(self, relative_offset, range_length): if self.length is not None and range_length + relative_offset > self.length: diff --git a/b2sdk/v0/__init__.py b/b2sdk/v0/__init__.py index 79d3f2a5a..3bb0dab67 100644 --- a/b2sdk/v0/__init__.py +++ b/b2sdk/v0/__init__.py @@ -13,5 +13,6 @@ from b2sdk.v0.api import B2Api from b2sdk.v0.bucket import Bucket from b2sdk.v0.bucket import BucketFactory -from .sync import make_folder_sync_actions -from .sync import sync_folders +from b2sdk.v0.sync import Synchronizer +from b2sdk.v0.sync import make_folder_sync_actions +from b2sdk.v0.sync import sync_folders diff --git a/b2sdk/v1/__init__.py b/b2sdk/v1/__init__.py index 7f1a45138..ee04e47d9 100644 --- a/b2sdk/v1/__init__.py +++ b/b2sdk/v1/__init__.py @@ -125,6 +125,7 @@ # sync from b2sdk.sync.action import AbstractAction +from b2sdk.sync.action import B2CopyAction from b2sdk.sync.action import B2DeleteAction from b2sdk.sync.action import B2DownloadAction from b2sdk.sync.action import B2HideAction @@ -145,6 +146,9 @@ from b2sdk.sync.policy import DownAndDeletePolicy from b2sdk.sync.policy import DownAndKeepDaysPolicy from b2sdk.sync.policy import DownPolicy +from b2sdk.sync.policy import CopyPolicy +from b2sdk.sync.policy import CopyAndDeletePolicy +from b2sdk.sync.policy import CopyAndKeepDaysPolicy from b2sdk.sync.policy import UpAndDeletePolicy from b2sdk.sync.policy import UpAndKeepDaysPolicy from b2sdk.sync.policy import UpPolicy diff --git a/noxfile.py b/noxfile.py index 8349b6fa2..4a797450f 100644 --- a/noxfile.py +++ b/noxfile.py @@ -102,10 +102,10 @@ def unit(session): """Run unit tests.""" install_myself(session) session.install(*REQUIREMENTS_TEST) - session.run( - 'pytest', '--cov=b2sdk', '--cov-branch', '--cov-report=xml', '--doctest-modules', - *session.posargs, 'test/unit' - ) + args = ['--cov=b2sdk', '--cov-branch', '--cov-report=xml', '--doctest-modules'] + session.run('pytest', '--api=v1', *args, *session.posargs, 'test/unit') + session.run('pytest', '--api=v0', '--cov-append', *args, *session.posargs, 'test/unit') + if not session.posargs: session.notify('cover') diff --git a/test/unit/conftest.py b/test/unit/conftest.py new file mode 100644 index 000000000..a696956ed --- /dev/null +++ b/test/unit/conftest.py @@ -0,0 +1,62 @@ +###################################################################### +# +# File: test/unit/conftest.py +# +# Copyright 2020 Backblaze Inc. All Rights Reserved. +# +# License https://www.backblaze.com/using_b2_code.html +# +###################################################################### +import os +import sys +from glob import glob +from pathlib import Path + +import pytest + +pytest.register_assert_rewrite('test.unit') + + +def get_api_versions(): + return [ + str(Path(p).parent.name) for p in sorted(glob(str(Path(__file__).parent / 'v*/apiver/'))) + ] + + +API_VERSIONS = get_api_versions() + + +@pytest.hookimpl +def pytest_addoption(parser): + parser.addoption( + '--api', + default=API_VERSIONS[-1], + choices=API_VERSIONS, + help='version of the API', + ) + + +@pytest.hookimpl +def pytest_configure(config): + sys.path.insert(0, str(Path(__file__).parent / config.getoption('--api') / 'apiver')) + + +@pytest.hookimpl +def pytest_report_header(config): + return 'b2sdk apiver: %s' % config.getoption('--api') + + +@pytest.hookimpl(tryfirst=True) +def pytest_ignore_collect(path, config): + path = str(path) + ver = config.getoption('--api') + other_versions = [v for v in API_VERSIONS if v != ver] + for other_version in other_versions: + if other_version + os.sep in path: + return True + return False + + +@pytest.fixture(scope='session') +def apiver(request): + return request.config.getoption('--api') diff --git a/test/unit/sync/fixtures.py b/test/unit/sync/fixtures.py new file mode 100644 index 000000000..2211e5c08 --- /dev/null +++ b/test/unit/sync/fixtures.py @@ -0,0 +1,138 @@ +###################################################################### +# +# File: test/unit/sync/fixtures.py +# +# Copyright 2020 Backblaze Inc. All Rights Reserved. +# +# License https://www.backblaze.com/using_b2_code.html +# +###################################################################### + +import pytest + +from apiver_deps import AbstractFolder, File, FileVersion +from apiver_deps import CompareVersionMode, NewerFileSyncMode, KeepOrDeleteMode +from apiver_deps import DEFAULT_SCAN_MANAGER, Synchronizer + + +class FakeFolder(AbstractFolder): + def __init__(self, f_type, files=None): + if files is None: + files = [] + + self.f_type = f_type + self.files = files + + def all_files(self, reporter, policies_manager=DEFAULT_SCAN_MANAGER): + for single_file in self.files: + if single_file.name.endswith('/'): + if policies_manager.should_exclude_directory(single_file.name): + continue + else: + if policies_manager.should_exclude_file(single_file.name): + continue + yield single_file + + def folder_type(self): + return self.f_type + + def make_full_path(self, name): + if self.f_type == 'local': + return '/dir/' + name + else: + return 'folder/' + name + + def __str__(self): + return '%s(%s, %s)' % (self.__class__.__name__, self.f_type, self.make_full_path('')) + + +def local_file(name, mod_times, size=10): + """ + Makes a File object for a b2 file, with one FileVersion for + each modification time given in mod_times. + """ + versions = [ + FileVersion('/dir/%s' % (name,), name, mod_time, 'upload', size) for mod_time in mod_times + ] + return File(name, versions) + + +def b2_file(name, mod_times, size=10): + """ + Makes a File object for a b2 file, with one FileVersion for + each modification time given in mod_times. + + Positive modification times are uploads, and negative modification + times are hides. It's a hack, but it works. + + b2_file('a.txt', [300, -200, 100]) + + Is the same as: + + File( + 'a.txt', + [ + FileVersion('id_a_300', 'a.txt', 300, 'upload'), + FileVersion('id_a_200', 'a.txt', 200, 'hide'), + FileVersion('id_a_100', 'a.txt', 100, 'upload') + ] + ) + """ + versions = [ + FileVersion( + 'id_%s_%d' % (name[0], abs(mod_time)), + 'folder/' + name, + abs(mod_time), + 'upload' if 0 < mod_time else 'hide', + size, + ) for mod_time in mod_times + ] # yapf disable + return File(name, versions) + + +@pytest.fixture(scope='module') +def folder_factory(): + def get_folder(f_type, *files): + def get_files(): + nonlocal files + for file in files: + if f_type == 'local': + yield local_file(*file) + else: + yield b2_file(*file) + + return FakeFolder(f_type, list(get_files())) + + return get_folder + + +@pytest.fixture(scope='module') +def synchronizer_factory(): + def get_synchronizer( + policies_manager=DEFAULT_SCAN_MANAGER, + dry_run=False, + allow_empty_source=False, + newer_file_mode=NewerFileSyncMode.RAISE_ERROR, + keep_days_or_delete=KeepOrDeleteMode.NO_DELETE, + keep_days=None, + compare_version_mode=CompareVersionMode.MODTIME, + compare_threshold=None, + ): + return Synchronizer( + 1, + policies_manager=policies_manager, + dry_run=dry_run, + allow_empty_source=allow_empty_source, + newer_file_mode=newer_file_mode, + keep_days_or_delete=keep_days_or_delete, + keep_days=keep_days, + compare_version_mode=compare_version_mode, + compare_threshold=compare_threshold, + ) + + return get_synchronizer + + +@pytest.fixture +def synchronizer(synchronizer_factory): + return synchronizer_factory() diff --git a/test/unit/sync/test_base.py b/test/unit/sync/test_base.py deleted file mode 100644 index 1ff01fd26..000000000 --- a/test/unit/sync/test_base.py +++ /dev/null @@ -1,36 +0,0 @@ -###################################################################### -# -# File: test/unit/sync/test_base.py -# -# Copyright 2020 Backblaze Inc. All Rights Reserved. -# -# License https://www.backblaze.com/using_b2_code.html -# -###################################################################### - -from contextlib import contextmanager -import re -import unittest - - -class TestBase(unittest.TestCase): - @contextmanager - def assertRaises(self, exc, msg=None): - try: - yield - except exc as e: - if msg is not None: - if msg != str(e): - assert False, "expected message '%s', but got '%s'" % (msg, str(e)) - else: - assert False, 'should have thrown %s' % (exc,) - - @contextmanager - def assertRaisesRegexp(self, expected_exception, expected_regexp): - try: - yield - except expected_exception as e: - if not re.search(expected_regexp, str(e)): - assert False, "expected message '%s', but got '%s'" % (expected_regexp, str(e)) - else: - assert False, 'should have thrown %s' % (expected_exception,) diff --git a/test/unit/sync/test_exception.py b/test/unit/sync/test_exception.py index ab110c0de..522845025 100644 --- a/test/unit/sync/test_exception.py +++ b/test/unit/sync/test_exception.py @@ -7,9 +7,8 @@ # License https://www.backblaze.com/using_b2_code.html # ###################################################################### -from .test_base import TestBase -from .deps_exception import ( +from apiver_deps_exception import ( EnvironmentEncodingError, InvalidArgument, IncompleteSync, @@ -17,7 +16,7 @@ ) -class TestExceptions(TestBase): +class TestSyncExceptions: def test_environment_encoding_error(self): try: raise EnvironmentEncodingError('fred', 'george') diff --git a/test/unit/sync/test_sync.py b/test/unit/sync/test_sync.py new file mode 100644 index 000000000..fc34d8e64 --- /dev/null +++ b/test/unit/sync/test_sync.py @@ -0,0 +1,668 @@ +###################################################################### +# +# File: test/unit/sync/test_sync.py +# +# Copyright 2020 Backblaze Inc. All Rights Reserved. +# +# License https://www.backblaze.com/using_b2_code.html +# +###################################################################### + +from enum import Enum +from functools import partial + +from apiver_deps_exception import CommandError, DestFileNewer, InvalidArgument + +from .fixtures import * + +DAY = 86400000 # milliseconds +TODAY = DAY * 100 # an arbitrary reference time for testing + + +class TestSynchronizer: + class IllegalEnum(Enum): + ILLEGAL = 5100 + + @pytest.fixture(autouse=True) + def setup(self, folder_factory, mocker): + self.folder_factory = folder_factory + self.local_folder_factory = partial(folder_factory, 'local') + self.b2_folder_factory = partial(folder_factory, 'b2') + self.reporter = mocker.MagicMock() + + def assert_folder_sync_actions(self, synchronizer, src_folder, dst_folder, expected_actions): + """ + Checks the actions generated for one file. The file may or may not + exist at the source, and may or may not exist at the destination. + + The source and destination files may have multiple versions. + """ + actions = list( + synchronizer.make_folder_sync_actions( + src_folder, + dst_folder, + TODAY, + self.reporter, + ) + ) + assert expected_actions == [str(a) for a in actions] + + @pytest.mark.parametrize( + 'args', [ + { + 'newer_file_mode': IllegalEnum.ILLEGAL + }, + { + 'keep_days_or_delete': IllegalEnum.ILLEGAL + }, + ], + ids=[ + 'newer_file_mode', + 'keep_days_or_delete', + ] + ) + def test_illegal_args(self, synchronizer_factory, apiver, args): + exceptions = { + 'v1': InvalidArgument, + 'v0': CommandError, + } + + with pytest.raises(exceptions[apiver]): + synchronizer_factory(**args) + + def test_illegal(self, synchronizer): + with pytest.raises(ValueError): + src = self.local_folder_factory() + dst = self.local_folder_factory() + self.assert_folder_sync_actions(synchronizer, src, dst, []) + + # src: absent, dst: absent + + @pytest.mark.parametrize( + 'src_type,dst_type', + [ + ('local', 'b2'), + ('b2', 'local'), + ('b2', 'b2'), + ], + ) + def test_empty(self, synchronizer, src_type, dst_type): + src = self.folder_factory(src_type) + dst = self.folder_factory(dst_type) + self.assert_folder_sync_actions(synchronizer, src, dst, []) + + # # src: present, dst: absent + + @pytest.mark.parametrize( + 'src_type,dst_type,expected', + [ + ('local', 'b2', ['b2_upload(/dir/a.txt, folder/a.txt, 100)']), + ('b2', 'local', ['b2_download(folder/a.txt, id_a_100, /dir/a.txt, 100)']), + ('b2', 'b2', ['b2_copy(folder/a.txt, id_a_100, folder/a.txt, 100)']), + ], + ) + def test_not_there(self, synchronizer, src_type, dst_type, expected): + src = self.folder_factory(src_type, ('a.txt', [100])) + dst = self.folder_factory(dst_type) + self.assert_folder_sync_actions(synchronizer, src, dst, expected) + + @pytest.mark.parametrize( + 'src_type,expected', + [ + ('local', ['b2_upload(/dir/directory/a.txt, folder/directory/a.txt, 100)']), + ('b2', ['b2_copy(folder/directory/a.txt, id_d_100, folder/directory/a.txt, 100)']), + ], + ) + def test_dir_not_there_b2_keepdays( + self, synchronizer_factory, src_type, expected + ): # reproduces issue 220 + src = self.folder_factory(src_type, ('directory/a.txt', [100])) + dst = self.b2_folder_factory() + synchronizer = synchronizer_factory( + keep_days_or_delete=KeepOrDeleteMode.KEEP_BEFORE_DELETE, keep_days=1 + ) + self.assert_folder_sync_actions(synchronizer, src, dst, expected) + + @pytest.mark.parametrize( + 'src_type,expected', + [ + ('local', ['b2_upload(/dir/directory/a.txt, folder/directory/a.txt, 100)']), + ('b2', ['b2_copy(folder/directory/a.txt, id_d_100, folder/directory/a.txt, 100)']), + ], + ) + def test_dir_not_there_b2_delete( + self, synchronizer_factory, src_type, expected + ): # reproduces issue 220 + src = self.folder_factory(src_type, ('directory/a.txt', [100])) + dst = self.b2_folder_factory() + synchronizer = synchronizer_factory(keep_days_or_delete=KeepOrDeleteMode.DELETE) + self.assert_folder_sync_actions(synchronizer, src, dst, expected) + + # # src: absent, dst: present + + @pytest.mark.parametrize( + 'src_type,dst_type', + [ + ('local', 'b2'), + ('b2', 'local'), + ('b2', 'b2'), + ], + ) + def test_no_delete(self, synchronizer, src_type, dst_type): + src = self.folder_factory(src_type) + dst = self.folder_factory(dst_type, ('a.txt', [100])) + self.assert_folder_sync_actions(synchronizer, src, dst, []) + + @pytest.mark.parametrize( + 'src_type,dst_type,expected', + [ + ('local', 'b2', ['b2_delete(folder/a.txt, id_a_100, )']), + ('b2', 'local', ['local_delete(/dir/a.txt)']), + ('b2', 'b2', ['b2_delete(folder/a.txt, id_a_100, )']), + ], + ) + def test_delete(self, synchronizer_factory, src_type, dst_type, expected): + synchronizer = synchronizer_factory(keep_days_or_delete=KeepOrDeleteMode.DELETE) + src = self.folder_factory(src_type) + dst = self.folder_factory(dst_type, ('a.txt', [100])) + self.assert_folder_sync_actions(synchronizer, src, dst, expected) + + @pytest.mark.parametrize( + 'src_type,dst_type,expected', + [ + ('local', 'b2', ['b2_delete(folder/a.txt, id_a_100, )']), + ('b2', 'local', ['local_delete(/dir/a.txt)']), + ('b2', 'b2', ['b2_delete(folder/a.txt, id_a_100, )']), + ], + ) + def test_delete_large(self, synchronizer_factory, src_type, dst_type, expected): + synchronizer = synchronizer_factory(keep_days_or_delete=KeepOrDeleteMode.DELETE) + src = self.folder_factory(src_type) + dst = self.folder_factory(dst_type, ('a.txt', [100], 10737418240)) + self.assert_folder_sync_actions(synchronizer, src, dst, expected) + + @pytest.mark.parametrize( + 'src_type', + [ + 'local', + 'b2', + ], + ) + def test_delete_multiple_versions(self, synchronizer_factory, src_type): + synchronizer = synchronizer_factory(keep_days_or_delete=KeepOrDeleteMode.DELETE) + src = self.folder_factory(src_type) + dst = self.b2_folder_factory(('a.txt', [100, 200])) + expected = [ + 'b2_delete(folder/a.txt, id_a_100, )', + 'b2_delete(folder/a.txt, id_a_200, (old version))' + ] + self.assert_folder_sync_actions(synchronizer, src, dst, expected) + + @pytest.mark.parametrize( + 'src_type', + [ + 'local', + 'b2', + ], + ) + def test_delete_hide_b2_multiple_versions(self, synchronizer_factory, src_type): + synchronizer = synchronizer_factory( + keep_days_or_delete=KeepOrDeleteMode.KEEP_BEFORE_DELETE, keep_days=1 + ) + src = self.folder_factory(src_type) + dst = self.b2_folder_factory(('a.txt', [TODAY, TODAY - 2 * DAY, TODAY - 4 * DAY])) + expected = [ + 'b2_hide(folder/a.txt)', 'b2_delete(folder/a.txt, id_a_8294400000, (old version))' + ] + self.assert_folder_sync_actions(synchronizer, src, dst, expected) + + @pytest.mark.parametrize( + 'src_type', + [ + 'local', + 'b2', + ], + ) + def test_delete_hide_b2_multiple_versions_old(self, synchronizer_factory, src_type): + synchronizer = synchronizer_factory( + keep_days_or_delete=KeepOrDeleteMode.KEEP_BEFORE_DELETE, keep_days=2 + ) + src = self.folder_factory(src_type) + dst = self.b2_folder_factory(('a.txt', [TODAY - 1 * DAY, TODAY - 3 * DAY, TODAY - 5 * DAY])) + expected = [ + 'b2_hide(folder/a.txt)', 'b2_delete(folder/a.txt, id_a_8208000000, (old version))' + ] + self.assert_folder_sync_actions(synchronizer, src, dst, expected) + + @pytest.mark.parametrize( + 'src_type', + [ + 'local', + 'b2', + ], + ) + def test_already_hidden_multiple_versions_keep(self, synchronizer, src_type): + src = self.folder_factory(src_type) + dst = self.b2_folder_factory(('a.txt', [-TODAY, TODAY - 2 * DAY, TODAY - 4 * DAY])) + self.assert_folder_sync_actions(synchronizer, src, dst, []) + + @pytest.mark.parametrize( + 'src_type', + [ + 'local', + 'b2', + ], + ) + def test_already_hidden_multiple_versions_keep_days(self, synchronizer_factory, src_type): + synchronizer = synchronizer_factory( + keep_days_or_delete=KeepOrDeleteMode.KEEP_BEFORE_DELETE, keep_days=1 + ) + src = self.folder_factory(src_type) + dst = self.b2_folder_factory(('a.txt', [-TODAY, TODAY - 2 * DAY, TODAY - 4 * DAY])) + expected = ['b2_delete(folder/a.txt, id_a_8294400000, (old version))'] + self.assert_folder_sync_actions(synchronizer, src, dst, expected) + + @pytest.mark.parametrize( + 'src_type', + [ + 'local', + 'b2', + ], + ) + def test_already_hidden_multiple_versions_keep_days_one_old( + self, synchronizer_factory, src_type + ): + synchronizer = synchronizer_factory( + keep_days_or_delete=KeepOrDeleteMode.KEEP_BEFORE_DELETE, keep_days=5 + ) + src = self.folder_factory(src_type) + dst = self.b2_folder_factory( + ('a.txt', [-(TODAY - 2 * DAY), TODAY - 4 * DAY, TODAY - 6 * DAY]) + ) + self.assert_folder_sync_actions(synchronizer, src, dst, []) + + @pytest.mark.parametrize( + 'src_type', + [ + 'local', + 'b2', + ], + ) + def test_already_hidden_multiple_versions_keep_days_two_old( + self, synchronizer_factory, src_type + ): + synchronizer = synchronizer_factory( + keep_days_or_delete=KeepOrDeleteMode.KEEP_BEFORE_DELETE, keep_days=2 + ) + src = self.folder_factory(src_type) + dst = self.b2_folder_factory( + ('a.txt', [-(TODAY - 2 * DAY), TODAY - 4 * DAY, TODAY - 6 * DAY]) + ) + expected = ['b2_delete(folder/a.txt, id_a_8121600000, (old version))'] + self.assert_folder_sync_actions(synchronizer, src, dst, expected) + + @pytest.mark.parametrize( + 'src_type', + [ + 'local', + 'b2', + ], + ) + def test_already_hidden_multiple_versions_keep_days_delete_hide_marker( + self, synchronizer_factory, src_type + ): + synchronizer = synchronizer_factory( + keep_days_or_delete=KeepOrDeleteMode.KEEP_BEFORE_DELETE, keep_days=1 + ) + src = self.folder_factory(src_type) + dst = self.b2_folder_factory( + ('a.txt', [-(TODAY - 2 * DAY), TODAY - 4 * DAY, TODAY - 6 * DAY]) + ) + expected = [ + 'b2_delete(folder/a.txt, id_a_8467200000, (hide marker))', + 'b2_delete(folder/a.txt, id_a_8294400000, (old version))', + 'b2_delete(folder/a.txt, id_a_8121600000, (old version))' + ] + self.assert_folder_sync_actions(synchronizer, src, dst, expected) + + @pytest.mark.parametrize( + 'src_type', + [ + 'local', + 'b2', + ], + ) + def test_already_hidden_multiple_versions_keep_days_old_delete( + self, synchronizer_factory, src_type + ): + synchronizer = synchronizer_factory( + keep_days_or_delete=KeepOrDeleteMode.KEEP_BEFORE_DELETE, keep_days=1 + ) + src = self.folder_factory(src_type) + dst = self.b2_folder_factory(('a.txt', [-TODAY + 2 * DAY, TODAY - 4 * DAY])) + expected = [ + 'b2_delete(folder/a.txt, id_a_8467200000, (hide marker))', + 'b2_delete(folder/a.txt, id_a_8294400000, (old version))' + ] + self.assert_folder_sync_actions(synchronizer, src, dst, expected) + + @pytest.mark.parametrize( + 'src_type', + [ + 'local', + 'b2', + ], + ) + def test_already_hidden_multiple_versions_delete(self, synchronizer_factory, src_type): + synchronizer = synchronizer_factory(keep_days_or_delete=KeepOrDeleteMode.DELETE) + src = self.folder_factory(src_type) + dst = self.b2_folder_factory(('a.txt', [-TODAY, TODAY - 2 * DAY, TODAY - 4 * DAY])) + expected = [ + 'b2_delete(folder/a.txt, id_a_8640000000, (hide marker))', + 'b2_delete(folder/a.txt, id_a_8467200000, (old version))', + 'b2_delete(folder/a.txt, id_a_8294400000, (old version))' + ] + self.assert_folder_sync_actions(synchronizer, src, dst, expected) + + # # src same as dst + + @pytest.mark.parametrize( + 'src_type,dst_type', + [ + ('local', 'b2'), + ('b2', 'local'), + ('b2', 'b2'), + ], + ) + def test_same(self, synchronizer, src_type, dst_type): + src = self.folder_factory(src_type, ('a.txt', [100])) + dst = self.folder_factory(dst_type, ('a.txt', [100])) + self.assert_folder_sync_actions(synchronizer, src, dst, []) + + @pytest.mark.parametrize( + 'src_type', + [ + 'local', + 'b2', + ], + ) + def test_same_leave_old_version(self, synchronizer, src_type): + src = self.folder_factory(src_type, ('a.txt', [TODAY])) + dst = self.b2_folder_factory(('a.txt', [TODAY, TODAY - 3 * DAY])) + self.assert_folder_sync_actions(synchronizer, src, dst, []) + + @pytest.mark.parametrize( + 'src_type', + [ + 'local', + 'b2', + ], + ) + def test_same_clean_old_version(self, synchronizer_factory, src_type): + synchronizer = synchronizer_factory( + keep_days_or_delete=KeepOrDeleteMode.KEEP_BEFORE_DELETE, keep_days=1 + ) + src = self.folder_factory(src_type, ('a.txt', [TODAY - 3 * DAY])) + dst = self.b2_folder_factory(('a.txt', [TODAY - 3 * DAY, TODAY - 4 * DAY])) + expected = ['b2_delete(folder/a.txt, id_a_8294400000, (old version))'] + self.assert_folder_sync_actions(synchronizer, src, dst, expected) + + @pytest.mark.parametrize( + 'src_type', + [ + 'local', + 'b2', + ], + ) + def test_keep_days_no_change_with_old_file(self, synchronizer_factory, src_type): + synchronizer = synchronizer_factory( + keep_days_or_delete=KeepOrDeleteMode.KEEP_BEFORE_DELETE, keep_days=1 + ) + src = self.folder_factory(src_type, ('a.txt', [TODAY - 3 * DAY])) + dst = self.b2_folder_factory(('a.txt', [TODAY - 3 * DAY])) + self.assert_folder_sync_actions(synchronizer, src, dst, []) + + @pytest.mark.parametrize( + 'src_type', + [ + 'local', + 'b2', + ], + ) + def test_same_delete_old_versions(self, synchronizer_factory, src_type): + synchronizer = synchronizer_factory(keep_days_or_delete=KeepOrDeleteMode.DELETE) + src = self.folder_factory(src_type, ('a.txt', [TODAY])) + dst = self.b2_folder_factory(('a.txt', [TODAY, TODAY - 3 * DAY])) + expected = ['b2_delete(folder/a.txt, id_a_8380800000, (old version))'] + self.assert_folder_sync_actions(synchronizer, src, dst, expected) + + # # src newer than dst + + @pytest.mark.parametrize( + 'src_type,dst_type,expected', + [ + ('local', 'b2', ['b2_upload(/dir/a.txt, folder/a.txt, 200)']), + ('b2', 'local', ['b2_download(folder/a.txt, id_a_200, /dir/a.txt, 200)']), + ('b2', 'b2', ['b2_copy(folder/a.txt, id_a_200, folder/a.txt, 200)']), + ], + ) + def test_never(self, synchronizer, src_type, dst_type, expected): + src = self.folder_factory(src_type, ('a.txt', [200])) + dst = self.folder_factory(dst_type, ('a.txt', [100])) + self.assert_folder_sync_actions(synchronizer, src, dst, expected) + + @pytest.mark.parametrize( + 'src_type,expected', + [ + ( + 'local', [ + 'b2_upload(/dir/a.txt, folder/a.txt, 8640000000)', + 'b2_delete(folder/a.txt, id_a_8208000000, (old version))', + ] + ), + ( + 'b2', [ + 'b2_copy(folder/a.txt, id_a_8640000000, folder/a.txt, 8640000000)', + 'b2_delete(folder/a.txt, id_a_8208000000, (old version))', + ] + ), + ], + ) + def test_newer_clean_old_versions(self, synchronizer_factory, src_type, expected): + synchronizer = synchronizer_factory( + keep_days_or_delete=KeepOrDeleteMode.KEEP_BEFORE_DELETE, keep_days=2 + ) + src = self.folder_factory(src_type, ('a.txt', [TODAY])) + dst = self.b2_folder_factory(('a.txt', [TODAY - 1 * DAY, TODAY - 3 * DAY, TODAY - 5 * DAY])) + self.assert_folder_sync_actions(synchronizer, src, dst, expected) + + @pytest.mark.parametrize( + 'src_type,expected', + [ + ( + 'local', [ + 'b2_upload(/dir/a.txt, folder/a.txt, 8640000000)', + 'b2_delete(folder/a.txt, id_a_8553600000, (old version))', + 'b2_delete(folder/a.txt, id_a_8380800000, (old version))', + ] + ), + ( + 'b2', [ + 'b2_copy(folder/a.txt, id_a_8640000000, folder/a.txt, 8640000000)', + 'b2_delete(folder/a.txt, id_a_8553600000, (old version))', + 'b2_delete(folder/a.txt, id_a_8380800000, (old version))', + ] + ), + ], + ) + def test_newer_delete_old_versions(self, synchronizer_factory, src_type, expected): + synchronizer = synchronizer_factory(keep_days_or_delete=KeepOrDeleteMode.DELETE) + src = self.folder_factory(src_type, ('a.txt', [TODAY])) + dst = self.b2_folder_factory(('a.txt', [TODAY - 1 * DAY, TODAY - 3 * DAY])) + self.assert_folder_sync_actions(synchronizer, src, dst, expected) + + # # src older than dst + + @pytest.mark.parametrize( + 'src_type,dst_type,expected', + [ + ('local', 'b2', ['b2_upload(/dir/a.txt, folder/a.txt, 200)']), + ('b2', 'local', ['b2_download(folder/a.txt, id_a_200, /dir/a.txt, 200)']), + ('b2', 'b2', ['b2_copy(folder/a.txt, id_a_200, folder/a.txt, 200)']), + ], + ) + def test_older(self, synchronizer, apiver, src_type, dst_type, expected): + src = self.folder_factory(src_type, ('a.txt', [100])) + dst = self.folder_factory(dst_type, ('a.txt', [200])) + with pytest.raises(DestFileNewer) as excinfo: + self.assert_folder_sync_actions(synchronizer, src, dst, expected) + messages = { + 'v1': 'source file is older than destination: %s://a.txt with a time of 100 ' + 'cannot be synced to %s://a.txt with a time of 200, ' + 'unless a valid newer_file_mode is provided', + 'v0': 'source file is older than destination: %s://a.txt with a time of 100 ' + 'cannot be synced to %s://a.txt with a time of 200, ' + 'unless --skipNewer or --replaceNewer is provided', + } # yapf: disable + + assert str(excinfo.value) == messages[apiver] % (src_type, dst_type) + + @pytest.mark.parametrize( + 'src_type,dst_type', + [ + ('local', 'b2'), + ('b2', 'local'), + ('b2', 'b2'), + ], + ) + def test_older_skip(self, synchronizer_factory, src_type, dst_type): + synchronizer = synchronizer_factory(newer_file_mode=NewerFileSyncMode.SKIP) + src = self.folder_factory(src_type, ('a.txt', [100])) + dst = self.folder_factory(dst_type, ('a.txt', [200])) + self.assert_folder_sync_actions(synchronizer, src, dst, []) + + @pytest.mark.parametrize( + 'src_type,dst_type,expected', + [ + ('local', 'b2', ['b2_upload(/dir/a.txt, folder/a.txt, 100)']), + ('b2', 'local', ['b2_download(folder/a.txt, id_a_100, /dir/a.txt, 100)']), + ('b2', 'b2', ['b2_copy(folder/a.txt, id_a_100, folder/a.txt, 100)']), + ], + ) + def test_older_replace(self, synchronizer_factory, src_type, dst_type, expected): + synchronizer = synchronizer_factory(newer_file_mode=NewerFileSyncMode.REPLACE) + src = self.folder_factory(src_type, ('a.txt', [100])) + dst = self.folder_factory(dst_type, ('a.txt', [200])) + self.assert_folder_sync_actions(synchronizer, src, dst, expected) + + @pytest.mark.parametrize( + 'src_type,expected', + [ + ( + 'local', [ + 'b2_upload(/dir/a.txt, folder/a.txt, 100)', + 'b2_delete(folder/a.txt, id_a_200, (old version))', + ] + ), + ( + 'b2', [ + 'b2_copy(folder/a.txt, id_a_100, folder/a.txt, 100)', + 'b2_delete(folder/a.txt, id_a_200, (old version))', + ] + ), + ], + ) + def test_older_replace_delete(self, synchronizer_factory, src_type, expected): + synchronizer = synchronizer_factory( + newer_file_mode=NewerFileSyncMode.REPLACE, keep_days_or_delete=KeepOrDeleteMode.DELETE + ) + src = self.folder_factory(src_type, ('a.txt', [100])) + dst = self.b2_folder_factory(('a.txt', [200])) + self.assert_folder_sync_actions(synchronizer, src, dst, expected) + + # # compareVersions option + + @pytest.mark.parametrize( + 'src_type,dst_type', + [ + ('local', 'b2'), + ('b2', 'local'), + ('b2', 'b2'), + ], + ) + def test_compare_none_newer(self, synchronizer_factory, src_type, dst_type): + synchronizer = synchronizer_factory(compare_version_mode=CompareVersionMode.NONE) + src = self.folder_factory(src_type, ('a.txt', [200])) + dst = self.folder_factory(dst_type, ('a.txt', [100])) + self.assert_folder_sync_actions(synchronizer, src, dst, []) + + @pytest.mark.parametrize( + 'src_type,dst_type', + [ + ('local', 'b2'), + ('b2', 'local'), + ('b2', 'b2'), + ], + ) + def test_compare_none_older(self, synchronizer_factory, src_type, dst_type): + synchronizer = synchronizer_factory(compare_version_mode=CompareVersionMode.NONE) + src = self.folder_factory(src_type, ('a.txt', [100])) + dst = self.folder_factory(dst_type, ('a.txt', [200])) + self.assert_folder_sync_actions(synchronizer, src, dst, []) + + @pytest.mark.parametrize( + 'src_type,dst_type', + [ + ('local', 'b2'), + ('b2', 'local'), + ('b2', 'b2'), + ], + ) + def test_compare_size_equal(self, synchronizer_factory, src_type, dst_type): + synchronizer = synchronizer_factory(compare_version_mode=CompareVersionMode.SIZE) + src = self.folder_factory(src_type, ('a.txt', [200], 10)) + dst = self.folder_factory(dst_type, ('a.txt', [100], 10)) + self.assert_folder_sync_actions(synchronizer, src, dst, []) + + @pytest.mark.parametrize( + 'src_type,dst_type,expected', + [ + ('local', 'b2', ['b2_upload(/dir/a.txt, folder/a.txt, 200)']), + ('b2', 'local', ['b2_download(folder/a.txt, id_a_200, /dir/a.txt, 200)']), + ('b2', 'b2', ['b2_copy(folder/a.txt, id_a_200, folder/a.txt, 200)']), + ], + ) + def test_compare_size_not_equal(self, synchronizer_factory, src_type, dst_type, expected): + synchronizer = synchronizer_factory(compare_version_mode=CompareVersionMode.SIZE) + src = self.folder_factory(src_type, ('a.txt', [200], 11)) + dst = self.folder_factory(dst_type, ('a.txt', [100], 10)) + self.assert_folder_sync_actions(synchronizer, src, dst, expected) + + @pytest.mark.parametrize( + 'src_type,dst_type,expected', + [ + ( + 'local', 'b2', [ + 'b2_upload(/dir/a.txt, folder/a.txt, 200)', + 'b2_delete(folder/a.txt, id_a_100, (old version))' + ] + ), + ('b2', 'local', ['b2_download(folder/a.txt, id_a_200, /dir/a.txt, 200)']), + ( + 'b2', 'b2', [ + 'b2_copy(folder/a.txt, id_a_200, folder/a.txt, 200)', + 'b2_delete(folder/a.txt, id_a_100, (old version))' + ] + ), + ], + ) + def test_compare_size_not_equal_delete( + self, synchronizer_factory, src_type, dst_type, expected + ): + synchronizer = synchronizer_factory( + compare_version_mode=CompareVersionMode.SIZE, + keep_days_or_delete=KeepOrDeleteMode.DELETE + ) + src = self.folder_factory(src_type, ('a.txt', [200], 11)) + dst = self.folder_factory(dst_type, ('a.txt', [100], 10)) + self.assert_folder_sync_actions(synchronizer, src, dst, expected) diff --git a/test/unit/v0/apiver/__init__.py b/test/unit/v0/apiver/__init__.py new file mode 100644 index 000000000..f213f55a6 --- /dev/null +++ b/test/unit/v0/apiver/__init__.py @@ -0,0 +1,12 @@ +###################################################################### +# +# File: test/unit/v0/apiver/__init__.py +# +# Copyright 2020 Backblaze Inc. All Rights Reserved. +# +# License https://www.backblaze.com/using_b2_code.html +# +###################################################################### + +# configured by pytest using `--api` option +# check test/unit/conftest.py:pytest_configure for details diff --git a/test/unit/sync/deps_exception.py b/test/unit/v0/apiver/apiver_deps.py similarity index 77% rename from test/unit/sync/deps_exception.py rename to test/unit/v0/apiver/apiver_deps.py index 336ce76f4..c9ff0c6c4 100644 --- a/test/unit/sync/deps_exception.py +++ b/test/unit/v0/apiver/apiver_deps.py @@ -1,6 +1,6 @@ ###################################################################### # -# File: test/unit/sync/deps_exception.py +# File: test/unit/v0/apiver/apiver_deps.py # # Copyright 2020 Backblaze Inc. All Rights Reserved. # @@ -8,4 +8,4 @@ # ###################################################################### -from b2sdk.sync.exception import * +from b2sdk.v0 import * diff --git a/test/unit/v0/apiver/apiver_deps_exception.py b/test/unit/v0/apiver/apiver_deps_exception.py new file mode 100644 index 000000000..74614a306 --- /dev/null +++ b/test/unit/v0/apiver/apiver_deps_exception.py @@ -0,0 +1,11 @@ +###################################################################### +# +# File: test/unit/v0/apiver/apiver_deps_exception.py +# +# Copyright 2020 Backblaze Inc. All Rights Reserved. +# +# License https://www.backblaze.com/using_b2_code.html +# +###################################################################### + +from b2sdk.v0.exception import * diff --git a/test/unit/v0/deps.py b/test/unit/v0/deps.py index 051e469e8..5d36cb216 100644 --- a/test/unit/v0/deps.py +++ b/test/unit/v0/deps.py @@ -8,4 +8,9 @@ # ###################################################################### -from b2sdk.v0 import * +# TODO: This module is used in old-style unit tests, written separately for v0 and v1. +# It will be removed when all test are rewritten for the new style, like e.g. test/unit/sync/. + +# configured by pytest using `--api` option +# check test/unit/conftest.py:pytest_configure for details +from apiver_deps import * diff --git a/test/unit/v0/deps_exception.py b/test/unit/v0/deps_exception.py index a11f71b29..4d60a21cf 100644 --- a/test/unit/v0/deps_exception.py +++ b/test/unit/v0/deps_exception.py @@ -8,4 +8,9 @@ # ###################################################################### -from b2sdk.v0.exception import * +# TODO: This module is used in old-style unit tests, written separately for v0 and v1. +# It will be removed when all test are rewritten for the new style, like e.g. test/unit/sync/. + +# configured by pytest using `--api` option +# check test/unit/conftest.py:pytest_configure for details +from apiver_deps_exception import * diff --git a/test/unit/v0/test_sync.py b/test/unit/v0/test_sync.py index 0ef733650..146d840c1 100644 --- a/test/unit/v0/test_sync.py +++ b/test/unit/v0/test_sync.py @@ -21,7 +21,6 @@ from .test_base import TestBase -from .deps_exception import CommandError, DestFileNewer from .deps_exception import UnSyncableFilename from .deps import FileVersionInfo from .deps import AbstractFolder, B2Folder, LocalFolder @@ -732,39 +731,6 @@ def __init__( self.excludeAllSymlinks = excludeAllSymlinks -def b2_file(name, mod_times, size=10): - """ - Makes a File object for a b2 file, with one FileVersion for - each modification time given in mod_times. - - Positive modification times are uploads, and negative modification - times are hides. It's a hack, but it works. - - b2_file('a.txt', [300, -200, 100]) - - Is the same as: - - File( - 'a.txt', - [ - FileVersion('id_a_300', 'a.txt', 300, 'upload'), - FileVersion('id_a_200', 'a.txt', 200, 'hide'), - FileVersion('id_a_100', 'a.txt', 100, 'upload') - ] - ) - """ - versions = [ - FileVersion( - 'id_%s_%d' % (name[0], abs(mod_time)), - 'folder/' + name, - abs(mod_time), - 'upload' if 0 < mod_time else 'hide', - size, - ) for mod_time in mod_times - ] # yapf disable - return File(name, versions) - - def local_file(name, mod_times, size=10): """ Makes a File object for a b2 file, with one FileVersion for @@ -830,337 +796,6 @@ def test_file_exclusions_inclusions_with_delete(self): self._check_folder_sync(expected_actions, fakeargs) -class TestMakeSyncActions(TestSync): - def test_illegal_b2_to_b2(self): - b2_folder = FakeFolder('b2', []) - with self.assertRaises(NotImplementedError): - list(make_folder_sync_actions(b2_folder, b2_folder, FakeArgs(), 0, self.reporter)) - - def test_illegal_local_to_local(self): - local_folder = FakeFolder('local', []) - with self.assertRaises(NotImplementedError): - list(make_folder_sync_actions(local_folder, local_folder, FakeArgs(), 0, self.reporter)) - - def test_illegal_skip_and_replace(self): - with self.assertRaises(CommandError): - self._check_local_to_b2(None, None, FakeArgs(skipNewer=True, replaceNewer=True), []) - - def test_illegal_delete_and_keep_days(self): - with self.assertRaises(CommandError): - self._check_local_to_b2(None, None, FakeArgs(delete=True, keepDays=1), []) - - # src: absent, dst: absent - - def test_empty_b2(self): - self._check_local_to_b2(None, None, FakeArgs(), []) - - def test_empty_local(self): - self._check_b2_to_local(None, None, FakeArgs(), []) - - # src: present, dst: absent - - def test_not_there_b2(self): - src_file = local_file('a.txt', [100]) - self._check_local_to_b2( - src_file, None, FakeArgs(), ['b2_upload(/dir/a.txt, folder/a.txt, 100)'] - ) - - def test_dir_not_there_b2_keepdays(self): # reproduces issue 220 - src_file = b2_file('directory/a.txt', [100]) - actions = ['b2_upload(/dir/directory/a.txt, folder/directory/a.txt, 100)'] - self._check_local_to_b2(src_file, None, FakeArgs(keepDays=1), actions) - - def test_dir_not_there_b2_delete(self): # reproduces issue 218 - src_file = b2_file('directory/a.txt', [100]) - actions = ['b2_upload(/dir/directory/a.txt, folder/directory/a.txt, 100)'] - self._check_local_to_b2(src_file, None, FakeArgs(delete=True), actions) - - def test_not_there_local(self): - src_file = b2_file('a.txt', [100]) - actions = ['b2_download(folder/a.txt, id_a_100, /dir/a.txt, 100)'] - self._check_b2_to_local(src_file, None, FakeArgs(), actions) - - # src: absent, dst: present - - def test_no_delete_b2(self): - dst_file = b2_file('a.txt', [100]) - self._check_local_to_b2(None, dst_file, FakeArgs(), []) - - def test_no_delete_local(self): - dst_file = local_file('a.txt', [100]) - self._check_b2_to_local(None, dst_file, FakeArgs(), []) - - def test_delete_b2(self): - dst_file = b2_file('a.txt', [100]) - actions = ['b2_delete(folder/a.txt, id_a_100, )'] - self._check_local_to_b2(None, dst_file, FakeArgs(delete=True), actions) - - def test_delete_large_b2(self): - dst_file = b2_file('a.txt', [100]) - actions = ['b2_delete(folder/a.txt, id_a_100, )'] - self._check_local_to_b2(None, dst_file, FakeArgs(delete=True), actions) - - def test_delete_b2_multiple_versions(self): - dst_file = b2_file('a.txt', [100, 200]) - actions = [ - 'b2_delete(folder/a.txt, id_a_100, )', - 'b2_delete(folder/a.txt, id_a_200, (old version))' - ] - self._check_local_to_b2(None, dst_file, FakeArgs(delete=True), actions) - - def test_delete_hide_b2_multiple_versions(self): - dst_file = b2_file('a.txt', [TODAY, TODAY - 2 * DAY, TODAY - 4 * DAY]) - actions = [ - 'b2_hide(folder/a.txt)', 'b2_delete(folder/a.txt, id_a_8294400000, (old version))' - ] - self._check_local_to_b2(None, dst_file, FakeArgs(keepDays=1), actions) - - def test_delete_hide_b2_multiple_versions_old(self): - dst_file = b2_file('a.txt', [TODAY - 1 * DAY, TODAY - 3 * DAY, TODAY - 5 * DAY]) - actions = [ - 'b2_hide(folder/a.txt)', 'b2_delete(folder/a.txt, id_a_8208000000, (old version))' - ] - self._check_local_to_b2(None, dst_file, FakeArgs(keepDays=2), actions) - - def test_already_hidden_multiple_versions_keep(self): - dst_file = b2_file('a.txt', [-TODAY, TODAY - 2 * DAY, TODAY - 4 * DAY]) - self._check_local_to_b2(None, dst_file, FakeArgs(), []) - - def test_already_hidden_multiple_versions_keep_days(self): - dst_file = b2_file('a.txt', [-TODAY, TODAY - 2 * DAY, TODAY - 4 * DAY]) - actions = ['b2_delete(folder/a.txt, id_a_8294400000, (old version))'] - self._check_local_to_b2(None, dst_file, FakeArgs(keepDays=1), actions) - - def test_already_hidden_multiple_versions_keep_days_one_old(self): - # The 6-day-old file should be preserved, because it was visible - # 5 days ago. - dst_file = b2_file('a.txt', [-(TODAY - 2 * DAY), TODAY - 4 * DAY, TODAY - 6 * DAY]) - actions = [] - self._check_local_to_b2(None, dst_file, FakeArgs(keepDays=5), actions) - - def test_already_hidden_multiple_versions_keep_days_two_old(self): - dst_file = b2_file('a.txt', [-(TODAY - 2 * DAY), TODAY - 4 * DAY, TODAY - 6 * DAY]) - actions = ['b2_delete(folder/a.txt, id_a_8121600000, (old version))'] - self._check_local_to_b2(None, dst_file, FakeArgs(keepDays=2), actions) - - def test_already_hidden_multiple_versions_keep_days_delete_hide_marker(self): - dst_file = b2_file('a.txt', [-(TODAY - 2 * DAY), TODAY - 4 * DAY, TODAY - 6 * DAY]) - actions = [ - 'b2_delete(folder/a.txt, id_a_8467200000, (hide marker))', - 'b2_delete(folder/a.txt, id_a_8294400000, (old version))', - 'b2_delete(folder/a.txt, id_a_8121600000, (old version))' - ] - self._check_local_to_b2(None, dst_file, FakeArgs(keepDays=1), actions) - - def test_already_hidden_multiple_versions_keep_days_old_delete(self): - dst_file = b2_file('a.txt', [-TODAY + 2 * DAY, TODAY - 4 * DAY]) - actions = [ - 'b2_delete(folder/a.txt, id_a_8467200000, (hide marker))', - 'b2_delete(folder/a.txt, id_a_8294400000, (old version))' - ] - self._check_local_to_b2(None, dst_file, FakeArgs(keepDays=1), actions) - - def test_already_hidden_multiple_versions_delete(self): - dst_file = b2_file('a.txt', [-TODAY, TODAY - 2 * DAY, TODAY - 4 * DAY]) - actions = [ - 'b2_delete(folder/a.txt, id_a_8640000000, (hide marker))', - 'b2_delete(folder/a.txt, id_a_8467200000, (old version))', - 'b2_delete(folder/a.txt, id_a_8294400000, (old version))' - ] - self._check_local_to_b2(None, dst_file, FakeArgs(delete=True), actions) - - def test_delete_local(self): - dst_file = local_file('a.txt', [100]) - self._check_b2_to_local(None, dst_file, FakeArgs(delete=True), ['local_delete(/dir/a.txt)']) - - # src same as dst - - def test_same_b2(self): - src_file = local_file('a.txt', [100]) - dst_file = b2_file('a.txt', [100]) - self._check_local_to_b2(src_file, dst_file, FakeArgs(), []) - - def test_same_local(self): - src_file = b2_file('a.txt', [100]) - dst_file = local_file('a.txt', [100]) - self._check_b2_to_local(src_file, dst_file, FakeArgs(), []) - - def test_same_leave_old_versions(self): - src_file = local_file('a.txt', [TODAY]) - dst_file = b2_file('a.txt', [TODAY, TODAY - 3 * DAY]) - self._check_local_to_b2(src_file, dst_file, FakeArgs(), []) - - def test_same_clean_old_versions(self): - src_file = local_file('a.txt', [TODAY - 3 * DAY]) - dst_file = b2_file('a.txt', [TODAY - 3 * DAY, TODAY - 4 * DAY]) - actions = ['b2_delete(folder/a.txt, id_a_8294400000, (old version))'] - self._check_local_to_b2(src_file, dst_file, FakeArgs(keepDays=1), actions) - - def test_keep_days_no_change_with_old_file(self): - src_file = local_file('a.txt', [TODAY - 3 * DAY]) - dst_file = b2_file('a.txt', [TODAY - 3 * DAY]) - self._check_local_to_b2(src_file, dst_file, FakeArgs(keepDays=1), []) - - def test_same_delete_old_versions(self): - src_file = local_file('a.txt', [TODAY]) - dst_file = b2_file('a.txt', [TODAY, TODAY - 3 * DAY]) - actions = ['b2_delete(folder/a.txt, id_a_8380800000, (old version))'] - self._check_local_to_b2(src_file, dst_file, FakeArgs(delete=True), actions) - - # src newer than dst - - def test_newer_b2(self): - src_file = local_file('a.txt', [200]) - dst_file = b2_file('a.txt', [100]) - actions = ['b2_upload(/dir/a.txt, folder/a.txt, 200)'] - self._check_local_to_b2(src_file, dst_file, FakeArgs(), actions) - - def test_newer_b2_clean_old_versions(self): - src_file = local_file('a.txt', [TODAY]) - dst_file = b2_file('a.txt', [TODAY - 1 * DAY, TODAY - 3 * DAY, TODAY - 5 * DAY]) - actions = [ - 'b2_upload(/dir/a.txt, folder/a.txt, 8640000000)', - 'b2_delete(folder/a.txt, id_a_8208000000, (old version))' - ] - self._check_local_to_b2(src_file, dst_file, FakeArgs(keepDays=2), actions) - - def test_newer_b2_delete_old_versions(self): - src_file = local_file('a.txt', [TODAY]) - dst_file = b2_file('a.txt', [TODAY - 1 * DAY, TODAY - 3 * DAY]) - actions = [ - 'b2_upload(/dir/a.txt, folder/a.txt, 8640000000)', - 'b2_delete(folder/a.txt, id_a_8553600000, (old version))', - 'b2_delete(folder/a.txt, id_a_8380800000, (old version))' - ] # yapf disable - self._check_local_to_b2(src_file, dst_file, FakeArgs(delete=True), actions) - - def test_newer_local(self): - src_file = b2_file('a.txt', [200]) - dst_file = local_file('a.txt', [100]) - actions = ['b2_download(folder/a.txt, id_a_200, /dir/a.txt, 200)'] - self._check_b2_to_local(src_file, dst_file, FakeArgs(delete=True), actions) - - # src older than dst - - def test_older_b2(self): - src_file = local_file('a.txt', [100]) - dst_file = b2_file('a.txt', [200]) - try: - self._check_local_to_b2(src_file, dst_file, FakeArgs(), []) - self.fail('should have raised DestFileNewer') - except DestFileNewer as e: - self.assertEqual( - 'source file is older than destination: local://a.txt with a time of 100 cannot be synced to b2://a.txt with a time of 200, unless --skipNewer or --replaceNewer is provided', - str(e) - ) - - def test_older_b2_skip(self): - src_file = local_file('a.txt', [100]) - dst_file = b2_file('a.txt', [200]) - self._check_local_to_b2(src_file, dst_file, FakeArgs(skipNewer=True), []) - - def test_older_b2_replace(self): - src_file = local_file('a.txt', [100]) - dst_file = b2_file('a.txt', [200]) - actions = ['b2_upload(/dir/a.txt, folder/a.txt, 100)'] - self._check_local_to_b2(src_file, dst_file, FakeArgs(replaceNewer=True), actions) - - def test_older_b2_replace_delete(self): - src_file = local_file('a.txt', [100]) - dst_file = b2_file('a.txt', [200]) - args = FakeArgs(replaceNewer=True, delete=True) - actions = [ - 'b2_upload(/dir/a.txt, folder/a.txt, 100)', - 'b2_delete(folder/a.txt, id_a_200, (old version))' - ] - self._check_local_to_b2(src_file, dst_file, args, actions) - - def test_older_local(self): - src_file = b2_file('directory/a.txt', [100]) - dst_file = local_file('directory/a.txt', [200]) - try: - self._check_b2_to_local(src_file, dst_file, FakeArgs(), []) - self.fail('should have raised DestFileNewer') - except DestFileNewer as e: - self.assertEqual( - 'source file is older than destination: b2://directory/a.txt with a time of 100 cannot be synced to local://directory/a.txt with a time of 200, unless --skipNewer or --replaceNewer is provided', - str(e) - ) - - def test_older_local_skip(self): - src_file = b2_file('a.txt', [100]) - dst_file = local_file('a.txt', [200]) - self._check_b2_to_local(src_file, dst_file, FakeArgs(skipNewer=True), []) - - def test_older_local_replace(self): - src_file = b2_file('a.txt', [100]) - dst_file = local_file('a.txt', [200]) - actions = ['b2_download(folder/a.txt, id_a_100, /dir/a.txt, 100)'] - self._check_b2_to_local(src_file, dst_file, FakeArgs(replaceNewer=True), actions) - - # compareVersions option - - def test_compare_b2_none_newer(self): - src_file = local_file('a.txt', [200]) - dst_file = b2_file('a.txt', [100]) - self._check_local_to_b2(src_file, dst_file, FakeArgs(compareVersions='none'), []) - - def test_compare_b2_none_older(self): - src_file = local_file('a.txt', [100]) - dst_file = b2_file('a.txt', [200]) - self._check_local_to_b2(src_file, dst_file, FakeArgs(compareVersions='none'), []) - - def test_compare_b2_size_equal(self): - src_file = local_file('a.txt', [200], size=10) - dst_file = b2_file('a.txt', [100], size=10) - self._check_local_to_b2(src_file, dst_file, FakeArgs(compareVersions='size'), []) - - def test_compare_b2_size_not_equal(self): - src_file = local_file('a.txt', [200], size=11) - dst_file = b2_file('a.txt', [100], size=10) - actions = ['b2_upload(/dir/a.txt, folder/a.txt, 200)'] - self._check_local_to_b2(src_file, dst_file, FakeArgs(compareVersions='size'), actions) - - def test_compare_b2_size_not_equal_delete(self): - src_file = local_file('a.txt', [200], size=11) - dst_file = b2_file('a.txt', [100], size=10) - args = FakeArgs(compareVersions='size', delete=True) - actions = [ - 'b2_upload(/dir/a.txt, folder/a.txt, 200)', - 'b2_delete(folder/a.txt, id_a_100, (old version))' - ] - self._check_local_to_b2(src_file, dst_file, args, actions) - - # helper methods - - def _check_local_to_b2(self, src_file, dst_file, args, expected_actions): - self._check_one_file('local', src_file, 'b2', dst_file, args, expected_actions) - - def _check_b2_to_local(self, src_file, dst_file, args, expected_actions): - self._check_one_file('b2', src_file, 'local', dst_file, args, expected_actions) - - def _check_one_file(self, src_type, src_file, dst_type, dst_file, args, expected_actions): - """ - Checks the actions generated for one file. The file may or may not - exist at the source, and may or may not exist at the destination. - Passing in None means that the file does not exist. - - The source and destination files may have multiple versions. - """ - src_folder = FakeFolder(src_type, [src_file] if src_file else []) - dst_folder = FakeFolder(dst_type, [dst_file] if dst_file else []) - actions = list(make_folder_sync_actions(src_folder, dst_folder, args, TODAY, self.reporter)) - action_strs = [str(a) for a in actions] - if expected_actions != action_strs: - print('Expected:') - for a in expected_actions: - print(' ', a) - print('Actual:') - for a in action_strs: - print(' ', a) - self.assertEqual(expected_actions, [str(a) for a in actions]) - - class TestBoundedQueueExecutor(TestBase): def test_run_more_than_queue_size(self): """ diff --git a/test/unit/v1/apiver/__init__.py b/test/unit/v1/apiver/__init__.py new file mode 100644 index 000000000..2d3ee8086 --- /dev/null +++ b/test/unit/v1/apiver/__init__.py @@ -0,0 +1,12 @@ +###################################################################### +# +# File: test/unit/v1/apiver/__init__.py +# +# Copyright 2020 Backblaze Inc. All Rights Reserved. +# +# License https://www.backblaze.com/using_b2_code.html +# +###################################################################### + +# configured by pytest using `--api` option +# check test/unit/conftest.py:pytest_configure for details diff --git a/test/unit/v1/apiver/apiver_deps.py b/test/unit/v1/apiver/apiver_deps.py new file mode 100644 index 000000000..05d683ece --- /dev/null +++ b/test/unit/v1/apiver/apiver_deps.py @@ -0,0 +1,11 @@ +###################################################################### +# +# File: test/unit/v1/apiver/apiver_deps.py +# +# Copyright 2020 Backblaze Inc. All Rights Reserved. +# +# License https://www.backblaze.com/using_b2_code.html +# +###################################################################### + +from b2sdk.v1 import * diff --git a/test/unit/v1/apiver/apiver_deps_exception.py b/test/unit/v1/apiver/apiver_deps_exception.py new file mode 100644 index 000000000..ea1115e1c --- /dev/null +++ b/test/unit/v1/apiver/apiver_deps_exception.py @@ -0,0 +1,11 @@ +###################################################################### +# +# File: test/unit/v1/apiver/apiver_deps_exception.py +# +# Copyright 2020 Backblaze Inc. All Rights Reserved. +# +# License https://www.backblaze.com/using_b2_code.html +# +###################################################################### + +from b2sdk.v1.exception import * diff --git a/test/unit/v1/deps.py b/test/unit/v1/deps.py index b5d424b94..42e7e1916 100644 --- a/test/unit/v1/deps.py +++ b/test/unit/v1/deps.py @@ -8,4 +8,9 @@ # ###################################################################### -from b2sdk.v1 import * +# TODO: This module is used in old-style unit tests, written separately for v0 and v1. +# It will be removed when all test are rewritten for the new style, like e.g. test/unit/sync/. + +# configured by pytest using `--api` option +# check test/unit/conftest.py:pytest_configure for details +from apiver_deps import * diff --git a/test/unit/v1/deps_exception.py b/test/unit/v1/deps_exception.py index af5a4d578..aa8a2a650 100644 --- a/test/unit/v1/deps_exception.py +++ b/test/unit/v1/deps_exception.py @@ -8,4 +8,9 @@ # ###################################################################### -from b2sdk.v1.exception import * +# TODO: This module is used in old-style unit tests, written separately for v0 and v1. +# It will be removed when all test are rewritten for the new style, like e.g. test/unit/sync/. + +# configured by pytest using `--api` option +# check test/unit/conftest.py:pytest_configure for details +from apiver_deps_exception import * diff --git a/test/unit/v1/test_sync.py b/test/unit/v1/test_sync.py index 4d5b77881..2209459fc 100644 --- a/test/unit/v1/test_sync.py +++ b/test/unit/v1/test_sync.py @@ -22,7 +22,6 @@ from .test_base import TestBase -from .deps_exception import DestFileNewer from .deps_exception import UnSyncableFilename from .deps import FileVersionInfo from .deps import AbstractFolder, B2Folder, LocalFolder @@ -33,7 +32,6 @@ from .deps import parse_sync_folder from .deps import TempDir from .deps import Synchronizer -from .deps import InvalidArgument DAY = 86400000 # milliseconds TODAY = DAY * 100 # an arbitrary reference time for testing @@ -747,39 +745,6 @@ def get_synchronizer(self, policies_manager=DEFAULT_SCAN_MANAGER): ) -def b2_file(name, mod_times, size=10): - """ - Makes a File object for a b2 file, with one FileVersion for - each modification time given in mod_times. - - Positive modification times are uploads, and negative modification - times are hides. It's a hack, but it works. - - b2_file('a.txt', [300, -200, 100]) - - Is the same as: - - File( - 'a.txt', - [ - FileVersion('id_a_300', 'a.txt', 300, 'upload'), - FileVersion('id_a_200', 'a.txt', 200, 'hide'), - FileVersion('id_a_100', 'a.txt', 100, 'upload') - ] - ) - """ - versions = [ - FileVersion( - 'id_%s_%d' % (name[0], abs(mod_time)), - 'folder/' + name, - abs(mod_time), - 'upload' if 0 < mod_time else 'hide', - size, - ) for mod_time in mod_times - ] # yapf disable - return File(name, versions) - - def local_file(name, mod_times, size=10): """ Makes a File object for a b2 file, with one FileVersion for @@ -856,509 +821,6 @@ def test_file_exclusions_inclusions_with_delete(self): self._check_folder_sync(expected_actions, fakeargs) -class IllegalEnum(Enum): - ILLEGAL = 5100 - - -class TestMakeSyncActions(TestSync): - def test_illegal_b2_to_b2(self): - b2_folder = FakeFolder('b2', []) - with self.assertRaises(NotImplementedError): - fakeargs = FakeArgs() - syncronizer = fakeargs.get_synchronizer() - list(syncronizer.make_folder_sync_actions(b2_folder, b2_folder, 0, self.reporter)) - - def test_illegal_local_to_local(self): - local_folder = FakeFolder('local', []) - with self.assertRaises(NotImplementedError): - fakeargs = FakeArgs() - syncronizer = fakeargs.get_synchronizer() - list(syncronizer.make_folder_sync_actions(local_folder, local_folder, 0, self.reporter)) - - def test_illegal_newer_file_mode(self): - with self.assertRaises(InvalidArgument): - self._check_local_to_b2( - None, - None, - FakeArgs(newer_file_mode=IllegalEnum.ILLEGAL), - [], - ) - - def test_illegal_delete_and_keep_days(self): - with self.assertRaises(InvalidArgument): - self._check_local_to_b2( - None, - None, - FakeArgs(keep_days_or_delete=IllegalEnum.ILLEGAL), - [], - ) - - # src: absent, dst: absent - - def test_empty_b2(self): - self._check_local_to_b2(None, None, FakeArgs(), []) - - def test_empty_local(self): - self._check_b2_to_local(None, None, FakeArgs(), []) - - # src: present, dst: absent - - def test_not_there_b2(self): - src_file = local_file('a.txt', [100]) - self._check_local_to_b2( - src_file, None, FakeArgs(), ['b2_upload(/dir/a.txt, folder/a.txt, 100)'] - ) - - def test_dir_not_there_b2_keepdays(self): # reproduces issue 220 - src_file = b2_file('directory/a.txt', [100]) - actions = ['b2_upload(/dir/directory/a.txt, folder/directory/a.txt, 100)'] - self._check_local_to_b2( - src_file, - None, - FakeArgs(keep_days_or_delete=KeepOrDeleteMode.KEEP_BEFORE_DELETE, keep_days=1), - actions, - ) - - def test_dir_not_there_b2_delete(self): # reproduces issue 218 - src_file = b2_file('directory/a.txt', [100]) - actions = ['b2_upload(/dir/directory/a.txt, folder/directory/a.txt, 100)'] - self._check_local_to_b2( - src_file, - None, - FakeArgs(keep_days_or_delete=KeepOrDeleteMode.DELETE), - actions, - ) - - def test_not_there_local(self): - src_file = b2_file('a.txt', [100]) - actions = ['b2_download(folder/a.txt, id_a_100, /dir/a.txt, 100)'] - self._check_b2_to_local(src_file, None, FakeArgs(), actions) - - # src: absent, dst: present - - def test_no_delete_b2(self): - dst_file = b2_file('a.txt', [100]) - self._check_local_to_b2(None, dst_file, FakeArgs(), []) - - def test_no_delete_local(self): - dst_file = local_file('a.txt', [100]) - self._check_b2_to_local(None, dst_file, FakeArgs(), []) - - def test_delete_b2(self): - dst_file = b2_file('a.txt', [100]) - actions = ['b2_delete(folder/a.txt, id_a_100, )'] - self._check_local_to_b2( - None, - dst_file, - FakeArgs(keep_days_or_delete=KeepOrDeleteMode.DELETE), - actions, - ) - - def test_delete_large_b2(self): - dst_file = b2_file('a.txt', [100]) - actions = ['b2_delete(folder/a.txt, id_a_100, )'] - self._check_local_to_b2( - None, - dst_file, - FakeArgs(keep_days_or_delete=KeepOrDeleteMode.DELETE), - actions, - ) - - def test_delete_b2_multiple_versions(self): - dst_file = b2_file('a.txt', [100, 200]) - actions = [ - 'b2_delete(folder/a.txt, id_a_100, )', - 'b2_delete(folder/a.txt, id_a_200, (old version))' - ] - self._check_local_to_b2( - None, - dst_file, - FakeArgs(keep_days_or_delete=KeepOrDeleteMode.DELETE), - actions, - ) - - def test_delete_hide_b2_multiple_versions(self): - dst_file = b2_file('a.txt', [TODAY, TODAY - 2 * DAY, TODAY - 4 * DAY]) - actions = [ - 'b2_hide(folder/a.txt)', 'b2_delete(folder/a.txt, id_a_8294400000, (old version))' - ] - self._check_local_to_b2( - None, - dst_file, - FakeArgs(keep_days_or_delete=KeepOrDeleteMode.KEEP_BEFORE_DELETE, keep_days=1), - actions, - ) - - def test_delete_hide_b2_multiple_versions_old(self): - dst_file = b2_file('a.txt', [TODAY - 1 * DAY, TODAY - 3 * DAY, TODAY - 5 * DAY]) - actions = [ - 'b2_hide(folder/a.txt)', 'b2_delete(folder/a.txt, id_a_8208000000, (old version))' - ] - self._check_local_to_b2( - None, - dst_file, - FakeArgs(keep_days_or_delete=KeepOrDeleteMode.KEEP_BEFORE_DELETE, keep_days=2), - actions, - ) - - def test_already_hidden_multiple_versions_keep(self): - dst_file = b2_file('a.txt', [-TODAY, TODAY - 2 * DAY, TODAY - 4 * DAY]) - self._check_local_to_b2(None, dst_file, FakeArgs(), []) - - def test_already_hidden_multiple_versions_keep_days(self): - dst_file = b2_file('a.txt', [-TODAY, TODAY - 2 * DAY, TODAY - 4 * DAY]) - actions = ['b2_delete(folder/a.txt, id_a_8294400000, (old version))'] - self._check_local_to_b2( - None, - dst_file, - FakeArgs(keep_days_or_delete=KeepOrDeleteMode.KEEP_BEFORE_DELETE, keep_days=1), - actions, - ) - - def test_already_hidden_multiple_versions_keep_days_one_old(self): - # The 6-day-old file should be preserved, because it was visible - # 5 days ago. - dst_file = b2_file('a.txt', [-(TODAY - 2 * DAY), TODAY - 4 * DAY, TODAY - 6 * DAY]) - actions = [] - self._check_local_to_b2( - None, - dst_file, - FakeArgs(keep_days_or_delete=KeepOrDeleteMode.KEEP_BEFORE_DELETE, keep_days=5), - actions, - ) - - def test_already_hidden_multiple_versions_keep_days_two_old(self): - dst_file = b2_file('a.txt', [-(TODAY - 2 * DAY), TODAY - 4 * DAY, TODAY - 6 * DAY]) - actions = ['b2_delete(folder/a.txt, id_a_8121600000, (old version))'] - self._check_local_to_b2( - None, - dst_file, - FakeArgs(keep_days_or_delete=KeepOrDeleteMode.KEEP_BEFORE_DELETE, keep_days=2), - actions, - ) - - def test_already_hidden_multiple_versions_keep_days_delete_hide_marker(self): - dst_file = b2_file('a.txt', [-(TODAY - 2 * DAY), TODAY - 4 * DAY, TODAY - 6 * DAY]) - actions = [ - 'b2_delete(folder/a.txt, id_a_8467200000, (hide marker))', - 'b2_delete(folder/a.txt, id_a_8294400000, (old version))', - 'b2_delete(folder/a.txt, id_a_8121600000, (old version))' - ] - self._check_local_to_b2( - None, - dst_file, - FakeArgs(keep_days_or_delete=KeepOrDeleteMode.KEEP_BEFORE_DELETE, keep_days=1), - actions, - ) - - def test_already_hidden_multiple_versions_keep_days_old_delete(self): - dst_file = b2_file('a.txt', [-TODAY + 2 * DAY, TODAY - 4 * DAY]) - actions = [ - 'b2_delete(folder/a.txt, id_a_8467200000, (hide marker))', - 'b2_delete(folder/a.txt, id_a_8294400000, (old version))' - ] - self._check_local_to_b2( - None, - dst_file, - FakeArgs(keep_days_or_delete=KeepOrDeleteMode.KEEP_BEFORE_DELETE, keep_days=1), - actions, - ) - - def test_already_hidden_multiple_versions_delete(self): - dst_file = b2_file('a.txt', [-TODAY, TODAY - 2 * DAY, TODAY - 4 * DAY]) - actions = [ - 'b2_delete(folder/a.txt, id_a_8640000000, (hide marker))', - 'b2_delete(folder/a.txt, id_a_8467200000, (old version))', - 'b2_delete(folder/a.txt, id_a_8294400000, (old version))' - ] - self._check_local_to_b2( - None, - dst_file, - FakeArgs(keep_days_or_delete=KeepOrDeleteMode.DELETE), - actions, - ) - - def test_delete_local(self): - dst_file = local_file('a.txt', [100]) - self._check_b2_to_local( - None, - dst_file, - FakeArgs(keep_days_or_delete=KeepOrDeleteMode.DELETE), - ['local_delete(/dir/a.txt)'], - ) - - # src same as dst - - def test_same_b2(self): - src_file = local_file('a.txt', [100]) - dst_file = b2_file('a.txt', [100]) - self._check_local_to_b2(src_file, dst_file, FakeArgs(), []) - - def test_same_local(self): - src_file = b2_file('a.txt', [100]) - dst_file = local_file('a.txt', [100]) - self._check_b2_to_local(src_file, dst_file, FakeArgs(), []) - - def test_same_leave_old_versions(self): - src_file = local_file('a.txt', [TODAY]) - dst_file = b2_file('a.txt', [TODAY, TODAY - 3 * DAY]) - self._check_local_to_b2(src_file, dst_file, FakeArgs(), []) - - def test_same_clean_old_versions(self): - src_file = local_file('a.txt', [TODAY - 3 * DAY]) - dst_file = b2_file('a.txt', [TODAY - 3 * DAY, TODAY - 4 * DAY]) - actions = ['b2_delete(folder/a.txt, id_a_8294400000, (old version))'] - self._check_local_to_b2( - src_file, - dst_file, - FakeArgs(keep_days_or_delete=KeepOrDeleteMode.KEEP_BEFORE_DELETE, keep_days=1), - actions, - ) - - def test_keep_days_no_change_with_old_file(self): - src_file = local_file('a.txt', [TODAY - 3 * DAY]) - dst_file = b2_file('a.txt', [TODAY - 3 * DAY]) - self._check_local_to_b2( - src_file, - dst_file, - FakeArgs(keep_days_or_delete=KeepOrDeleteMode.KEEP_BEFORE_DELETE, keep_days=1), - [], - ) - - def test_same_delete_old_versions(self): - src_file = local_file('a.txt', [TODAY]) - dst_file = b2_file('a.txt', [TODAY, TODAY - 3 * DAY]) - actions = ['b2_delete(folder/a.txt, id_a_8380800000, (old version))'] - self._check_local_to_b2( - src_file, - dst_file, - FakeArgs(keep_days_or_delete=KeepOrDeleteMode.DELETE), - actions, - ) - - # src newer than dst - - def test_newer_b2(self): - src_file = local_file('a.txt', [200]) - dst_file = b2_file('a.txt', [100]) - actions = ['b2_upload(/dir/a.txt, folder/a.txt, 200)'] - self._check_local_to_b2(src_file, dst_file, FakeArgs(), actions) - - def test_newer_b2_clean_old_versions(self): - src_file = local_file('a.txt', [TODAY]) - dst_file = b2_file('a.txt', [TODAY - 1 * DAY, TODAY - 3 * DAY, TODAY - 5 * DAY]) - actions = [ - 'b2_upload(/dir/a.txt, folder/a.txt, 8640000000)', - 'b2_delete(folder/a.txt, id_a_8208000000, (old version))' - ] - self._check_local_to_b2( - src_file, - dst_file, - FakeArgs(keep_days_or_delete=KeepOrDeleteMode.KEEP_BEFORE_DELETE, keep_days=2), - actions, - ) - - def test_newer_b2_delete_old_versions(self): - src_file = local_file('a.txt', [TODAY]) - dst_file = b2_file('a.txt', [TODAY - 1 * DAY, TODAY - 3 * DAY]) - actions = [ - 'b2_upload(/dir/a.txt, folder/a.txt, 8640000000)', - 'b2_delete(folder/a.txt, id_a_8553600000, (old version))', - 'b2_delete(folder/a.txt, id_a_8380800000, (old version))' - ] # yapf disable - self._check_local_to_b2( - src_file, - dst_file, - FakeArgs(keep_days_or_delete=KeepOrDeleteMode.DELETE), - actions, - ) - - def test_newer_local(self): - src_file = b2_file('a.txt', [200]) - dst_file = local_file('a.txt', [100]) - actions = ['b2_download(folder/a.txt, id_a_200, /dir/a.txt, 200)'] - self._check_b2_to_local( - src_file, - dst_file, - FakeArgs(keep_days_or_delete=KeepOrDeleteMode.DELETE), - actions, - ) - - # src older than dst - - def test_older_b2(self): - src_file = local_file('a.txt', [100]) - dst_file = b2_file('a.txt', [200]) - try: - self._check_local_to_b2(src_file, dst_file, FakeArgs(), []) - self.fail('should have raised DestFileNewer') - except DestFileNewer as e: - self.assertEqual( - 'source file is older than destination: local://a.txt with a time of 100 cannot be synced to b2://a.txt with a time of 200, unless a valid newer_file_mode is provided', - str(e) - ) - - def test_older_b2_skip(self): - src_file = local_file('a.txt', [100]) - dst_file = b2_file('a.txt', [200]) - self._check_local_to_b2( - src_file, - dst_file, - FakeArgs(newer_file_mode=NewerFileSyncMode.SKIP), - [], - ) - - def test_older_b2_replace(self): - src_file = local_file('a.txt', [100]) - dst_file = b2_file('a.txt', [200]) - actions = ['b2_upload(/dir/a.txt, folder/a.txt, 100)'] - self._check_local_to_b2( - src_file, - dst_file, - FakeArgs(newer_file_mode=NewerFileSyncMode.REPLACE), - actions, - ) - - def test_older_b2_replace_delete(self): - src_file = local_file('a.txt', [100]) - dst_file = b2_file('a.txt', [200]) - args = FakeArgs( - newer_file_mode=NewerFileSyncMode.REPLACE, - keep_days_or_delete=KeepOrDeleteMode.DELETE, - ) - actions = [ - 'b2_upload(/dir/a.txt, folder/a.txt, 100)', - 'b2_delete(folder/a.txt, id_a_200, (old version))' - ] - self._check_local_to_b2(src_file, dst_file, args, actions) - - def test_older_local(self): - src_file = b2_file('directory/a.txt', [100]) - dst_file = local_file('directory/a.txt', [200]) - try: - self._check_b2_to_local(src_file, dst_file, FakeArgs(), []) - self.fail('should have raised DestFileNewer') - except DestFileNewer as e: - self.assertEqual( - 'source file is older than destination: b2://directory/a.txt with a time of 100 cannot be synced to local://directory/a.txt with a time of 200, unless a valid newer_file_mode is provided', - str(e) - ) - - def test_older_local_skip(self): - src_file = b2_file('a.txt', [100]) - dst_file = local_file('a.txt', [200]) - self._check_b2_to_local( - src_file, - dst_file, - FakeArgs(newer_file_mode=NewerFileSyncMode.SKIP), - [], - ) - - def test_older_local_replace(self): - src_file = b2_file('a.txt', [100]) - dst_file = local_file('a.txt', [200]) - actions = ['b2_download(folder/a.txt, id_a_100, /dir/a.txt, 100)'] - self._check_b2_to_local( - src_file, - dst_file, - FakeArgs(newer_file_mode=NewerFileSyncMode.REPLACE), - actions, - ) - - # compareVersions option - - def test_compare_b2_none_newer(self): - src_file = local_file('a.txt', [200]) - dst_file = b2_file('a.txt', [100]) - self._check_local_to_b2( - src_file, - dst_file, - FakeArgs(compare_version_mode=CompareVersionMode.NONE), - [], - ) - - def test_compare_b2_none_older(self): - src_file = local_file('a.txt', [100]) - dst_file = b2_file('a.txt', [200]) - self._check_local_to_b2( - src_file, - dst_file, - FakeArgs(compare_version_mode=CompareVersionMode.NONE), - [], - ) - - def test_compare_b2_size_equal(self): - src_file = local_file('a.txt', [200], size=10) - dst_file = b2_file('a.txt', [100], size=10) - self._check_local_to_b2( - src_file, - dst_file, - FakeArgs(compare_version_mode=CompareVersionMode.SIZE), - [], - ) - - def test_compare_b2_size_not_equal(self): - src_file = local_file('a.txt', [200], size=11) - dst_file = b2_file('a.txt', [100], size=10) - actions = ['b2_upload(/dir/a.txt, folder/a.txt, 200)'] - self._check_local_to_b2( - src_file, - dst_file, - FakeArgs(compare_version_mode=CompareVersionMode.SIZE), - actions, - ) - - def test_compare_b2_size_not_equal_delete(self): - src_file = local_file('a.txt', [200], size=11) - dst_file = b2_file('a.txt', [100], size=10) - args = FakeArgs( - compare_version_mode=CompareVersionMode.SIZE, - keep_days_or_delete=KeepOrDeleteMode.DELETE, - ) - actions = [ - 'b2_upload(/dir/a.txt, folder/a.txt, 200)', - 'b2_delete(folder/a.txt, id_a_100, (old version))' - ] - self._check_local_to_b2(src_file, dst_file, args, actions) - - # helper methods - - def _check_local_to_b2(self, src_file, dst_file, fakeargs, expected_actions): - self._check_one_file('local', src_file, 'b2', dst_file, fakeargs, expected_actions) - - def _check_b2_to_local(self, src_file, dst_file, fakeargs, expected_actions): - self._check_one_file('b2', src_file, 'local', dst_file, fakeargs, expected_actions) - - def _check_one_file(self, src_type, src_file, dst_type, dst_file, fakeargs, expected_actions): - """ - Checks the actions generated for one file. The file may or may not - exist at the source, and may or may not exist at the destination. - Passing in None means that the file does not exist. - - The source and destination files may have multiple versions. - """ - src_folder = FakeFolder(src_type, [src_file] if src_file else []) - dst_folder = FakeFolder(dst_type, [dst_file] if dst_file else []) - synchronizer = fakeargs.get_synchronizer() - actions = list( - synchronizer.make_folder_sync_actions( - src_folder, - dst_folder, - TODAY, - self.reporter, - ) - ) - action_strs = [str(a) for a in actions] - if expected_actions != action_strs: - print('Expected:') - for a in expected_actions: - print(' ', a) - print('Actual:') - for a in action_strs: - print(' ', a) - self.assertEqual(expected_actions, [str(a) for a in actions]) - - class TestBoundedQueueExecutor(TestBase): def test_run_more_than_queue_size(self): """