From 2fe70c6921361df2fb391e8b0c5bfa1cd11224c2 Mon Sep 17 00:00:00 2001 From: Pawel Polewicz Date: Wed, 11 Dec 2024 21:01:15 +0200 Subject: [PATCH 1/3] Add --exclude-if-uploaded-after sync option --- b2/_internal/console_tool.py | 10 ++++++++++ changelog.d/+9092be80.added.md | 1 + 2 files changed, 11 insertions(+) create mode 100644 changelog.d/+9092be80.added.md diff --git a/b2/_internal/console_tool.py b/b2/_internal/console_tool.py index 07884790..ddeec663 100644 --- a/b2/_internal/console_tool.py +++ b/b2/_internal/console_tool.py @@ -3007,6 +3007,8 @@ class Sync( Ignored files or file versions will not be taken for consideration during sync. The time should be given as a seconds timestamp (e.g. "1367900664") If you need milliseconds precision, put it after the comma (e.g. "1367900664.152") + Alternatively you can specify ``--exclude-if-uploaded-after`` to use the server-side + object creation timestamp rather than the modification time declared by the client. Files are considered to be the same if they have the same name and modification time. This behaviour can be changed using the @@ -3124,6 +3126,13 @@ def _setup_parser(cls, parser): default=None, metavar='TIMESTAMP' ) + add_normalized_argument( + parser, + '--exclude-if-uploaded-after', + type=parse_millis_from_float_timestamp, + default=None, + metavar='TIMESTAMP' + ) super()._setup_parser(parser) # add parameters from the mixins, and the parent class parser.add_argument('source') parser.add_argument('destination') @@ -3217,6 +3226,7 @@ def get_policies_manager_from_args(self, args): include_file_regexes=args.include_regex, exclude_all_symlinks=args.exclude_all_symlinks, exclude_modified_after=args.exclude_if_modified_after, + exclude_uploaded_after=args.exclude_if_uploaded_after, ) def get_synchronizer_from_args( diff --git a/changelog.d/+9092be80.added.md b/changelog.d/+9092be80.added.md new file mode 100644 index 00000000..b8abdc7e --- /dev/null +++ b/changelog.d/+9092be80.added.md @@ -0,0 +1 @@ +Add `--exclude-if-uploaded-after` to `sync`. From be5ddda96bfa150206c338399063794515cebec7 Mon Sep 17 00:00:00 2001 From: Maciej Lech Date: Wed, 18 Dec 2024 10:19:32 +0100 Subject: [PATCH 2/3] Add UTs --- test/unit/test_console_tool.py | 47 ++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/test/unit/test_console_tool.py b/test/unit/test_console_tool.py index fbbe92cb..108fba2e 100644 --- a/test/unit/test_console_tool.py +++ b/test/unit/test_console_tool.py @@ -15,6 +15,7 @@ from functools import lru_cache from io import StringIO from itertools import chain, product +from tempfile import TemporaryDirectory from test.helpers import skip_on_windows from typing import List, Optional from unittest import mock @@ -2210,6 +2211,52 @@ def test_sync_exclude_if_modified_after_exact(self): ] self._run_command(command, expected_stdout, '', 0) + def test_sync_exclude_if_uploaded_after_in_range(self): + self._authorize_account() + self._create_my_bucket() + + with TemporaryDirectory() as temp_dir: + for file, utime in (('test.txt', 1367900664152), ('test2.txt', 1367600664152)): + file_path = self._make_local_file(temp_dir, file) + command = [ + 'file', 'upload', '--no-progress', '--custom-upload-timestamp', + str(utime), 'my-bucket', file_path, file + ] + self._run_command(command, expected_status=0) + + with TemporaryDirectory() as temp_dir: + command = [ + 'sync', '--no-progress', '--exclude-if-uploaded-after', '1367700664.152', + 'b2://my-bucket', temp_dir + ] + expected_stdout = ''' + dnload test2.txt + ''' + self._run_command(command, expected_stdout, '', 0) + + def test_sync_exclude_if_uploaded_after_exact(self): + self._authorize_account() + self._create_my_bucket() + + with TemporaryDirectory() as temp_dir: + for file, utime in (('test.txt', 1367900664152), ('test2.txt', 1367600664152)): + file_path = self._make_local_file(temp_dir, file) + command = [ + 'file', 'upload', '--no-progress', '--custom-upload-timestamp', + str(utime), 'my-bucket', file_path, file + ] + self._run_command(command, expected_status=0) + + with TemporaryDirectory() as temp_dir: + command = [ + 'sync', '--no-progress', '--exclude-if-uploaded-after', '1367600664.152', + 'b2://my-bucket', temp_dir + ] + expected_stdout = ''' + dnload test2.txt + ''' + self._run_command(command, expected_stdout, '', 0) + def _test_sync_threads( self, threads=None, From 8caf29b89442d6487704a3b65b41f0ed8cc95d07 Mon Sep 17 00:00:00 2001 From: Maciej Lech Date: Wed, 18 Dec 2024 11:02:06 +0100 Subject: [PATCH 3/3] Update integration tests --- test/integration/test_b2_command_line.py | 30 +++++++++++++++++++++--- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/test/integration/test_b2_command_line.py b/test/integration/test_b2_command_line.py index 2082bc93..a2b6c1ed 100755 --- a/test/integration/test_b2_command_line.py +++ b/test/integration/test_b2_command_line.py @@ -1197,12 +1197,15 @@ def sync_down_helper(b2_tool, bucket_name, folder_in_bucket, sample_file, encryp upload_encryption_args, additional_env=upload_additional_env, ) + + # Sync all the files b2_tool.should_succeed( ['sync', b2_sync_point, local_path] + sync_encryption_args, additional_env=sync_additional_env, ) should_equal(['a', 'b'], sorted(os.listdir(local_path))) + # Put another file in B2 b2_tool.should_succeed( ['file', 'upload', '--no-progress', bucket_name, sample_file, b2_file_prefix + 'c'] + upload_encryption_args, @@ -1219,14 +1222,35 @@ def sync_down_helper(b2_tool, bucket_name, folder_in_bucket, sample_file, encryp additional_env=sync_additional_env, ) should_equal(['a', 'b'], sorted(os.listdir(local_path))) + + # Put another file in B2 with a custom upload timestamp + b2_tool.should_succeed( + [ + 'file', 'upload', '--no-progress', '--custom-upload-timestamp', '1367900664152', + bucket_name, sample_file, b2_file_prefix + 'd' + ] + upload_encryption_args, + additional_env=upload_additional_env, + ) + + # Sync the files with one file being excluded because of upload timestamp + b2_tool.should_succeed( + [ + 'sync', '--no-progress', '--exclude-if-uploaded-after', '1367900664142', + b2_sync_point, local_path + ] + sync_encryption_args, + additional_env=sync_additional_env, + ) + should_equal(['a', 'b'], sorted(os.listdir(local_path))) + # Sync all the files b2_tool.should_succeed( ['sync', '--no-progress', b2_sync_point, local_path] + sync_encryption_args, additional_env=sync_additional_env, ) - should_equal(['a', 'b', 'c'], sorted(os.listdir(local_path))) - with TempDir() as new_local_path: - if encryption and encryption.mode == EncryptionMode.SSE_C: + should_equal(['a', 'b', 'c', 'd'], sorted(os.listdir(local_path))) + + if encryption and encryption.mode == EncryptionMode.SSE_C: + with TempDir() as new_local_path: b2_tool.should_fail( ['sync', '--no-progress', b2_sync_point, new_local_path] + sync_encryption_args, expected_pattern='ValueError: Using SSE-C requires providing an encryption key via '