From 6d5b68287297aab2287964c82458e2dfd60ff8e7 Mon Sep 17 00:00:00 2001 From: Maciej Urbanski Date: Fri, 11 Oct 2024 17:45:48 +0200 Subject: [PATCH 1/2] add `b2 file server-side-copy` command --- b2/_internal/_cli/b2args.py | 9 +- b2/_internal/_utils/uri.py | 14 ++ b2/_internal/console_tool.py | 74 ++++-- .../+add_file_server_side_copy.added.md | 2 + test/integration/test_b2_command_line.py | 120 +++++----- .../test_file_server_side_copy.py | 225 ++++++++++++++++++ test/unit/test_console_tool.py | 215 ----------------- test/unit/test_copy.py | 13 +- 8 files changed, 375 insertions(+), 297 deletions(-) create mode 100644 changelog.d/+add_file_server_side_copy.added.md create mode 100644 test/unit/console_tool/test_file_server_side_copy.py diff --git a/b2/_internal/_cli/b2args.py b/b2/_internal/_cli/b2args.py index 61214e0f..fb026724 100644 --- a/b2/_internal/_cli/b2args.py +++ b/b2/_internal/_cli/b2args.py @@ -104,7 +104,6 @@ def b2id_or_file_like_b2_uri_or_bucket_name(value: str, *, B2ID_OR_B2_URI_OR_ALL_BUCKETS_ARG_TYPE = wrap_with_argument_type_error( functools.partial(parse_b2_uri, allow_all_buckets=True) ) -B2ID_OR_FILE_LIKE_B2_URI_ARG_TYPE = wrap_with_argument_type_error(b2id_or_file_like_b2_uri) def add_bucket_name_argument( @@ -204,13 +203,17 @@ def add_b2id_or_b2_bucket_uri_argument(parser: argparse.ArgumentParser, name="B2 return arg -def add_b2id_or_file_like_b2_uri_argument(parser: argparse.ArgumentParser, name="B2_URI"): +def add_b2id_or_file_like_b2_uri_argument( + parser: argparse.ArgumentParser, name="B2_URI", *, by_id: Optional[bool] = None +): """ Add a B2 URI pointing to a file as an argument to the parser. """ arg = parser.add_argument( name, - type=B2ID_OR_FILE_LIKE_B2_URI_ARG_TYPE, + type=wrap_with_argument_type_error( + functools.partial(b2id_or_file_like_b2_uri, by_id=by_id) + ), help="B2 URI pointing to a file, e.g. b2://yourBucket/file.txt or b2id://fileId", ) arg.completer = b2uri_file_completer diff --git a/b2/_internal/_utils/uri.py b/b2/_internal/_utils/uri.py index cdc935b5..71965d79 100644 --- a/b2/_internal/_utils/uri.py +++ b/b2/_internal/_utils/uri.py @@ -211,3 +211,17 @@ def _(self, uri: B2URI, *args, filters: Sequence[Filter] = (), **kwargs): @ls.register def _(self, uri: B2FileIdURI, *args, **kwargs): yield self.get_file_info_by_uri(uri), None + + @singledispatchmethod + def copy_by_uri(self, uri, *args, **kwargs): + raise NotImplementedError(f"Unsupported URI type: {type(uri)}") + + @copy_by_uri.register + def _(self, source: B2FileIdURI, destination: B2URI, *args, **kwargs): + destination_bucket = self.get_bucket_by_name(destination.bucket_name) + return destination_bucket.copy(source.file_id, destination.path, *args, **kwargs) + + @copy_by_uri.register + def _(self, source: B2URI, destination: B2URI, *args, **kwargs): + file_info = self.get_file_info_by_uri(source) + return self.copy_by_uri(B2FileIdURI(file_info.id_), destination, *args, **kwargs) diff --git a/b2/_internal/console_tool.py b/b2/_internal/console_tool.py index a40c0204..2c1cbc35 100644 --- a/b2/_internal/console_tool.py +++ b/b2/_internal/console_tool.py @@ -1466,7 +1466,7 @@ def _run(self, args): return 0 -class FileCopyByIdBase( +class FileServerSideCopyBase( HeaderFlagsMixin, DestinationSseMixin, SourceSseMixin, FileRetentionSettingMixin, LegalHoldMixin, Command ): @@ -1520,12 +1520,14 @@ def _setup_parser(cls, parser): add_normalized_argument(info_group, '--info', action='append') add_normalized_argument(info_group, '--no-info', action='store_true', default=False) - parser.add_argument('sourceFileId') - parser.add_argument('destinationBucketName') - parser.add_argument('b2FileName') - super()._setup_parser(parser) # add parameters from the mixins + def get_source_b2_uri(self, args) -> B2URIBase: + raise NotImplementedError + + def get_destination_b2_uri(self, args) -> B2URI: + raise NotImplementedError + def _run(self, args): file_infos = None if args.info: @@ -1540,7 +1542,9 @@ def _run(self, args): '--content-type and --info.' ) - bucket = self.api.get_bucket_by_name(args.destinationBucketName) + source_b2_uri = self.get_source_b2_uri(args) + destination_b2_uri = self.get_destination_b2_uri(args) + destination_encryption_setting = self._get_destination_sse_setting(args) source_encryption_setting = self._get_source_sse_setting(args) legal_hold = self._get_legal_hold_setting(args) @@ -1553,16 +1557,16 @@ def _run(self, args): else: range_args = {} source_file_info, source_content_type = self._determine_source_metadata( - source_file_id=args.sourceFileId, + source_b2_uri=source_b2_uri, source_encryption=source_encryption_setting, destination_encryption=destination_encryption_setting, target_content_type=args.content_type, target_file_info=file_infos, fetch_if_necessary=args.fetch_metadata, ) - file_version = bucket.copy( - args.sourceFileId, - args.b2FileName, + file_version = self.api.copy_by_uri( + source_b2_uri, + destination_b2_uri, **range_args, content_type=args.content_type, file_info=file_infos, @@ -1583,7 +1587,7 @@ def _is_ssec(self, encryption: EncryptionSetting | None): def _determine_source_metadata( self, - source_file_id: str, + source_b2_uri: B2URIBase, destination_encryption: EncryptionSetting | None, source_encryption: EncryptionSetting | None, target_file_info: dict | None, @@ -1602,10 +1606,25 @@ def _determine_source_metadata( 'Attempting to copy file with metadata while either source or destination uses ' 'SSE-C. Use --fetch-metadata to fetch source file metadata before copying.' ) - source_file_version = self.api.get_file_info(source_file_id) + source_file_version = self.api.get_file_info_by_uri(source_b2_uri) return source_file_version.file_info, source_file_version.content_type +class FileServerSideCopyLegacyBase(FileServerSideCopyBase): + @classmethod + def _setup_parser(cls, parser): + parser.add_argument('sourceFileId') + parser.add_argument('destinationBucketName') + parser.add_argument('b2FileName') + super()._setup_parser(parser) + + def get_source_b2_uri(self, args) -> B2URIBase: + return B2FileIdURI(args.sourceFileId) + + def get_destination_b2_uri(self, args) -> B2URI: + return B2URI(args.destinationBucketName, args.b2FileName) + + class BucketCreateBase(DefaultSseMixin, LifecycleRulesMixin, Command): """ Create a new bucket. @@ -5094,10 +5113,10 @@ class File(Command): .. code-block:: {NAME} file cat b2://yourBucket/file.txt - {NAME} file copy-by-id sourceFileId yourBucket file.txt {NAME} file download b2://yourBucket/file.txt localFile.txt {NAME} file hide b2://yourBucket/file.txt {NAME} file info b2://yourBucket/file.txt + {NAME} file server-side-copy b2://yourBucket/file.txt b2://otherBucket/file2.txt {NAME} file update --legal-hold off b2://yourBucket/file.txt {NAME} file upload yourBucket localFile.txt file.txt {NAME} file url b2://yourBucket/file.txt @@ -5136,9 +5155,28 @@ class FileDownload(B2URIFileArgMixin, FileDownloadBase): @File.subcommands_registry.register -class FileCopyById(FileCopyByIdBase): - __doc__ = FileCopyByIdBase.__doc__ +class FileServerSideCopy(FileServerSideCopyBase): + __doc__ = FileServerSideCopyBase.__doc__ + COMMAND_NAME = 'server-side-copy' + + @classmethod + def _setup_parser(cls, parser): + add_b2id_or_file_like_b2_uri_argument(parser, "sourceB2Uri") + add_b2id_or_file_like_b2_uri_argument(parser, "destinationB2Uri", by_id=False) + super()._setup_parser(parser) + + def get_source_b2_uri(self, args) -> B2URIBase: + return args.sourceB2Uri + + def get_destination_b2_uri(self, args) -> B2URI: + return args.destinationB2Uri + + +@File.subcommands_registry.register +class FileCopyById(CmdReplacedByMixin, FileServerSideCopyLegacyBase): + __doc__ = FileServerSideCopyBase.__doc__ COMMAND_NAME = 'copy-by-id' + replaced_by_cmd = (File, FileServerSideCopy) @File.subcommands_registry.register @@ -5216,9 +5254,9 @@ class DownloadFileByName(CmdReplacedByMixin, B2URIBucketNFilenameArgMixin, FileD replaced_by_cmd = (File, FileDownload) -class CopyFileById(CmdReplacedByMixin, FileCopyByIdBase): - __doc__ = FileCopyByIdBase.__doc__ - replaced_by_cmd = (File, FileCopyById) +class CopyFileById(CmdReplacedByMixin, FileServerSideCopyLegacyBase): + __doc__ = FileServerSideCopyBase.__doc__ + replaced_by_cmd = (File, FileServerSideCopy) class HideFile(CmdReplacedByMixin, HideFileBase): diff --git a/changelog.d/+add_file_server_side_copy.added.md b/changelog.d/+add_file_server_side_copy.added.md new file mode 100644 index 00000000..fce9d3c1 --- /dev/null +++ b/changelog.d/+add_file_server_side_copy.added.md @@ -0,0 +1,2 @@ +Add `b2 file server-side-copy b2id:/XXX` (also accepts `b2://bucket/objectName` syntax). +Add deprecation notice to `b2 file copy-by-id` - use `b2 file server-side-copy` instead in new scripts. diff --git a/test/integration/test_b2_command_line.py b/test/integration/test_b2_command_line.py index 0ab3dd6a..51812d4f 100755 --- a/test/integration/test_b2_command_line.py +++ b/test/integration/test_b2_command_line.py @@ -445,7 +445,10 @@ def test_basic(b2_tool, persistent_bucket, sample_file, tmp_path, b2_uri_args, a should_equal([], [f['fileName'] for f in list_of_files]) b2_tool.should_succeed( - ['file', 'copy-by-id', first_a_version['fileId'], bucket_name, f'{subfolder}x'] + [ + 'file', 'server-side-copy', f'b2id://{first_a_version["fileId"]}', + f'b2://{bucket_name}/{subfolder}x' + ] ) b2_tool.should_succeed( @@ -1593,14 +1596,19 @@ def test_sse_b2(b2_tool, persistent_bucket, sample_file, tmp_path, b2_uri_args): b2_tool.should_succeed( [ - 'file', 'copy-by-id', '--destination-server-side-encryption=SSE-B2', - encrypted_version['fileId'], bucket_name, f'{subfolder}/copied_encrypted' + 'file', + 'server-side-copy', + '--destination-server-side-encryption=SSE-B2', + f"b2id://{encrypted_version['fileId']}", + f"b2://{bucket_name}/{subfolder}/copied_encrypted", ] ) b2_tool.should_succeed( [ - 'file', 'copy-by-id', not_encrypted_version['fileId'], bucket_name, - f'{subfolder}/copied_not_encrypted' + 'file', + 'server-side-copy', + f"b2id://{not_encrypted_version['fileId']}", + f"b2://{bucket_name}/{subfolder}/copied_not_encrypted", ] ) @@ -1723,24 +1731,27 @@ def test_sse_c( assert read_file(dir_path / 'b') == read_file(sample_file) b2_tool.should_fail( - ['file', 'copy-by-id', file_version_info['fileId'], bucket_name, 'gonna-fail-anyway'], + [ + 'file', 'server-side-copy', f'b2id://{file_version_info["fileId"]}', + f'b2://{bucket_name}/gonna-fail-anyway' + ], expected_pattern= 'ERROR: The object was stored using a form of Server Side Encryption. The correct ' r'parameters must be provided to retrieve the object. \(bad_request\)' ) b2_tool.should_fail( [ - 'file', 'copy-by-id', '--source-server-side-encryption=SSE-C', - file_version_info['fileId'], bucket_name, 'gonna-fail-anyway' + 'file', 'server-side-copy', '--source-server-side-encryption=SSE-C', + f'b2id://{file_version_info["fileId"]}', f'b2://{bucket_name}/gonna-fail-anyway' ], expected_pattern='ValueError: Using SSE-C requires providing an encryption key via ' 'B2_SOURCE_SSE_C_KEY_B64 env var' ) b2_tool.should_fail( [ - 'file', 'copy-by-id', '--source-server-side-encryption=SSE-C', - '--destination-server-side-encryption=SSE-C', file_version_info['fileId'], bucket_name, - 'gonna-fail-anyway' + 'file', 'server-side-copy', '--source-server-side-encryption=SSE-C', + '--destination-server-side-encryption=SSE-C', f'b2id://{file_version_info["fileId"]}', + f'b2://{bucket_name}/gonna-fail-anyway' ], expected_pattern='ValueError: Using SSE-C requires providing an encryption key via ' 'B2_DESTINATION_SSE_C_KEY_B64 env var', @@ -1748,8 +1759,11 @@ def test_sse_c( ) b2_tool.should_fail( [ - 'file', 'copy-by-id', '--source-server-side-encryption=SSE-C', - file_version_info['fileId'], bucket_name, 'gonna-fail-anyway' + 'file', + 'server-side-copy', + '--source-server-side-encryption=SSE-C', + f'b2id://{file_version_info["fileId"]}', + f'b2://{bucket_name}/gonna-fail-anyway', ], additional_env={'B2_SOURCE_SSE_C_KEY_B64': base64.b64encode(secret).decode()}, expected_pattern= @@ -1759,11 +1773,10 @@ def test_sse_c( b2_tool.should_succeed( [ 'file', - 'copy-by-id', + 'server-side-copy', '--source-server-side-encryption=SSE-C', - file_version_info['fileId'], - bucket_name, - f'{subfolder}/not_encrypted_copied_from_encrypted_metadata_replace', + f'b2id://{file_version_info["fileId"]}', + f'b2://{bucket_name}/{subfolder}/not_encrypted_copied_from_encrypted_metadata_replace', '--info', 'a=b', '--content-type', @@ -1774,11 +1787,10 @@ def test_sse_c( b2_tool.should_succeed( [ 'file', - 'copy-by-id', + 'server-side-copy', '--source-server-side-encryption=SSE-C', - file_version_info['fileId'], - bucket_name, - f'{subfolder}/not_encrypted_copied_from_encrypted_metadata_replace_empty', + f'b2id://{file_version_info["fileId"]}', + f'b2://{bucket_name}/{subfolder}/not_encrypted_copied_from_encrypted_metadata_replace_empty', '--no-info', '--content-type', 'text/plain', @@ -1788,11 +1800,10 @@ def test_sse_c( b2_tool.should_succeed( [ 'file', - 'copy-by-id', + 'server-side-copy', '--source-server-side-encryption=SSE-C', - file_version_info['fileId'], - bucket_name, - f'{subfolder}/not_encrypted_copied_from_encrypted_metadata_pseudo_copy', + f'b2id://{file_version_info["fileId"]}', + f'b2://{bucket_name}/{subfolder}/not_encrypted_copied_from_encrypted_metadata_pseudo_copy', '--fetch-metadata', ], additional_env={'B2_SOURCE_SSE_C_KEY_B64': base64.b64encode(secret).decode()} @@ -1800,12 +1811,11 @@ def test_sse_c( b2_tool.should_succeed( [ 'file', - 'copy-by-id', + 'server-side-copy', '--source-server-side-encryption=SSE-C', '--destination-server-side-encryption=SSE-C', - file_version_info['fileId'], - bucket_name, - f'{subfolder}/encrypted_no_id_copied_from_encrypted', + f'b2id://{file_version_info["fileId"]}', + f'b2://{bucket_name}/{subfolder}/encrypted_no_id_copied_from_encrypted', '--fetch-metadata', ], additional_env={ @@ -1816,12 +1826,11 @@ def test_sse_c( b2_tool.should_succeed( [ 'file', - 'copy-by-id', + 'server-side-copy', '--source-server-side-encryption=SSE-C', '--destination-server-side-encryption=SSE-C', - file_version_info['fileId'], - bucket_name, - f'{subfolder}/encrypted_with_id_copied_from_encrypted_metadata_replace', + f'b2id://{file_version_info["fileId"]}', + f'b2://{bucket_name}/{subfolder}/encrypted_with_id_copied_from_encrypted_metadata_replace', '--no-info', '--content-type', 'text/plain', @@ -1835,12 +1844,11 @@ def test_sse_c( b2_tool.should_succeed( [ 'file', - 'copy-by-id', + 'server-side-copy', '--source-server-side-encryption=SSE-C', '--destination-server-side-encryption=SSE-C', - file_version_info['fileId'], - bucket_name, - f'{subfolder}/encrypted_with_id_copied_from_encrypted_metadata_pseudo_copy', + f'b2id://{file_version_info["fileId"]}', + f'b2://{bucket_name}/{subfolder}/encrypted_with_id_copied_from_encrypted_metadata_pseudo_copy', '--fetch-metadata', ], additional_env={ @@ -2268,10 +2276,9 @@ def test_file_lock( b2_tool.should_fail( [ 'file', - 'copy-by-id', - lockable_file['fileId'], - lock_disabled_bucket_name, - 'copied', + 'server-side-copy', + f'b2id://{lockable_file["fileId"]}', + f'b2://{lock_disabled_bucket_name}/copied', '--file-retention-mode', 'governance', '--retain-until', @@ -2284,10 +2291,9 @@ def test_file_lock( copied_file = b2_tool.should_succeed_json( [ 'file', - 'copy-by-id', - lockable_file['fileId'], - lock_enabled_bucket_name, - 'copied', + 'server-side-copy', + f"b2id://{lockable_file['fileId']}", + f'b2://{lock_enabled_bucket_name}/copied', '--file-retention-mode', 'governance', '--retain-until', @@ -2441,10 +2447,9 @@ def file_lock_without_perms_test( b2_tool.should_fail( [ 'file', - 'copy-by-id', - lockable_file_id, - lock_enabled_bucket_name, - 'copied', + 'server-side-copy', + f'b2id://{lockable_file_id}', + f'b2://{lock_enabled_bucket_name}/copied', '--file-retention-mode', 'governance', '--retain-until', @@ -2458,10 +2463,9 @@ def file_lock_without_perms_test( b2_tool.should_fail( [ 'file', - 'copy-by-id', - lockable_file_id, - lock_disabled_bucket_name, - 'copied', + 'server-side-copy', + f'b2id://{lockable_file_id}', + f'b2://{lock_disabled_bucket_name}/copied', '--file-retention-mode', 'governance', '--retain-until', @@ -3398,8 +3402,14 @@ def assert_expected(file_info, expected=expected_file_info): copied_version = b2_tool.should_succeed_json( [ - 'file', 'copy-by-id', '--quiet', *args, '--content-type', 'text/plain', - file_version['fileId'], bucket_name, f'{persistent_bucket.subfolder}/copied_file' + 'file', + 'server-side-copy', + '--quiet', + *args, + '--content-type', + 'text/plain', + f"b2id://{file_version['fileId']}", + f'b2://{bucket_name}/{persistent_bucket.subfolder}/copied_file', ] ) assert_expected(copied_version['fileInfo']) diff --git a/test/unit/console_tool/test_file_server_side_copy.py b/test/unit/console_tool/test_file_server_side_copy.py new file mode 100644 index 00000000..1845a6a3 --- /dev/null +++ b/test/unit/console_tool/test_file_server_side_copy.py @@ -0,0 +1,225 @@ +###################################################################### +# +# File: test/unit/console_tool/test_file_server_side_copy.py +# +# Copyright 2024 Backblaze Inc. All Rights Reserved. +# +# License https://www.backblaze.com/using_b2_code.html +# +###################################################################### +from __future__ import annotations + +import pytest + + +@pytest.mark.apiver +def test_copy_file_by_id(b2_cli, api_bucket, uploaded_file): + expected_json = { + "accountId": b2_cli.account_id, + "action": "copy", + "bucketId": api_bucket.id_, + "size": 11, + "contentSha1": "2aae6c35c94fcfb415dbe95f408b9ce91ee846ed", + "contentType": "b2/x-auto", + "fileId": "9998", + "fileInfo": { + "src_last_modified_millis": "1500111222000" + }, + "fileName": "file1_copy.txt", + "serverSideEncryption": { + "mode": "none" + }, + "uploadTimestamp": 5001 + } + b2_cli.run( + ['file', 'copy-by-id', '9999', 'my-bucket', 'file1_copy.txt'], + expected_json_in_stdout=expected_json, + expected_stderr= + 'WARNING: `copy-by-id` command is deprecated. Use `file server-side-copy` instead.\n', + ) + + +@pytest.mark.apiver +def test_file_server_side_copy__with_range(b2_cli, api_bucket, uploaded_file): + expected_json = { + "accountId": b2_cli.account_id, + "action": "copy", + "bucketId": api_bucket.id_, + "size": 5, + "contentSha1": "4f664540ff30b8d34e037298a84e4736be39d731", + "contentType": "b2/x-auto", + "fileId": "9998", + "fileInfo": { + "src_last_modified_millis": "1500111222000" + }, + "fileName": "file1_copy.txt", + "serverSideEncryption": { + "mode": "none" + }, + "uploadTimestamp": 5001 + } + b2_cli.run( + [ + 'file', 'server-side-copy', '--range', '3,7', f'b2id://{uploaded_file["fileId"]}', + 'b2://my-bucket/file1_copy.txt' + ], + expected_json_in_stdout=expected_json, + ) + + +@pytest.mark.apiver +def test_file_server_side_copy__invalid_metadata_copy_with_file_info( + b2_cli, api_bucket, uploaded_file +): + b2_cli.run( + [ + 'file', + 'server-side-copy', + '--info', + 'a=b', + 'b2id://9999', + 'b2://my-bucket/file1_copy.txt', + ], + '', + expected_stderr="ERROR: File info can be set only when content type is set\n", + expected_status=1, + ) + + +@pytest.mark.apiver +def test_file_server_side_copy__invalid_metadata_replace_file_info( + b2_cli, api_bucket, uploaded_file +): + b2_cli.run( + [ + 'file', + 'server-side-copy', + '--content-type', + 'text/plain', + 'b2id://9999', + 'b2://my-bucket/file1_copy.txt', + ], + '', + expected_stderr="ERROR: File info can be not set only when content type is not set\n", + expected_status=1, + ) + + # replace with content type and file info + expected_json = { + "accountId": b2_cli.account_id, + "action": "copy", + "bucketId": api_bucket.id_, + "size": 11, + "contentSha1": "2aae6c35c94fcfb415dbe95f408b9ce91ee846ed", + "contentType": "text/plain", + "fileId": "9998", + "fileInfo": { + "a": "b" + }, + "fileName": "file1_copy.txt", + "serverSideEncryption": { + "mode": "none" + }, + "uploadTimestamp": 5001 + } + b2_cli.run( + [ + 'file', + 'server-side-copy', + '--content-type', + 'text/plain', + '--info', + 'a=b', + 'b2id://9999', + 'b2://my-bucket/file1_copy.txt', + ], + expected_json_in_stdout=expected_json, + ) + + +@pytest.mark.apiver +def test_file_server_side_copy__unsatisfied_range(b2_cli, api_bucket, uploaded_file): + expected_stderr = "ERROR: The range in the request is outside the size of the file\n" + b2_cli.run( + [ + 'file', 'server-side-copy', '--range', '12,20', 'b2id://9999', + 'b2://my-bucket/file1_copy.txt' + ], + '', + expected_stderr, + 1, + ) + + # Copy in different bucket + b2_cli.run(['bucket', 'create', 'my-bucket1', 'allPublic'], 'bucket_1\n', '', 0) + expected_json = { + "accountId": b2_cli.account_id, + "action": "copy", + "bucketId": "bucket_1", + "size": 11, + "contentSha1": "2aae6c35c94fcfb415dbe95f408b9ce91ee846ed", + "contentType": "b2/x-auto", + "fileId": "9997", + "fileInfo": { + "src_last_modified_millis": "1500111222000" + }, + "fileName": "file1_copy.txt", + "serverSideEncryption": { + "mode": "none" + }, + "uploadTimestamp": 5001 + } + b2_cli.run( + ['file', 'server-side-copy', 'b2id://9999', 'b2://my-bucket1/file1_copy.txt'], + expected_json_in_stdout=expected_json, + ) + + +@pytest.mark.apiver +def test_copy_file_by_id__deprecated(b2_cli, api_bucket, uploaded_file): + expected_json = { + "accountId": b2_cli.account_id, + "action": "copy", + "bucketId": api_bucket.id_, + "size": 11, + "contentSha1": "2aae6c35c94fcfb415dbe95f408b9ce91ee846ed", + "contentType": "b2/x-auto", + "fileId": "9998", + "fileInfo": { + "src_last_modified_millis": "1500111222000" + }, + "fileName": "file1_copy_2.txt", + "serverSideEncryption": { + "mode": "none" + }, + "uploadTimestamp": 5001 + } + b2_cli.run( + ['copy-file-by-id', '9999', api_bucket.name, 'file1_copy_2.txt'], + expected_stderr= + 'WARNING: `copy-file-by-id` command is deprecated. Use `file server-side-copy` instead.\n', + expected_json_in_stdout=expected_json, + ) + + +@pytest.mark.apiver +def test_file_server_side_copy__by_b2_uri(b2_cli, api_bucket, uploaded_file): + b2_cli.run( + [ + "file", "server-side-copy", + f"b2://{uploaded_file['bucket']}/{uploaded_file['fileName']}", + f"b2://{uploaded_file['bucket']}/copy.bin" + ], + ) + assert [fv.file_name for fv, _ in api_bucket.ls()] == ['copy.bin', uploaded_file['fileName']] + + +@pytest.mark.apiver +def test_file_hide__by_b2id_uri(b2_cli, api_bucket, uploaded_file): + b2_cli.run( + [ + "file", "server-side-copy", f"b2id://{uploaded_file['fileId']}", + f"b2://{uploaded_file['bucket']}/copy.bin" + ], + ) + assert [fv.file_name for fv, _ in api_bucket.ls()] == ['copy.bin', uploaded_file['fileName']] diff --git a/test/unit/test_console_tool.py b/test/unit/test_console_tool.py index fae0a47a..fbbe92cb 100644 --- a/test/unit/test_console_tool.py +++ b/test/unit/test_console_tool.py @@ -1506,221 +1506,6 @@ def test_download_by_id_to_directory(self): def test_download_by_name_to_directory(self): self._test_download_to_directory(download_by='name') - def test_copy_file_by_id(self): - self._authorize_account() - self._create_my_bucket() - - with TempDir() as temp_dir: - local_file1 = self._make_local_file(temp_dir, 'file1.txt') - # For this test, use a mod time without millis. My mac truncates - # millis and just leaves seconds. - mod_time = 1500111222 - os.utime(local_file1, (mod_time, mod_time)) - self.assertEqual(1500111222, os.path.getmtime(local_file1)) - - # Upload a file - expected_stdout = ''' - URL by file name: http://download.example.com/file/my-bucket/file1.txt - URL by fileId: http://download.example.com/b2api/vx/b2_download_file_by_id?fileId=9999''' - expected_json = { - "action": "upload", - "contentSha1": "2aae6c35c94fcfb415dbe95f408b9ce91ee846ed", - "contentType": "b2/x-auto", - "fileId": "9999", - "fileInfo": { - "src_last_modified_millis": "1500111222000" - }, - "fileName": "file1.txt", - "serverSideEncryption": { - "mode": "none" - }, - "size": 11, - "uploadTimestamp": 5000 - } - - self._run_command( - ['file', 'upload', '--no-progress', 'my-bucket', local_file1, 'file1.txt'], - expected_json_in_stdout=expected_json, - remove_version=True, - expected_part_of_stdout=expected_stdout, - ) - - # Copy File - expected_json = { - "accountId": self.account_id, - "action": "copy", - "bucketId": "bucket_0", - "size": 11, - "contentSha1": "2aae6c35c94fcfb415dbe95f408b9ce91ee846ed", - "contentType": "b2/x-auto", - "fileId": "9998", - "fileInfo": { - "src_last_modified_millis": "1500111222000" - }, - "fileName": "file1_copy.txt", - "serverSideEncryption": { - "mode": "none" - }, - "uploadTimestamp": 5001 - } - self._run_command( - ['file', 'copy-by-id', '9999', 'my-bucket', 'file1_copy.txt'], - expected_json_in_stdout=expected_json, - ) - - # Copy File with range parameter - expected_json = { - "accountId": self.account_id, - "action": "copy", - "bucketId": "bucket_0", - "size": 5, - "contentSha1": "4f664540ff30b8d34e037298a84e4736be39d731", - "contentType": "b2/x-auto", - "fileId": "9997", - "fileInfo": { - "src_last_modified_millis": "1500111222000" - }, - "fileName": "file1_copy.txt", - "serverSideEncryption": { - "mode": "none" - }, - "uploadTimestamp": 5002 - } - self._run_command( - ['file', 'copy-by-id', '--range', '3,7', '9999', 'my-bucket', 'file1_copy.txt'], - expected_json_in_stdout=expected_json, - ) - - local_download1 = os.path.join(temp_dir, 'file1_copy.txt') - self._run_command( - ['file', 'download', '-q', 'b2://my-bucket/file1_copy.txt', local_download1] - ) - self.assertEqual(b'lo wo', self._read_file(local_download1)) - - # Invalid metadata copy with file info - expected_stderr = "ERROR: File info can be set only when content type is set\n" - self._run_command( - [ - 'file', - 'copy-by-id', - '--info', - 'a=b', - '9999', - 'my-bucket', - 'file1_copy.txt', - ], - '', - expected_stderr, - 1, - ) - - # Invalid metadata replace without file info - expected_stderr = "ERROR: File info can be not set only when content type is not set\n" - self._run_command( - [ - 'file', - 'copy-by-id', - '--content-type', - 'text/plain', - '9999', - 'my-bucket', - 'file1_copy.txt', - ], - '', - expected_stderr, - 1, - ) - - # replace with content type and file info - expected_json = { - "accountId": self.account_id, - "action": "copy", - "bucketId": "bucket_0", - "size": 11, - "contentSha1": "2aae6c35c94fcfb415dbe95f408b9ce91ee846ed", - "contentType": "text/plain", - "fileId": "9996", - "fileInfo": { - "a": "b" - }, - "fileName": "file1_copy.txt", - "serverSideEncryption": { - "mode": "none" - }, - "uploadTimestamp": 5003 - } - self._run_command( - [ - 'file', - 'copy-by-id', - '--content-type', - 'text/plain', - '--info', - 'a=b', - '9999', - 'my-bucket', - 'file1_copy.txt', - ], - expected_json_in_stdout=expected_json, - ) - - # UnsatisfiableRange - expected_stderr = "ERROR: The range in the request is outside the size of the file\n" - self._run_command( - ['file', 'copy-by-id', '--range', '12,20', '9999', 'my-bucket', 'file1_copy.txt'], - '', - expected_stderr, - 1, - ) - - # Copy in different bucket - self._run_command(['bucket', 'create', 'my-bucket1', 'allPublic'], 'bucket_1\n', '', 0) - expected_json = { - "accountId": self.account_id, - "action": "copy", - "bucketId": "bucket_1", - "size": 11, - "contentSha1": "2aae6c35c94fcfb415dbe95f408b9ce91ee846ed", - "contentType": "b2/x-auto", - "fileId": "9994", - "fileInfo": { - "src_last_modified_millis": "1500111222000" - }, - "fileName": "file1_copy.txt", - "serverSideEncryption": { - "mode": "none" - }, - "uploadTimestamp": 5004 - } - self._run_command( - ['file', 'copy-by-id', '9999', 'my-bucket1', 'file1_copy.txt'], - expected_json_in_stdout=expected_json, - ) - - expected_json = { - "accountId": self.account_id, - "action": "copy", - "bucketId": "bucket_1", - "size": 11, - "contentSha1": "2aae6c35c94fcfb415dbe95f408b9ce91ee846ed", - "contentType": "b2/x-auto", - "fileId": "9993", - "fileInfo": { - "src_last_modified_millis": "1500111222000" - }, - "fileName": "file1_copy_2.txt", - "serverSideEncryption": { - "mode": "none" - }, - "uploadTimestamp": 5005 - } - self._run_command( - ['copy-file-by-id', '9999', 'my-bucket1', 'file1_copy_2.txt'], - expected_stderr= - 'WARNING: `copy-file-by-id` command is deprecated. Use `file copy-by-id` instead.\n', - expected_json_in_stdout=expected_json, - ) - def test_get_download_auth_defaults(self): self._authorize_account() self._create_my_bucket() diff --git a/test/unit/test_copy.py b/test/unit/test_copy.py index fac26753..336bf53c 100644 --- a/test/unit/test_copy.py +++ b/test/unit/test_copy.py @@ -19,6 +19,7 @@ EncryptionSetting, ) +from b2._internal._utils.uri import B2FileIdURI from b2._internal.console_tool import FileCopyById from .test_base import TestBase @@ -32,7 +33,7 @@ def test_determine_source_metadata(self): copy_file_command = FileCopyById(mock_console_tool) result = copy_file_command._determine_source_metadata( - 'id', + B2FileIdURI('id'), destination_encryption=None, source_encryption=None, target_file_info=None, @@ -43,7 +44,7 @@ def test_determine_source_metadata(self): assert len(mock_api.method_calls) == 0 result = copy_file_command._determine_source_metadata( - 'id', + B2FileIdURI('id'), destination_encryption=SSE_B2_AES, source_encryption=SSE_B2_AES, target_file_info={}, @@ -54,7 +55,7 @@ def test_determine_source_metadata(self): assert len(mock_api.method_calls) == 0 result = copy_file_command._determine_source_metadata( - 'id', + B2FileIdURI('id'), destination_encryption=SSE_B2_AES, source_encryption=SSE_B2_AES, target_file_info={}, @@ -74,7 +75,7 @@ def test_determine_source_metadata(self): ) result = copy_file_command._determine_source_metadata( - 'id', + B2FileIdURI('id'), destination_encryption=destination_sse_c, source_encryption=source_sse_c, target_file_info={}, @@ -90,7 +91,7 @@ def test_determine_source_metadata(self): 'file metadata before copying.' ): copy_file_command._determine_source_metadata( - 'id', + B2FileIdURI('id'), destination_encryption=destination_sse_c, source_encryption=source_sse_c, target_file_info=None, @@ -100,7 +101,7 @@ def test_determine_source_metadata(self): assert len(mock_api.method_calls) == 0 result = copy_file_command._determine_source_metadata( - 'id', + B2FileIdURI('id'), destination_encryption=destination_sse_c, source_encryption=source_sse_c, target_file_info=None, From a59bcc19f395439b0e1c8b3ba6ff57d16371ef95 Mon Sep 17 00:00:00 2001 From: Maciej Urbanski Date: Sun, 13 Oct 2024 23:40:24 +0200 Subject: [PATCH 2/2] fix test docker job on Github Actions --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6fb13b67..0a68aa8c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -144,7 +144,7 @@ jobs: with: python-version: ${{ env.PYTHON_DEFAULT_VERSION }} - name: Install dependencies - run: sudo python -m pip install --upgrade nox pdm + run: python -m pip install --upgrade nox pdm - name: Generate Dockerfile run: nox -vs generate_dockerfile - name: Set up QEMU