From 36d05ffa1abd884cef608407e7166bb519c9f367 Mon Sep 17 00:00:00 2001 From: Adal Chiriliuc Date: Thu, 25 Apr 2024 01:32:21 +0300 Subject: [PATCH 01/22] added file info, url, cat, upload --- b2/_internal/_b2v4/registry.py | 3 +- b2/_internal/b2v3/registry.py | 1 + b2/_internal/console_tool.py | 111 ++++++++++++++++++------ changelog.d/+command-file.added.md | 1 + changelog.d/+command-file.deprecated.md | 1 + 5 files changed, 90 insertions(+), 27 deletions(-) create mode 100644 changelog.d/+command-file.added.md create mode 100644 changelog.d/+command-file.deprecated.md diff --git a/b2/_internal/_b2v4/registry.py b/b2/_internal/_b2v4/registry.py index 2c11f633..c1bcf78d 100644 --- a/b2/_internal/_b2v4/registry.py +++ b/b2/_internal/_b2v4/registry.py @@ -27,7 +27,7 @@ B2.register_subcommand(Cat) B2.register_subcommand(GetAccountInfo) B2.register_subcommand(GetBucket) -B2.register_subcommand(FileInfo) +B2.register_subcommand(FileInfo2) B2.register_subcommand(GetFileInfo) B2.register_subcommand(GetDownloadAuth) B2.register_subcommand(GetDownloadUrlWithAuth) @@ -60,3 +60,4 @@ B2.register_subcommand(Replication) B2.register_subcommand(Account) B2.register_subcommand(BucketCmd) +B2.register_subcommand(File) diff --git a/b2/_internal/b2v3/registry.py b/b2/_internal/b2v3/registry.py index 13041592..c7fa6449 100644 --- a/b2/_internal/b2v3/registry.py +++ b/b2/_internal/b2v3/registry.py @@ -140,3 +140,4 @@ class Ls(B2URIBucketNFolderNameArgMixin, BaseLs): B2.register_subcommand(Replication) B2.register_subcommand(Account) B2.register_subcommand(BucketCmd) +B2.register_subcommand(File) diff --git a/b2/_internal/console_tool.py b/b2/_internal/console_tool.py index ade898cc..8032704d 100644 --- a/b2/_internal/console_tool.py +++ b/b2/_internal/console_tool.py @@ -1955,7 +1955,7 @@ def _setup_parser(cls, parser): parser.add_argument('localFileName') -class Cat(B2URIFileArgMixin, DownloadCommand): +class FileCatBase(B2URIFileArgMixin, DownloadCommand): """ Download content of a file-like object identified by B2 URI directly to stdout. @@ -2071,15 +2071,6 @@ def _run(self, args): return 0 -class FileInfo(B2URIFileArgMixin, FileInfoBase): - __doc__ = FileInfoBase.__doc__ - - -class GetFileInfo(CmdReplacedByMixin, B2URIFileIDArgMixin, FileInfoBase): - __doc__ = FileInfoBase.__doc__ - replaced_by_cmd = FileInfo - - class BucketGetDownloadAuthBase(Command): """ Prints an authorization token that is valid only for downloading @@ -2775,7 +2766,7 @@ class Rm(B2IDOrB2URIMixin, BaseRm): """ -class GetUrlBase(Command): +class FileUrlBase(Command): """ Prints an URL that can be used to download the given file, if it is public. @@ -2787,20 +2778,6 @@ def _run(self, args): return 0 -class GetUrl(B2URIFileArgMixin, GetUrlBase): - __doc__ = GetUrlBase.__doc__ - - -class MakeUrl(CmdReplacedByMixin, B2URIFileIDArgMixin, GetUrlBase): - __doc__ = GetUrlBase.__doc__ - replaced_by_cmd = GetUrl - - -class MakeFriendlyUrl(CmdReplacedByMixin, B2URIBucketNFilenameArgMixin, GetUrlBase): - __doc__ = GetUrlBase.__doc__ - replaced_by_cmd = GetUrl - - class Sync( ThreadsMixin, DestinationSseMixin, @@ -3434,7 +3411,7 @@ class NotAnInputStream(Exception): pass -class UploadFile(UploadFileMixin, UploadModeMixin, Command): +class FileUploadBase(UploadFileMixin, UploadModeMixin, Command): """ Uploads one file to the given bucket. @@ -4917,6 +4894,88 @@ class NotificationRules(CmdReplacedByMixin, BucketNotificationRuleBase): replaced_by_cmd = (BucketCmd, BucketNotificationRule) +class File(Command): + """ + File management subcommands. + + For more information on each subcommand, use ``{NAME} key SUBCOMMAND --help``. + + Examples: + + .. code-block:: + + {NAME} file info + {NAME} file url + {NAME} file cat + {NAME} file upload + {NAME} file download + {NAME} file copy-by-id + {NAME} file hide + """ + subcommands_registry = ClassRegistry(attr_name='COMMAND_NAME') + + +@File.subcommands_registry.register +class FileInfo(B2URIFileArgMixin, FileInfoBase): + __doc__ = FileInfoBase.__doc__ + COMMAND_NAME = 'info' + + +@File.subcommands_registry.register +class FileUrl(B2URIFileArgMixin, FileUrlBase): + __doc__ = FileUrlBase.__doc__ + COMMAND_NAME = 'url' + + +@File.subcommands_registry.register +class FileCat(FileCatBase): + __doc__ = FileCatBase.__doc__ + COMMAND_NAME = 'cat' + + +@File.subcommands_registry.register +class FileUpload(FileUploadBase): + __doc__ = FileUploadBase.__doc__ + COMMAND_NAME = 'upload' + + +class FileInfo2(CmdReplacedByMixin, B2URIFileArgMixin, FileInfoBase): + __doc__ = FileInfoBase.__doc__ + replaced_by_cmd = File + # TODO we can't use 'file-info', gets transformed to 'file--info' + COMMAND_NAME = 'FileInfo' + + +class GetFileInfo(CmdReplacedByMixin, B2URIFileIDArgMixin, FileInfoBase): + __doc__ = FileInfoBase.__doc__ + replaced_by_cmd = File + + +class GetUrl(CmdReplacedByMixin, B2URIFileArgMixin, FileUrlBase): + __doc__ = FileUrlBase.__doc__ + replaced_by_cmd = File + + +class MakeUrl(CmdReplacedByMixin, B2URIFileIDArgMixin, FileUrlBase): + __doc__ = FileUrlBase.__doc__ + replaced_by_cmd = File + + +class MakeFriendlyUrl(CmdReplacedByMixin, B2URIBucketNFilenameArgMixin, FileUrlBase): + __doc__ = FileUrlBase.__doc__ + replaced_by_cmd = File + + +class Cat(CmdReplacedByMixin, FileCatBase): + __doc__ = FileCatBase.__doc__ + replaced_by_cmd = File + + +class UploadFile(CmdReplacedByMixin, FileUploadBase): + __doc__ = FileUploadBase.__doc__ + replaced_by_cmd = File + + class ConsoleTool: """ Implements the commands available in the B2 command-line tool diff --git a/changelog.d/+command-file.added.md b/changelog.d/+command-file.added.md new file mode 100644 index 00000000..71b2be3d --- /dev/null +++ b/changelog.d/+command-file.added.md @@ -0,0 +1 @@ +Add `file {info|url|cat|upload|download|copy-by-id|hide}` commands. \ No newline at end of file diff --git a/changelog.d/+command-file.deprecated.md b/changelog.d/+command-file.deprecated.md new file mode 100644 index 00000000..6d0a284e --- /dev/null +++ b/changelog.d/+command-file.deprecated.md @@ -0,0 +1 @@ +Deprecated `file-info`, `get-url`, `cat`, `upload-file`, `download-file`, `copy-file-by-id` and `hide-file`, use `file {info|url|cat|upload|download|copy-by-id|hide}` instead. \ No newline at end of file From adbb53f163a4801df54bf1a71b552f686eb1ee67 Mon Sep 17 00:00:00 2001 From: Adal Chiriliuc Date: Thu, 25 Apr 2024 17:43:29 +0300 Subject: [PATCH 02/22] add subcommand to replaced_by_cmd --- b2/_internal/console_tool.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/b2/_internal/console_tool.py b/b2/_internal/console_tool.py index 8032704d..f5e5da20 100644 --- a/b2/_internal/console_tool.py +++ b/b2/_internal/console_tool.py @@ -4941,39 +4941,39 @@ class FileUpload(FileUploadBase): class FileInfo2(CmdReplacedByMixin, B2URIFileArgMixin, FileInfoBase): __doc__ = FileInfoBase.__doc__ - replaced_by_cmd = File + replaced_by_cmd = (File, FileInfo) # TODO we can't use 'file-info', gets transformed to 'file--info' COMMAND_NAME = 'FileInfo' class GetFileInfo(CmdReplacedByMixin, B2URIFileIDArgMixin, FileInfoBase): __doc__ = FileInfoBase.__doc__ - replaced_by_cmd = File + replaced_by_cmd = (File, FileInfo) class GetUrl(CmdReplacedByMixin, B2URIFileArgMixin, FileUrlBase): __doc__ = FileUrlBase.__doc__ - replaced_by_cmd = File + replaced_by_cmd = (File, FileUrl) class MakeUrl(CmdReplacedByMixin, B2URIFileIDArgMixin, FileUrlBase): __doc__ = FileUrlBase.__doc__ - replaced_by_cmd = File + replaced_by_cmd = (File, FileUrl) class MakeFriendlyUrl(CmdReplacedByMixin, B2URIBucketNFilenameArgMixin, FileUrlBase): __doc__ = FileUrlBase.__doc__ - replaced_by_cmd = File + replaced_by_cmd = (File, FileUrl) class Cat(CmdReplacedByMixin, FileCatBase): __doc__ = FileCatBase.__doc__ - replaced_by_cmd = File + replaced_by_cmd = (File, FileCat) class UploadFile(CmdReplacedByMixin, FileUploadBase): __doc__ = FileUploadBase.__doc__ - replaced_by_cmd = File + replaced_by_cmd = (File, FileUpload) class ConsoleTool: From e35d1b95457508aa122bca54b1d76b6390d8abcf Mon Sep 17 00:00:00 2001 From: Adal Chiriliuc Date: Thu, 25 Apr 2024 18:06:58 +0300 Subject: [PATCH 03/22] use file subcommands --- test/integration/test_b2_command_line.py | 26 +++++++++++--------- test/integration/test_tqdm_closer.py | 2 ++ test/unit/console_tool/test_download_file.py | 10 +++++--- test/unit/console_tool/test_file_info.py | 7 +++--- test/unit/console_tool/test_get_url.py | 9 ++++--- test/unit/test_console_tool.py | 4 +-- 6 files changed, 34 insertions(+), 24 deletions(-) diff --git a/test/integration/test_b2_command_line.py b/test/integration/test_b2_command_line.py index 5a370471..d6fd675b 100755 --- a/test/integration/test_b2_command_line.py +++ b/test/integration/test_b2_command_line.py @@ -400,7 +400,9 @@ def test_basic(b2_tool, bucket_name, sample_file, tmp_path, b2_uri_args): ['ls', *b2_uri_args(bucket_name, 'b/')], f'^b/1{os.linesep}b/2{os.linesep}' ) - file_info = b2_tool.should_succeed_json(['file-info', f"b2id://{second_c_version['fileId']}"]) + file_info = b2_tool.should_succeed_json( + ['file', 'info', f"b2id://{second_c_version['fileId']}"] + ) expected_info = { 'color': 'blue', 'foo': 'bar=baz', @@ -413,10 +415,10 @@ def test_basic(b2_tool, bucket_name, sample_file, tmp_path, b2_uri_args): ['ls', *b2_uri_args(bucket_name)], f'^a{os.linesep}b/{os.linesep}c{os.linesep}d{os.linesep}' ) - b2_tool.should_succeed(['get-url', f"b2id://{second_c_version['fileId']}"]) + b2_tool.should_succeed(['file', 'url', f"b2id://{second_c_version['fileId']}"]) b2_tool.should_succeed( - ['get-url', f"b2://{bucket_name}/any-file-name"], + ['file', 'url', f"b2://{bucket_name}/any-file-name"], '^https://.*/file/{}/{}\r?$'.format( bucket_name, 'any-file-name', @@ -897,7 +899,7 @@ def sync_up_helper(b2_tool, bucket_name, dir_, encryption=None): return # that's enough, we've checked that encryption works, no need to repeat the whole sync suite c_id = find_file_id(file_versions, prefix + 'c') - file_info = b2_tool.should_succeed_json(['file-info', f"b2id://{c_id}"])['fileInfo'] + file_info = b2_tool.should_succeed_json(['file', 'info', f"b2id://{c_id}"])['fileInfo'] should_equal( file_mod_time_millis(dir_path / 'c'), int(file_info['src_last_modified_millis']) ) @@ -1477,11 +1479,13 @@ def test_sse_b2(b2_tool, bucket_name, sample_file, tmp_path, b2_uri_args): ) encrypted_version = list_of_files[0] - file_info = b2_tool.should_succeed_json(['file-info', f"b2id://{encrypted_version['fileId']}"]) + file_info = b2_tool.should_succeed_json( + ['file', 'info', f"b2id://{encrypted_version['fileId']}"] + ) should_equal({'algorithm': 'AES256', 'mode': 'SSE-B2'}, file_info['serverSideEncryption']) not_encrypted_version = list_of_files[1] file_info = b2_tool.should_succeed_json( - ['file-info', f"b2id://{not_encrypted_version['fileId']}"] + ['file', 'info', f"b2id://{not_encrypted_version['fileId']}"] ) should_equal({'mode': 'none'}, file_info['serverSideEncryption']) @@ -1509,13 +1513,13 @@ def test_sse_b2(b2_tool, bucket_name, sample_file, tmp_path, b2_uri_args): copied_encrypted_version = list_of_files[2] file_info = b2_tool.should_succeed_json( - ['file-info', f"b2id://{copied_encrypted_version['fileId']}"] + ['file', 'info', f"b2id://{copied_encrypted_version['fileId']}"] ) should_equal({'algorithm': 'AES256', 'mode': 'SSE-B2'}, file_info['serverSideEncryption']) copied_not_encrypted_version = list_of_files[3] file_info = b2_tool.should_succeed_json( - ['file-info', f"b2id://{copied_not_encrypted_version['fileId']}"] + ['file', 'info', f"b2id://{copied_not_encrypted_version['fileId']}"] ) should_equal({'mode': 'none'}, file_info['serverSideEncryption']) @@ -2967,7 +2971,7 @@ def _assert_file_lock_configuration( legal_hold: LegalHold | None = None ): - file_version = b2_tool.should_succeed_json(['file-info', f"b2id://{file_id}"]) + file_version = b2_tool.should_succeed_json(['file', 'info', f"b2id://{file_id}"]) if retention_mode is not None: if file_version['fileRetention']['mode'] == 'unknown': actual_file_retention = UNKNOWN_FILE_RETENTION_SETTING @@ -3097,9 +3101,9 @@ def test_download_file_to_directory( def test_cat(b2_tool, bucket_name, sample_filepath, tmp_path, uploaded_sample_file): assert b2_tool.should_succeed( - ['cat', f"b2://{bucket_name}/{uploaded_sample_file['fileName']}"], + ['file', 'cat', f"b2://{bucket_name}/{uploaded_sample_file['fileName']}"], ) == sample_filepath.read_text() - assert b2_tool.should_succeed(['cat', f"b2id://{uploaded_sample_file['fileId']}"] + assert b2_tool.should_succeed(['file', 'cat', f"b2id://{uploaded_sample_file['fileId']}"] ) == sample_filepath.read_text() diff --git a/test/integration/test_tqdm_closer.py b/test/integration/test_tqdm_closer.py index a97a4310..9786c1fa 100644 --- a/test/integration/test_tqdm_closer.py +++ b/test/integration/test_tqdm_closer.py @@ -21,6 +21,7 @@ def test_tqdm_closer(b2_tool, bucket, file_name): # test that stderr doesn't contain any warning, in particular warnings about multiprocessing resource tracker # leaking semaphores b2_tool.should_succeed([ + 'file', 'cat', f'b2://{bucket.name}/{file_name}', ]) @@ -29,6 +30,7 @@ def test_tqdm_closer(b2_tool, bucket, file_name): # that would mean that either Tqdm or python fixed the issue and _TqdmCloser can be disabled for fixed versions b2_tool.should_succeed( [ + 'file', 'cat', f'b2://{bucket.name}/{file_name}', ], diff --git a/test/unit/console_tool/test_download_file.py b/test/unit/console_tool/test_download_file.py index fd148310..b32b38d7 100644 --- a/test/unit/console_tool/test_download_file.py +++ b/test/unit/console_tool/test_download_file.py @@ -163,13 +163,15 @@ def test_download_file_by_name__to_stdout_by_alias( def test_cat__b2_uri(b2_cli, bucket, uploaded_stdout_txt, tmp_path, capfd): """Test download_file_by_name stdout alias support""" - b2_cli.run(['cat', '--no-progress', f"b2://{bucket}/{uploaded_stdout_txt['fileName']}"],) + b2_cli.run( + ['file', 'cat', '--no-progress', f"b2://{bucket}/{uploaded_stdout_txt['fileName']}"], + ) assert capfd.readouterr().out == uploaded_stdout_txt['content'] def test_cat__b2_uri__invalid(b2_cli, capfd): b2_cli.run( - ['cat', "nothing/meaningful"], + ['file', 'cat', "nothing/meaningful"], expected_stderr=None, expected_status=2, ) @@ -178,7 +180,7 @@ def test_cat__b2_uri__invalid(b2_cli, capfd): def test_cat__b2_uri__not_a_file(b2_cli, bucket, capfd): b2_cli.run( - ['cat', "b2://bucket/dir/subdir/"], + ['file', 'cat', "b2://bucket/dir/subdir/"], expected_stderr=None, expected_status=2, ) @@ -188,7 +190,7 @@ def test_cat__b2_uri__not_a_file(b2_cli, bucket, capfd): def test_cat__b2id_uri(b2_cli, bucket, uploaded_stdout_txt, tmp_path, capfd): """Test download_file_by_name stdout alias support""" - b2_cli.run(['cat', '--no-progress', "b2id://9999"],) + b2_cli.run(['file', 'cat', '--no-progress', "b2id://9999"],) assert capfd.readouterr().out == uploaded_stdout_txt['content'] diff --git a/test/unit/console_tool/test_file_info.py b/test/unit/console_tool/test_file_info.py index 022fbad0..9da815d2 100644 --- a/test/unit/console_tool/test_file_info.py +++ b/test/unit/console_tool/test_file_info.py @@ -42,14 +42,15 @@ def test_get_file_info(b2_cli, uploaded_file_version): b2_cli.run( ["get-file-info", uploaded_file_version["fileId"]], expected_json_in_stdout=uploaded_file_version, - expected_stderr='WARNING: `get-file-info` command is deprecated. Use `file-info` instead.\n', + expected_stderr='WARNING: `get-file-info` command is deprecated. Use `file info` instead.\n', ) def test_file_info__b2_uri(b2_cli, bucket, uploaded_download_version): b2_cli.run( [ - "file-info", + "file", + "info", f'b2://{bucket}/{uploaded_download_version["fileName"]}', ], expected_json_in_stdout=uploaded_download_version, @@ -58,6 +59,6 @@ def test_file_info__b2_uri(b2_cli, bucket, uploaded_download_version): def test_file_info__b2id_uri(b2_cli, uploaded_file_version): b2_cli.run( - ["file-info", f'b2id://{uploaded_file_version["fileId"]}'], + ["file", "info", f'b2id://{uploaded_file_version["fileId"]}'], expected_json_in_stdout=uploaded_file_version, ) diff --git a/test/unit/console_tool/test_get_url.py b/test/unit/console_tool/test_get_url.py index 9941d5f6..981d274f 100644 --- a/test/unit/console_tool/test_get_url.py +++ b/test/unit/console_tool/test_get_url.py @@ -24,7 +24,7 @@ def test_make_url(b2_cli, uploaded_file, uploaded_file_url_by_id): b2_cli.run( ["make-url", uploaded_file["fileId"]], expected_stdout=f"{uploaded_file_url_by_id}\n", - expected_stderr='WARNING: `make-url` command is deprecated. Use `get-url` instead.\n', + expected_stderr='WARNING: `make-url` command is deprecated. Use `file url` instead.\n', ) @@ -33,14 +33,15 @@ def test_make_friendly_url(b2_cli, bucket, uploaded_file, uploaded_file_url): ["make-friendly-url", bucket, uploaded_file["fileName"]], expected_stdout=f"{uploaded_file_url}\n", expected_stderr= - 'WARNING: `make-friendly-url` command is deprecated. Use `get-url` instead.\n', + 'WARNING: `make-friendly-url` command is deprecated. Use `file url` instead.\n', ) def test_get_url__b2_uri(b2_cli, bucket, uploaded_file, uploaded_file_url): b2_cli.run( [ - "get-url", + "file", + "url", f'b2://{bucket}/{uploaded_file["fileName"]}', ], expected_stdout=f"{uploaded_file_url}\n", @@ -49,6 +50,6 @@ def test_get_url__b2_uri(b2_cli, bucket, uploaded_file, uploaded_file_url): def test_get_url__b2id_uri(b2_cli, uploaded_file, uploaded_file_url_by_id): b2_cli.run( - ["get-url", f'b2id://{uploaded_file["fileId"]}'], + ["file", "url", f'b2id://{uploaded_file["fileId"]}'], expected_stdout=f"{uploaded_file_url_by_id}\n", ) diff --git a/test/unit/test_console_tool.py b/test/unit/test_console_tool.py index 99192533..a7516b60 100644 --- a/test/unit/test_console_tool.py +++ b/test/unit/test_console_tool.py @@ -1060,7 +1060,7 @@ def test_files(self): } self._run_command( - ['file-info', 'b2id://9999'], + ['file', 'info', 'b2id://9999'], expected_json_in_stdout=expected_json, ) @@ -1209,7 +1209,7 @@ def test_files_encrypted(self): } self._run_command( - ['file-info', 'b2id://9999'], + ['file', 'info', 'b2id://9999'], expected_json_in_stdout=expected_json, ) From 09769e6c5ab8e02833cd1bae2a51935fc75811e9 Mon Sep 17 00:00:00 2001 From: Adal Chiriliuc Date: Thu, 25 Apr 2024 18:16:20 +0300 Subject: [PATCH 04/22] use file upload subcommand --- README.md | 2 +- b2/_internal/console_tool.py | 8 +- test/integration/test_b2_command_line.py | 92 ++++++++++--------- test/unit/conftest.py | 4 +- test/unit/console_tool/test_download_file.py | 2 +- test/unit/console_tool/test_upload_file.py | 20 ++-- .../test_upload_unbound_stream.py | 2 +- test/unit/test_console_tool.py | 37 +++++--- 8 files changed, 92 insertions(+), 75 deletions(-) diff --git a/README.md b/README.md index 6e1c3789..837e83ae 100644 --- a/README.md +++ b/README.md @@ -150,7 +150,7 @@ cat source_file.txt | docker run -i --rm -v b2:/root backblazeit/b2:latest b2v3 or by mounting local files in the docker container: ```bash -docker run --rm -v b2:/root -v /home/user/path/to/data:/data backblazeit/b2:latest b2v3 upload-file bucket_name /data/source_file.txt target_file_name +docker run --rm -v b2:/root -v /home/user/path/to/data:/data backblazeit/b2:latest b2v3 file upload bucket_name /data/source_file.txt target_file_name ``` ## ApiVer CLI versions (`b2` vs `b2v3`, `b2v4`, etc.) diff --git a/b2/_internal/console_tool.py b/b2/_internal/console_tool.py index f5e5da20..5a477681 100644 --- a/b2/_internal/console_tool.py +++ b/b2/_internal/console_tool.py @@ -3376,7 +3376,7 @@ def execute_operation(self, **kwargs) -> b2sdk.file_version.FileVersion: def upload_file_kwargs_to_unbound_upload(self, **kwargs): """ - Translate upload_file kwargs to unbound_upload equivalents + Translate `file upload` kwargs to unbound_upload equivalents """ kwargs["large_file_sha1"] = kwargs.pop("sha1_sum", None) kwargs["buffers_count"] = kwargs["threads"] + 1 @@ -3420,7 +3420,7 @@ class FileUploadBase(UploadFileMixin, UploadModeMixin, Command): A FIFO file (such as named pipe) can be given instead of regular file. - By default, upload_file will compute the sha1 checksum of the file + By default, `file upload` will compute the sha1 checksum of the file to be uploaded. But, if you already have it, you can provide it on the command line to save a little time. @@ -3481,7 +3481,7 @@ class UploadUnboundStream(UploadFileMixin, Command): {UploadFileMixin} {MinPartSizeMixin} - As opposed to ``b2 upload-file``, ``b2 upload-unbound-stream`` cannot choose optimal `partSize` on its own. + As opposed to ``b2 file upload``, ``b2 upload-unbound-stream`` cannot choose optimal `partSize` on its own. So on memory constrained system it is best to use ``--part-size`` option to set it manually. During upload of unbound stream ``--part-size`` as well as ``--threads`` determine the amount of memory used. The maximum memory use for the upload buffers can be estimated at ``partSize * threads``, that is ~1GB by default. @@ -3540,7 +3540,7 @@ def execute_operation(self, local_file, bucket, threads, **kwargs): self._print_stderr( "WARNING: You are using a stream upload command to upload a regular file. " "While it will work, it is inefficient. " - "Use of upload-file command is recommended." + "Use of `file upload` command is recommended." ) input_stream = local_file diff --git a/test/integration/test_b2_command_line.py b/test/integration/test_b2_command_line.py index d6fd675b..24079701 100755 --- a/test/integration/test_b2_command_line.py +++ b/test/integration/test_b2_command_line.py @@ -273,7 +273,7 @@ def test_command_with_env_vars_reusing_existing_account_info( @pytest.fixture def uploaded_sample_file(b2_tool, bucket_name, sample_filepath): return b2_tool.should_succeed_json( - ['upload-file', '--quiet', bucket_name, + ['file', 'upload', '--quiet', bucket_name, str(sample_filepath), 'sample_file'] ) @@ -308,32 +308,32 @@ def test_basic(b2_tool, bucket_name, sample_file, tmp_path, b2_uri_args): [bucket_name], [b['bucketName'] for b in list_of_buckets if b['bucketName'] == bucket_name] ) - b2_tool.should_succeed(['upload-file', '--quiet', bucket_name, sample_file, 'a']) + b2_tool.should_succeed(['file', 'upload', '--quiet', bucket_name, sample_file, 'a']) b2_tool.should_succeed(['ls', '--long', '--replication', *b2_uri_args(bucket_name)]) - b2_tool.should_succeed(['upload-file', '--no-progress', bucket_name, sample_file, 'a']) - b2_tool.should_succeed(['upload-file', '--no-progress', bucket_name, sample_file, 'b/1']) - b2_tool.should_succeed(['upload-file', '--no-progress', bucket_name, sample_file, 'b/2']) + b2_tool.should_succeed(['file', 'upload', '--no-progress', bucket_name, sample_file, 'a']) + b2_tool.should_succeed(['file', 'upload', '--no-progress', bucket_name, sample_file, 'b/1']) + b2_tool.should_succeed(['file', 'upload', '--no-progress', bucket_name, sample_file, 'b/2']) b2_tool.should_succeed( [ - 'upload-file', '--no-progress', '--sha1', hex_sha1, '--info', 'foo=bar=baz', '--info', - 'color=blue', bucket_name, sample_file, 'c' + 'file', 'upload', '--no-progress', '--sha1', hex_sha1, '--info', 'foo=bar=baz', + '--info', 'color=blue', bucket_name, sample_file, 'c' ] ) b2_tool.should_fail( [ - 'upload-file', '--no-progress', '--sha1', hex_sha1, '--info', 'foo-bar', '--info', + 'file', 'upload', '--no-progress', '--sha1', hex_sha1, '--info', 'foo-bar', '--info', 'color=blue', bucket_name, sample_file, 'c' ], r'ERROR: Bad file info: foo-bar' ) b2_tool.should_succeed( [ - 'upload-file', '--no-progress', '--content-type', 'text/plain', bucket_name, + 'file', 'upload', '--no-progress', '--content-type', 'text/plain', bucket_name, sample_file, 'd' ] ) - b2_tool.should_succeed(['upload-file', '--no-progress', bucket_name, sample_file, 'rm']) - b2_tool.should_succeed(['upload-file', '--no-progress', bucket_name, sample_file, 'rm1']) + b2_tool.should_succeed(['file', 'upload', '--no-progress', bucket_name, sample_file, 'rm']) + b2_tool.should_succeed(['file', 'upload', '--no-progress', bucket_name, sample_file, 'rm1']) # with_wildcard allows us to target a single file. rm will be removed, rm1 will be left alone b2_tool.should_succeed( ['rm', '--recursive', '--with-wildcard', *b2_uri_args(bucket_name, 'rm')] @@ -532,7 +532,7 @@ def test_bucket(b2_tool, bucket_name): def test_key_restrictions(b2_tool, bucket_name, sample_file, bucket_factory, b2_uri_args): # A single file for rm to fail on. - b2_tool.should_succeed(['upload-file', '--no-progress', bucket_name, sample_file, 'test']) + b2_tool.should_succeed(['file', 'upload', '--no-progress', bucket_name, sample_file, 'test']) key_one_name = 'clt-testKey-01' + random_hex(6) created_key_stdout = b2_tool.should_succeed( @@ -1093,12 +1093,12 @@ def sync_down_helper(b2_tool, bucket_name, folder_in_bucket, sample_file, encryp # Put a couple files in B2 b2_tool.should_succeed( - ['upload-file', '--no-progress', bucket_name, sample_file, b2_file_prefix + 'a'] + + ['file', 'upload', '--no-progress', bucket_name, sample_file, b2_file_prefix + 'a'] + upload_encryption_args, additional_env=upload_additional_env, ) b2_tool.should_succeed( - ['upload-file', '--no-progress', bucket_name, sample_file, b2_file_prefix + 'b'] + + ['file', 'upload', '--no-progress', bucket_name, sample_file, b2_file_prefix + 'b'] + upload_encryption_args, additional_env=upload_additional_env, ) @@ -1109,7 +1109,7 @@ def sync_down_helper(b2_tool, bucket_name, folder_in_bucket, sample_file, encryp should_equal(['a', 'b'], sorted(os.listdir(local_path))) b2_tool.should_succeed( - ['upload-file', '--no-progress', bucket_name, sample_file, b2_file_prefix + 'c'] + + ['file', 'upload', '--no-progress', bucket_name, sample_file, b2_file_prefix + 'c'] + upload_encryption_args, additional_env=upload_additional_env, ) @@ -1303,19 +1303,19 @@ def run_sync_copy_with_basic_checks( ): b2_tool.should_succeed( [ - 'upload-file', '--no-progress', '--destination-server-side-encryption', 'SSE-B2', + 'file', 'upload', '--no-progress', '--destination-server-side-encryption', 'SSE-B2', bucket_name, sample_file, b2_file_prefix + 'a' ] ) b2_tool.should_succeed( - ['upload-file', '--no-progress', bucket_name, sample_file, b2_file_prefix + 'b'] + ['file', 'upload', '--no-progress', bucket_name, sample_file, b2_file_prefix + 'b'] ) elif source_encryption.mode == EncryptionMode.SSE_C: for suffix in ['a', 'b']: b2_tool.should_succeed( [ - 'upload-file', '--no-progress', '--destination-server-side-encryption', 'SSE-C', - bucket_name, sample_file, b2_file_prefix + suffix + 'file', 'upload', '--no-progress', '--destination-server-side-encryption', + 'SSE-C', bucket_name, sample_file, b2_file_prefix + suffix ], additional_env={ 'B2_DESTINATION_SSE_C_KEY_B64': @@ -1450,11 +1450,11 @@ def test_default_sse_b2__create_bucket(b2_tool, schedule_bucket_cleanup): def test_sse_b2(b2_tool, bucket_name, sample_file, tmp_path, b2_uri_args): b2_tool.should_succeed( [ - 'upload-file', '--destination-server-side-encryption=SSE-B2', '--quiet', bucket_name, + 'file', 'upload', '--destination-server-side-encryption=SSE-B2', '--quiet', bucket_name, sample_file, 'encrypted' ] ) - b2_tool.should_succeed(['upload-file', '--quiet', bucket_name, sample_file, 'not_encrypted']) + b2_tool.should_succeed(['file', 'upload', '--quiet', bucket_name, sample_file, 'not_encrypted']) b2_tool.should_succeed( ['download-file', '--quiet', f'b2://{bucket_name}/encrypted', tmp_path / 'encrypted'] @@ -1535,14 +1535,14 @@ def test_sse_c(b2_tool, bucket_name, is_running_on_docker, sample_file, tmp_path b2_tool.should_fail( [ - 'upload-file', '--no-progress', '--quiet', '--destination-server-side-encryption', + 'file', 'upload', '--no-progress', '--quiet', '--destination-server-side-encryption', 'SSE-C', bucket_name, sample_file, 'gonna-fail-anyway' ], 'Using SSE-C requires providing an encryption key via B2_DESTINATION_SSE_C_KEY_B64 env var' ) file_version_info = b2_tool.should_succeed_json( [ - 'upload-file', '--no-progress', '--quiet', '--destination-server-side-encryption', + 'file', 'upload', '--no-progress', '--quiet', '--destination-server-side-encryption', 'SSE-C', bucket_name, sample_file, 'uploaded_encrypted' ], additional_env={ @@ -1905,7 +1905,7 @@ def test_file_lock( now_millis = current_time_millis() not_lockable_file = b2_tool.should_succeed_json( # file in a lock disabled bucket - ['upload-file', '--quiet', lock_disabled_bucket_name, sample_file, 'a'] + ['file', 'upload', '--quiet', lock_disabled_bucket_name, sample_file, 'a'] ) _assert_file_lock_configuration( @@ -1917,7 +1917,8 @@ def test_file_lock( b2_tool.should_fail( [ - 'upload-file', + 'file', + 'upload', '--quiet', lock_disabled_bucket_name, sample_file, @@ -1976,7 +1977,7 @@ def test_file_lock( } lockable_file = b2_tool.should_succeed_json( # file in a lock enabled bucket - ['upload-file', '--no-progress', '--quiet', lock_enabled_bucket_name, sample_file, 'a'] + ['file', 'upload', '--no-progress', '--quiet', lock_enabled_bucket_name, sample_file, 'a'] ) b2_tool.should_fail( @@ -2082,7 +2083,8 @@ def test_file_lock( b2_tool.should_fail( [ - 'upload-file', + 'file', + 'upload', '--no-progress', '--quiet', lock_enabled_bucket_name, @@ -2098,7 +2100,8 @@ def test_file_lock( uploaded_file = b2_tool.should_succeed_json( [ - 'upload-file', + 'file', + 'upload', '--no-progress', '--quiet', lock_enabled_bucket_name, @@ -2254,7 +2257,8 @@ def file_lock_without_perms_test( b2_tool.should_fail( [ - 'upload-file', + 'file', + 'upload', '--no-progress', '--quiet', lock_enabled_bucket_name, @@ -2272,7 +2276,8 @@ def file_lock_without_perms_test( b2_tool.should_fail( [ - 'upload-file', + 'file', + 'upload', '--no-progress', '--quiet', lock_disabled_bucket_name, @@ -2324,7 +2329,8 @@ def file_lock_without_perms_test( def upload_locked_file(b2_tool, bucket_name, sample_file): return b2_tool.should_succeed_json( [ - 'upload-file', + 'file', + 'upload', '--no-progress', '--quiet', '--file-retention-mode', @@ -2699,7 +2705,7 @@ def test_replication_monitoring(b2_tool, bucket_name, sample_file, schedule_buck # ---------------- add test data ---------------- destination_bucket_name = bucket_name uploaded_a = b2_tool.should_succeed_json( - ['upload-file', '--quiet', destination_bucket_name, sample_file, 'one/a'] + ['file', 'upload', '--quiet', destination_bucket_name, sample_file, 'one/a'] ) # ---------------- set up replication destination ---------------- @@ -2770,11 +2776,12 @@ def test_replication_monitoring(b2_tool, bucket_name, sample_file, schedule_buck # make test data uploaded_a = b2_tool.should_succeed_json( - ['upload-file', '--quiet', source_bucket_name, sample_file, 'one/a'] + ['file', 'upload', '--quiet', source_bucket_name, sample_file, 'one/a'] ) b2_tool.should_succeed_json( [ - 'upload-file', + 'file', + 'upload', '--quiet', source_bucket_name, '--legal-hold', @@ -2789,7 +2796,7 @@ def test_replication_monitoring(b2_tool, bucket_name, sample_file, schedule_buck upload_encryption_args = ['--destination-server-side-encryption', 'SSE-B2'] upload_additional_env = {} b2_tool.should_succeed_json( - ['upload-file', '--quiet', source_bucket_name, sample_file, 'two/c'] + + ['file', 'upload', '--quiet', source_bucket_name, sample_file, 'two/c'] + upload_encryption_args, additional_env=upload_additional_env, ) @@ -2801,7 +2808,7 @@ def test_replication_monitoring(b2_tool, bucket_name, sample_file, schedule_buck 'B2_DESTINATION_SSE_C_KEY_ID': SSE_C_AES.key.key_id, } b2_tool.should_succeed_json( - ['upload-file', '--quiet', source_bucket_name, sample_file, 'two/d'] + + ['file', 'upload', '--quiet', source_bucket_name, sample_file, 'two/d'] + upload_encryption_args, additional_env=upload_additional_env, ) @@ -2809,7 +2816,8 @@ def test_replication_monitoring(b2_tool, bucket_name, sample_file, schedule_buck # encryption + legal hold b2_tool.should_succeed_json( [ - 'upload-file', + 'file', + 'upload', '--quiet', source_bucket_name, sample_file, @@ -2994,7 +3002,8 @@ def test_upload_file__custom_upload_time(b2_tool, bucket_name, sample_file, b2_u cut = 12345 cut_printable = '1970-01-01 00:00:12' args = [ - 'upload-file', + 'file', + 'upload', '--no-progress', '--custom-upload-time', str(cut), @@ -3023,12 +3032,12 @@ def test_upload_file__custom_upload_time(b2_tool, bucket_name, sample_file, b2_u @skip_on_windows def test_upload_file__stdin_pipe_operator(request, bash_runner, b2_tool, bucket_name): - """Test upload-file from stdin using pipe operator.""" + """Test `file upload` from stdin using pipe operator.""" content = request.node.name run = bash_runner( f'echo -n {content!r} ' f'| ' - f'{" ".join(b2_tool.parse_command(b2_tool.prepare_env()))} upload-file {bucket_name} - {request.node.name}.txt' + f'{" ".join(b2_tool.parse_command(b2_tool.prepare_env()))} file upload {bucket_name} - {request.node.name}.txt' ) assert hashlib.sha1(content.encode()).hexdigest() in run.stdout @@ -3131,7 +3140,8 @@ def assert_expected(file_info, expected=expected_file_info): status, stdout, stderr = b2_tool.execute( [ - 'upload-file', + 'file', + 'upload', '--quiet', '--no-progress', bucket_name, diff --git a/test/unit/conftest.py b/test/unit/conftest.py index bd8b0838..83131c0b 100644 --- a/test/unit/conftest.py +++ b/test/unit/conftest.py @@ -157,7 +157,7 @@ def local_file(tmp_path): @pytest.fixture def uploaded_file_with_control_chars(b2_cli, bucket_info, local_file): filename = '\u009bC\u009bC\u009bIfile.txt' - b2_cli.run(['upload-file', bucket_info["bucketName"], str(local_file), filename]) + b2_cli.run(['file', 'upload', bucket_info["bucketName"], str(local_file), filename]) return { 'bucket': bucket_info["bucketName"], 'bucketId': bucket_info["bucketId"], @@ -171,7 +171,7 @@ def uploaded_file_with_control_chars(b2_cli, bucket_info, local_file): @pytest.fixture def uploaded_file(b2_cli, bucket_info, local_file): filename = 'file1.txt' - b2_cli.run(['upload-file', '--quiet', bucket_info["bucketName"], str(local_file), filename]) + b2_cli.run(['file', 'upload', '--quiet', bucket_info["bucketName"], str(local_file), filename]) return { 'bucket': bucket_info["bucketName"], 'bucketId': bucket_info["bucketId"], diff --git a/test/unit/console_tool/test_download_file.py b/test/unit/console_tool/test_download_file.py index b32b38d7..7a5d0d87 100644 --- a/test/unit/console_tool/test_download_file.py +++ b/test/unit/console_tool/test_download_file.py @@ -140,7 +140,7 @@ def reader(): @pytest.fixture def uploaded_stdout_txt(b2_cli, bucket, local_file, tmp_path): local_file.write_text('non-mocked /dev/stdout test ignore me') - b2_cli.run(['upload-file', bucket, str(local_file), 'stdout.txt']) + b2_cli.run(['file', 'upload', bucket, str(local_file), 'stdout.txt']) return { 'bucket': bucket, 'fileName': 'stdout.txt', diff --git a/test/unit/console_tool/test_upload_file.py b/test/unit/console_tool/test_upload_file.py index 69107331..029d2736 100644 --- a/test/unit/console_tool/test_upload_file.py +++ b/test/unit/console_tool/test_upload_file.py @@ -14,7 +14,7 @@ def test_upload_file__file_info_src_last_modified_millis_and_headers(b2_cli, bucket, tmpdir): - """Test upload_file supports manually specifying file info src_last_modified_millis""" + """Test `file upload` supports manually specifying file info src_last_modified_millis""" filename = 'file1.txt' content = 'hello world' local_file1 = tmpdir.join('file1.txt') @@ -37,7 +37,7 @@ def test_upload_file__file_info_src_last_modified_millis_and_headers(b2_cli, buc } b2_cli.run( [ - 'upload-file', '--no-progress', '--info=src_last_modified_millis=1', 'my-bucket', + 'file', 'upload', '--no-progress', '--info=src_last_modified_millis=1', 'my-bucket', '--cache-control', 'max-age=3600', '--expires', 'Thu, 01 Dec 2050 16:00:00 GMT', '--content-language', 'en', '--content-disposition', 'attachment', '--content-encoding', 'gzip', @@ -50,7 +50,7 @@ def test_upload_file__file_info_src_last_modified_millis_and_headers(b2_cli, buc @skip_on_windows def test_upload_file__named_pipe(b2_cli, bucket, tmpdir, bg_executor): - """Test upload_file supports named pipes""" + """Test `file upload` supports named pipes""" filename = 'named_pipe.txt' content = 'hello world' local_file1 = tmpdir.join('file1.txt') @@ -68,7 +68,7 @@ def test_upload_file__named_pipe(b2_cli, bucket, tmpdir, bg_executor): "size": len(content), } b2_cli.run( - ['upload-file', '--no-progress', 'my-bucket', + ['file', 'upload', '--no-progress', 'my-bucket', str(local_file1), filename], expected_json_in_stdout=expected_json, remove_version=True, @@ -78,7 +78,7 @@ def test_upload_file__named_pipe(b2_cli, bucket, tmpdir, bg_executor): def test_upload_file__hyphen_file_instead_of_stdin(b2_cli, bucket, tmpdir, monkeypatch): - """Test upload_file will upload file named `-` instead of stdin by default""" + """Test `file upload` will upload file named `-` instead of stdin by default""" # TODO remove this in v4 assert b2.__version__ < '4', "`-` filename should not be supported in next major version of CLI" filename = 'stdin.txt' @@ -95,7 +95,7 @@ def test_upload_file__hyphen_file_instead_of_stdin(b2_cli, bucket, tmpdir, monke "size": len(content), } b2_cli.run( - ['upload-file', '--no-progress', 'my-bucket', '-', filename], + ['file', 'upload', '--no-progress', 'my-bucket', '-', filename], expected_json_in_stdout=expected_json, remove_version=True, expected_part_of_stdout=expected_stdout, @@ -105,7 +105,7 @@ def test_upload_file__hyphen_file_instead_of_stdin(b2_cli, bucket, tmpdir, monke def test_upload_file__stdin(b2_cli, bucket, tmpdir, mock_stdin): - """Test upload_file stdin alias support""" + """Test `file upload` stdin alias support""" content = "stdin input" filename = 'stdin.txt' @@ -120,7 +120,7 @@ def test_upload_file__stdin(b2_cli, bucket, tmpdir, mock_stdin): mock_stdin.close() b2_cli.run( - ['upload-file', '--no-progress', 'my-bucket', '-', filename], + ['file', 'upload', '--no-progress', 'my-bucket', '-', filename], expected_json_in_stdout=expected_json, remove_version=True, expected_part_of_stdout=expected_stdout, @@ -128,7 +128,7 @@ def test_upload_file__stdin(b2_cli, bucket, tmpdir, mock_stdin): def test_upload_file__threads_setting(b2_cli, bucket, tmp_path): - """Test upload_file supports setting number of threads""" + """Test `file upload` supports setting number of threads""" num_threads = 66 filename = 'file1.txt' content = 'hello world' @@ -147,7 +147,7 @@ def test_upload_file__threads_setting(b2_cli, bucket, tmp_path): b2_cli.run( [ - 'upload-file', '--no-progress', 'my-bucket', '--threads', + 'file', 'upload', '--no-progress', 'my-bucket', '--threads', str(num_threads), str(local_file1), 'file1.txt' ], diff --git a/test/unit/console_tool/test_upload_unbound_stream.py b/test/unit/console_tool/test_upload_unbound_stream.py index da9f6106..fc00f9ed 100644 --- a/test/unit/console_tool/test_upload_unbound_stream.py +++ b/test/unit/console_tool/test_upload_unbound_stream.py @@ -125,5 +125,5 @@ def test_upload_unbound_stream__regular_file(b2_cli, bucket, tmpdir): remove_version=True, expected_part_of_stdout=expected_stdout, expected_stderr= - "WARNING: You are using a stream upload command to upload a regular file. While it will work, it is inefficient. Use of upload-file command is recommended.\n", + "WARNING: You are using a stream upload command to upload a regular file. While it will work, it is inefficient. Use of `file upload` command is recommended.\n", ) diff --git a/test/unit/test_console_tool.py b/test/unit/test_console_tool.py index a7516b60..dab5293e 100644 --- a/test/unit/test_console_tool.py +++ b/test/unit/test_console_tool.py @@ -303,9 +303,11 @@ def test_e_c1_char_ls_default_escape_control_chars_setting(self): bad_str = "\u009b2K\u009b7Gb\u009b24Gx\u009b4GH" escaped_bad_str = "\\x9b2K\\x9b7Gb\\x9b24Gx\\x9b4GH" - self._run_command(['upload-file', '--no-progress', 'my-bucket-cc', local_file, bad_str]) self._run_command( - ['upload-file', '--no-progress', 'my-bucket-cc', local_file, "some_normal_text"] + ['file', 'upload', '--no-progress', 'my-bucket-cc', local_file, bad_str] + ) + self._run_command( + ['file', 'upload', '--no-progress', 'my-bucket-cc', local_file, "some_normal_text"] ) self._run_command( @@ -1029,7 +1031,7 @@ def test_files(self): self._run_command( [ - 'upload-file', '--no-progress', 'my-bucket', local_file1, 'file1.txt', + 'file', 'upload', '--no-progress', 'my-bucket', local_file1, 'file1.txt', '--cache-control=private, max-age=3600' ], expected_json_in_stdout=expected_json, @@ -1179,8 +1181,9 @@ def test_files_encrypted(self): self._run_command( [ - 'upload-file', '--no-progress', '--destination-server-side-encryption=SSE-B2', - 'my-bucket', local_file1, 'file1.txt' + 'file', 'upload', '--no-progress', + '--destination-server-side-encryption=SSE-B2', 'my-bucket', local_file1, + 'file1.txt' ], expected_json_in_stdout=expected_json, remove_version=True, @@ -1344,7 +1347,7 @@ def _test_download_to_directory(self, download_by: str): local_file_content = self._read_file(local_file) self._run_command( - ['upload-file', '--no-progress', 'my-bucket', local_file, source_filename], + ['file', 'upload', '--no-progress', 'my-bucket', local_file, source_filename], remove_version=True, ) @@ -1414,7 +1417,7 @@ def test_copy_file_by_id(self): } self._run_command( - ['upload-file', '--no-progress', 'my-bucket', local_file1, 'file1.txt'], + ['file', 'upload', '--no-progress', 'my-bucket', local_file1, 'file1.txt'], expected_json_in_stdout=expected_json, remove_version=True, expected_part_of_stdout=expected_stdout, @@ -1645,7 +1648,7 @@ def test_upload_large_file(self): self._run_command( [ - 'upload-file', '--no-progress', '--threads', '5', 'my-bucket', file_path, + 'file', 'upload', '--no-progress', '--threads', '5', 'my-bucket', file_path, 'test.txt' ], expected_json_in_stdout=expected_json, @@ -1689,8 +1692,9 @@ def test_upload_large_file_encrypted(self): self._run_command( [ - 'upload-file', '--no-progress', '--destination-server-side-encryption=SSE-B2', - '--threads', '5', 'my-bucket', file_path, 'test.txt' + 'file', 'upload', '--no-progress', + '--destination-server-side-encryption=SSE-B2', '--threads', '5', 'my-bucket', + file_path, 'test.txt' ], expected_json_in_stdout=expected_json, remove_version=True, @@ -1707,7 +1711,8 @@ def test_upload_incremental(self): file_path = pathlib.Path(temp_dir) / 'test.txt' incremental_upload_params = [ - 'upload-file', + 'file', + 'upload', '--no-progress', '--threads', '5', @@ -1838,7 +1843,7 @@ def test_get_bucket_one_item_show_size(self): "uploadTimestamp": 5000 } self._run_command( - ['upload-file', '--no-progress', 'my-bucket', local_file1, 'file1.txt'], + ['file', 'upload', '--no-progress', 'my-bucket', local_file1, 'file1.txt'], expected_json_in_stdout=expected_json, remove_version=True, expected_part_of_stdout=expected_stdout, @@ -2825,7 +2830,7 @@ def test_escape_c1_char_on_ls_long(self): escaped_cc_filename = '\\x9bT\\x9bE\\x9bS\\x9bTtest.txt' self._run_command( - ['upload-file', '--no-progress', 'my-bucket-0', local_file, cc_filename] + ['file', 'upload', '--no-progress', 'my-bucket-0', local_file, cc_filename] ) self._run_command( @@ -2852,10 +2857,12 @@ def test_escape_c1_char_ls(self): bad_str = "\u009b2K\u009b7Gb\u009b24Gx\u009b4GH" escaped_bad_str = "\\x9b2K\\x9b7Gb\\x9b24Gx\\x9b4GH" - self._run_command(['upload-file', '--no-progress', 'my-bucket-cc', local_file, bad_str]) + self._run_command( + ['file', 'upload', '--no-progress', 'my-bucket-cc', local_file, bad_str] + ) self._run_command( - ['upload-file', '--no-progress', 'my-bucket-cc', local_file, "some_normal_text"] + ['file', 'upload', '--no-progress', 'my-bucket-cc', local_file, "some_normal_text"] ) self._run_command( From bc60f7a8193552f315133e06fdc68f8d7c9692da Mon Sep 17 00:00:00 2001 From: Adal Chiriliuc Date: Thu, 25 Apr 2024 18:27:36 +0300 Subject: [PATCH 05/22] deduplicated and moved _setup_parser method into parent DownloadFileBase class --- b2/_internal/console_tool.py | 20 +++++--------------- 1 file changed, 5 insertions(+), 15 deletions(-) diff --git a/b2/_internal/console_tool.py b/b2/_internal/console_tool.py index 5a477681..c164c431 100644 --- a/b2/_internal/console_tool.py +++ b/b2/_internal/console_tool.py @@ -1901,6 +1901,11 @@ class DownloadFileBase( - **readFiles** """ + @classmethod + def _setup_parser(cls, parser): + super()._setup_parser(parser) + parser.add_argument('localFileName') + def _run(self, args): progress_listener = self.make_progress_listener( args.localFileName, args.no_progress or args.quiet @@ -1926,11 +1931,6 @@ def _run(self, args): class DownloadFile(B2URIFileArgMixin, DownloadFileBase): __doc__ = DownloadFileBase.__doc__ - @classmethod - def _setup_parser(cls, parser): - super()._setup_parser(parser) - parser.add_argument('localFileName') - def get_b2_uri_from_arg(self, args: argparse.Namespace) -> B2URIBase: return args.B2_URI @@ -1939,21 +1939,11 @@ class DownloadFileById(CmdReplacedByMixin, B2URIFileIDArgMixin, DownloadFileBase __doc__ = DownloadFileBase.__doc__ replaced_by_cmd = DownloadFile - @classmethod - def _setup_parser(cls, parser): - super()._setup_parser(parser) - parser.add_argument('localFileName') - class DownloadFileByName(CmdReplacedByMixin, B2URIBucketNFilenameArgMixin, DownloadFileBase): __doc__ = DownloadFileBase.__doc__ replaced_by_cmd = DownloadFile - @classmethod - def _setup_parser(cls, parser): - super()._setup_parser(parser) - parser.add_argument('localFileName') - class FileCatBase(B2URIFileArgMixin, DownloadCommand): """ From e7ff9f60204c5119314ca19bb9f2d2975064ac71 Mon Sep 17 00:00:00 2001 From: Adal Chiriliuc Date: Thu, 25 Apr 2024 19:12:47 +0300 Subject: [PATCH 06/22] use file download subcommand --- b2/_internal/console_tool.py | 40 ++++++++++--------- test/integration/test_autocomplete.py | 2 +- test/integration/test_b2_command_line.py | 42 +++++++++++++------- test/unit/_cli/test_autocomplete_cache.py | 2 +- test/unit/console_tool/test_download_file.py | 18 ++++----- test/unit/console_tool/test_help.py | 4 +- test/unit/test_console_tool.py | 14 ++++--- 7 files changed, 69 insertions(+), 53 deletions(-) diff --git a/b2/_internal/console_tool.py b/b2/_internal/console_tool.py index c164c431..149aae17 100644 --- a/b2/_internal/console_tool.py +++ b/b2/_internal/console_tool.py @@ -1881,7 +1881,7 @@ def get_local_output_filepath( return pathlib.Path(output_filepath_str) -class DownloadFileBase( +class FileDownloadBase( ThreadsMixin, MaxDownloadStreamsMixin, DownloadCommand, @@ -1928,23 +1928,6 @@ def _run(self, args): return 0 -class DownloadFile(B2URIFileArgMixin, DownloadFileBase): - __doc__ = DownloadFileBase.__doc__ - - def get_b2_uri_from_arg(self, args: argparse.Namespace) -> B2URIBase: - return args.B2_URI - - -class DownloadFileById(CmdReplacedByMixin, B2URIFileIDArgMixin, DownloadFileBase): - __doc__ = DownloadFileBase.__doc__ - replaced_by_cmd = DownloadFile - - -class DownloadFileByName(CmdReplacedByMixin, B2URIBucketNFilenameArgMixin, DownloadFileBase): - __doc__ = DownloadFileBase.__doc__ - replaced_by_cmd = DownloadFile - - class FileCatBase(B2URIFileArgMixin, DownloadCommand): """ Download content of a file-like object identified by B2 URI directly to stdout. @@ -4929,6 +4912,12 @@ class FileUpload(FileUploadBase): COMMAND_NAME = 'upload' +@File.subcommands_registry.register +class FileDownload(B2URIFileArgMixin, FileDownloadBase): + __doc__ = FileDownloadBase.__doc__ + COMMAND_NAME = 'download' + + class FileInfo2(CmdReplacedByMixin, B2URIFileArgMixin, FileInfoBase): __doc__ = FileInfoBase.__doc__ replaced_by_cmd = (File, FileInfo) @@ -4966,6 +4955,21 @@ class UploadFile(CmdReplacedByMixin, FileUploadBase): replaced_by_cmd = (File, FileUpload) +class DownloadFile(CmdReplacedByMixin, B2URIFileArgMixin, FileDownloadBase): + __doc__ = FileDownloadBase.__doc__ + replaced_by_cmd = (File, FileDownload) + + +class DownloadFileById(CmdReplacedByMixin, B2URIFileIDArgMixin, FileDownloadBase): + __doc__ = FileDownloadBase.__doc__ + replaced_by_cmd = (File, FileDownload) + + +class DownloadFileByName(CmdReplacedByMixin, B2URIBucketNFilenameArgMixin, FileDownloadBase): + __doc__ = FileDownloadBase.__doc__ + replaced_by_cmd = (File, FileDownload) + + class ConsoleTool: """ Implements the commands available in the B2 command-line tool diff --git a/test/integration/test_autocomplete.py b/test/integration/test_autocomplete.py index cd8cf73d..82420aba 100644 --- a/test/integration/test_autocomplete.py +++ b/test/integration/test_autocomplete.py @@ -97,7 +97,7 @@ def test_autocomplete_b2__download_file__b2uri( """Test that autocomplete suggests bucket names and file names.""" if is_running_on_docker: pytest.skip('Not supported on Docker') - shell.send(f'{cli_version} download_file \t\t') + shell.send(f'{cli_version} file download \t\t') shell.expect_exact("b2://", timeout=TIMEOUT) shell.send('b2://\t\t') shell.expect_exact(bucket_name, timeout=TIMEOUT) diff --git a/test/integration/test_b2_command_line.py b/test/integration/test_b2_command_line.py index 24079701..7516d81c 100755 --- a/test/integration/test_b2_command_line.py +++ b/test/integration/test_b2_command_line.py @@ -282,7 +282,7 @@ def test_download(b2_tool, bucket_name, sample_filepath, uploaded_sample_file, t output_a = tmp_path / 'a' b2_tool.should_succeed( [ - 'download-file', '--quiet', f"b2://{bucket_name}/{uploaded_sample_file['fileName']}", + 'file', 'download', '--quiet', f"b2://{bucket_name}/{uploaded_sample_file['fileName']}", str(output_a) ] ) @@ -290,7 +290,7 @@ def test_download(b2_tool, bucket_name, sample_filepath, uploaded_sample_file, t output_b = tmp_path / 'b' b2_tool.should_succeed( - ['download-file', '--quiet', f"b2id://{uploaded_sample_file['fileId']}", + ['file', 'download', '--quiet', f"b2id://{uploaded_sample_file['fileId']}", str(output_b)] ) assert output_b.read_text() == sample_filepath.read_text() @@ -346,7 +346,9 @@ def test_basic(b2_tool, bucket_name, sample_file, tmp_path, b2_uri_args): ['rm', '--recursive', '--with-wildcard', *b2_uri_args(bucket_name, 'rm1')] ) - b2_tool.should_succeed(['download-file', '--quiet', f'b2://{bucket_name}/b/1', tmp_path / 'a']) + b2_tool.should_succeed( + ['file', 'download', '--quiet', f'b2://{bucket_name}/b/1', tmp_path / 'a'] + ) b2_tool.should_succeed(['hide-file', bucket_name, 'c']) @@ -1457,11 +1459,11 @@ def test_sse_b2(b2_tool, bucket_name, sample_file, tmp_path, b2_uri_args): b2_tool.should_succeed(['file', 'upload', '--quiet', bucket_name, sample_file, 'not_encrypted']) b2_tool.should_succeed( - ['download-file', '--quiet', f'b2://{bucket_name}/encrypted', tmp_path / 'encrypted'] + ['file', 'download', '--quiet', f'b2://{bucket_name}/encrypted', tmp_path / 'encrypted'] ) b2_tool.should_succeed( [ - 'download-file', '--quiet', f'b2://{bucket_name}/not_encrypted', + 'file', 'download', '--quiet', f'b2://{bucket_name}/not_encrypted', tmp_path / 'not_encrypted' ] ) @@ -1561,13 +1563,16 @@ def test_sse_c(b2_tool, bucket_name, is_running_on_docker, sample_file, tmp_path should_equal(sse_c_key_id, file_version_info['fileInfo'][SSE_C_KEY_ID_FILE_INFO_KEY_NAME]) b2_tool.should_fail( - ['download-file', '--quiet', f'b2://{bucket_name}/uploaded_encrypted', 'gonna_fail_anyway'], + [ + 'file', 'download', '--quiet', f'b2://{bucket_name}/uploaded_encrypted', + 'gonna_fail_anyway' + ], expected_pattern='ERROR: The object was stored using a form of Server Side Encryption. The ' r'correct parameters must be provided to retrieve the object. \(bad_request\)' ) b2_tool.should_fail( [ - 'download-file', '--quiet', '--source-server-side-encryption', 'SSE-C', + 'file', 'download', '--quiet', '--source-server-side-encryption', 'SSE-C', f'b2://{bucket_name}/uploaded_encrypted', 'gonna_fail_anyway' ], expected_pattern='ValueError: Using SSE-C requires providing an encryption key via ' @@ -1575,7 +1580,7 @@ def test_sse_c(b2_tool, bucket_name, is_running_on_docker, sample_file, tmp_path ) b2_tool.should_fail( [ - 'download-file', '--quiet', '--source-server-side-encryption', 'SSE-C', + 'file', 'download', '--quiet', '--source-server-side-encryption', 'SSE-C', f'b2://{bucket_name}/uploaded_encrypted', 'gonna_fail_anyway' ], expected_pattern='ERROR: Wrong or no SSE-C key provided when reading a file.', @@ -1584,7 +1589,8 @@ def test_sse_c(b2_tool, bucket_name, is_running_on_docker, sample_file, tmp_path with contextlib.nullcontext(tmp_path) as dir_path: b2_tool.should_succeed( [ - 'download-file', + 'file', + 'download', '--no-progress', '--quiet', '--source-server-side-encryption', @@ -1597,7 +1603,8 @@ def test_sse_c(b2_tool, bucket_name, is_running_on_docker, sample_file, tmp_path assert read_file(dir_path / 'a') == read_file(sample_file) b2_tool.should_succeed( [ - 'download-file', + 'file', + 'download', '--no-progress', '--quiet', '--source-server-side-encryption', @@ -3061,10 +3068,13 @@ def test_download_file_stdout( b2_tool, bucket_name, sample_filepath, tmp_path, uploaded_sample_file ): assert b2_tool.should_succeed( - ['download-file', '--quiet', f"b2://{bucket_name}/{uploaded_sample_file['fileName']}", '-'], + [ + 'file', 'download', '--quiet', f"b2://{bucket_name}/{uploaded_sample_file['fileName']}", + '-' + ], ) == sample_filepath.read_text() assert b2_tool.should_succeed( - ['download-file', '--quiet', f"b2id://{uploaded_sample_file['fileId']}", '-'], + ['file', 'download', '--quiet', f"b2id://{uploaded_sample_file['fileId']}", '-'], ) == sample_filepath.read_text() @@ -3079,7 +3089,8 @@ def test_download_file_to_directory( sample_file_content = sample_filepath.read_text() b2_tool.should_succeed( [ - 'download-file', + 'file', + 'download', '--quiet', f"b2://{bucket_name}/{uploaded_sample_file['fileName']}", str(target_directory), @@ -3091,7 +3102,8 @@ def test_download_file_to_directory( b2_tool.should_succeed( [ - 'download-file', + 'file', + 'download', '--quiet', f"b2id://{uploaded_sample_file['fileId']}", str(target_directory), @@ -3169,7 +3181,7 @@ def assert_expected(file_info, expected=expected_file_info): assert_expected(copied_version['fileInfo']) download_output = b2_tool.should_succeed( - ['download-file', f"b2id://{file_version['fileId']}", tmp_path / 'downloaded_file'] + ['file', 'download', f"b2id://{file_version['fileId']}", tmp_path / 'downloaded_file'] ) assert re.search(r'CacheControl: *max-age=3600', download_output) assert re.search(r'ContentDisposition: *attachment', download_output) diff --git a/test/unit/_cli/test_autocomplete_cache.py b/test/unit/_cli/test_autocomplete_cache.py index 4ce1ad73..41e0c736 100644 --- a/test/unit/_cli/test_autocomplete_cache.py +++ b/test/unit/_cli/test_autocomplete_cache.py @@ -227,7 +227,7 @@ def test_complete_with_file_uri_suggestions( tracker=autocomplete_cache.VersionTracker(), store=autocomplete_cache.HomeCachePickleStore(tmp_path), ) - with autocomplete_runner(f'b2 download-file b2://{bucket}/'): + with autocomplete_runner(f'b2 file download b2://{bucket}/'): exit, argcomplete_output = argcomplete_result() assert exit == 0 assert file_name in argcomplete_output diff --git a/test/unit/console_tool/test_download_file.py b/test/unit/console_tool/test_download_file.py index 7a5d0d87..6ff3c7bf 100644 --- a/test/unit/console_tool/test_download_file.py +++ b/test/unit/console_tool/test_download_file.py @@ -40,7 +40,7 @@ def test_download_file_by_uri__flag_support(b2_cli, uploaded_file, tmp_path, fla output_path = tmp_path / 'output.txt' b2_cli.run( - ['download-file', flag, 'b2id://9999', + ['file', 'download', flag, 'b2id://9999', str(output_path)], expected_stdout=expected_stdout.format(output_path=pathlib.Path(output_path).resolve()) ) @@ -55,7 +55,7 @@ def test_download_file_by_uri__b2_uri_support(b2_cli, uploaded_file, tmp_path, b output_path = tmp_path / 'output.txt' b2_cli.run( - ['download-file', b2_uri, str(output_path)], + ['file', 'download', b2_uri, str(output_path)], expected_stdout=EXPECTED_STDOUT_DOWNLOAD.format( output_path=pathlib.Path(output_path).resolve() ) @@ -82,7 +82,7 @@ def test_download_file_by_name(b2_cli, local_file, uploaded_file, tmp_path, flag output_path=pathlib.Path(output_path).resolve() ), expected_stderr= - 'WARNING: `download-file-by-name` command is deprecated. Use `download-file` instead.\n', + 'WARNING: `download-file-by-name` command is deprecated. Use `file download` instead.\n', ) assert output_path.read_text() == uploaded_file['content'] @@ -101,7 +101,7 @@ def test_download_file_by_id(b2_cli, uploaded_file, tmp_path, flag, expected_std ['download-file-by-id', flag, '9999', str(output_path)], expected_stdout=expected_stdout.format(output_path=pathlib.Path(output_path).resolve()), expected_stderr= - 'WARNING: `download-file-by-id` command is deprecated. Use `download-file` instead.\n', + 'WARNING: `download-file-by-id` command is deprecated. Use `file download` instead.\n', ) assert output_path.read_text() == uploaded_file['content'] @@ -131,7 +131,7 @@ def reader(): output_path=pathlib.Path(output_path).resolve() ), expected_stderr= - 'WARNING: `download-file-by-name` command is deprecated. Use `download-file` instead.\n', + 'WARNING: `download-file-by-name` command is deprecated. Use `file download` instead.\n', ) reader_future.result(timeout=1) assert output_string == uploaded_file['content'] @@ -151,18 +151,17 @@ def uploaded_stdout_txt(b2_cli, bucket, local_file, tmp_path): def test_download_file_by_name__to_stdout_by_alias( b2_cli, bucket, uploaded_stdout_txt, tmp_path, capfd ): - """Test download_file_by_name stdout alias support""" + """Test download-file-by-name stdout alias support""" b2_cli.run( ['download-file-by-name', '--no-progress', bucket, uploaded_stdout_txt['fileName'], '-'], expected_stderr= - 'WARNING: `download-file-by-name` command is deprecated. Use `download-file` instead.\n', + 'WARNING: `download-file-by-name` command is deprecated. Use `file download` instead.\n', ) assert capfd.readouterr().out == uploaded_stdout_txt['content'] assert not pathlib.Path('-').exists() def test_cat__b2_uri(b2_cli, bucket, uploaded_stdout_txt, tmp_path, capfd): - """Test download_file_by_name stdout alias support""" b2_cli.run( ['file', 'cat', '--no-progress', f"b2://{bucket}/{uploaded_stdout_txt['fileName']}"], ) @@ -189,7 +188,6 @@ def test_cat__b2_uri__not_a_file(b2_cli, bucket, capfd): def test_cat__b2id_uri(b2_cli, bucket, uploaded_stdout_txt, tmp_path, capfd): - """Test download_file_by_name stdout alias support""" b2_cli.run(['file', 'cat', '--no-progress', "b2id://9999"],) assert capfd.readouterr().out == uploaded_stdout_txt['content'] @@ -200,7 +198,7 @@ def test__download_file__threads(b2_cli, local_file, uploaded_file, tmp_path): b2_cli.run( [ - 'download-file', '--no-progress', '--threads', + 'file', 'download', '--no-progress', '--threads', str(num_threads), 'b2://my-bucket/file1.txt', str(output_path) ] diff --git a/test/unit/console_tool/test_help.py b/test/unit/console_tool/test_help.py index 4dfca6c6..d908c1e3 100644 --- a/test/unit/console_tool/test_help.py +++ b/test/unit/console_tool/test_help.py @@ -16,8 +16,8 @@ # --help shouldn't show deprecated commands ( "--help", - [" b2 download-file ", "-h", "--help-all"], - [" download-file-by-name ", "(DEPRECATED)"], + [" b2 file ", "-h", "--help-all"], + [" b2 download-file-by-name ", "(DEPRECATED)"], ), # --help-all should show deprecated commands, but marked as deprecated ( diff --git a/test/unit/test_console_tool.py b/test/unit/test_console_tool.py index dab5293e..3a1a1454 100644 --- a/test/unit/test_console_tool.py +++ b/test/unit/test_console_tool.py @@ -1237,7 +1237,7 @@ def test_files_encrypted(self): ) self._run_command( - ['download-file', '--no-progress', 'b2://my-bucket/file1.txt', local_download1], + ['file', 'download', '--no-progress', 'b2://my-bucket/file1.txt', local_download1], expected_stdout, '', 0 ) self.assertEqual(b'hello world', self._read_file(local_download1)) @@ -1249,8 +1249,8 @@ def test_files_encrypted(self): output_path=pathlib.Path(local_download2).resolve() ) self._run_command( - ['download-file', '--no-progress', 'b2id://9999', local_download2], expected_stdout, - '', 0 + ['file', 'download', '--no-progress', 'b2id://9999', local_download2], + expected_stdout, '', 0 ) self.assertEqual(b'hello world', self._read_file(local_download2)) @@ -1353,7 +1353,8 @@ def _test_download_to_directory(self, download_by: str): b2uri = f'b2://my-bucket/{source_filename}' if download_by == 'name' else 'b2id://9999' command = [ - 'download-file', + 'file', + 'download', '--no-progress', b2uri, ] @@ -1471,7 +1472,7 @@ def test_copy_file_by_id(self): local_download1 = os.path.join(temp_dir, 'file1_copy.txt') self._run_command( - ['download-file', '-q', 'b2://my-bucket/file1_copy.txt', local_download1] + ['file', 'download', '-q', 'b2://my-bucket/file1_copy.txt', local_download1] ) self.assertEqual(b'lo wo', self._read_file(local_download1)) @@ -1732,7 +1733,8 @@ def test_upload_incremental(self): downloaded_path = pathlib.Path(temp_dir) / 'out.txt' self._run_command( [ - 'download-file', + 'file', + 'download', '-q', 'b2://my-bucket/test.txt', str(downloaded_path), From 66685c281cb7e633b272a1bcfed970304955a38a Mon Sep 17 00:00:00 2001 From: Adal Chiriliuc Date: Thu, 25 Apr 2024 19:25:28 +0300 Subject: [PATCH 07/22] add file copy-by-id subcommand --- b2/_internal/console_tool.py | 14 +++++++++++++- test/unit/test_copy.py | 4 ++-- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/b2/_internal/console_tool.py b/b2/_internal/console_tool.py index 149aae17..985263a9 100644 --- a/b2/_internal/console_tool.py +++ b/b2/_internal/console_tool.py @@ -1409,7 +1409,7 @@ def _run(self, args): return 0 -class CopyFileById( +class FileCopyByIdBase( HeaderFlagsMixin, DestinationSseMixin, SourceSseMixin, FileRetentionSettingMixin, LegalHoldMixin, Command ): @@ -4918,6 +4918,13 @@ class FileDownload(B2URIFileArgMixin, FileDownloadBase): COMMAND_NAME = 'download' +@File.subcommands_registry.register +class FileCopyById(FileCopyByIdBase): + __doc__ = FileCopyByIdBase.__doc__ + # TODO we can't use 'copy-by-id', gets transformed to 'copy--by--id' + COMMAND_NAME = 'CopyById' + + class FileInfo2(CmdReplacedByMixin, B2URIFileArgMixin, FileInfoBase): __doc__ = FileInfoBase.__doc__ replaced_by_cmd = (File, FileInfo) @@ -4970,6 +4977,11 @@ class DownloadFileByName(CmdReplacedByMixin, B2URIBucketNFilenameArgMixin, FileD replaced_by_cmd = (File, FileDownload) +class CopyFileById(CmdReplacedByMixin, FileCopyByIdBase): + __doc__ = FileCopyByIdBase.__doc__ + replaced_by_cmd = (File, FileCopyById) + + class ConsoleTool: """ Implements the commands available in the B2 command-line tool diff --git a/test/unit/test_copy.py b/test/unit/test_copy.py index 718fd9a1..fac26753 100644 --- a/test/unit/test_copy.py +++ b/test/unit/test_copy.py @@ -19,7 +19,7 @@ EncryptionSetting, ) -from b2._internal.console_tool import CopyFileById +from b2._internal.console_tool import FileCopyById from .test_base import TestBase @@ -29,7 +29,7 @@ def test_determine_source_metadata(self): mock_api = mock.MagicMock() mock_console_tool = mock.MagicMock() mock_console_tool.api = mock_api - copy_file_command = CopyFileById(mock_console_tool) + copy_file_command = FileCopyById(mock_console_tool) result = copy_file_command._determine_source_metadata( 'id', From 2d31d7e169a94062f7bf635768d4dd2df0a623e5 Mon Sep 17 00:00:00 2001 From: Adal Chiriliuc Date: Thu, 25 Apr 2024 19:30:08 +0300 Subject: [PATCH 08/22] use file copy-by-id subcommand --- test/integration/test_b2_command_line.py | 53 +++++++++++++++--------- test/unit/test_console_tool.py | 17 ++++---- 2 files changed, 43 insertions(+), 27 deletions(-) diff --git a/test/integration/test_b2_command_line.py b/test/integration/test_b2_command_line.py index 7516d81c..465857f8 100755 --- a/test/integration/test_b2_command_line.py +++ b/test/integration/test_b2_command_line.py @@ -375,7 +375,7 @@ def test_basic(b2_tool, bucket_name, sample_file, tmp_path, b2_uri_args): ) should_equal([], [f['fileName'] for f in list_of_files]) - b2_tool.should_succeed(['copy-file-by-id', first_a_version['fileId'], bucket_name, 'x']) + b2_tool.should_succeed(['file', 'copy-by-id', first_a_version['fileId'], bucket_name, 'x']) b2_tool.should_succeed(['ls', *b2_uri_args(bucket_name)], '^a{0}b/{0}d{0}'.format(os.linesep)) # file_id, action, date, time, size(, replication), name @@ -1493,12 +1493,15 @@ def test_sse_b2(b2_tool, bucket_name, sample_file, tmp_path, b2_uri_args): b2_tool.should_succeed( [ - 'copy-file-by-id', '--destination-server-side-encryption=SSE-B2', + 'file', 'copy-by-id', '--destination-server-side-encryption=SSE-B2', encrypted_version['fileId'], bucket_name, 'copied_encrypted' ] ) b2_tool.should_succeed( - ['copy-file-by-id', not_encrypted_version['fileId'], bucket_name, 'copied_not_encrypted'] + [ + 'file', 'copy-by-id', not_encrypted_version['fileId'], bucket_name, + 'copied_not_encrypted' + ] ) list_of_files = b2_tool.should_succeed_json( @@ -1617,22 +1620,22 @@ def test_sse_c(b2_tool, bucket_name, is_running_on_docker, sample_file, tmp_path assert read_file(dir_path / 'b') == read_file(sample_file) b2_tool.should_fail( - ['copy-file-by-id', file_version_info['fileId'], bucket_name, 'gonna-fail-anyway'], + ['file', 'copy-by-id', file_version_info['fileId'], bucket_name, 'gonna-fail-anyway'], expected_pattern= 'ERROR: The object was stored using a form of Server Side Encryption. The correct ' r'parameters must be provided to retrieve the object. \(bad_request\)' ) b2_tool.should_fail( [ - 'copy-file-by-id', '--source-server-side-encryption=SSE-C', file_version_info['fileId'], - bucket_name, 'gonna-fail-anyway' + 'file', 'copy-by-id', '--source-server-side-encryption=SSE-C', + file_version_info['fileId'], bucket_name, 'gonna-fail-anyway' ], expected_pattern='ValueError: Using SSE-C requires providing an encryption key via ' 'B2_SOURCE_SSE_C_KEY_B64 env var' ) b2_tool.should_fail( [ - 'copy-file-by-id', '--source-server-side-encryption=SSE-C', + 'file', 'copy-by-id', '--source-server-side-encryption=SSE-C', '--destination-server-side-encryption=SSE-C', file_version_info['fileId'], bucket_name, 'gonna-fail-anyway' ], @@ -1642,8 +1645,8 @@ def test_sse_c(b2_tool, bucket_name, is_running_on_docker, sample_file, tmp_path ) b2_tool.should_fail( [ - 'copy-file-by-id', '--source-server-side-encryption=SSE-C', file_version_info['fileId'], - bucket_name, 'gonna-fail-anyway' + 'file', 'copy-by-id', '--source-server-side-encryption=SSE-C', + file_version_info['fileId'], bucket_name, 'gonna-fail-anyway' ], additional_env={'B2_SOURCE_SSE_C_KEY_B64': base64.b64encode(secret).decode()}, expected_pattern= @@ -1652,7 +1655,8 @@ def test_sse_c(b2_tool, bucket_name, is_running_on_docker, sample_file, tmp_path ) b2_tool.should_succeed( [ - 'copy-file-by-id', + 'file', + 'copy-by-id', '--source-server-side-encryption=SSE-C', file_version_info['fileId'], bucket_name, @@ -1666,7 +1670,8 @@ def test_sse_c(b2_tool, bucket_name, is_running_on_docker, sample_file, tmp_path ) b2_tool.should_succeed( [ - 'copy-file-by-id', + 'file', + 'copy-by-id', '--source-server-side-encryption=SSE-C', file_version_info['fileId'], bucket_name, @@ -1679,7 +1684,8 @@ def test_sse_c(b2_tool, bucket_name, is_running_on_docker, sample_file, tmp_path ) b2_tool.should_succeed( [ - 'copy-file-by-id', + 'file', + 'copy-by-id', '--source-server-side-encryption=SSE-C', file_version_info['fileId'], bucket_name, @@ -1690,7 +1696,8 @@ def test_sse_c(b2_tool, bucket_name, is_running_on_docker, sample_file, tmp_path ) b2_tool.should_succeed( [ - 'copy-file-by-id', + 'file', + 'copy-by-id', '--source-server-side-encryption=SSE-C', '--destination-server-side-encryption=SSE-C', file_version_info['fileId'], @@ -1705,7 +1712,8 @@ def test_sse_c(b2_tool, bucket_name, is_running_on_docker, sample_file, tmp_path ) b2_tool.should_succeed( [ - 'copy-file-by-id', + 'file', + 'copy-by-id', '--source-server-side-encryption=SSE-C', '--destination-server-side-encryption=SSE-C', file_version_info['fileId'], @@ -1723,7 +1731,8 @@ def test_sse_c(b2_tool, bucket_name, is_running_on_docker, sample_file, tmp_path ) b2_tool.should_succeed( [ - 'copy-file-by-id', + 'file', + 'copy-by-id', '--source-server-side-encryption=SSE-C', '--destination-server-side-encryption=SSE-C', file_version_info['fileId'], @@ -2133,7 +2142,8 @@ def test_file_lock( b2_tool.should_fail( [ - 'copy-file-by-id', + 'file', + 'copy-by-id', lockable_file['fileId'], lock_disabled_bucket_name, 'copied', @@ -2148,7 +2158,8 @@ def test_file_lock( copied_file = b2_tool.should_succeed_json( [ - 'copy-file-by-id', + 'file', + 'copy-by-id', lockable_file['fileId'], lock_enabled_bucket_name, 'copied', @@ -2302,7 +2313,8 @@ def file_lock_without_perms_test( b2_tool.should_fail( [ - 'copy-file-by-id', + 'file', + 'copy-by-id', lockable_file_id, lock_enabled_bucket_name, 'copied', @@ -2318,7 +2330,8 @@ def file_lock_without_perms_test( b2_tool.should_fail( [ - 'copy-file-by-id', + 'file', + 'copy-by-id', lockable_file_id, lock_disabled_bucket_name, 'copied', @@ -3174,7 +3187,7 @@ def assert_expected(file_info, expected=expected_file_info): copied_version = b2_tool.should_succeed_json( [ - 'copy-file-by-id', '--quiet', *args, '--content-type', 'text/plain', + 'file', 'copy-by-id', '--quiet', *args, '--content-type', 'text/plain', file_version['fileId'], bucket_name, 'copied_file' ] ) diff --git a/test/unit/test_console_tool.py b/test/unit/test_console_tool.py index 3a1a1454..18b65109 100644 --- a/test/unit/test_console_tool.py +++ b/test/unit/test_console_tool.py @@ -1443,7 +1443,7 @@ def test_copy_file_by_id(self): "uploadTimestamp": 5001 } self._run_command( - ['copy-file-by-id', '9999', 'my-bucket', 'file1_copy.txt'], + ['file', 'copy-by-id', '9999', 'my-bucket', 'file1_copy.txt'], expected_json_in_stdout=expected_json, ) @@ -1466,7 +1466,7 @@ def test_copy_file_by_id(self): "uploadTimestamp": 5002 } self._run_command( - ['copy-file-by-id', '--range', '3,7', '9999', 'my-bucket', 'file1_copy.txt'], + ['file', 'copy-by-id', '--range', '3,7', '9999', 'my-bucket', 'file1_copy.txt'], expected_json_in_stdout=expected_json, ) @@ -1480,7 +1480,8 @@ def test_copy_file_by_id(self): expected_stderr = "ERROR: File info can be set only when content type is set\n" self._run_command( [ - 'copy-file-by-id', + 'file', + 'copy-by-id', '--info', 'a=b', '9999', @@ -1496,7 +1497,8 @@ def test_copy_file_by_id(self): expected_stderr = "ERROR: File info can be not set only when content type is not set\n" self._run_command( [ - 'copy-file-by-id', + 'file', + 'copy-by-id', '--content-type', 'text/plain', '9999', @@ -1528,7 +1530,8 @@ def test_copy_file_by_id(self): } self._run_command( [ - 'copy-file-by-id', + 'file', + 'copy-by-id', '--content-type', 'text/plain', '--info', @@ -1543,7 +1546,7 @@ def test_copy_file_by_id(self): # UnsatisfiableRange expected_stderr = "ERROR: The range in the request is outside the size of the file\n" self._run_command( - ['copy-file-by-id', '--range', '12,20', '9999', 'my-bucket', 'file1_copy.txt'], + ['file', 'copy-by-id', '--range', '12,20', '9999', 'my-bucket', 'file1_copy.txt'], '', expected_stderr, 1, @@ -1569,7 +1572,7 @@ def test_copy_file_by_id(self): "uploadTimestamp": 5004 } self._run_command( - ['copy-file-by-id', '9999', 'my-bucket1', 'file1_copy.txt'], + ['file', 'copy-by-id', '9999', 'my-bucket1', 'file1_copy.txt'], expected_json_in_stdout=expected_json, ) From b0e9f79f2a0dfd80502fbebaee76a455906d59e2 Mon Sep 17 00:00:00 2001 From: Adal Chiriliuc Date: Thu, 25 Apr 2024 19:36:29 +0300 Subject: [PATCH 09/22] file hide subcommand --- b2/_internal/console_tool.py | 13 +++++++++++- test/integration/test_b2_command_line.py | 2 +- test/unit/_cli/test_autocomplete_cache.py | 4 ++-- test/unit/test_console_tool.py | 26 +++++++++++------------ 4 files changed, 28 insertions(+), 17 deletions(-) diff --git a/b2/_internal/console_tool.py b/b2/_internal/console_tool.py index 985263a9..ab2a6a69 100644 --- a/b2/_internal/console_tool.py +++ b/b2/_internal/console_tool.py @@ -2113,7 +2113,7 @@ def _run(self, args): return 0 -class HideFile(Command): +class FileHideBase(Command): """ Uploads a new, hidden, version of the given file. @@ -4925,6 +4925,12 @@ class FileCopyById(FileCopyByIdBase): COMMAND_NAME = 'CopyById' +@File.subcommands_registry.register +class FileHide(FileHideBase): + __doc__ = FileHideBase.__doc__ + COMMAND_NAME = 'hide' + + class FileInfo2(CmdReplacedByMixin, B2URIFileArgMixin, FileInfoBase): __doc__ = FileInfoBase.__doc__ replaced_by_cmd = (File, FileInfo) @@ -4982,6 +4988,11 @@ class CopyFileById(CmdReplacedByMixin, FileCopyByIdBase): replaced_by_cmd = (File, FileCopyById) +class HideFile(CmdReplacedByMixin, FileHideBase): + __doc__ = FileHideBase.__doc__ + replaced_by_cmd = (File, FileHide) + + class ConsoleTool: """ Implements the commands available in the B2 command-line tool diff --git a/test/integration/test_b2_command_line.py b/test/integration/test_b2_command_line.py index 465857f8..5c0824f1 100755 --- a/test/integration/test_b2_command_line.py +++ b/test/integration/test_b2_command_line.py @@ -350,7 +350,7 @@ def test_basic(b2_tool, bucket_name, sample_file, tmp_path, b2_uri_args): ['file', 'download', '--quiet', f'b2://{bucket_name}/b/1', tmp_path / 'a'] ) - b2_tool.should_succeed(['hide-file', bucket_name, 'c']) + b2_tool.should_succeed(['file', 'hide', bucket_name, 'c']) list_of_files = b2_tool.should_succeed_json( ['ls', '--json', '--recursive', *b2_uri_args(bucket_name)] diff --git a/test/unit/_cli/test_autocomplete_cache.py b/test/unit/_cli/test_autocomplete_cache.py index 41e0c736..383e224b 100644 --- a/test/unit/_cli/test_autocomplete_cache.py +++ b/test/unit/_cli/test_autocomplete_cache.py @@ -176,7 +176,7 @@ def test_complete_with_escaped_control_characters( store=autocomplete_cache.HomeCachePickleStore(tmp_path), ) - with autocomplete_runner(f'b2 hide-file {bucket} '): + with autocomplete_runner(f'b2 file hide {bucket} '): exit, argcomplete_output = argcomplete_result() assert exit == 0 assert escaped_cc_file_name in argcomplete_output @@ -200,7 +200,7 @@ def test_complete_with_file_suggestions( tracker=autocomplete_cache.VersionTracker(), store=autocomplete_cache.HomeCachePickleStore(tmp_path), ) - with autocomplete_runner(f'b2 hide-file {bucket} '): + with autocomplete_runner(f'b2 file hide {bucket} '): exit, argcomplete_output = argcomplete_result() assert exit == 0 assert file_name in argcomplete_output diff --git a/test/unit/test_console_tool.py b/test/unit/test_console_tool.py index 18b65109..73462221 100644 --- a/test/unit/test_console_tool.py +++ b/test/unit/test_console_tool.py @@ -1081,7 +1081,7 @@ def test_files(self): } self._run_command( - ['hide-file', 'my-bucket', 'file1.txt'], + ['file', 'hide', 'my-bucket', 'file1.txt'], expected_json_in_stdout=expected_json, ) @@ -1269,7 +1269,7 @@ def test_files_encrypted(self): } self._run_command( - ['hide-file', 'my-bucket', 'file1.txt'], + ['file', 'hide', 'my-bucket', 'file1.txt'], expected_json_in_stdout=expected_json, ) @@ -1982,10 +1982,10 @@ def test_get_bucket_with_hidden(self): # something has failed if the output of 'bucket get' does not match the canon. stdout, stderr = self._get_stdouterr() console_tool = self.console_tool_class(stdout, stderr) - console_tool.run_command(['b2', 'hide-file', 'my-bucket', 'hidden1']) - console_tool.run_command(['b2', 'hide-file', 'my-bucket', 'hidden2']) - console_tool.run_command(['b2', 'hide-file', 'my-bucket', 'hidden3']) - console_tool.run_command(['b2', 'hide-file', 'my-bucket', 'hidden4']) + console_tool.run_command(['b2', 'file', 'hide', 'my-bucket', 'hidden1']) + console_tool.run_command(['b2', 'file', 'hide', 'my-bucket', 'hidden2']) + console_tool.run_command(['b2', 'file', 'hide', 'my-bucket', 'hidden3']) + console_tool.run_command(['b2', 'file', 'hide', 'my-bucket', 'hidden4']) # Now check the output of `bucket get` against the canon. expected_json = { @@ -2043,13 +2043,13 @@ def test_get_bucket_complex(self): # something has failed if the output of 'bucket get' does not match the canon. stdout, stderr = self._get_stdouterr() console_tool = self.console_tool_class(stdout, stderr) - console_tool.run_command(['b2', 'hide-file', 'my-bucket', '1/hidden1']) - console_tool.run_command(['b2', 'hide-file', 'my-bucket', '1/hidden1']) - console_tool.run_command(['b2', 'hide-file', 'my-bucket', '1/hidden2']) - console_tool.run_command(['b2', 'hide-file', 'my-bucket', '1/2/hidden3']) - console_tool.run_command(['b2', 'hide-file', 'my-bucket', '1/2/hidden3']) - console_tool.run_command(['b2', 'hide-file', 'my-bucket', '1/2/hidden3']) - console_tool.run_command(['b2', 'hide-file', 'my-bucket', '1/2/hidden3']) + console_tool.run_command(['b2', 'file', 'hide', 'my-bucket', '1/hidden1']) + console_tool.run_command(['b2', 'file', 'hide', 'my-bucket', '1/hidden1']) + console_tool.run_command(['b2', 'file', 'hide', 'my-bucket', '1/hidden2']) + console_tool.run_command(['b2', 'file', 'hide', 'my-bucket', '1/2/hidden3']) + console_tool.run_command(['b2', 'file', 'hide', 'my-bucket', '1/2/hidden3']) + console_tool.run_command(['b2', 'file', 'hide', 'my-bucket', '1/2/hidden3']) + console_tool.run_command(['b2', 'file', 'hide', 'my-bucket', '1/2/hidden3']) # Now check the output of `bucket get` against the canon. expected_json = { From 8a076fc8c62615ea4f1c38831cebe08284080c75 Mon Sep 17 00:00:00 2001 From: Adal Chiriliuc Date: Thu, 25 Apr 2024 20:15:26 +0300 Subject: [PATCH 10/22] fixed typo --- b2/_internal/console_tool.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/b2/_internal/console_tool.py b/b2/_internal/console_tool.py index ab2a6a69..3f544915 100644 --- a/b2/_internal/console_tool.py +++ b/b2/_internal/console_tool.py @@ -4645,7 +4645,7 @@ class Replication(Command): """ Replication rule management subcommands. - For more information on each subcommand, use ``{NAME} key SUBCOMMAND --help``. + For more information on each subcommand, use ``{NAME} replication SUBCOMMAND --help``. Examples: @@ -4719,7 +4719,7 @@ class Account(Command): """ Account management subcommands. - For more information on each subcommand, use ``{NAME} key SUBCOMMAND --help``. + For more information on each subcommand, use ``{NAME} account SUBCOMMAND --help``. Examples: @@ -4769,7 +4769,7 @@ class BucketCmd(Command): """ Bucket management subcommands. - For more information on each subcommand, use ``{NAME} key SUBCOMMAND --help``. + For more information on each subcommand, use ``{NAME} bucket SUBCOMMAND --help``. Examples: From 315f77dcc7fef9e6c17e775e97da23454ba3ec7b Mon Sep 17 00:00:00 2001 From: Adal Chiriliuc Date: Thu, 25 Apr 2024 20:15:44 +0300 Subject: [PATCH 11/22] fixed typo --- b2/_internal/console_tool.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/b2/_internal/console_tool.py b/b2/_internal/console_tool.py index 3f544915..50c0fd74 100644 --- a/b2/_internal/console_tool.py +++ b/b2/_internal/console_tool.py @@ -4871,7 +4871,7 @@ class File(Command): """ File management subcommands. - For more information on each subcommand, use ``{NAME} key SUBCOMMAND --help``. + For more information on each subcommand, use ``{NAME} file SUBCOMMAND --help``. Examples: From 7c94990ec709d09e537975ce11f7ce8186048e95 Mon Sep 17 00:00:00 2001 From: Adal Chiriliuc Date: Thu, 25 Apr 2024 22:07:55 +0300 Subject: [PATCH 12/22] added file update subcommand --- b2/_internal/console_tool.py | 87 ++++++++++++++++++++++++- changelog.d/+command-file.added.md | 2 +- changelog.d/+command-file.deprecated.md | 2 +- 3 files changed, 87 insertions(+), 4 deletions(-) diff --git a/b2/_internal/console_tool.py b/b2/_internal/console_tool.py index 50c0fd74..03601841 100644 --- a/b2/_internal/console_tool.py +++ b/b2/_internal/console_tool.py @@ -3525,7 +3525,74 @@ def execute_operation(self, local_file, bucket, threads, **kwargs): return file_version -class UpdateFileLegalHold(FileIdAndOptionalFileNameMixin, Command): +class FileUpdateBase(B2URIFileArgMixin, LegalHoldMixin, Command): + """ + {LegalHoldMixin} + + Retention: + + Setting file retention settings requires the **writeFileRetentions** capability, and only works in bucket + with fileLockEnabled=true. Providing a ``retention-mode`` other than ``none`` requires providing ``retainUntil``, + which has to be a future timestamp in the form of an integer representing milliseconds since epoch. + + If a file already is in governance mode, disabling retention or shortening it's period requires providing + ``--bypass-governance``. + + If a file already is in compliance mode, disabling retention or shortening it's period is impossible. + + In both cases prolonging the retention period is possible. Changing from governance to compliance is also supported. + + {FILE_RETENTION_COMPATIBILITY_WARNING} + + Requires capability: + + - **readFiles** + """ + + @classmethod + def _setup_parser(cls, parser): + super()._setup_parser(parser) + + add_normalized_argument( + parser, + '--file-retention-mode', + default=None, + choices=(RetentionMode.COMPLIANCE.value, RetentionMode.GOVERNANCE.value, 'none') + ) + add_normalized_argument( + parser, + '--retain-until', + type=parse_millis_from_float_timestamp, + metavar='TIMESTAMP', + default=None + ) + add_normalized_argument(parser, '--bypass-governance', action='store_true', default=False) + + def _run(self, args): + b2_uri = self.get_b2_uri_from_arg(args) + file_version = self.api.get_file_info_by_uri(b2_uri) + + if args.legal_hold is not None: + self.api.update_file_legal_hold( + file_version.id_, file_version.file_name, LegalHold(args.legal_hold) + ) + + if args.file_retention_mode is not None: + if args.file_retention_mode == 'none': + file_retention = FileRetentionSetting(RetentionMode.NONE) + else: + file_retention = FileRetentionSetting( + RetentionMode(args.file_retention_mode), args.retain_until + ) + + self.api.update_file_retention( + file_version.id_, file_version.file_name, file_retention, args.bypass_governance + ) + + return 0 + + +class UpdateFileLegalHoldBase(FileIdAndOptionalFileNameMixin, Command): """ Only works in buckets with fileLockEnabled=true. @@ -3550,7 +3617,7 @@ def _run(self, args): return 0 -class UpdateFileRetention(FileIdAndOptionalFileNameMixin, Command): +class UpdateFileRetentionBase(FileIdAndOptionalFileNameMixin, Command): """ Only works in buckets with fileLockEnabled=true. Providing a ``retention-mode`` other than ``none`` requires providing ``retainUntil``, which has to be a future timestamp in the form of an integer representing milliseconds @@ -4931,6 +4998,12 @@ class FileHide(FileHideBase): COMMAND_NAME = 'hide' +@File.subcommands_registry.register +class FileUpdate(FileUpdateBase): + __doc__ = FileUpdateBase.__doc__ + COMMAND_NAME = 'update' + + class FileInfo2(CmdReplacedByMixin, B2URIFileArgMixin, FileInfoBase): __doc__ = FileInfoBase.__doc__ replaced_by_cmd = (File, FileInfo) @@ -4993,6 +5066,16 @@ class HideFile(CmdReplacedByMixin, FileHideBase): replaced_by_cmd = (File, FileHide) +class UpdateFileLegalHold(CmdReplacedByMixin, UpdateFileLegalHoldBase): + __doc__ = UpdateFileLegalHoldBase.__doc__ + replaced_by_cmd = (File, FileUpdate) + + +class UpdateFileRetention(CmdReplacedByMixin, UpdateFileRetentionBase): + __doc__ = UpdateFileRetentionBase.__doc__ + replaced_by_cmd = (File, FileUpdate) + + class ConsoleTool: """ Implements the commands available in the B2 command-line tool diff --git a/changelog.d/+command-file.added.md b/changelog.d/+command-file.added.md index 71b2be3d..31eeba6b 100644 --- a/changelog.d/+command-file.added.md +++ b/changelog.d/+command-file.added.md @@ -1 +1 @@ -Add `file {info|url|cat|upload|download|copy-by-id|hide}` commands. \ No newline at end of file +Add `file {info|url|cat|upload|download|copy-by-id|hide|update}` commands. \ No newline at end of file diff --git a/changelog.d/+command-file.deprecated.md b/changelog.d/+command-file.deprecated.md index 6d0a284e..baad1025 100644 --- a/changelog.d/+command-file.deprecated.md +++ b/changelog.d/+command-file.deprecated.md @@ -1 +1 @@ -Deprecated `file-info`, `get-url`, `cat`, `upload-file`, `download-file`, `copy-file-by-id` and `hide-file`, use `file {info|url|cat|upload|download|copy-by-id|hide}` instead. \ No newline at end of file +Deprecated `file-info`, `get-url`, `cat`, `upload-file`, `download-file`, `copy-file-by-id`, `hide-file`, `update-file-legal-hold` and `update-file-retention`, use `file {info|url|cat|upload|download|copy-by-id|hide|update}` instead. \ No newline at end of file From df038612153987045389d063020fde8d769205fe Mon Sep 17 00:00:00 2001 From: Adal Chiriliuc Date: Thu, 25 Apr 2024 23:12:55 +0300 Subject: [PATCH 13/22] updated tests --- test/integration/test_b2_command_line.py | 61 +++++++++++++++--------- 1 file changed, 38 insertions(+), 23 deletions(-) diff --git a/test/integration/test_b2_command_line.py b/test/integration/test_b2_command_line.py index 5c0824f1..1423edba 100755 --- a/test/integration/test_b2_command_line.py +++ b/test/integration/test_b2_command_line.py @@ -1996,6 +1996,7 @@ def test_file_lock( ['file', 'upload', '--no-progress', '--quiet', lock_enabled_bucket_name, sample_file, 'a'] ) + # deprecated command b2_tool.should_fail( [ 'update-file-retention', not_lockable_file['fileName'], not_lockable_file['fileId'], @@ -2004,11 +2005,21 @@ def test_file_lock( ], r'ERROR: The bucket is not file lock enabled \(bucket_missing_file_lock\)' ) + # deprecated command + update_file_retention_deprecated_pattern = re.compile( + re.escape( + 'WARNING: `update-file-retention` command is deprecated. Use `file update` instead.' + ) + ) b2_tool.should_succeed( # first let's try with a file name ['update-file-retention', lockable_file['fileName'], lockable_file['fileId'], 'governance', - '--retain-until', str(now_millis + ONE_DAY_MILLIS + ONE_HOUR_MILLIS)] + '--retain-until', str(now_millis + ONE_DAY_MILLIS + ONE_HOUR_MILLIS)], + expected_stderr_pattern=update_file_retention_deprecated_pattern, ) + lockable_b2uri = f"b2://{lock_enabled_bucket_name}/{lockable_file['fileName']}" + not_lockable_b2uri = f"b2://{lock_disabled_bucket_name}/{not_lockable_file['fileName']}" + _assert_file_lock_configuration( b2_tool, lockable_file['fileId'], @@ -2017,8 +2028,8 @@ def test_file_lock( ) b2_tool.should_succeed( # and now without a file name - ['update-file-retention', lockable_file['fileId'], 'governance', - '--retain-until', str(now_millis + ONE_DAY_MILLIS + 2 * ONE_HOUR_MILLIS)] + ['file', 'update', '--file-retention-mode', 'governance', + '--retain-until', str(now_millis + ONE_DAY_MILLIS + 2 * ONE_HOUR_MILLIS), lockable_b2uri], ) _assert_file_lock_configuration( @@ -2030,18 +2041,16 @@ def test_file_lock( b2_tool.should_fail( [ - 'update-file-retention', lockable_file['fileName'], lockable_file['fileId'], - 'governance', '--retain-until', - str(now_millis + ONE_HOUR_MILLIS) + 'file', 'update', '--file-retention-mode', 'governance', '--retain-until', + str(now_millis + ONE_HOUR_MILLIS), lockable_b2uri ], "ERROR: Auth token not authorized to write retention or file already in 'compliance' mode or " "bypassGovernance=true parameter missing", ) b2_tool.should_succeed( [ - 'update-file-retention', lockable_file['fileName'], lockable_file['fileId'], - 'governance', '--retain-until', - str(now_millis + ONE_HOUR_MILLIS), '--bypass-governance' + 'file', 'update', '--file-retention-mode', 'governance', '--retain-until', + str(now_millis + ONE_HOUR_MILLIS), '--bypass-governance', lockable_b2uri ], ) @@ -2053,15 +2062,12 @@ def test_file_lock( ) b2_tool.should_fail( - ['update-file-retention', lockable_file['fileName'], lockable_file['fileId'], 'none'], + ['file', 'update', '--file-retention-mode', 'none', lockable_b2uri], "ERROR: Auth token not authorized to write retention or file already in 'compliance' mode or " "bypassGovernance=true parameter missing", ) b2_tool.should_succeed( - [ - 'update-file-retention', lockable_file['fileName'], lockable_file['fileId'], 'none', - '--bypass-governance' - ], + ['file', 'update', '--file-retention-mode', 'none', '--bypass-governance', lockable_b2uri], ) _assert_file_lock_configuration( @@ -2069,18 +2075,25 @@ def test_file_lock( ) b2_tool.should_fail( - ['update-file-legal-hold', not_lockable_file['fileId'], 'on'], + ['file', 'update', '--legal-hold', 'on', not_lockable_b2uri], r'ERROR: The bucket is not file lock enabled \(bucket_missing_file_lock\)' ) + # deprecated command + update_file_legal_hold_deprecated_pattern = re.compile( + re.escape( + 'WARNING: `update-file-legal-hold` command is deprecated. Use `file update` instead.' + ) + ) b2_tool.should_succeed( # first let's try with a file name ['update-file-legal-hold', lockable_file['fileName'], lockable_file['fileId'], 'on'], + expected_stderr_pattern=update_file_legal_hold_deprecated_pattern, ) _assert_file_lock_configuration(b2_tool, lockable_file['fileId'], legal_hold=LegalHold.ON) b2_tool.should_succeed( # and now without a file name - ['update-file-legal-hold', lockable_file['fileId'], 'off'], + ['file', 'update', '--legal-hold', 'off', lockable_b2uri], ) _assert_file_lock_configuration(b2_tool, lockable_file['fileId'], legal_hold=LegalHold.OFF) @@ -2194,6 +2207,8 @@ def test_file_lock( lock_disabled_bucket_name, lockable_file['fileId'], not_lockable_file['fileId'], + lockable_b2uri, + not_lockable_b2uri, sample_file=sample_file ) @@ -2225,7 +2240,7 @@ def make_lock_disabled_key(b2_tool): def file_lock_without_perms_test( b2_tool, lock_enabled_bucket_name, lock_disabled_bucket_name, lockable_file_id, - not_lockable_file_id, sample_file + not_lockable_file_id, lockable_b2uri, not_lockable_b2uri, sample_file ): b2_tool.should_fail( @@ -2245,8 +2260,8 @@ def file_lock_without_perms_test( b2_tool.should_fail( [ - 'update-file-retention', lockable_file_id, 'governance', '--retain-until', - str(current_time_millis() + 7 * ONE_DAY_MILLIS) + 'file', 'update', '--file-retention-mode', 'governance', '--retain-until', + str(current_time_millis() + 7 * ONE_DAY_MILLIS), lockable_b2uri ], "ERROR: Auth token not authorized to write retention or file already in 'compliance' mode or " "bypassGovernance=true parameter missing", @@ -2254,21 +2269,21 @@ def file_lock_without_perms_test( b2_tool.should_fail( [ - 'update-file-retention', not_lockable_file_id, 'governance', '--retain-until', - str(current_time_millis() + 7 * ONE_DAY_MILLIS) + 'file', 'update', '--file-retention-mode', 'governance', '--retain-until', + str(current_time_millis() + 7 * ONE_DAY_MILLIS), not_lockable_b2uri ], "ERROR: Auth token not authorized to write retention or file already in 'compliance' mode or " "bypassGovernance=true parameter missing", ) b2_tool.should_fail( - ['update-file-legal-hold', lockable_file_id, 'on'], + ['file', 'update', '--legal-hold', 'on', lockable_b2uri], "ERROR: Auth token not authorized to write retention or file already in 'compliance' mode or " "bypassGovernance=true parameter missing", ) b2_tool.should_fail( - ['update-file-legal-hold', not_lockable_file_id, 'on'], + ['file', 'update', '--legal-hold', 'on', not_lockable_b2uri], "ERROR: Auth token not authorized to write retention or file already in 'compliance' mode or " "bypassGovernance=true parameter missing", ) From 91b5724c7f76c256f311a9bce760e276d50eb001 Mon Sep 17 00:00:00 2001 From: Adal Chiriliuc Date: Thu, 25 Apr 2024 23:33:04 +0300 Subject: [PATCH 14/22] updated doc string --- b2/_internal/console_tool.py | 1 + 1 file changed, 1 insertion(+) diff --git a/b2/_internal/console_tool.py b/b2/_internal/console_tool.py index 03601841..79bd6b0d 100644 --- a/b2/_internal/console_tool.py +++ b/b2/_internal/console_tool.py @@ -4951,6 +4951,7 @@ class File(Command): {NAME} file download {NAME} file copy-by-id {NAME} file hide + {NAME} file update """ subcommands_registry = ClassRegistry(attr_name='COMMAND_NAME') From b10f3fb0752efc5be62fe9167a8355ca2264b653 Mon Sep 17 00:00:00 2001 From: Adal Chiriliuc Date: Fri, 26 Apr 2024 14:26:38 +0300 Subject: [PATCH 15/22] updated file update doc string --- b2/_internal/console_tool.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/b2/_internal/console_tool.py b/b2/_internal/console_tool.py index 79bd6b0d..8048c812 100644 --- a/b2/_internal/console_tool.py +++ b/b2/_internal/console_tool.py @@ -3527,13 +3527,13 @@ def execute_operation(self, local_file, bucket, threads, **kwargs): class FileUpdateBase(B2URIFileArgMixin, LegalHoldMixin, Command): """ - {LegalHoldMixin} + Setting legal holds only works in bucket with fileLockEnabled=true. Retention: - Setting file retention settings requires the **writeFileRetentions** capability, and only works in bucket - with fileLockEnabled=true. Providing a ``retention-mode`` other than ``none`` requires providing ``retainUntil``, - which has to be a future timestamp in the form of an integer representing milliseconds since epoch. + Only works in bucket with fileLockEnabled=true. Providing a ``retention-mode`` other than ``none`` requires + providing ``retainUntil``, which has to be a future timestamp in the form of an integer representing milliseconds + since epoch. If a file already is in governance mode, disabling retention or shortening it's period requires providing ``--bypass-governance``. @@ -3547,6 +3547,9 @@ class FileUpdateBase(B2URIFileArgMixin, LegalHoldMixin, Command): Requires capability: - **readFiles** + - **writeFileLegalHolds** (if updating legal holds) + - **writeFileRetentions** (if updating retention) + - **bypassGovernance** (if --bypass-governance is used) """ @classmethod From 2895c67b6367fed4706381af02d39c21da7518de Mon Sep 17 00:00:00 2001 From: Adal Chiriliuc Date: Fri, 26 Apr 2024 15:37:19 +0300 Subject: [PATCH 16/22] deprecated commands unit tests --- test/unit/console_tool/test_download_file.py | 15 ++++++++ test/unit/console_tool/test_get_url.py | 8 ++++ test/unit/console_tool/test_upload_file.py | 24 ++++++++++++ test/unit/test_console_tool.py | 40 +++++++++++++++++++- 4 files changed, 86 insertions(+), 1 deletion(-) diff --git a/test/unit/console_tool/test_download_file.py b/test/unit/console_tool/test_download_file.py index 6ff3c7bf..5bb573cd 100644 --- a/test/unit/console_tool/test_download_file.py +++ b/test/unit/console_tool/test_download_file.py @@ -46,6 +46,15 @@ def test_download_file_by_uri__flag_support(b2_cli, uploaded_file, tmp_path, fla ) assert output_path.read_text() == uploaded_file['content'] + b2_cli.run( + ['download-file', flag, 'b2id://9999', + str(output_path)], + expected_stderr= + 'WARNING: `download-file` command is deprecated. Use `file download` instead.\n', + expected_stdout=expected_stdout.format(output_path=pathlib.Path(output_path).resolve()) + ) + assert output_path.read_text() == uploaded_file['content'] + @pytest.mark.parametrize('b2_uri', [ 'b2://my-bucket/file1.txt', @@ -191,6 +200,12 @@ def test_cat__b2id_uri(b2_cli, bucket, uploaded_stdout_txt, tmp_path, capfd): b2_cli.run(['file', 'cat', '--no-progress', "b2id://9999"],) assert capfd.readouterr().out == uploaded_stdout_txt['content'] + b2_cli.run( + ['cat', '--no-progress', "b2id://9999"], + expected_stderr='WARNING: `cat` command is deprecated. Use `file cat` instead.\n' + ) + assert capfd.readouterr().out == uploaded_stdout_txt['content'] + def test__download_file__threads(b2_cli, local_file, uploaded_file, tmp_path): num_threads = 13 diff --git a/test/unit/console_tool/test_get_url.py b/test/unit/console_tool/test_get_url.py index 981d274f..a2140585 100644 --- a/test/unit/console_tool/test_get_url.py +++ b/test/unit/console_tool/test_get_url.py @@ -20,6 +20,14 @@ def uploaded_file_url_by_id(uploaded_file): return f"http://download.example.com/b2api/v2/b2_download_file_by_id?fileId={uploaded_file['fileId']}" +def test_get_url(b2_cli, uploaded_file, uploaded_file_url_by_id): + b2_cli.run( + ["get-url", f"b2id://{uploaded_file['fileId']}"], + expected_stdout=f"{uploaded_file_url_by_id}\n", + expected_stderr='WARNING: `get-url` command is deprecated. Use `file url` instead.\n', + ) + + def test_make_url(b2_cli, uploaded_file, uploaded_file_url_by_id): b2_cli.run( ["make-url", uploaded_file["fileId"]], diff --git a/test/unit/console_tool/test_upload_file.py b/test/unit/console_tool/test_upload_file.py index 029d2736..6f4a65e8 100644 --- a/test/unit/console_tool/test_upload_file.py +++ b/test/unit/console_tool/test_upload_file.py @@ -127,6 +127,30 @@ def test_upload_file__stdin(b2_cli, bucket, tmpdir, mock_stdin): ) +def test_upload_file_deprecated__stdin(b2_cli, bucket, tmpdir, mock_stdin): + """Test `upload-file` stdin alias support""" + content = "stdin input deprecated" + filename = 'stdin-deprecated.txt' + + expected_stdout = f'URL by file name: http://download.example.com/file/my-bucket/{filename}' + expected_json = { + "action": "upload", + "contentSha1": "fcaa935e050efe0b5d7b26e65162b32b5e40aa81", + "fileName": filename, + "size": len(content), + } + mock_stdin.write(content) + mock_stdin.close() + + b2_cli.run( + ['upload-file', '--no-progress', 'my-bucket', '-', filename], + expected_stderr='WARNING: `upload-file` command is deprecated. Use `file upload` instead.\n', + expected_json_in_stdout=expected_json, + remove_version=True, + expected_part_of_stdout=expected_stdout, + ) + + def test_upload_file__threads_setting(b2_cli, bucket, tmp_path): """Test `file upload` supports setting number of threads""" num_threads = 66 diff --git a/test/unit/test_console_tool.py b/test/unit/test_console_tool.py index 73462221..e0dc837f 100644 --- a/test/unit/test_console_tool.py +++ b/test/unit/test_console_tool.py @@ -1216,6 +1216,20 @@ def test_files_encrypted(self): expected_json_in_stdout=expected_json, ) + self._run_command( + ['file-info', 'b2id://9999'], + expected_stderr= + 'WARNING: `file-info` command is deprecated. Use `file info` instead.\n', + expected_json_in_stdout=expected_json, + ) + + self._run_command( + ['get-file-info', '9999'], + expected_stderr= + 'WARNING: `get-file-info` command is deprecated. Use `file info` instead.\n', + expected_json_in_stdout=expected_json, + ) + # Download by name local_download1 = os.path.join(temp_dir, 'download1.txt') expected_stdout_template = ''' @@ -1576,6 +1590,30 @@ def test_copy_file_by_id(self): expected_json_in_stdout=expected_json, ) + expected_json = { + "accountId": self.account_id, + "action": "copy", + "bucketId": "bucket_1", + "size": 11, + "contentSha1": "2aae6c35c94fcfb415dbe95f408b9ce91ee846ed", + "contentType": "b2/x-auto", + "fileId": "9993", + "fileInfo": { + "src_last_modified_millis": "1500111222000" + }, + "fileName": "file1_copy_2.txt", + "serverSideEncryption": { + "mode": "none" + }, + "uploadTimestamp": 5005 + } + self._run_command( + ['copy-file-by-id', '9999', 'my-bucket1', 'file1_copy_2.txt'], + expected_stderr= + 'WARNING: `copy-file-by-id` command is deprecated. Use `file copy-by-id` instead.\n', + expected_json_in_stdout=expected_json, + ) + def test_get_download_auth_defaults(self): self._authorize_account() self._create_my_bucket() @@ -1985,7 +2023,7 @@ def test_get_bucket_with_hidden(self): console_tool.run_command(['b2', 'file', 'hide', 'my-bucket', 'hidden1']) console_tool.run_command(['b2', 'file', 'hide', 'my-bucket', 'hidden2']) console_tool.run_command(['b2', 'file', 'hide', 'my-bucket', 'hidden3']) - console_tool.run_command(['b2', 'file', 'hide', 'my-bucket', 'hidden4']) + console_tool.run_command(['b2', 'hide-file', 'my-bucket', 'hidden4']) # Now check the output of `bucket get` against the canon. expected_json = { From 68be0b88d5f21e34bf75886a1d9acf03e496700a Mon Sep 17 00:00:00 2001 From: Adal Chiriliuc Date: Fri, 26 Apr 2024 16:30:49 +0300 Subject: [PATCH 17/22] updated doc string --- b2/_internal/console_tool.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/b2/_internal/console_tool.py b/b2/_internal/console_tool.py index 8048c812..ce9c94de 100644 --- a/b2/_internal/console_tool.py +++ b/b2/_internal/console_tool.py @@ -4947,14 +4947,14 @@ class File(Command): .. code-block:: - {NAME} file info - {NAME} file url - {NAME} file cat - {NAME} file upload - {NAME} file download - {NAME} file copy-by-id - {NAME} file hide - {NAME} file update + {NAME} file info b2://yourBucket/file.txt + {NAME} file url b2://yourBucket/file.txt + {NAME} file cat b2://yourBucket/file.txt + {NAME} file upload yourBucket localFile.txt file.txt + {NAME} file download b2://yourBucket/file.txt localFile.txt + {NAME} file copy-by-id sourceFileId yourBucket file.txt + {NAME} file hide yourBucket file.txt + {NAME} file update --legal-hold off b2://yourBucket/file.txt """ subcommands_registry = ClassRegistry(attr_name='COMMAND_NAME') From f54dbd2484fda3ee1aeb5a7ca4f323a065c10c92 Mon Sep 17 00:00:00 2001 From: Adal Chiriliuc Date: Fri, 26 Apr 2024 18:55:01 +0300 Subject: [PATCH 18/22] fixed hyphen in command name --- b2/_internal/console_tool.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/b2/_internal/console_tool.py b/b2/_internal/console_tool.py index ce9c94de..897f01d4 100644 --- a/b2/_internal/console_tool.py +++ b/b2/_internal/console_tool.py @@ -901,7 +901,9 @@ def make_progress_listener(self, file_name: str, quiet: bool): @classmethod def name_and_alias(cls): - name = mixed_case_to_hyphens(cls.COMMAND_NAME or cls.__name__) + name = cls.COMMAND_NAME or cls.__name__ + if '-' not in name: + name = mixed_case_to_hyphens(name) alias = None if '-' in name: alias = name.replace('-', '_') @@ -4891,15 +4893,13 @@ class BucketDelete(BucketDeleteBase): @BucketCmd.subcommands_registry.register class BucketGetDownloadAuth(BucketGetDownloadAuthBase): __doc__ = BucketGetDownloadAuthBase.__doc__ - # TODO we can't use 'get-download-auth', gets transformed to 'get--download--auth' - COMMAND_NAME = 'GetDownloadAuth' + COMMAND_NAME = 'get-download-auth' @BucketCmd.subcommands_registry.register class BucketNotificationRule(BucketNotificationRuleBase): __doc__ = BucketNotificationRuleBase.__doc__ - # TODO we can't use 'notification-rule', gets transformed to 'notification--rule' - COMMAND_NAME = 'NotificationRule' + COMMAND_NAME = 'notification-rule' class ListBuckets(CmdReplacedByMixin, BucketListBase): @@ -4992,8 +4992,7 @@ class FileDownload(B2URIFileArgMixin, FileDownloadBase): @File.subcommands_registry.register class FileCopyById(FileCopyByIdBase): __doc__ = FileCopyByIdBase.__doc__ - # TODO we can't use 'copy-by-id', gets transformed to 'copy--by--id' - COMMAND_NAME = 'CopyById' + COMMAND_NAME = 'copy-by-id' @File.subcommands_registry.register @@ -5011,8 +5010,7 @@ class FileUpdate(FileUpdateBase): class FileInfo2(CmdReplacedByMixin, B2URIFileArgMixin, FileInfoBase): __doc__ = FileInfoBase.__doc__ replaced_by_cmd = (File, FileInfo) - # TODO we can't use 'file-info', gets transformed to 'file--info' - COMMAND_NAME = 'FileInfo' + COMMAND_NAME = 'file-info' class GetFileInfo(CmdReplacedByMixin, B2URIFileIDArgMixin, FileInfoBase): From 3fd6eed039f7a0558ab15a591fbd0d5e49e2a90a Mon Sep 17 00:00:00 2001 From: Adal Chiriliuc Date: Fri, 26 Apr 2024 18:56:04 +0300 Subject: [PATCH 19/22] sort command help lines --- b2/_internal/console_tool.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/b2/_internal/console_tool.py b/b2/_internal/console_tool.py index 897f01d4..0221b2d1 100644 --- a/b2/_internal/console_tool.py +++ b/b2/_internal/console_tool.py @@ -4947,14 +4947,14 @@ class File(Command): .. code-block:: - {NAME} file info b2://yourBucket/file.txt - {NAME} file url b2://yourBucket/file.txt {NAME} file cat b2://yourBucket/file.txt - {NAME} file upload yourBucket localFile.txt file.txt - {NAME} file download b2://yourBucket/file.txt localFile.txt {NAME} file copy-by-id sourceFileId yourBucket file.txt + {NAME} file download b2://yourBucket/file.txt localFile.txt {NAME} file hide yourBucket file.txt + {NAME} file info b2://yourBucket/file.txt {NAME} file update --legal-hold off b2://yourBucket/file.txt + {NAME} file upload yourBucket localFile.txt file.txt + {NAME} file url b2://yourBucket/file.txt """ subcommands_registry = ClassRegistry(attr_name='COMMAND_NAME') From b17e93c14e17240e94048d2def539be75fefdb65 Mon Sep 17 00:00:00 2001 From: Adal Chiriliuc Date: Fri, 26 Apr 2024 23:35:08 +0300 Subject: [PATCH 20/22] workaround to hide command aliases in usage --- b2/_internal/arg_parser.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/b2/_internal/arg_parser.py b/b2/_internal/arg_parser.py index f869af22..c13dc5ba 100644 --- a/b2/_internal/arg_parser.py +++ b/b2/_internal/arg_parser.py @@ -149,6 +149,31 @@ def print_help(self, *args, show_all: bool = False, **kwargs): ): super().print_help(*args, **kwargs) + def format_usage(self): + # TODO We don't want to list underscore aliases subcommands in the usage. + # Unfortunately the only way found was to temporarily remove the aliases, + # print the usage and then restore the aliases since the formatting is deep + # inside the Python argparse module. + # We restore the original dictionary which we don't modify, just in case + # someone else has taken a reference to it. + subparsers_action = None + original_choices = None + if self._subparsers is not None: + for action in self._subparsers._actions: + if isinstance(action, argparse._SubParsersAction): + subparsers_action = action + original_choices = action.choices + action.choices = { + key: choice + for key, choice in action.choices.items() if "_" not in key + } + # only one subparser supported + break + usage = super().format_usage() + if subparsers_action is not None: + subparsers_action.choices = original_choices + return usage + SUPPORT_CAMEL_CASE_ARGUMENTS = False From aaefec9d7589170a3ed5275b3d8f8adbb3b9d0b5 Mon Sep 17 00:00:00 2001 From: Adal Chiriliuc Date: Mon, 29 Apr 2024 20:25:08 +0300 Subject: [PATCH 21/22] deprecated download-url-with-auth and replaced with file url --- b2/_internal/console_tool.py | 38 ++++++++++++++++++- ...d-get-download-url-with-auth.deprecated.md | 1 + test/unit/test_console_tool.py | 12 ++++++ 3 files changed, 49 insertions(+), 2 deletions(-) create mode 100644 changelog.d/+command-get-download-url-with-auth.deprecated.md diff --git a/b2/_internal/console_tool.py b/b2/_internal/console_tool.py index 0221b2d1..7447958c 100644 --- a/b2/_internal/console_tool.py +++ b/b2/_internal/console_tool.py @@ -2079,7 +2079,7 @@ def _run(self, args): return 0 -class GetDownloadUrlWithAuth(Command): +class GetDownloadUrlWithAuthBase(Command): """ Prints a URL to download the given file. The URL includes an authorization token that allows downloads from the given bucket for files whose names @@ -2745,11 +2745,40 @@ class FileUrlBase(Command): """ Prints an URL that can be used to download the given file, if it is public. + + If it is private, you can use --with-auth to include an authorization + token in the URL that allows downloads from the given bucket for files + whose names start with the given file name. + + The URL will work for the given file, but is not specific to that file. Files + with longer names that start with the give file name can also be downloaded + with the same auth token. + + The token is valid for the duration specified, which defaults + to 86400 seconds (one day). + + + Requires capability: + + - **shareFiles** (if using --with-auth) """ + @classmethod + def _setup_parser(cls, parser): + add_normalized_argument(parser, '--with-auth', action='store_true') + parser.add_argument('--duration', type=int, default=86400) + super()._setup_parser(parser) + def _run(self, args): b2_uri = self.get_b2_uri_from_arg(args) - self._print(self.api.get_download_url_by_uri(b2_uri)) + url = self.api.get_download_url_by_uri(b2_uri) + if args.with_auth: + bucket = self.api.get_bucket_by_name(b2_uri.bucket_name) + auth_token = bucket.get_download_authorization( + file_name_prefix=b2_uri.path, valid_duration_in_seconds=args.duration + ) + url += '?Authorization=' + auth_token + self._print(url) return 0 @@ -5078,6 +5107,11 @@ class UpdateFileRetention(CmdReplacedByMixin, UpdateFileRetentionBase): replaced_by_cmd = (File, FileUpdate) +class GetDownloadUrlWithAuth(CmdReplacedByMixin, GetDownloadUrlWithAuthBase): + __doc__ = GetDownloadUrlWithAuthBase.__doc__ + replaced_by_cmd = (File, FileUrl) + + class ConsoleTool: """ Implements the commands available in the B2 command-line tool diff --git a/changelog.d/+command-get-download-url-with-auth.deprecated.md b/changelog.d/+command-get-download-url-with-auth.deprecated.md new file mode 100644 index 00000000..554a1edf --- /dev/null +++ b/changelog.d/+command-get-download-url-with-auth.deprecated.md @@ -0,0 +1 @@ +Deprecated `get-download-url-with-auth`, use `file url` instead. Added `--with-auth` and `--duration` options to `file url`. \ No newline at end of file diff --git a/test/unit/test_console_tool.py b/test/unit/test_console_tool.py index e0dc837f..cb3645df 100644 --- a/test/unit/test_console_tool.py +++ b/test/unit/test_console_tool.py @@ -1638,6 +1638,12 @@ def test_get_download_auth_url(self): self._run_command( ['get-download-url-with-auth', '--duration', '12345', 'my-bucket', 'my-file'], 'http://download.example.com/file/my-bucket/my-file?Authorization=fake_download_auth_token_bucket_0_my-file_12345\n', + 'WARNING: `get-download-url-with-auth` command is deprecated. Use `file url` instead.\n', + 0 + ) + self._run_command( + ['file', 'url', '--with-auth', '--duration', '12345', 'b2://my-bucket/my-file'], + 'http://download.example.com/file/my-bucket/my-file?Authorization=fake_download_auth_token_bucket_0_my-file_12345\n', '', 0 ) @@ -1647,6 +1653,12 @@ def test_get_download_auth_url_with_encoding(self): self._run_command( ['get-download-url-with-auth', '--duration', '12345', 'my-bucket', '\u81ea'], 'http://download.example.com/file/my-bucket/%E8%87%AA?Authorization=fake_download_auth_token_bucket_0_%E8%87%AA_12345\n', + 'WARNING: `get-download-url-with-auth` command is deprecated. Use `file url` instead.\n', + 0 + ) + self._run_command( + ['file', 'url', '--with-auth', '--duration', '12345', 'b2://my-bucket/\u81ea'], + 'http://download.example.com/file/my-bucket/%E8%87%AA?Authorization=fake_download_auth_token_bucket_0_%E8%87%AA_12345\n', '', 0 ) From e294c7a904166c0ed1d39392151bea945bf213f0 Mon Sep 17 00:00:00 2001 From: Adal Chiriliuc Date: Mon, 29 Apr 2024 23:48:43 +0300 Subject: [PATCH 22/22] deprecate delete-file-version --- b2/_internal/b2v3/rm.py | 1 + b2/_internal/console_tool.py | 13 ++- ...+command-delete-file-version.deprecated.md | 1 + test/integration/test_b2_command_line.py | 97 +++++++++++++++++-- test/unit/test_console_tool.py | 47 ++++++++- 5 files changed, 147 insertions(+), 12 deletions(-) create mode 100644 changelog.d/+command-delete-file-version.deprecated.md diff --git a/b2/_internal/b2v3/rm.py b/b2/_internal/b2v3/rm.py index bd33480a..841dfd72 100644 --- a/b2/_internal/b2v3/rm.py +++ b/b2/_internal/b2v3/rm.py @@ -56,4 +56,5 @@ class Rm(B2URIBucketNFolderNameArgMixin, BaseRm): - **listFiles** - **deleteFiles** + - **bypassGovernance** (if --bypass-governance is used) """ diff --git a/b2/_internal/console_tool.py b/b2/_internal/console_tool.py index 7447958c..7da72e73 100644 --- a/b2/_internal/console_tool.py +++ b/b2/_internal/console_tool.py @@ -1694,7 +1694,7 @@ def _run(self, args): return 0 -class DeleteFileVersion(FileIdAndOptionalFileNameMixin, Command): +class DeleteFileVersionBase(FileIdAndOptionalFileNameMixin, Command): """ Permanently and irrevocably deletes one version of a file. @@ -2535,7 +2535,8 @@ class BaseRm(ThreadsMixin, AbstractLsCommand, metaclass=ABCMeta): example if a file matching a pattern is uploaded during a run of ``rm`` command, it MIGHT be deleted (as "latest") instead of the one present when the ``rm`` run has started. - In order to safely delete a single file version, please use ``delete-file-version``. + If a file is in governance retention mode, and the retention period has not expired, + adding --bypass-governance is required. To list (but not remove) files to be deleted, use ``--dry-run``. You can also list files via ``ls`` command - the listing behaviour is exactly the same. @@ -2617,6 +2618,7 @@ def _run_removal(self, executor: Executor): self.runner.api.delete_file_version, file_version.id_, file_version.file_name, + self.args.bypass_governance, ) with self.mapping_lock: self.futures_mapping[future] = file_version @@ -2649,6 +2651,7 @@ def _removal_done(self, future: Future) -> None: @classmethod def _setup_parser(cls, parser): + add_normalized_argument(parser, '--bypass-governance', action='store_true', default=False) add_normalized_argument(parser, '--dry-run', action='store_true') add_normalized_argument(parser, '--queue-size', @@ -2738,6 +2741,7 @@ class Rm(B2IDOrB2URIMixin, BaseRm): - **listFiles** - **deleteFiles** + - **bypassGovernance** (if --bypass-governance is used) """ @@ -5112,6 +5116,11 @@ class GetDownloadUrlWithAuth(CmdReplacedByMixin, GetDownloadUrlWithAuthBase): replaced_by_cmd = (File, FileUrl) +class DeleteFileVersion(CmdReplacedByMixin, DeleteFileVersionBase): + __doc__ = DeleteFileVersionBase.__doc__ + replaced_by_cmd = Rm + + class ConsoleTool: """ Implements the commands available in the B2 command-line tool diff --git a/changelog.d/+command-delete-file-version.deprecated.md b/changelog.d/+command-delete-file-version.deprecated.md new file mode 100644 index 00000000..39c652db --- /dev/null +++ b/changelog.d/+command-delete-file-version.deprecated.md @@ -0,0 +1 @@ +Deprecated `delete-file-version`, use `rm` instead. Added `--bypass-governance` option to `rm`. \ No newline at end of file diff --git a/test/integration/test_b2_command_line.py b/test/integration/test_b2_command_line.py index 1423edba..f6f8d484 100755 --- a/test/integration/test_b2_command_line.py +++ b/test/integration/test_b2_command_line.py @@ -412,7 +412,12 @@ def test_basic(b2_tool, bucket_name, sample_file, tmp_path, b2_uri_args): } should_equal(expected_info, file_info['fileInfo']) - b2_tool.should_succeed(['delete-file-version', 'c', first_c_version['fileId']]) + b2_tool.should_succeed( + ['delete-file-version', 'c', first_c_version['fileId']], + expected_stderr_pattern=re.compile( + re.escape('WARNING: `delete-file-version` command is deprecated. Use `rm` instead.') + ) + ) b2_tool.should_succeed( ['ls', *b2_uri_args(bucket_name)], f'^a{os.linesep}b/{os.linesep}c{os.linesep}d{os.linesep}' ) @@ -2392,11 +2397,14 @@ def deleting_locked_files( "ERROR: Access Denied for application key " ) b2_tool.should_succeed([ # master key - 'delete-file-version', - locked_file['fileName'], - locked_file['fileId'], - '--bypass-governance' - ]) + 'delete-file-version', + locked_file['fileName'], + locked_file['fileId'], + '--bypass-governance' + ], expected_stderr_pattern=re.compile(re.escape( + 'WARNING: `delete-file-version` command is deprecated. Use `rm` instead.' + )) + ) locked_file = upload_locked_file(b2_tool, lock_enabled_bucket_name, sample_file) @@ -2414,6 +2422,76 @@ def deleting_locked_files( ], "ERROR: unauthorized for application key with capabilities '") +@pytest.mark.apiver(from_ver=4) +def test_deleting_locked_files_v4(b2_tool, sample_file, schedule_bucket_cleanup): + lock_enabled_bucket_name = b2_tool.generate_bucket_name() + schedule_bucket_cleanup(lock_enabled_bucket_name) + b2_tool.should_succeed( + [ + 'bucket', + 'create', + lock_enabled_bucket_name, + 'allPrivate', + '--file-lock-enabled', + *b2_tool.get_bucket_info_args(), + ], + ) + updated_bucket = b2_tool.should_succeed_json( + [ + 'bucket', + 'update', + lock_enabled_bucket_name, + 'allPrivate', + '--default-retention-mode', + 'governance', + '--default-retention-period', + '1 days', + ], + ) + assert updated_bucket['defaultRetention'] == { + 'mode': 'governance', + 'period': { + 'duration': 1, + 'unit': 'days', + }, + } + + locked_file = upload_locked_file(b2_tool, lock_enabled_bucket_name, sample_file) + b2_tool.should_fail( + [ # master key + 'rm', + f"b2id://{locked_file['fileId']}", + ], + " failed: Access Denied for application key " + ) + b2_tool.should_succeed( + [ # master key + 'rm', + '--bypass-governance', + f"b2id://{locked_file['fileId']}", + ] + ) + + locked_file = upload_locked_file(b2_tool, lock_enabled_bucket_name, sample_file) + + lock_disabled_key_id, lock_disabled_key = make_lock_disabled_key(b2_tool) + b2_tool.should_succeed( + [ + 'account', 'authorize', '--environment', b2_tool.realm, lock_disabled_key_id, + lock_disabled_key + ], + ) + + b2_tool.should_fail( + [ # lock disabled key + 'rm', + '--bypass-governance', + f"b2id://{locked_file['fileId']}", + ], + " failed: unauthorized for application key with capabilities '" + ) + + def test_profile_switch(b2_tool): # this test could be unit, but it adds a lot of complexity because of # necessity to pass mocked B2Api to ConsoleTool; it's much easier to @@ -2864,7 +2942,12 @@ def test_replication_monitoring(b2_tool, bucket_name, sample_file, schedule_buck ) # there is just one file, so clean after itself for faster execution - b2_tool.should_succeed(['delete-file-version', uploaded_a['fileName'], uploaded_a['fileId']]) + b2_tool.should_succeed( + ['delete-file-version', uploaded_a['fileName'], uploaded_a['fileId']], + expected_stderr_pattern=re.compile( + re.escape('WARNING: `delete-file-version` command is deprecated. Use `rm` instead.') + ) + ) # run stats command replication_status_deprecated_pattern = re.compile( diff --git a/test/unit/test_console_tool.py b/test/unit/test_console_tool.py index cb3645df..82f5e13c 100644 --- a/test/unit/test_console_tool.py +++ b/test/unit/test_console_tool.py @@ -994,6 +994,37 @@ def test_bucket_info_from_json(self): expected_json_in_stdout=expected_json, ) + @pytest.mark.apiver(from_ver=4) + def test_rm_fileid_v4(self): + + self._authorize_account() + self._run_command(['bucket', 'create', 'my-bucket', 'allPublic'], 'bucket_0\n', '', 0) + + with TempDir() as temp_dir: + local_file1 = self._make_local_file(temp_dir, 'file1.txt') + # For this test, use a mod time without millis. My mac truncates + # millis and just leaves seconds. + mod_time = 1500111222 + os.utime(local_file1, (mod_time, mod_time)) + self.assertEqual(1500111222, os.path.getmtime(local_file1)) + + # Upload a file + self._run_command( + [ + 'file', 'upload', '--no-progress', 'my-bucket', local_file1, 'file1.txt', + '--cache-control=private, max-age=3600' + ], + remove_version=True, + ) + + # Hide file + self._run_command(['file', 'hide', 'my-bucket', 'file1.txt'],) + + # Delete one file version + self._run_command(['rm', 'b2id://9998']) + # Delete one file version + self._run_command(['rm', 'b2id://9999']) + def test_files(self): self._authorize_account() @@ -1135,14 +1166,20 @@ def test_files(self): expected_json = {"action": "delete", "fileId": "9998", "fileName": "file1.txt"} self._run_command( - ['delete-file-version', 'file1.txt', '9998'], expected_json_in_stdout=expected_json + ['delete-file-version', 'file1.txt', '9998'], + expected_stderr= + 'WARNING: `delete-file-version` command is deprecated. Use `rm` instead.\n', + expected_json_in_stdout=expected_json ) # Delete one file version, not passing the name in expected_json = {"action": "delete", "fileId": "9999", "fileName": "file1.txt"} self._run_command( - ['delete-file-version', '9999'], expected_json_in_stdout=expected_json + ['delete-file-version', '9999'], + expected_stderr= + 'WARNING: `delete-file-version` command is deprecated. Use `rm` instead.\n', + expected_json_in_stdout=expected_json ) def test_files_encrypted(self): @@ -1337,7 +1374,9 @@ def test_files_encrypted(self): self._run_command( ['delete-file-version', 'file1.txt', '9998'], - expected_json_in_stdout=expected_json, + expected_stderr= + 'WARNING: `delete-file-version` command is deprecated. Use `rm` instead.\n', + expected_json_in_stdout=expected_json ) # Delete one file version, not passing the name in @@ -1345,6 +1384,8 @@ def test_files_encrypted(self): self._run_command( ['delete-file-version', '9999'], + expected_stderr= + 'WARNING: `delete-file-version` command is deprecated. Use `rm` instead.\n', expected_json_in_stdout=expected_json, )