From afa43113a2e4ecfbea57d0f5afb6f1ed6b0b2349 Mon Sep 17 00:00:00 2001 From: Maciej Urbanski Date: Tue, 21 Nov 2023 15:05:45 +0100 Subject: [PATCH 1/9] fix ConseoleToolTest failing on empty expected output --- test/unit/test_console_tool.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/unit/test_console_tool.py b/test/unit/test_console_tool.py index 649d7b56b..0e969c362 100644 --- a/test/unit/test_console_tool.py +++ b/test/unit/test_console_tool.py @@ -101,7 +101,7 @@ def _trim_leading_spaces(self, s): return '' # Count the leading spaces - space_count = min(self._leading_spaces(line) for line in lines if line != '') + space_count = min((self._leading_spaces(line) for line in lines if line != ''), default=0) # Remove the leading spaces from each line, based on the line # with the fewest leading spaces From 828f2ede0c51945706c7c03226c64261db96584a Mon Sep 17 00:00:00 2001 From: Maciej Urbanski Date: Tue, 21 Nov 2023 17:55:19 +0100 Subject: [PATCH 2/9] add B2 URI support --- b2/_cli/argcompleters.py | 36 +++ b2/_cli/b2args.py | 44 ++++ b2/_utils/python_compat.py | 49 ++++ b2/_utils/uri.py | 111 ++++++++- b2/console_tool.py | 228 ++++++++----------- b2/json_encoder.py | 4 +- doc/source/subcommands/download_file.rst | 8 + doc/source/subcommands/file_info.rst | 8 + doc/source/subcommands/get_url.rst | 8 + test/integration/test_autocomplete.py | 15 ++ test/integration/test_b2_command_line.py | 6 +- test/unit/_utils/test_uri.py | 43 +++- test/unit/console_tool/conftest.py | 42 +++- test/unit/console_tool/test_download_file.py | 64 ++++-- test/unit/console_tool/test_file_info.py | 62 +++++ test/unit/console_tool/test_get_url.py | 51 +++++ 16 files changed, 607 insertions(+), 172 deletions(-) create mode 100644 b2/_cli/b2args.py create mode 100644 b2/_utils/python_compat.py create mode 100644 doc/source/subcommands/download_file.rst create mode 100644 doc/source/subcommands/file_info.rst create mode 100644 doc/source/subcommands/get_url.rst create mode 100644 test/unit/console_tool/test_file_info.py create mode 100644 test/unit/console_tool/test_get_url.py diff --git a/b2/_cli/argcompleters.py b/b2/_cli/argcompleters.py index c6fe94410..8b6ff700b 100644 --- a/b2/_cli/argcompleters.py +++ b/b2/_cli/argcompleters.py @@ -14,6 +14,8 @@ from b2sdk.v2.api import B2Api from b2._cli.b2api import _get_b2api_for_profile +from b2._utils.python_compat import removeprefix +from b2._utils.uri import parse_b2_uri def _with_api(func): @@ -50,3 +52,37 @@ def file_name_completer(api: B2Api, parsed_args, **kwargs): folder_name or file_version.file_name for file_version, folder_name in islice(file_versions, LIST_FILE_NAMES_MAX_LIMIT) ] + + +@_with_api +def b2uri_file_completer(api: B2Api, prefix: str, **kwargs): + """ + Completes B2 URI pointing to a file-like object in a bucket. + """ + if prefix.startswith('b2://'): + prefix_without_scheme = removeprefix(prefix, 'b2://') + if '/' not in prefix_without_scheme: + return [f"b2://{bucket.name}/" for bucket in api.list_buckets(use_cache=True)] + + b2_uri = parse_b2_uri(prefix) + bucket = api.get_bucket_by_name(b2_uri.bucket_name) + file_versions = bucket.ls( + f"{b2_uri.path}*", + latest_only=True, + recursive=True, + fetch_count=LIST_FILE_NAMES_MAX_LIMIT, + with_wildcard=True, + ) + return [ + f"b2://{bucket.name}/{file_version.file_name}" + for file_version, folder_name in islice(file_versions, LIST_FILE_NAMES_MAX_LIMIT) + if file_version + ] + elif prefix.startswith('b2id://'): + # listing all files from all buckets is unreasonably expensive + return ["b2id://"] + else: + return [ + "b2://", + "b2id://", + ] diff --git a/b2/_cli/b2args.py b/b2/_cli/b2args.py new file mode 100644 index 000000000..3277f45de --- /dev/null +++ b/b2/_cli/b2args.py @@ -0,0 +1,44 @@ +###################################################################### +# +# File: b2/_cli/b2args.py +# +# Copyright 2023 Backblaze Inc. All Rights Reserved. +# +# License https://www.backblaze.com/using_b2_code.html +# +###################################################################### +""" +Utility functions for adding b2-specific arguments to an argparse parser. +""" +import argparse + +from b2._cli.argcompleters import b2uri_file_completer +from b2._utils.uri import B2URI, B2URIBase, parse_b2_uri +from b2.arg_parser import wrap_with_argument_type_error + + +def b2_file_uri(value: str) -> B2URIBase: + b2_uri = parse_b2_uri(value) + if isinstance(b2_uri, B2URI): + if b2_uri.is_dir(): + raise ValueError( + f"B2 URI pointing to a file-like object is required, but {value} was provided" + ) + return b2_uri + + return b2_uri + + +B2_URI_ARG_TYPE = wrap_with_argument_type_error(parse_b2_uri) +B2_URI_FILE_ARG_TYPE = wrap_with_argument_type_error(b2_file_uri) + + +def add_b2_file_argument(parser: argparse.ArgumentParser, name="B2_URI"): + """ + Add an argument to the parser that must be a B2 URI pointing to a file. + """ + parser.add_argument( + name, + type=B2_URI_FILE_ARG_TYPE, + help="B2 URI pointing to a file, e.g. b2://yourBucket/file.txt or b2id://fileId", + ).completer = b2uri_file_completer diff --git a/b2/_utils/python_compat.py b/b2/_utils/python_compat.py new file mode 100644 index 000000000..e7945e33a --- /dev/null +++ b/b2/_utils/python_compat.py @@ -0,0 +1,49 @@ +###################################################################### +# +# File: b2/_utils/python_compat.py +# +# Copyright 2023 Backblaze Inc. All Rights Reserved. +# +# License https://www.backblaze.com/using_b2_code.html +# +###################################################################### +""" +Utilities for compatibility with older Python versions. +""" +import functools +import sys + +if sys.version_info < (3, 9): + + def removeprefix(s: str, prefix: str) -> str: + return s[len(prefix):] if s.startswith(prefix) else s + +else: + removeprefix = str.removeprefix + +if sys.version_info < (3, 8): + + class singledispatchmethod: + """ + singledispatchmethod backport for Python 3.7. + + There are no guarantees for its completeness. + """ + + def __init__(self, method): + self.dispatcher = functools.singledispatch(method) + self.method = method + + def register(self, cls, method=None): + return self.dispatcher.register(cls, func=method) + + def __get__(self, obj, cls): + @functools.wraps(self.method) + def method_wrapper(arg, *args, **kwargs): + method_desc = self.dispatcher.dispatch(arg.__class__) + return method_desc.__get__(obj, cls)(arg, *args, **kwargs) + + method_wrapper.register = self.register + return method_wrapper +else: + singledispatchmethod = functools.singledispatchmethod diff --git a/b2/_utils/uri.py b/b2/_utils/uri.py index 537e3863a..b82daaa4c 100644 --- a/b2/_utils/uri.py +++ b/b2/_utils/uri.py @@ -14,22 +14,65 @@ import urllib from pathlib import Path +from b2sdk.v2 import ( + B2Api, + DownloadVersion, + FileVersion, +) + +from b2._utils.python_compat import removeprefix, singledispatchmethod + class B2URIBase: pass -@dataclasses.dataclass +@dataclasses.dataclass(frozen=True) class B2URI(B2URIBase): - bucket: str - path: str + """ + B2 URI designating a particular object by name & bucket or "subdirectory" in a bucket. + + Please note, both files and directories are symbolical concept, not a real one in B2, i.e. + there is no such thing as "directory" in B2, but it is possible to mimic it by using object names with non-trailing + slashes. + To make it possible, it is highly discouraged to use trailing slashes in object names. + """ + + bucket_name: str + path: str = "" + + def __post_init__(self): + path = removeprefix(self.path, "/") + self.__dict__["path"] = path # hack for a custom init in frozen dataclass def __str__(self) -> str: - return f"b2://{self.bucket}{self.path}" + return f"b2://{self.bucket_name}/{self.path}" + + def is_dir(self) -> bool | None: + """ + Return if the path is a directory. + Please note this is symbolical. + It is possible for file to have a trailing slash, but it is HIGHLY discouraged, and not supported by B2 CLI. + At the same time it is possible for a directory to not have a trailing slash, + which is discouraged, but allowed by B2 CLI. + This is done to mimic unix-like Path. -@dataclasses.dataclass + In practice, this means that `.is_dir() == True` will always be interpreted as "this is a directory", + but reverse is not necessary true, and `not uri.is_dir()` should be merely interpreted as + "this is a directory or a file". + + :return: True if the path is a directory, None if it is unknown + """ + return not self.path or self.path.endswith("/") or None + + +@dataclasses.dataclass(frozen=True) class B2FileIdURI(B2URIBase): + """ + B2 URI designating a particular file by its id. + """ + file_id: str def __str__(self) -> str: @@ -58,8 +101,62 @@ def _parse_b2_uri(uri, parsed: urllib.parse.ParseResult) -> B2URI | B2FileIdURI: ) if parsed.scheme == "b2": - return B2URI(bucket=parsed.netloc, path=parsed.path[1:]) + return B2URI(bucket_name=parsed.netloc, path=parsed.path) elif parsed.scheme == "b2id": - return B2FileIdURI(file_id=parsed.netloc) + file_id = parsed.netloc + if not file_id: + raise ValueError(f"File id was not provided in B2 URI: {uri!r}") + return B2FileIdURI(file_id=file_id) else: raise ValueError(f"Unsupported URI scheme: {parsed.scheme!r}") + + +class B2URIAdapter: + """ + Adapter for using B2URI with B2Api. + + When this matures enough methods from here should be moved to b2sdk B2Api class. + """ + + def __init__(self, api: B2Api): + self.api = api + + def __getattr__(self, name): + return getattr(self.api, name) + + @singledispatchmethod + def download_file_by_uri(self, uri, *args, **kwargs): + raise NotImplementedError(f"Unsupported URI type: {type(uri)}") + + @download_file_by_uri.register + def _(self, uri: B2URI, *args, **kwargs): + bucket = self.get_bucket_by_name(uri.bucket_name) + return bucket.download_file_by_name(uri.path, *args, **kwargs) + + @download_file_by_uri.register + def _(self, uri: B2FileIdURI, *args, **kwargs): + return self.download_file_by_id(uri.file_id, *args, **kwargs) + + @singledispatchmethod + def get_file_info_by_uri(self, uri, *args, **kwargs): + raise NotImplementedError(f"Unsupported URI type: {type(uri)}") + + @get_file_info_by_uri.register + def _(self, uri: B2URI, *args, **kwargs) -> DownloadVersion: + return self.get_file_info_by_name(uri.bucket_name, uri.path, *args, **kwargs) + + @get_file_info_by_uri.register + def _(self, uri: B2FileIdURI, *args, **kwargs) -> FileVersion: + return self.get_file_info(uri.file_id, *args, **kwargs) + + @singledispatchmethod + def get_download_url_by_uri(self, uri, *args, **kwargs): + raise NotImplementedError(f"Unsupported URI type: {type(uri)}") + + @get_download_url_by_uri.register + def _(self, uri: B2URI, *args, **kwargs) -> str: + return self.get_download_url_for_file_name(uri.bucket_name, uri.path, *args, **kwargs) + + @get_download_url_by_uri.register + def _(self, uri: B2FileIdURI, *args, **kwargs) -> str: + return self.get_download_url_for_fileid(uri.file_id, *args, **kwargs) diff --git a/b2/console_tool.py b/b2/console_tool.py index 4568c54ee..ec2448290 100644 --- a/b2/console_tool.py +++ b/b2/console_tool.py @@ -111,6 +111,7 @@ autocomplete_install, ) from b2._cli.b2api import _get_b2api_for_profile +from b2._cli.b2args import add_b2_file_argument from b2._cli.const import ( B2_APPLICATION_KEY_ENV_VAR, B2_APPLICATION_KEY_ID_ENV_VAR, @@ -124,14 +125,13 @@ ) from b2._cli.obj_loads import validated_loads from b2._cli.shell import detect_shell -from b2._utils.uri import B2URI, B2FileIdURI, B2URIBase, parse_b2_uri +from b2._utils.uri import B2URI, B2FileIdURI, B2URIAdapter, B2URIBase from b2.arg_parser import ( ArgumentParser, parse_comma_separated_list, parse_default_retention_period, parse_millis_from_float_timestamp, parse_range, - wrap_with_argument_type_error, ) from b2.json_encoder import B2CliJsonEncoder from b2.version import VERSION @@ -206,9 +206,6 @@ def local_path_to_b2_path(path): return path.replace(os.path.sep, '/') -B2_URI_ARG_TYPE = wrap_with_argument_type_error(parse_b2_uri) - - def keyboard_interrupt_handler(signum, frame): raise KeyboardInterrupt() @@ -573,6 +570,37 @@ def _get_file_name_from_args(self, args): return file_info.file_name +class B2URIFileArgMixin: + @classmethod + def _setup_parser(cls, parser): + add_b2_file_argument(parser) + super()._setup_parser(parser) + + def get_b2_uri_from_arg(self, args: argparse.Namespace) -> B2URIBase: + return args.B2_URI + + +class B2URIFileIDArgMixin: + @classmethod + def _setup_parser(cls, parser): + parser.add_argument('fileId') + super()._setup_parser(parser) + + def get_b2_uri_from_arg(self, args: argparse.Namespace) -> B2URIBase: + return B2FileIdURI(args.fileId) + + +class B2URIBucketNFilenameArgMixin: + @classmethod + def _setup_parser(cls, parser): + parser.add_argument('bucketName') + parser.add_argument('fileName') + super()._setup_parser(parser) + + def get_b2_uri_from_arg(self, args: argparse.Namespace) -> B2URIBase: + return B2URI(args.bucketName, args.fileName) + + class UploadModeMixin(Described): """ Use --incrementalMode to allow for incremental file uploads to safe bandwidth. This will only affect files, which @@ -667,7 +695,7 @@ class Command(Described): def __init__(self, console_tool): self.console_tool = console_tool - self.api = console_tool.api + self.api = B2URIAdapter(console_tool.api) self.stdout = console_tool.stdout self.stderr = console_tool.stderr self.quiet = False @@ -1407,7 +1435,9 @@ def run(self, args): return 0 -class DownloadCommand(Command): +class DownloadCommand( + ProgressMixin, SourceSseMixin, WriteBufferSizeMixin, SkipHashVerificationMixin, Command +): """ helper methods for returning results from download commands """ def _print_download_info(self, downloaded_file: DownloadedFile): @@ -1492,18 +1522,13 @@ def get_local_output_filepath(self, filename: str) -> pathlib.Path: return pathlib.Path(filename) -@B2.register_subcommand -class DownloadFileById( +class DownloadFileBase( ThreadsMixin, - ProgressMixin, - SourceSseMixin, - WriteBufferSizeMixin, - SkipHashVerificationMixin, MaxDownloadStreamsMixin, DownloadCommand, ): """ - Downloads the given file, and stores it in the given local file. + Downloads the given file-like object, and stores it in the given local file. {PROGRESSMIXIN} {THREADSMIXIN} @@ -1517,12 +1542,6 @@ class DownloadFileById( - **readFiles** """ - @classmethod - def _setup_parser(cls, parser): - parser.add_argument('fileId') - parser.add_argument('localFileName') - super()._setup_parser(parser) - def run(self, args): super().run(args) progress_listener = make_progress_listener( @@ -1530,8 +1549,10 @@ def run(self, args): ) encryption_setting = self._get_source_sse_setting(args) self._set_threads_from_args(args) - downloaded_file = self.api.download_file_by_id( - args.fileId, progress_listener, encryption=encryption_setting + + b2_uri = self.get_b2_uri_from_arg(args) + downloaded_file = self.api.download_file_by_uri( + b2_uri, progress_listener, encryption=encryption_setting ) self._print_download_info(downloaded_file) @@ -1543,67 +1564,42 @@ def run(self, args): @B2.register_subcommand -class DownloadFileByName( - ThreadsMixin, - ProgressMixin, - SourceSseMixin, - WriteBufferSizeMixin, - SkipHashVerificationMixin, - MaxDownloadStreamsMixin, - DownloadCommand, -): - """ - Downloads the given file, and stores it in the given local file. +class DownloadFile(B2URIFileArgMixin, DownloadFileBase): + __doc__ = DownloadFileBase.__doc__ - {PROGRESSMIXIN} - {THREADSMIXIN} - {SOURCESSEMIXIN} - {WRITEBUFFERSIZEMIXIN} - {SKIPHASHVERIFICATIONMIXIN} - {MAXDOWNLOADSTREAMSMIXIN} + @classmethod + def _setup_parser(cls, parser): + super()._setup_parser(parser) + parser.add_argument('localFileName') - Requires capability: + def get_b2_uri_from_arg(self, args: argparse.Namespace) -> B2URIBase: + return args.B2_URI - - **readFiles** - """ + +@B2.register_subcommand +class DownloadFileById(B2URIFileIDArgMixin, DownloadFileBase): + __doc__ = DownloadFileBase.__doc__ @classmethod def _setup_parser(cls, parser): - parser.add_argument('bucketName').completer = bucket_name_completer - parser.add_argument('b2FileName').completer = file_name_completer - parser.add_argument('localFileName') super()._setup_parser(parser) + parser.add_argument('localFileName') - def run(self, args): - super().run(args) - self._set_threads_from_args(args) - bucket = self.api.get_bucket_by_name(args.bucketName) - progress_listener = make_progress_listener( - args.localFileName, args.noProgress or args.quiet - ) - encryption_setting = self._get_source_sse_setting(args) - downloaded_file = bucket.download_file_by_name( - args.b2FileName, progress_listener, encryption=encryption_setting - ) - self._print_download_info(downloaded_file) - output_filepath = self.get_local_output_filepath(args.localFileName) - downloaded_file.save_to(output_filepath) - self._print('Download finished') +@B2.register_subcommand +class DownloadFileByName(B2URIBucketNFilenameArgMixin, DownloadFileBase): + __doc__ = DownloadFileBase.__doc__ - return 0 + @classmethod + def _setup_parser(cls, parser): + super()._setup_parser(parser) + parser.add_argument('localFileName') @B2.register_subcommand -class Cat( - ProgressMixin, - SourceSseMixin, - WriteBufferSizeMixin, - SkipHashVerificationMixin, - DownloadCommand, -): +class Cat(B2URIFileArgMixin, DownloadCommand): """ - Download content of a file identified by B2 URI directly to stdout. + Download content of a file-like object identified by B2 URI directly to stdout. {PROGRESSMIXIN} {SOURCESSEMIXIN} @@ -1615,36 +1611,16 @@ class Cat( - **readFiles** """ - @classmethod - def _setup_parser(cls, parser): - parser.add_argument( - 'b2uri', - type=B2_URI_ARG_TYPE, - help= - "B2 URI identifying the file to print, e.g. b2://yourBucket/file.txt or b2id://fileId", - ) - super()._setup_parser(parser) - - def download_by_b2_uri( - self, b2_uri: B2URIBase, args: argparse.Namespace, local_filename: str - ) -> DownloadedFile: - progress_listener = make_progress_listener(local_filename, args.noProgress or args.quiet) - encryption_setting = self._get_source_sse_setting(args) - if isinstance(b2_uri, B2FileIdURI): - download = functools.partial(self.api.download_file_by_id, b2_uri.file_id) - elif isinstance(b2_uri, B2URI): - bucket = self.api.get_bucket_by_name(b2_uri.bucket) - download = functools.partial(bucket.download_file_by_name, b2_uri.path) - else: # This should never happen since there are no more subclasses of B2URIBase - raise ValueError(f'Unsupported B2 URI: {b2_uri!r}') - - return download(progress_listener=progress_listener, encryption=encryption_setting) - def run(self, args): super().run(args) - downloaded_file = self.download_by_b2_uri(args.b2uri, args, '-') - output_filepath = self.get_local_output_filepath('-') - downloaded_file.save_to(output_filepath) + target_filename = '-' + progress_listener = make_progress_listener(target_filename, args.noProgress or args.quiet) + encryption_setting = self._get_source_sse_setting(args) + file_request = self.api.download_file_by_uri( + args.B2_URI, progress_listener=progress_listener, encryption=encryption_setting + ) + output_filepath = self.get_local_output_filepath(target_filename) + file_request.save_to(output_filepath) return 0 @@ -1740,28 +1716,33 @@ def run(self, args): return 1 -@B2.register_subcommand -class GetFileInfo(Command): +class FileInfoBase(Command): """ - Prints all of the information about the file, but not its contents. + Prints all of the information about the object, but not its contents. Requires capability: - **readFiles** """ - @classmethod - def _setup_parser(cls, parser): - parser.add_argument('fileId') - super()._setup_parser(parser) - def run(self, args): super().run(args) - file_version = self.api.get_file_info(args.fileId) + b2_uri = self.get_b2_uri_from_arg(args) + file_version = self.api.get_file_info_by_uri(b2_uri) self._print_json(file_version) return 0 +@B2.register_subcommand +class FileInfo(B2URIFileArgMixin, FileInfoBase): + __doc__ = FileInfoBase.__doc__ + + +@B2.register_subcommand +class GetFileInfo(B2URIFileIDArgMixin, FileInfoBase): + __doc__ = FileInfoBase.__doc__ + + @B2.register_subcommand class GetDownloadAuth(Command): """ @@ -2415,41 +2396,32 @@ def run(self, args): return 1 if failed_on_any_file else 0 -@B2.register_subcommand -class MakeUrl(Command): +class GetUrlBase(Command): """ Prints an URL that can be used to download the given file, if it is public. """ - @classmethod - def _setup_parser(cls, parser): - parser.add_argument('fileId') - super()._setup_parser(parser) - def run(self, args): super().run(args) - self._print(self.api.get_download_url_for_fileid(args.fileId)) + b2_uri = self.get_b2_uri_from_arg(args) + self._print(self.api.get_download_url_by_uri(b2_uri)) return 0 @B2.register_subcommand -class MakeFriendlyUrl(Command): - """ - Prints a short URL that can be used to download the given file, if - it is public. - """ +class GetUrl(B2URIFileArgMixin, GetUrlBase): + __doc__ = GetUrlBase.__doc__ - @classmethod - def _setup_parser(cls, parser): - parser.add_argument('bucketName').completer = bucket_name_completer - parser.add_argument('fileName').completer = file_name_completer - super()._setup_parser(parser) - def run(self, args): - super().run(args) - self._print(self.api.get_download_url_for_file_name(args.bucketName, args.fileName)) - return 0 +@B2.register_subcommand +class MakeUrl(B2URIFileIDArgMixin, GetUrlBase): + __doc__ = GetUrlBase.__doc__ + + +@B2.register_subcommand +class MakeFriendlyUrl(B2URIBucketNFilenameArgMixin, GetUrlBase): + __doc__ = GetUrlBase.__doc__ @B2.register_subcommand diff --git a/b2/json_encoder.py b/b2/json_encoder.py index a5f80a47e..81c182d92 100644 --- a/b2/json_encoder.py +++ b/b2/json_encoder.py @@ -11,7 +11,7 @@ import json from enum import Enum -from b2sdk.v2 import Bucket, FileIdAndName, FileVersion +from b2sdk.v2 import Bucket, DownloadVersion, FileIdAndName, FileVersion class B2CliJsonEncoder(json.JSONEncoder): @@ -27,7 +27,7 @@ class B2CliJsonEncoder(json.JSONEncoder): def default(self, obj): if isinstance(obj, set): return list(obj) - elif isinstance(obj, (FileVersion, FileIdAndName, Bucket)): + elif isinstance(obj, (DownloadVersion, FileVersion, FileIdAndName, Bucket)): return obj.as_dict() elif isinstance(obj, Enum): return obj.value diff --git a/doc/source/subcommands/download_file.rst b/doc/source/subcommands/download_file.rst new file mode 100644 index 000000000..6c80386b7 --- /dev/null +++ b/doc/source/subcommands/download_file.rst @@ -0,0 +1,8 @@ +Download-file command +*************************** + +.. argparse:: + :module: b2.console_tool + :func: get_parser + :prog: b2 + :path: download-file diff --git a/doc/source/subcommands/file_info.rst b/doc/source/subcommands/file_info.rst new file mode 100644 index 000000000..863d52265 --- /dev/null +++ b/doc/source/subcommands/file_info.rst @@ -0,0 +1,8 @@ +File-info command +********************* + +.. argparse:: + :module: b2.console_tool + :func: get_parser + :prog: b2 + :path: file-info diff --git a/doc/source/subcommands/get_url.rst b/doc/source/subcommands/get_url.rst new file mode 100644 index 000000000..537e4f481 --- /dev/null +++ b/doc/source/subcommands/get_url.rst @@ -0,0 +1,8 @@ +Get-url command +**************** + +.. argparse:: + :module: b2.console_tool + :func: get_parser + :prog: b2 + :path: get-url diff --git a/test/integration/test_autocomplete.py b/test/integration/test_autocomplete.py index e0c30e065..d798ba859 100644 --- a/test/integration/test_autocomplete.py +++ b/test/integration/test_autocomplete.py @@ -89,3 +89,18 @@ def test_autocomplete_b2_bucket_n_file_name( shell.expect_exact(bucket_name, timeout=TIMEOUT) shell.send(f'{bucket_name} \t\t') shell.expect_exact(file_name, timeout=TIMEOUT) + + +@skip_on_windows +def test_autocomplete_b2__download_file__b2uri( + autocomplete_installed, shell, b2_tool, bucket_name, file_name, is_running_on_docker +): + """Test that autocomplete suggests bucket names and file names.""" + if is_running_on_docker: + pytest.skip('Not supported on Docker') + shell.send('b2 download_file \t\t') + shell.expect_exact("b2://", timeout=TIMEOUT) + shell.send('b2://\t\t') + shell.expect_exact(bucket_name, timeout=TIMEOUT) + shell.send(f'{bucket_name}/\t\t') + shell.expect_exact(file_name, timeout=TIMEOUT) diff --git a/test/integration/test_b2_command_line.py b/test/integration/test_b2_command_line.py index e62e359d3..ae69478cb 100755 --- a/test/integration/test_b2_command_line.py +++ b/test/integration/test_b2_command_line.py @@ -176,7 +176,7 @@ def test_basic(b2_tool, bucket_name, sample_file, tmp_path): b2_tool.should_succeed(['ls', bucket_name, 'b'], '^b/1{0}b/2{0}'.format(os.linesep)) b2_tool.should_succeed(['ls', bucket_name, 'b/'], '^b/1{0}b/2{0}'.format(os.linesep)) - file_info = b2_tool.should_succeed_json(['get-file-info', second_c_version['fileId']]) + file_info = b2_tool.should_succeed_json(['file-info', f"b2id://{second_c_version['fileId']}"]) expected_info = { 'color': 'blue', 'foo': 'bar=baz', @@ -187,10 +187,10 @@ def test_basic(b2_tool, bucket_name, sample_file, tmp_path): b2_tool.should_succeed(['delete-file-version', 'c', first_c_version['fileId']]) b2_tool.should_succeed(['ls', bucket_name], '^a{0}b/{0}c{0}d{0}'.format(os.linesep)) - b2_tool.should_succeed(['make-url', second_c_version['fileId']]) + b2_tool.should_succeed(['get-url', f"b2id://{second_c_version['fileId']}"]) b2_tool.should_succeed( - ['make-friendly-url', bucket_name, 'any-file-name'], + ['get-url', f"b2://{bucket_name}/any-file-name"], '^https://.*/file/{}/{}\r?$'.format( bucket_name, 'any-file-name', diff --git a/test/unit/_utils/test_uri.py b/test/unit/_utils/test_uri.py index ef34d8a33..ee2834abe 100644 --- a/test/unit/_utils/test_uri.py +++ b/test/unit/_utils/test_uri.py @@ -14,9 +14,44 @@ from b2._utils.uri import B2URI, B2FileIdURI, parse_uri -def test_b2pathuri_str(): - uri = B2URI(bucket="testbucket", path="/path/to/file") - assert str(uri) == "b2://testbucket/path/to/file" +class TestB2URI: + def test__str__(self): + uri = B2URI(bucket_name="testbucket", path="/path/to/file") + assert str(uri) == "b2://testbucket/path/to/file" + + @pytest.mark.parametrize( + "path, expected", + [ + ("", True), + ("/", True), + ("path/", True), + ("path/subpath", None), + ], + ) + def test_is_dir(self, path, expected): + assert B2URI("bucket", path).is_dir() is expected + + def test__bucket_uris_is_normalized(self): + alternatives = [ + B2URI("bucket"), + B2URI("bucket", ""), + B2URI("bucket", "/"), + ] + assert len(set(alternatives)) == 1 + assert {str(uri) for uri in alternatives} == {"b2://bucket/"} # normalized + + @pytest.mark.parametrize( + "path, expected_uri_str", + [ + ("", "b2://bucket/"), + ("/", "b2://bucket/"), + ("path/", "b2://bucket/path/"), + ("path/subpath", "b2://bucket/path/subpath"), + ], + ) + def test__normalization(self, path, expected_uri_str): + assert str(B2URI("bucket", path)) == expected_uri_str + assert str(B2URI("bucket", path)) == str(B2URI("bucket", path)) # normalized def test_b2fileuri_str(): @@ -29,7 +64,7 @@ def test_b2fileuri_str(): [ ("some/local/path", Path("some/local/path")), ("./some/local/path", Path("some/local/path")), - ("b2://bucket/path/to/dir/", B2URI(bucket="bucket", path="path/to/dir/")), + ("b2://bucket/path/to/dir/", B2URI(bucket_name="bucket", path="path/to/dir/")), ("b2id://file123", B2FileIdURI(file_id="file123")), ], ) diff --git a/test/unit/console_tool/conftest.py b/test/unit/console_tool/conftest.py index ddf5ca4e2..aae3562fc 100644 --- a/test/unit/console_tool/conftest.py +++ b/test/unit/console_tool/conftest.py @@ -37,10 +37,19 @@ def authorized_b2_cli(b2_cli): @pytest.fixture -def bucket(b2_cli, authorized_b2_cli): +def bucket_info(b2_cli, authorized_b2_cli): bucket_name = "my-bucket" - b2_cli.run(['create-bucket', bucket_name, 'allPublic'], expected_stdout='bucket_0\n') - yield bucket_name + bucket_id = "bucket_0" + b2_cli.run(['create-bucket', bucket_name, 'allPublic'], expected_stdout=f'{bucket_id}\n') + return { + 'bucketName': bucket_name, + 'bucketId': bucket_id, + } + + +@pytest.fixture +def bucket(bucket_info): + return bucket_info['bucketName'] @pytest.fixture @@ -50,3 +59,30 @@ def mock_stdin(monkeypatch): in_f = open(in_, 'w') yield in_f in_f.close() + + +@pytest.fixture +def local_file(tmp_path): + """Set up a test file and return its path.""" + filename = 'file1.txt' + content = 'hello world' + local_file = tmp_path / filename + local_file.write_text(content) + + mod_time = 1500111222 + os.utime(local_file, (mod_time, mod_time)) + + return local_file + + +@pytest.fixture +def uploaded_file(b2_cli, bucket_info, local_file): + filename = 'file1.txt' + b2_cli.run(['upload-file', '--quiet', bucket_info["bucketName"], str(local_file), filename]) + return { + 'bucket': bucket_info["bucketName"], + 'bucketId': bucket_info["bucketId"], + 'fileName': filename, + 'fileId': '9999', + 'content': local_file.read_text(), + } diff --git a/test/unit/console_tool/test_download_file.py b/test/unit/console_tool/test_download_file.py index 326ddc9fd..13ae320f9 100644 --- a/test/unit/console_tool/test_download_file.py +++ b/test/unit/console_tool/test_download_file.py @@ -13,21 +13,6 @@ import pytest - -@pytest.fixture -def local_file(tmp_path): - """Set up a test file and return its path.""" - filename = 'file1.txt' - content = 'hello world' - local_file = tmp_path / filename - local_file.write_text(content) - - mod_time = 1500111222 - os.utime(local_file, (mod_time, mod_time)) - - return local_file - - EXPECTED_STDOUT_DOWNLOAD = ''' File name: file1.txt File id: 9999 @@ -43,15 +28,34 @@ def local_file(tmp_path): ''' -@pytest.fixture -def uploaded_file(b2_cli, bucket, local_file): - filename = 'file1.txt' - b2_cli.run(['upload-file', bucket, str(local_file), filename]) - return { - 'bucket': bucket, - 'fileName': filename, - 'content': local_file.read_text(), - } +@pytest.mark.parametrize( + 'flag,expected_stdout', [ + ('--noProgress', EXPECTED_STDOUT_DOWNLOAD), + ('-q', ''), + ('--quiet', ''), + ] +) +def test_download_file_by_uri__flag_support(b2_cli, uploaded_file, tmp_path, flag, expected_stdout): + output_path = tmp_path / 'output.txt' + + b2_cli.run( + ['download-file', flag, 'b2id://9999', + str(output_path)], expected_stdout=expected_stdout + ) + assert output_path.read_text() == uploaded_file['content'] + + +@pytest.mark.parametrize('b2_uri', [ + 'b2://my-bucket/file1.txt', + 'b2id://9999', +]) +def test_download_file_by_uri__b2_uri_support(b2_cli, uploaded_file, tmp_path, b2_uri): + output_path = tmp_path / 'output.txt' + + b2_cli.run( + ['download-file', b2_uri, str(output_path)], expected_stdout=EXPECTED_STDOUT_DOWNLOAD + ) + assert output_path.read_text() == uploaded_file['content'] @pytest.mark.parametrize( @@ -151,7 +155,17 @@ def test_cat__b2_uri__invalid(b2_cli, capfd): expected_stderr=None, expected_status=2, ) - assert "argument b2uri: Unsupported URI scheme: ''" in capfd.readouterr().err + assert "argument B2_URI: Unsupported URI scheme: ''" in capfd.readouterr().err + + +def test_cat__b2_uri__not_a_file(b2_cli, bucket, capfd): + b2_cli.run( + ['cat', "b2://bucket/dir/subdir/"], + expected_stderr=None, + expected_status=2, + ) + assert "argument B2_URI: B2 URI pointing to a file-like object is required" in capfd.readouterr( + ).err def test_cat__b2id_uri(b2_cli, bucket, uploaded_stdout_txt, tmp_path, capfd): diff --git a/test/unit/console_tool/test_file_info.py b/test/unit/console_tool/test_file_info.py new file mode 100644 index 000000000..4cfb7e84a --- /dev/null +++ b/test/unit/console_tool/test_file_info.py @@ -0,0 +1,62 @@ +###################################################################### +# +# File: test/unit/console_tool/test_file_info.py +# +# Copyright 2023 Backblaze Inc. All Rights Reserved. +# +# License https://www.backblaze.com/using_b2_code.html +# +###################################################################### +import pytest + + +@pytest.fixture +def uploaded_download_version(b2_cli, bucket_info, uploaded_file): + return { + "contentSha1": "2aae6c35c94fcfb415dbe95f408b9ce91ee846ed", + "contentType": "b2/x-auto", + "fileId": uploaded_file["fileId"], + "fileInfo": { + "src_last_modified_millis": "1500111222000" + }, + "fileName": "file1.txt", + "serverSideEncryption": { + "mode": "none" + }, + "size": 11, + "uploadTimestamp": 5000, + } + + +@pytest.fixture +def uploaded_file_version(b2_cli, bucket_info, uploaded_file, uploaded_download_version): + return { + **uploaded_download_version, + "accountId": b2_cli.account_id, + "action": "upload", + "bucketId": uploaded_file["bucketId"], + } + + +def test_get_file_info(b2_cli, uploaded_file_version): + b2_cli.run( + ["get-file-info", uploaded_file_version["fileId"]], + expected_json_in_stdout=uploaded_file_version, + ) + + +def test_file_info__b2_uri(b2_cli, bucket, uploaded_download_version): + b2_cli.run( + [ + "file-info", + f'b2://{bucket}/{uploaded_download_version["fileName"]}', + ], + expected_json_in_stdout=uploaded_download_version, + ) + + +def test_file_info__b2id_uri(b2_cli, uploaded_file_version): + b2_cli.run( + ["file-info", f'b2id://{uploaded_file_version["fileId"]}'], + expected_json_in_stdout=uploaded_file_version, + ) diff --git a/test/unit/console_tool/test_get_url.py b/test/unit/console_tool/test_get_url.py new file mode 100644 index 000000000..9c4ebf98b --- /dev/null +++ b/test/unit/console_tool/test_get_url.py @@ -0,0 +1,51 @@ +###################################################################### +# +# File: test/unit/console_tool/test_get_url.py +# +# Copyright 2023 Backblaze Inc. All Rights Reserved. +# +# License https://www.backblaze.com/using_b2_code.html +# +###################################################################### +import pytest + + +@pytest.fixture +def uploaded_file_url(bucket_info, uploaded_file): + return f"http://download.example.com/file/{bucket_info['bucketName']}/{uploaded_file['fileName']}" + + +@pytest.fixture +def uploaded_file_url_by_id(uploaded_file): + return f"http://download.example.com/b2api/v2/b2_download_file_by_id?fileId={uploaded_file['fileId']}" + + +def test_make_url(b2_cli, uploaded_file, uploaded_file_url_by_id): + b2_cli.run( + ["make-url", uploaded_file["fileId"]], + expected_stdout=f"{uploaded_file_url_by_id}\n", + ) + + +def test_make_friendly_url(b2_cli, bucket, uploaded_file, uploaded_file_url): + b2_cli.run( + ["make-friendly-url", bucket, uploaded_file["fileName"]], + expected_stdout=f"{uploaded_file_url}\n", + ) + + +def test_get_url__b2_uri(b2_cli, bucket, uploaded_file, uploaded_file_url): + b2_cli.run( + [ + "get-url", + f'b2://{bucket}/{uploaded_file["fileName"]}', + ], + expected_stdout=f"{uploaded_file_url}\n", + ) + + +def test_get_url__b2id_uri(b2_cli, uploaded_file, uploaded_file_url_by_id): + b2_cli.run( + ["get-url", f'b2id://{uploaded_file["fileId"]}'], + expected_stdout=f"{uploaded_file_url_by_id}\n", + ) From b157789016e272639cd7d919368c08e66953ff85 Mon Sep 17 00:00:00 2001 From: Maciej Urbanski Date: Tue, 21 Nov 2023 18:35:10 +0100 Subject: [PATCH 3/9] add cat command to the docs --- changelog.d/+cat.doc.md | 1 + doc/source/subcommands/cat.rst | 8 ++++++++ 2 files changed, 9 insertions(+) create mode 100644 changelog.d/+cat.doc.md create mode 100644 doc/source/subcommands/cat.rst diff --git a/changelog.d/+cat.doc.md b/changelog.d/+cat.doc.md new file mode 100644 index 000000000..32f9a0b72 --- /dev/null +++ b/changelog.d/+cat.doc.md @@ -0,0 +1 @@ +Add `cat` command to documentation \ No newline at end of file diff --git a/doc/source/subcommands/cat.rst b/doc/source/subcommands/cat.rst new file mode 100644 index 000000000..5f866985f --- /dev/null +++ b/doc/source/subcommands/cat.rst @@ -0,0 +1,8 @@ +Cat command +**************** + +.. argparse:: + :module: b2.console_tool + :func: get_parser + :prog: b2 + :path: cat From dad653dd36092cd56e8eaa7f114f63b80ac58634 Mon Sep 17 00:00:00 2001 From: Maciej Urbanski Date: Tue, 21 Nov 2023 18:46:19 +0100 Subject: [PATCH 4/9] add B2 URI commands changelog note --- changelog.d/+b2_uri_cmds.added.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/+b2_uri_cmds.added.md diff --git a/changelog.d/+b2_uri_cmds.added.md b/changelog.d/+b2_uri_cmds.added.md new file mode 100644 index 000000000..56fc2e13b --- /dev/null +++ b/changelog.d/+b2_uri_cmds.added.md @@ -0,0 +1 @@ +Add `download-file`, `file-info` and `get-url` commands using new B2 URI syntax allowing for referring to file-like objects by their bucket&name or ID. From 3a3037fe1ff08a91d8a05106e08c27e4dca4cca7 Mon Sep 17 00:00:00 2001 From: Maciej Urbanski Date: Tue, 21 Nov 2023 18:51:21 +0100 Subject: [PATCH 5/9] fix changelog example in CONTRIBUTING guide --- CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ed1289a12..5eb149f3d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -48,7 +48,7 @@ The `changelog.d` file name convention is: These files can either be created manually, or using `towncrier` e.g. - towncrier create -c 'write your description here' 157.fixed.md + towncrier create -c 'Add proper changelog example to CONTRIBUTING guide' 157.added.md `towncrier create` also takes care of duplicates automatically (if there is more than 1 news fragment of one type for a given github issue). From f541768a5ca9561d57188312cba8ba3a7931051c Mon Sep 17 00:00:00 2001 From: Maciej Urbanski Date: Tue, 21 Nov 2023 19:13:23 +0100 Subject: [PATCH 6/9] deprecate commands replaced by B2 URI using equivalents --- b2/arg_parser.py | 20 +++-- b2/console_tool.py | 90 ++++++++++++++------ changelog.d/+b2_uri_cmds.deprecated.md | 3 + test/integration/test_autocomplete.py | 21 +---- test/integration/test_b2_command_line.py | 56 ++++++------ test/unit/console_tool/test_download_file.py | 15 +++- test/unit/console_tool/test_file_info.py | 1 + test/unit/console_tool/test_get_url.py | 2 + test/unit/test_arg_parser.py | 4 +- test/unit/test_console_tool.py | 30 +++---- 10 files changed, 144 insertions(+), 98 deletions(-) create mode 100644 changelog.d/+b2_uri_cmds.deprecated.md diff --git a/b2/arg_parser.py b/b2/arg_parser.py index 9a0e0d4e4..9420ac51e 100644 --- a/b2/arg_parser.py +++ b/b2/arg_parser.py @@ -7,6 +7,7 @@ # License https://www.backblaze.com/using_b2_code.html # ###################################################################### +from __future__ import annotations import argparse import functools @@ -22,11 +23,11 @@ _arrow_version = tuple(int(p) for p in arrow.__version__.split(".")) -class RawTextHelpFormatter(argparse.RawTextHelpFormatter): +class B2RawTextHelpFormatter(argparse.RawTextHelpFormatter): """ CLI custom formatter. - It removes default "usage: " text and prints usage for all subcommands. + It removes default "usage: " text and prints usage for all (non-hidden) subcommands. """ def add_usage(self, usage, actions, groups, prefix=None): @@ -38,7 +39,8 @@ def add_argument(self, action): if isinstance(action, argparse._SubParsersAction) and action.help is not argparse.SUPPRESS: usages = [] for choice in self._unique_choice_values(action): - usages.append(choice.format_usage()) + if not getattr(choice, 'hidden', False): + usages.append(choice.format_usage()) self.add_text(''.join(usages)) else: super().add_argument(action) @@ -52,7 +54,7 @@ def _unique_choice_values(cls, action): yield value -class ArgumentParser(argparse.ArgumentParser): +class B2ArgumentParser(argparse.ArgumentParser): """ CLI custom parser. @@ -60,11 +62,17 @@ class ArgumentParser(argparse.ArgumentParser): and use help message in case of error. """ - def __init__(self, *args, for_docs=False, **kwargs): + def __init__(self, *args, for_docs: bool = False, hidden: bool = False, **kwargs): + """ + + :param for_docs: is this parser used for generating docs + :param hidden: should this parser be hidden from `--help` + """ self._raw_description = None self._description = None self._for_docs = for_docs - kwargs.setdefault('formatter_class', RawTextHelpFormatter) + self.hidden = hidden + kwargs.setdefault('formatter_class', B2RawTextHelpFormatter) super().__init__(*args, **kwargs) @property diff --git a/b2/console_tool.py b/b2/console_tool.py index ec2448290..2865a56c7 100644 --- a/b2/console_tool.py +++ b/b2/console_tool.py @@ -127,7 +127,7 @@ from b2._cli.shell import detect_shell from b2._utils.uri import B2URI, B2FileIdURI, B2URIAdapter, B2URIBase from b2.arg_parser import ( - ArgumentParser, + B2ArgumentParser, parse_comma_separated_list, parse_default_retention_period, parse_millis_from_float_timestamp, @@ -687,6 +687,8 @@ class Command(Described): # Set to True for commands that receive sensitive information in arguments FORBID_LOGGING_ARGUMENTS = False + hide_from_help = False + # The registry for the subcommands, should be reinitialized in subclass subcommands_registry = None @@ -716,28 +718,38 @@ def register_subcommand(cls, command_class): return decorator @classmethod - def get_parser(cls, subparsers=None, parents=None, for_docs=False): + def create_parser( + cls, subparsers: "argparse._SubParsersAction | None" = None, parents=None, for_docs=False + ) -> argparse.ArgumentParser: + """ + Creates a parser for the command. + + :param subparsers: subparsers object to which add new parser + :param parents: created ArgumentParser `parents`, see `argparse.ArgumentParser` + :param for_docs: if parser is to be used for documentation generation + :return: created parser + """ if parents is None: parents = [] description = cls._get_description() + name, alias = cls.name_and_alias() + parser_kwargs = dict( + prog=name, + description=description, + parents=parents, + for_docs=for_docs, + hidden=cls.hide_from_help, + ) + if subparsers is None: - name, _ = cls.name_and_alias() - parser = ArgumentParser( - prog=name, - description=description, - parents=parents, - for_docs=for_docs, - ) + parser = B2ArgumentParser(**parser_kwargs,) else: - name, alias = cls.name_and_alias() parser = subparsers.add_parser( - name, - description=description, - parents=parents, + parser_kwargs.pop('prog'), + **parser_kwargs, aliases=[alias] if alias is not None and not for_docs else (), - for_docs=for_docs, ) # Register class that will handle this particular command, for both name and alias. parser.set_defaults(command_class=cls) @@ -746,7 +758,7 @@ def get_parser(cls, subparsers=None, parents=None, for_docs=False): if cls.subcommands_registry: if not parents: - common_parser = ArgumentParser(add_help=False) + common_parser = B2ArgumentParser(add_help=False) common_parser.add_argument( '--debugLogs', action='store_true', help=argparse.SUPPRESS ) @@ -758,10 +770,15 @@ def get_parser(cls, subparsers=None, parents=None, for_docs=False): ) parents = [common_parser] - subparsers = parser.add_subparsers(prog=parser.prog, title='usages', dest='command') + subparsers = parser.add_subparsers( + prog=parser.prog, + title='usages', + dest='command', + parser_class=B2ArgumentParser, + ) subparsers.required = True for subcommand in cls.subcommands_registry.values(): - subcommand.get_parser(subparsers=subparsers, parents=parents, for_docs=for_docs) + subcommand.create_parser(subparsers=subparsers, parents=parents, for_docs=for_docs) return parser @@ -845,6 +862,26 @@ def __str__(self): return f'{self.__class__.__module__}.{self.__class__.__name__}' +class CmdReplacedByMixin: + hide_from_help = True + replaced_by_cmd: "type[Command]" + + def run(self, args): + self._print_stderr( + f'WARNING: {self.__class__.name_and_alias()[0]} command is deprecated. ' + f'Use {self.replaced_by_cmd.name_and_alias()[0]} instead.' + ) + return super().run(args) + + @classmethod + def _get_description(cls): + return ( + f'{super()._get_description()}\n\n' + f'.. warning::\n' + f' This command is deprecated. Use ``{cls.replaced_by_cmd.name_and_alias()[0]}`` instead.\n' + ) + + class B2(Command): """ This program provides command-line access to the B2 service. @@ -1577,8 +1614,9 @@ def get_b2_uri_from_arg(self, args: argparse.Namespace) -> B2URIBase: @B2.register_subcommand -class DownloadFileById(B2URIFileIDArgMixin, DownloadFileBase): +class DownloadFileById(CmdReplacedByMixin, B2URIFileIDArgMixin, DownloadFileBase): __doc__ = DownloadFileBase.__doc__ + replaced_by_cmd = DownloadFile @classmethod def _setup_parser(cls, parser): @@ -1587,8 +1625,9 @@ def _setup_parser(cls, parser): @B2.register_subcommand -class DownloadFileByName(B2URIBucketNFilenameArgMixin, DownloadFileBase): +class DownloadFileByName(CmdReplacedByMixin, B2URIBucketNFilenameArgMixin, DownloadFileBase): __doc__ = DownloadFileBase.__doc__ + replaced_by_cmd = DownloadFile @classmethod def _setup_parser(cls, parser): @@ -1739,8 +1778,9 @@ class FileInfo(B2URIFileArgMixin, FileInfoBase): @B2.register_subcommand -class GetFileInfo(B2URIFileIDArgMixin, FileInfoBase): +class GetFileInfo(CmdReplacedByMixin, B2URIFileIDArgMixin, FileInfoBase): __doc__ = FileInfoBase.__doc__ + replaced_by_cmd = FileInfo @B2.register_subcommand @@ -2415,13 +2455,15 @@ class GetUrl(B2URIFileArgMixin, GetUrlBase): @B2.register_subcommand -class MakeUrl(B2URIFileIDArgMixin, GetUrlBase): +class MakeUrl(CmdReplacedByMixin, B2URIFileIDArgMixin, GetUrlBase): __doc__ = GetUrlBase.__doc__ + replaced_by_cmd = GetUrl @B2.register_subcommand -class MakeFriendlyUrl(B2URIBucketNFilenameArgMixin, GetUrlBase): +class MakeFriendlyUrl(CmdReplacedByMixin, B2URIBucketNFilenameArgMixin, GetUrlBase): __doc__ = GetUrlBase.__doc__ + replaced_by_cmd = GetUrl @B2.register_subcommand @@ -3887,7 +3929,7 @@ def __init__(self, b2_api: B2Api | None, stdout, stderr): def run_command(self, argv): signal.signal(signal.SIGINT, keyboard_interrupt_handler) - parser = B2.get_parser() + parser = B2.create_parser() argcomplete.autocomplete(parser, default_completer=None) args = parser.parse_args(argv[1:]) self._setup_logging(args, argv) @@ -4025,7 +4067,7 @@ def _setup_logging(cls, args, argv): # used by Sphinx -get_parser = functools.partial(B2.get_parser, for_docs=True) +get_parser = functools.partial(B2.create_parser, for_docs=True) # TODO: import from b2sdk as soon as we rely on 1.0.0 diff --git a/changelog.d/+b2_uri_cmds.deprecated.md b/changelog.d/+b2_uri_cmds.deprecated.md new file mode 100644 index 000000000..98aeead6b --- /dev/null +++ b/changelog.d/+b2_uri_cmds.deprecated.md @@ -0,0 +1,3 @@ +Deprecated `download-file-by-id` and `download-file-by-name`, use `download-file` instead. +Deprecated `get-file-info`, use `file-info` instead. +Deprecated `make-url` and `make-friendly-url`, use `get-url` instead. diff --git a/test/integration/test_autocomplete.py b/test/integration/test_autocomplete.py index d798ba859..6e5b0bd38 100644 --- a/test/integration/test_autocomplete.py +++ b/test/integration/test_autocomplete.py @@ -60,7 +60,7 @@ def test_autocomplete_b2_commands(autocomplete_installed, is_running_on_docker, if is_running_on_docker: pytest.skip('Not supported on Docker') shell.send('b2 \t\t') - shell.expect_exact(["authorize-account", "download-file-by-id", "get-bucket"], timeout=TIMEOUT) + shell.expect_exact(["authorize-account", "download-file", "get-bucket"], timeout=TIMEOUT) @skip_on_windows @@ -69,28 +69,13 @@ def test_autocomplete_b2_only_matching_commands( ): if is_running_on_docker: pytest.skip('Not supported on Docker') - shell.send('b2 download-\t\t') + shell.send('b2 delete-\t\t') - shell.expect_exact( - "file-by-", timeout=TIMEOUT - ) # common part of remaining cmds is autocompleted + shell.expect_exact("file", timeout=TIMEOUT) # common part of remaining cmds is autocompleted with pytest.raises(pexpect.exceptions.TIMEOUT): # no other commands are suggested shell.expect_exact("get-bucket", timeout=0.5) -@skip_on_windows -def test_autocomplete_b2_bucket_n_file_name( - autocomplete_installed, shell, b2_tool, bucket_name, file_name, is_running_on_docker -): - """Test that autocomplete suggests bucket names and file names.""" - if is_running_on_docker: - pytest.skip('Not supported on Docker') - shell.send('b2 download_file_by_name \t\t') - shell.expect_exact(bucket_name, timeout=TIMEOUT) - shell.send(f'{bucket_name} \t\t') - shell.expect_exact(file_name, timeout=TIMEOUT) - - @skip_on_windows def test_autocomplete_b2__download_file__b2uri( autocomplete_installed, shell, b2_tool, bucket_name, file_name, is_running_on_docker diff --git a/test/integration/test_b2_command_line.py b/test/integration/test_b2_command_line.py index ae69478cb..5c5c0896d 100755 --- a/test/integration/test_b2_command_line.py +++ b/test/integration/test_b2_command_line.py @@ -67,7 +67,7 @@ def test_download(b2_tool, bucket_name, sample_filepath, uploaded_sample_file, t output_a = tmp_path / 'a' b2_tool.should_succeed( [ - 'download-file-by-name', '--quiet', bucket_name, uploaded_sample_file['fileName'], + 'download-file', '--quiet', f"b2://{bucket_name}/{uploaded_sample_file['fileName']}", str(output_a) ] ) @@ -75,7 +75,7 @@ def test_download(b2_tool, bucket_name, sample_filepath, uploaded_sample_file, t output_b = tmp_path / 'b' b2_tool.should_succeed( - ['download-file-by-id', '--quiet', uploaded_sample_file['fileId'], + ['download-file', '--quiet', f"b2id://{uploaded_sample_file['fileId']}", str(output_b)] ) assert output_b.read_text() == sample_filepath.read_text() @@ -127,9 +127,7 @@ def test_basic(b2_tool, bucket_name, sample_file, tmp_path): should_equal(['rm1'], [f['fileName'] for f in list_of_files]) b2_tool.should_succeed(['rm', '--recursive', '--withWildcard', bucket_name, 'rm1']) - b2_tool.should_succeed( - ['download-file-by-name', '--noProgress', '--quiet', bucket_name, 'b/1', tmp_path / 'a'] - ) + b2_tool.should_succeed(['download-file', '--quiet', f'b2://{bucket_name}/b/1', tmp_path / 'a']) b2_tool.should_succeed(['hide-file', bucket_name, 'c']) @@ -597,7 +595,7 @@ def sync_up_helper(b2_tool, bucket_name, dir_, encryption=None): return # that's enough, we've checked that encryption works, no need to repeat the whole sync suite c_id = find_file_id(file_versions, prefix + 'c') - file_info = b2_tool.should_succeed_json(['get-file-info', c_id])['fileInfo'] + file_info = b2_tool.should_succeed_json(['file-info', f"b2id://{c_id}"])['fileInfo'] should_equal( file_mod_time_millis(dir_path / 'c'), int(file_info['src_last_modified_millis']) ) @@ -1151,12 +1149,12 @@ def test_sse_b2(b2_tool, bucket_name, sample_file, tmp_path): b2_tool.should_succeed(['upload-file', '--quiet', bucket_name, sample_file, 'not_encrypted']) b2_tool.should_succeed( - ['download-file-by-name', '--quiet', bucket_name, 'encrypted', tmp_path / 'encrypted'] + ['download-file', '--quiet', f'b2://{bucket_name}/encrypted', tmp_path / 'encrypted'] ) b2_tool.should_succeed( [ - 'download-file-by-name', '--quiet', bucket_name, 'not_encrypted', - tmp_path / 'not_encypted' + 'download-file', '--quiet', f'b2://{bucket_name}/not_encrypted', + tmp_path / 'not_encrypted' ] ) @@ -1171,10 +1169,12 @@ def test_sse_b2(b2_tool, bucket_name, sample_file, tmp_path): ) encrypted_version = list_of_files[0] - file_info = b2_tool.should_succeed_json(['get-file-info', encrypted_version['fileId']]) + file_info = b2_tool.should_succeed_json(['file-info', f"b2id://{encrypted_version['fileId']}"]) should_equal({'algorithm': 'AES256', 'mode': 'SSE-B2'}, file_info['serverSideEncryption']) not_encrypted_version = list_of_files[1] - file_info = b2_tool.should_succeed_json(['get-file-info', not_encrypted_version['fileId']]) + file_info = b2_tool.should_succeed_json( + ['file-info', f"b2id://{not_encrypted_version['fileId']}"] + ) should_equal({'mode': 'none'}, file_info['serverSideEncryption']) b2_tool.should_succeed( @@ -1198,12 +1198,14 @@ def test_sse_b2(b2_tool, bucket_name, sample_file, tmp_path): ) copied_encrypted_version = list_of_files[2] - file_info = b2_tool.should_succeed_json(['get-file-info', copied_encrypted_version['fileId']]) + file_info = b2_tool.should_succeed_json( + ['file-info', f"b2id://{copied_encrypted_version['fileId']}"] + ) should_equal({'algorithm': 'AES256', 'mode': 'SSE-B2'}, file_info['serverSideEncryption']) copied_not_encrypted_version = list_of_files[3] file_info = b2_tool.should_succeed_json( - ['get-file-info', copied_not_encrypted_version['fileId']] + ['file-info', f"b2id://{copied_not_encrypted_version['fileId']}"] ) should_equal({'mode': 'none'}, file_info['serverSideEncryption']) @@ -1245,25 +1247,22 @@ def test_sse_c(b2_tool, bucket_name, is_running_on_docker, sample_file, tmp_path should_equal(sse_c_key_id, file_version_info['fileInfo'][SSE_C_KEY_ID_FILE_INFO_KEY_NAME]) b2_tool.should_fail( - [ - 'download-file-by-name', '--quiet', bucket_name, 'uploaded_encrypted', - 'gonna_fail_anyway' - ], + ['download-file', '--quiet', f'b2://{bucket_name}/uploaded_encrypted', 'gonna_fail_anyway'], expected_pattern='ERROR: The object was stored using a form of Server Side Encryption. The ' r'correct parameters must be provided to retrieve the object. \(bad_request\)' ) b2_tool.should_fail( [ - 'download-file-by-name', '--quiet', '--sourceServerSideEncryption', 'SSE-C', - bucket_name, 'uploaded_encrypted', 'gonna_fail_anyway' + 'download-file', '--quiet', '--sourceServerSideEncryption', 'SSE-C', + f'b2://{bucket_name}/uploaded_encrypted', 'gonna_fail_anyway' ], expected_pattern='ValueError: Using SSE-C requires providing an encryption key via ' 'B2_SOURCE_SSE_C_KEY_B64 env var' ) b2_tool.should_fail( [ - 'download-file-by-name', '--quiet', '--sourceServerSideEncryption', 'SSE-C', - bucket_name, 'uploaded_encrypted', 'gonna_fail_anyway' + 'download-file', '--quiet', '--sourceServerSideEncryption', 'SSE-C', + f'b2://{bucket_name}/uploaded_encrypted', 'gonna_fail_anyway' ], expected_pattern='ERROR: Wrong or no SSE-C key provided when reading a file.', additional_env={'B2_SOURCE_SSE_C_KEY_B64': base64.b64encode(os.urandom(32)).decode()} @@ -1271,13 +1270,12 @@ def test_sse_c(b2_tool, bucket_name, is_running_on_docker, sample_file, tmp_path with contextlib.nullcontext(tmp_path) as dir_path: b2_tool.should_succeed( [ - 'download-file-by-name', + 'download-file', '--noProgress', '--quiet', '--sourceServerSideEncryption', 'SSE-C', - bucket_name, - 'uploaded_encrypted', + f'b2://{bucket_name}/uploaded_encrypted', dir_path / 'a', ], additional_env={'B2_SOURCE_SSE_C_KEY_B64': base64.b64encode(secret).decode()} @@ -1285,12 +1283,12 @@ def test_sse_c(b2_tool, bucket_name, is_running_on_docker, sample_file, tmp_path assert read_file(dir_path / 'a') == read_file(sample_file) b2_tool.should_succeed( [ - 'download-file-by-id', + 'download-file', '--noProgress', '--quiet', '--sourceServerSideEncryption', 'SSE-C', - file_version_info['fileId'], + f"b2id://{file_version_info['fileId']}", dir_path / 'b', ], additional_env={'B2_SOURCE_SSE_C_KEY_B64': base64.b64encode(secret).decode()} @@ -2595,7 +2593,7 @@ def _assert_file_lock_configuration( legal_hold: LegalHold | None = None ): - file_version = b2_tool.should_succeed_json(['get-file-info', file_id]) + file_version = b2_tool.should_succeed_json(['file-info', f"b2id://{file_id}"]) if retention_mode is not None: if file_version['fileRetention']['mode'] == 'unknown': actual_file_retention = UNKNOWN_FILE_RETENTION_SETTING @@ -2677,10 +2675,10 @@ def test_download_file_stdout( b2_tool, bucket_name, sample_filepath, tmp_path, uploaded_sample_file ): assert b2_tool.should_succeed( - ['download-file-by-name', '--quiet', bucket_name, uploaded_sample_file['fileName'], '-'], + ['download-file', '--quiet', f"b2://{bucket_name}/{uploaded_sample_file['fileName']}", '-'], ).replace("\r", "") == sample_filepath.read_text() assert b2_tool.should_succeed( - ['download-file-by-id', '--quiet', uploaded_sample_file['fileId'], '-'], + ['download-file', '--quiet', f"b2id://{uploaded_sample_file['fileId']}", '-'], ).replace("\r", "") == sample_filepath.read_text() diff --git a/test/unit/console_tool/test_download_file.py b/test/unit/console_tool/test_download_file.py index 13ae320f9..a7a942cff 100644 --- a/test/unit/console_tool/test_download_file.py +++ b/test/unit/console_tool/test_download_file.py @@ -73,7 +73,9 @@ def test_download_file_by_name(b2_cli, local_file, uploaded_file, tmp_path, flag 'download-file-by-name', uploaded_file['bucket'], uploaded_file['fileName'], str(output_path) ], - expected_stdout=EXPECTED_STDOUT_DOWNLOAD + expected_stdout=EXPECTED_STDOUT_DOWNLOAD, + expected_stderr= + 'WARNING: download-file-by-name command is deprecated. Use download-file instead.\n', ) assert output_path.read_text() == uploaded_file['content'] @@ -89,7 +91,10 @@ def test_download_file_by_id(b2_cli, uploaded_file, tmp_path, flag, expected_std output_path = tmp_path / 'output.txt' b2_cli.run( - ['download-file-by-id', flag, '9999', str(output_path)], expected_stdout=expected_stdout + ['download-file-by-id', flag, '9999', str(output_path)], + expected_stdout=expected_stdout, + expected_stderr= + 'WARNING: download-file-by-id command is deprecated. Use download-file instead.\n', ) assert output_path.read_text() == uploaded_file['content'] @@ -115,7 +120,9 @@ def reader(): uploaded_file['fileName'], str(output_path) ], - expected_stdout=EXPECTED_STDOUT_DOWNLOAD + expected_stdout=EXPECTED_STDOUT_DOWNLOAD, + expected_stderr= + 'WARNING: download-file-by-name command is deprecated. Use download-file instead.\n', ) reader_future.result(timeout=1) assert output_string == uploaded_file['content'] @@ -138,6 +145,8 @@ def test_download_file_by_name__to_stdout_by_alias( """Test download_file_by_name stdout alias support""" b2_cli.run( ['download-file-by-name', '--noProgress', bucket, uploaded_stdout_txt['fileName'], '-'], + expected_stderr= + 'WARNING: download-file-by-name command is deprecated. Use download-file instead.\n', ) assert capfd.readouterr().out == uploaded_stdout_txt['content'] assert not pathlib.Path('-').exists() diff --git a/test/unit/console_tool/test_file_info.py b/test/unit/console_tool/test_file_info.py index 4cfb7e84a..e3fed5325 100644 --- a/test/unit/console_tool/test_file_info.py +++ b/test/unit/console_tool/test_file_info.py @@ -42,6 +42,7 @@ def test_get_file_info(b2_cli, uploaded_file_version): b2_cli.run( ["get-file-info", uploaded_file_version["fileId"]], expected_json_in_stdout=uploaded_file_version, + expected_stderr='WARNING: get-file-info command is deprecated. Use file-info instead.\n', ) diff --git a/test/unit/console_tool/test_get_url.py b/test/unit/console_tool/test_get_url.py index 9c4ebf98b..740d773fb 100644 --- a/test/unit/console_tool/test_get_url.py +++ b/test/unit/console_tool/test_get_url.py @@ -24,6 +24,7 @@ def test_make_url(b2_cli, uploaded_file, uploaded_file_url_by_id): b2_cli.run( ["make-url", uploaded_file["fileId"]], expected_stdout=f"{uploaded_file_url_by_id}\n", + expected_stderr='WARNING: make-url command is deprecated. Use get-url instead.\n', ) @@ -31,6 +32,7 @@ def test_make_friendly_url(b2_cli, bucket, uploaded_file, uploaded_file_url): b2_cli.run( ["make-friendly-url", bucket, uploaded_file["fileName"]], expected_stdout=f"{uploaded_file_url}\n", + expected_stderr='WARNING: make-friendly-url command is deprecated. Use get-url instead.\n', ) diff --git a/test/unit/test_arg_parser.py b/test/unit/test_arg_parser.py index a615f82b1..127551e8d 100644 --- a/test/unit/test_arg_parser.py +++ b/test/unit/test_arg_parser.py @@ -12,7 +12,7 @@ import sys from b2.arg_parser import ( - ArgumentParser, + B2ArgumentParser, parse_comma_separated_list, parse_millis_from_float_timestamp, parse_range, @@ -61,7 +61,7 @@ def check_help_string(self, command_class, command_name): help_string = command_class.__doc__ # create a parser with a help message that is based on the command_class.__doc__ string - parser = ArgumentParser(description=help_string) + parser = B2ArgumentParser(description=help_string) try: old_stdout = sys.stdout diff --git a/test/unit/test_console_tool.py b/test/unit/test_console_tool.py index 0e969c362..dd58ac162 100644 --- a/test/unit/test_console_tool.py +++ b/test/unit/test_console_tool.py @@ -927,7 +927,7 @@ def test_files(self): } self._run_command( - ['get-file-info', '9999'], + ['file-info', 'b2id://9999'], expected_json_in_stdout=expected_json, ) @@ -1074,7 +1074,7 @@ def test_files_encrypted(self): } self._run_command( - ['get-file-info', '9999'], + ['file-info', 'b2id://9999'], expected_json_in_stdout=expected_json, ) @@ -1095,10 +1095,8 @@ def test_files_encrypted(self): ''' self._run_command( - [ - 'download-file-by-name', '--noProgress', 'my-bucket', 'file1.txt', - local_download1 - ], expected_stdout, '', 0 + ['download-file', '--noProgress', 'b2://my-bucket/file1.txt', local_download1], + expected_stdout, '', 0 ) self.assertEqual(b'hello world', self._read_file(local_download1)) self.assertEqual(mod_time, int(round(os.path.getmtime(local_download1)))) @@ -1106,7 +1104,7 @@ def test_files_encrypted(self): # Download file by ID. (Same expected output as downloading by name) local_download2 = os.path.join(temp_dir, 'download2.txt') self._run_command( - ['download-file-by-id', '--noProgress', '9999', local_download2], expected_stdout, + ['download-file', '--noProgress', 'b2id://9999', local_download2], expected_stdout, '', 0 ) self.assertEqual(b'hello world', self._read_file(local_download2)) @@ -1207,7 +1205,11 @@ def _test_download_threads(self, download_by, num_threads): command += ['9999'] if download_by == 'id' else ['my-bucket', 'file.txt'] local_download = os.path.join(temp_dir, 'download.txt') command += [local_download] - self._run_command(command) + self._run_command( + command, + expected_stderr= + f'WARNING: download-file-by-{download_by} command is deprecated. Use download-file instead.\n' + ) self.assertEqual(b'hello world', self._read_file(local_download)) def test_download_by_id_1_thread(self): @@ -1309,10 +1311,7 @@ def test_copy_file_by_id(self): local_download1 = os.path.join(temp_dir, 'file1_copy.txt') self._run_command( - [ - 'download-file-by-name', '--noProgress', 'my-bucket', 'file1_copy.txt', - local_download1 - ] + ['download-file', '-q', 'b2://my-bucket/file1_copy.txt', local_download1] ) self.assertEqual(b'lo wo', self._read_file(local_download1)) @@ -1568,10 +1567,9 @@ def test_upload_incremental(self): downloaded_path = pathlib.Path(temp_dir) / 'out.txt' self._run_command( [ - 'download-file-by-name', - '--noProgress', - 'my-bucket', - 'test.txt', + 'download-file', + '-q', + 'b2://my-bucket/test.txt', str(downloaded_path), ] ) From 1b4de69648f42ba43ee58c0a41c7dffbb800ec9a Mon Sep 17 00:00:00 2001 From: Maciej Urbanski Date: Tue, 21 Nov 2023 19:13:34 +0100 Subject: [PATCH 7/9] update b2 --help in README --- README.md | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 922de2be3..18888fa0f 100644 --- a/README.md +++ b/README.md @@ -74,12 +74,11 @@ b2 create-key [-h] [--bucket BUCKET] [--namePrefix NAMEPREFIX] [--duration DURAT b2 delete-bucket [-h] bucketName b2 delete-file-version [-h] [--bypassGovernance] [fileName] fileId b2 delete-key [-h] applicationKeyId -b2 download-file-by-id [-h] [--threads THREADS] [--noProgress] [--sourceServerSideEncryption {SSE-C}] [--sourceServerSideEncryptionAlgorithm {AES256}] [--write-buffer-size BYTES] [--skip-hash-verification] [--max-download-streams-per-file MAX_DOWNLOAD_STREAMS_PER_FILE] fileId localFileName -b2 download-file-by-name [-h] [--threads THREADS] [--noProgress] [--sourceServerSideEncryption {SSE-C}] [--sourceServerSideEncryptionAlgorithm {AES256}] [--write-buffer-size BYTES] [--skip-hash-verification] [--max-download-streams-per-file MAX_DOWNLOAD_STREAMS_PER_FILE] bucketName b2FileName localFileName -b2 cat [-h] [--noProgress] [--sourceServerSideEncryption {SSE-C}] [--sourceServerSideEncryptionAlgorithm {AES256}] [--write-buffer-size BYTES] [--skip-hash-verification] b2uri +b2 download-file [-h] [--threads THREADS] [--max-download-streams-per-file MAX_DOWNLOAD_STREAMS_PER_FILE] [--noProgress] [--sourceServerSideEncryption {SSE-C}] [--sourceServerSideEncryptionAlgorithm {AES256}] [--write-buffer-size BYTES] [--skip-hash-verification] B2_URI localFileName +b2 cat [-h] [--noProgress] [--sourceServerSideEncryption {SSE-C}] [--sourceServerSideEncryptionAlgorithm {AES256}] [--write-buffer-size BYTES] [--skip-hash-verification] B2_URI b2 get-account-info [-h] b2 get-bucket [-h] [--showSize] bucketName -b2 get-file-info [-h] fileId +b2 file-info [-h] B2_URI b2 get-download-auth [-h] [--prefix PREFIX] [--duration DURATION] bucketName b2 get-download-url-with-auth [-h] [--duration DURATION] bucketName fileName b2 hide-file [-h] bucketName fileName @@ -89,8 +88,7 @@ b2 list-parts [-h] largeFileId b2 list-unfinished-large-files [-h] bucketName b2 ls [-h] [--long] [--json] [--replication] [--versions] [-r] [--withWildcard] bucketName [folderName] b2 rm [-h] [--dryRun] [--queueSize QUEUESIZE] [--noProgress] [--failFast] [--threads THREADS] [--versions] [-r] [--withWildcard] bucketName [folderName] -b2 make-url [-h] fileId -b2 make-friendly-url [-h] bucketName fileName +b2 get-url [-h] B2_URI b2 sync [-h] [--noProgress] [--dryRun] [--allowEmptySource] [--excludeAllSymlinks] [--syncThreads SYNCTHREADS] [--downloadThreads DOWNLOADTHREADS] [--uploadThreads UPLOADTHREADS] [--compareVersions {none,modTime,size}] [--compareThreshold MILLIS] [--excludeRegex REGEX] [--includeRegex REGEX] [--excludeDirRegex REGEX] [--excludeIfModifiedAfter TIMESTAMP] [--threads THREADS] [--destinationServerSideEncryption {SSE-B2,SSE-C}] [--destinationServerSideEncryptionAlgorithm {AES256}] [--sourceServerSideEncryption {SSE-C}] [--sourceServerSideEncryptionAlgorithm {AES256}] [--write-buffer-size BYTES] [--skip-hash-verification] [--max-download-streams-per-file MAX_DOWNLOAD_STREAMS_PER_FILE] [--incrementalMode] [--skipNewer | --replaceNewer] [--delete | --keepDays DAYS] source destination b2 update-bucket [-h] [--bucketInfo BUCKETINFO] [--corsRules CORSRULES] [--defaultRetentionMode {compliance,governance,none}] [--defaultRetentionPeriod period] [--replication REPLICATION] [--fileLockEnabled] [--defaultServerSideEncryption {SSE-B2,none}] [--defaultServerSideEncryptionAlgorithm {AES256}] [--lifecycleRule LIFECYCLERULES | --lifecycleRules LIFECYCLERULES] bucketName [{allPublic,allPrivate}] b2 upload-file [-h] [--contentType CONTENTTYPE] [--sha1 SHA1] [--cache-control CACHE_CONTROL] [--info INFO] [--custom-upload-timestamp CUSTOM_UPLOAD_TIMESTAMP] [--minPartSize MINPARTSIZE] [--threads THREADS] [--noProgress] [--destinationServerSideEncryption {SSE-B2,SSE-C}] [--destinationServerSideEncryptionAlgorithm {AES256}] [--legalHold {on,off}] [--fileRetentionMode {compliance,governance}] [--retainUntil TIMESTAMP] [--incrementalMode] bucketName localFilePath b2FileName From cab42e2a735343ad493e7bab40f777b7a1ac244b Mon Sep 17 00:00:00 2001 From: Maciej Urbanski Date: Wed, 22 Nov 2023 09:18:10 +0100 Subject: [PATCH 8/9] use imperative mood in internal docstring --- b2/_cli/argcompleters.py | 2 +- b2/_cli/b2args.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/b2/_cli/argcompleters.py b/b2/_cli/argcompleters.py index 8b6ff700b..a3d12dca8 100644 --- a/b2/_cli/argcompleters.py +++ b/b2/_cli/argcompleters.py @@ -57,7 +57,7 @@ def file_name_completer(api: B2Api, parsed_args, **kwargs): @_with_api def b2uri_file_completer(api: B2Api, prefix: str, **kwargs): """ - Completes B2 URI pointing to a file-like object in a bucket. + Complete B2 URI pointing to a file-like object in a bucket. """ if prefix.startswith('b2://'): prefix_without_scheme = removeprefix(prefix, 'b2://') diff --git a/b2/_cli/b2args.py b/b2/_cli/b2args.py index 3277f45de..4c9b4e2a1 100644 --- a/b2/_cli/b2args.py +++ b/b2/_cli/b2args.py @@ -35,7 +35,7 @@ def b2_file_uri(value: str) -> B2URIBase: def add_b2_file_argument(parser: argparse.ArgumentParser, name="B2_URI"): """ - Add an argument to the parser that must be a B2 URI pointing to a file. + Add a B2 URI pointing to a file as an argument to the parser. """ parser.add_argument( name, From 853ba16dbc4a57b25dbdd5bf9e56b257859b4175 Mon Sep 17 00:00:00 2001 From: Maciej Urbanski Date: Thu, 23 Nov 2023 21:01:08 +0100 Subject: [PATCH 9/9] add --help-all --- b2/arg_parser.py | 48 ++++++++++++++++++++++++++--- b2/console_tool.py | 26 ++++++++++------ test/unit/console_tool/test_help.py | 43 ++++++++++++++++++++++++++ 3 files changed, 104 insertions(+), 13 deletions(-) create mode 100644 test/unit/console_tool/test_help.py diff --git a/b2/arg_parser.py b/b2/arg_parser.py index 9420ac51e..480a251fb 100644 --- a/b2/arg_parser.py +++ b/b2/arg_parser.py @@ -15,6 +15,7 @@ import re import sys import textwrap +import unittest.mock import arrow from b2sdk.v2 import RetentionPeriod @@ -30,6 +31,10 @@ class B2RawTextHelpFormatter(argparse.RawTextHelpFormatter): It removes default "usage: " text and prints usage for all (non-hidden) subcommands. """ + def __init__(self, *args, show_all: bool = False, **kwargs): + super().__init__(*args, **kwargs) + self.show_all = show_all + def add_usage(self, usage, actions, groups, prefix=None): if prefix is None: prefix = '' @@ -39,7 +44,11 @@ def add_argument(self, action): if isinstance(action, argparse._SubParsersAction) and action.help is not argparse.SUPPRESS: usages = [] for choice in self._unique_choice_values(action): - if not getattr(choice, 'hidden', False): + deprecated = getattr(choice, 'deprecated', False) + if deprecated: + if self.show_all: + usages.append(f'(DEPRECATED) {choice.format_usage()}') + else: usages.append(choice.format_usage()) self.add_text(''.join(usages)) else: @@ -54,6 +63,14 @@ def _unique_choice_values(cls, action): yield value +class _HelpAllAction(argparse._HelpAction): + """Like argparse._HelpAction but prints help for all subcommands (even deprecated ones).""" + + def __call__(self, parser, namespace, values, option_string=None): + parser.print_help(show_all=True) + parser.exit() + + class B2ArgumentParser(argparse.ArgumentParser): """ CLI custom parser. @@ -62,18 +79,32 @@ class B2ArgumentParser(argparse.ArgumentParser): and use help message in case of error. """ - def __init__(self, *args, for_docs: bool = False, hidden: bool = False, **kwargs): + def __init__( + self, + *args, + add_help_all: bool = True, + for_docs: bool = False, + deprecated: bool = False, + **kwargs + ): """ :param for_docs: is this parser used for generating docs - :param hidden: should this parser be hidden from `--help` + :param deprecated: is this option deprecated """ self._raw_description = None self._description = None self._for_docs = for_docs - self.hidden = hidden + self.deprecated = deprecated kwargs.setdefault('formatter_class', B2RawTextHelpFormatter) super().__init__(*args, **kwargs) + if add_help_all: + self.register('action', 'help_all', _HelpAllAction) + self.add_argument( + '--help-all', + help='show help for all options, including deprecated ones', + action='help_all', + ) @property def description(self): @@ -113,6 +144,15 @@ def _get_encoding(cls): # locales are improperly configured return 'ascii' + def print_help(self, *args, show_all: bool = False, **kwargs): + """ + Print help message. + """ + with unittest.mock.patch.object( + self, 'formatter_class', functools.partial(B2RawTextHelpFormatter, show_all=show_all) + ): + super().print_help(*args, **kwargs) + def parse_comma_separated_list(s): """ diff --git a/b2/console_tool.py b/b2/console_tool.py index 2865a56c7..37564fe1a 100644 --- a/b2/console_tool.py +++ b/b2/console_tool.py @@ -687,7 +687,7 @@ class Command(Described): # Set to True for commands that receive sensitive information in arguments FORBID_LOGGING_ARGUMENTS = False - hide_from_help = False + deprecated = False # The registry for the subcommands, should be reinitialized in subclass subcommands_registry = None @@ -719,7 +719,11 @@ def register_subcommand(cls, command_class): @classmethod def create_parser( - cls, subparsers: "argparse._SubParsersAction | None" = None, parents=None, for_docs=False + cls, + subparsers: argparse._SubParsersAction | None = None, + parents=None, + for_docs=False, + name: str | None = None ) -> argparse.ArgumentParser: """ Creates a parser for the command. @@ -734,22 +738,26 @@ def create_parser( description = cls._get_description() - name, alias = cls.name_and_alias() + if name: + alias = None + else: + name, alias = cls.name_and_alias() parser_kwargs = dict( prog=name, description=description, parents=parents, for_docs=for_docs, - hidden=cls.hide_from_help, + deprecated=cls.deprecated, ) if subparsers is None: - parser = B2ArgumentParser(**parser_kwargs,) + parser = B2ArgumentParser(**parser_kwargs) else: parser = subparsers.add_parser( parser_kwargs.pop('prog'), **parser_kwargs, aliases=[alias] if alias is not None and not for_docs else (), + add_help_all=False, ) # Register class that will handle this particular command, for both name and alias. parser.set_defaults(command_class=cls) @@ -758,7 +766,7 @@ def create_parser( if cls.subcommands_registry: if not parents: - common_parser = B2ArgumentParser(add_help=False) + common_parser = B2ArgumentParser(add_help=False, add_help_all=False) common_parser.add_argument( '--debugLogs', action='store_true', help=argparse.SUPPRESS ) @@ -863,8 +871,8 @@ def __str__(self): class CmdReplacedByMixin: - hide_from_help = True - replaced_by_cmd: "type[Command]" + deprecated = True + replaced_by_cmd: type[Command] def run(self, args): self._print_stderr( @@ -3929,7 +3937,7 @@ def __init__(self, b2_api: B2Api | None, stdout, stderr): def run_command(self, argv): signal.signal(signal.SIGINT, keyboard_interrupt_handler) - parser = B2.create_parser() + parser = B2.create_parser(name=argv[0]) argcomplete.autocomplete(parser, default_completer=None) args = parser.parse_args(argv[1:]) self._setup_logging(args, argv) diff --git a/test/unit/console_tool/test_help.py b/test/unit/console_tool/test_help.py new file mode 100644 index 000000000..4dfca6c61 --- /dev/null +++ b/test/unit/console_tool/test_help.py @@ -0,0 +1,43 @@ +###################################################################### +# +# File: test/unit/console_tool/test_help.py +# +# Copyright 2023 Backblaze Inc. All Rights Reserved. +# +# License https://www.backblaze.com/using_b2_code.html +# +###################################################################### +import pytest + + +@pytest.mark.parametrize( + "flag, included, excluded", + [ + # --help shouldn't show deprecated commands + ( + "--help", + [" b2 download-file ", "-h", "--help-all"], + [" download-file-by-name ", "(DEPRECATED)"], + ), + # --help-all should show deprecated commands, but marked as deprecated + ( + "--help-all", + ["(DEPRECATED) b2 download-file-by-name ", "-h", "--help-all"], + [], + ), + ], +) +def test_help(b2_cli, flag, included, excluded, capsys): + b2_cli.run([flag], expected_stdout=None) + + out = capsys.readouterr().out + + found = set() + for i in included: + if i in out: + found.add(i) + for e in excluded: + if e in out: + found.add(e) + assert found.issuperset(included), f"expected {included!r} in {out!r}" + assert found.isdisjoint(excluded), f"expected {excluded!r} not in {out!r}"