From d946c1eee930fbc9663bbb32e6d3a009129dc210 Mon Sep 17 00:00:00 2001 From: Maciej Urbanski Date: Thu, 21 Mar 2024 19:52:45 +0100 Subject: [PATCH 01/10] prevent CD from breaking in case if DOCKER secrets are not configured --- .github/workflows/cd.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index 2874aa33b..9ce190afb 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -25,7 +25,7 @@ jobs: run: | export IS_PRERELEASE=$([[ ${{ github.ref }} =~ [^0-9]$ ]] && echo true || echo false) echo "prerelease=$IS_PRERELEASE" >> $GITHUB_OUTPUT - export PUBLISH_DOCKER=$([[ $IS_PRERELEASE == 'false' && ${{ secrets.DOCKERHUB_USERNAME }} != '' ]] && echo true || echo false) + export PUBLISH_DOCKER=$([[ $IS_PRERELEASE == 'false' && "${{ secrets.DOCKERHUB_USERNAME }}" != '' ]] && echo true || echo false) echo "publish_docker=$PUBLISH_DOCKER" >> $GITHUB_OUTPUT - uses: actions/checkout@v4 with: From ee9d0b390d2493fcb2440c9a4c27ef16086668db Mon Sep 17 00:00:00 2001 From: Maciej Urbanski Date: Fri, 9 Feb 2024 14:50:23 +0100 Subject: [PATCH 02/10] add yaml support --- b2/_internal/_cli/obj_loads.py | 14 ++++-- changelog.d/+yaml.added.md | 1 + pyproject.toml | 1 + test/unit/_cli/test_obj_loads.py | 81 ++++++++++++++++++++++++++++++++ 4 files changed, 94 insertions(+), 3 deletions(-) create mode 100644 changelog.d/+yaml.added.md create mode 100644 test/unit/_cli/test_obj_loads.py diff --git a/b2/_internal/_cli/obj_loads.py b/b2/_internal/_cli/obj_loads.py index 5ad2b48ef..67fdfb4c9 100644 --- a/b2/_internal/_cli/obj_loads.py +++ b/b2/_internal/_cli/obj_loads.py @@ -14,6 +14,7 @@ import json from typing import TypeVar +import yaml from b2sdk.v2 import get_b2sdk_doc_urls try: @@ -43,15 +44,22 @@ def describe_type(type_) -> str: def validated_loads(data: str, expected_type: type[T] | None = None) -> T: + try: + try: + val = json.loads(data) + except json.JSONDecodeError: + val = yaml.safe_load(data) + except yaml.YAMLError as e: + raise argparse.ArgumentTypeError(f'{data!r} is not a valid JSON/YAML: {e}') from e + if expected_type is not None and pydantic is not None: ta = TypeAdapter(expected_type) try: - val = ta.validate_json(data) + val = ta.validate_python(data) except ValidationError as e: errors = convert_error_to_human_readable(e) raise argparse.ArgumentTypeError( f'Invalid value inputted, expected {describe_type(expected_type)}, got {data!r}, more detail below:\n{errors}' ) from e - else: - val = json.loads(data) + return val diff --git a/changelog.d/+yaml.added.md b/changelog.d/+yaml.added.md new file mode 100644 index 000000000..af144c44d --- /dev/null +++ b/changelog.d/+yaml.added.md @@ -0,0 +1 @@ +Add `yaml` format fallback in addition to `json` format supported in some of the arguments. \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 72cf0e16a..9a3153eaf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,6 +35,7 @@ dependencies = [ "tqdm>=4.65.0,<5", "platformdirs>=3.11.0,<5", "setuptools>=60; python_version < '3.10'", # required by phx-class-registry<4.1 + "PyYAML>=6,<7", ] [project.optional-dependencies] diff --git a/test/unit/_cli/test_obj_loads.py b/test/unit/_cli/test_obj_loads.py new file mode 100644 index 000000000..275373d26 --- /dev/null +++ b/test/unit/_cli/test_obj_loads.py @@ -0,0 +1,81 @@ +###################################################################### +# +# File: test/unit/_cli/test_obj_loads.py +# +# Copyright 2024 Backblaze Inc. All Rights Reserved. +# +# License https://www.backblaze.com/using_b2_code.html +# +###################################################################### +import argparse + +import pytest + +from b2._internal._cli.obj_loads import validated_loads + + +@pytest.mark.parametrize( + "input_, expected_val", + [ + # json + ('{"a": 1}', { + "a": 1 + }), + ('{"a": 1, "b": 2}', { + "a": 1, + "b": 2 + }), + ('{"a": 1, "b": 2, "c": 3}', { + "a": 1, + "b": 2, + "c": 3 + }), + # yaml + ("a: 1", { + "a": 1 + }), + ("a: 1\nb: 2", { + "a": 1, + "b": 2 + }), + ("a: 1\nb: 2\nc: 3", { + "a": 1, + "b": 2, + "c": 3 + }), + # yaml one-liners + ("a: 1", { + "a": 1 + }), + ("{a: 1,b: 2}", { + "a": 1, + "b": 2 + }), + ("{a: test,b: 2,sub:{c: 3}}", { + "a": "test", + "b": 2, + "sub": { + "c": 3 + } + }), + ], +) +def test_validated_loads(input_, expected_val): + assert validated_loads(input_) == expected_val + + +@pytest.mark.parametrize( + "input_, error_msg", + [ + # not valid json nor yaml + ("{", "'{' is not a valid JSON/YAML:"), + # not-valid yaml + ( + "a: 1, a: 2", + r"a: 1, a: 2\' is not a valid JSON/YAML: mapping values are not allowed here", + ), + ], +) +def test_validated_loads__invalid_syntax(input_, error_msg): + with pytest.raises(argparse.ArgumentTypeError, match=error_msg): + validated_loads(input_) From 1919dab732dae0c9d749e42eb01f97d2137260ec Mon Sep 17 00:00:00 2001 From: Maciej Urbanski Date: Mon, 12 Feb 2024 10:42:21 +0100 Subject: [PATCH 03/10] added notifications --- b2/_internal/_b2v4/registry.py | 1 + b2/_internal/_cli/argcompleters.py | 9 +- b2/_internal/_cli/b2args.py | 41 +- b2/_internal/_utils/uri.py | 15 +- b2/_internal/b2v3/registry.py | 1 + b2/_internal/console_tool.py | 474 +++++++++++++++++- changelog.d/+notification_rules.added.md | 1 + test/integration/test_b2_command_line.py | 62 +++ .../console_tool/test_authorize_account.py | 2 + .../console_tool/test_notification_rules.py | 258 ++++++++++ test/unit/test_console_tool.py | 2 +- 11 files changed, 836 insertions(+), 30 deletions(-) create mode 100644 changelog.d/+notification_rules.added.md create mode 100644 test/unit/console_tool/test_notification_rules.py diff --git a/b2/_internal/_b2v4/registry.py b/b2/_internal/_b2v4/registry.py index d6deb14e3..36dabfaaa 100644 --- a/b2/_internal/_b2v4/registry.py +++ b/b2/_internal/_b2v4/registry.py @@ -55,3 +55,4 @@ B2.register_subcommand(Version) B2.register_subcommand(License) B2.register_subcommand(InstallAutocomplete) +B2.register_subcommand(NotificationRules) diff --git a/b2/_internal/_cli/argcompleters.py b/b2/_internal/_cli/argcompleters.py index 06a67b320..ebf62b0b4 100644 --- a/b2/_internal/_cli/argcompleters.py +++ b/b2/_internal/_cli/argcompleters.py @@ -7,11 +7,11 @@ # License https://www.backblaze.com/using_b2_code.html # ###################################################################### +import itertools # We import all the necessary modules lazily in completers in order # to avoid upfront cost of the imports when argcompleter is used for # autocompletions. - from itertools import islice @@ -20,7 +20,12 @@ def bucket_name_completer(prefix, parsed_args, **kwargs): from b2._internal._cli.b2api import _get_b2api_for_profile api = _get_b2api_for_profile(getattr(parsed_args, 'profile', None)) - res = [unprintable_to_hex(bucket.name) for bucket in api.list_buckets(use_cache=True)] + res = [ + unprintable_to_hex(bucket_name_alias) + for bucket_name_alias in itertools.chain.from_iterable( + (bucket.name, f"b2://{bucket.name}") for bucket in api.list_buckets(use_cache=True) + ) + ] return res diff --git a/b2/_internal/_cli/b2args.py b/b2/_internal/_cli/b2args.py index 2a71a0cd7..61e84d886 100644 --- a/b2/_internal/_cli/b2args.py +++ b/b2/_internal/_cli/b2args.py @@ -16,12 +16,12 @@ from typing import Optional, Tuple from b2._internal._cli.arg_parser_types import wrap_with_argument_type_error -from b2._internal._cli.argcompleters import b2uri_file_completer +from b2._internal._cli.argcompleters import b2uri_file_completer, bucket_name_completer from b2._internal._cli.const import ( B2_APPLICATION_KEY_ENV_VAR, B2_APPLICATION_KEY_ID_ENV_VAR, ) -from b2._internal._utils.uri import B2URI, B2URIBase, parse_b2_uri +from b2._internal._utils.uri import B2URI, B2URIBase, parse_b2_uri, parse_uri def b2id_or_file_like_b2_uri(value: str) -> B2URIBase: @@ -36,6 +36,18 @@ def b2id_or_file_like_b2_uri(value: str) -> B2URIBase: return b2_uri +def parse_bucket_name(value: str) -> str: + uri = parse_uri(value) + if isinstance(uri, B2URI): + if uri.path: + raise ValueError( + f"Expected a bucket name, but {value!r} was provided which contains path part: {uri.path!r}" + ) + return uri.bucket_name + return str(value) + + +B2_BUCKET_NAME_ARG_TYPE = wrap_with_argument_type_error(parse_bucket_name) B2ID_OR_B2_URI_ARG_TYPE = wrap_with_argument_type_error(parse_b2_uri) B2ID_OR_B2_URI_OR_ALL_BUCKETS_ARG_TYPE = wrap_with_argument_type_error( functools.partial(parse_b2_uri, allow_all_buckets=True) @@ -43,6 +55,31 @@ def b2id_or_file_like_b2_uri(value: str) -> B2URIBase: B2ID_OR_FILE_LIKE_B2_URI_ARG_TYPE = wrap_with_argument_type_error(b2id_or_file_like_b2_uri) +def add_bucket_name_argument( + parser: argparse.ArgumentParser, name="bucketName", help="Target bucket name", nargs=None +): + parser.add_argument( + name, type=B2_BUCKET_NAME_ARG_TYPE, help=help, nargs=nargs + ).completer = bucket_name_completer + + +def add_b2_uri_argument( + parser: argparse.ArgumentParser, + name="B2_URI", + help="B2 URI pointing to a bucket with optional path, e.g. b2://yourBucket, b2://yourBucket/file.txt, b2://yourBucket/folderName/", +): + """ + Add B2 URI as an argument to the parser. + + B2 URI can point to a bucket optionally with a object name prefix (directory). + """ + parser.add_argument( + name, + type=wrap_with_argument_type_error(functools.partial(parse_b2_uri, allow_b2id=False)), + help=help, + ).completer = b2uri_file_completer + + def add_b2id_or_b2_uri_argument( parser: argparse.ArgumentParser, name="B2_URI", *, allow_all_buckets: bool = False ): diff --git a/b2/_internal/_utils/uri.py b/b2/_internal/_utils/uri.py index cdee5f60e..fb61f088a 100644 --- a/b2/_internal/_utils/uri.py +++ b/b2/_internal/_utils/uri.py @@ -97,21 +97,28 @@ def parse_uri(uri: str, *, allow_all_buckets: bool = False) -> Path | B2URI | B2 return _parse_b2_uri(uri, parsed, allow_all_buckets=allow_all_buckets) -def parse_b2_uri(uri: str, *, allow_all_buckets: bool = False) -> B2URI | B2FileIdURI: +def parse_b2_uri( + uri: str, *, allow_all_buckets: bool = False, allow_b2id: bool = True +) -> B2URI | B2FileIdURI: """ Parse B2 URI. :param uri: string to parse :param allow_all_buckets: if True, allow `b2://` without a bucket name to refer to all buckets + :param allow_b2id: if True, allow `b2id://` to refer to a file by its id :return: B2 URI :raises ValueError: if the URI is invalid """ parsed = urllib.parse.urlsplit(uri) - return _parse_b2_uri(uri, parsed, allow_all_buckets=allow_all_buckets) + return _parse_b2_uri(uri, parsed, allow_all_buckets=allow_all_buckets, allow_b2id=allow_b2id) def _parse_b2_uri( - uri, parsed: urllib.parse.SplitResult, allow_all_buckets: bool = False + uri, + parsed: urllib.parse.SplitResult, + *, + allow_all_buckets: bool = False, + allow_b2id: bool = True ) -> B2URI | B2FileIdURI: if parsed.scheme in ("b2", "b2id"): path = urllib.parse.urlunsplit(parsed._replace(scheme="", netloc="")) @@ -130,7 +137,7 @@ def _parse_b2_uri( if parsed.scheme == "b2": return B2URI(bucket_name=parsed.netloc, path=removeprefix(path, "/")) - elif parsed.scheme == "b2id": + elif parsed.scheme == "b2id" and allow_b2id: return B2FileIdURI(file_id=parsed.netloc) else: raise ValueError(f"Unsupported URI scheme: {parsed.scheme!r}") diff --git a/b2/_internal/b2v3/registry.py b/b2/_internal/b2v3/registry.py index 01186c93f..665ec1e1e 100644 --- a/b2/_internal/b2v3/registry.py +++ b/b2/_internal/b2v3/registry.py @@ -135,3 +135,4 @@ class Ls(B2URIBucketNFolderNameArgMixin, BaseLs): B2.register_subcommand(Version) B2.register_subcommand(License) B2.register_subcommand(InstallAutocomplete) +B2.register_subcommand(NotificationRules) diff --git a/b2/_internal/console_tool.py b/b2/_internal/console_tool.py index ce89841fc..f217672b6 100644 --- a/b2/_internal/console_tool.py +++ b/b2/_internal/console_tool.py @@ -12,6 +12,7 @@ # ruff: noqa: E402 from __future__ import annotations +import copy import tempfile from b2._internal._cli.autocomplete_cache import AUTOCOMPLETE # noqa @@ -52,6 +53,7 @@ import b2sdk import requests import rst2ansi +import yaml from b2sdk.v2 import ( ALL_CAPABILITIES, B2_ACCOUNT_INFO_DEFAULT_FILE, @@ -100,6 +102,7 @@ escape_control_chars, get_included_sources, make_progress_listener, + notification_rule_response_to_request, parse_sync_folder, points_to_fifo, substitute_control_chars, @@ -124,7 +127,7 @@ parse_millis_from_float_timestamp, parse_range, ) -from b2._internal._cli.argcompleters import bucket_name_completer, file_name_completer +from b2._internal._cli.argcompleters import file_name_completer from b2._internal._cli.autocomplete_install import ( SUPPORTED_SHELLS, AutocompleteInstallError, @@ -132,8 +135,10 @@ ) from b2._internal._cli.b2api import _get_b2api_for_profile, _get_inmemory_b2api from b2._internal._cli.b2args import ( + add_b2_uri_argument, add_b2id_or_b2_uri_argument, add_b2id_or_file_like_b2_uri_argument, + add_bucket_name_argument, get_keyid_and_key_from_env_vars, ) from b2._internal._cli.const import ( @@ -170,6 +175,27 @@ VERSION_0_COMPATIBILITY = False +def filter_out_empty_values(v, empty_marker=None): + if isinstance(v, dict): + d = {} + for k, v in v.items(): + new_v = filter_out_empty_values(v, empty_marker=empty_marker) + if new_v is not empty_marker: + d[k] = new_v + return d or empty_marker + return v + + +def override_dict(base_dict, override): + result = copy.deepcopy(base_dict) + for k, v in override.items(): + if isinstance(v, dict): + result[k] = override_dict(result.get(k, {}), v) + else: + result[k] = v + return result + + class NoControlCharactersStdout: def __init__(self, stdout): self.stdout = stdout @@ -304,6 +330,7 @@ def _get_description(cls, **kwargs): for klass in cls.mro() if klass is not cls and klass.__doc__ and issubclass(klass, Described) } + mro_docs.update(**{name.upper(): value for name, value in mro_docs.items()}) return cls.__doc__.format(**kwargs, **DOC_STRING_DATA, **mro_docs) @classmethod @@ -311,6 +338,21 @@ def lazy_get_description(cls, **kwargs): return DescriptionGetter(cls, **kwargs) +class JSONOptionMixin(Described): + """ + Use ``--json`` to get machine-readable output. + Unless ``--json`` is used, the output is human-readable, and may change from one minor version to the next. + Therefore, for scripting, it is strongly encouraged to use ``--json``. + """ + + @classmethod + def _setup_parser(cls, parser): + parser.add_argument( + '--json', action='store_true', help='output in JSON format to use in scripts' + ) + super()._setup_parser(parser) # noqa + + class DefaultSseMixin(Described): """ If you want server-side encryption for all of the files that are uploaded to a bucket, @@ -672,11 +714,21 @@ def get_b2_uri_from_arg(self, args: argparse.Namespace) -> B2URIBase: return B2FileIdURI(args.fileId) +class B2URIBucketArgMixin: + @classmethod + def _setup_parser(cls, parser): + add_bucket_name_argument(parser) + super()._setup_parser(parser) + + def get_b2_uri_from_arg(self, args: argparse.Namespace) -> B2URIBase: + return B2URI(args.bucketName) + + class B2URIBucketNFilenameArgMixin: @classmethod def _setup_parser(cls, parser): - parser.add_argument('bucketName') - parser.add_argument('fileName') + add_bucket_name_argument(parser) + parser.add_argument('fileName').completion = file_name_completer super()._setup_parser(parser) def get_b2_uri_from_arg(self, args: argparse.Namespace) -> B2URIBase: @@ -688,10 +740,7 @@ class B2URIBucketNFolderNameArgMixin: @classmethod def _setup_parser(cls, parser): - parser.add_argument( - 'bucketName', - nargs='?' if cls.ALLOW_ALL_BUCKETS else None, - ).completer = bucket_name_completer + add_bucket_name_argument(parser, nargs='?' if cls.ALLOW_ALL_BUCKETS else None) parser.add_argument('folderName', nargs='?').completer = file_name_completer super()._setup_parser(parser) @@ -820,6 +869,7 @@ def __exit__(self, exc_type, exc_val, exc_tb): class Command(Described, metaclass=ABCMeta): + COMMAND_NAME: str | None = None # Set to True for commands that receive sensitive information in arguments FORBID_LOGGING_ARGUMENTS = False @@ -837,6 +887,7 @@ def __init__(self, console_tool): self.stdout = console_tool.stdout self.stderr = console_tool.stderr self.quiet = False + self.escape_control_characters = True self.exit_stack = contextlib.ExitStack() def make_progress_listener(self, file_name: str, quiet: bool): @@ -848,7 +899,7 @@ def make_progress_listener(self, file_name: str, quiet: bool): @classmethod def name_and_alias(cls): - name = mixed_case_to_hyphens(cls.__name__) + name = mixed_case_to_hyphens(cls.COMMAND_NAME or cls.__name__) alias = None if '-' in name: alias = name.replace('-', '_') @@ -957,6 +1008,7 @@ def create_parser( def run(self, args): self.quiet = args.quiet + self.escape_control_characters = args.escape_control_characters with self.exit_stack: return self._run(args) @@ -984,6 +1036,9 @@ def _print_json(self, data) -> None: enforce_output=True ) + def _print_human_readable_structure(self, data) -> None: + return self._print(yaml.dump(data, sort_keys=True).rstrip()) + def _print( self, *args, @@ -1021,7 +1076,14 @@ def _print_standard_descriptor( :param end: end of the line characters; None for default newline """ if not self.quiet or enforce_output: - self._print_helper(descriptor, descriptor.encoding, descriptor_name, *args, end=end) + self._print_helper( + descriptor, + descriptor.encoding, + descriptor_name, + *args, + end=end, + sanitize=self.escape_control_characters + ) @classmethod def _print_helper( @@ -1030,8 +1092,11 @@ def _print_helper( descriptor_encoding: str, descriptor_name: str, *args, + sanitize: bool = True, end: str | None = None ): + if sanitize: + args = tuple(unprintable_to_hex(arg) or '' for arg in args) try: descriptor.write(' '.join(args)) except UnicodeEncodeError: @@ -1280,7 +1345,7 @@ class CancelAllUnfinishedLargeFiles(Command): @classmethod def _setup_parser(cls, parser): - parser.add_argument('bucketName').completer = bucket_name_completer + add_bucket_name_argument(parser) super()._setup_parser(parser) def _run(self, args): @@ -1509,7 +1574,7 @@ def _setup_parser(cls, parser): "If given, the bucket will have the file lock mechanism enabled. This parameter cannot be changed after bucket creation." ) parser.add_argument('--replication', type=validated_loads) - parser.add_argument('bucketName') + add_bucket_name_argument(parser) parser.add_argument('bucketType', choices=CREATE_BUCKET_TYPES) super()._setup_parser(parser) # add parameters from the mixins @@ -1601,7 +1666,7 @@ class DeleteBucket(Command): @classmethod def _setup_parser(cls, parser): - parser.add_argument('bucketName').completer = bucket_name_completer + add_bucket_name_argument(parser) super()._setup_parser(parser) def _run(self, args): @@ -1941,7 +2006,7 @@ class GetBucket(Command): @classmethod def _setup_parser(cls, parser): add_normalized_argument(parser, '--show-size', action='store_true') - parser.add_argument('bucketName').completer = bucket_name_completer + add_bucket_name_argument(parser) super()._setup_parser(parser) def _run(self, args): @@ -2019,7 +2084,7 @@ class GetDownloadAuth(Command): def _setup_parser(cls, parser): parser.add_argument('--prefix', default='') parser.add_argument('--duration', type=int, default=86400) - parser.add_argument('bucketName').completer = bucket_name_completer + add_bucket_name_argument(parser) super()._setup_parser(parser) def _run(self, args): @@ -2052,7 +2117,7 @@ class GetDownloadUrlWithAuth(Command): @classmethod def _setup_parser(cls, parser): parser.add_argument('--duration', type=int, default=86400) - parser.add_argument('bucketName').completer = bucket_name_completer + add_bucket_name_argument(parser) parser.add_argument('fileName').completer = file_name_completer super()._setup_parser(parser) @@ -2078,7 +2143,7 @@ class HideFile(Command): @classmethod def _setup_parser(cls, parser): - parser.add_argument('bucketName').completer = bucket_name_completer + add_bucket_name_argument(parser) parser.add_argument('fileName').completer = file_name_completer super()._setup_parser(parser) @@ -2243,7 +2308,7 @@ class ListUnfinishedLargeFiles(Command): @classmethod def _setup_parser(cls, parser): - parser.add_argument('bucketName').completer = bucket_name_completer + add_bucket_name_argument(parser) super()._setup_parser(parser) def _run(self, args): @@ -3151,7 +3216,7 @@ def _setup_parser(cls, parser): help= "If given, the bucket will have the file lock mechanism enabled. This parameter cannot be changed back." ) - parser.add_argument('bucketName').completer = bucket_name_completer + add_bucket_name_argument(parser) parser.add_argument('bucketType', nargs='?', choices=CREATE_BUCKET_TYPES) super()._setup_parser(parser) # add parameters from the mixins and the parent class @@ -3249,9 +3314,7 @@ def _setup_parser(cls, parser): type=int, help="overrides object creation date. Expressed as a number of milliseconds since epoch." ) - parser.add_argument( - 'bucketName', help="name of the bucket where the file will be stored" - ).completer = bucket_name_completer + add_bucket_name_argument(parser, help="name of the bucket where the file will be stored") parser.add_argument('localFilePath', help="path of the local file or stream to be uploaded") parser.add_argument('b2FileName', help="name file will be given when stored in B2") @@ -4174,6 +4237,375 @@ def _run(self, args): return 0 +class NotificationRules(Command): + """ + Bucket notification rules management subcommands. + + For more information on each subcommand, use ``{NAME} notification-rules SUBCOMMAND --help``. + + Examples: + + .. code-block:: + + {NAME} notification-rules create b2://bucketName/optionalSubPath/ ruleName --event-type "b2:ObjectCreated:*" --webhook-url https://example.com/webhook + {NAME} notification-rules list b2://bucketName + {NAME} notification-rules update b2://bucketName/newPath/ ruleName --disable --event-type "b2:ObjectCreated:*" --event-type "b2:ObjectHidden:*" + {NAME} notification-rules delete b2://bucketName ruleName + """ + subcommands_registry = ClassRegistry(attr_name='COMMAND_NAME') + + +@NotificationRules.subcommands_registry.register +class NotificationRulesList(JSONOptionMixin, Command): + """ + Allows listing bucket notification rules of the given bucket. + + + {JSONOptionMixin} + + Examples: + + .. code-block:: + + {NAME} notification-rules list b2://bucketName + + + Requires capability: + + - **readBucketNotifications** + """ + COMMAND_NAME = 'list' + + @classmethod + def _setup_parser(cls, parser): + add_b2_uri_argument( + parser, + help= + "B2 URI of the bucket with optional path prefix, e.g. b2://bucketName or b2://bucketName/optionalSubPath/" + ) + super()._setup_parser(parser) + + def _run(self, args): + bucket = self.api.get_bucket_by_name(args.B2_URI.bucket_name) + rules = sorted( + ( + rule for rule in bucket.get_notification_rules() + if rule["objectNamePrefix"].startswith(args.B2_URI.path) + ), + key=lambda rule: rule["name"] + ) + if args.json: + self._print_json(rules) + else: + if rules: + self._print(f'Notification rules for {args.B2_URI} :') + self._print_human_readable_structure(rules) + else: + self._print(f'No notification rules for {args.B2_URI}') + return 0 + + +class NotificationRulesCreateBase(JSONOptionMixin, Command): + @classmethod + def _validate_secret(cls, value: str) -> str: + if not re.match(r'^[a-zA-Z0-9]{32}$', value): + raise argparse.ArgumentTypeError( + f'the secret has to be exactly 32 alphanumeric characters, got: {value!r}' + ) + return value + + @classmethod + def setup_rule_fields_parser(cls, parser, creation: bool): + add_b2_uri_argument( + parser, + help= + "B2 URI of the bucket with optional path prefix, e.g. b2://bucketName or b2://bucketName/optionalSubPath/" + ) + parser.add_argument('ruleName', help="Name of the rule") + parser.add_argument( + '--event-type', + action='append', + help= + "Events scope, e.g., 'b2:ObjectCreated:*'. Can be used multiple times to set multiple scopes.", + required=creation + ) + parser.add_argument( + '--webhook-url', help="URL to send the notification to", required=creation + ) + parser.add_argument( + '--sign-secret', + help="optional signature key consisting of 32 alphanumeric characters ", + type=cls._validate_secret, + default=None, + ) + parser.add_argument( + '--custom-header', + action='append', + help= + "Custom header to be sent with the notification. Can be used multiple times to set multiple headers. Format: HEADER_NAME=VALUE" + ) + parser.add_argument( + '--enable', + action='store_true', + help="Flag to enable the notification rule", + default=None + ) + parser.add_argument( + '--disable', + action='store_false', + help="Flag to disable the notification rule", + dest='enable' + ) + + def get_rule_from_args(self, args): + custom_headers = None + if args.custom_header is not None: + custom_headers = {} + for header in args.custom_header: + try: + name, value = header.split('=', 1) + except ValueError: + name, value = header, '' + custom_headers[name] = value + + rule = { + 'name': args.ruleName, + 'eventTypes': args.event_type, + 'isEnabled': args.enable, + 'objectNamePrefix': args.B2_URI.path, + 'targetConfiguration': + { + 'url': args.webhook_url, + 'customHeaders': custom_headers, + 'hmacSha256SigningSecret': args.sign_secret, + }, + } + return filter_out_empty_values(rule) + + def print_rule(self, args, rule): + if args.json: + self._print_json(rule) + else: + self._print_human_readable_structure(rule) + + +class NotificationRulesUpdateBase(NotificationRulesCreateBase): + def _run(self, args): + bucket = self.api.get_bucket_by_name(args.B2_URI.bucket_name) + rules_by_name = {rule["name"]: rule for rule in bucket.get_notification_rules()} + rule = rules_by_name.get(args.ruleName) + if not rule: + raise CommandError( + f'rule with name {args.ruleName!r} does not exist on bucket {bucket.name!r}, ' + f'available rules: {sorted(rules_by_name)}' + ) + + rules_by_name[args.ruleName] = override_dict( + rule, + self.get_rule_from_args(args), + ) + + rules = bucket.set_notification_rules( + [notification_rule_response_to_request(rule) for rule in rules_by_name.values()] + ) + rule = next(rule for rule in rules if rule["name"] == args.ruleName) + self.print_rule(args=args, rule=rule) + return 0 + + +@NotificationRules.subcommands_registry.register +class NotificationRulesCreate(NotificationRulesCreateBase): + """ + Allows creating bucket notification rules for the given bucket. + + + Examples: + + .. code-block:: + + {NAME} notification-rules create b2://bucketName/optionalSubPath/ ruleName --event-type "b2:ObjectCreated:*" --webhook-url https://example.com/webhook + + + Requires capability: + + - **readBucketNotifications** + - **writeBucketNotifications** + """ + COMMAND_NAME = 'create' + + NEW_RULE_DEFAULTS = { + 'isEnabled': True, + 'objectNamePrefix': '', + 'targetConfiguration': { + 'targetType': 'webhook', + }, + } + + @classmethod + def _setup_parser(cls, parser): + cls.setup_rule_fields_parser(parser, creation=True) + super()._setup_parser(parser) + + def _run(self, args): + bucket = self.api.get_bucket_by_name(args.B2_URI.bucket_name) + rules_by_name = {rule["name"]: rule for rule in bucket.get_notification_rules()} + if args.ruleName in rules_by_name: + raise CommandError( + f'rule with name {args.ruleName!r} already exists on bucket {bucket.name!r}' + ) + + rule = override_dict( + self.NEW_RULE_DEFAULTS, + self.get_rule_from_args(args), + ) + rules_by_name[args.ruleName] = rule + + rules = bucket.set_notification_rules( + [ + notification_rule_response_to_request(rule) + for rule in sorted(rules_by_name.values(), key=lambda r: r["name"]) + ] + ) + rule = next(rule for rule in rules if rule["name"] == args.ruleName) + self.print_rule(args=args, rule=rule) + return 0 + + +@NotificationRules.subcommands_registry.register +class NotificationRulesUpdate(NotificationRulesUpdateBase): + """ + Allows updating notification rule of the given bucket. + + + Examples: + + .. code-block:: + + {NAME} notification-rules update b2://bucketName/newPath/ ruleName --disable --event-type "b2:ObjectCreated:*" --event-type "b2:ObjectHidden:*" + {NAME} notification-rules update b2://bucketName/newPath/ ruleName --enable + + + Requires capability: + + - **readBucketNotifications** + - **writeBucketNotifications** + """ + + COMMAND_NAME = 'update' + + @classmethod + def _setup_parser(cls, parser): + cls.setup_rule_fields_parser(parser, creation=False) + super()._setup_parser(parser) + + +@NotificationRules.subcommands_registry.register +class NotificationRulesEnable(NotificationRulesUpdateBase): + """ + Allows enabling notification rule of the given bucket. + + + Examples: + + .. code-block:: + + {NAME} notification-rules enable b2://bucketName/ ruleName + + + Requires capability: + + - **readBucketNotifications** + - **writeBucketNotifications** + """ + + COMMAND_NAME = 'enable' + + @classmethod + def _setup_parser(cls, parser): + add_b2_uri_argument( + parser, help="B2 URI of the bucket to enable the rule for, e.g. b2://bucketName" + ) + parser.add_argument('ruleName', help="Name of the rule to enable") + super()._setup_parser(parser) + + def get_rule_from_args(self, args): + logger.warning("WARNING: ignoring path from %r", args.B2_URI) + return {'name': args.ruleName, 'isEnabled': True} + + +@NotificationRules.subcommands_registry.register +class NotificationRulesDisable(NotificationRulesUpdateBase): + """ + Allows disabling notification rule of the given bucket. + + + Examples: + + .. code-block:: + + {NAME} notification-rules disable b2://bucketName/ ruleName + + + Requires capability: + + - **readBucketNotifications** + - **writeBucketNotifications** + """ + + COMMAND_NAME = 'disable' + + @classmethod + def _setup_parser(cls, parser): + add_b2_uri_argument( + parser, help="B2 URI of the bucket to enable the rule for, e.g. b2://bucketName" + ) + parser.add_argument('ruleName', help="Name of the rule to enable") + super()._setup_parser(parser) + + def get_rule_from_args(self, args): + logger.warning("WARNING: ignoring path from %r", args.B2_URI) + return {'name': args.ruleName, 'isEnabled': False} + + +@NotificationRules.subcommands_registry.register +class NotificationRulesDelete(Command): + """ + Allows deleting bucket notification rule of the given bucket. + + Requires capability: + + - **readBucketNotifications** + - **writeBucketNotifications** + """ + + COMMAND_NAME = 'delete' + + @classmethod + def _setup_parser(cls, parser): + add_b2_uri_argument( + parser, help="B2 URI of the bucket to delete the rule from, e.g. b2://bucketName" + ) + parser.add_argument('ruleName', help="Name of the rule to delete") + super()._setup_parser(parser) + + def _run(self, args): + bucket = self.api.get_bucket_by_name(args.B2_URI.bucket_name) + rules_by_name = {rule["name"]: rule for rule in bucket.get_notification_rules()} + + try: + del rules_by_name[args.ruleName] + except KeyError: + raise CommandError( + f'no such rule to delete: {args.ruleName!r}, ' + f'available rules: {sorted(rules_by_name.keys())!r}; No rules have been deleted.' + ) + bucket.set_notification_rules( + [notification_rule_response_to_request(rule) for rule in rules_by_name.values()] + ) + self._print(f'Rule {args.ruleName!r} has been deleted from {args.B2_URI}') + return 0 + + class ConsoleTool: """ Implements the commands available in the B2 command-line tool diff --git a/changelog.d/+notification_rules.added.md b/changelog.d/+notification_rules.added.md new file mode 100644 index 000000000..6b49b0441 --- /dev/null +++ b/changelog.d/+notification_rules.added.md @@ -0,0 +1 @@ +Add `notification-rules` commands for manipulating Bucket notification rules. \ No newline at end of file diff --git a/test/integration/test_b2_command_line.py b/test/integration/test_b2_command_line.py index a62cfd470..a8772e840 100755 --- a/test/integration/test_b2_command_line.py +++ b/test/integration/test_b2_command_line.py @@ -3067,3 +3067,65 @@ def assert_expected(file_info, expected=expected_file_info): assert re.search(r'ContentEncoding: *gzip', download_output) assert re.search(r'ContentLanguage: *en', download_output) assert re.search(r'Expires: *Thu, 01 Dec 2050 16:00:00 GMT', download_output) + + +def test_notification_rules(b2_tool, bucket_name): + assert b2_tool.should_succeed_json( + ["notification-rules", "list", f"b2://{bucket_name}", "--json"] + ) == [] + + notification_rule = { + "eventTypes": ["b2:ObjectCreated:*"], + "isEnabled": True, + "name": "test-rule", + "objectNamePrefix": "", + "targetConfiguration": + { + "customHeaders": None, + "targetType": "webhook", + "url": "https://example.com/webhook", + } + } + # add rule + created_rule = b2_tool.should_succeed_json( + [ + "notification-rules", + "create", + "--json", + f"b2://{bucket_name}", + "test-rule", + "--webhook-url", + "https://example.com/webhook", + "--event-type", + "b2:ObjectCreated:*", + ] + ) + expected_rules = [{**notification_rule, "isSuspended": False, "suspensionReason": ""}] + assert created_rule == expected_rules[0] + + # modify rule + modified_rule = b2_tool.should_succeed_json( + [ + "notification-rules", + "update", + "--json", + f"b2://{bucket_name}/prefix", + "test-rule", + "--disable", + ] + ) + expected_rules[0].update({"objectNamePrefix": "prefix", "isEnabled": False}) + assert modified_rule == expected_rules[0] + + # read updated rules + assert b2_tool.should_succeed_json( + ["notification-rules", "list", f"b2://{bucket_name}", "--json"] + ) == expected_rules + + # delete rule by name + assert b2_tool.should_succeed( + ["notification-rules", "delete", f"b2://{bucket_name}", "test-rule"], + ) == f"Rule 'test-rule' has been deleted from b2://{bucket_name}/\n" + assert b2_tool.should_succeed_json( + ["notification-rules", "list", f"b2://{bucket_name}", "--json"] + ) == [] diff --git a/test/unit/console_tool/test_authorize_account.py b/test/unit/console_tool/test_authorize_account.py index e4ce9bfb5..4d32c8c54 100644 --- a/test/unit/console_tool/test_authorize_account.py +++ b/test/unit/console_tool/test_authorize_account.py @@ -159,6 +159,8 @@ def test_authorize_account_prints_account_info(b2_cli): 'shareFiles', 'writeFiles', 'deleteFiles', + 'readBucketNotifications', + 'writeBucketNotifications', ], 'namePrefix': None, }, diff --git a/test/unit/console_tool/test_notification_rules.py b/test/unit/console_tool/test_notification_rules.py new file mode 100644 index 000000000..0b3dafc82 --- /dev/null +++ b/test/unit/console_tool/test_notification_rules.py @@ -0,0 +1,258 @@ +###################################################################### +# +# File: test/unit/console_tool/test_notification_rules.py +# +# Copyright 2023 Backblaze Inc. All Rights Reserved. +# +# License https://www.backblaze.com/using_b2_code.html +# +###################################################################### +import json + +import pytest + + +@pytest.fixture() +def bucket_notification_rule(b2_cli, bucket): + rule = { + "eventTypes": ["b2:ObjectCreated:*"], + "isEnabled": True, + "isSuspended": False, + "name": "test-rule", + "objectNamePrefix": "", + "suspensionReason": "", + "targetConfiguration": { + "targetType": "webhook", + "url": "https://example.com/webhook", + }, + } + _, stdout, _ = b2_cli.run( + [ + "notification-rules", + "create", + "--json", + f"b2://{bucket}", + "test-rule", + "--webhook-url", + "https://example.com/webhook", + "--event-type", + "b2:ObjectCreated:*", + ], + ) + actual_rule = json.loads(stdout) + assert actual_rule == rule + return actual_rule + + +def test_notification_rules__list_all(b2_cli, bucket, bucket_notification_rule): + _, stdout, _ = b2_cli.run([ + "notification-rules", + "list", + f"b2://{bucket}", + ]) + assert ( + stdout == f"""\ +Notification rules for b2://{bucket}/ : +- eventTypes: + - b2:ObjectCreated:* + isEnabled: true + isSuspended: false + name: test-rule + objectNamePrefix: '' + suspensionReason: '' + targetConfiguration: + targetType: webhook + url: https://example.com/webhook +""" + ) + + +def test_notification_rules__list_all_json(b2_cli, bucket, bucket_notification_rule): + _, stdout, _ = b2_cli.run([ + "notification-rules", + "list", + "--json", + f"b2://{bucket}", + ]) + assert json.loads(stdout) == [bucket_notification_rule] + + +def test_notification_rules__update(b2_cli, bucket, bucket_notification_rule): + bucket_notification_rule["isEnabled"] = False + _, stdout, _ = b2_cli.run( + [ + "notification-rules", + "update", + "--json", + f"b2://{bucket}", + bucket_notification_rule["name"], + "--disable", + "--custom-header", + "X-Custom-Header=value=1", + ], + ) + bucket_notification_rule["targetConfiguration"]["customHeaders"] = { + "X-Custom-Header": "value=1" + } + assert json.loads(stdout) == bucket_notification_rule + + +def test_notification_rules__update__no_such_rule(b2_cli, bucket, bucket_notification_rule): + b2_cli.run( + [ + "notification-rules", + "update", + f"b2://{bucket}", + f'{bucket_notification_rule["name"]}-unexisting', + "--disable", + ], + expected_stderr=( + "ERROR: rule with name 'test-rule-unexisting' does not exist on bucket " + "'my-bucket', available rules: ['test-rule']\n" + ), + expected_status=1, + ) + + +def test_notification_rules__update__custom_header_malformed( + b2_cli, bucket, bucket_notification_rule +): + bucket_notification_rule["isEnabled"] = False + _, stdout, _ = b2_cli.run( + [ + "notification-rules", + "update", + "--json", + f"b2://{bucket}", + bucket_notification_rule["name"], + "--disable", + "--custom-header", + "X-Custom-Header: value", + ], + ) + bucket_notification_rule["targetConfiguration"]["customHeaders"] = { + "X-Custom-Header: value": "" + } + assert json.loads(stdout) == bucket_notification_rule + + +def test_notification_rules__delete(b2_cli, bucket, bucket_notification_rule): + _, stdout, _ = b2_cli.run( + [ + "notification-rules", + "delete", + f"b2://{bucket}", + bucket_notification_rule["name"], + ], + ) + assert stdout == "Rule 'test-rule' has been deleted from b2://my-bucket/\n" + + +def test_notification_rules__delete_no_such_rule(b2_cli, bucket, bucket_notification_rule): + b2_cli.run( + [ + "notification-rules", + "delete", + f"b2://{bucket}", + f'{bucket_notification_rule["name"]}-unexisting', + ], + expected_stderr=( + "ERROR: no such rule to delete: 'test-rule-unexisting', available rules: ['test-rule'];" + " No rules have been deleted.\n" + ), + expected_status=1, + ) + + +@pytest.mark.parametrize( + "args,expected_stdout", + [ + (["-q"], ""), + ([], "No notification rules for b2://my-bucket/\n"), + (["--json"], "[]\n"), + ], +) +def test_notification_rules__no_rules(b2_cli, bucket, args, expected_stdout): + b2_cli.run( + ["notification-rules", "list", f"b2://{bucket}", *args], + expected_stdout=expected_stdout, + ) + + +def test_notification_rules__disable_enable(b2_cli, bucket, bucket_notification_rule): + _, stdout, _ = b2_cli.run( + [ + "notification-rules", + "disable", + "--json", + f"b2://{bucket}", + bucket_notification_rule["name"], + ], + ) + assert json.loads(stdout) == {**bucket_notification_rule, "isEnabled": False} + + _, stdout, _ = b2_cli.run( + [ + "notification-rules", + "enable", + "--json", + f"b2://{bucket}", + bucket_notification_rule["name"], + ], + ) + assert json.loads(stdout) == {**bucket_notification_rule, "isEnabled": True} + + +@pytest.mark.parametrize( + "command", + ["disable", "enable"], +) +def test_notification_rules__disable_enable__no_such_rule( + b2_cli, bucket, bucket_notification_rule, command +): + b2_cli.run( + [ + "notification-rules", + command, + f"b2://{bucket}", + f'{bucket_notification_rule["name"]}-unexisting', + ], + expected_stderr=( + "ERROR: rule with name 'test-rule-unexisting' does not exist on bucket " + "'my-bucket', available rules: ['test-rule']\n" + ), + expected_status=1, + ) + + +def test_notification_rules__sign_secret(b2_cli, bucket, bucket_notification_rule): + _, _, stderr = b2_cli.run( + [ + "notification-rules", + "update", + "--json", + f"b2://{bucket}", + bucket_notification_rule["name"], + "--sign-secret", + "new-secret", + ], + expected_status=2, + ) + assert "" in stderr + + _, stdout, _ = b2_cli.run( + [ + "notification-rules", + "update", + "--json", + f"b2://{bucket}", + bucket_notification_rule["name"], + "--sign-secret", + "7" * 32, + ], + ) + bucket_notification_rule["targetConfiguration"]["hmacSha256SigningSecret"] = "7" * 32 + assert json.loads(stdout) == bucket_notification_rule + + assert json.loads(b2_cli.run(["notification-rules", "list", "--json", f"b2://{bucket}"],)[1] + ) == [bucket_notification_rule] diff --git a/test/unit/test_console_tool.py b/test/unit/test_console_tool.py index 66526171e..044b8e4ea 100644 --- a/test/unit/test_console_tool.py +++ b/test/unit/test_console_tool.py @@ -268,7 +268,7 @@ def _run_command( self.assertNotIn(unexpected_part_of_stdout, actual_stdout) if expected_stderr is not None: self.assertEqual(expected_stderr, actual_stderr, 'stderr') - assert expected_status == actual_status, 'exit status code' + assert expected_status == actual_status return actual_status, actual_stdout, actual_stderr @classmethod From c2bb22d4a5dd23930123d1c10f5c5dc49917992c Mon Sep 17 00:00:00 2001 From: Maciej Urbanski Date: Thu, 11 Apr 2024 12:46:40 +0200 Subject: [PATCH 04/10] simplify how warnings are printed in CLI output --- b2/_internal/console_tool.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/b2/_internal/console_tool.py b/b2/_internal/console_tool.py index f217672b6..55012869b 100644 --- a/b2/_internal/console_tool.py +++ b/b2/_internal/console_tool.py @@ -14,6 +14,7 @@ import copy import tempfile +import warnings from b2._internal._cli.autocomplete_cache import AUTOCOMPLETE # noqa from b2._internal._utils.python_compat import removeprefix @@ -4771,6 +4772,10 @@ def _setup_logging(cls, args, argv): # logs from ALL loggers sent to the log file should be formatted this way logging.root.addHandler(handler) + if not args.debug_logs and not args.verbose: + warnings.showwarning = lambda message, category, *arg_, **_: print( + f'{category.__name__}: {message}', file=sys.stderr + ) logger.info(r'// %s %s %s \\', SEPARATOR, VERSION.center(8), SEPARATOR) logger.debug('platform is %s', platform.platform()) From c8df06163069623d168fbfb547b6e696db1fad52 Mon Sep 17 00:00:00 2001 From: Maciej Urbanski Date: Thu, 11 Apr 2024 19:03:21 +0200 Subject: [PATCH 05/10] remove yaml dependency --- b2/_internal/_cli/obj_dumps.py | 72 +++++++++++++++++++ b2/_internal/_cli/obj_loads.py | 17 ++--- b2/_internal/console_tool.py | 6 +- changelog.d/+yaml.added.md | 1 - pyproject.toml | 1 - test/unit/_cli/test_obj_dumps.py | 71 ++++++++++++++++++ test/unit/_cli/test_obj_loads.py | 35 +-------- .../console_tool/test_notification_rules.py | 6 +- 8 files changed, 157 insertions(+), 52 deletions(-) create mode 100644 b2/_internal/_cli/obj_dumps.py delete mode 100644 changelog.d/+yaml.added.md create mode 100644 test/unit/_cli/test_obj_dumps.py diff --git a/b2/_internal/_cli/obj_dumps.py b/b2/_internal/_cli/obj_dumps.py new file mode 100644 index 000000000..65bec6770 --- /dev/null +++ b/b2/_internal/_cli/obj_dumps.py @@ -0,0 +1,72 @@ +###################################################################### +# +# File: b2/_internal/_cli/obj_dumps.py +# +# Copyright 2024 Backblaze Inc. All Rights Reserved. +# +# License https://www.backblaze.com/using_b2_code.html +# +###################################################################### +import io + +from b2sdk.v2 import ( + unprintable_to_hex, +) + +_simple_repr_map = { + False: "false", + None: "null", + True: "true", +} +_simple_repr_map_values = set(_simple_repr_map.values()) | {"yes", "no"} + + +def _yaml_simple_repr(obj): + """ + Like YAML for simple types, but also escapes control characters for safety. + """ + if obj is None or isinstance(obj, (bool, float)): + simple_repr = _simple_repr_map.get(obj) + if simple_repr: + return simple_repr + obj_repr = unprintable_to_hex(str(obj)) + if isinstance( + obj, str + ) and (obj == "" or obj_repr.lower() in _simple_repr_map_values or obj_repr.isdigit()): + obj_repr = repr(obj) # add quotes to distinguish from numbers and booleans + return obj_repr + + +def _id_name_first_key(item): + try: + return ("id", "name").index(str(item[0]).lower()), item[0], item[1] + except ValueError: + return 2, item[0], item[1] + + +def _dump(data, indent=0, skip=False, *, output): + prefix = " " * indent + if isinstance(data, dict): + for idx, (key, value) in enumerate(sorted(data.items(), key=_id_name_first_key)): + output.write(f"{'' if skip and idx == 0 else prefix}{_yaml_simple_repr(key)}: ") + if isinstance(value, (dict, list)): + output.write("\n") + _dump(value, indent + 2, output=output) + else: + _dump(value, 0, True, output=output) + elif isinstance(data, list): + for idx, item in enumerate(data): + output.write(f"{'' if skip and idx == 0 else prefix}- ") + _dump(item, indent + 2, True, output=output) + else: + output.write(f"{'' if skip else prefix}{_yaml_simple_repr(data)}\n") + + +def readable_yaml_dump(data, output: io.TextIOBase) -> None: + """ + Print YAML-like human-readable representation of the data. + + :param data: The data to be printed. Can be a list, dict, or any basic datatype. + :param output: An output stream derived from io.TextIOBase where the data is to be printed. + """ + _dump(data, output=output) diff --git a/b2/_internal/_cli/obj_loads.py b/b2/_internal/_cli/obj_loads.py index 67fdfb4c9..a9b771980 100644 --- a/b2/_internal/_cli/obj_loads.py +++ b/b2/_internal/_cli/obj_loads.py @@ -14,7 +14,6 @@ import json from typing import TypeVar -import yaml from b2sdk.v2 import get_b2sdk_doc_urls try: @@ -44,22 +43,18 @@ def describe_type(type_) -> str: def validated_loads(data: str, expected_type: type[T] | None = None) -> T: - try: - try: - val = json.loads(data) - except json.JSONDecodeError: - val = yaml.safe_load(data) - except yaml.YAMLError as e: - raise argparse.ArgumentTypeError(f'{data!r} is not a valid JSON/YAML: {e}') from e - if expected_type is not None and pydantic is not None: ta = TypeAdapter(expected_type) try: - val = ta.validate_python(data) + val = ta.validate_json(data) except ValidationError as e: errors = convert_error_to_human_readable(e) raise argparse.ArgumentTypeError( f'Invalid value inputted, expected {describe_type(expected_type)}, got {data!r}, more detail below:\n{errors}' ) from e - + else: + try: + val = json.loads(data) + except json.JSONDecodeError as e: + raise argparse.ArgumentTypeError(f'{data!r} is not a valid JSON value') from e return val diff --git a/b2/_internal/console_tool.py b/b2/_internal/console_tool.py index 55012869b..57dcd07be 100644 --- a/b2/_internal/console_tool.py +++ b/b2/_internal/console_tool.py @@ -54,7 +54,6 @@ import b2sdk import requests import rst2ansi -import yaml from b2sdk.v2 import ( ALL_CAPABILITIES, B2_ACCOUNT_INFO_DEFAULT_FILE, @@ -154,6 +153,7 @@ CREATE_BUCKET_TYPES, DEFAULT_THREADS, ) +from b2._internal._cli.obj_dumps import readable_yaml_dump from b2._internal._cli.obj_loads import validated_loads from b2._internal._cli.shell import detect_shell, resolve_short_call_name from b2._internal._utils.uri import B2URI, B2FileIdURI, B2URIAdapter, B2URIBase @@ -1038,7 +1038,9 @@ def _print_json(self, data) -> None: ) def _print_human_readable_structure(self, data) -> None: - return self._print(yaml.dump(data, sort_keys=True).rstrip()) + output = io.StringIO() + readable_yaml_dump(data, output) + return self._print(output.getvalue().rstrip()) def _print( self, diff --git a/changelog.d/+yaml.added.md b/changelog.d/+yaml.added.md deleted file mode 100644 index af144c44d..000000000 --- a/changelog.d/+yaml.added.md +++ /dev/null @@ -1 +0,0 @@ -Add `yaml` format fallback in addition to `json` format supported in some of the arguments. \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 9a3153eaf..72cf0e16a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,7 +35,6 @@ dependencies = [ "tqdm>=4.65.0,<5", "platformdirs>=3.11.0,<5", "setuptools>=60; python_version < '3.10'", # required by phx-class-registry<4.1 - "PyYAML>=6,<7", ] [project.optional-dependencies] diff --git a/test/unit/_cli/test_obj_dumps.py b/test/unit/_cli/test_obj_dumps.py new file mode 100644 index 000000000..849e18ca1 --- /dev/null +++ b/test/unit/_cli/test_obj_dumps.py @@ -0,0 +1,71 @@ +###################################################################### +# +# File: test/unit/_cli/test_obj_dumps.py +# +# Copyright 2024 Backblaze Inc. All Rights Reserved. +# +# License https://www.backblaze.com/using_b2_code.html +# +###################################################################### +from io import StringIO + +import pytest + +from b2._internal._cli.obj_dumps import readable_yaml_dump + +# Test cases as tuples: (input_data, expected_output) +test_cases = [ + ({"key": "value"}, "key: value\n"), + ([{"a": 1, "b": 2}], "- a: 1\n b: 2\n"), + ([1, 2, "false"], "- 1\n- 2\n- 'false'\n"), + ({"true": True, "null": None}, "'null': null\n'true': true\n"), + ([''], "- ''\n"), + ( + # make sure id and name are first, rest should be sorted alphabetically + [ + {"b": 2, "a": 1, "name": 4, "id": 3}, + ], + "- id: 3\n name: 4\n a: 1\n b: 2\n", + ), + ( # nested data + [ + { + "name": "John Doe", + "age": 30, + "addresses": [ + { + "street": "123 Elm St", + "city": "Somewhere", + }, + { + "street": "456 Oak St", + }, + ], + "address": { + "street": "789 Pine St", + "city": "Anywhere", + "zip": "67890", + }, + } + ], + ( + "- name: John Doe\n" + " address: \n" + " city: Anywhere\n" + " street: 789 Pine St\n" + " zip: '67890'\n" + " addresses: \n" + " - city: Somewhere\n" + " street: 123 Elm St\n" + " - street: 456 Oak St\n" + " age: 30\n" + ), + ), +] + + +@pytest.mark.parametrize("input_data,expected", test_cases) +def test_readable_yaml_dump(input_data, expected): + output = StringIO() + readable_yaml_dump(input_data, output) + assert output.getvalue() == expected diff --git a/test/unit/_cli/test_obj_loads.py b/test/unit/_cli/test_obj_loads.py index 275373d26..06bcfcc4e 100644 --- a/test/unit/_cli/test_obj_loads.py +++ b/test/unit/_cli/test_obj_loads.py @@ -30,34 +30,6 @@ "b": 2, "c": 3 }), - # yaml - ("a: 1", { - "a": 1 - }), - ("a: 1\nb: 2", { - "a": 1, - "b": 2 - }), - ("a: 1\nb: 2\nc: 3", { - "a": 1, - "b": 2, - "c": 3 - }), - # yaml one-liners - ("a: 1", { - "a": 1 - }), - ("{a: 1,b: 2}", { - "a": 1, - "b": 2 - }), - ("{a: test,b: 2,sub:{c: 3}}", { - "a": "test", - "b": 2, - "sub": { - "c": 3 - } - }), ], ) def test_validated_loads(input_, expected_val): @@ -68,12 +40,7 @@ def test_validated_loads(input_, expected_val): "input_, error_msg", [ # not valid json nor yaml - ("{", "'{' is not a valid JSON/YAML:"), - # not-valid yaml - ( - "a: 1, a: 2", - r"a: 1, a: 2\' is not a valid JSON/YAML: mapping values are not allowed here", - ), + ("{", "'{' is not a valid JSON value"), ], ) def test_validated_loads__invalid_syntax(input_, error_msg): diff --git a/test/unit/console_tool/test_notification_rules.py b/test/unit/console_tool/test_notification_rules.py index 0b3dafc82..07d310075 100644 --- a/test/unit/console_tool/test_notification_rules.py +++ b/test/unit/console_tool/test_notification_rules.py @@ -53,11 +53,11 @@ def test_notification_rules__list_all(b2_cli, bucket, bucket_notification_rule): assert ( stdout == f"""\ Notification rules for b2://{bucket}/ : -- eventTypes: - - b2:ObjectCreated:* +- name: test-rule + eventTypes: + - b2:ObjectCreated:* isEnabled: true isSuspended: false - name: test-rule objectNamePrefix: '' suspensionReason: '' targetConfiguration: From d8b75c02ea676e608c5262f1012a51d691101396 Mon Sep 17 00:00:00 2001 From: Maciej Urbanski Date: Thu, 11 Apr 2024 19:21:11 +0200 Subject: [PATCH 06/10] event notifications feature private preview annoucment --- b2/_internal/console_tool.py | 22 +++++++++++++++++++--- changelog.d/+notification_rules.added.md | 3 ++- doc/source/index.rst | 2 ++ 3 files changed, 23 insertions(+), 4 deletions(-) diff --git a/b2/_internal/console_tool.py b/b2/_internal/console_tool.py index 57dcd07be..70a959042 100644 --- a/b2/_internal/console_tool.py +++ b/b2/_internal/console_tool.py @@ -4240,10 +4240,21 @@ def _run(self, args): return 0 -class NotificationRules(Command): +class NotificationRulesWarningMixin(Described): + """ + .. warning:: + + Event Notifications feature is in \"Private Preview\" state and may change without notice. + See https://www.backblaze.com/blog/announcing-event-notifications/ for details. + """ + + +class NotificationRules(NotificationRulesWarningMixin, Command): """ Bucket notification rules management subcommands. + {NotificationRulesWarningMixin} + For more information on each subcommand, use ``{NAME} notification-rules SUBCOMMAND --help``. Examples: @@ -4259,10 +4270,11 @@ class NotificationRules(Command): @NotificationRules.subcommands_registry.register -class NotificationRulesList(JSONOptionMixin, Command): +class NotificationRulesList(JSONOptionMixin, NotificationRulesWarningMixin, Command): """ Allows listing bucket notification rules of the given bucket. + {NotificationRulesWarningMixin} {JSONOptionMixin} @@ -4308,7 +4320,7 @@ def _run(self, args): return 0 -class NotificationRulesCreateBase(JSONOptionMixin, Command): +class NotificationRulesCreateBase(JSONOptionMixin, NotificationRulesWarningMixin, Command): @classmethod def _validate_secret(cls, value: str) -> str: if not re.match(r'^[a-zA-Z0-9]{32}$', value): @@ -4421,6 +4433,7 @@ class NotificationRulesCreate(NotificationRulesCreateBase): """ Allows creating bucket notification rules for the given bucket. + {NotificationRulesWarningMixin} Examples: @@ -4479,6 +4492,7 @@ class NotificationRulesUpdate(NotificationRulesUpdateBase): """ Allows updating notification rule of the given bucket. + {NotificationRulesWarningMixin} Examples: @@ -4507,6 +4521,7 @@ class NotificationRulesEnable(NotificationRulesUpdateBase): """ Allows enabling notification rule of the given bucket. + {NotificationRulesWarningMixin} Examples: @@ -4541,6 +4556,7 @@ class NotificationRulesDisable(NotificationRulesUpdateBase): """ Allows disabling notification rule of the given bucket. + {NotificationRulesWarningMixin} Examples: diff --git a/changelog.d/+notification_rules.added.md b/changelog.d/+notification_rules.added.md index 6b49b0441..77c95a6a2 100644 --- a/changelog.d/+notification_rules.added.md +++ b/changelog.d/+notification_rules.added.md @@ -1 +1,2 @@ -Add `notification-rules` commands for manipulating Bucket notification rules. \ No newline at end of file +Add `notification-rules` commands for manipulating Bucket notification rules as part of Event Notifications feature Private Preview. +See https://www.backblaze.com/blog/announcing-event-notifications/ for details. diff --git a/doc/source/index.rst b/doc/source/index.rst index 519083040..ae9dc36c8 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -1,3 +1,5 @@ +.. note:: **Event Notifications** feature is now in **Private Preview**. See https://www.backblaze.com/blog/announcing-event-notifications/ for details. + ######################################### Overview ######################################### From 1b4b4ae7ce2b715bdabf9cac72eef7f78d30676f Mon Sep 17 00:00:00 2001 From: Maciej Urbanski Date: Fri, 12 Apr 2024 13:03:25 +0200 Subject: [PATCH 07/10] simplify yaml_simple_repr for numbers --- b2/_internal/_cli/obj_dumps.py | 9 +++++---- test/unit/_cli/test_obj_dumps.py | 1 + 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/b2/_internal/_cli/obj_dumps.py b/b2/_internal/_cli/obj_dumps.py index 65bec6770..3a80de8f6 100644 --- a/b2/_internal/_cli/obj_dumps.py +++ b/b2/_internal/_cli/obj_dumps.py @@ -25,10 +25,11 @@ def _yaml_simple_repr(obj): """ Like YAML for simple types, but also escapes control characters for safety. """ - if obj is None or isinstance(obj, (bool, float)): - simple_repr = _simple_repr_map.get(obj) - if simple_repr: - return simple_repr + if isinstance(obj, (int, float)) and not isinstance(obj, bool): + return str(obj) + simple_repr = _simple_repr_map.get(obj) + if simple_repr: + return simple_repr obj_repr = unprintable_to_hex(str(obj)) if isinstance( obj, str diff --git a/test/unit/_cli/test_obj_dumps.py b/test/unit/_cli/test_obj_dumps.py index 849e18ca1..932b4bb3a 100644 --- a/test/unit/_cli/test_obj_dumps.py +++ b/test/unit/_cli/test_obj_dumps.py @@ -19,6 +19,7 @@ ([{"a": 1, "b": 2}], "- a: 1\n b: 2\n"), ([1, 2, "false"], "- 1\n- 2\n- 'false'\n"), ({"true": True, "null": None}, "'null': null\n'true': true\n"), + ([1., 0.567], "- 1.0\n- 0.567\n"), ([''], "- ''\n"), ( # make sure id and name are first, rest should be sorted alphabetically From c553c3367691586fe948a810ec4891d91be7415b Mon Sep 17 00:00:00 2001 From: Maciej Urbanski Date: Mon, 15 Apr 2024 13:05:43 +0200 Subject: [PATCH 08/10] cleanup notification code --- b2/_internal/console_tool.py | 1 - test/unit/console_tool/test_notification_rules.py | 3 +-- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/b2/_internal/console_tool.py b/b2/_internal/console_tool.py index 70a959042..9376eb4ab 100644 --- a/b2/_internal/console_tool.py +++ b/b2/_internal/console_tool.py @@ -331,7 +331,6 @@ def _get_description(cls, **kwargs): for klass in cls.mro() if klass is not cls and klass.__doc__ and issubclass(klass, Described) } - mro_docs.update(**{name.upper(): value for name, value in mro_docs.items()}) return cls.__doc__.format(**kwargs, **DOC_STRING_DATA, **mro_docs) @classmethod diff --git a/test/unit/console_tool/test_notification_rules.py b/test/unit/console_tool/test_notification_rules.py index 07d310075..efe6436c2 100644 --- a/test/unit/console_tool/test_notification_rules.py +++ b/test/unit/console_tool/test_notification_rules.py @@ -226,7 +226,7 @@ def test_notification_rules__disable_enable__no_such_rule( def test_notification_rules__sign_secret(b2_cli, bucket, bucket_notification_rule): - _, _, stderr = b2_cli.run( + b2_cli.run( [ "notification-rules", "update", @@ -238,7 +238,6 @@ def test_notification_rules__sign_secret(b2_cli, bucket, bucket_notification_rul ], expected_status=2, ) - assert "" in stderr _, stdout, _ = b2_cli.run( [ From 300f253769187d0ad3671fd1dfe0394e7021ec13 Mon Sep 17 00:00:00 2001 From: Maciej Urbanski Date: Mon, 15 Apr 2024 15:36:04 +0200 Subject: [PATCH 09/10] update b2sdk to one supporting Event Notification rules --- pdm.lock | 8 ++++---- pyproject.toml | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/pdm.lock b/pdm.lock index adede3aec..eab8803b2 100644 --- a/pdm.lock +++ b/pdm.lock @@ -5,7 +5,7 @@ groups = ["default", "bundle", "doc", "format", "full", "license", "lint", "release", "test"] strategy = ["cross_platform", "inherit_metadata"] lock_version = "4.4.1" -content_hash = "sha256:454c2cab0c928d6f9b44476c771b5fbd0247cc294a322d21eb19d72df97ffbc9" +content_hash = "sha256:c41ed236c40d0db7d7eeaf871f99aad2ff11e96179c48761a035ff3e43f314c9" [[package]] name = "alabaster" @@ -100,7 +100,7 @@ files = [ [[package]] name = "b2sdk" -version = "2.0.0" +version = "2.1.0" requires_python = ">=3.7" summary = "Backblaze B2 SDK" groups = ["default"] @@ -111,8 +111,8 @@ dependencies = [ "typing-extensions>=4.7.1; python_version < \"3.12\"", ] files = [ - {file = "b2sdk-2.0.0-py3-none-any.whl", hash = "sha256:e1be966e312f08cb33edefcd2449cb16c6cef66994e5a353f07bdbd310e139a6"}, - {file = "b2sdk-2.0.0.tar.gz", hash = "sha256:0e49cd0fdc989b1bdf8a2509b06badb41e5b9384ac509ab82d09d677037ea93e"}, + {file = "b2sdk-2.1.0-py3-none-any.whl", hash = "sha256:c88c9ca94034b5c490884280f921df10a3fa98a757eccca3fb57fb257fb04bde"}, + {file = "b2sdk-2.1.0.tar.gz", hash = "sha256:39116cc539ffa09c45eb9802b96416efafd255698d514303bdf3b7f7cf105f3f"}, ] [[package]] diff --git a/pyproject.toml b/pyproject.toml index 72cf0e16a..72362505b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,7 +25,7 @@ classifiers = [ dependencies = [ "argcomplete>=2,<4", "arrow>=1.0.2,<2.0.0", - "b2sdk>=2.0.0,<3", + "b2sdk>=2.1.0,<3", "docutils>=0.18.1", "idna~=3.4; platform_system == 'Java'", "importlib-metadata>=3.3; python_version < '3.8'", From 0982d7da4b5f9554a4aaf578679dd34beac4822b Mon Sep 17 00:00:00 2001 From: Maciej Urbanski Date: Mon, 15 Apr 2024 16:04:23 +0200 Subject: [PATCH 10/10] fix `b2 ls b2://` regression under b2v3 --- b2/_internal/_cli/b2args.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/b2/_internal/_cli/b2args.py b/b2/_internal/_cli/b2args.py index 61e84d886..1e2c26cbf 100644 --- a/b2/_internal/_cli/b2args.py +++ b/b2/_internal/_cli/b2args.py @@ -36,8 +36,8 @@ def b2id_or_file_like_b2_uri(value: str) -> B2URIBase: return b2_uri -def parse_bucket_name(value: str) -> str: - uri = parse_uri(value) +def parse_bucket_name(value: str, allow_all_buckets: bool = False) -> str: + uri = parse_uri(value, allow_all_buckets=allow_all_buckets) if isinstance(uri, B2URI): if uri.path: raise ValueError( @@ -47,7 +47,6 @@ def parse_bucket_name(value: str) -> str: return str(value) -B2_BUCKET_NAME_ARG_TYPE = wrap_with_argument_type_error(parse_bucket_name) B2ID_OR_B2_URI_ARG_TYPE = wrap_with_argument_type_error(parse_b2_uri) B2ID_OR_B2_URI_OR_ALL_BUCKETS_ARG_TYPE = wrap_with_argument_type_error( functools.partial(parse_b2_uri, allow_all_buckets=True) @@ -59,7 +58,12 @@ def add_bucket_name_argument( parser: argparse.ArgumentParser, name="bucketName", help="Target bucket name", nargs=None ): parser.add_argument( - name, type=B2_BUCKET_NAME_ARG_TYPE, help=help, nargs=nargs + name, + type=wrap_with_argument_type_error( + functools.partial(parse_bucket_name, allow_all_buckets=nargs == "?") + ), + help=help, + nargs=nargs ).completer = bucket_name_completer