From db7a28d10f4089ceeaab9193d5828501899471cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vykintas=20Baltru=C5=A1aitis?= Date: Fri, 24 Nov 2023 17:36:03 +0200 Subject: [PATCH 01/14] Cache autocomplete suggestions --- b2/_cli/argcompleters.py | 35 +-- b2/arg_parser.py | 50 ---- b2/arg_parser_types.py | 62 +++++ b2/autocomplete_cache.py | 162 +++++++++++ b2/console_tool.py | 9 +- pyproject.toml | 1 + test/integration/autocomplete/__init__.py | 9 + .../autocomplete/module_loading_b2sdk.py | 17 ++ .../{ => autocomplete}/test_autocomplete.py | 2 +- .../autocomplete/test_autocomplete_cache.py | 256 ++++++++++++++++++ test/unit/test_arg_parser.py | 4 +- 11 files changed, 527 insertions(+), 80 deletions(-) create mode 100644 b2/arg_parser_types.py create mode 100644 b2/autocomplete_cache.py create mode 100644 test/integration/autocomplete/__init__.py create mode 100644 test/integration/autocomplete/module_loading_b2sdk.py rename test/integration/{ => autocomplete}/test_autocomplete.py (97%) create mode 100644 test/integration/autocomplete/test_autocomplete_cache.py diff --git a/b2/_cli/argcompleters.py b/b2/_cli/argcompleters.py index a4510a35e..7ef5b316f 100644 --- a/b2/_cli/argcompleters.py +++ b/b2/_cli/argcompleters.py @@ -7,38 +7,27 @@ # License https://www.backblaze.com/using_b2_code.html # ###################################################################### -from functools import wraps -from itertools import islice -from b2sdk.v2.api import B2Api -from b2._cli.b2api import _get_b2api_for_profile -from b2._cli.const import LIST_FILE_NAMES_MAX_LIMIT +def bucket_name_completer(prefix, parsed_args, **kwargs): + from b2._cli.b2api import _get_b2api_for_profile + api = _get_b2api_for_profile(getattr(parsed_args, 'profile', None)) + res = [bucket.name for bucket in api.list_buckets(use_cache=True)] + return res -def _with_api(func): - """Decorator to inject B2Api instance into argcompleter function.""" - - @wraps(func) - def wrapper(prefix, parsed_args, **kwargs): - api = _get_b2api_for_profile(parsed_args.profile) - return func(prefix=prefix, parsed_args=parsed_args, api=api, **kwargs) - - return wrapper - - -@_with_api -def bucket_name_completer(api: B2Api, **kwargs): - return [bucket.name for bucket in api.list_buckets(use_cache=True)] - - -@_with_api -def file_name_completer(api: B2Api, parsed_args, **kwargs): +def file_name_completer(prefix, parsed_args, **kwargs): """ Completes file names in a bucket. To limit delay & cost only lists files returned from by single call to b2_list_file_names """ + from itertools import islice + + from b2._cli.b2api import _get_b2api_for_profile + from b2._cli.const import LIST_FILE_NAMES_MAX_LIMIT + + api = _get_b2api_for_profile(parsed_args.profile) bucket = api.get_bucket_by_name(parsed_args.bucketName) file_versions = bucket.ls( getattr(parsed_args, 'folderName', None) or '', diff --git a/b2/arg_parser.py b/b2/arg_parser.py index 3f61ae3aa..bc1914268 100644 --- a/b2/arg_parser.py +++ b/b2/arg_parser.py @@ -10,16 +10,11 @@ import argparse import locale -import re import sys import textwrap -import arrow -from b2sdk.v2 import RetentionPeriod from rst2ansi import rst2ansi -_arrow_version = tuple(int(p) for p in arrow.__version__.split(".")) - class RawTextHelpFormatter(argparse.RawTextHelpFormatter): """ @@ -103,48 +98,3 @@ def _get_encoding(cls): # locales are improperly configured return 'ascii' - - -def parse_comma_separated_list(s): - """ - Parse comma-separated list. - """ - return [word.strip() for word in s.split(",")] - - -def parse_millis_from_float_timestamp(s): - """ - Parse timestamp, e.g. 1367900664 or 1367900664.152 - """ - parsed = arrow.get(float(s)) - if _arrow_version < (1, 0, 0): - return int(parsed.format("XSSS")) - else: - return int(parsed.format("x")[:13]) - - -def parse_range(s): - """ - Parse optional integer range - """ - bytes_range = None - if s is not None: - bytes_range = s.split(',') - if len(bytes_range) != 2: - raise argparse.ArgumentTypeError('the range must have 2 values: start,end') - bytes_range = ( - int(bytes_range[0]), - int(bytes_range[1]), - ) - - return bytes_range - - -def parse_default_retention_period(s): - unit_part = '(' + ')|('.join(RetentionPeriod.KNOWN_UNITS) + ')' - m = re.match(r'^(?P\d+) (?P%s)$' % (unit_part), s) - if not m: - raise argparse.ArgumentTypeError( - 'default retention period must be in the form of "X days|years "' - ) - return RetentionPeriod(**{m.group('unit'): int(m.group('duration'))}) diff --git a/b2/arg_parser_types.py b/b2/arg_parser_types.py new file mode 100644 index 000000000..dd0a4303e --- /dev/null +++ b/b2/arg_parser_types.py @@ -0,0 +1,62 @@ +###################################################################### +# +# File: b2/arg_parser_types.py +# +# Copyright 2020 Backblaze Inc. All Rights Reserved. +# +# License https://www.backblaze.com/using_b2_code.html +# +###################################################################### + +import argparse +import re + +import arrow +from b2sdk.v2 import RetentionPeriod + +_arrow_version = tuple(int(p) for p in arrow.__version__.split(".")) + + +def parse_comma_separated_list(s): + """ + Parse comma-separated list. + """ + return [word.strip() for word in s.split(",")] + + +def parse_millis_from_float_timestamp(s): + """ + Parse timestamp, e.g. 1367900664 or 1367900664.152 + """ + parsed = arrow.get(float(s)) + if _arrow_version < (1, 0, 0): + return int(parsed.format("XSSS")) + else: + return int(parsed.format("x")[:13]) + + +def parse_range(s): + """ + Parse optional integer range + """ + bytes_range = None + if s is not None: + bytes_range = s.split(',') + if len(bytes_range) != 2: + raise argparse.ArgumentTypeError('the range must have 2 values: start,end') + bytes_range = ( + int(bytes_range[0]), + int(bytes_range[1]), + ) + + return bytes_range + + +def parse_default_retention_period(s): + unit_part = '(' + ')|('.join(RetentionPeriod.KNOWN_UNITS) + ')' + m = re.match(r'^(?P\d+) (?P%s)$' % (unit_part), s) + if not m: + raise argparse.ArgumentTypeError( + 'default retention period must be in the form of "X days|years "' + ) + return RetentionPeriod(**{m.group('unit'): int(m.group('duration'))}) \ No newline at end of file diff --git a/b2/autocomplete_cache.py b/b2/autocomplete_cache.py new file mode 100644 index 000000000..1c26e793b --- /dev/null +++ b/b2/autocomplete_cache.py @@ -0,0 +1,162 @@ +###################################################################### +# +# File: b2/autocomplete_cache.py +# +# Copyright 2020 Backblaze Inc. All Rights Reserved. +# +# License https://www.backblaze.com/using_b2_code.html +# +###################################################################### + +import abc +import argparse +import hashlib +import os +import pathlib +import pickle +from typing import Callable, Iterable + +import argcomplete + + +def identity(x): + return x + + +class StateTracker(abc.ABC): + @abc.abstractmethod + def current_state_identifier(self) -> str: + raise NotImplementedError() + + +class PickleStore(abc.ABC): + @abc.abstractmethod + def get_pickle(self, identifier: str) -> bytes | None: + raise NotImplementedError() + + @abc.abstractmethod + def set_pickle(self, identifier: str, data: bytes) -> None: + raise NotImplementedError() + + +class FileSetStateTrakcer(StateTracker): + _files: list[pathlib.Path] + + def __init__(self, files: Iterable[pathlib.Path]) -> None: + self._files = list(files) + + def _one_file_hash(self, file: pathlib.Path) -> str: + with open(file, 'rb') as f: + return hashlib.md5(str(file.absolute).encode('utf-8') + f.read()).hexdigest() + + def current_state_identifier(self) -> str: + return hashlib.md5( + b''.join(self._one_file_hash(file).encode('ascii') for file in self._files) + ).hexdigest() + + +class HomeCachePickleStore(PickleStore): + _dir: pathlib.Path + + def __init__(self, dir: pathlib.Path | None = None) -> None: + self._dir = dir + + def _dir_or_default(self) -> pathlib.Path: + if self._dir is not None: + return self._dir + cache_home = os.environ.get('XDG_CACHE_HOME') + if cache_home: + return pathlib.Path(cache_home) / 'b2' / 'autocomplete' + home = os.environ.get('HOME') + if not home: + raise RuntimeError( + 'Neither $HOME not $XDG_CACHE_HOME is set, cannot determine cache directory' + ) + return pathlib.Path(home) / '.cache' / 'b2' / 'autocomplete' + + def _fname(self, identifier: str) -> str: + return f"b2-autocomplete-cache-{identifier}.pickle" + + def get_pickle(self, identifier: str) -> bytes | None: + path = self._dir_or_default() / self._fname(identifier) + if path.exists(): + with open(path, 'rb') as f: + return f.read() + + def set_pickle(self, identifier: str, data: bytes) -> None: + """Sets the pickle for identifier if it doesn't exist. + When a new pickle is added, old ones are removed.""" + + dir = self._dir_or_default() + os.makedirs(dir, exist_ok=True) + path = dir / self._fname(identifier) + for file in dir.glob('b2-autocomplete-cache-*.pickle'): + file.unlink() + with open(path, 'wb') as f: + f.write(data) + + +class AutocompleteCache: + _tracker: StateTracker + _store: PickleStore + _unpickle: Callable[[bytes], argparse.ArgumentParser] + + def __init__( + self, + tracker: StateTracker, + store: PickleStore, + unpickle: Callable[[bytes], argparse.ArgumentParser] | None = None + ): + self._tracker = tracker + self._store = store + self._unpickle = unpickle or pickle.loads + + def _is_autocomplete_run(self) -> bool: + return '_ARGCOMPLETE' in os.environ + + def autocomplete_from_cache(self, uncached_args: dict | None = None) -> None: + if not self._is_autocomplete_run(): + return + + try: + identifier = self._tracker.current_state_identifier() + pickle_data = self._store.get_pickle(identifier) + if pickle_data: + parser = self._unpickle(pickle_data) + argcomplete.autocomplete(parser, **(uncached_args or {})) + except Exception: + # Autocomplete from cache failed but maybe we can autocomplete from scratch + return + + def _clean_parser(self, parser: argparse.ArgumentParser) -> None: + parser.register('type', None, identity) + for action in parser._actions: + if action.type not in [str, int]: + action.type = None + for action in parser._action_groups: + for key in parser._defaults: + action.set_defaults(**{key: None}) + parser.description = None + if parser._subparsers: + for group_action in parser._subparsers._group_actions: + for parser in group_action.choices.values(): + self._clean_parser(parser) + + def cache_and_autocomplete( + self, parser: argparse.ArgumentParser, uncached_args: dict | None = None + ) -> None: + if not self._is_autocomplete_run(): + return + + try: + identifier = self._tracker.current_state_identifier() + self._clean_parser(parser) + self._store.set_pickle(identifier, pickle.dumps(parser)) + finally: + argcomplete.autocomplete(parser, **(uncached_args or {})) + + +AUTOCOMPLETE = AutocompleteCache( + tracker=FileSetStateTrakcer(pathlib.Path(__file__).parent.glob('**/*.py')), + store=HomeCachePickleStore() +) diff --git a/b2/console_tool.py b/b2/console_tool.py index c9f8d0eac..864272969 100644 --- a/b2/console_tool.py +++ b/b2/console_tool.py @@ -9,6 +9,8 @@ # License https://www.backblaze.com/using_b2_code.html # ###################################################################### +from .autocomplete_cache import AUTOCOMPLETE # noqa +AUTOCOMPLETE.autocomplete_from_cache() import argparse import base64 @@ -39,7 +41,6 @@ from enum import Enum from typing import Any, BinaryIO, Dict, List, Optional, Tuple -import argcomplete import b2sdk import requests import rst2ansi @@ -122,8 +123,8 @@ from b2._cli.obj_loads import validated_loads from b2._cli.shell import detect_shell from b2._utils.filesystem import points_to_fifo -from b2.arg_parser import ( - ArgumentParser, +from b2.arg_parser import ArgumentParser +from b2.arg_parser_types import ( parse_comma_separated_list, parse_default_retention_period, parse_millis_from_float_timestamp, @@ -3779,7 +3780,7 @@ def __init__(self, b2_api: Optional[B2Api], stdout, stderr): def run_command(self, argv): signal.signal(signal.SIGINT, keyboard_interrupt_handler) parser = B2.get_parser() - argcomplete.autocomplete(parser, default_completer=None) + AUTOCOMPLETE.cache_and_autocomplete(parser) args = parser.parse_args(argv[1:]) self._setup_logging(args, argv) diff --git a/pyproject.toml b/pyproject.toml index 27a9965f2..946c55740 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,3 +19,4 @@ line-length = 100 [tool.ruff.per-file-ignores] "__init__.py" = ["F401"] "test/**" = ["D", "F403", "F405"] +"b2/console_tool.py" = ["E402"] diff --git a/test/integration/autocomplete/__init__.py b/test/integration/autocomplete/__init__.py new file mode 100644 index 000000000..89e74a09f --- /dev/null +++ b/test/integration/autocomplete/__init__.py @@ -0,0 +1,9 @@ +###################################################################### +# +# File: test/integration/autocomplete/__init__.py +# +# Copyright 2019 Backblaze Inc. All Rights Reserved. +# +# License https://www.backblaze.com/using_b2_code.html +# +###################################################################### diff --git a/test/integration/autocomplete/module_loading_b2sdk.py b/test/integration/autocomplete/module_loading_b2sdk.py new file mode 100644 index 000000000..acf60a231 --- /dev/null +++ b/test/integration/autocomplete/module_loading_b2sdk.py @@ -0,0 +1,17 @@ +###################################################################### +# +# File: test/integration/autocomplete/module_loading_b2sdk.py +# +# Copyright 2023 Backblaze Inc. All Rights Reserved. +# +# License https://www.backblaze.com/using_b2_code.html +# +###################################################################### + +# This is a helper module for test_autocomplete_cache.py + +from b2sdk.v2 import B2Api # noqa + + +def function(): + pass diff --git a/test/integration/test_autocomplete.py b/test/integration/autocomplete/test_autocomplete.py similarity index 97% rename from test/integration/test_autocomplete.py rename to test/integration/autocomplete/test_autocomplete.py index e0c30e065..18c787097 100644 --- a/test/integration/test_autocomplete.py +++ b/test/integration/autocomplete/test_autocomplete.py @@ -1,6 +1,6 @@ ###################################################################### # -# File: test/integration/test_autocomplete.py +# File: test/integration/autocomplete/test_autocomplete.py # # Copyright 2023 Backblaze Inc. All Rights Reserved. # diff --git a/test/integration/autocomplete/test_autocomplete_cache.py b/test/integration/autocomplete/test_autocomplete_cache.py new file mode 100644 index 000000000..f85382f08 --- /dev/null +++ b/test/integration/autocomplete/test_autocomplete_cache.py @@ -0,0 +1,256 @@ +###################################################################### +# +# File: test/integration/autocomplete/test_autocomplete_cache.py +# +# Copyright 2023 Backblaze Inc. All Rights Reserved. +# +# License https://www.backblaze.com/using_b2_code.html +# +###################################################################### + +import contextlib +import importlib +import io +import os +import pathlib +import pickle +import sys +from typing import Any + +import argcomplete +import pytest + +import b2._cli.argcompleters +import b2.arg_parser +import b2.console_tool +from b2 import autocomplete_cache + + +class Exit: + """A mocked exit method callable. Instead of actually exiting, + it just stores the exit code and returns.""" + + code: int | None + + @property + def success(self): + return self.code == 0 + + @property + def empty(self): + return self.code is None + + def __init__(self): + self.code = None + + def __call__(self, n: int): + self.code = n + + +@pytest.fixture +def autocomplete_runner(monkeypatch): + def fdopen(fd, *args, **kwargs): + # argcomplete package tries to open fd 9 for debugging which causes + # pytest to later raise errors about bad file descriptors. + if fd == 9: + return sys.stderr + return os.fdopen(fd, *args, **kwargs) + + @contextlib.contextmanager + def runner(command: str): + with monkeypatch.context() as m: + m.setenv('COMP_LINE', command) + m.setenv('COMP_POINT', str(len(command))) + m.setenv('_ARGCOMPLETE_IFS', ' ') + m.setenv('_ARGCOMPLETE', '1') + m.setattr('os.fdopen', fdopen) + yield + + return runner + + +def argcomplete_result(): + parser = b2.console_tool.B2.get_parser() + exit, output = Exit(), io.StringIO() + argcomplete.autocomplete(parser, exit_method=exit, output_stream=output) + return exit.code, output.getvalue() + + +def cached_complete_result(cache: autocomplete_cache.AutocompleteCache): + exit, output = Exit(), io.StringIO() + cache.autocomplete_from_cache(uncached_args={'exit_method': exit, 'output_stream': output}) + return exit.code, output.getvalue() + + +def uncached_complete_result(cache: autocomplete_cache.AutocompleteCache): + exit, output = Exit(), io.StringIO() + parser = b2.console_tool.B2.get_parser() + cache.cache_and_autocomplete( + parser, uncached_args={ + 'exit_method': exit, + 'output_stream': output + } + ) + return exit.code, output.getvalue() + + +def test_complete_main_command(autocomplete_runner, tmpdir): + cache = autocomplete_cache.AutocompleteCache( + tracker=autocomplete_cache.FileSetStateTrakcer([pathlib.Path(__file__)]), + store=autocomplete_cache.HomeCachePickleStore(pathlib.Path(tmpdir)), + ) + with autocomplete_runner('b2 '): + exit, argcomplete_output = argcomplete_result() + assert exit == 0 + assert 'get-bucket' in argcomplete_output + + with autocomplete_runner('b2 '): + exit, output = cached_complete_result(cache) + # Nothing has been cached yet, we expect simple return, not an exit + assert exit is None + assert not output + + with autocomplete_runner('b2 '): + exit, output = uncached_complete_result(cache) + assert exit == 0 + assert output == argcomplete_output + + with autocomplete_runner('b2 '): + exit, output = cached_complete_result(cache) + assert exit == 0 + assert output == argcomplete_output + + +def test_complete_with_bucket_suggestions(autocomplete_runner, tmpdir, bucket_name, b2_tool): + cache = autocomplete_cache.AutocompleteCache( + tracker=autocomplete_cache.FileSetStateTrakcer([pathlib.Path(__file__)]), + store=autocomplete_cache.HomeCachePickleStore(pathlib.Path(tmpdir)), + ) + with autocomplete_runner('b2 get-bucket '): + exit, argcomplete_output = argcomplete_result() + assert exit == 0 + assert bucket_name in argcomplete_output + + exit, output = cached_complete_result(cache) + assert exit is None + assert output == '' + + exit, output = uncached_complete_result(cache) + assert exit == 0 + assert output == argcomplete_output + + exit, output = cached_complete_result(cache) + assert exit == 0 + assert output == argcomplete_output + + +def test_complete_with_file_suggestions( + autocomplete_runner, tmpdir, bucket_name, file_name, b2_tool +): + cache = autocomplete_cache.AutocompleteCache( + tracker=autocomplete_cache.FileSetStateTrakcer([pathlib.Path(__file__)]), + store=autocomplete_cache.HomeCachePickleStore(pathlib.Path(tmpdir)), + ) + with autocomplete_runner(f'b2 download-file-by-name {bucket_name} '): + exit, argcomplete_output = argcomplete_result() + assert exit == 0 + assert file_name in argcomplete_output + + exit, output = cached_complete_result(cache) + assert exit is None + assert output == '' + + exit, output = uncached_complete_result(cache) + assert exit == 0 + assert output == argcomplete_output + + exit, output = cached_complete_result(cache) + assert exit == 0 + assert output == argcomplete_output + + +def test_hasher(tmpdir): + path_1 = pathlib.Path(tmpdir) / 'test_1' + path_1.write_text('test_1', 'ascii') + path_2 = pathlib.Path(tmpdir) / 'test_2' + path_2.write_text('test_2', 'ascii') + path_3 = pathlib.Path(tmpdir) / 'test_3' + path_3.write_text('test_3', 'ascii') + tracker_1 = autocomplete_cache.FileSetStateTrakcer([path_1, path_2]) + tracker_2 = autocomplete_cache.FileSetStateTrakcer([path_2, path_3]) + assert tracker_1.current_state_identifier() != tracker_2.current_state_identifier() + + +def test_pickle_store(tmpdir): + dir = pathlib.Path(tmpdir) + store = autocomplete_cache.HomeCachePickleStore(dir) + + store.set_pickle('test_1', b'test_data_1') + assert store.get_pickle('test_1') == b'test_data_1' + assert store.get_pickle('test_2') is None + assert len(list(dir.glob('**'))) == 1 + + store.set_pickle('test_2', b'test_data_2') + assert store.get_pickle('test_2') == b'test_data_2' + assert store.get_pickle('test_1') is None + assert len(list(dir.glob('**'))) == 1 + + +class Unpickler(pickle.Unpickler): + """This Unpickler will raise an exception if loading the pickled object + imports any b2sdk module.""" + + _modules_to_load: set[str] + + def load(self): + self._modules_to_load = set() + + b2_modules = [module for module in sys.modules if 'b2sdk' in module] + for key in b2_modules: + del sys.modules[key] + + result = super().load() + + for module in self._modules_to_load: + importlib.import_module(module) + importlib.reload(sys.modules[module]) + + if any('b2sdk' in module for module in sys.modules): + raise RuntimeError("Loading the pickled object imported b2sdk module") + return result + + def find_class(self, module: str, name: str) -> Any: + self._modules_to_load.add(module) + return super().find_class(module, name) + + +def unpickle(data: bytes) -> Any: + """Unpickling function that raises RunTimError if unpickled + object depends on b2sdk.""" + return Unpickler(io.BytesIO(data)).load() + + +def test_unpickle(): + """This tests ensures that Unpickler works as expected: + prevents successful unpickling of objects that depend on loading + modules from b2sdk.""" + from .module_loading_b2sdk import function + pickled = pickle.dumps(function) + with pytest.raises(RuntimeError): + unpickle(pickled) + + +def test_that_autocomplete_cache_loading_does_not_load_b2sdk(autocomplete_runner, tmpdir): + cache = autocomplete_cache.AutocompleteCache( + tracker=autocomplete_cache.FileSetStateTrakcer([pathlib.Path(__file__)]), + store=autocomplete_cache.HomeCachePickleStore(pathlib.Path(tmpdir)), + unpickle=unpickle, # using our unpickling function that fails if b2sdk is loaded + ) + with autocomplete_runner('b2 '): + exit, uncached_output = uncached_complete_result(cache) + assert exit == 0 + assert 'get-bucket' in uncached_output + + exit, output = cached_complete_result(cache) + assert exit == 0 + assert output == uncached_output diff --git a/test/unit/test_arg_parser.py b/test/unit/test_arg_parser.py index a615f82b1..9ef324dc1 100644 --- a/test/unit/test_arg_parser.py +++ b/test/unit/test_arg_parser.py @@ -11,8 +11,8 @@ import argparse import sys -from b2.arg_parser import ( - ArgumentParser, +from b2.arg_parser import ArgumentParser +from b2.arg_parser_types import ( parse_comma_separated_list, parse_millis_from_float_timestamp, parse_range, From 28694e2bb6522ca0c2d8056391ef98fd1d8900df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vykintas=20Baltru=C5=A1aitis?= Date: Fri, 24 Nov 2023 20:00:18 +0200 Subject: [PATCH 02/14] Add town crier update --- b2/_cli/argcompleters.py | 4 ++++ changelog.d/+improve_autocomplete_performance.fixed.md | 1 + 2 files changed, 5 insertions(+) create mode 100644 changelog.d/+improve_autocomplete_performance.fixed.md diff --git a/b2/_cli/argcompleters.py b/b2/_cli/argcompleters.py index d3edfd658..dc070eebe 100644 --- a/b2/_cli/argcompleters.py +++ b/b2/_cli/argcompleters.py @@ -8,6 +8,10 @@ # ###################################################################### +# We import all the necessary modules lazily in completers in order +# to avoid upfront cost of the imports when argcompleter is used for +# autocompletions. + from itertools import islice diff --git a/changelog.d/+improve_autocomplete_performance.fixed.md b/changelog.d/+improve_autocomplete_performance.fixed.md new file mode 100644 index 000000000..b5424b2c8 --- /dev/null +++ b/changelog.d/+improve_autocomplete_performance.fixed.md @@ -0,0 +1 @@ +Added autocomplete suggestion caching to improve autocomplete performance. \ No newline at end of file From 1c6a14feaa16b6cd66a83dcef75531698740bcf8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vykintas=20Baltru=C5=A1aitis?= Date: Fri, 24 Nov 2023 20:22:24 +0200 Subject: [PATCH 03/14] Fix imports --- test/unit/test_arg_parser.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/unit/test_arg_parser.py b/test/unit/test_arg_parser.py index 127551e8d..e918b0bd8 100644 --- a/test/unit/test_arg_parser.py +++ b/test/unit/test_arg_parser.py @@ -11,8 +11,8 @@ import argparse import sys -from b2.arg_parser import ( - B2ArgumentParser, +from b2.arg_parser import B2ArgumentParser +from b2.arg_parser_types import ( parse_comma_separated_list, parse_millis_from_float_timestamp, parse_range, From 26a3480112c9525704bc8a58673c60de671d9d83 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vykintas=20Baltru=C5=A1aitis?= Date: Fri, 24 Nov 2023 20:28:02 +0200 Subject: [PATCH 04/14] from __future__ import annotations --- b2/autocomplete_cache.py | 1 + test/integration/autocomplete/test_autocomplete_cache.py | 1 + 2 files changed, 2 insertions(+) diff --git a/b2/autocomplete_cache.py b/b2/autocomplete_cache.py index 1c26e793b..d75bd9a3a 100644 --- a/b2/autocomplete_cache.py +++ b/b2/autocomplete_cache.py @@ -7,6 +7,7 @@ # License https://www.backblaze.com/using_b2_code.html # ###################################################################### +from __future__ import annotations import abc import argparse diff --git a/test/integration/autocomplete/test_autocomplete_cache.py b/test/integration/autocomplete/test_autocomplete_cache.py index a61b5dd90..cc61fd2d4 100644 --- a/test/integration/autocomplete/test_autocomplete_cache.py +++ b/test/integration/autocomplete/test_autocomplete_cache.py @@ -7,6 +7,7 @@ # License https://www.backblaze.com/using_b2_code.html # ###################################################################### +from __future__ import annotations import contextlib import importlib From a9f1b35c808a7260068442a8040a4e56bf2e0848 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vykintas=20Baltru=C5=A1aitis?= Date: Sun, 26 Nov 2023 19:46:02 +0200 Subject: [PATCH 05/14] Autocomplete cache improvements * Invalidate cache when VERSION changes instead of looking at the file hashes. * Use platformdirs for determining cache directory. * Move some modules to under internal _cli module. --- b2/{ => _cli}/arg_parser_types.py | 2 +- b2/{ => _cli}/autocomplete_cache.py | 49 ++++++------------- b2/_cli/b2args.py | 2 +- b2/console_tool.py | 14 +++--- requirements.txt | 1 + test/integration/autocomplete/__init__.py | 2 +- .../autocomplete/fixture/__init__.py | 9 ++++ .../{ => fixture}/module_loading_b2sdk.py | 2 +- .../autocomplete/test_autocomplete_cache.py | 28 +++-------- test/unit/test_arg_parser.py | 4 +- 10 files changed, 47 insertions(+), 66 deletions(-) rename b2/{ => _cli}/arg_parser_types.py (98%) rename b2/{ => _cli}/autocomplete_cache.py (74%) create mode 100644 test/integration/autocomplete/fixture/__init__.py rename test/integration/autocomplete/{ => fixture}/module_loading_b2sdk.py (84%) diff --git a/b2/arg_parser_types.py b/b2/_cli/arg_parser_types.py similarity index 98% rename from b2/arg_parser_types.py rename to b2/_cli/arg_parser_types.py index b0a792086..93d0357ff 100644 --- a/b2/arg_parser_types.py +++ b/b2/_cli/arg_parser_types.py @@ -1,6 +1,6 @@ ###################################################################### # -# File: b2/arg_parser_types.py +# File: b2/_cli/arg_parser_types.py # # Copyright 2020 Backblaze Inc. All Rights Reserved. # diff --git a/b2/autocomplete_cache.py b/b2/_cli/autocomplete_cache.py similarity index 74% rename from b2/autocomplete_cache.py rename to b2/_cli/autocomplete_cache.py index d75bd9a3a..cdd5dfef1 100644 --- a/b2/autocomplete_cache.py +++ b/b2/_cli/autocomplete_cache.py @@ -1,6 +1,6 @@ ###################################################################### # -# File: b2/autocomplete_cache.py +# File: b2/_cli/autocomplete_cache.py # # Copyright 2020 Backblaze Inc. All Rights Reserved. # @@ -11,13 +11,15 @@ import abc import argparse -import hashlib import os import pathlib import pickle -from typing import Callable, Iterable +from typing import Callable import argcomplete +import platformdirs + +from b2.version import VERSION def identity(x): @@ -40,20 +42,9 @@ def set_pickle(self, identifier: str, data: bytes) -> None: raise NotImplementedError() -class FileSetStateTrakcer(StateTracker): - _files: list[pathlib.Path] - - def __init__(self, files: Iterable[pathlib.Path]) -> None: - self._files = list(files) - - def _one_file_hash(self, file: pathlib.Path) -> str: - with open(file, 'rb') as f: - return hashlib.md5(str(file.absolute).encode('utf-8') + f.read()).hexdigest() - +class VersionTracker(StateTracker): def current_state_identifier(self) -> str: - return hashlib.md5( - b''.join(self._one_file_hash(file).encode('ascii') for file in self._files) - ).hexdigest() + return VERSION class HomeCachePickleStore(PickleStore): @@ -62,24 +53,19 @@ class HomeCachePickleStore(PickleStore): def __init__(self, dir: pathlib.Path | None = None) -> None: self._dir = dir - def _dir_or_default(self) -> pathlib.Path: - if self._dir is not None: + def _cache_dir(self) -> pathlib.Path: + if self._dir: return self._dir - cache_home = os.environ.get('XDG_CACHE_HOME') - if cache_home: - return pathlib.Path(cache_home) / 'b2' / 'autocomplete' - home = os.environ.get('HOME') - if not home: - raise RuntimeError( - 'Neither $HOME not $XDG_CACHE_HOME is set, cannot determine cache directory' - ) - return pathlib.Path(home) / '.cache' / 'b2' / 'autocomplete' + self._dir = pathlib.Path( + platformdirs.user_cache_dir(appname='b2', appauthor='backblaze') + ) / 'autocomplete' + return self._dir def _fname(self, identifier: str) -> str: return f"b2-autocomplete-cache-{identifier}.pickle" def get_pickle(self, identifier: str) -> bytes | None: - path = self._dir_or_default() / self._fname(identifier) + path = self._cache_dir() / self._fname(identifier) if path.exists(): with open(path, 'rb') as f: return f.read() @@ -88,7 +74,7 @@ def set_pickle(self, identifier: str, data: bytes) -> None: """Sets the pickle for identifier if it doesn't exist. When a new pickle is added, old ones are removed.""" - dir = self._dir_or_default() + dir = self._cache_dir() os.makedirs(dir, exist_ok=True) path = dir / self._fname(identifier) for file in dir.glob('b2-autocomplete-cache-*.pickle'): @@ -157,7 +143,4 @@ def cache_and_autocomplete( argcomplete.autocomplete(parser, **(uncached_args or {})) -AUTOCOMPLETE = AutocompleteCache( - tracker=FileSetStateTrakcer(pathlib.Path(__file__).parent.glob('**/*.py')), - store=HomeCachePickleStore() -) +AUTOCOMPLETE = AutocompleteCache(tracker=VersionTracker(), store=HomeCachePickleStore()) diff --git a/b2/_cli/b2args.py b/b2/_cli/b2args.py index e0c0a15da..d85379f32 100644 --- a/b2/_cli/b2args.py +++ b/b2/_cli/b2args.py @@ -12,9 +12,9 @@ """ import argparse +from b2._cli.arg_parser_types import wrap_with_argument_type_error from b2._cli.argcompleters import b2uri_file_completer from b2._utils.uri import B2URI, B2URIBase, parse_b2_uri -from b2.arg_parser_types import wrap_with_argument_type_error def b2_file_uri(value: str) -> B2URIBase: diff --git a/b2/console_tool.py b/b2/console_tool.py index be9b50e23..3316c64f8 100644 --- a/b2/console_tool.py +++ b/b2/console_tool.py @@ -11,7 +11,7 @@ ###################################################################### from __future__ import annotations -from .autocomplete_cache import AUTOCOMPLETE # noqa +from b2._cli.autocomplete_cache import AUTOCOMPLETE # noqa AUTOCOMPLETE.autocomplete_from_cache() @@ -107,6 +107,12 @@ from class_registry import ClassRegistry from tabulate import tabulate +from b2._cli.arg_parser_types import ( + parse_comma_separated_list, + parse_default_retention_period, + parse_millis_from_float_timestamp, + parse_range, +) from b2._cli.argcompleters import bucket_name_completer, file_name_completer from b2._cli.autocomplete_install import ( SUPPORTED_SHELLS, @@ -130,12 +136,6 @@ from b2._cli.shell import detect_shell from b2._utils.uri import B2URI, B2FileIdURI, B2URIAdapter, B2URIBase from b2.arg_parser import B2ArgumentParser -from b2.arg_parser_types import ( - parse_comma_separated_list, - parse_default_retention_period, - parse_millis_from_float_timestamp, - parse_range, -) from b2.json_encoder import B2CliJsonEncoder from b2.version import VERSION diff --git a/requirements.txt b/requirements.txt index 79f6c88b3..e6f3e7a0d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,3 +8,4 @@ phx-class-registry~=4.0 rst2ansi==0.1.5 tabulate==0.9.0 tqdm~=4.65.0 +platformdirs==4.0.0 \ No newline at end of file diff --git a/test/integration/autocomplete/__init__.py b/test/integration/autocomplete/__init__.py index 89e74a09f..7aa0f03f4 100644 --- a/test/integration/autocomplete/__init__.py +++ b/test/integration/autocomplete/__init__.py @@ -2,7 +2,7 @@ # # File: test/integration/autocomplete/__init__.py # -# Copyright 2019 Backblaze Inc. All Rights Reserved. +# Copyright 2023 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # diff --git a/test/integration/autocomplete/fixture/__init__.py b/test/integration/autocomplete/fixture/__init__.py new file mode 100644 index 000000000..b4507640c --- /dev/null +++ b/test/integration/autocomplete/fixture/__init__.py @@ -0,0 +1,9 @@ +###################################################################### +# +# File: test/integration/autocomplete/fixture/__init__.py +# +# Copyright 2023 Backblaze Inc. All Rights Reserved. +# +# License https://www.backblaze.com/using_b2_code.html +# +###################################################################### diff --git a/test/integration/autocomplete/module_loading_b2sdk.py b/test/integration/autocomplete/fixture/module_loading_b2sdk.py similarity index 84% rename from test/integration/autocomplete/module_loading_b2sdk.py rename to test/integration/autocomplete/fixture/module_loading_b2sdk.py index acf60a231..1d44f036f 100644 --- a/test/integration/autocomplete/module_loading_b2sdk.py +++ b/test/integration/autocomplete/fixture/module_loading_b2sdk.py @@ -1,6 +1,6 @@ ###################################################################### # -# File: test/integration/autocomplete/module_loading_b2sdk.py +# File: test/integration/autocomplete/fixture/module_loading_b2sdk.py # # Copyright 2023 Backblaze Inc. All Rights Reserved. # diff --git a/test/integration/autocomplete/test_autocomplete_cache.py b/test/integration/autocomplete/test_autocomplete_cache.py index cc61fd2d4..d2b104b57 100644 --- a/test/integration/autocomplete/test_autocomplete_cache.py +++ b/test/integration/autocomplete/test_autocomplete_cache.py @@ -24,7 +24,7 @@ import b2._cli.argcompleters import b2.arg_parser import b2.console_tool -from b2 import autocomplete_cache +from b2._cli import autocomplete_cache class Exit: @@ -97,7 +97,7 @@ def uncached_complete_result(cache: autocomplete_cache.AutocompleteCache): def test_complete_main_command(autocomplete_runner, tmpdir): cache = autocomplete_cache.AutocompleteCache( - tracker=autocomplete_cache.FileSetStateTrakcer([pathlib.Path(__file__)]), + tracker=autocomplete_cache.VersionTracker(), store=autocomplete_cache.HomeCachePickleStore(pathlib.Path(tmpdir)), ) with autocomplete_runner('b2 '): @@ -124,7 +124,7 @@ def test_complete_main_command(autocomplete_runner, tmpdir): def test_complete_with_bucket_suggestions(autocomplete_runner, tmpdir, bucket_name, b2_tool): cache = autocomplete_cache.AutocompleteCache( - tracker=autocomplete_cache.FileSetStateTrakcer([pathlib.Path(__file__)]), + tracker=autocomplete_cache.VersionTracker(), store=autocomplete_cache.HomeCachePickleStore(pathlib.Path(tmpdir)), ) with autocomplete_runner('b2 get-bucket '): @@ -145,7 +145,7 @@ def test_complete_with_file_suggestions( autocomplete_runner, tmpdir, bucket_name, file_name, b2_tool ): cache = autocomplete_cache.AutocompleteCache( - tracker=autocomplete_cache.FileSetStateTrakcer([pathlib.Path(__file__)]), + tracker=autocomplete_cache.VersionTracker(), store=autocomplete_cache.HomeCachePickleStore(pathlib.Path(tmpdir)), ) with autocomplete_runner(f'b2 hide-file {bucket_name} '): @@ -170,7 +170,7 @@ def test_complete_with_file_uri_suggestions( autocomplete_runner, tmpdir, bucket_name, file_name, b2_tool ): cache = autocomplete_cache.AutocompleteCache( - tracker=autocomplete_cache.FileSetStateTrakcer([pathlib.Path(__file__)]), + tracker=autocomplete_cache.VersionTracker(), store=autocomplete_cache.HomeCachePickleStore(pathlib.Path(tmpdir)), ) with autocomplete_runner(f'b2 download-file b2://{bucket_name}/'): @@ -187,18 +187,6 @@ def test_complete_with_file_uri_suggestions( assert output == argcomplete_output -def test_hasher(tmpdir): - path_1 = pathlib.Path(tmpdir) / 'test_1' - path_1.write_text('test_1', 'ascii') - path_2 = pathlib.Path(tmpdir) / 'test_2' - path_2.write_text('test_2', 'ascii') - path_3 = pathlib.Path(tmpdir) / 'test_3' - path_3.write_text('test_3', 'ascii') - tracker_1 = autocomplete_cache.FileSetStateTrakcer([path_1, path_2]) - tracker_2 = autocomplete_cache.FileSetStateTrakcer([path_2, path_3]) - assert tracker_1.current_state_identifier() != tracker_2.current_state_identifier() - - def test_pickle_store(tmpdir): dir = pathlib.Path(tmpdir) store = autocomplete_cache.HomeCachePickleStore(dir) @@ -243,7 +231,7 @@ def find_class(self, module: str, name: str) -> Any: def unpickle(data: bytes) -> Any: - """Unpickling function that raises RunTimError if unpickled + """Unpickling function that raises RuntimeError if unpickled object depends on b2sdk.""" return Unpickler(io.BytesIO(data)).load() @@ -252,7 +240,7 @@ def test_unpickle(): """This tests ensures that Unpickler works as expected: prevents successful unpickling of objects that depend on loading modules from b2sdk.""" - from .module_loading_b2sdk import function + from .fixture.module_loading_b2sdk import function pickled = pickle.dumps(function) with pytest.raises(RuntimeError): unpickle(pickled) @@ -260,7 +248,7 @@ def test_unpickle(): def test_that_autocomplete_cache_loading_does_not_load_b2sdk(autocomplete_runner, tmpdir): cache = autocomplete_cache.AutocompleteCache( - tracker=autocomplete_cache.FileSetStateTrakcer([pathlib.Path(__file__)]), + tracker=autocomplete_cache.VersionTracker(), store=autocomplete_cache.HomeCachePickleStore(pathlib.Path(tmpdir)), unpickle=unpickle, # using our unpickling function that fails if b2sdk is loaded ) diff --git a/test/unit/test_arg_parser.py b/test/unit/test_arg_parser.py index e918b0bd8..53195f16e 100644 --- a/test/unit/test_arg_parser.py +++ b/test/unit/test_arg_parser.py @@ -11,12 +11,12 @@ import argparse import sys -from b2.arg_parser import B2ArgumentParser -from b2.arg_parser_types import ( +from b2._cli.arg_parser_types import ( parse_comma_separated_list, parse_millis_from_float_timestamp, parse_range, ) +from b2.arg_parser import B2ArgumentParser from b2.console_tool import B2 from .test_base import TestBase From 352ae4a36da6c9dd01fe2bb8367cc747cb5207d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vykintas=20Baltru=C5=A1aitis?= Date: Mon, 27 Nov 2023 10:03:11 +0200 Subject: [PATCH 06/14] Relax platformdirs dependency --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index e6f3e7a0d..1e9e5ed8b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,4 +8,4 @@ phx-class-registry~=4.0 rst2ansi==0.1.5 tabulate==0.9.0 tqdm~=4.65.0 -platformdirs==4.0.0 \ No newline at end of file +platformdirs~=4.0.0 \ No newline at end of file From 9ca42db532452e28ae2754bfdba2b2db22d88b4e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vykintas=20Baltru=C5=A1aitis?= Date: Wed, 29 Nov 2023 15:07:35 +0200 Subject: [PATCH 07/14] Move autocomplete cache tests to unit --- noxfile.py | 1 + requirements.txt | 2 +- .../autocomplete/fixture/__init__.py | 9 ----- .../{autocomplete => }/test_autocomplete.py | 2 +- .../_cli/fixtures}/__init__.py | 2 +- .../_cli/fixtures}/module_loading_b2sdk.py | 2 +- .../_cli}/test_autocomplete_cache.py | 37 +++++++++++++------ test/unit/fixtures/__init__.py | 9 ----- 8 files changed, 31 insertions(+), 33 deletions(-) delete mode 100644 test/integration/autocomplete/fixture/__init__.py rename test/integration/{autocomplete => }/test_autocomplete.py (97%) rename test/{integration/autocomplete => unit/_cli/fixtures}/__init__.py (83%) rename test/{integration/autocomplete/fixture => unit/_cli/fixtures}/module_loading_b2sdk.py (84%) rename test/{integration/autocomplete => unit/_cli}/test_autocomplete_cache.py (88%) delete mode 100644 test/unit/fixtures/__init__.py diff --git a/noxfile.py b/noxfile.py index 2c0d5c974..f92b64ce1 100644 --- a/noxfile.py +++ b/noxfile.py @@ -51,6 +51,7 @@ "pexpect==4.8.0", "pytest==6.2.5", "pytest-cov==3.0.0", + 'pytest-forked==1.4.0', 'pytest-xdist==2.5.0', 'backoff==2.1.2', 'more_itertools==8.13.0', diff --git a/requirements.txt b/requirements.txt index 1e9e5ed8b..15bee5734 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,4 +8,4 @@ phx-class-registry~=4.0 rst2ansi==0.1.5 tabulate==0.9.0 tqdm~=4.65.0 -platformdirs~=4.0.0 \ No newline at end of file +platformdirs>=4.0.0,<5 \ No newline at end of file diff --git a/test/integration/autocomplete/fixture/__init__.py b/test/integration/autocomplete/fixture/__init__.py deleted file mode 100644 index b4507640c..000000000 --- a/test/integration/autocomplete/fixture/__init__.py +++ /dev/null @@ -1,9 +0,0 @@ -###################################################################### -# -# File: test/integration/autocomplete/fixture/__init__.py -# -# Copyright 2023 Backblaze Inc. All Rights Reserved. -# -# License https://www.backblaze.com/using_b2_code.html -# -###################################################################### diff --git a/test/integration/autocomplete/test_autocomplete.py b/test/integration/test_autocomplete.py similarity index 97% rename from test/integration/autocomplete/test_autocomplete.py rename to test/integration/test_autocomplete.py index 239af092d..6e5b0bd38 100644 --- a/test/integration/autocomplete/test_autocomplete.py +++ b/test/integration/test_autocomplete.py @@ -1,6 +1,6 @@ ###################################################################### # -# File: test/integration/autocomplete/test_autocomplete.py +# File: test/integration/test_autocomplete.py # # Copyright 2023 Backblaze Inc. All Rights Reserved. # diff --git a/test/integration/autocomplete/__init__.py b/test/unit/_cli/fixtures/__init__.py similarity index 83% rename from test/integration/autocomplete/__init__.py rename to test/unit/_cli/fixtures/__init__.py index 7aa0f03f4..9428f8b12 100644 --- a/test/integration/autocomplete/__init__.py +++ b/test/unit/_cli/fixtures/__init__.py @@ -1,6 +1,6 @@ ###################################################################### # -# File: test/integration/autocomplete/__init__.py +# File: test/unit/_cli/fixtures/__init__.py # # Copyright 2023 Backblaze Inc. All Rights Reserved. # diff --git a/test/integration/autocomplete/fixture/module_loading_b2sdk.py b/test/unit/_cli/fixtures/module_loading_b2sdk.py similarity index 84% rename from test/integration/autocomplete/fixture/module_loading_b2sdk.py rename to test/unit/_cli/fixtures/module_loading_b2sdk.py index 1d44f036f..9d0a40c9e 100644 --- a/test/integration/autocomplete/fixture/module_loading_b2sdk.py +++ b/test/unit/_cli/fixtures/module_loading_b2sdk.py @@ -1,6 +1,6 @@ ###################################################################### # -# File: test/integration/autocomplete/fixture/module_loading_b2sdk.py +# File: test/unit/_cli/fixtures/module_loading_b2sdk.py # # Copyright 2023 Backblaze Inc. All Rights Reserved. # diff --git a/test/integration/autocomplete/test_autocomplete_cache.py b/test/unit/_cli/test_autocomplete_cache.py similarity index 88% rename from test/integration/autocomplete/test_autocomplete_cache.py rename to test/unit/_cli/test_autocomplete_cache.py index d2b104b57..817ce4d07 100644 --- a/test/integration/autocomplete/test_autocomplete_cache.py +++ b/test/unit/_cli/test_autocomplete_cache.py @@ -1,6 +1,6 @@ ###################################################################### # -# File: test/integration/autocomplete/test_autocomplete_cache.py +# File: test/unit/_cli/test_autocomplete_cache.py # # Copyright 2023 Backblaze Inc. All Rights Reserved. # @@ -10,6 +10,7 @@ from __future__ import annotations import contextlib +import copy import importlib import io import os @@ -26,6 +27,8 @@ import b2.console_tool from b2._cli import autocomplete_cache +from ..console_tool.conftest import * # noqa + class Exit: """A mocked exit method callable. Instead of actually exiting, @@ -49,7 +52,7 @@ def __call__(self, n: int): @pytest.fixture -def autocomplete_runner(monkeypatch): +def autocomplete_runner(monkeypatch, b2_cli): def fdopen(fd, *args, **kwargs): # argcomplete package tries to open fd 9 for debugging which causes # pytest to later raise errors about bad file descriptors. @@ -65,13 +68,18 @@ def runner(command: str): m.setenv('_ARGCOMPLETE_IFS', ' ') m.setenv('_ARGCOMPLETE', '1') m.setattr('os.fdopen', fdopen) + + def _get_b2api_for_profile(profile: str): + return b2_cli.b2_api + + m.setattr('b2._cli.b2api._get_b2api_for_profile', _get_b2api_for_profile) yield return runner def argcomplete_result(): - parser = b2.console_tool.B2.create_parser() + parser = copy.deepcopy(b2.console_tool.B2.create_parser()) exit, output = Exit(), io.StringIO() argcomplete.autocomplete(parser, exit_method=exit, output_stream=output) return exit.code, output.getvalue() @@ -85,7 +93,7 @@ def cached_complete_result(cache: autocomplete_cache.AutocompleteCache): def uncached_complete_result(cache: autocomplete_cache.AutocompleteCache): exit, output = Exit(), io.StringIO() - parser = b2.console_tool.B2.create_parser() + parser = copy.deepcopy(b2.console_tool.B2.create_parser()) cache.cache_and_autocomplete( parser, uncached_args={ 'exit_method': exit, @@ -95,6 +103,7 @@ def uncached_complete_result(cache: autocomplete_cache.AutocompleteCache): return exit.code, output.getvalue() +@pytest.mark.forked def test_complete_main_command(autocomplete_runner, tmpdir): cache = autocomplete_cache.AutocompleteCache( tracker=autocomplete_cache.VersionTracker(), @@ -122,7 +131,8 @@ def test_complete_main_command(autocomplete_runner, tmpdir): assert output == argcomplete_output -def test_complete_with_bucket_suggestions(autocomplete_runner, tmpdir, bucket_name, b2_tool): +@pytest.mark.forked +def test_complete_with_bucket_suggestions(autocomplete_runner, tmpdir, bucket, authorized_b2_cli): cache = autocomplete_cache.AutocompleteCache( tracker=autocomplete_cache.VersionTracker(), store=autocomplete_cache.HomeCachePickleStore(pathlib.Path(tmpdir)), @@ -130,7 +140,7 @@ def test_complete_with_bucket_suggestions(autocomplete_runner, tmpdir, bucket_na with autocomplete_runner('b2 get-bucket '): exit, argcomplete_output = argcomplete_result() assert exit == 0 - assert bucket_name in argcomplete_output + assert bucket in argcomplete_output exit, output = uncached_complete_result(cache) assert exit == 0 @@ -141,14 +151,16 @@ def test_complete_with_bucket_suggestions(autocomplete_runner, tmpdir, bucket_na assert output == argcomplete_output +@pytest.mark.forked def test_complete_with_file_suggestions( - autocomplete_runner, tmpdir, bucket_name, file_name, b2_tool + autocomplete_runner, tmpdir, bucket, uploaded_file, authorized_b2_cli ): + file_name = uploaded_file['fileName'] cache = autocomplete_cache.AutocompleteCache( tracker=autocomplete_cache.VersionTracker(), store=autocomplete_cache.HomeCachePickleStore(pathlib.Path(tmpdir)), ) - with autocomplete_runner(f'b2 hide-file {bucket_name} '): + with autocomplete_runner(f'b2 hide-file {bucket} '): exit, argcomplete_output = argcomplete_result() assert exit == 0 assert file_name in argcomplete_output @@ -166,14 +178,16 @@ def test_complete_with_file_suggestions( assert output == argcomplete_output +@pytest.mark.forked def test_complete_with_file_uri_suggestions( - autocomplete_runner, tmpdir, bucket_name, file_name, b2_tool + autocomplete_runner, tmpdir, bucket, uploaded_file, authorized_b2_cli ): + file_name = uploaded_file['fileName'] cache = autocomplete_cache.AutocompleteCache( tracker=autocomplete_cache.VersionTracker(), store=autocomplete_cache.HomeCachePickleStore(pathlib.Path(tmpdir)), ) - with autocomplete_runner(f'b2 download-file b2://{bucket_name}/'): + with autocomplete_runner(f'b2 download-file b2://{bucket}/'): exit, argcomplete_output = argcomplete_result() assert exit == 0 assert file_name in argcomplete_output @@ -240,12 +254,13 @@ def test_unpickle(): """This tests ensures that Unpickler works as expected: prevents successful unpickling of objects that depend on loading modules from b2sdk.""" - from .fixture.module_loading_b2sdk import function + from .fixtures.module_loading_b2sdk import function pickled = pickle.dumps(function) with pytest.raises(RuntimeError): unpickle(pickled) +@pytest.mark.forked def test_that_autocomplete_cache_loading_does_not_load_b2sdk(autocomplete_runner, tmpdir): cache = autocomplete_cache.AutocompleteCache( tracker=autocomplete_cache.VersionTracker(), diff --git a/test/unit/fixtures/__init__.py b/test/unit/fixtures/__init__.py deleted file mode 100644 index 8a1e1ffdc..000000000 --- a/test/unit/fixtures/__init__.py +++ /dev/null @@ -1,9 +0,0 @@ -###################################################################### -# -# File: test/unit/fixtures/__init__.py -# -# Copyright 2019 Backblaze Inc. All Rights Reserved. -# -# License https://www.backblaze.com/using_b2_code.html -# -###################################################################### From fcf4ffd5ba094fbc8144ef9e3893287ef4bcf7f0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vykintas=20Baltru=C5=A1aitis?= Date: Wed, 29 Nov 2023 15:12:00 +0200 Subject: [PATCH 08/14] Remove unnecessary deepcopy --- test/unit/_cli/test_autocomplete_cache.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/test/unit/_cli/test_autocomplete_cache.py b/test/unit/_cli/test_autocomplete_cache.py index 817ce4d07..dacfee62b 100644 --- a/test/unit/_cli/test_autocomplete_cache.py +++ b/test/unit/_cli/test_autocomplete_cache.py @@ -10,7 +10,6 @@ from __future__ import annotations import contextlib -import copy import importlib import io import os @@ -79,7 +78,7 @@ def _get_b2api_for_profile(profile: str): def argcomplete_result(): - parser = copy.deepcopy(b2.console_tool.B2.create_parser()) + parser = b2.console_tool.B2.create_parser() exit, output = Exit(), io.StringIO() argcomplete.autocomplete(parser, exit_method=exit, output_stream=output) return exit.code, output.getvalue() @@ -93,7 +92,7 @@ def cached_complete_result(cache: autocomplete_cache.AutocompleteCache): def uncached_complete_result(cache: autocomplete_cache.AutocompleteCache): exit, output = Exit(), io.StringIO() - parser = copy.deepcopy(b2.console_tool.B2.create_parser()) + parser = b2.console_tool.B2.create_parser() cache.cache_and_autocomplete( parser, uncached_args={ 'exit_method': exit, From 90089e59abb2f48bce4ac99f10a7ba405263e243 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vykintas=20Baltru=C5=A1aitis?= Date: Wed, 29 Nov 2023 15:24:04 +0200 Subject: [PATCH 09/14] Skip autocomplete cache tests on Windows --- test/unit/_cli/test_autocomplete_cache.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/test/unit/_cli/test_autocomplete_cache.py b/test/unit/_cli/test_autocomplete_cache.py index dacfee62b..5130132fa 100644 --- a/test/unit/_cli/test_autocomplete_cache.py +++ b/test/unit/_cli/test_autocomplete_cache.py @@ -26,6 +26,7 @@ import b2.console_tool from b2._cli import autocomplete_cache +from ...helpers import skip_on_windows from ..console_tool.conftest import * # noqa @@ -102,6 +103,7 @@ def uncached_complete_result(cache: autocomplete_cache.AutocompleteCache): return exit.code, output.getvalue() +@skip_on_windows @pytest.mark.forked def test_complete_main_command(autocomplete_runner, tmpdir): cache = autocomplete_cache.AutocompleteCache( @@ -130,6 +132,7 @@ def test_complete_main_command(autocomplete_runner, tmpdir): assert output == argcomplete_output +@skip_on_windows @pytest.mark.forked def test_complete_with_bucket_suggestions(autocomplete_runner, tmpdir, bucket, authorized_b2_cli): cache = autocomplete_cache.AutocompleteCache( @@ -150,6 +153,7 @@ def test_complete_with_bucket_suggestions(autocomplete_runner, tmpdir, bucket, a assert output == argcomplete_output +@skip_on_windows @pytest.mark.forked def test_complete_with_file_suggestions( autocomplete_runner, tmpdir, bucket, uploaded_file, authorized_b2_cli @@ -177,6 +181,7 @@ def test_complete_with_file_suggestions( assert output == argcomplete_output +@skip_on_windows @pytest.mark.forked def test_complete_with_file_uri_suggestions( autocomplete_runner, tmpdir, bucket, uploaded_file, authorized_b2_cli @@ -259,6 +264,7 @@ def test_unpickle(): unpickle(pickled) +@skip_on_windows @pytest.mark.forked def test_that_autocomplete_cache_loading_does_not_load_b2sdk(autocomplete_runner, tmpdir): cache = autocomplete_cache.AutocompleteCache( From 3923335f2cee218439be91460cd7b3562304b95f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vykintas=20Baltru=C5=A1aitis?= Date: Wed, 29 Nov 2023 16:10:50 +0200 Subject: [PATCH 10/14] Fix skipping forked tests on Windows --- b2/console_tool.py | 2 +- test/unit/_cli/test_autocomplete_cache.py | 29 ++++++++++++++--------- 2 files changed, 19 insertions(+), 12 deletions(-) diff --git a/b2/console_tool.py b/b2/console_tool.py index 3316c64f8..9a2d8657c 100644 --- a/b2/console_tool.py +++ b/b2/console_tool.py @@ -2747,7 +2747,7 @@ def run(self, args): with SyncReport(self.stdout, args.noProgress or args.quiet) as reporter: try: synchronizer.sync_folders( - source_folder=source, + source_folder=source, dest_folder=destination, now_millis=current_time_millis(), reporter=reporter, diff --git a/test/unit/_cli/test_autocomplete_cache.py b/test/unit/_cli/test_autocomplete_cache.py index 5130132fa..579946aa7 100644 --- a/test/unit/_cli/test_autocomplete_cache.py +++ b/test/unit/_cli/test_autocomplete_cache.py @@ -7,6 +7,11 @@ # License https://www.backblaze.com/using_b2_code.html # ###################################################################### + +# Most of the tests in this module are running in a forked process +# because argcomplete and autocomplete_cache mess with global state, +# making the argument parser unusable for other tests. + from __future__ import annotations import contextlib @@ -26,9 +31,16 @@ import b2.console_tool from b2._cli import autocomplete_cache -from ...helpers import skip_on_windows from ..console_tool.conftest import * # noqa +# We can't use pytest.mark.skipif to skip forked tests because with pytest-forked, +# there is an attempt to fork even if the test is marked as skipped. +# See https://github.com/pytest-dev/pytest-forked/issues/44 +if sys.platform == "win32": + forked = pytest.mark.skip(reason="Tests can't be run forked on windows") +else: + forked = pytest.mark.forked + class Exit: """A mocked exit method callable. Instead of actually exiting, @@ -103,8 +115,7 @@ def uncached_complete_result(cache: autocomplete_cache.AutocompleteCache): return exit.code, output.getvalue() -@skip_on_windows -@pytest.mark.forked +@forked def test_complete_main_command(autocomplete_runner, tmpdir): cache = autocomplete_cache.AutocompleteCache( tracker=autocomplete_cache.VersionTracker(), @@ -132,8 +143,7 @@ def test_complete_main_command(autocomplete_runner, tmpdir): assert output == argcomplete_output -@skip_on_windows -@pytest.mark.forked +@forked def test_complete_with_bucket_suggestions(autocomplete_runner, tmpdir, bucket, authorized_b2_cli): cache = autocomplete_cache.AutocompleteCache( tracker=autocomplete_cache.VersionTracker(), @@ -153,8 +163,7 @@ def test_complete_with_bucket_suggestions(autocomplete_runner, tmpdir, bucket, a assert output == argcomplete_output -@skip_on_windows -@pytest.mark.forked +@forked def test_complete_with_file_suggestions( autocomplete_runner, tmpdir, bucket, uploaded_file, authorized_b2_cli ): @@ -181,8 +190,7 @@ def test_complete_with_file_suggestions( assert output == argcomplete_output -@skip_on_windows -@pytest.mark.forked +@forked def test_complete_with_file_uri_suggestions( autocomplete_runner, tmpdir, bucket, uploaded_file, authorized_b2_cli ): @@ -264,8 +272,7 @@ def test_unpickle(): unpickle(pickled) -@skip_on_windows -@pytest.mark.forked +@forked def test_that_autocomplete_cache_loading_does_not_load_b2sdk(autocomplete_runner, tmpdir): cache = autocomplete_cache.AutocompleteCache( tracker=autocomplete_cache.VersionTracker(), From 7a7d3045c0391aa47eebefcf738b9f9ed75fb0f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vykintas=20Baltru=C5=A1aitis?= Date: Wed, 29 Nov 2023 16:12:24 +0200 Subject: [PATCH 11/14] Run formatter --- b2/console_tool.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/b2/console_tool.py b/b2/console_tool.py index 9a2d8657c..3316c64f8 100644 --- a/b2/console_tool.py +++ b/b2/console_tool.py @@ -2747,7 +2747,7 @@ def run(self, args): with SyncReport(self.stdout, args.noProgress or args.quiet) as reporter: try: synchronizer.sync_folders( - source_folder=source, + source_folder=source, dest_folder=destination, now_millis=current_time_millis(), reporter=reporter, From 09faad725d72a1b6565777c7e3e094c78e32755c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vykintas=20Baltru=C5=A1aitis?= Date: Wed, 29 Nov 2023 16:40:19 +0200 Subject: [PATCH 12/14] Use tmp_path instead of tmpdir --- test/unit/_cli/test_autocomplete_cache.py | 24 +++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/test/unit/_cli/test_autocomplete_cache.py b/test/unit/_cli/test_autocomplete_cache.py index 579946aa7..580ec18fd 100644 --- a/test/unit/_cli/test_autocomplete_cache.py +++ b/test/unit/_cli/test_autocomplete_cache.py @@ -116,10 +116,10 @@ def uncached_complete_result(cache: autocomplete_cache.AutocompleteCache): @forked -def test_complete_main_command(autocomplete_runner, tmpdir): +def test_complete_main_command(autocomplete_runner, tmp_path): cache = autocomplete_cache.AutocompleteCache( tracker=autocomplete_cache.VersionTracker(), - store=autocomplete_cache.HomeCachePickleStore(pathlib.Path(tmpdir)), + store=autocomplete_cache.HomeCachePickleStore(tmp_path), ) with autocomplete_runner('b2 '): exit, argcomplete_output = argcomplete_result() @@ -144,10 +144,10 @@ def test_complete_main_command(autocomplete_runner, tmpdir): @forked -def test_complete_with_bucket_suggestions(autocomplete_runner, tmpdir, bucket, authorized_b2_cli): +def test_complete_with_bucket_suggestions(autocomplete_runner, tmp_path, bucket, authorized_b2_cli): cache = autocomplete_cache.AutocompleteCache( tracker=autocomplete_cache.VersionTracker(), - store=autocomplete_cache.HomeCachePickleStore(pathlib.Path(tmpdir)), + store=autocomplete_cache.HomeCachePickleStore(tmp_path), ) with autocomplete_runner('b2 get-bucket '): exit, argcomplete_output = argcomplete_result() @@ -165,12 +165,12 @@ def test_complete_with_bucket_suggestions(autocomplete_runner, tmpdir, bucket, a @forked def test_complete_with_file_suggestions( - autocomplete_runner, tmpdir, bucket, uploaded_file, authorized_b2_cli + autocomplete_runner, tmp_path, bucket, uploaded_file, authorized_b2_cli ): file_name = uploaded_file['fileName'] cache = autocomplete_cache.AutocompleteCache( tracker=autocomplete_cache.VersionTracker(), - store=autocomplete_cache.HomeCachePickleStore(pathlib.Path(tmpdir)), + store=autocomplete_cache.HomeCachePickleStore(tmp_path), ) with autocomplete_runner(f'b2 hide-file {bucket} '): exit, argcomplete_output = argcomplete_result() @@ -192,12 +192,12 @@ def test_complete_with_file_suggestions( @forked def test_complete_with_file_uri_suggestions( - autocomplete_runner, tmpdir, bucket, uploaded_file, authorized_b2_cli + autocomplete_runner, tmp_path, bucket, uploaded_file, authorized_b2_cli ): file_name = uploaded_file['fileName'] cache = autocomplete_cache.AutocompleteCache( tracker=autocomplete_cache.VersionTracker(), - store=autocomplete_cache.HomeCachePickleStore(pathlib.Path(tmpdir)), + store=autocomplete_cache.HomeCachePickleStore(tmp_path), ) with autocomplete_runner(f'b2 download-file b2://{bucket}/'): exit, argcomplete_output = argcomplete_result() @@ -213,8 +213,8 @@ def test_complete_with_file_uri_suggestions( assert output == argcomplete_output -def test_pickle_store(tmpdir): - dir = pathlib.Path(tmpdir) +def test_pickle_store(tmp_path): + dir = tmp_path store = autocomplete_cache.HomeCachePickleStore(dir) store.set_pickle('test_1', b'test_data_1') @@ -273,10 +273,10 @@ def test_unpickle(): @forked -def test_that_autocomplete_cache_loading_does_not_load_b2sdk(autocomplete_runner, tmpdir): +def test_that_autocomplete_cache_loading_does_not_load_b2sdk(autocomplete_runner, tmp_path): cache = autocomplete_cache.AutocompleteCache( tracker=autocomplete_cache.VersionTracker(), - store=autocomplete_cache.HomeCachePickleStore(pathlib.Path(tmpdir)), + store=autocomplete_cache.HomeCachePickleStore(tmp_path), unpickle=unpickle, # using our unpickling function that fails if b2sdk is loaded ) with autocomplete_runner('b2 '): From 30493f6b2d27c5fb622032034812f9f3dced54c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vykintas=20Baltru=C5=A1aitis?= Date: Wed, 29 Nov 2023 16:44:09 +0200 Subject: [PATCH 13/14] Remove unused import --- test/unit/_cli/test_autocomplete_cache.py | 1 - 1 file changed, 1 deletion(-) diff --git a/test/unit/_cli/test_autocomplete_cache.py b/test/unit/_cli/test_autocomplete_cache.py index 580ec18fd..a93ab819b 100644 --- a/test/unit/_cli/test_autocomplete_cache.py +++ b/test/unit/_cli/test_autocomplete_cache.py @@ -18,7 +18,6 @@ import importlib import io import os -import pathlib import pickle import sys from typing import Any From ca9e198257a60809b5a3e56affe723cbb57339c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vykintas=20Baltru=C5=A1aitis?= Date: Wed, 29 Nov 2023 16:55:59 +0200 Subject: [PATCH 14/14] Move common fixtures to /test/unit --- test/unit/_cli/test_autocomplete_cache.py | 2 - test/unit/conftest.py | 77 ++++++++++++++++++++ test/unit/console_tool/conftest.py | 88 ----------------------- 3 files changed, 77 insertions(+), 90 deletions(-) delete mode 100644 test/unit/console_tool/conftest.py diff --git a/test/unit/_cli/test_autocomplete_cache.py b/test/unit/_cli/test_autocomplete_cache.py index a93ab819b..5fb397233 100644 --- a/test/unit/_cli/test_autocomplete_cache.py +++ b/test/unit/_cli/test_autocomplete_cache.py @@ -30,8 +30,6 @@ import b2.console_tool from b2._cli import autocomplete_cache -from ..console_tool.conftest import * # noqa - # We can't use pytest.mark.skipif to skip forked tests because with pytest-forked, # there is an attempt to fork even if the test is marked as skipped. # See https://github.com/pytest-dev/pytest-forked/issues/44 diff --git a/test/unit/conftest.py b/test/unit/conftest.py index c21344b75..998f07487 100644 --- a/test/unit/conftest.py +++ b/test/unit/conftest.py @@ -7,7 +7,10 @@ # License https://www.backblaze.com/using_b2_code.html # ###################################################################### +import os +import sys from test.unit.helpers import RunOrDieExecutor +from test.unit.test_console_tool import BaseConsoleToolTest from unittest import mock import pytest @@ -25,3 +28,77 @@ def bg_executor(): """Executor for running background tasks in tests""" with RunOrDieExecutor() as executor: yield executor + + +class ConsoleToolTester(BaseConsoleToolTest): + def authorize(self): + self._authorize_account() + + def run(self, *args, **kwargs): + return self._run_command(*args, **kwargs) + + +@pytest.fixture +def b2_cli(): + cli_tester = ConsoleToolTester() + cli_tester.setUp() + yield cli_tester + cli_tester.tearDown() + + +@pytest.fixture +def authorized_b2_cli(b2_cli): + b2_cli.authorize() + yield b2_cli + + +@pytest.fixture +def bucket_info(b2_cli, authorized_b2_cli): + bucket_name = "my-bucket" + bucket_id = "bucket_0" + b2_cli.run(['create-bucket', bucket_name, 'allPublic'], expected_stdout=f'{bucket_id}\n') + return { + 'bucketName': bucket_name, + 'bucketId': bucket_id, + } + + +@pytest.fixture +def bucket(bucket_info): + return bucket_info['bucketName'] + + +@pytest.fixture +def mock_stdin(monkeypatch): + out_, in_ = os.pipe() + monkeypatch.setattr(sys, 'stdin', os.fdopen(out_)) + in_f = open(in_, 'w') + yield in_f + in_f.close() + + +@pytest.fixture +def local_file(tmp_path): + """Set up a test file and return its path.""" + filename = 'file1.txt' + content = 'hello world' + local_file = tmp_path / filename + local_file.write_text(content) + + mod_time = 1500111222 + os.utime(local_file, (mod_time, mod_time)) + + return local_file + + +@pytest.fixture +def uploaded_file(b2_cli, bucket_info, local_file): + filename = 'file1.txt' + b2_cli.run(['upload-file', '--quiet', bucket_info["bucketName"], str(local_file), filename]) + return { + 'bucket': bucket_info["bucketName"], + 'bucketId': bucket_info["bucketId"], + 'fileName': filename, + 'fileId': '9999', + 'content': local_file.read_text(), + } diff --git a/test/unit/console_tool/conftest.py b/test/unit/console_tool/conftest.py deleted file mode 100644 index aae3562fc..000000000 --- a/test/unit/console_tool/conftest.py +++ /dev/null @@ -1,88 +0,0 @@ -###################################################################### -# -# File: test/unit/console_tool/conftest.py -# -# Copyright 2023 Backblaze Inc. All Rights Reserved. -# -# License https://www.backblaze.com/using_b2_code.html -# -###################################################################### -import os -import sys -from test.unit.test_console_tool import BaseConsoleToolTest - -import pytest - - -class ConsoleToolTester(BaseConsoleToolTest): - def authorize(self): - self._authorize_account() - - def run(self, *args, **kwargs): - return self._run_command(*args, **kwargs) - - -@pytest.fixture -def b2_cli(): - cli_tester = ConsoleToolTester() - cli_tester.setUp() - yield cli_tester - cli_tester.tearDown() - - -@pytest.fixture -def authorized_b2_cli(b2_cli): - b2_cli.authorize() - yield b2_cli - - -@pytest.fixture -def bucket_info(b2_cli, authorized_b2_cli): - bucket_name = "my-bucket" - bucket_id = "bucket_0" - b2_cli.run(['create-bucket', bucket_name, 'allPublic'], expected_stdout=f'{bucket_id}\n') - return { - 'bucketName': bucket_name, - 'bucketId': bucket_id, - } - - -@pytest.fixture -def bucket(bucket_info): - return bucket_info['bucketName'] - - -@pytest.fixture -def mock_stdin(monkeypatch): - out_, in_ = os.pipe() - monkeypatch.setattr(sys, 'stdin', os.fdopen(out_)) - in_f = open(in_, 'w') - yield in_f - in_f.close() - - -@pytest.fixture -def local_file(tmp_path): - """Set up a test file and return its path.""" - filename = 'file1.txt' - content = 'hello world' - local_file = tmp_path / filename - local_file.write_text(content) - - mod_time = 1500111222 - os.utime(local_file, (mod_time, mod_time)) - - return local_file - - -@pytest.fixture -def uploaded_file(b2_cli, bucket_info, local_file): - filename = 'file1.txt' - b2_cli.run(['upload-file', '--quiet', bucket_info["bucketName"], str(local_file), filename]) - return { - 'bucket': bucket_info["bucketName"], - 'bucketId': bucket_info["bucketId"], - 'fileName': filename, - 'fileId': '9999', - 'content': local_file.read_text(), - }