diff --git a/b2/arg_parser.py b/b2/arg_parser.py index 45d60673f..9a0e0d4e4 100644 --- a/b2/arg_parser.py +++ b/b2/arg_parser.py @@ -160,9 +160,7 @@ def wrap_with_argument_type_error(func, translator=str, exc_type=ValueError): def wrapper(*args, **kwargs): try: return func(*args, **kwargs) - except Exception as e: - if isinstance(e, exc_type): - raise argparse.ArgumentTypeError(translator(e)) - raise + except exc_type as e: + raise argparse.ArgumentTypeError(translator(e)) return wrapper diff --git a/test/integration/cleanup_buckets.py b/test/integration/cleanup_buckets.py index 45adf743a..51d216528 100644 --- a/test/integration/cleanup_buckets.py +++ b/test/integration/cleanup_buckets.py @@ -13,4 +13,4 @@ def test_cleanup_buckets(b2_api): # this is not a test, but it is intended to be called # via pytest because it reuses fixtures which have everything # set up - b2_api.clean_buckets() + pass # b2_api calls b2_api.clean_buckets() in its finalizer diff --git a/test/integration/conftest.py b/test/integration/conftest.py index a94ca504d..b1b44b9fb 100755 --- a/test/integration/conftest.py +++ b/test/integration/conftest.py @@ -7,8 +7,9 @@ # License https://www.backblaze.com/using_b2_code.html # ###################################################################### +from __future__ import annotations -import contextlib +import logging import os import pathlib import subprocess @@ -18,16 +19,38 @@ from tempfile import TemporaryDirectory import pytest -from b2sdk.exception import BadRequest, BucketIdNotFound -from b2sdk.v2 import B2_ACCOUNT_INFO_ENV_VAR, XDG_CONFIG_HOME_ENV_VAR +from b2sdk.v2 import B2_ACCOUNT_INFO_ENV_VAR, XDG_CONFIG_HOME_ENV_VAR, Bucket -from .helpers import Api, CommandLine, bucket_name_part +from .helpers import NODE_DESCRIPTION, RNG_SEED, Api, CommandLine, bucket_name_part, random_token + +logger = logging.getLogger(__name__) GENERAL_BUCKET_NAME_PREFIX = 'clitst' TEMPDIR = tempfile.gettempdir() ROOT_PATH = pathlib.Path(__file__).parent.parent.parent +@pytest.fixture(scope='session', autouse=True) +def summary_notes(request, worker_id): + capmanager = request.config.pluginmanager.getplugin("capturemanager") + with capmanager.global_and_fixture_disabled(): + log_handler = logging.StreamHandler(sys.stderr) + log_fmt = logging.Formatter(f'{worker_id} %(asctime)s %(levelname).1s %(message)s') + log_handler.setFormatter(log_fmt) + logger.addHandler(log_handler) + + class Notes: + def append(self, note): + logger.info(note) + + return Notes() + + +@pytest.fixture(scope='session', autouse=True) +def node_stats(summary_notes): + summary_notes.append(f"NODE={NODE_DESCRIPTION} seed={RNG_SEED}") + + @pytest.hookimpl def pytest_addoption(parser): parser.addoption( @@ -64,19 +87,19 @@ def realm() -> str: yield environ.get('B2_TEST_ENVIRONMENT', 'production') -@pytest.fixture(scope='function') -def bucket(b2_api) -> str: - try: - bucket = b2_api.create_bucket() - except BadRequest as e: - if e.code != 'too_many_buckets': - raise - num_buckets = b2_api.count_and_print_buckets() - print('current number of buckets:', num_buckets) - raise - yield bucket - with contextlib.suppress(BucketIdNotFound): - b2_api.clean_bucket(bucket) +@pytest.fixture +def bucket(bucket_factory) -> Bucket: + return bucket_factory() + + +@pytest.fixture +def bucket_factory(b2_api, schedule_bucket_cleanup): + def create_bucket(**kwargs): + new_bucket = b2_api.create_bucket(**kwargs) + schedule_bucket_cleanup(new_bucket.name, new_bucket.bucket_dict) + return new_bucket + + yield create_bucket @pytest.fixture(scope='function') @@ -86,7 +109,7 @@ def bucket_name(bucket) -> str: @pytest.fixture(scope='function') def file_name(bucket) -> str: - file_ = bucket.upload_bytes(b'test_file', f'{bucket_name_part(8)}.txt') + file_ = bucket.upload_bytes(b'test_file', f'{random_token(8)}.txt') yield file_.file_name @@ -111,44 +134,55 @@ def this_run_bucket_name_prefix() -> str: yield GENERAL_BUCKET_NAME_PREFIX + bucket_name_part(8) -@pytest.fixture(scope='module') -def monkey_patch(): - """ Module-scope monkeypatching (original `monkeypatch` is function-scope) """ - from _pytest.monkeypatch import MonkeyPatch - monkey = MonkeyPatch() - yield monkey - monkey.undo() +@pytest.fixture(scope='session') +def monkeysession(): + with pytest.MonkeyPatch.context() as mp: + yield mp -@pytest.fixture(scope='module', autouse=True) -def auto_change_account_info_dir(monkey_patch) -> dir: +@pytest.fixture(scope='session', autouse=True) +def auto_change_account_info_dir(monkeysession) -> dir: """ - Automatically for the whole module testing: + Automatically for the whole testing: 1) temporary remove B2_APPLICATION_KEY and B2_APPLICATION_KEY_ID from environment 2) create a temporary directory for storing account info database """ - monkey_patch.delenv('B2_APPLICATION_KEY_ID', raising=False) - monkey_patch.delenv('B2_APPLICATION_KEY', raising=False) + monkeysession.delenv('B2_APPLICATION_KEY_ID', raising=False) + monkeysession.delenv('B2_APPLICATION_KEY', raising=False) # make b2sdk use temp dir for storing default & per-profile account information with TemporaryDirectory() as temp_dir: - monkey_patch.setenv(B2_ACCOUNT_INFO_ENV_VAR, path.join(temp_dir, '.b2_account_info')) - monkey_patch.setenv(XDG_CONFIG_HOME_ENV_VAR, temp_dir) + monkeysession.setenv(B2_ACCOUNT_INFO_ENV_VAR, path.join(temp_dir, '.b2_account_info')) + monkeysession.setenv(XDG_CONFIG_HOME_ENV_VAR, temp_dir) yield temp_dir -@pytest.fixture(scope='module') -def b2_api(application_key_id, application_key, realm, this_run_bucket_name_prefix) -> Api: - yield Api( - application_key_id, application_key, realm, GENERAL_BUCKET_NAME_PREFIX, - this_run_bucket_name_prefix +@pytest.fixture(scope='session') +def b2_api( + application_key_id, + application_key, + realm, + this_run_bucket_name_prefix, + auto_change_account_info_dir, + summary_notes, +) -> Api: + api = Api( + application_key_id, + application_key, + realm, + general_bucket_name_prefix=GENERAL_BUCKET_NAME_PREFIX, + this_run_bucket_name_prefix=this_run_bucket_name_prefix, ) + yield api + api.clean_buckets() + summary_notes.append(f"Buckets names used during this tests: {api.bucket_name_log!r}") @pytest.fixture(scope='module') def global_b2_tool( - request, application_key_id, application_key, realm, this_run_bucket_name_prefix + request, application_key_id, application_key, realm, this_run_bucket_name_prefix, b2_api, + auto_change_account_info_dir ) -> CommandLine: tool = CommandLine( request.config.getoption('--sut'), @@ -157,9 +191,10 @@ def global_b2_tool( realm, this_run_bucket_name_prefix, request.config.getoption('--env-file-cmd-placeholder'), + api_wrapper=b2_api, ) tool.reauthorize(check_key_capabilities=True) # reauthorize for the first time (with check) - return tool + yield tool @pytest.fixture(scope='function') @@ -169,6 +204,23 @@ def b2_tool(global_b2_tool): return global_b2_tool +@pytest.fixture +def schedule_bucket_cleanup(global_b2_tool): + """ + Explicitly ask for buckets cleanup after the test + + This should be only used when testing `create-bucket` command; otherwise use `bucket_factory` fixture. + """ + buckets_to_clean = {} + + def add_bucket_to_cleanup(bucket_name, bucket_dict: dict | None = None): + buckets_to_clean[bucket_name] = bucket_dict + + yield add_bucket_to_cleanup + for bucket, bucket_dict_ in buckets_to_clean.items(): + global_b2_tool.cleanup_bucket(bucket, bucket_dict_) + + @pytest.fixture(autouse=True, scope='session') def sample_filepath(): """Copy the README.md file to /tmp so that docker tests can access it""" @@ -223,12 +275,12 @@ def b2_in_path(tmp_path_factory): @pytest.fixture(scope="module") -def env(b2_in_path, homedir, monkey_patch, is_running_on_docker): +def env(b2_in_path, homedir, monkeysession, is_running_on_docker): """Get ENV for running b2 command from shell level.""" if not is_running_on_docker: - monkey_patch.setenv('PATH', b2_in_path) - monkey_patch.setenv('HOME', str(homedir)) - monkey_patch.setenv('SHELL', "/bin/bash") # fix for running under github actions + monkeysession.setenv('PATH', b2_in_path) + monkeysession.setenv('HOME', str(homedir)) + monkeysession.setenv('SHELL', "/bin/bash") # fix for running under github actions yield os.environ diff --git a/test/integration/helpers.py b/test/integration/helpers.py index 7eb867a3e..5cef790fc 100755 --- a/test/integration/helpers.py +++ b/test/integration/helpers.py @@ -7,6 +7,9 @@ # License https://www.backblaze.com/using_b2_code.html # ###################################################################### +from __future__ import annotations + +import dataclasses import json import logging import os @@ -14,6 +17,7 @@ import platform import random import re +import secrets import shutil import string import subprocess @@ -26,13 +30,13 @@ from os import environ, linesep, path from pathlib import Path from tempfile import gettempdir, mkdtemp, mktemp -from typing import List, Optional, Union from unittest.mock import MagicMock import backoff -from b2sdk._v3.exception import BucketIdNotFound as v3BucketIdNotFound from b2sdk.v2 import ( ALL_CAPABILITIES, + BUCKET_NAME_CHARS_UNIQ, + BUCKET_NAME_LENGTH_RANGE, NO_RETENTION_FILE_SETTING, B2Api, Bucket, @@ -48,63 +52,80 @@ fix_windows_path_limit, ) from b2sdk.v2.exception import ( + BadRequest, BucketIdNotFound, - DuplicateBucketName, FileNotPresent, TooManyRequests, + v3BucketIdNotFound, ) from b2.console_tool import Command, current_time_millis logger = logging.getLogger(__name__) -BUCKET_CLEANUP_PERIOD_MILLIS = timedelta(hours=6).total_seconds() * 1000 +# A large period is set here to avoid issues related to clock skew or other time-related issues under CI +BUCKET_CLEANUP_PERIOD_MILLIS = timedelta(days=1).total_seconds() * 1000 ONE_HOUR_MILLIS = 60 * 60 * 1000 ONE_DAY_MILLIS = ONE_HOUR_MILLIS * 24 -BUCKET_NAME_LENGTH = 50 -BUCKET_NAME_CHARS = string.ascii_letters + string.digits + '-' +BUCKET_NAME_LENGTH = BUCKET_NAME_LENGTH_RANGE[1] BUCKET_CREATED_AT_MILLIS = 'created_at_millis' +NODE_DESCRIPTION = f"{platform.node()}: {platform.platform()}" + + +def get_seed(): + """ + Get seed for random number generator. + + GH Actions machines seem to offer a very limited entropy pool + """ + return b''.join( + ( + secrets.token_bytes(32), + str(time.time_ns()).encode(), + NODE_DESCRIPTION.encode(), + str(os.getpid()).encode(), # needed due to pytest-xdist + str(environ).encode('utf8', errors='ignore' + ), # especially helpful under GitHub (and similar) CI + ) + ) + + +RNG = random.Random(get_seed()) +RNG_SEED = RNG.randint(0, 2 << 31) +RNG_COUNTER = 0 + +if sys.version_info < (3, 9): + RNG.randbytes = lambda n: RNG.getrandbits(n * 8).to_bytes(n, 'little') + SSE_NONE = EncryptionSetting(mode=EncryptionMode.NONE,) SSE_B2_AES = EncryptionSetting( mode=EncryptionMode.SSE_B2, algorithm=EncryptionAlgorithm.AES256, ) +_SSE_KEY = RNG.randbytes(32) SSE_C_AES = EncryptionSetting( mode=EncryptionMode.SSE_C, algorithm=EncryptionAlgorithm.AES256, - key=EncryptionKey(secret=os.urandom(32), key_id='user-generated-key-id') + key=EncryptionKey(secret=_SSE_KEY, key_id='user-generated-key-id') ) SSE_C_AES_2 = EncryptionSetting( mode=EncryptionMode.SSE_C, algorithm=EncryptionAlgorithm.AES256, - key=EncryptionKey(secret=os.urandom(32), key_id='another-user-generated-key-id') -) - -RNG_SEED = '_'.join( - [ - os.getenv('GITHUB_REPOSITORY', ''), - os.getenv('GITHUB_SHA', ''), - os.getenv('GITHUB_RUN_ID', ''), - os.getenv('GITHUB_RUN_ATTEMPT', ''), - os.getenv('GITHUB_JOB', ''), - os.getenv('GITHUB_ACTION', ''), - str(os.getpid()), # for local runs with xdist - str(time.time()), - ] + key=EncryptionKey(secret=_SSE_KEY, key_id='another-user-generated-key-id') ) -RNG = random.Random(RNG_SEED) -RNG_COUNTER = 0 +def random_token(length: int, chars=string.ascii_letters) -> str: + return ''.join(RNG.choice(chars) for _ in range(length)) def bucket_name_part(length: int) -> str: assert length >= 1 global RNG_COUNTER RNG_COUNTER += 1 - name_part = ''.join(RNG.choice(BUCKET_NAME_CHARS) for _ in range(length)) + name_part = random_token(length, BUCKET_NAME_CHARS_UNIQ) logger.info('RNG_SEED: %s', RNG_SEED) logger.info('RNG_COUNTER: %i, length: %i', RNG_COUNTER, length) logger.info('name_part: %s', name_part) @@ -120,6 +141,7 @@ class Api: this_run_bucket_name_prefix: str api: B2Api = None + bucket_name_log: list[str] = dataclasses.field(default_factory=list) def __post_init__(self): info = InMemoryAccountInfo() @@ -130,55 +152,67 @@ def __post_init__(self): self.this_run_bucket_name_prefix ) > 5, self.this_run_bucket_name_prefix - def create_bucket(self) -> Bucket: - for _ in range(10): - bucket_name = self.this_run_bucket_name_prefix + bucket_name_part( - BUCKET_NAME_LENGTH - len(self.this_run_bucket_name_prefix) - ) - print('Creating bucket:', bucket_name) - try: - return self.api.create_bucket( - bucket_name, - 'allPublic', - bucket_info={BUCKET_CREATED_AT_MILLIS: str(current_time_millis())}, - ) - except DuplicateBucketName: - pass - print() + def new_bucket_name(self) -> str: + bucket_name = self.this_run_bucket_name_prefix + bucket_name_part( + BUCKET_NAME_LENGTH - len(self.this_run_bucket_name_prefix) + ) + self.bucket_name_log.append(bucket_name) + return bucket_name - raise ValueError('Failed to create bucket due to name collision') + def new_bucket_info(self) -> dict: + return { + BUCKET_CREATED_AT_MILLIS: str(current_time_millis()), + "created_by": NODE_DESCRIPTION, + } - def _should_remove_bucket(self, bucket: Bucket): + @backoff.on_exception( + backoff.expo, + BadRequest, # there is no specialized exception for too_many_buckets + max_time=60, + max_value=5, + ) + def create_bucket(self, bucket_type: str = 'allPublic', **kwargs) -> Bucket: + bucket_name = self.new_bucket_name() + return self.api.create_bucket( + bucket_name, + bucket_type=bucket_type, + bucket_info=self.new_bucket_info(), + **kwargs, + ) + + def _should_remove_bucket(self, bucket: Bucket) -> tuple[bool, str]: if bucket.name.startswith(self.this_run_bucket_name_prefix): return True, 'it is a bucket for this very run' - OLD_PATTERN = 'test-b2-cli-' - if bucket.name.startswith(self.general_bucket_name_prefix) or bucket.name.startswith(OLD_PATTERN): # yapf: disable + if bucket.name.startswith(self.general_bucket_name_prefix): if BUCKET_CREATED_AT_MILLIS in bucket.bucket_info: delete_older_than = current_time_millis() - BUCKET_CLEANUP_PERIOD_MILLIS - this_bucket_creation_time = bucket.bucket_info[BUCKET_CREATED_AT_MILLIS] - if int(this_bucket_creation_time) < delete_older_than: + this_bucket_creation_time = int(bucket.bucket_info[BUCKET_CREATED_AT_MILLIS]) + if this_bucket_creation_time < delete_older_than: return True, f"this_bucket_creation_time={this_bucket_creation_time} < delete_older_than={delete_older_than}" + return False, f"this_bucket_creation_time={this_bucket_creation_time} >= delete_older_than={delete_older_than}" else: return True, 'undefined ' + BUCKET_CREATED_AT_MILLIS - return False, '' + return False, f'does not start with {self.general_bucket_name_prefix!r}' - def clean_buckets(self): - buckets = self.api.list_buckets() + def clean_buckets(self, quick=False): + # even with use_cache=True, if cache is empty API call will be made + buckets = self.api.list_buckets(use_cache=quick) print('Total bucket count:', len(buckets)) + remaining_buckets = [] for bucket in buckets: should_remove, why = self._should_remove_bucket(bucket) if not should_remove: - print(f'Skipping bucket removal: "{bucket.name}"') + print(f'Skipping bucket removal {bucket.name!r} because {why}') + remaining_buckets.append(bucket) continue print('Trying to remove bucket:', bucket.name, 'because', why) try: self.clean_bucket(bucket) - except (BucketIdNotFound, v3BucketIdNotFound): + except BucketIdNotFound: print(f'It seems that bucket {bucket.name} has already been removed') - buckets = self.api.list_buckets() - print('Total bucket count after cleanup:', len(buckets)) - for bucket in buckets: + print('Total bucket count after cleanup:', len(remaining_buckets)) + for bucket in remaining_buckets: print(bucket) @backoff.on_exception( @@ -186,10 +220,18 @@ def clean_buckets(self): TooManyRequests, max_tries=8, ) - def clean_bucket(self, bucket: Union[Bucket, str]): + def clean_bucket(self, bucket: Bucket | str): if isinstance(bucket, str): bucket = self.api.get_bucket_by_name(bucket) + # try optimistic bucket removal first, since it is completely free (as opposed to `ls` call) + try: + return self.api.delete_bucket(bucket) + except (BucketIdNotFound, v3BucketIdNotFound): + return # bucket was already removed + except BadRequest as exc: + assert exc.code == 'cannot_delete_non_empty_bucket' + files_leftover = False file_versions = bucket.ls(latest_only=False, recursive=True) @@ -353,8 +395,14 @@ class CommandLine: ] def __init__( - self, command, account_id, application_key, realm, bucket_name_prefix, - env_file_cmd_placeholder + self, + command, + account_id, + application_key, + realm, + bucket_name_prefix, + env_file_cmd_placeholder, + api_wrapper: Api, ): self.command = command self.account_id = account_id @@ -363,14 +411,12 @@ def __init__( self.bucket_name_prefix = bucket_name_prefix self.env_file_cmd_placeholder = env_file_cmd_placeholder self.env_var_test_context = EnvVarTestContext(SqliteAccountInfo().filename) - self.account_info_file_name = SqliteAccountInfo().filename + self.api_wrapper = api_wrapper def generate_bucket_name(self): - return self.bucket_name_prefix + bucket_name_part( - BUCKET_NAME_LENGTH - len(self.bucket_name_prefix) - ) + return self.api_wrapper.new_bucket_name() - def run_command(self, args, additional_env: Optional[dict] = None): + def run_command(self, args, additional_env: dict | None = None): """ Runs the command with the given arguments, returns a tuple in form of (succeeded, stdout) @@ -380,9 +426,9 @@ def run_command(self, args, additional_env: Optional[dict] = None): def should_succeed( self, - args: Optional[List[str]], - expected_pattern: Optional[str] = None, - additional_env: Optional[dict] = None, + args: list[str] | None, + expected_pattern: str | None = None, + additional_env: dict | None = None, ) -> str: """ Runs the command-line with the given arguments. Raises an exception @@ -404,7 +450,7 @@ def should_succeed( return stdout @classmethod - def prepare_env(self, additional_env: Optional[dict] = None): + def prepare_env(self, additional_env: dict | None = None): environ['PYTHONPATH'] = '.' environ['PYTHONIOENCODING'] = 'utf-8' env = environ.copy() @@ -431,8 +477,8 @@ def parse_command(self, env): def execute( self, - args: Optional[List[Union[str, Path, int]]] = None, - additional_env: Optional[dict] = None, + args: list[str | Path | int] | None = None, + additional_env: dict | None = None, ): """ :param cmd: a command to run @@ -445,7 +491,7 @@ def execute( env = self.prepare_env(additional_env) command = self.parse_command(env) - args: List[str] = [str(arg) for arg in args] if args else [] + args: list[str] = [str(arg) for arg in args] if args else [] command.extend(args) print('Running:', ' '.join(command)) @@ -476,7 +522,7 @@ def execute( print_output(p.returncode, stdout_decoded, stderr_decoded) return p.returncode, stdout_decoded, stderr_decoded - def should_succeed_json(self, args, additional_env: Optional[dict] = None): + def should_succeed_json(self, args, additional_env: dict | None = None): """ Runs the command-line with the given arguments. Raises an exception if there was an error; otherwise, treats the stdout as JSON and returns @@ -489,7 +535,7 @@ def should_succeed_json(self, args, additional_env: Optional[dict] = None): raise ValueError(f'{result} is not a valid json') return loaded_result - def should_fail(self, args, expected_pattern, additional_env: Optional[dict] = None): + def should_fail(self, args, expected_pattern, additional_env: dict | None = None): """ Runs the command-line with the given args, expecting the given pattern to appear in stderr. @@ -519,6 +565,26 @@ def reauthorize(self, check_key_capabilities=False): def list_file_versions(self, bucket_name): return self.should_succeed_json(['ls', '--json', '--recursive', '--versions', bucket_name]) + def cleanup_bucket(self, bucket_name: str, bucket_dict: dict | None = None) -> None: + """ + Cleanup bucket + + Since bucket was being handled by the tool, it is safe to assume it is cached in its cache and we don't + need to call C class API list_buckets endpoint to get it. + """ + if not bucket_dict: + try: + bucket_dict = self.should_succeed_json(['get-bucket', bucket_name]) + except (ValueError, AssertionError): # bucket doesn't exist + return + + bucket = self.api_wrapper.api.BUCKET_CLASS( + api=self.api_wrapper.api, + id_=bucket_dict['bucketId'], + name=bucket_name, + ) + self.api_wrapper.clean_bucket(bucket) + class TempDir: def __init__(self): @@ -529,11 +595,6 @@ def __init__(self): ) self.dirpath = None - def get_dir(self): - assert self.dirpath is not None, \ - "can't call get_dir() before entering the context manager" - return self.dirpath - def __enter__(self): self.dirpath = mkdtemp() return Path(self.dirpath) @@ -542,27 +603,23 @@ def __exit__(self, exc_type, exc_val, exc_tb): shutil.rmtree(fix_windows_path_limit(self.dirpath)) -def read_file(path: Union[str, Path]): +def read_file(path: str | Path): with open(path, 'rb') as f: return f.read() -def write_file(path: Union[str, Path], contents: bytes): +def write_file(path: str | Path, contents: bytes): with open(path, 'wb') as f: f.write(contents) -def file_mod_time_millis(path: Union[str, Path]): - if isinstance(path, Path): - path = str(path) +def file_mod_time_millis(path: str | Path) -> int: return int(os.path.getmtime(path) * 1000) -def set_file_mod_time_millis(path: Union[str, Path], time): - if isinstance(path, Path): - path = str(path) +def set_file_mod_time_millis(path: str | Path, time): os.utime(path, (os.path.getatime(path), time / 1000)) def random_hex(length): - return ''.join(random.choice('0123456789abcdef') for _ in range(length)) + return ''.join(RNG.choice('0123456789abcdef') for _ in range(length)) diff --git a/test/integration/test_b2_command_line.py b/test/integration/test_b2_command_line.py index d1177f68d..b0df6ed8b 100755 --- a/test/integration/test_b2_command_line.py +++ b/test/integration/test_b2_command_line.py @@ -10,6 +10,7 @@ ###################################################################### import base64 +import contextlib import hashlib import itertools import json @@ -85,7 +86,7 @@ def test_download(b2_tool, bucket_name, sample_filepath, uploaded_sample_file, t assert output_b.read_text() == sample_filepath.read_text() -def test_basic(b2_tool, bucket_name, sample_file, is_running_on_docker): +def test_basic(b2_tool, bucket_name, sample_file, is_running_on_docker, tmp_path): file_mod_time_str = str(file_mod_time_millis(sample_file)) @@ -131,13 +132,9 @@ def test_basic(b2_tool, bucket_name, sample_file, is_running_on_docker): should_equal(['rm1'], [f['fileName'] for f in list_of_files]) b2_tool.should_succeed(['rm', '--recursive', '--withWildcard', bucket_name, 'rm1']) - with TempDir() as dir_path: - b2_tool.should_succeed( - [ - 'download-file-by-name', '--noProgress', '--quiet', bucket_name, 'b/1', - dir_path / 'a' - ] - ) + b2_tool.should_succeed( + ['download-file-by-name', '--noProgress', '--quiet', bucket_name, 'b/1', tmp_path / 'a'] + ) b2_tool.should_succeed(['hide-file', bucket_name, 'c']) @@ -280,10 +277,8 @@ def test_bucket(b2_tool, bucket_name): ] -def test_key_restrictions(b2_api, b2_tool, bucket_name, sample_file): - - second_bucket_name = b2_tool.generate_bucket_name() - b2_tool.should_succeed(['create-bucket', second_bucket_name, 'allPublic', *get_bucketinfo()],) +def test_key_restrictions(b2_tool, bucket_name, sample_file, bucket_factory): + second_bucket_name = bucket_factory().name # A single file for rm to fail on. b2_tool.should_succeed(['upload-file', '--noProgress', bucket_name, sample_file, 'test']) @@ -346,7 +341,6 @@ def test_key_restrictions(b2_api, b2_tool, bucket_name, sample_file): b2_tool.application_key ] ) - b2_api.clean_bucket(second_bucket_name) b2_tool.should_succeed(['delete-key', key_one_id]) b2_tool.should_succeed(['delete-key', key_two_id]) @@ -836,13 +830,15 @@ def sync_down_helper(b2_tool, bucket_name, folder_in_bucket, sample_file, encryp ) -def test_sync_copy(b2_api, b2_tool, bucket_name, sample_file): - prepare_and_run_sync_copy_tests(b2_api, b2_tool, bucket_name, 'sync', sample_file=sample_file) +def test_sync_copy(bucket_factory, b2_tool, bucket_name, sample_file): + prepare_and_run_sync_copy_tests( + bucket_factory, b2_tool, bucket_name, 'sync', sample_file=sample_file + ) -def test_sync_copy_no_prefix_default_encryption(b2_api, b2_tool, bucket_name, sample_file): +def test_sync_copy_no_prefix_default_encryption(bucket_factory, b2_tool, bucket_name, sample_file): prepare_and_run_sync_copy_tests( - b2_api, + bucket_factory, b2_tool, bucket_name, '', @@ -852,9 +848,9 @@ def test_sync_copy_no_prefix_default_encryption(b2_api, b2_tool, bucket_name, sa ) -def test_sync_copy_no_prefix_no_encryption(b2_api, b2_tool, bucket_name, sample_file): +def test_sync_copy_no_prefix_no_encryption(bucket_factory, b2_tool, bucket_name, sample_file): prepare_and_run_sync_copy_tests( - b2_api, + bucket_factory, b2_tool, bucket_name, '', @@ -864,9 +860,9 @@ def test_sync_copy_no_prefix_no_encryption(b2_api, b2_tool, bucket_name, sample_ ) -def test_sync_copy_no_prefix_sse_b2(b2_api, b2_tool, bucket_name, sample_file): +def test_sync_copy_no_prefix_sse_b2(bucket_factory, b2_tool, bucket_name, sample_file): prepare_and_run_sync_copy_tests( - b2_api, + bucket_factory, b2_tool, bucket_name, '', @@ -876,9 +872,9 @@ def test_sync_copy_no_prefix_sse_b2(b2_api, b2_tool, bucket_name, sample_file): ) -def test_sync_copy_no_prefix_sse_c(b2_api, b2_tool, bucket_name, sample_file): +def test_sync_copy_no_prefix_sse_c(bucket_factory, b2_tool, bucket_name, sample_file): prepare_and_run_sync_copy_tests( - b2_api, + bucket_factory, b2_tool, bucket_name, '', @@ -922,7 +918,7 @@ def test_sync_copy_sse_c_single_bucket(b2_tool, bucket_name, sample_file): def prepare_and_run_sync_copy_tests( - b2_api, + bucket_factory, b2_tool, bucket_name, folder_in_bucket, @@ -938,10 +934,7 @@ def prepare_and_run_sync_copy_tests( else: b2_file_prefix = '' - other_bucket_name = b2_tool.generate_bucket_name() - success, _ = b2_tool.run_command( - ['create-bucket', other_bucket_name, 'allPublic', *get_bucketinfo()] - ) + other_bucket_name = bucket_factory().name other_b2_sync_point = 'b2:%s' % other_bucket_name if folder_in_bucket: @@ -977,8 +970,6 @@ def prepare_and_run_sync_copy_tests( file_version_summary_with_encryption(file_versions), ) - b2_api.clean_bucket(other_bucket_name) - def run_sync_copy_with_basic_checks( b2_tool, @@ -1095,7 +1086,7 @@ def test_sync_long_path(b2_tool, bucket_name): should_equal(['+ ' + long_path], file_version_summary(file_versions)) -def test_default_sse_b2(b2_api, b2_tool, bucket_name): +def test_default_sse_b2(b2_tool, bucket_name, schedule_bucket_cleanup): # Set default encryption via update-bucket bucket_info = b2_tool.should_succeed_json(['get-bucket', bucket_name]) bucket_default_sse = {'mode': 'none'} @@ -1119,6 +1110,7 @@ def test_default_sse_b2(b2_api, b2_tool, bucket_name): # Set default encryption via create-bucket second_bucket_name = b2_tool.generate_bucket_name() + schedule_bucket_cleanup(second_bucket_name) b2_tool.should_succeed( [ 'create-bucket', @@ -1134,10 +1126,9 @@ def test_default_sse_b2(b2_api, b2_tool, bucket_name): 'mode': 'SSE-B2', } should_equal(second_bucket_default_sse, second_bucket_info['defaultServerSideEncryption']) - b2_api.clean_bucket(second_bucket_name) -def test_sse_b2(b2_tool, bucket_name, sample_file): +def test_sse_b2(b2_tool, bucket_name, sample_file, tmp_path): b2_tool.should_succeed( [ 'upload-file', '--destinationServerSideEncryption=SSE-B2', '--quiet', bucket_name, @@ -1145,16 +1136,16 @@ def test_sse_b2(b2_tool, bucket_name, sample_file): ] ) b2_tool.should_succeed(['upload-file', '--quiet', bucket_name, sample_file, 'not_encrypted']) - with TempDir() as dir_path: - b2_tool.should_succeed( - ['download-file-by-name', '--quiet', bucket_name, 'encrypted', dir_path / 'encrypted'] - ) - b2_tool.should_succeed( - [ - 'download-file-by-name', '--quiet', bucket_name, 'not_encrypted', - dir_path / 'not_encypted' - ] - ) + + b2_tool.should_succeed( + ['download-file-by-name', '--quiet', bucket_name, 'encrypted', tmp_path / 'encrypted'] + ) + b2_tool.should_succeed( + [ + 'download-file-by-name', '--quiet', bucket_name, 'not_encrypted', + tmp_path / 'not_encypted' + ] + ) list_of_files = b2_tool.should_succeed_json(['ls', '--json', '--recursive', bucket_name]) should_equal( @@ -1204,7 +1195,7 @@ def test_sse_b2(b2_tool, bucket_name, sample_file): should_equal({'mode': 'none'}, file_info['serverSideEncryption']) -def test_sse_c(b2_tool, bucket_name, is_running_on_docker, sample_file): +def test_sse_c(b2_tool, bucket_name, is_running_on_docker, sample_file, tmp_path): sse_c_key_id = 'user-generated-key-id \nąóźćż\nœøΩ≈ç\nßäöü' if is_running_on_docker: @@ -1264,7 +1255,7 @@ def test_sse_c(b2_tool, bucket_name, is_running_on_docker, sample_file): expected_pattern='ERROR: Wrong or no SSE-C key provided when reading a file.', additional_env={'B2_SOURCE_SSE_C_KEY_B64': base64.b64encode(os.urandom(32)).decode()} ) - with TempDir() as dir_path: + with contextlib.nullcontext(tmp_path) as dir_path: b2_tool.should_succeed( [ 'download-file-by-name', @@ -1573,16 +1564,11 @@ def test_license(b2_tool, with_packages): SOFTWARE.""" in license_text.replace(os.linesep, '\n'), repr(license_text[-2000:]) -def test_file_lock(b2_tool, application_key_id, application_key, b2_api, sample_file): - lock_disabled_bucket_name = b2_tool.generate_bucket_name() - b2_tool.should_succeed( - [ - 'create-bucket', - lock_disabled_bucket_name, - 'allPrivate', - *get_bucketinfo(), - ], - ) +def test_file_lock( + b2_tool, application_key_id, application_key, sample_file, bucket_factory, + schedule_bucket_cleanup +): + lock_disabled_bucket_name = bucket_factory(bucket_type='allPrivate').name now_millis = current_time_millis() @@ -1626,6 +1612,7 @@ def test_file_lock(b2_tool, application_key_id, application_key, b2_api, sample_ ], r'ERROR: The bucket is not file lock enabled \(bucket_missing_file_lock\)' ) lock_enabled_bucket_name = b2_tool.generate_bucket_name() + schedule_bucket_cleanup(lock_enabled_bucket_name) b2_tool.should_succeed( [ 'create-bucket', @@ -1862,17 +1849,6 @@ def test_file_lock(b2_tool, application_key_id, application_key, b2_api, sample_ b2_tool, lock_enabled_bucket_name, lock_disabled_key_id, lock_disabled_key, sample_file ) - # ---- perform test cleanup ---- - b2_tool.should_succeed( - ['authorize-account', '--environment', b2_tool.realm, application_key_id, application_key], - ) - buckets = [ - bucket for bucket in b2_api.api.list_buckets() - if bucket.name in {lock_enabled_bucket_name, lock_disabled_bucket_name} - ] - for bucket in buckets: - b2_api.clean_bucket(bucket) - def make_lock_disabled_key(b2_tool): key_name = 'no-perms-for-file-lock' + random_hex(6) @@ -2119,7 +2095,7 @@ def test_profile_switch(b2_tool): os.environ[B2_ACCOUNT_INFO_ENV_VAR] = B2_ACCOUNT_INFO -def test_replication_basic(b2_api, b2_tool, bucket_name): +def test_replication_basic(b2_tool, bucket_name, schedule_bucket_cleanup): key_one_name = 'clt-testKey-01' + random_hex(6) created_key_stdout = b2_tool.should_succeed( [ @@ -2203,6 +2179,7 @@ def test_replication_basic(b2_api, b2_tool, bucket_name): # create a source bucket and set up replication to destination bucket source_bucket_name = b2_tool.generate_bucket_name() + schedule_bucket_cleanup(source_bucket_name) b2_tool.should_succeed( [ 'create-bucket', @@ -2268,11 +2245,11 @@ def test_replication_basic(b2_api, b2_tool, bucket_name): b2_tool.should_succeed(['delete-key', key_one_id]) b2_tool.should_succeed(['delete-key', key_two_id]) - b2_api.clean_bucket(source_bucket_name) -def test_replication_setup(b2_api, b2_tool, bucket_name): +def test_replication_setup(b2_tool, bucket_name, schedule_bucket_cleanup): source_bucket_name = b2_tool.generate_bucket_name() + schedule_bucket_cleanup(source_bucket_name) b2_tool.should_succeed( [ 'create-bucket', @@ -2324,13 +2301,12 @@ def test_replication_setup(b2_api, b2_tool, bucket_name): 'sourceToDestinationKeyMapping'].items(): b2_tool.should_succeed(['delete-key', key_one_id]) b2_tool.should_succeed(['delete-key', key_two_id]) - b2_api.clean_bucket(source_bucket_name) assert destination_bucket_old['replication']['asReplicationDestination'][ 'sourceToDestinationKeyMapping'] == destination_bucket['replication'][ 'asReplicationDestination']['sourceToDestinationKeyMapping'] -def test_replication_monitoring(b2_tool, bucket_name, b2_api, sample_file): +def test_replication_monitoring(b2_tool, bucket_name, sample_file, schedule_bucket_cleanup): # ---------------- set up keys ---------------- key_one_name = 'clt-testKey-01' + random_hex(6) @@ -2410,6 +2386,7 @@ def test_replication_monitoring(b2_tool, bucket_name, b2_api, sample_file): # create a source bucket and set up replication to destination bucket source_bucket_name = b2_tool.generate_bucket_name() + schedule_bucket_cleanup(source_bucket_name) b2_tool.should_succeed( [ 'create-bucket', @@ -2559,10 +2536,8 @@ def test_replication_monitoring(b2_tool, bucket_name, b2_api, sample_file): } for first, second in itertools.product(['FAILED', 'PENDING'], ['FAILED', 'PENDING']) ] - b2_api.clean_bucket(source_bucket_name) - -def test_enable_file_lock_first_retention_second(b2_tool, b2_api, bucket_name): +def test_enable_file_lock_first_retention_second(b2_tool, bucket_name): # enable file lock only b2_tool.should_succeed(['update-bucket', bucket_name, '--fileLockEnabled']) @@ -2577,10 +2552,8 @@ def test_enable_file_lock_first_retention_second(b2_tool, b2_api, bucket_name): # attempt to re-enable should be a noop b2_tool.should_succeed(['update-bucket', bucket_name, '--fileLockEnabled']) - b2_api.clean_bucket(bucket_name) - -def test_enable_file_lock_and_set_retention_at_once(b2_tool, b2_api, bucket_name): +def test_enable_file_lock_and_set_retention_at_once(b2_tool, bucket_name): # attempt setting retention without file lock enabled b2_tool.should_fail( [ @@ -2600,8 +2573,6 @@ def test_enable_file_lock_and_set_retention_at_once(b2_tool, b2_api, bucket_name # attempt to re-enable should be a noop b2_tool.should_succeed(['update-bucket', bucket_name, '--fileLockEnabled']) - b2_api.clean_bucket(bucket_name) - def _assert_file_lock_configuration( b2_tool,