From 155382564bdfb6f1267e153b4a421036f96552f7 Mon Sep 17 00:00:00 2001 From: kgwizdz Date: Fri, 28 Apr 2023 06:15:34 +0200 Subject: [PATCH 1/2] Add common logic for creating and cleaning buckets in tests --- CHANGELOG.md | 3 + b2sdk/_test_manager/__init__.py | 9 ++ b2sdk/_test_manager/api.py | 188 +++++++++++++++++++++++++ b2sdk/_test_manager/bucket_tracking.py | 32 +++++ noxfile.py | 4 +- requirements-dev.txt | 1 + setup.py | 5 +- test/integration/__init__.py | 7 +- test/integration/base.py | 64 +-------- test/integration/bucket_cleaner.py | 90 ------------ test/integration/cleanup_buckets.py | 6 +- test/integration/fixtures/__init__.py | 7 +- test/integration/helpers.py | 35 ----- test/integration/test_download.py | 8 +- test/integration/test_upload.py | 2 +- 15 files changed, 266 insertions(+), 195 deletions(-) create mode 100644 b2sdk/_test_manager/__init__.py create mode 100644 b2sdk/_test_manager/api.py create mode 100644 b2sdk/_test_manager/bucket_tracking.py create mode 100644 requirements-dev.txt delete mode 100644 test/integration/bucket_cleaner.py delete mode 100644 test/integration/helpers.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 26f0f9b66..f5e62cb99 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added +* Wrapper for B2Api class which can be used for test purposes + ## [1.21.0] - 2023-04-17 ### Added diff --git a/b2sdk/_test_manager/__init__.py b/b2sdk/_test_manager/__init__.py new file mode 100644 index 000000000..fd7483f18 --- /dev/null +++ b/b2sdk/_test_manager/__init__.py @@ -0,0 +1,9 @@ +###################################################################### +# +# File: b2sdk/_test_manager/__init__.py +# +# Copyright 2023 Backblaze Inc. All Rights Reserved. +# +# License https://www.backblaze.com/using_b2_code.html +# +###################################################################### diff --git a/b2sdk/_test_manager/api.py b/b2sdk/_test_manager/api.py new file mode 100644 index 000000000..158746a23 --- /dev/null +++ b/b2sdk/_test_manager/api.py @@ -0,0 +1,188 @@ +###################################################################### +# +# File: b2sdk/_test_manager/api.py +# +# Copyright 2023 Backblaze Inc. All Rights Reserved. +# +# License https://www.backblaze.com/using_b2_code.html +# +###################################################################### +import contextlib +import random +import string +import time + +from datetime import datetime +from os import environ +from typing import Union + +import backoff + +from .bucket_tracking import BucketTrackingMixin +from .._v3.exception import BucketIdNotFound as v3BucketIdNotFound +from ..v2 import NO_RETENTION_FILE_SETTING, B2Api, Bucket, InMemoryAccountInfo, InMemoryCache, LegalHold, RetentionMode +from ..v2.exception import BucketIdNotFound, DuplicateBucketName, FileNotPresent, TooManyRequests, NonExistentBucket + +ONE_HOUR_MILLIS = 60 * 60 * 1000 +ONE_DAY_MILLIS = ONE_HOUR_MILLIS * 24 + +BUCKET_NAME_LENGTH = 50 +BUCKET_NAME_CHARS = string.ascii_letters + string.digits + '-' + +BUCKET_NAME_PREFIX = 'b2tst' + +# RUNNER_NAME is the only variable exposed by the GitHub CI that was changing for each matrix entry. +# Example values are "GitHub Actions N" (with N being a whole number, starting from 2) and "Hosted Agent". +# Here, we're using these names as long as time as seeds to start the random number generator. +# Name fraction is used for runners inside the same matrix, time fraction is used for runners in different runs. +# To avoid collision when the same runners are fired in different commits at the same time we also use GITHUB_SHA +random.seed( + environ.get('RUNNER_NAME', 'local') + environ.get('GITHUB_SHA', 'local') + str(time.time_ns()) +) + + +def generate_bucket_name() -> str: + suffix_length = BUCKET_NAME_LENGTH - len(BUCKET_NAME_PREFIX) + suffix = ''.join(random.choice(BUCKET_NAME_CHARS) for _ in range(suffix_length)) + return f"{BUCKET_NAME_PREFIX}{suffix}" + + +def current_time_millis() -> int: + return int(round(time.time() * 1000)) + + +class Api(BucketTrackingMixin, B2Api): + """ + B2Api wrapper which should only be used for testing purposes! + """ + + def __init__(self, account_id: str, application_key: str, realm: str, *args, **kwargs): + info = InMemoryAccountInfo() + cache = InMemoryCache() + super().__init__(info, cache=cache, *args, **kwargs) + self.authorize_account(realm, account_id, application_key) + + @backoff.on_exception( + backoff.constant, + DuplicateBucketName, + max_tries=8, + ) + def create_test_bucket(self, bucket_type="allPublic", **kwargs) -> Bucket: + bucket_name = generate_bucket_name() + print('Creating bucket:', bucket_name) + try: + return self.create_bucket(bucket_name, bucket_type, **kwargs) + except DuplicateBucketName: + self._duplicated_bucket_name_debug_info(bucket_name) + raise + + @backoff.on_exception( + backoff.expo, + TooManyRequests, + max_tries=8, + ) + def clean_bucket(self, bucket: Union[Bucket, str]) -> None: + if isinstance(bucket, str): + bucket = self.get_bucket_by_name(bucket) + + files_leftover = False + file_versions = bucket.ls(latest_only=False, recursive=True) + + for file_version_info, _ in file_versions: + if file_version_info.file_retention: + if file_version_info.file_retention.mode == RetentionMode.GOVERNANCE: + print('Removing retention from file version:', file_version_info.id_) + self.update_file_retention( + file_version_info.id_, file_version_info.file_name, + NO_RETENTION_FILE_SETTING, True + ) + elif file_version_info.file_retention.mode == RetentionMode.COMPLIANCE: + if file_version_info.file_retention.retain_until > current_time_millis(): # yapf: disable + print( + 'File version: %s cannot be removed due to compliance mode retention' % + (file_version_info.id_,) + ) + files_leftover = True + continue + elif file_version_info.file_retention.mode == RetentionMode.NONE: + pass + else: + raise ValueError( + 'Unknown retention mode: %s' % (file_version_info.file_retention.mode,) + ) + if file_version_info.legal_hold.is_on(): + print('Removing legal hold from file version:', file_version_info.id_) + self.update_file_legal_hold( + file_version_info.id_, file_version_info.file_name, LegalHold.OFF + ) + print('Removing file version:', file_version_info.id_) + try: + self.delete_file_version(file_version_info.id_, file_version_info.file_name) + except FileNotPresent: + print( + 'It seems that file version %s has already been removed' % + (file_version_info.id_,) + ) + + if files_leftover: + print('Unable to remove bucket because some retained files remain') + else: + print('Removing bucket:', bucket.name) + try: + self.delete_bucket(bucket) + except BucketIdNotFound: + print('It seems that bucket %s has already been removed' % (bucket.name,)) + print() + + def clean_buckets(self) -> None: + for bucket in self.buckets: + with contextlib.suppress(BucketIdNotFound, v3BucketIdNotFound, NonExistentBucket): + self.clean_bucket(bucket) + self.buckets = [] + + def clean_all_buckets(self) -> None: + buckets = self.list_buckets() + print(f'Total bucket count: {len(buckets)}') + + for bucket in buckets: + if not bucket.name.startswith(BUCKET_NAME_PREFIX): + print(f'Skipping bucket removal: "{bucket.name}"') + continue + + print(f'Removing bucket: "{bucket.name}"') + try: + self.clean_bucket(bucket) + except (BucketIdNotFound, v3BucketIdNotFound): + print(f'It seems that bucket "{bucket.name}" has already been removed') + + buckets = self.list_buckets() + print(f'Total bucket count after cleanup: {len(buckets)}') + for bucket in buckets: + print(bucket) + + def count_and_print_buckets(self) -> int: + buckets = self.list_buckets() + count = len(buckets) + print(f'Total bucket count at {datetime.now()}: {count}') + for i, bucket in enumerate(buckets, start=1): + print(f'- {i}\t{bucket.name} [{bucket.id_}]') + return count + + def _duplicated_bucket_name_debug_info(self, bucket_name: str) -> None: + # Trying to obtain as much information as possible about this bucket. + print(' DUPLICATED BUCKET DEBUG START '.center(60, '=')) + bucket = self.get_bucket_by_name(bucket_name) + + print('Bucket metadata:') + bucket_dict = bucket.as_dict() + for info_key, info in bucket_dict.items(): + print('\t%s: "%s"' % (info_key, info)) + + print('All files (and their versions) inside the bucket:') + ls_generator = bucket.ls(recursive=True, latest_only=False) + for file_version, _directory in ls_generator: + # as_dict() is bound to have more info than we can use, + # but maybe some of it will cast some light on the issue. + print('\t%s (%s)' % (file_version.file_name, file_version.as_dict())) + + print(' DUPLICATED BUCKET DEBUG END '.center(60, '=')) diff --git a/b2sdk/_test_manager/bucket_tracking.py b/b2sdk/_test_manager/bucket_tracking.py new file mode 100644 index 000000000..2687762d3 --- /dev/null +++ b/b2sdk/_test_manager/bucket_tracking.py @@ -0,0 +1,32 @@ +###################################################################### +# +# File: b2sdk/_test_manager/bucket_tracking.py +# +# Copyright 2023 Backblaze Inc. All Rights Reserved. +# +# License https://www.backblaze.com/using_b2_code.html +# +###################################################################### +from b2sdk.v2 import Bucket + + +class BucketTrackingMixin: + """ + Mixin class for B2Api, which enables bucket tracking. + This mixin will add a `buckets` member to the B2Api instance and will use it track created and + deleted buckets. The main purpose of this are tests -- the `buckets` member can be used in test + teardown to ensure proper bucket cleanup. + """ + + def __init__(self, *args, **kwargs): + self.buckets = [] + super().__init__(*args, **kwargs) + + def create_bucket(self, name: str, *args, **kwargs) -> Bucket: + bucket = super().create_bucket(name, *args, **kwargs) + self.buckets.append(bucket) + return bucket + + def delete_bucket(self, bucket: Bucket): + super().delete_bucket(bucket) + self.buckets = [b for b in self.buckets if b.id_ != bucket.id_] diff --git a/noxfile.py b/noxfile.py index 7fd5d27e5..4adef21c9 100644 --- a/noxfile.py +++ b/noxfile.py @@ -134,7 +134,7 @@ def unit(session): @nox.session(python=PYTHON_VERSIONS) def integration(session): """Run integration tests.""" - install_myself(session) + install_myself(session, ['dev']) session.run('pip', 'install', *REQUIREMENTS_TEST) session.run('pytest', '-s', *session.posargs, 'test/integration') @@ -142,7 +142,7 @@ def integration(session): @nox.session(python=PYTHON_DEFAULT_VERSION) def cleanup_old_buckets(session): """Remove buckets from previous test runs.""" - install_myself(session) + install_myself(session, ['dev']) session.run('pip', 'install', *REQUIREMENTS_TEST) session.run('python', '-m', 'test.integration.cleanup_buckets') diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 000000000..8c06c864a --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1 @@ +backoff>=1.4.0 \ No newline at end of file diff --git a/setup.py b/setup.py index 486522f28..9f70bc8b2 100644 --- a/setup.py +++ b/setup.py @@ -108,7 +108,10 @@ def read_requirements(extra=None): # dependencies). You can install these using the following syntax, # for example: # $ pip install -e .[dev,test] - extras_require={'doc': read_requirements('doc')}, + extras_require={ + 'doc': read_requirements('doc'), + 'dev': read_requirements('dev') + }, setup_requires=['setuptools_scm<6.0'], use_scm_version=True, diff --git a/test/integration/__init__.py b/test/integration/__init__.py index 1bc44711a..0475eb7ec 100644 --- a/test/integration/__init__.py +++ b/test/integration/__init__.py @@ -8,9 +8,10 @@ # ###################################################################### import os +from typing import Tuple -def get_b2_auth_data(): +def get_b2_auth_data() -> Tuple[str, str]: application_key_id = os.environ.get('B2_TEST_APPLICATION_KEY_ID') if application_key_id is None: raise ValueError('B2_TEST_APPLICATION_KEY_ID is not set.') @@ -19,3 +20,7 @@ def get_b2_auth_data(): if application_key is None: raise ValueError('B2_TEST_APPLICATION_KEY is not set.') return application_key_id, application_key + + +def get_realm() -> str: + return os.environ.get('B2_TEST_ENVIRONMENT', 'production') diff --git a/test/integration/base.py b/test/integration/base.py index ff6245665..ebc22ef23 100644 --- a/test/integration/base.py +++ b/test/integration/base.py @@ -13,10 +13,7 @@ import pytest -from b2sdk.v2 import current_time_millis -from b2sdk.v2.exception import DuplicateBucketName -from .bucket_cleaner import BucketCleaner -from .helpers import GENERAL_BUCKET_NAME_PREFIX, BUCKET_NAME_LENGTH, BUCKET_CREATED_AT_MILLIS, bucket_name_part, authorize +from b2sdk._test_manager.api import Api class IntegrationTestBase: @@ -26,30 +23,11 @@ def set_http_debug(self): http.client.HTTPConnection.debuglevel = 1 @pytest.fixture(autouse=True) - def save_settings(self, dont_cleanup_old_buckets, b2_auth_data): - type(self).dont_cleanup_old_buckets = dont_cleanup_old_buckets - type(self).b2_auth_data = b2_auth_data - - @classmethod - def setup_class(cls): - cls.this_run_bucket_name_prefix = GENERAL_BUCKET_NAME_PREFIX + bucket_name_part(8) - - @classmethod - def teardown_class(cls): - BucketCleaner( - cls.dont_cleanup_old_buckets, - *cls.b2_auth_data, - current_run_prefix=cls.this_run_bucket_name_prefix - ).cleanup_buckets() - - @pytest.fixture(autouse=True) - def setup_method(self): - self.b2_api, self.info = authorize(self.b2_auth_data) - - def generate_bucket_name(self): - return self.this_run_bucket_name_prefix + bucket_name_part( - BUCKET_NAME_LENGTH - len(self.this_run_bucket_name_prefix) - ) + def setup_method(self, b2_auth_data, realm): + self.b2_api = Api(*b2_auth_data, realm) + self.info = self.b2_api.account_info + yield + self.b2_api.clean_buckets() def write_zeros(self, file, number): line = b'0' * 1000 + b'\n' @@ -60,32 +38,4 @@ def write_zeros(self, file, number): written += line_len def create_bucket(self): - bucket_name = self.generate_bucket_name() - try: - return self.b2_api.create_bucket( - bucket_name, - 'allPublic', - bucket_info={BUCKET_CREATED_AT_MILLIS: str(current_time_millis())} - ) - except DuplicateBucketName: - self._duplicated_bucket_name_debug_info(bucket_name) - raise - - def _duplicated_bucket_name_debug_info(self, bucket_name: str) -> None: - # Trying to obtain as much information as possible about this bucket. - print(' DUPLICATED BUCKET DEBUG START '.center(60, '=')) - bucket = self.b2_api.get_bucket_by_name(bucket_name) - - print('Bucket metadata:') - bucket_dict = bucket.as_dict() - for info_key, info in bucket_dict.items(): - print('\t%s: "%s"' % (info_key, info)) - - print('All files (and their versions) inside the bucket:') - ls_generator = bucket.ls(recursive=True, latest_only=False) - for file_version, _directory in ls_generator: - # as_dict() is bound to have more info than we can use, - # but maybe some of it will cast some light on the issue. - print('\t%s (%s)' % (file_version.file_name, file_version.as_dict())) - - print(' DUPLICATED BUCKET DEBUG END '.center(60, '=')) + return self.b2_api.create_test_bucket() diff --git a/test/integration/bucket_cleaner.py b/test/integration/bucket_cleaner.py deleted file mode 100644 index e1b1e5783..000000000 --- a/test/integration/bucket_cleaner.py +++ /dev/null @@ -1,90 +0,0 @@ -###################################################################### -# -# File: test/integration/bucket_cleaner.py -# -# Copyright 2022 Backblaze Inc. All Rights Reserved. -# -# License https://www.backblaze.com/using_b2_code.html -# -###################################################################### - -from typing import Optional - -from b2sdk.v2 import * - -from .helpers import GENERAL_BUCKET_NAME_PREFIX, BUCKET_CREATED_AT_MILLIS, authorize - -ONE_HOUR_MILLIS = 60 * 60 * 1000 - - -class BucketCleaner: - def __init__( - self, - dont_cleanup_old_buckets: bool, - b2_application_key_id: str, - b2_application_key: str, - current_run_prefix: Optional[str] = None - ): - self.current_run_prefix = current_run_prefix - self.dont_cleanup_old_buckets = dont_cleanup_old_buckets - self.b2_application_key_id = b2_application_key_id - self.b2_application_key = b2_application_key - - def _should_remove_bucket(self, bucket: Bucket): - if self.current_run_prefix and bucket.name.startswith(self.current_run_prefix): - return True - if self.dont_cleanup_old_buckets: - return False - if bucket.name.startswith(GENERAL_BUCKET_NAME_PREFIX): - if BUCKET_CREATED_AT_MILLIS in bucket.bucket_info: - if int(bucket.bucket_info[BUCKET_CREATED_AT_MILLIS] - ) < current_time_millis() - ONE_HOUR_MILLIS: - return True - return False - - def cleanup_buckets(self): - b2_api, _ = authorize((self.b2_application_key_id, self.b2_application_key)) - buckets = b2_api.list_buckets() - for bucket in buckets: - if not self._should_remove_bucket(bucket): - print('Skipping bucket removal:', bucket.name) - else: - print('Trying to remove bucket:', bucket.name) - files_leftover = False - file_versions = bucket.ls(latest_only=False, recursive=True) - for file_version_info, _ in file_versions: - if file_version_info.file_retention: - if file_version_info.file_retention.mode == RetentionMode.GOVERNANCE: - print('Removing retention from file version:', file_version_info.id_) - b2_api.update_file_retention( - file_version_info.id_, file_version_info.file_name, - NO_RETENTION_FILE_SETTING, True - ) - elif file_version_info.file_retention.mode == RetentionMode.COMPLIANCE: - if file_version_info.file_retention.retain_until > current_time_millis(): # yapf: disable - print( - 'File version: %s cannot be removed due to compliance mode retention' - % (file_version_info.id_,) - ) - files_leftover = True - continue - elif file_version_info.file_retention.mode == RetentionMode.NONE: - pass - else: - raise ValueError( - 'Unknown retention mode: %s' % - (file_version_info.file_retention.mode,) - ) - if file_version_info.legal_hold.is_on(): - print('Removing legal hold from file version:', file_version_info.id_) - b2_api.update_file_legal_hold( - file_version_info.id_, file_version_info.file_name, LegalHold.OFF - ) - print('Removing file version:', file_version_info.id_) - b2_api.delete_file_version(file_version_info.id_, file_version_info.file_name) - - if files_leftover: - print('Unable to remove bucket because some retained files remain') - else: - print('Removing bucket:', bucket.name) - b2_api.delete_bucket(bucket) diff --git a/test/integration/cleanup_buckets.py b/test/integration/cleanup_buckets.py index 02bb16ce0..72501c95c 100755 --- a/test/integration/cleanup_buckets.py +++ b/test/integration/cleanup_buckets.py @@ -8,10 +8,10 @@ # ###################################################################### -from . import get_b2_auth_data -from .bucket_cleaner import BucketCleaner +from . import get_b2_auth_data, get_realm +from b2sdk._test_manager.api import Api from .test_raw_api import cleanup_old_buckets if __name__ == '__main__': cleanup_old_buckets() - BucketCleaner(False, *get_b2_auth_data()).cleanup_buckets() + Api(*get_b2_auth_data(), get_realm()).clean_all_buckets() diff --git a/test/integration/fixtures/__init__.py b/test/integration/fixtures/__init__.py index 2d12d6e8c..37cf29e38 100644 --- a/test/integration/fixtures/__init__.py +++ b/test/integration/fixtures/__init__.py @@ -11,7 +11,7 @@ import os import pytest -from .. import get_b2_auth_data +from .. import get_b2_auth_data, get_realm @pytest.fixture @@ -20,3 +20,8 @@ def b2_auth_data(): return get_b2_auth_data() except ValueError as ex: pytest.fail(ex.args[0]) + + +@pytest.fixture +def realm(): + return get_realm() diff --git a/test/integration/helpers.py b/test/integration/helpers.py deleted file mode 100644 index 60ea3539e..000000000 --- a/test/integration/helpers.py +++ /dev/null @@ -1,35 +0,0 @@ -###################################################################### -# -# File: test/integration/helpers.py -# -# Copyright 2022 Backblaze Inc. All Rights Reserved. -# -# License https://www.backblaze.com/using_b2_code.html -# -###################################################################### - -from typing import Optional -import os -import random -import string - -import pytest - -from b2sdk.v2 import * - -GENERAL_BUCKET_NAME_PREFIX = 'sdktst' -BUCKET_NAME_CHARS = string.ascii_letters + string.digits + '-' -BUCKET_NAME_LENGTH = 50 -BUCKET_CREATED_AT_MILLIS = 'created_at_millis' - - -def bucket_name_part(length): - return ''.join(random.choice(BUCKET_NAME_CHARS) for _ in range(length)) - - -def authorize(b2_auth_data, api_config=DEFAULT_HTTP_API_CONFIG): - info = InMemoryAccountInfo() - b2_api = B2Api(info, api_config=api_config) - realm = os.environ.get('B2_TEST_ENVIRONMENT', 'production') - b2_api.authorize_account(realm, *b2_auth_data) - return b2_api, info diff --git a/test/integration/test_download.py b/test/integration/test_download.py index a093d24be..acf3c918e 100644 --- a/test/integration/test_download.py +++ b/test/integration/test_download.py @@ -15,11 +15,11 @@ from typing import Optional, Tuple from unittest import mock +from b2sdk._test_manager.api import Api from b2sdk.v2 import * from b2sdk.utils import Sha1HexDigest from .fixtures import * # pyflakes: disable -from .helpers import authorize from .base import IntegrationTestBase @@ -93,7 +93,7 @@ def test_small_unverified(self): pprint(f.download_version._get_args_for_clone()) assert not f.download_version.content_sha1_verified - def test_gzip(self): + def test_gzip(self, b2_auth_data, realm): bucket = self.create_bucket() with TempDir() as temp_dir: temp_dir = pathlib.Path(temp_dir) @@ -116,8 +116,8 @@ def test_gzip(self): source_data = sf.read() assert downloaded_data == source_data - decompressing_api, _ = authorize( - self.b2_auth_data, B2HttpApiConfig(decode_content=True) + decompressing_api = Api( + *b2_auth_data, realm, api_config=B2HttpApiConfig(decode_content=True) ) decompressing_api.download_file_by_id(file_id=file_version.id_).save_to( str(downloaded_uncompressed_file) diff --git a/test/integration/test_upload.py b/test/integration/test_upload.py index 34b88b8f5..7a41b3322 100644 --- a/test/integration/test_upload.py +++ b/test/integration/test_upload.py @@ -11,7 +11,7 @@ import io from typing import Optional -from .fixtures import b2_auth_data # noqa +from .fixtures import * # noqa from .base import IntegrationTestBase From 872ac146a72deb8196b2c8278da61c0d9019e4d4 Mon Sep 17 00:00:00 2001 From: kgwizdz Date: Tue, 9 May 2023 02:24:09 +0200 Subject: [PATCH 2/2] Change directory of test utils & add minor fixes --- CHANGELOG.md | 2 +- b2sdk/{_test_manager => test}/__init__.py | 2 +- .../api.py => test/api_test_manager.py} | 82 +++++++------------ .../bucket_tracking.py | 2 +- requirements-dev.txt | 2 +- setup.py | 2 +- test/integration/base.py | 4 +- test/integration/cleanup_buckets.py | 4 +- test/integration/test_download.py | 4 +- test/integration/test_upload.py | 2 +- 10 files changed, 42 insertions(+), 64 deletions(-) rename b2sdk/{_test_manager => test}/__init__.py (86%) rename b2sdk/{_test_manager/api.py => test/api_test_manager.py} (62%) rename b2sdk/{_test_manager => test}/bucket_tracking.py (95%) diff --git a/CHANGELOG.md b/CHANGELOG.md index f5e62cb99..3fa304767 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] -### Added +### Infrastructure * Wrapper for B2Api class which can be used for test purposes ## [1.21.0] - 2023-04-17 diff --git a/b2sdk/_test_manager/__init__.py b/b2sdk/test/__init__.py similarity index 86% rename from b2sdk/_test_manager/__init__.py rename to b2sdk/test/__init__.py index fd7483f18..f4a62b184 100644 --- a/b2sdk/_test_manager/__init__.py +++ b/b2sdk/test/__init__.py @@ -1,6 +1,6 @@ ###################################################################### # -# File: b2sdk/_test_manager/__init__.py +# File: b2sdk/test/__init__.py # # Copyright 2023 Backblaze Inc. All Rights Reserved. # diff --git a/b2sdk/_test_manager/api.py b/b2sdk/test/api_test_manager.py similarity index 62% rename from b2sdk/_test_manager/api.py rename to b2sdk/test/api_test_manager.py index 158746a23..82462f456 100644 --- a/b2sdk/_test_manager/api.py +++ b/b2sdk/test/api_test_manager.py @@ -1,16 +1,14 @@ ###################################################################### # -# File: b2sdk/_test_manager/api.py +# File: b2sdk/test/api_test_manager.py # # Copyright 2023 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### -import contextlib -import random -import string import time +import uuid from datetime import datetime from os import environ @@ -19,39 +17,25 @@ import backoff from .bucket_tracking import BucketTrackingMixin -from .._v3.exception import BucketIdNotFound as v3BucketIdNotFound -from ..v2 import NO_RETENTION_FILE_SETTING, B2Api, Bucket, InMemoryAccountInfo, InMemoryCache, LegalHold, RetentionMode -from ..v2.exception import BucketIdNotFound, DuplicateBucketName, FileNotPresent, TooManyRequests, NonExistentBucket - -ONE_HOUR_MILLIS = 60 * 60 * 1000 -ONE_DAY_MILLIS = ONE_HOUR_MILLIS * 24 - -BUCKET_NAME_LENGTH = 50 -BUCKET_NAME_CHARS = string.ascii_letters + string.digits + '-' - -BUCKET_NAME_PREFIX = 'b2tst' - -# RUNNER_NAME is the only variable exposed by the GitHub CI that was changing for each matrix entry. -# Example values are "GitHub Actions N" (with N being a whole number, starting from 2) and "Hosted Agent". -# Here, we're using these names as long as time as seeds to start the random number generator. -# Name fraction is used for runners inside the same matrix, time fraction is used for runners in different runs. -# To avoid collision when the same runners are fired in different commits at the same time we also use GITHUB_SHA -random.seed( - environ.get('RUNNER_NAME', 'local') + environ.get('GITHUB_SHA', 'local') + str(time.time_ns()) +from b2sdk.v2 import ( + NO_RETENTION_FILE_SETTING, B2Api, Bucket, InMemoryAccountInfo, InMemoryCache, LegalHold, + RetentionMode ) +from b2sdk.v2.exception import BucketIdNotFound, DuplicateBucketName, FileNotPresent, TooManyRequests, NonExistentBucket + +SHORT_SHA = environ.get('GITHUB_SHA', 'local')[:10] +BUCKET_NAME_PREFIX = f"b2test-{SHORT_SHA}" def generate_bucket_name() -> str: - suffix_length = BUCKET_NAME_LENGTH - len(BUCKET_NAME_PREFIX) - suffix = ''.join(random.choice(BUCKET_NAME_CHARS) for _ in range(suffix_length)) - return f"{BUCKET_NAME_PREFIX}{suffix}" + return f"{BUCKET_NAME_PREFIX}-{uuid.uuid4()}" def current_time_millis() -> int: return int(round(time.time() * 1000)) -class Api(BucketTrackingMixin, B2Api): +class ApiTestManager(BucketTrackingMixin, B2Api): """ B2Api wrapper which should only be used for testing purposes! """ @@ -69,7 +53,7 @@ def __init__(self, account_id: str, application_key: str, realm: str, *args, **k ) def create_test_bucket(self, bucket_type="allPublic", **kwargs) -> Bucket: bucket_name = generate_bucket_name() - print('Creating bucket:', bucket_name) + print(f'Creating bucket: {bucket_name}') try: return self.create_bucket(bucket_name, bucket_type, **kwargs) except DuplicateBucketName: @@ -91,16 +75,17 @@ def clean_bucket(self, bucket: Union[Bucket, str]) -> None: for file_version_info, _ in file_versions: if file_version_info.file_retention: if file_version_info.file_retention.mode == RetentionMode.GOVERNANCE: - print('Removing retention from file version:', file_version_info.id_) + print(f'Removing retention from file version: {file_version_info.id_}') self.update_file_retention( - file_version_info.id_, file_version_info.file_name, - NO_RETENTION_FILE_SETTING, True + file_version_info.id_, + file_version_info.file_name, + NO_RETENTION_FILE_SETTING, + bypass_governance=True ) elif file_version_info.file_retention.mode == RetentionMode.COMPLIANCE: if file_version_info.file_retention.retain_until > current_time_millis(): # yapf: disable print( - 'File version: %s cannot be removed due to compliance mode retention' % - (file_version_info.id_,) + f'File version: {file_version_info.id_} cannot be removed due to compliance mode retention' ) files_leftover = True continue @@ -108,36 +93,35 @@ def clean_bucket(self, bucket: Union[Bucket, str]) -> None: pass else: raise ValueError( - 'Unknown retention mode: %s' % (file_version_info.file_retention.mode,) + f'Unknown retention mode: {file_version_info.file_retention.mode}' ) if file_version_info.legal_hold.is_on(): - print('Removing legal hold from file version:', file_version_info.id_) + print(f'Removing legal hold from file version: {file_version_info.id_}') self.update_file_legal_hold( file_version_info.id_, file_version_info.file_name, LegalHold.OFF ) - print('Removing file version:', file_version_info.id_) + print(f'Removing file version: {file_version_info.id_}') try: self.delete_file_version(file_version_info.id_, file_version_info.file_name) except FileNotPresent: print( - 'It seems that file version %s has already been removed' % - (file_version_info.id_,) + f'It seems that file version {file_version_info.id_} has already been removed' ) if files_leftover: print('Unable to remove bucket because some retained files remain') else: - print('Removing bucket:', bucket.name) + print(f'Removing bucket: {bucket.name}') try: self.delete_bucket(bucket) - except BucketIdNotFound: - print('It seems that bucket %s has already been removed' % (bucket.name,)) + except (BucketIdNotFound, NonExistentBucket): + print(f'It seems that bucket {bucket.name} has already been removed') print() def clean_buckets(self) -> None: + self.count_and_print_buckets() for bucket in self.buckets: - with contextlib.suppress(BucketIdNotFound, v3BucketIdNotFound, NonExistentBucket): - self.clean_bucket(bucket) + self.clean_bucket(bucket) self.buckets = [] def clean_all_buckets(self) -> None: @@ -148,25 +132,19 @@ def clean_all_buckets(self) -> None: if not bucket.name.startswith(BUCKET_NAME_PREFIX): print(f'Skipping bucket removal: "{bucket.name}"') continue - - print(f'Removing bucket: "{bucket.name}"') - try: - self.clean_bucket(bucket) - except (BucketIdNotFound, v3BucketIdNotFound): - print(f'It seems that bucket "{bucket.name}" has already been removed') + self.clean_bucket(bucket) buckets = self.list_buckets() print(f'Total bucket count after cleanup: {len(buckets)}') for bucket in buckets: print(bucket) - def count_and_print_buckets(self) -> int: - buckets = self.list_buckets() + def count_and_print_buckets(self) -> None: + buckets = self.buckets count = len(buckets) print(f'Total bucket count at {datetime.now()}: {count}') for i, bucket in enumerate(buckets, start=1): print(f'- {i}\t{bucket.name} [{bucket.id_}]') - return count def _duplicated_bucket_name_debug_info(self, bucket_name: str) -> None: # Trying to obtain as much information as possible about this bucket. diff --git a/b2sdk/_test_manager/bucket_tracking.py b/b2sdk/test/bucket_tracking.py similarity index 95% rename from b2sdk/_test_manager/bucket_tracking.py rename to b2sdk/test/bucket_tracking.py index 2687762d3..ae1c66d14 100644 --- a/b2sdk/_test_manager/bucket_tracking.py +++ b/b2sdk/test/bucket_tracking.py @@ -1,6 +1,6 @@ ###################################################################### # -# File: b2sdk/_test_manager/bucket_tracking.py +# File: b2sdk/test/bucket_tracking.py # # Copyright 2023 Backblaze Inc. All Rights Reserved. # diff --git a/requirements-dev.txt b/requirements-dev.txt index 8c06c864a..28a7e406e 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1 +1 @@ -backoff>=1.4.0 \ No newline at end of file +backoff>=1.4.0,<3.0.0 \ No newline at end of file diff --git a/setup.py b/setup.py index 9f70bc8b2..e2c58a423 100644 --- a/setup.py +++ b/setup.py @@ -110,7 +110,7 @@ def read_requirements(extra=None): # $ pip install -e .[dev,test] extras_require={ 'doc': read_requirements('doc'), - 'dev': read_requirements('dev') + 'dev': read_requirements('dev'), }, setup_requires=['setuptools_scm<6.0'], use_scm_version=True, diff --git a/test/integration/base.py b/test/integration/base.py index ebc22ef23..b3d1c5b95 100644 --- a/test/integration/base.py +++ b/test/integration/base.py @@ -13,7 +13,7 @@ import pytest -from b2sdk._test_manager.api import Api +from b2sdk.test.api_test_manager import ApiTestManager class IntegrationTestBase: @@ -24,7 +24,7 @@ def set_http_debug(self): @pytest.fixture(autouse=True) def setup_method(self, b2_auth_data, realm): - self.b2_api = Api(*b2_auth_data, realm) + self.b2_api = ApiTestManager(*b2_auth_data, realm) self.info = self.b2_api.account_info yield self.b2_api.clean_buckets() diff --git a/test/integration/cleanup_buckets.py b/test/integration/cleanup_buckets.py index 72501c95c..bb16cadeb 100755 --- a/test/integration/cleanup_buckets.py +++ b/test/integration/cleanup_buckets.py @@ -9,9 +9,9 @@ ###################################################################### from . import get_b2_auth_data, get_realm -from b2sdk._test_manager.api import Api +from b2sdk.test.api_test_manager import ApiTestManager from .test_raw_api import cleanup_old_buckets if __name__ == '__main__': cleanup_old_buckets() - Api(*get_b2_auth_data(), get_realm()).clean_all_buckets() + ApiTestManager(*get_b2_auth_data(), get_realm()).clean_all_buckets() diff --git a/test/integration/test_download.py b/test/integration/test_download.py index acf3c918e..473aafef2 100644 --- a/test/integration/test_download.py +++ b/test/integration/test_download.py @@ -15,7 +15,7 @@ from typing import Optional, Tuple from unittest import mock -from b2sdk._test_manager.api import Api +from b2sdk.test.api_test_manager import ApiTestManager from b2sdk.v2 import * from b2sdk.utils import Sha1HexDigest @@ -116,7 +116,7 @@ def test_gzip(self, b2_auth_data, realm): source_data = sf.read() assert downloaded_data == source_data - decompressing_api = Api( + decompressing_api = ApiTestManager( *b2_auth_data, realm, api_config=B2HttpApiConfig(decode_content=True) ) decompressing_api.download_file_by_id(file_id=file_version.id_).save_to( diff --git a/test/integration/test_upload.py b/test/integration/test_upload.py index 7a41b3322..5b82c4ee7 100644 --- a/test/integration/test_upload.py +++ b/test/integration/test_upload.py @@ -11,7 +11,7 @@ import io from typing import Optional -from .fixtures import * # noqa +from .fixtures import b2_auth_data, realm from .base import IntegrationTestBase