diff --git a/CHANGELOG.md b/CHANGELOG.md index 26f0f9b66..3fa304767 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Infrastructure +* Wrapper for B2Api class which can be used for test purposes + ## [1.21.0] - 2023-04-17 ### Added diff --git a/b2sdk/test/__init__.py b/b2sdk/test/__init__.py new file mode 100644 index 000000000..f4a62b184 --- /dev/null +++ b/b2sdk/test/__init__.py @@ -0,0 +1,9 @@ +###################################################################### +# +# File: b2sdk/test/__init__.py +# +# Copyright 2023 Backblaze Inc. All Rights Reserved. +# +# License https://www.backblaze.com/using_b2_code.html +# +###################################################################### diff --git a/b2sdk/test/api_test_manager.py b/b2sdk/test/api_test_manager.py new file mode 100644 index 000000000..82462f456 --- /dev/null +++ b/b2sdk/test/api_test_manager.py @@ -0,0 +1,166 @@ +###################################################################### +# +# File: b2sdk/test/api_test_manager.py +# +# Copyright 2023 Backblaze Inc. All Rights Reserved. +# +# License https://www.backblaze.com/using_b2_code.html +# +###################################################################### +import time +import uuid + +from datetime import datetime +from os import environ +from typing import Union + +import backoff + +from .bucket_tracking import BucketTrackingMixin +from b2sdk.v2 import ( + NO_RETENTION_FILE_SETTING, B2Api, Bucket, InMemoryAccountInfo, InMemoryCache, LegalHold, + RetentionMode +) +from b2sdk.v2.exception import BucketIdNotFound, DuplicateBucketName, FileNotPresent, TooManyRequests, NonExistentBucket + +SHORT_SHA = environ.get('GITHUB_SHA', 'local')[:10] +BUCKET_NAME_PREFIX = f"b2test-{SHORT_SHA}" + + +def generate_bucket_name() -> str: + return f"{BUCKET_NAME_PREFIX}-{uuid.uuid4()}" + + +def current_time_millis() -> int: + return int(round(time.time() * 1000)) + + +class ApiTestManager(BucketTrackingMixin, B2Api): + """ + B2Api wrapper which should only be used for testing purposes! + """ + + def __init__(self, account_id: str, application_key: str, realm: str, *args, **kwargs): + info = InMemoryAccountInfo() + cache = InMemoryCache() + super().__init__(info, cache=cache, *args, **kwargs) + self.authorize_account(realm, account_id, application_key) + + @backoff.on_exception( + backoff.constant, + DuplicateBucketName, + max_tries=8, + ) + def create_test_bucket(self, bucket_type="allPublic", **kwargs) -> Bucket: + bucket_name = generate_bucket_name() + print(f'Creating bucket: {bucket_name}') + try: + return self.create_bucket(bucket_name, bucket_type, **kwargs) + except DuplicateBucketName: + self._duplicated_bucket_name_debug_info(bucket_name) + raise + + @backoff.on_exception( + backoff.expo, + TooManyRequests, + max_tries=8, + ) + def clean_bucket(self, bucket: Union[Bucket, str]) -> None: + if isinstance(bucket, str): + bucket = self.get_bucket_by_name(bucket) + + files_leftover = False + file_versions = bucket.ls(latest_only=False, recursive=True) + + for file_version_info, _ in file_versions: + if file_version_info.file_retention: + if file_version_info.file_retention.mode == RetentionMode.GOVERNANCE: + print(f'Removing retention from file version: {file_version_info.id_}') + self.update_file_retention( + file_version_info.id_, + file_version_info.file_name, + NO_RETENTION_FILE_SETTING, + bypass_governance=True + ) + elif file_version_info.file_retention.mode == RetentionMode.COMPLIANCE: + if file_version_info.file_retention.retain_until > current_time_millis(): # yapf: disable + print( + f'File version: {file_version_info.id_} cannot be removed due to compliance mode retention' + ) + files_leftover = True + continue + elif file_version_info.file_retention.mode == RetentionMode.NONE: + pass + else: + raise ValueError( + f'Unknown retention mode: {file_version_info.file_retention.mode}' + ) + if file_version_info.legal_hold.is_on(): + print(f'Removing legal hold from file version: {file_version_info.id_}') + self.update_file_legal_hold( + file_version_info.id_, file_version_info.file_name, LegalHold.OFF + ) + print(f'Removing file version: {file_version_info.id_}') + try: + self.delete_file_version(file_version_info.id_, file_version_info.file_name) + except FileNotPresent: + print( + f'It seems that file version {file_version_info.id_} has already been removed' + ) + + if files_leftover: + print('Unable to remove bucket because some retained files remain') + else: + print(f'Removing bucket: {bucket.name}') + try: + self.delete_bucket(bucket) + except (BucketIdNotFound, NonExistentBucket): + print(f'It seems that bucket {bucket.name} has already been removed') + print() + + def clean_buckets(self) -> None: + self.count_and_print_buckets() + for bucket in self.buckets: + self.clean_bucket(bucket) + self.buckets = [] + + def clean_all_buckets(self) -> None: + buckets = self.list_buckets() + print(f'Total bucket count: {len(buckets)}') + + for bucket in buckets: + if not bucket.name.startswith(BUCKET_NAME_PREFIX): + print(f'Skipping bucket removal: "{bucket.name}"') + continue + self.clean_bucket(bucket) + + buckets = self.list_buckets() + print(f'Total bucket count after cleanup: {len(buckets)}') + for bucket in buckets: + print(bucket) + + def count_and_print_buckets(self) -> None: + buckets = self.buckets + count = len(buckets) + print(f'Total bucket count at {datetime.now()}: {count}') + for i, bucket in enumerate(buckets, start=1): + print(f'- {i}\t{bucket.name} [{bucket.id_}]') + + def _duplicated_bucket_name_debug_info(self, bucket_name: str) -> None: + # Trying to obtain as much information as possible about this bucket. + print(' DUPLICATED BUCKET DEBUG START '.center(60, '=')) + bucket = self.get_bucket_by_name(bucket_name) + + print('Bucket metadata:') + bucket_dict = bucket.as_dict() + for info_key, info in bucket_dict.items(): + print('\t%s: "%s"' % (info_key, info)) + + print('All files (and their versions) inside the bucket:') + ls_generator = bucket.ls(recursive=True, latest_only=False) + for file_version, _directory in ls_generator: + # as_dict() is bound to have more info than we can use, + # but maybe some of it will cast some light on the issue. + print('\t%s (%s)' % (file_version.file_name, file_version.as_dict())) + + print(' DUPLICATED BUCKET DEBUG END '.center(60, '=')) diff --git a/b2sdk/test/bucket_tracking.py b/b2sdk/test/bucket_tracking.py new file mode 100644 index 000000000..ae1c66d14 --- /dev/null +++ b/b2sdk/test/bucket_tracking.py @@ -0,0 +1,32 @@ +###################################################################### +# +# File: b2sdk/test/bucket_tracking.py +# +# Copyright 2023 Backblaze Inc. All Rights Reserved. +# +# License https://www.backblaze.com/using_b2_code.html +# +###################################################################### +from b2sdk.v2 import Bucket + + +class BucketTrackingMixin: + """ + Mixin class for B2Api, which enables bucket tracking. + This mixin will add a `buckets` member to the B2Api instance and will use it track created and + deleted buckets. The main purpose of this are tests -- the `buckets` member can be used in test + teardown to ensure proper bucket cleanup. + """ + + def __init__(self, *args, **kwargs): + self.buckets = [] + super().__init__(*args, **kwargs) + + def create_bucket(self, name: str, *args, **kwargs) -> Bucket: + bucket = super().create_bucket(name, *args, **kwargs) + self.buckets.append(bucket) + return bucket + + def delete_bucket(self, bucket: Bucket): + super().delete_bucket(bucket) + self.buckets = [b for b in self.buckets if b.id_ != bucket.id_] diff --git a/noxfile.py b/noxfile.py index 7fd5d27e5..4adef21c9 100644 --- a/noxfile.py +++ b/noxfile.py @@ -134,7 +134,7 @@ def unit(session): @nox.session(python=PYTHON_VERSIONS) def integration(session): """Run integration tests.""" - install_myself(session) + install_myself(session, ['dev']) session.run('pip', 'install', *REQUIREMENTS_TEST) session.run('pytest', '-s', *session.posargs, 'test/integration') @@ -142,7 +142,7 @@ def integration(session): @nox.session(python=PYTHON_DEFAULT_VERSION) def cleanup_old_buckets(session): """Remove buckets from previous test runs.""" - install_myself(session) + install_myself(session, ['dev']) session.run('pip', 'install', *REQUIREMENTS_TEST) session.run('python', '-m', 'test.integration.cleanup_buckets') diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 000000000..28a7e406e --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1 @@ +backoff>=1.4.0,<3.0.0 \ No newline at end of file diff --git a/setup.py b/setup.py index 486522f28..e2c58a423 100644 --- a/setup.py +++ b/setup.py @@ -108,7 +108,10 @@ def read_requirements(extra=None): # dependencies). You can install these using the following syntax, # for example: # $ pip install -e .[dev,test] - extras_require={'doc': read_requirements('doc')}, + extras_require={ + 'doc': read_requirements('doc'), + 'dev': read_requirements('dev'), + }, setup_requires=['setuptools_scm<6.0'], use_scm_version=True, diff --git a/test/integration/__init__.py b/test/integration/__init__.py index 1bc44711a..0475eb7ec 100644 --- a/test/integration/__init__.py +++ b/test/integration/__init__.py @@ -8,9 +8,10 @@ # ###################################################################### import os +from typing import Tuple -def get_b2_auth_data(): +def get_b2_auth_data() -> Tuple[str, str]: application_key_id = os.environ.get('B2_TEST_APPLICATION_KEY_ID') if application_key_id is None: raise ValueError('B2_TEST_APPLICATION_KEY_ID is not set.') @@ -19,3 +20,7 @@ def get_b2_auth_data(): if application_key is None: raise ValueError('B2_TEST_APPLICATION_KEY is not set.') return application_key_id, application_key + + +def get_realm() -> str: + return os.environ.get('B2_TEST_ENVIRONMENT', 'production') diff --git a/test/integration/base.py b/test/integration/base.py index ff6245665..b3d1c5b95 100644 --- a/test/integration/base.py +++ b/test/integration/base.py @@ -13,10 +13,7 @@ import pytest -from b2sdk.v2 import current_time_millis -from b2sdk.v2.exception import DuplicateBucketName -from .bucket_cleaner import BucketCleaner -from .helpers import GENERAL_BUCKET_NAME_PREFIX, BUCKET_NAME_LENGTH, BUCKET_CREATED_AT_MILLIS, bucket_name_part, authorize +from b2sdk.test.api_test_manager import ApiTestManager class IntegrationTestBase: @@ -26,30 +23,11 @@ def set_http_debug(self): http.client.HTTPConnection.debuglevel = 1 @pytest.fixture(autouse=True) - def save_settings(self, dont_cleanup_old_buckets, b2_auth_data): - type(self).dont_cleanup_old_buckets = dont_cleanup_old_buckets - type(self).b2_auth_data = b2_auth_data - - @classmethod - def setup_class(cls): - cls.this_run_bucket_name_prefix = GENERAL_BUCKET_NAME_PREFIX + bucket_name_part(8) - - @classmethod - def teardown_class(cls): - BucketCleaner( - cls.dont_cleanup_old_buckets, - *cls.b2_auth_data, - current_run_prefix=cls.this_run_bucket_name_prefix - ).cleanup_buckets() - - @pytest.fixture(autouse=True) - def setup_method(self): - self.b2_api, self.info = authorize(self.b2_auth_data) - - def generate_bucket_name(self): - return self.this_run_bucket_name_prefix + bucket_name_part( - BUCKET_NAME_LENGTH - len(self.this_run_bucket_name_prefix) - ) + def setup_method(self, b2_auth_data, realm): + self.b2_api = ApiTestManager(*b2_auth_data, realm) + self.info = self.b2_api.account_info + yield + self.b2_api.clean_buckets() def write_zeros(self, file, number): line = b'0' * 1000 + b'\n' @@ -60,32 +38,4 @@ def write_zeros(self, file, number): written += line_len def create_bucket(self): - bucket_name = self.generate_bucket_name() - try: - return self.b2_api.create_bucket( - bucket_name, - 'allPublic', - bucket_info={BUCKET_CREATED_AT_MILLIS: str(current_time_millis())} - ) - except DuplicateBucketName: - self._duplicated_bucket_name_debug_info(bucket_name) - raise - - def _duplicated_bucket_name_debug_info(self, bucket_name: str) -> None: - # Trying to obtain as much information as possible about this bucket. - print(' DUPLICATED BUCKET DEBUG START '.center(60, '=')) - bucket = self.b2_api.get_bucket_by_name(bucket_name) - - print('Bucket metadata:') - bucket_dict = bucket.as_dict() - for info_key, info in bucket_dict.items(): - print('\t%s: "%s"' % (info_key, info)) - - print('All files (and their versions) inside the bucket:') - ls_generator = bucket.ls(recursive=True, latest_only=False) - for file_version, _directory in ls_generator: - # as_dict() is bound to have more info than we can use, - # but maybe some of it will cast some light on the issue. - print('\t%s (%s)' % (file_version.file_name, file_version.as_dict())) - - print(' DUPLICATED BUCKET DEBUG END '.center(60, '=')) + return self.b2_api.create_test_bucket() diff --git a/test/integration/bucket_cleaner.py b/test/integration/bucket_cleaner.py deleted file mode 100644 index e1b1e5783..000000000 --- a/test/integration/bucket_cleaner.py +++ /dev/null @@ -1,90 +0,0 @@ -###################################################################### -# -# File: test/integration/bucket_cleaner.py -# -# Copyright 2022 Backblaze Inc. All Rights Reserved. -# -# License https://www.backblaze.com/using_b2_code.html -# -###################################################################### - -from typing import Optional - -from b2sdk.v2 import * - -from .helpers import GENERAL_BUCKET_NAME_PREFIX, BUCKET_CREATED_AT_MILLIS, authorize - -ONE_HOUR_MILLIS = 60 * 60 * 1000 - - -class BucketCleaner: - def __init__( - self, - dont_cleanup_old_buckets: bool, - b2_application_key_id: str, - b2_application_key: str, - current_run_prefix: Optional[str] = None - ): - self.current_run_prefix = current_run_prefix - self.dont_cleanup_old_buckets = dont_cleanup_old_buckets - self.b2_application_key_id = b2_application_key_id - self.b2_application_key = b2_application_key - - def _should_remove_bucket(self, bucket: Bucket): - if self.current_run_prefix and bucket.name.startswith(self.current_run_prefix): - return True - if self.dont_cleanup_old_buckets: - return False - if bucket.name.startswith(GENERAL_BUCKET_NAME_PREFIX): - if BUCKET_CREATED_AT_MILLIS in bucket.bucket_info: - if int(bucket.bucket_info[BUCKET_CREATED_AT_MILLIS] - ) < current_time_millis() - ONE_HOUR_MILLIS: - return True - return False - - def cleanup_buckets(self): - b2_api, _ = authorize((self.b2_application_key_id, self.b2_application_key)) - buckets = b2_api.list_buckets() - for bucket in buckets: - if not self._should_remove_bucket(bucket): - print('Skipping bucket removal:', bucket.name) - else: - print('Trying to remove bucket:', bucket.name) - files_leftover = False - file_versions = bucket.ls(latest_only=False, recursive=True) - for file_version_info, _ in file_versions: - if file_version_info.file_retention: - if file_version_info.file_retention.mode == RetentionMode.GOVERNANCE: - print('Removing retention from file version:', file_version_info.id_) - b2_api.update_file_retention( - file_version_info.id_, file_version_info.file_name, - NO_RETENTION_FILE_SETTING, True - ) - elif file_version_info.file_retention.mode == RetentionMode.COMPLIANCE: - if file_version_info.file_retention.retain_until > current_time_millis(): # yapf: disable - print( - 'File version: %s cannot be removed due to compliance mode retention' - % (file_version_info.id_,) - ) - files_leftover = True - continue - elif file_version_info.file_retention.mode == RetentionMode.NONE: - pass - else: - raise ValueError( - 'Unknown retention mode: %s' % - (file_version_info.file_retention.mode,) - ) - if file_version_info.legal_hold.is_on(): - print('Removing legal hold from file version:', file_version_info.id_) - b2_api.update_file_legal_hold( - file_version_info.id_, file_version_info.file_name, LegalHold.OFF - ) - print('Removing file version:', file_version_info.id_) - b2_api.delete_file_version(file_version_info.id_, file_version_info.file_name) - - if files_leftover: - print('Unable to remove bucket because some retained files remain') - else: - print('Removing bucket:', bucket.name) - b2_api.delete_bucket(bucket) diff --git a/test/integration/cleanup_buckets.py b/test/integration/cleanup_buckets.py index 02bb16ce0..bb16cadeb 100755 --- a/test/integration/cleanup_buckets.py +++ b/test/integration/cleanup_buckets.py @@ -8,10 +8,10 @@ # ###################################################################### -from . import get_b2_auth_data -from .bucket_cleaner import BucketCleaner +from . import get_b2_auth_data, get_realm +from b2sdk.test.api_test_manager import ApiTestManager from .test_raw_api import cleanup_old_buckets if __name__ == '__main__': cleanup_old_buckets() - BucketCleaner(False, *get_b2_auth_data()).cleanup_buckets() + ApiTestManager(*get_b2_auth_data(), get_realm()).clean_all_buckets() diff --git a/test/integration/fixtures/__init__.py b/test/integration/fixtures/__init__.py index 2d12d6e8c..37cf29e38 100644 --- a/test/integration/fixtures/__init__.py +++ b/test/integration/fixtures/__init__.py @@ -11,7 +11,7 @@ import os import pytest -from .. import get_b2_auth_data +from .. import get_b2_auth_data, get_realm @pytest.fixture @@ -20,3 +20,8 @@ def b2_auth_data(): return get_b2_auth_data() except ValueError as ex: pytest.fail(ex.args[0]) + + +@pytest.fixture +def realm(): + return get_realm() diff --git a/test/integration/helpers.py b/test/integration/helpers.py deleted file mode 100644 index 60ea3539e..000000000 --- a/test/integration/helpers.py +++ /dev/null @@ -1,35 +0,0 @@ -###################################################################### -# -# File: test/integration/helpers.py -# -# Copyright 2022 Backblaze Inc. All Rights Reserved. -# -# License https://www.backblaze.com/using_b2_code.html -# -###################################################################### - -from typing import Optional -import os -import random -import string - -import pytest - -from b2sdk.v2 import * - -GENERAL_BUCKET_NAME_PREFIX = 'sdktst' -BUCKET_NAME_CHARS = string.ascii_letters + string.digits + '-' -BUCKET_NAME_LENGTH = 50 -BUCKET_CREATED_AT_MILLIS = 'created_at_millis' - - -def bucket_name_part(length): - return ''.join(random.choice(BUCKET_NAME_CHARS) for _ in range(length)) - - -def authorize(b2_auth_data, api_config=DEFAULT_HTTP_API_CONFIG): - info = InMemoryAccountInfo() - b2_api = B2Api(info, api_config=api_config) - realm = os.environ.get('B2_TEST_ENVIRONMENT', 'production') - b2_api.authorize_account(realm, *b2_auth_data) - return b2_api, info diff --git a/test/integration/test_download.py b/test/integration/test_download.py index a093d24be..473aafef2 100644 --- a/test/integration/test_download.py +++ b/test/integration/test_download.py @@ -15,11 +15,11 @@ from typing import Optional, Tuple from unittest import mock +from b2sdk.test.api_test_manager import ApiTestManager from b2sdk.v2 import * from b2sdk.utils import Sha1HexDigest from .fixtures import * # pyflakes: disable -from .helpers import authorize from .base import IntegrationTestBase @@ -93,7 +93,7 @@ def test_small_unverified(self): pprint(f.download_version._get_args_for_clone()) assert not f.download_version.content_sha1_verified - def test_gzip(self): + def test_gzip(self, b2_auth_data, realm): bucket = self.create_bucket() with TempDir() as temp_dir: temp_dir = pathlib.Path(temp_dir) @@ -116,8 +116,8 @@ def test_gzip(self): source_data = sf.read() assert downloaded_data == source_data - decompressing_api, _ = authorize( - self.b2_auth_data, B2HttpApiConfig(decode_content=True) + decompressing_api = ApiTestManager( + *b2_auth_data, realm, api_config=B2HttpApiConfig(decode_content=True) ) decompressing_api.download_file_by_id(file_id=file_version.id_).save_to( str(downloaded_uncompressed_file) diff --git a/test/integration/test_upload.py b/test/integration/test_upload.py index 34b88b8f5..5b82c4ee7 100644 --- a/test/integration/test_upload.py +++ b/test/integration/test_upload.py @@ -11,7 +11,7 @@ import io from typing import Optional -from .fixtures import b2_auth_data # noqa +from .fixtures import b2_auth_data, realm from .base import IntegrationTestBase