From 4ec50f3dabb418ad87ac400068a673cbdd143893 Mon Sep 17 00:00:00 2001 From: Ee Durbin Date: Mon, 10 Apr 2023 16:27:04 -0400 Subject: [PATCH] start at tests --- tests/unit/packaging/test_services.py | 215 ++++++++++++++++++++++++++ tests/unit/test_b2.py | 51 ++++++ warehouse/b2.py | 4 +- 3 files changed, 268 insertions(+), 2 deletions(-) create mode 100644 tests/unit/test_b2.py diff --git a/tests/unit/packaging/test_services.py b/tests/unit/packaging/test_services.py index dc300ccde570..7418e41d6734 100644 --- a/tests/unit/packaging/test_services.py +++ b/tests/unit/packaging/test_services.py @@ -13,6 +13,7 @@ import io import os.path +import b2sdk.v2 import boto3.session import botocore.exceptions import pretend @@ -24,12 +25,15 @@ from warehouse.packaging.interfaces import IDocsStorage, IFileStorage, ISimpleStorage from warehouse.packaging.services import ( + B2FileStorage, GCSFileStorage, GCSSimpleStorage, GenericLocalBlobStorage, + LocalArchiveFileStorage, LocalDocsStorage, LocalFileStorage, LocalSimpleStorage, + S3ArchiveFileStorage, S3DocsStorage, S3FileStorage, project_service_factory, @@ -76,6 +80,22 @@ def test_stores_file(self, tmpdir): with open(os.path.join(storage_dir, "foo/bar.txt"), "rb") as fp: assert fp.read() == b"Test File!" + def test_stores_and_gets_metadata(self, tmpdir): + filename = str(tmpdir.join("testfile.txt")) + with open(filename, "wb") as fp: + fp.write(b"Test File!") + + storage_dir = str(tmpdir.join("storage")) + storage = LocalFileStorage(storage_dir) + storage.store("foo/bar.txt", filename, meta={"foo": "bar", "wu": "tang"}) + + with open(os.path.join(storage_dir, "foo/bar.txt"), "rb") as fp: + assert fp.read() == b"Test File!" + with open(os.path.join(storage_dir, "foo/bar.txt.meta"), "rb") as fp: + assert fp.read() == b'{"foo": "bar", "wu": "tang"}' + + assert storage.get_metadata("foo/bar.txt") == {"foo": "bar", "wu": "tang"} + def test_stores_two_files(self, tmpdir): filename1 = str(tmpdir.join("testfile1.txt")) with open(filename1, "wb") as fp: @@ -97,6 +117,18 @@ def test_stores_two_files(self, tmpdir): assert fp.read() == b"Second Test File!" +class TestLocalArchiveFileStorage: + def test_verify_service(self): + assert verifyClass(IFileStorage, LocalArchiveFileStorage) + + def test_create_service(self): + request = pretend.stub( + registry=pretend.stub(settings={"archive_files.path": "/the/one/two/"}) + ) + storage = LocalArchiveFileStorage.create_service(None, request) + assert storage.base == "/the/one/two/" + + class TestLocalDocsStorage: def test_verify_service(self): assert verifyClass(IDocsStorage, LocalDocsStorage) @@ -202,6 +234,130 @@ def test_stores_two_files(self, tmpdir): assert fp.read() == b"Second Test File!" +class TestB2FileStorage: + def test_verify_service(self): + assert verifyClass(IFileStorage, B2FileStorage) + + def test_basic_init(self): + bucket = pretend.stub() + prefix = "segakcap" + storage = B2FileStorage(bucket, prefix=prefix) + assert storage.bucket is bucket + assert storage.prefix == "segakcap" + + def test_create_service(self): + bucket_stub = pretend.stub() + mock_b2_api = pretend.stub( + get_bucket_by_name=pretend.call_recorder(lambda bucket_name: bucket_stub) + ) + + request = pretend.stub( + find_service=pretend.call_recorder(lambda name: mock_b2_api), + registry=pretend.stub(settings={"files.bucket": "froblob"}), + ) + storage = B2FileStorage.create_service(None, request) + + assert request.find_service.calls == [pretend.call(name="b2.api")] + assert storage.bucket == bucket_stub + assert mock_b2_api.get_bucket_by_name.calls == [pretend.call("froblob")] + + def test_gets_file(self): + bucket_stub = pretend.stub( + download_file_by_name=pretend.call_recorder( + lambda path: pretend.stub( + save=lambda file_obj: file_obj.write(b"my contents") + ) + ) + ) + mock_b2_api = pretend.stub(get_bucket_by_name=lambda bucket_name: bucket_stub) + + request = pretend.stub( + find_service=pretend.call_recorder(lambda name: mock_b2_api), + registry=pretend.stub(settings={"files.bucket": "froblob"}), + ) + storage = B2FileStorage.create_service(None, request) + + file_object = storage.get("file.txt") + + assert file_object.read() == b"my contents" + assert bucket_stub.download_file_by_name.calls == [pretend.call("file.txt")] + + def test_gets_metadata(self): + bucket_stub = pretend.stub( + get_file_info_by_name=pretend.call_recorder( + lambda path: pretend.stub(file_info={"foo": "bar", "wu": "tang"}) + ) + ) + mock_b2_api = pretend.stub(get_bucket_by_name=lambda bucket_name: bucket_stub) + + request = pretend.stub( + find_service=pretend.call_recorder(lambda name: mock_b2_api), + registry=pretend.stub(settings={"files.bucket": "froblob"}), + ) + storage = B2FileStorage.create_service(None, request) + + metadata = storage.get_metadata("file.txt") + + assert metadata == {"foo": "bar", "wu": "tang"} + assert bucket_stub.get_file_info_by_name.calls == [pretend.call("file.txt")] + + def test_raises_when_key_non_existent(self): + def raiser(path): + raise b2sdk.exception.FileNotPresent() + + bucket_stub = pretend.stub(download_file_by_name=raiser) + mock_b2_api = pretend.stub(get_bucket_by_name=lambda bucket_name: bucket_stub) + + request = pretend.stub( + find_service=pretend.call_recorder(lambda name: mock_b2_api), + registry=pretend.stub(settings={"files.bucket": "froblob"}), + ) + storage = B2FileStorage.create_service(None, request) + + with pytest.raises(FileNotFoundError): + storage.get("file.txt") + + def test_get_metadata_raises_when_key_non_existent(self): + def raiser(path): + raise b2sdk.exception.FileNotPresent() + + bucket_stub = pretend.stub(get_file_info_by_name=raiser) + mock_b2_api = pretend.stub(get_bucket_by_name=lambda bucket_name: bucket_stub) + + request = pretend.stub( + find_service=pretend.call_recorder(lambda name: mock_b2_api), + registry=pretend.stub(settings={"files.bucket": "froblob"}), + ) + storage = B2FileStorage.create_service(None, request) + + with pytest.raises(FileNotFoundError): + storage.get_metadata("file.txt") + + def test_stores_file(self, tmpdir): + filename = str(tmpdir.join("testfile.txt")) + with open(filename, "wb") as fp: + fp.write(b"Test File!") + + bucket_stub = pretend.stub( + upload_local_file=pretend.call_recorder( + lambda local_file=None, file_name=None, file_infos=None: None + ) + ) + mock_b2_api = pretend.stub(get_bucket_by_name=lambda bucket_name: bucket_stub) + + request = pretend.stub( + find_service=pretend.call_recorder(lambda name: mock_b2_api), + registry=pretend.stub(settings={"files.bucket": "froblob"}), + ) + storage = B2FileStorage.create_service(None, request) + + storage.store("foo/bar.txt", filename) + + assert bucket_stub.upload_local_file.calls == [ + pretend.call(local_file=filename, file_name="foo/bar.txt", file_infos=None) + ] + + class TestS3FileStorage: def test_verify_service(self): assert verifyClass(IFileStorage, S3FileStorage) @@ -234,6 +390,16 @@ def test_gets_file(self): assert file_object.read() == b"my contents" assert bucket.Object.calls == [pretend.call("file.txt")] + def test_gets_metadata(self): + s3key = pretend.stub(metadata={"foo": "bar", "wu": "tang"}) + bucket = pretend.stub(Object=pretend.call_recorder(lambda path: s3key)) + storage = S3FileStorage(bucket) + + metadata = storage.get_metadata("file.txt") + + assert metadata == {"foo": "bar", "wu": "tang"} + assert bucket.Object.calls == [pretend.call("file.txt")] + def test_raises_when_key_non_existent(self): def raiser(): raise botocore.exceptions.ClientError( @@ -249,6 +415,18 @@ def raiser(): assert bucket.Object.calls == [pretend.call("file.txt")] + def test_get_metadata_raises_when_key_non_existent(self): + def raiser(*a, **kw): + raise botocore.exceptions.ClientError( + {"Error": {"Code": "NoSuchKey", "Message": "No Key!"}}, "some operation" + ) + + bucket = pretend.stub(Object=raiser) + storage = S3FileStorage(bucket) + + with pytest.raises(FileNotFoundError): + storage.get_metadata("file.txt") + def test_passes_up_error_when_not_no_such_key(self): def raiser(): raise botocore.exceptions.ClientError( @@ -263,6 +441,19 @@ def raiser(): with pytest.raises(botocore.exceptions.ClientError): storage.get("file.txt") + def test_get_metadata_passes_up_error_when_not_no_such_key(self): + def raiser(*a, **kw): + raise botocore.exceptions.ClientError( + {"Error": {"Code": "SomeOtherError", "Message": "Who Knows!"}}, + "some operation", + ) + + bucket = pretend.stub(Object=raiser) + storage = S3FileStorage(bucket) + + with pytest.raises(botocore.exceptions.ClientError): + storage.get_metadata("file.txt") + def test_stores_file(self, tmpdir): filename = str(tmpdir.join("testfile.txt")) with open(filename, "wb") as fp: @@ -337,6 +528,24 @@ def test_hashed_path_without_prefix(self): assert bucket.Object.calls == [pretend.call("ab/file.txt")] +class TestS3ArchiveFileStorage: + def test_verify_service(self): + assert verifyClass(IFileStorage, S3ArchiveFileStorage) + + def test_create_service(self): + session = boto3.session.Session( + aws_access_key_id="foo", aws_secret_access_key="bar" + ) + request = pretend.stub( + find_service=pretend.call_recorder(lambda name: session), + registry=pretend.stub(settings={"archive_files.bucket": "froblob"}), + ) + storage = S3ArchiveFileStorage.create_service(None, request) + + assert request.find_service.calls == [pretend.call(name="aws.session")] + assert storage.bucket.name == "froblob" + + class TestGCSFileStorage: def test_verify_service(self): assert verifyClass(IFileStorage, GCSFileStorage) @@ -365,6 +574,12 @@ def test_gets_file_raises(self): with pytest.raises(NotImplementedError): storage.get("file.txt") + def test_get_metadata_raises(self): + storage = GCSFileStorage(pretend.stub()) + + with pytest.raises(NotImplementedError): + storage.get_metadata("file.txt") + def test_stores_file(self, tmpdir): filename = str(tmpdir.join("testfile.txt")) with open(filename, "wb") as fp: diff --git a/tests/unit/test_b2.py b/tests/unit/test_b2.py new file mode 100644 index 000000000000..ab376b44b226 --- /dev/null +++ b/tests/unit/test_b2.py @@ -0,0 +1,51 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import b2sdk.v2 +import pretend +import pytest + +from warehouse import b2 + + +def test_b2_api_factory(monkeypatch): + mock_in_memory_account_info = pretend.call_recorder(lambda: "InMemoryAccountInfo") + monkeypatch.setattr(b2sdk.v2, "InMemoryAccountInfo", mock_in_memory_account_info) + mock_b2_api = pretend.stub( + authorize_account=pretend.call_recorder(lambda mode, key_id, key: None) + ) + mock_b2_api_class = pretend.call_recorder(lambda account_info: mock_b2_api) + monkeypatch.setattr(b2sdk.v2, "B2Api", mock_b2_api_class) + + request = pretend.stub( + registry=pretend.stub( + settings={"b2.application_key_id": "key_id", "b2.application_key": "key"} + ) + ) + + assert b2.b2_api_factory(None, request) is mock_b2_api + assert mock_b2_api_class.calls == [pretend.call("InMemoryAccountInfo")] + assert mock_b2_api.authorize_account.calls == [ + pretend.call("production", "key_id", "key") + ] + + +def test_includeme(): + config = pretend.stub( + register_service_factory=pretend.call_recorder(lambda factory, name: None) + ) + + b2.includeme(config) + + assert config.register_service_factory.calls == [ + pretend.call(b2.b2_api_factory, name="b2.api") + ] diff --git a/warehouse/b2.py b/warehouse/b2.py index fe5c45413db7..3b6da1271249 100644 --- a/warehouse/b2.py +++ b/warehouse/b2.py @@ -10,11 +10,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -from b2sdk.v2 import B2Api, InMemoryAccountInfo +import b2sdk.v2 def b2_api_factory(context, request): - b2_api = B2Api(InMemoryAccountInfo()) + b2_api = b2sdk.v2.B2Api(b2sdk.v2.InMemoryAccountInfo()) b2_api.authorize_account( "production", request.registry.settings["b2.application_key_id"],