start at tests

tests/unit/packaging/test_services.py

@@ -13,6 +13,7 @@
 import io
 import os.path
 
+import b2sdk.v2
 import boto3.session
 import botocore.exceptions
 import pretend
@@ -24,12 +25,15 @@
 
 from warehouse.packaging.interfaces import IDocsStorage, IFileStorage, ISimpleStorage
 from warehouse.packaging.services import (
+    B2FileStorage,
     GCSFileStorage,
     GCSSimpleStorage,
     GenericLocalBlobStorage,
+    LocalArchiveFileStorage,
     LocalDocsStorage,
     LocalFileStorage,
     LocalSimpleStorage,
+    S3ArchiveFileStorage,
     S3DocsStorage,
     S3FileStorage,
     project_service_factory,
@@ -76,6 +80,22 @@ def test_stores_file(self, tmpdir):
         with open(os.path.join(storage_dir, "foo/bar.txt"), "rb") as fp:
             assert fp.read() == b"Test File!"
 
+    def test_stores_and_gets_metadata(self, tmpdir):
+        filename = str(tmpdir.join("testfile.txt"))
+        with open(filename, "wb") as fp:
+            fp.write(b"Test File!")
+
+        storage_dir = str(tmpdir.join("storage"))
+        storage = LocalFileStorage(storage_dir)
+        storage.store("foo/bar.txt", filename, meta={"foo": "bar", "wu": "tang"})
+
+        with open(os.path.join(storage_dir, "foo/bar.txt"), "rb") as fp:
+            assert fp.read() == b"Test File!"
+        with open(os.path.join(storage_dir, "foo/bar.txt.meta"), "rb") as fp:
+            assert fp.read() == b'{"foo": "bar", "wu": "tang"}'
+
+        assert storage.get_metadata("foo/bar.txt") == {"foo": "bar", "wu": "tang"}
+
     def test_stores_two_files(self, tmpdir):
         filename1 = str(tmpdir.join("testfile1.txt"))
         with open(filename1, "wb") as fp:
@@ -97,6 +117,18 @@ def test_stores_two_files(self, tmpdir):
             assert fp.read() == b"Second Test File!"
 
 
+class TestLocalArchiveFileStorage:
+    def test_verify_service(self):
+        assert verifyClass(IFileStorage, LocalArchiveFileStorage)
+
+    def test_create_service(self):
+        request = pretend.stub(
+            registry=pretend.stub(settings={"archive_files.path": "/the/one/two/"})
+        )
+        storage = LocalArchiveFileStorage.create_service(None, request)
+        assert storage.base == "/the/one/two/"
+
+
 class TestLocalDocsStorage:
     def test_verify_service(self):
         assert verifyClass(IDocsStorage, LocalDocsStorage)
@@ -202,6 +234,130 @@ def test_stores_two_files(self, tmpdir):
             assert fp.read() == b"Second Test File!"
 
 
+class TestB2FileStorage:
+    def test_verify_service(self):
+        assert verifyClass(IFileStorage, B2FileStorage)
+
+    def test_basic_init(self):
+        bucket = pretend.stub()
+        prefix = "segakcap"
+        storage = B2FileStorage(bucket, prefix=prefix)
+        assert storage.bucket is bucket
+        assert storage.prefix == "segakcap"
+
+    def test_create_service(self):
+        bucket_stub = pretend.stub()
+        mock_b2_api = pretend.stub(
+            get_bucket_by_name=pretend.call_recorder(lambda bucket_name: bucket_stub)
+        )
+
+        request = pretend.stub(
+            find_service=pretend.call_recorder(lambda name: mock_b2_api),
+            registry=pretend.stub(settings={"files.bucket": "froblob"}),
+        )
+        storage = B2FileStorage.create_service(None, request)
+
+        assert request.find_service.calls == [pretend.call(name="b2.api")]
+        assert storage.bucket == bucket_stub
+        assert mock_b2_api.get_bucket_by_name.calls == [pretend.call("froblob")]
+
+    def test_gets_file(self):
+        bucket_stub = pretend.stub(
+            download_file_by_name=pretend.call_recorder(
+                lambda path: pretend.stub(
+                    save=lambda file_obj: file_obj.write(b"my contents")
+                )
+            )
+        )
+        mock_b2_api = pretend.stub(get_bucket_by_name=lambda bucket_name: bucket_stub)
+
+        request = pretend.stub(
+            find_service=pretend.call_recorder(lambda name: mock_b2_api),
+            registry=pretend.stub(settings={"files.bucket": "froblob"}),
+        )
+        storage = B2FileStorage.create_service(None, request)
+
+        file_object = storage.get("file.txt")
+
+        assert file_object.read() == b"my contents"
+        assert bucket_stub.download_file_by_name.calls == [pretend.call("file.txt")]
+
+    def test_gets_metadata(self):
+        bucket_stub = pretend.stub(
+            get_file_info_by_name=pretend.call_recorder(
+                lambda path: pretend.stub(file_info={"foo": "bar", "wu": "tang"})
+            )
+        )
+        mock_b2_api = pretend.stub(get_bucket_by_name=lambda bucket_name: bucket_stub)
+
+        request = pretend.stub(
+            find_service=pretend.call_recorder(lambda name: mock_b2_api),
+            registry=pretend.stub(settings={"files.bucket": "froblob"}),
+        )
+        storage = B2FileStorage.create_service(None, request)
+
+        metadata = storage.get_metadata("file.txt")
+
+        assert metadata == {"foo": "bar", "wu": "tang"}
+        assert bucket_stub.get_file_info_by_name.calls == [pretend.call("file.txt")]
+
+    def test_raises_when_key_non_existent(self):
+        def raiser(path):
+            raise b2sdk.exception.FileNotPresent()
+
+        bucket_stub = pretend.stub(download_file_by_name=raiser)
+        mock_b2_api = pretend.stub(get_bucket_by_name=lambda bucket_name: bucket_stub)
+
+        request = pretend.stub(
+            find_service=pretend.call_recorder(lambda name: mock_b2_api),
+            registry=pretend.stub(settings={"files.bucket": "froblob"}),
+        )
+        storage = B2FileStorage.create_service(None, request)
+
+        with pytest.raises(FileNotFoundError):
+            storage.get("file.txt")
+
+    def test_get_metadata_raises_when_key_non_existent(self):
+        def raiser(path):
+            raise b2sdk.exception.FileNotPresent()
+
+        bucket_stub = pretend.stub(get_file_info_by_name=raiser)
+        mock_b2_api = pretend.stub(get_bucket_by_name=lambda bucket_name: bucket_stub)
+
+        request = pretend.stub(
+            find_service=pretend.call_recorder(lambda name: mock_b2_api),
+            registry=pretend.stub(settings={"files.bucket": "froblob"}),
+        )
+        storage = B2FileStorage.create_service(None, request)
+
+        with pytest.raises(FileNotFoundError):
+            storage.get_metadata("file.txt")
+
+    def test_stores_file(self, tmpdir):
+        filename = str(tmpdir.join("testfile.txt"))
+        with open(filename, "wb") as fp:
+            fp.write(b"Test File!")
+
+        bucket_stub = pretend.stub(
+            upload_local_file=pretend.call_recorder(
+                lambda local_file=None, file_name=None, file_infos=None: None
+            )
+        )
+        mock_b2_api = pretend.stub(get_bucket_by_name=lambda bucket_name: bucket_stub)
+
+        request = pretend.stub(
+            find_service=pretend.call_recorder(lambda name: mock_b2_api),
+            registry=pretend.stub(settings={"files.bucket": "froblob"}),
+        )
+        storage = B2FileStorage.create_service(None, request)
+
+        storage.store("foo/bar.txt", filename)
+
+        assert bucket_stub.upload_local_file.calls == [
+            pretend.call(local_file=filename, file_name="foo/bar.txt", file_infos=None)
+        ]
+
+
 class TestS3FileStorage:
     def test_verify_service(self):
         assert verifyClass(IFileStorage, S3FileStorage)
@@ -234,6 +390,16 @@ def test_gets_file(self):
         assert file_object.read() == b"my contents"
         assert bucket.Object.calls == [pretend.call("file.txt")]
 
+    def test_gets_metadata(self):
+        s3key = pretend.stub(metadata={"foo": "bar", "wu": "tang"})
+        bucket = pretend.stub(Object=pretend.call_recorder(lambda path: s3key))
+        storage = S3FileStorage(bucket)
+
+        metadata = storage.get_metadata("file.txt")
+
+        assert metadata == {"foo": "bar", "wu": "tang"}
+        assert bucket.Object.calls == [pretend.call("file.txt")]
+
     def test_raises_when_key_non_existent(self):
         def raiser():
             raise botocore.exceptions.ClientError(
@@ -249,6 +415,18 @@ def raiser():
 
         assert bucket.Object.calls == [pretend.call("file.txt")]
 
+    def test_get_metadata_raises_when_key_non_existent(self):
+        def raiser(*a, **kw):
+            raise botocore.exceptions.ClientError(
+                {"Error": {"Code": "NoSuchKey", "Message": "No Key!"}}, "some operation"
+            )
+
+        bucket = pretend.stub(Object=raiser)
+        storage = S3FileStorage(bucket)
+
+        with pytest.raises(FileNotFoundError):
+            storage.get_metadata("file.txt")
+
     def test_passes_up_error_when_not_no_such_key(self):
         def raiser():
             raise botocore.exceptions.ClientError(
@@ -263,6 +441,19 @@ def raiser():
         with pytest.raises(botocore.exceptions.ClientError):
             storage.get("file.txt")
 
+    def test_get_metadata_passes_up_error_when_not_no_such_key(self):
+        def raiser(*a, **kw):
+            raise botocore.exceptions.ClientError(
+                {"Error": {"Code": "SomeOtherError", "Message": "Who Knows!"}},
+                "some operation",
+            )
+
+        bucket = pretend.stub(Object=raiser)
+        storage = S3FileStorage(bucket)
+
+        with pytest.raises(botocore.exceptions.ClientError):
+            storage.get_metadata("file.txt")
+
     def test_stores_file(self, tmpdir):
         filename = str(tmpdir.join("testfile.txt"))
         with open(filename, "wb") as fp:
@@ -337,6 +528,24 @@ def test_hashed_path_without_prefix(self):
         assert bucket.Object.calls == [pretend.call("ab/file.txt")]
 
 
+class TestS3ArchiveFileStorage:
+    def test_verify_service(self):
+        assert verifyClass(IFileStorage, S3ArchiveFileStorage)
+
+    def test_create_service(self):
+        session = boto3.session.Session(
+            aws_access_key_id="foo", aws_secret_access_key="bar"
+        )
+        request = pretend.stub(
+            find_service=pretend.call_recorder(lambda name: session),
+            registry=pretend.stub(settings={"archive_files.bucket": "froblob"}),
+        )
+        storage = S3ArchiveFileStorage.create_service(None, request)
+
+        assert request.find_service.calls == [pretend.call(name="aws.session")]
+        assert storage.bucket.name == "froblob"
+
+
 class TestGCSFileStorage:
     def test_verify_service(self):
         assert verifyClass(IFileStorage, GCSFileStorage)
@@ -365,6 +574,12 @@ def test_gets_file_raises(self):
         with pytest.raises(NotImplementedError):
             storage.get("file.txt")
 
+    def test_get_metadata_raises(self):
+        storage = GCSFileStorage(pretend.stub())
+
+        with pytest.raises(NotImplementedError):
+            storage.get_metadata("file.txt")
+
     def test_stores_file(self, tmpdir):
         filename = str(tmpdir.join("testfile.txt"))
         with open(filename, "wb") as fp:

tests/unit/test_b2.py

@@ -0,0 +1,51 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import b2sdk.v2
+import pretend
+import pytest
+
+from warehouse import b2
+
+
+def test_b2_api_factory(monkeypatch):
+    mock_in_memory_account_info = pretend.call_recorder(lambda: "InMemoryAccountInfo")
+    monkeypatch.setattr(b2sdk.v2, "InMemoryAccountInfo", mock_in_memory_account_info)
+    mock_b2_api = pretend.stub(
+        authorize_account=pretend.call_recorder(lambda mode, key_id, key: None)
+    )
+    mock_b2_api_class = pretend.call_recorder(lambda account_info: mock_b2_api)
+    monkeypatch.setattr(b2sdk.v2, "B2Api", mock_b2_api_class)
+
+    request = pretend.stub(
+        registry=pretend.stub(
+            settings={"b2.application_key_id": "key_id", "b2.application_key": "key"}
+        )
+    )
+
+    assert b2.b2_api_factory(None, request) is mock_b2_api
+    assert mock_b2_api_class.calls == [pretend.call("InMemoryAccountInfo")]
+    assert mock_b2_api.authorize_account.calls == [
+        pretend.call("production", "key_id", "key")
+    ]
+
+
+def test_includeme():
+    config = pretend.stub(
+        register_service_factory=pretend.call_recorder(lambda factory, name: None)
+    )
+
+    b2.includeme(config)
+
+    assert config.register_service_factory.calls == [
+        pretend.call(b2.b2_api_factory, name="b2.api")
+    ]

warehouse/b2.py

@@ -10,11 +10,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from b2sdk.v2 import B2Api, InMemoryAccountInfo
+import b2sdk.v2
 
 
 def b2_api_factory(context, request):
-    b2_api = B2Api(InMemoryAccountInfo())
+    b2_api = b2sdk.v2.B2Api(b2sdk.v2.InMemoryAccountInfo())
     b2_api.authorize_account(
         "production",
         request.registry.settings["b2.application_key_id"],