tests and linting

tests/conftest.py

@@ -89,6 +89,7 @@ def metrics_timing(*args, **kwargs):
 def metrics():
     return pretend.stub(
         event=pretend.call_recorder(lambda *args, **kwargs: None),
+        gauge=pretend.call_recorder(lambda *args, **kwargs: None),
         increment=pretend.call_recorder(lambda *args, **kwargs: None),
         histogram=pretend.call_recorder(lambda *args, **kwargs: None),
         timing=pretend.call_recorder(lambda *args, **kwargs: None),

tests/unit/packaging/test_services.py

@@ -13,6 +13,7 @@
 import io
 import os.path
 
+import b2sdk.v2
 import boto3.session
 import botocore.exceptions
 import pretend
@@ -24,12 +25,15 @@
 
 from warehouse.packaging.interfaces import IDocsStorage, IFileStorage, ISimpleStorage
 from warehouse.packaging.services import (
+    B2FileStorage,
     GCSFileStorage,
     GCSSimpleStorage,
     GenericLocalBlobStorage,
+    LocalArchiveFileStorage,
     LocalDocsStorage,
     LocalFileStorage,
     LocalSimpleStorage,
+    S3ArchiveFileStorage,
     S3DocsStorage,
     S3FileStorage,
     project_service_factory,
@@ -76,6 +80,22 @@ def test_stores_file(self, tmpdir):
         with open(os.path.join(storage_dir, "foo/bar.txt"), "rb") as fp:
             assert fp.read() == b"Test File!"
 
+    def test_stores_and_gets_metadata(self, tmpdir):
+        filename = str(tmpdir.join("testfile.txt"))
+        with open(filename, "wb") as fp:
+            fp.write(b"Test File!")
+
+        storage_dir = str(tmpdir.join("storage"))
+        storage = LocalFileStorage(storage_dir)
+        storage.store("foo/bar.txt", filename, meta={"foo": "bar", "wu": "tang"})
+
+        with open(os.path.join(storage_dir, "foo/bar.txt"), "rb") as fp:
+            assert fp.read() == b"Test File!"
+        with open(os.path.join(storage_dir, "foo/bar.txt.meta"), "rb") as fp:
+            assert fp.read() == b'{"foo": "bar", "wu": "tang"}'
+
+        assert storage.get_metadata("foo/bar.txt") == {"foo": "bar", "wu": "tang"}
+
     def test_stores_two_files(self, tmpdir):
         filename1 = str(tmpdir.join("testfile1.txt"))
         with open(filename1, "wb") as fp:
@@ -97,6 +117,18 @@ def test_stores_two_files(self, tmpdir):
             assert fp.read() == b"Second Test File!"
 
 
+class TestLocalArchiveFileStorage:
+    def test_verify_service(self):
+        assert verifyClass(IFileStorage, LocalArchiveFileStorage)
+
+    def test_create_service(self):
+        request = pretend.stub(
+            registry=pretend.stub(settings={"archive_files.path": "/the/one/two/"})
+        )
+        storage = LocalArchiveFileStorage.create_service(None, request)
+        assert storage.base == "/the/one/two/"
+
+
 class TestLocalDocsStorage:
     def test_verify_service(self):
         assert verifyClass(IDocsStorage, LocalDocsStorage)
@@ -202,6 +234,130 @@ def test_stores_two_files(self, tmpdir):
             assert fp.read() == b"Second Test File!"
 
 
+class TestB2FileStorage:
+    def test_verify_service(self):
+        assert verifyClass(IFileStorage, B2FileStorage)
+
+    def test_basic_init(self):
+        bucket = pretend.stub()
+        prefix = "segakcap"
+        storage = B2FileStorage(bucket, prefix=prefix)
+        assert storage.bucket is bucket
+        assert storage.prefix == "segakcap"
+
+    def test_create_service(self):
+        bucket_stub = pretend.stub()
+        mock_b2_api = pretend.stub(
+            get_bucket_by_name=pretend.call_recorder(lambda bucket_name: bucket_stub)
+        )
+
+        request = pretend.stub(
+            find_service=pretend.call_recorder(lambda name: mock_b2_api),
+            registry=pretend.stub(settings={"files.bucket": "froblob"}),
+        )
+        storage = B2FileStorage.create_service(None, request)
+
+        assert request.find_service.calls == [pretend.call(name="b2.api")]
+        assert storage.bucket == bucket_stub
+        assert mock_b2_api.get_bucket_by_name.calls == [pretend.call("froblob")]
+
+    def test_gets_file(self):
+        bucket_stub = pretend.stub(
+            download_file_by_name=pretend.call_recorder(
+                lambda path: pretend.stub(
+                    save=lambda file_obj: file_obj.write(b"my contents")
+                )
+            )
+        )
+        mock_b2_api = pretend.stub(get_bucket_by_name=lambda bucket_name: bucket_stub)
+
+        request = pretend.stub(
+            find_service=pretend.call_recorder(lambda name: mock_b2_api),
+            registry=pretend.stub(settings={"files.bucket": "froblob"}),
+        )
+        storage = B2FileStorage.create_service(None, request)
+
+        file_object = storage.get("file.txt")
+
+        assert file_object.read() == b"my contents"
+        assert bucket_stub.download_file_by_name.calls == [pretend.call("file.txt")]
+
+    def test_gets_metadata(self):
+        bucket_stub = pretend.stub(
+            get_file_info_by_name=pretend.call_recorder(
+                lambda path: pretend.stub(file_info={"foo": "bar", "wu": "tang"})
+            )
+        )
+        mock_b2_api = pretend.stub(get_bucket_by_name=lambda bucket_name: bucket_stub)
+
+        request = pretend.stub(
+            find_service=pretend.call_recorder(lambda name: mock_b2_api),
+            registry=pretend.stub(settings={"files.bucket": "froblob"}),
+        )
+        storage = B2FileStorage.create_service(None, request)
+
+        metadata = storage.get_metadata("file.txt")
+
+        assert metadata == {"foo": "bar", "wu": "tang"}
+        assert bucket_stub.get_file_info_by_name.calls == [pretend.call("file.txt")]
+
+    def test_raises_when_key_non_existent(self):
+        def raiser(path):
+            raise b2sdk.exception.FileNotPresent()
+
+        bucket_stub = pretend.stub(download_file_by_name=raiser)
+        mock_b2_api = pretend.stub(get_bucket_by_name=lambda bucket_name: bucket_stub)
+
+        request = pretend.stub(
+            find_service=pretend.call_recorder(lambda name: mock_b2_api),
+            registry=pretend.stub(settings={"files.bucket": "froblob"}),
+        )
+        storage = B2FileStorage.create_service(None, request)
+
+        with pytest.raises(FileNotFoundError):
+            storage.get("file.txt")
+
+    def test_get_metadata_raises_when_key_non_existent(self):
+        def raiser(path):
+            raise b2sdk.exception.FileNotPresent()
+
+        bucket_stub = pretend.stub(get_file_info_by_name=raiser)
+        mock_b2_api = pretend.stub(get_bucket_by_name=lambda bucket_name: bucket_stub)
+
+        request = pretend.stub(
+            find_service=pretend.call_recorder(lambda name: mock_b2_api),
+            registry=pretend.stub(settings={"files.bucket": "froblob"}),
+        )
+        storage = B2FileStorage.create_service(None, request)
+
+        with pytest.raises(FileNotFoundError):
+            storage.get_metadata("file.txt")
+
+    def test_stores_file(self, tmpdir):
+        filename = str(tmpdir.join("testfile.txt"))
+        with open(filename, "wb") as fp:
+            fp.write(b"Test File!")
+
+        bucket_stub = pretend.stub(
+            upload_local_file=pretend.call_recorder(
+                lambda local_file=None, file_name=None, file_infos=None: None
+            )
+        )
+        mock_b2_api = pretend.stub(get_bucket_by_name=lambda bucket_name: bucket_stub)
+
+        request = pretend.stub(
+            find_service=pretend.call_recorder(lambda name: mock_b2_api),
+            registry=pretend.stub(settings={"files.bucket": "froblob"}),
+        )
+        storage = B2FileStorage.create_service(None, request)
+
+        storage.store("foo/bar.txt", filename)
+
+        assert bucket_stub.upload_local_file.calls == [
+            pretend.call(local_file=filename, file_name="foo/bar.txt", file_infos=None)
+        ]
+
+
 class TestS3FileStorage:
     def test_verify_service(self):
         assert verifyClass(IFileStorage, S3FileStorage)
@@ -234,6 +390,16 @@ def test_gets_file(self):
         assert file_object.read() == b"my contents"
         assert bucket.Object.calls == [pretend.call("file.txt")]
 
+    def test_gets_metadata(self):
+        s3key = pretend.stub(metadata={"foo": "bar", "wu": "tang"})
+        bucket = pretend.stub(Object=pretend.call_recorder(lambda path: s3key))
+        storage = S3FileStorage(bucket)
+
+        metadata = storage.get_metadata("file.txt")
+
+        assert metadata == {"foo": "bar", "wu": "tang"}
+        assert bucket.Object.calls == [pretend.call("file.txt")]
+
     def test_raises_when_key_non_existent(self):
         def raiser():
             raise botocore.exceptions.ClientError(
@@ -249,6 +415,18 @@ def raiser():
 
         assert bucket.Object.calls == [pretend.call("file.txt")]
 
+    def test_get_metadata_raises_when_key_non_existent(self):
+        def raiser(*a, **kw):
+            raise botocore.exceptions.ClientError(
+                {"Error": {"Code": "NoSuchKey", "Message": "No Key!"}}, "some operation"
+            )
+
+        bucket = pretend.stub(Object=raiser)
+        storage = S3FileStorage(bucket)
+
+        with pytest.raises(FileNotFoundError):
+            storage.get_metadata("file.txt")
+
     def test_passes_up_error_when_not_no_such_key(self):
         def raiser():
             raise botocore.exceptions.ClientError(
@@ -263,6 +441,19 @@ def raiser():
         with pytest.raises(botocore.exceptions.ClientError):
             storage.get("file.txt")
 
+    def test_get_metadata_passes_up_error_when_not_no_such_key(self):
+        def raiser(*a, **kw):
+            raise botocore.exceptions.ClientError(
+                {"Error": {"Code": "SomeOtherError", "Message": "Who Knows!"}},
+                "some operation",
+            )
+
+        bucket = pretend.stub(Object=raiser)
+        storage = S3FileStorage(bucket)
+
+        with pytest.raises(botocore.exceptions.ClientError):
+            storage.get_metadata("file.txt")
+
     def test_stores_file(self, tmpdir):
         filename = str(tmpdir.join("testfile.txt"))
         with open(filename, "wb") as fp:
@@ -337,6 +528,24 @@ def test_hashed_path_without_prefix(self):
         assert bucket.Object.calls == [pretend.call("ab/file.txt")]
 
 
+class TestS3ArchiveFileStorage:
+    def test_verify_service(self):
+        assert verifyClass(IFileStorage, S3ArchiveFileStorage)
+
+    def test_create_service(self):
+        session = boto3.session.Session(
+            aws_access_key_id="foo", aws_secret_access_key="bar"
+        )
+        request = pretend.stub(
+            find_service=pretend.call_recorder(lambda name: session),
+            registry=pretend.stub(settings={"archive_files.bucket": "froblob"}),
+        )
+        storage = S3ArchiveFileStorage.create_service(None, request)
+
+        assert request.find_service.calls == [pretend.call(name="aws.session")]
+        assert storage.bucket.name == "froblob"
+
+
 class TestGCSFileStorage:
     def test_verify_service(self):
         assert verifyClass(IFileStorage, GCSFileStorage)
@@ -365,6 +574,12 @@ def test_gets_file_raises(self):
         with pytest.raises(NotImplementedError):
             storage.get("file.txt")
 
+    def test_get_metadata_raises(self):
+        storage = GCSFileStorage(pretend.stub())
+
+        with pytest.raises(NotImplementedError):
+            storage.get_metadata("file.txt")
+
     def test_stores_file(self, tmpdir):
         filename = str(tmpdir.join("testfile.txt"))
         with open(filename, "wb") as fp:

tests/unit/packaging/test_tasks.py

@@ -10,6 +10,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import tempfile
+
+from contextlib import contextmanager
 from itertools import product
 
 import pretend
@@ -23,9 +26,11 @@
 from warehouse.accounts.models import WebAuthn
 from warehouse.packaging.models import Description
 from warehouse.packaging.tasks import (
+    check_file_archive_tasks_outstanding,
     compute_2fa_mandate,
     compute_2fa_metrics,
     sync_bigquery_release_files,
+    sync_file_to_archive,
     update_bigquery_release_files,
     update_description_html,
 )
@@ -43,6 +48,55 @@
 )
 
 
+def test_sync_file_to_archive(db_request, monkeypatch):
+    file = FileFactory(archived=False)
+    primary_stub = pretend.stub(
+        get_metadata=pretend.call_recorder(lambda path: {"fizz": "buzz"}),
+        get=pretend.call_recorder(
+            lambda path: pretend.stub(read=lambda: b"my content")
+        ),
+    )
+    archive_stub = pretend.stub(
+        store=pretend.call_recorder(lambda filename, path, meta=None: None)
+    )
+    db_request.find_service = pretend.call_recorder(
+        lambda iface, name=None: {"primary": primary_stub, "archive": archive_stub}[
+            name
+        ]
+    )
+
+    @contextmanager
+    def mock_named_temporary_file():
+        yield pretend.stub(
+            name="/tmp/wutang",
+            write=pretend.call_recorder(lambda bites: None),
+            flush=pretend.call_recorder(lambda: None),
+        )
+
+    monkeypatch.setattr(tempfile, "NamedTemporaryFile", mock_named_temporary_file)
+
+    sync_file_to_archive(db_request, file.id)
+
+    assert file.archived
+
+    assert primary_stub.get_metadata.calls == [pretend.call(file.path)]
+    assert primary_stub.get.calls == [pretend.call(file.path)]
+    assert archive_stub.store.calls == [
+        pretend.call(file.path, "/tmp/wutang", meta={"fizz": "buzz"})
+    ]
+
+
+def test_check_file_archive_tasks_outstanding(db_request, metrics):
+    [FileFactory(archived=True) for _ in range(12)]
+    [FileFactory(archived=False) for _ in range(3)]
+
+    check_file_archive_tasks_outstanding(db_request)
+
+    assert metrics.gauge.calls == [
+        pretend.call("warehouse.packaging.files.not_archived", 3)
+    ]
+
+
 def test_update_description_html(monkeypatch, db_request):
     current_version = "24.0"
     previous_version = "23.0"