From d85432e1a06bd58a4e9870f5b050ed3dc3a0275c Mon Sep 17 00:00:00 2001 From: Ee Durbin Date: Thu, 11 May 2023 13:33:56 -0400 Subject: [PATCH] ensure metadata and pgp files are archived along with the actual distribution file also resolves https://github.com/pypi/warehouse/issues/13414 --- tests/unit/packaging/test_tasks.py | 60 +++++++++++++++++++++++++++++- warehouse/packaging/tasks.py | 22 ++++++++--- 2 files changed, 75 insertions(+), 7 deletions(-) diff --git a/tests/unit/packaging/test_tasks.py b/tests/unit/packaging/test_tasks.py index 58850369c84a..a3474686ade5 100644 --- a/tests/unit/packaging/test_tasks.py +++ b/tests/unit/packaging/test_tasks.py @@ -84,7 +84,65 @@ def mock_named_temporary_file(): assert primary_stub.get_metadata.calls == [pretend.call(file.path)] assert primary_stub.get.calls == [pretend.call(file.path)] assert archive_stub.store.calls == [ - pretend.call(file.path, "/tmp/wutang", meta={"fizz": "buzz"}) + pretend.call(file.path, "/tmp/wutang", meta={"fizz": "buzz"}), + ] + else: + assert primary_stub.get_metadata.calls == [] + assert primary_stub.get.calls == [] + assert archive_stub.store.calls == [] + + +@pytest.mark.parametrize("archived", [True, False]) +def test_sync_file_to_archive_includes_bonus_files(db_request, monkeypatch, archived): + file = FileFactory( + archived=archived, + has_signature=True, + metadata_file_sha256_digest="deadbeefdeadbeefdeadbeefdeadbeef", + ) + primary_stub = pretend.stub( + get_metadata=pretend.call_recorder(lambda path: {"fizz": "buzz"}), + get=pretend.call_recorder( + lambda path: pretend.stub(read=lambda: b"my content") + ), + ) + archive_stub = pretend.stub( + store=pretend.call_recorder(lambda filename, path, meta=None: None) + ) + db_request.find_service = pretend.call_recorder( + lambda iface, name=None: {"primary": primary_stub, "archive": archive_stub}[ + name + ] + ) + + @contextmanager + def mock_named_temporary_file(): + yield pretend.stub( + name="/tmp/wutang", + write=lambda bites: None, + flush=lambda: None, + ) + + monkeypatch.setattr(tempfile, "NamedTemporaryFile", mock_named_temporary_file) + + sync_file_to_archive(db_request, file.id) + + assert file.archived + + if not archived: + assert primary_stub.get_metadata.calls == [ + pretend.call(file.path), + pretend.call(file.metadata_path), + pretend.call(file.pgp_path), + ] + assert primary_stub.get.calls == [ + pretend.call(file.path), + pretend.call(file.metadata_path), + pretend.call(file.pgp_path), + ] + assert archive_stub.store.calls == [ + pretend.call(file.path, "/tmp/wutang", meta={"fizz": "buzz"}), + pretend.call(file.metadata_path, "/tmp/wutang", meta={"fizz": "buzz"}), + pretend.call(file.pgp_path, "/tmp/wutang", meta={"fizz": "buzz"}), ] else: assert primary_stub.get_metadata.calls == [] diff --git a/warehouse/packaging/tasks.py b/warehouse/packaging/tasks.py index 6951e367b9b9..fc23847d0c5a 100644 --- a/warehouse/packaging/tasks.py +++ b/warehouse/packaging/tasks.py @@ -29,18 +29,28 @@ from warehouse.utils import readme +def _copy_file_to_archive(primary_storage, archive_storage, path): + metadata = primary_storage.get_metadata(path) + file_obj = primary_storage.get(path) + with tempfile.NamedTemporaryFile() as file_for_archive: + file_for_archive.write(file_obj.read()) + file_for_archive.flush() + archive_storage.store(path, file_for_archive.name, meta=metadata) + + @tasks.task(ignore_result=True, acks_late=True) def sync_file_to_archive(request, file_id): file = request.db.get(File, file_id) if not file.archived: primary_storage = request.find_service(IFileStorage, name="primary") archive_storage = request.find_service(IFileStorage, name="archive") - metadata = primary_storage.get_metadata(file.path) - file_obj = primary_storage.get(file.path) - with tempfile.NamedTemporaryFile() as file_for_archive: - file_for_archive.write(file_obj.read()) - file_for_archive.flush() - archive_storage.store(file.path, file_for_archive.name, meta=metadata) + + _copy_file_to_archive(primary_storage, archive_storage, file.path) + if file.metadata_file_sha256_digest is not None: + _copy_file_to_archive(primary_storage, archive_storage, file.metadata_path) + if file.has_signature: + _copy_file_to_archive(primary_storage, archive_storage, file.pgp_path) + file.archived = True