From 0dbd9417e9c12c9a28e9bb59368c2aa7b030b22c Mon Sep 17 00:00:00 2001 From: Piotr Date: Sun, 3 Mar 2024 21:13:20 +0100 Subject: [PATCH] Add --validate option to download (#7) --- src/salesforce_archivist/archivist.py | 6 +++-- src/salesforce_archivist/cli.py | 9 ++++--- test/salesforce/test_download.py | 34 +++++++++++++++++++++++++++ test/test_archivist.py | 22 +++++++++++++++++ 4 files changed, 66 insertions(+), 5 deletions(-) diff --git a/src/salesforce_archivist/archivist.py b/src/salesforce_archivist/archivist.py index 4e376ce..738aaf7 100644 --- a/src/salesforce_archivist/archivist.py +++ b/src/salesforce_archivist/archivist.py @@ -165,7 +165,7 @@ def __init__( self._sf_client = sf_client self._max_workers = max_workers - def download(self) -> None: + def download(self) -> bool: downloaded_content_versions_list = DownloadedContentVersionList(self._data_dir) if downloaded_content_versions_list.data_file_exist(): downloaded_content_versions_list.load_data_from_file() @@ -211,8 +211,9 @@ def download(self) -> None: ), fg=color, ) + return global_stats["errors"] == 0 - def validate(self) -> None: + def validate(self) -> bool: validated_versions_list = ValidatedContentVersionList(self._data_dir) if validated_versions_list.data_file_exist(): validated_versions_list.load_data_from_file() @@ -252,6 +253,7 @@ def validate(self) -> None: ), fg=color, ) + return global_stats["invalid"] == 0 @staticmethod def _print_msg(msg: str, obj_type: str, fg: str | None = None) -> None: diff --git a/src/salesforce_archivist/cli.py b/src/salesforce_archivist/cli.py index ce46747..d15064b 100644 --- a/src/salesforce_archivist/cli.py +++ b/src/salesforce_archivist/cli.py @@ -24,8 +24,9 @@ def cli(ctx: Context) -> None: @cli.command() +@click.option("--validate", is_flag=True, default=False, help="Trigger validation after download.") @click.pass_context -def download(ctx: Context) -> None: +def download(ctx: Context, validate: bool) -> None: config: ArchivistConfig = ctx.obj["config"] sf_client = SalesforceClient( instance_url=config.auth.instance_url, @@ -40,7 +41,8 @@ def download(ctx: Context) -> None: max_api_usage_percent=config.max_api_usage_percent, max_workers=config.max_workers, ) - archivist.download() + if not archivist.download() or validate and not archivist.validate(): + ctx.exit(code=1) @cli.command() @@ -60,7 +62,8 @@ def validate(ctx: Context) -> None: max_api_usage_percent=config.max_api_usage_percent, max_workers=config.max_workers, ) - archivist.validate() + if not archivist.validate(): + ctx.exit(code=1) if __name__ == "__main__": diff --git a/test/salesforce/test_download.py b/test/salesforce/test_download.py index d6b6646..f947154 100644 --- a/test/salesforce/test_download.py +++ b/test/salesforce/test_download.py @@ -246,6 +246,40 @@ def test_content_version_downloader_download_will_gracefully_shutdown(shutdown_m shutdown_mock.assert_has_calls([call(wait=True), call(wait=True, cancel_futures=True)]) +@patch.object(ContentVersionDownloader, "download_content_version_from_sf", side_effect=RuntimeError) +def test_content_version_downloader_download_will_return_download_stats(download_mock): + archivist_obj = ArchivistObject(data_dir="/fake/dir", obj_type="User") + link_list = ContentDocumentLinkList(data_dir=archivist_obj.data_dir) + link = ContentDocumentLink(linked_entity_id="LID", content_document_id="DOC1") + link_list.add_link(doc_link=link) + version_list = ContentVersionList(data_dir=archivist_obj.data_dir) + version_list.add_version( + version=ContentVersion( + id="VID1", + document_id=link.content_document_id, + checksum="c1", + extension="ext1", + title="version1", + version_number=1, + ) + ) + download_content_version_list = DownloadContentVersionList( + document_link_list=link_list, content_version_list=version_list, data_dir=archivist_obj.data_dir + ) + downloaded_version_list = DownloadedContentVersionList(data_dir=archivist_obj.data_dir) + sf_client = Mock() + + downloader = ContentVersionDownloader( + sf_client=sf_client, + downloaded_version_list=downloaded_version_list, + ) + result = downloader.download(download_list=download_content_version_list) + assert isinstance(result, DownloadStats) + assert result.total == 1 + assert result.processed == 1 + assert result.errors == 1 + + @patch("os.path.exists") def test_content_version_downloader_download_content_version_from_sf_will_add_already_downloaded_version_to_list( exist_mock, diff --git a/test/test_archivist.py b/test/test_archivist.py index 8bdbf0a..1d1f728 100644 --- a/test/test_archivist.py +++ b/test/test_archivist.py @@ -248,6 +248,28 @@ def test_archivist_download_will_load_lists_and_call_download_method( assert download_mock.call_count == 2 +@patch.object(Salesforce, "load_content_document_link_list") +@patch.object(Salesforce, "load_content_version_list") +@patch.object(Salesforce, "download_files") +def test_archivist_download_will_return_correct_bool_value( + download_mock, load_version_list_mock, load_doc_link_list_mock +): + stats_error = DownloadStats() + stats_error.initialize(total=1) + stats_error.add_processed(error=True) + stats_ok = DownloadStats() + for stats, expected_return in [(stats_error, False), (stats_ok, True)]: + download_mock.return_value = stats + archivist = Archivist( + data_dir="/fake/dir", + objects=[ + ArchivistObject(data_dir="/fakse/dir", obj_type="User"), + ], + sf_client=MagicMock(), + ) + assert archivist.download() == expected_return + + @patch.object(ValidatedContentVersionList, "data_file_exist", side_effect=[False, True]) @patch.object(ValidatedContentVersionList, "load_data_from_file") def test_archivist_validate_will_load_validated_list_if_possible(load_mock, exist_mock):