From 7d34554de70979d4adb8b4744b4c6a1c2beb5967 Mon Sep 17 00:00:00 2001 From: Sebastian Diaz Date: Wed, 20 Nov 2024 12:39:32 +0100 Subject: [PATCH 01/12] add pseudo workflow to delivery --- cg/cli/deliver/base.py | 12 ++-- cg/cli/deliver/utils.py | 9 +++ cg/constants/constants.py | 7 ++ .../deliver_files_service_factory.py | 64 ++++++------------- 4 files changed, 43 insertions(+), 49 deletions(-) diff --git a/cg/cli/deliver/base.py b/cg/cli/deliver/base.py index b7bb5c4072..c086b5ac3a 100644 --- a/cg/cli/deliver/base.py +++ b/cg/cli/deliver/base.py @@ -6,7 +6,7 @@ import click from cg.apps.tb import TrailblazerAPI -from cg.cli.deliver.utils import deliver_raw_data_for_analyses +from cg.cli.deliver.utils import deliver_raw_data_for_analyses, get_pseudo_workflow from cg.cli.utils import CLICK_CONTEXT_SETTINGS from cg.constants import Workflow from cg.constants.cli_options import DRY_RUN @@ -18,9 +18,7 @@ from cg.services.deliver_files.deliver_files_service.deliver_files_service_factory import ( DeliveryServiceFactory, ) -from cg.services.deliver_files.rsync.service import ( - DeliveryRsyncService, -) +from cg.services.deliver_files.rsync.service import DeliveryRsyncService from cg.store.models import Analysis, Case LOG = logging.getLogger(__name__) @@ -89,9 +87,10 @@ def deliver_case( if not case: LOG.error(f"Could not find case with id {case_id}") return + pseudo_workflow: str = get_pseudo_workflow(case=cases[0]) delivery_service: DeliverFilesService = service_builder.build_delivery_service( delivery_type=delivery_type if delivery_type else case.data_delivery, - workflow=case.data_analysis, + workflow=pseudo_workflow, ) delivery_service.deliver_files_for_case( case=case, delivery_base_path=Path(inbox), dry_run=dry_run @@ -124,9 +123,10 @@ def deliver_ticket( if not cases: LOG.error(f"Could not find case connected to ticket {ticket}") return + pseudo_workflow: str = get_pseudo_workflow(case=cases[0]) delivery_service: DeliverFilesService = service_builder.build_delivery_service( delivery_type=delivery_type if delivery_type else cases[0].data_delivery, - workflow=cases[0].data_analysis, + workflow=pseudo_workflow, ) delivery_service.deliver_files_for_ticket( ticket_id=ticket, delivery_base_path=Path(inbox), dry_run=dry_run diff --git a/cg/cli/deliver/utils.py b/cg/cli/deliver/utils.py index 215e30d386..e2c649d828 100644 --- a/cg/cli/deliver/utils.py +++ b/cg/cli/deliver/utils.py @@ -3,6 +3,7 @@ from pathlib import Path from cg.constants import Workflow +from cg.constants.constants import MICROBIAL_APP_TAGS from cg.services.deliver_files.deliver_files_service.deliver_files_service import ( DeliverFilesService, ) @@ -43,3 +44,11 @@ def deliver_raw_data_for_analyses( ) LOG.error(f"Could not deliver files for analysis {analysis.id}: {error}") continue + + +def get_pseudo_workflow(case: Case) -> str: + """Return the literal '' if the case is Microbial-fastq, otherwise return the case workflow.""" + tag: str = case.samples[0].application_version.application.tag + if case.data_analysis == Workflow.RAW_DATA and tag in MICROBIAL_APP_TAGS: + return "microbial-fastq" + return case.data_analysis diff --git a/cg/constants/constants.py b/cg/constants/constants.py index 427621b7c7..72859d040d 100644 --- a/cg/constants/constants.py +++ b/cg/constants/constants.py @@ -258,6 +258,13 @@ class MicrosaltAppTags(StrEnum): PREP_CATEGORY: str = "mic" +MICROBIAL_APP_TAGS = [ + MicrosaltAppTags.MWRNXTR003, + MicrosaltAppTags.MWXNXTR003, + MicrosaltAppTags.VWGNXTR001, +] + + class MutantQC: EXTERNAL_NEGATIVE_CONTROL_READS_THRESHOLD: int = 100000 INTERNAL_NEGATIVE_CONTROL_READS_THRESHOLD: int = 2000 diff --git a/cg/services/deliver_files/deliver_files_service/deliver_files_service_factory.py b/cg/services/deliver_files/deliver_files_service/deliver_files_service_factory.py index 9f357abd8c..13d2561975 100644 --- a/cg/services/deliver_files/deliver_files_service/deliver_files_service_factory.py +++ b/cg/services/deliver_files/deliver_files_service/deliver_files_service_factory.py @@ -1,60 +1,38 @@ """Module for the factory of the deliver files service.""" from typing import Type + from cg.apps.housekeeper.hk import HousekeeperAPI from cg.apps.tb import TrailblazerAPI -from cg.constants import Workflow, DataDelivery +from cg.constants import DataDelivery, Workflow from cg.services.analysis_service.analysis_service import AnalysisService -from cg.services.deliver_files.file_filter.sample_service import SampleFileFilter -from cg.services.deliver_files.tag_fetcher.bam_service import ( - BamDeliveryTagsFetcher, -) -from cg.services.fastq_concatenation_service.fastq_concatenation_service import ( - FastqConcatenationService, -) from cg.services.deliver_files.deliver_files_service.deliver_files_service import ( DeliverFilesService, ) from cg.services.deliver_files.deliver_files_service.exc import DeliveryTypeNotSupported -from cg.services.deliver_files.tag_fetcher.abstract import ( - FetchDeliveryFileTagsService, -) -from cg.services.deliver_files.tag_fetcher.sample_and_case_service import ( - SampleAndCaseDeliveryTagsFetcher, -) -from cg.services.deliver_files.file_fetcher.analysis_service import ( - AnalysisDeliveryFileFetcher, -) -from cg.services.deliver_files.file_fetcher.abstract import ( - FetchDeliveryFilesService, -) +from cg.services.deliver_files.file_fetcher.abstract import FetchDeliveryFilesService from cg.services.deliver_files.file_fetcher.analysis_raw_data_service import ( RawDataAndAnalysisDeliveryFileFetcher, ) -from cg.services.deliver_files.file_fetcher.raw_data_service import ( - RawDataDeliveryFileFetcher, -) -from cg.services.deliver_files.file_formatter.service import ( - DeliveryFileFormatter, -) - -from cg.services.deliver_files.file_formatter.abstract import ( - DeliveryFileFormattingService, -) -from cg.services.deliver_files.file_formatter.utils.case_service import ( - CaseFileFormatter, -) +from cg.services.deliver_files.file_fetcher.analysis_service import AnalysisDeliveryFileFetcher +from cg.services.deliver_files.file_fetcher.raw_data_service import RawDataDeliveryFileFetcher +from cg.services.deliver_files.file_filter.sample_service import SampleFileFilter +from cg.services.deliver_files.file_formatter.abstract import DeliveryFileFormattingService +from cg.services.deliver_files.file_formatter.service import DeliveryFileFormatter +from cg.services.deliver_files.file_formatter.utils.case_service import CaseFileFormatter from cg.services.deliver_files.file_formatter.utils.sample_concatenation_service import ( SampleFileConcatenationFormatter, ) -from cg.services.deliver_files.file_formatter.utils.sample_service import ( - SampleFileFormatter, -) -from cg.services.deliver_files.file_mover.service import ( - DeliveryFilesMover, +from cg.services.deliver_files.file_formatter.utils.sample_service import SampleFileFormatter +from cg.services.deliver_files.file_mover.service import DeliveryFilesMover +from cg.services.deliver_files.rsync.service import DeliveryRsyncService +from cg.services.deliver_files.tag_fetcher.abstract import FetchDeliveryFileTagsService +from cg.services.deliver_files.tag_fetcher.bam_service import BamDeliveryTagsFetcher +from cg.services.deliver_files.tag_fetcher.sample_and_case_service import ( + SampleAndCaseDeliveryTagsFetcher, ) -from cg.services.deliver_files.rsync.service import ( - DeliveryRsyncService, +from cg.services.fastq_concatenation_service.fastq_concatenation_service import ( + FastqConcatenationService, ) from cg.store.store import Store @@ -104,10 +82,10 @@ def _get_file_fetcher(self, delivery_type: DataDelivery) -> FetchDeliveryFilesSe @staticmethod def _get_sample_file_formatter( - workflow: Workflow, + workflow: str, ) -> SampleFileFormatter | SampleFileConcatenationFormatter: """Get the file formatter service based on the workflow.""" - if workflow in [Workflow.MICROSALT]: + if workflow in [Workflow.MICROSALT, "microbial-fastq"]: return SampleFileConcatenationFormatter(FastqConcatenationService()) return SampleFileFormatter() @@ -126,7 +104,7 @@ def _validate_delivery_type(delivery_type: DataDelivery): ) def build_delivery_service( - self, workflow: Workflow, delivery_type: DataDelivery + self, workflow: str, delivery_type: DataDelivery ) -> DeliverFilesService: """Build a delivery service based on the workflow and delivery type.""" delivery_type: DataDelivery = self._sanitise_delivery_type(delivery_type) From 5027303c7cb43c523db3c75a384cc1b62402fd3c Mon Sep 17 00:00:00 2001 From: Sebastian Diaz Date: Thu, 21 Nov 2024 10:06:55 +0100 Subject: [PATCH 02/12] refactor service factory to receive case --- cg/cli/deliver/base.py | 22 ++++++++-------- cg/cli/deliver/utils.py | 2 +- cg/meta/upload/upload_api.py | 2 +- .../deliver_files_service_factory.py | 26 ++++++++++++++++--- 4 files changed, 35 insertions(+), 17 deletions(-) diff --git a/cg/cli/deliver/base.py b/cg/cli/deliver/base.py index c086b5ac3a..5cdbda5618 100644 --- a/cg/cli/deliver/base.py +++ b/cg/cli/deliver/base.py @@ -87,10 +87,9 @@ def deliver_case( if not case: LOG.error(f"Could not find case with id {case_id}") return - pseudo_workflow: str = get_pseudo_workflow(case=cases[0]) delivery_service: DeliverFilesService = service_builder.build_delivery_service( - delivery_type=delivery_type if delivery_type else case.data_delivery, - workflow=pseudo_workflow, + case=case, + delivery_type=delivery_type, ) delivery_service.deliver_files_for_case( case=case, delivery_base_path=Path(inbox), dry_run=dry_run @@ -99,8 +98,8 @@ def deliver_case( @deliver.command( name="ticket", - help="Deliver all case files for cases in a ticket based on delivery type to the customer inbox on the HPC " - "and start an Rsync job to clinical-delivery. " + help="Deliver all case files for cases in a ticket based on delivery type to the customer" + "inbox on the HPC and start an Rsync job to clinical-delivery. " "NOTE: the dry-run flag will copy files to the customer inbox on Hasta, " "but will not perform the Rsync job.", ) @@ -115,7 +114,8 @@ def deliver_ticket( dry_run: bool, ): """ - Deliver all case files based on delivery type to the customer inbox on the HPC for cases connected to a ticket. + Deliver all case files based on delivery type to the customer inbox on the HPC for cases + connected to a ticket. """ inbox: str = context.delivery_path service_builder: DeliveryServiceFactory = context.delivery_service_factory @@ -123,10 +123,9 @@ def deliver_ticket( if not cases: LOG.error(f"Could not find case connected to ticket {ticket}") return - pseudo_workflow: str = get_pseudo_workflow(case=cases[0]) delivery_service: DeliverFilesService = service_builder.build_delivery_service( - delivery_type=delivery_type if delivery_type else cases[0].data_delivery, - workflow=pseudo_workflow, + case=cases[0], + delivery_type=delivery_type, ) delivery_service.deliver_files_for_ticket( ticket_id=ticket, delivery_base_path=Path(inbox), dry_run=dry_run @@ -173,8 +172,8 @@ def deliver_sample_raw_data( LOG.error(f"Could not find case with id {case_id}") return delivery_service: DeliverFilesService = service_builder.build_delivery_service( + case=case, delivery_type=delivery_type, - workflow=case.data_analysis, ) delivery_service.deliver_files_for_sample( case=case, sample_id=sample_id, delivery_base_path=Path(inbox), dry_run=dry_run @@ -186,7 +185,8 @@ def deliver_sample_raw_data( @DRY_RUN def deliver_auto_raw_data(context: CGConfig, dry_run: bool): """ - Deliver all case files for the raw data workflow to the customer inbox on the HPC and start a Rsync job. + Deliver all case files for the raw data workflow to the customer inbox on the HPC and start a + Rsync job. 1. get all cases with analysis type fastq that need to be delivered 2. check if their upload has started 3. if not, start the upload diff --git a/cg/cli/deliver/utils.py b/cg/cli/deliver/utils.py index e2c649d828..4fda7202b6 100644 --- a/cg/cli/deliver/utils.py +++ b/cg/cli/deliver/utils.py @@ -28,8 +28,8 @@ def deliver_raw_data_for_analyses( try: case: Case = analysis.case delivery_service: DeliverFilesService = service_builder.build_delivery_service( + case=case, delivery_type=case.data_delivery, - workflow=Workflow.RAW_DATA, ) delivery_service.deliver_files_for_case( diff --git a/cg/meta/upload/upload_api.py b/cg/meta/upload/upload_api.py index 3e047e8449..4593395ad1 100644 --- a/cg/meta/upload/upload_api.py +++ b/cg/meta/upload/upload_api.py @@ -97,8 +97,8 @@ def upload_files_to_customer_inbox(self, case: Case) -> None: """Uploads the analysis files to the customer inbox.""" factory_service: DeliveryServiceFactory = self.config.delivery_service_factory delivery_service: DeliverFilesService = factory_service.build_delivery_service( + case=case, delivery_type=case.data_delivery, - workflow=case.data_analysis, ) delivery_service.deliver_files_for_case( case=case, delivery_base_path=Path(self.config.delivery_path) diff --git a/cg/services/deliver_files/deliver_files_service/deliver_files_service_factory.py b/cg/services/deliver_files/deliver_files_service/deliver_files_service_factory.py index 13d2561975..15fc9f6d49 100644 --- a/cg/services/deliver_files/deliver_files_service/deliver_files_service_factory.py +++ b/cg/services/deliver_files/deliver_files_service/deliver_files_service_factory.py @@ -5,6 +5,7 @@ from cg.apps.housekeeper.hk import HousekeeperAPI from cg.apps.tb import TrailblazerAPI from cg.constants import DataDelivery, Workflow +from cg.constants.constants import MICROBIAL_APP_TAGS from cg.services.analysis_service.analysis_service import AnalysisService from cg.services.deliver_files.deliver_files_service.deliver_files_service import ( DeliverFilesService, @@ -34,6 +35,7 @@ from cg.services.fastq_concatenation_service.fastq_concatenation_service import ( FastqConcatenationService, ) +from cg.store.models import Case from cg.store.store import Store @@ -100,18 +102,23 @@ def _validate_delivery_type(delivery_type: DataDelivery): ]: return raise DeliveryTypeNotSupported( - f"Delivery type {delivery_type} is not supported. Supported delivery types are {DataDelivery.FASTQ}, {DataDelivery.ANALYSIS_FILES}, {DataDelivery.FASTQ_ANALYSIS}, {DataDelivery.BAM}." + f"Delivery type {delivery_type} is not supported. Supported delivery types are" + f" {DataDelivery.FASTQ}, {DataDelivery.ANALYSIS_FILES}," + f" {DataDelivery.FASTQ_ANALYSIS}, {DataDelivery.BAM}." ) def build_delivery_service( - self, workflow: str, delivery_type: DataDelivery + self, case: Case, delivery_type: DataDelivery ) -> DeliverFilesService: """Build a delivery service based on the workflow and delivery type.""" - delivery_type: DataDelivery = self._sanitise_delivery_type(delivery_type) + delivery_type: DataDelivery = self._sanitise_delivery_type( + delivery_type if delivery_type else case.data_delivery + ) self._validate_delivery_type(delivery_type) file_fetcher: FetchDeliveryFilesService = self._get_file_fetcher(delivery_type) + converted_workflow: Workflow = self._convert_workflow(case) sample_file_formatter: SampleFileFormatter | SampleFileConcatenationFormatter = ( - self._get_sample_file_formatter(workflow) + self._get_sample_file_formatter(converted_workflow) ) file_formatter: DeliveryFileFormattingService = DeliveryFileFormatter( case_file_formatter=CaseFileFormatter(), sample_file_formatter=sample_file_formatter @@ -127,6 +134,17 @@ def build_delivery_service( analysis_service=self.analysis_service, ) + @staticmethod + def _convert_workflow(case: Case) -> Workflow: + """Converts a workflow with the introduction of the microbial-fastq delivery type an + unsupported combination of delivery type and workflow setup is required. This function + makes sure that a raw data workflow with microbial fastq delivery type is treated as a + microsalt workflow so that the microbial-fastq sample files can be concatenated.""" + tag: str = case.samples[0].application_version.application.tag + if case.data_analysis == Workflow.RAW_DATA and tag in MICROBIAL_APP_TAGS: + return Workflow.MICROSALT + return case.data_analysis + @staticmethod def _sanitise_delivery_type(delivery_type: DataDelivery) -> DataDelivery: """Sanitise the delivery type.""" From 1b9521c5c267734c0c6f74e4262042b1e397f871 Mon Sep 17 00:00:00 2001 From: Sebastian Diaz Date: Thu, 21 Nov 2024 10:18:53 +0100 Subject: [PATCH 03/12] remove pseudo-workflow function --- cg/cli/deliver/base.py | 2 +- cg/cli/deliver/utils.py | 8 -------- 2 files changed, 1 insertion(+), 9 deletions(-) diff --git a/cg/cli/deliver/base.py b/cg/cli/deliver/base.py index 5cdbda5618..39ba54e19e 100644 --- a/cg/cli/deliver/base.py +++ b/cg/cli/deliver/base.py @@ -6,7 +6,7 @@ import click from cg.apps.tb import TrailblazerAPI -from cg.cli.deliver.utils import deliver_raw_data_for_analyses, get_pseudo_workflow +from cg.cli.deliver.utils import deliver_raw_data_for_analyses from cg.cli.utils import CLICK_CONTEXT_SETTINGS from cg.constants import Workflow from cg.constants.cli_options import DRY_RUN diff --git a/cg/cli/deliver/utils.py b/cg/cli/deliver/utils.py index 4fda7202b6..bb1de9ea39 100644 --- a/cg/cli/deliver/utils.py +++ b/cg/cli/deliver/utils.py @@ -44,11 +44,3 @@ def deliver_raw_data_for_analyses( ) LOG.error(f"Could not deliver files for analysis {analysis.id}: {error}") continue - - -def get_pseudo_workflow(case: Case) -> str: - """Return the literal '' if the case is Microbial-fastq, otherwise return the case workflow.""" - tag: str = case.samples[0].application_version.application.tag - if case.data_analysis == Workflow.RAW_DATA and tag in MICROBIAL_APP_TAGS: - return "microbial-fastq" - return case.data_analysis From 5665cae5cfa405e86d81245571dac5ba9457b41e Mon Sep 17 00:00:00 2001 From: Sebastian Diaz Date: Thu, 21 Nov 2024 10:20:48 +0100 Subject: [PATCH 04/12] implemented suggestion by CO --- .../deliver_files_service/deliver_files_service_factory.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cg/services/deliver_files/deliver_files_service/deliver_files_service_factory.py b/cg/services/deliver_files/deliver_files_service/deliver_files_service_factory.py index 15fc9f6d49..65cd467f53 100644 --- a/cg/services/deliver_files/deliver_files_service/deliver_files_service_factory.py +++ b/cg/services/deliver_files/deliver_files_service/deliver_files_service_factory.py @@ -87,7 +87,7 @@ def _get_sample_file_formatter( workflow: str, ) -> SampleFileFormatter | SampleFileConcatenationFormatter: """Get the file formatter service based on the workflow.""" - if workflow in [Workflow.MICROSALT, "microbial-fastq"]: + if workflow in [Workflow.MICROSALT]: return SampleFileConcatenationFormatter(FastqConcatenationService()) return SampleFileFormatter() From c0dcbb6335b02122865d9e759943f8de5f8693b0 Mon Sep 17 00:00:00 2001 From: Sebastian Diaz Date: Thu, 21 Nov 2024 11:16:55 +0100 Subject: [PATCH 05/12] fix tests --- cg/cli/deliver/utils.py | 2 - .../deliver_files_service_factory.py | 4 +- .../test_service_builder.py | 117 ++++++++++-------- 3 files changed, 66 insertions(+), 57 deletions(-) diff --git a/cg/cli/deliver/utils.py b/cg/cli/deliver/utils.py index bb1de9ea39..14e8255c51 100644 --- a/cg/cli/deliver/utils.py +++ b/cg/cli/deliver/utils.py @@ -2,8 +2,6 @@ from datetime import datetime from pathlib import Path -from cg.constants import Workflow -from cg.constants.constants import MICROBIAL_APP_TAGS from cg.services.deliver_files.deliver_files_service.deliver_files_service import ( DeliverFilesService, ) diff --git a/cg/services/deliver_files/deliver_files_service/deliver_files_service_factory.py b/cg/services/deliver_files/deliver_files_service/deliver_files_service_factory.py index 65cd467f53..47abde387f 100644 --- a/cg/services/deliver_files/deliver_files_service/deliver_files_service_factory.py +++ b/cg/services/deliver_files/deliver_files_service/deliver_files_service_factory.py @@ -108,9 +108,9 @@ def _validate_delivery_type(delivery_type: DataDelivery): ) def build_delivery_service( - self, case: Case, delivery_type: DataDelivery + self, case: Case, delivery_type: DataDelivery | None = None ) -> DeliverFilesService: - """Build a delivery service based on the workflow and delivery type.""" + """Build a delivery service based on a case.""" delivery_type: DataDelivery = self._sanitise_delivery_type( delivery_type if delivery_type else case.data_delivery ) diff --git a/tests/services/file_delivery/delivery_file_service/test_service_builder.py b/tests/services/file_delivery/delivery_file_service/test_service_builder.py index bc945ed581..d2c6798ec7 100644 --- a/tests/services/file_delivery/delivery_file_service/test_service_builder.py +++ b/tests/services/file_delivery/delivery_file_service/test_service_builder.py @@ -1,73 +1,77 @@ -import pytest from unittest.mock import MagicMock -from cg.constants import Workflow, DataDelivery +import pytest +from pydantic.v1 import BaseModel + +from cg.constants import DataDelivery, Workflow from cg.services.deliver_files.deliver_files_service.deliver_files_service import ( DeliverFilesService, ) from cg.services.deliver_files.deliver_files_service.deliver_files_service_factory import ( DeliveryServiceFactory, ) -from cg.services.deliver_files.tag_fetcher.sample_and_case_service import ( - SampleAndCaseDeliveryTagsFetcher, -) -from cg.services.deliver_files.file_fetcher.analysis_service import ( - AnalysisDeliveryFileFetcher, -) +from cg.services.deliver_files.file_fetcher.abstract import FetchDeliveryFilesService from cg.services.deliver_files.file_fetcher.analysis_raw_data_service import ( RawDataAndAnalysisDeliveryFileFetcher, ) -from cg.services.deliver_files.file_fetcher.raw_data_service import ( - RawDataDeliveryFileFetcher, -) +from cg.services.deliver_files.file_fetcher.analysis_service import AnalysisDeliveryFileFetcher +from cg.services.deliver_files.file_fetcher.raw_data_service import RawDataDeliveryFileFetcher from cg.services.deliver_files.file_formatter.utils.sample_concatenation_service import ( SampleFileConcatenationFormatter, ) -from cg.services.deliver_files.file_formatter.utils.sample_service import ( - SampleFileFormatter, -) -from cg.services.deliver_files.file_mover.service import ( - DeliveryFilesMover, +from cg.services.deliver_files.file_formatter.utils.sample_service import SampleFileFormatter +from cg.services.deliver_files.file_mover.service import DeliveryFilesMover +from cg.services.deliver_files.tag_fetcher.abstract import FetchDeliveryFileTagsService +from cg.services.deliver_files.tag_fetcher.sample_and_case_service import ( + SampleAndCaseDeliveryTagsFetcher, ) +from cg.store.store import Store + + +class DeliveryScenario(BaseModel): + app_tag: str + data_analysis: Workflow + delivery_type: DataDelivery + expected_tag_fetcher: type[FetchDeliveryFileTagsService] + expected_file_fetcher: type[FetchDeliveryFilesService] + expected_file_mover: type[DeliveryFilesMover] + expected_sample_file_formatter: type[SampleFileFormatter] @pytest.mark.parametrize( - "workflow,delivery_type,expected_tag_fetcher,expected_file_fetcher,expected_file_mover,expected_sample_file_formatter", + "scenario", [ - ( - Workflow.MICROSALT, - DataDelivery.FASTQ, - SampleAndCaseDeliveryTagsFetcher, - RawDataDeliveryFileFetcher, - DeliveryFilesMover, - SampleFileConcatenationFormatter, + DeliveryScenario( + app_tag="MWRNXTR003", + data_analysis=Workflow.RAW_DATA, + delivery_type=DataDelivery.FASTQ, + expected_tag_fetcher=SampleAndCaseDeliveryTagsFetcher, + expected_file_fetcher=RawDataDeliveryFileFetcher, + expected_file_mover=DeliveryFilesMover, + expected_sample_file_formatter=SampleFileConcatenationFormatter, ), - ( - Workflow.MUTANT, - DataDelivery.ANALYSIS_FILES, - SampleAndCaseDeliveryTagsFetcher, - AnalysisDeliveryFileFetcher, - DeliveryFilesMover, - SampleFileFormatter, + DeliveryScenario( + app_tag="VWGDPTR001", + data_analysis=Workflow.MUTANT, + delivery_type=DataDelivery.ANALYSIS_FILES, + expected_tag_fetcher=SampleAndCaseDeliveryTagsFetcher, + expected_file_fetcher=AnalysisDeliveryFileFetcher, + expected_file_mover=DeliveryFilesMover, + expected_sample_file_formatter=SampleFileFormatter, ), - ( - Workflow.BALSAMIC, - DataDelivery.FASTQ_ANALYSIS, - SampleAndCaseDeliveryTagsFetcher, - RawDataAndAnalysisDeliveryFileFetcher, - DeliveryFilesMover, - SampleFileFormatter, + DeliveryScenario( + app_tag="PANKTTR020", + data_analysis=Workflow.BALSAMIC, + delivery_type=DataDelivery.FASTQ_ANALYSIS, + expected_tag_fetcher=SampleAndCaseDeliveryTagsFetcher, + expected_file_fetcher=RawDataAndAnalysisDeliveryFileFetcher, + expected_file_mover=DeliveryFilesMover, + expected_sample_file_formatter=SampleFileFormatter, ), ], + ids=["microbial-fastq", "SARS-COV2", "Targeted"], ) -def test_build_delivery_service( - workflow, - delivery_type, - expected_tag_fetcher, - expected_file_fetcher, - expected_file_mover, - expected_sample_file_formatter, -): +def test_build_delivery_service(scenario: DeliveryScenario, store: Store): # GIVEN a delivery service builder with mocked store and hk_api builder = DeliveryServiceFactory( store=MagicMock(), @@ -77,15 +81,22 @@ def test_build_delivery_service( analysis_service=MagicMock(), ) + # GIVEN a case with mocked sample app tag + case_mock = MagicMock() + case_mock.data_delivery = scenario.delivery_type + case_mock.data_analysis = scenario.data_analysis + case_mock.samples = [ + MagicMock(application_version=MagicMock(application=MagicMock(tag=scenario.app_tag))) + ] + # WHEN building a delivery service - delivery_service: DeliverFilesService = builder.build_delivery_service( - workflow=workflow, delivery_type=delivery_type - ) + delivery_service: DeliverFilesService = builder.build_delivery_service(case=case_mock) # THEN the correct file formatter and file fetcher services are used - assert isinstance(delivery_service.file_manager.tags_fetcher, expected_tag_fetcher) - assert isinstance(delivery_service.file_manager, expected_file_fetcher) - assert isinstance(delivery_service.file_mover, expected_file_mover) + assert isinstance(delivery_service.file_manager.tags_fetcher, scenario.expected_tag_fetcher) + assert isinstance(delivery_service.file_manager, scenario.expected_file_fetcher) + assert isinstance(delivery_service.file_mover, scenario.expected_file_mover) assert isinstance( - delivery_service.file_formatter.sample_file_formatter, expected_sample_file_formatter + delivery_service.file_formatter.sample_file_formatter, + scenario.expected_sample_file_formatter, ) From a9ad9ed18abadd73b99499a731fcf473ddd468af Mon Sep 17 00:00:00 2001 From: Sebastian Diaz Date: Thu, 21 Nov 2024 11:31:44 +0100 Subject: [PATCH 06/12] fix type hint --- .../deliver_files_service/deliver_files_service_factory.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cg/services/deliver_files/deliver_files_service/deliver_files_service_factory.py b/cg/services/deliver_files/deliver_files_service/deliver_files_service_factory.py index 47abde387f..5a47416531 100644 --- a/cg/services/deliver_files/deliver_files_service/deliver_files_service_factory.py +++ b/cg/services/deliver_files/deliver_files_service/deliver_files_service_factory.py @@ -84,7 +84,7 @@ def _get_file_fetcher(self, delivery_type: DataDelivery) -> FetchDeliveryFilesSe @staticmethod def _get_sample_file_formatter( - workflow: str, + workflow: Workflow, ) -> SampleFileFormatter | SampleFileConcatenationFormatter: """Get the file formatter service based on the workflow.""" if workflow in [Workflow.MICROSALT]: From fb3a93caf508a59e5c6750e67dbe8274c00cf2f1 Mon Sep 17 00:00:00 2001 From: Sebastian Diaz Date: Thu, 21 Nov 2024 11:48:31 +0100 Subject: [PATCH 07/12] remove unused store --- .../file_delivery/delivery_file_service/test_service_builder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/services/file_delivery/delivery_file_service/test_service_builder.py b/tests/services/file_delivery/delivery_file_service/test_service_builder.py index d2c6798ec7..da58b1761c 100644 --- a/tests/services/file_delivery/delivery_file_service/test_service_builder.py +++ b/tests/services/file_delivery/delivery_file_service/test_service_builder.py @@ -71,7 +71,7 @@ class DeliveryScenario(BaseModel): ], ids=["microbial-fastq", "SARS-COV2", "Targeted"], ) -def test_build_delivery_service(scenario: DeliveryScenario, store: Store): +def test_build_delivery_service(scenario: DeliveryScenario): # GIVEN a delivery service builder with mocked store and hk_api builder = DeliveryServiceFactory( store=MagicMock(), From 9246508bac07eb662fc1506740b2bfaa8bb5db33 Mon Sep 17 00:00:00 2001 From: Sebastian Diaz Date: Thu, 21 Nov 2024 13:24:50 +0100 Subject: [PATCH 08/12] remove unused import --- .../delivery_file_service/test_service_builder.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/services/file_delivery/delivery_file_service/test_service_builder.py b/tests/services/file_delivery/delivery_file_service/test_service_builder.py index da58b1761c..cb0da8666f 100644 --- a/tests/services/file_delivery/delivery_file_service/test_service_builder.py +++ b/tests/services/file_delivery/delivery_file_service/test_service_builder.py @@ -1,7 +1,7 @@ from unittest.mock import MagicMock import pytest -from pydantic.v1 import BaseModel +from pydantic import BaseModel from cg.constants import DataDelivery, Workflow from cg.services.deliver_files.deliver_files_service.deliver_files_service import ( @@ -25,7 +25,6 @@ from cg.services.deliver_files.tag_fetcher.sample_and_case_service import ( SampleAndCaseDeliveryTagsFetcher, ) -from cg.store.store import Store class DeliveryScenario(BaseModel): From fc0b4dfd20a12d310213a3d177db5575c2db5333 Mon Sep 17 00:00:00 2001 From: Sebastian Diaz Date: Thu, 21 Nov 2024 16:03:02 +0100 Subject: [PATCH 09/12] implement comments --- .../deliver_files_service_factory.py | 10 +++++----- .../delivery_file_service/test_service_builder.py | 12 +++++++----- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/cg/services/deliver_files/deliver_files_service/deliver_files_service_factory.py b/cg/services/deliver_files/deliver_files_service/deliver_files_service_factory.py index 5a47416531..9346ecd12c 100644 --- a/cg/services/deliver_files/deliver_files_service/deliver_files_service_factory.py +++ b/cg/services/deliver_files/deliver_files_service/deliver_files_service_factory.py @@ -82,12 +82,13 @@ def _get_file_fetcher(self, delivery_type: DataDelivery) -> FetchDeliveryFilesSe tags_fetcher=file_tag_fetcher, ) - @staticmethod def _get_sample_file_formatter( - workflow: Workflow, + self, + case: Case, ) -> SampleFileFormatter | SampleFileConcatenationFormatter: """Get the file formatter service based on the workflow.""" - if workflow in [Workflow.MICROSALT]: + converted_workflow: Workflow = self._convert_workflow(case) + if converted_workflow in [Workflow.MICROSALT]: return SampleFileConcatenationFormatter(FastqConcatenationService()) return SampleFileFormatter() @@ -116,9 +117,8 @@ def build_delivery_service( ) self._validate_delivery_type(delivery_type) file_fetcher: FetchDeliveryFilesService = self._get_file_fetcher(delivery_type) - converted_workflow: Workflow = self._convert_workflow(case) sample_file_formatter: SampleFileFormatter | SampleFileConcatenationFormatter = ( - self._get_sample_file_formatter(converted_workflow) + self._get_sample_file_formatter(case) ) file_formatter: DeliveryFileFormattingService = DeliveryFileFormatter( case_file_formatter=CaseFileFormatter(), sample_file_formatter=sample_file_formatter diff --git a/tests/services/file_delivery/delivery_file_service/test_service_builder.py b/tests/services/file_delivery/delivery_file_service/test_service_builder.py index cb0da8666f..9a49ce0d0f 100644 --- a/tests/services/file_delivery/delivery_file_service/test_service_builder.py +++ b/tests/services/file_delivery/delivery_file_service/test_service_builder.py @@ -27,7 +27,9 @@ ) -class DeliveryScenario(BaseModel): +class DeliveryServiceScenario(BaseModel): + """Model holding the necessary parameters to build a specific delivery service.""" + app_tag: str data_analysis: Workflow delivery_type: DataDelivery @@ -40,7 +42,7 @@ class DeliveryScenario(BaseModel): @pytest.mark.parametrize( "scenario", [ - DeliveryScenario( + DeliveryServiceScenario( app_tag="MWRNXTR003", data_analysis=Workflow.RAW_DATA, delivery_type=DataDelivery.FASTQ, @@ -49,7 +51,7 @@ class DeliveryScenario(BaseModel): expected_file_mover=DeliveryFilesMover, expected_sample_file_formatter=SampleFileConcatenationFormatter, ), - DeliveryScenario( + DeliveryServiceScenario( app_tag="VWGDPTR001", data_analysis=Workflow.MUTANT, delivery_type=DataDelivery.ANALYSIS_FILES, @@ -58,7 +60,7 @@ class DeliveryScenario(BaseModel): expected_file_mover=DeliveryFilesMover, expected_sample_file_formatter=SampleFileFormatter, ), - DeliveryScenario( + DeliveryServiceScenario( app_tag="PANKTTR020", data_analysis=Workflow.BALSAMIC, delivery_type=DataDelivery.FASTQ_ANALYSIS, @@ -70,7 +72,7 @@ class DeliveryScenario(BaseModel): ], ids=["microbial-fastq", "SARS-COV2", "Targeted"], ) -def test_build_delivery_service(scenario: DeliveryScenario): +def test_build_delivery_service(scenario: DeliveryServiceScenario): # GIVEN a delivery service builder with mocked store and hk_api builder = DeliveryServiceFactory( store=MagicMock(), From af476ca1e11e2f02bd61b9b188946c1d4651e3a6 Mon Sep 17 00:00:00 2001 From: Sebastian Diaz Date: Fri, 22 Nov 2024 10:38:54 +0100 Subject: [PATCH 10/12] change constants for fetching app tags from statusdb --- .../deliver_files_service.py | 16 +- .../deliver_files_service_factory.py | 13 +- .../deliver_files/file_formatter/service.py | 24 +-- cg/store/crud/read.py | 13 ++ .../filters/status_application_filters.py | 8 +- tests/conftest.py | 23 +-- tests/fixture_plugins/store_fixtures.py | 140 ++++++++++++++++++ tests/meta/workflow/mutant/conftest.py | 118 +-------------- .../test_service_builder.py | 9 +- tests/store/conftest.py | 47 +----- tests/store/crud/conftest.py | 15 -- tests/store/crud/read/test_read.py | 21 ++- 12 files changed, 210 insertions(+), 237 deletions(-) create mode 100644 tests/fixture_plugins/store_fixtures.py diff --git a/cg/services/deliver_files/deliver_files_service/deliver_files_service.py b/cg/services/deliver_files/deliver_files_service/deliver_files_service.py index 17e3cf6b32..6b45433885 100644 --- a/cg/services/deliver_files/deliver_files_service/deliver_files_service.py +++ b/cg/services/deliver_files/deliver_files_service/deliver_files_service.py @@ -9,21 +9,13 @@ from cg.services.deliver_files.deliver_files_service.error_handling import ( handle_no_delivery_files_error, ) -from cg.services.deliver_files.file_fetcher.abstract import ( - FetchDeliveryFilesService, -) +from cg.services.deliver_files.file_fetcher.abstract import FetchDeliveryFilesService from cg.services.deliver_files.file_fetcher.models import DeliveryFiles from cg.services.deliver_files.file_filter.abstract import FilterDeliveryFilesService -from cg.services.deliver_files.file_formatter.abstract import ( - DeliveryFileFormattingService, -) -from cg.services.deliver_files.file_formatter.models import ( - FormattedFiles, -) +from cg.services.deliver_files.file_formatter.abstract import DeliveryFileFormattingService +from cg.services.deliver_files.file_formatter.models import FormattedFiles from cg.services.deliver_files.file_mover.service import DeliveryFilesMover -from cg.services.deliver_files.rsync.service import ( - DeliveryRsyncService, -) +from cg.services.deliver_files.rsync.service import DeliveryRsyncService from cg.store.exc import EntryNotFoundError from cg.store.models import Case from cg.store.store import Store diff --git a/cg/services/deliver_files/deliver_files_service/deliver_files_service_factory.py b/cg/services/deliver_files/deliver_files_service/deliver_files_service_factory.py index 9346ecd12c..b3c0a3aef9 100644 --- a/cg/services/deliver_files/deliver_files_service/deliver_files_service_factory.py +++ b/cg/services/deliver_files/deliver_files_service/deliver_files_service_factory.py @@ -5,7 +5,7 @@ from cg.apps.housekeeper.hk import HousekeeperAPI from cg.apps.tb import TrailblazerAPI from cg.constants import DataDelivery, Workflow -from cg.constants.constants import MICROBIAL_APP_TAGS +from cg.constants.constants import MICROBIAL_APP_TAGS, PrepCategory from cg.services.analysis_service.analysis_service import AnalysisService from cg.services.deliver_files.deliver_files_service.deliver_files_service import ( DeliverFilesService, @@ -134,14 +134,19 @@ def build_delivery_service( analysis_service=self.analysis_service, ) - @staticmethod - def _convert_workflow(case: Case) -> Workflow: + def _convert_workflow(self, case: Case) -> Workflow: """Converts a workflow with the introduction of the microbial-fastq delivery type an unsupported combination of delivery type and workflow setup is required. This function makes sure that a raw data workflow with microbial fastq delivery type is treated as a microsalt workflow so that the microbial-fastq sample files can be concatenated.""" tag: str = case.samples[0].application_version.application.tag - if case.data_analysis == Workflow.RAW_DATA and tag in MICROBIAL_APP_TAGS: + microbial_tags: list[str] = [ + application.tag + for application in self.store.get_active_applications_by_prep_category( + prep_category=PrepCategory.MICROBIAL + ) + ] + if case.data_analysis == Workflow.RAW_DATA and tag in microbial_tags: return Workflow.MICROSALT return case.data_analysis diff --git a/cg/services/deliver_files/file_formatter/service.py b/cg/services/deliver_files/file_formatter/service.py index 1aa3377c01..2265db4f2e 100644 --- a/cg/services/deliver_files/file_formatter/service.py +++ b/cg/services/deliver_files/file_formatter/service.py @@ -2,28 +2,14 @@ import os from pathlib import Path -from cg.constants.delivery import INBOX_NAME -from cg.services.deliver_files.file_fetcher.models import ( - CaseFile, - DeliveryFiles, - SampleFile, -) -from cg.services.deliver_files.file_formatter.abstract import ( - DeliveryFileFormattingService, -) -from cg.services.deliver_files.file_formatter.models import ( - FormattedFile, - FormattedFiles, -) -from cg.services.deliver_files.file_formatter.utils.case_service import ( - CaseFileFormatter, -) +from cg.services.deliver_files.file_fetcher.models import CaseFile, DeliveryFiles, SampleFile +from cg.services.deliver_files.file_formatter.abstract import DeliveryFileFormattingService +from cg.services.deliver_files.file_formatter.models import FormattedFile, FormattedFiles +from cg.services.deliver_files.file_formatter.utils.case_service import CaseFileFormatter from cg.services.deliver_files.file_formatter.utils.sample_concatenation_service import ( SampleFileConcatenationFormatter, ) -from cg.services.deliver_files.file_formatter.utils.sample_service import ( - SampleFileFormatter, -) +from cg.services.deliver_files.file_formatter.utils.sample_service import SampleFileFormatter LOG = logging.getLogger(__name__) diff --git a/cg/store/crud/read.py b/cg/store/crud/read.py index 29cbbf33f6..6dbdc85ded 100644 --- a/cg/store/crud/read.py +++ b/cg/store/crud/read.py @@ -862,6 +862,19 @@ def get_current_application_version_by_tag(self, tag: str) -> ApplicationVersion valid_from=dt.datetime.now(), ).first() + def get_active_applications_by_prep_category( + self, prep_category: PrepCategory + ) -> list[Application]: + """Return all active applications by prep category.""" + return apply_application_filter( + applications=self._get_query(table=Application), + filter_functions=[ + ApplicationFilter.BY_PREP_CATEGORIES, + ApplicationFilter.IS_NOT_ARCHIVED, + ], + prep_categories=[prep_category], + ).all() + def get_bed_version_by_file_name(self, bed_version_file_name: str) -> BedVersion: """Return bed version with file name.""" return apply_bed_version_filter( diff --git a/cg/store/filters/status_application_filters.py b/cg/store/filters/status_application_filters.py index 870844c45d..727f4c00fd 100644 --- a/cg/store/filters/status_application_filters.py +++ b/cg/store/filters/status_application_filters.py @@ -3,7 +3,7 @@ from sqlalchemy.orm import Query -from cg.constants import PrepCategory, Workflow +from cg.constants import PrepCategory from cg.store.models import Application @@ -42,9 +42,7 @@ def apply_application_filter( filter_functions: list[Callable], applications: Query, tag: str = None, - prep_category: str = None, - prep_categories: list[Workflow] = None, - entry_id: int = None, + prep_categories: list[PrepCategory] = None, ) -> Query: """Apply filtering functions to the sample queries and return filtered results.""" @@ -52,9 +50,7 @@ def apply_application_filter( applications: Query = filter_function( applications=applications, tag=tag, - prep_category=prep_category, prep_categories=prep_categories, - entry_id=entry_id, ) return applications diff --git a/tests/conftest.py b/tests/conftest.py index 66dfa7b90e..8c43041d26 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -30,13 +30,7 @@ from cg.apps.tb.dto.summary_response import AnalysisSummary, StatusSummary from cg.clients.freshdesk.freshdesk_client import FreshdeskClient from cg.constants import FileExtensions, SequencingFileTag, Workflow -from cg.constants.constants import ( - CaseActions, - CustomerId, - FileFormat, - GenomeVersion, - Strandedness, -) +from cg.constants.constants import CaseActions, CustomerId, FileFormat, GenomeVersion, Strandedness from cg.constants.gene_panel import GenePanelMasterList from cg.constants.housekeeper_tags import HK_DELIVERY_REPORT_TAG from cg.constants.priority import SlurmQos @@ -57,22 +51,14 @@ from cg.models import CompressionData from cg.models.cg_config import CGConfig, PDCArchivingDirectory from cg.models.downsample.downsample_data import DownsampleData -from cg.models.raredisease.raredisease import ( - RarediseaseParameters, - RarediseaseSampleSheetHeaders, -) +from cg.models.raredisease.raredisease import RarediseaseParameters, RarediseaseSampleSheetHeaders from cg.models.rnafusion.rnafusion import RnafusionParameters, RnafusionSampleSheetEntry from cg.models.run_devices.illumina_run_directory_data import IlluminaRunDirectoryData -from cg.models.taxprofiler.taxprofiler import ( - TaxprofilerParameters, - TaxprofilerSampleSheetEntry, -) +from cg.models.taxprofiler.taxprofiler import TaxprofilerParameters, TaxprofilerSampleSheetEntry from cg.models.tomte.tomte import TomteParameters, TomteSampleSheetHeaders from cg.services.deliver_files.rsync.service import DeliveryRsyncService from cg.services.illumina.backup.encrypt_service import IlluminaRunEncryptionService -from cg.services.illumina.data_transfer.data_transfer_service import ( - IlluminaDataTransferService, -) +from cg.services.illumina.data_transfer.data_transfer_service import IlluminaDataTransferService from cg.store.database import create_all_tables, drop_all_tables, initialize_database from cg.store.models import ( Application, @@ -146,6 +132,7 @@ "tests.fixture_plugins.pacbio_fixtures.unprocessed_runs_fixtures", "tests.fixture_plugins.quality_controller_fixtures.sequencing_qc_check_scenario", "tests.fixture_plugins.quality_controller_fixtures.sequencing_qc_fixtures", + "tests.fixture_plugins.store_fixtures", "tests.fixture_plugins.timestamp_fixtures", ] diff --git a/tests/fixture_plugins/store_fixtures.py b/tests/fixture_plugins/store_fixtures.py new file mode 100644 index 0000000000..fcfae8cd7c --- /dev/null +++ b/tests/fixture_plugins/store_fixtures.py @@ -0,0 +1,140 @@ +from typing import Generator + +import pytest + +from cg.constants.constants import ControlOptions +from cg.store.store import Store +from tests.store_helpers import StoreHelpers + + +@pytest.fixture +def applications_store(store: Store, helpers: StoreHelpers) -> Store: + """Return a store populated with applications from excel file""" + app_tags: list[str] = ["PGOTTTR020", "PGOTTTR030", "PGOTTTR040"] + for app_tag in app_tags: + helpers.ensure_application(store=store, tag=app_tag) + return store + + +@pytest.fixture +def microbial_store(store: Store, helpers: StoreHelpers) -> Generator[Store, None, None]: + """Populate a store with microbial application tags""" + microbial_active_apptags = ["MWRNXTR003", "MWGNXTR003", "MWMNXTR003", "MWLNXTR003"] + microbial_inactive_apptags = ["MWXNXTR003", "VWGNXTR001", "VWLNXTR001"] + + for app_tag in microbial_active_apptags: + helpers.ensure_application(store=store, tag=app_tag, prep_category="mic", is_archived=False) + + for app_tag in microbial_inactive_apptags: + helpers.ensure_application(store=store, tag=app_tag, prep_category="mic", is_archived=True) + + return store + + +@pytest.fixture +def mutant_store(store: Store, helpers: StoreHelpers) -> Store: + # Add mutant application and application_version + application = helpers.add_application( + store=store, application_tag="VWGDPTR001", target_reads=2000000, percent_reads_guaranteed=1 + ) + + # Add cases + case_qc_pass = helpers.add_case(store=store, name="case_qc_pass", internal_id="case_qc_pass") + case_qc_fail = helpers.add_case(store=store, name="case_qc_fail", internal_id="case_qc_fail") + + case_qc_fail_with_failing_controls = helpers.add_case( + store=store, + name="case_qc_fail_with_failing_controls", + internal_id="case_qc_fail_with_failing_controls", + ) + + # Add samples + sample_qc_pass = helpers.add_sample( + store=store, + internal_id="sample_qc_pass", + name="sample_qc_pass", + control=ControlOptions.EMPTY, + reads=861966, + application_tag=application.tag, + ) + + sample_qc_fail = helpers.add_sample( + store=store, + internal_id="sample_qc_fail", + name="sample_qc_fail", + control=ControlOptions.EMPTY, + reads=438776, + application_tag=application.tag, + ) + + external_negative_control_qc_pass = helpers.add_sample( + store=store, + internal_id="external_negative_control_qc_pass", + name="external_negative_control_qc_pass", + control=ControlOptions.NEGATIVE, + reads=20674, + application_tag=application.tag, + ) + + internal_negative_control_qc_pass = helpers.add_sample( + store=store, + internal_id="internal_negative_control_qc_pass", + name="internal_negative_control_qc_pass", + control=ControlOptions.NEGATIVE, + reads=0, + application_tag=application.tag, + ) + + sample_qc_pass_with_failing_controls = helpers.add_sample( + store=store, + internal_id="sample_qc_pass_with_failing_controls", + name="sample_qc_pass", + control=ControlOptions.EMPTY, + reads=861966, + application_tag=application.tag, + ) + + internal_negative_control_qc_fail = helpers.add_sample( + store=store, + internal_id="internal_negative_control_qc_fail", + name="internal_negative_control_qc_fail", + control=ControlOptions.NEGATIVE, + reads=3000, + application_tag=application.tag, + ) + + external_negative_control_qc_fail = helpers.add_sample( + store=store, + internal_id="external_negative_control_qc_fail", + name="external_negative_control_qc_fail", + control=ControlOptions.NEGATIVE, + reads=200000, + application_tag=application.tag, + ) + + # Add CaseSample relationships + # case_qc_pass + helpers.add_relationship(store=store, case=case_qc_pass, sample=sample_qc_pass) + helpers.add_relationship( + store=store, case=case_qc_pass, sample=external_negative_control_qc_pass + ) + + # case_qc_fail + helpers.add_relationship(store=store, case=case_qc_fail, sample=sample_qc_fail) + helpers.add_relationship( + store=store, case=case_qc_fail, sample=external_negative_control_qc_pass + ) + + # case_qc_fail_with_failing_controls + helpers.add_relationship( + store=store, + case=case_qc_fail_with_failing_controls, + sample=sample_qc_pass_with_failing_controls, + ) + helpers.add_relationship( + store=store, + case=case_qc_fail_with_failing_controls, + sample=external_negative_control_qc_fail, + ) + + return store diff --git a/tests/meta/workflow/mutant/conftest.py b/tests/meta/workflow/mutant/conftest.py index 9ad37700a3..7361cf2a35 100644 --- a/tests/meta/workflow/mutant/conftest.py +++ b/tests/meta/workflow/mutant/conftest.py @@ -1,133 +1,23 @@ -import pytest - from pathlib import Path +import pytest + +from cg.constants.constants import MutantQC from cg.meta.workflow.mutant.quality_controller.metrics_parser_utils import ( _get_validated_results_list, parse_samples_results, ) from cg.meta.workflow.mutant.quality_controller.models import ( - SamplePoolAndResults, ParsedSampleResults, + SamplePoolAndResults, SamplesQualityResults, ) from cg.meta.workflow.mutant.quality_controller.quality_controller import MutantQualityController from cg.store.models import Case, Sample from cg.store.store import Store -from cg.constants.constants import ControlOptions, MutantQC -from tests.store_helpers import StoreHelpers from tests.mocks.limsmock import LimsSample, LimsUDF, MockLimsAPI -@pytest.fixture -def mutant_store(store: Store, helpers: StoreHelpers) -> Store: - # Add mutant application and application_version - application = helpers.add_application( - store=store, application_tag="VWGDPTR001", target_reads=2000000, percent_reads_guaranteed=1 - ) - - # Add cases - case_qc_pass = helpers.add_case(store=store, name="case_qc_pass", internal_id="case_qc_pass") - case_qc_fail = helpers.add_case(store=store, name="case_qc_fail", internal_id="case_qc_fail") - - case_qc_fail_with_failing_controls = helpers.add_case( - store=store, - name="case_qc_fail_with_failing_controls", - internal_id="case_qc_fail_with_failing_controls", - ) - - # Add samples - sample_qc_pass = helpers.add_sample( - store=store, - internal_id="sample_qc_pass", - name="sample_qc_pass", - control=ControlOptions.EMPTY, - reads=861966, - application_tag=application.tag, - ) - - sample_qc_fail = helpers.add_sample( - store=store, - internal_id="sample_qc_fail", - name="sample_qc_fail", - control=ControlOptions.EMPTY, - reads=438776, - application_tag=application.tag, - ) - - external_negative_control_qc_pass = helpers.add_sample( - store=store, - internal_id="external_negative_control_qc_pass", - name="external_negative_control_qc_pass", - control=ControlOptions.NEGATIVE, - reads=20674, - application_tag=application.tag, - ) - - internal_negative_control_qc_pass = helpers.add_sample( - store=store, - internal_id="internal_negative_control_qc_pass", - name="internal_negative_control_qc_pass", - control=ControlOptions.NEGATIVE, - reads=0, - application_tag=application.tag, - ) - - sample_qc_pass_with_failing_controls = helpers.add_sample( - store=store, - internal_id="sample_qc_pass_with_failing_controls", - name="sample_qc_pass", - control=ControlOptions.EMPTY, - reads=861966, - application_tag=application.tag, - ) - - internal_negative_control_qc_fail = helpers.add_sample( - store=store, - internal_id="internal_negative_control_qc_fail", - name="internal_negative_control_qc_fail", - control=ControlOptions.NEGATIVE, - reads=3000, - application_tag=application.tag, - ) - - external_negative_control_qc_fail = helpers.add_sample( - store=store, - internal_id="external_negative_control_qc_fail", - name="external_negative_control_qc_fail", - control=ControlOptions.NEGATIVE, - reads=200000, - application_tag=application.tag, - ) - - # Add CaseSample relationships - # case_qc_pass - helpers.add_relationship(store=store, case=case_qc_pass, sample=sample_qc_pass) - helpers.add_relationship( - store=store, case=case_qc_pass, sample=external_negative_control_qc_pass - ) - - # case_qc_fail - helpers.add_relationship(store=store, case=case_qc_fail, sample=sample_qc_fail) - helpers.add_relationship( - store=store, case=case_qc_fail, sample=external_negative_control_qc_pass - ) - - # case_qc_fail_with_failing_controls - helpers.add_relationship( - store=store, - case=case_qc_fail_with_failing_controls, - sample=sample_qc_pass_with_failing_controls, - ) - helpers.add_relationship( - store=store, - case=case_qc_fail_with_failing_controls, - sample=external_negative_control_qc_fail, - ) - - return store - - @pytest.fixture def mutant_lims(lims_api: MockLimsAPI) -> MockLimsAPI: # Get samples diff --git a/tests/services/file_delivery/delivery_file_service/test_service_builder.py b/tests/services/file_delivery/delivery_file_service/test_service_builder.py index 9a49ce0d0f..3ecba5b6f2 100644 --- a/tests/services/file_delivery/delivery_file_service/test_service_builder.py +++ b/tests/services/file_delivery/delivery_file_service/test_service_builder.py @@ -1,6 +1,7 @@ from unittest.mock import MagicMock import pytest +from _pytest.fixtures import FixtureRequest from pydantic import BaseModel from cg.constants import DataDelivery, Workflow @@ -37,6 +38,7 @@ class DeliveryServiceScenario(BaseModel): expected_file_fetcher: type[FetchDeliveryFilesService] expected_file_mover: type[DeliveryFilesMover] expected_sample_file_formatter: type[SampleFileFormatter] + store_name: str @pytest.mark.parametrize( @@ -50,6 +52,7 @@ class DeliveryServiceScenario(BaseModel): expected_file_fetcher=RawDataDeliveryFileFetcher, expected_file_mover=DeliveryFilesMover, expected_sample_file_formatter=SampleFileConcatenationFormatter, + store_name="microbial_store", ), DeliveryServiceScenario( app_tag="VWGDPTR001", @@ -59,6 +62,7 @@ class DeliveryServiceScenario(BaseModel): expected_file_fetcher=AnalysisDeliveryFileFetcher, expected_file_mover=DeliveryFilesMover, expected_sample_file_formatter=SampleFileFormatter, + store_name="mutant_store", ), DeliveryServiceScenario( app_tag="PANKTTR020", @@ -68,14 +72,15 @@ class DeliveryServiceScenario(BaseModel): expected_file_fetcher=RawDataAndAnalysisDeliveryFileFetcher, expected_file_mover=DeliveryFilesMover, expected_sample_file_formatter=SampleFileFormatter, + store_name="applications_store", ), ], ids=["microbial-fastq", "SARS-COV2", "Targeted"], ) -def test_build_delivery_service(scenario: DeliveryServiceScenario): +def test_build_delivery_service(scenario: DeliveryServiceScenario, request: FixtureRequest): # GIVEN a delivery service builder with mocked store and hk_api builder = DeliveryServiceFactory( - store=MagicMock(), + store=request.getfixturevalue(scenario.store_name), hk_api=MagicMock(), rsync_service=MagicMock(), tb_service=MagicMock(), diff --git a/tests/store/conftest.py b/tests/store/conftest.py index 8ccc0fe4ae..0c6c679dc1 100644 --- a/tests/store/conftest.py +++ b/tests/store/conftest.py @@ -12,9 +12,7 @@ from cg.constants.priority import PriorityTerms from cg.constants.subject import PhenotypeStatus, Sex from cg.services.illumina.data_transfer.models import IlluminaFlowCellDTO -from cg.services.orders.store_order_services.store_pool_order import ( - StorePoolOrderService, -) +from cg.services.orders.store_order_services.store_pool_order import StorePoolOrderService from cg.store.models import ( Analysis, Application, @@ -141,40 +139,6 @@ def _get_item(name: str, internal_id: str, well_position: str, organism: str) -> } -@pytest.fixture(name="microbial_store") -def microbial_store( - base_store: Store, microbial_submitted_order: dict -) -> Generator[Store, None, None]: - """Set up a microbial store instance.""" - customer: Customer = base_store.get_customer_by_internal_id( - customer_internal_id=microbial_submitted_order["customer"] - ) - - for sample_data in microbial_submitted_order["items"]: - application_version = base_store.get_application_by_tag( - sample_data["application"] - ).versions[0] - organism: Organism = Organism( - internal_id=sample_data["organism"], name=sample_data["organism"] - ) - base_store.session.add(organism) - sample = base_store.add_sample( - name=sample_data["name"], - sex=Sex.UNKNOWN, - comment=sample_data["comment"], - priority=sample_data["priority"], - reads=sample_data["reads"], - reference_genome=sample_data["reference_genome"], - ) - sample.application_version = application_version - sample.customer = customer - sample.organism = organism - base_store.session.add(sample) - - base_store.session.commit() - yield base_store - - @pytest.fixture(name="sample") def sample_obj(analysis_store) -> Sample: """Return a sample models object.""" @@ -335,15 +299,6 @@ def store_with_application_limitations( return store_with_an_application_with_and_without_attributes -@pytest.fixture(name="applications_store") -def applications_store(store: Store, helpers: StoreHelpers) -> Store: - """Return a store populated with applications from excel file""" - app_tags: list[str] = ["PGOTTTR020", "PGOTTTR030", "PGOTTTR040"] - for app_tag in app_tags: - helpers.ensure_application(store=store, tag=app_tag) - return store - - @pytest.fixture(name="store_with_different_application_versions") def store_with_different_application_versions( applications_store: Store, diff --git a/tests/store/crud/conftest.py b/tests/store/crud/conftest.py index 38aadb3038..4e7dbc7b23 100644 --- a/tests/store/crud/conftest.py +++ b/tests/store/crud/conftest.py @@ -399,21 +399,6 @@ def expected_number_of_applications() -> int: return 7 -@pytest.fixture -def microbial_store(store: Store, helpers: StoreHelpers) -> Generator[Store, None, None]: - """Populate a store with microbial application tags""" - microbial_active_apptags = ["MWRNXTR003", "MWGNXTR003", "MWMNXTR003", "MWLNXTR003"] - microbial_inactive_apptags = ["MWXNXTR003", "VWGNXTR001", "VWLNXTR001"] - - for app_tag in microbial_active_apptags: - helpers.ensure_application(store=store, tag=app_tag, prep_category="mic", is_archived=False) - - for app_tag in microbial_inactive_apptags: - helpers.ensure_application(store=store, tag=app_tag, prep_category="mic", is_archived=True) - - return store - - @pytest.fixture def max_nr_of_samples() -> int: """Return maximum numbers of samples""" diff --git a/tests/store/crud/read/test_read.py b/tests/store/crud/read/test_read.py index 4804680b54..e69fc6be8e 100644 --- a/tests/store/crud/read/test_read.py +++ b/tests/store/crud/read/test_read.py @@ -5,7 +5,7 @@ from sqlalchemy.orm import Query from cg.constants import SequencingRunDataAvailability -from cg.constants.constants import CaseActions, MicrosaltAppTags, Workflow +from cg.constants.constants import CaseActions, MicrosaltAppTags, PrepCategory, Workflow from cg.constants.subject import PhenotypeStatus from cg.exc import CgError from cg.services.orders.order_service.models import OrderQueryParams @@ -91,6 +91,25 @@ def test_get_active_beds_when_archived(base_store: Store): assert not list(active_beds) +def test_get_active_applications_by_prep_category(microbial_store: Store): + """Test that non-archived and correct applications are returned for the given prep category.""" + # GIVEN a store with applications with a given prep category + prep_category = PrepCategory.MICROBIAL + + # WHEN fetching the active applications for a given prep category + applications: list[Application] = microbial_store.get_active_applications_by_prep_category( + prep_category=prep_category + ) + + # THEN some applications are returned + assert applications + + # THEN the applications should have the given prep category and not be archived + for application in applications: + assert application.prep_category == prep_category + assert not application.is_archived + + def test_get_application_by_tag(microbial_store: Store, tag: str = MicrosaltAppTags.MWRNXTR003): """Test function to return the application by tag.""" From f0d7edaed6b8e88867bbf1932af1fe05448da0cf Mon Sep 17 00:00:00 2001 From: Sebastian Diaz Date: Fri, 22 Nov 2024 10:41:09 +0100 Subject: [PATCH 11/12] delete constant --- cg/constants/constants.py | 7 ------- .../deliver_files_service/deliver_files_service_factory.py | 2 +- 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/cg/constants/constants.py b/cg/constants/constants.py index 72859d040d..427621b7c7 100644 --- a/cg/constants/constants.py +++ b/cg/constants/constants.py @@ -258,13 +258,6 @@ class MicrosaltAppTags(StrEnum): PREP_CATEGORY: str = "mic" -MICROBIAL_APP_TAGS = [ - MicrosaltAppTags.MWRNXTR003, - MicrosaltAppTags.MWXNXTR003, - MicrosaltAppTags.VWGNXTR001, -] - - class MutantQC: EXTERNAL_NEGATIVE_CONTROL_READS_THRESHOLD: int = 100000 INTERNAL_NEGATIVE_CONTROL_READS_THRESHOLD: int = 2000 diff --git a/cg/services/deliver_files/deliver_files_service/deliver_files_service_factory.py b/cg/services/deliver_files/deliver_files_service/deliver_files_service_factory.py index b3c0a3aef9..d685d74d6c 100644 --- a/cg/services/deliver_files/deliver_files_service/deliver_files_service_factory.py +++ b/cg/services/deliver_files/deliver_files_service/deliver_files_service_factory.py @@ -5,7 +5,7 @@ from cg.apps.housekeeper.hk import HousekeeperAPI from cg.apps.tb import TrailblazerAPI from cg.constants import DataDelivery, Workflow -from cg.constants.constants import MICROBIAL_APP_TAGS, PrepCategory +from cg.constants.constants import PrepCategory from cg.services.analysis_service.analysis_service import AnalysisService from cg.services.deliver_files.deliver_files_service.deliver_files_service import ( DeliverFilesService, From 5c8e60d4a9c77373bb408aaa2473477744c6b490 Mon Sep 17 00:00:00 2001 From: Sebastian Diaz Date: Fri, 22 Nov 2024 10:47:52 +0100 Subject: [PATCH 12/12] reorder functions --- .../deliver_files_service_factory.py | 92 +++++++++---------- 1 file changed, 46 insertions(+), 46 deletions(-) diff --git a/cg/services/deliver_files/deliver_files_service/deliver_files_service_factory.py b/cg/services/deliver_files/deliver_files_service/deliver_files_service_factory.py index d685d74d6c..593a677918 100644 --- a/cg/services/deliver_files/deliver_files_service/deliver_files_service_factory.py +++ b/cg/services/deliver_files/deliver_files_service/deliver_files_service_factory.py @@ -56,6 +56,36 @@ def __init__( self.tb_service = tb_service self.analysis_service = analysis_service + @staticmethod + def _sanitise_delivery_type(delivery_type: DataDelivery) -> DataDelivery: + """Sanitise the delivery type.""" + if delivery_type in [DataDelivery.FASTQ_QC, DataDelivery.FASTQ_SCOUT]: + return DataDelivery.FASTQ + if delivery_type in [DataDelivery.ANALYSIS_SCOUT]: + return DataDelivery.ANALYSIS_FILES + if delivery_type in [ + DataDelivery.FASTQ_ANALYSIS_SCOUT, + DataDelivery.FASTQ_QC_ANALYSIS, + ]: + return DataDelivery.FASTQ_ANALYSIS + return delivery_type + + @staticmethod + def _validate_delivery_type(delivery_type: DataDelivery): + """Check if the delivery type is supported. Raises DeliveryTypeNotSupported error.""" + if delivery_type in [ + DataDelivery.FASTQ, + DataDelivery.ANALYSIS_FILES, + DataDelivery.FASTQ_ANALYSIS, + DataDelivery.BAM, + ]: + return + raise DeliveryTypeNotSupported( + f"Delivery type {delivery_type} is not supported. Supported delivery types are" + f" {DataDelivery.FASTQ}, {DataDelivery.ANALYSIS_FILES}," + f" {DataDelivery.FASTQ_ANALYSIS}, {DataDelivery.BAM}." + ) + @staticmethod def _get_file_tag_fetcher(delivery_type: DataDelivery) -> FetchDeliveryFileTagsService: """Get the file tag fetcher based on the delivery type.""" @@ -82,6 +112,22 @@ def _get_file_fetcher(self, delivery_type: DataDelivery) -> FetchDeliveryFilesSe tags_fetcher=file_tag_fetcher, ) + def _convert_workflow(self, case: Case) -> Workflow: + """Converts a workflow with the introduction of the microbial-fastq delivery type an + unsupported combination of delivery type and workflow setup is required. This function + makes sure that a raw data workflow with microbial fastq delivery type is treated as a + microsalt workflow so that the microbial-fastq sample files can be concatenated.""" + tag: str = case.samples[0].application_version.application.tag + microbial_tags: list[str] = [ + application.tag + for application in self.store.get_active_applications_by_prep_category( + prep_category=PrepCategory.MICROBIAL + ) + ] + if case.data_analysis == Workflow.RAW_DATA and tag in microbial_tags: + return Workflow.MICROSALT + return case.data_analysis + def _get_sample_file_formatter( self, case: Case, @@ -92,22 +138,6 @@ def _get_sample_file_formatter( return SampleFileConcatenationFormatter(FastqConcatenationService()) return SampleFileFormatter() - @staticmethod - def _validate_delivery_type(delivery_type: DataDelivery): - """Check if the delivery type is supported. Raises DeliveryTypeNotSupported error.""" - if delivery_type in [ - DataDelivery.FASTQ, - DataDelivery.ANALYSIS_FILES, - DataDelivery.FASTQ_ANALYSIS, - DataDelivery.BAM, - ]: - return - raise DeliveryTypeNotSupported( - f"Delivery type {delivery_type} is not supported. Supported delivery types are" - f" {DataDelivery.FASTQ}, {DataDelivery.ANALYSIS_FILES}," - f" {DataDelivery.FASTQ_ANALYSIS}, {DataDelivery.BAM}." - ) - def build_delivery_service( self, case: Case, delivery_type: DataDelivery | None = None ) -> DeliverFilesService: @@ -133,33 +163,3 @@ def build_delivery_service( tb_service=self.tb_service, analysis_service=self.analysis_service, ) - - def _convert_workflow(self, case: Case) -> Workflow: - """Converts a workflow with the introduction of the microbial-fastq delivery type an - unsupported combination of delivery type and workflow setup is required. This function - makes sure that a raw data workflow with microbial fastq delivery type is treated as a - microsalt workflow so that the microbial-fastq sample files can be concatenated.""" - tag: str = case.samples[0].application_version.application.tag - microbial_tags: list[str] = [ - application.tag - for application in self.store.get_active_applications_by_prep_category( - prep_category=PrepCategory.MICROBIAL - ) - ] - if case.data_analysis == Workflow.RAW_DATA and tag in microbial_tags: - return Workflow.MICROSALT - return case.data_analysis - - @staticmethod - def _sanitise_delivery_type(delivery_type: DataDelivery) -> DataDelivery: - """Sanitise the delivery type.""" - if delivery_type in [DataDelivery.FASTQ_QC, DataDelivery.FASTQ_SCOUT]: - return DataDelivery.FASTQ - if delivery_type in [DataDelivery.ANALYSIS_SCOUT]: - return DataDelivery.ANALYSIS_FILES - if delivery_type in [ - DataDelivery.FASTQ_ANALYSIS_SCOUT, - DataDelivery.FASTQ_QC_ANALYSIS, - ]: - return DataDelivery.FASTQ_ANALYSIS - return delivery_type