From aa690ed10e39fe33da284c745ecb89d46dbaa7a4 Mon Sep 17 00:00:00 2001 From: linglp Date: Tue, 25 Jun 2024 11:46:09 -0400 Subject: [PATCH 001/233] add workflow dispatch --- .github/workflows/scan_repo.yml | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 .github/workflows/scan_repo.yml diff --git a/.github/workflows/scan_repo.yml b/.github/workflows/scan_repo.yml new file mode 100644 index 000000000..d062f3df7 --- /dev/null +++ b/.github/workflows/scan_repo.yml @@ -0,0 +1,32 @@ +# borrowed from mono repo: https://github.com/Sage-Bionetworks/sage-monorepo/blob/main/.github/workflows/scan-repo.yml +name: Scan Git repo +on: + push: + branches: + - develop + pull_request: + workflow_dispatch: + +jobs: + trivy: + name: Trivy + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v3 + + - name: Run Trivy vulnerability scanner in repo mode + uses: aquasecurity/trivy-action@master + with: + scan-type: 'fs' + ignore-unfixed: true + format: 'sarif' + output: 'trivy-results.sarif' + severity: 'CRITICAL,HIGH' + limit-severities-for-sarif: true + + - name: Upload Trivy scan results to GitHub Security tab + uses: github/codeql-action/upload-sarif@v2 + with: + sarif_file: 'trivy-results.sarif' + category: Git Repository \ No newline at end of file From 45933e6064412ac26acacc69ded8c5d4b5f62cae Mon Sep 17 00:00:00 2001 From: linglp Date: Tue, 25 Jun 2024 11:59:30 -0400 Subject: [PATCH 002/233] update to use v3 --- .github/workflows/scan_repo.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/scan_repo.yml b/.github/workflows/scan_repo.yml index d062f3df7..d7b5c36c0 100644 --- a/.github/workflows/scan_repo.yml +++ b/.github/workflows/scan_repo.yml @@ -26,7 +26,7 @@ jobs: limit-severities-for-sarif: true - name: Upload Trivy scan results to GitHub Security tab - uses: github/codeql-action/upload-sarif@v2 + uses: github/codeql-action/upload-sarif@v3 with: sarif_file: 'trivy-results.sarif' category: Git Repository \ No newline at end of file From a5082a1f8a6a578791cd84aa4c1d83a29f91c6c4 Mon Sep 17 00:00:00 2001 From: linglp Date: Tue, 25 Jun 2024 12:06:21 -0400 Subject: [PATCH 003/233] severity set to medium --- .github/workflows/scan_repo.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/scan_repo.yml b/.github/workflows/scan_repo.yml index d7b5c36c0..c28233203 100644 --- a/.github/workflows/scan_repo.yml +++ b/.github/workflows/scan_repo.yml @@ -1,4 +1,4 @@ -# borrowed from mono repo: https://github.com/Sage-Bionetworks/sage-monorepo/blob/main/.github/workflows/scan-repo.yml +# Modified from mono repo: https://github.com/Sage-Bionetworks/sage-monorepo/blob/main/.github/workflows/scan-repo.yml name: Scan Git repo on: push: @@ -22,7 +22,7 @@ jobs: ignore-unfixed: true format: 'sarif' output: 'trivy-results.sarif' - severity: 'CRITICAL,HIGH' + severity: 'CRITICAL,HIGH,MEDIUM' limit-severities-for-sarif: true - name: Upload Trivy scan results to GitHub Security tab From 0e93b5bd8d1ebc5d912e191e540d4aa909654d6b Mon Sep 17 00:00:00 2001 From: linglp Date: Tue, 25 Jun 2024 14:39:17 -0400 Subject: [PATCH 004/233] add comment --- .github/workflows/scan_repo.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/scan_repo.yml b/.github/workflows/scan_repo.yml index c28233203..ffe69c9f2 100644 --- a/.github/workflows/scan_repo.yml +++ b/.github/workflows/scan_repo.yml @@ -1,4 +1,5 @@ # Modified from mono repo: https://github.com/Sage-Bionetworks/sage-monorepo/blob/main/.github/workflows/scan-repo.yml +# Also, reference: https://github.com/aquasecurity/trivy-action?tab=readme-ov-file#using-trivy-to-scan-your-git-repo name: Scan Git repo on: push: @@ -18,7 +19,9 @@ jobs: - name: Run Trivy vulnerability scanner in repo mode uses: aquasecurity/trivy-action@master with: + # the scan targets the file system. scan-type: 'fs' + # it will ignore vulnerabilities without a fix. ignore-unfixed: true format: 'sarif' output: 'trivy-results.sarif' From 2a8881c42911df7df563d5a0830e0f9a44ba9c83 Mon Sep 17 00:00:00 2001 From: linglp Date: Wed, 26 Jun 2024 13:41:51 -0400 Subject: [PATCH 005/233] fix key error --- schematic/store/synapse.py | 92 ++++++++++++++------------------------ tests/test_store.py | 73 +++++++++++++++++++++++++----- 2 files changed, 95 insertions(+), 70 deletions(-) diff --git a/schematic/store/synapse.py b/schematic/store/synapse.py index 71711bbae..bd9bda254 100644 --- a/schematic/store/synapse.py +++ b/schematic/store/synapse.py @@ -1,78 +1,50 @@ """Synapse storage class""" +import asyncio import atexit -from copy import deepcopy -from dataclasses import dataclass import logging -import numpy as np -import pandas as pd import os import re import secrets import shutil -import synapseclient -from synapseclient.api import get_entity_id_bundle2 import uuid # used to generate unique names for entities - -from tenacity import ( - retry, - stop_after_attempt, - wait_chain, - wait_fixed, - retry_if_exception_type, -) +from copy import deepcopy +from dataclasses import asdict, dataclass from time import sleep - # allows specifying explicit variable types -from typing import Dict, List, Tuple, Sequence, Union, Optional, Any, Set - -from synapseclient import ( - Synapse, - File, - Folder, - Table, - Schema, - EntityViewSchema, - EntityViewType, - Column, - as_table_columns, -) -from synapseclient.entity import File -from synapseclient.table import CsvFileTable, build_table, Schema -from synapseclient.core.exceptions import ( - SynapseHTTPError, - SynapseAuthenticationError, - SynapseUnmetAccessRestrictions, - SynapseHTTPError, -) -import synapseutils +from typing import Any, Dict, List, Optional, Sequence, Set, Tuple, Union +import numpy as np +import pandas as pd +import synapseclient +import synapseutils +from opentelemetry import trace from schematic_db.rdb.synapse_database import SynapseDatabase +from synapseclient import (Column, EntityViewSchema, EntityViewType, File, + Folder, Schema, Synapse, Table, as_table_columns) +from synapseclient.api import get_entity_id_bundle2 +from synapseclient.core.exceptions import (SynapseAuthenticationError, + SynapseHTTPError, + SynapseUnmetAccessRestrictions) +from synapseclient.entity import File +from synapseclient.models.annotations import Annotations +from synapseclient.table import CsvFileTable, Schema, build_table +from tenacity import (retry, retry_if_exception_type, stop_after_attempt, + wait_chain, wait_fixed) +from schematic.configuration.configuration import CONFIG +from schematic.exceptions import AccessCredentialsError from schematic.schemas.data_model_graph import DataModelGraphExplorer - -from schematic.utils.df_utils import update_df, load_df, col_in_dataframe -from schematic.utils.validate_utils import comma_separated_list_regex, rule_in_rule_list - +from schematic.store.base import BaseStorage +from schematic.utils.df_utils import col_in_dataframe, load_df, update_df # entity_type_mapping, get_dir_size, create_temp_folder, check_synapse_cache_size, and clear_synapse_cache functions are used for AWS deployment # Please do not remove these import statements -from schematic.utils.general import ( - entity_type_mapping, - get_dir_size, - create_temp_folder, - check_synapse_cache_size, - clear_synapse_cache, -) - +from schematic.utils.general import (check_synapse_cache_size, + clear_synapse_cache, create_temp_folder, + entity_type_mapping, get_dir_size) from schematic.utils.schema_utils import get_class_label_from_display_name - -from schematic.store.base import BaseStorage -from schematic.exceptions import AccessCredentialsError -from schematic.configuration.configuration import CONFIG -from synapseclient.models.annotations import Annotations -import asyncio -from dataclasses import asdict -from opentelemetry import trace +from schematic.utils.validate_utils import (comma_separated_list_regex, + rule_in_rule_list) logger = logging.getLogger("Synapse storage") @@ -1704,10 +1676,12 @@ async def _process_store_annos(self, requests: Set[asyncio.Task]) -> None: if isinstance(annos, Annotations): annos_dict = asdict(annos) - entity_id = annos_dict["id"] + normalized_annos = {k.lower(): v for k, v in annos_dict.items()} + entity_id = normalized_annos["id"] logger.info(f"Successfully stored annotations for {entity_id}") else: - entity_id = annos["EntityId"] + normalized_annos = {k.lower(): v for k, v in annos.items()} + entity_id = normalized_annos["entityid"] logger.info( f"Obtained and processed annotations for {entity_id} entity" ) diff --git a/tests/test_store.py b/tests/test_store.py index cd5f4385b..edf1d3116 100644 --- a/tests/test_store.py +++ b/tests/test_store.py @@ -2,31 +2,31 @@ from __future__ import annotations +import asyncio import logging import math import os -from time import sleep -from typing import Generator, Any -from unittest.mock import patch import shutil -import asyncio +from time import sleep +from typing import Any, Generator +from unittest.mock import AsyncMock, patch import pandas as pd import pytest +from pandas.testing import assert_frame_equal from synapseclient import EntityViewSchema, Folder from synapseclient.entity import File -from pandas.testing import assert_frame_equal +from synapseclient.models import Annotations from schematic.configuration.configuration import Configuration -from schematic.schemas.data_model_graph import DataModelGraph, DataModelGraphExplorer +from schematic.schemas.data_model_graph import (DataModelGraph, + DataModelGraphExplorer) from schematic.schemas.data_model_parser import DataModelParser -from tests.conftest import Helpers - from schematic.store.base import BaseStorage -from schematic.store.synapse import DatasetFileView, ManifestDownload, SynapseStorage +from schematic.store.synapse import (DatasetFileView, ManifestDownload, + SynapseStorage) from schematic.utils.general import check_synapse_cache_size -from unittest.mock import AsyncMock -from synapseclient.models import Annotations +from tests.conftest import Helpers logging.basicConfig(level=logging.DEBUG) logger = logging.getLogger(__name__) @@ -633,6 +633,57 @@ async def mock_success_coro(): await synapse_store._process_store_annos(new_tasks) mock_store_async2.assert_called_once() + async def test_process_store_annos_success_get_entity_id_variants( + self, synapse_store: SynapseStorage + ) -> None: + "mock annotations obtained after gettinng annotations have different annotations and formatting" + annotations_variants = [ + {"EntityId": ["mock_syn_id"], "Id": ["mock_string"]}, + {"entityId": ["mock_syn_id"], "id": ["mock_string"]}, + {"entityid": ["mock_syn_id"], "id": ["mock_string"]}, + {"ENTITYID": ["mock_syn_id"], "ID": ["mock_string"]}, + ] + for anno_variant in annotations_variants: + mock_annos_dict = { + "annotations": { + "id": "mock_syn_id", + "etag": "mock etag", + "annotations": { + "Id": {"type": "STRING", "value": ["mock value"]}, + "EntityId": {"type": "STRING", "value": ["mock_syn_id"]}, + "SampleID": {"type": "STRING", "value": [""]}, + "Component": {"type": "STRING", "value": ["mock value"]}, + }, + }, + "FileFormat": "mock format", + "Component": "mock component", + **anno_variant, + } + mock_stored_annos = Annotations( + annotations={ + **anno_variant, + "SampleID": [""], + "Component": ["mock value"], + "FileFormat": ["mock_format"], + }, + etag="mock etag", + id="mock syn_id", + ) + + async def mock_success_coro(): + return mock_annos_dict + + # make sure that the else statement is working + new_tasks = set() + with patch( + "schematic.store.synapse.SynapseStorage.store_async_annotation", + new_callable=AsyncMock, + return_value=mock_stored_annos, + ) as mock_store_async2: + new_tasks.add(asyncio.create_task(mock_success_coro())) + await synapse_store._process_store_annos(new_tasks) + mock_store_async2.assert_called_once() + class TestDatasetFileView: def test_init(self, dataset_id, dataset_fileview, synapse_store): From 6a505a4693e3eb28fc19ad3f67dd3a3c488b3442 Mon Sep 17 00:00:00 2001 From: linglp Date: Wed, 26 Jun 2024 13:56:33 -0400 Subject: [PATCH 006/233] run black --- schematic/store/synapse.py | 45 ++++++++++++++++++++++++++++---------- tests/test_store.py | 6 ++--- 2 files changed, 35 insertions(+), 16 deletions(-) diff --git a/schematic/store/synapse.py b/schematic/store/synapse.py index bd9bda254..6a1c71a96 100644 --- a/schematic/store/synapse.py +++ b/schematic/store/synapse.py @@ -11,6 +11,7 @@ from copy import deepcopy from dataclasses import asdict, dataclass from time import sleep + # allows specifying explicit variable types from typing import Any, Dict, List, Optional, Sequence, Set, Tuple, Union @@ -20,31 +21,51 @@ import synapseutils from opentelemetry import trace from schematic_db.rdb.synapse_database import SynapseDatabase -from synapseclient import (Column, EntityViewSchema, EntityViewType, File, - Folder, Schema, Synapse, Table, as_table_columns) +from synapseclient import ( + Column, + EntityViewSchema, + EntityViewType, + File, + Folder, + Schema, + Synapse, + Table, + as_table_columns, +) from synapseclient.api import get_entity_id_bundle2 -from synapseclient.core.exceptions import (SynapseAuthenticationError, - SynapseHTTPError, - SynapseUnmetAccessRestrictions) +from synapseclient.core.exceptions import ( + SynapseAuthenticationError, + SynapseHTTPError, + SynapseUnmetAccessRestrictions, +) from synapseclient.entity import File from synapseclient.models.annotations import Annotations from synapseclient.table import CsvFileTable, Schema, build_table -from tenacity import (retry, retry_if_exception_type, stop_after_attempt, - wait_chain, wait_fixed) +from tenacity import ( + retry, + retry_if_exception_type, + stop_after_attempt, + wait_chain, + wait_fixed, +) from schematic.configuration.configuration import CONFIG from schematic.exceptions import AccessCredentialsError from schematic.schemas.data_model_graph import DataModelGraphExplorer from schematic.store.base import BaseStorage from schematic.utils.df_utils import col_in_dataframe, load_df, update_df + # entity_type_mapping, get_dir_size, create_temp_folder, check_synapse_cache_size, and clear_synapse_cache functions are used for AWS deployment # Please do not remove these import statements -from schematic.utils.general import (check_synapse_cache_size, - clear_synapse_cache, create_temp_folder, - entity_type_mapping, get_dir_size) +from schematic.utils.general import ( + check_synapse_cache_size, + clear_synapse_cache, + create_temp_folder, + entity_type_mapping, + get_dir_size, +) from schematic.utils.schema_utils import get_class_label_from_display_name -from schematic.utils.validate_utils import (comma_separated_list_regex, - rule_in_rule_list) +from schematic.utils.validate_utils import comma_separated_list_regex, rule_in_rule_list logger = logging.getLogger("Synapse storage") diff --git a/tests/test_store.py b/tests/test_store.py index edf1d3116..896a9ebb1 100644 --- a/tests/test_store.py +++ b/tests/test_store.py @@ -19,12 +19,10 @@ from synapseclient.models import Annotations from schematic.configuration.configuration import Configuration -from schematic.schemas.data_model_graph import (DataModelGraph, - DataModelGraphExplorer) +from schematic.schemas.data_model_graph import DataModelGraph, DataModelGraphExplorer from schematic.schemas.data_model_parser import DataModelParser from schematic.store.base import BaseStorage -from schematic.store.synapse import (DatasetFileView, ManifestDownload, - SynapseStorage) +from schematic.store.synapse import DatasetFileView, ManifestDownload, SynapseStorage from schematic.utils.general import check_synapse_cache_size from tests.conftest import Helpers From aa4840c15062eff8e7cbd6ecaefa4efb003998f2 Mon Sep 17 00:00:00 2001 From: linglp Date: Wed, 26 Jun 2024 17:39:04 -0400 Subject: [PATCH 007/233] add async decorator --- schematic/store/synapse.py | 31 ++++++++++++++++++++++--------- 1 file changed, 22 insertions(+), 9 deletions(-) diff --git a/schematic/store/synapse.py b/schematic/store/synapse.py index 71711bbae..64a2b08f8 100644 --- a/schematic/store/synapse.py +++ b/schematic/store/synapse.py @@ -337,6 +337,19 @@ def wrapper(*args, **kwargs): return wrapper + def async_missing_entity_handler(method): + async def wrapper(*args, **kwargs): + try: + return await method(*args, **kwargs) + except SynapseHTTPError as ex: + str_message = str(ex).replace("\n", "") + if "trash" in str_message or "does not exist" in str_message: + logging.warning(str_message) + return None + else: + raise ex + return wrapper + def getStorageFileviewTable(self): """Returns the storageFileviewTable obtained during initialization.""" return self.storageFileviewTable @@ -1393,7 +1406,7 @@ async def store_async_annotation(self, annotation_dict: dict) -> Annotations: ) return await annotation_class.store_async(self.syn) - @missing_entity_handler + @async_missing_entity_handler async def format_row_annotations( self, dmge, row, entityId: str, hideBlanks: bool, annotation_keys: str ): @@ -1707,16 +1720,16 @@ async def _process_store_annos(self, requests: Set[asyncio.Task]) -> None: entity_id = annos_dict["id"] logger.info(f"Successfully stored annotations for {entity_id}") else: - entity_id = annos["EntityId"] - logger.info( - f"Obtained and processed annotations for {entity_id} entity" - ) - if annos: + if annos: + entity_id = annos["EntityId"] + logger.info( + f"Obtained and processed annotations for {entity_id} entity" + ) requests.add( - asyncio.create_task( - self.store_async_annotation(annotation_dict=annos) + asyncio.create_task( + self.store_async_annotation(annotation_dict=annos) + ) ) - ) except Exception as e: raise RuntimeError(f"failed with { repr(e) }.") from e From 2d922dddd4be69ac81169851704b65d5ba17482e Mon Sep 17 00:00:00 2001 From: linglp Date: Wed, 26 Jun 2024 17:39:25 -0400 Subject: [PATCH 008/233] add async decorator --- schematic/store/synapse.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/schematic/store/synapse.py b/schematic/store/synapse.py index 64a2b08f8..9f2219d66 100644 --- a/schematic/store/synapse.py +++ b/schematic/store/synapse.py @@ -348,6 +348,7 @@ async def wrapper(*args, **kwargs): return None else: raise ex + return wrapper def getStorageFileviewTable(self): @@ -1720,16 +1721,16 @@ async def _process_store_annos(self, requests: Set[asyncio.Task]) -> None: entity_id = annos_dict["id"] logger.info(f"Successfully stored annotations for {entity_id}") else: - if annos: + if annos: entity_id = annos["EntityId"] logger.info( f"Obtained and processed annotations for {entity_id} entity" ) requests.add( - asyncio.create_task( - self.store_async_annotation(annotation_dict=annos) - ) + asyncio.create_task( + self.store_async_annotation(annotation_dict=annos) ) + ) except Exception as e: raise RuntimeError(f"failed with { repr(e) }.") from e From 6dd85d19597fe78ea56f0874cb7fabebbacd5470 Mon Sep 17 00:00:00 2001 From: linglp Date: Thu, 27 Jun 2024 13:33:55 -0400 Subject: [PATCH 009/233] add typing and test --- schematic/store/synapse.py | 84 +++++++++++++++++++------------------- tests/test_store.py | 38 +++++++++++++++++ 2 files changed, 79 insertions(+), 43 deletions(-) diff --git a/schematic/store/synapse.py b/schematic/store/synapse.py index 9f2219d66..569389b16 100644 --- a/schematic/store/synapse.py +++ b/schematic/store/synapse.py @@ -1,78 +1,71 @@ """Synapse storage class""" +import asyncio import atexit -from copy import deepcopy -from dataclasses import dataclass import logging -import numpy as np -import pandas as pd import os import re import secrets import shutil -import synapseclient -from synapseclient.api import get_entity_id_bundle2 import uuid # used to generate unique names for entities - -from tenacity import ( - retry, - stop_after_attempt, - wait_chain, - wait_fixed, - retry_if_exception_type, -) +from copy import deepcopy +from dataclasses import asdict, dataclass from time import sleep # allows specifying explicit variable types -from typing import Dict, List, Tuple, Sequence, Union, Optional, Any, Set +from typing import Any, Dict, List, Optional, Sequence, Set, Tuple, Union +import numpy as np +import pandas as pd +import synapseclient +import synapseutils +from opentelemetry import trace +from schematic_db.rdb.synapse_database import SynapseDatabase from synapseclient import ( - Synapse, + Column, + EntityViewSchema, + EntityViewType, File, Folder, - Table, Schema, - EntityViewSchema, - EntityViewType, - Column, + Synapse, + Table, as_table_columns, ) -from synapseclient.entity import File -from synapseclient.table import CsvFileTable, build_table, Schema +from synapseclient.api import get_entity_id_bundle2 from synapseclient.core.exceptions import ( - SynapseHTTPError, SynapseAuthenticationError, - SynapseUnmetAccessRestrictions, SynapseHTTPError, + SynapseUnmetAccessRestrictions, +) +from synapseclient.entity import File +from synapseclient.models.annotations import Annotations +from synapseclient.table import CsvFileTable, Schema, build_table +from tenacity import ( + retry, + retry_if_exception_type, + stop_after_attempt, + wait_chain, + wait_fixed, ) -import synapseutils - -from schematic_db.rdb.synapse_database import SynapseDatabase +from schematic.configuration.configuration import CONFIG +from schematic.exceptions import AccessCredentialsError from schematic.schemas.data_model_graph import DataModelGraphExplorer - -from schematic.utils.df_utils import update_df, load_df, col_in_dataframe -from schematic.utils.validate_utils import comma_separated_list_regex, rule_in_rule_list +from schematic.store.base import BaseStorage +from schematic.utils.df_utils import col_in_dataframe, load_df, update_df # entity_type_mapping, get_dir_size, create_temp_folder, check_synapse_cache_size, and clear_synapse_cache functions are used for AWS deployment # Please do not remove these import statements from schematic.utils.general import ( - entity_type_mapping, - get_dir_size, - create_temp_folder, check_synapse_cache_size, clear_synapse_cache, + create_temp_folder, + entity_type_mapping, + get_dir_size, ) - from schematic.utils.schema_utils import get_class_label_from_display_name - -from schematic.store.base import BaseStorage -from schematic.exceptions import AccessCredentialsError -from schematic.configuration.configuration import CONFIG -from synapseclient.models.annotations import Annotations -import asyncio -from dataclasses import asdict -from opentelemetry import trace +from schematic.utils.validate_utils import comma_separated_list_regex, rule_in_rule_list logger = logging.getLogger("Synapse storage") @@ -1409,7 +1402,12 @@ async def store_async_annotation(self, annotation_dict: dict) -> Annotations: @async_missing_entity_handler async def format_row_annotations( - self, dmge, row, entityId: str, hideBlanks: bool, annotation_keys: str + self, + dmge: DataModelGraphExplorer, + row: pd.Series, + entityId: str, + hideBlanks: bool, + annotation_keys: str, ): # prepare metadata for Synapse storage (resolve display name into a name that Synapse annotations support (e.g no spaces, parenthesis) # note: the removal of special characters, will apply only to annotation keys; we are not altering the manifest diff --git a/tests/test_store.py b/tests/test_store.py index cd5f4385b..48d1afb5f 100644 --- a/tests/test_store.py +++ b/tests/test_store.py @@ -15,8 +15,10 @@ import pytest from synapseclient import EntityViewSchema, Folder from synapseclient.entity import File +from synapseclient.core.exceptions import SynapseHTTPError from pandas.testing import assert_frame_equal + from schematic.configuration.configuration import Configuration from schematic.schemas.data_model_graph import DataModelGraph, DataModelGraphExplorer from schematic.schemas.data_model_parser import DataModelParser @@ -451,6 +453,42 @@ def test_add_entity_id_and_filename_without_component_col(self, synapse_store): ) assert_frame_equal(manifest_to_return, expected_df) + @pytest.mark.parametrize( + "hideBlanks, annotation_keys", + [ + (True, "display_label"), + (False, "display_label"), + (True, "class_label"), + (False, "class_label"), + ], + ) + async def test_format_row_annotations_entity_id_trash_can( + self, caplog, dmge, synapse_store, hideBlanks, annotation_keys + ): + """make sure that missing_entity_handler gets triggered when entity is in the trash can""" + with patch( + "schematic.store.synapse.SynapseStorage.get_async_annotation", + side_effect=SynapseHTTPError("entity syn123 is in the trash can"), + new_callable=AsyncMock, + ): + mock_row_dict = { + "Component": "MockComponent", + "Mock_id": 1, + "Id": "Mock_id", + "entityId": "mock_syn_id", + } + mock_row = pd.Series(mock_row_dict) + with caplog.at_level(logging.WARNING): + formatted_annotations = await synapse_store.format_row_annotations( + dmge, + mock_row, + entityId="mock_syn_id", + hideBlanks=hideBlanks, + annotation_keys=annotation_keys, + ) + assert "entity syn123 is in the trash can" in caplog.text + assert formatted_annotations == None + def test_get_files_metadata_from_dataset(self, synapse_store): patch_get_children = [ ("syn123", "parent_folder/test_A.txt"), From a21388401f0bff3d05e6354451392ddfb6626d45 Mon Sep 17 00:00:00 2001 From: linglp Date: Thu, 27 Jun 2024 13:40:21 -0400 Subject: [PATCH 010/233] add docstring and return type --- schematic/store/synapse.py | 98 ++++++++++++++++++++++---------------- 1 file changed, 56 insertions(+), 42 deletions(-) diff --git a/schematic/store/synapse.py b/schematic/store/synapse.py index 569389b16..a275ef47a 100644 --- a/schematic/store/synapse.py +++ b/schematic/store/synapse.py @@ -1,71 +1,78 @@ """Synapse storage class""" -import asyncio import atexit +from copy import deepcopy +from dataclasses import dataclass import logging +import numpy as np +import pandas as pd import os import re import secrets import shutil +import synapseclient +from synapseclient.api import get_entity_id_bundle2 import uuid # used to generate unique names for entities -from copy import deepcopy -from dataclasses import asdict, dataclass + +from tenacity import ( + retry, + stop_after_attempt, + wait_chain, + wait_fixed, + retry_if_exception_type, +) from time import sleep # allows specifying explicit variable types -from typing import Any, Dict, List, Optional, Sequence, Set, Tuple, Union +from typing import Dict, List, Tuple, Sequence, Union, Optional, Any, Set -import numpy as np -import pandas as pd -import synapseclient -import synapseutils -from opentelemetry import trace -from schematic_db.rdb.synapse_database import SynapseDatabase from synapseclient import ( - Column, - EntityViewSchema, - EntityViewType, + Synapse, File, Folder, - Schema, - Synapse, Table, + Schema, + EntityViewSchema, + EntityViewType, + Column, as_table_columns, ) -from synapseclient.api import get_entity_id_bundle2 +from synapseclient.entity import File +from synapseclient.table import CsvFileTable, build_table, Schema from synapseclient.core.exceptions import ( - SynapseAuthenticationError, SynapseHTTPError, + SynapseAuthenticationError, SynapseUnmetAccessRestrictions, + SynapseHTTPError, ) -from synapseclient.entity import File -from synapseclient.models.annotations import Annotations -from synapseclient.table import CsvFileTable, Schema, build_table -from tenacity import ( - retry, - retry_if_exception_type, - stop_after_attempt, - wait_chain, - wait_fixed, -) +import synapseutils + +from schematic_db.rdb.synapse_database import SynapseDatabase -from schematic.configuration.configuration import CONFIG -from schematic.exceptions import AccessCredentialsError from schematic.schemas.data_model_graph import DataModelGraphExplorer -from schematic.store.base import BaseStorage -from schematic.utils.df_utils import col_in_dataframe, load_df, update_df + +from schematic.utils.df_utils import update_df, load_df, col_in_dataframe +from schematic.utils.validate_utils import comma_separated_list_regex, rule_in_rule_list # entity_type_mapping, get_dir_size, create_temp_folder, check_synapse_cache_size, and clear_synapse_cache functions are used for AWS deployment # Please do not remove these import statements from schematic.utils.general import ( - check_synapse_cache_size, - clear_synapse_cache, - create_temp_folder, entity_type_mapping, get_dir_size, + create_temp_folder, + check_synapse_cache_size, + clear_synapse_cache, ) + from schematic.utils.schema_utils import get_class_label_from_display_name -from schematic.utils.validate_utils import comma_separated_list_regex, rule_in_rule_list + +from schematic.store.base import BaseStorage +from schematic.exceptions import AccessCredentialsError +from schematic.configuration.configuration import CONFIG +from synapseclient.models.annotations import Annotations +import asyncio +from dataclasses import asdict +from opentelemetry import trace logger = logging.getLogger("Synapse storage") @@ -1402,13 +1409,20 @@ async def store_async_annotation(self, annotation_dict: dict) -> Annotations: @async_missing_entity_handler async def format_row_annotations( - self, - dmge: DataModelGraphExplorer, - row: pd.Series, - entityId: str, - hideBlanks: bool, - annotation_keys: str, - ): + self, dmge:DataModelGraphExplorer, row:pd.Series, entityId:str, hideBlanks: bool, annotation_keys: str + ) -> Union[None, Dict[str,Any]]: + """Format row annotations + + Args: + dmge (DataModelGraphExplorer): data moodel graph explorer object + row (pd.Series): row of the manifest + entityId (str): entity id of the manifest + hideBlanks (bool): when true, does not upload annotation keys with blank values. When false, upload Annotation keys with empty string values + annotation_keys (str): display_label/class_label + + Returns: + Union[None, Dict[str,]]: if entity id is in trash can, return None. Otherwise, return the annotations + """ # prepare metadata for Synapse storage (resolve display name into a name that Synapse annotations support (e.g no spaces, parenthesis) # note: the removal of special characters, will apply only to annotation keys; we are not altering the manifest # this could create a divergence between manifest column and annotations. this should be ok for most use cases. From 0685529a4d34743a88d43584130391bb26f03172 Mon Sep 17 00:00:00 2001 From: linglp Date: Thu, 27 Jun 2024 13:40:50 -0400 Subject: [PATCH 011/233] add parameter docstring --- schematic/store/synapse.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/schematic/store/synapse.py b/schematic/store/synapse.py index a275ef47a..79c2ca1b4 100644 --- a/schematic/store/synapse.py +++ b/schematic/store/synapse.py @@ -1409,15 +1409,20 @@ async def store_async_annotation(self, annotation_dict: dict) -> Annotations: @async_missing_entity_handler async def format_row_annotations( - self, dmge:DataModelGraphExplorer, row:pd.Series, entityId:str, hideBlanks: bool, annotation_keys: str - ) -> Union[None, Dict[str,Any]]: - """Format row annotations + self, + dmge: DataModelGraphExplorer, + row: pd.Series, + entityId: str, + hideBlanks: bool, + annotation_keys: str, + ) -> Union[None, Dict[str, Any]]: + """Format row annotations Args: dmge (DataModelGraphExplorer): data moodel graph explorer object - row (pd.Series): row of the manifest + row (pd.Series): row of the manifest entityId (str): entity id of the manifest - hideBlanks (bool): when true, does not upload annotation keys with blank values. When false, upload Annotation keys with empty string values + hideBlanks (bool): when true, does not upload annotation keys with blank values. When false, upload Annotation keys with empty string values annotation_keys (str): display_label/class_label Returns: From 2649e14092874e24cd39693e0b17a77168d04d2e Mon Sep 17 00:00:00 2001 From: linglp Date: Thu, 27 Jun 2024 14:02:32 -0400 Subject: [PATCH 012/233] when annos is emptystoring not be triggered --- schematic/store/synapse.py | 1 + tests/test_store.py | 19 +++++++++++++++++++ 2 files changed, 20 insertions(+) diff --git a/schematic/store/synapse.py b/schematic/store/synapse.py index 79c2ca1b4..020d87a1b 100644 --- a/schematic/store/synapse.py +++ b/schematic/store/synapse.py @@ -1738,6 +1738,7 @@ async def _process_store_annos(self, requests: Set[asyncio.Task]) -> None: entity_id = annos_dict["id"] logger.info(f"Successfully stored annotations for {entity_id}") else: + # store annotations if they are not None if annos: entity_id = annos["EntityId"] logger.info( diff --git a/tests/test_store.py b/tests/test_store.py index 48d1afb5f..ec1b52b6c 100644 --- a/tests/test_store.py +++ b/tests/test_store.py @@ -671,6 +671,25 @@ async def mock_success_coro(): await synapse_store._process_store_annos(new_tasks) mock_store_async2.assert_called_once() + async def test_process_store_annos_get_annos_empty( + self, synapse_store: SynapseStorage + ) -> None: + """ "test _process_store_annos function and make sure that task of storing annotations wont be triggered when annotations are empty""" + + # make sure that the else statement is working + # and that the task of storing annotations is not triggered when annotations are empty + async def mock_success_coro(): + return None + + with patch( + "schematic.store.synapse.SynapseStorage.store_async_annotation", + new_callable=AsyncMock, + ) as mock_store_async: + new_tasks = set() + new_tasks.add(asyncio.create_task(mock_success_coro())) + await synapse_store._process_store_annos(new_tasks) + mock_store_async.assert_not_called() + class TestDatasetFileView: def test_init(self, dataset_id, dataset_fileview, synapse_store): From f05aa66968345e1d7eae45417c4001f17b1ae1ba Mon Sep 17 00:00:00 2001 From: linglp Date: Thu, 27 Jun 2024 14:17:10 -0400 Subject: [PATCH 013/233] add typing --- tests/test_store.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_store.py b/tests/test_store.py index ec1b52b6c..07f370499 100644 --- a/tests/test_store.py +++ b/tests/test_store.py @@ -463,7 +463,7 @@ def test_add_entity_id_and_filename_without_component_col(self, synapse_store): ], ) async def test_format_row_annotations_entity_id_trash_can( - self, caplog, dmge, synapse_store, hideBlanks, annotation_keys + self, caplog:pytest.LogCaptureFixture, dmge: DataModelGraph, synapse_store: SynapseStorage, hideBlanks: bool, annotation_keys: str ): """make sure that missing_entity_handler gets triggered when entity is in the trash can""" with patch( From c7e4cfe173fc7dc032255b331b437bbbb305519c Mon Sep 17 00:00:00 2001 From: linglp Date: Thu, 27 Jun 2024 14:17:33 -0400 Subject: [PATCH 014/233] add typing --- tests/test_store.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/test_store.py b/tests/test_store.py index 07f370499..80228215f 100644 --- a/tests/test_store.py +++ b/tests/test_store.py @@ -463,7 +463,12 @@ def test_add_entity_id_and_filename_without_component_col(self, synapse_store): ], ) async def test_format_row_annotations_entity_id_trash_can( - self, caplog:pytest.LogCaptureFixture, dmge: DataModelGraph, synapse_store: SynapseStorage, hideBlanks: bool, annotation_keys: str + self, + caplog: pytest.LogCaptureFixture, + dmge: DataModelGraph, + synapse_store: SynapseStorage, + hideBlanks: bool, + annotation_keys: str, ): """make sure that missing_entity_handler gets triggered when entity is in the trash can""" with patch( From 667d865649fe5b629bf38cf01797758880a873da Mon Sep 17 00:00:00 2001 From: linglp Date: Thu, 27 Jun 2024 14:21:29 -0400 Subject: [PATCH 015/233] add docstring and reformat --- schematic/store/synapse.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/schematic/store/synapse.py b/schematic/store/synapse.py index 020d87a1b..a0b529c52 100644 --- a/schematic/store/synapse.py +++ b/schematic/store/synapse.py @@ -338,7 +338,9 @@ def wrapper(*args, **kwargs): return wrapper def async_missing_entity_handler(method): - async def wrapper(*args, **kwargs): + """Decorator to handle missing entities in async methods.""" + + async def wrapper(*args: Any, **kwargs: Any) -> Any: try: return await method(*args, **kwargs) except SynapseHTTPError as ex: From d10c434e1914d051fdde7825e3e99e8037b2058a Mon Sep 17 00:00:00 2001 From: linglp Date: Thu, 27 Jun 2024 14:24:40 -0400 Subject: [PATCH 016/233] add a return type --- tests/test_store.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_store.py b/tests/test_store.py index 80228215f..72d466595 100644 --- a/tests/test_store.py +++ b/tests/test_store.py @@ -469,7 +469,7 @@ async def test_format_row_annotations_entity_id_trash_can( synapse_store: SynapseStorage, hideBlanks: bool, annotation_keys: str, - ): + ) -> None: """make sure that missing_entity_handler gets triggered when entity is in the trash can""" with patch( "schematic.store.synapse.SynapseStorage.get_async_annotation", @@ -683,7 +683,7 @@ async def test_process_store_annos_get_annos_empty( # make sure that the else statement is working # and that the task of storing annotations is not triggered when annotations are empty - async def mock_success_coro(): + async def mock_success_coro() -> None: return None with patch( From 613beff9b2f13b5340b325afcff92faef3894af6 Mon Sep 17 00:00:00 2001 From: linglp Date: Thu, 27 Jun 2024 14:28:35 -0400 Subject: [PATCH 017/233] add type hint to func --- tests/test_store.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_store.py b/tests/test_store.py index 896a9ebb1..483d7d90b 100644 --- a/tests/test_store.py +++ b/tests/test_store.py @@ -668,7 +668,7 @@ async def test_process_store_annos_success_get_entity_id_variants( id="mock syn_id", ) - async def mock_success_coro(): + async def mock_success_coro() -> dict[str, Any]: return mock_annos_dict # make sure that the else statement is working From 2fa9075a9542cab37ffbd53c300626ba95af8687 Mon Sep 17 00:00:00 2001 From: linglp Date: Fri, 28 Jun 2024 13:01:39 -0400 Subject: [PATCH 018/233] run black --- tests/test_store.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_store.py b/tests/test_store.py index 22a659f57..ce2e7cdd5 100644 --- a/tests/test_store.py +++ b/tests/test_store.py @@ -725,6 +725,7 @@ async def mock_success_coro() -> dict[str, Any]: new_tasks.add(asyncio.create_task(mock_success_coro())) await synapse_store._process_store_annos(new_tasks) mock_store_async2.assert_called_once() + async def test_process_store_annos_get_annos_empty( self, synapse_store: SynapseStorage ) -> None: From 0fd04d2b925e1f684c5fc3fef478a58cec4445bd Mon Sep 17 00:00:00 2001 From: linglp Date: Fri, 28 Jun 2024 22:28:39 -0400 Subject: [PATCH 019/233] Revert "merge with develop" This reverts commit 47312f8dbe4b77ef5ad63df3c43b3ce7240f5a89, reversing changes made to 2fa9075a9542cab37ffbd53c300626ba95af8687. --- poetry.lock | 54 +++++++++---- pyproject.toml | 4 +- schematic/store/synapse.py | 155 ++++++++++++++++++------------------- tests/test_store.py | 77 +++++++++++++----- 4 files changed, 175 insertions(+), 115 deletions(-) diff --git a/poetry.lock b/poetry.lock index 203239b7a..efab907c7 100644 --- a/poetry.lock +++ b/poetry.lock @@ -190,6 +190,19 @@ files = [ [package.dependencies] typing-extensions = {version = ">=4.0.0", markers = "python_version < \"3.11\""} +[[package]] +name = "asyncio" +version = "3.4.3" +description = "reference implementation of PEP 3156" +optional = false +python-versions = "*" +files = [ + {file = "asyncio-3.4.3-cp33-none-win32.whl", hash = "sha256:b62c9157d36187eca799c378e572c969f0da87cd5fc42ca372d92cdb06e7e1de"}, + {file = "asyncio-3.4.3-cp33-none-win_amd64.whl", hash = "sha256:c46a87b48213d7464f22d9a497b9eef8c1928b68320a2fa94240f969f6fec08c"}, + {file = "asyncio-3.4.3-py3-none-any.whl", hash = "sha256:c4d18b22701821de07bd6aea8b53d21449ec0ec5680645e5317062ea21817d2d"}, + {file = "asyncio-3.4.3.tar.gz", hash = "sha256:83360ff8bc97980e4ff25c964c7bd3923d333d177aa4f7fb736b019f26c7cb41"}, +] + [[package]] name = "asyncio-atexit" version = "1.0.1" @@ -3230,6 +3243,24 @@ tomli = {version = ">=1.0.0", markers = "python_version < \"3.11\""} [package.extras] testing = ["argcomplete", "attrs (>=19.2.0)", "hypothesis (>=3.56)", "mock", "nose", "pygments (>=2.7.2)", "requests", "setuptools", "xmlschema"] +[[package]] +name = "pytest-asyncio" +version = "0.23.7" +description = "Pytest support for asyncio" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pytest_asyncio-0.23.7-py3-none-any.whl", hash = "sha256:009b48127fbe44518a547bddd25611551b0e43ccdbf1e67d12479f569832c20b"}, + {file = "pytest_asyncio-0.23.7.tar.gz", hash = "sha256:5f5c72948f4c49e7db4f29f2521d4031f1c27f86e57b046126654083d4770268"}, +] + +[package.dependencies] +pytest = ">=7.0.0,<9" + +[package.extras] +docs = ["sphinx (>=5.3)", "sphinx-rtd-theme (>=1.0)"] +testing = ["coverage (>=6.2)", "hypothesis (>=5.7.1)"] + [[package]] name = "pytest-cov" version = "4.1.0" @@ -3400,7 +3431,6 @@ files = [ {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, - {file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"}, {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, @@ -3408,16 +3438,8 @@ files = [ {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, - {file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"}, {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, - {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, - {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, - {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"}, - {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, - {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, - {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, - {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"}, {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, @@ -3434,7 +3456,6 @@ files = [ {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, - {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"}, {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, @@ -3442,7 +3463,6 @@ files = [ {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, - {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"}, {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, @@ -4379,13 +4399,13 @@ Jinja2 = ">=2.0" [[package]] name = "synapseclient" -version = "4.2.0" +version = "4.3.0" description = "A client for Synapse, a collaborative, open-source research platform that allows teams to share data, track analyses, and collaborate." optional = false python-versions = ">=3.8" files = [ - {file = "synapseclient-4.2.0-py3-none-any.whl", hash = "sha256:ab5bc9c2bf5b90f271f1a9478eff7e9fca3e573578401ac706383ddb984d7a13"}, - {file = "synapseclient-4.2.0.tar.gz", hash = "sha256:89222661125de1795b1a096cf8c58b8115c19d6b0fa5846ed2a41cdb394ef773"}, + {file = "synapseclient-4.3.0-py3-none-any.whl", hash = "sha256:5d8107cfff4031a0a46d60a3c9a8120300190fa27df4983d883dc951d8bd885f"}, + {file = "synapseclient-4.3.0.tar.gz", hash = "sha256:a1149a64b3281669d42c69e210677a902478b8f6b302966d518473c7384f6387"}, ] [package.dependencies] @@ -4405,11 +4425,11 @@ urllib3 = ">=1.26.18,<2" [package.extras] boto3 = ["boto3 (>=1.7.0,<2.0)"] -dev = ["black", "flake8 (>=3.7.0,<4.0)", "func-timeout (>=4.3,<5.0)", "pre-commit", "pytest (>=6.0.0,<7.0)", "pytest-asyncio (>=0.19,<1.0)", "pytest-cov (>=4.1.0,<4.2.0)", "pytest-mock (>=3.0,<4.0)", "pytest-rerunfailures (>=12.0,<13.0)", "pytest-socket (>=0.6.0,<0.7.0)", "pytest-xdist[psutil] (>=2.2,<3.0.0)"] +dev = ["black", "flake8 (>=3.7.0,<4.0)", "func-timeout (>=4.3,<5.0)", "pandas (>=1.5,<3.0)", "pre-commit", "pytest (>=7.0.0,<8.0)", "pytest-asyncio (>=0.23.6,<1.0)", "pytest-cov (>=4.1.0,<4.2.0)", "pytest-mock (>=3.0,<4.0)", "pytest-rerunfailures (>=12.0,<13.0)", "pytest-socket (>=0.6.0,<0.7.0)", "pytest-xdist[psutil] (>=2.2,<3.0.0)"] docs = ["markdown-include (>=0.8.1,<0.9.0)", "mkdocs (>=1.5.3)", "mkdocs-material (>=9.4.14)", "mkdocs-open-in-new-tab (>=1.0.3,<1.1.0)", "mkdocstrings (>=0.24.0)", "mkdocstrings-python (>=1.7.5)", "termynal (>=0.11.1)"] pandas = ["pandas (>=1.5,<3.0)"] pysftp = ["pysftp (>=0.2.8,<0.3)"] -tests = ["flake8 (>=3.7.0,<4.0)", "func-timeout (>=4.3,<5.0)", "pytest (>=6.0.0,<7.0)", "pytest-asyncio (>=0.19,<1.0)", "pytest-cov (>=4.1.0,<4.2.0)", "pytest-mock (>=3.0,<4.0)", "pytest-rerunfailures (>=12.0,<13.0)", "pytest-socket (>=0.6.0,<0.7.0)", "pytest-xdist[psutil] (>=2.2,<3.0.0)"] +tests = ["flake8 (>=3.7.0,<4.0)", "func-timeout (>=4.3,<5.0)", "pandas (>=1.5,<3.0)", "pytest (>=7.0.0,<8.0)", "pytest-asyncio (>=0.23.6,<1.0)", "pytest-cov (>=4.1.0,<4.2.0)", "pytest-mock (>=3.0,<4.0)", "pytest-rerunfailures (>=12.0,<13.0)", "pytest-socket (>=0.6.0,<0.7.0)", "pytest-xdist[psutil] (>=2.2,<3.0.0)"] [[package]] name = "tabulate" @@ -4944,4 +4964,4 @@ aws = ["uWSGI"] [metadata] lock-version = "2.0" python-versions = ">=3.9.0,<3.11" -content-hash = "5bf0c831977694ea541db24481181ec1980ec9589a2adbd9f30ed0fe7f2b2742" +content-hash = "a3048c0808e73fd19f5175897e9dda47a2a593422dd4744886615ac453a42139" diff --git a/pyproject.toml b/pyproject.toml index 3c2795140..8d941b8ae 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -54,7 +54,7 @@ pygsheets = "^2.0.4" PyYAML = "^6.0.0" rdflib = "^6.0.0" setuptools = "^66.0.0" -synapseclient = "^4.1.0" +synapseclient = "^4.3.0" tenacity = "^8.0.1" toml = "^0.10.2" great-expectations = "^0.15.0" @@ -74,6 +74,8 @@ Flask = {version = "2.1.3", optional = true} Flask-Cors = {version = "^3.0.10", optional = true} uWSGI = {version = "^2.0.21", optional = true} Jinja2 = {version = ">2.11.3", optional = true} +asyncio = "^3.4.3" +pytest-asyncio = "^0.23.7" jaeger-client = {version = "^4.8.0", optional = true} flask-opentracing = {version="^2.0.0", optional = true} diff --git a/schematic/store/synapse.py b/schematic/store/synapse.py index edd238942..73e32d067 100644 --- a/schematic/store/synapse.py +++ b/schematic/store/synapse.py @@ -7,16 +7,20 @@ import re import secrets import shutil -import synapseclient -from synapseclient.api import get_entity_id_bundle2 import uuid # used to generate unique names for entities from copy import deepcopy from dataclasses import asdict, dataclass from time import sleep # allows specifying explicit variable types -from typing import Dict, List, Tuple, Sequence, Union, Optional, Any, Set +from typing import Any, Dict, List, Optional, Sequence, Set, Tuple, Union +import numpy as np +import pandas as pd +import synapseclient +import synapseutils +from opentelemetry import trace +from schematic_db.rdb.synapse_database import SynapseDatabase from synapseclient import ( Column, EntityViewSchema, @@ -61,14 +65,7 @@ get_dir_size, ) from schematic.utils.schema_utils import get_class_label_from_display_name - -from schematic.store.base import BaseStorage -from schematic.exceptions import AccessCredentialsError -from schematic.configuration.configuration import CONFIG -from synapseclient.models.annotations import Annotations -import asyncio -from dataclasses import asdict -from opentelemetry import trace +from schematic.utils.validate_utils import comma_separated_list_regex, rule_in_rule_list logger = logging.getLogger("Synapse storage") @@ -723,7 +720,6 @@ def fill_in_entity_id_filename( new_files = self._get_file_entityIds( dataset_files=dataset_files, only_new_files=True, manifest=manifest ) - # update manifest so that it contains new dataset files new_files = pd.DataFrame(new_files) manifest = ( @@ -1406,10 +1402,27 @@ async def store_async_annotation(self, annotation_dict: dict) -> Annotations: ) return await annotation_class.store_async(self.syn) - @missing_entity_handler + @async_missing_entity_handler async def format_row_annotations( - self, dmge, row, entityId: str, hideBlanks: bool, annotation_keys: str - ): + self, + dmge: DataModelGraphExplorer, + row: pd.Series, + entityId: str, + hideBlanks: bool, + annotation_keys: str, + ) -> Union[None, Dict[str, Any]]: + """Format row annotations + + Args: + dmge (DataModelGraphExplorer): data moodel graph explorer object + row (pd.Series): row of the manifest + entityId (str): entity id of the manifest + hideBlanks (bool): when true, does not upload annotation keys with blank values. When false, upload Annotation keys with empty string values + annotation_keys (str): display_label/class_label + + Returns: + Union[None, Dict[str,]]: if entity id is in trash can, return None. Otherwise, return the annotations + """ # prepare metadata for Synapse storage (resolve display name into a name that Synapse annotations support (e.g no spaces, parenthesis) # note: the removal of special characters, will apply only to annotation keys; we are not altering the manifest # this could create a divergence between manifest column and annotations. this should be ok for most use cases. @@ -1439,7 +1452,8 @@ async def format_row_annotations( metadataSyn[keySyn] = v # set annotation(s) for the various objects/items in a dataset on Synapse - annos = self.syn.get_annotations(entityId) + annos = await self.get_async_annotation(entityId) + csv_list_regex = comma_separated_list_regex() for anno_k, anno_v in metadataSyn.items(): # Remove keys with nan or empty string values from dict of annotations to be uploaded @@ -1676,37 +1690,6 @@ def _generate_table_name(self, manifest): table_name = "synapse_storage_manifest_table" return table_name, component_name - @tracer.start_as_current_span("SynapseStorage::_add_annotations") - def _add_annotations( - self, - dmge, - row, - entityId: str, - hideBlanks: bool, - annotation_keys: str, - ): - """Helper function to format and add annotations to entities in Synapse. - Args: - dmge: DataModelGraphExplorer object, - row: current row of manifest being processed - entityId (str): synapseId of entity to add annotations to - hideBlanks: Boolean flag that does not upload annotation keys with blank values when true. Uploads Annotation keys with empty string values when false. - annotation_keys: (str) display_label/class_label(default), Determines labeling syle for annotation keys. class_label will format the display - name as upper camelcase, and strip blacklisted characters, display_label will strip blacklisted characters including spaces, to retain - display label formatting while ensuring the label is formatted properly for Synapse annotations. - Returns: - Annotations are added to entities in Synapse, no return. - """ - # Format annotations for Synapse - annos = self.format_row_annotations( - dmge, row, entityId, hideBlanks, annotation_keys - ) - - if annos: - # Store annotations for an entity folder - self.syn.set_annotations(annos) - return - def _create_entity_id(self, idx, row, manifest, datasetId): """Helper function to generate an entityId and add it to the appropriate row in the manifest. Args: @@ -1747,14 +1730,17 @@ async def _process_store_annos(self, requests: Set[asyncio.Task]) -> None: if isinstance(annos, Annotations): annos_dict = asdict(annos) - entity_id = annos_dict["id"] + normalized_annos = {k.lower(): v for k, v in annos_dict.items()} + entity_id = normalized_annos["id"] logger.info(f"Successfully stored annotations for {entity_id}") else: - entity_id = annos["EntityId"] - logger.info( - f"Obtained and processed annotations for {entity_id} entity" - ) + # store annotations if they are not None if annos: + normalized_annos = {k.lower(): v for k, v in annos.items()} + entity_id = normalized_annos["entityid"] + logger.info( + f"Obtained and processed annotations for {entity_id} entity" + ) requests.add( asyncio.create_task( self.store_async_annotation(annotation_dict=annos) @@ -1764,7 +1750,7 @@ async def _process_store_annos(self, requests: Set[asyncio.Task]) -> None: raise RuntimeError(f"failed with { repr(e) }.") from e @tracer.start_as_current_span("SynapseStorage::add_annotations_to_entities_files") - def add_annotations_to_entities_files( + async def add_annotations_to_entities_files( self, dmge, manifest, @@ -1805,6 +1791,7 @@ def add_annotations_to_entities_files( ).drop("entityId_x", axis=1) # Fill `entityId` for each row if missing and annotate entity as appropriate + requests = set() for idx, row in manifest.iterrows(): if not row["entityId"] and ( manifest_record_type == "file_and_entities" @@ -1824,8 +1811,14 @@ def add_annotations_to_entities_files( # Adding annotations to connected files. if entityId: - self._add_annotations(dmge, row, entityId, hideBlanks, annotation_keys) - logger.info(f"Added annotations to entity: {entityId}") + # Format annotations for Synapse + annos_task = asyncio.create_task( + self.format_row_annotations( + dmge, row, entityId, hideBlanks, annotation_keys + ) + ) + requests.add(annos_task) + await self._process_store_annos(requests) return manifest @tracer.start_as_current_span("SynapseStorage::upload_manifest_as_table") @@ -1879,14 +1872,16 @@ def upload_manifest_as_table( ) if file_annotations_upload: - manifest = self.add_annotations_to_entities_files( - dmge, - manifest, - manifest_record_type, - datasetId, - hideBlanks, - manifest_synapse_table_id, - annotation_keys, + manifest = asyncio.run( + self.add_annotations_to_entities_files( + dmge, + manifest, + manifest_record_type, + datasetId, + hideBlanks, + manifest_synapse_table_id, + annotation_keys, + ) ) # Load manifest to synapse as a CSV File manifest_synapse_file_id = self.upload_manifest_file( @@ -1953,13 +1948,15 @@ def upload_manifest_as_csv( manifest_synapse_file_id (str): SynID of manifest csv uploaded to synapse. """ if file_annotations_upload: - manifest = self.add_annotations_to_entities_files( - dmge, - manifest, - manifest_record_type, - datasetId, - hideBlanks, - annotation_keys=annotation_keys, + manifest = asyncio.run( + self.add_annotations_to_entities_files( + dmge, + manifest, + manifest_record_type, + datasetId, + hideBlanks, + annotation_keys=annotation_keys, + ) ) # Load manifest to synapse as a CSV File @@ -2031,14 +2028,16 @@ def upload_manifest_combo( ) if file_annotations_upload: - manifest = self.add_annotations_to_entities_files( - dmge, - manifest, - manifest_record_type, - datasetId, - hideBlanks, - manifest_synapse_table_id, - annotation_keys=annotation_keys, + manifest = asyncio.run( + self.add_annotations_to_entities_files( + dmge, + manifest, + manifest_record_type, + datasetId, + hideBlanks, + manifest_synapse_table_id, + annotation_keys=annotation_keys, + ) ) # Load manifest to synapse as a CSV File diff --git a/tests/test_store.py b/tests/test_store.py index f0cf55052..ce2e7cdd5 100644 --- a/tests/test_store.py +++ b/tests/test_store.py @@ -1198,7 +1198,7 @@ class TestManifestUpload: ), ], ) - def test_add_annotations_to_entities_files( + async def test_add_annotations_to_entities_files( self, synapse_store: SynapseStorage, dmge: DataModelGraphExplorer, @@ -1218,27 +1218,49 @@ def test_add_annotations_to_entities_files( expected_filenames (list(str)): expected list of file names expected_entity_ids (list(str)): expected list of entity ids """ + + async def mock_format_row_annos(): + return + + async def mock_process_store_annos(requests): + return + with patch( "schematic.store.synapse.SynapseStorage.getFilesInStorageDataset", return_value=files_in_dataset, ): - manifest_df = pd.DataFrame(original_manifest) + with patch( + "schematic.store.synapse.SynapseStorage.format_row_annotations", + return_value=mock_format_row_annos, + new_callable=AsyncMock, + ) as mock_format_row: + with patch( + "schematic.store.synapse.SynapseStorage._process_store_annos", + return_value=mock_process_store_annos, + new_callable=AsyncMock, + ) as mock_process_store: + manifest_df = pd.DataFrame(original_manifest) + + new_df = await synapse_store.add_annotations_to_entities_files( + dmge, + manifest_df, + manifest_record_type="entity", + datasetId="mock id", + hideBlanks=True, + ) - new_df = synapse_store.add_annotations_to_entities_files( - dmge, - manifest_df, - manifest_record_type="entity", - datasetId="mock id", - hideBlanks=True, - ) - file_names_lst = new_df["Filename"].tolist() - entity_ids_lst = new_df["entityId"].tolist() + file_names_lst = new_df["Filename"].tolist() + entity_ids_lst = new_df["entityId"].tolist() + + # test entityId and Id columns get added + assert "entityId" in new_df.columns + assert "Id" in new_df.columns + assert file_names_lst == expected_filenames + assert entity_ids_lst == expected_entity_ids - # test entityId and Id columns get added - assert "entityId" in new_df.columns - assert "Id" in new_df.columns - assert file_names_lst == expected_filenames - assert entity_ids_lst == expected_entity_ids + # make sure async function gets called as expected + assert mock_format_row.call_count == len(expected_entity_ids) + assert mock_process_store.call_count == 1 @pytest.mark.parametrize( "mock_manifest_file_path", @@ -1322,9 +1344,14 @@ def test_upload_manifest_as_csv( hide_blanks: bool, restrict: bool, ) -> None: + async def mock_add_annotations_to_entities_files(): + return + with ( patch( - "schematic.store.synapse.SynapseStorage.add_annotations_to_entities_files" + "schematic.store.synapse.SynapseStorage.add_annotations_to_entities_files", + return_value=mock_add_annotations_to_entities_files, + new_callable=AsyncMock, ) as add_anno_mock, patch( "schematic.store.synapse.SynapseStorage.upload_manifest_file", @@ -1372,13 +1399,19 @@ def test_upload_manifest_as_table( manifest_record_type: str, ) -> None: mock_df = pd.DataFrame() + + async def mock_add_annotations_to_entities_files(): + return + with ( patch( "schematic.store.synapse.SynapseStorage.uploadDB", return_value=["mock_table_id", mock_df, "mock_table_manifest"], ) as update_db_mock, patch( - "schematic.store.synapse.SynapseStorage.add_annotations_to_entities_files" + "schematic.store.synapse.SynapseStorage.add_annotations_to_entities_files", + return_value=mock_add_annotations_to_entities_files, + new_callable=AsyncMock, ) as add_anno_mock, patch( "schematic.store.synapse.SynapseStorage.upload_manifest_file", @@ -1432,13 +1465,19 @@ def test_upload_manifest_combo( mock_df = pd.DataFrame() manifest_path = helpers.get_data_path("mock_manifests/test_BulkRNAseq.csv") manifest_df = helpers.get_data_frame(manifest_path) + + async def mock_add_annotations_to_entities_files(): + return + with ( patch( "schematic.store.synapse.SynapseStorage.uploadDB", return_value=["mock_table_id", mock_df, "mock_table_manifest"], ) as update_db_mock, patch( - "schematic.store.synapse.SynapseStorage.add_annotations_to_entities_files" + "schematic.store.synapse.SynapseStorage.add_annotations_to_entities_files", + return_value=mock_add_annotations_to_entities_files, + new_callable=AsyncMock, ) as add_anno_mock, patch( "schematic.store.synapse.SynapseStorage.upload_manifest_file", From 03fbe32dfa7d9cd5f11368605ef47253251899c8 Mon Sep 17 00:00:00 2001 From: linglp Date: Fri, 28 Jun 2024 22:55:49 -0400 Subject: [PATCH 020/233] reorg commit --- schematic/store/synapse.py | 45 +++++++++++--------------------------- 1 file changed, 13 insertions(+), 32 deletions(-) diff --git a/schematic/store/synapse.py b/schematic/store/synapse.py index 73e32d067..e713d7b03 100644 --- a/schematic/store/synapse.py +++ b/schematic/store/synapse.py @@ -21,51 +21,31 @@ import synapseutils from opentelemetry import trace from schematic_db.rdb.synapse_database import SynapseDatabase -from synapseclient import ( - Column, - EntityViewSchema, - EntityViewType, - File, - Folder, - Schema, - Synapse, - Table, - as_table_columns, -) +from synapseclient import (Column, EntityViewSchema, EntityViewType, File, + Folder, Schema, Synapse, Table, as_table_columns) from synapseclient.api import get_entity_id_bundle2 -from synapseclient.core.exceptions import ( - SynapseAuthenticationError, - SynapseHTTPError, - SynapseUnmetAccessRestrictions, -) +from synapseclient.core.exceptions import (SynapseAuthenticationError, + SynapseHTTPError, + SynapseUnmetAccessRestrictions) from synapseclient.entity import File from synapseclient.models.annotations import Annotations from synapseclient.table import CsvFileTable, Schema, build_table -from tenacity import ( - retry, - retry_if_exception_type, - stop_after_attempt, - wait_chain, - wait_fixed, -) +from tenacity import (retry, retry_if_exception_type, stop_after_attempt, + wait_chain, wait_fixed) from schematic.configuration.configuration import CONFIG from schematic.exceptions import AccessCredentialsError from schematic.schemas.data_model_graph import DataModelGraphExplorer from schematic.store.base import BaseStorage from schematic.utils.df_utils import col_in_dataframe, load_df, update_df - # entity_type_mapping, get_dir_size, create_temp_folder, check_synapse_cache_size, and clear_synapse_cache functions are used for AWS deployment # Please do not remove these import statements -from schematic.utils.general import ( - check_synapse_cache_size, - clear_synapse_cache, - create_temp_folder, - entity_type_mapping, - get_dir_size, -) +from schematic.utils.general import (check_synapse_cache_size, + clear_synapse_cache, create_temp_folder, + entity_type_mapping, get_dir_size) from schematic.utils.schema_utils import get_class_label_from_display_name -from schematic.utils.validate_utils import comma_separated_list_regex, rule_in_rule_list +from schematic.utils.validate_utils import (comma_separated_list_regex, + rule_in_rule_list) logger = logging.getLogger("Synapse storage") @@ -720,6 +700,7 @@ def fill_in_entity_id_filename( new_files = self._get_file_entityIds( dataset_files=dataset_files, only_new_files=True, manifest=manifest ) + # update manifest so that it contains new dataset files new_files = pd.DataFrame(new_files) manifest = ( From 9132c5e98866324b07fee1f7a59faccf8c48ecd9 Mon Sep 17 00:00:00 2001 From: linglp Date: Fri, 28 Jun 2024 22:58:16 -0400 Subject: [PATCH 021/233] reformat --- schematic/store/synapse.py | 44 +++++++++++++++++++++++++++----------- 1 file changed, 32 insertions(+), 12 deletions(-) diff --git a/schematic/store/synapse.py b/schematic/store/synapse.py index e713d7b03..a6f9260d4 100644 --- a/schematic/store/synapse.py +++ b/schematic/store/synapse.py @@ -21,31 +21,51 @@ import synapseutils from opentelemetry import trace from schematic_db.rdb.synapse_database import SynapseDatabase -from synapseclient import (Column, EntityViewSchema, EntityViewType, File, - Folder, Schema, Synapse, Table, as_table_columns) +from synapseclient import ( + Column, + EntityViewSchema, + EntityViewType, + File, + Folder, + Schema, + Synapse, + Table, + as_table_columns, +) from synapseclient.api import get_entity_id_bundle2 -from synapseclient.core.exceptions import (SynapseAuthenticationError, - SynapseHTTPError, - SynapseUnmetAccessRestrictions) +from synapseclient.core.exceptions import ( + SynapseAuthenticationError, + SynapseHTTPError, + SynapseUnmetAccessRestrictions, +) from synapseclient.entity import File from synapseclient.models.annotations import Annotations from synapseclient.table import CsvFileTable, Schema, build_table -from tenacity import (retry, retry_if_exception_type, stop_after_attempt, - wait_chain, wait_fixed) +from tenacity import ( + retry, + retry_if_exception_type, + stop_after_attempt, + wait_chain, + wait_fixed, +) from schematic.configuration.configuration import CONFIG from schematic.exceptions import AccessCredentialsError from schematic.schemas.data_model_graph import DataModelGraphExplorer from schematic.store.base import BaseStorage from schematic.utils.df_utils import col_in_dataframe, load_df, update_df + # entity_type_mapping, get_dir_size, create_temp_folder, check_synapse_cache_size, and clear_synapse_cache functions are used for AWS deployment # Please do not remove these import statements -from schematic.utils.general import (check_synapse_cache_size, - clear_synapse_cache, create_temp_folder, - entity_type_mapping, get_dir_size) +from schematic.utils.general import ( + check_synapse_cache_size, + clear_synapse_cache, + create_temp_folder, + entity_type_mapping, + get_dir_size, +) from schematic.utils.schema_utils import get_class_label_from_display_name -from schematic.utils.validate_utils import (comma_separated_list_regex, - rule_in_rule_list) +from schematic.utils.validate_utils import comma_separated_list_regex, rule_in_rule_list logger = logging.getLogger("Synapse storage") From 9d9f19b86ba086b17adc0e6a08f12362c6880e0a Mon Sep 17 00:00:00 2001 From: GiaJordan Date: Tue, 9 Jul 2024 13:17:19 -0700 Subject: [PATCH 022/233] add new rule --- schematic/utils/validate_rules_utils.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/schematic/utils/validate_rules_utils.py b/schematic/utils/validate_rules_utils.py index e5192ce8a..a1308f617 100644 --- a/schematic/utils/validate_rules_utils.py +++ b/schematic/utils/validate_rules_utils.py @@ -1,10 +1,10 @@ """validate rules utils""" import logging -from typing import TypedDict, Optional, Literal -from typing_extensions import assert_never -from jsonschema import ValidationError +from typing import Literal, Optional, TypedDict +from jsonschema import ValidationError +from typing_extensions import assert_never logger = logging.getLogger(__name__) @@ -149,6 +149,13 @@ def validation_rule_info() -> dict[str, Rule]: "default_message_level": None, "fixed_arg": None, }, + "filenameExists": { + "arguments": (0, 0), + "type": "content_validation", + "complementary_rules": None, + "default_message_level": "error", + "fixed_arg": None, + }, } From 0c858f00444fe8f0c55ffabe3e9f464a21b3d0a2 Mon Sep 17 00:00:00 2001 From: GiaJordan Date: Tue, 9 Jul 2024 13:18:37 -0700 Subject: [PATCH 023/233] add new rule to model --- tests/data/example.model.csv | 5 +++-- tests/data/example.model.jsonld | 29 ++++++++++++++++++++++++++++- 2 files changed, 31 insertions(+), 3 deletions(-) diff --git a/tests/data/example.model.csv b/tests/data/example.model.csv index a99a404f1..a85cf8cbf 100644 --- a/tests/data/example.model.csv +++ b/tests/data/example.model.csv @@ -12,7 +12,7 @@ Biospecimen,,,"Sample ID, Patient ID, Tissue Status, Component",,FALSE,DataType, Sample ID,,,,,TRUE,DataProperty,,, Tissue Status,,"Healthy, Malignant",,,TRUE,DataProperty,,, Bulk RNA-seq Assay,,,"Filename, Sample ID, File Format, Component",,FALSE,DataType,Biospecimen,, -Filename,,,,,TRUE,DataProperty,,, +Filename,,,,,TRUE,DataProperty,,,#MockFilename filenameExists^^ File Format,,"FASTQ, BAM, CRAM, CSV/TSV",,,TRUE,DataProperty,,, BAM,,,Genome Build,,FALSE,ValidValue,,, CRAM,,,"Genome Build, Genome FASTA",,FALSE,ValidValue,,, @@ -51,4 +51,5 @@ Check Date,,,,,TRUE,DataProperty,,,date Check NA,,,,,TRUE,DataProperty,,,int::IsNA MockRDB,,,"Component, MockRDB_id, SourceManifest",,FALSE,DataType,,, MockRDB_id,,,,,TRUE,DataProperty,,,int -SourceManifest,,,,,TRUE,DataProperty,,, \ No newline at end of file +SourceManifest,,,,,TRUE,DataProperty,,, +MockFilename,,,"Component, Filename",,FALSE,DataType,,, diff --git a/tests/data/example.model.jsonld b/tests/data/example.model.jsonld index 44cea61d5..3f13b188e 100644 --- a/tests/data/example.model.jsonld +++ b/tests/data/example.model.jsonld @@ -614,7 +614,9 @@ }, "sms:displayName": "Filename", "sms:required": "sms:true", - "sms:validationRules": [] + "sms:validationRules": { + "MockFilename": "filenameExists" + } }, { "@id": "bts:FileFormat", @@ -1719,6 +1721,31 @@ "sms:displayName": "SourceManifest", "sms:required": "sms:true", "sms:validationRules": [] + }, + { + "@id": "bts:MockFilename", + "@type": "rdfs:Class", + "rdfs:comment": "TBD", + "rdfs:label": "MockFilename", + "rdfs:subClassOf": [ + { + "@id": "bts:DataType" + } + ], + "schema:isPartOf": { + "@id": "http://schema.biothings.io" + }, + "sms:displayName": "MockFilename", + "sms:required": "sms:false", + "sms:requiresDependency": [ + { + "@id": "bts:Component" + }, + { + "@id": "bts:Filename" + } + ], + "sms:validationRules": [] } ], "@id": "http://schema.biothings.io/#0.1" From 64d7e41c6730b41136cfc79254483171dd6db190 Mon Sep 17 00:00:00 2001 From: GiaJordan Date: Tue, 9 Jul 2024 14:00:43 -0700 Subject: [PATCH 024/233] pull login ops into own method --- schematic/models/validate_attribute.py | 42 +++++++++++++------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/schematic/models/validate_attribute.py b/schematic/models/validate_attribute.py index 6ed613bae..d7f135865 100644 --- a/schematic/models/validate_attribute.py +++ b/schematic/models/validate_attribute.py @@ -4,7 +4,7 @@ from time import perf_counter # allows specifying explicit variable types -from typing import Any, Optional, Literal, Union +from typing import Any, Literal, Optional, Union from urllib import error from urllib.parse import urlparse from urllib.request import Request, urlopen @@ -12,22 +12,20 @@ import numpy as np import pandas as pd from jsonschema import ValidationError +from synapseclient.core.exceptions import SynapseNoCredentialsError from schematic.schemas.data_model_graph import DataModelGraphExplorer - from schematic.store.synapse import SynapseStorage from schematic.utils.validate_rules_utils import validation_rule_info from schematic.utils.validate_utils import ( comma_separated_list_regex, - parse_str_series_to_list, - np_array_to_str_list, + get_list_robustness, iterable_to_str_list, + np_array_to_str_list, + parse_str_series_to_list, rule_in_rule_list, - get_list_robustness, ) -from synapseclient.core.exceptions import SynapseNoCredentialsError - logger = logging.getLogger(__name__) MessageLevelType = Literal["warning", "error"] @@ -731,6 +729,17 @@ class ValidateAttribute(object): def __init__(self, dmge: DataModelGraphExplorer) -> None: self.dmge = dmge + def _login(self, project_scope: list[str], access_token: str = None): + # login + try: + self.synStore = SynapseStorage( + access_token=access_token, project_scope=project_scope + ) + except SynapseNoCredentialsError as e: + raise ValueError( + "No Synapse credentials were provided. Credentials must be provided to utilize cross-manfiest validation functionality." + ) from e + def get_no_entry(self, entry: str, node_display_name: str) -> bool: """Helper function to check if the entry is blank or contains a not applicable type string (and NA is permitted) Args: @@ -777,22 +786,14 @@ def get_target_manifests( target_manifest_ids = [] target_dataset_ids = [] - # login - try: - synStore = SynapseStorage( - access_token=access_token, project_scope=project_scope - ) - except SynapseNoCredentialsError as e: - raise ValueError( - "No Synapse credentials were provided. Credentials must be provided to utilize cross-manfiest validation functionality." - ) from e + self._login(project_scope=project_scope, access_token=access_token) # Get list of all projects user has access to - projects = synStore.getStorageProjects(project_scope=project_scope) + projects = self.synStore.getStorageProjects(project_scope=project_scope) for project in projects: # get all manifests associated with datasets in the projects - target_datasets = synStore.getProjectManifests(projectId=project[0]) + target_datasets = self.synStore.getProjectManifests(projectId=project[0]) # If the manifest includes the target component, include synID in list for target_dataset in target_datasets: @@ -805,7 +806,7 @@ def get_target_manifests( logger.debug( f"Cross manifest gathering elapsed time {perf_counter()-t_manifest_search}" ) - return synStore, target_manifest_ids, target_dataset_ids + return target_manifest_ids, target_dataset_ids def list_validation( self, @@ -1779,7 +1780,6 @@ def _run_validation_across_target_manifests( # Get IDs of manifests with target component ( - synStore, target_manifest_ids, target_dataset_ids, ) = self.get_target_manifests(target_component, project_scope, access_token) @@ -1793,7 +1793,7 @@ def _run_validation_across_target_manifests( target_manifest_ids, target_dataset_ids ): # Pull manifest from Synapse - entity = synStore.getDatasetManifest( + entity = self.synStore.getDatasetManifest( datasetId=target_dataset_id, downloadFile=True ) # Load manifest From f0c2ec6c0cf05d2b88d8bcb40b2b0bd9c4680fc9 Mon Sep 17 00:00:00 2001 From: GiaJordan Date: Tue, 9 Jul 2024 14:02:31 -0700 Subject: [PATCH 025/233] add new rule method --- schematic/models/validate_attribute.py | 9 +++++++ schematic/models/validate_manifest.py | 34 ++++++++++++++------------ 2 files changed, 28 insertions(+), 15 deletions(-) diff --git a/schematic/models/validate_attribute.py b/schematic/models/validate_attribute.py index d7f135865..78912338c 100644 --- a/schematic/models/validate_attribute.py +++ b/schematic/models/validate_attribute.py @@ -1951,3 +1951,12 @@ def cross_validation( logger.debug(f"cross manifest validation time {perf_counter()-start_time}") return errors, warnings + + def content_validation( + self, + val_rule, + manifest, + project_scope, + access_token, + ): + return diff --git a/schematic/models/validate_manifest.py b/schematic/models/validate_manifest.py index 686842d20..cfdfc169c 100644 --- a/schematic/models/validate_manifest.py +++ b/schematic/models/validate_manifest.py @@ -1,35 +1,33 @@ import json -from statistics import mode -from tabnanny import check -from jsonschema import Draft7Validator, exceptions, ValidationError import logging - -import numpy as np import os -import pandas as pd import re import sys -from time import perf_counter from numbers import Number +from statistics import mode +from tabnanny import check +from time import perf_counter # allows specifying explicit variable types -from typing import Any, Dict, Optional, Text, List -from urllib.parse import urlparse -from urllib.request import urlopen, OpenerDirector, HTTPDefaultErrorHandler -from urllib.request import Request +from typing import Any, Dict, List, Optional, Text from urllib import error +from urllib.parse import urlparse +from urllib.request import HTTPDefaultErrorHandler, OpenerDirector, Request, urlopen -from schematic.models.validate_attribute import ValidateAttribute, GenerateError +import numpy as np +import pandas as pd +from jsonschema import Draft7Validator, ValidationError, exceptions +from schematic.models.GE_Helpers import GreatExpectationsHelpers +from schematic.models.validate_attribute import GenerateError, ValidateAttribute from schematic.schemas.data_model_graph import DataModelGraphExplorer from schematic.store.synapse import SynapseStorage -from schematic.models.GE_Helpers import GreatExpectationsHelpers +from schematic.utils.schema_utils import extract_component_validation_rules from schematic.utils.validate_rules_utils import validation_rule_info from schematic.utils.validate_utils import ( - rule_in_rule_list, convert_nan_entries_to_empty_strings, + rule_in_rule_list, ) -from schematic.utils.schema_utils import extract_component_validation_rules logger = logging.getLogger(__name__) @@ -153,6 +151,7 @@ def validate_manifest_rules( "matchAtLeastOne.*", "matchExactlyOne.*", "matchNone.*", + "filenameExists", ] in_house_rules = [ @@ -166,6 +165,7 @@ def validate_manifest_rules( "matchAtLeastOne.*", "matchExactlyOne.*", "matchNone.*", + "filenameExists", ] # initialize error and warning handling lists. @@ -270,6 +270,10 @@ def validate_manifest_rules( vr_errors, vr_warnings = validation_method( rule, manifest[col], project_scope, access_token ) + elif validation_type == "filenameExists": + vr_errors, vr_warnings = validation_method( + rule, manifest, project_scope, access_token + ) else: vr_errors, vr_warnings = validation_method( rule, From c1860fc77325efa321293cb38a9750acee47dbc4 Mon Sep 17 00:00:00 2001 From: GiaJordan Date: Tue, 9 Jul 2024 14:12:30 -0700 Subject: [PATCH 026/233] add login --- schematic/models/validate_attribute.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/schematic/models/validate_attribute.py b/schematic/models/validate_attribute.py index 78912338c..8840de3fe 100644 --- a/schematic/models/validate_attribute.py +++ b/schematic/models/validate_attribute.py @@ -1959,4 +1959,6 @@ def content_validation( project_scope, access_token, ): + self._login(project_scope=project_scope, access_token=access_token) + return From f01413c10c2c3cc14ad04b6100436290b278749a Mon Sep 17 00:00:00 2001 From: GiaJordan Date: Wed, 10 Jul 2024 15:23:23 -0700 Subject: [PATCH 027/233] add test for fileview query builder --- tests/test_store.py | 47 ++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 42 insertions(+), 5 deletions(-) diff --git a/tests/test_store.py b/tests/test_store.py index 5ac61f3d0..cec7d43ee 100644 --- a/tests/test_store.py +++ b/tests/test_store.py @@ -5,25 +5,24 @@ import logging import math import os +import shutil from time import sleep -from typing import Generator, Any +from typing import Any, Generator from unittest.mock import patch -import shutil import pandas as pd import pytest +from pandas.testing import assert_frame_equal from synapseclient import EntityViewSchema, Folder from synapseclient.entity import File -from pandas.testing import assert_frame_equal from schematic.configuration.configuration import Configuration from schematic.schemas.data_model_graph import DataModelGraph, DataModelGraphExplorer from schematic.schemas.data_model_parser import DataModelParser -from tests.conftest import Helpers - from schematic.store.base import BaseStorage from schematic.store.synapse import DatasetFileView, ManifestDownload, SynapseStorage from schematic.utils.general import check_synapse_cache_size +from tests.conftest import Helpers logging.basicConfig(level=logging.DEBUG) logger = logging.getLogger(__name__) @@ -152,6 +151,44 @@ def test_login(self) -> None: assert synapse_client.cache.cache_root_dir == "test_cache_dir" shutil.rmtree("test_cache_dir") + @pytest.mark.parametrize( + "project_scope,columns,where_clauses,expected", + [ + ([], [], [], "SELECT * FROM syn23643253 ;"), + ( + ["syn23643250"], + [], + [], + "SELECT * FROM syn23643253 WHERE projectId IN ('syn23643250', '') ;", + ), + ( + ["syn23643250"], + ["name", "id", "path"], + [], + "SELECT name,id,path FROM syn23643253 WHERE projectId IN ('syn23643250', '') ;", + ), + ( + ["syn23643250"], + ["name", "id", "path"], + ["parentId='syn61682648'", "type='file'"], + "SELECT name,id,path FROM syn23643253 WHERE parentId='syn61682648' AND type='file' AND projectId IN ('syn23643250', '') ;", + ), + ], + ) + def test_build_query( + self, + synapse_store: SynapseStorage, + project_scope: list, + columns: list, + where_clauses: list, + expected: str, + ) -> None: + assert synapse_store.storageFileview == "syn23643253" + if project_scope: + synapse_store.project_scope = project_scope + query = synapse_store._build_query(columns, where_clauses) + assert query == expected + def test_getFileAnnotations(self, synapse_store: SynapseStorage) -> None: expected_dict = { "author": "bruno, milen, sujay", From a1f02c0b2e40675e74a3dc46e0b73aa9876b74ea Mon Sep 17 00:00:00 2001 From: GiaJordan Date: Wed, 10 Jul 2024 15:33:07 -0700 Subject: [PATCH 028/233] add method for building fileview queries --- schematic/store/synapse.py | 125 +++++++++++++++++++++---------------- 1 file changed, 71 insertions(+), 54 deletions(-) diff --git a/schematic/store/synapse.py b/schematic/store/synapse.py index 688bbce4f..d4e893ed9 100644 --- a/schematic/store/synapse.py +++ b/schematic/store/synapse.py @@ -1,74 +1,69 @@ """Synapse storage class""" import atexit -from copy import deepcopy -from dataclasses import dataclass import logging -import numpy as np -import pandas as pd import os import re import secrets import shutil -import synapseclient import uuid # used to generate unique names for entities - -from tenacity import ( - retry, - stop_after_attempt, - wait_chain, - wait_fixed, - retry_if_exception_type, -) -from time import sleep +from copy import deepcopy +from dataclasses import dataclass +from sys import getsizeof +from time import perf_counter, sleep # allows specifying explicit variable types -from typing import Dict, List, Tuple, Sequence, Union, Optional +from typing import Dict, List, Optional, Sequence, Tuple, Union +import numpy as np +import pandas as pd +import synapseclient +import synapseutils +from opentelemetry import trace +from schematic_db.rdb.synapse_database import SynapseDatabase from synapseclient import ( - Synapse, + Column, + EntityViewSchema, + EntityViewType, File, Folder, - Table, Schema, - EntityViewSchema, - EntityViewType, - Column, + Synapse, + Table, as_table_columns, ) -from synapseclient.entity import File -from synapseclient.table import CsvFileTable, build_table, Schema from synapseclient.core.exceptions import ( - SynapseHTTPError, SynapseAuthenticationError, - SynapseUnmetAccessRestrictions, SynapseHTTPError, + SynapseUnmetAccessRestrictions, +) +from synapseclient.entity import File +from synapseclient.table import CsvFileTable, Schema, build_table +from tenacity import ( + retry, + retry_if_exception_type, + stop_after_attempt, + wait_chain, + wait_fixed, ) -import synapseutils - -from schematic_db.rdb.synapse_database import SynapseDatabase +from schematic.configuration.configuration import CONFIG +from schematic.exceptions import AccessCredentialsError from schematic.schemas.data_model_graph import DataModelGraphExplorer - -from schematic.utils.df_utils import update_df, load_df, col_in_dataframe -from schematic.utils.validate_utils import comma_separated_list_regex, rule_in_rule_list +from schematic.store.base import BaseStorage +from schematic.utils.df_utils import col_in_dataframe, load_df, update_df # entity_type_mapping, get_dir_size, create_temp_folder, check_synapse_cache_size, and clear_synapse_cache functions are used for AWS deployment # Please do not remove these import statements from schematic.utils.general import ( - entity_type_mapping, - get_dir_size, - create_temp_folder, check_synapse_cache_size, clear_synapse_cache, + create_temp_folder, + entity_type_mapping, + get_dir_size, ) - from schematic.utils.schema_utils import get_class_label_from_display_name - -from schematic.store.base import BaseStorage -from schematic.exceptions import AccessCredentialsError -from schematic.configuration.configuration import CONFIG -from opentelemetry import trace +from schematic.utils.validate_utils import comma_separated_list_regex, rule_in_rule_list logger = logging.getLogger("Synapse storage") @@ -218,7 +213,7 @@ def __init__( self.storageFileview = CONFIG.synapse_master_fileview_id self.manifest = CONFIG.synapse_manifest_basename self.root_synapse_cache = self.syn.cache.cache_root_dir - self._query_fileview() + self.query_fileview() def _purge_synapse_cache( self, maximum_storage_allowed_cache_gb: int = 1, minute_buffer: int = 15 @@ -251,24 +246,46 @@ def _purge_synapse_cache( # instead of guessing how much space that we left, print out .synapseCache here logger.info(f"the total size of .synapseCache is: {nbytes} bytes") - @tracer.start_as_current_span("SynapseStorage::_query_fileview") - def _query_fileview(self): + @tracer.start_as_current_span("SynapseStorage::query_fileview") + def query_fileview( + self, + columns: List = [], + where_clauses: List = [], + ): self._purge_synapse_cache() + + self.storageFileview = CONFIG.synapse_master_fileview_id + self.manifest = CONFIG.synapse_manifest_basename + + fileview_query = self._build_query(columns=columns, where_clauses=where_clauses) + try: - self.storageFileview = CONFIG.synapse_master_fileview_id - self.manifest = CONFIG.synapse_manifest_basename - if self.project_scope: - self.storageFileviewTable = self.syn.tableQuery( - f"SELECT * FROM {self.storageFileview} WHERE projectId IN {tuple(self.project_scope + [''])}" - ).asDataFrame() - else: - # get data in administrative fileview for this pipeline - self.storageFileviewTable = self.syn.tableQuery( - "SELECT * FROM " + self.storageFileview - ).asDataFrame() + self.storageFileviewTable = self.syn.tableQuery( + query=fileview_query, + ).asDataFrame() except SynapseHTTPError: raise AccessCredentialsError(self.storageFileview) + def _build_query(self, columns: list = [], where_clauses: list = []): + if self.project_scope: + project_scope_clause = f"projectId IN {tuple(self.project_scope + [''])}" + where_clauses.append(project_scope_clause) + + if where_clauses: + where_clauses = " AND ".join(where_clauses) + where_clauses = f"WHERE {where_clauses} ;" + else: + where_clauses = ";" + + if columns: + columns = ",".join(columns) + else: + columns = "*" + + query = f"SELECT {columns} FROM {self.storageFileview} {where_clauses}" + + return query + @staticmethod def login( synapse_cache_path: Optional[str] = None, @@ -2294,7 +2311,7 @@ def getDatasetProject(self, datasetId: str) -> str: # re-query if no datasets found if dataset_row.empty: sleep(5) - self._query_fileview() + self.query_fileview() # Subset main file view dataset_index = self.storageFileviewTable["id"] == datasetId dataset_row = self.storageFileviewTable[dataset_index] From e18d8ec566715e0373e20095dca7b24b27aec174 Mon Sep 17 00:00:00 2001 From: GiaJordan Date: Thu, 11 Jul 2024 10:17:21 -0700 Subject: [PATCH 029/233] only login once --- schematic/models/validate_attribute.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/schematic/models/validate_attribute.py b/schematic/models/validate_attribute.py index 8840de3fe..ba11446ca 100644 --- a/schematic/models/validate_attribute.py +++ b/schematic/models/validate_attribute.py @@ -786,7 +786,8 @@ def get_target_manifests( target_manifest_ids = [] target_dataset_ids = [] - self._login(project_scope=project_scope, access_token=access_token) + if not hasattr(self, "synStore"): + self._login(project_scope=project_scope, access_token=access_token) # Get list of all projects user has access to projects = self.synStore.getStorageProjects(project_scope=project_scope) @@ -1961,4 +1962,8 @@ def content_validation( ): self._login(project_scope=project_scope, access_token=access_token) + # filename in dataset? + + # filenames match with entity IDs in dataset + return From d59cca040343b9ff5e9ada8d7410c973fa58753c Mon Sep 17 00:00:00 2001 From: GiaJordan Date: Thu, 11 Jul 2024 10:26:50 -0700 Subject: [PATCH 030/233] change list param init --- schematic/store/synapse.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/schematic/store/synapse.py b/schematic/store/synapse.py index d4e893ed9..41d5c835d 100644 --- a/schematic/store/synapse.py +++ b/schematic/store/synapse.py @@ -249,8 +249,8 @@ def _purge_synapse_cache( @tracer.start_as_current_span("SynapseStorage::query_fileview") def query_fileview( self, - columns: List = [], - where_clauses: List = [], + columns: Optional[list] = None, + where_clauses: Optional[list] = None, ): self._purge_synapse_cache() @@ -266,7 +266,14 @@ def query_fileview( except SynapseHTTPError: raise AccessCredentialsError(self.storageFileview) - def _build_query(self, columns: list = [], where_clauses: list = []): + def _build_query( + self, columns: Optional[list] = None, where_clauses: Optional[list] = None + ): + if columns is None: + columns = [] + if where_clauses is None: + where_clauses = [] + if self.project_scope: project_scope_clause = f"projectId IN {tuple(self.project_scope + [''])}" where_clauses.append(project_scope_clause) From d71784a36fa8e04a5133ab9c1e0bc17ff310d0b8 Mon Sep 17 00:00:00 2001 From: GiaJordan Date: Thu, 11 Jul 2024 11:02:23 -0700 Subject: [PATCH 031/233] update schemas test --- tests/test_schemas.py | 65 ++++++++++++++++++++++++------------------- 1 file changed, 36 insertions(+), 29 deletions(-) diff --git a/tests/test_schemas.py b/tests/test_schemas.py index f80449b18..6870cbef5 100644 --- a/tests/test_schemas.py +++ b/tests/test_schemas.py @@ -1,46 +1,41 @@ -from copy import deepcopy import json import logging -import networkx as nx -import numpy as np import os -import pandas as pd -import pytest import random +from copy import deepcopy from typing import Optional -from schematic.schemas.data_model_edges import DataModelEdges -from schematic.schemas.data_model_nodes import DataModelNodes -from schematic.schemas.data_model_relationships import DataModelRelationships - -from schematic.utils.df_utils import load_df -from schematic.utils.schema_utils import ( - get_label_from_display_name, - get_attribute_display_name_from_label, - convert_bool_to_str, - parse_validation_rules, - DisplayLabelType, - get_json_schema_log_file_path, -) -from schematic.utils.io_utils import load_json +import networkx as nx +import numpy as np +import pandas as pd +import pytest -from schematic.schemas.data_model_graph import DataModelGraph -from schematic.schemas.data_model_nodes import DataModelNodes from schematic.schemas.data_model_edges import DataModelEdges -from schematic.schemas.data_model_graph import DataModelGraphExplorer -from schematic.schemas.data_model_relationships import DataModelRelationships +from schematic.schemas.data_model_graph import DataModelGraph, DataModelGraphExplorer +from schematic.schemas.data_model_json_schema import DataModelJSONSchema from schematic.schemas.data_model_jsonld import ( - DataModelJsonLD, - convert_graph_to_jsonld, BaseTemplate, - PropertyTemplate, ClassTemplate, + DataModelJsonLD, + PropertyTemplate, + convert_graph_to_jsonld, ) -from schematic.schemas.data_model_json_schema import DataModelJSONSchema +from schematic.schemas.data_model_nodes import DataModelNodes from schematic.schemas.data_model_parser import ( - DataModelParser, DataModelCSVParser, DataModelJSONLDParser, + DataModelParser, +) +from schematic.schemas.data_model_relationships import DataModelRelationships +from schematic.utils.df_utils import load_df +from schematic.utils.io_utils import load_json +from schematic.utils.schema_utils import ( + DisplayLabelType, + convert_bool_to_str, + get_attribute_display_name_from_label, + get_json_schema_log_file_path, + get_label_from_display_name, + parse_validation_rules, ) logging.basicConfig(level=logging.DEBUG) @@ -898,12 +893,24 @@ def test_run_rel_functions(self, helpers, data_model, rel_func, test_dn, test_bo if "::" in rule[0]: assert parsed_vrs[ind] == rule[0].split("::") elif "^^" in rule[0]: + component_with_specific_rules = [] component_rule_sets = rule[0].split("^^") components = [ cr.split(" ")[0].replace("#", "") for cr in component_rule_sets ] - assert components == [k for k in parsed_vrs[0].keys()] + if "" in components: + components.remove("") + for parsed_rule in parsed_vrs: + if isinstance(parsed_rule, dict): + for k in parsed_rule.keys(): + component_with_specific_rules.append(k) + assert all( + [ + component in component_with_specific_rules + for component in components + ] + ) else: assert parsed_vrs[ind] == rule elif DATA_MODEL_DICT[data_model] == "JSONLD": From 356eb62c0b9ef8e5d70278bb35a723515a2f7795 Mon Sep 17 00:00:00 2001 From: linglp Date: Thu, 11 Jul 2024 14:11:51 -0400 Subject: [PATCH 032/233] edit helper --- schematic/models/GE_Helpers.py | 55 +++++++++++++++++++++------------- tests/test_ge_helpers.py | 54 +++++++++++++++++++++++++++++++++ 2 files changed, 89 insertions(+), 20 deletions(-) create mode 100644 tests/test_ge_helpers.py diff --git a/schematic/models/GE_Helpers.py b/schematic/models/GE_Helpers.py index d73354f31..1323fe769 100644 --- a/schematic/models/GE_Helpers.py +++ b/schematic/models/GE_Helpers.py @@ -1,21 +1,17 @@ -from statistics import mode -from tabnanny import check import logging import os import re -import numpy as np +from statistics import mode +from tabnanny import check # allows specifying explicit variable types -from typing import Any, Dict, Optional, Text, List -from urllib.parse import urlparse -from urllib.request import urlopen, OpenerDirector, HTTPDefaultErrorHandler -from urllib.request import Request +from typing import Any, Dict, List, Optional, Text from urllib import error -from attr import attr - -from ruamel import yaml +from urllib.parse import urlparse +from urllib.request import HTTPDefaultErrorHandler, OpenerDirector, Request, urlopen -import great_expectations as ge +import numpy as np +from attr import attr from great_expectations.core.expectation_configuration import ExpectationConfiguration from great_expectations.data_context import BaseDataContext from great_expectations.data_context.types.base import ( @@ -27,18 +23,17 @@ ExpectationSuiteIdentifier, ) from great_expectations.exceptions.exceptions import GreatExpectationsError +from ruamel import yaml - +import great_expectations as ge from schematic.models.validate_attribute import GenerateError from schematic.schemas.data_model_graph import DataModelGraphExplorer - from schematic.utils.schema_utils import extract_component_validation_rules - from schematic.utils.validate_utils import ( - rule_in_rule_list, - np_array_to_str_list, iterable_to_str_list, + np_array_to_str_list, required_is_only_rule, + rule_in_rule_list, ) logger = logging.getLogger(__name__) @@ -147,6 +142,29 @@ def build_context(self): # self.context.test_yaml_config(yaml.dump(datasource_config)) self.context.add_datasource(**datasource_config) + def add_expectation_suite_if_not_exists(self): + """ + Purpose: + Add expectation suite if it does not exist + Input: + Returns: + saves expectation suite and identifier to self + """ + self.expectation_suite_name = "Manifest_test_suite" + suite_names = self.context.list_expectation_suite_names() + print("suite name", suite_names) + if self.expectation_suite_name not in suite_names: + self.suite = self.context.add_expectation_suite( + expectation_suite_name=self.expectation_suite_name, + ) + # in gh actions, sometimes the suite has already been added. + # if that's the case, get the existing one + else: + self.suite = self.context.get_expectation_suite( + expectation_suite_name=self.expectation_suite_name + ) + return self.suite + def build_expectation_suite( self, ): @@ -162,10 +180,7 @@ def build_expectation_suite( """ # create blank expectation suite - self.expectation_suite_name = "Manifest_test_suite" - self.suite = self.context.add_expectation_suite( - expectation_suite_name=self.expectation_suite_name, - ) + self.suite = self.add_expectation_suite_if_not_exists() # build expectation configurations for each expectation for col in self.manifest.columns: diff --git a/tests/test_ge_helpers.py b/tests/test_ge_helpers.py new file mode 100644 index 000000000..90d103e30 --- /dev/null +++ b/tests/test_ge_helpers.py @@ -0,0 +1,54 @@ +from unittest.mock import MagicMock + +import pandas as pd +import pytest + +from schematic.models.GE_Helpers import GreatExpectationsHelpers + + +@pytest.fixture(scope="class") +def mock_ge_helpers(helpers): + dmge = helpers.get_data_model_graph_explorer(path="example.model.jsonld") + unimplemented_expectations = ["url"] + test_manifest_path = helpers.get_data_path("mock_manifests/Valid_Test_Manifest.csv") + manifest = pd.read_csv(test_manifest_path) + + ge_helpers = GreatExpectationsHelpers( + dmge=dmge, + unimplemented_expectations=unimplemented_expectations, + manifest=manifest, + manifestPath=test_manifest_path, + ) + yield ge_helpers + + +class TestGreatExpectationsHelpers: + def test_add_expectation_suite_if_not_exists_does_not_exist(self, mock_ge_helpers): + # mock context provided by ge_helpers + mock_ge_helpers.context = MagicMock() + mock_ge_helpers.context.list_expectation_suite_names.return_value = [] + + # Call the method + result = mock_ge_helpers.add_expectation_suite_if_not_exists() + + # Make sure the method of creating expectation suites if it doesn't exist + mock_ge_helpers.context.list_expectation_suite_names.assert_called_once() + mock_ge_helpers.context.add_expectation_suite.assert_called_once_with( + expectation_suite_name="Manifest_test_suite" + ) + + def test_add_expectation_suite_if_not_exists_does_exist(self, mock_ge_helpers): + # mock context provided by ge_helpers + mock_ge_helpers.context = MagicMock() + mock_ge_helpers.context.list_expectation_suite_names.return_value = [ + "Manifest_test_suite" + ] + + # Call the method + result = mock_ge_helpers.add_expectation_suite_if_not_exists() + + # Make sure the method of getting existing suites gets called + mock_ge_helpers.context.list_expectation_suite_names.assert_called_once() + mock_ge_helpers.context.get_expectation_suite.assert_called_once_with( + expectation_suite_name="Manifest_test_suite" + ) From 98d566eda369796afc8ae61dd3903c4ede36d594 Mon Sep 17 00:00:00 2001 From: GiaJordan Date: Thu, 11 Jul 2024 11:17:00 -0700 Subject: [PATCH 033/233] add docstrings --- schematic/store/synapse.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/schematic/store/synapse.py b/schematic/store/synapse.py index 41d5c835d..5a4fafef7 100644 --- a/schematic/store/synapse.py +++ b/schematic/store/synapse.py @@ -251,7 +251,14 @@ def query_fileview( self, columns: Optional[list] = None, where_clauses: Optional[list] = None, - ): + ) -> None: + """ + Method to query the Synapse FileView and store the results in a pandas DataFrame. The results are stored in the storageFileviewTable attribute. + Is called once during initialization of the SynapseStorage object and can be called again later to specify a specific, more limited scope for validation purposes. + Args: + columns (Optional[list], optional): List of columns to be selected from the table. Defaults behavior is to request all columns. + where_clauses (Optional[list], optional): List of where clauses to be used to scope the query. Defaults to None. + """ self._purge_synapse_cache() self.storageFileview = CONFIG.synapse_master_fileview_id @@ -269,6 +276,16 @@ def query_fileview( def _build_query( self, columns: Optional[list] = None, where_clauses: Optional[list] = None ): + """ + Method to build a query for Synapse FileViews + Args: + columns (Optional[list], optional): List of columns to be selected from the table. Defaults behavior is to request all columns. + where_clauses (Optional[list], optional): List of where clauses to be used to scope the query. Defaults to None. + self.storageFileview (str): Synapse FileView ID + self.project_scope (Optional[list], optional): List of project IDs to be used to scope the query. Defaults to None. Gets added to where_clauses, more included for backwards compatability and as a more user friendly way of subsetting the view in a simple way. + Returns: + query (str): A query string to be used to query the Synapse FileView + """ if columns is None: columns = [] if where_clauses is None: From e88c1f479e0b4dc742be092f0278533f3985d4d7 Mon Sep 17 00:00:00 2001 From: GiaJordan Date: Thu, 11 Jul 2024 11:17:25 -0700 Subject: [PATCH 034/233] add docstrings --- schematic/store/synapse.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/schematic/store/synapse.py b/schematic/store/synapse.py index 5a4fafef7..5626b1a1c 100644 --- a/schematic/store/synapse.py +++ b/schematic/store/synapse.py @@ -282,7 +282,8 @@ def _build_query( columns (Optional[list], optional): List of columns to be selected from the table. Defaults behavior is to request all columns. where_clauses (Optional[list], optional): List of where clauses to be used to scope the query. Defaults to None. self.storageFileview (str): Synapse FileView ID - self.project_scope (Optional[list], optional): List of project IDs to be used to scope the query. Defaults to None. Gets added to where_clauses, more included for backwards compatability and as a more user friendly way of subsetting the view in a simple way. + self.project_scope (Optional[list], optional): List of project IDs to be used to scope the query. Defaults to None. + Gets added to where_clauses, more included for backwards compatability and as a more user friendly way of subsetting the view in a simple way. Returns: query (str): A query string to be used to query the Synapse FileView """ From 706669dcf8b178810ccec66bd83abc5a47031a10 Mon Sep 17 00:00:00 2001 From: GiaJordan Date: Thu, 11 Jul 2024 11:21:28 -0700 Subject: [PATCH 035/233] update rule args --- schematic/utils/validate_rules_utils.py | 2 +- tests/data/example.model.csv | 2 +- tests/data/example.model.jsonld | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/schematic/utils/validate_rules_utils.py b/schematic/utils/validate_rules_utils.py index a1308f617..189b62635 100644 --- a/schematic/utils/validate_rules_utils.py +++ b/schematic/utils/validate_rules_utils.py @@ -150,7 +150,7 @@ def validation_rule_info() -> dict[str, Rule]: "fixed_arg": None, }, "filenameExists": { - "arguments": (0, 0), + "arguments": (1, 1), "type": "content_validation", "complementary_rules": None, "default_message_level": "error", diff --git a/tests/data/example.model.csv b/tests/data/example.model.csv index a85cf8cbf..32bbb9977 100644 --- a/tests/data/example.model.csv +++ b/tests/data/example.model.csv @@ -12,7 +12,7 @@ Biospecimen,,,"Sample ID, Patient ID, Tissue Status, Component",,FALSE,DataType, Sample ID,,,,,TRUE,DataProperty,,, Tissue Status,,"Healthy, Malignant",,,TRUE,DataProperty,,, Bulk RNA-seq Assay,,,"Filename, Sample ID, File Format, Component",,FALSE,DataType,Biospecimen,, -Filename,,,,,TRUE,DataProperty,,,#MockFilename filenameExists^^ +Filename,,,,,TRUE,DataProperty,,,#MockFilename filenameExists syn61682648^^ File Format,,"FASTQ, BAM, CRAM, CSV/TSV",,,TRUE,DataProperty,,, BAM,,,Genome Build,,FALSE,ValidValue,,, CRAM,,,"Genome Build, Genome FASTA",,FALSE,ValidValue,,, diff --git a/tests/data/example.model.jsonld b/tests/data/example.model.jsonld index 3f13b188e..1c7910d3c 100644 --- a/tests/data/example.model.jsonld +++ b/tests/data/example.model.jsonld @@ -615,7 +615,7 @@ "sms:displayName": "Filename", "sms:required": "sms:true", "sms:validationRules": { - "MockFilename": "filenameExists" + "MockFilename": "filenameExists syn61682648" } }, { From a708a859e892764b140ebf81f19f73ff7d022113 Mon Sep 17 00:00:00 2001 From: linglp Date: Thu, 11 Jul 2024 16:11:14 -0400 Subject: [PATCH 036/233] remove print statement --- schematic/models/GE_Helpers.py | 1 - 1 file changed, 1 deletion(-) diff --git a/schematic/models/GE_Helpers.py b/schematic/models/GE_Helpers.py index 1323fe769..50f91912a 100644 --- a/schematic/models/GE_Helpers.py +++ b/schematic/models/GE_Helpers.py @@ -152,7 +152,6 @@ def add_expectation_suite_if_not_exists(self): """ self.expectation_suite_name = "Manifest_test_suite" suite_names = self.context.list_expectation_suite_names() - print("suite name", suite_names) if self.expectation_suite_name not in suite_names: self.suite = self.context.add_expectation_suite( expectation_suite_name=self.expectation_suite_name, From 3dc1f6cbc907246253a070d322b663800f118cfe Mon Sep 17 00:00:00 2001 From: linglp Date: Thu, 11 Jul 2024 16:43:00 -0400 Subject: [PATCH 037/233] add return type ExpectationSuite --- schematic/models/GE_Helpers.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/schematic/models/GE_Helpers.py b/schematic/models/GE_Helpers.py index 50f91912a..99c3728f5 100644 --- a/schematic/models/GE_Helpers.py +++ b/schematic/models/GE_Helpers.py @@ -12,6 +12,7 @@ import numpy as np from attr import attr +from great_expectations.core import ExpectationSuite from great_expectations.core.expectation_configuration import ExpectationConfiguration from great_expectations.data_context import BaseDataContext from great_expectations.data_context.types.base import ( @@ -142,7 +143,7 @@ def build_context(self): # self.context.test_yaml_config(yaml.dump(datasource_config)) self.context.add_datasource(**datasource_config) - def add_expectation_suite_if_not_exists(self): + def add_expectation_suite_if_not_exists(self) -> ExpectationSuite: """ Purpose: Add expectation suite if it does not exist From f95fa0d0a4885bc41be8f25ac7a6f985acc8b5e5 Mon Sep 17 00:00:00 2001 From: linglp Date: Thu, 11 Jul 2024 17:18:03 -0400 Subject: [PATCH 038/233] add type hints --- tests/test_ge_helpers.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/tests/test_ge_helpers.py b/tests/test_ge_helpers.py index 90d103e30..bd6bc9187 100644 --- a/tests/test_ge_helpers.py +++ b/tests/test_ge_helpers.py @@ -1,13 +1,18 @@ +from typing import Generator from unittest.mock import MagicMock import pandas as pd import pytest from schematic.models.GE_Helpers import GreatExpectationsHelpers +from tests.conftest import Helpers @pytest.fixture(scope="class") -def mock_ge_helpers(helpers): +def mock_ge_helpers( + helpers: Helpers, +) -> Generator[GreatExpectationsHelpers, None, None]: + """Fixture for creating a GreatExpectationsHelpers object""" dmge = helpers.get_data_model_graph_explorer(path="example.model.jsonld") unimplemented_expectations = ["url"] test_manifest_path = helpers.get_data_path("mock_manifests/Valid_Test_Manifest.csv") @@ -23,7 +28,10 @@ def mock_ge_helpers(helpers): class TestGreatExpectationsHelpers: - def test_add_expectation_suite_if_not_exists_does_not_exist(self, mock_ge_helpers): + def test_add_expectation_suite_if_not_exists_does_not_exist( + self, mock_ge_helpers: Generator[GreatExpectationsHelpers, None, None] + ) -> None: + """test add_expectation_suite_if_not_exists method when the expectation suite does not exists""" # mock context provided by ge_helpers mock_ge_helpers.context = MagicMock() mock_ge_helpers.context.list_expectation_suite_names.return_value = [] @@ -37,7 +45,10 @@ def test_add_expectation_suite_if_not_exists_does_not_exist(self, mock_ge_helper expectation_suite_name="Manifest_test_suite" ) - def test_add_expectation_suite_if_not_exists_does_exist(self, mock_ge_helpers): + def test_add_expectation_suite_if_not_exists_does_exist( + self, mock_ge_helpers: Generator[GreatExpectationsHelpers, None, None] + ) -> None: + """test add_expectation_suite_if_not_exists method when the expectation suite does exists""" # mock context provided by ge_helpers mock_ge_helpers.context = MagicMock() mock_ge_helpers.context.list_expectation_suite_names.return_value = [ From 23c064d2d2352ac36ff38d662a38d15014e299b4 Mon Sep 17 00:00:00 2001 From: GiaJordan Date: Thu, 11 Jul 2024 14:20:43 -0700 Subject: [PATCH 039/233] save query string --- schematic/store/synapse.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/schematic/store/synapse.py b/schematic/store/synapse.py index 5626b1a1c..689e9f0c3 100644 --- a/schematic/store/synapse.py +++ b/schematic/store/synapse.py @@ -264,11 +264,11 @@ def query_fileview( self.storageFileview = CONFIG.synapse_master_fileview_id self.manifest = CONFIG.synapse_manifest_basename - fileview_query = self._build_query(columns=columns, where_clauses=where_clauses) + self._build_query(columns=columns, where_clauses=where_clauses) try: self.storageFileviewTable = self.syn.tableQuery( - query=fileview_query, + query=self.fileview_query, ).asDataFrame() except SynapseHTTPError: raise AccessCredentialsError(self.storageFileview) @@ -284,8 +284,6 @@ def _build_query( self.storageFileview (str): Synapse FileView ID self.project_scope (Optional[list], optional): List of project IDs to be used to scope the query. Defaults to None. Gets added to where_clauses, more included for backwards compatability and as a more user friendly way of subsetting the view in a simple way. - Returns: - query (str): A query string to be used to query the Synapse FileView """ if columns is None: columns = [] @@ -307,9 +305,11 @@ def _build_query( else: columns = "*" - query = f"SELECT {columns} FROM {self.storageFileview} {where_clauses}" + self.fileview_query = ( + f"SELECT {columns} FROM {self.storageFileview} {where_clauses}" + ) - return query + return @staticmethod def login( From 1c6fa51701988d80fda5695eea984dacd8b030b3 Mon Sep 17 00:00:00 2001 From: GiaJordan Date: Thu, 11 Jul 2024 14:21:38 -0700 Subject: [PATCH 040/233] change fixutre for query test --- tests/test_store.py | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/tests/test_store.py b/tests/test_store.py index cec7d43ee..e55308c13 100644 --- a/tests/test_store.py +++ b/tests/test_store.py @@ -113,6 +113,17 @@ def dmge( yield dmge +@pytest.fixture +def synapse_store_special_scope(request): + access_token = os.getenv("SYNAPSE_ACCESS_TOKEN") + if access_token: + synapse_store = SynapseStorage(access_token=access_token) + else: + synapse_store = SynapseStorage() + + yield synapse_store + + def raise_final_error(retry_state): return retry_state.outcome.result() @@ -177,17 +188,17 @@ def test_login(self) -> None: ) def test_build_query( self, - synapse_store: SynapseStorage, + synapse_store_special_scope: SynapseStorage, project_scope: list, columns: list, where_clauses: list, expected: str, ) -> None: - assert synapse_store.storageFileview == "syn23643253" + assert synapse_store_special_scope.storageFileview == "syn23643253" if project_scope: - synapse_store.project_scope = project_scope - query = synapse_store._build_query(columns, where_clauses) - assert query == expected + synapse_store_special_scope.project_scope = project_scope + synapse_store_special_scope._build_query(columns, where_clauses) + assert synapse_store_special_scope.fileview_query == expected def test_getFileAnnotations(self, synapse_store: SynapseStorage) -> None: expected_dict = { From d96f097da343cb71b31c81bc616c6e49ff04189c Mon Sep 17 00:00:00 2001 From: GiaJordan Date: Thu, 11 Jul 2024 15:22:38 -0700 Subject: [PATCH 041/233] update filepaths for manifests --- schematic/store/synapse.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/schematic/store/synapse.py b/schematic/store/synapse.py index 689e9f0c3..9d5f3080d 100644 --- a/schematic/store/synapse.py +++ b/schematic/store/synapse.py @@ -526,6 +526,8 @@ def getFilesInStorageDataset( self.syn, datasetId, includeTypes=["folder", "file"] ) + project = self.getDatasetProject(datasetId) + project_name = self.syn.get(project, downloadFile=False).name file_list = [] # iterate over all results @@ -541,7 +543,10 @@ def getFilesInStorageDataset( if fullpath: # append directory path to filename - filename = (dirpath[0] + "/" + filename[0], filename[1]) + filename = ( + project_name + "/" + dirpath[0] + "/" + filename[0], + filename[1], + ) # add file name file id tuple, rearranged so that id is first and name follows file_list.append(filename[::-1]) From 2121ca9e58d6ee24a3b4eb4b966dbc71ba6ffb49 Mon Sep 17 00:00:00 2001 From: GiaJordan Date: Thu, 11 Jul 2024 15:23:08 -0700 Subject: [PATCH 042/233] update manifest tests with new paths --- tests/test_manifest.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/tests/test_manifest.py b/tests/test_manifest.py index c34148dab..5829dcde1 100644 --- a/tests/test_manifest.py +++ b/tests/test_manifest.py @@ -1,16 +1,16 @@ +import logging import os import shutil -import logging -import pytest +from unittest.mock import MagicMock, Mock, patch + import pandas as pd -from unittest.mock import Mock -from unittest.mock import patch -from unittest.mock import MagicMock +import pytest + +from schematic.configuration.configuration import Configuration from schematic.manifest.generator import ManifestGenerator -from schematic.schemas.data_model_parser import DataModelParser from schematic.schemas.data_model_graph import DataModelGraph, DataModelGraphExplorer from schematic.schemas.data_model_json_schema import DataModelJSONSchema -from schematic.configuration.configuration import Configuration +from schematic.schemas.data_model_parser import DataModelParser from schematic.utils.google_api_utils import execute_google_api_requests from schematic_api.api import create_app @@ -213,9 +213,9 @@ def test_get_manifest_first_time(self, manifest): # Confirm contents of Filename column assert output["Filename"].tolist() == [ - "TestDataset-Annotations-v3/Sample_A.txt", - "TestDataset-Annotations-v3/Sample_B.txt", - "TestDataset-Annotations-v3/Sample_C.txt", + "schematic - main/TestDataset-Annotations-v3/Sample_A.txt", + "schematic - main/TestDataset-Annotations-v3/Sample_B.txt", + "schematic - main/TestDataset-Annotations-v3/Sample_C.txt", ] # Test dimensions of data frame From f85d0d828c03f61e72fa1d05d6fcab0254a86d1c Mon Sep 17 00:00:00 2001 From: GiaJordan Date: Thu, 11 Jul 2024 15:29:25 -0700 Subject: [PATCH 043/233] update params --- tests/test_store.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/test_store.py b/tests/test_store.py index e55308c13..948c44fa8 100644 --- a/tests/test_store.py +++ b/tests/test_store.py @@ -165,17 +165,17 @@ def test_login(self) -> None: @pytest.mark.parametrize( "project_scope,columns,where_clauses,expected", [ - ([], [], [], "SELECT * FROM syn23643253 ;"), + (None, None, None, "SELECT * FROM syn23643253 ;"), ( ["syn23643250"], - [], - [], + None, + None, "SELECT * FROM syn23643253 WHERE projectId IN ('syn23643250', '') ;", ), ( ["syn23643250"], ["name", "id", "path"], - [], + None, "SELECT name,id,path FROM syn23643253 WHERE projectId IN ('syn23643250', '') ;", ), ( From 06b73a66a1e3c21b9c65f102e4b8a3c593312727 Mon Sep 17 00:00:00 2001 From: GiaJordan Date: Fri, 12 Jul 2024 10:20:35 -0700 Subject: [PATCH 044/233] update paths in filebased test manifests --- ...lkRNAseq_component_based_required_rule_test.manifest.csv | 6 +++--- tests/data/mock_manifests/test_BulkRNAseq.csv | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/data/mock_manifests/BulkRNAseq_component_based_required_rule_test.manifest.csv b/tests/data/mock_manifests/BulkRNAseq_component_based_required_rule_test.manifest.csv index d0cd7a01c..9818cb966 100644 --- a/tests/data/mock_manifests/BulkRNAseq_component_based_required_rule_test.manifest.csv +++ b/tests/data/mock_manifests/BulkRNAseq_component_based_required_rule_test.manifest.csv @@ -1,4 +1,4 @@ Filename,Sample ID,File Format,Component,Genome Build,Genome FASTA -test/file/name.png,fake_id,BAM,BulkRNA-seqAssay,GRCh37, -test/file/name_2.png,fake_id_2,CRAM,BulkRNA-seqAssay,GRCh38, -test/file/name_3.png,fake_id_3,,BulkRNA-seqAssay,, \ No newline at end of file +schematic - main/test/file/name.png,fake_id,BAM,BulkRNA-seqAssay,GRCh37, +schematic - main/test/file/name_2.png,fake_id_2,CRAM,BulkRNA-seqAssay,GRCh38, +schematic - main/test/file/name_3.png,fake_id_3,,BulkRNA-seqAssay,, diff --git a/tests/data/mock_manifests/test_BulkRNAseq.csv b/tests/data/mock_manifests/test_BulkRNAseq.csv index facfa3f6a..3dfddf90d 100644 --- a/tests/data/mock_manifests/test_BulkRNAseq.csv +++ b/tests/data/mock_manifests/test_BulkRNAseq.csv @@ -1,3 +1,3 @@ Filename,Sample ID,File Format,Component,Genome Build,Genome FASTA -TestRNA-seqDataset1/TestRNA-seq-dummy-dataset.rtf,ABCD,BAM,BulkRNA-seqAssay,GRCh38, -TestRNA-seqDataset1/TestRNA-seq-dummy-dataset2.rtf,EFGH,CRAM,BulkRNA-seqAssay,GRCm39, +schematic - main/TestRNA-seqDataset1/TestRNA-seq-dummy-dataset.rtf,ABCD,BAM,BulkRNA-seqAssay,GRCh38, +schematic - main/TestRNA-seqDataset1/TestRNA-seq-dummy-dataset2.rtf,EFGH,CRAM,BulkRNA-seqAssay,GRCm39, From 2a9b30d18a1c3656c43a7b4cd83ef461d7a7c598 Mon Sep 17 00:00:00 2001 From: GiaJordan Date: Fri, 12 Jul 2024 10:27:28 -0700 Subject: [PATCH 045/233] update paths in tests --- tests/test_store.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_store.py b/tests/test_store.py index 948c44fa8..6304ac24a 100644 --- a/tests/test_store.py +++ b/tests/test_store.py @@ -315,7 +315,7 @@ def test_getDatasetAnnotations(self, dataset_id, synapse_store, force_batch): expected_df = pd.DataFrame.from_records( [ { - "Filename": "TestDataset-Annotations-v3/Sample_A.txt", + "Filename": "schematic - main/TestDataset-Annotations-v3/Sample_A.txt", "author": "bruno, milen, sujay", "impact": "42.9", "confidence": "high", @@ -325,13 +325,13 @@ def test_getDatasetAnnotations(self, dataset_id, synapse_store, force_batch): "IsImportantText": "TRUE", }, { - "Filename": "TestDataset-Annotations-v3/Sample_B.txt", + "Filename": "schematic - main/TestDataset-Annotations-v3/Sample_B.txt", "confidence": "low", "FileFormat": "csv", "date": "2020-02-01", }, { - "Filename": "TestDataset-Annotations-v3/Sample_C.txt", + "Filename": "schematic - main/TestDataset-Annotations-v3/Sample_C.txt", "FileFormat": "fastq", "IsImportantBool": "False", "IsImportantText": "FALSE", From a86865b1171e25f15e5d1f6a44e01a0d4392fe94 Mon Sep 17 00:00:00 2001 From: GiaJordan Date: Fri, 12 Jul 2024 11:16:24 -0700 Subject: [PATCH 046/233] update files in dataset test --- tests/test_store.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/tests/test_store.py b/tests/test_store.py index 6304ac24a..20ed7c21a 100644 --- a/tests/test_store.py +++ b/tests/test_store.py @@ -366,8 +366,11 @@ def test_getDatasetProject(self, dataset_id, synapse_store): ( True, [ - ("syn126", "parent_folder/test_file"), - ("syn125", "parent_folder/test_folder/test_file_2"), + ("syn126", "schematic - main/parent_folder/test_file"), + ( + "syn125", + "schematic - main/parent_folder/test_folder/test_file_2", + ), ], ), (False, [("syn126", "test_file"), ("syn125", "test_file_2")]), @@ -386,11 +389,18 @@ def test_getFilesInStorageDataset(self, synapse_store, full_path, expected): [("test_file_2", "syn125")], ), ] - with patch("synapseutils.walk_functions._help_walk", return_value=mock_return): + with patch( + "synapseutils.walk_functions._help_walk", return_value=mock_return + ) as mock_walk_patch, patch( + "schematic.store.synapse.SynapseStorage.getDatasetProject", + return_value="syn23643250", + ) as mock_project_id_patch, patch( + "synapseclient.entity.Entity.__getattr__", return_value="schematic - main" + ) as mock_project_name_patch: file_list = synapse_store.getFilesInStorageDataset( datasetId="syn_mock", fileNames=None, fullpath=full_path ) - assert file_list == expected + assert file_list == expected @pytest.mark.parametrize("downloadFile", [True, False]) def test_getDatasetManifest(self, synapse_store, downloadFile): From c1700064d59b9c1297e7d92390e54c51e4763d1a Mon Sep 17 00:00:00 2001 From: GiaJordan Date: Mon, 15 Jul 2024 09:45:30 -0700 Subject: [PATCH 047/233] change rule group --- schematic/models/validate_attribute.py | 8 +------- schematic/utils/validate_rules_utils.py | 2 +- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/schematic/models/validate_attribute.py b/schematic/models/validate_attribute.py index ba11446ca..c9851da49 100644 --- a/schematic/models/validate_attribute.py +++ b/schematic/models/validate_attribute.py @@ -1953,17 +1953,11 @@ def cross_validation( return errors, warnings - def content_validation( + def filename_validation( self, val_rule, manifest, project_scope, access_token, ): - self._login(project_scope=project_scope, access_token=access_token) - - # filename in dataset? - - # filenames match with entity IDs in dataset - return diff --git a/schematic/utils/validate_rules_utils.py b/schematic/utils/validate_rules_utils.py index 189b62635..5da5510ab 100644 --- a/schematic/utils/validate_rules_utils.py +++ b/schematic/utils/validate_rules_utils.py @@ -151,7 +151,7 @@ def validation_rule_info() -> dict[str, Rule]: }, "filenameExists": { "arguments": (1, 1), - "type": "content_validation", + "type": "filename_validation", "complementary_rules": None, "default_message_level": "error", "fixed_arg": None, From 17381b1f86d493fc5ab0d2e75fdad7a33cb09faa Mon Sep 17 00:00:00 2001 From: Gianna Jordan <61707471+GiaJordan@users.noreply.github.com> Date: Tue, 16 Jul 2024 10:22:48 -0700 Subject: [PATCH 048/233] run google creds tests --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 0b1a152ef..e12303a7e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -122,7 +122,7 @@ jobs: run: > source .venv/bin/activate; pytest --durations=0 --cov-report=term --cov-report=html:htmlcov --cov=schematic/ - -m "not (google_credentials_needed or schematic_api or table_operations)" --reruns 2 -n auto + -m "not (schematic_api or table_operations)" --reruns 2 -n auto - name: Upload pytest test results uses: actions/upload-artifact@v2 From b28b859f982f7681ed1f692da8caa38268fde341 Mon Sep 17 00:00:00 2001 From: linglp Date: Wed, 17 Jul 2024 11:03:50 -0400 Subject: [PATCH 049/233] update to use client 4.3.1 --- poetry.lock | 34 ++++------------------------------ pyproject.toml | 2 +- 2 files changed, 5 insertions(+), 31 deletions(-) diff --git a/poetry.lock b/poetry.lock index efab907c7..5e8cb5f28 100644 --- a/poetry.lock +++ b/poetry.lock @@ -585,17 +585,6 @@ files = [ click = ">=4.0" PyYAML = ">=3.11" -[[package]] -name = "cloudpickle" -version = "3.0.0" -description = "Pickler class to extend the standard pickle.Pickler functionality" -optional = false -python-versions = ">=3.8" -files = [ - {file = "cloudpickle-3.0.0-py3-none-any.whl", hash = "sha256:246ee7d0c295602a036e86369c77fecda4ab17b506496730f2f576d9016fd9c7"}, - {file = "cloudpickle-3.0.0.tar.gz", hash = "sha256:996d9a482c6fb4f33c1a35335cf8afd065d2a56e973270364840712d9131a882"}, -] - [[package]] name = "colorama" version = "0.4.6" @@ -2091,20 +2080,6 @@ files = [ {file = "lazy_object_proxy-1.10.0-pp310.pp311.pp312.pp38.pp39-none-any.whl", hash = "sha256:80fa48bd89c8f2f456fc0765c11c23bf5af827febacd2f523ca5bc1893fcc09d"}, ] -[[package]] -name = "loky" -version = "3.0.0" -description = "A robust implementation of concurrent.futures.ProcessPoolExecutor" -optional = false -python-versions = "*" -files = [ - {file = "loky-3.0.0-py2.py3-none-any.whl", hash = "sha256:1d5a4d778c7ff09c919aa3fbf2d879a2c7ac936a545c615af40e080a1c902b82"}, - {file = "loky-3.0.0.tar.gz", hash = "sha256:fd8750b24b283a579bafaf0631d114aa4487c682aef6fce01fa3635336297fdf"}, -] - -[package.dependencies] -cloudpickle = "*" - [[package]] name = "makefun" version = "1.15.2" @@ -4399,13 +4374,13 @@ Jinja2 = ">=2.0" [[package]] name = "synapseclient" -version = "4.3.0" +version = "4.3.1" description = "A client for Synapse, a collaborative, open-source research platform that allows teams to share data, track analyses, and collaborate." optional = false python-versions = ">=3.8" files = [ - {file = "synapseclient-4.3.0-py3-none-any.whl", hash = "sha256:5d8107cfff4031a0a46d60a3c9a8120300190fa27df4983d883dc951d8bd885f"}, - {file = "synapseclient-4.3.0.tar.gz", hash = "sha256:a1149a64b3281669d42c69e210677a902478b8f6b302966d518473c7384f6387"}, + {file = "synapseclient-4.3.1-py3-none-any.whl", hash = "sha256:515fff80092c4acee010e272ae313533ae31f7cbe0a590f540f98fd10a18177b"}, + {file = "synapseclient-4.3.1.tar.gz", hash = "sha256:9d1c2cd1d6fe4fabb386290c0eed20944ab7e44e6713db40f19cf28babe3be3c"}, ] [package.dependencies] @@ -4413,7 +4388,6 @@ async-lru = ">=2.0.4,<2.1.0" asyncio-atexit = ">=1.0.1,<1.1.0" deprecated = ">=1.2.4,<2.0" httpx = ">=0.27.0,<0.28.0" -loky = ">=3.0.0,<3.1.0" nest-asyncio = ">=1.6.0,<1.7.0" opentelemetry-api = ">=1.21.0,<1.22.0" opentelemetry-exporter-otlp-proto-http = ">=1.21.0,<1.22.0" @@ -4964,4 +4938,4 @@ aws = ["uWSGI"] [metadata] lock-version = "2.0" python-versions = ">=3.9.0,<3.11" -content-hash = "a3048c0808e73fd19f5175897e9dda47a2a593422dd4744886615ac453a42139" +content-hash = "9904142a9dec88658394b216905455419b80b58d242a1bd9b1e2ffe9c3cff40d" diff --git a/pyproject.toml b/pyproject.toml index 8d941b8ae..d875f7dcf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -54,7 +54,7 @@ pygsheets = "^2.0.4" PyYAML = "^6.0.0" rdflib = "^6.0.0" setuptools = "^66.0.0" -synapseclient = "^4.3.0" +synapseclient = "4.3.1" tenacity = "^8.0.1" toml = "^0.10.2" great-expectations = "^0.15.0" From 14beea03eb64a9a1f2d8c0ee4a6156c98bd9c0e0 Mon Sep 17 00:00:00 2001 From: GiaJordan Date: Wed, 17 Jul 2024 09:51:12 -0700 Subject: [PATCH 050/233] add test manifest --- tests/data/mock_manifests/InvalidFilenameManifest.csv | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 tests/data/mock_manifests/InvalidFilenameManifest.csv diff --git a/tests/data/mock_manifests/InvalidFilenameManifest.csv b/tests/data/mock_manifests/InvalidFilenameManifest.csv new file mode 100644 index 000000000..211f2a12c --- /dev/null +++ b/tests/data/mock_manifests/InvalidFilenameManifest.csv @@ -0,0 +1,6 @@ +Component,Filename,entityId +MockFilename,schematic - main/MockFilenameComponent/txt1.txt,syn61682653 +MockFilename,schematic - main/MockFilenameComponent/txt2.txt,syn61682660 +MockFilename,schematic - main/MockFilenameComponent/txt3.txt,syn61682662 +MockFilename,schematic - main/MockFilenameComponent/txt4.txt,syn6168265 +MockFilename,schematic - main/MockFilenameComponent/txt5.txt, From c53161b9cf86d3e300e60d51ab200ee38ce77a00 Mon Sep 17 00:00:00 2001 From: GiaJordan Date: Wed, 17 Jul 2024 09:51:57 -0700 Subject: [PATCH 051/233] add test --- tests/test_validation.py | 58 +++++++++++++++++++++++++++++++++------- 1 file changed, 48 insertions(+), 10 deletions(-) diff --git a/tests/test_validation.py b/tests/test_validation.py index 9ea47b973..271577a82 100644 --- a/tests/test_validation.py +++ b/tests/test_validation.py @@ -1,21 +1,20 @@ -import os +import itertools import logging +import os import re -import networkx as nx +from pathlib import Path + import jsonschema +import networkx as nx import pytest -from pathlib import Path -import itertools -from schematic.models.validate_attribute import ValidateAttribute, GenerateError -from schematic.models.validate_manifest import ValidateManifest from schematic.models.metadata import MetadataModel -from schematic.store.synapse import SynapseStorage - -from schematic.schemas.data_model_parser import DataModelParser +from schematic.models.validate_attribute import GenerateError, ValidateAttribute +from schematic.models.validate_manifest import ValidateManifest from schematic.schemas.data_model_graph import DataModelGraph, DataModelGraphExplorer from schematic.schemas.data_model_json_schema import DataModelJSONSchema - +from schematic.schemas.data_model_parser import DataModelParser +from schematic.store.synapse import SynapseStorage from schematic.utils.validate_rules_utils import validation_rule_info logging.basicConfig(level=logging.DEBUG) @@ -692,6 +691,45 @@ def test_in_house_validation(self, helpers, dmge): in warnings ) + def test_filename_manifest(self, helpers, dmge): + metadataModel = get_metadataModel(helpers, model_name="example.model.jsonld") + + manifestPath = helpers.get_data_path( + "mock_manifests/InvalidFilenameManifest.csv" + ) + rootNode = "MockFilename" + + errors, warnings = metadataModel.validateModelManifest( + manifestPath=manifestPath, + rootNode=rootNode, + project_scope=["syn23643250"], + ) + + # Check errors + assert ( + GenerateError.generate_filename_error( + val_rule="filenameExists syn61682648", + attribute_name="Filename", + row_num="3", + invalid_entry="schematic - main/MockFilenameComponent/txt4.txt", + error_type="mismatched entityId", + dmge=dmge, + )[0] + in errors + ) + + assert ( + GenerateError.generate_filename_error( + val_rule="filenameExists syn61682648", + attribute_name="Filename", + row_num="4", + invalid_entry="schematic - main/MockFilenameComponent/txt5.txt", + error_type="path does not exist", + dmge=dmge, + )[0] + in errors + ) + def test_missing_column(self, helpers, dmge: DataModelGraph): """Test that a manifest missing a column returns the proper error.""" model_name = "example.model.csv" From ee17e426d3c76a649b00f100aa1740b540aeb9fc Mon Sep 17 00:00:00 2001 From: GiaJordan Date: Wed, 17 Jul 2024 09:54:24 -0700 Subject: [PATCH 052/233] add rule logic --- schematic/models/validate_attribute.py | 71 +++++++++++++++++++++++++- 1 file changed, 70 insertions(+), 1 deletion(-) diff --git a/schematic/models/validate_attribute.py b/schematic/models/validate_attribute.py index c9851da49..e9e9e7bca 100644 --- a/schematic/models/validate_attribute.py +++ b/schematic/models/validate_attribute.py @@ -1,6 +1,7 @@ import builtins import logging import re +from copy import deepcopy from time import perf_counter # allows specifying explicit variable types @@ -456,6 +457,30 @@ def generate_no_value_in_manifest_error( warnings.append(nv_warnings) return errors, warnings + def generate_filename_error( + val_rule: str, + attribute_name: str, + row_num: str, + invalid_entry: Any, + error_type: str, + dmge: DataModelGraphExplorer, + ) -> tuple[list[str], list[str]]: + if error_type == "path does not exist": + error_message = f"The file path '{invalid_entry}' on row {row_num} does not exist in the file view." + elif error_type == "mismatched entityId": + error_message = f"The entityId for file path '{invalid_entry}' on row {row_num} does not match the entityId for the file in the file view" + + error_list, warning_list = GenerateError.raise_and_store_message( + dmge=dmge, + val_rule=val_rule, + error_row=row_num, + error_col=attribute_name, + error_message=error_message, + error_val=invalid_entry, + ) + + return error_list, warning_list + def _get_rule_attributes( val_rule: str, error_col_name: str, dmge: DataModelGraphExplorer ) -> tuple[list, str, MessageLevelType, bool, bool, bool]: @@ -1960,4 +1985,48 @@ def filename_validation( project_scope, access_token, ): - return + errors = [] + warnings = [] + + where_clauses = [] + self._login(project_scope=project_scope, access_token=access_token) + rule_parts = val_rule.split(" ") + + dataset_clause = f"parentId='{rule_parts[1]}'" + where_clauses.append(dataset_clause) + + self.synStore.query_fileview( + columns=["id", "path"], where_clauses=where_clauses + ) + fileview = self.synStore.storageFileviewTable.reset_index(drop=True) + # filename in dataset? + files_in_view = manifest["Filename"].isin(fileview["path"]) + # filenames match with entity IDs in dataset + joined_df = manifest.merge( + fileview, how="outer", left_on="Filename", right_on="path" + ) + entity_id_match = joined_df["id"] == joined_df["entityId"] + + manifest_with_errors = deepcopy(manifest) + manifest_with_errors["Error"] = pd.NA + + manifest_with_errors.loc[~entity_id_match, "Error"] = "mismatched entityId" + manifest_with_errors.loc[~files_in_view, "Error"] = "path does not exist" + + invalid_entries = manifest_with_errors.loc[ + manifest_with_errors["Error"].notna() + ] + for index, data in invalid_entries.iterrows(): + vr_errors, vr_warnings = GenerateError.generate_filename_error( + val_rule=val_rule, + attribute_name="Filename", + row_num=str(index), + invalid_entry=data["Filename"], + error_type=data["Error"], + dmge=self.dmge, + ) + if vr_errors: + errors.append(vr_errors) + if vr_warnings: + warnings.append(vr_warnings) + return errors, warnings From c51b830084285f874294396a7f840a454565edd7 Mon Sep 17 00:00:00 2001 From: GiaJordan Date: Thu, 18 Jul 2024 10:53:43 -0700 Subject: [PATCH 053/233] add docstrings --- schematic/models/validate_attribute.py | 35 +++++++++++++++++++++++--- 1 file changed, 31 insertions(+), 4 deletions(-) diff --git a/schematic/models/validate_attribute.py b/schematic/models/validate_attribute.py index e9e9e7bca..145fc824a 100644 --- a/schematic/models/validate_attribute.py +++ b/schematic/models/validate_attribute.py @@ -465,6 +465,21 @@ def generate_filename_error( error_type: str, dmge: DataModelGraphExplorer, ) -> tuple[list[str], list[str]]: + """ + Purpose: + Generate an logging error as well as a stored error message, when + a filename error is encountered. + Args: + val_rule: str, rule as defined in the schema for the component. + attribute_name: str, attribute being validated + row_num: str, row where the error was detected + invalid_entry: str, value that caused the error + error_type: str, type of error encountered + dmge: DataModelGraphExplorer object + Returns: + Errors: list[str] Error details for further storage. + warnings: list[str] Warning details for further storage. + """ if error_type == "path does not exist": error_message = f"The file path '{invalid_entry}' on row {row_num} does not exist in the file view." elif error_type == "mismatched entityId": @@ -1980,11 +1995,23 @@ def cross_validation( def filename_validation( self, - val_rule, - manifest, - project_scope, - access_token, + val_rule: str, + manifest: pd.core.frame.DataFrame, + access_token: str, + project_scope: Optional[list] = None, ): + """ + Purpose: + Validate the filenames in the manifest against the data paths in the fileview. + Args: + val_rule: str, Validation rule for the component + manifest: pd.core.frame.DataFrame, manifest + access_token: str, Asset Store access token + project_scope: Optional[list] = None: Projects to limit the scope of cross manifest validation to. + Returns: + errors: list[str] Error details for further storage. + warnings: list[str] Warning details for further storage. + """ errors = [] warnings = [] From bbf1c3f14b1a32863101def188759b7290c987cc Mon Sep 17 00:00:00 2001 From: GiaJordan Date: Thu, 18 Jul 2024 11:20:38 -0700 Subject: [PATCH 054/233] update store tests --- tests/test_store.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/tests/test_store.py b/tests/test_store.py index 20ed7c21a..f5417c1da 100644 --- a/tests/test_store.py +++ b/tests/test_store.py @@ -172,6 +172,12 @@ def test_login(self) -> None: None, "SELECT * FROM syn23643253 WHERE projectId IN ('syn23643250', '') ;", ), + ( + None, + None, + ["projectId IN ('syn23643250')"], + "SELECT * FROM syn23643253 WHERE projectId IN ('syn23643250') ;", + ), ( ["syn23643250"], ["name", "id", "path"], @@ -186,7 +192,7 @@ def test_login(self) -> None: ), ], ) - def test_build_query( + def test_view_query( self, synapse_store_special_scope: SynapseStorage, project_scope: list, @@ -194,11 +200,16 @@ def test_build_query( where_clauses: list, expected: str, ) -> None: + # Ensure correct view is being utilized assert synapse_store_special_scope.storageFileview == "syn23643253" - if project_scope: - synapse_store_special_scope.project_scope = project_scope - synapse_store_special_scope._build_query(columns, where_clauses) + + synapse_store_special_scope.project_scope = project_scope + + synapse_store_special_scope.query_fileview(columns, where_clauses) + # tests ._build_query() assert synapse_store_special_scope.fileview_query == expected + # tests that the query was valid and successful, that a view subset has actually been retrived + assert synapse_store_special_scope.storageFileviewTable.empty is False def test_getFileAnnotations(self, synapse_store: SynapseStorage) -> None: expected_dict = { From bfde6ce30c7a352a90a3ca97d9826c9dd77cc29e Mon Sep 17 00:00:00 2001 From: GiaJordan Date: Thu, 18 Jul 2024 12:17:30 -0700 Subject: [PATCH 055/233] update query logic --- schematic/store/synapse.py | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/schematic/store/synapse.py b/schematic/store/synapse.py index 9d5f3080d..a381646bb 100644 --- a/schematic/store/synapse.py +++ b/schematic/store/synapse.py @@ -193,6 +193,7 @@ def __init__( access_token: Optional[str] = None, project_scope: Optional[list] = None, synapse_cache_path: Optional[str] = None, + perform_query: Optional[bool] = True, ) -> None: """Initializes a SynapseStorage object. @@ -213,7 +214,8 @@ def __init__( self.storageFileview = CONFIG.synapse_master_fileview_id self.manifest = CONFIG.synapse_manifest_basename self.root_synapse_cache = self.syn.cache.cache_root_dir - self.query_fileview() + if perform_query: + self.query_fileview() def _purge_synapse_cache( self, maximum_storage_allowed_cache_gb: int = 1, minute_buffer: int = 15 @@ -264,14 +266,24 @@ def query_fileview( self.storageFileview = CONFIG.synapse_master_fileview_id self.manifest = CONFIG.synapse_manifest_basename + self.new_query_different = True + + previous_query_built = hasattr(self, "fileview_query") + if previous_query_built: + previous_query = self.fileview_query + self._build_query(columns=columns, where_clauses=where_clauses) - try: - self.storageFileviewTable = self.syn.tableQuery( - query=self.fileview_query, - ).asDataFrame() - except SynapseHTTPError: - raise AccessCredentialsError(self.storageFileview) + if previous_query_built: + self.new_query_different = self.fileview_query != previous_query + + if self.new_query_different: + try: + self.storageFileviewTable = self.syn.tableQuery( + query=self.fileview_query, + ).asDataFrame() + except SynapseHTTPError: + raise AccessCredentialsError(self.storageFileview) def _build_query( self, columns: Optional[list] = None, where_clauses: Optional[list] = None From fcaf91fd15e36ed81cdd3b23774f06150654af76 Mon Sep 17 00:00:00 2001 From: GiaJordan Date: Thu, 18 Jul 2024 12:17:47 -0700 Subject: [PATCH 056/233] update tests --- tests/test_store.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_store.py b/tests/test_store.py index f5417c1da..d7da1af75 100644 --- a/tests/test_store.py +++ b/tests/test_store.py @@ -117,9 +117,9 @@ def dmge( def synapse_store_special_scope(request): access_token = os.getenv("SYNAPSE_ACCESS_TOKEN") if access_token: - synapse_store = SynapseStorage(access_token=access_token) + synapse_store = SynapseStorage(access_token=access_token, perform_query=False) else: - synapse_store = SynapseStorage() + synapse_store = SynapseStorage(perform_query=False) yield synapse_store From cb2db9e98cdd19a894cfa57aff636df4cd4b210e Mon Sep 17 00:00:00 2001 From: GiaJordan Date: Thu, 18 Jul 2024 15:10:34 -0700 Subject: [PATCH 057/233] update login logic --- schematic/models/validate_attribute.py | 32 +++++++++++++++++--------- 1 file changed, 21 insertions(+), 11 deletions(-) diff --git a/schematic/models/validate_attribute.py b/schematic/models/validate_attribute.py index 145fc824a..1d2d12a3b 100644 --- a/schematic/models/validate_attribute.py +++ b/schematic/models/validate_attribute.py @@ -769,16 +769,27 @@ class ValidateAttribute(object): def __init__(self, dmge: DataModelGraphExplorer) -> None: self.dmge = dmge - def _login(self, project_scope: list[str], access_token: str = None): + def _login( + self, + access_token: Optional[str] = None, + project_scope: Optional[list[str]] = None, + columns: Optional[list] = None, + where_clauses: Optional[list] = None, + ): # login - try: - self.synStore = SynapseStorage( - access_token=access_token, project_scope=project_scope - ) - except SynapseNoCredentialsError as e: - raise ValueError( - "No Synapse credentials were provided. Credentials must be provided to utilize cross-manfiest validation functionality." - ) from e + if hasattr(self, "synStore"): + if self.synstore.project_scope != project_scope: + self.synStore.project_scope = project_scope + self.synStore.query_fileview(columns=columns, where_clauses=where_clauses) + else: + try: + self.synStore = SynapseStorage( + access_token=access_token, project_scope=project_scope + ) + except SynapseNoCredentialsError as e: + raise ValueError( + "No Synapse credentials were provided. Credentials must be provided to utilize cross-manfiest validation functionality." + ) from e def get_no_entry(self, entry: str, node_display_name: str) -> bool: """Helper function to check if the entry is blank or contains a not applicable type string (and NA is permitted) @@ -826,8 +837,7 @@ def get_target_manifests( target_manifest_ids = [] target_dataset_ids = [] - if not hasattr(self, "synStore"): - self._login(project_scope=project_scope, access_token=access_token) + self._login(project_scope=project_scope, access_token=access_token) # Get list of all projects user has access to projects = self.synStore.getStorageProjects(project_scope=project_scope) From e750ece49a0d9d47b55ee77b00964798a783934b Mon Sep 17 00:00:00 2001 From: GiaJordan Date: Thu, 18 Jul 2024 15:47:56 -0700 Subject: [PATCH 058/233] update var name --- schematic/models/validate_attribute.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/schematic/models/validate_attribute.py b/schematic/models/validate_attribute.py index 1d2d12a3b..f5aa03568 100644 --- a/schematic/models/validate_attribute.py +++ b/schematic/models/validate_attribute.py @@ -763,7 +763,7 @@ class ValidateAttribute(object): See functions for more details. TODO: - Add year validator - - Add string length validator + - Add string length validatorn """ def __init__(self, dmge: DataModelGraphExplorer) -> None: @@ -778,7 +778,7 @@ def _login( ): # login if hasattr(self, "synStore"): - if self.synstore.project_scope != project_scope: + if self.synStore.project_scope != project_scope: self.synStore.project_scope = project_scope self.synStore.query_fileview(columns=columns, where_clauses=where_clauses) else: From c284eba1e67b4b5e0cecf2b3ba0098706e967d60 Mon Sep 17 00:00:00 2001 From: GiaJordan Date: Thu, 18 Jul 2024 16:29:08 -0700 Subject: [PATCH 059/233] update function call --- schematic/models/validate_manifest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/schematic/models/validate_manifest.py b/schematic/models/validate_manifest.py index cfdfc169c..2ffc1d3b0 100644 --- a/schematic/models/validate_manifest.py +++ b/schematic/models/validate_manifest.py @@ -272,7 +272,7 @@ def validate_manifest_rules( ) elif validation_type == "filenameExists": vr_errors, vr_warnings = validation_method( - rule, manifest, project_scope, access_token + rule, manifest, access_token, project_scope ) else: vr_errors, vr_warnings = validation_method( From e579ca40d1838c41b8284ff7d57cca60458ea7d3 Mon Sep 17 00:00:00 2001 From: linglp Date: Fri, 19 Jul 2024 12:29:31 -0400 Subject: [PATCH 060/233] change it to delete suite if exists; --- schematic/models/GE_Helpers.py | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/schematic/models/GE_Helpers.py b/schematic/models/GE_Helpers.py index 99c3728f5..9eda117a8 100644 --- a/schematic/models/GE_Helpers.py +++ b/schematic/models/GE_Helpers.py @@ -152,17 +152,24 @@ def add_expectation_suite_if_not_exists(self) -> ExpectationSuite: saves expectation suite and identifier to self """ self.expectation_suite_name = "Manifest_test_suite" + # Get a list of all expectation suites suite_names = self.context.list_expectation_suite_names() - if self.expectation_suite_name not in suite_names: - self.suite = self.context.add_expectation_suite( - expectation_suite_name=self.expectation_suite_name, - ) - # in gh actions, sometimes the suite has already been added. - # if that's the case, get the existing one - else: - self.suite = self.context.get_expectation_suite( - expectation_suite_name=self.expectation_suite_name - ) + # Get a list of all checkpoints + all_checkpoints = self.context.list_checkpoints() + + # if the suite exists, delete it + if self.expectation_suite_name in suite_names: + self.context.delete_expectation_suite(self.expectation_suite_name) + + # also delete all the checkpoints associated with the suite + if all_checkpoints: + for checkpoint_name in all_checkpoints: + self.context.delete_checkpoint(checkpoint_name) + + self.suite = self.context.add_expectation_suite( + expectation_suite_name=self.expectation_suite_name, + ) + return self.suite def build_expectation_suite( From e5d096325004f95a12d8955d58608781da5c3da1 Mon Sep 17 00:00:00 2001 From: linglp Date: Fri, 19 Jul 2024 14:17:26 -0400 Subject: [PATCH 061/233] fix test --- tests/test_ge_helpers.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/tests/test_ge_helpers.py b/tests/test_ge_helpers.py index bd6bc9187..041824732 100644 --- a/tests/test_ge_helpers.py +++ b/tests/test_ge_helpers.py @@ -54,12 +54,19 @@ def test_add_expectation_suite_if_not_exists_does_exist( mock_ge_helpers.context.list_expectation_suite_names.return_value = [ "Manifest_test_suite" ] + mock_ge_helpers.context.list_checkpoints.return_value = ["test_checkpoint"] # Call the method result = mock_ge_helpers.add_expectation_suite_if_not_exists() - # Make sure the method of getting existing suites gets called + # Make sure the method of deleting suites gets called mock_ge_helpers.context.list_expectation_suite_names.assert_called_once() - mock_ge_helpers.context.get_expectation_suite.assert_called_once_with( + mock_ge_helpers.context.delete_expectation_suite.assert_called_once_with( + "Manifest_test_suite" + ) + mock_ge_helpers.context.add_expectation_suite.assert_called_once_with( expectation_suite_name="Manifest_test_suite" ) + mock_ge_helpers.context.delete_checkpoint.assert_called_once_with( + "test_checkpoint" + ) From 1b6f69408c7b96791253917eeb80026a9910390b Mon Sep 17 00:00:00 2001 From: linglp Date: Fri, 19 Jul 2024 17:32:32 -0400 Subject: [PATCH 062/233] add severity low --- .github/workflows/scan_repo.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/scan_repo.yml b/.github/workflows/scan_repo.yml index ffe69c9f2..6a93beee7 100644 --- a/.github/workflows/scan_repo.yml +++ b/.github/workflows/scan_repo.yml @@ -25,7 +25,7 @@ jobs: ignore-unfixed: true format: 'sarif' output: 'trivy-results.sarif' - severity: 'CRITICAL,HIGH,MEDIUM' + severity: 'CRITICAL,HIGH,MEDIUM,LOW' limit-severities-for-sarif: true - name: Upload Trivy scan results to GitHub Security tab From 2597dd17d8b7af6fcb06f5a57011f0853eeae862 Mon Sep 17 00:00:00 2001 From: GiaJordan Date: Mon, 22 Jul 2024 09:28:13 -0700 Subject: [PATCH 063/233] update rule spec --- schematic/utils/validate_rules_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/schematic/utils/validate_rules_utils.py b/schematic/utils/validate_rules_utils.py index 5da5510ab..859b8d220 100644 --- a/schematic/utils/validate_rules_utils.py +++ b/schematic/utils/validate_rules_utils.py @@ -150,7 +150,7 @@ def validation_rule_info() -> dict[str, Rule]: "fixed_arg": None, }, "filenameExists": { - "arguments": (1, 1), + "arguments": (2, 1), "type": "filename_validation", "complementary_rules": None, "default_message_level": "error", From 4c9551d2227130c98a52dac7e1847990e0a9d57d Mon Sep 17 00:00:00 2001 From: GiaJordan Date: Mon, 22 Jul 2024 11:45:23 -0700 Subject: [PATCH 064/233] add test for login --- tests/test_validation.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/tests/test_validation.py b/tests/test_validation.py index 271577a82..a3a7f1f8a 100644 --- a/tests/test_validation.py +++ b/tests/test_validation.py @@ -1069,3 +1069,26 @@ def test_rule_combinations( restrict_rules=False, project_scope=None, ) + + +class TestValidateAttributeObject: + def test_login(self, helpers, dmge): + """ + Tests that sequential logins update the view query as necessary + """ + validate_attribute = ValidateAttribute(dmge) + validate_attribute._login() + + assert ( + validate_attribute.synStore.fileview_query == "SELECT * FROM syn23643253 ;" + ) + + validate_attribute._login( + project_scope=["syn23643250"], + columns=["name", "id", "path"], + where_clauses=["parentId='syn61682648'", "type='file'"], + ) + assert ( + validate_attribute.synStore.fileview_query + == "SELECT name,id,path FROM syn23643253 WHERE parentId='syn61682648' AND type='file' AND projectId IN ('syn23643250', '') ;" + ) From b2495a34a4d707a3bc9cef080e0b9207df52aa52 Mon Sep 17 00:00:00 2001 From: GiaJordan Date: Mon, 22 Jul 2024 14:32:39 -0700 Subject: [PATCH 065/233] run black --- tests/test_store.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/tests/test_store.py b/tests/test_store.py index b6f2d0d19..b89735e2e 100644 --- a/tests/test_store.py +++ b/tests/test_store.py @@ -15,12 +15,9 @@ import pytest from pandas.testing import assert_frame_equal from synapseclient import EntityViewSchema, Folder +from synapseclient.core.exceptions import SynapseHTTPError from synapseclient.entity import File from synapseclient.models import Annotations -from synapseclient.core.exceptions import SynapseHTTPError -from pandas.testing import assert_frame_equal - - from schematic.configuration.configuration import Configuration from schematic.schemas.data_model_graph import DataModelGraph, DataModelGraphExplorer From 7aa67918d8894f20dc6ecd4964ed3e1d5966be5a Mon Sep 17 00:00:00 2001 From: GiaJordan Date: Mon, 22 Jul 2024 14:33:48 -0700 Subject: [PATCH 066/233] run isort --- schematic/store/synapse.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/schematic/store/synapse.py b/schematic/store/synapse.py index 9c3037c79..70685b4d9 100644 --- a/schematic/store/synapse.py +++ b/schematic/store/synapse.py @@ -33,14 +33,12 @@ as_table_columns, ) from synapseclient.api import get_entity_id_bundle2 - from synapseclient.core.exceptions import ( SynapseAuthenticationError, SynapseHTTPError, SynapseUnmetAccessRestrictions, ) from synapseclient.entity import File - from synapseclient.models.annotations import Annotations from synapseclient.table import CsvFileTable, Schema, build_table from tenacity import ( From 22d1d30ede70b0268d41fba59a231d443f6dc2f9 Mon Sep 17 00:00:00 2001 From: GiaJordan Date: Tue, 23 Jul 2024 09:27:00 -0700 Subject: [PATCH 067/233] fix typo --- schematic/models/validate_attribute.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/schematic/models/validate_attribute.py b/schematic/models/validate_attribute.py index f5aa03568..2d00daded 100644 --- a/schematic/models/validate_attribute.py +++ b/schematic/models/validate_attribute.py @@ -763,7 +763,7 @@ class ValidateAttribute(object): See functions for more details. TODO: - Add year validator - - Add string length validatorn + - Add string length validator """ def __init__(self, dmge: DataModelGraphExplorer) -> None: From b650375052667f553c81e4fd75ac607a2b9c8d88 Mon Sep 17 00:00:00 2001 From: GiaJordan Date: Tue, 23 Jul 2024 09:28:14 -0700 Subject: [PATCH 068/233] update imports --- schematic/store/synapse.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/schematic/store/synapse.py b/schematic/store/synapse.py index 70685b4d9..8e0d1b4c5 100644 --- a/schematic/store/synapse.py +++ b/schematic/store/synapse.py @@ -10,7 +10,7 @@ import uuid # used to generate unique names for entities from copy import deepcopy from dataclasses import asdict, dataclass -from time import perf_counter, sleep +from time import sleep # allows specifying explicit variable types from typing import Any, Dict, List, Optional, Sequence, Set, Tuple, Union From 35853b0cdf5a32e99880f4c311970afc2eb6da6d Mon Sep 17 00:00:00 2001 From: GiaJordan Date: Tue, 23 Jul 2024 09:34:29 -0700 Subject: [PATCH 069/233] update filename val test --- tests/test_validation.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/test_validation.py b/tests/test_validation.py index a3a7f1f8a..c2b1268ed 100644 --- a/tests/test_validation.py +++ b/tests/test_validation.py @@ -730,6 +730,9 @@ def test_filename_manifest(self, helpers, dmge): in errors ) + assert len(errors) == 2 + assert len(warnings) == 0 + def test_missing_column(self, helpers, dmge: DataModelGraph): """Test that a manifest missing a column returns the proper error.""" model_name = "example.model.csv" From 5ef5121551a69a9fbb4c581b6f7b90f4ea998d2e Mon Sep 17 00:00:00 2001 From: GiaJordan Date: Tue, 23 Jul 2024 09:44:46 -0700 Subject: [PATCH 070/233] add comments --- schematic/store/synapse.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/schematic/store/synapse.py b/schematic/store/synapse.py index 8e0d1b4c5..d432e2e71 100644 --- a/schematic/store/synapse.py +++ b/schematic/store/synapse.py @@ -210,6 +210,8 @@ def __init__( synapse_cache_path (Optional[str], optional): Location of synapse cache. Defaults to None. + TODO: + Consider necessity of adding "columns" and "where_clauses" params to the constructor. Currently with how `query_fileview` is implemented, these params are not needed at this step but could be useful in the future if the need for more scoped querys expands. """ self.syn = self.login(synapse_cache_path, token, access_token) self.project_scope = project_scope @@ -268,17 +270,20 @@ def query_fileview( self.storageFileview = CONFIG.synapse_master_fileview_id self.manifest = CONFIG.synapse_manifest_basename + # Initialize to assume that the new fileview query will be different from what may already be stored. Initializes to True because generally one will not have already been performed self.new_query_different = True + # If a query has already been performed, store the query previous_query_built = hasattr(self, "fileview_query") if previous_query_built: previous_query = self.fileview_query + # Build a query with the current given parameters and check to see if it is different from the previous self._build_query(columns=columns, where_clauses=where_clauses) - if previous_query_built: self.new_query_different = self.fileview_query != previous_query + # Only perform the query if it is different from the previous query if self.new_query_different: try: self.storageFileviewTable = self.syn.tableQuery( From b5f5dfbf38cb27582f348da66f8f4f3ed54330a6 Mon Sep 17 00:00:00 2001 From: GiaJordan Date: Tue, 23 Jul 2024 10:06:00 -0700 Subject: [PATCH 071/233] update store init and login --- schematic/models/validate_attribute.py | 16 +++++++++++----- schematic/store/synapse.py | 4 +++- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/schematic/models/validate_attribute.py b/schematic/models/validate_attribute.py index 2d00daded..1fe19c956 100644 --- a/schematic/models/validate_attribute.py +++ b/schematic/models/validate_attribute.py @@ -776,7 +776,7 @@ def _login( columns: Optional[list] = None, where_clauses: Optional[list] = None, ): - # login + # if the ValidateAttribute object already has a SynapseStorage object, just requery the fileview, if not then login if hasattr(self, "synStore"): if self.synStore.project_scope != project_scope: self.synStore.project_scope = project_scope @@ -784,7 +784,10 @@ def _login( else: try: self.synStore = SynapseStorage( - access_token=access_token, project_scope=project_scope + access_token=access_token, + project_scope=project_scope, + columns=columns, + where_clauses=where_clauses, ) except SynapseNoCredentialsError as e: raise ValueError( @@ -2026,15 +2029,18 @@ def filename_validation( warnings = [] where_clauses = [] - self._login(project_scope=project_scope, access_token=access_token) rule_parts = val_rule.split(" ") dataset_clause = f"parentId='{rule_parts[1]}'" where_clauses.append(dataset_clause) - self.synStore.query_fileview( - columns=["id", "path"], where_clauses=where_clauses + self._login( + project_scope=project_scope, + access_token=access_token, + columns=["id", "path"], + where_clauses=where_clauses, ) + fileview = self.synStore.storageFileviewTable.reset_index(drop=True) # filename in dataset? files_in_view = manifest["Filename"].isin(fileview["path"]) diff --git a/schematic/store/synapse.py b/schematic/store/synapse.py index d432e2e71..db3ab10ad 100644 --- a/schematic/store/synapse.py +++ b/schematic/store/synapse.py @@ -196,6 +196,8 @@ def __init__( project_scope: Optional[list] = None, synapse_cache_path: Optional[str] = None, perform_query: Optional[bool] = True, + columns: Optional[list] = None, + where_clauses: Optional[list] = None, ) -> None: """Initializes a SynapseStorage object. @@ -219,7 +221,7 @@ def __init__( self.manifest = CONFIG.synapse_manifest_basename self.root_synapse_cache = self.syn.cache.cache_root_dir if perform_query: - self.query_fileview() + self.query_fileview(columns=columns, where_clauses=where_clauses) def _purge_synapse_cache( self, maximum_storage_allowed_cache_gb: int = 1, minute_buffer: int = 15 From 16231c66b0f077fff27ad821560bd49c25fafa97 Mon Sep 17 00:00:00 2001 From: GiaJordan Date: Tue, 23 Jul 2024 10:16:52 -0700 Subject: [PATCH 072/233] cover edge case --- schematic/models/validate_attribute.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/schematic/models/validate_attribute.py b/schematic/models/validate_attribute.py index 1fe19c956..8aef229fd 100644 --- a/schematic/models/validate_attribute.py +++ b/schematic/models/validate_attribute.py @@ -2048,6 +2048,9 @@ def filename_validation( joined_df = manifest.merge( fileview, how="outer", left_on="Filename", right_on="path" ) + # cover case where there are more files in dataset than in manifest + joined_df = joined_df.loc[~joined_df["Component"].isna()].reset_index(drop=True) + entity_id_match = joined_df["id"] == joined_df["entityId"] manifest_with_errors = deepcopy(manifest) From 3451b656bed33971e6d4c438e6611e5b0799a762 Mon Sep 17 00:00:00 2001 From: GiaJordan Date: Tue, 23 Jul 2024 10:18:34 -0700 Subject: [PATCH 073/233] comments --- schematic/models/validate_attribute.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/schematic/models/validate_attribute.py b/schematic/models/validate_attribute.py index 8aef229fd..74f5c2db2 100644 --- a/schematic/models/validate_attribute.py +++ b/schematic/models/validate_attribute.py @@ -2053,12 +2053,13 @@ def filename_validation( entity_id_match = joined_df["id"] == joined_df["entityId"] + # update manifest with types of errors identified manifest_with_errors = deepcopy(manifest) manifest_with_errors["Error"] = pd.NA - manifest_with_errors.loc[~entity_id_match, "Error"] = "mismatched entityId" manifest_with_errors.loc[~files_in_view, "Error"] = "path does not exist" + # Generate errors invalid_entries = manifest_with_errors.loc[ manifest_with_errors["Error"].notna() ] From a33ab6904a11d335bc01f53e6e5354a89ccf762a Mon Sep 17 00:00:00 2001 From: linglp Date: Tue, 30 Jul 2024 14:10:23 -0400 Subject: [PATCH 074/233] update instructions --- README.md | 50 +++++++++++++++++++------------------------------- 1 file changed, 19 insertions(+), 31 deletions(-) diff --git a/README.md b/README.md index 3d0bf04ca..403169e9b 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ - [Introduction](#introduction) - [Installation](#installation) - [Installation Requirements](#installation-requirements) - - [Installation guide for data curator app](#installation-guide-for-data-curator-app) + - [Installation guide for Schematic CLI users](#installation-guide-for-schematic-cli-users) - [Installation guide for developers/contributors](#installation-guide-for-developerscontributors) - [Other Contribution Guidelines](#other-contribution-guidelines) - [Update readthedocs documentation](#update-readthedocs-documentation) @@ -25,19 +25,31 @@ SCHEMATIC is an acronym for _Schema Engine for Manifest Ingress and Curation_. T Note: Our credential policy for Google credentials in order to create Google sheet files from Schematic, see tutorial ['HERE'](https://scribehow.com/shared/Get_Credentials_for_Google_Drive_and_Google_Sheets_APIs_to_use_with_schematicpy__yqfcJz_rQVeyTcg0KQCINA). If you plan to use `config.yml`, please ensure that the path of `schematic_service_account_creds.json` is indicated there (see `google_sheets > service_account_creds` section) +## Installation guide for Schematic CLI users +1. Verifying Python Version Compatibility +To ensure compatibility with Schematic, please follow these steps: -## Installation guide for data curator app +Check your own Python version: +``` +python3 --version +``` + +Check the Supported Python Version: Open the pyproject.toml file in the Schematic repository to find the version of Python that is supported. You can view this file directly on GitHub [here](https://github.com/Sage-Bionetworks/schematic/blob/main/pyproject.toml#L39). + +Switching Python Versions: If your current Python version is not supported by Schematic, you can switch to the supported version using tools like [pyenv](https://github.com/pyenv/pyenv?tab=readme-ov-file#switch-between-python-versions). Follow the instructions in the pyenv documentation to install and switch between Python versions easily. -Create and activate a virtual environment within which you can install the package: +Please check pyproject.toml file to see what version of Python that schematic supports (See [here](https://github.com/Sage-Bionetworks/schematic/blob/main/pyproject.toml#L39)). If the version of Python that you are using is not supported by Schematic, please use tools such as [pyenv](https://github.com/pyenv/pyenv?tab=readme-ov-file#switch-between-python-versions) to switch to the Python version supported by Schematic. +2. Setting Up the Virtual Environment +After switching to the version of Python supported by Schematic, please activate a virtual environment within which you can install the package: ``` python3 -m venv .venv source .venv/bin/activate ``` +Note: Python 3 has built-in support for virtual environments with the venv module, so you no longer need to install virtualenv. -Note: Python 3 has a built-in support for virtual environment [venv](https://docs.python.org/3/library/venv.html#module-venv) so you no longer need to install virtualenv. - -Install and update the package using [pip](https://pip.pypa.io/en/stable/quickstart/): +3. Installing Schematic +Install the package using [pip](https://pip.pypa.io/en/stable/quickstart/): ``` python3 -m pip install schematicpy @@ -68,21 +80,10 @@ poetry shell ``` 4. Install the dependencies by doing: ``` -poetry install +poetry install --all-extras ``` This command will install the dependencies based on what we specify in poetry.lock. If this step is taking a long time, try to go back to step 2 and check your version of poetry. Alternatively, you could also try deleting the lock file and regenerate it by doing `poetry install` (Please note this method should be used as a last resort because this would force other developers to change their development environment) -If you want to install the API you will need to install those dependencies as well: - -``` -poetry install --extras "api" -``` - -If you want to install the uwsgi: - -``` -poetry install --extras "api" -``` 5. Fill in credential files: *Note*: If you won't interact with Synapse, please ignore this section. @@ -219,19 +220,6 @@ For new features, bugs, enhancements *Note*: Make sure you have the latest version of the `develop` branch on your local machine. -## Installation Guide - Docker - -1. Install docker from https://www.docker.com/ .
-2. Identify docker image of interest from [Schematic DockerHub](https://hub.docker.com/r/sagebionetworks/schematic/tags)
- Ex `docker pull sagebionetworks/schematic:latest` from the CLI or, run `docker compose up` after cloning the schematic github repo
- in this case, `sagebionetworks/schematic:latest` is the name of the image chosen -3. Run Schematic Command with `docker run `.
- - For more information on flags for `docker run` and what they do, visit the [Docker Documentation](https://docs.docker.com/engine/reference/commandline/run/)
- - These example commands assume that you have navigated to the directory you want to run schematic from. To specify your working directory, use `$(pwd)` on MacOS/Linux or `%cd%` on Windows.
- - If not using the latest image, then the full name should be specified: ie `sagebionetworks/schematic:commit-e611e4a`
- - If using local image created by `docker compose up`, then the docker image name should be changed: i.e. `schematic_schematic`
- - Using the `--name` flag sets the name of the container running locally on your machine
- ### Example For REST API
#### Use file path of `config.yml` to run API endpoints: From 8d0de83d644110c404b3297a3f2a0d599586ba07 Mon Sep 17 00:00:00 2001 From: linglp Date: Tue, 30 Jul 2024 14:13:19 -0400 Subject: [PATCH 075/233] update font --- README.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 403169e9b..6649d3037 100644 --- a/README.md +++ b/README.md @@ -26,7 +26,8 @@ SCHEMATIC is an acronym for _Schema Engine for Manifest Ingress and Curation_. T Note: Our credential policy for Google credentials in order to create Google sheet files from Schematic, see tutorial ['HERE'](https://scribehow.com/shared/Get_Credentials_for_Google_Drive_and_Google_Sheets_APIs_to_use_with_schematicpy__yqfcJz_rQVeyTcg0KQCINA). If you plan to use `config.yml`, please ensure that the path of `schematic_service_account_creds.json` is indicated there (see `google_sheets > service_account_creds` section) ## Installation guide for Schematic CLI users -1. Verifying Python Version Compatibility +1. **Verifying Python Version Compatibility** + To ensure compatibility with Schematic, please follow these steps: Check your own Python version: @@ -40,7 +41,7 @@ Switching Python Versions: If your current Python version is not supported by Sc Please check pyproject.toml file to see what version of Python that schematic supports (See [here](https://github.com/Sage-Bionetworks/schematic/blob/main/pyproject.toml#L39)). If the version of Python that you are using is not supported by Schematic, please use tools such as [pyenv](https://github.com/pyenv/pyenv?tab=readme-ov-file#switch-between-python-versions) to switch to the Python version supported by Schematic. -2. Setting Up the Virtual Environment +2. **Setting Up the Virtual Environment** After switching to the version of Python supported by Schematic, please activate a virtual environment within which you can install the package: ``` python3 -m venv .venv @@ -48,7 +49,7 @@ source .venv/bin/activate ``` Note: Python 3 has built-in support for virtual environments with the venv module, so you no longer need to install virtualenv. -3. Installing Schematic +3. **Installing Schematic** Install the package using [pip](https://pip.pypa.io/en/stable/quickstart/): ``` From 07510b6a8c67a93fffb8d8a6f486206a64ec40da Mon Sep 17 00:00:00 2001 From: linglp Date: Tue, 30 Jul 2024 14:14:41 -0400 Subject: [PATCH 076/233] add line --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 6649d3037..f3469728f 100644 --- a/README.md +++ b/README.md @@ -42,6 +42,7 @@ Switching Python Versions: If your current Python version is not supported by Sc Please check pyproject.toml file to see what version of Python that schematic supports (See [here](https://github.com/Sage-Bionetworks/schematic/blob/main/pyproject.toml#L39)). If the version of Python that you are using is not supported by Schematic, please use tools such as [pyenv](https://github.com/pyenv/pyenv?tab=readme-ov-file#switch-between-python-versions) to switch to the Python version supported by Schematic. 2. **Setting Up the Virtual Environment** + After switching to the version of Python supported by Schematic, please activate a virtual environment within which you can install the package: ``` python3 -m venv .venv @@ -50,6 +51,7 @@ source .venv/bin/activate Note: Python 3 has built-in support for virtual environments with the venv module, so you no longer need to install virtualenv. 3. **Installing Schematic** + Install the package using [pip](https://pip.pypa.io/en/stable/quickstart/): ``` From 5cbbd19314fdfce6a458b41f28656aa9a6eeb157 Mon Sep 17 00:00:00 2001 From: linglp Date: Tue, 30 Jul 2024 14:29:12 -0400 Subject: [PATCH 077/233] cli example usage --- README.md | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index f3469728f..9eea09424 100644 --- a/README.md +++ b/README.md @@ -302,7 +302,31 @@ You can **create bug and feature requests** through [Sage Bionetwork's FAIR Data - **Provide screenshots of the expected or actual behaviour** where applicable. # Command Line Usage -Please visit more documentation [here](https://sage-schematic.readthedocs.io/en/develop/cli_reference.html) +1. Generate a new manifest as a google sheet + +``` +schematic manifest -c /path/to/config.yml get -dt -s +``` + +2. Grab an existing manifest from synapse + +``` +schematic manifest -c /path/to/config.yml get -dt -d -s +``` + +3. Validate a manifest + +``` +schematic model -c /path/to/config.yml validate -dt -mp +``` + +4. Submit a manifest as a file + +``` +schematic model -c /path/to/config.yml submit -mp -d -vc -mrt file_only +``` + +Please visit more documentation [here](https://sage-schematic.readthedocs.io/en/develop/cli_reference.html) for more information. From 47907fb53ce4f59899a0e29fccff2047d297b073 Mon Sep 17 00:00:00 2001 From: linglp Date: Tue, 30 Jul 2024 15:04:20 -0400 Subject: [PATCH 078/233] remove repeated part --- README.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/README.md b/README.md index 9eea09424..2849dc4fa 100644 --- a/README.md +++ b/README.md @@ -39,8 +39,6 @@ Check the Supported Python Version: Open the pyproject.toml file in the Schemati Switching Python Versions: If your current Python version is not supported by Schematic, you can switch to the supported version using tools like [pyenv](https://github.com/pyenv/pyenv?tab=readme-ov-file#switch-between-python-versions). Follow the instructions in the pyenv documentation to install and switch between Python versions easily. -Please check pyproject.toml file to see what version of Python that schematic supports (See [here](https://github.com/Sage-Bionetworks/schematic/blob/main/pyproject.toml#L39)). If the version of Python that you are using is not supported by Schematic, please use tools such as [pyenv](https://github.com/pyenv/pyenv?tab=readme-ov-file#switch-between-python-versions) to switch to the Python version supported by Schematic. - 2. **Setting Up the Virtual Environment** After switching to the version of Python supported by Schematic, please activate a virtual environment within which you can install the package: From 30806f18d1e65fecf06597310282eb90eeb95f98 Mon Sep 17 00:00:00 2001 From: linglp Date: Tue, 30 Jul 2024 16:20:13 -0400 Subject: [PATCH 079/233] add sequence diagram for create_manifests --- .github/workflows/pdoc.yml | 1 + schematic/manifest/generator.py | 68 ++++++++++++++++++++++----------- 2 files changed, 47 insertions(+), 22 deletions(-) diff --git a/.github/workflows/pdoc.yml b/.github/workflows/pdoc.yml index 187b5adb1..7ffd73bbb 100644 --- a/.github/workflows/pdoc.yml +++ b/.github/workflows/pdoc.yml @@ -5,6 +5,7 @@ on: push: branches: - develop + - develop-add-mermaid workflow_dispatch: # Allow manually triggering the workflow # security: restrict permissions for CI jobs. diff --git a/schematic/manifest/generator.py b/schematic/manifest/generator.py index d7eb16c30..1664c48ee 100644 --- a/schematic/manifest/generator.py +++ b/schematic/manifest/generator.py @@ -1,38 +1,38 @@ -from collections import OrderedDict import json import logging -import networkx as nx -from openpyxl.styles import Font, Alignment, PatternFill -from openpyxl import load_workbook -from openpyxl.utils.dataframe import dataframe_to_rows import os -import pandas as pd +from collections import OrderedDict from pathlib import Path -import pygsheets as ps from tempfile import NamedTemporaryFile -from typing import Any, Dict, List, Optional, Tuple, Union, BinaryIO, Literal +from typing import Any, BinaryIO, Dict, List, Literal, Optional, Tuple, Union + +import networkx as nx +import pandas as pd +import pygsheets as ps +from openpyxl import load_workbook +from openpyxl.styles import Alignment, Font, PatternFill +from openpyxl.utils.dataframe import dataframe_to_rows +from opentelemetry import trace +from schematic.configuration.configuration import CONFIG from schematic.schemas.data_model_graph import DataModelGraph, DataModelGraphExplorer -from schematic.schemas.data_model_parser import DataModelParser from schematic.schemas.data_model_json_schema import DataModelJSONSchema +from schematic.schemas.data_model_parser import DataModelParser +# TODO: This module should only be aware of the store interface +# we shouldn't need to expose Synapse functionality explicitly +from schematic.store.synapse import SynapseStorage +from schematic.utils.df_utils import load_df, update_df from schematic.utils.google_api_utils import ( - execute_google_api_requests, build_service_account_creds, + execute_google_api_requests, + export_manifest_drive_service, +) +from schematic.utils.schema_utils import ( + DisplayLabelType, + extract_component_validation_rules, ) -from schematic.utils.df_utils import update_df, load_df -from schematic.utils.schema_utils import extract_component_validation_rules from schematic.utils.validate_utils import rule_in_rule_list -from schematic.utils.schema_utils import DisplayLabelType - -# TODO: This module should only be aware of the store interface -# we shouldn't need to expose Synapse functionality explicitly -from schematic.store.synapse import SynapseStorage - -from schematic.configuration.configuration import CONFIG -from schematic.utils.google_api_utils import export_manifest_drive_service - -from opentelemetry import trace logger = logging.getLogger(__name__) tracer = trace.get_tracer("Schematic") @@ -1657,6 +1657,30 @@ def create_manifests( Returns: Union[List[str], List[pd.DataFrame]]: a list of Googlesheet URLs, a list of pandas dataframes or excel file paths + + ::: mermaid + sequenceDiagram + participant User + participant Function + participant DataModelParser + participant DataModelGraph + participant ManifestGenerator + User->>Function: call create_manifests + Function->>Function: check dataset_ids and validate inputs + Function->>DataModelParser: parse data model + DataModelParser-->>Function: return parsed data model + Function->>DataModelGraph: generate graph + DataModelGraph-->>Function: return graph data model + alt data_types == "all manifests" + Function->>ManifestGenerator: create manifests for all components + else + loop for each data_type + Function->>ManifestGenerator: create single manifest + end + end + ManifestGenerator-->>Function: return results + Function-->>User: return manifests based on output_format + ::: """ if dataset_ids: # Check that the number of submitted data_types matches From 81fd161a0351769f34984234b5aeddbf9bd61c78 Mon Sep 17 00:00:00 2001 From: linglp Date: Tue, 30 Jul 2024 16:42:09 -0400 Subject: [PATCH 080/233] update github pdoc version --- .github/workflows/pdoc.yml | 5 ++--- poetry.lock | 12 ++++++------ pyproject.toml | 2 +- 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/.github/workflows/pdoc.yml b/.github/workflows/pdoc.yml index 7ffd73bbb..51924c4b4 100644 --- a/.github/workflows/pdoc.yml +++ b/.github/workflows/pdoc.yml @@ -5,7 +5,6 @@ on: push: branches: - develop - - develop-add-mermaid workflow_dispatch: # Allow manually triggering the workflow # security: restrict permissions for CI jobs. @@ -73,9 +72,9 @@ jobs: run: poetry install --no-interaction --all-extras # create documentation - - run: poetry add pdoc@13.0.0 + - run: poetry add pdoc@14.6.0 - run: poetry show pdoc - - run: poetry run pdoc --docformat google -o docs/schematic schematic/manifest schematic/models schematic/schemas schematic/store schematic/utils schematic/visualization + - run: poetry run pdoc --docformat google --mermaid -o docs/schematic schematic/manifest schematic/models schematic/schemas schematic/store schematic/utils schematic/visualization - uses: actions/upload-pages-artifact@v1 with: diff --git a/poetry.lock b/poetry.lock index 5e8cb5f28..642c0203e 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2787,13 +2787,13 @@ files = [ [[package]] name = "pdoc" -version = "12.3.1" +version = "14.6.0" description = "API Documentation for Python Projects" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "pdoc-12.3.1-py3-none-any.whl", hash = "sha256:c3f24f31286e634de9c76fa6e67bd5c0c5e74360b41dc91e6b82499831eb52d8"}, - {file = "pdoc-12.3.1.tar.gz", hash = "sha256:453236f225feddb8a9071428f1982a78d74b9b3da4bc4433aedb64dbd0cc87ab"}, + {file = "pdoc-14.6.0-py3-none-any.whl", hash = "sha256:36c42c546a317d8e3e8c0b39645f24161374de0c7066ccaae76628d721e49ba5"}, + {file = "pdoc-14.6.0.tar.gz", hash = "sha256:6e98a24c5e0ca5d188397969cf82581836eaef13f172fc3820047bfe15c61c9a"}, ] [package.dependencies] @@ -2802,7 +2802,7 @@ MarkupSafe = "*" pygments = ">=2.12.0" [package.extras] -dev = ["black", "hypothesis", "mypy", "pytest", "pytest-cov", "pytest-timeout", "ruff", "tox", "types-pygments"] +dev = ["hypothesis", "mypy", "pdoc-pyo3-sample-library (==1.0.11)", "pygments (>=2.14.0)", "pytest", "pytest-cov", "pytest-timeout", "ruff", "tox", "types-pygments"] [[package]] name = "pexpect" @@ -4938,4 +4938,4 @@ aws = ["uWSGI"] [metadata] lock-version = "2.0" python-versions = ">=3.9.0,<3.11" -content-hash = "9904142a9dec88658394b216905455419b80b58d242a1bd9b1e2ffe9c3cff40d" +content-hash = "e03232582ef853981ff34ea68dad7f8bbe5fb847728505864bbd083f170bac60" diff --git a/pyproject.toml b/pyproject.toml index d875f7dcf..3b2c3be1e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -62,7 +62,7 @@ sphinx-click = "^4.0.0" itsdangerous = "^2.0.0" openpyxl = "^3.0.9" "backports.zoneinfo" = {markers = "python_version < \"3.9\"", version = "^0.2.1"} -pdoc = "^12.2.0" +pdoc = "^14.0.0" dateparser = "^1.1.4" pandarallel = "^1.6.4" schematic-db = {version = "0.0.41", extras = ["synapse"]} From 264ac1cc8604866df9babf48cc345244578bf8dc Mon Sep 17 00:00:00 2001 From: linglp Date: Tue, 30 Jul 2024 16:57:44 -0400 Subject: [PATCH 081/233] add another diagram --- schematic/manifest/generator.py | 68 +++++++++++++++++++++++---------- 1 file changed, 47 insertions(+), 21 deletions(-) diff --git a/schematic/manifest/generator.py b/schematic/manifest/generator.py index 1664c48ee..4a04ece32 100644 --- a/schematic/manifest/generator.py +++ b/schematic/manifest/generator.py @@ -1658,29 +1658,29 @@ def create_manifests( Returns: Union[List[str], List[pd.DataFrame]]: a list of Googlesheet URLs, a list of pandas dataframes or excel file paths - ::: mermaid + ```mermaid sequenceDiagram - participant User - participant Function - participant DataModelParser - participant DataModelGraph - participant ManifestGenerator - User->>Function: call create_manifests - Function->>Function: check dataset_ids and validate inputs - Function->>DataModelParser: parse data model - DataModelParser-->>Function: return parsed data model - Function->>DataModelGraph: generate graph - DataModelGraph-->>Function: return graph data model - alt data_types == "all manifests" - Function->>ManifestGenerator: create manifests for all components - else - loop for each data_type - Function->>ManifestGenerator: create single manifest + participant User + participant Function + participant DataModelParser + participant DataModelGraph + participant ManifestGenerator + User->>Function: call create_manifests + Function->>Function: check dataset_ids and validate inputs + Function->>DataModelParser: parse data model + DataModelParser-->>Function: return parsed data model + Function->>DataModelGraph: generate graph + DataModelGraph-->>Function: return graph data model + alt data_types == "all manifests" + Function->>ManifestGenerator: create manifests for all components + else + loop for each data_type + Function->>ManifestGenerator: create single manifest + end end - end - ManifestGenerator-->>Function: return results - Function-->>User: return manifests based on output_format - ::: + ManifestGenerator-->>Function: return results + Function-->>User: return manifests based on output_format + ``` """ if dataset_ids: # Check that the number of submitted data_types matches @@ -1807,6 +1807,32 @@ def get_manifest( Returns: Googlesheet URL, pandas dataframe, or an Excel spreadsheet + + ```mermaid + flowchart TD + A[Start] --> B{Dataset ID provided?} + B -- No --> C[Get Empty Manifest] + C --> D{Output Format is 'excel'?} + D -- Yes --> E[Export to Excel] + D -- No --> F[Return Manifest URL] + B -- Yes --> G[Instantiate SynapseStorage] + G --> H[Update Dataset Manifest Files] + H --> I[Get Empty Manifest URL] + I --> J{Manifest Record exists?} + J -- Yes --> K[Update Dataframe] + K --> L[Handle Output Format Logic] + L --> M[Return Result] + J -- No --> N[Get Annotations] + N --> O{Annotations Empty?} + O -- Yes --> P[Get Empty Manifest Dataframe] + P --> Q[Update Dataframe] + O -- No --> R[Get Manifest with Annotations] + R --> S[Update Dataframe] + S --> L + M --> T[End] + F --> T + E --> T + ``` """ # Handle case when no dataset ID is provided if not dataset_id: From 0060095e43557a0f700f2ed45c1b55c1d9f600f6 Mon Sep 17 00:00:00 2001 From: linglp Date: Tue, 30 Jul 2024 17:05:20 -0400 Subject: [PATCH 082/233] more mermaid diagram --- schematic/manifest/generator.py | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/schematic/manifest/generator.py b/schematic/manifest/generator.py index 4a04ece32..f9ef5c7fb 100644 --- a/schematic/manifest/generator.py +++ b/schematic/manifest/generator.py @@ -1440,6 +1440,19 @@ def get_manifest_with_annotations( Returns: Tuple[ps.Spreadsheet, pd.DataFrame]: Both the Google Sheet URL and the corresponding data frame is returned. + + ```Mermaid + flowchart TD + A[Start] --> B[Map Annotation Names to Display Names] + B --> C[Convert Annotations to Dictionary] + C --> D[Set Additional Metadata] + D --> E[Generate Empty Manifest] + E --> F[Get DataFrame by URL] + F --> G[Update DataFrame] + G --> H[Set DataFrame by URL] + H --> I[Return Values] + I --> J[End] + ``` """ # Map annotation labels to display names to match manifest columns annotations = self.map_annotation_names_to_display_names(annotations) @@ -1543,6 +1556,24 @@ def _handle_output_format_logic( a pandas dataframe, file path of an excel spreadsheet, or a google sheet URL TODO: Depreciate sheet URL and add google_sheet as an output_format choice. + + ```Mermaid + flowchart TD + A[Start] --> B{Output Format is 'dataframe'?} + B -- Yes --> C[Return DataFrame] + B -- No --> D{Output Format is 'excel'?} + D -- Yes --> E[Export to Excel] + E --> F[Populate Excel] + F --> G[Return Excel Path] + D -- No --> H{Sheet URL is set?} + H -- Yes --> I[Set DataFrame by URL] + I --> J[Return Sheet URL] + H -- No --> K[Default Return DataFrame] + C --> L[End] + G --> L + J --> L + K --> L + ``` """ # if the output type gets set to "dataframe", return a data frame From 7f4afe7399106299c5584a578f6706e972fc06be Mon Sep 17 00:00:00 2001 From: linglp Date: Tue, 30 Jul 2024 17:07:38 -0400 Subject: [PATCH 083/233] remove diagram --- schematic/manifest/generator.py | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/schematic/manifest/generator.py b/schematic/manifest/generator.py index f9ef5c7fb..fb77d4e33 100644 --- a/schematic/manifest/generator.py +++ b/schematic/manifest/generator.py @@ -1440,19 +1440,6 @@ def get_manifest_with_annotations( Returns: Tuple[ps.Spreadsheet, pd.DataFrame]: Both the Google Sheet URL and the corresponding data frame is returned. - - ```Mermaid - flowchart TD - A[Start] --> B[Map Annotation Names to Display Names] - B --> C[Convert Annotations to Dictionary] - C --> D[Set Additional Metadata] - D --> E[Generate Empty Manifest] - E --> F[Get DataFrame by URL] - F --> G[Update DataFrame] - G --> H[Set DataFrame by URL] - H --> I[Return Values] - I --> J[End] - ``` """ # Map annotation labels to display names to match manifest columns annotations = self.map_annotation_names_to_display_names(annotations) From b9da261c9b1b5d56f79205bcfe6cf58d9a8f06a0 Mon Sep 17 00:00:00 2001 From: linglp Date: Tue, 30 Jul 2024 17:25:01 -0400 Subject: [PATCH 084/233] lower case --- schematic/manifest/generator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/schematic/manifest/generator.py b/schematic/manifest/generator.py index fb77d4e33..c92d8c7a3 100644 --- a/schematic/manifest/generator.py +++ b/schematic/manifest/generator.py @@ -1544,7 +1544,7 @@ def _handle_output_format_logic( TODO: Depreciate sheet URL and add google_sheet as an output_format choice. - ```Mermaid + ```mermaid flowchart TD A[Start] --> B{Output Format is 'dataframe'?} B -- Yes --> C[Return DataFrame] From 89c114e98fccf5bbd0a46a1e8390e8d2a2540f6b Mon Sep 17 00:00:00 2001 From: linglp Date: Wed, 31 Jul 2024 16:10:42 -0400 Subject: [PATCH 085/233] change to use something from the conftest --- tests/test_ge_helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_ge_helpers.py b/tests/test_ge_helpers.py index 041824732..6e0b3e01c 100644 --- a/tests/test_ge_helpers.py +++ b/tests/test_ge_helpers.py @@ -16,7 +16,7 @@ def mock_ge_helpers( dmge = helpers.get_data_model_graph_explorer(path="example.model.jsonld") unimplemented_expectations = ["url"] test_manifest_path = helpers.get_data_path("mock_manifests/Valid_Test_Manifest.csv") - manifest = pd.read_csv(test_manifest_path) + manifest = helpers.get_data_frame(test_manifest_path) ge_helpers = GreatExpectationsHelpers( dmge=dmge, From bf888658d80748a8a7e9e272edf5f1908b541414 Mon Sep 17 00:00:00 2001 From: lakikowolfe Date: Thu, 1 Aug 2024 11:43:06 -0700 Subject: [PATCH 086/233] add csv data model to tests --- tests/test_api.py | 107 +++++++++++++++++++++++----------------------- tests/test_cli.py | 21 ++++----- 2 files changed, 65 insertions(+), 63 deletions(-) diff --git a/tests/test_api.py b/tests/test_api.py index 97183186f..9b7b1c077 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -75,11 +75,12 @@ def test_manifest_json(helpers): ) yield test_manifest_path - -@pytest.fixture(scope="class") -def data_model_jsonld(): - data_model_jsonld = "https://raw.githubusercontent.com/Sage-Bionetworks/schematic/develop/tests/data/example.model.jsonld" - yield data_model_jsonld +@pytest.fixture(scope="class", + params=["example.model.jsonld", + "example.model.csv"]) +def data_model_url(request): + data_model_url = "https://raw.githubusercontent.com/Sage-Bionetworks/schematic/develop/tests/data/" + request.param + yield data_model_url @pytest.fixture(scope="class") @@ -320,9 +321,9 @@ def test_if_in_assetview(self, request_headers, client, entity_id): @pytest.mark.schematic_api class TestMetadataModelOperation: @pytest.mark.parametrize("as_graph", [True, False]) - def test_component_requirement(self, client, data_model_jsonld, as_graph): + def test_component_requirement(self, client, data_model_url, as_graph): params = { - "schema_url": data_model_jsonld, + "schema_url": data_model_url, "source_component": "BulkRNA-seqAssay", "as_graph": as_graph, } @@ -369,8 +370,8 @@ def test_get_property_label_from_display_name(self, client, strict_camel_case): @pytest.mark.schematic_api class TestDataModelGraphExplorerOperation: - def test_get_schema(self, client, data_model_jsonld): - params = {"schema_url": data_model_jsonld, "data_model_labels": "class_label"} + def test_get_schema(self, client, data_model_url): + params = {"schema_url": data_model_url, "data_model_labels": "class_label"} response = client.get( "http://localhost:3001/v1/schemas/get/schema", query_string=params ) @@ -383,9 +384,9 @@ def test_get_schema(self, client, data_model_jsonld): if os.path.exists(response_dt): os.remove(response_dt) - def test_if_node_required(test, client, data_model_jsonld): + def test_if_node_required(test, client, data_model_url): params = { - "schema_url": data_model_jsonld, + "schema_url": data_model_url, "node_display_name": "FamilyHistory", "data_model_labels": "class_label", } @@ -397,9 +398,9 @@ def test_if_node_required(test, client, data_model_jsonld): assert response.status_code == 200 assert response_dta == True - def test_get_node_validation_rules(test, client, data_model_jsonld): + def test_get_node_validation_rules(test, client, data_model_url): params = { - "schema_url": data_model_jsonld, + "schema_url": data_model_url, "node_display_name": "CheckRegexList", } response = client.get( @@ -411,9 +412,9 @@ def test_get_node_validation_rules(test, client, data_model_jsonld): assert "list" in response_dta assert "regex match [a-f]" in response_dta - def test_get_nodes_display_names(test, client, data_model_jsonld): + def test_get_nodes_display_names(test, client, data_model_url): params = { - "schema_url": data_model_jsonld, + "schema_url": data_model_url, "node_list": ["FamilyHistory", "Biospecimen"], } response = client.get( @@ -427,8 +428,8 @@ def test_get_nodes_display_names(test, client, data_model_jsonld): @pytest.mark.parametrize( "relationship", ["parentOf", "requiresDependency", "rangeValue", "domainValue"] ) - def test_get_subgraph_by_edge(self, client, data_model_jsonld, relationship): - params = {"schema_url": data_model_jsonld, "relationship": relationship} + def test_get_subgraph_by_edge(self, client, data_model_url, relationship): + params = {"schema_url": data_model_url, "relationship": relationship} response = client.get( "http://localhost:3001/v1/schemas/get/graph_by_edge_type", @@ -439,10 +440,10 @@ def test_get_subgraph_by_edge(self, client, data_model_jsonld, relationship): @pytest.mark.parametrize("return_display_names", [True, False]) @pytest.mark.parametrize("node_label", ["FamilyHistory", "TissueStatus"]) def test_get_node_range( - self, client, data_model_jsonld, return_display_names, node_label + self, client, data_model_url, return_display_names, node_label ): params = { - "schema_url": data_model_jsonld, + "schema_url": data_model_url, "return_display_names": return_display_names, "node_label": node_label, } @@ -467,7 +468,7 @@ def test_get_node_range( def test_node_dependencies( self, client, - data_model_jsonld, + data_model_url, source_node, return_display_names, return_schema_ordered, @@ -476,7 +477,7 @@ def test_node_dependencies( return_schema_ordered = False params = { - "schema_url": data_model_jsonld, + "schema_url": data_model_url, "source_node": source_node, "return_display_names": return_display_names, "return_schema_ordered": return_schema_ordered, @@ -544,7 +545,7 @@ def ifPandasDataframe(self, response_dt): def test_generate_existing_manifest( self, client, - data_model_jsonld, + data_model_url, data_type, output_format, caplog, @@ -561,7 +562,7 @@ def test_generate_existing_manifest( dataset_id = None # if "all manifests", dataset id is None params = { - "schema_url": data_model_jsonld, + "schema_url": data_model_url, "asset_view": "syn23643253", "title": "Example", "data_type": data_type, @@ -629,13 +630,13 @@ def test_generate_new_manifest( self, caplog, client, - data_model_jsonld, + data_model_url, data_type, output_format, request_headers, ): params = { - "schema_url": data_model_jsonld, + "schema_url": data_model_url, "asset_view": "syn23643253", "title": "Example", "data_type": data_type, @@ -731,10 +732,10 @@ def test_generate_new_manifest( ], ) def test_generate_manifest_file_based_annotations( - self, client, use_annotations, expected, data_model_jsonld + self, client, use_annotations, expected, data_model_url ): params = { - "schema_url": data_model_jsonld, + "schema_url": data_model_url, "data_type": "BulkRNA-seqAssay", "dataset_id": "syn25614635", "asset_view": "syn51707141", @@ -781,10 +782,10 @@ def test_generate_manifest_file_based_annotations( # test case: generate a manifest with annotations when use_annotations is set to True for a component that is not file-based # the dataset folder does not contain an existing manifest def test_generate_manifest_not_file_based_with_annotations( - self, client, data_model_jsonld + self, client, data_model_url ): params = { - "schema_url": data_model_jsonld, + "schema_url": data_model_url, "data_type": "Patient", "dataset_id": "syn25614635", "asset_view": "syn51707141", @@ -816,9 +817,9 @@ def test_generate_manifest_not_file_based_with_annotations( ] ) - def test_generate_manifest_data_type_not_found(self, client, data_model_jsonld): + def test_generate_manifest_data_type_not_found(self, client, data_model_url): params = { - "schema_url": data_model_jsonld, + "schema_url": data_model_url, "data_type": "wrong data type", "use_annotations": False, } @@ -829,13 +830,13 @@ def test_generate_manifest_data_type_not_found(self, client, data_model_jsonld): assert response.status_code == 500 assert "LookupError" in str(response.data) - def test_populate_manifest(self, client, data_model_jsonld, test_manifest_csv): + def test_populate_manifest(self, client, data_model_url, test_manifest_csv): # test manifest test_manifest_data = open(test_manifest_csv, "rb") params = { "data_type": "MockComponent", - "schema_url": data_model_jsonld, + "schema_url": data_model_url, "title": "Example", "csv_file": test_manifest_data, } @@ -861,14 +862,14 @@ def test_populate_manifest(self, client, data_model_jsonld, test_manifest_csv): ) def test_validate_manifest( self, - data_model_jsonld, + data_model_url, client, json_str, restrict_rules, test_manifest_csv, request_headers, ): - params = {"schema_url": data_model_jsonld, "restrict_rules": restrict_rules} + params = {"schema_url": data_model_url, "restrict_rules": restrict_rules} if json_str: params["json_str"] = json_str @@ -1056,11 +1057,11 @@ def test_dataset_manifest_download( @pytest.mark.synapse_credentials_needed @pytest.mark.submission def test_submit_manifest_table_and_file_replace( - self, client, request_headers, data_model_jsonld, test_manifest_submit + self, client, request_headers, data_model_url, test_manifest_submit ): """Testing submit manifest in a csv format as a table and a file. Only replace the table""" params = { - "schema_url": data_model_jsonld, + "schema_url": data_model_url, "data_type": "Biospecimen", "restrict_rules": False, "hide_blanks": False, @@ -1094,14 +1095,14 @@ def test_submit_manifest_file_only_replace( helpers, client, request_headers, - data_model_jsonld, + data_model_url, data_type, manifest_path_fixture, request, ): """Testing submit manifest in a csv format as a file""" params = { - "schema_url": data_model_jsonld, + "schema_url": data_model_url, "data_type": data_type, "restrict_rules": False, "manifest_record_type": "file_only", @@ -1144,12 +1145,12 @@ def test_submit_manifest_file_only_replace( @pytest.mark.synapse_credentials_needed @pytest.mark.submission def test_submit_manifest_json_str_replace( - self, client, request_headers, data_model_jsonld + self, client, request_headers, data_model_url ): """Submit json str as a file""" json_str = '[{"Sample ID": 123, "Patient ID": 1,"Tissue Status": "Healthy","Component": "Biospecimen"}]' params = { - "schema_url": data_model_jsonld, + "schema_url": data_model_url, "data_type": "Biospecimen", "json_str": json_str, "restrict_rules": False, @@ -1172,10 +1173,10 @@ def test_submit_manifest_json_str_replace( @pytest.mark.synapse_credentials_needed @pytest.mark.submission def test_submit_manifest_w_file_and_entities( - self, client, request_headers, data_model_jsonld, test_manifest_submit + self, client, request_headers, data_model_url, test_manifest_submit ): params = { - "schema_url": data_model_jsonld, + "schema_url": data_model_url, "data_type": "Biospecimen", "restrict_rules": False, "manifest_record_type": "file_and_entities", @@ -1202,11 +1203,11 @@ def test_submit_manifest_table_and_file_upsert( self, client, request_headers, - data_model_jsonld, + data_model_url, test_upsert_manifest_csv, ): params = { - "schema_url": data_model_jsonld, + "schema_url": data_model_url, "data_type": "MockRDB", "restrict_rules": False, "manifest_record_type": "table_and_file", @@ -1229,8 +1230,8 @@ def test_submit_manifest_table_and_file_upsert( @pytest.mark.schematic_api class TestSchemaVisualization: - def test_visualize_attributes(self, client, data_model_jsonld): - params = {"schema_url": data_model_jsonld} + def test_visualize_attributes(self, client, data_model_url): + params = {"schema_url": data_model_url} response = client.get( "http://localhost:3001/v1/visualize/attributes", query_string=params @@ -1240,10 +1241,10 @@ def test_visualize_attributes(self, client, data_model_jsonld): @pytest.mark.parametrize("figure_type", ["component", "dependency"]) def test_visualize_tangled_tree_layers( - self, client, figure_type, data_model_jsonld + self, client, figure_type, data_model_url ): # TODO: Determine a 2nd data model to use for this test, test both models sequentially, add checks for content of response - params = {"schema_url": data_model_jsonld, "figure_type": figure_type} + params = {"schema_url": data_model_url, "figure_type": figure_type} response = client.get( "http://localhost:3001/v1/visualize/tangled_tree/layers", @@ -1260,10 +1261,10 @@ def test_visualize_tangled_tree_layers( ], ) def test_visualize_component( - self, client, data_model_jsonld, component, response_text + self, client, data_model_url, component, response_text ): params = { - "schema_url": data_model_jsonld, + "schema_url": data_model_url, "component": component, "include_index": False, "data_model_labels": "class_label", @@ -1288,7 +1289,7 @@ class TestValidationBenchmark: def test_validation_performance( self, helpers, - benchmark_data_model_jsonld, + benchmark_data_model_url, client, test_invalid_manifest, MockComponent_attribute, @@ -1309,7 +1310,7 @@ def test_validation_performance( # Set paramters for endpoint params = { - "schema_url": benchmark_data_model_jsonld, + "schema_url": benchmark_data_model_url, "data_type": "MockComponent", } headers = {"Content-Type": "multipart/form-data", "Accept": "application/json"} diff --git a/tests/test_cli.py b/tests/test_cli.py index 308f9c73f..11f640bc6 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -19,10 +19,11 @@ def runner() -> CliRunner: return CliRunner() -@pytest.fixture -def data_model_jsonld(helpers): - data_model_jsonld = helpers.get_data_path("example.model.jsonld") - yield data_model_jsonld +@pytest.fixture(params=["example.model.jsonld", + "example.model.csv"]) +def data_model_url(request, helpers): + data_model_url = helpers.get_data_path(request.param) + yield data_model_url class TestSchemaCli: @@ -77,7 +78,7 @@ def test_schema_convert_cli(self, runner, helpers): # by default this should download the manifest as a CSV file @pytest.mark.google_credentials_needed def test_get_example_manifest_default( - self, runner, helpers, config: Configuration, data_model_jsonld + self, runner, helpers, config: Configuration, data_model_url ): output_path = helpers.get_data_path("example.Patient.manifest.csv") config.load_config("config_example.yml") @@ -91,7 +92,7 @@ def test_get_example_manifest_default( "--data_type", "Patient", "--path_to_data_model", - data_model_jsonld, + data_model_url, ], ) @@ -102,7 +103,7 @@ def test_get_example_manifest_default( # use google drive to export @pytest.mark.google_credentials_needed def test_get_example_manifest_csv( - self, runner, helpers, config: Configuration, data_model_jsonld + self, runner, helpers, config: Configuration, data_model_url ): output_path = helpers.get_data_path("test.csv") config.load_config("config_example.yml") @@ -116,7 +117,7 @@ def test_get_example_manifest_csv( "--data_type", "Patient", "--path_to_data_model", - data_model_jsonld, + data_model_url, "--output_csv", output_path, ], @@ -127,7 +128,7 @@ def test_get_example_manifest_csv( # get manifest as an excel spreadsheet @pytest.mark.google_credentials_needed def test_get_example_manifest_excel( - self, runner, helpers, config: Configuration, data_model_jsonld + self, runner, helpers, config: Configuration, data_model_url ): output_path = helpers.get_data_path("test.xlsx") config.load_config("config_example.yml") @@ -141,7 +142,7 @@ def test_get_example_manifest_excel( "--data_type", "Patient", "--path_to_data_model", - data_model_jsonld, + data_model_url, "--output_xlsx", output_path, ], From 36564dc4e5ba2a2717aa29735f79064827501353 Mon Sep 17 00:00:00 2001 From: linglp Date: Thu, 1 Aug 2024 16:41:13 -0400 Subject: [PATCH 087/233] update mermaid chart --- schematic/manifest/generator.py | 28 ++++++++++++++++++++-------- 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/schematic/manifest/generator.py b/schematic/manifest/generator.py index c92d8c7a3..c67bd16b3 100644 --- a/schematic/manifest/generator.py +++ b/schematic/manifest/generator.py @@ -1829,7 +1829,7 @@ def get_manifest( ```mermaid flowchart TD A[Start] --> B{Dataset ID provided?} - B -- No --> C[Get Empty Manifest] + B -- No --> C[Get Empty Manifest URL] C --> D{Output Format is 'excel'?} D -- Yes --> E[Export to Excel] D -- No --> F[Return Manifest URL] @@ -1840,13 +1840,25 @@ def get_manifest( J -- Yes --> K[Update Dataframe] K --> L[Handle Output Format Logic] L --> M[Return Result] - J -- No --> N[Get Annotations] - N --> O{Annotations Empty?} - O -- Yes --> P[Get Empty Manifest Dataframe] - P --> Q[Update Dataframe] - O -- No --> R[Get Manifest with Annotations] - R --> S[Update Dataframe] - S --> L + J -- No --> AN{Use Annotations?} + + AN -- No --> Q[Create dataframe from empty manifest on Google] + Q --> AJ{Manifest file-based?} + AJ -- Yes --> P[Add entityId and filename to manifest df] + AJ -- No --> R[Use dataframe from an empty manifest] + + P --> L[Handle Output Format Logic] + R --> L[Handle Output Format Logic] + + AN -- Yes --> AM{Manifest file-based?} + AM -- No --> L[Handle Output Format Logic] + AM -- Yes --> AO[Process Annotations] + AO --> AP{Annotations Empty?} + AP -- Yes --> AQ[Create dataframe from an empty manifest on Google] + AQ --> AR[Update dataframe] + AP -- No --> AS[Get Manifest with Annotations] + AS --> AR + AR --> L[Handle Output Format Logic] M --> T[End] F --> T E --> T From d4ee92f46de36db68f1d7d88bd77e81c7cd445c3 Mon Sep 17 00:00:00 2001 From: GiaJordan Date: Mon, 12 Aug 2024 10:26:13 -0700 Subject: [PATCH 088/233] update login call --- schematic/store/synapse.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/schematic/store/synapse.py b/schematic/store/synapse.py index db3ab10ad..cacffcc1d 100644 --- a/schematic/store/synapse.py +++ b/schematic/store/synapse.py @@ -360,7 +360,7 @@ def login( if token: syn = synapseclient.Synapse(cache_root_dir=synapse_cache_path) try: - syn.login(sessionToken=token, silent=True) + syn.login(authToken=token, silent=True) except SynapseHTTPError as exc: raise ValueError( "Please make sure you are logged into synapse.org." @@ -368,7 +368,7 @@ def login( elif access_token: try: syn = synapseclient.Synapse(cache_root_dir=synapse_cache_path) - syn.default_headers["Authorization"] = f"Bearer {access_token}" + syn.login(authToken=access_token, silent=True) except SynapseHTTPError as exc: raise ValueError( "No access to resources. Please make sure that your token is correct" From 5cfca8352a9f705bb809f6d3d41539597b4ff0f8 Mon Sep 17 00:00:00 2001 From: GiaJordan Date: Mon, 12 Aug 2024 11:32:23 -0700 Subject: [PATCH 089/233] update store fixtures --- tests/conftest.py | 19 ++++++------------- tests/test_store.py | 8 +------- 2 files changed, 7 insertions(+), 20 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 62c2cb3e3..0416a55ee 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,19 +1,18 @@ """Fixtures and helpers for use across all tests""" -import os import logging +import os +import shutil import sys from typing import Generator -import shutil import pytest from dotenv import load_dotenv -from schematic.schemas.data_model_parser import DataModelParser -from schematic.schemas.data_model_graph import DataModelGraph, DataModelGraphExplorer - from schematic.configuration.configuration import CONFIG -from schematic.utils.df_utils import load_df +from schematic.schemas.data_model_graph import DataModelGraph, DataModelGraphExplorer +from schematic.schemas.data_model_parser import DataModelParser from schematic.store.synapse import SynapseStorage +from schematic.utils.df_utils import load_df load_dotenv() @@ -118,13 +117,7 @@ def config(): @pytest.fixture(scope="session") def synapse_store(request): - access_token = os.getenv("SYNAPSE_ACCESS_TOKEN") - if access_token: - synapse_store = SynapseStorage(access_token=access_token) - else: - synapse_store = SynapseStorage() - - yield synapse_store + yield SynapseStorage() # These fixtures make copies of existing test manifests. diff --git a/tests/test_store.py b/tests/test_store.py index b89735e2e..8bfeaed34 100644 --- a/tests/test_store.py +++ b/tests/test_store.py @@ -118,13 +118,7 @@ def dmge( @pytest.fixture def synapse_store_special_scope(request): - access_token = os.getenv("SYNAPSE_ACCESS_TOKEN") - if access_token: - synapse_store = SynapseStorage(access_token=access_token, perform_query=False) - else: - synapse_store = SynapseStorage(perform_query=False) - - yield synapse_store + yield SynapseStorage(perform_query=False) def raise_final_error(retry_state): From 372a90f33e32c504e0166893321da72fd74d0d17 Mon Sep 17 00:00:00 2001 From: GiaJordan Date: Mon, 12 Aug 2024 13:07:53 -0700 Subject: [PATCH 090/233] update store fixtures --- tests/conftest.py | 2 +- tests/test_store.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 0416a55ee..9ec4a4ef8 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -116,7 +116,7 @@ def config(): @pytest.fixture(scope="session") -def synapse_store(request): +def synapse_store(): yield SynapseStorage() diff --git a/tests/test_store.py b/tests/test_store.py index 8bfeaed34..708b6c36c 100644 --- a/tests/test_store.py +++ b/tests/test_store.py @@ -117,7 +117,7 @@ def dmge( @pytest.fixture -def synapse_store_special_scope(request): +def synapse_store_special_scope(): yield SynapseStorage(perform_query=False) From 08fdbe752a3d1dcdcdb24a14b1c173e8533dbef7 Mon Sep 17 00:00:00 2001 From: GiaJordan Date: Mon, 12 Aug 2024 13:21:07 -0700 Subject: [PATCH 091/233] simplify login method --- schematic/store/synapse.py | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/schematic/store/synapse.py b/schematic/store/synapse.py index cacffcc1d..c3d95c2bc 100644 --- a/schematic/store/synapse.py +++ b/schematic/store/synapse.py @@ -335,7 +335,6 @@ def _build_query( @staticmethod def login( synapse_cache_path: Optional[str] = None, - token: Optional[str] = None, access_token: Optional[str] = None, ) -> synapseclient.Synapse: """Login to Synapse @@ -353,19 +352,11 @@ def login( synapseclient.Synapse: A Synapse object that is logged in """ # If no token is provided, try retrieving access token from environment - if not token and not access_token: + if not access_token: access_token = os.getenv("SYNAPSE_ACCESS_TOKEN") # login using a token - if token: - syn = synapseclient.Synapse(cache_root_dir=synapse_cache_path) - try: - syn.login(authToken=token, silent=True) - except SynapseHTTPError as exc: - raise ValueError( - "Please make sure you are logged into synapse.org." - ) from exc - elif access_token: + if access_token: try: syn = synapseclient.Synapse(cache_root_dir=synapse_cache_path) syn.login(authToken=access_token, silent=True) From 3387eefd5de9d1315073f02ea39db6072f2336fe Mon Sep 17 00:00:00 2001 From: GiaJordan Date: Mon, 12 Aug 2024 13:38:25 -0700 Subject: [PATCH 092/233] simplify login method --- schematic/store/synapse.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/schematic/store/synapse.py b/schematic/store/synapse.py index c3d95c2bc..e694c1e21 100644 --- a/schematic/store/synapse.py +++ b/schematic/store/synapse.py @@ -215,7 +215,7 @@ def __init__( TODO: Consider necessity of adding "columns" and "where_clauses" params to the constructor. Currently with how `query_fileview` is implemented, these params are not needed at this step but could be useful in the future if the need for more scoped querys expands. """ - self.syn = self.login(synapse_cache_path, token, access_token) + self.syn = self.login(synapse_cache_path, access_token) self.project_scope = project_scope self.storageFileview = CONFIG.synapse_master_fileview_id self.manifest = CONFIG.synapse_manifest_basename @@ -340,12 +340,10 @@ def login( """Login to Synapse Args: - token (Optional[str], optional): A Synapse token. Defaults to None. access_token (Optional[str], optional): A synapse access token. Defaults to None. synapse_cache_path (Optional[str]): location of synapse cache Raises: - ValueError: If unable to login with token ValueError: If unable to loging with access token Returns: From 02ea6840f50e1dc1cdbad5f598e59149d80c637a Mon Sep 17 00:00:00 2001 From: lakikowolfe Date: Thu, 15 Aug 2024 10:39:35 -0700 Subject: [PATCH 093/233] data_model_url to data_model_path --- tests/test_cli.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/tests/test_cli.py b/tests/test_cli.py index 11f640bc6..722906ee5 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -21,9 +21,9 @@ def runner() -> CliRunner: @pytest.fixture(params=["example.model.jsonld", "example.model.csv"]) -def data_model_url(request, helpers): - data_model_url = helpers.get_data_path(request.param) - yield data_model_url +def data_model_path(request, helpers): + data_model_path = helpers.get_data_path(request.param) + yield data_model_path class TestSchemaCli: @@ -78,7 +78,7 @@ def test_schema_convert_cli(self, runner, helpers): # by default this should download the manifest as a CSV file @pytest.mark.google_credentials_needed def test_get_example_manifest_default( - self, runner, helpers, config: Configuration, data_model_url + self, runner, helpers, config: Configuration, data_model_path ): output_path = helpers.get_data_path("example.Patient.manifest.csv") config.load_config("config_example.yml") @@ -92,7 +92,7 @@ def test_get_example_manifest_default( "--data_type", "Patient", "--path_to_data_model", - data_model_url, + data_model_path, ], ) @@ -103,7 +103,7 @@ def test_get_example_manifest_default( # use google drive to export @pytest.mark.google_credentials_needed def test_get_example_manifest_csv( - self, runner, helpers, config: Configuration, data_model_url + self, runner, helpers, config: Configuration, data_model_path ): output_path = helpers.get_data_path("test.csv") config.load_config("config_example.yml") @@ -117,7 +117,7 @@ def test_get_example_manifest_csv( "--data_type", "Patient", "--path_to_data_model", - data_model_url, + data_model_path, "--output_csv", output_path, ], @@ -128,7 +128,7 @@ def test_get_example_manifest_csv( # get manifest as an excel spreadsheet @pytest.mark.google_credentials_needed def test_get_example_manifest_excel( - self, runner, helpers, config: Configuration, data_model_url + self, runner, helpers, config: Configuration, data_model_path ): output_path = helpers.get_data_path("test.xlsx") config.load_config("config_example.yml") @@ -142,7 +142,7 @@ def test_get_example_manifest_excel( "--data_type", "Patient", "--path_to_data_model", - data_model_url, + data_model_path, "--output_xlsx", output_path, ], From 1d48c368f3a7bc811ccf234651838c3b342156c2 Mon Sep 17 00:00:00 2001 From: GiaJordan Date: Fri, 16 Aug 2024 09:51:13 -0700 Subject: [PATCH 094/233] update existing file paths --- schematic/store/synapse.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/schematic/store/synapse.py b/schematic/store/synapse.py index db3ab10ad..fd253196d 100644 --- a/schematic/store/synapse.py +++ b/schematic/store/synapse.py @@ -788,12 +788,20 @@ def fill_in_entity_id_filename( # the columns Filename and entityId are assumed to be present in manifest schema # TODO: use idiomatic panda syntax if dataset_files: + all_files = self._get_file_entityIds( + dataset_files=dataset_files, only_new_files=False, manifest=manifest + ) new_files = self._get_file_entityIds( dataset_files=dataset_files, only_new_files=True, manifest=manifest ) - # update manifest so that it contains new dataset files + all_files = pd.DataFrame(all_files) new_files = pd.DataFrame(new_files) + existing_files = manifest["entityId"].isin(all_files["entityId"]) + manifest.loc[existing_files, "Filename"] = all_files["Filename"] + + # update manifest so that it contains new dataset files + manifest = ( pd.concat([manifest, new_files], sort=False) .reset_index() From e2d92cc075bf99638001da851c0e358a45a90522 Mon Sep 17 00:00:00 2001 From: GiaJordan Date: Fri, 16 Aug 2024 09:56:54 -0700 Subject: [PATCH 095/233] update logic for when no manfiest is passed in --- schematic/store/synapse.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/schematic/store/synapse.py b/schematic/store/synapse.py index fd253196d..cf5e561da 100644 --- a/schematic/store/synapse.py +++ b/schematic/store/synapse.py @@ -797,8 +797,10 @@ def fill_in_entity_id_filename( all_files = pd.DataFrame(all_files) new_files = pd.DataFrame(new_files) - existing_files = manifest["entityId"].isin(all_files["entityId"]) - manifest.loc[existing_files, "Filename"] = all_files["Filename"] + + if not manifest.empty: + existing_files = manifest["entityId"].isin(all_files["entityId"]) + manifest.loc[existing_files, "Filename"] = all_files["Filename"] # update manifest so that it contains new dataset files From dc7fac4b5c85d798167166be2bda75a50b629403 Mon Sep 17 00:00:00 2001 From: GiaJordan Date: Fri, 16 Aug 2024 11:15:38 -0700 Subject: [PATCH 096/233] only update paths when there is a mismatch --- schematic/store/synapse.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/schematic/store/synapse.py b/schematic/store/synapse.py index cf5e561da..e28b83e2c 100644 --- a/schematic/store/synapse.py +++ b/schematic/store/synapse.py @@ -799,8 +799,20 @@ def fill_in_entity_id_filename( new_files = pd.DataFrame(new_files) if not manifest.empty: - existing_files = manifest["entityId"].isin(all_files["entityId"]) - manifest.loc[existing_files, "Filename"] = all_files["Filename"] + synpase_files_in_manifest = all_files["entityId"].isin( + manifest["entityId"] + ) + file_paths_match = ( + manifest["Filename"] + == all_files.loc[synpase_files_in_manifest, "Filename"] + ).all() + if not file_paths_match: + manifest_files_in_synapse = manifest["entityId"].isin( + all_files["entityId"] + ) + manifest.loc[manifest_files_in_synapse, "Filename"] = all_files.loc[ + synpase_files_in_manifest, "Filename" + ] # update manifest so that it contains new dataset files From 2ecb5d6e40174c51fd12bc0af3dca03daeb2a425 Mon Sep 17 00:00:00 2001 From: GiaJordan Date: Fri, 16 Aug 2024 14:20:39 -0700 Subject: [PATCH 097/233] update test --- tests/test_store.py | 56 +++++++++++++++++++++++++++++++++++---------- 1 file changed, 44 insertions(+), 12 deletions(-) diff --git a/tests/test_store.py b/tests/test_store.py index b89735e2e..3602beec4 100644 --- a/tests/test_store.py +++ b/tests/test_store.py @@ -432,27 +432,59 @@ def test_getDatasetManifest(self, synapse_store, downloadFile): assert manifest_data == "syn51204513" @pytest.mark.parametrize( - "existing_manifest_df", + "existing_manifest_df,fill_in", [ - pd.DataFrame(), - pd.DataFrame( - { - "Filename": ["existing_mock_file_path"], - "entityId": ["existing_mock_entity_id"], - } + ( + pd.DataFrame(), + [ + { + "Filename": ["mock_file_path"], + "entityId": ["mock_entity_id"], + }, + { + "Filename": ["mock_file_path"], + "entityId": ["mock_entity_id"], + }, + ], + ), + ( + pd.DataFrame( + { + "Filename": ["existing_mock_file_path"], + "entityId": ["existing_mock_entity_id"], + } + ), + [ + { + "Filename": ["existing_mock_file_path", "mock_file_path"], + "entityId": ["existing_mock_entity_id", "mock_entity_id"], + }, + { + "Filename": ["mock_file_path"], + "entityId": ["mock_entity_id"], + }, + ], ), ], ) - def test_fill_in_entity_id_filename(self, synapse_store, existing_manifest_df): + def test_fill_in_entity_id_filename( + self, synapse_store, existing_manifest_df, fill_in + ): + fill_in_first_return_value = { + "Filename": ["mock_file_path"], + "entityId": ["mock_entity_id"], + } + fill_in_first_return_value = { + "Filename": ["existing_mock_file_path", "mock_file_path"], + "entityId": ["existing_mock_entity_id", "mock_entity_id"], + } + with patch( "schematic.store.synapse.SynapseStorage.getFilesInStorageDataset", return_value=["syn123", "syn124", "syn125"], ) as mock_get_file_storage, patch( "schematic.store.synapse.SynapseStorage._get_file_entityIds", - return_value={ - "Filename": ["mock_file_path"], - "entityId": ["mock_entity_id"], - }, + side_effect=fill_in, ) as mock_get_file_entity_id: dataset_files, new_manifest = synapse_store.fill_in_entity_id_filename( datasetId="test_syn_id", manifest=existing_manifest_df From 0bd91bf6ae4c6c8bc990b41505b17f73244732aa Mon Sep 17 00:00:00 2001 From: GiaJordan Date: Fri, 16 Aug 2024 14:21:42 -0700 Subject: [PATCH 098/233] clean and update name --- tests/test_store.py | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/tests/test_store.py b/tests/test_store.py index 3602beec4..96e1654d1 100644 --- a/tests/test_store.py +++ b/tests/test_store.py @@ -432,7 +432,7 @@ def test_getDatasetManifest(self, synapse_store, downloadFile): assert manifest_data == "syn51204513" @pytest.mark.parametrize( - "existing_manifest_df,fill_in", + "existing_manifest_df,fill_in_return_value", [ ( pd.DataFrame(), @@ -468,23 +468,14 @@ def test_getDatasetManifest(self, synapse_store, downloadFile): ], ) def test_fill_in_entity_id_filename( - self, synapse_store, existing_manifest_df, fill_in + self, synapse_store, existing_manifest_df, fill_in_return_value ): - fill_in_first_return_value = { - "Filename": ["mock_file_path"], - "entityId": ["mock_entity_id"], - } - fill_in_first_return_value = { - "Filename": ["existing_mock_file_path", "mock_file_path"], - "entityId": ["existing_mock_entity_id", "mock_entity_id"], - } - with patch( "schematic.store.synapse.SynapseStorage.getFilesInStorageDataset", return_value=["syn123", "syn124", "syn125"], ) as mock_get_file_storage, patch( "schematic.store.synapse.SynapseStorage._get_file_entityIds", - side_effect=fill_in, + side_effect=fill_in_return_value, ) as mock_get_file_entity_id: dataset_files, new_manifest = synapse_store.fill_in_entity_id_filename( datasetId="test_syn_id", manifest=existing_manifest_df From 3d5941c27f1a07112c2d762b308bf9e18f3a7c33 Mon Sep 17 00:00:00 2001 From: GiaJordan Date: Tue, 20 Aug 2024 11:49:16 -0700 Subject: [PATCH 099/233] add test for submission --- tests/test_metadata.py | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/tests/test_metadata.py b/tests/test_metadata.py index bf0c4d97b..8065f793a 100644 --- a/tests/test_metadata.py +++ b/tests/test_metadata.py @@ -2,8 +2,9 @@ import logging import os -from typing import Optional, Generator +from contextlib import contextmanager from pathlib import Path +from typing import Generator, Optional from unittest.mock import patch import pytest @@ -15,6 +16,11 @@ logger = logging.getLogger(__name__) +@contextmanager +def does_not_raise(): + yield + + def metadata_model(helpers, data_model_labels): metadata_model = MetadataModel( inputMModelLocation=helpers.get_data_path("example.model.jsonld"), @@ -143,3 +149,20 @@ def test_submit_metadata_manifest( hide_blanks=hide_blanks, ) assert mock_manifest_id == "mock manifest id" + + def test_submit_filebased_manifest(self, helpers): + meta_data_model = metadata_model(helpers, "class_label") + + manifest_path = helpers.get_data_path( + "mock_manifests/filepath_submission_test_manifest.csv" + ) + + with does_not_raise(): + manifest_id = meta_data_model.submit_metadata_manifest( + manifest_path=manifest_path, + dataset_id="syn62276880", + manifest_record_type="file_only", + restrict_rules=False, + file_annotations_upload=True, + ) + assert manifest_id == "syn62280543" From 53ab10c5418fa9a8bb3ddb0fbc138792430bf96c Mon Sep 17 00:00:00 2001 From: GiaJordan Date: Tue, 20 Aug 2024 12:58:30 -0700 Subject: [PATCH 100/233] add test manifest --- .../data/mock_manifests/filepath_submission_test_manifest.csv | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 tests/data/mock_manifests/filepath_submission_test_manifest.csv diff --git a/tests/data/mock_manifests/filepath_submission_test_manifest.csv b/tests/data/mock_manifests/filepath_submission_test_manifest.csv new file mode 100644 index 000000000..9def5eccc --- /dev/null +++ b/tests/data/mock_manifests/filepath_submission_test_manifest.csv @@ -0,0 +1,3 @@ +Filename,Sample ID,File Format,Component,Genome Build,Genome FASTA,Id,entityId +schematic - main/Test Filename Upload/txt1.txt,,,BulkRNA-seqAssay,,,01ded8fc-0915-4959-85ab-64e9644c8787,syn62276954 +schematic - main/Test Filename Upload/txt2.txt,,,BulkRNA-seqAssay,,,fd122bb5-3353-4c94-b1f5-0bb93a3e9fc9,syn62276956 From 9931d740b2f5f6f6c70f4ed507f0e74c73db0333 Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Tue, 20 Aug 2024 15:04:30 -0700 Subject: [PATCH 101/233] update python client to 4.4.0 --- poetry.lock | 19 +++++++++++++++---- pyproject.toml | 2 +- schematic/store/synapse.py | 2 +- 3 files changed, 17 insertions(+), 6 deletions(-) diff --git a/poetry.lock b/poetry.lock index 5e8cb5f28..4c4f8b094 100644 --- a/poetry.lock +++ b/poetry.lock @@ -3406,6 +3406,7 @@ files = [ {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, + {file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"}, {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, @@ -3413,8 +3414,16 @@ files = [ {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, + {file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"}, {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, + {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, + {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, + {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"}, {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, @@ -3431,6 +3440,7 @@ files = [ {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, + {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"}, {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, @@ -3438,6 +3448,7 @@ files = [ {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, + {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"}, {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, @@ -4374,13 +4385,13 @@ Jinja2 = ">=2.0" [[package]] name = "synapseclient" -version = "4.3.1" +version = "4.4.0" description = "A client for Synapse, a collaborative, open-source research platform that allows teams to share data, track analyses, and collaborate." optional = false python-versions = ">=3.8" files = [ - {file = "synapseclient-4.3.1-py3-none-any.whl", hash = "sha256:515fff80092c4acee010e272ae313533ae31f7cbe0a590f540f98fd10a18177b"}, - {file = "synapseclient-4.3.1.tar.gz", hash = "sha256:9d1c2cd1d6fe4fabb386290c0eed20944ab7e44e6713db40f19cf28babe3be3c"}, + {file = "synapseclient-4.4.0-py3-none-any.whl", hash = "sha256:efbf8ac46909a5ce50e54aa4c81850e876754bd3c58b26c6a43850fbca96a01b"}, + {file = "synapseclient-4.4.0.tar.gz", hash = "sha256:331d0740a8cebf29a231d5ead35cda164fc74d7d3a8470803e623f2c33e897b5"}, ] [package.dependencies] @@ -4938,4 +4949,4 @@ aws = ["uWSGI"] [metadata] lock-version = "2.0" python-versions = ">=3.9.0,<3.11" -content-hash = "9904142a9dec88658394b216905455419b80b58d242a1bd9b1e2ffe9c3cff40d" +content-hash = "450ff82615a78cd55b7c526f314898aa38e3186810f2e465d02117ea4fb9d10b" diff --git a/pyproject.toml b/pyproject.toml index d875f7dcf..9e601371c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -54,7 +54,7 @@ pygsheets = "^2.0.4" PyYAML = "^6.0.0" rdflib = "^6.0.0" setuptools = "^66.0.0" -synapseclient = "4.3.1" +synapseclient = "4.4.0" tenacity = "^8.0.1" toml = "^0.10.2" great-expectations = "^0.15.0" diff --git a/schematic/store/synapse.py b/schematic/store/synapse.py index e694c1e21..b4eed1740 100644 --- a/schematic/store/synapse.py +++ b/schematic/store/synapse.py @@ -1461,7 +1461,7 @@ async def store_async_annotation(self, annotation_dict: dict) -> Annotations: etag=annotation_dict["annotations"]["etag"], id=annotation_dict["annotations"]["id"], ) - return await annotation_class.store_async(self.syn) + return await annotation_class.store_async(synapse_client=self.syn) @async_missing_entity_handler async def format_row_annotations( From fa9b263446ea1f71c6ef5ad1277a9487ee848be4 Mon Sep 17 00:00:00 2001 From: GiaJordan Date: Tue, 20 Aug 2024 15:49:31 -0700 Subject: [PATCH 102/233] redo matching logic --- schematic/store/synapse.py | 36 +++++++++++++++++++----------------- 1 file changed, 19 insertions(+), 17 deletions(-) diff --git a/schematic/store/synapse.py b/schematic/store/synapse.py index e28b83e2c..243f8e931 100644 --- a/schematic/store/synapse.py +++ b/schematic/store/synapse.py @@ -798,30 +798,32 @@ def fill_in_entity_id_filename( all_files = pd.DataFrame(all_files) new_files = pd.DataFrame(new_files) - if not manifest.empty: - synpase_files_in_manifest = all_files["entityId"].isin( - manifest["entityId"] - ) - file_paths_match = ( - manifest["Filename"] - == all_files.loc[synpase_files_in_manifest, "Filename"] - ).all() - if not file_paths_match: - manifest_files_in_synapse = manifest["entityId"].isin( - all_files["entityId"] - ) - manifest.loc[manifest_files_in_synapse, "Filename"] = all_files.loc[ - synpase_files_in_manifest, "Filename" - ] - # update manifest so that it contains new dataset files - manifest = ( pd.concat([manifest, new_files], sort=False) .reset_index() .drop("index", axis=1) ) + manifest_reindex = manifest.set_index("entityId") + all_files_reindex = all_files.set_index("entityId") + all_files_reindex_like_manifest = all_files_reindex.reindex_like( + manifest_reindex + ) + + file_paths_match = ( + manifest_reindex["Filename"] + == all_files_reindex_like_manifest["Filename"] + ) + + if not file_paths_match.all(): + manifest_reindex.loc[ + ~file_paths_match, "Filename" + ] = all_files_reindex_like_manifest.loc[~file_paths_match, "Filename"] + manifest = manifest_reindex.reset_index() + entityIdCol = manifest.pop("entityId") + manifest.insert(len(manifest.columns), "entityId", entityIdCol) + manifest = manifest.fillna("") return dataset_files, manifest From 48a48fd297609640999b41e576560b0585ed9627 Mon Sep 17 00:00:00 2001 From: GiaJordan Date: Tue, 20 Aug 2024 16:22:21 -0700 Subject: [PATCH 103/233] update test assertion --- tests/test_manifest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_manifest.py b/tests/test_manifest.py index 5829dcde1..3ce9e58a0 100644 --- a/tests/test_manifest.py +++ b/tests/test_manifest.py @@ -798,4 +798,4 @@ def test_get_manifest_with_files(self, helpers, component, datasetId): assert n_rows == 4 elif component == "BulkRNA-seqAssay": assert filename_in_manifest_columns - assert n_rows == 3 + assert n_rows == 4 From abf127711bf3efda8cb46008435dfdc1a518ca81 Mon Sep 17 00:00:00 2001 From: GiaJordan Date: Wed, 21 Aug 2024 09:21:02 -0700 Subject: [PATCH 104/233] update tests for filepaths as well --- tests/test_manifest.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/tests/test_manifest.py b/tests/test_manifest.py index 3ce9e58a0..b4daaaf76 100644 --- a/tests/test_manifest.py +++ b/tests/test_manifest.py @@ -769,7 +769,9 @@ def test_create_manifests( ) def test_get_manifest_with_files(self, helpers, component, datasetId): """ - Test to ensure that when generating a record based manifset that has files in the dataset that the files are not added to the manifest as well + Test to ensure that + when generating a record based manifset that has files in the dataset that the files are not added to the manifest as well + when generating a file based manifest from a dataset thathas had files added that the files are added correctly """ path_to_data_model = helpers.get_data_path("example.model.jsonld") @@ -799,3 +801,14 @@ def test_get_manifest_with_files(self, helpers, component, datasetId): elif component == "BulkRNA-seqAssay": assert filename_in_manifest_columns assert n_rows == 4 + + expected_files = pd.Series( + [ + "schematic - main/BulkRNASeq and files/txt1.txt", + "schematic - main/BulkRNASeq and files/txt2.txt", + "schematic - main/BulkRNASeq and files/txt4.txt", + "schematic - main/BulkRNASeq and files/txt3.txt", + ], + name="Filename", + ) + assert expected_files.equals(manifest["Filename"]) From eb66a379f68720808a7f9561ff71ac5549090307 Mon Sep 17 00:00:00 2001 From: GiaJordan Date: Wed, 21 Aug 2024 09:53:06 -0700 Subject: [PATCH 105/233] add exceptions and messages --- schematic/store/synapse.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/schematic/store/synapse.py b/schematic/store/synapse.py index 243f8e931..0b69b14df 100644 --- a/schematic/store/synapse.py +++ b/schematic/store/synapse.py @@ -291,8 +291,19 @@ def query_fileview( self.storageFileviewTable = self.syn.tableQuery( query=self.fileview_query, ).asDataFrame() - except SynapseHTTPError: - raise AccessCredentialsError(self.storageFileview) + except SynapseHTTPError as exc: + exception_text = str(exc) + if "Unknown column path" in exception_text: + raise ValueError( + "The path column has not been added to the fileview. Please make sure that the fileview is up to date. You can add the path column to the fileview by follwing the instructions in the validation rules documentation (https://sagebionetworks.jira.com/wiki/spaces/SCHEM/pages/2645262364/Data+Model+Validation+Rules#Filename-Validation)." + ) from exc + elif "Unknown column" in exception_text: + missing_columns = exception_text.split("Unknown column ")[-1] + raise ValueError( + f"The column(s) ({missing_columns}) specified in the query do not exist in the fileview. Please make sure that the column names are correct and that all expected columns have been added to the fileview." + ) from exc + else: + raise AccessCredentialsError(self.storageFileview) def _build_query( self, columns: Optional[list] = None, where_clauses: Optional[list] = None From e0d6b8b8093ccc8781a49ca62c186713c619c399 Mon Sep 17 00:00:00 2001 From: GiaJordan Date: Wed, 21 Aug 2024 13:45:59 -0700 Subject: [PATCH 106/233] add tests for exceptions raised --- tests/test_store.py | 36 +++++++++++++++++++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/tests/test_store.py b/tests/test_store.py index 96e1654d1..f91234b27 100644 --- a/tests/test_store.py +++ b/tests/test_store.py @@ -7,6 +7,7 @@ import math import os import shutil +from contextlib import contextmanager from time import sleep from typing import Any, Generator from unittest.mock import AsyncMock, patch @@ -19,7 +20,7 @@ from synapseclient.entity import File from synapseclient.models import Annotations -from schematic.configuration.configuration import Configuration +from schematic.configuration.configuration import CONFIG, Configuration from schematic.schemas.data_model_graph import DataModelGraph, DataModelGraphExplorer from schematic.schemas.data_model_parser import DataModelParser from schematic.store.base import BaseStorage @@ -31,6 +32,11 @@ logger = logging.getLogger(__name__) +@contextmanager +def does_not_raise(): + yield + + @pytest.fixture def test_download_manifest_id(): yield "syn51203973" @@ -214,6 +220,34 @@ def test_view_query( # tests that the query was valid and successful, that a view subset has actually been retrived assert synapse_store_special_scope.storageFileviewTable.empty is False + @pytest.mark.parametrize( + "asset_view,columns,expectation", + [ + ("syn62339865", ["path"], pytest.raises(ValueError)), + ("syn62340177", ["id"], pytest.raises(ValueError)), + ("syn23643253", ["id", "path"], does_not_raise()), + ], + ) + def test_view_query_exception( + self, + synapse_store_special_scope: SynapseStorage, + asset_view: str, + columns: list, + expectation: str, + ) -> None: + project_scope = ["syn23643250"] + + # set to use appropriate test view and set scope + CONFIG.synapse_master_fileview_id = asset_view + synapse_store_special_scope.project_scope = project_scope + + # ensure approriate exception is raised + with expectation: + synapse_store_special_scope.query_fileview(columns) + + # reset config to default fileview + CONFIG.synapse_master_fileview_id = "syn23643253" + def test_getFileAnnotations(self, synapse_store: SynapseStorage) -> None: expected_dict = { "author": "bruno, milen, sujay", From 1c5621f4a31dbb7bc3feda4c5dad30184435f9db Mon Sep 17 00:00:00 2001 From: GiaJordan Date: Wed, 21 Aug 2024 14:50:59 -0700 Subject: [PATCH 107/233] update test --- tests/test_api.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/tests/test_api.py b/tests/test_api.py index 97183186f..4798faf2c 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -12,13 +12,11 @@ import pytest from schematic.configuration.configuration import Configuration -from schematic.schemas.data_model_parser import DataModelParser from schematic.schemas.data_model_graph import DataModelGraph, DataModelGraphExplorer +from schematic.schemas.data_model_parser import DataModelParser from schematic.schemas.data_model_relationships import DataModelRelationships - from schematic_api.api import create_app - logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) @@ -763,9 +761,9 @@ def test_generate_manifest_file_based_annotations( # make sure Filename, entityId, and component get filled with correct value assert google_sheet_df["Filename"].to_list() == [ - "TestDataset-Annotations-v3/Sample_A.txt", - "TestDataset-Annotations-v3/Sample_B.txt", - "TestDataset-Annotations-v3/Sample_C.txt", + "schematic - main/TestDataset-Annotations-v3/Sample_A.txt", + "schematic - main/TestDataset-Annotations-v3/Sample_B.txt", + "schematic - main/TestDataset-Annotations-v3/Sample_C.txt", ] assert google_sheet_df["entityId"].to_list() == [ "syn25614636", From 4c2edd99a536deb21d8a055a87d67b70a18fd9b0 Mon Sep 17 00:00:00 2001 From: GiaJordan Date: Thu, 22 Aug 2024 13:47:08 -0700 Subject: [PATCH 108/233] Revert "update test" This reverts commit 1c5621f4a31dbb7bc3feda4c5dad30184435f9db. --- tests/test_api.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/test_api.py b/tests/test_api.py index 4798faf2c..97183186f 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -12,11 +12,13 @@ import pytest from schematic.configuration.configuration import Configuration -from schematic.schemas.data_model_graph import DataModelGraph, DataModelGraphExplorer from schematic.schemas.data_model_parser import DataModelParser +from schematic.schemas.data_model_graph import DataModelGraph, DataModelGraphExplorer from schematic.schemas.data_model_relationships import DataModelRelationships + from schematic_api.api import create_app + logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) @@ -761,9 +763,9 @@ def test_generate_manifest_file_based_annotations( # make sure Filename, entityId, and component get filled with correct value assert google_sheet_df["Filename"].to_list() == [ - "schematic - main/TestDataset-Annotations-v3/Sample_A.txt", - "schematic - main/TestDataset-Annotations-v3/Sample_B.txt", - "schematic - main/TestDataset-Annotations-v3/Sample_C.txt", + "TestDataset-Annotations-v3/Sample_A.txt", + "TestDataset-Annotations-v3/Sample_B.txt", + "TestDataset-Annotations-v3/Sample_C.txt", ] assert google_sheet_df["entityId"].to_list() == [ "syn25614636", From 1ae2d901ab066cfa44c3f3292001f75fbc680d1d Mon Sep 17 00:00:00 2001 From: GiaJordan Date: Thu, 22 Aug 2024 13:48:32 -0700 Subject: [PATCH 109/233] update test --- tests/test_store.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/tests/test_store.py b/tests/test_store.py index f91234b27..0a62048da 100644 --- a/tests/test_store.py +++ b/tests/test_store.py @@ -242,11 +242,12 @@ def test_view_query_exception( synapse_store_special_scope.project_scope = project_scope # ensure approriate exception is raised - with expectation: - synapse_store_special_scope.query_fileview(columns) - - # reset config to default fileview - CONFIG.synapse_master_fileview_id = "syn23643253" + try: + with expectation: + synapse_store_special_scope.query_fileview(columns) + finally: + # reset config to default fileview + CONFIG.synapse_master_fileview_id = "syn23643253" def test_getFileAnnotations(self, synapse_store: SynapseStorage) -> None: expected_dict = { From 0b60395304888b621a2a6e4137e63b12d48fb2bd Mon Sep 17 00:00:00 2001 From: GiaJordan Date: Thu, 22 Aug 2024 15:51:33 -0700 Subject: [PATCH 110/233] add comments --- schematic/store/synapse.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/schematic/store/synapse.py b/schematic/store/synapse.py index 0b69b14df..d41b427d7 100644 --- a/schematic/store/synapse.py +++ b/schematic/store/synapse.py @@ -816,21 +816,26 @@ def fill_in_entity_id_filename( .drop("index", axis=1) ) + # Reindex manifest and new files dataframes according to entityIds to align file paths and metadata manifest_reindex = manifest.set_index("entityId") all_files_reindex = all_files.set_index("entityId") all_files_reindex_like_manifest = all_files_reindex.reindex_like( manifest_reindex ) + # Check if individual file paths in manifest and from synapse match file_paths_match = ( manifest_reindex["Filename"] == all_files_reindex_like_manifest["Filename"] ) + # If all the paths do not match, update the manifest with the filepaths from synapse if not file_paths_match.all(): manifest_reindex.loc[ ~file_paths_match, "Filename" ] = all_files_reindex_like_manifest.loc[~file_paths_match, "Filename"] + + # reformat manifest for further use manifest = manifest_reindex.reset_index() entityIdCol = manifest.pop("entityId") manifest.insert(len(manifest.columns), "entityId", entityIdCol) From 37be2e5932f8201afd658d103daecbb5b12d729f Mon Sep 17 00:00:00 2001 From: linglp Date: Mon, 26 Aug 2024 11:40:24 -0400 Subject: [PATCH 111/233] update synapse test --- tests/test_store.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_store.py b/tests/test_store.py index 708b6c36c..0bdfa0871 100644 --- a/tests/test_store.py +++ b/tests/test_store.py @@ -641,7 +641,7 @@ async def test_store_async_annotation(self, synapse_store: SynapseStorage) -> No ) as mock_store_async: result = await synapse_store.store_async_annotation(annos_dict) - mock_store_async.assert_called_once_with(synapse_store.syn) + mock_store_async.assert_called_once_with(synapse_client=synapse_store.syn) assert result == expected_dict assert isinstance(result, Annotations) From 3bf7f3da77c345eedad8edac4842c408f60da1b5 Mon Sep 17 00:00:00 2001 From: linglp Date: Mon, 26 Aug 2024 12:57:04 -0400 Subject: [PATCH 112/233] remove -n auto and see if test will get skipped or not --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e12303a7e..f72d562c9 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -122,7 +122,7 @@ jobs: run: > source .venv/bin/activate; pytest --durations=0 --cov-report=term --cov-report=html:htmlcov --cov=schematic/ - -m "not (schematic_api or table_operations)" --reruns 2 -n auto + -m "not (schematic_api or table_operations)" --reruns 2 - name: Upload pytest test results uses: actions/upload-artifact@v2 From e7018e99de84b740345186987789763e138b65b6 Mon Sep 17 00:00:00 2001 From: linglp Date: Mon, 26 Aug 2024 13:03:13 -0400 Subject: [PATCH 113/233] fix manifest test --- tests/test_manifest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_manifest.py b/tests/test_manifest.py index 5829dcde1..3ce9e58a0 100644 --- a/tests/test_manifest.py +++ b/tests/test_manifest.py @@ -798,4 +798,4 @@ def test_get_manifest_with_files(self, helpers, component, datasetId): assert n_rows == 4 elif component == "BulkRNA-seqAssay": assert filename_in_manifest_columns - assert n_rows == 3 + assert n_rows == 4 From 9935b019bf06d8312a55244f8286381721cb703d Mon Sep 17 00:00:00 2001 From: linglp Date: Mon, 26 Aug 2024 13:05:55 -0400 Subject: [PATCH 114/233] revert changes to the workflow --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index f72d562c9..e12303a7e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -122,7 +122,7 @@ jobs: run: > source .venv/bin/activate; pytest --durations=0 --cov-report=term --cov-report=html:htmlcov --cov=schematic/ - -m "not (schematic_api or table_operations)" --reruns 2 + -m "not (schematic_api or table_operations)" --reruns 2 -n auto - name: Upload pytest test results uses: actions/upload-artifact@v2 From e3905cf3dcd8aacdd9cd2ded3f6956ade955c350 Mon Sep 17 00:00:00 2001 From: BryanFauble <17128019+BryanFauble@users.noreply.github.com> Date: Mon, 26 Aug 2024 10:41:01 -0700 Subject: [PATCH 115/233] [FDS-2127] Update URL validation to use requests.options to verify connectivity (#1472) * Update URL validation to use requests.options to verify connectivity --- README.md | 25 +++++-- schematic/models/validate_attribute.py | 26 +++---- tests/conftest.py | 7 ++ tests/integration/test_validate_attribute.py | 75 ++++++++++++++++++++ tests/test_validation.py | 14 +--- 5 files changed, 116 insertions(+), 31 deletions(-) create mode 100644 tests/integration/test_validate_attribute.py diff --git a/README.md b/README.md index 2849dc4fa..cf1cd96f6 100644 --- a/README.md +++ b/README.md @@ -2,17 +2,28 @@ [![Build Status](https://img.shields.io/endpoint.svg?url=https%3A%2F%2Factions-badge.atrox.dev%2FSage-Bionetworks%2Fschematic%2Fbadge%3Fref%3Ddevelop&style=flat)](https://actions-badge.atrox.dev/Sage-Bionetworks/schematic/goto?ref=develop) [![Documentation Status](https://readthedocs.org/projects/sage-schematic/badge/?version=develop)](https://sage-schematic.readthedocs.io/en/develop/?badge=develop) [![PyPI version](https://badge.fury.io/py/schematicpy.svg)](https://badge.fury.io/py/schematicpy) # Table of contents +- [Schematic](#schematic) +- [Table of contents](#table-of-contents) - [Introduction](#introduction) - [Installation](#installation) - [Installation Requirements](#installation-requirements) - [Installation guide for Schematic CLI users](#installation-guide-for-schematic-cli-users) - [Installation guide for developers/contributors](#installation-guide-for-developerscontributors) + - [Development environment setup](#development-environment-setup) + - [Development process instruction](#development-process-instruction) + - [Example For REST API ](#example-for-rest-api-) + - [Use file path of `config.yml` to run API endpoints:](#use-file-path-of-configyml-to-run-api-endpoints) + - [Use content of `config.yml` and `schematic_service_account_creds.json`as an environment variable to run API endpoints:](#use-content-of-configyml-and-schematic_service_account_credsjsonas-an-environment-variable-to-run-api-endpoints) + - [Example For Schematic on mac/linux ](#example-for-schematic-on-maclinux-) + - [Example For Schematic on Windows ](#example-for-schematic-on-windows-) - [Other Contribution Guidelines](#other-contribution-guidelines) - - [Update readthedocs documentation](#update-readthedocs-documentation) + - [Updating readthedocs documentation](#updating-readthedocs-documentation) + - [Update toml file and lock file](#update-toml-file-and-lock-file) + - [Reporting bugs or feature requests](#reporting-bugs-or-feature-requests) - [Command Line Usage](#command-line-usage) - [Testing](#testing) - [Updating Synapse test resources](#updating-synapse-test-resources) -- [Code Style](#code-style) +- [Code style](#code-style) - [Contributors](#contributors) # Introduction @@ -90,13 +101,15 @@ This command will install the dependencies based on what we specify in poetry.lo *Note*: If you won't interact with Synapse, please ignore this section. There are two main configuration files that need to be edited: -config.yml -and [synapseConfig](https://raw.githubusercontent.com/Sage-Bionetworks/synapsePythonClient/v2.3.0-rc/synapseclient/.synapseConfig) +- config.yml +- [synapseConfig](https://raw.githubusercontent.com/Sage-Bionetworks/synapsePythonClient/master/synapseclient/.synapseConfig) Configure .synapseConfig File -Download a copy of the ``.synapseConfig`` file, open the file in the -editor of your choice and edit the `username` and `authtoken` attribute under the `authentication` section +Download a copy of the ``.synapseConfig`` file, open the file in the editor of your +choice and edit the `username` and `authtoken` attribute under the `authentication` +section. **Note:** You must place the file at the root of the project like +`{project_root}/.synapseConfig` in order for any authenticated tests to work. *Note*: You could also visit [configparser](https://docs.python.org/3/library/configparser.html#module-configparser>) doc to see the format that `.synapseConfig` must have. For instance: >[authentication]
username = ABC
authtoken = abc diff --git a/schematic/models/validate_attribute.py b/schematic/models/validate_attribute.py index 74f5c2db2..a4f79b036 100644 --- a/schematic/models/validate_attribute.py +++ b/schematic/models/validate_attribute.py @@ -1,4 +1,3 @@ -import builtins import logging import re from copy import deepcopy @@ -6,12 +5,11 @@ # allows specifying explicit variable types from typing import Any, Literal, Optional, Union -from urllib import error from urllib.parse import urlparse -from urllib.request import Request, urlopen import numpy as np import pandas as pd +import requests from jsonschema import ValidationError from synapseclient.core.exceptions import SynapseNoCredentialsError @@ -1127,16 +1125,16 @@ def type_validation( def url_validation( self, val_rule: str, - manifest_col: str, + manifest_col: pd.Series, ) -> tuple[list[list[str]], list[list[str]]]: """ Purpose: Validate URL's submitted for a particular attribute in a manifest. Determine if the URL is valid and contains attributes specified in the - schema. + schema. Additionally, the server must be reachable to be deemed as valid. Input: - val_rule: str, Validation rule - - manifest_col: pd.core.series.Series, column for a given + - manifest_col: pd.Series, column for a given attribute in the manifest Output: This function will return errors when the user input value @@ -1154,8 +1152,9 @@ def url_validation( ) if entry_has_value: # Check if a random phrase, string or number was added and - # log the appropriate error. Specifically, Raise an error if the value added is not a string or no part - # of the string can be parsed as a part of a URL. + # log the appropriate error. Specifically, Raise an error if the value + # added is not a string or no part of the string can be parsed as a + # part of a URL. if not isinstance(url, str) or not ( urlparse(url).scheme + urlparse(url).netloc @@ -1186,10 +1185,13 @@ def url_validation( try: # Check that the URL points to a working webpage # if not log the appropriate error. - request = Request(url) - response = urlopen(request) valid_url = True - response_code = response.getcode() + response = requests.options(url, allow_redirects=True) + logger.debug( + "Validated URL [URL: %s, status_code: %s]", + url, + response.status_code, + ) except: valid_url = False url_error = "invalid_url" @@ -1207,7 +1209,7 @@ def url_validation( errors.append(vr_errors) if vr_warnings: warnings.append(vr_warnings) - if valid_url == True: + if valid_url: # If the URL works, check to see if it contains the proper arguments # as specified in the schema. for arg in url_args: diff --git a/tests/conftest.py b/tests/conftest.py index 9ec4a4ef8..9a0b5789d 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -142,3 +142,10 @@ def temporary_file_copy(request, helpers: Helpers) -> Generator[str, None, None] # Teardown if os.path.exists(temp_csv_path): os.remove(temp_csv_path) + + +@pytest.fixture(name="dmge", scope="function") +def DMGE(helpers: Helpers) -> DataModelGraphExplorer: + """Fixture to instantiate a DataModelGraphExplorer object.""" + dmge = helpers.get_data_model_graph_explorer(path="example.model.jsonld") + return dmge diff --git a/tests/integration/test_validate_attribute.py b/tests/integration/test_validate_attribute.py new file mode 100644 index 000000000..36c1f9bab --- /dev/null +++ b/tests/integration/test_validate_attribute.py @@ -0,0 +1,75 @@ +import pandas as pd + +from schematic.models.validate_attribute import ValidateAttribute +from schematic.schemas.data_model_graph import DataModelGraphExplorer + +CHECK_URL_NODE_NAME = "Check URL" +VALIDATION_RULE_URL = "url" + + +class TestValidateAttribute: + """Integration tests for the ValidateAttribute class.""" + + def test_url_validation_valid_url(self, dmge: DataModelGraphExplorer) -> None: + # GIVEN a valid URL: + url = "https://github.com/Sage-Bionetworks/schematic" + + # AND a pd.core.series.Series that contains this URL + content = pd.Series(data=[url], name=CHECK_URL_NODE_NAME) + + # AND a validation attribute + validator = ValidateAttribute(dmge=dmge) + + # WHEN the URL is validated + result = validator.url_validation( + val_rule=VALIDATION_RULE_URL, manifest_col=content + ) + + # THEN the result should pass validation + assert result == ([], []) + + def test_url_validation_valid_doi(self, dmge: DataModelGraphExplorer) -> None: + # GIVEN a valid URL: + url = "https://doi.org/10.1158/0008-5472.can-23-0128" + + # AND a pd.core.series.Series that contains this URL + content = pd.Series(data=[url], name=CHECK_URL_NODE_NAME) + + # AND a validation attribute + validator = ValidateAttribute(dmge=dmge) + + # WHEN the URL is validated + result = validator.url_validation( + val_rule=VALIDATION_RULE_URL, manifest_col=content + ) + + # THEN the result should pass validation + assert result == ([], []) + + def test_url_validation_invalid_url(self, dmge: DataModelGraphExplorer) -> None: + # GIVEN an invalid URL: + url = "http://googlef.com/" + + # AND a pd.core.series.Series that contains this URL + content = pd.Series(data=[url], name=CHECK_URL_NODE_NAME) + + # AND a validation attribute + validator = ValidateAttribute(dmge=dmge) + + # WHEN the URL is validated + result = validator.url_validation( + val_rule=VALIDATION_RULE_URL, manifest_col=content + ) + + # THEN the result should not pass validation + assert result == ( + [ + [ + "2", + "Check URL", + "For the attribute 'Check URL', on row 2, the URL provided (http://googlef.com/) does not conform to the standards of a URL. Please make sure you are entering a real, working URL as required by the Schema.", + "http://googlef.com/", + ] + ], + [], + ) diff --git a/tests/test_validation.py b/tests/test_validation.py index c2b1268ed..d6b3971be 100644 --- a/tests/test_validation.py +++ b/tests/test_validation.py @@ -1,11 +1,7 @@ -import itertools import logging import os import re -from pathlib import Path -import jsonschema -import networkx as nx import pytest from schematic.models.metadata import MetadataModel @@ -13,20 +9,12 @@ from schematic.models.validate_manifest import ValidateManifest from schematic.schemas.data_model_graph import DataModelGraph, DataModelGraphExplorer from schematic.schemas.data_model_json_schema import DataModelJSONSchema -from schematic.schemas.data_model_parser import DataModelParser -from schematic.store.synapse import SynapseStorage from schematic.utils.validate_rules_utils import validation_rule_info logging.basicConfig(level=logging.DEBUG) logger = logging.getLogger(__name__) -@pytest.fixture(name="dmge") -def DMGE(helpers): - dmge = helpers.get_data_model_graph_explorer(path="example.model.jsonld") - yield dmge - - def get_metadataModel(helpers, model_name: str): metadataModel = MetadataModel( inputMModelLocation=helpers.get_data_path(model_name), @@ -1075,7 +1063,7 @@ def test_rule_combinations( class TestValidateAttributeObject: - def test_login(self, helpers, dmge): + def test_login(self, dmge: DataModelGraphExplorer) -> None: """ Tests that sequential logins update the view query as necessary """ From 6feea81f8d5102043bbd69bb090e73c868b82945 Mon Sep 17 00:00:00 2001 From: GiaJordan Date: Mon, 26 Aug 2024 14:24:07 -0700 Subject: [PATCH 116/233] update exception message --- schematic/store/synapse.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/schematic/store/synapse.py b/schematic/store/synapse.py index d41b427d7..e02ff40b5 100644 --- a/schematic/store/synapse.py +++ b/schematic/store/synapse.py @@ -295,7 +295,7 @@ def query_fileview( exception_text = str(exc) if "Unknown column path" in exception_text: raise ValueError( - "The path column has not been added to the fileview. Please make sure that the fileview is up to date. You can add the path column to the fileview by follwing the instructions in the validation rules documentation (https://sagebionetworks.jira.com/wiki/spaces/SCHEM/pages/2645262364/Data+Model+Validation+Rules#Filename-Validation)." + "The path column has not been added to the fileview. Please make sure that the fileview is up to date. You can add the path column to the fileview by follwing the instructions in the validation rules documentation." ) from exc elif "Unknown column" in exception_text: missing_columns = exception_text.split("Unknown column ")[-1] From 976645a26cb809a433f8c6dc41c1e966f8b584fa Mon Sep 17 00:00:00 2001 From: GiaJordan Date: Mon, 26 Aug 2024 14:31:56 -0700 Subject: [PATCH 117/233] update type hint --- tests/test_store.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_store.py b/tests/test_store.py index 0a62048da..409dda5fa 100644 --- a/tests/test_store.py +++ b/tests/test_store.py @@ -232,7 +232,7 @@ def test_view_query_exception( self, synapse_store_special_scope: SynapseStorage, asset_view: str, - columns: list, + columns: list[str], expectation: str, ) -> None: project_scope = ["syn23643250"] From 208f22534588a68bfae24f6677371b5df74d63b2 Mon Sep 17 00:00:00 2001 From: GiaJordan Date: Mon, 26 Aug 2024 14:39:34 -0700 Subject: [PATCH 118/233] update comment --- tests/test_store.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_store.py b/tests/test_store.py index 409dda5fa..bb8d7ea8b 100644 --- a/tests/test_store.py +++ b/tests/test_store.py @@ -246,7 +246,7 @@ def test_view_query_exception( with expectation: synapse_store_special_scope.query_fileview(columns) finally: - # reset config to default fileview + # reset config to default fileview so subsequent tests do not use the test fileviews utilized by this test CONFIG.synapse_master_fileview_id = "syn23643253" def test_getFileAnnotations(self, synapse_store: SynapseStorage) -> None: From d0628b2be4bbbd5790a59c35d62294c9afbbbe84 Mon Sep 17 00:00:00 2001 From: GiaJordan Date: Mon, 26 Aug 2024 15:04:17 -0700 Subject: [PATCH 119/233] change structure of --- schematic/store/synapse.py | 72 ++++++++++++++++++++------------------ 1 file changed, 37 insertions(+), 35 deletions(-) diff --git a/schematic/store/synapse.py b/schematic/store/synapse.py index 0e71a7fd9..a287859f5 100644 --- a/schematic/store/synapse.py +++ b/schematic/store/synapse.py @@ -787,47 +787,49 @@ def fill_in_entity_id_filename( # note that if there is an existing manifest and there are files in the dataset # the columns Filename and entityId are assumed to be present in manifest schema # TODO: use idiomatic panda syntax - if dataset_files: - all_files = self._get_file_entityIds( - dataset_files=dataset_files, only_new_files=False, manifest=manifest - ) - new_files = self._get_file_entityIds( - dataset_files=dataset_files, only_new_files=True, manifest=manifest - ) + if not dataset_files: + manifest = manifest.fillna("") + return dataset_files, manifest - all_files = pd.DataFrame(all_files) - new_files = pd.DataFrame(new_files) + all_files = self._get_file_entityIds( + dataset_files=dataset_files, only_new_files=False, manifest=manifest + ) + new_files = self._get_file_entityIds( + dataset_files=dataset_files, only_new_files=True, manifest=manifest + ) - # update manifest so that it contains new dataset files - manifest = ( - pd.concat([manifest, new_files], sort=False) - .reset_index() - .drop("index", axis=1) - ) + all_files = pd.DataFrame(all_files) + new_files = pd.DataFrame(new_files) - # Reindex manifest and new files dataframes according to entityIds to align file paths and metadata - manifest_reindex = manifest.set_index("entityId") - all_files_reindex = all_files.set_index("entityId") - all_files_reindex_like_manifest = all_files_reindex.reindex_like( - manifest_reindex - ) + # update manifest so that it contains new dataset files + manifest = ( + pd.concat([manifest, new_files], sort=False) + .reset_index() + .drop("index", axis=1) + ) - # Check if individual file paths in manifest and from synapse match - file_paths_match = ( - manifest_reindex["Filename"] - == all_files_reindex_like_manifest["Filename"] - ) + # Reindex manifest and new files dataframes according to entityIds to align file paths and metadata + manifest_reindex = manifest.set_index("entityId") + all_files_reindex = all_files.set_index("entityId") + all_files_reindex_like_manifest = all_files_reindex.reindex_like( + manifest_reindex + ) + + # Check if individual file paths in manifest and from synapse match + file_paths_match = ( + manifest_reindex["Filename"] == all_files_reindex_like_manifest["Filename"] + ) - # If all the paths do not match, update the manifest with the filepaths from synapse - if not file_paths_match.all(): - manifest_reindex.loc[ - ~file_paths_match, "Filename" - ] = all_files_reindex_like_manifest.loc[~file_paths_match, "Filename"] + # If all the paths do not match, update the manifest with the filepaths from synapse + if not file_paths_match.all(): + manifest_reindex.loc[ + ~file_paths_match, "Filename" + ] = all_files_reindex_like_manifest.loc[~file_paths_match, "Filename"] - # reformat manifest for further use - manifest = manifest_reindex.reset_index() - entityIdCol = manifest.pop("entityId") - manifest.insert(len(manifest.columns), "entityId", entityIdCol) + # reformat manifest for further use + manifest = manifest_reindex.reset_index() + entityIdCol = manifest.pop("entityId") + manifest.insert(len(manifest.columns), "entityId", entityIdCol) manifest = manifest.fillna("") return dataset_files, manifest From 34d17607dbf711774e62ae3c184fe1a7c00c7169 Mon Sep 17 00:00:00 2001 From: BryanFauble <17128019+BryanFauble@users.noreply.github.com> Date: Tue, 27 Aug 2024 09:46:36 -0700 Subject: [PATCH 120/233] [FDS-2294] Prevent including project name twice in walked path (#1474) * Prevent including project name twice in walked path --- schematic/store/synapse.py | 42 +++-- schematic_api/api/routes.py | 19 +- tests/test_api.py | 349 +++++++++++++++++++----------------- tests/test_store.py | 5 +- 4 files changed, 223 insertions(+), 192 deletions(-) diff --git a/schematic/store/synapse.py b/schematic/store/synapse.py index b4eed1740..e62f11e01 100644 --- a/schematic/store/synapse.py +++ b/schematic/store/synapse.py @@ -547,7 +547,8 @@ def getFilesInStorageDataset( Raises: ValueError: Dataset ID not found. """ - # select all files within a given storage dataset folder (top level folder in a Synapse storage project or folder marked with contentType = 'dataset') + # select all files within a given storage dataset folder (top level folder in + # a Synapse storage project or folder marked with contentType = 'dataset') walked_path = synapseutils.walk( self.syn, datasetId, includeTypes=["folder", "file"] ) @@ -557,25 +558,36 @@ def getFilesInStorageDataset( file_list = [] # iterate over all results - for dirpath, dirname, filenames in walked_path: + for dirpath, _, path_filenames in walked_path: # iterate over all files in a folder - for filename in filenames: - if (not "manifest" in filename[0] and not fileNames) or ( - fileNames and filename[0] in fileNames + for path_filename in path_filenames: + if ("manifest" not in path_filename[0] and not fileNames) or ( + fileNames and path_filename[0] in fileNames ): - # don't add manifest to list of files unless it is specified in the list of specified fileNames; return all found files + # don't add manifest to list of files unless it is specified in the + # list of specified fileNames; return all found files # except the manifest if no fileNames have been specified # TODO: refactor for clarity/maintainability if fullpath: # append directory path to filename - filename = ( - project_name + "/" + dirpath[0] + "/" + filename[0], - filename[1], - ) + if dirpath[0].startswith(f"{project_name}/"): + path_filename = ( + dirpath[0] + "/" + path_filename[0], + path_filename[1], + ) + else: + path_filename = ( + project_name + + "/" + + dirpath[0] + + "/" + + path_filename[0], + path_filename[1], + ) # add file name file id tuple, rearranged so that id is first and name follows - file_list.append(filename[::-1]) + file_list.append(path_filename[::-1]) return file_list @@ -655,8 +667,8 @@ def getDatasetManifest( manifest_data = ManifestDownload.download_manifest( md, newManifestName=newManifestName, manifest_df=manifest ) - ## TO DO: revisit how downstream code handle manifest_data. If the downstream code would break when manifest_data is an empty string, - ## then we should catch the error here without returning an empty string. + # TO DO: revisit how downstream code handle manifest_data. If the downstream code would break when manifest_data is an empty string, + # then we should catch the error here without returning an empty string. if not manifest_data: logger.debug( f"No manifest data returned. Please check if you have successfully downloaded manifest: {manifest_syn_id}" @@ -3024,6 +3036,8 @@ def _fix_int_columns(self): for col in int_columns: # Coercing to string because NaN is a floating point value # and cannot exist alongside integers in a column - to_int_fn = lambda x: "" if np.isnan(x) else str(int(x)) + def to_int_fn(x): + return "" if np.isnan(x) else str(int(x)) + self.table[col] = self.table[col].apply(to_int_fn) return self.table diff --git a/schematic_api/api/routes.py b/schematic_api/api/routes.py index ae73428eb..59a6cd55e 100644 --- a/schematic_api/api/routes.py +++ b/schematic_api/api/routes.py @@ -1,4 +1,3 @@ -import json import logging import os import pathlib @@ -8,12 +7,10 @@ import time import urllib.request from functools import wraps -from json.decoder import JSONDecodeError -from typing import Any, List, Optional +from typing import List, Tuple import connexion import pandas as pd -from connexion.decorators.uri_parsing import Swagger2URIParser from flask import current_app as app from flask import request, send_from_directory from flask_cors import cross_origin @@ -28,14 +25,6 @@ Span, ) from opentelemetry.sdk.trace.sampling import ALWAYS_OFF -from synapseclient.core.exceptions import ( - SynapseAuthenticationError, - SynapseHTTPError, - SynapseNoCredentialsError, - SynapseTimeoutError, - SynapseUnmetAccessRestrictions, -) -from werkzeug.debug import DebuggedApplication from schematic.configuration.configuration import CONFIG from schematic.manifest.generator import ManifestGenerator @@ -457,7 +446,7 @@ def validate_manifest_route( return res_dict -#####profile validate manifest route function +# profile validate manifest route function @trace_function_params() def submit_manifest_route( schema_url, @@ -596,7 +585,9 @@ def get_storage_projects_datasets(asset_view, project_id): return sorted_dataset_lst -def get_files_storage_dataset(asset_view, dataset_id, full_path, file_names=None): +def get_files_storage_dataset( + asset_view: str, dataset_id: str, full_path: bool, file_names: List[str] = None +) -> List[Tuple[str, str]]: # Access token now stored in request header access_token = get_access_token() diff --git a/tests/test_api.py b/tests/test_api.py index 97183186f..d20a77074 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -3,35 +3,35 @@ import logging import os import re -import time from math import ceil from time import perf_counter +from typing import Dict, Generator, List, Tuple, Union -import numpy as np +import flask import pandas as pd # third party library import import pytest +from flask.testing import FlaskClient from schematic.configuration.configuration import Configuration -from schematic.schemas.data_model_parser import DataModelParser from schematic.schemas.data_model_graph import DataModelGraph, DataModelGraphExplorer -from schematic.schemas.data_model_relationships import DataModelRelationships - +from schematic.schemas.data_model_parser import DataModelParser from schematic_api.api import create_app - logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) +BENCHMARK_DATA_MODEL_JSON_LD = "https://raw.githubusercontent.com/Sage-Bionetworks/schematic/develop/tests/data/example.single_rule.model.jsonld" +DATA_MODEL_JSON_LD = "https://raw.githubusercontent.com/Sage-Bionetworks/schematic/develop/tests/data/example.model.jsonld" + -## TO DO: Clean up url and use a global variable SERVER_URL @pytest.fixture(scope="class") -def app(): +def app() -> flask.Flask: app = create_app() - yield app + return app @pytest.fixture(scope="class") -def client(app): +def client(app: flask.Flask) -> Generator[FlaskClient, None, None]: app.config["SCHEMATIC_CONFIG"] = None with app.test_client() as client: @@ -39,62 +39,49 @@ def client(app): @pytest.fixture(scope="class") -def test_manifest_csv(helpers): +def test_manifest_csv(helpers) -> str: test_manifest_path = helpers.get_data_path("mock_manifests/Valid_Test_Manifest.csv") - yield test_manifest_path + return test_manifest_path @pytest.fixture(scope="class") -def test_manifest_submit(helpers): +def test_manifest_submit(helpers) -> str: test_manifest_path = helpers.get_data_path( "mock_manifests/example_biospecimen_test.csv" ) - yield test_manifest_path + return test_manifest_path @pytest.fixture(scope="class") -def test_invalid_manifest(helpers): +def test_invalid_manifest(helpers) -> pd.DataFrame: test_invalid_manifest = helpers.get_data_frame( "mock_manifests/Invalid_Test_Manifest.csv", preserve_raw_input=False ) - yield test_invalid_manifest + return test_invalid_manifest @pytest.fixture(scope="class") -def test_upsert_manifest_csv(helpers): +def test_upsert_manifest_csv(helpers) -> str: test_upsert_manifest_path = helpers.get_data_path( "mock_manifests/rdb_table_manifest.csv" ) - yield test_upsert_manifest_path + return test_upsert_manifest_path @pytest.fixture(scope="class") -def test_manifest_json(helpers): +def test_manifest_json(helpers) -> str: test_manifest_path = helpers.get_data_path( "mock_manifests/Example.Patient.manifest.json" ) - yield test_manifest_path - - -@pytest.fixture(scope="class") -def data_model_jsonld(): - data_model_jsonld = "https://raw.githubusercontent.com/Sage-Bionetworks/schematic/develop/tests/data/example.model.jsonld" - yield data_model_jsonld - - -@pytest.fixture(scope="class") -def benchmark_data_model_jsonld(): - benchmark_data_model_jsonld = "https://raw.githubusercontent.com/Sage-Bionetworks/schematic/develop/tests/data/example.single_rule.model.jsonld" - yield benchmark_data_model_jsonld + return test_manifest_path -def get_MockComponent_attribute(): +def get_MockComponent_attribute() -> Generator[str, None, None]: """ Yield all of the mock conponent attributes one at a time TODO: pull in jsonld from fixture """ - schema_url = "https://raw.githubusercontent.com/Sage-Bionetworks/schematic/develop/tests/data/example.single_rule.model.jsonld" - data_model_parser = DataModelParser(path_to_data_model=schema_url) + data_model_parser = DataModelParser(path_to_data_model=BENCHMARK_DATA_MODEL_JSON_LD) # Parse Model parsed_data_model = data_model_parser.parse_model() @@ -123,24 +110,26 @@ def syn_token(config: Configuration): token = os.environ["SYNAPSE_ACCESS_TOKEN"] else: token = config_parser["authentication"]["authtoken"] - yield token + return token @pytest.fixture -def request_headers(syn_token): +def request_headers(syn_token: str) -> Dict[str, str]: headers = {"Authorization": "Bearer " + syn_token} - yield headers + return headers @pytest.fixture -def request_invalid_headers(): +def request_invalid_headers() -> Dict[str, str]: headers = {"Authorization": "Bearer invalid headers"} - yield headers + return headers @pytest.mark.schematic_api class TestSynapseStorage: - def test_invalid_authentication(self, client, request_invalid_headers): + def test_invalid_authentication( + self, client: FlaskClient, request_invalid_headers: Dict[str, str] + ) -> None: response = client.get( "http://localhost:3001/v1/storage/assets/tables", query_string={"asset_view": "syn23643253", "return_type": "csv"}, @@ -148,7 +137,9 @@ def test_invalid_authentication(self, client, request_invalid_headers): ) assert response.status_code == 401 - def test_insufficent_auth(self, client, request_headers): + def test_insufficent_auth( + self, client: FlaskClient, request_headers: Dict[str, str] + ) -> None: response = client.get( "http://localhost:3001/v1/storage/assets/tables", query_string={"asset_view": "syn23643252", "return_type": "csv"}, @@ -158,7 +149,9 @@ def test_insufficent_auth(self, client, request_headers): @pytest.mark.synapse_credentials_needed @pytest.mark.parametrize("return_type", ["json", "csv"]) - def test_get_storage_assets_tables(self, client, return_type, request_headers): + def test_get_storage_assets_tables( + self, client: FlaskClient, return_type, request_headers: Dict[str, str] + ): params = {"asset_view": "syn23643253", "return_type": return_type} response = client.get( @@ -186,7 +179,13 @@ def test_get_storage_assets_tables(self, client, return_type, request_headers): @pytest.mark.synapse_credentials_needed @pytest.mark.parametrize("full_path", [True, False]) @pytest.mark.parametrize("file_names", [None, "Sample_A.txt"]) - def test_get_dataset_files(self, full_path, file_names, request_headers, client): + def test_get_dataset_files( + self, + full_path: bool, + file_names: Union[str, None], + request_headers: Dict[str, str], + client: FlaskClient, + ) -> None: params = { "asset_view": "syn23643253", "dataset_id": "syn23643250", @@ -203,17 +202,19 @@ def test_get_dataset_files(self, full_path, file_names, request_headers, client) ) assert response.status_code == 200 - response_dt = json.loads(response.data) + response_dt: List[Tuple[str, str]] = json.loads(response.data) # would show full file path .txt in result if full_path: if file_names: assert ( ["syn23643255", "schematic - main/DataTypeX/Sample_A.txt"] + in response_dt and [ "syn24226530", "schematic - main/TestDatasets/TestDataset-Annotations/Sample_A.txt", ] + in response_dt and [ "syn25057024", "schematic - main/TestDatasets/TestDataset-Annotations-v2/Sample_A.txt", @@ -228,23 +229,25 @@ def test_get_dataset_files(self, full_path, file_names, request_headers, client) else: if file_names: assert ( - ["syn23643255", "Sample_A.txt"] - and ["syn24226530", "Sample_A.txt"] + ["syn23643255", "Sample_A.txt"] in response_dt + and ["syn24226530", "Sample_A.txt"] in response_dt and ["syn25057024", "Sample_A.txt"] in response_dt ) - assert ["syn23643256", "Sample_C.txt"] and [ + assert ["syn23643256", "Sample_C.txt"] not in response_dt and [ "syn24226531", "Sample_B.txt", ] not in response_dt else: assert ( - ["syn23643256", "Sample_C.txt"] - and ["syn24226530", "Sample_A.txt"] + ["syn23643256", "Sample_C.txt"] in response_dt + and ["syn24226530", "Sample_A.txt"] in response_dt and ["syn24226531", "Sample_B.txt"] in response_dt ) @pytest.mark.synapse_credentials_needed - def test_get_storage_project_dataset(self, request_headers, client): + def test_get_storage_project_dataset( + self, request_headers: Dict[str, str], client: FlaskClient + ) -> None: params = {"asset_view": "syn23643253", "project_id": "syn26251192"} response = client.get( @@ -257,7 +260,9 @@ def test_get_storage_project_dataset(self, request_headers, client): assert ["syn26251193", "Issue522"] in response_dt @pytest.mark.synapse_credentials_needed - def test_get_storage_project_manifests(self, request_headers, client): + def test_get_storage_project_manifests( + self, request_headers: Dict[str, str], client: FlaskClient + ) -> None: params = {"asset_view": "syn23643253", "project_id": "syn30988314"} response = client.get( @@ -269,7 +274,9 @@ def test_get_storage_project_manifests(self, request_headers, client): assert response.status_code == 200 @pytest.mark.synapse_credentials_needed - def test_get_storage_projects(self, request_headers, client): + def test_get_storage_projects( + self, request_headers: Dict[str, str], client: FlaskClient + ) -> None: params = {"asset_view": "syn23643253"} response = client.get( @@ -282,7 +289,9 @@ def test_get_storage_projects(self, request_headers, client): @pytest.mark.synapse_credentials_needed @pytest.mark.parametrize("entity_id", ["syn34640850", "syn23643253", "syn24992754"]) - def test_get_entity_type(self, request_headers, client, entity_id): + def test_get_entity_type( + self, request_headers: Dict[str, str], client: FlaskClient, entity_id: str + ) -> None: params = {"asset_view": "syn23643253", "entity_id": entity_id} response = client.get( "http://localhost:3001/v1/storage/entity/type", @@ -301,7 +310,9 @@ def test_get_entity_type(self, request_headers, client, entity_id): @pytest.mark.synapse_credentials_needed @pytest.mark.parametrize("entity_id", ["syn30988314", "syn27221721"]) - def test_if_in_assetview(self, request_headers, client, entity_id): + def test_if_in_assetview( + self, request_headers: Dict[str, str], client: FlaskClient, entity_id: str + ) -> None: params = {"asset_view": "syn23643253", "entity_id": entity_id} response = client.get( "http://localhost:3001/v1/storage/if_in_asset_view", @@ -320,9 +331,9 @@ def test_if_in_assetview(self, request_headers, client, entity_id): @pytest.mark.schematic_api class TestMetadataModelOperation: @pytest.mark.parametrize("as_graph", [True, False]) - def test_component_requirement(self, client, data_model_jsonld, as_graph): + def test_component_requirement(self, client: FlaskClient, as_graph: bool) -> None: params = { - "schema_url": data_model_jsonld, + "schema_url": DATA_MODEL_JSON_LD, "source_component": "BulkRNA-seqAssay", "as_graph": as_graph, } @@ -347,7 +358,9 @@ def test_component_requirement(self, client, data_model_jsonld, as_graph): @pytest.mark.schematic_api class TestUtilsOperation: @pytest.mark.parametrize("strict_camel_case", [True, False]) - def test_get_property_label_from_display_name(self, client, strict_camel_case): + def test_get_property_label_from_display_name( + self, client: FlaskClient, strict_camel_case: bool + ) -> None: params = { "display_name": "mocular entity", "strict_camel_case": strict_camel_case, @@ -369,8 +382,8 @@ def test_get_property_label_from_display_name(self, client, strict_camel_case): @pytest.mark.schematic_api class TestDataModelGraphExplorerOperation: - def test_get_schema(self, client, data_model_jsonld): - params = {"schema_url": data_model_jsonld, "data_model_labels": "class_label"} + def test_get_schema(self, client: FlaskClient) -> None: + params = {"schema_url": DATA_MODEL_JSON_LD, "data_model_labels": "class_label"} response = client.get( "http://localhost:3001/v1/schemas/get/schema", query_string=params ) @@ -383,9 +396,9 @@ def test_get_schema(self, client, data_model_jsonld): if os.path.exists(response_dt): os.remove(response_dt) - def test_if_node_required(test, client, data_model_jsonld): + def test_if_node_required(test, client: FlaskClient) -> None: params = { - "schema_url": data_model_jsonld, + "schema_url": DATA_MODEL_JSON_LD, "node_display_name": "FamilyHistory", "data_model_labels": "class_label", } @@ -397,9 +410,9 @@ def test_if_node_required(test, client, data_model_jsonld): assert response.status_code == 200 assert response_dta == True - def test_get_node_validation_rules(test, client, data_model_jsonld): + def test_get_node_validation_rules(test, client: FlaskClient) -> None: params = { - "schema_url": data_model_jsonld, + "schema_url": DATA_MODEL_JSON_LD, "node_display_name": "CheckRegexList", } response = client.get( @@ -411,9 +424,9 @@ def test_get_node_validation_rules(test, client, data_model_jsonld): assert "list" in response_dta assert "regex match [a-f]" in response_dta - def test_get_nodes_display_names(test, client, data_model_jsonld): + def test_get_nodes_display_names(test, client: FlaskClient) -> None: params = { - "schema_url": data_model_jsonld, + "schema_url": DATA_MODEL_JSON_LD, "node_list": ["FamilyHistory", "Biospecimen"], } response = client.get( @@ -427,8 +440,8 @@ def test_get_nodes_display_names(test, client, data_model_jsonld): @pytest.mark.parametrize( "relationship", ["parentOf", "requiresDependency", "rangeValue", "domainValue"] ) - def test_get_subgraph_by_edge(self, client, data_model_jsonld, relationship): - params = {"schema_url": data_model_jsonld, "relationship": relationship} + def test_get_subgraph_by_edge(self, client: FlaskClient, relationship: str) -> None: + params = {"schema_url": DATA_MODEL_JSON_LD, "relationship": relationship} response = client.get( "http://localhost:3001/v1/schemas/get/graph_by_edge_type", @@ -439,10 +452,10 @@ def test_get_subgraph_by_edge(self, client, data_model_jsonld, relationship): @pytest.mark.parametrize("return_display_names", [True, False]) @pytest.mark.parametrize("node_label", ["FamilyHistory", "TissueStatus"]) def test_get_node_range( - self, client, data_model_jsonld, return_display_names, node_label - ): + self, client: FlaskClient, return_display_names: bool, node_label: str + ) -> None: params = { - "schema_url": data_model_jsonld, + "schema_url": DATA_MODEL_JSON_LD, "return_display_names": return_display_names, "node_label": node_label, } @@ -466,17 +479,16 @@ def test_get_node_range( @pytest.mark.parametrize("source_node", ["Patient", "Biospecimen"]) def test_node_dependencies( self, - client, - data_model_jsonld, - source_node, - return_display_names, - return_schema_ordered, - ): + client: FlaskClient, + source_node: str, + return_display_names: Union[bool, None], + return_schema_ordered: Union[bool, None], + ) -> None: return_display_names = True return_schema_ordered = False params = { - "schema_url": data_model_jsonld, + "schema_url": DATA_MODEL_JSON_LD, "source_node": source_node, "return_display_names": return_display_names, "return_schema_ordered": return_schema_ordered, @@ -519,7 +531,7 @@ def test_node_dependencies( @pytest.mark.schematic_api class TestManifestOperation: - def ifExcelExists(self, response, file_name): + def ifExcelExists(self, response, file_name) -> None: # return one excel file d = response.headers["content-disposition"] fname = re.findall("filename=(.+)", d)[0] @@ -543,13 +555,12 @@ def ifPandasDataframe(self, response_dt): ) def test_generate_existing_manifest( self, - client, - data_model_jsonld, - data_type, - output_format, - caplog, - request_headers, - ): + client: FlaskClient, + data_type: str, + output_format: str, + caplog: pytest.LogCaptureFixture, + request_headers: Dict[str, str], + ) -> None: # set dataset if data_type == "Patient": dataset_id = ["syn51730545"] # Mock Patient Manifest folder on synapse @@ -561,7 +572,7 @@ def test_generate_existing_manifest( dataset_id = None # if "all manifests", dataset id is None params = { - "schema_url": data_model_jsonld, + "schema_url": DATA_MODEL_JSON_LD, "asset_view": "syn23643253", "title": "Example", "data_type": data_type, @@ -627,15 +638,14 @@ def test_generate_existing_manifest( ) def test_generate_new_manifest( self, - caplog, - client, - data_model_jsonld, - data_type, - output_format, - request_headers, - ): + caplog: pytest.LogCaptureFixture, + client: FlaskClient, + data_type: str, + output_format: str, + request_headers: Dict[str, str], + ) -> None: params = { - "schema_url": data_model_jsonld, + "schema_url": DATA_MODEL_JSON_LD, "asset_view": "syn23643253", "title": "Example", "data_type": data_type, @@ -731,10 +741,10 @@ def test_generate_new_manifest( ], ) def test_generate_manifest_file_based_annotations( - self, client, use_annotations, expected, data_model_jsonld - ): + self, client: FlaskClient, use_annotations: bool, expected: list[str] + ) -> None: params = { - "schema_url": data_model_jsonld, + "schema_url": DATA_MODEL_JSON_LD, "data_type": "BulkRNA-seqAssay", "dataset_id": "syn25614635", "asset_view": "syn51707141", @@ -781,10 +791,10 @@ def test_generate_manifest_file_based_annotations( # test case: generate a manifest with annotations when use_annotations is set to True for a component that is not file-based # the dataset folder does not contain an existing manifest def test_generate_manifest_not_file_based_with_annotations( - self, client, data_model_jsonld - ): + self, client: FlaskClient + ) -> None: params = { - "schema_url": data_model_jsonld, + "schema_url": DATA_MODEL_JSON_LD, "data_type": "Patient", "dataset_id": "syn25614635", "asset_view": "syn51707141", @@ -816,9 +826,9 @@ def test_generate_manifest_not_file_based_with_annotations( ] ) - def test_generate_manifest_data_type_not_found(self, client, data_model_jsonld): + def test_generate_manifest_data_type_not_found(self, client: FlaskClient) -> None: params = { - "schema_url": data_model_jsonld, + "schema_url": DATA_MODEL_JSON_LD, "data_type": "wrong data type", "use_annotations": False, } @@ -829,13 +839,15 @@ def test_generate_manifest_data_type_not_found(self, client, data_model_jsonld): assert response.status_code == 500 assert "LookupError" in str(response.data) - def test_populate_manifest(self, client, data_model_jsonld, test_manifest_csv): + def test_populate_manifest( + self, client: FlaskClient, test_manifest_csv: str + ) -> None: # test manifest test_manifest_data = open(test_manifest_csv, "rb") params = { "data_type": "MockComponent", - "schema_url": data_model_jsonld, + "schema_url": DATA_MODEL_JSON_LD, "title": "Example", "csv_file": test_manifest_data, } @@ -861,14 +873,13 @@ def test_populate_manifest(self, client, data_model_jsonld, test_manifest_csv): ) def test_validate_manifest( self, - data_model_jsonld, - client, - json_str, - restrict_rules, - test_manifest_csv, - request_headers, - ): - params = {"schema_url": data_model_jsonld, "restrict_rules": restrict_rules} + client: FlaskClient, + json_str: Union[str, None], + restrict_rules: Union[bool, None], + test_manifest_csv: str, + request_headers: Dict[str, str], + ) -> None: + params = {"schema_url": DATA_MODEL_JSON_LD, "restrict_rules": restrict_rules} if json_str: params["json_str"] = json_str @@ -908,7 +919,9 @@ def test_validate_manifest( assert "warnings" in response_dt.keys() @pytest.mark.synapse_credentials_needed - def test_get_datatype_manifest(self, client, request_headers): + def test_get_datatype_manifest( + self, client: FlaskClient, request_headers: Dict[str, str] + ) -> None: params = {"asset_view": "syn23643253", "manifest_id": "syn27600110"} response = client.get( @@ -944,14 +957,14 @@ def test_get_datatype_manifest(self, client, request_headers): def test_manifest_download( self, config: Configuration, - client, - request_headers, - manifest_id, - new_manifest_name, - as_json, - expected_component, - expected_file_name, - ): + client: FlaskClient, + request_headers: Dict[str, str], + manifest_id: str, + new_manifest_name: str, + as_json: Union[bool, None], + expected_component: str, + expected_file_name: str, + ) -> None: params = { "manifest_id": manifest_id, "new_manifest_name": new_manifest_name, @@ -1006,7 +1019,9 @@ def test_manifest_download( @pytest.mark.synapse_credentials_needed # test downloading a manifest with access restriction and see if the correct error message got raised - def test_download_access_restricted_manifest(self, client, request_headers): + def test_download_access_restricted_manifest( + self, client: FlaskClient, request_headers: Dict[str, str] + ) -> None: params = {"manifest_id": "syn29862078"} response = client.get( @@ -1023,8 +1038,12 @@ def test_download_access_restricted_manifest(self, client, request_headers): @pytest.mark.parametrize("as_json", [None, True, False]) @pytest.mark.parametrize("new_manifest_name", [None, "Test"]) def test_dataset_manifest_download( - self, client, as_json, request_headers, new_manifest_name - ): + self, + client: FlaskClient, + as_json: Union[bool, None], + request_headers: Dict[str, str], + new_manifest_name: Union[str, None], + ) -> None: params = { "asset_view": "syn28559058", "dataset_id": "syn28268700", @@ -1056,11 +1075,14 @@ def test_dataset_manifest_download( @pytest.mark.synapse_credentials_needed @pytest.mark.submission def test_submit_manifest_table_and_file_replace( - self, client, request_headers, data_model_jsonld, test_manifest_submit - ): + self, + client: FlaskClient, + request_headers: Dict[str, str], + test_manifest_submit: str, + ) -> None: """Testing submit manifest in a csv format as a table and a file. Only replace the table""" params = { - "schema_url": data_model_jsonld, + "schema_url": DATA_MODEL_JSON_LD, "data_type": "Biospecimen", "restrict_rules": False, "hide_blanks": False, @@ -1092,16 +1114,15 @@ def test_submit_manifest_table_and_file_replace( def test_submit_manifest_file_only_replace( self, helpers, - client, - request_headers, - data_model_jsonld, - data_type, - manifest_path_fixture, - request, - ): + client: FlaskClient, + request_headers: Dict[str, str], + data_type: str, + manifest_path_fixture: str, + request: pytest.FixtureRequest, + ) -> None: """Testing submit manifest in a csv format as a file""" params = { - "schema_url": data_model_jsonld, + "schema_url": DATA_MODEL_JSON_LD, "data_type": data_type, "restrict_rules": False, "manifest_record_type": "file_only", @@ -1144,12 +1165,12 @@ def test_submit_manifest_file_only_replace( @pytest.mark.synapse_credentials_needed @pytest.mark.submission def test_submit_manifest_json_str_replace( - self, client, request_headers, data_model_jsonld - ): + self, client: FlaskClient, request_headers: Dict[str, str] + ) -> None: """Submit json str as a file""" json_str = '[{"Sample ID": 123, "Patient ID": 1,"Tissue Status": "Healthy","Component": "Biospecimen"}]' params = { - "schema_url": data_model_jsonld, + "schema_url": DATA_MODEL_JSON_LD, "data_type": "Biospecimen", "json_str": json_str, "restrict_rules": False, @@ -1172,10 +1193,13 @@ def test_submit_manifest_json_str_replace( @pytest.mark.synapse_credentials_needed @pytest.mark.submission def test_submit_manifest_w_file_and_entities( - self, client, request_headers, data_model_jsonld, test_manifest_submit - ): + self, + client: FlaskClient, + request_headers: Dict[str, str], + test_manifest_submit: str, + ) -> None: params = { - "schema_url": data_model_jsonld, + "schema_url": DATA_MODEL_JSON_LD, "data_type": "Biospecimen", "restrict_rules": False, "manifest_record_type": "file_and_entities", @@ -1200,13 +1224,12 @@ def test_submit_manifest_w_file_and_entities( @pytest.mark.submission def test_submit_manifest_table_and_file_upsert( self, - client, - request_headers, - data_model_jsonld, - test_upsert_manifest_csv, - ): + client: FlaskClient, + request_headers: Dict[str, str], + test_upsert_manifest_csv: str, + ) -> None: params = { - "schema_url": data_model_jsonld, + "schema_url": DATA_MODEL_JSON_LD, "data_type": "MockRDB", "restrict_rules": False, "manifest_record_type": "table_and_file", @@ -1214,7 +1237,8 @@ def test_submit_manifest_table_and_file_upsert( "dataset_id": "syn51514551", "table_manipulation": "upsert", "data_model_labels": "class_label", - "table_column_names": "display_name", # have to set table_column_names to display_name to ensure upsert feature works + # have to set table_column_names to display_name to ensure upsert feature works + "table_column_names": "display_name", } # test uploading a csv file @@ -1229,8 +1253,8 @@ def test_submit_manifest_table_and_file_upsert( @pytest.mark.schematic_api class TestSchemaVisualization: - def test_visualize_attributes(self, client, data_model_jsonld): - params = {"schema_url": data_model_jsonld} + def test_visualize_attributes(self, client: FlaskClient) -> None: + params = {"schema_url": DATA_MODEL_JSON_LD} response = client.get( "http://localhost:3001/v1/visualize/attributes", query_string=params @@ -1240,10 +1264,10 @@ def test_visualize_attributes(self, client, data_model_jsonld): @pytest.mark.parametrize("figure_type", ["component", "dependency"]) def test_visualize_tangled_tree_layers( - self, client, figure_type, data_model_jsonld - ): + self, client: FlaskClient, figure_type: str + ) -> None: # TODO: Determine a 2nd data model to use for this test, test both models sequentially, add checks for content of response - params = {"schema_url": data_model_jsonld, "figure_type": figure_type} + params = {"schema_url": DATA_MODEL_JSON_LD, "figure_type": figure_type} response = client.get( "http://localhost:3001/v1/visualize/tangled_tree/layers", @@ -1260,10 +1284,10 @@ def test_visualize_tangled_tree_layers( ], ) def test_visualize_component( - self, client, data_model_jsonld, component, response_text - ): + self, client: FlaskClient, component: str, response_text: str + ) -> None: params = { - "schema_url": data_model_jsonld, + "schema_url": DATA_MODEL_JSON_LD, "component": component, "include_index": False, "data_model_labels": "class_label", @@ -1288,11 +1312,10 @@ class TestValidationBenchmark: def test_validation_performance( self, helpers, - benchmark_data_model_jsonld, - client, - test_invalid_manifest, - MockComponent_attribute, - ): + client: FlaskClient, + test_invalid_manifest: pd.DataFrame, + MockComponent_attribute: Generator[str, None, None], + ) -> None: """ Test to benchamrk performance of validation rules on large manifests Test loads the invalid_test_manifest.csv and isolates one attribute at a time @@ -1309,7 +1332,7 @@ def test_validation_performance( # Set paramters for endpoint params = { - "schema_url": benchmark_data_model_jsonld, + "schema_url": BENCHMARK_DATA_MODEL_JSON_LD, "data_type": "MockComponent", } headers = {"Content-Type": "multipart/form-data", "Accept": "application/json"} diff --git a/tests/test_store.py b/tests/test_store.py index 0bdfa0871..cefa6d56a 100644 --- a/tests/test_store.py +++ b/tests/test_store.py @@ -392,7 +392,10 @@ def test_getFilesInStorageDataset(self, synapse_store, full_path, expected): [("test_file", "syn126")], ), ( - (os.path.join("parent_folder", "test_folder"), "syn124"), + ( + os.path.join("schematic - main", "parent_folder", "test_folder"), + "syn124", + ), [], [("test_file_2", "syn125")], ), From c2b832d314348740b4b44c2f2ee59bbc34e0b57d Mon Sep 17 00:00:00 2001 From: GiaJordan Date: Tue, 27 Aug 2024 09:58:16 -0700 Subject: [PATCH 121/233] move test to integration dir --- tests/integration/test_metadata_model.py | 42 ++++++++++++++++++++++++ tests/test_metadata.py | 22 ------------- 2 files changed, 42 insertions(+), 22 deletions(-) create mode 100644 tests/integration/test_metadata_model.py diff --git a/tests/integration/test_metadata_model.py b/tests/integration/test_metadata_model.py new file mode 100644 index 000000000..29167c991 --- /dev/null +++ b/tests/integration/test_metadata_model.py @@ -0,0 +1,42 @@ +import logging +from contextlib import contextmanager +from unittest.mock import patch + +from schematic.models.metadata import MetadataModel + +logging.basicConfig(level=logging.DEBUG) +logger = logging.getLogger(__name__) + + +@contextmanager +def does_not_raise(): + yield + + +def metadata_model(helpers, data_model_labels): + metadata_model = MetadataModel( + inputMModelLocation=helpers.get_data_path("example.model.jsonld"), + data_model_labels=data_model_labels, + inputMModelLocationType="local", + ) + + return metadata_model + + +class TestMetadataModel: + def test_submit_filebased_manifest(self, helpers): + meta_data_model = metadata_model(helpers, "class_label") + + manifest_path = helpers.get_data_path( + "mock_manifests/filepath_submission_test_manifest.csv" + ) + + with does_not_raise(): + manifest_id = meta_data_model.submit_metadata_manifest( + manifest_path=manifest_path, + dataset_id="syn62276880", + manifest_record_type="file_only", + restrict_rules=False, + file_annotations_upload=True, + ) + assert manifest_id == "syn62280543" diff --git a/tests/test_metadata.py b/tests/test_metadata.py index 8065f793a..b192f5387 100644 --- a/tests/test_metadata.py +++ b/tests/test_metadata.py @@ -16,11 +16,6 @@ logger = logging.getLogger(__name__) -@contextmanager -def does_not_raise(): - yield - - def metadata_model(helpers, data_model_labels): metadata_model = MetadataModel( inputMModelLocation=helpers.get_data_path("example.model.jsonld"), @@ -149,20 +144,3 @@ def test_submit_metadata_manifest( hide_blanks=hide_blanks, ) assert mock_manifest_id == "mock manifest id" - - def test_submit_filebased_manifest(self, helpers): - meta_data_model = metadata_model(helpers, "class_label") - - manifest_path = helpers.get_data_path( - "mock_manifests/filepath_submission_test_manifest.csv" - ) - - with does_not_raise(): - manifest_id = meta_data_model.submit_metadata_manifest( - manifest_path=manifest_path, - dataset_id="syn62276880", - manifest_record_type="file_only", - restrict_rules=False, - file_annotations_upload=True, - ) - assert manifest_id == "syn62280543" From 2dd6fdf7108c0c8b1d08ee5f7f6be5d2300fcaf2 Mon Sep 17 00:00:00 2001 From: GiaJordan Date: Tue, 27 Aug 2024 10:05:50 -0700 Subject: [PATCH 122/233] move function to conftest --- tests/conftest.py | 11 +++++++++++ tests/integration/test_metadata_model.py | 11 +---------- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 9a0b5789d..6ce21df5d 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -9,6 +9,7 @@ from dotenv import load_dotenv from schematic.configuration.configuration import CONFIG +from schematic.models.metadata import MetadataModel from schematic.schemas.data_model_graph import DataModelGraph, DataModelGraphExplorer from schematic.schemas.data_model_parser import DataModelParser from schematic.store.synapse import SynapseStorage @@ -149,3 +150,13 @@ def DMGE(helpers: Helpers) -> DataModelGraphExplorer: """Fixture to instantiate a DataModelGraphExplorer object.""" dmge = helpers.get_data_model_graph_explorer(path="example.model.jsonld") return dmge + + +def metadata_model(helpers, data_model_labels): + metadata_model = MetadataModel( + inputMModelLocation=helpers.get_data_path("example.model.jsonld"), + data_model_labels=data_model_labels, + inputMModelLocationType="local", + ) + + return metadata_model diff --git a/tests/integration/test_metadata_model.py b/tests/integration/test_metadata_model.py index 29167c991..1ed8899f5 100644 --- a/tests/integration/test_metadata_model.py +++ b/tests/integration/test_metadata_model.py @@ -3,6 +3,7 @@ from unittest.mock import patch from schematic.models.metadata import MetadataModel +from tests.conftest import metadata_model logging.basicConfig(level=logging.DEBUG) logger = logging.getLogger(__name__) @@ -13,16 +14,6 @@ def does_not_raise(): yield -def metadata_model(helpers, data_model_labels): - metadata_model = MetadataModel( - inputMModelLocation=helpers.get_data_path("example.model.jsonld"), - data_model_labels=data_model_labels, - inputMModelLocationType="local", - ) - - return metadata_model - - class TestMetadataModel: def test_submit_filebased_manifest(self, helpers): meta_data_model = metadata_model(helpers, "class_label") From 096fe550ba5810ad50a08986a927bdc8193d34f4 Mon Sep 17 00:00:00 2001 From: GiaJordan Date: Tue, 27 Aug 2024 11:11:38 -0700 Subject: [PATCH 123/233] update exception message --- schematic/store/synapse.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/schematic/store/synapse.py b/schematic/store/synapse.py index 69af178fc..c9ca36c10 100644 --- a/schematic/store/synapse.py +++ b/schematic/store/synapse.py @@ -298,9 +298,9 @@ def query_fileview( "The path column has not been added to the fileview. Please make sure that the fileview is up to date. You can add the path column to the fileview by follwing the instructions in the validation rules documentation." ) from exc elif "Unknown column" in exception_text: - missing_columns = exception_text.split("Unknown column ")[-1] + missing_column = exception_text.split("Unknown column ")[-1] raise ValueError( - f"The column(s) ({missing_columns}) specified in the query do not exist in the fileview. Please make sure that the column names are correct and that all expected columns have been added to the fileview." + f"The columns {missing_column} specified in the query do not exist in the fileview. Please make sure that the column names are correct and that all expected columns have been added to the fileview." ) from exc else: raise AccessCredentialsError(self.storageFileview) From 05a791eb3180128c0ca81e60907fa84ae7bc8cb7 Mon Sep 17 00:00:00 2001 From: GiaJordan Date: Tue, 27 Aug 2024 11:12:14 -0700 Subject: [PATCH 124/233] update query tests --- tests/test_store.py | 30 +++++++++++++++++++----------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/tests/test_store.py b/tests/test_store.py index 024581411..4f13c10da 100644 --- a/tests/test_store.py +++ b/tests/test_store.py @@ -208,18 +208,26 @@ def test_view_query( synapse_store_special_scope.project_scope = project_scope - synapse_store_special_scope.query_fileview(columns, where_clauses) - # tests ._build_query() - assert synapse_store_special_scope.fileview_query == expected - # tests that the query was valid and successful, that a view subset has actually been retrived - assert synapse_store_special_scope.storageFileviewTable.empty is False + with does_not_raise(): + synapse_store_special_scope.query_fileview(columns, where_clauses) + # tests ._build_query() + assert synapse_store_special_scope.fileview_query == expected + # tests that the query was valid and successful, that a view subset has actually been retrived + assert synapse_store_special_scope.storageFileviewTable.empty is False @pytest.mark.parametrize( - "asset_view,columns,expectation", + "asset_view,columns,message", [ - ("syn62339865", ["path"], pytest.raises(ValueError)), - ("syn62340177", ["id"], pytest.raises(ValueError)), - ("syn23643253", ["id", "path"], does_not_raise()), + ( + "syn62339865", + ["path"], + r"The path column has not been added to the fileview. .*", + ), + ( + "syn62340177", + ["id"], + r"The columns id specified in the query do not exist in the fileview. .*", + ), ], ) def test_view_query_exception( @@ -227,7 +235,7 @@ def test_view_query_exception( synapse_store_special_scope: SynapseStorage, asset_view: str, columns: list[str], - expectation: str, + message: str, ) -> None: project_scope = ["syn23643250"] @@ -237,7 +245,7 @@ def test_view_query_exception( # ensure approriate exception is raised try: - with expectation: + with pytest.raises(ValueError, match=message): synapse_store_special_scope.query_fileview(columns) finally: # reset config to default fileview so subsequent tests do not use the test fileviews utilized by this test From 2b84f4d7b386c3e731584863c5f02a2168d14ac8 Mon Sep 17 00:00:00 2001 From: GiaJordan Date: Tue, 27 Aug 2024 11:32:59 -0700 Subject: [PATCH 125/233] update comments --- tests/test_store.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/tests/test_store.py b/tests/test_store.py index 4f13c10da..bebad9920 100644 --- a/tests/test_store.py +++ b/tests/test_store.py @@ -203,16 +203,19 @@ def test_view_query( where_clauses: list, expected: str, ) -> None: - # Ensure correct view is being utilized + # GIVEN a the correct fileview assert synapse_store_special_scope.storageFileview == "syn23643253" + # AND the approrpiate project scope synapse_store_special_scope.project_scope = project_scope + # WHEN the query is built and run + # THEN it should complete without raising an exception with does_not_raise(): synapse_store_special_scope.query_fileview(columns, where_clauses) - # tests ._build_query() + # AND the query string should be as expected assert synapse_store_special_scope.fileview_query == expected - # tests that the query was valid and successful, that a view subset has actually been retrived + # AND query should have recieved a non-empty table assert synapse_store_special_scope.storageFileviewTable.empty is False @pytest.mark.parametrize( @@ -239,16 +242,18 @@ def test_view_query_exception( ) -> None: project_scope = ["syn23643250"] - # set to use appropriate test view and set scope + # GIVEN the appropriate test file view CONFIG.synapse_master_fileview_id = asset_view + # AND the approrpiate project scope synapse_store_special_scope.project_scope = project_scope - # ensure approriate exception is raised + # WHEN the query is built and run try: + # THEN it should raise a ValueError with the appropriate message with pytest.raises(ValueError, match=message): synapse_store_special_scope.query_fileview(columns) finally: - # reset config to default fileview so subsequent tests do not use the test fileviews utilized by this test + # AND the fileview should be reset to the default so the other tests are not affected regardless of the outcome of the query CONFIG.synapse_master_fileview_id = "syn23643253" def test_getFileAnnotations(self, synapse_store: SynapseStorage) -> None: From fe39f580bbc58d17dedbd4d92d87d3c414ddb908 Mon Sep 17 00:00:00 2001 From: linglp Date: Tue, 27 Aug 2024 14:52:48 -0400 Subject: [PATCH 126/233] try a new test data model --- tests/data/test-model.csv | 57 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 tests/data/test-model.csv diff --git a/tests/data/test-model.csv b/tests/data/test-model.csv new file mode 100644 index 000000000..230addbd8 --- /dev/null +++ b/tests/data/test-model.csv @@ -0,0 +1,57 @@ +Attribute,Description,Valid Values,DependsOn,Properties,Required,Parent,DependsOn Component,Source,Validation Rules +Component,,,,,TRUE,,,, +Patient,,,"Patient ID, Sex, Year of Birth, Diagnosis, Component",,FALSE,DataType,,, +Patient ID,,,,,TRUE,DataProperty,,,#Patient unique warning^^#Biospecimen unique error +Sex,,"Female, Male, Other",,,TRUE,DataProperty,,, +Year of Birth,,,,,FALSE,DataProperty,,, +Diagnosis,,"Healthy, Cancer",,,TRUE,DataProperty,,, +Cancer,,,"Cancer Type, Family History",,FALSE,ValidValue,,, +Cancer Type,,"Breast, Colorectal, Lung, Prostate, Skin",,,TRUE,DataProperty,,, +Family History,,"Breast, Colorectal, Lung, Prostate, Skin",,,TRUE,DataProperty,,,list strict +Biospecimen,,,"Sample ID, Patient ID, Tissue Status, Component",,FALSE,DataType,Patient,, +Sample ID,,,,,TRUE,DataProperty,,, +Tissue Status,,"Healthy, Malignant",,,TRUE,DataProperty,,, +Bulk RNA-seq Assay,,,"Filename, Sample ID, File Format, Component",,FALSE,DataType,Biospecimen,, +Filename,,,,,TRUE,DataProperty,,,#MockFilename filenameExists syn61682648^^ +File Format,,"FASTQ, BAM, CRAM, CSV/TSV",,,TRUE,DataProperty,,, +BAM,,,Genome Build,,FALSE,ValidValue,,, +CRAM,,,"Genome Build, Genome FASTA",,FALSE,ValidValue,,, +CSV/TSV,,,Genome Build,,FALSE,ValidValue,,, +Genome Build,,"GRCh37, GRCh38, GRCm38, GRCm39",,,TRUE,DataProperty,,, +Genome FASTA,,,,,TRUE,DataProperty,,, +MockComponent,,,"Component, Check List, Check List Enum, Check List Like, Check List Like Enum, Check List Strict, Check List Enum Strict, Check Regex List, Check Regex List Like, Check Regex List Strict, Check Regex Single, Check Regex Format, Check Regex Integer, Check Num, Check Float, Check Int, Check String, Check URL,Check Match at Least, Check Match at Least values, Check Match Exactly, Check Match Exactly values, Check Match None, Check Match None values, Check Recommended, Check Ages, Check Unique, Check Range, Check Date, Check NA",,FALSE,DataType,,, +Check List,,,,,TRUE,DataProperty,,,list +Check List Enum,,"ab, cd, ef, gh",,,TRUE,DataProperty,,,list +Check List Like,,,,,TRUE,DataProperty,,,list like +Check List Like Enum,,"ab, cd, ef, gh",,,TRUE,DataProperty,,,list like +Check List Strict,,,,,TRUE,DataProperty,,,list strict +Check List Enum Strict,,"ab, cd, ef, gh",,,TRUE,DataProperty,,,list strict +Check Regex List,,,,,TRUE,DataProperty,,,list::regex match [a-f] +Check Regex List Strict,,,,,TRUE,DataProperty,,,list strict::regex match [a-f] +Check Regex List Like,,,,,TRUE,DataProperty,,,list like::regex match [a-f] +Check Regex Single,,,,,TRUE,DataProperty,,,regex search [a-f] +Check Regex Format,,,,,TRUE,DataProperty,,,regex match [a-f] +Check Regex Integer,,,,,TRUE,DataProperty,,,regex search ^\d+$ +Check Num,,,,,TRUE,DataProperty,,,num +Check Float,,,,,TRUE,DataProperty,,,float +Check Int,,,,,TRUE,DataProperty,,,int +Check String,,,,,TRUE,DataProperty,,,str +Check URL,,,,,TRUE,DataProperty,,,url +Check Match at Least,,,,,TRUE,DataProperty,,,matchAtLeastOne Patient.PatientID set +Check Match Exactly,,,,,TRUE,DataProperty,,,matchExactlyOne MockComponent.checkMatchExactly set +Check Match None,,,,,TRUE,DataProperty,,,matchNone MockComponent.checkMatchNone set error +Check Match at Least values,,,,,TRUE,DataProperty,,,matchAtLeastOne MockComponent.checkMatchatLeastvalues value +Check Match Exactly values,,,,,TRUE,DataProperty,,,matchExactlyOne MockComponent.checkMatchExactlyvalues value +Check Match None values,,,,,TRUE,DataProperty,,,matchNone MockComponent.checkMatchNonevalues value error +Check Recommended,,,,,FALSE,DataProperty,,,recommended +Check Ages,,,,,TRUE,DataProperty,,,protectAges +MockCombinedRule,,,,,TRUE,DataProperty,,,#MockComponent unique error::regex match ^[a-zA-Z0-9_-]+$ error::required +Check Unique,,,,,TRUE,DataProperty,,,unique error +Check Range,,,,,TRUE,DataProperty,,,inRange 50 100 error +Check Date,,,,,TRUE,DataProperty,,,date +Check NA,,,,,TRUE,DataProperty,,,int::IsNA +MockRDB,,,"Component, MockRDB_id, SourceManifest",,FALSE,DataType,,, +MockRDB_id,,,,,TRUE,DataProperty,,,int +SourceManifest,,,,,TRUE,DataProperty,,, +MockFilename,,,"Component, Filename",,FALSE,DataType,,, +MockComponent,,,"Component, MockCombinedRule",,FALSE,DataType,,, \ No newline at end of file From b4e20dccfb52bf9a861fe8d919850349aca408e2 Mon Sep 17 00:00:00 2001 From: linglp Date: Tue, 27 Aug 2024 15:03:49 -0400 Subject: [PATCH 127/233] change component name --- tests/data/test-model.csv | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/data/test-model.csv b/tests/data/test-model.csv index 230addbd8..2e4db1e3a 100644 --- a/tests/data/test-model.csv +++ b/tests/data/test-model.csv @@ -45,7 +45,7 @@ Check Match Exactly values,,,,,TRUE,DataProperty,,,matchExactlyOne MockComponent Check Match None values,,,,,TRUE,DataProperty,,,matchNone MockComponent.checkMatchNonevalues value error Check Recommended,,,,,FALSE,DataProperty,,,recommended Check Ages,,,,,TRUE,DataProperty,,,protectAges -MockCombinedRule,,,,,TRUE,DataProperty,,,#MockComponent unique error::regex match ^[a-zA-Z0-9_-]+$ error::required +MockCombinedRule,,,,,TRUE,DataProperty,,,#AnotherMockComponent unique error::regex match ^[a-zA-Z0-9_-]+$ error::required Check Unique,,,,,TRUE,DataProperty,,,unique error Check Range,,,,,TRUE,DataProperty,,,inRange 50 100 error Check Date,,,,,TRUE,DataProperty,,,date @@ -54,4 +54,4 @@ MockRDB,,,"Component, MockRDB_id, SourceManifest",,FALSE,DataType,,, MockRDB_id,,,,,TRUE,DataProperty,,,int SourceManifest,,,,,TRUE,DataProperty,,, MockFilename,,,"Component, Filename",,FALSE,DataType,,, -MockComponent,,,"Component, MockCombinedRule",,FALSE,DataType,,, \ No newline at end of file +AnotherMockComponent,,,"Component, MockCombinedRule",,FALSE,DataType,,, \ No newline at end of file From cb0f0fbe329f6c61ea165dbd21f0dceb00a6d660 Mon Sep 17 00:00:00 2001 From: GiaJordan Date: Tue, 27 Aug 2024 12:56:06 -0700 Subject: [PATCH 128/233] update imports --- tests/test_metadata.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_metadata.py b/tests/test_metadata.py index b192f5387..fca1d3db5 100644 --- a/tests/test_metadata.py +++ b/tests/test_metadata.py @@ -2,7 +2,6 @@ import logging import os -from contextlib import contextmanager from pathlib import Path from typing import Generator, Optional from unittest.mock import patch From a916641319b0d7dc37991fcffd087609c353751b Mon Sep 17 00:00:00 2001 From: GiaJordan Date: Tue, 27 Aug 2024 12:56:43 -0700 Subject: [PATCH 129/233] update get manifest test --- tests/test_manifest.py | 67 +++++++++++++++++++++++++++++------------- 1 file changed, 46 insertions(+), 21 deletions(-) diff --git a/tests/test_manifest.py b/tests/test_manifest.py index b4daaaf76..06bd7b168 100644 --- a/tests/test_manifest.py +++ b/tests/test_manifest.py @@ -763,24 +763,52 @@ def test_create_manifests( assert all_results == expected_result @pytest.mark.parametrize( - "component,datasetId", - [("Biospecimen", "syn61260107"), ("BulkRNA-seqAssay", "syn61374924")], + "component,datasetId,expected_file_based,expected_rows,expected_files", + [ + ("Biospecimen", "syn61260107", False, 4, None), + ( + "BulkRNA-seqAssay", + "syn61374924", + True, + 4, + pd.Series( + [ + "schematic - main/BulkRNASeq and files/txt1.txt", + "schematic - main/BulkRNASeq and files/txt2.txt", + "schematic - main/BulkRNASeq and files/txt4.txt", + "schematic - main/BulkRNASeq and files/txt3.txt", + ], + name="Filename", + ), + ), + ], ids=["Record based", "File based"], ) - def test_get_manifest_with_files(self, helpers, component, datasetId): + def test_get_manifest_with_files( + self, + helpers, + component, + datasetId, + expected_file_based, + expected_rows, + expected_files, + ): """ Test to ensure that when generating a record based manifset that has files in the dataset that the files are not added to the manifest as well when generating a file based manifest from a dataset thathas had files added that the files are added correctly """ + # GIVEN the example data model path_to_data_model = helpers.get_data_path("example.model.jsonld") + # AND a graph data model graph_data_model = generate_graph_data_model( helpers, path_to_data_model=path_to_data_model, data_model_labels="class_label", ) + # AND a manifest generator generator = ManifestGenerator( path_to_data_model=path_to_data_model, graph=graph_data_model, @@ -788,27 +816,24 @@ def test_get_manifest_with_files(self, helpers, component, datasetId): use_annotations=True, ) + # WHEN a manifest is generated for the appropriate dataset as a dataframe manifest = generator.get_manifest( dataset_id=datasetId, output_format="dataframe" ) - filename_in_manifest_columns = "Filename" in manifest.columns + # AND it is determined if the manifest is filebased + is_file_based = "Filename" in manifest.columns + + # AND the number of rows are checked n_rows = manifest.shape[0] - if component == "Biospecimen": - assert not filename_in_manifest_columns - assert n_rows == 4 - elif component == "BulkRNA-seqAssay": - assert filename_in_manifest_columns - assert n_rows == 4 - - expected_files = pd.Series( - [ - "schematic - main/BulkRNASeq and files/txt1.txt", - "schematic - main/BulkRNASeq and files/txt2.txt", - "schematic - main/BulkRNASeq and files/txt4.txt", - "schematic - main/BulkRNASeq and files/txt3.txt", - ], - name="Filename", - ) - assert expected_files.equals(manifest["Filename"]) + # THEN the manifest should have the expected number of rows + assert n_rows == expected_rows + + # AND the manifest should be filebased or not as expected + assert is_file_based == expected_file_based + + # AND if the manifest is file based + if expected_file_based: + # THEN the manifest should have the expected files + assert manifest["Filename"].equals(expected_files) From e3107a535d1284398442cdf4faa295c3d981b910 Mon Sep 17 00:00:00 2001 From: GiaJordan Date: Tue, 27 Aug 2024 13:02:17 -0700 Subject: [PATCH 130/233] cover other cases in query test --- tests/test_store.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/test_store.py b/tests/test_store.py index bebad9920..a06d8e95f 100644 --- a/tests/test_store.py +++ b/tests/test_store.py @@ -187,6 +187,18 @@ def test_login(self) -> None: None, "SELECT name,id,path FROM syn23643253 WHERE projectId IN ('syn23643250', '') ;", ), + ( + None, + ["name", "id", "path"], + ["parentId='syn61682648'", "type='file'"], + "SELECT name,id,path FROM syn23643253 WHERE parentId='syn61682648' AND type='file' ;", + ), + ( + ["syn23643250"], + None, + ["parentId='syn61682648'", "type='file'"], + "SELECT * FROM syn23643253 WHERE parentId='syn61682648' AND type='file' AND projectId IN ('syn23643250', '') ;", + ), ( ["syn23643250"], ["name", "id", "path"], From 57ec340f0b6df364aba23fcee757a989121150ba Mon Sep 17 00:00:00 2001 From: GiaJordan Date: Tue, 27 Aug 2024 13:10:56 -0700 Subject: [PATCH 131/233] update query test --- tests/test_store.py | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/tests/test_store.py b/tests/test_store.py index a06d8e95f..3517439af 100644 --- a/tests/test_store.py +++ b/tests/test_store.py @@ -122,7 +122,7 @@ def dmge( yield dmge -@pytest.fixture +@pytest.fixture(scope="module") def synapse_store_special_scope(): yield SynapseStorage(perform_query=False) @@ -166,44 +166,57 @@ def test_login(self) -> None: shutil.rmtree("test_cache_dir") @pytest.mark.parametrize( - "project_scope,columns,where_clauses,expected", + "project_scope,columns,where_clauses,expected,expected_new_query", [ - (None, None, None, "SELECT * FROM syn23643253 ;"), + (None, None, None, "SELECT * FROM syn23643253 ;", True), ( ["syn23643250"], None, None, "SELECT * FROM syn23643253 WHERE projectId IN ('syn23643250', '') ;", + True, ), ( None, None, ["projectId IN ('syn23643250')"], "SELECT * FROM syn23643253 WHERE projectId IN ('syn23643250') ;", + True, ), ( ["syn23643250"], ["name", "id", "path"], None, "SELECT name,id,path FROM syn23643253 WHERE projectId IN ('syn23643250', '') ;", + True, ), ( None, ["name", "id", "path"], ["parentId='syn61682648'", "type='file'"], "SELECT name,id,path FROM syn23643253 WHERE parentId='syn61682648' AND type='file' ;", + True, ), ( ["syn23643250"], None, ["parentId='syn61682648'", "type='file'"], "SELECT * FROM syn23643253 WHERE parentId='syn61682648' AND type='file' AND projectId IN ('syn23643250', '') ;", + True, + ), + ( + ["syn23643250"], + ["name", "id", "path"], + ["parentId='syn61682648'", "type='file'"], + "SELECT name,id,path FROM syn23643253 WHERE parentId='syn61682648' AND type='file' AND projectId IN ('syn23643250', '') ;", + True, ), ( ["syn23643250"], ["name", "id", "path"], ["parentId='syn61682648'", "type='file'"], "SELECT name,id,path FROM syn23643253 WHERE parentId='syn61682648' AND type='file' AND projectId IN ('syn23643250', '') ;", + False, ), ], ) @@ -214,6 +227,7 @@ def test_view_query( columns: list, where_clauses: list, expected: str, + expected_new_query: bool, ) -> None: # GIVEN a the correct fileview assert synapse_store_special_scope.storageFileview == "syn23643253" @@ -229,6 +243,8 @@ def test_view_query( assert synapse_store_special_scope.fileview_query == expected # AND query should have recieved a non-empty table assert synapse_store_special_scope.storageFileviewTable.empty is False + # AND the query should be new if expected + assert synapse_store_special_scope.new_query_different == expected_new_query @pytest.mark.parametrize( "asset_view,columns,message", From bb043f6a2cfd724a847d05db5674c842fbc71fc0 Mon Sep 17 00:00:00 2001 From: GiaJordan Date: Tue, 27 Aug 2024 13:14:48 -0700 Subject: [PATCH 132/233] change structure of test --- tests/test_store.py | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/tests/test_store.py b/tests/test_store.py index 3517439af..38db2d74a 100644 --- a/tests/test_store.py +++ b/tests/test_store.py @@ -505,7 +505,7 @@ def test_getDatasetManifest(self, synapse_store, downloadFile): assert manifest_data == "syn51204513" @pytest.mark.parametrize( - "existing_manifest_df,fill_in_return_value", + "existing_manifest_df,fill_in_return_value,expected_df", [ ( pd.DataFrame(), @@ -519,6 +519,9 @@ def test_getDatasetManifest(self, synapse_store, downloadFile): "entityId": ["mock_entity_id"], }, ], + pd.DataFrame( + {"Filename": ["mock_file_path"], "entityId": ["mock_entity_id"]} + ), ), ( pd.DataFrame( @@ -537,11 +540,17 @@ def test_getDatasetManifest(self, synapse_store, downloadFile): "entityId": ["mock_entity_id"], }, ], + pd.DataFrame( + { + "Filename": ["existing_mock_file_path", "mock_file_path"], + "entityId": ["existing_mock_entity_id", "mock_entity_id"], + } + ), ), ], ) def test_fill_in_entity_id_filename( - self, synapse_store, existing_manifest_df, fill_in_return_value + self, synapse_store, existing_manifest_df, fill_in_return_value, expected_df ): with patch( "schematic.store.synapse.SynapseStorage.getFilesInStorageDataset", @@ -553,17 +562,7 @@ def test_fill_in_entity_id_filename( dataset_files, new_manifest = synapse_store.fill_in_entity_id_filename( datasetId="test_syn_id", manifest=existing_manifest_df ) - if not existing_manifest_df.empty: - expected_df = pd.DataFrame( - { - "Filename": ["existing_mock_file_path", "mock_file_path"], - "entityId": ["existing_mock_entity_id", "mock_entity_id"], - } - ) - else: - expected_df = pd.DataFrame( - {"Filename": ["mock_file_path"], "entityId": ["mock_entity_id"]} - ) + assert_frame_equal(new_manifest, expected_df) assert dataset_files == ["syn123", "syn124", "syn125"] From ed9837bd71c472642ff82afe5ec00fff06a11ca8 Mon Sep 17 00:00:00 2001 From: GiaJordan Date: Tue, 27 Aug 2024 13:26:05 -0700 Subject: [PATCH 133/233] update mock data names --- tests/test_store.py | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/tests/test_store.py b/tests/test_store.py index 38db2d74a..d2bf6d57a 100644 --- a/tests/test_store.py +++ b/tests/test_store.py @@ -511,16 +511,19 @@ def test_getDatasetManifest(self, synapse_store, downloadFile): pd.DataFrame(), [ { - "Filename": ["mock_file_path"], - "entityId": ["mock_entity_id"], + "Filename": ["new_mock_file_path"], + "entityId": ["new_mock_entity_id"], }, { - "Filename": ["mock_file_path"], - "entityId": ["mock_entity_id"], + "Filename": ["new_mock_file_path"], + "entityId": ["new_mock_entity_id"], }, ], pd.DataFrame( - {"Filename": ["mock_file_path"], "entityId": ["mock_entity_id"]} + { + "Filename": ["new_mock_file_path"], + "entityId": ["new_mock_entity_id"], + } ), ), ( @@ -532,18 +535,18 @@ def test_getDatasetManifest(self, synapse_store, downloadFile): ), [ { - "Filename": ["existing_mock_file_path", "mock_file_path"], - "entityId": ["existing_mock_entity_id", "mock_entity_id"], + "Filename": ["existing_mock_file_path", "new_mock_file_path"], + "entityId": ["existing_mock_entity_id", "new_mock_entity_id"], }, { - "Filename": ["mock_file_path"], - "entityId": ["mock_entity_id"], + "Filename": ["new_mock_file_path"], + "entityId": ["new_mock_entity_id"], }, ], pd.DataFrame( { - "Filename": ["existing_mock_file_path", "mock_file_path"], - "entityId": ["existing_mock_entity_id", "mock_entity_id"], + "Filename": ["existing_mock_file_path", "new_mock_file_path"], + "entityId": ["existing_mock_entity_id", "new_mock_entity_id"], } ), ), From 7d89e6e157bba24aacb18277533810840d14097e Mon Sep 17 00:00:00 2001 From: GiaJordan Date: Tue, 27 Aug 2024 13:57:20 -0700 Subject: [PATCH 134/233] update entityId logic with manifest merge --- schematic/store/synapse.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/schematic/store/synapse.py b/schematic/store/synapse.py index e62f11e01..99876b62d 100644 --- a/schematic/store/synapse.py +++ b/schematic/store/synapse.py @@ -216,6 +216,9 @@ def __init__( Consider necessity of adding "columns" and "where_clauses" params to the constructor. Currently with how `query_fileview` is implemented, these params are not needed at this step but could be useful in the future if the need for more scoped querys expands. """ self.syn = self.login(synapse_cache_path, access_token) + # TODO REMOVE + self.syn.setEndpoints(**synapseclient.client.STAGING_ENDPOINTS) + self.project_scope = project_scope self.storageFileview = CONFIG.synapse_master_fileview_id self.manifest = CONFIG.synapse_manifest_basename @@ -1861,7 +1864,17 @@ async def add_annotations_to_entities_files( # Merge dataframes to add entityIds manifest = manifest.merge( file_df, how="left", on="Filename", suffixes=["_x", None] - ).drop("entityId_x", axis=1) + ) + + # drop the duplicate entity column with NA values + col_to_drop = "entityId_x" + if manifest.entityId.isnull().all(): + col_to_drop = "entityId" + + # If the original entityId column is empty after the merge, drop it and rename the duplicate column + manifest.drop(columns=[col_to_drop], inplace=True) + if col_to_drop == "entityId": + manifest.rename(columns={"entityId_x": "entityId"}, inplace=True) # Fill `entityId` for each row if missing and annotate entity as appropriate requests = set() From cf500651b4ddff710da0b97e7eefdfce5f2ee825 Mon Sep 17 00:00:00 2001 From: GiaJordan Date: Tue, 27 Aug 2024 14:19:28 -0700 Subject: [PATCH 135/233] add logic to use class or display label --- schematic/store/synapse.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/schematic/store/synapse.py b/schematic/store/synapse.py index 99876b62d..f984590ce 100644 --- a/schematic/store/synapse.py +++ b/schematic/store/synapse.py @@ -1540,14 +1540,21 @@ async def format_row_annotations( annos.pop(anno_k) if anno_k in annos.keys() else annos # Otherwise save annotation as approrpriate else: + if annotation_keys == "display_label": + node_validation_rules = dmge.get_node_validation_rules( + node_display_name=anno_k + ) + else: + node_validation_rules = dmge.get_node_validation_rules( + node_label=anno_k + ) + if isinstance(anno_v, float) and np.isnan(anno_v): annos[anno_k] = "" elif ( isinstance(anno_v, str) and re.fullmatch(csv_list_regex, anno_v) - and rule_in_rule_list( - "list", dmge.get_node_validation_rules(anno_k) - ) + and rule_in_rule_list("list", node_validation_rules) ): annos[anno_k] = anno_v.split(",") else: From 5c42e1b23b3805a5b3fa563127d960dec167d223 Mon Sep 17 00:00:00 2001 From: GiaJordan Date: Tue, 27 Aug 2024 14:47:09 -0700 Subject: [PATCH 136/233] mark async tests --- tests/test_store.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/test_store.py b/tests/test_store.py index cefa6d56a..9d71451da 100644 --- a/tests/test_store.py +++ b/tests/test_store.py @@ -517,6 +517,7 @@ def test_add_entity_id_and_filename_without_component_col(self, synapse_store): ) assert_frame_equal(manifest_to_return, expected_df) + @pytest.mark.asyncio @pytest.mark.parametrize( "hideBlanks, annotation_keys", [ @@ -588,6 +589,7 @@ def test_get_files_metadata_from_dataset(self, synapse_store): "entityId": ["syn123", "syn456"], } + @pytest.mark.asyncio async def test_get_async_annotation(self, synapse_store: SynapseStorage) -> None: """test get annotation async function""" mock_syn_id = "syn1234" @@ -607,6 +609,7 @@ async def test_get_async_annotation(self, synapse_store: SynapseStorage) -> None ) assert result == "mock" + @pytest.mark.asyncio async def test_store_async_annotation(self, synapse_store: SynapseStorage) -> None: """test store annotations async function""" annos_dict = { @@ -648,6 +651,7 @@ async def test_store_async_annotation(self, synapse_store: SynapseStorage) -> No assert result == expected_dict assert isinstance(result, Annotations) + @pytest.mark.asyncio async def test_process_store_annos_failure( self, synapse_store: SynapseStorage ) -> None: @@ -665,6 +669,7 @@ async def mock_failure_coro(): with pytest.raises(RuntimeError, match="failed with"): await synapse_store._process_store_annos(tasks) + @pytest.mark.asyncio async def test_process_store_annos_success_store( self, synapse_store: SynapseStorage ) -> None: @@ -695,6 +700,7 @@ async def mock_success_coro(): # make sure that the if statement is working mock_store_async1.assert_not_called() + @pytest.mark.asyncio async def test_process_store_annos_success_get( self, synapse_store: SynapseStorage ) -> None: @@ -740,6 +746,7 @@ async def mock_success_coro(): await synapse_store._process_store_annos(new_tasks) mock_store_async2.assert_called_once() + @pytest.mark.asyncio async def test_process_store_annos_success_get_entity_id_variants( self, synapse_store: SynapseStorage ) -> None: @@ -791,6 +798,7 @@ async def mock_success_coro() -> dict[str, Any]: await synapse_store._process_store_annos(new_tasks) mock_store_async2.assert_called_once() + @pytest.mark.asyncio async def test_process_store_annos_get_annos_empty( self, synapse_store: SynapseStorage ) -> None: @@ -1213,6 +1221,7 @@ def test_entity_type_checking(self, synapse_store, entity_id, caplog): class TestManifestUpload: """Test manifest upload""" + @pytest.mark.asyncio @pytest.mark.parametrize( "original_manifest, files_in_dataset, expected_entity_ids, expected_filenames", [ From 885b0afc1bd485c4dbdcfca90fdb5d4226b2f723 Mon Sep 17 00:00:00 2001 From: GiaJordan Date: Tue, 27 Aug 2024 14:55:59 -0700 Subject: [PATCH 137/233] add pytest.ini file --- pytest.ini | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 pytest.ini diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 000000000..faf59a4ad --- /dev/null +++ b/pytest.ini @@ -0,0 +1,3 @@ +[pytest] +python_files = test_*.py +asyncio_mode = auto \ No newline at end of file From 8b0de403611b52c5ae8904d93d1feb2441635890 Mon Sep 17 00:00:00 2001 From: GiaJordan Date: Tue, 27 Aug 2024 14:56:30 -0700 Subject: [PATCH 138/233] Revert "mark async tests" This reverts commit 5c42e1b23b3805a5b3fa563127d960dec167d223. --- tests/test_store.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/tests/test_store.py b/tests/test_store.py index 9d71451da..cefa6d56a 100644 --- a/tests/test_store.py +++ b/tests/test_store.py @@ -517,7 +517,6 @@ def test_add_entity_id_and_filename_without_component_col(self, synapse_store): ) assert_frame_equal(manifest_to_return, expected_df) - @pytest.mark.asyncio @pytest.mark.parametrize( "hideBlanks, annotation_keys", [ @@ -589,7 +588,6 @@ def test_get_files_metadata_from_dataset(self, synapse_store): "entityId": ["syn123", "syn456"], } - @pytest.mark.asyncio async def test_get_async_annotation(self, synapse_store: SynapseStorage) -> None: """test get annotation async function""" mock_syn_id = "syn1234" @@ -609,7 +607,6 @@ async def test_get_async_annotation(self, synapse_store: SynapseStorage) -> None ) assert result == "mock" - @pytest.mark.asyncio async def test_store_async_annotation(self, synapse_store: SynapseStorage) -> None: """test store annotations async function""" annos_dict = { @@ -651,7 +648,6 @@ async def test_store_async_annotation(self, synapse_store: SynapseStorage) -> No assert result == expected_dict assert isinstance(result, Annotations) - @pytest.mark.asyncio async def test_process_store_annos_failure( self, synapse_store: SynapseStorage ) -> None: @@ -669,7 +665,6 @@ async def mock_failure_coro(): with pytest.raises(RuntimeError, match="failed with"): await synapse_store._process_store_annos(tasks) - @pytest.mark.asyncio async def test_process_store_annos_success_store( self, synapse_store: SynapseStorage ) -> None: @@ -700,7 +695,6 @@ async def mock_success_coro(): # make sure that the if statement is working mock_store_async1.assert_not_called() - @pytest.mark.asyncio async def test_process_store_annos_success_get( self, synapse_store: SynapseStorage ) -> None: @@ -746,7 +740,6 @@ async def mock_success_coro(): await synapse_store._process_store_annos(new_tasks) mock_store_async2.assert_called_once() - @pytest.mark.asyncio async def test_process_store_annos_success_get_entity_id_variants( self, synapse_store: SynapseStorage ) -> None: @@ -798,7 +791,6 @@ async def mock_success_coro() -> dict[str, Any]: await synapse_store._process_store_annos(new_tasks) mock_store_async2.assert_called_once() - @pytest.mark.asyncio async def test_process_store_annos_get_annos_empty( self, synapse_store: SynapseStorage ) -> None: @@ -1221,7 +1213,6 @@ def test_entity_type_checking(self, synapse_store, entity_id, caplog): class TestManifestUpload: """Test manifest upload""" - @pytest.mark.asyncio @pytest.mark.parametrize( "original_manifest, files_in_dataset, expected_entity_ids, expected_filenames", [ From 701c9e5eec148aa1aec24059d188363846c7e65b Mon Sep 17 00:00:00 2001 From: linglp Date: Wed, 28 Aug 2024 03:38:35 -0400 Subject: [PATCH 139/233] get_node_validation_rules better error handling and add a unit test --- schematic/schemas/data_model_graph.py | 24 +++++----- tests/test_schemas.py | 64 ++++++++++++++++++++++++++- 2 files changed, 76 insertions(+), 12 deletions(-) diff --git a/schematic/schemas/data_model_graph.py b/schematic/schemas/data_model_graph.py index 4aae01a22..511f5309e 100644 --- a/schematic/schemas/data_model_graph.py +++ b/schematic/schemas/data_model_graph.py @@ -1,25 +1,24 @@ """DataModel Graph""" import logging -from typing import Optional, Union, Any +from typing import Any, Optional, Union -import networkx as nx # type: ignore import graphviz # type: ignore +import networkx as nx # type: ignore from opentelemetry import trace from schematic.schemas.data_model_edges import DataModelEdges from schematic.schemas.data_model_nodes import DataModelNodes from schematic.schemas.data_model_relationships import DataModelRelationships - +from schematic.utils.general import unlist from schematic.utils.schema_utils import ( - get_property_label_from_display_name, - get_class_label_from_display_name, DisplayLabelType, extract_component_validation_rules, + get_class_label_from_display_name, + get_property_label_from_display_name, ) -from schematic.utils.general import unlist -from schematic.utils.viz_utils import visualize from schematic.utils.validate_utils import rule_in_rule_list +from schematic.utils.viz_utils import visualize logger = logging.getLogger(__name__) @@ -780,13 +779,18 @@ def get_node_validation_rules( node_label: Label of the node for which you need to look up. node_display_name: Display name of the node which you want to get the label for. Returns: - A set of validation rules associated with node, as a list. + A set of validation rules associated with node, as a list or a dictionary. """ if not node_label: - assert node_display_name is not None + if node_display_name is None: + raise ValueError( + "Either node_label or node_display_name must be provided." + ) + + # try search node label using display name node_label = self.get_node_label(node_display_name) - if not node_label: + if not node_label or node_label not in self.graph.nodes: return [] node_validation_rules = self.graph.nodes[node_label]["validationRules"] diff --git a/tests/test_schemas.py b/tests/test_schemas.py index 6870cbef5..0c9dd5723 100644 --- a/tests/test_schemas.py +++ b/tests/test_schemas.py @@ -616,8 +616,68 @@ def test_get_node_range(self): def test_get_node_required(self): return - def test_get_node_validation_rules(self): - return + @pytest.mark.parametrize( + "data_model", list(DATA_MODEL_DICT.keys()), ids=list(DATA_MODEL_DICT.values()) + ) + @pytest.mark.parametrize( + "node_label, node_display_name, expected_validation_rule", + [ + # Test case 1: node label is provided + ( + "PatientID", + None, + {"Biospecimen": "unique error", "Patient": "unique warning"}, + ), + ( + "CheckRegexListStrict", + None, + ["list strict", "regex match [a-f]"], + ), + # test case 2: node label is not valid and display name is not provided + ( + "invalid node label", + None, + [], + ), + # Test case 3: node label and node display name are not provided + ( + None, + None, + None, + ), + # Test case 4: node label and display label is not in graph + ( + None, + "invalid display label", + [], + ), + # Test case 5: node label is not provided but a valid display label is provided + ( + None, + "Patient ID", + {"Biospecimen": "unique error", "Patient": "unique warning"}, + ), + ], + ) + def test_get_node_validation_rules( + self, + helpers, + data_model, + node_label, + node_display_name, + expected_validation_rule, + ): + DMGE = helpers.get_data_model_graph_explorer(path=data_model) + if not node_label and not node_display_name: + with pytest.raises(ValueError): + node_validation_rules = DMGE.get_node_validation_rules( + node_label=node_label, node_display_name=node_display_name + ) + else: + node_validation_rules = DMGE.get_node_validation_rules( + node_label=node_label, node_display_name=node_display_name + ) + assert node_validation_rules == expected_validation_rule def test_get_subgraph_by_edge_type(self): return From b91caf189f095ab72eebe0101234ff1e220fc163 Mon Sep 17 00:00:00 2001 From: linglp Date: Wed, 28 Aug 2024 03:54:50 -0400 Subject: [PATCH 140/233] update get_node_validation_rules and test --- schematic/schemas/data_model_graph.py | 9 ++++- tests/test_schemas.py | 56 +++++++++++++++++---------- 2 files changed, 42 insertions(+), 23 deletions(-) diff --git a/schematic/schemas/data_model_graph.py b/schematic/schemas/data_model_graph.py index 511f5309e..61eef7b4a 100644 --- a/schematic/schemas/data_model_graph.py +++ b/schematic/schemas/data_model_graph.py @@ -790,10 +790,15 @@ def get_node_validation_rules( # try search node label using display name node_label = self.get_node_label(node_display_name) - if not node_label or node_label not in self.graph.nodes: + if not node_label: return [] - node_validation_rules = self.graph.nodes[node_label]["validationRules"] + try: + node_validation_rules = self.graph.nodes[node_label]["validationRules"] + except: + raise ValueError( + f"{node_label} is not in the graph, please check that you are providing the proper node label" + ) return node_validation_rules diff --git a/tests/test_schemas.py b/tests/test_schemas.py index 0c9dd5723..c72f2aa28 100644 --- a/tests/test_schemas.py +++ b/tests/test_schemas.py @@ -633,51 +633,65 @@ def test_get_node_required(self): None, ["list strict", "regex match [a-f]"], ), - # test case 2: node label is not valid and display name is not provided + # Test case 2: node label is not provided and display label is not part of the schema ( - "invalid node label", None, + "invalid display label", [], ), - # Test case 3: node label and node display name are not provided + # Test case 3: node label is not provided but a valid display label is provided ( None, - None, - None, + "Patient ID", + {"Biospecimen": "unique error", "Patient": "unique warning"}, ), - # Test case 4: node label and display label is not in graph + ], + ) + def test_get_node_validation_rules_valid( + self, + helpers, + data_model, + node_label, + node_display_name, + expected_validation_rule, + ): + DMGE = helpers.get_data_model_graph_explorer(path=data_model) + + node_validation_rules = DMGE.get_node_validation_rules( + node_label=node_label, node_display_name=node_display_name + ) + assert node_validation_rules == expected_validation_rule + + @pytest.mark.parametrize( + "data_model", list(DATA_MODEL_DICT.keys()), ids=list(DATA_MODEL_DICT.values()) + ) + @pytest.mark.parametrize( + "node_label, node_display_name", + [ + # Test case 1: node label and node display name are not provided ( None, - "invalid display label", - [], + None, ), - # Test case 5: node label is not provided but a valid display label is provided + # Test case 2: node label is not valid and display name is not provided ( + "invalid node", None, - "Patient ID", - {"Biospecimen": "unique error", "Patient": "unique warning"}, ), ], ) - def test_get_node_validation_rules( + def test_get_node_validation_rules_invalid( self, helpers, data_model, node_label, node_display_name, - expected_validation_rule, ): DMGE = helpers.get_data_model_graph_explorer(path=data_model) - if not node_label and not node_display_name: - with pytest.raises(ValueError): - node_validation_rules = DMGE.get_node_validation_rules( - node_label=node_label, node_display_name=node_display_name - ) - else: - node_validation_rules = DMGE.get_node_validation_rules( + with pytest.raises(ValueError): + DMGE.get_node_validation_rules( node_label=node_label, node_display_name=node_display_name ) - assert node_validation_rules == expected_validation_rule def test_get_subgraph_by_edge_type(self): return From def3bde331bdc5a74b933ad30ee90e8866f85758 Mon Sep 17 00:00:00 2001 From: BryanFauble <17128019+BryanFauble@users.noreply.github.com> Date: Wed, 28 Aug 2024 09:13:20 -0700 Subject: [PATCH 141/233] [FDS-2294 ]Run schematic_api tests, skip rule benchmark, add more tests, and set up sonarcloud code coverage (#1476) * Run schematic_api tests, skip rule benchmark, add more tests, and set up sonarcloud code coverage --- .github/workflows/test.yml | 48 ++++++++++++- schematic/visualization/tangled_tree.py | 21 +++--- sonar-project.properties | 5 ++ tests/test_api.py | 96 +++++++++++++++++++++++-- tests/test_viz.py | 31 +++++++- 5 files changed, 184 insertions(+), 17 deletions(-) create mode 100644 sonar-project.properties diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e12303a7e..2bea3258b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -121,8 +121,9 @@ jobs: SERVICE_ACCOUNT_CREDS: ${{ secrets.SERVICE_ACCOUNT_CREDS }} run: > source .venv/bin/activate; - pytest --durations=0 --cov-report=term --cov-report=html:htmlcov --cov=schematic/ - -m "not (schematic_api or table_operations)" --reruns 2 -n auto + pytest --durations=0 --cov-report=term --cov-report=html:htmlcov --cov-report=xml:coverage.xml --cov=schematic/ + -m "not (rule_benchmark or table_operations)" --reruns 2 -n auto + - name: Upload pytest test results uses: actions/upload-artifact@v2 @@ -131,3 +132,46 @@ jobs: path: htmlcov # Use always() to always run this step to publish test results when there are test failures if: ${{ always() }} + - name: Upload XML coverage report + id: upload_coverage_report + uses: actions/upload-artifact@v4 + # Only upload a single python version to pass along to sonarcloud + if: ${{ contains(fromJSON('["3.10"]'), matrix.python-version) && always() }} + with: + name: coverage-report + path: coverage.xml + + sonarcloud: + needs: [test] + if: ${{ always() && !cancelled()}} + name: SonarCloud + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 # Shallow clones should be disabled for a better relevancy of analysis + - name: Check coverage-report artifact existence + id: check_coverage_report + uses: LIT-Protocol/artifact-exists-action@v0 + with: + name: "coverage-report" + - name: Download coverage report + uses: actions/download-artifact@v4 + if: steps.check_coverage_report.outputs.exists == 'true' + with: + name: coverage-report + - name: Check coverage.xml file existence + id: check_coverage_xml + uses: andstor/file-existence-action@v3 + with: + files: "coverage.xml" + # This is a workaround described in https://community.sonarsource.com/t/sonar-on-github-actions-with-python-coverage-source-issue/36057 + - name: Override Coverage Source Path for Sonar + if: steps.check_coverage_xml.outputs.files_exists == 'true' + run: sed -i "s/\/home\/runner\/work\/schematic\/schematic\/schematic<\/source>/\/github\/workspace\/schematic<\/source>/g" coverage.xml + - name: SonarCloud Scan + uses: SonarSource/sonarcloud-github-action@master + if: ${{ always() }} + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} diff --git a/schematic/visualization/tangled_tree.py b/schematic/visualization/tangled_tree.py index 59ba62139..95eac7e75 100644 --- a/schematic/visualization/tangled_tree.py +++ b/schematic/visualization/tangled_tree.py @@ -3,24 +3,24 @@ # pylint: disable=logging-fstring-interpolation # pylint: disable=too-many-lines -from io import StringIO +import ast import json import logging import os +from io import StringIO from os import path -from typing import Optional, Literal, TypedDict, Union -from typing_extensions import assert_never +from typing import Literal, Optional, TypedDict, Union import networkx as nx # type: ignore -from networkx.classes.reportviews import NodeView, EdgeDataView # type: ignore import pandas as pd +from networkx.classes.reportviews import EdgeDataView, NodeView # type: ignore +from typing_extensions import assert_never -from schematic.visualization.attributes_explorer import AttributesExplorer -from schematic.schemas.data_model_parser import DataModelParser from schematic.schemas.data_model_graph import DataModelGraph, DataModelGraphExplorer +from schematic.schemas.data_model_parser import DataModelParser from schematic.utils.io_utils import load_json from schematic.utils.schema_utils import DisplayLabelType - +from schematic.visualization.attributes_explorer import AttributesExplorer logger = logging.getLogger(__name__) @@ -258,11 +258,16 @@ def _get_ca_alias(self, conditional_requirements: list[str]) -> dict[str, str]: value: attribute """ ca_alias: dict[str, str] = {} + extracted_conditional_requirements = [] + for conditional_requirement in conditional_requirements: + extracted_conditional_requirements.extend( + ast.literal_eval(node_or_string=conditional_requirement) + ) # clean up conditional requirements conditional_requirements = [ self._remove_unwanted_characters_from_conditional_statement(req) - for req in conditional_requirements + for req in extracted_conditional_requirements ] for req in conditional_requirements: diff --git a/sonar-project.properties b/sonar-project.properties new file mode 100644 index 000000000..6aeb19c1c --- /dev/null +++ b/sonar-project.properties @@ -0,0 +1,5 @@ +sonar.projectKey=Sage-Bionetworks_schematic +sonar.organization=sage-bionetworks +sonar.python.coverage.reportPaths=coverage.xml +sonar.sources=schematic +sonar.tests=tests diff --git a/tests/test_api.py b/tests/test_api.py index d20a77074..6ce515622 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -773,9 +773,9 @@ def test_generate_manifest_file_based_annotations( # make sure Filename, entityId, and component get filled with correct value assert google_sheet_df["Filename"].to_list() == [ - "TestDataset-Annotations-v3/Sample_A.txt", - "TestDataset-Annotations-v3/Sample_B.txt", - "TestDataset-Annotations-v3/Sample_C.txt", + "schematic - main/TestDataset-Annotations-v3/Sample_A.txt", + "schematic - main/TestDataset-Annotations-v3/Sample_B.txt", + "schematic - main/TestDataset-Annotations-v3/Sample_C.txt", ] assert google_sheet_df["entityId"].to_list() == [ "syn25614636", @@ -1276,6 +1276,94 @@ def test_visualize_tangled_tree_layers( assert response.status_code == 200 + response_data = json.loads(response.data) + + if figure_type == "component": + assert len(response_data) == 3 + expected_data = [ + { + "id": "Patient", + "parents": [], + "direct_children": ["Biospecimen"], + "children": ["Biospecimen", "BulkRNA-seqAssay"], + }, + { + "id": "Biospecimen", + "parents": ["Patient"], + "direct_children": ["BulkRNA-seqAssay"], + "children": ["BulkRNA-seqAssay"], + }, + { + "id": "BulkRNA-seqAssay", + "parents": ["Biospecimen"], + "direct_children": [], + "children": [], + }, + ] + for data_list in response_data: + for data_point in data_list: + assert any( + data_point["id"] == expected["id"] + and data_point["parents"] == expected["parents"] + and data_point["direct_children"] == expected["direct_children"] + and set(data_point["children"]) == set(expected["children"]) + for expected in expected_data + ) + elif figure_type == "dependency": + assert len(response_data) == 3 + expected_data = [ + { + "id": "BulkRNA-seqAssay", + "parents": [], + "direct_children": ["SampleID", "Filename", "FileFormat"], + "children": [], + }, + { + "id": "SampleID", + "parents": ["BulkRNA-seqAssay"], + "direct_children": [], + "children": [], + }, + { + "id": "FileFormat", + "parents": ["BulkRNA-seqAssay"], + "direct_children": [ + "GenomeBuild", + "GenomeBuild", + "GenomeBuild", + "GenomeFASTA", + ], + "children": [], + }, + { + "id": "Filename", + "parents": ["BulkRNA-seqAssay"], + "direct_children": [], + "children": [], + }, + { + "id": "GenomeBuild", + "parents": ["FileFormat", "FileFormat", "FileFormat"], + "direct_children": [], + "children": [], + }, + { + "id": "GenomeFASTA", + "parents": ["FileFormat"], + "direct_children": [], + "children": [], + }, + ] + for data_list in response_data: + for data_point in data_list: + assert any( + data_point["id"] == expected["id"] + and data_point["parents"] == expected["parents"] + and data_point["direct_children"] == expected["direct_children"] + and set(data_point["children"]) == set(expected["children"]) + for expected in expected_data + ) + @pytest.mark.parametrize( "component, response_text", [ @@ -1379,7 +1467,7 @@ def test_validation_performance( # Log and check time and ensure successful response logger.warning( - f"validation endpiont response time {round(response_time,2)} seconds." + f"validation endpoint response time {round(response_time,2)} seconds." ) assert response.status_code == 200 assert response_time < 5.00 diff --git a/tests/test_viz.py b/tests/test_viz.py index f40aca332..b94d79688 100644 --- a/tests/test_viz.py +++ b/tests/test_viz.py @@ -1,9 +1,9 @@ -from io import StringIO import json -import os -import pandas as pd import logging +import os +from io import StringIO +import pandas as pd import pytest from schematic.visualization.attributes_explorer import AttributesExplorer @@ -124,6 +124,31 @@ def test_text(self, helpers, tangled_tree): assert actual_patient_text == expected_patient_text assert actual_Biospecimen_text == expected_Biospecimen_text + @pytest.mark.parametrize( + "conditional_requirements, expected", + [ + # Test case 1: Multiple file formats + ( + [ + "['File Format is \"BAM\"', 'File Format is \"CRAM\"', 'File Format is \"CSV/TSV\"']" + ], + {"BAM": "FileFormat", "CRAM": "FileFormat", "CSV/TSV": "FileFormat"}, + ), + # Test case 2: Single file format + (["['File Format is \"CRAM\"']"], {"CRAM": "FileFormat"}), + # Test case 3: with "OR" keyword + ( + ['[\'File Format is "BAM" OR "CRAM" OR "CSV/TSV"\']'], + {"BAM": "File Format", "CRAM": "File Format", "CSV/TSV": "File Format"}, + ), + ], + ) + def test_get_ca_alias( + self, helpers, tangled_tree, conditional_requirements, expected + ): + ca_alias = tangled_tree._get_ca_alias(conditional_requirements) + assert ca_alias == expected + def test_layers(self, helpers, tangled_tree): layers_str = tangled_tree.get_tangled_tree_layers(save_file=False)[0] From dfc0e13b4d6f42070dfca0148d1e20f5a398a6f8 Mon Sep 17 00:00:00 2001 From: GiaJordan Date: Wed, 28 Aug 2024 10:08:22 -0700 Subject: [PATCH 142/233] update 'does_not_raise' --- tests/integration/test_metadata_model.py | 8 +------- tests/test_store.py | 7 +------ 2 files changed, 2 insertions(+), 13 deletions(-) diff --git a/tests/integration/test_metadata_model.py b/tests/integration/test_metadata_model.py index 1ed8899f5..bd7e55242 100644 --- a/tests/integration/test_metadata_model.py +++ b/tests/integration/test_metadata_model.py @@ -1,19 +1,13 @@ import logging -from contextlib import contextmanager +from contextlib import nullcontext as does_not_raise from unittest.mock import patch -from schematic.models.metadata import MetadataModel from tests.conftest import metadata_model logging.basicConfig(level=logging.DEBUG) logger = logging.getLogger(__name__) -@contextmanager -def does_not_raise(): - yield - - class TestMetadataModel: def test_submit_filebased_manifest(self, helpers): meta_data_model = metadata_model(helpers, "class_label") diff --git a/tests/test_store.py b/tests/test_store.py index d2bf6d57a..94e897dc0 100644 --- a/tests/test_store.py +++ b/tests/test_store.py @@ -7,7 +7,7 @@ import math import os import shutil -from contextlib import contextmanager +from contextlib import nullcontext as does_not_raise from time import sleep from typing import Any, Generator from unittest.mock import AsyncMock, patch @@ -32,11 +32,6 @@ logger = logging.getLogger(__name__) -@contextmanager -def does_not_raise(): - yield - - @pytest.fixture def test_download_manifest_id(): yield "syn51203973" From 250d71de63cc7840886563697cdc08856958dd45 Mon Sep 17 00:00:00 2001 From: GiaJordan Date: Wed, 28 Aug 2024 10:09:25 -0700 Subject: [PATCH 143/233] update imports --- tests/integration/test_metadata_model.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/integration/test_metadata_model.py b/tests/integration/test_metadata_model.py index bd7e55242..7276812e2 100644 --- a/tests/integration/test_metadata_model.py +++ b/tests/integration/test_metadata_model.py @@ -1,6 +1,5 @@ import logging from contextlib import nullcontext as does_not_raise -from unittest.mock import patch from tests.conftest import metadata_model From 2b33dead710868750e2175632747fb6f324f50a4 Mon Sep 17 00:00:00 2001 From: Lingling <55448354+linglp@users.noreply.github.com> Date: Wed, 28 Aug 2024 16:05:42 -0400 Subject: [PATCH 144/233] Delete tests/data/test-model.csv --- tests/data/test-model.csv | 57 --------------------------------------- 1 file changed, 57 deletions(-) delete mode 100644 tests/data/test-model.csv diff --git a/tests/data/test-model.csv b/tests/data/test-model.csv deleted file mode 100644 index 2e4db1e3a..000000000 --- a/tests/data/test-model.csv +++ /dev/null @@ -1,57 +0,0 @@ -Attribute,Description,Valid Values,DependsOn,Properties,Required,Parent,DependsOn Component,Source,Validation Rules -Component,,,,,TRUE,,,, -Patient,,,"Patient ID, Sex, Year of Birth, Diagnosis, Component",,FALSE,DataType,,, -Patient ID,,,,,TRUE,DataProperty,,,#Patient unique warning^^#Biospecimen unique error -Sex,,"Female, Male, Other",,,TRUE,DataProperty,,, -Year of Birth,,,,,FALSE,DataProperty,,, -Diagnosis,,"Healthy, Cancer",,,TRUE,DataProperty,,, -Cancer,,,"Cancer Type, Family History",,FALSE,ValidValue,,, -Cancer Type,,"Breast, Colorectal, Lung, Prostate, Skin",,,TRUE,DataProperty,,, -Family History,,"Breast, Colorectal, Lung, Prostate, Skin",,,TRUE,DataProperty,,,list strict -Biospecimen,,,"Sample ID, Patient ID, Tissue Status, Component",,FALSE,DataType,Patient,, -Sample ID,,,,,TRUE,DataProperty,,, -Tissue Status,,"Healthy, Malignant",,,TRUE,DataProperty,,, -Bulk RNA-seq Assay,,,"Filename, Sample ID, File Format, Component",,FALSE,DataType,Biospecimen,, -Filename,,,,,TRUE,DataProperty,,,#MockFilename filenameExists syn61682648^^ -File Format,,"FASTQ, BAM, CRAM, CSV/TSV",,,TRUE,DataProperty,,, -BAM,,,Genome Build,,FALSE,ValidValue,,, -CRAM,,,"Genome Build, Genome FASTA",,FALSE,ValidValue,,, -CSV/TSV,,,Genome Build,,FALSE,ValidValue,,, -Genome Build,,"GRCh37, GRCh38, GRCm38, GRCm39",,,TRUE,DataProperty,,, -Genome FASTA,,,,,TRUE,DataProperty,,, -MockComponent,,,"Component, Check List, Check List Enum, Check List Like, Check List Like Enum, Check List Strict, Check List Enum Strict, Check Regex List, Check Regex List Like, Check Regex List Strict, Check Regex Single, Check Regex Format, Check Regex Integer, Check Num, Check Float, Check Int, Check String, Check URL,Check Match at Least, Check Match at Least values, Check Match Exactly, Check Match Exactly values, Check Match None, Check Match None values, Check Recommended, Check Ages, Check Unique, Check Range, Check Date, Check NA",,FALSE,DataType,,, -Check List,,,,,TRUE,DataProperty,,,list -Check List Enum,,"ab, cd, ef, gh",,,TRUE,DataProperty,,,list -Check List Like,,,,,TRUE,DataProperty,,,list like -Check List Like Enum,,"ab, cd, ef, gh",,,TRUE,DataProperty,,,list like -Check List Strict,,,,,TRUE,DataProperty,,,list strict -Check List Enum Strict,,"ab, cd, ef, gh",,,TRUE,DataProperty,,,list strict -Check Regex List,,,,,TRUE,DataProperty,,,list::regex match [a-f] -Check Regex List Strict,,,,,TRUE,DataProperty,,,list strict::regex match [a-f] -Check Regex List Like,,,,,TRUE,DataProperty,,,list like::regex match [a-f] -Check Regex Single,,,,,TRUE,DataProperty,,,regex search [a-f] -Check Regex Format,,,,,TRUE,DataProperty,,,regex match [a-f] -Check Regex Integer,,,,,TRUE,DataProperty,,,regex search ^\d+$ -Check Num,,,,,TRUE,DataProperty,,,num -Check Float,,,,,TRUE,DataProperty,,,float -Check Int,,,,,TRUE,DataProperty,,,int -Check String,,,,,TRUE,DataProperty,,,str -Check URL,,,,,TRUE,DataProperty,,,url -Check Match at Least,,,,,TRUE,DataProperty,,,matchAtLeastOne Patient.PatientID set -Check Match Exactly,,,,,TRUE,DataProperty,,,matchExactlyOne MockComponent.checkMatchExactly set -Check Match None,,,,,TRUE,DataProperty,,,matchNone MockComponent.checkMatchNone set error -Check Match at Least values,,,,,TRUE,DataProperty,,,matchAtLeastOne MockComponent.checkMatchatLeastvalues value -Check Match Exactly values,,,,,TRUE,DataProperty,,,matchExactlyOne MockComponent.checkMatchExactlyvalues value -Check Match None values,,,,,TRUE,DataProperty,,,matchNone MockComponent.checkMatchNonevalues value error -Check Recommended,,,,,FALSE,DataProperty,,,recommended -Check Ages,,,,,TRUE,DataProperty,,,protectAges -MockCombinedRule,,,,,TRUE,DataProperty,,,#AnotherMockComponent unique error::regex match ^[a-zA-Z0-9_-]+$ error::required -Check Unique,,,,,TRUE,DataProperty,,,unique error -Check Range,,,,,TRUE,DataProperty,,,inRange 50 100 error -Check Date,,,,,TRUE,DataProperty,,,date -Check NA,,,,,TRUE,DataProperty,,,int::IsNA -MockRDB,,,"Component, MockRDB_id, SourceManifest",,FALSE,DataType,,, -MockRDB_id,,,,,TRUE,DataProperty,,,int -SourceManifest,,,,,TRUE,DataProperty,,, -MockFilename,,,"Component, Filename",,FALSE,DataType,,, -AnotherMockComponent,,,"Component, MockCombinedRule",,FALSE,DataType,,, \ No newline at end of file From b1c6bcae873b39c4d9f6c62c71aa63dfcfcffb10 Mon Sep 17 00:00:00 2001 From: lakikowolfe Date: Wed, 28 Aug 2024 16:50:42 -0700 Subject: [PATCH 145/233] add csv data model to manifest tests --- tests/test_manifest.py | 96 ++++++++++++++++++------------------------ 1 file changed, 42 insertions(+), 54 deletions(-) diff --git a/tests/test_manifest.py b/tests/test_manifest.py index c34148dab..704b1ae7e 100644 --- a/tests/test_manifest.py +++ b/tests/test_manifest.py @@ -38,7 +38,12 @@ def generate_graph_data_model(helpers, path_to_data_model, data_model_labels): return graph_data_model - +@pytest.fixture(params=["example.model.jsonld", + "example.model.csv"]) +def data_model_path(request, helpers): + data_model_path = helpers.get_data_path(request.param) + yield data_model_path + @pytest.fixture( params=[ (True, "Patient"), @@ -53,21 +58,19 @@ def generate_graph_data_model(helpers, path_to_data_model, data_model_labels): "skip_annotations-BulkRNAseqAssay", ], ) -def manifest_generator(helpers, request): +def manifest_generator(helpers, request, data_model_path): # Rename request param for readability use_annotations, data_type = request.param - path_to_data_model = helpers.get_data_path("example.model.jsonld") - # Get graph data model graph_data_model = generate_graph_data_model( helpers, - path_to_data_model=path_to_data_model, + path_to_data_model=data_model_path, data_model_labels="class_label", ) manifest_generator = ManifestGenerator( - path_to_data_model=path_to_data_model, + path_to_data_model=data_model_path, graph=graph_data_model, root=data_type, use_annotations=use_annotations, @@ -121,20 +124,19 @@ def app(): class TestManifestGenerator: - def test_init(self, helpers): - path_to_data_model = helpers.get_data_path("example.model.jsonld") + def test_init(self, helpers, data_model_path): # Get graph data model graph_data_model = generate_graph_data_model( helpers, - path_to_data_model=path_to_data_model, + path_to_data_model=data_model_path, data_model_labels="class_label", ) generator = ManifestGenerator( graph=graph_data_model, title="mock_title", - path_to_data_model=path_to_data_model, + path_to_data_model=data_model_path, root="Patient", ) @@ -155,23 +157,22 @@ def test_init(self, helpers): ], ids=["DataType not found in Schema", "No DataType provided"], ) - def test_missing_root_error(self, helpers, data_type, exc, exc_message): + def test_missing_root_error(self, helpers, data_type, exc, exc_message, data_model_path): """ Test for errors when either no DataType is provided or when a DataType is provided but not found in the schema """ - path_to_data_model = helpers.get_data_path("example.model.jsonld") # Get graph data model graph_data_model = generate_graph_data_model( helpers, - path_to_data_model=path_to_data_model, + path_to_data_model=data_model_path, data_model_labels="class_label", ) # A LookupError should be raised and include message when the component cannot be found with pytest.raises(exc) as e: generator = ManifestGenerator( - path_to_data_model=helpers.get_data_path("example.model.jsonld"), + path_to_data_model=data_model_path, graph=graph_data_model, root=data_type, use_annotations=False, @@ -236,7 +237,7 @@ def test_get_manifest_first_time(self, manifest): @pytest.mark.parametrize("sheet_url", [None, True, False]) @pytest.mark.parametrize("dataset_id", [None, "syn27600056"]) @pytest.mark.google_credentials_needed - def test_get_manifest_excel(self, helpers, sheet_url, output_format, dataset_id): + def test_get_manifest_excel(self, helpers, sheet_url, output_format, dataset_id, data_model_path): """ Purpose: the goal of this test is to make sure that output_format parameter and sheet_url parameter could function well; In addition, this test also makes sure that getting a manifest with an existing dataset_id is working @@ -246,17 +247,16 @@ def test_get_manifest_excel(self, helpers, sheet_url, output_format, dataset_id) data_type = "Patient" # Get path to data model - path_to_data_model = helpers.get_data_path("example.model.jsonld") # Get graph data model graph_data_model = generate_graph_data_model( helpers, - path_to_data_model=path_to_data_model, + path_to_data_model=data_model_path, data_model_labels="class_label", ) generator = ManifestGenerator( - path_to_data_model=path_to_data_model, + path_to_data_model=data_model_path, graph=graph_data_model, root=data_type, use_annotations=False, @@ -296,7 +296,7 @@ def test_get_manifest_excel(self, helpers, sheet_url, output_format, dataset_id) [("syn27600056"), ("syn52397659")], ids=["Annotations present", "Annotations not present"], ) - def test_get_manifest_no_annos(self, helpers, dataset_id): + def test_get_manifest_no_annos(self, helpers, dataset_id, data_model_path): """ Test to cover manifest generation under the case where use_annotations is True but there are no annotations in the dataset @@ -305,19 +305,16 @@ def test_get_manifest_no_annos(self, helpers, dataset_id): # Use a non-file based DataType data_type = "Patient" - # Get path to data model - path_to_data_model = helpers.get_data_path("example.model.jsonld") - # Get graph data model graph_data_model = generate_graph_data_model( helpers, - path_to_data_model=path_to_data_model, + path_to_data_model=data_model_path, data_model_labels="class_label", ) # Instantiate object with use_annotations set to True generator = ManifestGenerator( - path_to_data_model=path_to_data_model, + path_to_data_model=data_model_path, graph=graph_data_model, root=data_type, use_annotations=True, @@ -418,23 +415,21 @@ def test_gather_all_fields(self, simple_manifest_generator): {"Filename": [], "Component": []}, {"Component": ["BulkRNA-seqAssay"]}, ), - ], + ], ) def test_add_root_to_component_without_additional_metadata( - self, helpers, data_type, required_metadata_fields, expected + self, helpers, data_type, required_metadata_fields, expected, data_model_path ): - # Get path to data model - path_to_data_model = helpers.get_data_path("example.model.jsonld") # Get graph data model graph_data_model = generate_graph_data_model( helpers, - path_to_data_model=path_to_data_model, + path_to_data_model=data_model_path, data_model_labels="class_label", ) manifest_generator = ManifestGenerator( - path_to_data_model=path_to_data_model, + path_to_data_model=data_model_path, graph=graph_data_model, root=data_type, ) @@ -460,20 +455,18 @@ def test_add_root_to_component_without_additional_metadata( ], ) def test_add_root_to_component_with_additional_metadata( - self, helpers, additional_metadata + self, helpers, additional_metadata, data_model_path ): - # Get path to data model - path_to_data_model = helpers.get_data_path("example.model.jsonld") # Get graph data model graph_data_model = generate_graph_data_model( helpers, - path_to_data_model=path_to_data_model, + path_to_data_model=data_model_path, data_model_labels="class_label", ) manifest_generator = ManifestGenerator( - path_to_data_model=path_to_data_model, + path_to_data_model=data_model_path, graph=graph_data_model, root="BulkRNA-seqAssay", ) @@ -538,7 +531,7 @@ def test_get_missing_columns( ], ) @pytest.mark.google_credentials_needed - def test_update_dataframe_with_existing_df(self, helpers, existing_manifest): + def test_update_dataframe_with_existing_df(self, helpers, existing_manifest, data_model_path): """ Tests the following discrepancies with an existing schema: - schema has matching columns to existing_df @@ -549,18 +542,16 @@ def test_update_dataframe_with_existing_df(self, helpers, existing_manifest): data_type = "Patient" sheet_url = True - path_to_data_model = helpers.get_data_path("example.model.jsonld") - # Get graph data model graph_data_model = generate_graph_data_model( helpers, - path_to_data_model=path_to_data_model, + path_to_data_model=data_model_path, data_model_labels="class_label", ) # Instantiate the Manifest Generator. generator = ManifestGenerator( - path_to_data_model=path_to_data_model, + path_to_data_model=data_model_path, graph=graph_data_model, root=data_type, use_annotations=False, @@ -684,23 +675,22 @@ def test_populate_existing_excel_spreadsheet( "return_output", ["Mock excel file path", "Mock google sheet link"] ) def test_create_single_manifest( - self, simple_manifest_generator, helpers, return_output + self, simple_manifest_generator, helpers, return_output, data_model_path ): with patch( "schematic.manifest.generator.ManifestGenerator.get_manifest", return_value=return_output, ): - json_ld_path = helpers.get_data_path("example.model.jsonld") data_type = "Patient" graph_data_model = generate_graph_data_model( helpers, - path_to_data_model=json_ld_path, + path_to_data_model=data_model_path, data_model_labels="class_label", ) result = simple_manifest_generator.create_single_manifest( - path_to_data_model=json_ld_path, + path_to_data_model=data_model_path, graph_data_model=graph_data_model, data_type=data_type, output_format="google_sheet", @@ -712,15 +702,14 @@ def test_create_single_manifest( "test_data_types", [["Patient", "Biospecimen"], ["all manifests"]] ) def test_create_manifests_raise_errors( - self, simple_manifest_generator, helpers, test_data_types + self, simple_manifest_generator, helpers, test_data_types, data_model_path ): with pytest.raises(ValueError) as exception_info: - json_ld_path = helpers.get_data_path("example.model.jsonld") data_types = test_data_types dataset_ids = ["syn123456"] simple_manifest_generator.create_manifests( - path_to_data_model=json_ld_path, + path_to_data_model=data_model_path, data_types=data_types, dataset_ids=dataset_ids, output_format="google_sheet", @@ -746,14 +735,15 @@ def test_create_manifests( test_data_types, dataset_ids, expected_result, + data_model_path ): with patch( "schematic.manifest.generator.ManifestGenerator.create_single_manifest", return_value="mock google sheet link", ): - json_ld_path = helpers.get_data_path("example.model.jsonld") + all_results = simple_manifest_generator.create_manifests( - path_to_data_model=json_ld_path, + path_to_data_model=data_model_path, data_types=test_data_types, dataset_ids=dataset_ids, output_format="google_sheet", @@ -767,20 +757,18 @@ def test_create_manifests( [("Biospecimen", "syn61260107"), ("BulkRNA-seqAssay", "syn61374924")], ids=["Record based", "File based"], ) - def test_get_manifest_with_files(self, helpers, component, datasetId): + def test_get_manifest_with_files(self, helpers, component, datasetId, data_model_path): """ Test to ensure that when generating a record based manifset that has files in the dataset that the files are not added to the manifest as well """ - path_to_data_model = helpers.get_data_path("example.model.jsonld") - graph_data_model = generate_graph_data_model( helpers, - path_to_data_model=path_to_data_model, + path_to_data_model=data_model_path, data_model_labels="class_label", ) generator = ManifestGenerator( - path_to_data_model=path_to_data_model, + path_to_data_model=data_model_path, graph=graph_data_model, root=component, use_annotations=True, From 7b82453774a2c82eb2ae0f12506ade2cf26e38ee Mon Sep 17 00:00:00 2001 From: GiaJordan Date: Thu, 29 Aug 2024 10:05:02 -0700 Subject: [PATCH 146/233] modifiy query exception raised --- schematic/store/synapse.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/schematic/store/synapse.py b/schematic/store/synapse.py index c9ca36c10..139002a20 100644 --- a/schematic/store/synapse.py +++ b/schematic/store/synapse.py @@ -296,12 +296,12 @@ def query_fileview( if "Unknown column path" in exception_text: raise ValueError( "The path column has not been added to the fileview. Please make sure that the fileview is up to date. You can add the path column to the fileview by follwing the instructions in the validation rules documentation." - ) from exc + ) elif "Unknown column" in exception_text: missing_column = exception_text.split("Unknown column ")[-1] raise ValueError( f"The columns {missing_column} specified in the query do not exist in the fileview. Please make sure that the column names are correct and that all expected columns have been added to the fileview." - ) from exc + ) else: raise AccessCredentialsError(self.storageFileview) From 06532e050d5caa703bb1b0ec8d7ba50578488ab7 Mon Sep 17 00:00:00 2001 From: GiaJordan Date: Thu, 29 Aug 2024 10:13:51 -0700 Subject: [PATCH 147/233] mock config for query exception tests --- tests/test_store.py | 31 ++++++++++++++++++------------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/tests/test_store.py b/tests/test_store.py index 94e897dc0..9ad00a052 100644 --- a/tests/test_store.py +++ b/tests/test_store.py @@ -10,7 +10,7 @@ from contextlib import nullcontext as does_not_raise from time import sleep from typing import Any, Generator -from unittest.mock import AsyncMock, patch +from unittest.mock import AsyncMock, MagicMock, patch import pandas as pd import pytest @@ -258,26 +258,31 @@ def test_view_query( ) def test_view_query_exception( self, - synapse_store_special_scope: SynapseStorage, asset_view: str, columns: list[str], message: str, ) -> None: + # GIVEN a project scope project_scope = ["syn23643250"] - # GIVEN the appropriate test file view - CONFIG.synapse_master_fileview_id = asset_view - # AND the approrpiate project scope - synapse_store_special_scope.project_scope = project_scope - - # WHEN the query is built and run - try: + # AND a test configuration + TEST_CONFIG = Configuration() + with patch( + "schematic.store.synapse.CONFIG", return_value=TEST_CONFIG + ) as mock_config: + # AND the appropriate test file view + mock_config.synapse_master_fileview_id = asset_view + # AND a real path to the synapse config file + mock_config.synapse_configuration_path = CONFIG.synapse_configuration_path + # AND a unique synapse storage object that uses the values modified in the test config + synapse_store = SynapseStorage(perform_query=False) + # AND the given project scope + synapse_store.project_scope = project_scope + + # WHEN the query is built and run # THEN it should raise a ValueError with the appropriate message with pytest.raises(ValueError, match=message): - synapse_store_special_scope.query_fileview(columns) - finally: - # AND the fileview should be reset to the default so the other tests are not affected regardless of the outcome of the query - CONFIG.synapse_master_fileview_id = "syn23643253" + synapse_store.query_fileview(columns) def test_getFileAnnotations(self, synapse_store: SynapseStorage) -> None: expected_dict = { From 5dc117b5aabac8592245501f418fe834619453b9 Mon Sep 17 00:00:00 2001 From: GiaJordan Date: Thu, 29 Aug 2024 11:06:11 -0700 Subject: [PATCH 148/233] update test data --- .../data/mock_manifests/filepath_submission_test_manifest.csv | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/data/mock_manifests/filepath_submission_test_manifest.csv b/tests/data/mock_manifests/filepath_submission_test_manifest.csv index 9def5eccc..11e0d5077 100644 --- a/tests/data/mock_manifests/filepath_submission_test_manifest.csv +++ b/tests/data/mock_manifests/filepath_submission_test_manifest.csv @@ -1,3 +1,3 @@ Filename,Sample ID,File Format,Component,Genome Build,Genome FASTA,Id,entityId -schematic - main/Test Filename Upload/txt1.txt,,,BulkRNA-seqAssay,,,01ded8fc-0915-4959-85ab-64e9644c8787,syn62276954 -schematic - main/Test Filename Upload/txt2.txt,,,BulkRNA-seqAssay,,,fd122bb5-3353-4c94-b1f5-0bb93a3e9fc9,syn62276956 +schematic - main/Test Filename Upload/txt1.txt,1,,BulkRNA-seqAssay,,,01ded8fc-0915-4959-85ab-64e9644c8787,syn62276954 +schematic - main/Test Filename Upload/txt2.txt,2,,BulkRNA-seqAssay,,,fd122bb5-3353-4c94-b1f5-0bb93a3e9fc9,syn62276956 From b2b09ca947a8cb3e982e391b7637464d46e12429 Mon Sep 17 00:00:00 2001 From: GiaJordan Date: Thu, 29 Aug 2024 11:06:28 -0700 Subject: [PATCH 149/233] update test --- tests/integration/test_metadata_model.py | 33 ++++++++++++++++++++++-- 1 file changed, 31 insertions(+), 2 deletions(-) diff --git a/tests/integration/test_metadata_model.py b/tests/integration/test_metadata_model.py index 7276812e2..26602bb92 100644 --- a/tests/integration/test_metadata_model.py +++ b/tests/integration/test_metadata_model.py @@ -1,6 +1,9 @@ import logging from contextlib import nullcontext as does_not_raise +from pytest_mock import MockerFixture + +from schematic.store.synapse import SynapseStorage from tests.conftest import metadata_model logging.basicConfig(level=logging.DEBUG) @@ -8,19 +11,45 @@ class TestMetadataModel: - def test_submit_filebased_manifest(self, helpers): + def test_submit_filebased_manifest(self, helpers, mocker: MockerFixture): + # spys + spy_upload_file_as_csv = mocker.spy(SynapseStorage, "upload_manifest_as_csv") + spy_upload_file_as_table = mocker.spy( + SynapseStorage, "upload_manifest_as_table" + ) + spy_upload_file_combo = mocker.spy(SynapseStorage, "upload_manifest_combo") + spy_add_annotations = mocker.spy( + SynapseStorage, "add_annotations_to_entities_files" + ) + + # GIVEN a metadata model object using class labels meta_data_model = metadata_model(helpers, "class_label") + # AND a filebased test manifset manifest_path = helpers.get_data_path( "mock_manifests/filepath_submission_test_manifest.csv" ) + # WHEN the manifest it submitted + # THEN submission should complete without error with does_not_raise(): manifest_id = meta_data_model.submit_metadata_manifest( manifest_path=manifest_path, dataset_id="syn62276880", - manifest_record_type="file_only", + manifest_record_type="file_and_entities", restrict_rules=False, file_annotations_upload=True, + hide_blanks=False, ) + + # AND the manifest should be submitted to the correct place assert manifest_id == "syn62280543" + + # AND the manifest should be uploaded as a CSV + spy_upload_file_as_csv.assert_called_once() + # AND annotations should be added to the files + spy_add_annotations.assert_called_once() + + # AND the manifest should not be uploaded as a table or combination of table, file, and entities + spy_upload_file_as_table.assert_not_called() + spy_upload_file_combo.assert_not_called() From 8c77ccfce7bfa780e5a3f7f2551d37ebd01bd074 Mon Sep 17 00:00:00 2001 From: GiaJordan Date: Thu, 29 Aug 2024 11:23:03 -0700 Subject: [PATCH 150/233] update test file --- .../data/mock_manifests/filepath_submission_test_manifest.csv | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/data/mock_manifests/filepath_submission_test_manifest.csv b/tests/data/mock_manifests/filepath_submission_test_manifest.csv index 11e0d5077..3b1a349fc 100644 --- a/tests/data/mock_manifests/filepath_submission_test_manifest.csv +++ b/tests/data/mock_manifests/filepath_submission_test_manifest.csv @@ -1,3 +1,3 @@ Filename,Sample ID,File Format,Component,Genome Build,Genome FASTA,Id,entityId -schematic - main/Test Filename Upload/txt1.txt,1,,BulkRNA-seqAssay,,,01ded8fc-0915-4959-85ab-64e9644c8787,syn62276954 -schematic - main/Test Filename Upload/txt2.txt,2,,BulkRNA-seqAssay,,,fd122bb5-3353-4c94-b1f5-0bb93a3e9fc9,syn62276956 +schematic - main/Test Filename Upload/txt1.txt,1.0,,BulkRNA-seqAssay,,,01ded8fc-0915-4959-85ab-64e9644c8787,syn62276954 +schematic - main/Test Filename Upload/txt2.txt,2.0,,BulkRNA-seqAssay,,,fd122bb5-3353-4c94-b1f5-0bb93a3e9fc9,syn62276956 From ce985be311b028df476ecc1410360a5533716c59 Mon Sep 17 00:00:00 2001 From: linglp Date: Fri, 30 Aug 2024 13:31:40 -0400 Subject: [PATCH 151/233] add type hinting --- schematic/schemas/data_model_graph.py | 2 +- tests/test_schemas.py | 13 +++++++------ 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/schematic/schemas/data_model_graph.py b/schematic/schemas/data_model_graph.py index 61eef7b4a..c06670f16 100644 --- a/schematic/schemas/data_model_graph.py +++ b/schematic/schemas/data_model_graph.py @@ -772,7 +772,7 @@ def get_node_required( def get_node_validation_rules( self, node_label: Optional[str] = None, node_display_name: Optional[str] = None - ) -> Union[list, dict[str, str]]: + ) -> Union[list[str], dict[str, str]]: """Get validation rules associated with a node, Args: diff --git a/tests/test_schemas.py b/tests/test_schemas.py index c72f2aa28..25df1e9ba 100644 --- a/tests/test_schemas.py +++ b/tests/test_schemas.py @@ -3,7 +3,7 @@ import os import random from copy import deepcopy -from typing import Optional +from typing import Optional, Union import networkx as nx import numpy as np @@ -37,6 +37,7 @@ get_label_from_display_name, parse_validation_rules, ) +from tests.conftest import Helpers logging.basicConfig(level=logging.DEBUG) logger = logging.getLogger(__name__) @@ -649,11 +650,11 @@ def test_get_node_required(self): ) def test_get_node_validation_rules_valid( self, - helpers, - data_model, - node_label, - node_display_name, - expected_validation_rule, + helpers: Helpers, + data_model: str, + node_label: Optional[str], + node_display_name: Optional[str], + expected_validation_rule: Union[list[str], dict[str, str]], ): DMGE = helpers.get_data_model_graph_explorer(path=data_model) From d85baa6ee5b41a8d5df7b3dcc55567cab5ad548e Mon Sep 17 00:00:00 2001 From: linglp Date: Fri, 30 Aug 2024 13:37:42 -0400 Subject: [PATCH 152/233] add exception type --- schematic/schemas/data_model_graph.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/schematic/schemas/data_model_graph.py b/schematic/schemas/data_model_graph.py index c06670f16..e78875a81 100644 --- a/schematic/schemas/data_model_graph.py +++ b/schematic/schemas/data_model_graph.py @@ -795,7 +795,7 @@ def get_node_validation_rules( try: node_validation_rules = self.graph.nodes[node_label]["validationRules"] - except: + except KeyError: raise ValueError( f"{node_label} is not in the graph, please check that you are providing the proper node label" ) From 2d60374a2913e554c40b09611eec33d21a692a6e Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Fri, 30 Aug 2024 11:46:55 -0700 Subject: [PATCH 153/233] added unit tests for validate attribute class --- schematic/models/validate_attribute.py | 201 ++-- tests/integration/test_validate_attribute.py | 10 + tests/unit/test_validate_attribute.py | 1115 ++++++++++++++++++ 3 files changed, 1236 insertions(+), 90 deletions(-) create mode 100644 tests/unit/test_validate_attribute.py diff --git a/schematic/models/validate_attribute.py b/schematic/models/validate_attribute.py index a4f79b036..f5444ad08 100644 --- a/schematic/models/validate_attribute.py +++ b/schematic/models/validate_attribute.py @@ -12,6 +12,7 @@ import requests from jsonschema import ValidationError from synapseclient.core.exceptions import SynapseNoCredentialsError +from synapseclient import File from schematic.schemas.data_model_graph import DataModelGraphExplorer from schematic.store.synapse import SynapseStorage @@ -831,9 +832,52 @@ def get_entry_has_value( node_display_name, ) + def _get_target_manifest_dataframes( + self, + target_component: str, + project_scope: Optional[list[str]] = None, + access_token: Optional[str] = None, + ) -> dict[str, pd.DataFrame]: + """Returns target manifest dataframes in the form of a dictionary + + Args: + target_component (str): The component to get manifests for + project_scope (Optional[list[str]], optional): + Projects to limit the scope of cross manifest validation to. Defaults to None. + access_token (Optional[str], optional): Asset Store access token. Defaults to None. + + Returns: + dict[str, pd.DataFrame]: Keys are synapse ids, values are datframes of the synapse id + """ + manifest_ids, dataset_ids = self.get_target_manifests( + target_component, project_scope, access_token + ) + manifests: list[pd.DataFrame] = [] + for dataset_id in dataset_ids: + entity: File = self.synStore.getDatasetManifest( + datasetId=dataset_id, downloadFile=True + ) + manifests.append(pd.read_csv(entity.path)) + return dict(zip(manifest_ids, manifests)) + def get_target_manifests( - self, target_component: str, project_scope: list[str], access_token: str = None - ): + self, + target_component: str, + project_scope: Optional[list[str]], + access_token: Optional[str] = None, + ) -> tuple[list[str], list[str]]: + """Gets a list of synapse ids of mainfests to check against + + Args: + target_component (str): Manifet ids are gotten fo this type + project_scope (Optional[list[str]]): Projects to limit the scope + of cross manifest validation to. Defaults to None. + access_token (Optional[str], optional): Synapse access token Defaults to None. + + Returns: + tuple[list[str], list[str]]: + A list of manifest synapse ids, and their dataset synapse ids + """ t_manifest_search = perf_counter() target_manifest_ids = [] target_dataset_ids = [] @@ -1314,19 +1358,19 @@ def _gather_set_warnings_errors( val_rule: str, source_attribute: str, set_validation_store: tuple[ - dict[str, pd.core.series.Series], + dict[str, pd.Series], list[str], - dict[str, pd.core.series.Series], + dict[str, pd.Series], ], - ) -> tuple[[list[str], list[str]]]: + ) -> tuple[list[str], list[str]]: """Based on the cross manifest validation rule, and in set rule scope, pass variables to _get_cross_errors_warnings to log appropriate error or warning. Args: val_rule, str: Validation Rule source_attribute, str: Source manifest column name - set_validation_store, tuple[dict[str, pd.core.series.Series], list[string], - dict[str, pd.core.series.Series]]: + set_validation_store, tuple[dict[str, pd.Series], list[string], + dict[str, pd.Series]]: contains the missing_manifest_log, present_manifest_log, and repeat_manifest_log dmge: DataModelGraphExplorer Object. @@ -1336,7 +1380,8 @@ def _gather_set_warnings_errors( warnings, list[str]: list of warnings to raise, as appropriate, if values in current manifest do not pass relevant cross mannifest validation across the target manifest(s) """ - errors, warnings = [], [] + errors: list[str] = [] + warnings: list[str] = [] ( missing_manifest_log, @@ -1476,18 +1521,13 @@ def _gather_value_warnings_errors( self, val_rule: str, source_attribute: str, - value_validation_store: tuple[ - dict[str, pd.core.series.Series], - dict[str, pd.core.series.Series], - dict[str, pd.core.series.Series], - ], - ) -> tuple[[list[str], list[str]]]: + value_validation_store: tuple[pd.Series, pd.Series, pd.Series], + ) -> tuple[list[str], list[str]]: """For value rule scope, find invalid rows and entries, and generate appropriate errors and warnings Args: val_rule, str: Validation rule source_attribute, str: source manifest column name - value_validation_store, tuple(dict[str, pd.core.series.Series], dict[str, pd.core.series.Series], - dict[str, pd.core.series.Series]): + value_validation_store, tuple(pd.Series, pd.Series, pd.Series]): contains missing_values, duplicated_values, and repeat values Returns: errors, list[str]: list of errors to raise, as appropriate, if values in current manifest do @@ -1564,20 +1604,20 @@ def _run_validation_across_targets_set( self, val_rule: str, column_names: dict[str, str], - manifest_col: pd.core.series.Series, + manifest_col: pd.Series, target_attribute: str, - target_manifest: pd.core.series.Series, + target_manifest: pd.DataFrame, target_manifest_id: str, - missing_manifest_log: dict[str, pd.core.series.Series], + missing_manifest_log: dict[str, pd.Series], present_manifest_log: list[str], - repeat_manifest_log: dict[str, pd.core.series.Series], + repeat_manifest_log: dict[str, pd.Series], target_attribute_in_manifest_list: list[bool], target_manifest_empty: list[bool], ) -> tuple[ tuple[ - dict[str, pd.core.series.Series], + dict[str, pd.Series], list[str], - dict[str, pd.core.series.Series], + dict[str, pd.Series], ], list[bool], list[bool], @@ -1586,26 +1626,26 @@ def _run_validation_across_targets_set( Args: val_rule, str: Validation rule column_names, dict[str,str]: {stripped_col_name:original_column_name} - target_column, pd.core.series.Series: Empty target_column to fill out in this function - manifest_col, pd.core.series.Series: Source manifest column + target_column, pd.Series: Empty target_column to fill out in this function + manifest_col, pd.Series: Source manifest column target_attribute, str: current target attribute - target_column, pd.core.series.Series: Current target column - target_manifest, pd.core.series.Series: Current target manifest + target_column, pd.Series: Current target column + target_manifest, pd.DataFrame: Current target manifest target_manifest_id, str: Current target manifest Synapse ID - missing_manifest_log, dict[str, pd.core.series.Series]: + missing_manifest_log, dict[str, pd.Series]: Log of manifests with missing values, {synapse_id: index,missing value}, updated. present_manifest_log, list[str] Log of present manifests, [synapse_id present manifest], updated. - repeat_manifest_log, dict[str, pd.core.series.Series] + repeat_manifest_log, dict[str, pd.Series] Log of manifests with repeat values, {synapse_id: index,repeat value}, updated. Returns: tuple( - missing_manifest_log, dict[str, pd.core.series.Series]: + missing_manifest_log, dict[str, pd.Series]: Log of manifests with missing values, {synapse_id: index,missing value}, updated. present_manifest_log, list[str] Log of present manifests, [synapse_id present manifest], updated. - repeat_manifest_log, dict[str, pd.core.series.Series] + repeat_manifest_log, dict[str, pd.Series] Log of manifests with repeat values, {synapse_id: index,repeat value}, updated.) target_attribute_in_manifest, bool: True if the target attribute is in the current manifest. """ @@ -1656,11 +1696,11 @@ def _gather_target_columns_value( self, column_names: dict[str, str], target_attribute: str, - concatenated_target_column: pd.core.series.Series, - target_manifest: pd.core.series.Series, + concatenated_target_column: pd.Series, + target_manifest: pd.DataFrame, target_attribute_in_manifest_list: list[bool], target_manifest_empty: list[bool], - ) -> tuple[pd.core.series.Series, list[bool], list[bool],]: + ) -> tuple[pd.Series, list[bool], list[bool],]: """A helper function for creating a concatenating all target attribute columns across all target manifest. This function checks if the target attribute is in the current target manifest. If it is, and is the first manifest with this column, start recording it, if it has already been recorded from @@ -1668,11 +1708,11 @@ def _gather_target_columns_value( Args: column_names, dict: {stripped_col_name:original_column_name} target_attribute, str: current target attribute - concatenated_target_column, pd.core.series.Series: target column in the process of being built, possibly + concatenated_target_column, pd.Series: target column in the process of being built, possibly passed through this function multiple times based on the number of manifests - target_manifest, pd.core.series.Series: current target manifest + target_manifest, pd.DataFrame: current target manifest Returns: - concatenated_target_column, pd.core.series.Series: All target columns concatenated into a single column + concatenated_target_column, pd.Series: All target columns concatenated into a single column """ # Check if the target_attribute is in the current target manifest. target_attribute_in_manifest = False @@ -1713,20 +1753,20 @@ def _gather_target_columns_value( def _run_validation_across_targets_value( self, - manifest_col: pd.core.series.Series, - concatenated_target_column: pd.core.series.Series, - ) -> tuple[[pd.core.series.Series, pd.core.series.Series, pd.core.series.Series]]: + manifest_col: pd.Series, + concatenated_target_column: pd.Series, + ) -> tuple[pd.Series, pd.Series, pd.Series]: """Get missing values, duplicated values and repeat values assesed comapring the source manifest to all the values in all target columns. Args: - manifest_col, pd.core.series.Series: Current source manifest column - concatenated_target_column, pd.core.series.Series: All target columns concatenated into a single column + manifest_col, pd.Series: Current source manifest column + concatenated_target_column, pd.Series: All target columns concatenated into a single column Returns: - missing_values, pd.core.series.Series: values that are present in the source manifest, but not present + missing_values, pd.Series: values that are present in the source manifest, but not present in the target manifest - duplicated_values, pd.core.series.Series: values that duplicated in the concatenated target column, and + duplicated_values, pd.Series: values that duplicated in the concatenated target column, and also present in the source manifest column - repeat_values, pd.core.series.Series: values that are repeated between the manifest column and + repeat_values, pd.Series: values that are repeated between the manifest column and concatenated target column """ # Find values that are present in the source manifest, but not present in the target manifest @@ -1744,12 +1784,10 @@ def _run_validation_across_targets_value( return missing_values, duplicated_values, repeat_values - def _get_column_names( - self, target_manifest: pd.core.series.Series - ) -> dict[str, str]: + def _get_column_names(self, target_manifest: pd.DataFrame) -> dict[str, str]: """Convert manifest column names into validation rule input format Args: - target_manifest, pd.core.series.Series: Current target manifest + target_manifest, pd.DataFrame: Current target manifest Returns: column_names, dict[str,str]: {stripped_col_name:original_column_name} """ @@ -1771,27 +1809,21 @@ def _get_rule_scope(self, val_rule: str) -> ScopeTypes: def _run_validation_across_target_manifests( self, - project_scope: Optional[list[str]], rule_scope: ScopeTypes, - access_token: str, val_rule: str, - manifest_col: pd.core.series.Series, - target_column: pd.core.series.Series, + manifest_col: pd.Series, + target_column: pd.Series, + access_token: Optional[str] = None, + project_scope: Optional[list[str]] = None, ) -> tuple[ float, Union[ - Union[ - tuple[ - dict[str, pd.core.series.Series], - list[str], - dict[str, pd.core.series.Series], - ], - tuple[ - dict[str, pd.core.series.Series], - dict[str, pd.core.series.Series], - dict[str, pd.core.series.Series], - ], + tuple[ + dict[str, pd.Series], + list[str], + dict[str, pd.Series], ], + tuple[pd.Series, pd.Series, pd.Series], bool, str, ], @@ -1801,10 +1833,10 @@ def _run_validation_across_target_manifests( Args: project_scope, Optional[list]: Projects to limit the scope of cross manifest validation to. rule_scope, ScopeTypes: The scope of the rule, taken from validation rule - access_token, str: Asset Store access token + access_token, Optional[str]: Asset Store access token val_rule, str: Validation rule. - manifest_col, pd.core.series.Series: Source manifest column for a given source component - target_column, pd.core.series.Series: Empty target_column to fill out in this function + manifest_col, pd.Series: Source manifest column for a given source component + target_column, pd.Series: Empty target_column to fill out in this function Returns: start_time, float: start time in fractional seconds valdiation_output: @@ -1813,9 +1845,9 @@ def _run_validation_across_target_manifests( "values not recorded in targets stored", str, will return a string if targets were found, but there was no data in the target. Union[ - tuple[dict[str, pd.core.series.Series], list[str], dict[str, pd.core.series.Series]], - tuple[dict[str, pd.core.series.Series], dict[str, pd.core.series.Series], - dict[str, pd.core.series.Series]]: + tuple[dict[str, pd.Series], list[str], dict[str, pd.Series]], + tuple[dict[str, pd.Series], dict[str, pd.Series], + dict[str, pd.Series]]: validation outputs, exact types depend on scope, """ # Initialize variables @@ -1834,27 +1866,16 @@ def _run_validation_across_target_manifests( [target_component, target_attribute] = val_rule.lower().split(" ")[1].split(".") target_column.name = target_attribute - # Get IDs of manifests with target component - ( - target_manifest_ids, - target_dataset_ids, - ) = self.get_target_manifests(target_component, project_scope, access_token) - # Start timer start_time = perf_counter() + manifest_dict = self._get_target_manifest_dataframes( + target_component, project_scope, access_token + ) + # For each target manifest, gather target manifest column and compare to the source manifest column # Save relevant data as appropriate for the given scope - for target_manifest_id, target_dataset_id in zip( - target_manifest_ids, target_dataset_ids - ): - # Pull manifest from Synapse - entity = self.synStore.getDatasetManifest( - datasetId=target_dataset_id, downloadFile=True - ) - # Load manifest - target_manifest = pd.read_csv(entity.path) - + for target_manifest_id, target_manifest in manifest_dict.items(): # Get manifest column names column_names = self._get_column_names(target_manifest=target_manifest) @@ -1936,9 +1957,9 @@ def _run_validation_across_target_manifests( def cross_validation( self, val_rule: str, - manifest_col: pd.core.series.Series, - project_scope: Optional[list[str]], - access_token: str, + manifest_col: pd.Series, + project_scope: Optional[list[str]] = None, + access_token: Optional[str] = None, ) -> list[list[str]]: """ Purpose: @@ -1946,11 +1967,11 @@ def cross_validation( by project scope, if provided). Args: val_rule, str: Validation rule - manifest_col, pd.core.series.Series: column for a given + manifest_col, pd.Series: column for a given attribute in the manifest project_scope, Optional[list] = None: Projects to limit the scope of cross manifest validation to. dmge: DataModelGraphExplorer Object - access_token, str: Asset Store access token + access_token, Optional[str]: Asset Store access token Returns: errors, warnings, list[list[str]]: raise warnings and errors as appropriate if values in current manifest do no pass relevant cross mannifest validation across the target manifest(s) diff --git a/tests/integration/test_validate_attribute.py b/tests/integration/test_validate_attribute.py index 36c1f9bab..d4d59eefc 100644 --- a/tests/integration/test_validate_attribute.py +++ b/tests/integration/test_validate_attribute.py @@ -73,3 +73,13 @@ def test_url_validation_invalid_url(self, dmge: DataModelGraphExplorer) -> None: ], [], ) + + def test__get_target_manifest_dataframes( + self, dmge: DataModelGraphExplorer + ) -> None: + """Testing for ValidateAttribute._get_target_manifest_dataframes""" + validator = ValidateAttribute(dmge=dmge) + manifests = validator._get_target_manifest_dataframes( # pylint:disable= protected-access + "patient", project_scope=["syn54126707"] + ) + assert list(manifests.keys()) == ["syn54126997", "syn54127001"] diff --git a/tests/unit/test_validate_attribute.py b/tests/unit/test_validate_attribute.py new file mode 100644 index 000000000..009ecc5cb --- /dev/null +++ b/tests/unit/test_validate_attribute.py @@ -0,0 +1,1115 @@ +"""Unit testing for the ValidateAttribute class""" + +from typing import Generator +from unittest.mock import patch + +import pytest +from pandas import Series, DataFrame, concat + +from schematic.models.validate_attribute import ValidateAttribute +from schematic.schemas.data_model_graph import DataModelGraphExplorer +import schematic.models.validate_attribute + +# pylint: disable=protected-access + + +@pytest.fixture(name="cross_val_df1") +def fixture_cross_val_df1() -> Generator[DataFrame, None, None]: + """Yields a dataframe""" + df = DataFrame( + { + "PatientID": ["A", "B", "C"], + "component": ["comp1", "comp1", "comp1"], + "id": ["id1", "id2", "id3"], + "entityid": ["x", "x", "x"], + } + ) + yield df + + +@pytest.fixture(name="cross_val_df2") +def fixture_cross_val_df2(cross_val_df1: DataFrame) -> Generator[DataFrame, None, None]: + """Yields dataframe df1 with an extra row""" + df = concat( + [ + cross_val_df1, + DataFrame( + { + "PatientID": ["D"], + "component": ["comp1"], + "id": ["id4"], + "entityid": ["x"], + } + ), + ] + ) + yield df + + +@pytest.fixture(name="cross_val_df3") +def fixture_cross_val_df3() -> Generator[DataFrame, None, None]: + """Yields empty dataframe""" + df = DataFrame( + { + "PatientID": [], + "component": [], + "id": [], + "entityid": [], + } + ) + yield df + + +@pytest.fixture(name="cross_val_col_names") +def fixture_cross_val_col_names() -> Generator[dict[str, str], None, None]: + """ + Yields: + Generator[dict[str, str], None, None]: A dicitonary of column names + keys are the label, and values are the display name + """ + column_names = { + "patientid": "PatientID", + "component": "component", + "id": "id", + "entityid": "entityid", + } + yield column_names + + +@pytest.fixture(name="va_obj") +def fixture_va_obj( + dmge: DataModelGraphExplorer, +) -> Generator[ValidateAttribute, None, None]: + """Yield a ValidateAttribute object""" + yield ValidateAttribute(dmge) + + +class TestValidateAttributeObject: + """Testing for ValidateAttribute class with all Synapse calls mocked""" + + def test_cross_validation_match_atleast_one_set_rules_passing( + self, va_obj: ValidateAttribute, cross_val_df1: DataFrame + ): + """Tests for cross manifest validation for matchAtLeastOne set rules""" + val_rule = "matchAtLeastOne Patient.PatientID set error" + + with patch.object( + schematic.models.validate_attribute.ValidateAttribute, + "_get_target_manifest_dataframes", + return_value={"syn1": cross_val_df1}, + ): + assert va_obj.cross_validation(val_rule, Series(["A", "B", "C"])) == ( + [], + [], + ) + assert va_obj.cross_validation(val_rule, Series(["A", "B", "C", "C"])) == ( + [], + [], + ) + assert va_obj.cross_validation( + val_rule, Series(["A", "B", "C", "A", "B", "C"]) + ) == ( + [], + [], + ) + assert va_obj.cross_validation( + val_rule, Series(["A", "B"], index=[0, 1], name="PatientID") + ) == ( + [], + [], + ) + assert va_obj.cross_validation( + val_rule, Series([], index=[], name="PatientID") + ) == ( + [], + [], + ) + + def test_cross_validation_match_atleast_one_set_rules_errors( + self, va_obj: ValidateAttribute, cross_val_df1: DataFrame + ): + """Tests for cross manifest validation for matchAtLeastOne set rules""" + val_rule = "matchAtLeastOne Patient.PatientID set error" + + with patch.object( + schematic.models.validate_attribute.ValidateAttribute, + "_get_target_manifest_dataframes", + return_value={"syn1": cross_val_df1}, + ): + errors, warnings = va_obj.cross_validation( + val_rule, + Series(["A", "B", "C", "D"], index=[0, 1, 2, 3], name="PatientID"), + ) + assert len(warnings) == 0 + assert len(errors) == 1 + + errors, warnings = va_obj.cross_validation( + val_rule, Series([""], index=[0], name="PatientID") + ) + assert len(warnings) == 0 + assert len(errors) == 1 + + def test_cross_validation_match_atleast_one_set_rules_warnings( + self, va_obj: ValidateAttribute, cross_val_df1: DataFrame + ): + """Tests for cross manifest validation for matchAtLeastOne set rules""" + val_rule = "matchAtLeastOne Patient.PatientID set warning" + + with patch.object( + schematic.models.validate_attribute.ValidateAttribute, + "_get_target_manifest_dataframes", + return_value={"syn1": cross_val_df1}, + ): + errors, warnings = va_obj.cross_validation( + val_rule, + Series(["A", "B", "C", "D"], index=[0, 1, 2, 3], name="PatientID"), + ) + assert len(warnings) == 1 + assert len(errors) == 0 + + def test_cross_validation_match_exactly_one_set_rules_passing( + self, va_obj: ValidateAttribute, cross_val_df1: DataFrame + ): + """Tests for cross manifest validation for matchExactlyOne set rules""" + val_rule = "matchExactlyOne Patient.PatientID set error" + + with patch.object( + schematic.models.validate_attribute.ValidateAttribute, + "_get_target_manifest_dataframes", + return_value={"syn1": cross_val_df1}, + ): + assert va_obj.cross_validation(val_rule, Series(["A", "B", "C"])) == ( + [], + [], + ) + assert va_obj.cross_validation(val_rule, Series(["A", "B", "C", "C"])) == ( + [], + [], + ) + + assert va_obj.cross_validation(val_rule, Series(["A", "B"])) == ([], []) + + assert va_obj.cross_validation(val_rule, Series(["A", "B", "C", "D"])) == ( + [], + [], + ) + + def test_cross_validation_match_exactly_one_set_rules_errors( + self, va_obj: ValidateAttribute, cross_val_df1: DataFrame + ): + """Tests for cross manifest validation for matchExactlyOne set rules""" + val_rule = "matchExactlyOne Patient.PatientID set error" + + with patch.object( + schematic.models.validate_attribute.ValidateAttribute, + "_get_target_manifest_dataframes", + return_value={"syn1": cross_val_df1, "syn2": cross_val_df1}, + ): + errors, _ = va_obj.cross_validation( + val_rule, Series(["A", "B", "C"], index=[0, 1, 2], name="PatientID") + ) + assert len(errors) == 1 + + def test_cross_validation_match_none_set_rules_passing( + self, va_obj: ValidateAttribute, cross_val_df1: DataFrame + ): + """Tests for cross manifest validation for matchNone set rules""" + val_rule = "matchNone Patient.PatientID set error" + + with patch.object( + schematic.models.validate_attribute.ValidateAttribute, + "_get_target_manifest_dataframes", + return_value={"syn1": cross_val_df1}, + ): + assert va_obj.cross_validation(val_rule, Series(["D"])) == ( + [], + [], + ) + + def test_cross_validation_match_none_set_rules_errors( + self, va_obj: ValidateAttribute, cross_val_df1: DataFrame + ): + """Tests for cross manifest validation for matchNone set rules""" + val_rule = "matchNone Patient.PatientID set error" + + with patch.object( + schematic.models.validate_attribute.ValidateAttribute, + "_get_target_manifest_dataframes", + return_value={"syn1": cross_val_df1}, + ): + errors, _ = va_obj.cross_validation( + val_rule, + Series(["A", "B", "C"], index=[0, 1, 2], name="PatientID"), + ) + assert len(errors) == 1 + + errors, _ = va_obj.cross_validation( + val_rule, + Series(["A", "B", "C", "C"], index=[0, 1, 2, 3], name="PatientID"), + ) + assert len(errors) == 1 + + errors, _ = va_obj.cross_validation( + val_rule, Series(["A", "B"], index=[0, 1], name="PatientID") + ) + assert len(errors) == 1 + + def test_cross_validation_value_match_atleast_one_rules_passing( + self, + va_obj: ValidateAttribute, + cross_val_df1: DataFrame, + cross_val_df2: DataFrame, + ): + """Tests for cross manifest validation for matchAtLeastOne value rules""" + val_rule = "matchAtLeastOne Patient.PatientID value error" + + with patch.object( + schematic.models.validate_attribute.ValidateAttribute, + "_get_target_manifest_dataframes", + return_value={"syn1": cross_val_df1}, + ): + assert va_obj.cross_validation(val_rule, Series([])) == ([], []) + assert va_obj.cross_validation(val_rule, Series(["A"])) == ([], []) + assert va_obj.cross_validation(val_rule, Series(["A", "A"])) == ([], []) + assert va_obj.cross_validation(val_rule, Series(["A", "B"])) == ([], []) + assert va_obj.cross_validation(val_rule, Series(["A", "B", "C"])) == ( + [], + [], + ) + assert va_obj.cross_validation(val_rule, Series(["A", "B", "C", "C"])) == ( + [], + [], + ) + + with patch.object( + schematic.models.validate_attribute.ValidateAttribute, + "_get_target_manifest_dataframes", + return_value={"syn1": cross_val_df1, "syn2": cross_val_df2}, + ): + assert va_obj.cross_validation(val_rule, Series(["A", "B", "C", "D"])) == ( + [], + [], + ) + + def test_cross_validation_value_match_atleast_one_rules_errors( + self, va_obj: ValidateAttribute, cross_val_df1: DataFrame + ): + """Tests for cross manifest validation for matchAtLeastOne value rules""" + val_rule = "matchAtLeastOne Patient.PatientID value error" + + with patch.object( + schematic.models.validate_attribute.ValidateAttribute, + "_get_target_manifest_dataframes", + return_value={"syn1": cross_val_df1}, + ): + errors, _ = va_obj.cross_validation( + val_rule, Series(["D"], index=[0], name="PatientID") + ) + assert len(errors) == 1 + + def test_cross_validation_match_exactly_one_value_rules_passing( + self, va_obj: ValidateAttribute, cross_val_df1: DataFrame + ): + """Tests for cross manifest validation for matchExactlyOne value rules""" + val_rule = "matchExactlyOne Patient.PatientID value error" + + with patch.object( + schematic.models.validate_attribute.ValidateAttribute, + "_get_target_manifest_dataframes", + return_value={"syn1": cross_val_df1}, + ): + assert va_obj.cross_validation(val_rule, Series([])) == ([], []) + assert va_obj.cross_validation(val_rule, Series(["A"])) == ([], []) + assert va_obj.cross_validation(val_rule, Series(["A", "A"])) == ([], []) + assert va_obj.cross_validation(val_rule, Series(["A", "B"])) == ([], []) + assert va_obj.cross_validation(val_rule, Series(["A", "B", "C"])) == ( + [], + [], + ) + assert va_obj.cross_validation(val_rule, Series(["A", "B", "C", "C"])) == ( + [], + [], + ) + + with patch.object( + schematic.models.validate_attribute.ValidateAttribute, + "_get_target_manifest_dataframes", + return_value={"syn1": cross_val_df1, "syn2": cross_val_df1}, + ): + assert va_obj.cross_validation(val_rule, Series([])) == ([], []) + + def test_cross_validation_match_exactly_one_value_rules_errors( + self, va_obj: ValidateAttribute, cross_val_df1: DataFrame + ): + """Tests for cross manifest validation for matchExactlyOne value rules""" + val_rule = "matchExactlyOne Patient.PatientID value error" + + with patch.object( + schematic.models.validate_attribute.ValidateAttribute, + "_get_target_manifest_dataframes", + return_value={"syn1": cross_val_df1}, + ): + errors, _ = va_obj.cross_validation( + val_rule, Series(["D"], index=[0], name="PatientID") + ) + assert len(errors) == 1 + + with patch.object( + schematic.models.validate_attribute.ValidateAttribute, + "_get_target_manifest_dataframes", + return_value={"syn1": cross_val_df1, "syn2": cross_val_df1}, + ): + errors, _ = va_obj.cross_validation( + val_rule, Series(["A"], index=[0], name="PatientID") + ) + assert len(errors) == 1 + + errors, _ = va_obj.cross_validation( + val_rule, Series(["D"], index=[0], name="PatientID") + ) + assert len(errors) == 1 + + def test_cross_validation_match_none_value_rules_passing( + self, va_obj: ValidateAttribute, cross_val_df1: DataFrame + ): + """Tests for cross manifest validation for matchNone value rules""" + val_rule = "matchNone Patient.PatientID value error" + + with patch.object( + schematic.models.validate_attribute.ValidateAttribute, + "_get_target_manifest_dataframes", + return_value={"syn1": cross_val_df1}, + ): + assert va_obj.cross_validation(val_rule, Series([])) == ([], []) + assert va_obj.cross_validation(val_rule, Series(["D"])) == ([], []) + + def test_cross_validation_match_none_value_rules_errors( + self, va_obj: ValidateAttribute, cross_val_df1: DataFrame + ): + """Tests for cross manifest validation for matchNone value rules""" + val_rule = "matchNone Patient.PatientID value error" + + with patch.object( + schematic.models.validate_attribute.ValidateAttribute, + "_get_target_manifest_dataframes", + return_value={"syn1": cross_val_df1}, + ): + errors, _ = va_obj.cross_validation( + val_rule, Series(["A"], index=[0], name="PatientID") + ) + assert len(errors) == 1 + + def test__run_validation_across_target_manifests_failures( + self, + va_obj: ValidateAttribute, + cross_val_df1: DataFrame, + cross_val_df3: DataFrame, + ) -> None: + """Tests for ValidateAttribute._run_validation_across_target_manifests with failures""" + # This shows that when no target manifests are found to check against, False is returned + with patch.object( + schematic.models.validate_attribute.ValidateAttribute, + "_get_target_manifest_dataframes", + return_value={}, + ): + _, result = va_obj._run_validation_across_target_manifests( + rule_scope="value", + val_rule="xxx Patient.PatientID xxx", + manifest_col=Series([]), + target_column=Series([]), + ) + assert result is False + + # This shows that when the only target manifest is empty, only a message is returned + with patch.object( + schematic.models.validate_attribute.ValidateAttribute, + "_get_target_manifest_dataframes", + return_value={"syn1": cross_val_df3}, + ): + _, result = va_obj._run_validation_across_target_manifests( + rule_scope="value", + val_rule="xxx Patient.PatientID xxx", + manifest_col=Series([]), + target_column=Series([]), + ) + + assert result == "values not recorded in targets stored" + + # This shows that when the only target manifest is empty, only a message is returned + # even if the tested column has values + with patch.object( + schematic.models.validate_attribute.ValidateAttribute, + "_get_target_manifest_dataframes", + return_value={"syn1": cross_val_df3}, + ): + _, result = va_obj._run_validation_across_target_manifests( + rule_scope="value", + val_rule="xxx Patient.PatientID xxx", + manifest_col=Series(["A", "B", "C"]), + target_column=Series([]), + ) + + assert result == "values not recorded in targets stored" + + # This shows that when any target manifest is empty, only a message is returned + with patch.object( + schematic.models.validate_attribute.ValidateAttribute, + "_get_target_manifest_dataframes", + return_value={"syn1": cross_val_df1, "syn2": cross_val_df3}, + ): + _, result = va_obj._run_validation_across_target_manifests( + rule_scope="value", + val_rule="xxx Patient.PatientID xxx", + manifest_col=Series([]), + target_column=Series([]), + ) + + assert result == "values not recorded in targets stored" + + def test__run_validation_across_target_manifests_value_rules( + self, va_obj: ValidateAttribute, cross_val_df1: DataFrame + ) -> None: + """Tests for ValidateAttribute._run_validation_across_target_manifests with value rule""" + + # This tests when an empty column is validated there are no missing values to be returned + with patch.object( + schematic.models.validate_attribute.ValidateAttribute, + "_get_target_manifest_dataframes", + return_value={"syn1": cross_val_df1}, + ): + _, validation_output = va_obj._run_validation_across_target_manifests( + rule_scope="value", + val_rule="xxx Patient.PatientID xxx", + manifest_col=Series([]), + target_column=Series([]), + ) + assert isinstance(validation_output, tuple) + assert isinstance(validation_output[0], Series) + assert validation_output[0].empty + assert isinstance(validation_output[1], Series) + assert validation_output[1].empty + assert isinstance(validation_output[2], Series) + assert validation_output[2].empty + + def test__run_validation_across_target_manifests_set_rules_match_atleast_one( + self, + va_obj: ValidateAttribute, + cross_val_df1: DataFrame, + ) -> None: + """ + Tests for ValidateAttribute._run_validation_across_target_manifests + with matchAtleastOne set rule + """ + # These tests show when a column that is empty or partial/full match to + # the target manifest, the output only the present manifest log is updated + with patch.object( + schematic.models.validate_attribute.ValidateAttribute, + "_get_target_manifest_dataframes", + return_value={"syn1": cross_val_df1}, + ): + _, validation_output = va_obj._run_validation_across_target_manifests( + rule_scope="set", + val_rule="matchAtLeastOne Patient.PatientID set error", + manifest_col=Series([]), + target_column=Series([]), + ) + assert isinstance(validation_output, tuple) + assert validation_output[0] == {} + assert validation_output[1] == ["syn1"] + assert validation_output[2] == {} + + _, validation_output = va_obj._run_validation_across_target_manifests( + rule_scope="set", + val_rule="matchAtLeastOne Patient.PatientID set error", + manifest_col=Series(["A", "B", "C"]), + target_column=Series([]), + ) + assert isinstance(validation_output, tuple) + assert validation_output[0] == {} + assert validation_output[1] == ["syn1"] + assert validation_output[2] == {} + + _, validation_output = va_obj._run_validation_across_target_manifests( + rule_scope="set", + val_rule="matchAtLeastOne Patient.PatientID set error", + manifest_col=Series(["A"]), + target_column=Series([]), + ) + + # This test shows that if there is an extra value ("D") in the column being validated + # it gets added to the missing_manifest_log + with patch.object( + schematic.models.validate_attribute.ValidateAttribute, + "_get_target_manifest_dataframes", + return_value={"syn1": cross_val_df1}, + ): + _, validation_output = va_obj._run_validation_across_target_manifests( + rule_scope="set", + val_rule="matchAtLeastOne Patient.PatientID set error", + manifest_col=Series(["A", "B", "C", "D"]), + target_column=Series([]), + ) + assert isinstance(validation_output, tuple) + assert list(validation_output[0].keys()) == ["syn1"] + assert validation_output[0]["syn1"].to_list() == ["D"] + assert validation_output[1] == [] + assert validation_output[2] == {} + + def test__run_validation_across_target_manifests_set_rules_match_none( + self, + va_obj: ValidateAttribute, + cross_val_df1: DataFrame, + ) -> None: + """ + Tests for ValidateAttribute._run_validation_across_target_manifests + with matchNone set rule + """ + + # This test shows nothing happens when the column is empty or has no shared values + # witht the target manifest + with patch.object( + schematic.models.validate_attribute.ValidateAttribute, + "_get_target_manifest_dataframes", + return_value={"syn1": cross_val_df1}, + ): + _, validation_output = va_obj._run_validation_across_target_manifests( + rule_scope="set", + val_rule="matchNone Patient.PatientID set error", + manifest_col=Series([]), + target_column=Series([]), + ) + assert isinstance(validation_output, tuple) + assert validation_output[0] == {} + assert validation_output[1] == [] + assert validation_output[2] == {} + + _, validation_output = va_obj._run_validation_across_target_manifests( + rule_scope="set", + val_rule="matchNone Patient.PatientID set error", + manifest_col=Series(["D"]), + target_column=Series([]), + ) + assert isinstance(validation_output, tuple) + assert validation_output[0] == {} + assert validation_output[1] == [] + assert validation_output[2] == {} + + # These tests when any values match they are put into the repeat log + with patch.object( + schematic.models.validate_attribute.ValidateAttribute, + "_get_target_manifest_dataframes", + return_value={"syn1": cross_val_df1}, + ): + _, validation_output = va_obj._run_validation_across_target_manifests( + rule_scope="set", + val_rule="matchNone Patient.PatientID set error", + manifest_col=Series(["A", "B", "C"]), + target_column=Series([]), + ) + assert isinstance(validation_output, tuple) + assert validation_output[0] == {} + assert validation_output[1] == [] + assert list(validation_output[2].keys()) == ["syn1"] + assert validation_output[2]["syn1"].to_list() == ["A", "B", "C"] + + _, validation_output = va_obj._run_validation_across_target_manifests( + rule_scope="set", + val_rule="matchNone Patient.PatientID set error", + manifest_col=Series(["A"]), + target_column=Series([]), + ) + assert isinstance(validation_output, tuple) + assert validation_output[0] == {} + assert validation_output[1] == [] + assert list(validation_output[2].keys()) == ["syn1"] + assert validation_output[2]["syn1"].to_list() == ["A"] + + def test__run_validation_across_target_manifests_set_rules_exactly_one( + self, + va_obj: ValidateAttribute, + cross_val_df1: DataFrame, + ) -> None: + """ + Tests for ValidateAttribute._run_validation_across_target_manifests with + matchExactlyOne set rule + """ + # These tests show when an empty a partial match or full match column is used, + # the output only contains the targeted synapse id as a present value + with patch.object( + schematic.models.validate_attribute.ValidateAttribute, + "_get_target_manifest_dataframes", + return_value={"syn1": cross_val_df1}, + ): + _, validation_output = va_obj._run_validation_across_target_manifests( + rule_scope="set", + val_rule="matchExactlyOne Patient.PatientID set error", + manifest_col=Series([]), + target_column=Series([]), + ) + assert isinstance(validation_output, tuple) + assert validation_output[0] == {} + assert validation_output[1] == ["syn1"] + assert validation_output[2] == {} + + _, validation_output = va_obj._run_validation_across_target_manifests( + rule_scope="set", + val_rule="matchExactlyOne Patient.PatientID set error", + manifest_col=Series(["A", "B", "C"]), + target_column=Series([]), + ) + assert isinstance(validation_output, tuple) + assert validation_output[0] == {} + assert validation_output[1] == ["syn1"] + assert validation_output[2] == {} + + _, validation_output = va_obj._run_validation_across_target_manifests( + rule_scope="set", + val_rule="matchExactlyOne Patient.PatientID set error", + manifest_col=Series(["A"]), + target_column=Series([]), + ) + assert isinstance(validation_output, tuple) + assert validation_output[0] == {} + assert validation_output[1] == ["syn1"] + assert validation_output[2] == {} + + # These tests shows that if there is an extra value ("D") in the column being validated + # it gets added to the missing manifest values dict + with patch.object( + schematic.models.validate_attribute.ValidateAttribute, + "_get_target_manifest_dataframes", + return_value={"syn1": cross_val_df1}, + ): + _, validation_output = va_obj._run_validation_across_target_manifests( + rule_scope="set", + val_rule="matchExactlyOne Patient.PatientID set error", + manifest_col=Series(["A", "B", "C", "D"]), + target_column=Series([]), + ) + assert isinstance(validation_output, tuple) + assert list(validation_output[0].keys()) == ["syn1"] + assert validation_output[0]["syn1"].to_list() == ["D"] + assert validation_output[1] == [] + assert validation_output[2] == {} + + # This tests shows when a manifest macthes more than one manifest, both are added + # to the present manifest log + with patch.object( + schematic.models.validate_attribute.ValidateAttribute, + "_get_target_manifest_dataframes", + return_value={"syn1": cross_val_df1, "syn2": cross_val_df1}, + ): + _, validation_output = va_obj._run_validation_across_target_manifests( + rule_scope="set", + val_rule="matchExactlyOne Patient.PatientID set error", + manifest_col=Series([]), + target_column=Series(["A", "B", "C"]), + ) + assert isinstance(validation_output, tuple) + assert validation_output[0] == {} + assert validation_output[1] == ["syn1", "syn2"] + assert validation_output[2] == {} + + def test__run_validation_across_targets_value( + self, va_obj: ValidateAttribute + ) -> None: + """Tests for ValidateAttribute._run_validation_across_targets_value""" + + validation_output = va_obj._run_validation_across_targets_value( + manifest_col=Series(["A", "B", "C"]), + concatenated_target_column=Series(["A", "B", "C"]), + ) + assert validation_output[0].empty + assert validation_output[1].empty + assert validation_output[2].to_list() == ["A", "B", "C"] + + validation_output = va_obj._run_validation_across_targets_value( + manifest_col=Series(["C"]), + concatenated_target_column=Series(["A", "B", "B", "C", "C"]), + ) + assert validation_output[0].empty + assert validation_output[1].to_list() == ["C"] + assert validation_output[2].to_list() == ["C"] + + validation_output = va_obj._run_validation_across_targets_value( + manifest_col=Series(["A", "B", "C"]), + concatenated_target_column=Series(["A"]), + ) + assert validation_output[0].to_list() == ["B", "C"] + assert validation_output[1].empty + assert validation_output[2].to_list() == ["A"] + + def test__gather_value_warnings_errors_passing( + self, va_obj: ValidateAttribute + ) -> None: + """Tests for ValidateAttribute._gather_value_warnings_errors""" + errors, warnings = va_obj._gather_value_warnings_errors( + val_rule="matchAtLeastOne Patient.PatientID value error", + source_attribute="PatientID", + value_validation_store=(Series(), Series(["A", "B", "C"]), Series()), + ) + assert len(warnings) == 0 + assert len(errors) == 0 + + errors, warnings = va_obj._gather_value_warnings_errors( + val_rule="matchAtLeastOne Patient.PatientID value error", + source_attribute="PatientID", + value_validation_store=( + Series(), + Series(["A", "B", "C"]), + Series(["A", "B", "C"]), + ), + ) + assert len(warnings) == 0 + assert len(errors) == 0 + + errors, warnings = va_obj._gather_value_warnings_errors( + val_rule="matchAtLeastOne comp.att value error", + source_attribute="att", + value_validation_store=(Series(), Series(), Series()), + ) + assert len(errors) == 0 + assert len(warnings) == 0 + + def test__gather_value_warnings_errors_with_errors( + self, va_obj: ValidateAttribute + ) -> None: + """Tests for ValidateAttribute._gather_value_warnings_errors""" + + errors, warnings = va_obj._gather_value_warnings_errors( + val_rule="matchAtLeastOne Patient.PatientID value error", + source_attribute="PatientID", + value_validation_store=(Series(["A", "B", "C"]), Series(), Series()), + ) + assert len(warnings) == 0 + assert len(errors) == 1 + assert len(errors[0]) == 4 + assert errors[0][1] == "PatientID" + assert errors[0][2] == ( + "Value(s) ['A', 'B', 'C'] from row(s) ['2', '3', '4'] of the attribute " + "PatientID in the source manifest are missing." + ) + + def test__run_validation_across_targets_set( + self, + va_obj: ValidateAttribute, + cross_val_col_names: dict[str, str], + cross_val_df1: DataFrame, + ) -> None: + """Tests for ValidateAttribute._run_validation_across_targets_set for matchAtLeastOne""" + + output, bool_list1, bool_list2 = va_obj._run_validation_across_targets_set( + val_rule="matchAtleastOne, Patient.PatientID, set", + column_names=cross_val_col_names, + manifest_col=Series(["A", "B", "C"]), + target_attribute="patientid", + target_manifest=cross_val_df1, + target_manifest_id="syn1", + missing_manifest_log={}, + present_manifest_log=[], + repeat_manifest_log={}, + target_attribute_in_manifest_list=[], + target_manifest_empty=[], + ) + assert output[0] == {} + assert output[1] == ["syn1"] + assert output[2] == {} + assert bool_list1 == [True] + assert bool_list2 == [False] + + output, bool_list1, bool_list2 = va_obj._run_validation_across_targets_set( + val_rule="matchAtleastOne, Patient.PatientID, set", + column_names=cross_val_col_names, + manifest_col=Series(["A", "B", "C"]), + target_attribute="patientid", + target_manifest=cross_val_df1, + target_manifest_id="syn2", + missing_manifest_log={}, + present_manifest_log=["syn1"], + repeat_manifest_log={}, + target_attribute_in_manifest_list=[], + target_manifest_empty=[], + ) + assert output[0] == {} + assert output[1] == ["syn1", "syn2"] + assert output[2] == {} + assert bool_list1 == [True] + assert bool_list2 == [False] + + output, bool_list1, bool_list2 = va_obj._run_validation_across_targets_set( + val_rule="matchAtleastOne, Patient.PatientID, set", + column_names=cross_val_col_names, + manifest_col=Series(["A", "B", "C", "D"]), + target_attribute="patientid", + target_manifest=cross_val_df1, + target_manifest_id="syn1", + missing_manifest_log={}, + present_manifest_log=[], + repeat_manifest_log={}, + target_attribute_in_manifest_list=[], + target_manifest_empty=[], + ) + assert list(output[0].keys()) == ["syn1"] + assert list(output[0].values())[0].to_list() == ["D"] + assert output[1] == [] + assert output[2] == {} + assert bool_list1 == [True] + assert bool_list2 == [False] + + output, bool_list1, bool_list2 = va_obj._run_validation_across_targets_set( + val_rule="matchAtleastOne, Patient.PatientID, set", + column_names=cross_val_col_names, + manifest_col=Series(["A", "B", "C", "E"]), + target_attribute="patientid", + target_manifest=cross_val_df1, + target_manifest_id="syn2", + missing_manifest_log={"syn1": Series(["D"])}, + present_manifest_log=[], + repeat_manifest_log={}, + target_attribute_in_manifest_list=[], + target_manifest_empty=[], + ) + assert list(output[0].keys()) == ["syn1", "syn2"] + assert output[0]["syn1"].to_list() == ["D"] + assert output[0]["syn2"].to_list() == ["E"] + assert output[1] == [] + assert output[2] == {} + assert bool_list1 == [True] + assert bool_list2 == [False] + + output, bool_list1, bool_list2 = va_obj._run_validation_across_targets_set( + val_rule="matchNone, Patient.PatientID, set", + column_names=cross_val_col_names, + manifest_col=Series(["A", "B", "C"]), + target_attribute="patientid", + target_manifest=cross_val_df1, + target_manifest_id="syn1", + missing_manifest_log={}, + present_manifest_log=[], + repeat_manifest_log={}, + target_attribute_in_manifest_list=[], + target_manifest_empty=[], + ) + assert output[0] == {} + assert output[1] == [] + assert list(output[2].keys()) == ["syn1"] + assert output[2]["syn1"].to_list() == ["A", "B", "C"] + assert bool_list1 == [True] + assert bool_list2 == [False] + + output, bool_list1, bool_list2 = va_obj._run_validation_across_targets_set( + val_rule="matchNone, Patient.PatientID, set", + column_names=cross_val_col_names, + manifest_col=Series(["A"]), + target_attribute="patientid", + target_manifest=cross_val_df1, + target_manifest_id="syn2", + missing_manifest_log={}, + present_manifest_log=[], + repeat_manifest_log={"syn1": Series(["A", "B", "C"])}, + target_attribute_in_manifest_list=[], + target_manifest_empty=[], + ) + assert output[0] == {} + assert output[1] == [] + assert list(output[2].keys()) == ["syn1", "syn2"] + assert output[2]["syn1"].to_list() == ["A", "B", "C"] + assert output[2]["syn2"].to_list() == ["A"] + assert bool_list1 == [True] + assert bool_list2 == [False] + + def test__gather_set_warnings_errors_match_atleast_one_passes( + self, va_obj: ValidateAttribute + ) -> None: + """Tests for ValidateAttribute._gather_set_warnings_errors for matchAtLeastOne""" + + errors, warnings = va_obj._gather_set_warnings_errors( + val_rule="matchAtLeastOne Patient.PatientID set error", + source_attribute="PatientID", + set_validation_store=({}, [], {}), + ) + assert len(warnings) == 0 + assert len(errors) == 0 + + errors, warnings = va_obj._gather_set_warnings_errors( + val_rule="matchAtLeastOne Patient.PatientID set error", + source_attribute="PatientID", + set_validation_store=({}, ["syn1"], {}), + ) + assert len(warnings) == 0 + assert len(errors) == 0 + + errors, warnings = va_obj._gather_set_warnings_errors( + val_rule="matchAtLeastOne Patient.PatientID set error", + source_attribute="PatientID", + set_validation_store=({}, ["syn1", "syn2"], {}), + ) + assert len(warnings) == 0 + assert len(errors) == 0 + + errors, warnings = va_obj._gather_set_warnings_errors( + val_rule="matchAtLeastOne Patient.PatientID set error", + source_attribute="PatientID", + set_validation_store=( + {"syn1": Series(["A"])}, + ["syn1"], + {"syn2": Series(["B"])}, + ), + ) + assert len(warnings) == 0 + assert len(errors) == 0 + + def test__gather_set_warnings_errors_match_atleast_one_errors( + self, va_obj: ValidateAttribute + ) -> None: + """Tests for ValidateAttribute._gather_set_warnings_errors for matchAtLeastOne""" + + errors, warnings = va_obj._gather_set_warnings_errors( + val_rule="matchAtLeastOne Patient.PatientID set error", + source_attribute="PatientID", + set_validation_store=({"syn1": Series(["A"])}, [], {}), + ) + assert len(warnings) == 0 + assert len(errors) == 1 + assert errors[0][0] == ["2"] + assert errors[0][1] == "PatientID" + assert errors[0][2] == ( + "Value(s) ['A'] from row(s) ['2'] of the attribute PatientID in the source " + "manifest are missing. Manifest(s) ['syn1'] are missing the value(s)." + ) + assert errors[0][3] == ["A"] + + def test__gather_set_warnings_errors_match_exactly_one_passes( + self, va_obj: ValidateAttribute + ) -> None: + """Tests for ValidateAttribute._gather_set_warnings_errors for matchExactlyOne""" + + errors, warnings = va_obj._gather_set_warnings_errors( + val_rule="matchExactlyOne Patient.PatientID set error", + source_attribute="PatientID", + set_validation_store=({}, [], {}), + ) + assert len(warnings) == 0 + assert len(errors) == 0 + + errors, warnings = va_obj._gather_set_warnings_errors( + val_rule="matchExactlyOne Patient.PatientID set error", + source_attribute="PatientID", + set_validation_store=({}, ["syn1"], {}), + ) + assert len(warnings) == 0 + assert len(errors) == 0 + + errors, warnings = va_obj._gather_set_warnings_errors( + val_rule="matchExactlyOne Patient.PatientID set error", + source_attribute="PatientID", + set_validation_store=( + {"syn1": Series(["A"])}, + ["syn1"], + {"syn2": Series(["B"])}, + ), + ) + assert len(warnings) == 0 + assert len(errors) == 0 + + def test__gather_set_warnings_errors_match_exactly_one_errors( + self, va_obj: ValidateAttribute + ) -> None: + """Tests for ValidateAttribute._gather_set_warnings_errors for matchExactlyOne""" + + errors, warnings = va_obj._gather_set_warnings_errors( + val_rule="matchExactlyOne Patient.PatientID set error", + source_attribute="PatientID", + set_validation_store=({}, ["syn1", "syn2"], {}), + ) + assert len(warnings) == 0 + assert len(errors) == 1 + assert not errors[0][0] + assert errors[0][1] == "PatientID" + assert errors[0][2] == ( + "All values from attribute PatientID in the source manifest are present in 2 " + "manifests instead of only 1. Manifests ['syn1', 'syn2'] match the values in " + "the source attribute." + ) + assert not errors[0][3] + + def test__gather_set_warnings_errors_match_none_passes( + self, va_obj: ValidateAttribute + ) -> None: + """Tests for ValidateAttribute._gather_set_warnings_errors for matchNone""" + + errors, warnings = va_obj._gather_set_warnings_errors( + val_rule="matchNone Patient.PatientID set error", + source_attribute="PatientID", + set_validation_store=({}, [], {}), + ) + assert len(warnings) == 0 + assert len(errors) == 0 + + errors, warnings = va_obj._gather_set_warnings_errors( + val_rule="matchNone Patient.PatientID set error", + source_attribute="PatientID", + set_validation_store=({"syn1": Series(["A"])}, ["syn1"], {}), + ) + assert len(warnings) == 0 + assert len(errors) == 0 + + def test__gather_set_warnings_errors_match_none_errors( + self, va_obj: ValidateAttribute + ) -> None: + """Tests for ValidateAttribute._gather_set_warnings_errors for matchNone""" + + errors, warnings = va_obj._gather_set_warnings_errors( + val_rule="matchNone Patient.PatientID set error", + source_attribute="PatientID", + set_validation_store=({}, [], {"syn1": Series(["A"])}), + ) + assert len(warnings) == 0 + assert len(errors) == 1 + assert errors[0][0] == ["2"] + assert errors[0][1] == "PatientID" + assert errors[0][2] == ( + "Value(s) ['A'] from row(s) ['2'] for the attribute PatientID " + "in the source manifest are not unique. " + "Manifest(s) ['syn1'] contain duplicate values." + ) + assert errors[0][3] == ["A"] + + errors, warnings = va_obj._gather_set_warnings_errors( + val_rule="matchNone Patient.PatientID set error", + source_attribute="PatientID", + set_validation_store=( + {}, + [], + {"syn1": Series(["A"]), "syn2": Series(["B"])}, + ), + ) + assert len(warnings) == 0 + assert len(errors) == 1 + assert errors[0][0] == ["2"] + assert errors[0][1] == "PatientID" + possible_errors = [ + ( + "Value(s) ['A', 'B'] from row(s) ['2'] for the attribute PatientID in the source " + "manifest are not unique. Manifest(s) ['syn1', 'syn2'] contain duplicate values." + ), + ( + "Value(s) ['B', 'A'] from row(s) ['2'] for the attribute PatientID in the source " + "manifest are not unique. Manifest(s) ['syn1', 'syn2'] contain duplicate values." + ), + ] + assert errors[0][2] in possible_errors + possible_missing_values = [["A", "B"], ["B", "A"]] + assert errors[0][3] in possible_missing_values + + def test__get_column_names(self, va_obj: ValidateAttribute) -> None: + """Tests for ValidateAttribute._get_column_names""" + assert not va_obj._get_column_names(DataFrame()) + assert va_obj._get_column_names(DataFrame({"col1": []})) == {"col1": "col1"} + assert va_obj._get_column_names(DataFrame({"col1": [], "col2": []})) == { + "col1": "col1", + "col2": "col2", + } + assert va_obj._get_column_names(DataFrame({"COL 1": []})) == {"col1": "COL 1"} + assert va_obj._get_column_names(DataFrame({"ColId": []})) == {"colid": "ColId"} + assert va_obj._get_column_names(DataFrame({"ColID": []})) == {"colid": "ColID"} From 206a348aa8699b5db6b16207fe84552ad17fc511 Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Fri, 30 Aug 2024 13:53:11 -0700 Subject: [PATCH 154/233] added init files to test folders --- tests/integration/__init__.py | 0 tests/unit/__init__.py | 0 2 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 tests/integration/__init__.py create mode 100644 tests/unit/__init__.py diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/unit/__init__.py b/tests/unit/__init__.py new file mode 100644 index 000000000..e69de29bb From e87a9d30fb827a564b97c73e1c0ddcf8db4bced1 Mon Sep 17 00:00:00 2001 From: linglp Date: Fri, 30 Aug 2024 20:26:20 -0400 Subject: [PATCH 155/233] fix typing --- schematic/schemas/data_model_graph.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/schematic/schemas/data_model_graph.py b/schematic/schemas/data_model_graph.py index e78875a81..d649e6306 100644 --- a/schematic/schemas/data_model_graph.py +++ b/schematic/schemas/data_model_graph.py @@ -772,7 +772,7 @@ def get_node_required( def get_node_validation_rules( self, node_label: Optional[str] = None, node_display_name: Optional[str] = None - ) -> Union[list[str], dict[str, str]]: + ) -> Union[list, dict[str, str]]: """Get validation rules associated with a node, Args: From f517062abede474d44dba919afa962986dc14c9a Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Tue, 3 Sep 2024 08:22:52 -0700 Subject: [PATCH 156/233] paramaterized tests --- schematic/models/validate_attribute.py | 1 + tests/unit/test_validate_attribute.py | 1605 ++++++++++++++---------- 2 files changed, 910 insertions(+), 696 deletions(-) diff --git a/schematic/models/validate_attribute.py b/schematic/models/validate_attribute.py index f5444ad08..8ff1cf0ea 100644 --- a/schematic/models/validate_attribute.py +++ b/schematic/models/validate_attribute.py @@ -1952,6 +1952,7 @@ def _run_validation_across_target_manifests( concatenated_target_column=target_column, ) validation_store = (missing_values, duplicated_values, repeat_values) + return (start_time, validation_store) def cross_validation( diff --git a/tests/unit/test_validate_attribute.py b/tests/unit/test_validate_attribute.py index 009ecc5cb..26ab9f167 100644 --- a/tests/unit/test_validate_attribute.py +++ b/tests/unit/test_validate_attribute.py @@ -2,15 +2,110 @@ from typing import Generator from unittest.mock import patch - import pytest + from pandas import Series, DataFrame, concat +import numpy as np from schematic.models.validate_attribute import ValidateAttribute from schematic.schemas.data_model_graph import DataModelGraphExplorer import schematic.models.validate_attribute # pylint: disable=protected-access +# pylint: disable=too-many-public-methods +# pylint: disable=too-many-arguments + +MATCH_ATLEAST_ONE_SET_RULES = [ + "matchAtLeastOne Patient.PatientID set error", + "matchAtLeastOne Patient.PatientID set warning", +] +MATCH_EXACTLY_ONE_SET_RULES = [ + "matchExactlyOne Patient.PatientID set error", + "matchExactlyOne Patient.PatientID set warning", +] +MATCH_NONE_SET_RULES = [ + "matchNone Patient.PatientID set error", + "matchNone Patient.PatientID set warning", +] +ALL_SET_RULES = ( + MATCH_ATLEAST_ONE_SET_RULES + MATCH_EXACTLY_ONE_SET_RULES + MATCH_NONE_SET_RULES +) +MATCH_ATLEAST_ONE_VALUE_RULES = [ + "matchAtLeastOne Patient.PatientID value error", + "matchAtLeastOne Patient.PatientID value warning", +] +MATCH_EXACTLY_ONE_VALUE_RULES = [ + "matchExactlyOne Patient.PatientID value error", + "matchExactlyOne Patient.PatientID value warning", +] +MATCH_NONE_VALUE_RULES = [ + "matchNone Patient.PatientID value error", + "matchNone Patient.PatientID value warning", +] +ALL_VALUE_RULES = ( + MATCH_ATLEAST_ONE_VALUE_RULES + + MATCH_EXACTLY_ONE_VALUE_RULES + + MATCH_NONE_VALUE_RULES +) +EXACTLY_ATLEAST_PASSING_SERIES = [ + Series(["A", "B", "C"], index=[0, 1, 2], name="PatientID"), + Series(["A", "B", "C", "C"], index=[0, 1, 2, 3], name="PatientID"), + Series(["A", "B", "C", "A", "B", "C"], index=[0, 1, 2, 3, 4, 5], name="PatientID"), + Series(["A", "B"], index=[0, 1], name="PatientID"), + Series([], name="PatientID"), +] + +TEST_DF1 = DataFrame( + { + "PatientID": ["A", "B", "C"], + "component": ["comp1", "comp1", "comp1"], + "id": ["id1", "id2", "id3"], + "entityid": ["x", "x", "x"], + } +) + +TEST_DF2 = DataFrame( + { + "PatientID": ["D", "E", "F"], + "component": ["comp1", "comp1", "comp1"], + "id": ["id1", "id2", "id3"], + "entityid": ["x", "x", "x"], + } +) + +TEST_DF_MISSING_VALS = DataFrame( + { + "PatientID": [np.isnan, ""], + "component": ["comp1", "comp1"], + "id": ["id1", "id2"], + "entityid": ["x", "x"], + } +) + +TEST_DF_MISSING_PATIENT = DataFrame( + { + "component": ["comp1", "comp1"], + "id": ["id1", "id2"], + "entityid": ["x", "x"], + } +) + +TEST_DF_EMPTY_COLS = DataFrame( + { + "PatientID": [], + "component": [], + "id": [], + "entityid": [], + } +) + + +@pytest.fixture(name="va_obj") +def fixture_va_obj( + dmge: DataModelGraphExplorer, +) -> Generator[ValidateAttribute, None, None]: + """Yield a ValidateAttribute object""" + yield ValidateAttribute(dmge) @pytest.fixture(name="cross_val_df1") @@ -76,398 +171,427 @@ def fixture_cross_val_col_names() -> Generator[dict[str, str], None, None]: yield column_names -@pytest.fixture(name="va_obj") -def fixture_va_obj( - dmge: DataModelGraphExplorer, -) -> Generator[ValidateAttribute, None, None]: - """Yield a ValidateAttribute object""" - yield ValidateAttribute(dmge) - - class TestValidateAttributeObject: """Testing for ValidateAttribute class with all Synapse calls mocked""" - def test_cross_validation_match_atleast_one_set_rules_passing( - self, va_obj: ValidateAttribute, cross_val_df1: DataFrame - ): - """Tests for cross manifest validation for matchAtLeastOne set rules""" - val_rule = "matchAtLeastOne Patient.PatientID set error" - - with patch.object( - schematic.models.validate_attribute.ValidateAttribute, - "_get_target_manifest_dataframes", - return_value={"syn1": cross_val_df1}, - ): - assert va_obj.cross_validation(val_rule, Series(["A", "B", "C"])) == ( - [], - [], - ) - assert va_obj.cross_validation(val_rule, Series(["A", "B", "C", "C"])) == ( - [], - [], - ) - assert va_obj.cross_validation( - val_rule, Series(["A", "B", "C", "A", "B", "C"]) - ) == ( - [], - [], - ) - assert va_obj.cross_validation( - val_rule, Series(["A", "B"], index=[0, 1], name="PatientID") - ) == ( - [], - [], - ) - assert va_obj.cross_validation( - val_rule, Series([], index=[], name="PatientID") - ) == ( - [], - [], - ) + ################## + # cross_validation + ################## - def test_cross_validation_match_atleast_one_set_rules_errors( - self, va_obj: ValidateAttribute, cross_val_df1: DataFrame + @pytest.mark.parametrize("series", EXACTLY_ATLEAST_PASSING_SERIES) + @pytest.mark.parametrize("rule", MATCH_ATLEAST_ONE_SET_RULES) + def test_cross_validation_match_atleast_one_set_rules_passing( + self, + va_obj: ValidateAttribute, + cross_val_df1: DataFrame, + series: Series, + rule: str, ): - """Tests for cross manifest validation for matchAtLeastOne set rules""" - val_rule = "matchAtLeastOne Patient.PatientID set error" - + """ + Tests for ValidateAttribute.cross_validation using matchAtLeastOne rule + These tests pass with no errors or warnings + """ with patch.object( schematic.models.validate_attribute.ValidateAttribute, "_get_target_manifest_dataframes", return_value={"syn1": cross_val_df1}, ): - errors, warnings = va_obj.cross_validation( - val_rule, - Series(["A", "B", "C", "D"], index=[0, 1, 2, 3], name="PatientID"), - ) - assert len(warnings) == 0 - assert len(errors) == 1 - - errors, warnings = va_obj.cross_validation( - val_rule, Series([""], index=[0], name="PatientID") - ) - assert len(warnings) == 0 - assert len(errors) == 1 + assert va_obj.cross_validation(rule, series) == ([], []) - def test_cross_validation_match_atleast_one_set_rules_warnings( - self, va_obj: ValidateAttribute, cross_val_df1: DataFrame + @pytest.mark.parametrize("series", EXACTLY_ATLEAST_PASSING_SERIES) + @pytest.mark.parametrize("rule", MATCH_EXACTLY_ONE_SET_RULES) + def test_cross_validation_match_exactly_one_set_rules_passing( + self, + va_obj: ValidateAttribute, + cross_val_df1: DataFrame, + series: Series, + rule: str, ): - """Tests for cross manifest validation for matchAtLeastOne set rules""" - val_rule = "matchAtLeastOne Patient.PatientID set warning" - + """ + Tests for ValidateAttribute.cross_validation using matchExactlyOne rule + These tests pass with no errors or warnings + """ with patch.object( schematic.models.validate_attribute.ValidateAttribute, "_get_target_manifest_dataframes", return_value={"syn1": cross_val_df1}, ): - errors, warnings = va_obj.cross_validation( - val_rule, - Series(["A", "B", "C", "D"], index=[0, 1, 2, 3], name="PatientID"), - ) - assert len(warnings) == 1 - assert len(errors) == 0 + assert va_obj.cross_validation(rule, series) == ([], []) - def test_cross_validation_match_exactly_one_set_rules_passing( - self, va_obj: ValidateAttribute, cross_val_df1: DataFrame + @pytest.mark.parametrize( + "series", + [ + Series(["A", "B", "C", "D"], index=[0, 1, 2, 3], name="PatientID"), + Series([np.nan], index=[0], name="PatientID"), + Series([""], index=[0], name="PatientID"), + Series([1], index=[0], name="PatientID"), + ], + ) + @pytest.mark.parametrize("rule", MATCH_ATLEAST_ONE_SET_RULES) + def test_cross_validation_match_atleast_one_set_rules_errors( + self, + va_obj: ValidateAttribute, + cross_val_df1: DataFrame, + series: Series, + rule: str, ): - """Tests for cross manifest validation for matchExactlyOne set rules""" - val_rule = "matchExactlyOne Patient.PatientID set error" - + """ + Tests for ValidateAttribute.cross_validation using matchAtLeastOne rule + These tests fail with either one error or warning depending on the rule + """ with patch.object( schematic.models.validate_attribute.ValidateAttribute, "_get_target_manifest_dataframes", return_value={"syn1": cross_val_df1}, ): - assert va_obj.cross_validation(val_rule, Series(["A", "B", "C"])) == ( - [], - [], - ) - assert va_obj.cross_validation(val_rule, Series(["A", "B", "C", "C"])) == ( - [], - [], - ) - - assert va_obj.cross_validation(val_rule, Series(["A", "B"])) == ([], []) - - assert va_obj.cross_validation(val_rule, Series(["A", "B", "C", "D"])) == ( - [], - [], - ) - + errors, warnings = va_obj.cross_validation(rule, series) + if rule.endswith("error"): + assert warnings == [] + assert len(errors) == 1 + else: + assert len(warnings) == 1 + assert errors == [] + + @pytest.mark.parametrize( + "series", + [ + Series(["A", "B", "C"], index=[0, 1, 2], name="PatientID"), + Series(["A", "B", "C", "C"], index=[0, 1, 2, 3], name="PatientID"), + ], + ) + @pytest.mark.parametrize("rule", MATCH_EXACTLY_ONE_SET_RULES) def test_cross_validation_match_exactly_one_set_rules_errors( - self, va_obj: ValidateAttribute, cross_val_df1: DataFrame + self, + va_obj: ValidateAttribute, + cross_val_df1: DataFrame, + series: Series, + rule: str, ): - """Tests for cross manifest validation for matchExactlyOne set rules""" - val_rule = "matchExactlyOne Patient.PatientID set error" - + """ + Tests for ValidateAttribute.cross_validation using matchExactlyOne rule + These tests fail with either one error or warning depending on the rule + """ with patch.object( schematic.models.validate_attribute.ValidateAttribute, "_get_target_manifest_dataframes", return_value={"syn1": cross_val_df1, "syn2": cross_val_df1}, ): - errors, _ = va_obj.cross_validation( - val_rule, Series(["A", "B", "C"], index=[0, 1, 2], name="PatientID") - ) - assert len(errors) == 1 - + errors, warnings = va_obj.cross_validation(rule, series) + if rule.endswith("error"): + assert warnings == [] + assert len(errors) == 1 + else: + assert len(warnings) == 1 + assert errors == [] + + @pytest.mark.parametrize( + "series", + [ + Series(["D"], index=[0], name="PatientID"), + Series(["D", "D"], index=[0, 1], name="PatientID"), + Series([np.nan], index=[0], name="PatientID"), + Series([""], index=[0], name="PatientID"), + Series([1], index=[0], name="PatientID"), + ], + ) + @pytest.mark.parametrize("rule", MATCH_NONE_SET_RULES) def test_cross_validation_match_none_set_rules_passing( - self, va_obj: ValidateAttribute, cross_val_df1: DataFrame + self, + va_obj: ValidateAttribute, + cross_val_df1: DataFrame, + series: Series, + rule: str, ): - """Tests for cross manifest validation for matchNone set rules""" - val_rule = "matchNone Patient.PatientID set error" - + """ + Tests for cross manifest validation for matchNone set rules + These tests pass with no errors or warnings + """ with patch.object( schematic.models.validate_attribute.ValidateAttribute, "_get_target_manifest_dataframes", return_value={"syn1": cross_val_df1}, ): - assert va_obj.cross_validation(val_rule, Series(["D"])) == ( - [], - [], - ) + assert va_obj.cross_validation(rule, series) == ([], []) + @pytest.mark.parametrize( + "series", + [ + Series(["A", "B", "C"], index=[0, 1, 2], name="PatientID"), + Series(["A", "B", "C", "D"], index=[0, 1, 2, 3], name="PatientID"), + Series(["A", "B"], index=[0, 1], name="PatientID"), + Series(["A"], index=[0], name="PatientID"), + ], + ) + @pytest.mark.parametrize("rule", MATCH_NONE_SET_RULES) def test_cross_validation_match_none_set_rules_errors( - self, va_obj: ValidateAttribute, cross_val_df1: DataFrame + self, + va_obj: ValidateAttribute, + cross_val_df1: DataFrame, + series: Series, + rule: str, ): - """Tests for cross manifest validation for matchNone set rules""" - val_rule = "matchNone Patient.PatientID set error" - + """ + Tests for cross manifest validation for matchNone set rules + These tests fail with either one error or warning depending on the rule + """ with patch.object( schematic.models.validate_attribute.ValidateAttribute, "_get_target_manifest_dataframes", return_value={"syn1": cross_val_df1}, ): - errors, _ = va_obj.cross_validation( - val_rule, - Series(["A", "B", "C"], index=[0, 1, 2], name="PatientID"), - ) - assert len(errors) == 1 - - errors, _ = va_obj.cross_validation( - val_rule, - Series(["A", "B", "C", "C"], index=[0, 1, 2, 3], name="PatientID"), - ) - assert len(errors) == 1 - - errors, _ = va_obj.cross_validation( - val_rule, Series(["A", "B"], index=[0, 1], name="PatientID") - ) - assert len(errors) == 1 - + errors, warnings = va_obj.cross_validation(rule, series) + if rule.endswith("error"): + assert warnings == [] + assert len(errors) == 1 + else: + assert len(warnings) == 1 + assert errors == [] + + @pytest.mark.parametrize("rule", MATCH_ATLEAST_ONE_VALUE_RULES) + @pytest.mark.parametrize( + "tested_column", + [ + ([]), + (["A"]), + (["A", "A"]), + (["A", "B"]), + (["A", "B", "C"]), + (["A", "B", "C", "C"]), + ], + ) def test_cross_validation_value_match_atleast_one_rules_passing( self, va_obj: ValidateAttribute, cross_val_df1: DataFrame, - cross_val_df2: DataFrame, + rule: str, + tested_column: list, ): - """Tests for cross manifest validation for matchAtLeastOne value rules""" - val_rule = "matchAtLeastOne Patient.PatientID value error" - + """ + Tests ValidateAttribute.cross_validation + These tests show what columns pass for matchAtLeastOne + """ with patch.object( schematic.models.validate_attribute.ValidateAttribute, "_get_target_manifest_dataframes", return_value={"syn1": cross_val_df1}, ): - assert va_obj.cross_validation(val_rule, Series([])) == ([], []) - assert va_obj.cross_validation(val_rule, Series(["A"])) == ([], []) - assert va_obj.cross_validation(val_rule, Series(["A", "A"])) == ([], []) - assert va_obj.cross_validation(val_rule, Series(["A", "B"])) == ([], []) - assert va_obj.cross_validation(val_rule, Series(["A", "B", "C"])) == ( - [], - [], - ) - assert va_obj.cross_validation(val_rule, Series(["A", "B", "C", "C"])) == ( - [], - [], - ) - - with patch.object( - schematic.models.validate_attribute.ValidateAttribute, - "_get_target_manifest_dataframes", - return_value={"syn1": cross_val_df1, "syn2": cross_val_df2}, - ): - assert va_obj.cross_validation(val_rule, Series(["A", "B", "C", "D"])) == ( - [], - [], - ) + assert va_obj.cross_validation(rule, Series(tested_column)) == ([], []) + @pytest.mark.parametrize("rule", MATCH_ATLEAST_ONE_VALUE_RULES) + @pytest.mark.parametrize( + "tested_column", + [ + Series(["D"], index=[0], name="PatientID"), + Series(["D", "D"], index=[0, 1], name="PatientID"), + Series(["D", "F"], index=[0, 1], name="PatientID"), + Series([np.nan], index=[0], name="PatientID"), + Series([1], index=[0], name="PatientID"), + ], + ) def test_cross_validation_value_match_atleast_one_rules_errors( - self, va_obj: ValidateAttribute, cross_val_df1: DataFrame + self, + va_obj: ValidateAttribute, + cross_val_df1: DataFrame, + rule: str, + tested_column: Series, ): - """Tests for cross manifest validation for matchAtLeastOne value rules""" - val_rule = "matchAtLeastOne Patient.PatientID value error" - + """ + Tests ValidateAttribute.cross_validation + These tests show what columns fail for matchAtLeastOne + """ with patch.object( schematic.models.validate_attribute.ValidateAttribute, "_get_target_manifest_dataframes", return_value={"syn1": cross_val_df1}, ): - errors, _ = va_obj.cross_validation( - val_rule, Series(["D"], index=[0], name="PatientID") - ) - assert len(errors) == 1 - + errors, warnings = va_obj.cross_validation(rule, tested_column) + if rule.endswith("error"): + assert len(errors) == 1 + assert warnings == [] + else: + assert errors == [] + assert len(warnings) == 1 + + @pytest.mark.parametrize("rule", MATCH_EXACTLY_ONE_VALUE_RULES) + @pytest.mark.parametrize( + "tested_column", + [ + ([]), + (["A"]), + (["A", "A"]), + (["A", "B"]), + (["A", "B", "C"]), + (["A", "B", "C", "C"]), + ], + ) def test_cross_validation_match_exactly_one_value_rules_passing( - self, va_obj: ValidateAttribute, cross_val_df1: DataFrame + self, + va_obj: ValidateAttribute, + cross_val_df1: DataFrame, + rule: str, + tested_column: list, ): - """Tests for cross manifest validation for matchExactlyOne value rules""" - val_rule = "matchExactlyOne Patient.PatientID value error" - + """ + Tests ValidateAttribute.cross_validation + These tests show what columns pass for matchExactlyOne + """ with patch.object( schematic.models.validate_attribute.ValidateAttribute, "_get_target_manifest_dataframes", return_value={"syn1": cross_val_df1}, ): - assert va_obj.cross_validation(val_rule, Series([])) == ([], []) - assert va_obj.cross_validation(val_rule, Series(["A"])) == ([], []) - assert va_obj.cross_validation(val_rule, Series(["A", "A"])) == ([], []) - assert va_obj.cross_validation(val_rule, Series(["A", "B"])) == ([], []) - assert va_obj.cross_validation(val_rule, Series(["A", "B", "C"])) == ( - [], - [], - ) - assert va_obj.cross_validation(val_rule, Series(["A", "B", "C", "C"])) == ( - [], - [], - ) - - with patch.object( - schematic.models.validate_attribute.ValidateAttribute, - "_get_target_manifest_dataframes", - return_value={"syn1": cross_val_df1, "syn2": cross_val_df1}, - ): - assert va_obj.cross_validation(val_rule, Series([])) == ([], []) + assert va_obj.cross_validation(rule, Series(tested_column)) == ([], []) - def test_cross_validation_match_exactly_one_value_rules_errors( - self, va_obj: ValidateAttribute, cross_val_df1: DataFrame + @pytest.mark.parametrize("rule", MATCH_EXACTLY_ONE_VALUE_RULES) + @pytest.mark.parametrize( + "tested_column", + [ + Series(["D"], index=[0], name="PatientID"), + Series(["D", "D"], index=[0, 1], name="PatientID"), + Series(["D", "F"], index=[0, 1], name="PatientID"), + Series([1], index=[0], name="PatientID"), + ], + ) + def test_cross_validation_value_match_exactly_one_rules_errors( + self, + va_obj: ValidateAttribute, + cross_val_df1: DataFrame, + rule: str, + tested_column: Series, ): - """Tests for cross manifest validation for matchExactlyOne value rules""" - val_rule = "matchExactlyOne Patient.PatientID value error" - + """ + Tests ValidateAttribute.cross_validation + These tests show what columns fail for matchExactlyOne + """ with patch.object( schematic.models.validate_attribute.ValidateAttribute, "_get_target_manifest_dataframes", return_value={"syn1": cross_val_df1}, ): - errors, _ = va_obj.cross_validation( - val_rule, Series(["D"], index=[0], name="PatientID") - ) - assert len(errors) == 1 - - with patch.object( - schematic.models.validate_attribute.ValidateAttribute, - "_get_target_manifest_dataframes", - return_value={"syn1": cross_val_df1, "syn2": cross_val_df1}, - ): - errors, _ = va_obj.cross_validation( - val_rule, Series(["A"], index=[0], name="PatientID") - ) - assert len(errors) == 1 - - errors, _ = va_obj.cross_validation( - val_rule, Series(["D"], index=[0], name="PatientID") - ) - assert len(errors) == 1 - + errors, warnings = va_obj.cross_validation(rule, tested_column) + if rule.endswith("error"): + assert len(errors) == 1 + assert warnings == [] + else: + assert errors == [] + assert len(warnings) == 1 + + @pytest.mark.parametrize("rule", MATCH_NONE_VALUE_RULES) + @pytest.mark.parametrize( + "tested_column", + [([]), (["D"]), (["D", "D"]), (["D", "F"]), ([1]), ([np.nan])], + ) def test_cross_validation_match_none_value_rules_passing( - self, va_obj: ValidateAttribute, cross_val_df1: DataFrame + self, + va_obj: ValidateAttribute, + cross_val_df1: DataFrame, + rule: str, + tested_column: list, ): - """Tests for cross manifest validation for matchNone value rules""" - val_rule = "matchNone Patient.PatientID value error" - + """ + Tests ValidateAttribute.cross_validation + These tests show what columns pass for matchNone + """ with patch.object( schematic.models.validate_attribute.ValidateAttribute, "_get_target_manifest_dataframes", return_value={"syn1": cross_val_df1}, ): - assert va_obj.cross_validation(val_rule, Series([])) == ([], []) - assert va_obj.cross_validation(val_rule, Series(["D"])) == ([], []) + assert va_obj.cross_validation(rule, Series(tested_column)) == ([], []) - def test_cross_validation_match_none_value_rules_errors( - self, va_obj: ValidateAttribute, cross_val_df1: DataFrame + @pytest.mark.parametrize("rule", MATCH_NONE_VALUE_RULES) + @pytest.mark.parametrize( + "tested_column", + [ + Series(["A"], index=[0], name="PatientID"), + Series(["A", "B"], index=[0, 1], name="PatientID"), + Series(["A", "A"], index=[0, 1], name="PatientID"), + ], + ) + def test_cross_validation_value_match_none_rules_errors( + self, + va_obj: ValidateAttribute, + cross_val_df1: DataFrame, + rule: str, + tested_column: Series, ): - """Tests for cross manifest validation for matchNone value rules""" - val_rule = "matchNone Patient.PatientID value error" - + """ + Tests ValidateAttribute.cross_validation + These tests show what columns fail for matchNone + """ with patch.object( schematic.models.validate_attribute.ValidateAttribute, "_get_target_manifest_dataframes", return_value={"syn1": cross_val_df1}, ): - errors, _ = va_obj.cross_validation( - val_rule, Series(["A"], index=[0], name="PatientID") - ) - assert len(errors) == 1 - - def test__run_validation_across_target_manifests_failures( + errors, warnings = va_obj.cross_validation(rule, tested_column) + if rule.endswith("error"): + assert len(errors) == 1 + assert warnings == [] + else: + assert errors == [] + assert len(warnings) == 1 + + ######################################### + # _run_validation_across_target_manifests + ######################################### + + @pytest.mark.parametrize("input_column", [(Series([])), (Series(["A"]))]) + @pytest.mark.parametrize("rule", ALL_SET_RULES) + @pytest.mark.parametrize( + "target_manifests", [({"syn1": TEST_DF_MISSING_PATIENT}), ({})] + ) + def test__run_validation_across_target_manifests_return_false( self, va_obj: ValidateAttribute, - cross_val_df1: DataFrame, - cross_val_df3: DataFrame, + input_column: Series, + rule: str, + target_manifests: dict[str, DataFrame], ) -> None: - """Tests for ValidateAttribute._run_validation_across_target_manifests with failures""" - # This shows that when no target manifests are found to check against, False is returned + """ + Tests for ValidateAttribute._run_validation_across_target_manifests that return False + These tests show that when no target manifests are found to check against, or the target + manifest is missing the target column, False is returned + """ with patch.object( schematic.models.validate_attribute.ValidateAttribute, "_get_target_manifest_dataframes", - return_value={}, + return_value=target_manifests, ): _, result = va_obj._run_validation_across_target_manifests( rule_scope="value", - val_rule="xxx Patient.PatientID xxx", - manifest_col=Series([]), + val_rule=rule, + manifest_col=input_column, target_column=Series([]), ) assert result is False - # This shows that when the only target manifest is empty, only a message is returned - with patch.object( - schematic.models.validate_attribute.ValidateAttribute, - "_get_target_manifest_dataframes", - return_value={"syn1": cross_val_df3}, - ): - _, result = va_obj._run_validation_across_target_manifests( - rule_scope="value", - val_rule="xxx Patient.PatientID xxx", - manifest_col=Series([]), - target_column=Series([]), - ) - - assert result == "values not recorded in targets stored" - - # This shows that when the only target manifest is empty, only a message is returned - # even if the tested column has values - with patch.object( - schematic.models.validate_attribute.ValidateAttribute, - "_get_target_manifest_dataframes", - return_value={"syn1": cross_val_df3}, - ): - _, result = va_obj._run_validation_across_target_manifests( - rule_scope="value", - val_rule="xxx Patient.PatientID xxx", - manifest_col=Series(["A", "B", "C"]), - target_column=Series([]), - ) - - assert result == "values not recorded in targets stored" + @pytest.mark.parametrize("input_column", [(Series([])), (Series(["A"]))]) + @pytest.mark.parametrize("rule", ALL_SET_RULES) + def test__run_validation_across_target_manifests_return_msg( + self, va_obj: ValidateAttribute, input_column: Series, rule: str + ) -> None: + """ + Tests for ValidateAttribute._run_validation_across_target_manifests that return a string + These tests show that if at least one target manifest does'nt have - # This shows that when any target manifest is empty, only a message is returned + """ with patch.object( schematic.models.validate_attribute.ValidateAttribute, "_get_target_manifest_dataframes", - return_value={"syn1": cross_val_df1, "syn2": cross_val_df3}, + return_value={"syn1": TEST_DF1, "syn2": TEST_DF_EMPTY_COLS}, ): _, result = va_obj._run_validation_across_target_manifests( rule_scope="value", - val_rule="xxx Patient.PatientID xxx", - manifest_col=Series([]), + val_rule=rule, + manifest_col=input_column, target_column=Series([]), ) - assert result == "values not recorded in targets stored" - def test__run_validation_across_target_manifests_value_rules( - self, va_obj: ValidateAttribute, cross_val_df1: DataFrame + @pytest.mark.parametrize("rule", ALL_VALUE_RULES) + def test__run_validation_across_target_manifests_value_scope( + self, va_obj: ValidateAttribute, cross_val_df1: DataFrame, rule: str ) -> None: """Tests for ValidateAttribute._run_validation_across_target_manifests with value rule""" @@ -479,7 +603,7 @@ def test__run_validation_across_target_manifests_value_rules( ): _, validation_output = va_obj._run_validation_across_target_manifests( rule_scope="value", - val_rule="xxx Patient.PatientID xxx", + val_rule=rule, manifest_col=Series([]), target_column=Series([]), ) @@ -491,17 +615,40 @@ def test__run_validation_across_target_manifests_value_rules( assert isinstance(validation_output[2], Series) assert validation_output[2].empty - def test__run_validation_across_target_manifests_set_rules_match_atleast_one( + @pytest.mark.parametrize( + "input_column, missing_ids, present_ids, repeat_ids", + [ + ([], [], ["syn1"], []), + (["A"], [], ["syn1"], []), + (["A", "A"], [], ["syn1"], []), + (["A", "B", "C"], [], ["syn1"], []), + (["D"], ["syn1"], [], []), + (["D", "D"], ["syn1"], [], []), + (["D", "E"], ["syn1"], [], []), + ([1], ["syn1"], [], []), + ], + ) + @pytest.mark.parametrize( + "rule", MATCH_ATLEAST_ONE_SET_RULES + MATCH_EXACTLY_ONE_SET_RULES + ) + def test__run_validation_across_target_manifests_match_atleast_exactly_with_one_target( self, va_obj: ValidateAttribute, cross_val_df1: DataFrame, + input_column: list, + missing_ids: list[str], + present_ids: list[str], + repeat_ids: list[str], + rule: str, ) -> None: """ Tests for ValidateAttribute._run_validation_across_target_manifests - with matchAtleastOne set rule + using matchAtleastOne set and matchExactlyOne rule. + This shows that these rules behave the same. + If all values in the column match the target manifest, the manifest id gets added + to the present ids list. + Otherwise the maniferst id gets added to the missing ids list """ - # These tests show when a column that is empty or partial/full match to - # the target manifest, the output only the present manifest log is updated with patch.object( schematic.models.validate_attribute.ValidateAttribute, "_get_target_manifest_dataframes", @@ -509,92 +656,90 @@ def test__run_validation_across_target_manifests_set_rules_match_atleast_one( ): _, validation_output = va_obj._run_validation_across_target_manifests( rule_scope="set", - val_rule="matchAtLeastOne Patient.PatientID set error", - manifest_col=Series([]), + val_rule=rule, + manifest_col=Series(input_column), target_column=Series([]), ) assert isinstance(validation_output, tuple) - assert validation_output[0] == {} - assert validation_output[1] == ["syn1"] - assert validation_output[2] == {} + assert list(validation_output[0].keys()) == missing_ids + assert validation_output[1] == present_ids + assert list(validation_output[2].keys()) == repeat_ids - _, validation_output = va_obj._run_validation_across_target_manifests( - rule_scope="set", - val_rule="matchAtLeastOne Patient.PatientID set error", - manifest_col=Series(["A", "B", "C"]), - target_column=Series([]), - ) - assert isinstance(validation_output, tuple) - assert validation_output[0] == {} - assert validation_output[1] == ["syn1"] - assert validation_output[2] == {} - - _, validation_output = va_obj._run_validation_across_target_manifests( - rule_scope="set", - val_rule="matchAtLeastOne Patient.PatientID set error", - manifest_col=Series(["A"]), - target_column=Series([]), - ) - - # This test shows that if there is an extra value ("D") in the column being validated - # it gets added to the missing_manifest_log - with patch.object( - schematic.models.validate_attribute.ValidateAttribute, - "_get_target_manifest_dataframes", - return_value={"syn1": cross_val_df1}, - ): - _, validation_output = va_obj._run_validation_across_target_manifests( - rule_scope="set", - val_rule="matchAtLeastOne Patient.PatientID set error", - manifest_col=Series(["A", "B", "C", "D"]), - target_column=Series([]), - ) - assert isinstance(validation_output, tuple) - assert list(validation_output[0].keys()) == ["syn1"] - assert validation_output[0]["syn1"].to_list() == ["D"] - assert validation_output[1] == [] - assert validation_output[2] == {} - - def test__run_validation_across_target_manifests_set_rules_match_none( + @pytest.mark.parametrize( + "input_column, missing_ids, present_ids, repeat_ids", + [ + ([], [], ["syn1", "syn2"], []), + (["A"], [], ["syn1", "syn2"], []), + (["D"], ["syn1", "syn2"], [], []), + ], + ) + @pytest.mark.parametrize( + "rule", MATCH_ATLEAST_ONE_SET_RULES + MATCH_EXACTLY_ONE_SET_RULES + ) + def test__run_validation_across_target_manifests_match_atleast_exactly_with_two_targets( self, va_obj: ValidateAttribute, cross_val_df1: DataFrame, + input_column: list, + missing_ids: list[str], + present_ids: list[str], + repeat_ids: list[str], + rule: str, ) -> None: """ Tests for ValidateAttribute._run_validation_across_target_manifests - with matchNone set rule + using matchAtleastOne set and matchExactlyOne rule. + This shows these rules behave the same. + This also shows that when thare are multiple target mnaifests they both get added to + either the present of missing manifest ids """ - - # This test shows nothing happens when the column is empty or has no shared values - # witht the target manifest with patch.object( schematic.models.validate_attribute.ValidateAttribute, "_get_target_manifest_dataframes", - return_value={"syn1": cross_val_df1}, + return_value={"syn1": cross_val_df1, "syn2": cross_val_df1}, ): _, validation_output = va_obj._run_validation_across_target_manifests( rule_scope="set", - val_rule="matchNone Patient.PatientID set error", - manifest_col=Series([]), + val_rule=rule, + manifest_col=Series(input_column), target_column=Series([]), ) assert isinstance(validation_output, tuple) - assert validation_output[0] == {} - assert validation_output[1] == [] - assert validation_output[2] == {} + assert list(validation_output[0].keys()) == missing_ids + assert validation_output[1] == present_ids + assert list(validation_output[2].keys()) == repeat_ids - _, validation_output = va_obj._run_validation_across_target_manifests( - rule_scope="set", - val_rule="matchNone Patient.PatientID set error", - manifest_col=Series(["D"]), - target_column=Series([]), - ) - assert isinstance(validation_output, tuple) - assert validation_output[0] == {} - assert validation_output[1] == [] - assert validation_output[2] == {} + @pytest.mark.parametrize( + "input_column, missing_ids, present_ids, repeat_ids", + [ + ([], [], [], []), + (["D"], [], [], []), + (["D", "D"], [], [], []), + (["D", "E"], [], [], []), + ([1], [], [], []), + (["A"], [], [], ["syn1"]), + (["A", "A"], [], [], ["syn1"]), + (["A", "B", "C"], [], [], ["syn1"]), + ], + ) + @pytest.mark.parametrize("rule", MATCH_NONE_SET_RULES) + def test__run_validation_across_target_manifests_set_rules_match_none_with_one_target( + self, + va_obj: ValidateAttribute, + cross_val_df1: DataFrame, + input_column: list, + missing_ids: list[str], + present_ids: list[str], + repeat_ids: list[str], + rule: str, + ) -> None: + """ + Tests for ValidateAttribute._run_validation_across_target_manifests + using matchNone set rule + When there are nt matching values, no id get added + When there are mathcing values the id gets added to the repeat ids + """ - # These tests when any values match they are put into the repeat log with patch.object( schematic.models.validate_attribute.ValidateAttribute, "_get_target_manifest_dataframes", @@ -602,280 +747,214 @@ def test__run_validation_across_target_manifests_set_rules_match_none( ): _, validation_output = va_obj._run_validation_across_target_manifests( rule_scope="set", - val_rule="matchNone Patient.PatientID set error", - manifest_col=Series(["A", "B", "C"]), + val_rule=rule, + manifest_col=Series(input_column), target_column=Series([]), ) assert isinstance(validation_output, tuple) - assert validation_output[0] == {} - assert validation_output[1] == [] - assert list(validation_output[2].keys()) == ["syn1"] - assert validation_output[2]["syn1"].to_list() == ["A", "B", "C"] + assert list(validation_output[0].keys()) == missing_ids + assert validation_output[1] == present_ids + assert list(validation_output[2].keys()) == repeat_ids - _, validation_output = va_obj._run_validation_across_target_manifests( - rule_scope="set", - val_rule="matchNone Patient.PatientID set error", - manifest_col=Series(["A"]), - target_column=Series([]), - ) - assert isinstance(validation_output, tuple) - assert validation_output[0] == {} - assert validation_output[1] == [] - assert list(validation_output[2].keys()) == ["syn1"] - assert validation_output[2]["syn1"].to_list() == ["A"] - - def test__run_validation_across_target_manifests_set_rules_exactly_one( + @pytest.mark.parametrize( + "input_column, missing_ids, present_ids, repeat_ids", + [ + ([], [], [], []), + (["D"], [], [], []), + (["D", "D"], [], [], []), + (["D", "E"], [], [], []), + ([1], [], [], []), + (["A"], [], [], ["syn1", "syn2"]), + (["A", "A"], [], [], ["syn1", "syn2"]), + (["A", "B", "C"], [], [], ["syn1", "syn2"]), + ], + ) + @pytest.mark.parametrize("rule", MATCH_NONE_SET_RULES) + def test__run_validation_across_target_manifests_set_rules_match_none_with_two_targets( self, va_obj: ValidateAttribute, cross_val_df1: DataFrame, + input_column: list, + missing_ids: list[str], + present_ids: list[str], + repeat_ids: list[str], + rule: str, ) -> None: """ - Tests for ValidateAttribute._run_validation_across_target_manifests with - matchExactlyOne set rule + Tests for ValidateAttribute._run_validation_across_target_manifests + using matchNone set rule + When there are nt matching values, no id get added + When there are mathcing values the id gets added to the repeat ids """ - # These tests show when an empty a partial match or full match column is used, - # the output only contains the targeted synapse id as a present value - with patch.object( - schematic.models.validate_attribute.ValidateAttribute, - "_get_target_manifest_dataframes", - return_value={"syn1": cross_val_df1}, - ): - _, validation_output = va_obj._run_validation_across_target_manifests( - rule_scope="set", - val_rule="matchExactlyOne Patient.PatientID set error", - manifest_col=Series([]), - target_column=Series([]), - ) - assert isinstance(validation_output, tuple) - assert validation_output[0] == {} - assert validation_output[1] == ["syn1"] - assert validation_output[2] == {} - _, validation_output = va_obj._run_validation_across_target_manifests( - rule_scope="set", - val_rule="matchExactlyOne Patient.PatientID set error", - manifest_col=Series(["A", "B", "C"]), - target_column=Series([]), - ) - assert isinstance(validation_output, tuple) - assert validation_output[0] == {} - assert validation_output[1] == ["syn1"] - assert validation_output[2] == {} - - _, validation_output = va_obj._run_validation_across_target_manifests( - rule_scope="set", - val_rule="matchExactlyOne Patient.PatientID set error", - manifest_col=Series(["A"]), - target_column=Series([]), - ) - assert isinstance(validation_output, tuple) - assert validation_output[0] == {} - assert validation_output[1] == ["syn1"] - assert validation_output[2] == {} - - # These tests shows that if there is an extra value ("D") in the column being validated - # it gets added to the missing manifest values dict with patch.object( schematic.models.validate_attribute.ValidateAttribute, "_get_target_manifest_dataframes", - return_value={"syn1": cross_val_df1}, + return_value={"syn1": cross_val_df1, "syn2": cross_val_df1}, ): _, validation_output = va_obj._run_validation_across_target_manifests( rule_scope="set", - val_rule="matchExactlyOne Patient.PatientID set error", - manifest_col=Series(["A", "B", "C", "D"]), + val_rule=rule, + manifest_col=Series(input_column), target_column=Series([]), ) assert isinstance(validation_output, tuple) - assert list(validation_output[0].keys()) == ["syn1"] - assert validation_output[0]["syn1"].to_list() == ["D"] - assert validation_output[1] == [] - assert validation_output[2] == {} + assert list(validation_output[0].keys()) == missing_ids + assert validation_output[1] == present_ids + assert list(validation_output[2].keys()) == repeat_ids - # This tests shows when a manifest macthes more than one manifest, both are added - # to the present manifest log - with patch.object( - schematic.models.validate_attribute.ValidateAttribute, - "_get_target_manifest_dataframes", - return_value={"syn1": cross_val_df1, "syn2": cross_val_df1}, - ): - _, validation_output = va_obj._run_validation_across_target_manifests( - rule_scope="set", - val_rule="matchExactlyOne Patient.PatientID set error", - manifest_col=Series([]), - target_column=Series(["A", "B", "C"]), - ) - assert isinstance(validation_output, tuple) - assert validation_output[0] == {} - assert validation_output[1] == ["syn1", "syn2"] - assert validation_output[2] == {} + ###################################### + # _run_validation_across_targets_value + ###################################### + @pytest.mark.parametrize( + "tested_column, target_column, missing, duplicated, repeat", + [ + (["A", "B", "C"], ["A", "B", "C"], [], [], ["A", "B", "C"]), + (["A", "B", "C", "C"], ["A", "B", "C"], [], [], ["A", "B", "C", "C"]), + (["A", "B"], ["A", "B", "C"], [], [], ["A", "B"]), + (["C"], ["C", "C"], [], ["C"], ["C"]), + (["C"], ["C", "C", "C"], [], ["C"], ["C"]), + (["A", "B", "C", "D"], ["A", "B", "C"], ["D"], [], ["A", "B", "C"]), + ( + ["A", "B", "C", "D", "D"], + ["A", "B", "C"], + ["D", "D"], + [], + ["A", "B", "C"], + ), + (["D"], ["A", "B", "C"], ["D"], [], []), + ], + ) def test__run_validation_across_targets_value( - self, va_obj: ValidateAttribute + self, + va_obj: ValidateAttribute, + tested_column: list, + target_column: list, + missing: list, + duplicated: list, + repeat: list, ) -> None: - """Tests for ValidateAttribute._run_validation_across_targets_value""" - - validation_output = va_obj._run_validation_across_targets_value( - manifest_col=Series(["A", "B", "C"]), - concatenated_target_column=Series(["A", "B", "C"]), - ) - assert validation_output[0].empty - assert validation_output[1].empty - assert validation_output[2].to_list() == ["A", "B", "C"] - - validation_output = va_obj._run_validation_across_targets_value( - manifest_col=Series(["C"]), - concatenated_target_column=Series(["A", "B", "B", "C", "C"]), - ) - assert validation_output[0].empty - assert validation_output[1].to_list() == ["C"] - assert validation_output[2].to_list() == ["C"] + """ + Tests for ValidateAttribute._run_validation_across_targets_value + These tests show: + To get repeat values, a value must appear in both the tested and target column + To get duplicated values, a value must appear more than once in the target column + To get missing values, a value must appear in the tested column, but not the target column + """ validation_output = va_obj._run_validation_across_targets_value( - manifest_col=Series(["A", "B", "C"]), - concatenated_target_column=Series(["A"]), + manifest_col=Series(tested_column), + concatenated_target_column=Series(target_column), ) - assert validation_output[0].to_list() == ["B", "C"] - assert validation_output[1].empty - assert validation_output[2].to_list() == ["A"] - - def test__gather_value_warnings_errors_passing( - self, va_obj: ValidateAttribute - ) -> None: - """Tests for ValidateAttribute._gather_value_warnings_errors""" - errors, warnings = va_obj._gather_value_warnings_errors( - val_rule="matchAtLeastOne Patient.PatientID value error", - source_attribute="PatientID", - value_validation_store=(Series(), Series(["A", "B", "C"]), Series()), - ) - assert len(warnings) == 0 - assert len(errors) == 0 - - errors, warnings = va_obj._gather_value_warnings_errors( - val_rule="matchAtLeastOne Patient.PatientID value error", - source_attribute="PatientID", - value_validation_store=( - Series(), - Series(["A", "B", "C"]), - Series(["A", "B", "C"]), - ), - ) - assert len(warnings) == 0 - assert len(errors) == 0 - - errors, warnings = va_obj._gather_value_warnings_errors( - val_rule="matchAtLeastOne comp.att value error", - source_attribute="att", - value_validation_store=(Series(), Series(), Series()), - ) - assert len(errors) == 0 - assert len(warnings) == 0 - - def test__gather_value_warnings_errors_with_errors( - self, va_obj: ValidateAttribute - ) -> None: - """Tests for ValidateAttribute._gather_value_warnings_errors""" + assert validation_output[0].to_list() == missing + assert validation_output[1].to_list() == duplicated + assert validation_output[2].to_list() == repeat - errors, warnings = va_obj._gather_value_warnings_errors( - val_rule="matchAtLeastOne Patient.PatientID value error", - source_attribute="PatientID", - value_validation_store=(Series(["A", "B", "C"]), Series(), Series()), - ) - assert len(warnings) == 0 - assert len(errors) == 1 - assert len(errors[0]) == 4 - assert errors[0][1] == "PatientID" - assert errors[0][2] == ( - "Value(s) ['A', 'B', 'C'] from row(s) ['2', '3', '4'] of the attribute " - "PatientID in the source manifest are missing." - ) + #################################### + # _run_validation_across_targets_set + #################################### - def test__run_validation_across_targets_set( + @pytest.mark.parametrize("tested_column", [(), ("A"), ("A", "A"), ("A", "B")]) + @pytest.mark.parametrize( + "rule", MATCH_ATLEAST_ONE_SET_RULES + MATCH_EXACTLY_ONE_SET_RULES + ) + @pytest.mark.parametrize( + "target_id, present_log_input, present_log_expected", + [ + ("syn1", [], ["syn1"]), + ("syn2", ["syn1"], ["syn1", "syn2"]), + ("syn3", ["syn1"], ["syn1", "syn3"]), + ], + ) + def test__run_validation_across_targets_set_match_exactly_atleaset_one_no_missing_values( self, va_obj: ValidateAttribute, cross_val_col_names: dict[str, str], cross_val_df1: DataFrame, + rule: str, + tested_column: list, + target_id: str, + present_log_input: list[str], + present_log_expected: list[str], ) -> None: - """Tests for ValidateAttribute._run_validation_across_targets_set for matchAtLeastOne""" - - output, bool_list1, bool_list2 = va_obj._run_validation_across_targets_set( - val_rule="matchAtleastOne, Patient.PatientID, set", - column_names=cross_val_col_names, - manifest_col=Series(["A", "B", "C"]), - target_attribute="patientid", - target_manifest=cross_val_df1, - target_manifest_id="syn1", - missing_manifest_log={}, - present_manifest_log=[], - repeat_manifest_log={}, - target_attribute_in_manifest_list=[], - target_manifest_empty=[], - ) - assert output[0] == {} - assert output[1] == ["syn1"] - assert output[2] == {} - assert bool_list1 == [True] - assert bool_list2 == [False] + """ + This test shows that for matchAtLeastOne and matchExactlyOne rules that as long as all + values in the tested column are in the target manifest, only the present manifest list + is updated + """ output, bool_list1, bool_list2 = va_obj._run_validation_across_targets_set( - val_rule="matchAtleastOne, Patient.PatientID, set", + val_rule=rule, column_names=cross_val_col_names, - manifest_col=Series(["A", "B", "C"]), + manifest_col=Series(tested_column), target_attribute="patientid", target_manifest=cross_val_df1, - target_manifest_id="syn2", + target_manifest_id=target_id, missing_manifest_log={}, - present_manifest_log=["syn1"], + present_manifest_log=present_log_input.copy(), repeat_manifest_log={}, target_attribute_in_manifest_list=[], target_manifest_empty=[], ) assert output[0] == {} - assert output[1] == ["syn1", "syn2"] + assert output[1] == present_log_expected assert output[2] == {} assert bool_list1 == [True] assert bool_list2 == [False] + @pytest.mark.parametrize( + "rule", MATCH_ATLEAST_ONE_SET_RULES + MATCH_EXACTLY_ONE_SET_RULES + ) + @pytest.mark.parametrize( + "tested_column, target_id, present_log_input, present_log_expected", + [ + (["D"], "syn1", [], []), + (["D", "D"], "syn2", [], []), + (["D", "F"], "syn3", [], []), + ], + ) + def test__run_validation_across_targets_set_match_exactly_atleaset_one_missing_values( + self, + va_obj: ValidateAttribute, + cross_val_col_names: dict[str, str], + cross_val_df1: DataFrame, + rule: str, + tested_column: list, + target_id: str, + present_log_input: list[str], + present_log_expected: list[str], + ) -> None: + """ + This test shows that for matchAtLeastOne and matchExactlyOne rules, + that missing values get added + """ output, bool_list1, bool_list2 = va_obj._run_validation_across_targets_set( - val_rule="matchAtleastOne, Patient.PatientID, set", + val_rule=rule, column_names=cross_val_col_names, - manifest_col=Series(["A", "B", "C", "D"]), + manifest_col=Series(tested_column), target_attribute="patientid", target_manifest=cross_val_df1, - target_manifest_id="syn1", + target_manifest_id=target_id, missing_manifest_log={}, - present_manifest_log=[], + present_manifest_log=present_log_input.copy(), repeat_manifest_log={}, target_attribute_in_manifest_list=[], target_manifest_empty=[], ) - assert list(output[0].keys()) == ["syn1"] - assert list(output[0].values())[0].to_list() == ["D"] - assert output[1] == [] + assert output[0][target_id].to_list() == tested_column + assert output[1] == present_log_expected assert output[2] == {} assert bool_list1 == [True] assert bool_list2 == [False] - output, bool_list1, bool_list2 = va_obj._run_validation_across_targets_set( - val_rule="matchAtleastOne, Patient.PatientID, set", - column_names=cross_val_col_names, - manifest_col=Series(["A", "B", "C", "E"]), - target_attribute="patientid", - target_manifest=cross_val_df1, - target_manifest_id="syn2", - missing_manifest_log={"syn1": Series(["D"])}, - present_manifest_log=[], - repeat_manifest_log={}, - target_attribute_in_manifest_list=[], - target_manifest_empty=[], - ) - assert list(output[0].keys()) == ["syn1", "syn2"] - assert output[0]["syn1"].to_list() == ["D"] - assert output[0]["syn2"].to_list() == ["E"] - assert output[1] == [] - assert output[2] == {} - assert bool_list1 == [True] - assert bool_list2 == [False] + def test__run_validation_across_targets_set_match_none( + self, + va_obj: ValidateAttribute, + cross_val_col_names: dict[str, str], + cross_val_df1: DataFrame, + ) -> None: + """Tests for ValidateAttribute._run_validation_across_targets_set for matchAtLeastOne""" output, bool_list1, bool_list2 = va_obj._run_validation_across_targets_set( val_rule="matchNone, Patient.PatientID, set", @@ -918,46 +997,110 @@ def test__run_validation_across_targets_set( assert bool_list1 == [True] assert bool_list2 == [False] - def test__gather_set_warnings_errors_match_atleast_one_passes( - self, va_obj: ValidateAttribute - ) -> None: - """Tests for ValidateAttribute._gather_set_warnings_errors for matchAtLeastOne""" + ############################### + # _gather_value_warnings_errors + ############################### - errors, warnings = va_obj._gather_set_warnings_errors( - val_rule="matchAtLeastOne Patient.PatientID set error", + @pytest.mark.parametrize( + "rule, missing, duplicated, repeat", + [ + ("matchAtLeastOne Patient.PatientID value error", [], [], []), + ("matchAtLeastOne Patient.PatientID value error", [], ["A"], []), + ("matchAtLeastOne Patient.PatientID value error", [], ["A", "A"], []), + ("matchAtLeastOne Patient.PatientID value error", [], ["A", "B", "C"], []), + ("matchExactlyOne Patient.PatientID value error", [], [], []), + ("matchNone Patient.PatientID value error", [], [], []), + ], + ) + def test__gather_value_warnings_errors_passing( + self, + va_obj: ValidateAttribute, + rule: str, + missing: list, + duplicated: list, + repeat: list, + ) -> None: + """ + Tests for ValidateAttribute._gather_value_warnings_errors + For matchAtLeastOne to pass there must be no mssing values + For matchExactlyOne there must be no missing or duplicated values + For matchNone there must be no repeat values + """ + assert va_obj._gather_value_warnings_errors( + val_rule=rule, source_attribute="PatientID", - set_validation_store=({}, [], {}), - ) - assert len(warnings) == 0 - assert len(errors) == 0 + value_validation_store=( + Series(missing), + Series(duplicated), + Series(repeat), + ), + ) == ([], []) - errors, warnings = va_obj._gather_set_warnings_errors( - val_rule="matchAtLeastOne Patient.PatientID set error", - source_attribute="PatientID", - set_validation_store=({}, ["syn1"], {}), - ) - assert len(warnings) == 0 - assert len(errors) == 0 + @pytest.mark.parametrize( + "rule, missing, duplicated, repeat", + [ + ("matchAtLeastOne Patient.PatientID value error", ["A"], [], []), + ("matchAtLeastOne Patient.PatientID value warning", ["A"], [], []), + ("matchExactlyOne Patient.PatientID value error", ["A"], [], []), + ("matchExactlyOne Patient.PatientID value warning", ["A"], [], []), + ("matchExactlyOne Patient.PatientID value error", [], ["B"], []), + ("matchExactlyOne Patient.PatientID value warning", [], ["B"], []), + ("matchNonePatient.PatientID value error", [], [], ["A"]), + ("matchNone Patient.PatientID value warning", [], [], ["A"]), + ], + ) + def test__gather_value_warnings_errors_with_errors( + self, + va_obj: ValidateAttribute, + rule: str, + missing: list, + duplicated: list, + repeat: list, + ) -> None: + """Tests for ValidateAttribute._gather_value_warnings_errors""" - errors, warnings = va_obj._gather_set_warnings_errors( - val_rule="matchAtLeastOne Patient.PatientID set error", + errors, warnings = va_obj._gather_value_warnings_errors( + val_rule=rule, source_attribute="PatientID", - set_validation_store=({}, ["syn1", "syn2"], {}), + value_validation_store=( + Series(missing, name="PatientID"), + Series(duplicated, name="PatientID"), + Series(repeat, name="PatientID"), + ), ) - assert len(warnings) == 0 - assert len(errors) == 0 + if rule.endswith("error"): + assert warnings == [] + assert len(errors) == 1 + else: + assert len(warnings) == 1 + assert errors == [] - errors, warnings = va_obj._gather_set_warnings_errors( - val_rule="matchAtLeastOne Patient.PatientID set error", + ############################# + # _gather_set_warnings_errors + ############################# + + @pytest.mark.parametrize( + "validation_tuple", + [ + (({}, [], {})), + (({}, ["syn1"], {})), + (({"syn1": Series(["A"])}, ["syn1"], {"syn2": Series(["B"])})), + ], + ) + @pytest.mark.parametrize("rule", MATCH_EXACTLY_ONE_SET_RULES) + def test__gather_set_warnings_errors_match_atleast_one_passes( + self, + va_obj: ValidateAttribute, + validation_tuple: tuple[dict[str, Series], list[str], dict[str, Series]], + rule: str, + ) -> None: + """Tests for ValidateAttribute._gather_set_warnings_errors for matchAtLeastOne""" + + assert va_obj._gather_set_warnings_errors( + val_rule=rule, source_attribute="PatientID", - set_validation_store=( - {"syn1": Series(["A"])}, - ["syn1"], - {"syn2": Series(["B"])}, - ), - ) - assert len(warnings) == 0 - assert len(errors) == 0 + set_validation_store=validation_tuple, + ) == ([], []) def test__gather_set_warnings_errors_match_atleast_one_errors( self, va_obj: ValidateAttribute @@ -979,137 +1122,207 @@ def test__gather_set_warnings_errors_match_atleast_one_errors( ) assert errors[0][3] == ["A"] + @pytest.mark.parametrize( + "validation_tuple", + [ + (({}, [], {})), + (({}, ["syn1"], {})), + ({"syn1": Series(["A"])}, ["syn1"], {"syn2": Series(["B"])}), + ], + ) + @pytest.mark.parametrize("rule", MATCH_EXACTLY_ONE_SET_RULES) def test__gather_set_warnings_errors_match_exactly_one_passes( - self, va_obj: ValidateAttribute + self, + va_obj: ValidateAttribute, + validation_tuple: tuple[dict[str, Series], list[str], dict[str, Series]], + rule: str, ) -> None: """Tests for ValidateAttribute._gather_set_warnings_errors for matchExactlyOne""" - - errors, warnings = va_obj._gather_set_warnings_errors( - val_rule="matchExactlyOne Patient.PatientID set error", + assert va_obj._gather_set_warnings_errors( + val_rule=rule, source_attribute="PatientID", - set_validation_store=({}, [], {}), - ) - assert len(warnings) == 0 - assert len(errors) == 0 + set_validation_store=validation_tuple, + ) == ([], []) - errors, warnings = va_obj._gather_set_warnings_errors( - val_rule="matchExactlyOne Patient.PatientID set error", - source_attribute="PatientID", - set_validation_store=({}, ["syn1"], {}), - ) - assert len(warnings) == 0 - assert len(errors) == 0 - - errors, warnings = va_obj._gather_set_warnings_errors( - val_rule="matchExactlyOne Patient.PatientID set error", - source_attribute="PatientID", - set_validation_store=( - {"syn1": Series(["A"])}, - ["syn1"], - {"syn2": Series(["B"])}, + @pytest.mark.parametrize( + "input_store, expected_list", + [ + ( + ({}, ["syn1", "syn2"], {}), + [ + [ + None, + "PatientID", + ( + "All values from attribute PatientID in the source manifest are " + "present in 2 manifests instead of only 1. Manifests ['syn1', 'syn2'] " + "match the values in the source attribute." + ), + None, + ] + ], ), - ) - assert len(warnings) == 0 - assert len(errors) == 0 - + ( + ({}, ["syn1", "syn2", "syn3"], {}), + [ + [ + None, + "PatientID", + ( + "All values from attribute PatientID in the source manifest are " + "present in 3 manifests instead of only 1. Manifests " + "['syn1', 'syn2', 'syn3'] match the values in the source attribute." + ), + None, + ] + ], + ), + ], + ) + @pytest.mark.parametrize("rule", MATCH_EXACTLY_ONE_SET_RULES) def test__gather_set_warnings_errors_match_exactly_one_errors( - self, va_obj: ValidateAttribute + self, + va_obj: ValidateAttribute, + input_store: tuple[dict[str, Series], list[str], dict[str, Series]], + expected_list: list[str], + rule: str, ) -> None: """Tests for ValidateAttribute._gather_set_warnings_errors for matchExactlyOne""" errors, warnings = va_obj._gather_set_warnings_errors( - val_rule="matchExactlyOne Patient.PatientID set error", + val_rule=rule, source_attribute="PatientID", - set_validation_store=({}, ["syn1", "syn2"], {}), + set_validation_store=input_store, ) - assert len(warnings) == 0 - assert len(errors) == 1 - assert not errors[0][0] - assert errors[0][1] == "PatientID" - assert errors[0][2] == ( - "All values from attribute PatientID in the source manifest are present in 2 " - "manifests instead of only 1. Manifests ['syn1', 'syn2'] match the values in " - "the source attribute." - ) - assert not errors[0][3] - + if rule.endswith("error"): + assert warnings == [] + assert errors == expected_list + else: + assert warnings == expected_list + assert errors == [] + + @pytest.mark.parametrize( + "validation_tuple", [(({}, [], {})), (({"syn1": Series(["A"])}, ["syn1"], {}))] + ) + @pytest.mark.parametrize("rule", MATCH_NONE_SET_RULES) def test__gather_set_warnings_errors_match_none_passes( - self, va_obj: ValidateAttribute + self, + va_obj: ValidateAttribute, + validation_tuple: tuple[dict[str, Series], list[str], dict[str, Series]], + rule: str, ) -> None: """Tests for ValidateAttribute._gather_set_warnings_errors for matchNone""" - errors, warnings = va_obj._gather_set_warnings_errors( - val_rule="matchNone Patient.PatientID set error", - source_attribute="PatientID", - set_validation_store=({}, [], {}), - ) - assert len(warnings) == 0 - assert len(errors) == 0 - - errors, warnings = va_obj._gather_set_warnings_errors( - val_rule="matchNone Patient.PatientID set error", + assert va_obj._gather_set_warnings_errors( + val_rule=rule, source_attribute="PatientID", - set_validation_store=({"syn1": Series(["A"])}, ["syn1"], {}), - ) - assert len(warnings) == 0 - assert len(errors) == 0 + set_validation_store=validation_tuple, + ) == ([], []) + @pytest.mark.parametrize( + "input_store, expected_list", + [ + ( + ({}, [], {"syn1": Series(["A"])}), + [ + [ + ["2"], + "PatientID", + ( + "Value(s) ['A'] from row(s) ['2'] for the attribute PatientID " + "in the source manifest are not unique. " + "Manifest(s) ['syn1'] contain duplicate values." + ), + ["A"], + ] + ], + ), + ( + ({"x": Series(["A"])}, ["x"], {"syn1": Series(["A"])}), + [ + [ + ["2"], + "PatientID", + ( + "Value(s) ['A'] from row(s) ['2'] for the attribute PatientID " + "in the source manifest are not unique. " + "Manifest(s) ['syn1'] contain duplicate values." + ), + ["A"], + ] + ], + ), + ( + ({}, [], {"syn2": Series(["B"])}), + [ + [ + ["2"], + "PatientID", + ( + "Value(s) ['B'] from row(s) ['2'] for the attribute PatientID " + "in the source manifest are not unique. " + "Manifest(s) ['syn2'] contain duplicate values." + ), + ["B"], + ] + ], + ), + ], + ) + @pytest.mark.parametrize("rule", MATCH_NONE_SET_RULES) def test__gather_set_warnings_errors_match_none_errors( - self, va_obj: ValidateAttribute + self, + va_obj: ValidateAttribute, + input_store: tuple[dict[str, Series], list[str], dict[str, Series]], + expected_list: list[str], + rule: str, ) -> None: - """Tests for ValidateAttribute._gather_set_warnings_errors for matchNone""" - - errors, warnings = va_obj._gather_set_warnings_errors( - val_rule="matchNone Patient.PatientID set error", - source_attribute="PatientID", - set_validation_store=({}, [], {"syn1": Series(["A"])}), - ) - assert len(warnings) == 0 - assert len(errors) == 1 - assert errors[0][0] == ["2"] - assert errors[0][1] == "PatientID" - assert errors[0][2] == ( - "Value(s) ['A'] from row(s) ['2'] for the attribute PatientID " - "in the source manifest are not unique. " - "Manifest(s) ['syn1'] contain duplicate values." - ) - assert errors[0][3] == ["A"] + """ + Tests for ValidateAttribute._gather_set_warnings_errors for matchNone + This test shows that only the repeat_manifest_log matters + NOTE: when the repeat repeat_manifest_log is longer than one the order + of the values and synapse ids in the msg are inconsistent, making that + case hard to test + """ errors, warnings = va_obj._gather_set_warnings_errors( - val_rule="matchNone Patient.PatientID set error", + val_rule=rule, source_attribute="PatientID", - set_validation_store=( - {}, - [], - {"syn1": Series(["A"]), "syn2": Series(["B"])}, - ), + set_validation_store=input_store, ) - assert len(warnings) == 0 - assert len(errors) == 1 - assert errors[0][0] == ["2"] - assert errors[0][1] == "PatientID" - possible_errors = [ - ( - "Value(s) ['A', 'B'] from row(s) ['2'] for the attribute PatientID in the source " - "manifest are not unique. Manifest(s) ['syn1', 'syn2'] contain duplicate values." - ), + if rule.endswith("error"): + assert warnings == [] + assert errors == expected_list + else: + assert warnings == expected_list + assert errors == [] + + ################### + # _get_column_names + ################### + + @pytest.mark.parametrize( + "input_dict, expected_dict", + [ + ({}, {}), + ({"col1": []}, {"col1": "col1"}), + ({"COL 1": []}, {"col1": "COL 1"}), + ({"ColId": []}, {"colid": "ColId"}), + ({"ColID": []}, {"colid": "ColID"}), ( - "Value(s) ['B', 'A'] from row(s) ['2'] for the attribute PatientID in the source " - "manifest are not unique. Manifest(s) ['syn1', 'syn2'] contain duplicate values." + {"col1": [], "col2": []}, + { + "col1": "col1", + "col2": "col2", + }, ), - ] - assert errors[0][2] in possible_errors - possible_missing_values = [["A", "B"], ["B", "A"]] - assert errors[0][3] in possible_missing_values - - def test__get_column_names(self, va_obj: ValidateAttribute) -> None: + ], + ) + def test__get_column_names( + self, + va_obj: ValidateAttribute, + input_dict: dict[str, list], + expected_dict: dict[str, str], + ) -> None: """Tests for ValidateAttribute._get_column_names""" - assert not va_obj._get_column_names(DataFrame()) - assert va_obj._get_column_names(DataFrame({"col1": []})) == {"col1": "col1"} - assert va_obj._get_column_names(DataFrame({"col1": [], "col2": []})) == { - "col1": "col1", - "col2": "col2", - } - assert va_obj._get_column_names(DataFrame({"COL 1": []})) == {"col1": "COL 1"} - assert va_obj._get_column_names(DataFrame({"ColId": []})) == {"colid": "ColId"} - assert va_obj._get_column_names(DataFrame({"ColID": []})) == {"colid": "ColID"} + assert va_obj._get_column_names(DataFrame(input_dict)) == expected_dict From ea7081110ffd5b15a39cfcb938d9a5d4b7c4e700 Mon Sep 17 00:00:00 2001 From: andrewelamb Date: Tue, 3 Sep 2024 10:23:35 -0700 Subject: [PATCH 157/233] Revert "add csv data model to unit tests" --- tests/test_api.py | 102 ++++++++++++++++------------------------- tests/test_cli.py | 21 ++++----- tests/test_manifest.py | 96 +++++++++++++++++++++----------------- 3 files changed, 104 insertions(+), 115 deletions(-) diff --git a/tests/test_api.py b/tests/test_api.py index e5015df26..6ce515622 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -76,19 +76,6 @@ def test_manifest_json(helpers) -> str: return test_manifest_path -@pytest.fixture(scope="class", - params=["example.model.jsonld", - "example.model.csv"]) -def data_model_url(request): - data_model_url = "https://raw.githubusercontent.com/Sage-Bionetworks/schematic/develop/tests/data/" + request.param - yield data_model_url - - -@pytest.fixture(scope="class") -def benchmark_data_model_jsonld(): - benchmark_data_model_jsonld = "https://raw.githubusercontent.com/Sage-Bionetworks/schematic/develop/tests/data/example.single_rule.model.jsonld" - yield benchmark_data_model_jsonld - def get_MockComponent_attribute() -> Generator[str, None, None]: """ Yield all of the mock conponent attributes one at a time @@ -344,9 +331,9 @@ def test_if_in_assetview( @pytest.mark.schematic_api class TestMetadataModelOperation: @pytest.mark.parametrize("as_graph", [True, False]) - def test_component_requirement(self, data_model_url, client: FlaskClient, as_graph: bool) -> None: + def test_component_requirement(self, client: FlaskClient, as_graph: bool) -> None: params = { - "schema_url": data_model_url, + "schema_url": DATA_MODEL_JSON_LD, "source_component": "BulkRNA-seqAssay", "as_graph": as_graph, } @@ -395,8 +382,8 @@ def test_get_property_label_from_display_name( @pytest.mark.schematic_api class TestDataModelGraphExplorerOperation: - def test_get_schema(self, data_model_url, client: FlaskClient) -> None: - params = {"schema_url": data_model_url, "data_model_labels": "class_label"} + def test_get_schema(self, client: FlaskClient) -> None: + params = {"schema_url": DATA_MODEL_JSON_LD, "data_model_labels": "class_label"} response = client.get( "http://localhost:3001/v1/schemas/get/schema", query_string=params ) @@ -409,9 +396,9 @@ def test_get_schema(self, data_model_url, client: FlaskClient) -> None: if os.path.exists(response_dt): os.remove(response_dt) - def test_if_node_required(test, data_model_url, client: FlaskClient) -> None: + def test_if_node_required(test, client: FlaskClient) -> None: params = { - "schema_url": data_model_url, + "schema_url": DATA_MODEL_JSON_LD, "node_display_name": "FamilyHistory", "data_model_labels": "class_label", } @@ -423,9 +410,9 @@ def test_if_node_required(test, data_model_url, client: FlaskClient) -> None: assert response.status_code == 200 assert response_dta == True - def test_get_node_validation_rules(test, data_model_url, client: FlaskClient) -> None: + def test_get_node_validation_rules(test, client: FlaskClient) -> None: params = { - "schema_url": data_model_url, + "schema_url": DATA_MODEL_JSON_LD, "node_display_name": "CheckRegexList", } response = client.get( @@ -437,9 +424,9 @@ def test_get_node_validation_rules(test, data_model_url, client: FlaskClient) -> assert "list" in response_dta assert "regex match [a-f]" in response_dta - def test_get_nodes_display_names(test, data_model_url, client: FlaskClient) -> None: + def test_get_nodes_display_names(test, client: FlaskClient) -> None: params = { - "schema_url": data_model_url, + "schema_url": DATA_MODEL_JSON_LD, "node_list": ["FamilyHistory", "Biospecimen"], } response = client.get( @@ -453,8 +440,8 @@ def test_get_nodes_display_names(test, data_model_url, client: FlaskClient) -> N @pytest.mark.parametrize( "relationship", ["parentOf", "requiresDependency", "rangeValue", "domainValue"] ) - def test_get_subgraph_by_edge(self, data_model_url, client: FlaskClient, relationship: str) -> None: - params = {"schema_url": data_model_url, "relationship": relationship} + def test_get_subgraph_by_edge(self, client: FlaskClient, relationship: str) -> None: + params = {"schema_url": DATA_MODEL_JSON_LD, "relationship": relationship} response = client.get( "http://localhost:3001/v1/schemas/get/graph_by_edge_type", @@ -465,10 +452,10 @@ def test_get_subgraph_by_edge(self, data_model_url, client: FlaskClient, relatio @pytest.mark.parametrize("return_display_names", [True, False]) @pytest.mark.parametrize("node_label", ["FamilyHistory", "TissueStatus"]) def test_get_node_range( - self, data_model_url, client: FlaskClient, return_display_names: bool, node_label: str + self, client: FlaskClient, return_display_names: bool, node_label: str ) -> None: params = { - "schema_url": data_model_url, + "schema_url": DATA_MODEL_JSON_LD, "return_display_names": return_display_names, "node_label": node_label, } @@ -492,7 +479,6 @@ def test_get_node_range( @pytest.mark.parametrize("source_node", ["Patient", "Biospecimen"]) def test_node_dependencies( self, - data_model_url, client: FlaskClient, source_node: str, return_display_names: Union[bool, None], @@ -502,7 +488,7 @@ def test_node_dependencies( return_schema_ordered = False params = { - "schema_url": data_model_url, + "schema_url": DATA_MODEL_JSON_LD, "source_node": source_node, "return_display_names": return_display_names, "return_schema_ordered": return_schema_ordered, @@ -568,8 +554,7 @@ def ifPandasDataframe(self, response_dt): ["Biospecimen", "Patient", "all manifests", ["Biospecimen", "Patient"]], ) def test_generate_existing_manifest( - self,, - data_model_url, + self, client: FlaskClient, data_type: str, output_format: str, @@ -587,7 +572,7 @@ def test_generate_existing_manifest( dataset_id = None # if "all manifests", dataset id is None params = { - "schema_url": data_model_url, + "schema_url": DATA_MODEL_JSON_LD, "asset_view": "syn23643253", "title": "Example", "data_type": data_type, @@ -653,7 +638,6 @@ def test_generate_existing_manifest( ) def test_generate_new_manifest( self, - data_model_url caplog: pytest.LogCaptureFixture, client: FlaskClient, data_type: str, @@ -661,7 +645,7 @@ def test_generate_new_manifest( request_headers: Dict[str, str], ) -> None: params = { - "schema_url": data_model_url, + "schema_url": DATA_MODEL_JSON_LD, "asset_view": "syn23643253", "title": "Example", "data_type": data_type, @@ -757,10 +741,10 @@ def test_generate_new_manifest( ], ) def test_generate_manifest_file_based_annotations( - self, data_model_url, client: FlaskClient, use_annotations: bool, expected: list[str] + self, client: FlaskClient, use_annotations: bool, expected: list[str] ) -> None: params = { - "schema_url": data_model_url, + "schema_url": DATA_MODEL_JSON_LD, "data_type": "BulkRNA-seqAssay", "dataset_id": "syn25614635", "asset_view": "syn51707141", @@ -807,10 +791,10 @@ def test_generate_manifest_file_based_annotations( # test case: generate a manifest with annotations when use_annotations is set to True for a component that is not file-based # the dataset folder does not contain an existing manifest def test_generate_manifest_not_file_based_with_annotations( - self, data_model_url, client: FlaskClient + self, client: FlaskClient ) -> None: params = { - "schema_url": data_model_url, + "schema_url": DATA_MODEL_JSON_LD, "data_type": "Patient", "dataset_id": "syn25614635", "asset_view": "syn51707141", @@ -842,9 +826,9 @@ def test_generate_manifest_not_file_based_with_annotations( ] ) - def test_generate_manifest_data_type_not_found(self, data_model_url, client: FlaskClient) -> None: + def test_generate_manifest_data_type_not_found(self, client: FlaskClient) -> None: params = { - "schema_url": data_model_url, + "schema_url": DATA_MODEL_JSON_LD, "data_type": "wrong data type", "use_annotations": False, } @@ -856,14 +840,14 @@ def test_generate_manifest_data_type_not_found(self, data_model_url, client: Fla assert "LookupError" in str(response.data) def test_populate_manifest( - self, data_model_url, client: FlaskClient, test_manifest_csv: str + self, client: FlaskClient, test_manifest_csv: str ) -> None: # test manifest test_manifest_data = open(test_manifest_csv, "rb") params = { "data_type": "MockComponent", - "schema_url": data_model_url, + "schema_url": DATA_MODEL_JSON_LD, "title": "Example", "csv_file": test_manifest_data, } @@ -889,14 +873,13 @@ def test_populate_manifest( ) def test_validate_manifest( self, - data_model_url, client: FlaskClient, json_str: Union[str, None], restrict_rules: Union[bool, None], test_manifest_csv: str, request_headers: Dict[str, str], ) -> None: - params = {"schema_url": data_model_url, "restrict_rules": restrict_rules} + params = {"schema_url": DATA_MODEL_JSON_LD, "restrict_rules": restrict_rules} if json_str: params["json_str"] = json_str @@ -1093,14 +1076,13 @@ def test_dataset_manifest_download( @pytest.mark.submission def test_submit_manifest_table_and_file_replace( self, - data_model_url, client: FlaskClient, request_headers: Dict[str, str], test_manifest_submit: str, ) -> None: """Testing submit manifest in a csv format as a table and a file. Only replace the table""" params = { - "schema_url": data_model_url, + "schema_url": DATA_MODEL_JSON_LD, "data_type": "Biospecimen", "restrict_rules": False, "hide_blanks": False, @@ -1132,7 +1114,6 @@ def test_submit_manifest_table_and_file_replace( def test_submit_manifest_file_only_replace( self, helpers, - data_model_url, client: FlaskClient, request_headers: Dict[str, str], data_type: str, @@ -1141,7 +1122,7 @@ def test_submit_manifest_file_only_replace( ) -> None: """Testing submit manifest in a csv format as a file""" params = { - "schema_url": data_model_url, + "schema_url": DATA_MODEL_JSON_LD, "data_type": data_type, "restrict_rules": False, "manifest_record_type": "file_only", @@ -1184,12 +1165,12 @@ def test_submit_manifest_file_only_replace( @pytest.mark.synapse_credentials_needed @pytest.mark.submission def test_submit_manifest_json_str_replace( - self, cdata_model_url, lient: FlaskClient, request_headers: Dict[str, str] + self, client: FlaskClient, request_headers: Dict[str, str] ) -> None: """Submit json str as a file""" json_str = '[{"Sample ID": 123, "Patient ID": 1,"Tissue Status": "Healthy","Component": "Biospecimen"}]' params = { - "schema_url": data_model_url, + "schema_url": DATA_MODEL_JSON_LD, "data_type": "Biospecimen", "json_str": json_str, "restrict_rules": False, @@ -1213,13 +1194,12 @@ def test_submit_manifest_json_str_replace( @pytest.mark.submission def test_submit_manifest_w_file_and_entities( self, - data_model_url, client: FlaskClient, request_headers: Dict[str, str], test_manifest_submit: str, ) -> None: params = { - "schema_url": data_model_url, + "schema_url": DATA_MODEL_JSON_LD, "data_type": "Biospecimen", "restrict_rules": False, "manifest_record_type": "file_and_entities", @@ -1244,13 +1224,12 @@ def test_submit_manifest_w_file_and_entities( @pytest.mark.submission def test_submit_manifest_table_and_file_upsert( self, - data_model_url, client: FlaskClient, request_headers: Dict[str, str], test_upsert_manifest_csv: str, ) -> None: params = { - "schema_url": data_model_url, + "schema_url": DATA_MODEL_JSON_LD, "data_type": "MockRDB", "restrict_rules": False, "manifest_record_type": "table_and_file", @@ -1274,8 +1253,8 @@ def test_submit_manifest_table_and_file_upsert( @pytest.mark.schematic_api class TestSchemaVisualization: - def test_visualize_attributes(self, data_model_url, client: FlaskClient) -> None: - params = {"schema_url": data_model_url} + def test_visualize_attributes(self, client: FlaskClient) -> None: + params = {"schema_url": DATA_MODEL_JSON_LD} response = client.get( "http://localhost:3001/v1/visualize/attributes", query_string=params @@ -1285,10 +1264,10 @@ def test_visualize_attributes(self, data_model_url, client: FlaskClient) -> None @pytest.mark.parametrize("figure_type", ["component", "dependency"]) def test_visualize_tangled_tree_layers( - self, client: FlaskClient, figure_type: str, data_model_url + self, client: FlaskClient, figure_type: str ) -> None: # TODO: Determine a 2nd data model to use for this test, test both models sequentially, add checks for content of response - params = {"schema_url": data_model_url, "figure_type": figure_type} + params = {"schema_url": DATA_MODEL_JSON_LD, "figure_type": figure_type} response = client.get( "http://localhost:3001/v1/visualize/tangled_tree/layers", @@ -1393,10 +1372,10 @@ def test_visualize_tangled_tree_layers( ], ) def test_visualize_component( - self, data_model_url, client: FlaskClient, component: str, response_text: str + self, client: FlaskClient, component: str, response_text: str ) -> None: params = { - "schema_url": data_model_url, + "schema_url": DATA_MODEL_JSON_LD, "component": component, "include_index": False, "data_model_labels": "class_label", @@ -1421,7 +1400,6 @@ class TestValidationBenchmark: def test_validation_performance( self, helpers, - data_model_url, client: FlaskClient, test_invalid_manifest: pd.DataFrame, MockComponent_attribute: Generator[str, None, None], @@ -1442,7 +1420,7 @@ def test_validation_performance( # Set paramters for endpoint params = { - "schema_url": benchmark_data_model_url, + "schema_url": BENCHMARK_DATA_MODEL_JSON_LD, "data_type": "MockComponent", } headers = {"Content-Type": "multipart/form-data", "Accept": "application/json"} diff --git a/tests/test_cli.py b/tests/test_cli.py index 722906ee5..308f9c73f 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -19,11 +19,10 @@ def runner() -> CliRunner: return CliRunner() -@pytest.fixture(params=["example.model.jsonld", - "example.model.csv"]) -def data_model_path(request, helpers): - data_model_path = helpers.get_data_path(request.param) - yield data_model_path +@pytest.fixture +def data_model_jsonld(helpers): + data_model_jsonld = helpers.get_data_path("example.model.jsonld") + yield data_model_jsonld class TestSchemaCli: @@ -78,7 +77,7 @@ def test_schema_convert_cli(self, runner, helpers): # by default this should download the manifest as a CSV file @pytest.mark.google_credentials_needed def test_get_example_manifest_default( - self, runner, helpers, config: Configuration, data_model_path + self, runner, helpers, config: Configuration, data_model_jsonld ): output_path = helpers.get_data_path("example.Patient.manifest.csv") config.load_config("config_example.yml") @@ -92,7 +91,7 @@ def test_get_example_manifest_default( "--data_type", "Patient", "--path_to_data_model", - data_model_path, + data_model_jsonld, ], ) @@ -103,7 +102,7 @@ def test_get_example_manifest_default( # use google drive to export @pytest.mark.google_credentials_needed def test_get_example_manifest_csv( - self, runner, helpers, config: Configuration, data_model_path + self, runner, helpers, config: Configuration, data_model_jsonld ): output_path = helpers.get_data_path("test.csv") config.load_config("config_example.yml") @@ -117,7 +116,7 @@ def test_get_example_manifest_csv( "--data_type", "Patient", "--path_to_data_model", - data_model_path, + data_model_jsonld, "--output_csv", output_path, ], @@ -128,7 +127,7 @@ def test_get_example_manifest_csv( # get manifest as an excel spreadsheet @pytest.mark.google_credentials_needed def test_get_example_manifest_excel( - self, runner, helpers, config: Configuration, data_model_path + self, runner, helpers, config: Configuration, data_model_jsonld ): output_path = helpers.get_data_path("test.xlsx") config.load_config("config_example.yml") @@ -142,7 +141,7 @@ def test_get_example_manifest_excel( "--data_type", "Patient", "--path_to_data_model", - data_model_path, + data_model_jsonld, "--output_xlsx", output_path, ], diff --git a/tests/test_manifest.py b/tests/test_manifest.py index 8a294435b..06bd7b168 100644 --- a/tests/test_manifest.py +++ b/tests/test_manifest.py @@ -38,12 +38,7 @@ def generate_graph_data_model(helpers, path_to_data_model, data_model_labels): return graph_data_model -@pytest.fixture(params=["example.model.jsonld", - "example.model.csv"]) -def data_model_path(request, helpers): - data_model_path = helpers.get_data_path(request.param) - yield data_model_path - + @pytest.fixture( params=[ (True, "Patient"), @@ -58,19 +53,21 @@ def data_model_path(request, helpers): "skip_annotations-BulkRNAseqAssay", ], ) -def manifest_generator(helpers, request, data_model_path): +def manifest_generator(helpers, request): # Rename request param for readability use_annotations, data_type = request.param + path_to_data_model = helpers.get_data_path("example.model.jsonld") + # Get graph data model graph_data_model = generate_graph_data_model( helpers, - path_to_data_model=data_model_path, + path_to_data_model=path_to_data_model, data_model_labels="class_label", ) manifest_generator = ManifestGenerator( - path_to_data_model=data_model_path, + path_to_data_model=path_to_data_model, graph=graph_data_model, root=data_type, use_annotations=use_annotations, @@ -124,19 +121,20 @@ def app(): class TestManifestGenerator: - def test_init(self, helpers, data_model_path): + def test_init(self, helpers): + path_to_data_model = helpers.get_data_path("example.model.jsonld") # Get graph data model graph_data_model = generate_graph_data_model( helpers, - path_to_data_model=data_model_path, + path_to_data_model=path_to_data_model, data_model_labels="class_label", ) generator = ManifestGenerator( graph=graph_data_model, title="mock_title", - path_to_data_model=data_model_path, + path_to_data_model=path_to_data_model, root="Patient", ) @@ -157,22 +155,23 @@ def test_init(self, helpers, data_model_path): ], ids=["DataType not found in Schema", "No DataType provided"], ) - def test_missing_root_error(self, helpers, data_type, exc, exc_message, data_model_path): + def test_missing_root_error(self, helpers, data_type, exc, exc_message): """ Test for errors when either no DataType is provided or when a DataType is provided but not found in the schema """ + path_to_data_model = helpers.get_data_path("example.model.jsonld") # Get graph data model graph_data_model = generate_graph_data_model( helpers, - path_to_data_model=data_model_path, + path_to_data_model=path_to_data_model, data_model_labels="class_label", ) # A LookupError should be raised and include message when the component cannot be found with pytest.raises(exc) as e: generator = ManifestGenerator( - path_to_data_model=data_model_path, + path_to_data_model=helpers.get_data_path("example.model.jsonld"), graph=graph_data_model, root=data_type, use_annotations=False, @@ -237,7 +236,7 @@ def test_get_manifest_first_time(self, manifest): @pytest.mark.parametrize("sheet_url", [None, True, False]) @pytest.mark.parametrize("dataset_id", [None, "syn27600056"]) @pytest.mark.google_credentials_needed - def test_get_manifest_excel(self, helpers, sheet_url, output_format, dataset_id, data_model_path): + def test_get_manifest_excel(self, helpers, sheet_url, output_format, dataset_id): """ Purpose: the goal of this test is to make sure that output_format parameter and sheet_url parameter could function well; In addition, this test also makes sure that getting a manifest with an existing dataset_id is working @@ -247,16 +246,17 @@ def test_get_manifest_excel(self, helpers, sheet_url, output_format, dataset_id, data_type = "Patient" # Get path to data model + path_to_data_model = helpers.get_data_path("example.model.jsonld") # Get graph data model graph_data_model = generate_graph_data_model( helpers, - path_to_data_model=data_model_path, + path_to_data_model=path_to_data_model, data_model_labels="class_label", ) generator = ManifestGenerator( - path_to_data_model=data_model_path, + path_to_data_model=path_to_data_model, graph=graph_data_model, root=data_type, use_annotations=False, @@ -296,7 +296,7 @@ def test_get_manifest_excel(self, helpers, sheet_url, output_format, dataset_id, [("syn27600056"), ("syn52397659")], ids=["Annotations present", "Annotations not present"], ) - def test_get_manifest_no_annos(self, helpers, dataset_id, data_model_path): + def test_get_manifest_no_annos(self, helpers, dataset_id): """ Test to cover manifest generation under the case where use_annotations is True but there are no annotations in the dataset @@ -305,16 +305,19 @@ def test_get_manifest_no_annos(self, helpers, dataset_id, data_model_path): # Use a non-file based DataType data_type = "Patient" + # Get path to data model + path_to_data_model = helpers.get_data_path("example.model.jsonld") + # Get graph data model graph_data_model = generate_graph_data_model( helpers, - path_to_data_model=data_model_path, + path_to_data_model=path_to_data_model, data_model_labels="class_label", ) # Instantiate object with use_annotations set to True generator = ManifestGenerator( - path_to_data_model=data_model_path, + path_to_data_model=path_to_data_model, graph=graph_data_model, root=data_type, use_annotations=True, @@ -415,21 +418,23 @@ def test_gather_all_fields(self, simple_manifest_generator): {"Filename": [], "Component": []}, {"Component": ["BulkRNA-seqAssay"]}, ), - ], + ], ) def test_add_root_to_component_without_additional_metadata( - self, helpers, data_type, required_metadata_fields, expected, data_model_path + self, helpers, data_type, required_metadata_fields, expected ): + # Get path to data model + path_to_data_model = helpers.get_data_path("example.model.jsonld") # Get graph data model graph_data_model = generate_graph_data_model( helpers, - path_to_data_model=data_model_path, + path_to_data_model=path_to_data_model, data_model_labels="class_label", ) manifest_generator = ManifestGenerator( - path_to_data_model=data_model_path, + path_to_data_model=path_to_data_model, graph=graph_data_model, root=data_type, ) @@ -455,18 +460,20 @@ def test_add_root_to_component_without_additional_metadata( ], ) def test_add_root_to_component_with_additional_metadata( - self, helpers, additional_metadata, data_model_path + self, helpers, additional_metadata ): + # Get path to data model + path_to_data_model = helpers.get_data_path("example.model.jsonld") # Get graph data model graph_data_model = generate_graph_data_model( helpers, - path_to_data_model=data_model_path, + path_to_data_model=path_to_data_model, data_model_labels="class_label", ) manifest_generator = ManifestGenerator( - path_to_data_model=data_model_path, + path_to_data_model=path_to_data_model, graph=graph_data_model, root="BulkRNA-seqAssay", ) @@ -531,7 +538,7 @@ def test_get_missing_columns( ], ) @pytest.mark.google_credentials_needed - def test_update_dataframe_with_existing_df(self, helpers, existing_manifest, data_model_path): + def test_update_dataframe_with_existing_df(self, helpers, existing_manifest): """ Tests the following discrepancies with an existing schema: - schema has matching columns to existing_df @@ -542,16 +549,18 @@ def test_update_dataframe_with_existing_df(self, helpers, existing_manifest, dat data_type = "Patient" sheet_url = True + path_to_data_model = helpers.get_data_path("example.model.jsonld") + # Get graph data model graph_data_model = generate_graph_data_model( helpers, - path_to_data_model=data_model_path, + path_to_data_model=path_to_data_model, data_model_labels="class_label", ) # Instantiate the Manifest Generator. generator = ManifestGenerator( - path_to_data_model=data_model_path, + path_to_data_model=path_to_data_model, graph=graph_data_model, root=data_type, use_annotations=False, @@ -675,22 +684,23 @@ def test_populate_existing_excel_spreadsheet( "return_output", ["Mock excel file path", "Mock google sheet link"] ) def test_create_single_manifest( - self, simple_manifest_generator, helpers, return_output, data_model_path + self, simple_manifest_generator, helpers, return_output ): with patch( "schematic.manifest.generator.ManifestGenerator.get_manifest", return_value=return_output, ): + json_ld_path = helpers.get_data_path("example.model.jsonld") data_type = "Patient" graph_data_model = generate_graph_data_model( helpers, - path_to_data_model=data_model_path, + path_to_data_model=json_ld_path, data_model_labels="class_label", ) result = simple_manifest_generator.create_single_manifest( - path_to_data_model=data_model_path, + path_to_data_model=json_ld_path, graph_data_model=graph_data_model, data_type=data_type, output_format="google_sheet", @@ -702,14 +712,15 @@ def test_create_single_manifest( "test_data_types", [["Patient", "Biospecimen"], ["all manifests"]] ) def test_create_manifests_raise_errors( - self, simple_manifest_generator, helpers, test_data_types, data_model_path + self, simple_manifest_generator, helpers, test_data_types ): with pytest.raises(ValueError) as exception_info: + json_ld_path = helpers.get_data_path("example.model.jsonld") data_types = test_data_types dataset_ids = ["syn123456"] simple_manifest_generator.create_manifests( - path_to_data_model=data_model_path, + path_to_data_model=json_ld_path, data_types=data_types, dataset_ids=dataset_ids, output_format="google_sheet", @@ -735,15 +746,14 @@ def test_create_manifests( test_data_types, dataset_ids, expected_result, - data_model_path ): with patch( "schematic.manifest.generator.ManifestGenerator.create_single_manifest", return_value="mock google sheet link", ): - + json_ld_path = helpers.get_data_path("example.model.jsonld") all_results = simple_manifest_generator.create_manifests( - path_to_data_model=data_model_path, + path_to_data_model=json_ld_path, data_types=test_data_types, dataset_ids=dataset_ids, output_format="google_sheet", @@ -782,23 +792,25 @@ def test_get_manifest_with_files( expected_file_based, expected_rows, expected_files, - data_model_path ): """ Test to ensure that when generating a record based manifset that has files in the dataset that the files are not added to the manifest as well when generating a file based manifest from a dataset thathas had files added that the files are added correctly """ + # GIVEN the example data model + path_to_data_model = helpers.get_data_path("example.model.jsonld") + # AND a graph data model graph_data_model = generate_graph_data_model( helpers, - path_to_data_model=data_model_path, + path_to_data_model=path_to_data_model, data_model_labels="class_label", ) # AND a manifest generator generator = ManifestGenerator( - path_to_data_model=data_model_path, + path_to_data_model=path_to_data_model, graph=graph_data_model, root=component, use_annotations=True, From 2e78ea1b5e7f8d52c46bd99e32366836aa371810 Mon Sep 17 00:00:00 2001 From: GiaJordan Date: Tue, 3 Sep 2024 13:12:28 -0700 Subject: [PATCH 158/233] update rule spec --- schematic/utils/validate_rules_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/schematic/utils/validate_rules_utils.py b/schematic/utils/validate_rules_utils.py index 859b8d220..5da5510ab 100644 --- a/schematic/utils/validate_rules_utils.py +++ b/schematic/utils/validate_rules_utils.py @@ -150,7 +150,7 @@ def validation_rule_info() -> dict[str, Rule]: "fixed_arg": None, }, "filenameExists": { - "arguments": (2, 1), + "arguments": (1, 1), "type": "filename_validation", "complementary_rules": None, "default_message_level": "error", From 628c1facb8f07738c9493205f4c1e896b0437775 Mon Sep 17 00:00:00 2001 From: GiaJordan Date: Tue, 3 Sep 2024 13:12:59 -0700 Subject: [PATCH 159/233] update data model --- tests/data/example.model.csv | 2 +- tests/data/example.model.jsonld | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/data/example.model.csv b/tests/data/example.model.csv index 32bbb9977..a85cf8cbf 100644 --- a/tests/data/example.model.csv +++ b/tests/data/example.model.csv @@ -12,7 +12,7 @@ Biospecimen,,,"Sample ID, Patient ID, Tissue Status, Component",,FALSE,DataType, Sample ID,,,,,TRUE,DataProperty,,, Tissue Status,,"Healthy, Malignant",,,TRUE,DataProperty,,, Bulk RNA-seq Assay,,,"Filename, Sample ID, File Format, Component",,FALSE,DataType,Biospecimen,, -Filename,,,,,TRUE,DataProperty,,,#MockFilename filenameExists syn61682648^^ +Filename,,,,,TRUE,DataProperty,,,#MockFilename filenameExists^^ File Format,,"FASTQ, BAM, CRAM, CSV/TSV",,,TRUE,DataProperty,,, BAM,,,Genome Build,,FALSE,ValidValue,,, CRAM,,,"Genome Build, Genome FASTA",,FALSE,ValidValue,,, diff --git a/tests/data/example.model.jsonld b/tests/data/example.model.jsonld index 1c7910d3c..3f13b188e 100644 --- a/tests/data/example.model.jsonld +++ b/tests/data/example.model.jsonld @@ -615,7 +615,7 @@ "sms:displayName": "Filename", "sms:required": "sms:true", "sms:validationRules": { - "MockFilename": "filenameExists syn61682648" + "MockFilename": "filenameExists" } }, { From fbe28b9fbad2ff521dd9b890204cd2526e1aed8e Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Tue, 3 Sep 2024 13:13:33 -0700 Subject: [PATCH 160/233] impove test docstring --- tests/integration/test_validate_attribute.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/integration/test_validate_attribute.py b/tests/integration/test_validate_attribute.py index d4d59eefc..f6d6823b4 100644 --- a/tests/integration/test_validate_attribute.py +++ b/tests/integration/test_validate_attribute.py @@ -77,7 +77,10 @@ def test_url_validation_invalid_url(self, dmge: DataModelGraphExplorer) -> None: def test__get_target_manifest_dataframes( self, dmge: DataModelGraphExplorer ) -> None: - """Testing for ValidateAttribute._get_target_manifest_dataframes""" + """ + This test checks that the method successfully returns manifests from Synapse + + """ validator = ValidateAttribute(dmge=dmge) manifests = validator._get_target_manifest_dataframes( # pylint:disable= protected-access "patient", project_scope=["syn54126707"] From e3a3568bf577aadca191dc0ac0d01beeea249566 Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Tue, 3 Sep 2024 13:16:26 -0700 Subject: [PATCH 161/233] ran black --- tests/integration/test_validate_attribute.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/test_validate_attribute.py b/tests/integration/test_validate_attribute.py index f6d6823b4..b6d3b74b1 100644 --- a/tests/integration/test_validate_attribute.py +++ b/tests/integration/test_validate_attribute.py @@ -79,7 +79,7 @@ def test__get_target_manifest_dataframes( ) -> None: """ This test checks that the method successfully returns manifests from Synapse - + """ validator = ValidateAttribute(dmge=dmge) manifests = validator._get_target_manifest_dataframes( # pylint:disable= protected-access From 90b507823868f2e53d8f2b29eee1e6e985c8925f Mon Sep 17 00:00:00 2001 From: GiaJordan Date: Tue, 3 Sep 2024 13:16:52 -0700 Subject: [PATCH 162/233] update callback util --- schematic/utils/cli_utils.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/schematic/utils/cli_utils.py b/schematic/utils/cli_utils.py index 07c97af90..8999a594f 100644 --- a/schematic/utils/cli_utils.py +++ b/schematic/utils/cli_utils.py @@ -4,10 +4,9 @@ # pylint: disable=anomalous-backslash-in-string import logging - -from typing import Any, Mapping, Sequence, Union, Optional -from functools import reduce import re +from functools import reduce +from typing import Any, Mapping, Optional, Sequence, Union logger = logging.getLogger(__name__) @@ -69,13 +68,13 @@ def parse_syn_ids( if not syn_ids: return None - project_regex = re.compile("(syn\d+\,?)+") - valid = project_regex.fullmatch(syn_ids) + synID_regex = re.compile("(syn\d+\,?)+") + valid = synID_regex.fullmatch(syn_ids) if not valid: raise ValueError( f"The provided list of project synID(s): {syn_ids}, is not formatted correctly. " - "\nPlease check your list of projects for errors." + "\nPlease check your list of synIDs for errors." ) return syn_ids.split(",") From 6ea6f8648b64bffaf1206b7db8db88384de6936e Mon Sep 17 00:00:00 2001 From: GiaJordan Date: Tue, 3 Sep 2024 13:57:27 -0700 Subject: [PATCH 163/233] add param --- schematic/models/commands.py | 31 +++++++++++++++++++------- schematic/models/metadata.py | 21 +++++++++-------- schematic/models/validate_attribute.py | 4 ++-- schematic/models/validate_manifest.py | 6 +++-- 4 files changed, 41 insertions(+), 21 deletions(-) diff --git a/schematic/models/commands.py b/schematic/models/commands.py index 2119291b8..aefc305e7 100644 --- a/schematic/models/commands.py +++ b/schematic/models/commands.py @@ -1,26 +1,25 @@ #!/usr/bin/env python3 -from typing import get_args -from gc import callbacks import logging import sys +from gc import callbacks from time import perf_counter +from typing import get_args import click import click_log - from jsonschema import ValidationError +from schematic.configuration.configuration import CONFIG +from schematic.exceptions import MissingConfigValueError +from schematic.help import model_commands from schematic.models.metadata import MetadataModel from schematic.utils.cli_utils import ( log_value_from_config, - query_dict, - parse_syn_ids, parse_comma_str_to_list, + parse_syn_ids, + query_dict, ) -from schematic.help import model_commands -from schematic.exceptions import MissingConfigValueError -from schematic.configuration.configuration import CONFIG from schematic.utils.schema_utils import DisplayLabelType logger = logging.getLogger("schematic") @@ -110,6 +109,12 @@ def model(ctx, config): # use as `schematic model ...` callback=parse_syn_ids, help=query_dict(model_commands, ("model", "validate", "project_scope")), ) +@click.option( + "-ds", + "--dataset_scope", + default=None, + help=query_dict(model_commands, ("model", "validate", "dataset_scope")), +) @click.option( "--table_manipulation", "-tm", @@ -150,6 +155,7 @@ def submit_manifest( hide_blanks, restrict_rules, project_scope, + dataset_scope, table_manipulation, data_model_labels, table_column_names, @@ -177,6 +183,7 @@ def submit_manifest( restrict_rules=restrict_rules, hide_blanks=hide_blanks, project_scope=project_scope, + dataset_scope=dataset_scope, table_manipulation=table_manipulation, table_column_names=table_column_names, annotation_keys=annotation_keys, @@ -227,6 +234,12 @@ def submit_manifest( callback=parse_syn_ids, help=query_dict(model_commands, ("model", "validate", "project_scope")), ) +@click.option( + "-ds", + "--dataset_scope", + default=None, + help=query_dict(model_commands, ("model", "validate", "dataset_scope")), +) @click.option( "--data_model_labels", "-dml", @@ -241,6 +254,7 @@ def validate_manifest( json_schema, restrict_rules, project_scope, + dataset_scope, data_model_labels, ): """ @@ -276,6 +290,7 @@ def validate_manifest( jsonSchema=json_schema, restrict_rules=restrict_rules, project_scope=project_scope, + dataset_scope=dataset_scope, ) if not errors: diff --git a/schematic/models/metadata.py b/schematic/models/metadata.py index 2747f81d7..582a00168 100644 --- a/schematic/models/metadata.py +++ b/schematic/models/metadata.py @@ -1,26 +1,25 @@ -import os import logging -import networkx as nx +import os from os.path import exists -from jsonschema import ValidationError # allows specifying explicit variable types -from typing import Any, Dict, Optional, Text, List +from typing import Any, Dict, List, Optional, Text + +import networkx as nx +from jsonschema import ValidationError +from opentelemetry import trace from schematic.manifest.generator import ManifestGenerator +from schematic.models.validate_manifest import validate_all from schematic.schemas.data_model_graph import DataModelGraph, DataModelGraphExplorer -from schematic.schemas.data_model_parser import DataModelParser from schematic.schemas.data_model_json_schema import DataModelJSONSchema +from schematic.schemas.data_model_parser import DataModelParser # TODO: This module should only be aware of the store interface # we shouldn't need to expose Synapse functionality explicitly from schematic.store.synapse import SynapseStorage - from schematic.utils.df_utils import load_df -from schematic.models.validate_manifest import validate_all -from opentelemetry import trace - logger = logging.getLogger(__name__) tracer = trace.get_tracer("Schematic") @@ -200,6 +199,7 @@ def validateModelManifest( restrict_rules: bool = False, jsonSchema: Optional[str] = None, project_scope: Optional[List] = None, + dataset_scope: Optional[str] = None, access_token: Optional[str] = None, ) -> tuple[list, list]: """Check if provided annotations manifest dataframe satisfies all model requirements. @@ -287,6 +287,7 @@ def validateModelManifest( jsonSchema=jsonSchema, restrict_rules=restrict_rules, project_scope=project_scope, + dataset_scope=dataset_scope, access_token=access_token, ) return errors, warnings @@ -332,6 +333,7 @@ def submit_metadata_manifest( # pylint: disable=too-many-arguments, too-many-lo file_annotations_upload: bool = True, hide_blanks: bool = False, project_scope: Optional[list] = None, + dataset_scope: Optional[str] = None, table_manipulation: str = "replace", table_column_names: str = "class_label", annotation_keys: str = "class_label", @@ -396,6 +398,7 @@ def submit_metadata_manifest( # pylint: disable=too-many-arguments, too-many-lo rootNode=validate_component, restrict_rules=restrict_rules, project_scope=project_scope, + dataset_scope=dataset_scope, access_token=access_token, ) diff --git a/schematic/models/validate_attribute.py b/schematic/models/validate_attribute.py index a4f79b036..83eea50db 100644 --- a/schematic/models/validate_attribute.py +++ b/schematic/models/validate_attribute.py @@ -2014,6 +2014,7 @@ def filename_validation( manifest: pd.core.frame.DataFrame, access_token: str, project_scope: Optional[list] = None, + dataset_scope: Optional[str] = None, ): """ Purpose: @@ -2031,9 +2032,8 @@ def filename_validation( warnings = [] where_clauses = [] - rule_parts = val_rule.split(" ") - dataset_clause = f"parentId='{rule_parts[1]}'" + dataset_clause = f"parentId='{dataset_scope}'" where_clauses.append(dataset_clause) self._login( diff --git a/schematic/models/validate_manifest.py b/schematic/models/validate_manifest.py index 2ffc1d3b0..a45db5bf6 100644 --- a/schematic/models/validate_manifest.py +++ b/schematic/models/validate_manifest.py @@ -105,6 +105,7 @@ def validate_manifest_rules( dmge: DataModelGraphExplorer, restrict_rules: bool, project_scope: list[str], + dataset_scope: Optional[str], access_token: Optional[str] = None, ) -> (pd.core.frame.DataFrame, list[list[str]]): """ @@ -272,7 +273,7 @@ def validate_manifest_rules( ) elif validation_type == "filenameExists": vr_errors, vr_warnings = validation_method( - rule, manifest, access_token, project_scope + rule, manifest, access_token, project_scope, dataset_scope ) else: vr_errors, vr_warnings = validation_method( @@ -351,12 +352,13 @@ def validate_all( jsonSchema, restrict_rules, project_scope: List, + dataset_scope: List, access_token: str, ): # Run Validation Rules vm = ValidateManifest(errors, manifest, manifestPath, dmge, jsonSchema) manifest, vmr_errors, vmr_warnings = vm.validate_manifest_rules( - manifest, dmge, restrict_rules, project_scope, access_token + manifest, dmge, restrict_rules, project_scope, dataset_scope, access_token ) if vmr_errors: From 81c1343bf8543929606ec79f9fa017c80c9d132d Mon Sep 17 00:00:00 2001 From: GiaJordan Date: Tue, 3 Sep 2024 14:10:23 -0700 Subject: [PATCH 164/233] update api --- schematic_api/api/openapi/api.yaml | 16 ++++++++++++++++ schematic_api/api/routes.py | 4 ++++ 2 files changed, 20 insertions(+) diff --git a/schematic_api/api/openapi/api.yaml b/schematic_api/api/openapi/api.yaml index 15a3540d1..f14c9a927 100644 --- a/schematic_api/api/openapi/api.yaml +++ b/schematic_api/api/openapi/api.yaml @@ -303,6 +303,14 @@ paths: description: List, a subset of the projects contained within the asset view that are relevant for the current operation. Speeds up some operations that interact with Synapse. Relevant for validating manifests involving cross-manifest validation, but optional. example: ['syn23643250', 'syn47218127', 'syn47218347'] required: false + - in: query + name: dataset_scope + schema: + type: string + nullable: true + description: Specify a dataset to validate against for filename validation. + example: 'syn61682648' + required: false operationId: schematic_api.api.routes.validate_manifest_route responses: @@ -459,6 +467,14 @@ paths: description: List, a subset of the projects contained within the asset view that are relevant for the current operation. Speeds up some operations that interact with Synapse. example: ['syn23643250', 'syn47218127', 'syn47218347'] required: false + - in: query + name: dataset_scope + schema: + type: string + nullable: true + description: Specify a dataset to validate against for filename validation. + example: 'syn61682648' + required: false operationId: schematic_api.api.routes.submit_manifest_route responses: "200": diff --git a/schematic_api/api/routes.py b/schematic_api/api/routes.py index 59a6cd55e..a286f30bc 100644 --- a/schematic_api/api/routes.py +++ b/schematic_api/api/routes.py @@ -399,6 +399,7 @@ def validate_manifest_route( json_str=None, asset_view=None, project_scope=None, + dataset_scope=None, ): # Access token now stored in request header access_token = get_access_token() @@ -439,6 +440,7 @@ def validate_manifest_route( restrict_rules=restrict_rules, project_scope=project_scope, access_token=access_token, + dataset_scope=dataset_scope, ) res_dict = {"errors": errors, "warnings": warnings} @@ -458,6 +460,7 @@ def submit_manifest_route( data_type=None, hide_blanks=False, project_scope=None, + dataset_scope=None, table_column_names=None, annotation_keys=None, file_annotations_upload: bool = True, @@ -515,6 +518,7 @@ def submit_manifest_route( hide_blanks=hide_blanks, table_manipulation=table_manipulation, project_scope=project_scope, + dataset_scope=dataset_scope, table_column_names=table_column_names, annotation_keys=annotation_keys, file_annotations_upload=file_annotations_upload, From 22a9bb7beb2f7dd83b69f70c24f2d89e1867ca03 Mon Sep 17 00:00:00 2001 From: GiaJordan Date: Tue, 3 Sep 2024 14:11:29 -0700 Subject: [PATCH 165/233] update commands help --- schematic/help.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/schematic/help.py b/schematic/help.py index 7bf8c29a0..65c030ac8 100644 --- a/schematic/help.py +++ b/schematic/help.py @@ -3,10 +3,10 @@ #!/usr/bin/env python3 from typing import get_args + from schematic.utils.schema_utils import DisplayLabelType from schematic.visualization.tangled_tree import FigureType, TextType - DATA_MODEL_LABELS_DICT = { "display_label": "use the display name as a label, if it is valid (contains no blacklisted characters) otherwise will default to class_label.", "class_label": "default, use standard class or property label.", @@ -200,6 +200,9 @@ "project_scope": ( "Specify a comma-separated list of projects to search through for cross manifest validation." ), + "dataset_scope": ( + "Specify a dataset to validate against for filename validation." + ), "data_model_labels": DATA_MODEL_LABELS_HELP, }, } From 3e905a2fa528ac7023d703bfbd6d3953c9a0d1ba Mon Sep 17 00:00:00 2001 From: GiaJordan Date: Wed, 4 Sep 2024 09:48:46 -0700 Subject: [PATCH 166/233] update filename validation test --- tests/test_validation.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_validation.py b/tests/test_validation.py index d6b3971be..418a4b021 100644 --- a/tests/test_validation.py +++ b/tests/test_validation.py @@ -691,6 +691,7 @@ def test_filename_manifest(self, helpers, dmge): manifestPath=manifestPath, rootNode=rootNode, project_scope=["syn23643250"], + dataset_scope="syn61682648", ) # Check errors From 561185ac8d85741c5b8732df19815c60a75b03f2 Mon Sep 17 00:00:00 2001 From: linglp Date: Wed, 4 Sep 2024 14:30:24 -0400 Subject: [PATCH 167/233] edit error message --- schematic/schemas/data_model_graph.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/schematic/schemas/data_model_graph.py b/schematic/schemas/data_model_graph.py index d649e6306..55feec3c2 100644 --- a/schematic/schemas/data_model_graph.py +++ b/schematic/schemas/data_model_graph.py @@ -797,7 +797,7 @@ def get_node_validation_rules( node_validation_rules = self.graph.nodes[node_label]["validationRules"] except KeyError: raise ValueError( - f"{node_label} is not in the graph, please check that you are providing the proper node label" + f"{node_label} is not in the graph, please provide a proper node label" ) return node_validation_rules From 017025bec953bd24ef22cb6570fd76d9b25ad21a Mon Sep 17 00:00:00 2001 From: linglp Date: Wed, 4 Sep 2024 14:36:18 -0400 Subject: [PATCH 168/233] fix pylint error --- schematic/schemas/data_model_graph.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/schematic/schemas/data_model_graph.py b/schematic/schemas/data_model_graph.py index 55feec3c2..01a5f32ea 100644 --- a/schematic/schemas/data_model_graph.py +++ b/schematic/schemas/data_model_graph.py @@ -795,10 +795,10 @@ def get_node_validation_rules( try: node_validation_rules = self.graph.nodes[node_label]["validationRules"] - except KeyError: + except KeyError as exec: raise ValueError( f"{node_label} is not in the graph, please provide a proper node label" - ) + ) from exec return node_validation_rules From ae886a9827422cb4741495626828ec2dce2b5327 Mon Sep 17 00:00:00 2001 From: linglp Date: Wed, 4 Sep 2024 15:46:51 -0400 Subject: [PATCH 169/233] use from e --- schematic/schemas/data_model_graph.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/schematic/schemas/data_model_graph.py b/schematic/schemas/data_model_graph.py index 01a5f32ea..dab4b2d26 100644 --- a/schematic/schemas/data_model_graph.py +++ b/schematic/schemas/data_model_graph.py @@ -795,10 +795,10 @@ def get_node_validation_rules( try: node_validation_rules = self.graph.nodes[node_label]["validationRules"] - except KeyError as exec: + except KeyError as e: raise ValueError( f"{node_label} is not in the graph, please provide a proper node label" - ) from exec + ) from e return node_validation_rules From 692845714be5a9b9c3ed21ecc875108a1105e88e Mon Sep 17 00:00:00 2001 From: GiaJordan Date: Wed, 4 Sep 2024 12:47:33 -0700 Subject: [PATCH 170/233] add project scope to validation test --- tests/test_api.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/test_api.py b/tests/test_api.py index 6ce515622..c7c61874e 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -879,7 +879,11 @@ def test_validate_manifest( test_manifest_csv: str, request_headers: Dict[str, str], ) -> None: - params = {"schema_url": DATA_MODEL_JSON_LD, "restrict_rules": restrict_rules} + params = { + "schema_url": DATA_MODEL_JSON_LD, + "restrict_rules": restrict_rules, + "project_scope": "syn54126707", + } if json_str: params["json_str"] = json_str From a2b6fb386efe6134cf4ddd1fcbbcf67bbf6bf4fc Mon Sep 17 00:00:00 2001 From: GiaJordan Date: Wed, 4 Sep 2024 12:58:29 -0700 Subject: [PATCH 171/233] Revert "update callback util" This reverts commit 90b507823868f2e53d8f2b29eee1e6e985c8925f. --- schematic/utils/cli_utils.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/schematic/utils/cli_utils.py b/schematic/utils/cli_utils.py index 8999a594f..07c97af90 100644 --- a/schematic/utils/cli_utils.py +++ b/schematic/utils/cli_utils.py @@ -4,9 +4,10 @@ # pylint: disable=anomalous-backslash-in-string import logging -import re + +from typing import Any, Mapping, Sequence, Union, Optional from functools import reduce -from typing import Any, Mapping, Optional, Sequence, Union +import re logger = logging.getLogger(__name__) @@ -68,13 +69,13 @@ def parse_syn_ids( if not syn_ids: return None - synID_regex = re.compile("(syn\d+\,?)+") - valid = synID_regex.fullmatch(syn_ids) + project_regex = re.compile("(syn\d+\,?)+") + valid = project_regex.fullmatch(syn_ids) if not valid: raise ValueError( f"The provided list of project synID(s): {syn_ids}, is not formatted correctly. " - "\nPlease check your list of synIDs for errors." + "\nPlease check your list of projects for errors." ) return syn_ids.split(",") From acaf897b7fb832e86514321e6893c39a1b34088b Mon Sep 17 00:00:00 2001 From: linglp Date: Wed, 4 Sep 2024 16:28:27 -0400 Subject: [PATCH 172/233] move test to unit test folder --- tests/test_schemas.py | 77 -------------------------- tests/unit/test_data_model_graph.py | 86 +++++++++++++++++++++++++++++ 2 files changed, 86 insertions(+), 77 deletions(-) create mode 100644 tests/unit/test_data_model_graph.py diff --git a/tests/test_schemas.py b/tests/test_schemas.py index 25df1e9ba..f816b8881 100644 --- a/tests/test_schemas.py +++ b/tests/test_schemas.py @@ -617,83 +617,6 @@ def test_get_node_range(self): def test_get_node_required(self): return - @pytest.mark.parametrize( - "data_model", list(DATA_MODEL_DICT.keys()), ids=list(DATA_MODEL_DICT.values()) - ) - @pytest.mark.parametrize( - "node_label, node_display_name, expected_validation_rule", - [ - # Test case 1: node label is provided - ( - "PatientID", - None, - {"Biospecimen": "unique error", "Patient": "unique warning"}, - ), - ( - "CheckRegexListStrict", - None, - ["list strict", "regex match [a-f]"], - ), - # Test case 2: node label is not provided and display label is not part of the schema - ( - None, - "invalid display label", - [], - ), - # Test case 3: node label is not provided but a valid display label is provided - ( - None, - "Patient ID", - {"Biospecimen": "unique error", "Patient": "unique warning"}, - ), - ], - ) - def test_get_node_validation_rules_valid( - self, - helpers: Helpers, - data_model: str, - node_label: Optional[str], - node_display_name: Optional[str], - expected_validation_rule: Union[list[str], dict[str, str]], - ): - DMGE = helpers.get_data_model_graph_explorer(path=data_model) - - node_validation_rules = DMGE.get_node_validation_rules( - node_label=node_label, node_display_name=node_display_name - ) - assert node_validation_rules == expected_validation_rule - - @pytest.mark.parametrize( - "data_model", list(DATA_MODEL_DICT.keys()), ids=list(DATA_MODEL_DICT.values()) - ) - @pytest.mark.parametrize( - "node_label, node_display_name", - [ - # Test case 1: node label and node display name are not provided - ( - None, - None, - ), - # Test case 2: node label is not valid and display name is not provided - ( - "invalid node", - None, - ), - ], - ) - def test_get_node_validation_rules_invalid( - self, - helpers, - data_model, - node_label, - node_display_name, - ): - DMGE = helpers.get_data_model_graph_explorer(path=data_model) - with pytest.raises(ValueError): - DMGE.get_node_validation_rules( - node_label=node_label, node_display_name=node_display_name - ) - def test_get_subgraph_by_edge_type(self): return diff --git a/tests/unit/test_data_model_graph.py b/tests/unit/test_data_model_graph.py new file mode 100644 index 000000000..27b55b6cf --- /dev/null +++ b/tests/unit/test_data_model_graph.py @@ -0,0 +1,86 @@ +from typing import Optional, Union + +import pytest + +from tests.conftest import Helpers + +DATA_MODEL_DICT = {"example.model.csv": "CSV", "example.model.jsonld": "JSONLD"} + + +class TestDataModelGraphExplorer: + @pytest.mark.parametrize( + "data_model", list(DATA_MODEL_DICT.keys()), ids=list(DATA_MODEL_DICT.values()) + ) + @pytest.mark.parametrize( + "node_label, node_display_name, expected_validation_rule", + [ + # Test case 1: node label is provided + ( + "PatientID", + None, + {"Biospecimen": "unique error", "Patient": "unique warning"}, + ), + ( + "CheckRegexListStrict", + None, + ["list strict", "regex match [a-f]"], + ), + # Test case 2: node label is not provided and display label is not part of the schema + ( + None, + "invalid display label", + [], + ), + # Test case 3: node label is not provided but a valid display label is provided + ( + None, + "Patient ID", + {"Biospecimen": "unique error", "Patient": "unique warning"}, + ), + ], + ) + def test_get_node_validation_rules_valid( + self, + helpers: Helpers, + data_model: str, + node_label: Optional[str], + node_display_name: Optional[str], + expected_validation_rule: Union[list[str], dict[str, str]], + ): + DMGE = helpers.get_data_model_graph_explorer(path=data_model) + + node_validation_rules = DMGE.get_node_validation_rules( + node_label=node_label, node_display_name=node_display_name + ) + assert node_validation_rules == expected_validation_rule + + @pytest.mark.parametrize( + "data_model", list(DATA_MODEL_DICT.keys()), ids=list(DATA_MODEL_DICT.values()) + ) + @pytest.mark.parametrize( + "node_label, node_display_name", + [ + # Test case 1: node label and node display name are not provided + ( + None, + None, + ), + # Test case 2: node label is not valid and display name is not provided + ( + "invalid node", + None, + ), + ], + ) + def test_get_node_validation_rules_invalid( + self, + helpers, + data_model, + node_label, + node_display_name, + ): + DMGE = helpers.get_data_model_graph_explorer(path=data_model) + with pytest.raises(ValueError): + DMGE.get_node_validation_rules( + node_label=node_label, node_display_name=node_display_name + ) From e974fd613fd5fbd2e3837fd01863d2179f9f30ae Mon Sep 17 00:00:00 2001 From: linglp Date: Wed, 4 Sep 2024 16:30:15 -0400 Subject: [PATCH 173/233] remove unnecessary import --- tests/test_schemas.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/test_schemas.py b/tests/test_schemas.py index f816b8881..409c653b0 100644 --- a/tests/test_schemas.py +++ b/tests/test_schemas.py @@ -3,7 +3,6 @@ import os import random from copy import deepcopy -from typing import Optional, Union import networkx as nx import numpy as np @@ -37,7 +36,6 @@ get_label_from_display_name, parse_validation_rules, ) -from tests.conftest import Helpers logging.basicConfig(level=logging.DEBUG) logger = logging.getLogger(__name__) From ce267f8bf96a556b5bd663e48873cc9479560355 Mon Sep 17 00:00:00 2001 From: linglp Date: Wed, 4 Sep 2024 16:36:16 -0400 Subject: [PATCH 174/233] use key_error instead of e --- schematic/schemas/data_model_graph.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/schematic/schemas/data_model_graph.py b/schematic/schemas/data_model_graph.py index dab4b2d26..64cb59a65 100644 --- a/schematic/schemas/data_model_graph.py +++ b/schematic/schemas/data_model_graph.py @@ -795,10 +795,10 @@ def get_node_validation_rules( try: node_validation_rules = self.graph.nodes[node_label]["validationRules"] - except KeyError as e: + except KeyError as key_error: raise ValueError( f"{node_label} is not in the graph, please provide a proper node label" - ) from e + ) from key_error return node_validation_rules From 6be7c7482d020c88425e248912a80c578c489dfd Mon Sep 17 00:00:00 2001 From: GiaJordan Date: Thu, 5 Sep 2024 09:53:57 -0700 Subject: [PATCH 175/233] update default value --- schematic/models/validate_manifest.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/schematic/models/validate_manifest.py b/schematic/models/validate_manifest.py index a45db5bf6..d3f4a34fa 100644 --- a/schematic/models/validate_manifest.py +++ b/schematic/models/validate_manifest.py @@ -105,7 +105,7 @@ def validate_manifest_rules( dmge: DataModelGraphExplorer, restrict_rules: bool, project_scope: list[str], - dataset_scope: Optional[str], + dataset_scope: Optional[str] = None, access_token: Optional[str] = None, ) -> (pd.core.frame.DataFrame, list[list[str]]): """ @@ -352,7 +352,7 @@ def validate_all( jsonSchema, restrict_rules, project_scope: List, - dataset_scope: List, + dataset_scope: str, access_token: str, ): # Run Validation Rules From a861fbfa94865e747df5127fd8f17339b8eb46b7 Mon Sep 17 00:00:00 2001 From: GiaJordan Date: Thu, 5 Sep 2024 10:27:23 -0700 Subject: [PATCH 176/233] update params but keep what is parameterized the same --- tests/test_api.py | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/tests/test_api.py b/tests/test_api.py index c7c61874e..8e2e9dc86 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -76,6 +76,11 @@ def test_manifest_json(helpers) -> str: return test_manifest_path +@pytest.fixture(scope="class") +def patient_manifest_json_str() -> str: + return '[{"Patient ID": 123, "Sex": "Female", "Year of Birth": "", "Diagnosis": "Healthy", "Component": "Patient", "Cancer Type": "Breast", "Family History": "Breast, Lung"}]' + + def get_MockComponent_attribute() -> Generator[str, None, None]: """ Yield all of the mock conponent attributes one at a time @@ -863,21 +868,25 @@ def test_populate_manifest( assert isinstance(response_dt[0], str) assert response_dt[0].startswith("https://docs.google.com/") - @pytest.mark.parametrize("restrict_rules", [False, True, None]) @pytest.mark.parametrize( - "json_str", + "json_str_fixture,restrict_rules", [ - None, - '[{"Patient ID": 123, "Sex": "Female", "Year of Birth": "", "Diagnosis": "Healthy", "Component": "Patient", "Cancer Type": "Breast", "Family History": "Breast, Lung"}]', + (None, False), + (None, True), + (None, None), + ("patient_manifest_json_str", False), + ("patient_manifest_json_str", True), + ("patient_manifest_json_str", None), ], ) def test_validate_manifest( self, client: FlaskClient, - json_str: Union[str, None], + json_str_fixture: Union[str, None], restrict_rules: Union[bool, None], test_manifest_csv: str, request_headers: Dict[str, str], + request: pytest.FixtureRequest, ) -> None: params = { "schema_url": DATA_MODEL_JSON_LD, @@ -885,8 +894,8 @@ def test_validate_manifest( "project_scope": "syn54126707", } - if json_str: - params["json_str"] = json_str + if json_str_fixture: + params["json_str"] = request.getfixturevalue(json_str_fixture) params["data_type"] = "Patient" response = client.post( "http://localhost:3001/v1/model/validate", query_string=params From 25a42c0f0574073d716d01bce5442d99e96636c0 Mon Sep 17 00:00:00 2001 From: GiaJordan Date: Thu, 5 Sep 2024 11:03:19 -0700 Subject: [PATCH 177/233] update test structure/parameterization --- tests/test_api.py | 88 ++++++++++++++++++++++++++--------------------- 1 file changed, 49 insertions(+), 39 deletions(-) diff --git a/tests/test_api.py b/tests/test_api.py index 8e2e9dc86..6c55f29db 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -39,7 +39,7 @@ def client(app: flask.Flask) -> Generator[FlaskClient, None, None]: @pytest.fixture(scope="class") -def test_manifest_csv(helpers) -> str: +def valid_test_manifest_csv(helpers) -> str: test_manifest_path = helpers.get_data_path("mock_manifests/Valid_Test_Manifest.csv") return test_manifest_path @@ -845,10 +845,10 @@ def test_generate_manifest_data_type_not_found(self, client: FlaskClient) -> Non assert "LookupError" in str(response.data) def test_populate_manifest( - self, client: FlaskClient, test_manifest_csv: str + self, client: FlaskClient, valid_test_manifest_csv: str ) -> None: # test manifest - test_manifest_data = open(test_manifest_csv, "rb") + test_manifest_data = open(valid_test_manifest_csv, "rb") params = { "data_type": "MockComponent", @@ -869,14 +869,14 @@ def test_populate_manifest( assert response_dt[0].startswith("https://docs.google.com/") @pytest.mark.parametrize( - "json_str_fixture,restrict_rules", + "json_str_fixture,restrict_rules,data_type,update_headers,test_manifest_fixture", [ - (None, False), - (None, True), - (None, None), - ("patient_manifest_json_str", False), - ("patient_manifest_json_str", True), - ("patient_manifest_json_str", None), + (None, False, "MockComponent", True, "valid_test_manifest_csv"), + (None, True, "MockComponent", True, "valid_test_manifest_csv"), + (None, None, "MockComponent", True, "valid_test_manifest_csv"), + ("patient_manifest_json_str", False, "Patient", False, None), + ("patient_manifest_json_str", True, "Patient", False, None), + ("patient_manifest_json_str", None, "Patient", False, None), ], ) def test_validate_manifest( @@ -884,53 +884,63 @@ def test_validate_manifest( client: FlaskClient, json_str_fixture: Union[str, None], restrict_rules: Union[bool, None], - test_manifest_csv: str, + data_type: str, + update_headers: bool, + test_manifest_fixture: str, request_headers: Dict[str, str], request: pytest.FixtureRequest, ) -> None: + # GIVEN a set of common prameters params = { "schema_url": DATA_MODEL_JSON_LD, "restrict_rules": restrict_rules, "project_scope": "syn54126707", } - if json_str_fixture: - params["json_str"] = request.getfixturevalue(json_str_fixture) - params["data_type"] = "Patient" - response = client.post( - "http://localhost:3001/v1/model/validate", query_string=params - ) - response_dt = json.loads(response.data) - assert response.status_code == 200 - - else: - params["data_type"] = "MockComponent" + # AND a set of test specific parameters + params["data_type"] = data_type + # AND the appropriate headers for the test + if update_headers: request_headers.update( {"Content-Type": "multipart/form-data", "Accept": "application/json"} ) - # test uploading a csv file - response_csv = client.post( - "http://localhost:3001/v1/model/validate", - query_string=params, - data={"file_name": (open(test_manifest_csv, "rb"), "test.csv")}, - headers=request_headers, - ) - response_dt = json.loads(response_csv.data) - assert response_csv.status_code == 200 + # AND a test manifest as a json string + params["json_str"] = ( + request.getfixturevalue(json_str_fixture) if json_str_fixture else None + ) - # test uploading a json file - # change data type to patient since the testing json manifest is using Patient component - # WILL DEPRECATE uploading a json file for validation - # params["data_type"] = "Patient" - # response_json = client.post('http://localhost:3001/v1/model/validate', query_string=params, data={"file_name": (open(test_manifest_json, 'rb'), "test.json")}, headers=headers) - # response_dt = json.loads(response_json.data) - # assert response_json.status_code == 200 + # OR a test manifest as a file + data = None + if test_manifest_fixture: + test_manifest_path = request.getfixturevalue(test_manifest_fixture) + data = {"file_name": (open(test_manifest_path, "rb"), "test.csv")} + # WHEN the manifest is validated + response = client.post( + "http://localhost:3001/v1/model/validate", + query_string=params, + data=data, + headers=request_headers, + ) + + # THEN the request should be successful + assert response.status_code == 200 + + # AND the response should contain the expected error and warning lists + response_dt = json.loads(response.data) assert "errors" in response_dt.keys() assert "warnings" in response_dt.keys() + # test uploading a json file + # change data type to patient since the testing json manifest is using Patient component + # WILL DEPRECATE uploading a json file for validation + # params["data_type"] = "Patient" + # response_json = client.post('http://localhost:3001/v1/model/validate', query_string=params, data={"file_name": (open(test_manifest_json, 'rb'), "test.json")}, headers=headers) + # response_dt = json.loads(response_json.data) + # assert response_json.status_code == 200 + @pytest.mark.synapse_credentials_needed def test_get_datatype_manifest( self, client: FlaskClient, request_headers: Dict[str, str] @@ -1121,7 +1131,7 @@ def test_submit_manifest_table_and_file_replace( "data_type, manifest_path_fixture", [ ("Biospecimen", "test_manifest_submit"), - ("MockComponent", "test_manifest_csv"), + ("MockComponent", "valid_test_manifest_csv"), ], ) def test_submit_manifest_file_only_replace( From e274b86fdfb7a483dca9c75a478001907dbb3bc3 Mon Sep 17 00:00:00 2001 From: GiaJordan Date: Thu, 5 Sep 2024 11:04:50 -0700 Subject: [PATCH 178/233] remove comment block --- tests/test_api.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/tests/test_api.py b/tests/test_api.py index 6c55f29db..2ec9dfc97 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -933,14 +933,6 @@ def test_validate_manifest( assert "errors" in response_dt.keys() assert "warnings" in response_dt.keys() - # test uploading a json file - # change data type to patient since the testing json manifest is using Patient component - # WILL DEPRECATE uploading a json file for validation - # params["data_type"] = "Patient" - # response_json = client.post('http://localhost:3001/v1/model/validate', query_string=params, data={"file_name": (open(test_manifest_json, 'rb'), "test.json")}, headers=headers) - # response_dt = json.loads(response_json.data) - # assert response_json.status_code == 200 - @pytest.mark.synapse_credentials_needed def test_get_datatype_manifest( self, client: FlaskClient, request_headers: Dict[str, str] From 022df3ac32ba4442f1e70bef6dcef0cd20b5a599 Mon Sep 17 00:00:00 2001 From: GiaJordan Date: Thu, 5 Sep 2024 11:12:00 -0700 Subject: [PATCH 179/233] update param order --- tests/test_api.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/test_api.py b/tests/test_api.py index 2ec9dfc97..dbb6bae8d 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -869,24 +869,24 @@ def test_populate_manifest( assert response_dt[0].startswith("https://docs.google.com/") @pytest.mark.parametrize( - "json_str_fixture,restrict_rules,data_type,update_headers,test_manifest_fixture", + "json_str_fixture,test_manifest_fixture,restrict_rules,data_type,update_headers", [ - (None, False, "MockComponent", True, "valid_test_manifest_csv"), - (None, True, "MockComponent", True, "valid_test_manifest_csv"), - (None, None, "MockComponent", True, "valid_test_manifest_csv"), - ("patient_manifest_json_str", False, "Patient", False, None), - ("patient_manifest_json_str", True, "Patient", False, None), - ("patient_manifest_json_str", None, "Patient", False, None), + (None, "valid_test_manifest_csv", False, "MockComponent", True), + (None, "valid_test_manifest_csv", True, "MockComponent", True), + (None, "valid_test_manifest_csv", None, "MockComponent", True), + ("patient_manifest_json_str", None, False, "Patient", False), + ("patient_manifest_json_str", None, True, "Patient", False), + ("patient_manifest_json_str", None, None, "Patient", False), ], ) def test_validate_manifest( self, client: FlaskClient, json_str_fixture: Union[str, None], + test_manifest_fixture: Union[str, None], restrict_rules: Union[bool, None], data_type: str, update_headers: bool, - test_manifest_fixture: str, request_headers: Dict[str, str], request: pytest.FixtureRequest, ) -> None: From 77b4075790f717cca260b184a080f65817eeaa0c Mon Sep 17 00:00:00 2001 From: GiaJordan Date: Thu, 5 Sep 2024 11:27:35 -0700 Subject: [PATCH 180/233] update params --- tests/test_api.py | 43 ++++++++++++++++++++++++------------------- 1 file changed, 24 insertions(+), 19 deletions(-) diff --git a/tests/test_api.py b/tests/test_api.py index dbb6bae8d..5c41bc430 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -869,43 +869,42 @@ def test_populate_manifest( assert response_dt[0].startswith("https://docs.google.com/") @pytest.mark.parametrize( - "json_str_fixture,test_manifest_fixture,restrict_rules,data_type,update_headers", + "json_str_fixture, test_manifest_fixture, data_type, update_headers, project_scope, dataset_scope", [ - (None, "valid_test_manifest_csv", False, "MockComponent", True), - (None, "valid_test_manifest_csv", True, "MockComponent", True), - (None, "valid_test_manifest_csv", None, "MockComponent", True), - ("patient_manifest_json_str", None, False, "Patient", False), - ("patient_manifest_json_str", None, True, "Patient", False), - ("patient_manifest_json_str", None, None, "Patient", False), + ( + None, + "valid_test_manifest_csv", + "MockComponent", + True, + "syn54126707", + None, + ), + ("patient_manifest_json_str", None, "Patient", False, None, None), ], ) + @pytest.mark.parametrize("restrict_rules", [True, False, None]) def test_validate_manifest( self, client: FlaskClient, json_str_fixture: Union[str, None], test_manifest_fixture: Union[str, None], - restrict_rules: Union[bool, None], data_type: str, update_headers: bool, + project_scope: Union[str, None], + dataset_scope: Union[str, None], + restrict_rules: Union[bool, None], request_headers: Dict[str, str], request: pytest.FixtureRequest, ) -> None: - # GIVEN a set of common prameters + # GIVEN a set of appropriate test prameters params = { "schema_url": DATA_MODEL_JSON_LD, "restrict_rules": restrict_rules, - "project_scope": "syn54126707", + "project_scope": project_scope, + "dataset_scope": dataset_scope, + "data_type": data_type, } - # AND a set of test specific parameters - params["data_type"] = data_type - - # AND the appropriate headers for the test - if update_headers: - request_headers.update( - {"Content-Type": "multipart/form-data", "Accept": "application/json"} - ) - # AND a test manifest as a json string params["json_str"] = ( request.getfixturevalue(json_str_fixture) if json_str_fixture else None @@ -917,6 +916,12 @@ def test_validate_manifest( test_manifest_path = request.getfixturevalue(test_manifest_fixture) data = {"file_name": (open(test_manifest_path, "rb"), "test.csv")} + # AND the appropriate headers for the test + if update_headers: + request_headers.update( + {"Content-Type": "multipart/form-data", "Accept": "application/json"} + ) + # WHEN the manifest is validated response = client.post( "http://localhost:3001/v1/model/validate", From 70ce8e79de8205ab0483caea5a9197ef7b6cacd4 Mon Sep 17 00:00:00 2001 From: GiaJordan Date: Thu, 5 Sep 2024 11:40:35 -0700 Subject: [PATCH 181/233] add new test case to validation test --- tests/test_api.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/tests/test_api.py b/tests/test_api.py index 5c41bc430..72734ef50 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -44,6 +44,14 @@ def valid_test_manifest_csv(helpers) -> str: return test_manifest_path +@pytest.fixture(scope="class") +def invalid_filename_manifest_csv(helpers) -> str: + test_manifest_path = helpers.get_data_path( + "mock_manifests/InvalidFilenameManifest.csv" + ) + return test_manifest_path + + @pytest.fixture(scope="class") def test_manifest_submit(helpers) -> str: test_manifest_path = helpers.get_data_path( @@ -880,6 +888,14 @@ def test_populate_manifest( None, ), ("patient_manifest_json_str", None, "Patient", False, None, None), + ( + None, + "invalid_filename_manifest_csv", + "MockFilename", + True, + "syn23643250", + "syn61682648", + ), ], ) @pytest.mark.parametrize("restrict_rules", [True, False, None]) @@ -896,7 +912,7 @@ def test_validate_manifest( request_headers: Dict[str, str], request: pytest.FixtureRequest, ) -> None: - # GIVEN a set of appropriate test prameters + # GIVEN a set of test prameters params = { "schema_url": DATA_MODEL_JSON_LD, "restrict_rules": restrict_rules, From 177aba3e98e1e6c5adcb316eb84a4ad09daa599b Mon Sep 17 00:00:00 2001 From: GiaJordan Date: Thu, 5 Sep 2024 13:25:42 -0700 Subject: [PATCH 182/233] add test manifest --- tests/data/mock_manifests/ValidFilenameManifest.csv | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 tests/data/mock_manifests/ValidFilenameManifest.csv diff --git a/tests/data/mock_manifests/ValidFilenameManifest.csv b/tests/data/mock_manifests/ValidFilenameManifest.csv new file mode 100644 index 000000000..9ba58740b --- /dev/null +++ b/tests/data/mock_manifests/ValidFilenameManifest.csv @@ -0,0 +1,5 @@ +Component,Filename,entityId +MockFilename,schematic - main/TestSubmitMockFilename/txt1.txt,syn62822369 +MockFilename,schematic - main/TestSubmitMockFilename/txt2.txt,syn62822368 +MockFilename,schematic - main/TestSubmitMockFilename/txt3.txt,syn62822366 +MockFilename,schematic - main/TestSubmitMockFilename/txt4.txt,syn62822364 From a5f6a7313f994b0cd375a470a8b7378cc1278cb3 Mon Sep 17 00:00:00 2001 From: GiaJordan Date: Thu, 5 Sep 2024 13:27:08 -0700 Subject: [PATCH 183/233] add api submission test --- tests/test_api.py | 33 ++++++++++++++++++++++----------- 1 file changed, 22 insertions(+), 11 deletions(-) diff --git a/tests/test_api.py b/tests/test_api.py index 72734ef50..47ad6cc5b 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -44,6 +44,14 @@ def valid_test_manifest_csv(helpers) -> str: return test_manifest_path +@pytest.fixture(scope="class") +def valid_filename_manifest_csv(helpers) -> str: + test_manifest_path = helpers.get_data_path( + "mock_manifests/ValidFilenameManifest.csv" + ) + return test_manifest_path + + @pytest.fixture(scope="class") def invalid_filename_manifest_csv(helpers) -> str: test_manifest_path = helpers.get_data_path( @@ -1258,32 +1266,35 @@ def test_submit_manifest_w_file_and_entities( @pytest.mark.synapse_credentials_needed @pytest.mark.submission - def test_submit_manifest_table_and_file_upsert( + def test_submit_and_validate_filebased_manifest( self, client: FlaskClient, request_headers: Dict[str, str], - test_upsert_manifest_csv: str, + valid_filename_manifest_csv: str, ) -> None: + # GIVEN the appropriate upload parameters params = { "schema_url": DATA_MODEL_JSON_LD, - "data_type": "MockRDB", + "data_type": "MockFilename", "restrict_rules": False, - "manifest_record_type": "table_and_file", - "asset_view": "syn51514557", - "dataset_id": "syn51514551", - "table_manipulation": "upsert", + "manifest_record_type": "file_and_entities", + "asset_view": "syn23643253", + "dataset_id": "syn62822337", + "project_scope": "syn23643250", + "dataset_scope": "syn62822337", "data_model_labels": "class_label", - # have to set table_column_names to display_name to ensure upsert feature works - "table_column_names": "display_name", + "table_column_names": "class_label", } - # test uploading a csv file + # WHEN a filebased manifest is validated with the filenameExists rule and uploaded response_csv = client.post( "http://localhost:3001/v1/model/submit", query_string=params, - data={"file_name": (open(test_upsert_manifest_csv, "rb"), "test.csv")}, + data={"file_name": (open(valid_filename_manifest_csv, "rb"), "test.csv")}, headers=request_headers, ) + + # THEN the validation and submission should be successful assert response_csv.status_code == 200 From b3b01f10cf54f293c440bea3a994e30c09dbcfd2 Mon Sep 17 00:00:00 2001 From: GiaJordan Date: Thu, 5 Sep 2024 13:31:01 -0700 Subject: [PATCH 184/233] raise exception when no dataset provided --- schematic/models/validate_attribute.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/schematic/models/validate_attribute.py b/schematic/models/validate_attribute.py index 83eea50db..03f33c5d9 100644 --- a/schematic/models/validate_attribute.py +++ b/schematic/models/validate_attribute.py @@ -2028,6 +2028,12 @@ def filename_validation( errors: list[str] Error details for further storage. warnings: list[str] Warning details for further storage. """ + + if dataset_scope is None: + raise ValueError( + "A dataset is required to be specified for filename validation" + ) + errors = [] warnings = [] From e13990aef36fc1e99192ee9ce23b46c5d04b2c29 Mon Sep 17 00:00:00 2001 From: GiaJordan Date: Thu, 5 Sep 2024 13:33:53 -0700 Subject: [PATCH 185/233] add test for raised exception --- tests/test_validation.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/tests/test_validation.py b/tests/test_validation.py index 418a4b021..69ade10ca 100644 --- a/tests/test_validation.py +++ b/tests/test_validation.py @@ -722,6 +722,21 @@ def test_filename_manifest(self, helpers, dmge): assert len(errors) == 2 assert len(warnings) == 0 + def test_filename_manifest_exception(self, helpers, dmge): + metadataModel = get_metadataModel(helpers, model_name="example.model.jsonld") + + manifestPath = helpers.get_data_path( + "mock_manifests/InvalidFilenameManifest.csv" + ) + rootNode = "MockFilename" + + with pytest.raises(ValueError): + errors, warnings = metadataModel.validateModelManifest( + manifestPath=manifestPath, + rootNode=rootNode, + project_scope=["syn23643250"], + ) + def test_missing_column(self, helpers, dmge: DataModelGraph): """Test that a manifest missing a column returns the proper error.""" model_name = "example.model.csv" From ff6de7e18dc2c68aa7e9bc68e7631a6e4e43b2b1 Mon Sep 17 00:00:00 2001 From: GiaJordan Date: Thu, 5 Sep 2024 13:38:44 -0700 Subject: [PATCH 186/233] parametrize test --- tests/integration/test_metadata_model.py | 31 +++++++++++++++++++----- 1 file changed, 25 insertions(+), 6 deletions(-) diff --git a/tests/integration/test_metadata_model.py b/tests/integration/test_metadata_model.py index 26602bb92..04d2768b5 100644 --- a/tests/integration/test_metadata_model.py +++ b/tests/integration/test_metadata_model.py @@ -1,6 +1,7 @@ import logging from contextlib import nullcontext as does_not_raise +import pytest from pytest_mock import MockerFixture from schematic.store.synapse import SynapseStorage @@ -11,7 +12,26 @@ class TestMetadataModel: - def test_submit_filebased_manifest(self, helpers, mocker: MockerFixture): + @pytest.mark.parametrize( + "manifest_path, dataset_id, validate_component, expected_manifest_id", + [ + ( + "mock_manifests/filepath_submission_test_manifest.csv", + "syn62276880", + None, + "syn62280543", + ), + ], + ) + def test_submit_filebased_manifest( + self, + helpers, + manifest_path, + dataset_id, + validate_component, + expected_manifest_id, + mocker: MockerFixture, + ): # spys spy_upload_file_as_csv = mocker.spy(SynapseStorage, "upload_manifest_as_csv") spy_upload_file_as_table = mocker.spy( @@ -26,24 +46,23 @@ def test_submit_filebased_manifest(self, helpers, mocker: MockerFixture): meta_data_model = metadata_model(helpers, "class_label") # AND a filebased test manifset - manifest_path = helpers.get_data_path( - "mock_manifests/filepath_submission_test_manifest.csv" - ) + manifest_path = helpers.get_data_path(manifest_path) # WHEN the manifest it submitted # THEN submission should complete without error with does_not_raise(): manifest_id = meta_data_model.submit_metadata_manifest( manifest_path=manifest_path, - dataset_id="syn62276880", + dataset_id=dataset_id, manifest_record_type="file_and_entities", restrict_rules=False, file_annotations_upload=True, hide_blanks=False, + validate_component=validate_component, ) # AND the manifest should be submitted to the correct place - assert manifest_id == "syn62280543" + assert manifest_id == expected_manifest_id # AND the manifest should be uploaded as a CSV spy_upload_file_as_csv.assert_called_once() From 39f92364e2483071aff09e95891f85ccd05cb173 Mon Sep 17 00:00:00 2001 From: GiaJordan Date: Thu, 5 Sep 2024 13:53:36 -0700 Subject: [PATCH 187/233] add test condition --- tests/integration/test_metadata_model.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/tests/integration/test_metadata_model.py b/tests/integration/test_metadata_model.py index 04d2768b5..ff221d731 100644 --- a/tests/integration/test_metadata_model.py +++ b/tests/integration/test_metadata_model.py @@ -13,13 +13,21 @@ class TestMetadataModel: @pytest.mark.parametrize( - "manifest_path, dataset_id, validate_component, expected_manifest_id", + "manifest_path, dataset_id, validate_component, expected_manifest_id, dataset_scope", [ ( "mock_manifests/filepath_submission_test_manifest.csv", "syn62276880", None, "syn62280543", + None, + ), + ( + "mock_manifests/ValidFilenameManifest.csv", + "syn62822337", + "MockFilename", + "syn62822975", + "syn62822337", ), ], ) @@ -30,6 +38,7 @@ def test_submit_filebased_manifest( dataset_id, validate_component, expected_manifest_id, + dataset_scope, mocker: MockerFixture, ): # spys @@ -59,6 +68,7 @@ def test_submit_filebased_manifest( file_annotations_upload=True, hide_blanks=False, validate_component=validate_component, + dataset_scope=dataset_scope, ) # AND the manifest should be submitted to the correct place From b3debb1595f4ecfa13c7aa9de5d7bb20438ccfa5 Mon Sep 17 00:00:00 2001 From: linglp Date: Thu, 5 Sep 2024 17:22:16 -0400 Subject: [PATCH 188/233] pack else statement in a separate function; add tests --- schematic/store/synapse.py | 89 ++++++++++++------ tests/integration/test_store_synapse.py | 120 ++++++++++++++++++++++++ 2 files changed, 181 insertions(+), 28 deletions(-) create mode 100644 tests/integration/test_store_synapse.py diff --git a/schematic/store/synapse.py b/schematic/store/synapse.py index 7d7200311..c34c98eec 100644 --- a/schematic/store/synapse.py +++ b/schematic/store/synapse.py @@ -1520,6 +1520,59 @@ async def store_async_annotation(self, annotation_dict: dict) -> Annotations: ) return await annotation_class.store_async(synapse_client=self.syn) + def process_row_annotations( + self, + dmge: DataModelGraphExplorer, + metadataSyn: dict, + hideBlanks: bool, + csv_list_regex: str, + annos: dict, + annotation_keys: str, + ): + """processes metadata annotations + + Args: + dmge (DataModelGraphExplorer): data model graph explorer + metadataSyn (dict): metadata used for Synapse storage + hideBlanks (bool): if true, does not upload annotation keys with blank values. + csv_list_regex (str): + annos (dict): + annotation_keys (str): display_label/class_label + + Returns: + dict: annotations as a dictionary + """ + for anno_k, anno_v in metadataSyn.items(): + # Remove keys with nan or empty string values from dict of annotations to be uploaded + # if present on current data annotation + if hideBlanks and ( + (isinstance(anno_v, str) and anno_v.strip() == "") + or (isinstance(anno_v, float) and np.isnan(anno_v)) + ): + annos.pop(anno_k) if anno_k in annos.keys() else annos + # Otherwise save annotation as approrpriate + else: + if annotation_keys == "display_label": + node_validation_rules = dmge.get_node_validation_rules( + node_display_name=anno_k + ) + else: + node_validation_rules = dmge.get_node_validation_rules( + node_label=anno_k + ) + + if isinstance(anno_v, float) and np.isnan(anno_v): + annos[anno_k] = "" + elif ( + isinstance(anno_v, str) + and re.fullmatch(csv_list_regex, anno_v) + and rule_in_rule_list("list", node_validation_rules) + ): + annos[anno_k] = anno_v.split(",") + else: + annos[anno_k] = anno_v + return annos + @async_missing_entity_handler async def format_row_annotations( self, @@ -1573,34 +1626,14 @@ async def format_row_annotations( annos = await self.get_async_annotation(entityId) csv_list_regex = comma_separated_list_regex() - for anno_k, anno_v in metadataSyn.items(): - # Remove keys with nan or empty string values from dict of annotations to be uploaded - # if present on current data annotation - if hideBlanks and ( - anno_v == "" or (isinstance(anno_v, float) and np.isnan(anno_v)) - ): - annos.pop(anno_k) if anno_k in annos.keys() else annos - # Otherwise save annotation as approrpriate - else: - if annotation_keys == "display_label": - node_validation_rules = dmge.get_node_validation_rules( - node_display_name=anno_k - ) - else: - node_validation_rules = dmge.get_node_validation_rules( - node_label=anno_k - ) - - if isinstance(anno_v, float) and np.isnan(anno_v): - annos[anno_k] = "" - elif ( - isinstance(anno_v, str) - and re.fullmatch(csv_list_regex, anno_v) - and rule_in_rule_list("list", node_validation_rules) - ): - annos[anno_k] = anno_v.split(",") - else: - annos[anno_k] = anno_v + annos = self.process_row_annotations( + dmge=dmge, + metadataSyn=metadataSyn, + hideBlanks=hideBlanks, + csv_list_regex=csv_list_regex, + annos=annos, + annotation_keys=annotation_keys, + ) return annos diff --git a/tests/integration/test_store_synapse.py b/tests/integration/test_store_synapse.py new file mode 100644 index 000000000..cd087c9ff --- /dev/null +++ b/tests/integration/test_store_synapse.py @@ -0,0 +1,120 @@ +from unittest.mock import MagicMock + +import numpy as np +import pytest + +from schematic.schemas.data_model_graph import DataModelGraphExplorer +from schematic.utils.validate_utils import comma_separated_list_regex +from tests.conftest import Helpers + + +@pytest.fixture +def metadataSyn(): + return { + "key1": "value1", + "key2": np.nan, + "key3": "val1,val2,val3", # Simulate a CSV-like string + "key4": "another_value", + } + + +@pytest.fixture +def annos(): + return {"key1": "old_value1", "key2": "old_value2", "key3": "old_value3"} + + +@pytest.fixture(name="dmge", scope="function") +def DMGE(helpers: Helpers) -> DataModelGraphExplorer: + """Fixture to instantiate a DataModelGraphExplorer object.""" + dmge = helpers.get_data_model_graph_explorer(path="example.model.jsonld") + return dmge + + +class TestStoreSynapse: + @pytest.mark.parametrize("hideBlanks", [True, False]) + @pytest.mark.parametrize( + "label_options", + ["display_label", "class_label"], + ids=["display_label", "class_label"], + ) + def test_process_row_annotations_hide_blanks( + self, dmge, synapse_store, annos, hideBlanks, label_options + ): + metadata_syn_with_blanks = { + "PatientID": "value1", + "Sex": "value2", + "Diagnosis": "", # Blank value (empty string) + "FamilyHistory": 3, # Non-string value + "YearofBirth": np.nan, # Blank value (NaN) + "CancerType": " ", # Blank value (whitespace string) + } + annos = { + "PatientID": "value1", + "Sex": "value2", + "Diagnosis": "value3", + "FamilyHistory": "value4", + "YearofBirth": "value5", + "CancerType": "value6", + } + comma_separated_list = comma_separated_list_regex() + process_row_annos = synapse_store.process_row_annotations( + dmge=dmge, + metadataSyn=metadata_syn_with_blanks, + csv_list_regex=comma_separated_list, + hideBlanks=hideBlanks, + annos=annos, + annotation_keys=label_options, + ) + # make sure that empty keys are not added if hideBlanks is True + if hideBlanks: + assert ( + "Diagnosis" + and "YearofBirth" + and "CancerType" not in process_row_annos.keys() + ) + assert ( + "Diagnosis" + and "YearofBirth" + and "CancerType" + and "PatientID" + and "Sex" + and "FamilyHistory" in process_row_annos.keys() + ) + # make sure that annotations already in the dictionary are not overwritten + assert "PatientID" and "Sex" in process_row_annos.keys() + + @pytest.mark.parametrize( + "label_options", + ["display_label"], + ids=["display_label"], + ) + @pytest.mark.parametrize("hideBlanks", [True, False]) + def test_process_row_annotations_get_validation( + self, dmge, synapse_store, hideBlanks, label_options + ): + comma_separated_list = comma_separated_list_regex() + metadata_syn = {"PatientID": "value1", "Sex": "value2"} + annos = {"PatientID": "old_value", "Sex": "old_value"} + + dmge.get_node_validation_rules = MagicMock() + process_row_annos = synapse_store.process_row_annotations( + dmge=dmge, + metadataSyn=metadata_syn, + csv_list_regex=comma_separated_list, + hideBlanks=hideBlanks, + annos=annos, + annotation_keys=label_options, + ) + + # when the label is "display label", make sure that the get_node_validation_rules is called with the display name + if label_options == "display_label": + dmge.get_node_validation_rules.assert_called_once_with( + node_display_name="PatientID" + ) + dmge.get_node_validation_rules.assert_called_once_with( + node_display_name="Sex" + ) + # make sure that the get_node_validation_rules is called with the node label + else: + dmge.get_node_validation_rules.assert_any_call(node_label="PatientID") + dmge.get_node_validation_rules.assert_any_call(node_label="Sex") From cbbae61c257eb9e34301569a2a11c581d3d8d50c Mon Sep 17 00:00:00 2001 From: GiaJordan Date: Thu, 5 Sep 2024 14:51:06 -0700 Subject: [PATCH 189/233] add asset view param --- tests/test_api.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_api.py b/tests/test_api.py index 47ad6cc5b..c8cf78710 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -923,6 +923,7 @@ def test_validate_manifest( # GIVEN a set of test prameters params = { "schema_url": DATA_MODEL_JSON_LD, + "asset_view": "syn23643253", "restrict_rules": restrict_rules, "project_scope": project_scope, "dataset_scope": dataset_scope, From c6db5bb706f5f9ca715fbc80432f5f6a25976af0 Mon Sep 17 00:00:00 2001 From: BryanFauble <17128019+BryanFauble@users.noreply.github.com> Date: Fri, 6 Sep 2024 09:16:33 -0700 Subject: [PATCH 190/233] [FDS-2373] Update jwt verification + synpy update v4.4.0->v4.4.1 (#1493) * Update jwt verification + synpy update v4.4.0->v4.4.1 --- .gitignore | 2 +- poetry.lock | 1801 +++++++++-------- pyproject.toml | 3 +- schematic_api/api/openapi/api.yaml | 2 +- schematic_api/api/security_controller.py | 47 + schematic_api/api/security_controller_.py | 14 - tests/conftest.py | 16 +- tests/integration/test_security_controller.py | 59 + tests/test_api.py | 14 - 9 files changed, 1070 insertions(+), 888 deletions(-) create mode 100644 schematic_api/api/security_controller.py delete mode 100644 schematic_api/api/security_controller_.py create mode 100644 tests/integration/test_security_controller.py diff --git a/.gitignore b/.gitignore index fa0de2078..6d00e45d3 100644 --- a/.gitignore +++ b/.gitignore @@ -146,7 +146,7 @@ dmypy.json # End of https://www.toptal.com/developers/gitignore/api/python # Synapse configuration file -.synapseConfig +.synapseConfig* # Google services authorization credentials file credentials.json diff --git a/poetry.lock b/poetry.lock index 4c4f8b094..954deb011 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.7.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.8.3 and should not be changed by hand. [[package]] name = "alabaster" @@ -35,13 +35,13 @@ dev = ["black", "docutils", "flake8", "ipython", "m2r", "mistune (<2.0.0)", "pyt [[package]] name = "anyio" -version = "4.3.0" +version = "4.4.0" description = "High level compatibility layer for multiple asynchronous event loop implementations" optional = false python-versions = ">=3.8" files = [ - {file = "anyio-4.3.0-py3-none-any.whl", hash = "sha256:048e05d0f6caeed70d731f3db756d35dcc1f35747c8c403364a8332c630441b8"}, - {file = "anyio-4.3.0.tar.gz", hash = "sha256:f75253795a87df48568485fd18cdd2a3fa5c4f7c5be8e5e36637733fce06fed6"}, + {file = "anyio-4.4.0-py3-none-any.whl", hash = "sha256:c1b2d8f46a8a812513012e1107cb0e68c17159a7a594208005a57dc776e1bdc7"}, + {file = "anyio-4.4.0.tar.gz", hash = "sha256:5aadc6a1bbb7cdb0bede386cac5e2940f5e2ff3aa20277e991cf028e0585ce94"}, ] [package.dependencies] @@ -219,32 +219,32 @@ test = ["pytest", "uvloop"] [[package]] name = "attrs" -version = "23.2.0" +version = "24.2.0" description = "Classes Without Boilerplate" optional = false python-versions = ">=3.7" files = [ - {file = "attrs-23.2.0-py3-none-any.whl", hash = "sha256:99b87a485a5820b23b879f04c2305b44b951b502fd64be915879d77a7e8fc6f1"}, - {file = "attrs-23.2.0.tar.gz", hash = "sha256:935dc3b529c262f6cf76e50877d35a4bd3c1de194fd41f47a2b7ae8f19971f30"}, + {file = "attrs-24.2.0-py3-none-any.whl", hash = "sha256:81921eb96de3191c8258c199618104dd27ac608d9366f5e35d011eae1867ede2"}, + {file = "attrs-24.2.0.tar.gz", hash = "sha256:5cfb1b9148b5b086569baec03f20d7b6bf3bcacc9a42bebf87ffaaca362f6346"}, ] [package.extras] -cov = ["attrs[tests]", "coverage[toml] (>=5.3)"] -dev = ["attrs[tests]", "pre-commit"] -docs = ["furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier", "zope-interface"] -tests = ["attrs[tests-no-zope]", "zope-interface"] -tests-mypy = ["mypy (>=1.6)", "pytest-mypy-plugins"] -tests-no-zope = ["attrs[tests-mypy]", "cloudpickle", "hypothesis", "pympler", "pytest (>=4.3.0)", "pytest-xdist[psutil]"] +benchmark = ["cloudpickle", "hypothesis", "mypy (>=1.11.1)", "pympler", "pytest (>=4.3.0)", "pytest-codspeed", "pytest-mypy-plugins", "pytest-xdist[psutil]"] +cov = ["cloudpickle", "coverage[toml] (>=5.3)", "hypothesis", "mypy (>=1.11.1)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist[psutil]"] +dev = ["cloudpickle", "hypothesis", "mypy (>=1.11.1)", "pre-commit", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist[psutil]"] +docs = ["cogapp", "furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier (<24.7)"] +tests = ["cloudpickle", "hypothesis", "mypy (>=1.11.1)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist[psutil]"] +tests-mypy = ["mypy (>=1.11.1)", "pytest-mypy-plugins"] [[package]] name = "babel" -version = "2.15.0" +version = "2.16.0" description = "Internationalization utilities" optional = false python-versions = ">=3.8" files = [ - {file = "Babel-2.15.0-py3-none-any.whl", hash = "sha256:08706bdad8d0a3413266ab61bd6c34d0c28d6e1e7badf40a2cebe67644e2e1fb"}, - {file = "babel-2.15.0.tar.gz", hash = "sha256:8daf0e265d05768bc6c7a314cf1321e9a123afc328cc635c18622a2f30a04413"}, + {file = "babel-2.16.0-py3-none-any.whl", hash = "sha256:368b5b98b37c06b7daf6696391c3240c938b37767d4584413e8438c5c435fa8b"}, + {file = "babel-2.16.0.tar.gz", hash = "sha256:d1f3554ca26605fe173f3de0c65f750f5a42f924499bf134de6423582298e316"}, ] [package.extras] @@ -348,85 +348,100 @@ css = ["tinycss2 (>=1.1.0,<1.3)"] [[package]] name = "cachetools" -version = "5.3.3" +version = "5.5.0" description = "Extensible memoizing collections and decorators" optional = false python-versions = ">=3.7" files = [ - {file = "cachetools-5.3.3-py3-none-any.whl", hash = "sha256:0abad1021d3f8325b2fc1d2e9c8b9c9d57b04c3932657a72465447332c24d945"}, - {file = "cachetools-5.3.3.tar.gz", hash = "sha256:ba29e2dfa0b8b556606f097407ed1aa62080ee108ab0dc5ec9d6a723a007d105"}, + {file = "cachetools-5.5.0-py3-none-any.whl", hash = "sha256:02134e8439cdc2ffb62023ce1debca2944c3f289d66bb17ead3ab3dede74b292"}, + {file = "cachetools-5.5.0.tar.gz", hash = "sha256:2cc24fb4cbe39633fb7badd9db9ca6295d766d9c2995f245725a46715d050f2a"}, ] [[package]] name = "certifi" -version = "2024.2.2" +version = "2024.8.30" description = "Python package for providing Mozilla's CA Bundle." optional = false python-versions = ">=3.6" files = [ - {file = "certifi-2024.2.2-py3-none-any.whl", hash = "sha256:dc383c07b76109f368f6106eee2b593b04a011ea4d55f652c6ca24a754d1cdd1"}, - {file = "certifi-2024.2.2.tar.gz", hash = "sha256:0569859f95fc761b18b45ef421b1290a0f65f147e92a1e5eb3e635f9a5e4e66f"}, + {file = "certifi-2024.8.30-py3-none-any.whl", hash = "sha256:922820b53db7a7257ffbda3f597266d435245903d80737e34f8a45ff3e3230d8"}, + {file = "certifi-2024.8.30.tar.gz", hash = "sha256:bec941d2aa8195e248a60b31ff9f0558284cf01a52591ceda73ea9afffd69fd9"}, ] [[package]] name = "cffi" -version = "1.16.0" +version = "1.17.1" description = "Foreign Function Interface for Python calling C code." optional = false python-versions = ">=3.8" files = [ - {file = "cffi-1.16.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:6b3d6606d369fc1da4fd8c357d026317fbb9c9b75d36dc16e90e84c26854b088"}, - {file = "cffi-1.16.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ac0f5edd2360eea2f1daa9e26a41db02dd4b0451b48f7c318e217ee092a213e9"}, - {file = "cffi-1.16.0-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7e61e3e4fa664a8588aa25c883eab612a188c725755afff6289454d6362b9673"}, - {file = "cffi-1.16.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a72e8961a86d19bdb45851d8f1f08b041ea37d2bd8d4fd19903bc3083d80c896"}, - {file = "cffi-1.16.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5b50bf3f55561dac5438f8e70bfcdfd74543fd60df5fa5f62d94e5867deca684"}, - {file = "cffi-1.16.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7651c50c8c5ef7bdb41108b7b8c5a83013bfaa8a935590c5d74627c047a583c7"}, - {file = "cffi-1.16.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e4108df7fe9b707191e55f33efbcb2d81928e10cea45527879a4749cbe472614"}, - {file = "cffi-1.16.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:32c68ef735dbe5857c810328cb2481e24722a59a2003018885514d4c09af9743"}, - {file = "cffi-1.16.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:673739cb539f8cdaa07d92d02efa93c9ccf87e345b9a0b556e3ecc666718468d"}, - {file = "cffi-1.16.0-cp310-cp310-win32.whl", hash = "sha256:9f90389693731ff1f659e55c7d1640e2ec43ff725cc61b04b2f9c6d8d017df6a"}, - {file = "cffi-1.16.0-cp310-cp310-win_amd64.whl", hash = "sha256:e6024675e67af929088fda399b2094574609396b1decb609c55fa58b028a32a1"}, - {file = "cffi-1.16.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:b84834d0cf97e7d27dd5b7f3aca7b6e9263c56308ab9dc8aae9784abb774d404"}, - {file = "cffi-1.16.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1b8ebc27c014c59692bb2664c7d13ce7a6e9a629be20e54e7271fa696ff2b417"}, - {file = "cffi-1.16.0-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ee07e47c12890ef248766a6e55bd38ebfb2bb8edd4142d56db91b21ea68b7627"}, - {file = "cffi-1.16.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d8a9d3ebe49f084ad71f9269834ceccbf398253c9fac910c4fd7053ff1386936"}, - {file = "cffi-1.16.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e70f54f1796669ef691ca07d046cd81a29cb4deb1e5f942003f401c0c4a2695d"}, - {file = "cffi-1.16.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5bf44d66cdf9e893637896c7faa22298baebcd18d1ddb6d2626a6e39793a1d56"}, - {file = "cffi-1.16.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7b78010e7b97fef4bee1e896df8a4bbb6712b7f05b7ef630f9d1da00f6444d2e"}, - {file = "cffi-1.16.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:c6a164aa47843fb1b01e941d385aab7215563bb8816d80ff3a363a9f8448a8dc"}, - {file = "cffi-1.16.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e09f3ff613345df5e8c3667da1d918f9149bd623cd9070c983c013792a9a62eb"}, - {file = "cffi-1.16.0-cp311-cp311-win32.whl", hash = "sha256:2c56b361916f390cd758a57f2e16233eb4f64bcbeee88a4881ea90fca14dc6ab"}, - {file = "cffi-1.16.0-cp311-cp311-win_amd64.whl", hash = "sha256:db8e577c19c0fda0beb7e0d4e09e0ba74b1e4c092e0e40bfa12fe05b6f6d75ba"}, - {file = "cffi-1.16.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:fa3a0128b152627161ce47201262d3140edb5a5c3da88d73a1b790a959126956"}, - {file = "cffi-1.16.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:68e7c44931cc171c54ccb702482e9fc723192e88d25a0e133edd7aff8fcd1f6e"}, - {file = "cffi-1.16.0-cp312-cp312-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:abd808f9c129ba2beda4cfc53bde801e5bcf9d6e0f22f095e45327c038bfe68e"}, - {file = "cffi-1.16.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:88e2b3c14bdb32e440be531ade29d3c50a1a59cd4e51b1dd8b0865c54ea5d2e2"}, - {file = "cffi-1.16.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:fcc8eb6d5902bb1cf6dc4f187ee3ea80a1eba0a89aba40a5cb20a5087d961357"}, - {file = "cffi-1.16.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b7be2d771cdba2942e13215c4e340bfd76398e9227ad10402a8767ab1865d2e6"}, - {file = "cffi-1.16.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e715596e683d2ce000574bae5d07bd522c781a822866c20495e52520564f0969"}, - {file = "cffi-1.16.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:2d92b25dbf6cae33f65005baf472d2c245c050b1ce709cc4588cdcdd5495b520"}, - {file = "cffi-1.16.0-cp312-cp312-win32.whl", hash = "sha256:b2ca4e77f9f47c55c194982e10f058db063937845bb2b7a86c84a6cfe0aefa8b"}, - {file = "cffi-1.16.0-cp312-cp312-win_amd64.whl", hash = "sha256:68678abf380b42ce21a5f2abde8efee05c114c2fdb2e9eef2efdb0257fba1235"}, - {file = "cffi-1.16.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:0c9ef6ff37e974b73c25eecc13952c55bceed9112be2d9d938ded8e856138bcc"}, - {file = "cffi-1.16.0-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a09582f178759ee8128d9270cd1344154fd473bb77d94ce0aeb2a93ebf0feaf0"}, - {file = "cffi-1.16.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e760191dd42581e023a68b758769e2da259b5d52e3103c6060ddc02c9edb8d7b"}, - {file = "cffi-1.16.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:80876338e19c951fdfed6198e70bc88f1c9758b94578d5a7c4c91a87af3cf31c"}, - {file = "cffi-1.16.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a6a14b17d7e17fa0d207ac08642c8820f84f25ce17a442fd15e27ea18d67c59b"}, - {file = "cffi-1.16.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6602bc8dc6f3a9e02b6c22c4fc1e47aa50f8f8e6d3f78a5e16ac33ef5fefa324"}, - {file = "cffi-1.16.0-cp38-cp38-win32.whl", hash = "sha256:131fd094d1065b19540c3d72594260f118b231090295d8c34e19a7bbcf2e860a"}, - {file = "cffi-1.16.0-cp38-cp38-win_amd64.whl", hash = "sha256:31d13b0f99e0836b7ff893d37af07366ebc90b678b6664c955b54561fc36ef36"}, - {file = "cffi-1.16.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:582215a0e9adbe0e379761260553ba11c58943e4bbe9c36430c4ca6ac74b15ed"}, - {file = "cffi-1.16.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:b29ebffcf550f9da55bec9e02ad430c992a87e5f512cd63388abb76f1036d8d2"}, - {file = "cffi-1.16.0-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dc9b18bf40cc75f66f40a7379f6a9513244fe33c0e8aa72e2d56b0196a7ef872"}, - {file = "cffi-1.16.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9cb4a35b3642fc5c005a6755a5d17c6c8b6bcb6981baf81cea8bfbc8903e8ba8"}, - {file = "cffi-1.16.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b86851a328eedc692acf81fb05444bdf1891747c25af7529e39ddafaf68a4f3f"}, - {file = "cffi-1.16.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c0f31130ebc2d37cdd8e44605fb5fa7ad59049298b3f745c74fa74c62fbfcfc4"}, - {file = "cffi-1.16.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8f8e709127c6c77446a8c0a8c8bf3c8ee706a06cd44b1e827c3e6a2ee6b8c098"}, - {file = "cffi-1.16.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:748dcd1e3d3d7cd5443ef03ce8685043294ad6bd7c02a38d1bd367cfd968e000"}, - {file = "cffi-1.16.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:8895613bcc094d4a1b2dbe179d88d7fb4a15cee43c052e8885783fac397d91fe"}, - {file = "cffi-1.16.0-cp39-cp39-win32.whl", hash = "sha256:ed86a35631f7bfbb28e108dd96773b9d5a6ce4811cf6ea468bb6a359b256b1e4"}, - {file = "cffi-1.16.0-cp39-cp39-win_amd64.whl", hash = "sha256:3686dffb02459559c74dd3d81748269ffb0eb027c39a6fc99502de37d501faa8"}, - {file = "cffi-1.16.0.tar.gz", hash = "sha256:bcb3ef43e58665bbda2fb198698fcae6776483e0c4a631aa5647806c25e02cc0"}, + {file = "cffi-1.17.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:df8b1c11f177bc2313ec4b2d46baec87a5f3e71fc8b45dab2ee7cae86d9aba14"}, + {file = "cffi-1.17.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8f2cdc858323644ab277e9bb925ad72ae0e67f69e804f4898c070998d50b1a67"}, + {file = "cffi-1.17.1-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:edae79245293e15384b51f88b00613ba9f7198016a5948b5dddf4917d4d26382"}, + {file = "cffi-1.17.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:45398b671ac6d70e67da8e4224a065cec6a93541bb7aebe1b198a61b58c7b702"}, + {file = "cffi-1.17.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ad9413ccdeda48c5afdae7e4fa2192157e991ff761e7ab8fdd8926f40b160cc3"}, + {file = "cffi-1.17.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5da5719280082ac6bd9aa7becb3938dc9f9cbd57fac7d2871717b1feb0902ab6"}, + {file = "cffi-1.17.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2bb1a08b8008b281856e5971307cc386a8e9c5b625ac297e853d36da6efe9c17"}, + {file = "cffi-1.17.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:045d61c734659cc045141be4bae381a41d89b741f795af1dd018bfb532fd0df8"}, + {file = "cffi-1.17.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:6883e737d7d9e4899a8a695e00ec36bd4e5e4f18fabe0aca0efe0a4b44cdb13e"}, + {file = "cffi-1.17.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:6b8b4a92e1c65048ff98cfe1f735ef8f1ceb72e3d5f0c25fdb12087a23da22be"}, + {file = "cffi-1.17.1-cp310-cp310-win32.whl", hash = "sha256:c9c3d058ebabb74db66e431095118094d06abf53284d9c81f27300d0e0d8bc7c"}, + {file = "cffi-1.17.1-cp310-cp310-win_amd64.whl", hash = "sha256:0f048dcf80db46f0098ccac01132761580d28e28bc0f78ae0d58048063317e15"}, + {file = "cffi-1.17.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:a45e3c6913c5b87b3ff120dcdc03f6131fa0065027d0ed7ee6190736a74cd401"}, + {file = "cffi-1.17.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:30c5e0cb5ae493c04c8b42916e52ca38079f1b235c2f8ae5f4527b963c401caf"}, + {file = "cffi-1.17.1-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f75c7ab1f9e4aca5414ed4d8e5c0e303a34f4421f8a0d47a4d019ceff0ab6af4"}, + {file = "cffi-1.17.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a1ed2dd2972641495a3ec98445e09766f077aee98a1c896dcb4ad0d303628e41"}, + {file = "cffi-1.17.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:46bf43160c1a35f7ec506d254e5c890f3c03648a4dbac12d624e4490a7046cd1"}, + {file = "cffi-1.17.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a24ed04c8ffd54b0729c07cee15a81d964e6fee0e3d4d342a27b020d22959dc6"}, + {file = "cffi-1.17.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:610faea79c43e44c71e1ec53a554553fa22321b65fae24889706c0a84d4ad86d"}, + {file = "cffi-1.17.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:a9b15d491f3ad5d692e11f6b71f7857e7835eb677955c00cc0aefcd0669adaf6"}, + {file = "cffi-1.17.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:de2ea4b5833625383e464549fec1bc395c1bdeeb5f25c4a3a82b5a8c756ec22f"}, + {file = "cffi-1.17.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:fc48c783f9c87e60831201f2cce7f3b2e4846bf4d8728eabe54d60700b318a0b"}, + {file = "cffi-1.17.1-cp311-cp311-win32.whl", hash = "sha256:85a950a4ac9c359340d5963966e3e0a94a676bd6245a4b55bc43949eee26a655"}, + {file = "cffi-1.17.1-cp311-cp311-win_amd64.whl", hash = "sha256:caaf0640ef5f5517f49bc275eca1406b0ffa6aa184892812030f04c2abf589a0"}, + {file = "cffi-1.17.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:805b4371bf7197c329fcb3ead37e710d1bca9da5d583f5073b799d5c5bd1eee4"}, + {file = "cffi-1.17.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:733e99bc2df47476e3848417c5a4540522f234dfd4ef3ab7fafdf555b082ec0c"}, + {file = "cffi-1.17.1-cp312-cp312-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1257bdabf294dceb59f5e70c64a3e2f462c30c7ad68092d01bbbfb1c16b1ba36"}, + {file = "cffi-1.17.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:da95af8214998d77a98cc14e3a3bd00aa191526343078b530ceb0bd710fb48a5"}, + {file = "cffi-1.17.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d63afe322132c194cf832bfec0dc69a99fb9bb6bbd550f161a49e9e855cc78ff"}, + {file = "cffi-1.17.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f79fc4fc25f1c8698ff97788206bb3c2598949bfe0fef03d299eb1b5356ada99"}, + {file = "cffi-1.17.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b62ce867176a75d03a665bad002af8e6d54644fad99a3c70905c543130e39d93"}, + {file = "cffi-1.17.1-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:386c8bf53c502fff58903061338ce4f4950cbdcb23e2902d86c0f722b786bbe3"}, + {file = "cffi-1.17.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:4ceb10419a9adf4460ea14cfd6bc43d08701f0835e979bf821052f1805850fe8"}, + {file = "cffi-1.17.1-cp312-cp312-win32.whl", hash = "sha256:a08d7e755f8ed21095a310a693525137cfe756ce62d066e53f502a83dc550f65"}, + {file = "cffi-1.17.1-cp312-cp312-win_amd64.whl", hash = "sha256:51392eae71afec0d0c8fb1a53b204dbb3bcabcb3c9b807eedf3e1e6ccf2de903"}, + {file = "cffi-1.17.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:f3a2b4222ce6b60e2e8b337bb9596923045681d71e5a082783484d845390938e"}, + {file = "cffi-1.17.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:0984a4925a435b1da406122d4d7968dd861c1385afe3b45ba82b750f229811e2"}, + {file = "cffi-1.17.1-cp313-cp313-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d01b12eeeb4427d3110de311e1774046ad344f5b1a7403101878976ecd7a10f3"}, + {file = "cffi-1.17.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:706510fe141c86a69c8ddc029c7910003a17353970cff3b904ff0686a5927683"}, + {file = "cffi-1.17.1-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:de55b766c7aa2e2a3092c51e0483d700341182f08e67c63630d5b6f200bb28e5"}, + {file = "cffi-1.17.1-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c59d6e989d07460165cc5ad3c61f9fd8f1b4796eacbd81cee78957842b834af4"}, + {file = "cffi-1.17.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dd398dbc6773384a17fe0d3e7eeb8d1a21c2200473ee6806bb5e6a8e62bb73dd"}, + {file = "cffi-1.17.1-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:3edc8d958eb099c634dace3c7e16560ae474aa3803a5df240542b305d14e14ed"}, + {file = "cffi-1.17.1-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:72e72408cad3d5419375fc87d289076ee319835bdfa2caad331e377589aebba9"}, + {file = "cffi-1.17.1-cp313-cp313-win32.whl", hash = "sha256:e03eab0a8677fa80d646b5ddece1cbeaf556c313dcfac435ba11f107ba117b5d"}, + {file = "cffi-1.17.1-cp313-cp313-win_amd64.whl", hash = "sha256:f6a16c31041f09ead72d69f583767292f750d24913dadacf5756b966aacb3f1a"}, + {file = "cffi-1.17.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:636062ea65bd0195bc012fea9321aca499c0504409f413dc88af450b57ffd03b"}, + {file = "cffi-1.17.1-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c7eac2ef9b63c79431bc4b25f1cd649d7f061a28808cbc6c47b534bd789ef964"}, + {file = "cffi-1.17.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e221cf152cff04059d011ee126477f0d9588303eb57e88923578ace7baad17f9"}, + {file = "cffi-1.17.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:31000ec67d4221a71bd3f67df918b1f88f676f1c3b535a7eb473255fdc0b83fc"}, + {file = "cffi-1.17.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:6f17be4345073b0a7b8ea599688f692ac3ef23ce28e5df79c04de519dbc4912c"}, + {file = "cffi-1.17.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0e2b1fac190ae3ebfe37b979cc1ce69c81f4e4fe5746bb401dca63a9062cdaf1"}, + {file = "cffi-1.17.1-cp38-cp38-win32.whl", hash = "sha256:7596d6620d3fa590f677e9ee430df2958d2d6d6de2feeae5b20e82c00b76fbf8"}, + {file = "cffi-1.17.1-cp38-cp38-win_amd64.whl", hash = "sha256:78122be759c3f8a014ce010908ae03364d00a1f81ab5c7f4a7a5120607ea56e1"}, + {file = "cffi-1.17.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:b2ab587605f4ba0bf81dc0cb08a41bd1c0a5906bd59243d56bad7668a6fc6c16"}, + {file = "cffi-1.17.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:28b16024becceed8c6dfbc75629e27788d8a3f9030691a1dbf9821a128b22c36"}, + {file = "cffi-1.17.1-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1d599671f396c4723d016dbddb72fe8e0397082b0a77a4fab8028923bec050e8"}, + {file = "cffi-1.17.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ca74b8dbe6e8e8263c0ffd60277de77dcee6c837a3d0881d8c1ead7268c9e576"}, + {file = "cffi-1.17.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f7f5baafcc48261359e14bcd6d9bff6d4b28d9103847c9e136694cb0501aef87"}, + {file = "cffi-1.17.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:98e3969bcff97cae1b2def8ba499ea3d6f31ddfdb7635374834cf89a1a08ecf0"}, + {file = "cffi-1.17.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cdf5ce3acdfd1661132f2a9c19cac174758dc2352bfe37d98aa7512c6b7178b3"}, + {file = "cffi-1.17.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:9755e4345d1ec879e3849e62222a18c7174d65a6a92d5b346b1863912168b595"}, + {file = "cffi-1.17.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:f1e22e8c4419538cb197e4dd60acc919d7696e5ef98ee4da4e01d3f8cfa4cc5a"}, + {file = "cffi-1.17.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:c03e868a0b3bc35839ba98e74211ed2b05d2119be4e8a0f224fba9384f1fe02e"}, + {file = "cffi-1.17.1-cp39-cp39-win32.whl", hash = "sha256:e31ae45bc2e29f6b2abd0de1cc3b9d5205aa847cafaecb8af1476a609a2f6eb7"}, + {file = "cffi-1.17.1-cp39-cp39-win_amd64.whl", hash = "sha256:d016c76bdd850f3c626af19b0542c9677ba156e4ee4fccfdd7848803533ef662"}, + {file = "cffi-1.17.1.tar.gz", hash = "sha256:1c39c6016c32bc48dd54561950ebd6836e1670f2ae46128f67cf49e789c52824"}, ] [package.dependencies] @@ -645,63 +660,83 @@ tests = ["MarkupSafe (>=0.23)", "aiohttp (>=2.3.10,<4)", "aiohttp-jinja2 (>=0.14 [[package]] name = "coverage" -version = "7.5.1" +version = "7.6.1" description = "Code coverage measurement for Python" optional = false python-versions = ">=3.8" files = [ - {file = "coverage-7.5.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:c0884920835a033b78d1c73b6d3bbcda8161a900f38a488829a83982925f6c2e"}, - {file = "coverage-7.5.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:39afcd3d4339329c5f58de48a52f6e4e50f6578dd6099961cf22228feb25f38f"}, - {file = "coverage-7.5.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4a7b0ceee8147444347da6a66be737c9d78f3353b0681715b668b72e79203e4a"}, - {file = "coverage-7.5.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4a9ca3f2fae0088c3c71d743d85404cec8df9be818a005ea065495bedc33da35"}, - {file = "coverage-7.5.1-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5fd215c0c7d7aab005221608a3c2b46f58c0285a819565887ee0b718c052aa4e"}, - {file = "coverage-7.5.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:4bf0655ab60d754491004a5efd7f9cccefcc1081a74c9ef2da4735d6ee4a6223"}, - {file = "coverage-7.5.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:61c4bf1ba021817de12b813338c9be9f0ad5b1e781b9b340a6d29fc13e7c1b5e"}, - {file = "coverage-7.5.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:db66fc317a046556a96b453a58eced5024af4582a8dbdc0c23ca4dbc0d5b3146"}, - {file = "coverage-7.5.1-cp310-cp310-win32.whl", hash = "sha256:b016ea6b959d3b9556cb401c55a37547135a587db0115635a443b2ce8f1c7228"}, - {file = "coverage-7.5.1-cp310-cp310-win_amd64.whl", hash = "sha256:df4e745a81c110e7446b1cc8131bf986157770fa405fe90e15e850aaf7619bc8"}, - {file = "coverage-7.5.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:796a79f63eca8814ca3317a1ea443645c9ff0d18b188de470ed7ccd45ae79428"}, - {file = "coverage-7.5.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4fc84a37bfd98db31beae3c2748811a3fa72bf2007ff7902f68746d9757f3746"}, - {file = "coverage-7.5.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6175d1a0559986c6ee3f7fccfc4a90ecd12ba0a383dcc2da30c2b9918d67d8a3"}, - {file = "coverage-7.5.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1fc81d5878cd6274ce971e0a3a18a8803c3fe25457165314271cf78e3aae3aa2"}, - {file = "coverage-7.5.1-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:556cf1a7cbc8028cb60e1ff0be806be2eded2daf8129b8811c63e2b9a6c43bca"}, - {file = "coverage-7.5.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:9981706d300c18d8b220995ad22627647be11a4276721c10911e0e9fa44c83e8"}, - {file = "coverage-7.5.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:d7fed867ee50edf1a0b4a11e8e5d0895150e572af1cd6d315d557758bfa9c057"}, - {file = "coverage-7.5.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:ef48e2707fb320c8f139424a596f5b69955a85b178f15af261bab871873bb987"}, - {file = "coverage-7.5.1-cp311-cp311-win32.whl", hash = "sha256:9314d5678dcc665330df5b69c1e726a0e49b27df0461c08ca12674bcc19ef136"}, - {file = "coverage-7.5.1-cp311-cp311-win_amd64.whl", hash = "sha256:5fa567e99765fe98f4e7d7394ce623e794d7cabb170f2ca2ac5a4174437e90dd"}, - {file = "coverage-7.5.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:b6cf3764c030e5338e7f61f95bd21147963cf6aa16e09d2f74f1fa52013c1206"}, - {file = "coverage-7.5.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:2ec92012fefebee89a6b9c79bc39051a6cb3891d562b9270ab10ecfdadbc0c34"}, - {file = "coverage-7.5.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:16db7f26000a07efcf6aea00316f6ac57e7d9a96501e990a36f40c965ec7a95d"}, - {file = "coverage-7.5.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:beccf7b8a10b09c4ae543582c1319c6df47d78fd732f854ac68d518ee1fb97fa"}, - {file = "coverage-7.5.1-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8748731ad392d736cc9ccac03c9845b13bb07d020a33423fa5b3a36521ac6e4e"}, - {file = "coverage-7.5.1-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:7352b9161b33fd0b643ccd1f21f3a3908daaddf414f1c6cb9d3a2fd618bf2572"}, - {file = "coverage-7.5.1-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:7a588d39e0925f6a2bff87154752481273cdb1736270642aeb3635cb9b4cad07"}, - {file = "coverage-7.5.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:68f962d9b72ce69ea8621f57551b2fa9c70509af757ee3b8105d4f51b92b41a7"}, - {file = "coverage-7.5.1-cp312-cp312-win32.whl", hash = "sha256:f152cbf5b88aaeb836127d920dd0f5e7edff5a66f10c079157306c4343d86c19"}, - {file = "coverage-7.5.1-cp312-cp312-win_amd64.whl", hash = "sha256:5a5740d1fb60ddf268a3811bcd353de34eb56dc24e8f52a7f05ee513b2d4f596"}, - {file = "coverage-7.5.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:e2213def81a50519d7cc56ed643c9e93e0247f5bbe0d1247d15fa520814a7cd7"}, - {file = "coverage-7.5.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:5037f8fcc2a95b1f0e80585bd9d1ec31068a9bcb157d9750a172836e98bc7a90"}, - {file = "coverage-7.5.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5c3721c2c9e4c4953a41a26c14f4cef64330392a6d2d675c8b1db3b645e31f0e"}, - {file = "coverage-7.5.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ca498687ca46a62ae590253fba634a1fe9836bc56f626852fb2720f334c9e4e5"}, - {file = "coverage-7.5.1-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0cdcbc320b14c3e5877ee79e649677cb7d89ef588852e9583e6b24c2e5072661"}, - {file = "coverage-7.5.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:57e0204b5b745594e5bc14b9b50006da722827f0b8c776949f1135677e88d0b8"}, - {file = "coverage-7.5.1-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:8fe7502616b67b234482c3ce276ff26f39ffe88adca2acf0261df4b8454668b4"}, - {file = "coverage-7.5.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:9e78295f4144f9dacfed4f92935fbe1780021247c2fabf73a819b17f0ccfff8d"}, - {file = "coverage-7.5.1-cp38-cp38-win32.whl", hash = "sha256:1434e088b41594baa71188a17533083eabf5609e8e72f16ce8c186001e6b8c41"}, - {file = "coverage-7.5.1-cp38-cp38-win_amd64.whl", hash = "sha256:0646599e9b139988b63704d704af8e8df7fa4cbc4a1f33df69d97f36cb0a38de"}, - {file = "coverage-7.5.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:4cc37def103a2725bc672f84bd939a6fe4522310503207aae4d56351644682f1"}, - {file = "coverage-7.5.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:fc0b4d8bfeabd25ea75e94632f5b6e047eef8adaed0c2161ada1e922e7f7cece"}, - {file = "coverage-7.5.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0d0a0f5e06881ecedfe6f3dd2f56dcb057b6dbeb3327fd32d4b12854df36bf26"}, - {file = "coverage-7.5.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9735317685ba6ec7e3754798c8871c2f49aa5e687cc794a0b1d284b2389d1bd5"}, - {file = "coverage-7.5.1-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d21918e9ef11edf36764b93101e2ae8cc82aa5efdc7c5a4e9c6c35a48496d601"}, - {file = "coverage-7.5.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:c3e757949f268364b96ca894b4c342b41dc6f8f8b66c37878aacef5930db61be"}, - {file = "coverage-7.5.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:79afb6197e2f7f60c4824dd4b2d4c2ec5801ceb6ba9ce5d2c3080e5660d51a4f"}, - {file = "coverage-7.5.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:d1d0d98d95dd18fe29dc66808e1accf59f037d5716f86a501fc0256455219668"}, - {file = "coverage-7.5.1-cp39-cp39-win32.whl", hash = "sha256:1cc0fe9b0b3a8364093c53b0b4c0c2dd4bb23acbec4c9240b5f284095ccf7981"}, - {file = "coverage-7.5.1-cp39-cp39-win_amd64.whl", hash = "sha256:dde0070c40ea8bb3641e811c1cfbf18e265d024deff6de52c5950677a8fb1e0f"}, - {file = "coverage-7.5.1-pp38.pp39.pp310-none-any.whl", hash = "sha256:6537e7c10cc47c595828b8a8be04c72144725c383c4702703ff4e42e44577312"}, - {file = "coverage-7.5.1.tar.gz", hash = "sha256:54de9ef3a9da981f7af93eafde4ede199e0846cd819eb27c88e2b712aae9708c"}, + {file = "coverage-7.6.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:b06079abebbc0e89e6163b8e8f0e16270124c154dc6e4a47b413dd538859af16"}, + {file = "coverage-7.6.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:cf4b19715bccd7ee27b6b120e7e9dd56037b9c0681dcc1adc9ba9db3d417fa36"}, + {file = "coverage-7.6.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e61c0abb4c85b095a784ef23fdd4aede7a2628478e7baba7c5e3deba61070a02"}, + {file = "coverage-7.6.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:fd21f6ae3f08b41004dfb433fa895d858f3f5979e7762d052b12aef444e29afc"}, + {file = "coverage-7.6.1-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8f59d57baca39b32db42b83b2a7ba6f47ad9c394ec2076b084c3f029b7afca23"}, + {file = "coverage-7.6.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:a1ac0ae2b8bd743b88ed0502544847c3053d7171a3cff9228af618a068ed9c34"}, + {file = "coverage-7.6.1-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:e6a08c0be454c3b3beb105c0596ebdc2371fab6bb90c0c0297f4e58fd7e1012c"}, + {file = "coverage-7.6.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:f5796e664fe802da4f57a168c85359a8fbf3eab5e55cd4e4569fbacecc903959"}, + {file = "coverage-7.6.1-cp310-cp310-win32.whl", hash = "sha256:7bb65125fcbef8d989fa1dd0e8a060999497629ca5b0efbca209588a73356232"}, + {file = "coverage-7.6.1-cp310-cp310-win_amd64.whl", hash = "sha256:3115a95daa9bdba70aea750db7b96b37259a81a709223c8448fa97727d546fe0"}, + {file = "coverage-7.6.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:7dea0889685db8550f839fa202744652e87c60015029ce3f60e006f8c4462c93"}, + {file = "coverage-7.6.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:ed37bd3c3b063412f7620464a9ac1314d33100329f39799255fb8d3027da50d3"}, + {file = "coverage-7.6.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d85f5e9a5f8b73e2350097c3756ef7e785f55bd71205defa0bfdaf96c31616ff"}, + {file = "coverage-7.6.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9bc572be474cafb617672c43fe989d6e48d3c83af02ce8de73fff1c6bb3c198d"}, + {file = "coverage-7.6.1-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0c0420b573964c760df9e9e86d1a9a622d0d27f417e1a949a8a66dd7bcee7bc6"}, + {file = "coverage-7.6.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1f4aa8219db826ce6be7099d559f8ec311549bfc4046f7f9fe9b5cea5c581c56"}, + {file = "coverage-7.6.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:fc5a77d0c516700ebad189b587de289a20a78324bc54baee03dd486f0855d234"}, + {file = "coverage-7.6.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:b48f312cca9621272ae49008c7f613337c53fadca647d6384cc129d2996d1133"}, + {file = "coverage-7.6.1-cp311-cp311-win32.whl", hash = "sha256:1125ca0e5fd475cbbba3bb67ae20bd2c23a98fac4e32412883f9bcbaa81c314c"}, + {file = "coverage-7.6.1-cp311-cp311-win_amd64.whl", hash = "sha256:8ae539519c4c040c5ffd0632784e21b2f03fc1340752af711f33e5be83a9d6c6"}, + {file = "coverage-7.6.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:95cae0efeb032af8458fc27d191f85d1717b1d4e49f7cb226cf526ff28179778"}, + {file = "coverage-7.6.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:5621a9175cf9d0b0c84c2ef2b12e9f5f5071357c4d2ea6ca1cf01814f45d2391"}, + {file = "coverage-7.6.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:260933720fdcd75340e7dbe9060655aff3af1f0c5d20f46b57f262ab6c86a5e8"}, + {file = "coverage-7.6.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:07e2ca0ad381b91350c0ed49d52699b625aab2b44b65e1b4e02fa9df0e92ad2d"}, + {file = "coverage-7.6.1-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c44fee9975f04b33331cb8eb272827111efc8930cfd582e0320613263ca849ca"}, + {file = "coverage-7.6.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:877abb17e6339d96bf08e7a622d05095e72b71f8afd8a9fefc82cf30ed944163"}, + {file = "coverage-7.6.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:3e0cadcf6733c09154b461f1ca72d5416635e5e4ec4e536192180d34ec160f8a"}, + {file = "coverage-7.6.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:c3c02d12f837d9683e5ab2f3d9844dc57655b92c74e286c262e0fc54213c216d"}, + {file = "coverage-7.6.1-cp312-cp312-win32.whl", hash = "sha256:e05882b70b87a18d937ca6768ff33cc3f72847cbc4de4491c8e73880766718e5"}, + {file = "coverage-7.6.1-cp312-cp312-win_amd64.whl", hash = "sha256:b5d7b556859dd85f3a541db6a4e0167b86e7273e1cdc973e5b175166bb634fdb"}, + {file = "coverage-7.6.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:a4acd025ecc06185ba2b801f2de85546e0b8ac787cf9d3b06e7e2a69f925b106"}, + {file = "coverage-7.6.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:a6d3adcf24b624a7b778533480e32434a39ad8fa30c315208f6d3e5542aeb6e9"}, + {file = "coverage-7.6.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d0c212c49b6c10e6951362f7c6df3329f04c2b1c28499563d4035d964ab8e08c"}, + {file = "coverage-7.6.1-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6e81d7a3e58882450ec4186ca59a3f20a5d4440f25b1cff6f0902ad890e6748a"}, + {file = "coverage-7.6.1-cp313-cp313-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:78b260de9790fd81e69401c2dc8b17da47c8038176a79092a89cb2b7d945d060"}, + {file = "coverage-7.6.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:a78d169acd38300060b28d600344a803628c3fd585c912cacc9ea8790fe96862"}, + {file = "coverage-7.6.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:2c09f4ce52cb99dd7505cd0fc8e0e37c77b87f46bc9c1eb03fe3bc9991085388"}, + {file = "coverage-7.6.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:6878ef48d4227aace338d88c48738a4258213cd7b74fd9a3d4d7582bb1d8a155"}, + {file = "coverage-7.6.1-cp313-cp313-win32.whl", hash = "sha256:44df346d5215a8c0e360307d46ffaabe0f5d3502c8a1cefd700b34baf31d411a"}, + {file = "coverage-7.6.1-cp313-cp313-win_amd64.whl", hash = "sha256:8284cf8c0dd272a247bc154eb6c95548722dce90d098c17a883ed36e67cdb129"}, + {file = "coverage-7.6.1-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:d3296782ca4eab572a1a4eca686d8bfb00226300dcefdf43faa25b5242ab8a3e"}, + {file = "coverage-7.6.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:502753043567491d3ff6d08629270127e0c31d4184c4c8d98f92c26f65019962"}, + {file = "coverage-7.6.1-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6a89ecca80709d4076b95f89f308544ec8f7b4727e8a547913a35f16717856cb"}, + {file = "coverage-7.6.1-cp313-cp313t-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a318d68e92e80af8b00fa99609796fdbcdfef3629c77c6283566c6f02c6d6704"}, + {file = "coverage-7.6.1-cp313-cp313t-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:13b0a73a0896988f053e4fbb7de6d93388e6dd292b0d87ee51d106f2c11b465b"}, + {file = "coverage-7.6.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:4421712dbfc5562150f7554f13dde997a2e932a6b5f352edcce948a815efee6f"}, + {file = "coverage-7.6.1-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:166811d20dfea725e2e4baa71fffd6c968a958577848d2131f39b60043400223"}, + {file = "coverage-7.6.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:225667980479a17db1048cb2bf8bfb39b8e5be8f164b8f6628b64f78a72cf9d3"}, + {file = "coverage-7.6.1-cp313-cp313t-win32.whl", hash = "sha256:170d444ab405852903b7d04ea9ae9b98f98ab6d7e63e1115e82620807519797f"}, + {file = "coverage-7.6.1-cp313-cp313t-win_amd64.whl", hash = "sha256:b9f222de8cded79c49bf184bdbc06630d4c58eec9459b939b4a690c82ed05657"}, + {file = "coverage-7.6.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6db04803b6c7291985a761004e9060b2bca08da6d04f26a7f2294b8623a0c1a0"}, + {file = "coverage-7.6.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:f1adfc8ac319e1a348af294106bc6a8458a0f1633cc62a1446aebc30c5fa186a"}, + {file = "coverage-7.6.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a95324a9de9650a729239daea117df21f4b9868ce32e63f8b650ebe6cef5595b"}, + {file = "coverage-7.6.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b43c03669dc4618ec25270b06ecd3ee4fa94c7f9b3c14bae6571ca00ef98b0d3"}, + {file = "coverage-7.6.1-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8929543a7192c13d177b770008bc4e8119f2e1f881d563fc6b6305d2d0ebe9de"}, + {file = "coverage-7.6.1-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:a09ece4a69cf399510c8ab25e0950d9cf2b42f7b3cb0374f95d2e2ff594478a6"}, + {file = "coverage-7.6.1-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:9054a0754de38d9dbd01a46621636689124d666bad1936d76c0341f7d71bf569"}, + {file = "coverage-7.6.1-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:0dbde0f4aa9a16fa4d754356a8f2e36296ff4d83994b2c9d8398aa32f222f989"}, + {file = "coverage-7.6.1-cp38-cp38-win32.whl", hash = "sha256:da511e6ad4f7323ee5702e6633085fb76c2f893aaf8ce4c51a0ba4fc07580ea7"}, + {file = "coverage-7.6.1-cp38-cp38-win_amd64.whl", hash = "sha256:3f1156e3e8f2872197af3840d8ad307a9dd18e615dc64d9ee41696f287c57ad8"}, + {file = "coverage-7.6.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:abd5fd0db5f4dc9289408aaf34908072f805ff7792632250dcb36dc591d24255"}, + {file = "coverage-7.6.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:547f45fa1a93154bd82050a7f3cddbc1a7a4dd2a9bf5cb7d06f4ae29fe94eaf8"}, + {file = "coverage-7.6.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:645786266c8f18a931b65bfcefdbf6952dd0dea98feee39bd188607a9d307ed2"}, + {file = "coverage-7.6.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9e0b2df163b8ed01d515807af24f63de04bebcecbd6c3bfeff88385789fdf75a"}, + {file = "coverage-7.6.1-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:609b06f178fe8e9f89ef676532760ec0b4deea15e9969bf754b37f7c40326dbc"}, + {file = "coverage-7.6.1-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:702855feff378050ae4f741045e19a32d57d19f3e0676d589df0575008ea5004"}, + {file = "coverage-7.6.1-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:2bdb062ea438f22d99cba0d7829c2ef0af1d768d1e4a4f528087224c90b132cb"}, + {file = "coverage-7.6.1-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:9c56863d44bd1c4fe2abb8a4d6f5371d197f1ac0ebdee542f07f35895fc07f36"}, + {file = "coverage-7.6.1-cp39-cp39-win32.whl", hash = "sha256:6e2cd258d7d927d09493c8df1ce9174ad01b381d4729a9d8d4e38670ca24774c"}, + {file = "coverage-7.6.1-cp39-cp39-win_amd64.whl", hash = "sha256:06a737c882bd26d0d6ee7269b20b12f14a8704807a01056c80bb881a4b2ce6ca"}, + {file = "coverage-7.6.1-pp38.pp39.pp310-none-any.whl", hash = "sha256:e9a6e0eb86070e8ccaedfbd9d38fec54864f3125ab95419970575b42af7541df"}, + {file = "coverage-7.6.1.tar.gz", hash = "sha256:953510dfb7b12ab69d20135a0662397f077c59b1e6379a768e97c59d852ee51d"}, ] [package.dependencies] @@ -757,13 +792,13 @@ test-randomorder = ["pytest-randomly"] [[package]] name = "dataclasses-json" -version = "0.6.6" +version = "0.6.7" description = "Easily serialize dataclasses to and from JSON." optional = false python-versions = "<4.0,>=3.7" files = [ - {file = "dataclasses_json-0.6.6-py3-none-any.whl", hash = "sha256:e54c5c87497741ad454070ba0ed411523d46beb5da102e221efb873801b0ba85"}, - {file = "dataclasses_json-0.6.6.tar.gz", hash = "sha256:0c09827d26fffda27f1be2fed7a7a01a29c5ddcd2eb6393ad5ebf9d77e9deae8"}, + {file = "dataclasses_json-0.6.7-py3-none-any.whl", hash = "sha256:0dbf33f26c8d5305befd61b39d2b3414e8a407bedc2834dea9b8d642666fb40a"}, + {file = "dataclasses_json-0.6.7.tar.gz", hash = "sha256:b6b3e528266ea45b9535223bc53ca645f5208833c29229e847b3f26a1cc55fc0"}, ] [package.dependencies] @@ -794,33 +829,33 @@ langdetect = ["langdetect"] [[package]] name = "debugpy" -version = "1.8.1" +version = "1.8.5" description = "An implementation of the Debug Adapter Protocol for Python" optional = false python-versions = ">=3.8" files = [ - {file = "debugpy-1.8.1-cp310-cp310-macosx_11_0_x86_64.whl", hash = "sha256:3bda0f1e943d386cc7a0e71bfa59f4137909e2ed947fb3946c506e113000f741"}, - {file = "debugpy-1.8.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dda73bf69ea479c8577a0448f8c707691152e6c4de7f0c4dec5a4bc11dee516e"}, - {file = "debugpy-1.8.1-cp310-cp310-win32.whl", hash = "sha256:3a79c6f62adef994b2dbe9fc2cc9cc3864a23575b6e387339ab739873bea53d0"}, - {file = "debugpy-1.8.1-cp310-cp310-win_amd64.whl", hash = "sha256:7eb7bd2b56ea3bedb009616d9e2f64aab8fc7000d481faec3cd26c98a964bcdd"}, - {file = "debugpy-1.8.1-cp311-cp311-macosx_11_0_universal2.whl", hash = "sha256:016a9fcfc2c6b57f939673c874310d8581d51a0fe0858e7fac4e240c5eb743cb"}, - {file = "debugpy-1.8.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fd97ed11a4c7f6d042d320ce03d83b20c3fb40da892f994bc041bbc415d7a099"}, - {file = "debugpy-1.8.1-cp311-cp311-win32.whl", hash = "sha256:0de56aba8249c28a300bdb0672a9b94785074eb82eb672db66c8144fff673146"}, - {file = "debugpy-1.8.1-cp311-cp311-win_amd64.whl", hash = "sha256:1a9fe0829c2b854757b4fd0a338d93bc17249a3bf69ecf765c61d4c522bb92a8"}, - {file = "debugpy-1.8.1-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:3ebb70ba1a6524d19fa7bb122f44b74170c447d5746a503e36adc244a20ac539"}, - {file = "debugpy-1.8.1-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a2e658a9630f27534e63922ebf655a6ab60c370f4d2fc5c02a5b19baf4410ace"}, - {file = "debugpy-1.8.1-cp312-cp312-win32.whl", hash = "sha256:caad2846e21188797a1f17fc09c31b84c7c3c23baf2516fed5b40b378515bbf0"}, - {file = "debugpy-1.8.1-cp312-cp312-win_amd64.whl", hash = "sha256:edcc9f58ec0fd121a25bc950d4578df47428d72e1a0d66c07403b04eb93bcf98"}, - {file = "debugpy-1.8.1-cp38-cp38-macosx_11_0_x86_64.whl", hash = "sha256:7a3afa222f6fd3d9dfecd52729bc2e12c93e22a7491405a0ecbf9e1d32d45b39"}, - {file = "debugpy-1.8.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d915a18f0597ef685e88bb35e5d7ab968964b7befefe1aaea1eb5b2640b586c7"}, - {file = "debugpy-1.8.1-cp38-cp38-win32.whl", hash = "sha256:92116039b5500633cc8d44ecc187abe2dfa9b90f7a82bbf81d079fcdd506bae9"}, - {file = "debugpy-1.8.1-cp38-cp38-win_amd64.whl", hash = "sha256:e38beb7992b5afd9d5244e96ad5fa9135e94993b0c551ceebf3fe1a5d9beb234"}, - {file = "debugpy-1.8.1-cp39-cp39-macosx_11_0_x86_64.whl", hash = "sha256:bfb20cb57486c8e4793d41996652e5a6a885b4d9175dd369045dad59eaacea42"}, - {file = "debugpy-1.8.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:efd3fdd3f67a7e576dd869c184c5dd71d9aaa36ded271939da352880c012e703"}, - {file = "debugpy-1.8.1-cp39-cp39-win32.whl", hash = "sha256:58911e8521ca0c785ac7a0539f1e77e0ce2df753f786188f382229278b4cdf23"}, - {file = "debugpy-1.8.1-cp39-cp39-win_amd64.whl", hash = "sha256:6df9aa9599eb05ca179fb0b810282255202a66835c6efb1d112d21ecb830ddd3"}, - {file = "debugpy-1.8.1-py2.py3-none-any.whl", hash = "sha256:28acbe2241222b87e255260c76741e1fbf04fdc3b6d094fcf57b6c6f75ce1242"}, - {file = "debugpy-1.8.1.zip", hash = "sha256:f696d6be15be87aef621917585f9bb94b1dc9e8aced570db1b8a6fc14e8f9b42"}, + {file = "debugpy-1.8.5-cp310-cp310-macosx_12_0_x86_64.whl", hash = "sha256:7e4d594367d6407a120b76bdaa03886e9eb652c05ba7f87e37418426ad2079f7"}, + {file = "debugpy-1.8.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4413b7a3ede757dc33a273a17d685ea2b0c09dbd312cc03f5534a0fd4d40750a"}, + {file = "debugpy-1.8.5-cp310-cp310-win32.whl", hash = "sha256:dd3811bd63632bb25eda6bd73bea8e0521794cda02be41fa3160eb26fc29e7ed"}, + {file = "debugpy-1.8.5-cp310-cp310-win_amd64.whl", hash = "sha256:b78c1250441ce893cb5035dd6f5fc12db968cc07f91cc06996b2087f7cefdd8e"}, + {file = "debugpy-1.8.5-cp311-cp311-macosx_12_0_universal2.whl", hash = "sha256:606bccba19f7188b6ea9579c8a4f5a5364ecd0bf5a0659c8a5d0e10dcee3032a"}, + {file = "debugpy-1.8.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:db9fb642938a7a609a6c865c32ecd0d795d56c1aaa7a7a5722d77855d5e77f2b"}, + {file = "debugpy-1.8.5-cp311-cp311-win32.whl", hash = "sha256:4fbb3b39ae1aa3e5ad578f37a48a7a303dad9a3d018d369bc9ec629c1cfa7408"}, + {file = "debugpy-1.8.5-cp311-cp311-win_amd64.whl", hash = "sha256:345d6a0206e81eb68b1493ce2fbffd57c3088e2ce4b46592077a943d2b968ca3"}, + {file = "debugpy-1.8.5-cp312-cp312-macosx_12_0_universal2.whl", hash = "sha256:5b5c770977c8ec6c40c60d6f58cacc7f7fe5a45960363d6974ddb9b62dbee156"}, + {file = "debugpy-1.8.5-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c0a65b00b7cdd2ee0c2cf4c7335fef31e15f1b7056c7fdbce9e90193e1a8c8cb"}, + {file = "debugpy-1.8.5-cp312-cp312-win32.whl", hash = "sha256:c9f7c15ea1da18d2fcc2709e9f3d6de98b69a5b0fff1807fb80bc55f906691f7"}, + {file = "debugpy-1.8.5-cp312-cp312-win_amd64.whl", hash = "sha256:28ced650c974aaf179231668a293ecd5c63c0a671ae6d56b8795ecc5d2f48d3c"}, + {file = "debugpy-1.8.5-cp38-cp38-macosx_12_0_x86_64.whl", hash = "sha256:3df6692351172a42af7558daa5019651f898fc67450bf091335aa8a18fbf6f3a"}, + {file = "debugpy-1.8.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1cd04a73eb2769eb0bfe43f5bfde1215c5923d6924b9b90f94d15f207a402226"}, + {file = "debugpy-1.8.5-cp38-cp38-win32.whl", hash = "sha256:8f913ee8e9fcf9d38a751f56e6de12a297ae7832749d35de26d960f14280750a"}, + {file = "debugpy-1.8.5-cp38-cp38-win_amd64.whl", hash = "sha256:a697beca97dad3780b89a7fb525d5e79f33821a8bc0c06faf1f1289e549743cf"}, + {file = "debugpy-1.8.5-cp39-cp39-macosx_12_0_x86_64.whl", hash = "sha256:0a1029a2869d01cb777216af8c53cda0476875ef02a2b6ff8b2f2c9a4b04176c"}, + {file = "debugpy-1.8.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e84c276489e141ed0b93b0af648eef891546143d6a48f610945416453a8ad406"}, + {file = "debugpy-1.8.5-cp39-cp39-win32.whl", hash = "sha256:ad84b7cde7fd96cf6eea34ff6c4a1b7887e0fe2ea46e099e53234856f9d99a34"}, + {file = "debugpy-1.8.5-cp39-cp39-win_amd64.whl", hash = "sha256:7b0fe36ed9d26cb6836b0a51453653f8f2e347ba7348f2bbfe76bfeb670bfb1c"}, + {file = "debugpy-1.8.5-py2.py3-none-any.whl", hash = "sha256:55919dce65b471eff25901acf82d328bbd5b833526b6c1364bd5133754777a44"}, + {file = "debugpy-1.8.5.zip", hash = "sha256:b2112cfeb34b4507399d298fe7023a16656fc553ed5246536060ca7bd0e668d0"}, ] [[package]] @@ -937,13 +972,13 @@ files = [ [[package]] name = "exceptiongroup" -version = "1.2.1" +version = "1.2.2" description = "Backport of PEP 654 (exception groups)" optional = false python-versions = ">=3.7" files = [ - {file = "exceptiongroup-1.2.1-py3-none-any.whl", hash = "sha256:5258b9ed329c5bbdd31a309f53cbfb0b155341807f6ff7606a1e801a891b29ad"}, - {file = "exceptiongroup-1.2.1.tar.gz", hash = "sha256:a4785e48b045528f5bfe627b6ad554ff32def154f42372786903b7abcfe1aa16"}, + {file = "exceptiongroup-1.2.2-py3-none-any.whl", hash = "sha256:3111b9d131c238bec2f8f516e123e14ba243563fb135d3fe885990585aa7795b"}, + {file = "exceptiongroup-1.2.2.tar.gz", hash = "sha256:47c2edf7c6738fafb49fd34290706d1a1a2f4d1c6df275526b62cbb4aa5393cc"}, ] [package.extras] @@ -965,13 +1000,13 @@ testing = ["hatch", "pre-commit", "pytest", "tox"] [[package]] name = "executing" -version = "2.0.1" +version = "2.1.0" description = "Get the currently executing AST node of a frame, and other information" optional = false -python-versions = ">=3.5" +python-versions = ">=3.8" files = [ - {file = "executing-2.0.1-py2.py3-none-any.whl", hash = "sha256:eac49ca94516ccc753f9fb5ce82603156e590b27525a8bc32cce8ae302eb61bc"}, - {file = "executing-2.0.1.tar.gz", hash = "sha256:35afe2ce3affba8ee97f2d69927fa823b08b472b7b994e36a52a964b93d16147"}, + {file = "executing-2.1.0-py2.py3-none-any.whl", hash = "sha256:8d63781349375b5ebccc3142f4b30350c0cd9c79f921cde38be2be4637e98eaf"}, + {file = "executing-2.1.0.tar.gz", hash = "sha256:8ea27ddd260da8150fa5a708269c4a10e76161e2496ec3e587da9e3c0fe4b9ab"}, ] [package.extras] @@ -979,13 +1014,13 @@ tests = ["asttokens (>=2.1.0)", "coverage", "coverage-enable-subprocess", "ipyth [[package]] name = "fastjsonschema" -version = "2.19.1" +version = "2.20.0" description = "Fastest Python implementation of JSON schema" optional = false python-versions = "*" files = [ - {file = "fastjsonschema-2.19.1-py3-none-any.whl", hash = "sha256:3672b47bc94178c9f23dbb654bf47440155d4db9df5f7bc47643315f9c405cd0"}, - {file = "fastjsonschema-2.19.1.tar.gz", hash = "sha256:e3126a94bdc4623d3de4485f8d468a12f02a67921315ddc87836d6e456dc789d"}, + {file = "fastjsonschema-2.20.0-py3-none-any.whl", hash = "sha256:5875f0b0fa7a0043a91e93a9b8f793bcbbba9691e7fd83dca95c28ba26d21f0a"}, + {file = "fastjsonschema-2.20.0.tar.gz", hash = "sha256:3d48fc5300ee96f5d116f10fe6f28d938e6008f59a6a025c2649475b87f76a23"}, ] [package.extras] @@ -993,18 +1028,18 @@ devel = ["colorama", "json-spec", "jsonschema", "pylint", "pytest", "pytest-benc [[package]] name = "filelock" -version = "3.14.0" +version = "3.15.4" description = "A platform independent file lock." optional = false python-versions = ">=3.8" files = [ - {file = "filelock-3.14.0-py3-none-any.whl", hash = "sha256:43339835842f110ca7ae60f1e1c160714c5a6afd15a2873419ab185334975c0f"}, - {file = "filelock-3.14.0.tar.gz", hash = "sha256:6ea72da3be9b8c82afd3edcf99f2fffbb5076335a5ae4d03248bb5b6c3eae78a"}, + {file = "filelock-3.15.4-py3-none-any.whl", hash = "sha256:6ca1fffae96225dab4c6eaf1c4f4f28cd2568d3ec2a44e15a08520504de468e7"}, + {file = "filelock-3.15.4.tar.gz", hash = "sha256:2207938cbc1844345cb01a5a95524dae30f0ce089eba5b00378295a17e3e90cb"}, ] [package.extras] docs = ["furo (>=2023.9.10)", "sphinx (>=7.2.6)", "sphinx-autodoc-typehints (>=1.25.2)"] -testing = ["covdefaults (>=2.3)", "coverage (>=7.3.2)", "diff-cover (>=8.0.1)", "pytest (>=7.4.3)", "pytest-cov (>=4.1)", "pytest-mock (>=3.12)", "pytest-timeout (>=2.2)"] +testing = ["covdefaults (>=2.3)", "coverage (>=7.3.2)", "diff-cover (>=8.0.1)", "pytest (>=7.4.3)", "pytest-asyncio (>=0.21)", "pytest-cov (>=4.1)", "pytest-mock (>=3.12)", "pytest-timeout (>=2.2)", "virtualenv (>=20.26.2)"] typing = ["typing-extensions (>=4.8)"] [[package]] @@ -1091,20 +1126,20 @@ files = [ [[package]] name = "google-api-core" -version = "2.19.0" +version = "2.19.2" description = "Google API client core library" optional = false python-versions = ">=3.7" files = [ - {file = "google-api-core-2.19.0.tar.gz", hash = "sha256:cf1b7c2694047886d2af1128a03ae99e391108a08804f87cfd35970e49c9cd10"}, - {file = "google_api_core-2.19.0-py3-none-any.whl", hash = "sha256:8661eec4078c35428fd3f69a2c7ee29e342896b70f01d1a1cbcb334372dd6251"}, + {file = "google_api_core-2.19.2-py3-none-any.whl", hash = "sha256:53ec0258f2837dd53bbd3d3df50f5359281b3cc13f800c941dd15a9b5a415af4"}, + {file = "google_api_core-2.19.2.tar.gz", hash = "sha256:ca07de7e8aa1c98a8bfca9321890ad2340ef7f2eb136e558cee68f24b94b0a8f"}, ] [package.dependencies] google-auth = ">=2.14.1,<3.0.dev0" googleapis-common-protos = ">=1.56.2,<2.0.dev0" proto-plus = ">=1.22.3,<2.0.0dev" -protobuf = ">=3.19.5,<3.20.0 || >3.20.0,<3.20.1 || >3.20.1,<4.21.0 || >4.21.0,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<5.0.0.dev0" +protobuf = ">=3.19.5,<3.20.0 || >3.20.0,<3.20.1 || >3.20.1,<4.21.0 || >4.21.0,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<6.0.0.dev0" requests = ">=2.18.0,<3.0.0.dev0" [package.extras] @@ -1132,13 +1167,13 @@ uritemplate = ">=3.0.1,<5" [[package]] name = "google-auth" -version = "2.29.0" +version = "2.34.0" description = "Google Authentication Library" optional = false python-versions = ">=3.7" files = [ - {file = "google-auth-2.29.0.tar.gz", hash = "sha256:672dff332d073227550ffc7457868ac4218d6c500b155fe6cc17d2b13602c360"}, - {file = "google_auth-2.29.0-py2.py3-none-any.whl", hash = "sha256:d452ad095688cd52bae0ad6fafe027f6a6d6f560e810fec20914e17a09526415"}, + {file = "google_auth-2.34.0-py2.py3-none-any.whl", hash = "sha256:72fd4733b80b6d777dcde515628a9eb4a577339437012874ea286bca7261ee65"}, + {file = "google_auth-2.34.0.tar.gz", hash = "sha256:8eb87396435c19b20d32abd2f984e31c191a15284af72eb922f10e5bde9c04cc"}, ] [package.dependencies] @@ -1148,7 +1183,7 @@ rsa = ">=3.1.4,<5" [package.extras] aiohttp = ["aiohttp (>=3.6.2,<4.0.0.dev0)", "requests (>=2.20.0,<3.0.0.dev0)"] -enterprise-cert = ["cryptography (==36.0.2)", "pyopenssl (==22.0.0)"] +enterprise-cert = ["cryptography", "pyopenssl"] pyopenssl = ["cryptography (>=38.0.3)", "pyopenssl (>=20.0.0)"] reauth = ["pyu2f (>=0.1.5)"] requests = ["requests (>=2.20.0,<3.0.0.dev0)"] @@ -1188,17 +1223,17 @@ tool = ["click (>=6.0.0)"] [[package]] name = "googleapis-common-protos" -version = "1.63.0" +version = "1.65.0" description = "Common protobufs used in Google APIs" optional = false python-versions = ">=3.7" files = [ - {file = "googleapis-common-protos-1.63.0.tar.gz", hash = "sha256:17ad01b11d5f1d0171c06d3ba5c04c54474e883b66b949722b4938ee2694ef4e"}, - {file = "googleapis_common_protos-1.63.0-py2.py3-none-any.whl", hash = "sha256:ae45f75702f7c08b541f750854a678bd8f534a1a6bace6afe975f1d0a82d6632"}, + {file = "googleapis_common_protos-1.65.0-py2.py3-none-any.whl", hash = "sha256:2972e6c496f435b92590fd54045060867f3fe9be2c82ab148fc8885035479a63"}, + {file = "googleapis_common_protos-1.65.0.tar.gz", hash = "sha256:334a29d07cddc3aa01dee4988f9afd9b2916ee2ff49d6b757155dc0d197852c0"}, ] [package.dependencies] -protobuf = ">=3.19.5,<3.20.0 || >3.20.0,<3.20.1 || >3.20.1,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<5.0.0.dev0" +protobuf = ">=3.20.2,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<6.0.0.dev0" [package.extras] grpc = ["grpcio (>=1.44.0,<2.0.0.dev0)"] @@ -1412,13 +1447,13 @@ pyparsing = {version = ">=2.4.2,<3.0.0 || >3.0.0,<3.0.1 || >3.0.1,<3.0.2 || >3.0 [[package]] name = "httpx" -version = "0.27.0" +version = "0.27.2" description = "The next generation HTTP client." optional = false python-versions = ">=3.8" files = [ - {file = "httpx-0.27.0-py3-none-any.whl", hash = "sha256:71d5465162c13681bff01ad59b2cc68dd838ea1f10e51574bac27103f00c91a5"}, - {file = "httpx-0.27.0.tar.gz", hash = "sha256:a0cb88a46f32dc874e04ee956e4c2764aba2aa228f650b06788ba6bda2962ab5"}, + {file = "httpx-0.27.2-py3-none-any.whl", hash = "sha256:7bb2708e112d8fdd7829cd4243970f0c223274051cb35ee80c03301ee29a3df0"}, + {file = "httpx-0.27.2.tar.gz", hash = "sha256:f7c2be1d2f3c3c3160d441802406b206c2b76f5947b11115e6df10c6c65e66c2"}, ] [package.dependencies] @@ -1433,16 +1468,17 @@ brotli = ["brotli", "brotlicffi"] cli = ["click (==8.*)", "pygments (==2.*)", "rich (>=10,<14)"] http2 = ["h2 (>=3,<5)"] socks = ["socksio (==1.*)"] +zstd = ["zstandard (>=0.18.0)"] [[package]] name = "identify" -version = "2.5.36" +version = "2.6.0" description = "File identification library for Python" optional = false python-versions = ">=3.8" files = [ - {file = "identify-2.5.36-py2.py3-none-any.whl", hash = "sha256:37d93f380f4de590500d9dba7db359d0d3da95ffe7f9de1753faa159e71e7dfa"}, - {file = "identify-2.5.36.tar.gz", hash = "sha256:e5e00f54165f9047fbebeb4a560f9acfb8af4c88232be60a488e9b68d122745d"}, + {file = "identify-2.6.0-py2.py3-none-any.whl", hash = "sha256:e79ae4406387a9d300332b5fd366d8994f1525e8414984e1a59e058b2eda2dd0"}, + {file = "identify-2.6.0.tar.gz", hash = "sha256:cb171c685bdc31bcc4c1734698736a7d5b6c8bf2e0c15117f4d469c8640ae5cf"}, ] [package.extras] @@ -1450,13 +1486,13 @@ license = ["ukkonen"] [[package]] name = "idna" -version = "3.7" +version = "3.8" description = "Internationalized Domain Names in Applications (IDNA)" optional = false -python-versions = ">=3.5" +python-versions = ">=3.6" files = [ - {file = "idna-3.7-py3-none-any.whl", hash = "sha256:82fee1fc78add43492d3a1898bfa6d8a904cc97d8427f683ed8e798d07761aa0"}, - {file = "idna-3.7.tar.gz", hash = "sha256:028ff3aadf0609c1fd278d8ea3089299412a7a8b9bd005dd08b9f8285bcb5cfc"}, + {file = "idna-3.8-py3-none-any.whl", hash = "sha256:050b4e5baadcd44d760cedbd2b8e639f2ff89bbc7a5730fcc662954303377aac"}, + {file = "idna-3.8.tar.gz", hash = "sha256:d838c2c0ed6fced7693d5e8ab8e734d5f8fda53a039c0164afb0b82e771e3603"}, ] [[package]] @@ -1538,13 +1574,13 @@ tests = ["coverage[toml]", "pytest", "pytest-cov", "pytest-mock"] [[package]] name = "ipykernel" -version = "6.29.4" +version = "6.29.5" description = "IPython Kernel for Jupyter" optional = false python-versions = ">=3.8" files = [ - {file = "ipykernel-6.29.4-py3-none-any.whl", hash = "sha256:1181e653d95c6808039c509ef8e67c4126b3b3af7781496c7cbfb5ed938a27da"}, - {file = "ipykernel-6.29.4.tar.gz", hash = "sha256:3d44070060f9475ac2092b760123fadf105d2e2493c24848b6691a7c4f42af5c"}, + {file = "ipykernel-6.29.5-py3-none-any.whl", hash = "sha256:afdb66ba5aa354b09b91379bac28ae4afebbb30e8b39510c9690afb7a10421b5"}, + {file = "ipykernel-6.29.5.tar.gz", hash = "sha256:f093a22c4a40f8828f8e330a9c297cb93dcab13bd9678ded6de8e5cf81c56215"}, ] [package.dependencies] @@ -1608,21 +1644,21 @@ test-extra = ["curio", "matplotlib (!=3.2.0)", "nbformat", "numpy (>=1.22)", "pa [[package]] name = "ipywidgets" -version = "8.1.2" +version = "8.1.5" description = "Jupyter interactive widgets" optional = false python-versions = ">=3.7" files = [ - {file = "ipywidgets-8.1.2-py3-none-any.whl", hash = "sha256:bbe43850d79fb5e906b14801d6c01402857996864d1e5b6fa62dd2ee35559f60"}, - {file = "ipywidgets-8.1.2.tar.gz", hash = "sha256:d0b9b41e49bae926a866e613a39b0f0097745d2b9f1f3dd406641b4a57ec42c9"}, + {file = "ipywidgets-8.1.5-py3-none-any.whl", hash = "sha256:3290f526f87ae6e77655555baba4f36681c555b8bdbbff430b70e52c34c86245"}, + {file = "ipywidgets-8.1.5.tar.gz", hash = "sha256:870e43b1a35656a80c18c9503bbf2d16802db1cb487eec6fab27d683381dde17"}, ] [package.dependencies] comm = ">=0.1.3" ipython = ">=6.1.0" -jupyterlab-widgets = ">=3.0.10,<3.1.0" +jupyterlab-widgets = ">=3.0.12,<3.1.0" traitlets = ">=4.3.1" -widgetsnbextension = ">=4.0.10,<4.1.0" +widgetsnbextension = ">=4.0.12,<4.1.0" [package.extras] test = ["ipykernel", "jsonschema", "pytest (>=3.6.0)", "pytest-cov", "pytz"] @@ -1762,24 +1798,24 @@ jsonpointer = ">=1.9" [[package]] name = "jsonpointer" -version = "2.4" +version = "3.0.0" description = "Identify specific nodes in a JSON document (RFC 6901)" optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*, !=3.6.*" +python-versions = ">=3.7" files = [ - {file = "jsonpointer-2.4-py2.py3-none-any.whl", hash = "sha256:15d51bba20eea3165644553647711d150376234112651b4f1811022aecad7d7a"}, - {file = "jsonpointer-2.4.tar.gz", hash = "sha256:585cee82b70211fa9e6043b7bb89db6e1aa49524340dde8ad6b63206ea689d88"}, + {file = "jsonpointer-3.0.0-py2.py3-none-any.whl", hash = "sha256:13e088adc14fca8b6aa8177c044e12701e6ad4b28ff10e65f2267a90109c9942"}, + {file = "jsonpointer-3.0.0.tar.gz", hash = "sha256:2b2d729f2091522d61c3b31f82e11870f60b68f43fbc705cb76bf4b832af59ef"}, ] [[package]] name = "jsonschema" -version = "4.22.0" +version = "4.23.0" description = "An implementation of JSON Schema validation for Python" optional = false python-versions = ">=3.8" files = [ - {file = "jsonschema-4.22.0-py3-none-any.whl", hash = "sha256:ff4cfd6b1367a40e7bc6411caec72effadd3db0bbe5017de188f2d6108335802"}, - {file = "jsonschema-4.22.0.tar.gz", hash = "sha256:5b22d434a45935119af990552c862e5d6d564e8f6601206b305a61fdf661a2b7"}, + {file = "jsonschema-4.23.0-py3-none-any.whl", hash = "sha256:fbadb6f8b144a8f8cf9f0b89ba94501d143e50411a1278633f56a7acf7fd5566"}, + {file = "jsonschema-4.23.0.tar.gz", hash = "sha256:d71497fef26351a33265337fa77ffeb82423f3ea21283cd9467bb03999266bc4"}, ] [package.dependencies] @@ -1794,11 +1830,11 @@ rfc3339-validator = {version = "*", optional = true, markers = "extra == \"forma rfc3986-validator = {version = ">0.1.0", optional = true, markers = "extra == \"format-nongpl\""} rpds-py = ">=0.7.1" uri-template = {version = "*", optional = true, markers = "extra == \"format-nongpl\""} -webcolors = {version = ">=1.11", optional = true, markers = "extra == \"format-nongpl\""} +webcolors = {version = ">=24.6.0", optional = true, markers = "extra == \"format-nongpl\""} [package.extras] format = ["fqdn", "idna", "isoduration", "jsonpointer (>1.13)", "rfc3339-validator", "rfc3987", "uri-template", "webcolors (>=1.11)"] -format-nongpl = ["fqdn", "idna", "isoduration", "jsonpointer (>1.13)", "rfc3339-validator", "rfc3986-validator (>0.1.0)", "uri-template", "webcolors (>=1.11)"] +format-nongpl = ["fqdn", "idna", "isoduration", "jsonpointer (>1.13)", "rfc3339-validator", "rfc3986-validator (>0.1.0)", "uri-template", "webcolors (>=24.6.0)"] [[package]] name = "jsonschema-specifications" @@ -1816,13 +1852,13 @@ referencing = ">=0.31.0" [[package]] name = "jupyter-client" -version = "8.6.1" +version = "8.6.2" description = "Jupyter protocol implementation and client libraries" optional = false python-versions = ">=3.8" files = [ - {file = "jupyter_client-8.6.1-py3-none-any.whl", hash = "sha256:3b7bd22f058434e3b9a7ea4b1500ed47de2713872288c0d511d19926f99b459f"}, - {file = "jupyter_client-8.6.1.tar.gz", hash = "sha256:e842515e2bab8e19186d89fdfea7abd15e39dd581f94e399f00e2af5a1652d3f"}, + {file = "jupyter_client-8.6.2-py3-none-any.whl", hash = "sha256:50cbc5c66fd1b8f65ecb66bc490ab73217993632809b6e505687de18e9dea39f"}, + {file = "jupyter_client-8.6.2.tar.gz", hash = "sha256:2bda14d55ee5ba58552a8c53ae43d215ad9868853489213f37da060ced54d8df"}, ] [package.dependencies] @@ -1835,7 +1871,7 @@ traitlets = ">=5.3" [package.extras] docs = ["ipykernel", "myst-parser", "pydata-sphinx-theme", "sphinx (>=4)", "sphinx-autodoc-typehints", "sphinxcontrib-github-alt", "sphinxcontrib-spelling"] -test = ["coverage", "ipykernel (>=6.14)", "mypy", "paramiko", "pre-commit", "pytest", "pytest-cov", "pytest-jupyter[client] (>=0.4.1)", "pytest-timeout"] +test = ["coverage", "ipykernel (>=6.14)", "mypy", "paramiko", "pre-commit", "pytest (<8.2.0)", "pytest-cov", "pytest-jupyter[client] (>=0.4.1)", "pytest-timeout"] [[package]] name = "jupyter-core" @@ -1899,13 +1935,13 @@ jupyter-server = ">=1.1.2" [[package]] name = "jupyter-server" -version = "2.14.0" +version = "2.14.2" description = "The backend—i.e. core services, APIs, and REST endpoints—to Jupyter web applications." optional = false python-versions = ">=3.8" files = [ - {file = "jupyter_server-2.14.0-py3-none-any.whl", hash = "sha256:fb6be52c713e80e004fac34b35a0990d6d36ba06fd0a2b2ed82b899143a64210"}, - {file = "jupyter_server-2.14.0.tar.gz", hash = "sha256:659154cea512083434fd7c93b7fe0897af7a2fd0b9dd4749282b42eaac4ae677"}, + {file = "jupyter_server-2.14.2-py3-none-any.whl", hash = "sha256:47ff506127c2f7851a17bf4713434208fc490955d0e8632e95014a9a9afbeefd"}, + {file = "jupyter_server-2.14.2.tar.gz", hash = "sha256:66095021aa9638ced276c248b1d81862e4c50f292d575920bbe960de1c56b12b"}, ] [package.dependencies] @@ -1930,7 +1966,7 @@ traitlets = ">=5.6.0" websocket-client = ">=1.7" [package.extras] -docs = ["ipykernel", "jinja2", "jupyter-client", "jupyter-server", "myst-parser", "nbformat", "prometheus-client", "pydata-sphinx-theme", "send2trash", "sphinx-autodoc-typehints", "sphinxcontrib-github-alt", "sphinxcontrib-openapi (>=0.8.0)", "sphinxcontrib-spelling", "sphinxemoji", "tornado", "typing-extensions"] +docs = ["ipykernel", "jinja2", "jupyter-client", "myst-parser", "nbformat", "prometheus-client", "pydata-sphinx-theme", "send2trash", "sphinx-autodoc-typehints", "sphinxcontrib-github-alt", "sphinxcontrib-openapi (>=0.8.0)", "sphinxcontrib-spelling", "sphinxemoji", "tornado", "typing-extensions"] test = ["flaky", "ipykernel", "pre-commit", "pytest (>=7.0,<9)", "pytest-console-scripts", "pytest-jupyter[server] (>=0.7)", "pytest-timeout", "requests"] [[package]] @@ -1954,13 +1990,13 @@ test = ["jupyter-server (>=2.0.0)", "pytest (>=7.0)", "pytest-jupyter[server] (> [[package]] name = "jupyterlab" -version = "4.1.8" +version = "4.2.5" description = "JupyterLab computational environment" optional = false python-versions = ">=3.8" files = [ - {file = "jupyterlab-4.1.8-py3-none-any.whl", hash = "sha256:c3baf3a2f91f89d110ed5786cd18672b9a357129d4e389d2a0dead15e11a4d2c"}, - {file = "jupyterlab-4.1.8.tar.gz", hash = "sha256:3384aded8680e7ce504fd63b8bb89a39df21c9c7694d9e7dc4a68742cdb30f9b"}, + {file = "jupyterlab-4.2.5-py3-none-any.whl", hash = "sha256:73b6e0775d41a9fee7ee756c80f58a6bed4040869ccc21411dc559818874d321"}, + {file = "jupyterlab-4.2.5.tar.gz", hash = "sha256:ae7f3a1b8cb88b4f55009ce79fa7c06f99d70cd63601ee4aa91815d054f46f75"}, ] [package.dependencies] @@ -1975,16 +2011,17 @@ jupyter-server = ">=2.4.0,<3" jupyterlab-server = ">=2.27.1,<3" notebook-shim = ">=0.2" packaging = "*" +setuptools = ">=40.1.0" tomli = {version = ">=1.2.2", markers = "python_version < \"3.11\""} tornado = ">=6.2.0" traitlets = "*" [package.extras] -dev = ["build", "bump2version", "coverage", "hatch", "pre-commit", "pytest-cov", "ruff (==0.2.0)"] +dev = ["build", "bump2version", "coverage", "hatch", "pre-commit", "pytest-cov", "ruff (==0.3.5)"] docs = ["jsx-lexer", "myst-parser", "pydata-sphinx-theme (>=0.13.0)", "pytest", "pytest-check-links", "pytest-jupyter", "sphinx (>=1.8,<7.3.0)", "sphinx-copybutton"] -docs-screenshots = ["altair (==5.2.0)", "ipython (==8.16.1)", "ipywidgets (==8.1.1)", "jupyterlab-geojson (==3.4.0)", "jupyterlab-language-pack-zh-cn (==4.0.post6)", "matplotlib (==3.8.2)", "nbconvert (>=7.0.0)", "pandas (==2.2.0)", "scipy (==1.12.0)", "vega-datasets (==0.9.0)"] +docs-screenshots = ["altair (==5.3.0)", "ipython (==8.16.1)", "ipywidgets (==8.1.2)", "jupyterlab-geojson (==3.4.0)", "jupyterlab-language-pack-zh-cn (==4.1.post2)", "matplotlib (==3.8.3)", "nbconvert (>=7.0.0)", "pandas (==2.2.1)", "scipy (==1.12.0)", "vega-datasets (==0.9.0)"] test = ["coverage", "pytest (>=7.0)", "pytest-check-links (>=0.7)", "pytest-console-scripts", "pytest-cov", "pytest-jupyter (>=0.5.3)", "pytest-timeout", "pytest-tornasync", "requests", "requests-cache", "virtualenv"] -upgrade-extension = ["copier (>=8.0,<9.0)", "jinja2-time (<0.3)", "pydantic (<2.0)", "pyyaml-include (<2.0)", "tomli-w (<2.0)"] +upgrade-extension = ["copier (>=9,<10)", "jinja2-time (<0.3)", "pydantic (<3.0)", "pyyaml-include (<3.0)", "tomli-w (<2.0)"] [[package]] name = "jupyterlab-pygments" @@ -1999,13 +2036,13 @@ files = [ [[package]] name = "jupyterlab-server" -version = "2.27.1" +version = "2.27.3" description = "A set of server components for JupyterLab and JupyterLab like applications." optional = false python-versions = ">=3.8" files = [ - {file = "jupyterlab_server-2.27.1-py3-none-any.whl", hash = "sha256:f5e26156e5258b24d532c84e7c74cc212e203bff93eb856f81c24c16daeecc75"}, - {file = "jupyterlab_server-2.27.1.tar.gz", hash = "sha256:097b5ac709b676c7284ac9c5e373f11930a561f52cd5a86e4fc7e5a9c8a8631d"}, + {file = "jupyterlab_server-2.27.3-py3-none-any.whl", hash = "sha256:e697488f66c3db49df675158a77b3b017520d772c6e1548c7d9bcc5df7944ee4"}, + {file = "jupyterlab_server-2.27.3.tar.gz", hash = "sha256:eb36caca59e74471988f0ae25c77945610b887f777255aa21f8065def9e51ed4"}, ] [package.dependencies] @@ -2025,13 +2062,13 @@ test = ["hatch", "ipykernel", "openapi-core (>=0.18.0,<0.19.0)", "openapi-spec-v [[package]] name = "jupyterlab-widgets" -version = "3.0.10" +version = "3.0.13" description = "Jupyter interactive widgets for JupyterLab" optional = false python-versions = ">=3.7" files = [ - {file = "jupyterlab_widgets-3.0.10-py3-none-any.whl", hash = "sha256:dd61f3ae7a5a7f80299e14585ce6cf3d6925a96c9103c978eda293197730cb64"}, - {file = "jupyterlab_widgets-3.0.10.tar.gz", hash = "sha256:04f2ac04976727e4f9d0fa91cdc2f1ab860f965e504c29dbd6a65c882c9d04c0"}, + {file = "jupyterlab_widgets-3.0.13-py3-none-any.whl", hash = "sha256:e3cda2c233ce144192f1e29914ad522b2f4c40e77214b0cc97377ca3d323db54"}, + {file = "jupyterlab_widgets-3.0.13.tar.gz", hash = "sha256:a2966d385328c1942b683a8cd96b89b8dd82c8b8f81dda902bb2bc06d46f5bed"}, ] [[package]] @@ -2082,13 +2119,13 @@ files = [ [[package]] name = "makefun" -version = "1.15.2" +version = "1.15.4" description = "Small library to dynamically create python functions." optional = false python-versions = "*" files = [ - {file = "makefun-1.15.2-py2.py3-none-any.whl", hash = "sha256:1c83abfaefb6c3c7c83ed4a993b4a310af80adf6db15625b184b1f0f7545a041"}, - {file = "makefun-1.15.2.tar.gz", hash = "sha256:16f2a2b34d9ee0c2b578c960a1808c974e2822cf79f6e9b9c455aace10882d45"}, + {file = "makefun-1.15.4-py2.py3-none-any.whl", hash = "sha256:945d078a7e01a903f2cbef738b33e0ebc52b8d35fb7e20c528ed87b5c80db5b7"}, + {file = "makefun-1.15.4.tar.gz", hash = "sha256:9f9b9904e7c397759374a88f4c57781fbab2a458dec78df4b3ee6272cd9fb010"}, ] [[package]] @@ -2162,13 +2199,13 @@ files = [ [[package]] name = "marshmallow" -version = "3.21.2" +version = "3.22.0" description = "A lightweight library for converting complex datatypes to and from native Python datatypes." optional = false python-versions = ">=3.8" files = [ - {file = "marshmallow-3.21.2-py3-none-any.whl", hash = "sha256:70b54a6282f4704d12c0a41599682c5c5450e843b9ec406308653b47c59648a1"}, - {file = "marshmallow-3.21.2.tar.gz", hash = "sha256:82408deadd8b33d56338d2182d455db632c6313aa2af61916672146bb32edc56"}, + {file = "marshmallow-3.22.0-py3-none-any.whl", hash = "sha256:71a2dce49ef901c3f97ed296ae5051135fd3febd2bf43afe0ae9a82143a494d9"}, + {file = "marshmallow-3.22.0.tar.gz", hash = "sha256:4972f529104a220bb8637d595aa4c9762afbe7f7a77d82dc58c1615d70c5823e"}, ] [package.dependencies] @@ -2176,7 +2213,7 @@ packaging = ">=17.0" [package.extras] dev = ["marshmallow[tests]", "pre-commit (>=3.5,<4.0)", "tox"] -docs = ["alabaster (==0.7.16)", "autodocsumm (==0.2.12)", "sphinx (==7.3.7)", "sphinx-issues (==4.1.0)", "sphinx-version-warning (==1.1.2)"] +docs = ["alabaster (==1.0.0)", "autodocsumm (==0.2.13)", "sphinx (==8.0.2)", "sphinx-issues (==4.1.0)", "sphinx-version-warning (==1.1.2)"] tests = ["pytest", "pytz", "simplejson"] [[package]] @@ -2217,44 +2254,44 @@ files = [ [[package]] name = "mypy" -version = "1.10.0" +version = "1.11.2" description = "Optional static typing for Python" optional = false python-versions = ">=3.8" files = [ - {file = "mypy-1.10.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:da1cbf08fb3b851ab3b9523a884c232774008267b1f83371ace57f412fe308c2"}, - {file = "mypy-1.10.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:12b6bfc1b1a66095ab413160a6e520e1dc076a28f3e22f7fb25ba3b000b4ef99"}, - {file = "mypy-1.10.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9e36fb078cce9904c7989b9693e41cb9711e0600139ce3970c6ef814b6ebc2b2"}, - {file = "mypy-1.10.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:2b0695d605ddcd3eb2f736cd8b4e388288c21e7de85001e9f85df9187f2b50f9"}, - {file = "mypy-1.10.0-cp310-cp310-win_amd64.whl", hash = "sha256:cd777b780312ddb135bceb9bc8722a73ec95e042f911cc279e2ec3c667076051"}, - {file = "mypy-1.10.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:3be66771aa5c97602f382230165b856c231d1277c511c9a8dd058be4784472e1"}, - {file = "mypy-1.10.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8b2cbaca148d0754a54d44121b5825ae71868c7592a53b7292eeb0f3fdae95ee"}, - {file = "mypy-1.10.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1ec404a7cbe9fc0e92cb0e67f55ce0c025014e26d33e54d9e506a0f2d07fe5de"}, - {file = "mypy-1.10.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e22e1527dc3d4aa94311d246b59e47f6455b8729f4968765ac1eacf9a4760bc7"}, - {file = "mypy-1.10.0-cp311-cp311-win_amd64.whl", hash = "sha256:a87dbfa85971e8d59c9cc1fcf534efe664d8949e4c0b6b44e8ca548e746a8d53"}, - {file = "mypy-1.10.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:a781f6ad4bab20eef8b65174a57e5203f4be627b46291f4589879bf4e257b97b"}, - {file = "mypy-1.10.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:b808e12113505b97d9023b0b5e0c0705a90571c6feefc6f215c1df9381256e30"}, - {file = "mypy-1.10.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8f55583b12156c399dce2df7d16f8a5095291354f1e839c252ec6c0611e86e2e"}, - {file = "mypy-1.10.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:4cf18f9d0efa1b16478c4c129eabec36148032575391095f73cae2e722fcf9d5"}, - {file = "mypy-1.10.0-cp312-cp312-win_amd64.whl", hash = "sha256:bc6ac273b23c6b82da3bb25f4136c4fd42665f17f2cd850771cb600bdd2ebeda"}, - {file = "mypy-1.10.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:9fd50226364cd2737351c79807775136b0abe084433b55b2e29181a4c3c878c0"}, - {file = "mypy-1.10.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:f90cff89eea89273727d8783fef5d4a934be2fdca11b47def50cf5d311aff727"}, - {file = "mypy-1.10.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fcfc70599efde5c67862a07a1aaf50e55bce629ace26bb19dc17cece5dd31ca4"}, - {file = "mypy-1.10.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:075cbf81f3e134eadaf247de187bd604748171d6b79736fa9b6c9685b4083061"}, - {file = "mypy-1.10.0-cp38-cp38-win_amd64.whl", hash = "sha256:3f298531bca95ff615b6e9f2fc0333aae27fa48052903a0ac90215021cdcfa4f"}, - {file = "mypy-1.10.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:fa7ef5244615a2523b56c034becde4e9e3f9b034854c93639adb667ec9ec2976"}, - {file = "mypy-1.10.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:3236a4c8f535a0631f85f5fcdffba71c7feeef76a6002fcba7c1a8e57c8be1ec"}, - {file = "mypy-1.10.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4a2b5cdbb5dd35aa08ea9114436e0d79aceb2f38e32c21684dcf8e24e1e92821"}, - {file = "mypy-1.10.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:92f93b21c0fe73dc00abf91022234c79d793318b8a96faac147cd579c1671746"}, - {file = "mypy-1.10.0-cp39-cp39-win_amd64.whl", hash = "sha256:28d0e038361b45f099cc086d9dd99c15ff14d0188f44ac883010e172ce86c38a"}, - {file = "mypy-1.10.0-py3-none-any.whl", hash = "sha256:f8c083976eb530019175aabadb60921e73b4f45736760826aa1689dda8208aee"}, - {file = "mypy-1.10.0.tar.gz", hash = "sha256:3d087fcbec056c4ee34974da493a826ce316947485cef3901f511848e687c131"}, + {file = "mypy-1.11.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d42a6dd818ffce7be66cce644f1dff482f1d97c53ca70908dff0b9ddc120b77a"}, + {file = "mypy-1.11.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:801780c56d1cdb896eacd5619a83e427ce436d86a3bdf9112527f24a66618fef"}, + {file = "mypy-1.11.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:41ea707d036a5307ac674ea172875f40c9d55c5394f888b168033177fce47383"}, + {file = "mypy-1.11.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:6e658bd2d20565ea86da7d91331b0eed6d2eee22dc031579e6297f3e12c758c8"}, + {file = "mypy-1.11.2-cp310-cp310-win_amd64.whl", hash = "sha256:478db5f5036817fe45adb7332d927daa62417159d49783041338921dcf646fc7"}, + {file = "mypy-1.11.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:75746e06d5fa1e91bfd5432448d00d34593b52e7e91a187d981d08d1f33d4385"}, + {file = "mypy-1.11.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:a976775ab2256aadc6add633d44f100a2517d2388906ec4f13231fafbb0eccca"}, + {file = "mypy-1.11.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cd953f221ac1379050a8a646585a29574488974f79d8082cedef62744f0a0104"}, + {file = "mypy-1.11.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:57555a7715c0a34421013144a33d280e73c08df70f3a18a552938587ce9274f4"}, + {file = "mypy-1.11.2-cp311-cp311-win_amd64.whl", hash = "sha256:36383a4fcbad95f2657642a07ba22ff797de26277158f1cc7bd234821468b1b6"}, + {file = "mypy-1.11.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:e8960dbbbf36906c5c0b7f4fbf2f0c7ffb20f4898e6a879fcf56a41a08b0d318"}, + {file = "mypy-1.11.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:06d26c277962f3fb50e13044674aa10553981ae514288cb7d0a738f495550b36"}, + {file = "mypy-1.11.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6e7184632d89d677973a14d00ae4d03214c8bc301ceefcdaf5c474866814c987"}, + {file = "mypy-1.11.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:3a66169b92452f72117e2da3a576087025449018afc2d8e9bfe5ffab865709ca"}, + {file = "mypy-1.11.2-cp312-cp312-win_amd64.whl", hash = "sha256:969ea3ef09617aff826885a22ece0ddef69d95852cdad2f60c8bb06bf1f71f70"}, + {file = "mypy-1.11.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:37c7fa6121c1cdfcaac97ce3d3b5588e847aa79b580c1e922bb5d5d2902df19b"}, + {file = "mypy-1.11.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:4a8a53bc3ffbd161b5b2a4fff2f0f1e23a33b0168f1c0778ec70e1a3d66deb86"}, + {file = "mypy-1.11.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2ff93107f01968ed834f4256bc1fc4475e2fecf6c661260066a985b52741ddce"}, + {file = "mypy-1.11.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:edb91dded4df17eae4537668b23f0ff6baf3707683734b6a818d5b9d0c0c31a1"}, + {file = "mypy-1.11.2-cp38-cp38-win_amd64.whl", hash = "sha256:ee23de8530d99b6db0573c4ef4bd8f39a2a6f9b60655bf7a1357e585a3486f2b"}, + {file = "mypy-1.11.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:801ca29f43d5acce85f8e999b1e431fb479cb02d0e11deb7d2abb56bdaf24fd6"}, + {file = "mypy-1.11.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:af8d155170fcf87a2afb55b35dc1a0ac21df4431e7d96717621962e4b9192e70"}, + {file = "mypy-1.11.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f7821776e5c4286b6a13138cc935e2e9b6fde05e081bdebf5cdb2bb97c9df81d"}, + {file = "mypy-1.11.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:539c570477a96a4e6fb718b8d5c3e0c0eba1f485df13f86d2970c91f0673148d"}, + {file = "mypy-1.11.2-cp39-cp39-win_amd64.whl", hash = "sha256:3f14cd3d386ac4d05c5a39a51b84387403dadbd936e17cb35882134d4f8f0d24"}, + {file = "mypy-1.11.2-py3-none-any.whl", hash = "sha256:b499bc07dbdcd3de92b0a8b29fdf592c111276f6a12fe29c30f6c417dd546d12"}, + {file = "mypy-1.11.2.tar.gz", hash = "sha256:7f9993ad3e0ffdc95c2a14b66dee63729f021968bff8ad911867579c65d13a79"}, ] [package.dependencies] mypy-extensions = ">=1.0.0" tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""} -typing-extensions = ">=4.1.0" +typing-extensions = ">=4.6.0" [package.extras] dmypy = ["psutil (>=4.0)"] @@ -2385,40 +2422,37 @@ test = ["codecov (>=2.1)", "pytest (>=7.2)", "pytest-cov (>=4.0)"] [[package]] name = "nodeenv" -version = "1.8.0" +version = "1.9.1" description = "Node.js virtual environment builder" optional = false -python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*" +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" files = [ - {file = "nodeenv-1.8.0-py2.py3-none-any.whl", hash = "sha256:df865724bb3c3adc86b3876fa209771517b0cfe596beff01a92700e0e8be4cec"}, - {file = "nodeenv-1.8.0.tar.gz", hash = "sha256:d51e0c37e64fbf47d017feac3145cdbb58836d7eee8c6f6d3b6880c5456227d2"}, + {file = "nodeenv-1.9.1-py2.py3-none-any.whl", hash = "sha256:ba11c9782d29c27c70ffbdda2d7415098754709be8a7056d79a737cd901155c9"}, + {file = "nodeenv-1.9.1.tar.gz", hash = "sha256:6ec12890a2dab7946721edbfbcd91f3319c6ccc9aec47be7c7e6b7011ee6645f"}, ] -[package.dependencies] -setuptools = "*" - [[package]] name = "notebook" -version = "7.1.3" +version = "7.2.2" description = "Jupyter Notebook - A web-based notebook environment for interactive computing" optional = false python-versions = ">=3.8" files = [ - {file = "notebook-7.1.3-py3-none-any.whl", hash = "sha256:919b911e59f41f6e3857ce93c9d93535ba66bb090059712770e5968c07e1004d"}, - {file = "notebook-7.1.3.tar.gz", hash = "sha256:41fcebff44cf7bb9377180808bcbae066629b55d8c7722f1ebbe75ca44f9cfc1"}, + {file = "notebook-7.2.2-py3-none-any.whl", hash = "sha256:c89264081f671bc02eec0ed470a627ed791b9156cad9285226b31611d3e9fe1c"}, + {file = "notebook-7.2.2.tar.gz", hash = "sha256:2ef07d4220421623ad3fe88118d687bc0450055570cdd160814a59cf3a1c516e"}, ] [package.dependencies] jupyter-server = ">=2.4.0,<3" -jupyterlab = ">=4.1.1,<4.2" -jupyterlab-server = ">=2.22.1,<3" +jupyterlab = ">=4.2.0,<4.3" +jupyterlab-server = ">=2.27.1,<3" notebook-shim = ">=0.2,<0.3" tornado = ">=6.2.0" [package.extras] dev = ["hatch", "pre-commit"] docs = ["myst-parser", "nbsphinx", "pydata-sphinx-theme", "sphinx (>=1.3.6)", "sphinxcontrib-github-alt", "sphinxcontrib-spelling"] -test = ["importlib-resources (>=5.0)", "ipykernel", "jupyter-server[test] (>=2.4.0,<3)", "jupyterlab-server[test] (>=2.22.1,<3)", "nbval", "pytest (>=7.0)", "pytest-console-scripts", "pytest-timeout", "pytest-tornasync", "requests"] +test = ["importlib-resources (>=5.0)", "ipykernel", "jupyter-server[test] (>=2.4.0,<3)", "jupyterlab-server[test] (>=2.27.1,<3)", "nbval", "pytest (>=7.0)", "pytest-console-scripts", "pytest-timeout", "pytest-tornasync", "requests"] [[package]] name = "notebook-shim" @@ -2518,13 +2552,13 @@ signedtoken = ["cryptography (>=3.0.0)", "pyjwt (>=2.0.0,<3)"] [[package]] name = "openpyxl" -version = "3.1.2" +version = "3.1.5" description = "A Python library to read/write Excel 2010 xlsx/xlsm files" optional = false -python-versions = ">=3.6" +python-versions = ">=3.8" files = [ - {file = "openpyxl-3.1.2-py2.py3-none-any.whl", hash = "sha256:f91456ead12ab3c6c2e9491cf33ba6d08357d802192379bb482f1033ade496f5"}, - {file = "openpyxl-3.1.2.tar.gz", hash = "sha256:a6f5977418eff3b2d5500d54d9db50c8277a368436f4e4f8ddb1be3422870184"}, + {file = "openpyxl-3.1.5-py2.py3-none-any.whl", hash = "sha256:5282c12b107bffeef825f4617dc029afaf41d0ea60823bbb665ef3079dc79de2"}, + {file = "openpyxl-3.1.5.tar.gz", hash = "sha256:cf0e3cf56142039133628b5acffe8ef0c12bc902d2aadd3e0fe5878dc08d1050"}, ] [package.dependencies] @@ -2651,13 +2685,13 @@ files = [ [[package]] name = "packaging" -version = "24.0" +version = "24.1" description = "Core utilities for Python packages" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "packaging-24.0-py3-none-any.whl", hash = "sha256:2ddfb553fdf02fb784c234c7ba6ccc288296ceabec964ad2eae3777778130bc5"}, - {file = "packaging-24.0.tar.gz", hash = "sha256:eb82c5e3e56209074766e6885bb04b8c38a0c015d0a30036ebe7ece34c9989e9"}, + {file = "packaging-24.1-py3-none-any.whl", hash = "sha256:5b8f2217dbdbd2f7f384c41c628544e6d52f2d0f53c6d0c3ea61aa5d1d7ff124"}, + {file = "packaging-24.1.tar.gz", hash = "sha256:026ed72c8ed3fcce5bf8950572258698927fd1dbda10a5e981cdf0ac37f4f002"}, ] [[package]] @@ -2820,13 +2854,13 @@ ptyprocess = ">=0.5" [[package]] name = "platformdirs" -version = "4.2.1" +version = "4.2.2" description = "A small Python package for determining appropriate platform-specific dirs, e.g. a `user data dir`." optional = false python-versions = ">=3.8" files = [ - {file = "platformdirs-4.2.1-py3-none-any.whl", hash = "sha256:17d5a1161b3fd67b390023cb2d3b026bbd40abde6fdb052dfbd3a29c3ba22ee1"}, - {file = "platformdirs-4.2.1.tar.gz", hash = "sha256:031cd18d4ec63ec53e82dceaac0417d218a6863f7745dfcc9efe7793b7039bdf"}, + {file = "platformdirs-4.2.2-py3-none-any.whl", hash = "sha256:2d7a1657e36a80ea911db832a8a6ece5ee53d8de21edd5cc5879af6530b1bfee"}, + {file = "platformdirs-4.2.2.tar.gz", hash = "sha256:38b7b51f512eed9e84a22788b4bce1de17c0adb134d6becb09836e37d8654cd3"}, ] [package.extras] @@ -2851,13 +2885,13 @@ testing = ["pytest", "pytest-benchmark"] [[package]] name = "pre-commit" -version = "3.7.0" +version = "3.8.0" description = "A framework for managing and maintaining multi-language pre-commit hooks." optional = false python-versions = ">=3.9" files = [ - {file = "pre_commit-3.7.0-py2.py3-none-any.whl", hash = "sha256:5eae9e10c2b5ac51577c3452ec0a490455c45a0533f7960f993a0d01e59decab"}, - {file = "pre_commit-3.7.0.tar.gz", hash = "sha256:e209d61b8acdcf742404408531f0c37d49d2c734fd7cff2d6076083d191cb060"}, + {file = "pre_commit-3.8.0-py2.py3-none-any.whl", hash = "sha256:9a90a53bf82fdd8778d58085faf8d83df56e40dfe18f45b19446e26bf1b3a63f"}, + {file = "pre_commit-3.8.0.tar.gz", hash = "sha256:8bb6494d4a20423842e198980c9ecf9f96607a07ea29549e180eef9ae80fe7af"}, ] [package.dependencies] @@ -2883,13 +2917,13 @@ twisted = ["twisted"] [[package]] name = "prompt-toolkit" -version = "3.0.43" +version = "3.0.47" description = "Library for building powerful interactive command lines in Python" optional = false python-versions = ">=3.7.0" files = [ - {file = "prompt_toolkit-3.0.43-py3-none-any.whl", hash = "sha256:a11a29cb3bf0a28a387fe5122cdb649816a957cd9261dcedf8c9f1fef33eacf6"}, - {file = "prompt_toolkit-3.0.43.tar.gz", hash = "sha256:3527b7af26106cbc65a040bcc84839a3566ec1b051bb0bfe953631e704b0ff7d"}, + {file = "prompt_toolkit-3.0.47-py3-none-any.whl", hash = "sha256:0d7bfa67001d5e39d02c224b663abc33687405033a8c422d0d675a5a13361d10"}, + {file = "prompt_toolkit-3.0.47.tar.gz", hash = "sha256:1e1b29cb58080b1e69f207c893a1a7bf16d127a5c30c9d17a25a5d77792e5360"}, ] [package.dependencies] @@ -2897,39 +2931,39 @@ wcwidth = "*" [[package]] name = "proto-plus" -version = "1.23.0" +version = "1.24.0" description = "Beautiful, Pythonic protocol buffers." optional = false -python-versions = ">=3.6" +python-versions = ">=3.7" files = [ - {file = "proto-plus-1.23.0.tar.gz", hash = "sha256:89075171ef11988b3fa157f5dbd8b9cf09d65fffee97e29ce403cd8defba19d2"}, - {file = "proto_plus-1.23.0-py3-none-any.whl", hash = "sha256:a829c79e619e1cf632de091013a4173deed13a55f326ef84f05af6f50ff4c82c"}, + {file = "proto-plus-1.24.0.tar.gz", hash = "sha256:30b72a5ecafe4406b0d339db35b56c4059064e69227b8c3bda7462397f966445"}, + {file = "proto_plus-1.24.0-py3-none-any.whl", hash = "sha256:402576830425e5f6ce4c2a6702400ac79897dab0b4343821aa5188b0fab81a12"}, ] [package.dependencies] -protobuf = ">=3.19.0,<5.0.0dev" +protobuf = ">=3.19.0,<6.0.0dev" [package.extras] -testing = ["google-api-core[grpc] (>=1.31.5)"] +testing = ["google-api-core (>=1.31.5)"] [[package]] name = "protobuf" -version = "4.25.3" +version = "4.25.4" description = "" optional = false python-versions = ">=3.8" files = [ - {file = "protobuf-4.25.3-cp310-abi3-win32.whl", hash = "sha256:d4198877797a83cbfe9bffa3803602bbe1625dc30d8a097365dbc762e5790faa"}, - {file = "protobuf-4.25.3-cp310-abi3-win_amd64.whl", hash = "sha256:209ba4cc916bab46f64e56b85b090607a676f66b473e6b762e6f1d9d591eb2e8"}, - {file = "protobuf-4.25.3-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:f1279ab38ecbfae7e456a108c5c0681e4956d5b1090027c1de0f934dfdb4b35c"}, - {file = "protobuf-4.25.3-cp37-abi3-manylinux2014_aarch64.whl", hash = "sha256:e7cb0ae90dd83727f0c0718634ed56837bfeeee29a5f82a7514c03ee1364c019"}, - {file = "protobuf-4.25.3-cp37-abi3-manylinux2014_x86_64.whl", hash = "sha256:7c8daa26095f82482307bc717364e7c13f4f1c99659be82890dcfc215194554d"}, - {file = "protobuf-4.25.3-cp38-cp38-win32.whl", hash = "sha256:f4f118245c4a087776e0a8408be33cf09f6c547442c00395fbfb116fac2f8ac2"}, - {file = "protobuf-4.25.3-cp38-cp38-win_amd64.whl", hash = "sha256:c053062984e61144385022e53678fbded7aea14ebb3e0305ae3592fb219ccfa4"}, - {file = "protobuf-4.25.3-cp39-cp39-win32.whl", hash = "sha256:19b270aeaa0099f16d3ca02628546b8baefe2955bbe23224aaf856134eccf1e4"}, - {file = "protobuf-4.25.3-cp39-cp39-win_amd64.whl", hash = "sha256:e3c97a1555fd6388f857770ff8b9703083de6bf1f9274a002a332d65fbb56c8c"}, - {file = "protobuf-4.25.3-py3-none-any.whl", hash = "sha256:f0700d54bcf45424477e46a9f0944155b46fb0639d69728739c0e47bab83f2b9"}, - {file = "protobuf-4.25.3.tar.gz", hash = "sha256:25b5d0b42fd000320bd7830b349e3b696435f3b329810427a6bcce6a5492cc5c"}, + {file = "protobuf-4.25.4-cp310-abi3-win32.whl", hash = "sha256:db9fd45183e1a67722cafa5c1da3e85c6492a5383f127c86c4c4aa4845867dc4"}, + {file = "protobuf-4.25.4-cp310-abi3-win_amd64.whl", hash = "sha256:ba3d8504116a921af46499471c63a85260c1a5fc23333154a427a310e015d26d"}, + {file = "protobuf-4.25.4-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:eecd41bfc0e4b1bd3fa7909ed93dd14dd5567b98c941d6c1ad08fdcab3d6884b"}, + {file = "protobuf-4.25.4-cp37-abi3-manylinux2014_aarch64.whl", hash = "sha256:4c8a70fdcb995dcf6c8966cfa3a29101916f7225e9afe3ced4395359955d3835"}, + {file = "protobuf-4.25.4-cp37-abi3-manylinux2014_x86_64.whl", hash = "sha256:3319e073562e2515c6ddc643eb92ce20809f5d8f10fead3332f71c63be6a7040"}, + {file = "protobuf-4.25.4-cp38-cp38-win32.whl", hash = "sha256:7e372cbbda66a63ebca18f8ffaa6948455dfecc4e9c1029312f6c2edcd86c4e1"}, + {file = "protobuf-4.25.4-cp38-cp38-win_amd64.whl", hash = "sha256:051e97ce9fa6067a4546e75cb14f90cf0232dcb3e3d508c448b8d0e4265b61c1"}, + {file = "protobuf-4.25.4-cp39-cp39-win32.whl", hash = "sha256:90bf6fd378494eb698805bbbe7afe6c5d12c8e17fca817a646cd6a1818c696ca"}, + {file = "protobuf-4.25.4-cp39-cp39-win_amd64.whl", hash = "sha256:ac79a48d6b99dfed2729ccccee547b34a1d3d63289c71cef056653a846a2240f"}, + {file = "protobuf-4.25.4-py3-none-any.whl", hash = "sha256:bfbebc1c8e4793cfd58589acfb8a1026be0003e852b9da7db5a4285bde996978"}, + {file = "protobuf-4.25.4.tar.gz", hash = "sha256:0dc4a62cc4052a036ee2204d26fe4d835c62827c855c8a03f29fe6da146b380d"}, ] [[package]] @@ -2973,13 +3007,13 @@ files = [ [[package]] name = "pure-eval" -version = "0.2.2" +version = "0.2.3" description = "Safely evaluate AST nodes without side effects" optional = false python-versions = "*" files = [ - {file = "pure_eval-0.2.2-py3-none-any.whl", hash = "sha256:01eaab343580944bc56080ebe0a674b39ec44a945e6d09ba7db3cb8cec289350"}, - {file = "pure_eval-0.2.2.tar.gz", hash = "sha256:2b45320af6dfaa1750f543d714b6d1c520a1688dec6fd24d339063ce0aaa9ac3"}, + {file = "pure_eval-0.2.3-py3-none-any.whl", hash = "sha256:1db8e35b67b3d218d818ae653e27f06c3aa420901fa7b081ca98cbedc874e0d0"}, + {file = "pure_eval-0.2.3.tar.gz", hash = "sha256:5f4e983f40564c576c7c8635ae88db5956bb2229d7e9237d03b3c0b0190eaf42"}, ] [package.extras] @@ -3045,47 +3079,54 @@ files = [ [[package]] name = "pydantic" -version = "1.10.15" +version = "1.10.18" description = "Data validation and settings management using python type hints" optional = false python-versions = ">=3.7" files = [ - {file = "pydantic-1.10.15-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:22ed12ee588b1df028a2aa5d66f07bf8f8b4c8579c2e96d5a9c1f96b77f3bb55"}, - {file = "pydantic-1.10.15-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:75279d3cac98186b6ebc2597b06bcbc7244744f6b0b44a23e4ef01e5683cc0d2"}, - {file = "pydantic-1.10.15-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:50f1666a9940d3d68683c9d96e39640f709d7a72ff8702987dab1761036206bb"}, - {file = "pydantic-1.10.15-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:82790d4753ee5d00739d6cb5cf56bceb186d9d6ce134aca3ba7befb1eedbc2c8"}, - {file = "pydantic-1.10.15-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:d207d5b87f6cbefbdb1198154292faee8017d7495a54ae58db06762004500d00"}, - {file = "pydantic-1.10.15-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:e49db944fad339b2ccb80128ffd3f8af076f9f287197a480bf1e4ca053a866f0"}, - {file = "pydantic-1.10.15-cp310-cp310-win_amd64.whl", hash = "sha256:d3b5c4cbd0c9cb61bbbb19ce335e1f8ab87a811f6d589ed52b0254cf585d709c"}, - {file = "pydantic-1.10.15-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c3d5731a120752248844676bf92f25a12f6e45425e63ce22e0849297a093b5b0"}, - {file = "pydantic-1.10.15-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:c365ad9c394f9eeffcb30a82f4246c0006417f03a7c0f8315d6211f25f7cb654"}, - {file = "pydantic-1.10.15-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3287e1614393119c67bd4404f46e33ae3be3ed4cd10360b48d0a4459f420c6a3"}, - {file = "pydantic-1.10.15-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:be51dd2c8596b25fe43c0a4a59c2bee4f18d88efb8031188f9e7ddc6b469cf44"}, - {file = "pydantic-1.10.15-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:6a51a1dd4aa7b3f1317f65493a182d3cff708385327c1c82c81e4a9d6d65b2e4"}, - {file = "pydantic-1.10.15-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:4e316e54b5775d1eb59187f9290aeb38acf620e10f7fd2f776d97bb788199e53"}, - {file = "pydantic-1.10.15-cp311-cp311-win_amd64.whl", hash = "sha256:0d142fa1b8f2f0ae11ddd5e3e317dcac060b951d605fda26ca9b234b92214986"}, - {file = "pydantic-1.10.15-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:7ea210336b891f5ea334f8fc9f8f862b87acd5d4a0cbc9e3e208e7aa1775dabf"}, - {file = "pydantic-1.10.15-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3453685ccd7140715e05f2193d64030101eaad26076fad4e246c1cc97e1bb30d"}, - {file = "pydantic-1.10.15-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9bea1f03b8d4e8e86702c918ccfd5d947ac268f0f0cc6ed71782e4b09353b26f"}, - {file = "pydantic-1.10.15-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:005655cabc29081de8243126e036f2065bd7ea5b9dff95fde6d2c642d39755de"}, - {file = "pydantic-1.10.15-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:af9850d98fc21e5bc24ea9e35dd80a29faf6462c608728a110c0a30b595e58b7"}, - {file = "pydantic-1.10.15-cp37-cp37m-win_amd64.whl", hash = "sha256:d31ee5b14a82c9afe2bd26aaa405293d4237d0591527d9129ce36e58f19f95c1"}, - {file = "pydantic-1.10.15-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:5e09c19df304b8123938dc3c53d3d3be6ec74b9d7d0d80f4f4b5432ae16c2022"}, - {file = "pydantic-1.10.15-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:7ac9237cd62947db00a0d16acf2f3e00d1ae9d3bd602b9c415f93e7a9fc10528"}, - {file = "pydantic-1.10.15-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:584f2d4c98ffec420e02305cf675857bae03c9d617fcfdc34946b1160213a948"}, - {file = "pydantic-1.10.15-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:bbc6989fad0c030bd70a0b6f626f98a862224bc2b1e36bfc531ea2facc0a340c"}, - {file = "pydantic-1.10.15-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:d573082c6ef99336f2cb5b667b781d2f776d4af311574fb53d908517ba523c22"}, - {file = "pydantic-1.10.15-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:6bd7030c9abc80134087d8b6e7aa957e43d35714daa116aced57269a445b8f7b"}, - {file = "pydantic-1.10.15-cp38-cp38-win_amd64.whl", hash = "sha256:3350f527bb04138f8aff932dc828f154847fbdc7a1a44c240fbfff1b57f49a12"}, - {file = "pydantic-1.10.15-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:51d405b42f1b86703555797270e4970a9f9bd7953f3990142e69d1037f9d9e51"}, - {file = "pydantic-1.10.15-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:a980a77c52723b0dc56640ced396b73a024d4b74f02bcb2d21dbbac1debbe9d0"}, - {file = "pydantic-1.10.15-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:67f1a1fb467d3f49e1708a3f632b11c69fccb4e748a325d5a491ddc7b5d22383"}, - {file = "pydantic-1.10.15-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:676ed48f2c5bbad835f1a8ed8a6d44c1cd5a21121116d2ac40bd1cd3619746ed"}, - {file = "pydantic-1.10.15-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:92229f73400b80c13afcd050687f4d7e88de9234d74b27e6728aa689abcf58cc"}, - {file = "pydantic-1.10.15-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:2746189100c646682eff0bce95efa7d2e203420d8e1c613dc0c6b4c1d9c1fde4"}, - {file = "pydantic-1.10.15-cp39-cp39-win_amd64.whl", hash = "sha256:394f08750bd8eaad714718812e7fab615f873b3cdd0b9d84e76e51ef3b50b6b7"}, - {file = "pydantic-1.10.15-py3-none-any.whl", hash = "sha256:28e552a060ba2740d0d2aabe35162652c1459a0b9069fe0db7f4ee0e18e74d58"}, - {file = "pydantic-1.10.15.tar.gz", hash = "sha256:ca832e124eda231a60a041da4f013e3ff24949d94a01154b137fc2f2a43c3ffb"}, + {file = "pydantic-1.10.18-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:e405ffcc1254d76bb0e760db101ee8916b620893e6edfbfee563b3c6f7a67c02"}, + {file = "pydantic-1.10.18-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e306e280ebebc65040034bff1a0a81fd86b2f4f05daac0131f29541cafd80b80"}, + {file = "pydantic-1.10.18-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:11d9d9b87b50338b1b7de4ebf34fd29fdb0d219dc07ade29effc74d3d2609c62"}, + {file = "pydantic-1.10.18-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b661ce52c7b5e5f600c0c3c5839e71918346af2ef20062705ae76b5c16914cab"}, + {file = "pydantic-1.10.18-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:c20f682defc9ef81cd7eaa485879ab29a86a0ba58acf669a78ed868e72bb89e0"}, + {file = "pydantic-1.10.18-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:c5ae6b7c8483b1e0bf59e5f1843e4fd8fd405e11df7de217ee65b98eb5462861"}, + {file = "pydantic-1.10.18-cp310-cp310-win_amd64.whl", hash = "sha256:74fe19dda960b193b0eb82c1f4d2c8e5e26918d9cda858cbf3f41dd28549cb70"}, + {file = "pydantic-1.10.18-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:72fa46abace0a7743cc697dbb830a41ee84c9db8456e8d77a46d79b537efd7ec"}, + {file = "pydantic-1.10.18-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:ef0fe7ad7cbdb5f372463d42e6ed4ca9c443a52ce544472d8842a0576d830da5"}, + {file = "pydantic-1.10.18-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a00e63104346145389b8e8f500bc6a241e729feaf0559b88b8aa513dd2065481"}, + {file = "pydantic-1.10.18-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ae6fa2008e1443c46b7b3a5eb03800121868d5ab6bc7cda20b5df3e133cde8b3"}, + {file = "pydantic-1.10.18-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:9f463abafdc92635da4b38807f5b9972276be7c8c5121989768549fceb8d2588"}, + {file = "pydantic-1.10.18-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:3445426da503c7e40baccefb2b2989a0c5ce6b163679dd75f55493b460f05a8f"}, + {file = "pydantic-1.10.18-cp311-cp311-win_amd64.whl", hash = "sha256:467a14ee2183bc9c902579bb2f04c3d3dac00eff52e252850509a562255b2a33"}, + {file = "pydantic-1.10.18-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:efbc8a7f9cb5fe26122acba1852d8dcd1e125e723727c59dcd244da7bdaa54f2"}, + {file = "pydantic-1.10.18-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:24a4a159d0f7a8e26bf6463b0d3d60871d6a52eac5bb6a07a7df85c806f4c048"}, + {file = "pydantic-1.10.18-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b74be007703547dc52e3c37344d130a7bfacca7df112a9e5ceeb840a9ce195c7"}, + {file = "pydantic-1.10.18-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:fcb20d4cb355195c75000a49bb4a31d75e4295200df620f454bbc6bdf60ca890"}, + {file = "pydantic-1.10.18-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:46f379b8cb8a3585e3f61bf9ae7d606c70d133943f339d38b76e041ec234953f"}, + {file = "pydantic-1.10.18-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:cbfbca662ed3729204090c4d09ee4beeecc1a7ecba5a159a94b5a4eb24e3759a"}, + {file = "pydantic-1.10.18-cp312-cp312-win_amd64.whl", hash = "sha256:c6d0a9f9eccaf7f438671a64acf654ef0d045466e63f9f68a579e2383b63f357"}, + {file = "pydantic-1.10.18-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:3d5492dbf953d7d849751917e3b2433fb26010d977aa7a0765c37425a4026ff1"}, + {file = "pydantic-1.10.18-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fe734914977eed33033b70bfc097e1baaffb589517863955430bf2e0846ac30f"}, + {file = "pydantic-1.10.18-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:15fdbe568beaca9aacfccd5ceadfb5f1a235087a127e8af5e48df9d8a45ae85c"}, + {file = "pydantic-1.10.18-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:c3e742f62198c9eb9201781fbebe64533a3bbf6a76a91b8d438d62b813079dbc"}, + {file = "pydantic-1.10.18-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:19a3bd00b9dafc2cd7250d94d5b578edf7a0bd7daf102617153ff9a8fa37871c"}, + {file = "pydantic-1.10.18-cp37-cp37m-win_amd64.whl", hash = "sha256:2ce3fcf75b2bae99aa31bd4968de0474ebe8c8258a0110903478bd83dfee4e3b"}, + {file = "pydantic-1.10.18-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:335a32d72c51a313b33fa3a9b0fe283503272ef6467910338e123f90925f0f03"}, + {file = "pydantic-1.10.18-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:34a3613c7edb8c6fa578e58e9abe3c0f5e7430e0fc34a65a415a1683b9c32d9a"}, + {file = "pydantic-1.10.18-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e9ee4e6ca1d9616797fa2e9c0bfb8815912c7d67aca96f77428e316741082a1b"}, + {file = "pydantic-1.10.18-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:23e8ec1ce4e57b4f441fc91e3c12adba023fedd06868445a5b5f1d48f0ab3682"}, + {file = "pydantic-1.10.18-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:44ae8a3e35a54d2e8fa88ed65e1b08967a9ef8c320819a969bfa09ce5528fafe"}, + {file = "pydantic-1.10.18-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:d5389eb3b48a72da28c6e061a247ab224381435256eb541e175798483368fdd3"}, + {file = "pydantic-1.10.18-cp38-cp38-win_amd64.whl", hash = "sha256:069b9c9fc645474d5ea3653788b544a9e0ccd3dca3ad8c900c4c6eac844b4620"}, + {file = "pydantic-1.10.18-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:80b982d42515632eb51f60fa1d217dfe0729f008e81a82d1544cc392e0a50ddf"}, + {file = "pydantic-1.10.18-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:aad8771ec8dbf9139b01b56f66386537c6fe4e76c8f7a47c10261b69ad25c2c9"}, + {file = "pydantic-1.10.18-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:941a2eb0a1509bd7f31e355912eb33b698eb0051730b2eaf9e70e2e1589cae1d"}, + {file = "pydantic-1.10.18-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:65f7361a09b07915a98efd17fdec23103307a54db2000bb92095457ca758d485"}, + {file = "pydantic-1.10.18-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:6951f3f47cb5ca4da536ab161ac0163cab31417d20c54c6de5ddcab8bc813c3f"}, + {file = "pydantic-1.10.18-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:7a4c5eec138a9b52c67f664c7d51d4c7234c5ad65dd8aacd919fb47445a62c86"}, + {file = "pydantic-1.10.18-cp39-cp39-win_amd64.whl", hash = "sha256:49e26c51ca854286bffc22b69787a8d4063a62bf7d83dc21d44d2ff426108518"}, + {file = "pydantic-1.10.18-py3-none-any.whl", hash = "sha256:06a189b81ffc52746ec9c8c007f16e5167c8b0a696e1a726369327e3db7b2a82"}, + {file = "pydantic-1.10.18.tar.gz", hash = "sha256:baebdff1907d1d96a139c25136a9bb7d17e118f133a76a2ef3b845e831e3403a"}, ] [package.dependencies] @@ -3138,6 +3179,23 @@ google-auth-oauthlib = ">=0.7.1" [package.extras] pandas = ["pandas (>=0.14.0)"] +[[package]] +name = "pyjwt" +version = "2.9.0" +description = "JSON Web Token implementation in Python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "PyJWT-2.9.0-py3-none-any.whl", hash = "sha256:3b02fb0f44517787776cf48f2ae25d8e14f300e6d7545a4315cee571a415e850"}, + {file = "pyjwt-2.9.0.tar.gz", hash = "sha256:7e1e5b56cc735432a7369cbfa0efe50fa113ebecdc04ae6922deba8b84582d0c"}, +] + +[package.extras] +crypto = ["cryptography (>=3.4.0)"] +dev = ["coverage[toml] (==5.0.4)", "cryptography (>=3.4.0)", "pre-commit", "pytest (>=6.0.0,<7.0.0)", "sphinx", "sphinx-rtd-theme", "zope.interface"] +docs = ["sphinx", "sphinx-rtd-theme", "zope.interface"] +tests = ["coverage[toml] (==5.0.4)", "pytest (>=6.0.0,<7.0.0)"] + [[package]] name = "pylint" version = "2.17.7" @@ -3184,13 +3242,13 @@ test = ["flaky", "pretend", "pytest (>=3.0.1)"] [[package]] name = "pyparsing" -version = "3.1.2" +version = "3.1.4" description = "pyparsing module - Classes and methods to define and execute parsing grammars" optional = false python-versions = ">=3.6.8" files = [ - {file = "pyparsing-3.1.2-py3-none-any.whl", hash = "sha256:f9db75911801ed778fe61bb643079ff86601aca99fcae6345aa67292038fb742"}, - {file = "pyparsing-3.1.2.tar.gz", hash = "sha256:a1bac0ce561155ecc3ed78ca94d3c9378656ad4c94c1270de543f621420f94ad"}, + {file = "pyparsing-3.1.4-py3-none-any.whl", hash = "sha256:a6a7ee4235a3f944aa1fa2249307708f893fe5717dc603503c6c7969c070fb7c"}, + {file = "pyparsing-3.1.4.tar.gz", hash = "sha256:f86ec8d1a83f11977c9a6ea7598e8c27fc5cddfa5b07ea2241edbbde1d7bc032"}, ] [package.extras] @@ -3220,13 +3278,13 @@ testing = ["argcomplete", "attrs (>=19.2.0)", "hypothesis (>=3.56)", "mock", "no [[package]] name = "pytest-asyncio" -version = "0.23.7" +version = "0.23.8" description = "Pytest support for asyncio" optional = false python-versions = ">=3.8" files = [ - {file = "pytest_asyncio-0.23.7-py3-none-any.whl", hash = "sha256:009b48127fbe44518a547bddd25611551b0e43ccdbf1e67d12479f569832c20b"}, - {file = "pytest_asyncio-0.23.7.tar.gz", hash = "sha256:5f5c72948f4c49e7db4f29f2521d4031f1c27f86e57b046126654083d4770268"}, + {file = "pytest_asyncio-0.23.8-py3-none-any.whl", hash = "sha256:50265d892689a5faefb84df80819d1ecef566eb3549cf915dfb33569359d1ce2"}, + {file = "pytest_asyncio-0.23.8.tar.gz", hash = "sha256:759b10b33a6dc61cce40a8bd5205e302978bbbcc00e279a8b61d9a6a3c82e4d3"}, ] [package.dependencies] @@ -3396,159 +3454,182 @@ files = [ [[package]] name = "pyyaml" -version = "6.0.1" +version = "6.0.2" description = "YAML parser and emitter for Python" optional = false -python-versions = ">=3.6" +python-versions = ">=3.8" files = [ - {file = "PyYAML-6.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d858aa552c999bc8a8d57426ed01e40bef403cd8ccdd0fc5f6f04a00414cac2a"}, - {file = "PyYAML-6.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fd66fc5d0da6d9815ba2cebeb4205f95818ff4b79c3ebe268e75d961704af52f"}, - {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, - {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, - {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, - {file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"}, - {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, - {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, - {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, - {file = "PyYAML-6.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f003ed9ad21d6a4713f0a9b5a7a0a79e08dd0f221aff4525a2be4c346ee60aab"}, - {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, - {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, - {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, - {file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"}, - {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, - {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, - {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, - {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, - {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"}, - {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, - {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, - {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, - {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"}, - {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, - {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, - {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, - {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:afd7e57eddb1a54f0f1a974bc4391af8bcce0b444685d936840f125cf046d5bd"}, - {file = "PyYAML-6.0.1-cp36-cp36m-win32.whl", hash = "sha256:fca0e3a251908a499833aa292323f32437106001d436eca0e6e7833256674585"}, - {file = "PyYAML-6.0.1-cp36-cp36m-win_amd64.whl", hash = "sha256:f22ac1c3cac4dbc50079e965eba2c1058622631e526bd9afd45fedd49ba781fa"}, - {file = "PyYAML-6.0.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:b1275ad35a5d18c62a7220633c913e1b42d44b46ee12554e5fd39c70a243d6a3"}, - {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:18aeb1bf9a78867dc38b259769503436b7c72f7a1f1f4c93ff9a17de54319b27"}, - {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:596106435fa6ad000c2991a98fa58eeb8656ef2325d7e158344fb33864ed87e3"}, - {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:baa90d3f661d43131ca170712d903e6295d1f7a0f595074f151c0aed377c9b9c"}, - {file = "PyYAML-6.0.1-cp37-cp37m-win32.whl", hash = "sha256:9046c58c4395dff28dd494285c82ba00b546adfc7ef001486fbf0324bc174fba"}, - {file = "PyYAML-6.0.1-cp37-cp37m-win_amd64.whl", hash = "sha256:4fb147e7a67ef577a588a0e2c17b6db51dda102c71de36f8549b6816a96e1867"}, - {file = "PyYAML-6.0.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1d4c7e777c441b20e32f52bd377e0c409713e8bb1386e1099c2415f26e479595"}, - {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, - {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, - {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, - {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"}, - {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, - {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, - {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, - {file = "PyYAML-6.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c8098ddcc2a85b61647b2590f825f3db38891662cfc2fc776415143f599bb859"}, - {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, - {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, - {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, - {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"}, - {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, - {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, - {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, + {file = "PyYAML-6.0.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0a9a2848a5b7feac301353437eb7d5957887edbf81d56e903999a75a3d743086"}, + {file = "PyYAML-6.0.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:29717114e51c84ddfba879543fb232a6ed60086602313ca38cce623c1d62cfbf"}, + {file = "PyYAML-6.0.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8824b5a04a04a047e72eea5cec3bc266db09e35de6bdfe34c9436ac5ee27d237"}, + {file = "PyYAML-6.0.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7c36280e6fb8385e520936c3cb3b8042851904eba0e58d277dca80a5cfed590b"}, + {file = "PyYAML-6.0.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ec031d5d2feb36d1d1a24380e4db6d43695f3748343d99434e6f5f9156aaa2ed"}, + {file = "PyYAML-6.0.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:936d68689298c36b53b29f23c6dbb74de12b4ac12ca6cfe0e047bedceea56180"}, + {file = "PyYAML-6.0.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:23502f431948090f597378482b4812b0caae32c22213aecf3b55325e049a6c68"}, + {file = "PyYAML-6.0.2-cp310-cp310-win32.whl", hash = "sha256:2e99c6826ffa974fe6e27cdb5ed0021786b03fc98e5ee3c5bfe1fd5015f42b99"}, + {file = "PyYAML-6.0.2-cp310-cp310-win_amd64.whl", hash = "sha256:a4d3091415f010369ae4ed1fc6b79def9416358877534caf6a0fdd2146c87a3e"}, + {file = "PyYAML-6.0.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:cc1c1159b3d456576af7a3e4d1ba7e6924cb39de8f67111c735f6fc832082774"}, + {file = "PyYAML-6.0.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1e2120ef853f59c7419231f3bf4e7021f1b936f6ebd222406c3b60212205d2ee"}, + {file = "PyYAML-6.0.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5d225db5a45f21e78dd9358e58a98702a0302f2659a3c6cd320564b75b86f47c"}, + {file = "PyYAML-6.0.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5ac9328ec4831237bec75defaf839f7d4564be1e6b25ac710bd1a96321cc8317"}, + {file = "PyYAML-6.0.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ad2a3decf9aaba3d29c8f537ac4b243e36bef957511b4766cb0057d32b0be85"}, + {file = "PyYAML-6.0.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:ff3824dc5261f50c9b0dfb3be22b4567a6f938ccce4587b38952d85fd9e9afe4"}, + {file = "PyYAML-6.0.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:797b4f722ffa07cc8d62053e4cff1486fa6dc094105d13fea7b1de7d8bf71c9e"}, + {file = "PyYAML-6.0.2-cp311-cp311-win32.whl", hash = "sha256:11d8f3dd2b9c1207dcaf2ee0bbbfd5991f571186ec9cc78427ba5bd32afae4b5"}, + {file = "PyYAML-6.0.2-cp311-cp311-win_amd64.whl", hash = "sha256:e10ce637b18caea04431ce14fabcf5c64a1c61ec9c56b071a4b7ca131ca52d44"}, + {file = "PyYAML-6.0.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:c70c95198c015b85feafc136515252a261a84561b7b1d51e3384e0655ddf25ab"}, + {file = "PyYAML-6.0.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ce826d6ef20b1bc864f0a68340c8b3287705cae2f8b4b1d932177dcc76721725"}, + {file = "PyYAML-6.0.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1f71ea527786de97d1a0cc0eacd1defc0985dcf6b3f17bb77dcfc8c34bec4dc5"}, + {file = "PyYAML-6.0.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9b22676e8097e9e22e36d6b7bda33190d0d400f345f23d4065d48f4ca7ae0425"}, + {file = "PyYAML-6.0.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:80bab7bfc629882493af4aa31a4cfa43a4c57c83813253626916b8c7ada83476"}, + {file = "PyYAML-6.0.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:0833f8694549e586547b576dcfaba4a6b55b9e96098b36cdc7ebefe667dfed48"}, + {file = "PyYAML-6.0.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8b9c7197f7cb2738065c481a0461e50ad02f18c78cd75775628afb4d7137fb3b"}, + {file = "PyYAML-6.0.2-cp312-cp312-win32.whl", hash = "sha256:ef6107725bd54b262d6dedcc2af448a266975032bc85ef0172c5f059da6325b4"}, + {file = "PyYAML-6.0.2-cp312-cp312-win_amd64.whl", hash = "sha256:7e7401d0de89a9a855c839bc697c079a4af81cf878373abd7dc625847d25cbd8"}, + {file = "PyYAML-6.0.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:efdca5630322a10774e8e98e1af481aad470dd62c3170801852d752aa7a783ba"}, + {file = "PyYAML-6.0.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:50187695423ffe49e2deacb8cd10510bc361faac997de9efef88badc3bb9e2d1"}, + {file = "PyYAML-6.0.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0ffe8360bab4910ef1b9e87fb812d8bc0a308b0d0eef8c8f44e0254ab3b07133"}, + {file = "PyYAML-6.0.2-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:17e311b6c678207928d649faa7cb0d7b4c26a0ba73d41e99c4fff6b6c3276484"}, + {file = "PyYAML-6.0.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:70b189594dbe54f75ab3a1acec5f1e3faa7e8cf2f1e08d9b561cb41b845f69d5"}, + {file = "PyYAML-6.0.2-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:41e4e3953a79407c794916fa277a82531dd93aad34e29c2a514c2c0c5fe971cc"}, + {file = "PyYAML-6.0.2-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:68ccc6023a3400877818152ad9a1033e3db8625d899c72eacb5a668902e4d652"}, + {file = "PyYAML-6.0.2-cp313-cp313-win32.whl", hash = "sha256:bc2fa7c6b47d6bc618dd7fb02ef6fdedb1090ec036abab80d4681424b84c1183"}, + {file = "PyYAML-6.0.2-cp313-cp313-win_amd64.whl", hash = "sha256:8388ee1976c416731879ac16da0aff3f63b286ffdd57cdeb95f3f2e085687563"}, + {file = "PyYAML-6.0.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:24471b829b3bf607e04e88d79542a9d48bb037c2267d7927a874e6c205ca7e9a"}, + {file = "PyYAML-6.0.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d7fded462629cfa4b685c5416b949ebad6cec74af5e2d42905d41e257e0869f5"}, + {file = "PyYAML-6.0.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d84a1718ee396f54f3a086ea0a66d8e552b2ab2017ef8b420e92edbc841c352d"}, + {file = "PyYAML-6.0.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9056c1ecd25795207ad294bcf39f2db3d845767be0ea6e6a34d856f006006083"}, + {file = "PyYAML-6.0.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:82d09873e40955485746739bcb8b4586983670466c23382c19cffecbf1fd8706"}, + {file = "PyYAML-6.0.2-cp38-cp38-win32.whl", hash = "sha256:43fa96a3ca0d6b1812e01ced1044a003533c47f6ee8aca31724f78e93ccc089a"}, + {file = "PyYAML-6.0.2-cp38-cp38-win_amd64.whl", hash = "sha256:01179a4a8559ab5de078078f37e5c1a30d76bb88519906844fd7bdea1b7729ff"}, + {file = "PyYAML-6.0.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:688ba32a1cffef67fd2e9398a2efebaea461578b0923624778664cc1c914db5d"}, + {file = "PyYAML-6.0.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:a8786accb172bd8afb8be14490a16625cbc387036876ab6ba70912730faf8e1f"}, + {file = "PyYAML-6.0.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d8e03406cac8513435335dbab54c0d385e4a49e4945d2909a581c83647ca0290"}, + {file = "PyYAML-6.0.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f753120cb8181e736c57ef7636e83f31b9c0d1722c516f7e86cf15b7aa57ff12"}, + {file = "PyYAML-6.0.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3b1fdb9dc17f5a7677423d508ab4f243a726dea51fa5e70992e59a7411c89d19"}, + {file = "PyYAML-6.0.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:0b69e4ce7a131fe56b7e4d770c67429700908fc0752af059838b1cfb41960e4e"}, + {file = "PyYAML-6.0.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:a9f8c2e67970f13b16084e04f134610fd1d374bf477b17ec1599185cf611d725"}, + {file = "PyYAML-6.0.2-cp39-cp39-win32.whl", hash = "sha256:6395c297d42274772abc367baaa79683958044e5d3835486c16da75d2a694631"}, + {file = "PyYAML-6.0.2-cp39-cp39-win_amd64.whl", hash = "sha256:39693e1f8320ae4f43943590b49779ffb98acb81f788220ea932a6b6c51004d8"}, + {file = "pyyaml-6.0.2.tar.gz", hash = "sha256:d584d9ec91ad65861cc08d42e834324ef890a082e591037abe114850ff7bbc3e"}, ] [[package]] name = "pyzmq" -version = "26.0.3" +version = "26.2.0" description = "Python bindings for 0MQ" optional = false python-versions = ">=3.7" files = [ - {file = "pyzmq-26.0.3-cp310-cp310-macosx_10_15_universal2.whl", hash = "sha256:44dd6fc3034f1eaa72ece33588867df9e006a7303725a12d64c3dff92330f625"}, - {file = "pyzmq-26.0.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:acb704195a71ac5ea5ecf2811c9ee19ecdc62b91878528302dd0be1b9451cc90"}, - {file = "pyzmq-26.0.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5dbb9c997932473a27afa93954bb77a9f9b786b4ccf718d903f35da3232317de"}, - {file = "pyzmq-26.0.3-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6bcb34f869d431799c3ee7d516554797f7760cb2198ecaa89c3f176f72d062be"}, - {file = "pyzmq-26.0.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:38ece17ec5f20d7d9b442e5174ae9f020365d01ba7c112205a4d59cf19dc38ee"}, - {file = "pyzmq-26.0.3-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:ba6e5e6588e49139a0979d03a7deb9c734bde647b9a8808f26acf9c547cab1bf"}, - {file = "pyzmq-26.0.3-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:3bf8b000a4e2967e6dfdd8656cd0757d18c7e5ce3d16339e550bd462f4857e59"}, - {file = "pyzmq-26.0.3-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:2136f64fbb86451dbbf70223635a468272dd20075f988a102bf8a3f194a411dc"}, - {file = "pyzmq-26.0.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:e8918973fbd34e7814f59143c5f600ecd38b8038161239fd1a3d33d5817a38b8"}, - {file = "pyzmq-26.0.3-cp310-cp310-win32.whl", hash = "sha256:0aaf982e68a7ac284377d051c742610220fd06d330dcd4c4dbb4cdd77c22a537"}, - {file = "pyzmq-26.0.3-cp310-cp310-win_amd64.whl", hash = "sha256:f1a9b7d00fdf60b4039f4455afd031fe85ee8305b019334b72dcf73c567edc47"}, - {file = "pyzmq-26.0.3-cp310-cp310-win_arm64.whl", hash = "sha256:80b12f25d805a919d53efc0a5ad7c0c0326f13b4eae981a5d7b7cc343318ebb7"}, - {file = "pyzmq-26.0.3-cp311-cp311-macosx_10_15_universal2.whl", hash = "sha256:a72a84570f84c374b4c287183debc776dc319d3e8ce6b6a0041ce2e400de3f32"}, - {file = "pyzmq-26.0.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:7ca684ee649b55fd8f378127ac8462fb6c85f251c2fb027eb3c887e8ee347bcd"}, - {file = "pyzmq-26.0.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e222562dc0f38571c8b1ffdae9d7adb866363134299264a1958d077800b193b7"}, - {file = "pyzmq-26.0.3-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f17cde1db0754c35a91ac00b22b25c11da6eec5746431d6e5092f0cd31a3fea9"}, - {file = "pyzmq-26.0.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4b7c0c0b3244bb2275abe255d4a30c050d541c6cb18b870975553f1fb6f37527"}, - {file = "pyzmq-26.0.3-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:ac97a21de3712afe6a6c071abfad40a6224fd14fa6ff0ff8d0c6e6cd4e2f807a"}, - {file = "pyzmq-26.0.3-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:88b88282e55fa39dd556d7fc04160bcf39dea015f78e0cecec8ff4f06c1fc2b5"}, - {file = "pyzmq-26.0.3-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:72b67f966b57dbd18dcc7efbc1c7fc9f5f983e572db1877081f075004614fcdd"}, - {file = "pyzmq-26.0.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:f4b6cecbbf3b7380f3b61de3a7b93cb721125dc125c854c14ddc91225ba52f83"}, - {file = "pyzmq-26.0.3-cp311-cp311-win32.whl", hash = "sha256:eed56b6a39216d31ff8cd2f1d048b5bf1700e4b32a01b14379c3b6dde9ce3aa3"}, - {file = "pyzmq-26.0.3-cp311-cp311-win_amd64.whl", hash = "sha256:3191d312c73e3cfd0f0afdf51df8405aafeb0bad71e7ed8f68b24b63c4f36500"}, - {file = "pyzmq-26.0.3-cp311-cp311-win_arm64.whl", hash = "sha256:b6907da3017ef55139cf0e417c5123a84c7332520e73a6902ff1f79046cd3b94"}, - {file = "pyzmq-26.0.3-cp312-cp312-macosx_10_15_universal2.whl", hash = "sha256:068ca17214038ae986d68f4a7021f97e187ed278ab6dccb79f837d765a54d753"}, - {file = "pyzmq-26.0.3-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:7821d44fe07335bea256b9f1f41474a642ca55fa671dfd9f00af8d68a920c2d4"}, - {file = "pyzmq-26.0.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:eeb438a26d87c123bb318e5f2b3d86a36060b01f22fbdffd8cf247d52f7c9a2b"}, - {file = "pyzmq-26.0.3-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:69ea9d6d9baa25a4dc9cef5e2b77b8537827b122214f210dd925132e34ae9b12"}, - {file = "pyzmq-26.0.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7daa3e1369355766dea11f1d8ef829905c3b9da886ea3152788dc25ee6079e02"}, - {file = "pyzmq-26.0.3-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:6ca7a9a06b52d0e38ccf6bca1aeff7be178917893f3883f37b75589d42c4ac20"}, - {file = "pyzmq-26.0.3-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:1b7d0e124948daa4d9686d421ef5087c0516bc6179fdcf8828b8444f8e461a77"}, - {file = "pyzmq-26.0.3-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:e746524418b70f38550f2190eeee834db8850088c834d4c8406fbb9bc1ae10b2"}, - {file = "pyzmq-26.0.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:6b3146f9ae6af82c47a5282ac8803523d381b3b21caeae0327ed2f7ecb718798"}, - {file = "pyzmq-26.0.3-cp312-cp312-win32.whl", hash = "sha256:2b291d1230845871c00c8462c50565a9cd6026fe1228e77ca934470bb7d70ea0"}, - {file = "pyzmq-26.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:926838a535c2c1ea21c903f909a9a54e675c2126728c21381a94ddf37c3cbddf"}, - {file = "pyzmq-26.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:5bf6c237f8c681dfb91b17f8435b2735951f0d1fad10cc5dfd96db110243370b"}, - {file = "pyzmq-26.0.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:0c0991f5a96a8e620f7691e61178cd8f457b49e17b7d9cfa2067e2a0a89fc1d5"}, - {file = "pyzmq-26.0.3-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:dbf012d8fcb9f2cf0643b65df3b355fdd74fc0035d70bb5c845e9e30a3a4654b"}, - {file = "pyzmq-26.0.3-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:01fbfbeb8249a68d257f601deb50c70c929dc2dfe683b754659569e502fbd3aa"}, - {file = "pyzmq-26.0.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1c8eb19abe87029c18f226d42b8a2c9efdd139d08f8bf6e085dd9075446db450"}, - {file = "pyzmq-26.0.3-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:5344b896e79800af86ad643408ca9aa303a017f6ebff8cee5a3163c1e9aec987"}, - {file = "pyzmq-26.0.3-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:204e0f176fd1d067671157d049466869b3ae1fc51e354708b0dc41cf94e23a3a"}, - {file = "pyzmq-26.0.3-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:a42db008d58530efa3b881eeee4991146de0b790e095f7ae43ba5cc612decbc5"}, - {file = "pyzmq-26.0.3-cp37-cp37m-win32.whl", hash = "sha256:8d7a498671ca87e32b54cb47c82a92b40130a26c5197d392720a1bce1b3c77cf"}, - {file = "pyzmq-26.0.3-cp37-cp37m-win_amd64.whl", hash = "sha256:3b4032a96410bdc760061b14ed6a33613ffb7f702181ba999df5d16fb96ba16a"}, - {file = "pyzmq-26.0.3-cp38-cp38-macosx_10_15_universal2.whl", hash = "sha256:2cc4e280098c1b192c42a849de8de2c8e0f3a84086a76ec5b07bfee29bda7d18"}, - {file = "pyzmq-26.0.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:5bde86a2ed3ce587fa2b207424ce15b9a83a9fa14422dcc1c5356a13aed3df9d"}, - {file = "pyzmq-26.0.3-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:34106f68e20e6ff253c9f596ea50397dbd8699828d55e8fa18bd4323d8d966e6"}, - {file = "pyzmq-26.0.3-cp38-cp38-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:ebbbd0e728af5db9b04e56389e2299a57ea8b9dd15c9759153ee2455b32be6ad"}, - {file = "pyzmq-26.0.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f6b1d1c631e5940cac5a0b22c5379c86e8df6a4ec277c7a856b714021ab6cfad"}, - {file = "pyzmq-26.0.3-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:e891ce81edd463b3b4c3b885c5603c00141151dd9c6936d98a680c8c72fe5c67"}, - {file = "pyzmq-26.0.3-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:9b273ecfbc590a1b98f014ae41e5cf723932f3b53ba9367cfb676f838038b32c"}, - {file = "pyzmq-26.0.3-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:b32bff85fb02a75ea0b68f21e2412255b5731f3f389ed9aecc13a6752f58ac97"}, - {file = "pyzmq-26.0.3-cp38-cp38-win32.whl", hash = "sha256:f6c21c00478a7bea93caaaef9e7629145d4153b15a8653e8bb4609d4bc70dbfc"}, - {file = "pyzmq-26.0.3-cp38-cp38-win_amd64.whl", hash = "sha256:3401613148d93ef0fd9aabdbddb212de3db7a4475367f49f590c837355343972"}, - {file = "pyzmq-26.0.3-cp39-cp39-macosx_10_15_universal2.whl", hash = "sha256:2ed8357f4c6e0daa4f3baf31832df8a33334e0fe5b020a61bc8b345a3db7a606"}, - {file = "pyzmq-26.0.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c1c8f2a2ca45292084c75bb6d3a25545cff0ed931ed228d3a1810ae3758f975f"}, - {file = "pyzmq-26.0.3-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:b63731993cdddcc8e087c64e9cf003f909262b359110070183d7f3025d1c56b5"}, - {file = "pyzmq-26.0.3-cp39-cp39-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:b3cd31f859b662ac5d7f4226ec7d8bd60384fa037fc02aee6ff0b53ba29a3ba8"}, - {file = "pyzmq-26.0.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:115f8359402fa527cf47708d6f8a0f8234f0e9ca0cab7c18c9c189c194dbf620"}, - {file = "pyzmq-26.0.3-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:715bdf952b9533ba13dfcf1f431a8f49e63cecc31d91d007bc1deb914f47d0e4"}, - {file = "pyzmq-26.0.3-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:e1258c639e00bf5e8a522fec6c3eaa3e30cf1c23a2f21a586be7e04d50c9acab"}, - {file = "pyzmq-26.0.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:15c59e780be8f30a60816a9adab900c12a58d79c1ac742b4a8df044ab2a6d920"}, - {file = "pyzmq-26.0.3-cp39-cp39-win32.whl", hash = "sha256:d0cdde3c78d8ab5b46595054e5def32a755fc028685add5ddc7403e9f6de9879"}, - {file = "pyzmq-26.0.3-cp39-cp39-win_amd64.whl", hash = "sha256:ce828058d482ef860746bf532822842e0ff484e27f540ef5c813d516dd8896d2"}, - {file = "pyzmq-26.0.3-cp39-cp39-win_arm64.whl", hash = "sha256:788f15721c64109cf720791714dc14afd0f449d63f3a5487724f024345067381"}, - {file = "pyzmq-26.0.3-pp310-pypy310_pp73-macosx_10_9_x86_64.whl", hash = "sha256:2c18645ef6294d99b256806e34653e86236eb266278c8ec8112622b61db255de"}, - {file = "pyzmq-26.0.3-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7e6bc96ebe49604df3ec2c6389cc3876cabe475e6bfc84ced1bf4e630662cb35"}, - {file = "pyzmq-26.0.3-pp310-pypy310_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:971e8990c5cc4ddcff26e149398fc7b0f6a042306e82500f5e8db3b10ce69f84"}, - {file = "pyzmq-26.0.3-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d8416c23161abd94cc7da80c734ad7c9f5dbebdadfdaa77dad78244457448223"}, - {file = "pyzmq-26.0.3-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:082a2988364b60bb5de809373098361cf1dbb239623e39e46cb18bc035ed9c0c"}, - {file = "pyzmq-26.0.3-pp37-pypy37_pp73-macosx_10_9_x86_64.whl", hash = "sha256:d57dfbf9737763b3a60d26e6800e02e04284926329aee8fb01049635e957fe81"}, - {file = "pyzmq-26.0.3-pp37-pypy37_pp73-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:77a85dca4c2430ac04dc2a2185c2deb3858a34fe7f403d0a946fa56970cf60a1"}, - {file = "pyzmq-26.0.3-pp37-pypy37_pp73-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:4c82a6d952a1d555bf4be42b6532927d2a5686dd3c3e280e5f63225ab47ac1f5"}, - {file = "pyzmq-26.0.3-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4496b1282c70c442809fc1b151977c3d967bfb33e4e17cedbf226d97de18f709"}, - {file = "pyzmq-26.0.3-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:e4946d6bdb7ba972dfda282f9127e5756d4f299028b1566d1245fa0d438847e6"}, - {file = "pyzmq-26.0.3-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:03c0ae165e700364b266876d712acb1ac02693acd920afa67da2ebb91a0b3c09"}, - {file = "pyzmq-26.0.3-pp38-pypy38_pp73-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:3e3070e680f79887d60feeda051a58d0ac36622e1759f305a41059eff62c6da7"}, - {file = "pyzmq-26.0.3-pp38-pypy38_pp73-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:6ca08b840fe95d1c2bd9ab92dac5685f949fc6f9ae820ec16193e5ddf603c3b2"}, - {file = "pyzmq-26.0.3-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e76654e9dbfb835b3518f9938e565c7806976c07b37c33526b574cc1a1050480"}, - {file = "pyzmq-26.0.3-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:871587bdadd1075b112e697173e946a07d722459d20716ceb3d1bd6c64bd08ce"}, - {file = "pyzmq-26.0.3-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:d0a2d1bd63a4ad79483049b26514e70fa618ce6115220da9efdff63688808b17"}, - {file = "pyzmq-26.0.3-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0270b49b6847f0d106d64b5086e9ad5dc8a902413b5dbbb15d12b60f9c1747a4"}, - {file = "pyzmq-26.0.3-pp39-pypy39_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:703c60b9910488d3d0954ca585c34f541e506a091a41930e663a098d3b794c67"}, - {file = "pyzmq-26.0.3-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:74423631b6be371edfbf7eabb02ab995c2563fee60a80a30829176842e71722a"}, - {file = "pyzmq-26.0.3-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:4adfbb5451196842a88fda3612e2c0414134874bffb1c2ce83ab4242ec9e027d"}, - {file = "pyzmq-26.0.3-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:3516119f4f9b8671083a70b6afaa0a070f5683e431ab3dc26e9215620d7ca1ad"}, - {file = "pyzmq-26.0.3.tar.gz", hash = "sha256:dba7d9f2e047dfa2bca3b01f4f84aa5246725203d6284e3790f2ca15fba6b40a"}, + {file = "pyzmq-26.2.0-cp310-cp310-macosx_10_15_universal2.whl", hash = "sha256:ddf33d97d2f52d89f6e6e7ae66ee35a4d9ca6f36eda89c24591b0c40205a3629"}, + {file = "pyzmq-26.2.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:dacd995031a01d16eec825bf30802fceb2c3791ef24bcce48fa98ce40918c27b"}, + {file = "pyzmq-26.2.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:89289a5ee32ef6c439086184529ae060c741334b8970a6855ec0b6ad3ff28764"}, + {file = "pyzmq-26.2.0-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5506f06d7dc6ecf1efacb4a013b1f05071bb24b76350832c96449f4a2d95091c"}, + {file = "pyzmq-26.2.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8ea039387c10202ce304af74def5021e9adc6297067f3441d348d2b633e8166a"}, + {file = "pyzmq-26.2.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:a2224fa4a4c2ee872886ed00a571f5e967c85e078e8e8c2530a2fb01b3309b88"}, + {file = "pyzmq-26.2.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:28ad5233e9c3b52d76196c696e362508959741e1a005fb8fa03b51aea156088f"}, + {file = "pyzmq-26.2.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:1c17211bc037c7d88e85ed8b7d8f7e52db6dc8eca5590d162717c654550f7282"}, + {file = "pyzmq-26.2.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:b8f86dd868d41bea9a5f873ee13bf5551c94cf6bc51baebc6f85075971fe6eea"}, + {file = "pyzmq-26.2.0-cp310-cp310-win32.whl", hash = "sha256:46a446c212e58456b23af260f3d9fb785054f3e3653dbf7279d8f2b5546b21c2"}, + {file = "pyzmq-26.2.0-cp310-cp310-win_amd64.whl", hash = "sha256:49d34ab71db5a9c292a7644ce74190b1dd5a3475612eefb1f8be1d6961441971"}, + {file = "pyzmq-26.2.0-cp310-cp310-win_arm64.whl", hash = "sha256:bfa832bfa540e5b5c27dcf5de5d82ebc431b82c453a43d141afb1e5d2de025fa"}, + {file = "pyzmq-26.2.0-cp311-cp311-macosx_10_15_universal2.whl", hash = "sha256:8f7e66c7113c684c2b3f1c83cdd3376103ee0ce4c49ff80a648643e57fb22218"}, + {file = "pyzmq-26.2.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:3a495b30fc91db2db25120df5847d9833af237546fd59170701acd816ccc01c4"}, + {file = "pyzmq-26.2.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:77eb0968da535cba0470a5165468b2cac7772cfb569977cff92e240f57e31bef"}, + {file = "pyzmq-26.2.0-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6ace4f71f1900a548f48407fc9be59c6ba9d9aaf658c2eea6cf2779e72f9f317"}, + {file = "pyzmq-26.2.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:92a78853d7280bffb93df0a4a6a2498cba10ee793cc8076ef797ef2f74d107cf"}, + {file = "pyzmq-26.2.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:689c5d781014956a4a6de61d74ba97b23547e431e9e7d64f27d4922ba96e9d6e"}, + {file = "pyzmq-26.2.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:0aca98bc423eb7d153214b2df397c6421ba6373d3397b26c057af3c904452e37"}, + {file = "pyzmq-26.2.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:1f3496d76b89d9429a656293744ceca4d2ac2a10ae59b84c1da9b5165f429ad3"}, + {file = "pyzmq-26.2.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:5c2b3bfd4b9689919db068ac6c9911f3fcb231c39f7dd30e3138be94896d18e6"}, + {file = "pyzmq-26.2.0-cp311-cp311-win32.whl", hash = "sha256:eac5174677da084abf378739dbf4ad245661635f1600edd1221f150b165343f4"}, + {file = "pyzmq-26.2.0-cp311-cp311-win_amd64.whl", hash = "sha256:5a509df7d0a83a4b178d0f937ef14286659225ef4e8812e05580776c70e155d5"}, + {file = "pyzmq-26.2.0-cp311-cp311-win_arm64.whl", hash = "sha256:c0e6091b157d48cbe37bd67233318dbb53e1e6327d6fc3bb284afd585d141003"}, + {file = "pyzmq-26.2.0-cp312-cp312-macosx_10_15_universal2.whl", hash = "sha256:ded0fc7d90fe93ae0b18059930086c51e640cdd3baebdc783a695c77f123dcd9"}, + {file = "pyzmq-26.2.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:17bf5a931c7f6618023cdacc7081f3f266aecb68ca692adac015c383a134ca52"}, + {file = "pyzmq-26.2.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:55cf66647e49d4621a7e20c8d13511ef1fe1efbbccf670811864452487007e08"}, + {file = "pyzmq-26.2.0-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4661c88db4a9e0f958c8abc2b97472e23061f0bc737f6f6179d7a27024e1faa5"}, + {file = "pyzmq-26.2.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ea7f69de383cb47522c9c208aec6dd17697db7875a4674c4af3f8cfdac0bdeae"}, + {file = "pyzmq-26.2.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:7f98f6dfa8b8ccaf39163ce872bddacca38f6a67289116c8937a02e30bbe9711"}, + {file = "pyzmq-26.2.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:e3e0210287329272539eea617830a6a28161fbbd8a3271bf4150ae3e58c5d0e6"}, + {file = "pyzmq-26.2.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:6b274e0762c33c7471f1a7471d1a2085b1a35eba5cdc48d2ae319f28b6fc4de3"}, + {file = "pyzmq-26.2.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:29c6a4635eef69d68a00321e12a7d2559fe2dfccfa8efae3ffb8e91cd0b36a8b"}, + {file = "pyzmq-26.2.0-cp312-cp312-win32.whl", hash = "sha256:989d842dc06dc59feea09e58c74ca3e1678c812a4a8a2a419046d711031f69c7"}, + {file = "pyzmq-26.2.0-cp312-cp312-win_amd64.whl", hash = "sha256:2a50625acdc7801bc6f74698c5c583a491c61d73c6b7ea4dee3901bb99adb27a"}, + {file = "pyzmq-26.2.0-cp312-cp312-win_arm64.whl", hash = "sha256:4d29ab8592b6ad12ebbf92ac2ed2bedcfd1cec192d8e559e2e099f648570e19b"}, + {file = "pyzmq-26.2.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:9dd8cd1aeb00775f527ec60022004d030ddc51d783d056e3e23e74e623e33726"}, + {file = "pyzmq-26.2.0-cp313-cp313-macosx_10_15_universal2.whl", hash = "sha256:28c812d9757fe8acecc910c9ac9dafd2ce968c00f9e619db09e9f8f54c3a68a3"}, + {file = "pyzmq-26.2.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4d80b1dd99c1942f74ed608ddb38b181b87476c6a966a88a950c7dee118fdf50"}, + {file = "pyzmq-26.2.0-cp313-cp313-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8c997098cc65e3208eca09303630e84d42718620e83b733d0fd69543a9cab9cb"}, + {file = "pyzmq-26.2.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7ad1bc8d1b7a18497dda9600b12dc193c577beb391beae5cd2349184db40f187"}, + {file = "pyzmq-26.2.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:bea2acdd8ea4275e1278350ced63da0b166421928276c7c8e3f9729d7402a57b"}, + {file = "pyzmq-26.2.0-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:23f4aad749d13698f3f7b64aad34f5fc02d6f20f05999eebc96b89b01262fb18"}, + {file = "pyzmq-26.2.0-cp313-cp313-musllinux_1_1_i686.whl", hash = "sha256:a4f96f0d88accc3dbe4a9025f785ba830f968e21e3e2c6321ccdfc9aef755115"}, + {file = "pyzmq-26.2.0-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:ced65e5a985398827cc9276b93ef6dfabe0273c23de8c7931339d7e141c2818e"}, + {file = "pyzmq-26.2.0-cp313-cp313-win32.whl", hash = "sha256:31507f7b47cc1ead1f6e86927f8ebb196a0bab043f6345ce070f412a59bf87b5"}, + {file = "pyzmq-26.2.0-cp313-cp313-win_amd64.whl", hash = "sha256:70fc7fcf0410d16ebdda9b26cbd8bf8d803d220a7f3522e060a69a9c87bf7bad"}, + {file = "pyzmq-26.2.0-cp313-cp313-win_arm64.whl", hash = "sha256:c3789bd5768ab5618ebf09cef6ec2b35fed88709b104351748a63045f0ff9797"}, + {file = "pyzmq-26.2.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:034da5fc55d9f8da09015d368f519478a52675e558c989bfcb5cf6d4e16a7d2a"}, + {file = "pyzmq-26.2.0-cp313-cp313t-macosx_10_15_universal2.whl", hash = "sha256:c92d73464b886931308ccc45b2744e5968cbaade0b1d6aeb40d8ab537765f5bc"}, + {file = "pyzmq-26.2.0-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:794a4562dcb374f7dbbfb3f51d28fb40123b5a2abadee7b4091f93054909add5"}, + {file = "pyzmq-26.2.0-cp313-cp313t-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:aee22939bb6075e7afededabad1a56a905da0b3c4e3e0c45e75810ebe3a52672"}, + {file = "pyzmq-26.2.0-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2ae90ff9dad33a1cfe947d2c40cb9cb5e600d759ac4f0fd22616ce6540f72797"}, + {file = "pyzmq-26.2.0-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:43a47408ac52647dfabbc66a25b05b6a61700b5165807e3fbd40063fcaf46386"}, + {file = "pyzmq-26.2.0-cp313-cp313t-musllinux_1_1_aarch64.whl", hash = "sha256:25bf2374a2a8433633c65ccb9553350d5e17e60c8eb4de4d92cc6bd60f01d306"}, + {file = "pyzmq-26.2.0-cp313-cp313t-musllinux_1_1_i686.whl", hash = "sha256:007137c9ac9ad5ea21e6ad97d3489af654381324d5d3ba614c323f60dab8fae6"}, + {file = "pyzmq-26.2.0-cp313-cp313t-musllinux_1_1_x86_64.whl", hash = "sha256:470d4a4f6d48fb34e92d768b4e8a5cc3780db0d69107abf1cd7ff734b9766eb0"}, + {file = "pyzmq-26.2.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:3b55a4229ce5da9497dd0452b914556ae58e96a4381bb6f59f1305dfd7e53fc8"}, + {file = "pyzmq-26.2.0-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:9cb3a6460cdea8fe8194a76de8895707e61ded10ad0be97188cc8463ffa7e3a8"}, + {file = "pyzmq-26.2.0-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:8ab5cad923cc95c87bffee098a27856c859bd5d0af31bd346035aa816b081fe1"}, + {file = "pyzmq-26.2.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9ed69074a610fad1c2fda66180e7b2edd4d31c53f2d1872bc2d1211563904cd9"}, + {file = "pyzmq-26.2.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:cccba051221b916a4f5e538997c45d7d136a5646442b1231b916d0164067ea27"}, + {file = "pyzmq-26.2.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:0eaa83fc4c1e271c24eaf8fb083cbccef8fde77ec8cd45f3c35a9a123e6da097"}, + {file = "pyzmq-26.2.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:9edda2df81daa129b25a39b86cb57dfdfe16f7ec15b42b19bfac503360d27a93"}, + {file = "pyzmq-26.2.0-cp37-cp37m-win32.whl", hash = "sha256:ea0eb6af8a17fa272f7b98d7bebfab7836a0d62738e16ba380f440fceca2d951"}, + {file = "pyzmq-26.2.0-cp37-cp37m-win_amd64.whl", hash = "sha256:4ff9dc6bc1664bb9eec25cd17506ef6672d506115095411e237d571e92a58231"}, + {file = "pyzmq-26.2.0-cp38-cp38-macosx_10_15_universal2.whl", hash = "sha256:2eb7735ee73ca1b0d71e0e67c3739c689067f055c764f73aac4cc8ecf958ee3f"}, + {file = "pyzmq-26.2.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1a534f43bc738181aa7cbbaf48e3eca62c76453a40a746ab95d4b27b1111a7d2"}, + {file = "pyzmq-26.2.0-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:aedd5dd8692635813368e558a05266b995d3d020b23e49581ddd5bbe197a8ab6"}, + {file = "pyzmq-26.2.0-cp38-cp38-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:8be4700cd8bb02cc454f630dcdf7cfa99de96788b80c51b60fe2fe1dac480289"}, + {file = "pyzmq-26.2.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fcc03fa4997c447dce58264e93b5aa2d57714fbe0f06c07b7785ae131512732"}, + {file = "pyzmq-26.2.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:402b190912935d3db15b03e8f7485812db350d271b284ded2b80d2e5704be780"}, + {file = "pyzmq-26.2.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:8685fa9c25ff00f550c1fec650430c4b71e4e48e8d852f7ddcf2e48308038640"}, + {file = "pyzmq-26.2.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:76589c020680778f06b7e0b193f4b6dd66d470234a16e1df90329f5e14a171cd"}, + {file = "pyzmq-26.2.0-cp38-cp38-win32.whl", hash = "sha256:8423c1877d72c041f2c263b1ec6e34360448decfb323fa8b94e85883043ef988"}, + {file = "pyzmq-26.2.0-cp38-cp38-win_amd64.whl", hash = "sha256:76589f2cd6b77b5bdea4fca5992dc1c23389d68b18ccc26a53680ba2dc80ff2f"}, + {file = "pyzmq-26.2.0-cp39-cp39-macosx_10_15_universal2.whl", hash = "sha256:b1d464cb8d72bfc1a3adc53305a63a8e0cac6bc8c5a07e8ca190ab8d3faa43c2"}, + {file = "pyzmq-26.2.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:4da04c48873a6abdd71811c5e163bd656ee1b957971db7f35140a2d573f6949c"}, + {file = "pyzmq-26.2.0-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:d049df610ac811dcffdc147153b414147428567fbbc8be43bb8885f04db39d98"}, + {file = "pyzmq-26.2.0-cp39-cp39-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:05590cdbc6b902101d0e65d6a4780af14dc22914cc6ab995d99b85af45362cc9"}, + {file = "pyzmq-26.2.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c811cfcd6a9bf680236c40c6f617187515269ab2912f3d7e8c0174898e2519db"}, + {file = "pyzmq-26.2.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:6835dd60355593de10350394242b5757fbbd88b25287314316f266e24c61d073"}, + {file = "pyzmq-26.2.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:bc6bee759a6bddea5db78d7dcd609397449cb2d2d6587f48f3ca613b19410cfc"}, + {file = "pyzmq-26.2.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:c530e1eecd036ecc83c3407f77bb86feb79916d4a33d11394b8234f3bd35b940"}, + {file = "pyzmq-26.2.0-cp39-cp39-win32.whl", hash = "sha256:367b4f689786fca726ef7a6c5ba606958b145b9340a5e4808132cc65759abd44"}, + {file = "pyzmq-26.2.0-cp39-cp39-win_amd64.whl", hash = "sha256:e6fa2e3e683f34aea77de8112f6483803c96a44fd726d7358b9888ae5bb394ec"}, + {file = "pyzmq-26.2.0-cp39-cp39-win_arm64.whl", hash = "sha256:7445be39143a8aa4faec43b076e06944b8f9d0701b669df4af200531b21e40bb"}, + {file = "pyzmq-26.2.0-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:706e794564bec25819d21a41c31d4df2d48e1cc4b061e8d345d7fb4dd3e94072"}, + {file = "pyzmq-26.2.0-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8b435f2753621cd36e7c1762156815e21c985c72b19135dac43a7f4f31d28dd1"}, + {file = "pyzmq-26.2.0-pp310-pypy310_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:160c7e0a5eb178011e72892f99f918c04a131f36056d10d9c1afb223fc952c2d"}, + {file = "pyzmq-26.2.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2c4a71d5d6e7b28a47a394c0471b7e77a0661e2d651e7ae91e0cab0a587859ca"}, + {file = "pyzmq-26.2.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:90412f2db8c02a3864cbfc67db0e3dcdbda336acf1c469526d3e869394fe001c"}, + {file = "pyzmq-26.2.0-pp37-pypy37_pp73-macosx_10_9_x86_64.whl", hash = "sha256:2ea4ad4e6a12e454de05f2949d4beddb52460f3de7c8b9d5c46fbb7d7222e02c"}, + {file = "pyzmq-26.2.0-pp37-pypy37_pp73-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:fc4f7a173a5609631bb0c42c23d12c49df3966f89f496a51d3eb0ec81f4519d6"}, + {file = "pyzmq-26.2.0-pp37-pypy37_pp73-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:878206a45202247781472a2d99df12a176fef806ca175799e1c6ad263510d57c"}, + {file = "pyzmq-26.2.0-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:17c412bad2eb9468e876f556eb4ee910e62d721d2c7a53c7fa31e643d35352e6"}, + {file = "pyzmq-26.2.0-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:0d987a3ae5a71c6226b203cfd298720e0086c7fe7c74f35fa8edddfbd6597eed"}, + {file = "pyzmq-26.2.0-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:39887ac397ff35b7b775db7201095fc6310a35fdbae85bac4523f7eb3b840e20"}, + {file = "pyzmq-26.2.0-pp38-pypy38_pp73-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:fdb5b3e311d4d4b0eb8b3e8b4d1b0a512713ad7e6a68791d0923d1aec433d919"}, + {file = "pyzmq-26.2.0-pp38-pypy38_pp73-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:226af7dcb51fdb0109f0016449b357e182ea0ceb6b47dfb5999d569e5db161d5"}, + {file = "pyzmq-26.2.0-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0bed0e799e6120b9c32756203fb9dfe8ca2fb8467fed830c34c877e25638c3fc"}, + {file = "pyzmq-26.2.0-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:29c7947c594e105cb9e6c466bace8532dc1ca02d498684128b339799f5248277"}, + {file = "pyzmq-26.2.0-pp39-pypy39_pp73-macosx_10_15_x86_64.whl", hash = "sha256:cdeabcff45d1c219636ee2e54d852262e5c2e085d6cb476d938aee8d921356b3"}, + {file = "pyzmq-26.2.0-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:35cffef589bcdc587d06f9149f8d5e9e8859920a071df5a2671de2213bef592a"}, + {file = "pyzmq-26.2.0-pp39-pypy39_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:18c8dc3b7468d8b4bdf60ce9d7141897da103c7a4690157b32b60acb45e333e6"}, + {file = "pyzmq-26.2.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7133d0a1677aec369d67dd78520d3fa96dd7f3dcec99d66c1762870e5ea1a50a"}, + {file = "pyzmq-26.2.0-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:6a96179a24b14fa6428cbfc08641c779a53f8fcec43644030328f44034c7f1f4"}, + {file = "pyzmq-26.2.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:4f78c88905461a9203eac9faac157a2a0dbba84a0fd09fd29315db27be40af9f"}, + {file = "pyzmq-26.2.0.tar.gz", hash = "sha256:070672c258581c8e4f640b5159297580a9974b026043bd4ab0470be9ed324f1f"}, ] [package.dependencies] @@ -3592,101 +3673,101 @@ rpds-py = ">=0.7.0" [[package]] name = "regex" -version = "2024.5.10" +version = "2024.7.24" description = "Alternative regular expression module, to replace re." optional = false python-versions = ">=3.8" files = [ - {file = "regex-2024.5.10-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:eda3dd46df535da787ffb9036b5140f941ecb91701717df91c9daf64cabef953"}, - {file = "regex-2024.5.10-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:1d5bd666466c8f00a06886ce1397ba8b12371c1f1c6d1bef11013e9e0a1464a8"}, - {file = "regex-2024.5.10-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:32e5f3b8e32918bfbdd12eca62e49ab3031125c454b507127ad6ecbd86e62fca"}, - {file = "regex-2024.5.10-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:534efd2653ebc4f26fc0e47234e53bf0cb4715bb61f98c64d2774a278b58c846"}, - {file = "regex-2024.5.10-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:193b7c6834a06f722f0ce1ba685efe80881de7c3de31415513862f601097648c"}, - {file = "regex-2024.5.10-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:160ba087232c5c6e2a1e7ad08bd3a3f49b58c815be0504d8c8aacfb064491cd8"}, - {file = "regex-2024.5.10-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:951be1eae7b47660412dc4938777a975ebc41936d64e28081bf2e584b47ec246"}, - {file = "regex-2024.5.10-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d8a0f0ab5453e409586b11ebe91c672040bc804ca98d03a656825f7890cbdf88"}, - {file = "regex-2024.5.10-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:9e6d4d6ae1827b2f8c7200aaf7501c37cf3f3896c86a6aaf2566448397c823dd"}, - {file = "regex-2024.5.10-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:161a206c8f3511e2f5fafc9142a2cc25d7fe9a1ec5ad9b4ad2496a7c33e1c5d2"}, - {file = "regex-2024.5.10-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:44b3267cea873684af022822195298501568ed44d542f9a2d9bebc0212e99069"}, - {file = "regex-2024.5.10-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:560278c9975694e1f0bc50da187abf2cdc1e4890739ea33df2bc4a85eeef143e"}, - {file = "regex-2024.5.10-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:70364a097437dd0a90b31cd77f09f7387ad9ac60ef57590971f43b7fca3082a5"}, - {file = "regex-2024.5.10-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:42be5de7cc8c1edac55db92d82b68dc8e683b204d6f5414c5a51997a323d7081"}, - {file = "regex-2024.5.10-cp310-cp310-win32.whl", hash = "sha256:9a8625849387b9d558d528e263ecc9c0fbde86cfa5c2f0eef43fff480ae24d71"}, - {file = "regex-2024.5.10-cp310-cp310-win_amd64.whl", hash = "sha256:903350bf44d7e4116b4d5898b30b15755d61dcd3161e3413a49c7db76f0bee5a"}, - {file = "regex-2024.5.10-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:bf9596cba92ce7b1fd32c7b07c6e3212c7eed0edc271757e48bfcd2b54646452"}, - {file = "regex-2024.5.10-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:45cc13d398b6359a7708986386f72bd156ae781c3e83a68a6d4cee5af04b1ce9"}, - {file = "regex-2024.5.10-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:ad45f3bccfcb00868f2871dce02a755529838d2b86163ab8a246115e80cfb7d6"}, - {file = "regex-2024.5.10-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:33d19f0cde6838c81acffff25c7708e4adc7dd02896c9ec25c3939b1500a1778"}, - {file = "regex-2024.5.10-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:0a9f89d7db5ef6bdf53e5cc8e6199a493d0f1374b3171796b464a74ebe8e508a"}, - {file = "regex-2024.5.10-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8c6c71cf92b09e5faa72ea2c68aa1f61c9ce11cb66fdc5069d712f4392ddfd00"}, - {file = "regex-2024.5.10-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7467ad8b0eac0b28e52679e972b9b234b3de0ea5cee12eb50091d2b68145fe36"}, - {file = "regex-2024.5.10-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:bc0db93ad039fc2fe32ccd3dd0e0e70c4f3d6e37ae83f0a487e1aba939bd2fbd"}, - {file = "regex-2024.5.10-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:fa9335674d7c819674467c7b46154196c51efbaf5f5715187fd366814ba3fa39"}, - {file = "regex-2024.5.10-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:7dda3091838206969c2b286f9832dff41e2da545b99d1cfaea9ebd8584d02708"}, - {file = "regex-2024.5.10-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:504b5116e2bd1821efd815941edff7535e93372a098e156bb9dffde30264e798"}, - {file = "regex-2024.5.10-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:91b53dea84415e8115506cc62e441a2b54537359c63d856d73cb1abe05af4c9a"}, - {file = "regex-2024.5.10-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:1a3903128f9e17a500618e80c68165c78c741ebb17dd1a0b44575f92c3c68b02"}, - {file = "regex-2024.5.10-cp311-cp311-win32.whl", hash = "sha256:236cace6c1903effd647ed46ce6dd5d76d54985fc36dafc5256032886736c85d"}, - {file = "regex-2024.5.10-cp311-cp311-win_amd64.whl", hash = "sha256:12446827f43c7881decf2c126762e11425de5eb93b3b0d8b581344c16db7047a"}, - {file = "regex-2024.5.10-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:14905ed75c7a6edf423eb46c213ed3f4507c38115f1ed3c00f4ec9eafba50e58"}, - {file = "regex-2024.5.10-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:4fad420b14ae1970a1f322e8ae84a1d9d89375eb71e1b504060ab2d1bfe68f3c"}, - {file = "regex-2024.5.10-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:c46a76a599fcbf95f98755275c5527304cc4f1bb69919434c1e15544d7052910"}, - {file = "regex-2024.5.10-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0faecb6d5779753a6066a3c7a0471a8d29fe25d9981ca9e552d6d1b8f8b6a594"}, - {file = "regex-2024.5.10-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:aab65121229c2ecdf4a31b793d99a6a0501225bd39b616e653c87b219ed34a49"}, - {file = "regex-2024.5.10-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:50e7e96a527488334379e05755b210b7da4a60fc5d6481938c1fa053e0c92184"}, - {file = "regex-2024.5.10-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba034c8db4b264ef1601eb33cd23d87c5013b8fb48b8161debe2e5d3bd9156b0"}, - {file = "regex-2024.5.10-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:031219782d97550c2098d9a68ce9e9eaefe67d2d81d8ff84c8354f9c009e720c"}, - {file = "regex-2024.5.10-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:62b5f7910b639f3c1d122d408421317c351e213ca39c964ad4121f27916631c6"}, - {file = "regex-2024.5.10-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:cd832bd9b6120d6074f39bdfbb3c80e416848b07ac72910f1c7f03131a6debc3"}, - {file = "regex-2024.5.10-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:e91b1976358e17197157b405cab408a5f4e33310cda211c49fc6da7cffd0b2f0"}, - {file = "regex-2024.5.10-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:571452362d552de508c37191b6abbbb660028b8b418e2d68c20779e0bc8eaaa8"}, - {file = "regex-2024.5.10-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:5253dcb0bfda7214523de58b002eb0090cb530d7c55993ce5f6d17faf953ece7"}, - {file = "regex-2024.5.10-cp312-cp312-win32.whl", hash = "sha256:2f30a5ab8902f93930dc6f627c4dd5da2703333287081c85cace0fc6e21c25af"}, - {file = "regex-2024.5.10-cp312-cp312-win_amd64.whl", hash = "sha256:3799e36d60a35162bb35b2246d8bb012192b7437dff807ef79c14e7352706306"}, - {file = "regex-2024.5.10-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:bbdc5db2c98ac2bf1971ffa1410c87ca7a15800415f788971e8ba8520fc0fda9"}, - {file = "regex-2024.5.10-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6ccdeef4584450b6f0bddd5135354908dacad95425fcb629fe36d13e48b60f32"}, - {file = "regex-2024.5.10-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:29d839829209f3c53f004e1de8c3113efce6d98029f044fa5cfee666253ee7e6"}, - {file = "regex-2024.5.10-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0709ba544cf50bd5cb843df4b8bb6701bae2b70a8e88da9add8386cbca5c1385"}, - {file = "regex-2024.5.10-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:972b49f2fe1047b9249c958ec4fa1bdd2cf8ce305dc19d27546d5a38e57732d8"}, - {file = "regex-2024.5.10-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9cdbb1998da94607d5eec02566b9586f0e70d6438abf1b690261aac0edda7ab6"}, - {file = "regex-2024.5.10-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bf7c8ee4861d9ef5b1120abb75846828c811f932d63311596ad25fa168053e00"}, - {file = "regex-2024.5.10-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7d35d4cc9270944e95f9c88af757b0c9fc43f396917e143a5756608462c5223b"}, - {file = "regex-2024.5.10-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:8722f72068b3e1156a4b2e1afde6810f1fc67155a9fa30a4b9d5b4bc46f18fb0"}, - {file = "regex-2024.5.10-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:696639a73ca78a380acfaa0a1f6dd8220616a99074c05bba9ba8bb916914b224"}, - {file = "regex-2024.5.10-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:ea057306ab469130167014b662643cfaed84651c792948891d003cf0039223a5"}, - {file = "regex-2024.5.10-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:b43b78f9386d3d932a6ce5af4b45f393d2e93693ee18dc4800d30a8909df700e"}, - {file = "regex-2024.5.10-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:c43395a3b7cc9862801a65c6994678484f186ce13c929abab44fb8a9e473a55a"}, - {file = "regex-2024.5.10-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:0bc94873ba11e34837bffd7e5006703abeffc4514e2f482022f46ce05bd25e67"}, - {file = "regex-2024.5.10-cp38-cp38-win32.whl", hash = "sha256:1118ba9def608250250f4b3e3f48c62f4562ba16ca58ede491b6e7554bfa09ff"}, - {file = "regex-2024.5.10-cp38-cp38-win_amd64.whl", hash = "sha256:458d68d34fb74b906709735c927c029e62f7d06437a98af1b5b6258025223210"}, - {file = "regex-2024.5.10-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:15e593386ec6331e0ab4ac0795b7593f02ab2f4b30a698beb89fbdc34f92386a"}, - {file = "regex-2024.5.10-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:ca23b41355ba95929e9505ee04e55495726aa2282003ed9b012d86f857d3e49b"}, - {file = "regex-2024.5.10-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:2c8982ee19ccecabbaeac1ba687bfef085a6352a8c64f821ce2f43e6d76a9298"}, - {file = "regex-2024.5.10-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7117cb7d6ac7f2e985f3d18aa8a1728864097da1a677ffa69e970ca215baebf1"}, - {file = "regex-2024.5.10-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b66421f8878a0c82fc0c272a43e2121c8d4c67cb37429b764f0d5ad70b82993b"}, - {file = "regex-2024.5.10-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:224a9269f133564109ce668213ef3cb32bc72ccf040b0b51c72a50e569e9dc9e"}, - {file = "regex-2024.5.10-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ab98016541543692a37905871a5ffca59b16e08aacc3d7d10a27297b443f572d"}, - {file = "regex-2024.5.10-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:51d27844763c273a122e08a3e86e7aefa54ee09fb672d96a645ece0454d8425e"}, - {file = "regex-2024.5.10-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:853cc36e756ff673bf984e9044ccc8fad60b95a748915dddeab9488aea974c73"}, - {file = "regex-2024.5.10-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:4e7eaf9df15423d07b6050fb91f86c66307171b95ea53e2d87a7993b6d02c7f7"}, - {file = "regex-2024.5.10-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:169fd0acd7a259f58f417e492e93d0e15fc87592cd1e971c8c533ad5703b5830"}, - {file = "regex-2024.5.10-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:334b79ce9c08f26b4659a53f42892793948a613c46f1b583e985fd5a6bf1c149"}, - {file = "regex-2024.5.10-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:f03b1dbd4d9596dd84955bb40f7d885204d6aac0d56a919bb1e0ff2fb7e1735a"}, - {file = "regex-2024.5.10-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:cfa6d61a76c77610ba9274c1a90a453062bdf6887858afbe214d18ad41cf6bde"}, - {file = "regex-2024.5.10-cp39-cp39-win32.whl", hash = "sha256:249fbcee0a277c32a3ce36d8e36d50c27c968fdf969e0fbe342658d4e010fbc8"}, - {file = "regex-2024.5.10-cp39-cp39-win_amd64.whl", hash = "sha256:0ce56a923f4c01d7568811bfdffe156268c0a7aae8a94c902b92fe34c4bde785"}, - {file = "regex-2024.5.10.tar.gz", hash = "sha256:304e7e2418146ae4d0ef0e9ffa28f881f7874b45b4994cc2279b21b6e7ae50c8"}, + {file = "regex-2024.7.24-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:228b0d3f567fafa0633aee87f08b9276c7062da9616931382993c03808bb68ce"}, + {file = "regex-2024.7.24-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:3426de3b91d1bc73249042742f45c2148803c111d1175b283270177fdf669024"}, + {file = "regex-2024.7.24-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:f273674b445bcb6e4409bf8d1be67bc4b58e8b46fd0d560055d515b8830063cd"}, + {file = "regex-2024.7.24-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:23acc72f0f4e1a9e6e9843d6328177ae3074b4182167e34119ec7233dfeccf53"}, + {file = "regex-2024.7.24-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:65fd3d2e228cae024c411c5ccdffae4c315271eee4a8b839291f84f796b34eca"}, + {file = "regex-2024.7.24-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c414cbda77dbf13c3bc88b073a1a9f375c7b0cb5e115e15d4b73ec3a2fbc6f59"}, + {file = "regex-2024.7.24-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bf7a89eef64b5455835f5ed30254ec19bf41f7541cd94f266ab7cbd463f00c41"}, + {file = "regex-2024.7.24-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:19c65b00d42804e3fbea9708f0937d157e53429a39b7c61253ff15670ff62cb5"}, + {file = "regex-2024.7.24-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:7a5486ca56c8869070a966321d5ab416ff0f83f30e0e2da1ab48815c8d165d46"}, + {file = "regex-2024.7.24-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:6f51f9556785e5a203713f5efd9c085b4a45aecd2a42573e2b5041881b588d1f"}, + {file = "regex-2024.7.24-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:a4997716674d36a82eab3e86f8fa77080a5d8d96a389a61ea1d0e3a94a582cf7"}, + {file = "regex-2024.7.24-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:c0abb5e4e8ce71a61d9446040c1e86d4e6d23f9097275c5bd49ed978755ff0fe"}, + {file = "regex-2024.7.24-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:18300a1d78cf1290fa583cd8b7cde26ecb73e9f5916690cf9d42de569c89b1ce"}, + {file = "regex-2024.7.24-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:416c0e4f56308f34cdb18c3f59849479dde5b19febdcd6e6fa4d04b6c31c9faa"}, + {file = "regex-2024.7.24-cp310-cp310-win32.whl", hash = "sha256:fb168b5924bef397b5ba13aabd8cf5df7d3d93f10218d7b925e360d436863f66"}, + {file = "regex-2024.7.24-cp310-cp310-win_amd64.whl", hash = "sha256:6b9fc7e9cc983e75e2518496ba1afc524227c163e43d706688a6bb9eca41617e"}, + {file = "regex-2024.7.24-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:382281306e3adaaa7b8b9ebbb3ffb43358a7bbf585fa93821300a418bb975281"}, + {file = "regex-2024.7.24-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:4fdd1384619f406ad9037fe6b6eaa3de2749e2e12084abc80169e8e075377d3b"}, + {file = "regex-2024.7.24-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:3d974d24edb231446f708c455fd08f94c41c1ff4f04bcf06e5f36df5ef50b95a"}, + {file = "regex-2024.7.24-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a2ec4419a3fe6cf8a4795752596dfe0adb4aea40d3683a132bae9c30b81e8d73"}, + {file = "regex-2024.7.24-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:eb563dd3aea54c797adf513eeec819c4213d7dbfc311874eb4fd28d10f2ff0f2"}, + {file = "regex-2024.7.24-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:45104baae8b9f67569f0f1dca5e1f1ed77a54ae1cd8b0b07aba89272710db61e"}, + {file = "regex-2024.7.24-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:994448ee01864501912abf2bad9203bffc34158e80fe8bfb5b031f4f8e16da51"}, + {file = "regex-2024.7.24-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3fac296f99283ac232d8125be932c5cd7644084a30748fda013028c815ba3364"}, + {file = "regex-2024.7.24-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:7e37e809b9303ec3a179085415cb5f418ecf65ec98cdfe34f6a078b46ef823ee"}, + {file = "regex-2024.7.24-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:01b689e887f612610c869421241e075c02f2e3d1ae93a037cb14f88ab6a8934c"}, + {file = "regex-2024.7.24-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:f6442f0f0ff81775eaa5b05af8a0ffa1dda36e9cf6ec1e0d3d245e8564b684ce"}, + {file = "regex-2024.7.24-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:871e3ab2838fbcb4e0865a6e01233975df3a15e6fce93b6f99d75cacbd9862d1"}, + {file = "regex-2024.7.24-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:c918b7a1e26b4ab40409820ddccc5d49871a82329640f5005f73572d5eaa9b5e"}, + {file = "regex-2024.7.24-cp311-cp311-win32.whl", hash = "sha256:2dfbb8baf8ba2c2b9aa2807f44ed272f0913eeeba002478c4577b8d29cde215c"}, + {file = "regex-2024.7.24-cp311-cp311-win_amd64.whl", hash = "sha256:538d30cd96ed7d1416d3956f94d54e426a8daf7c14527f6e0d6d425fcb4cca52"}, + {file = "regex-2024.7.24-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:fe4ebef608553aff8deb845c7f4f1d0740ff76fa672c011cc0bacb2a00fbde86"}, + {file = "regex-2024.7.24-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:74007a5b25b7a678459f06559504f1eec2f0f17bca218c9d56f6a0a12bfffdad"}, + {file = "regex-2024.7.24-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:7df9ea48641da022c2a3c9c641650cd09f0cd15e8908bf931ad538f5ca7919c9"}, + {file = "regex-2024.7.24-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6a1141a1dcc32904c47f6846b040275c6e5de0bf73f17d7a409035d55b76f289"}, + {file = "regex-2024.7.24-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:80c811cfcb5c331237d9bad3bea2c391114588cf4131707e84d9493064d267f9"}, + {file = "regex-2024.7.24-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7214477bf9bd195894cf24005b1e7b496f46833337b5dedb7b2a6e33f66d962c"}, + {file = "regex-2024.7.24-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d55588cba7553f0b6ec33130bc3e114b355570b45785cebdc9daed8c637dd440"}, + {file = "regex-2024.7.24-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:558a57cfc32adcf19d3f791f62b5ff564922942e389e3cfdb538a23d65a6b610"}, + {file = "regex-2024.7.24-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:a512eed9dfd4117110b1881ba9a59b31433caed0c4101b361f768e7bcbaf93c5"}, + {file = "regex-2024.7.24-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:86b17ba823ea76256b1885652e3a141a99a5c4422f4a869189db328321b73799"}, + {file = "regex-2024.7.24-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:5eefee9bfe23f6df09ffb6dfb23809f4d74a78acef004aa904dc7c88b9944b05"}, + {file = "regex-2024.7.24-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:731fcd76bbdbf225e2eb85b7c38da9633ad3073822f5ab32379381e8c3c12e94"}, + {file = "regex-2024.7.24-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:eaef80eac3b4cfbdd6de53c6e108b4c534c21ae055d1dbea2de6b3b8ff3def38"}, + {file = "regex-2024.7.24-cp312-cp312-win32.whl", hash = "sha256:185e029368d6f89f36e526764cf12bf8d6f0e3a2a7737da625a76f594bdfcbfc"}, + {file = "regex-2024.7.24-cp312-cp312-win_amd64.whl", hash = "sha256:2f1baff13cc2521bea83ab2528e7a80cbe0ebb2c6f0bfad15be7da3aed443908"}, + {file = "regex-2024.7.24-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:66b4c0731a5c81921e938dcf1a88e978264e26e6ac4ec96a4d21ae0354581ae0"}, + {file = "regex-2024.7.24-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:88ecc3afd7e776967fa16c80f974cb79399ee8dc6c96423321d6f7d4b881c92b"}, + {file = "regex-2024.7.24-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:64bd50cf16bcc54b274e20235bf8edbb64184a30e1e53873ff8d444e7ac656b2"}, + {file = "regex-2024.7.24-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:eb462f0e346fcf41a901a126b50f8781e9a474d3927930f3490f38a6e73b6950"}, + {file = "regex-2024.7.24-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a82465ebbc9b1c5c50738536fdfa7cab639a261a99b469c9d4c7dcbb2b3f1e57"}, + {file = "regex-2024.7.24-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:68a8f8c046c6466ac61a36b65bb2395c74451df2ffb8458492ef49900efed293"}, + {file = "regex-2024.7.24-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dac8e84fff5d27420f3c1e879ce9929108e873667ec87e0c8eeb413a5311adfe"}, + {file = "regex-2024.7.24-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ba2537ef2163db9e6ccdbeb6f6424282ae4dea43177402152c67ef869cf3978b"}, + {file = "regex-2024.7.24-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:43affe33137fcd679bdae93fb25924979517e011f9dea99163f80b82eadc7e53"}, + {file = "regex-2024.7.24-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:c9bb87fdf2ab2370f21e4d5636e5317775e5d51ff32ebff2cf389f71b9b13750"}, + {file = "regex-2024.7.24-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:945352286a541406f99b2655c973852da7911b3f4264e010218bbc1cc73168f2"}, + {file = "regex-2024.7.24-cp38-cp38-musllinux_1_2_ppc64le.whl", hash = "sha256:8bc593dcce679206b60a538c302d03c29b18e3d862609317cb560e18b66d10cf"}, + {file = "regex-2024.7.24-cp38-cp38-musllinux_1_2_s390x.whl", hash = "sha256:3f3b6ca8eae6d6c75a6cff525c8530c60e909a71a15e1b731723233331de4169"}, + {file = "regex-2024.7.24-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:c51edc3541e11fbe83f0c4d9412ef6c79f664a3745fab261457e84465ec9d5a8"}, + {file = "regex-2024.7.24-cp38-cp38-win32.whl", hash = "sha256:d0a07763776188b4db4c9c7fb1b8c494049f84659bb387b71c73bbc07f189e96"}, + {file = "regex-2024.7.24-cp38-cp38-win_amd64.whl", hash = "sha256:8fd5afd101dcf86a270d254364e0e8dddedebe6bd1ab9d5f732f274fa00499a5"}, + {file = "regex-2024.7.24-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:0ffe3f9d430cd37d8fa5632ff6fb36d5b24818c5c986893063b4e5bdb84cdf24"}, + {file = "regex-2024.7.24-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:25419b70ba00a16abc90ee5fce061228206173231f004437730b67ac77323f0d"}, + {file = "regex-2024.7.24-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:33e2614a7ce627f0cdf2ad104797d1f68342d967de3695678c0cb84f530709f8"}, + {file = "regex-2024.7.24-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d33a0021893ede5969876052796165bab6006559ab845fd7b515a30abdd990dc"}, + {file = "regex-2024.7.24-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:04ce29e2c5fedf296b1a1b0acc1724ba93a36fb14031f3abfb7abda2806c1535"}, + {file = "regex-2024.7.24-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b16582783f44fbca6fcf46f61347340c787d7530d88b4d590a397a47583f31dd"}, + {file = "regex-2024.7.24-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:836d3cc225b3e8a943d0b02633fb2f28a66e281290302a79df0e1eaa984ff7c1"}, + {file = "regex-2024.7.24-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:438d9f0f4bc64e8dea78274caa5af971ceff0f8771e1a2333620969936ba10be"}, + {file = "regex-2024.7.24-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:973335b1624859cb0e52f96062a28aa18f3a5fc77a96e4a3d6d76e29811a0e6e"}, + {file = "regex-2024.7.24-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:c5e69fd3eb0b409432b537fe3c6f44ac089c458ab6b78dcec14478422879ec5f"}, + {file = "regex-2024.7.24-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:fbf8c2f00904eaf63ff37718eb13acf8e178cb940520e47b2f05027f5bb34ce3"}, + {file = "regex-2024.7.24-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:ae2757ace61bc4061b69af19e4689fa4416e1a04840f33b441034202b5cd02d4"}, + {file = "regex-2024.7.24-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:44fc61b99035fd9b3b9453f1713234e5a7c92a04f3577252b45feefe1b327759"}, + {file = "regex-2024.7.24-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:84c312cdf839e8b579f504afcd7b65f35d60b6285d892b19adea16355e8343c9"}, + {file = "regex-2024.7.24-cp39-cp39-win32.whl", hash = "sha256:ca5b2028c2f7af4e13fb9fc29b28d0ce767c38c7facdf64f6c2cd040413055f1"}, + {file = "regex-2024.7.24-cp39-cp39-win_amd64.whl", hash = "sha256:7c479f5ae937ec9985ecaf42e2e10631551d909f203e31308c12d703922742f9"}, + {file = "regex-2024.7.24.tar.gz", hash = "sha256:9cfd009eed1a46b27c14039ad5bbc5e71b6367c5b2e6d5f5da0ea91600817506"}, ] [[package]] name = "requests" -version = "2.31.0" +version = "2.32.3" description = "Python HTTP for Humans." optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "requests-2.31.0-py3-none-any.whl", hash = "sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f"}, - {file = "requests-2.31.0.tar.gz", hash = "sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1"}, + {file = "requests-2.32.3-py3-none-any.whl", hash = "sha256:70761cfe03c773ceb22aa2f671b4757976145175cdfca038c02654d061d6dcc6"}, + {file = "requests-2.32.3.tar.gz", hash = "sha256:55365417734eb18255590a9ff9eb97e9e1da868d4ccd6402399eaf68af20a760"}, ] [package.dependencies] @@ -3744,110 +3825,114 @@ files = [ [[package]] name = "rpds-py" -version = "0.18.1" +version = "0.20.0" description = "Python bindings to Rust's persistent data structures (rpds)" optional = false python-versions = ">=3.8" files = [ - {file = "rpds_py-0.18.1-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:d31dea506d718693b6b2cffc0648a8929bdc51c70a311b2770f09611caa10d53"}, - {file = "rpds_py-0.18.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:732672fbc449bab754e0b15356c077cc31566df874964d4801ab14f71951ea80"}, - {file = "rpds_py-0.18.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4a98a1f0552b5f227a3d6422dbd61bc6f30db170939bd87ed14f3c339aa6c7c9"}, - {file = "rpds_py-0.18.1-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:7f1944ce16401aad1e3f7d312247b3d5de7981f634dc9dfe90da72b87d37887d"}, - {file = "rpds_py-0.18.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:38e14fb4e370885c4ecd734f093a2225ee52dc384b86fa55fe3f74638b2cfb09"}, - {file = "rpds_py-0.18.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:08d74b184f9ab6289b87b19fe6a6d1a97fbfea84b8a3e745e87a5de3029bf944"}, - {file = "rpds_py-0.18.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d70129cef4a8d979caa37e7fe957202e7eee8ea02c5e16455bc9808a59c6b2f0"}, - {file = "rpds_py-0.18.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:ce0bb20e3a11bd04461324a6a798af34d503f8d6f1aa3d2aa8901ceaf039176d"}, - {file = "rpds_py-0.18.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:81c5196a790032e0fc2464c0b4ab95f8610f96f1f2fa3d4deacce6a79852da60"}, - {file = "rpds_py-0.18.1-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:f3027be483868c99b4985fda802a57a67fdf30c5d9a50338d9db646d590198da"}, - {file = "rpds_py-0.18.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:d44607f98caa2961bab4fa3c4309724b185b464cdc3ba6f3d7340bac3ec97cc1"}, - {file = "rpds_py-0.18.1-cp310-none-win32.whl", hash = "sha256:c273e795e7a0f1fddd46e1e3cb8be15634c29ae8ff31c196debb620e1edb9333"}, - {file = "rpds_py-0.18.1-cp310-none-win_amd64.whl", hash = "sha256:8352f48d511de5f973e4f2f9412736d7dea76c69faa6d36bcf885b50c758ab9a"}, - {file = "rpds_py-0.18.1-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:6b5ff7e1d63a8281654b5e2896d7f08799378e594f09cf3674e832ecaf396ce8"}, - {file = "rpds_py-0.18.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8927638a4d4137a289e41d0fd631551e89fa346d6dbcfc31ad627557d03ceb6d"}, - {file = "rpds_py-0.18.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:154bf5c93d79558b44e5b50cc354aa0459e518e83677791e6adb0b039b7aa6a7"}, - {file = "rpds_py-0.18.1-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:07f2139741e5deb2c5154a7b9629bc5aa48c766b643c1a6750d16f865a82c5fc"}, - {file = "rpds_py-0.18.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8c7672e9fba7425f79019db9945b16e308ed8bc89348c23d955c8c0540da0a07"}, - {file = "rpds_py-0.18.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:489bdfe1abd0406eba6b3bb4fdc87c7fa40f1031de073d0cfb744634cc8fa261"}, - {file = "rpds_py-0.18.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3c20f05e8e3d4fc76875fc9cb8cf24b90a63f5a1b4c5b9273f0e8225e169b100"}, - {file = "rpds_py-0.18.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:967342e045564cef76dfcf1edb700b1e20838d83b1aa02ab313e6a497cf923b8"}, - {file = "rpds_py-0.18.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:2cc7c1a47f3a63282ab0f422d90ddac4aa3034e39fc66a559ab93041e6505da7"}, - {file = "rpds_py-0.18.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:f7afbfee1157e0f9376c00bb232e80a60e59ed716e3211a80cb8506550671e6e"}, - {file = "rpds_py-0.18.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:9e6934d70dc50f9f8ea47081ceafdec09245fd9f6032669c3b45705dea096b88"}, - {file = "rpds_py-0.18.1-cp311-none-win32.whl", hash = "sha256:c69882964516dc143083d3795cb508e806b09fc3800fd0d4cddc1df6c36e76bb"}, - {file = "rpds_py-0.18.1-cp311-none-win_amd64.whl", hash = "sha256:70a838f7754483bcdc830444952fd89645569e7452e3226de4a613a4c1793fb2"}, - {file = "rpds_py-0.18.1-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:3dd3cd86e1db5aadd334e011eba4e29d37a104b403e8ca24dcd6703c68ca55b3"}, - {file = "rpds_py-0.18.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:05f3d615099bd9b13ecf2fc9cf2d839ad3f20239c678f461c753e93755d629ee"}, - {file = "rpds_py-0.18.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:35b2b771b13eee8729a5049c976197ff58a27a3829c018a04341bcf1ae409b2b"}, - {file = "rpds_py-0.18.1-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ee17cd26b97d537af8f33635ef38be873073d516fd425e80559f4585a7b90c43"}, - {file = "rpds_py-0.18.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b646bf655b135ccf4522ed43d6902af37d3f5dbcf0da66c769a2b3938b9d8184"}, - {file = "rpds_py-0.18.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:19ba472b9606c36716062c023afa2484d1e4220548751bda14f725a7de17b4f6"}, - {file = "rpds_py-0.18.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6e30ac5e329098903262dc5bdd7e2086e0256aa762cc8b744f9e7bf2a427d3f8"}, - {file = "rpds_py-0.18.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:d58ad6317d188c43750cb76e9deacf6051d0f884d87dc6518e0280438648a9ac"}, - {file = "rpds_py-0.18.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:e1735502458621921cee039c47318cb90b51d532c2766593be6207eec53e5c4c"}, - {file = "rpds_py-0.18.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:f5bab211605d91db0e2995a17b5c6ee5edec1270e46223e513eaa20da20076ac"}, - {file = "rpds_py-0.18.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:2fc24a329a717f9e2448f8cd1f960f9dac4e45b6224d60734edeb67499bab03a"}, - {file = "rpds_py-0.18.1-cp312-none-win32.whl", hash = "sha256:1805d5901779662d599d0e2e4159d8a82c0b05faa86ef9222bf974572286b2b6"}, - {file = "rpds_py-0.18.1-cp312-none-win_amd64.whl", hash = "sha256:720edcb916df872d80f80a1cc5ea9058300b97721efda8651efcd938a9c70a72"}, - {file = "rpds_py-0.18.1-cp38-cp38-macosx_10_12_x86_64.whl", hash = "sha256:c827576e2fa017a081346dce87d532a5310241648eb3700af9a571a6e9fc7e74"}, - {file = "rpds_py-0.18.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:aa3679e751408d75a0b4d8d26d6647b6d9326f5e35c00a7ccd82b78ef64f65f8"}, - {file = "rpds_py-0.18.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0abeee75434e2ee2d142d650d1e54ac1f8b01e6e6abdde8ffd6eeac6e9c38e20"}, - {file = "rpds_py-0.18.1-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ed402d6153c5d519a0faf1bb69898e97fb31613b49da27a84a13935ea9164dfc"}, - {file = "rpds_py-0.18.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:338dee44b0cef8b70fd2ef54b4e09bb1b97fc6c3a58fea5db6cc083fd9fc2724"}, - {file = "rpds_py-0.18.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7750569d9526199c5b97e5a9f8d96a13300950d910cf04a861d96f4273d5b104"}, - {file = "rpds_py-0.18.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:607345bd5912aacc0c5a63d45a1f73fef29e697884f7e861094e443187c02be5"}, - {file = "rpds_py-0.18.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:207c82978115baa1fd8d706d720b4a4d2b0913df1c78c85ba73fe6c5804505f0"}, - {file = "rpds_py-0.18.1-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:6d1e42d2735d437e7e80bab4d78eb2e459af48c0a46e686ea35f690b93db792d"}, - {file = "rpds_py-0.18.1-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:5463c47c08630007dc0fe99fb480ea4f34a89712410592380425a9b4e1611d8e"}, - {file = "rpds_py-0.18.1-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:06d218939e1bf2ca50e6b0ec700ffe755e5216a8230ab3e87c059ebb4ea06afc"}, - {file = "rpds_py-0.18.1-cp38-none-win32.whl", hash = "sha256:312fe69b4fe1ffbe76520a7676b1e5ac06ddf7826d764cc10265c3b53f96dbe9"}, - {file = "rpds_py-0.18.1-cp38-none-win_amd64.whl", hash = "sha256:9437ca26784120a279f3137ee080b0e717012c42921eb07861b412340f85bae2"}, - {file = "rpds_py-0.18.1-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:19e515b78c3fc1039dd7da0a33c28c3154458f947f4dc198d3c72db2b6b5dc93"}, - {file = "rpds_py-0.18.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:a7b28c5b066bca9a4eb4e2f2663012debe680f097979d880657f00e1c30875a0"}, - {file = "rpds_py-0.18.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:673fdbbf668dd958eff750e500495ef3f611e2ecc209464f661bc82e9838991e"}, - {file = "rpds_py-0.18.1-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:d960de62227635d2e61068f42a6cb6aae91a7fe00fca0e3aeed17667c8a34611"}, - {file = "rpds_py-0.18.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:352a88dc7892f1da66b6027af06a2e7e5d53fe05924cc2cfc56495b586a10b72"}, - {file = "rpds_py-0.18.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4e0ee01ad8260184db21468a6e1c37afa0529acc12c3a697ee498d3c2c4dcaf3"}, - {file = "rpds_py-0.18.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e4c39ad2f512b4041343ea3c7894339e4ca7839ac38ca83d68a832fc8b3748ab"}, - {file = "rpds_py-0.18.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:aaa71ee43a703c321906813bb252f69524f02aa05bf4eec85f0c41d5d62d0f4c"}, - {file = "rpds_py-0.18.1-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:6cd8098517c64a85e790657e7b1e509b9fe07487fd358e19431cb120f7d96338"}, - {file = "rpds_py-0.18.1-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:4adec039b8e2928983f885c53b7cc4cda8965b62b6596501a0308d2703f8af1b"}, - {file = "rpds_py-0.18.1-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:32b7daaa3e9389db3695964ce8e566e3413b0c43e3394c05e4b243a4cd7bef26"}, - {file = "rpds_py-0.18.1-cp39-none-win32.whl", hash = "sha256:2625f03b105328729f9450c8badda34d5243231eef6535f80064d57035738360"}, - {file = "rpds_py-0.18.1-cp39-none-win_amd64.whl", hash = "sha256:bf18932d0003c8c4d51a39f244231986ab23ee057d235a12b2684ea26a353590"}, - {file = "rpds_py-0.18.1-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:cbfbea39ba64f5e53ae2915de36f130588bba71245b418060ec3330ebf85678e"}, - {file = "rpds_py-0.18.1-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:a3d456ff2a6a4d2adcdf3c1c960a36f4fd2fec6e3b4902a42a384d17cf4e7a65"}, - {file = "rpds_py-0.18.1-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7700936ef9d006b7ef605dc53aa364da2de5a3aa65516a1f3ce73bf82ecfc7ae"}, - {file = "rpds_py-0.18.1-pp310-pypy310_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:51584acc5916212e1bf45edd17f3a6b05fe0cbb40482d25e619f824dccb679de"}, - {file = "rpds_py-0.18.1-pp310-pypy310_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:942695a206a58d2575033ff1e42b12b2aece98d6003c6bc739fbf33d1773b12f"}, - {file = "rpds_py-0.18.1-pp310-pypy310_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b906b5f58892813e5ba5c6056d6a5ad08f358ba49f046d910ad992196ea61397"}, - {file = "rpds_py-0.18.1-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f6f8e3fecca256fefc91bb6765a693d96692459d7d4c644660a9fff32e517843"}, - {file = "rpds_py-0.18.1-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:7732770412bab81c5a9f6d20aeb60ae943a9b36dcd990d876a773526468e7163"}, - {file = "rpds_py-0.18.1-pp310-pypy310_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:bd1105b50ede37461c1d51b9698c4f4be6e13e69a908ab7751e3807985fc0346"}, - {file = "rpds_py-0.18.1-pp310-pypy310_pp73-musllinux_1_2_i686.whl", hash = "sha256:618916f5535784960f3ecf8111581f4ad31d347c3de66d02e728de460a46303c"}, - {file = "rpds_py-0.18.1-pp310-pypy310_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:17c6d2155e2423f7e79e3bb18151c686d40db42d8645e7977442170c360194d4"}, - {file = "rpds_py-0.18.1-pp38-pypy38_pp73-macosx_10_12_x86_64.whl", hash = "sha256:6c4c4c3f878df21faf5fac86eda32671c27889e13570645a9eea0a1abdd50922"}, - {file = "rpds_py-0.18.1-pp38-pypy38_pp73-macosx_11_0_arm64.whl", hash = "sha256:fab6ce90574645a0d6c58890e9bcaac8d94dff54fb51c69e5522a7358b80ab64"}, - {file = "rpds_py-0.18.1-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:531796fb842b53f2695e94dc338929e9f9dbf473b64710c28af5a160b2a8927d"}, - {file = "rpds_py-0.18.1-pp38-pypy38_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:740884bc62a5e2bbb31e584f5d23b32320fd75d79f916f15a788d527a5e83644"}, - {file = "rpds_py-0.18.1-pp38-pypy38_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:998125738de0158f088aef3cb264a34251908dd2e5d9966774fdab7402edfab7"}, - {file = "rpds_py-0.18.1-pp38-pypy38_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e2be6e9dd4111d5b31ba3b74d17da54a8319d8168890fbaea4b9e5c3de630ae5"}, - {file = "rpds_py-0.18.1-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d0cee71bc618cd93716f3c1bf56653740d2d13ddbd47673efa8bf41435a60daa"}, - {file = "rpds_py-0.18.1-pp38-pypy38_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:2c3caec4ec5cd1d18e5dd6ae5194d24ed12785212a90b37f5f7f06b8bedd7139"}, - {file = "rpds_py-0.18.1-pp38-pypy38_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:27bba383e8c5231cd559affe169ca0b96ec78d39909ffd817f28b166d7ddd4d8"}, - {file = "rpds_py-0.18.1-pp38-pypy38_pp73-musllinux_1_2_i686.whl", hash = "sha256:a888e8bdb45916234b99da2d859566f1e8a1d2275a801bb8e4a9644e3c7e7909"}, - {file = "rpds_py-0.18.1-pp38-pypy38_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:6031b25fb1b06327b43d841f33842b383beba399884f8228a6bb3df3088485ff"}, - {file = "rpds_py-0.18.1-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:48c2faaa8adfacefcbfdb5f2e2e7bdad081e5ace8d182e5f4ade971f128e6bb3"}, - {file = "rpds_py-0.18.1-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:d85164315bd68c0806768dc6bb0429c6f95c354f87485ee3593c4f6b14def2bd"}, - {file = "rpds_py-0.18.1-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6afd80f6c79893cfc0574956f78a0add8c76e3696f2d6a15bca2c66c415cf2d4"}, - {file = "rpds_py-0.18.1-pp39-pypy39_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:fa242ac1ff583e4ec7771141606aafc92b361cd90a05c30d93e343a0c2d82a89"}, - {file = "rpds_py-0.18.1-pp39-pypy39_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d21be4770ff4e08698e1e8e0bce06edb6ea0626e7c8f560bc08222880aca6a6f"}, - {file = "rpds_py-0.18.1-pp39-pypy39_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5c45a639e93a0c5d4b788b2613bd637468edd62f8f95ebc6fcc303d58ab3f0a8"}, - {file = "rpds_py-0.18.1-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:910e71711d1055b2768181efa0a17537b2622afeb0424116619817007f8a2b10"}, - {file = "rpds_py-0.18.1-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:b9bb1f182a97880f6078283b3505a707057c42bf55d8fca604f70dedfdc0772a"}, - {file = "rpds_py-0.18.1-pp39-pypy39_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:1d54f74f40b1f7aaa595a02ff42ef38ca654b1469bef7d52867da474243cc633"}, - {file = "rpds_py-0.18.1-pp39-pypy39_pp73-musllinux_1_2_i686.whl", hash = "sha256:8d2e182c9ee01135e11e9676e9a62dfad791a7a467738f06726872374a83db49"}, - {file = "rpds_py-0.18.1-pp39-pypy39_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:636a15acc588f70fda1661234761f9ed9ad79ebed3f2125d44be0862708b666e"}, - {file = "rpds_py-0.18.1.tar.gz", hash = "sha256:dc48b479d540770c811fbd1eb9ba2bb66951863e448efec2e2c102625328e92f"}, + {file = "rpds_py-0.20.0-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:3ad0fda1635f8439cde85c700f964b23ed5fc2d28016b32b9ee5fe30da5c84e2"}, + {file = "rpds_py-0.20.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9bb4a0d90fdb03437c109a17eade42dfbf6190408f29b2744114d11586611d6f"}, + {file = "rpds_py-0.20.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c6377e647bbfd0a0b159fe557f2c6c602c159fc752fa316572f012fc0bf67150"}, + {file = "rpds_py-0.20.0-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:eb851b7df9dda52dc1415ebee12362047ce771fc36914586b2e9fcbd7d293b3e"}, + {file = "rpds_py-0.20.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1e0f80b739e5a8f54837be5d5c924483996b603d5502bfff79bf33da06164ee2"}, + {file = "rpds_py-0.20.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5a8c94dad2e45324fc74dce25e1645d4d14df9a4e54a30fa0ae8bad9a63928e3"}, + {file = "rpds_py-0.20.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f8e604fe73ba048c06085beaf51147eaec7df856824bfe7b98657cf436623daf"}, + {file = "rpds_py-0.20.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:df3de6b7726b52966edf29663e57306b23ef775faf0ac01a3e9f4012a24a4140"}, + {file = "rpds_py-0.20.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:cf258ede5bc22a45c8e726b29835b9303c285ab46fc7c3a4cc770736b5304c9f"}, + {file = "rpds_py-0.20.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:55fea87029cded5df854ca7e192ec7bdb7ecd1d9a3f63d5c4eb09148acf4a7ce"}, + {file = "rpds_py-0.20.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:ae94bd0b2f02c28e199e9bc51485d0c5601f58780636185660f86bf80c89af94"}, + {file = "rpds_py-0.20.0-cp310-none-win32.whl", hash = "sha256:28527c685f237c05445efec62426d285e47a58fb05ba0090a4340b73ecda6dee"}, + {file = "rpds_py-0.20.0-cp310-none-win_amd64.whl", hash = "sha256:238a2d5b1cad28cdc6ed15faf93a998336eb041c4e440dd7f902528b8891b399"}, + {file = "rpds_py-0.20.0-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:ac2f4f7a98934c2ed6505aead07b979e6f999389f16b714448fb39bbaa86a489"}, + {file = "rpds_py-0.20.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:220002c1b846db9afd83371d08d239fdc865e8f8c5795bbaec20916a76db3318"}, + {file = "rpds_py-0.20.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8d7919548df3f25374a1f5d01fbcd38dacab338ef5f33e044744b5c36729c8db"}, + {file = "rpds_py-0.20.0-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:758406267907b3781beee0f0edfe4a179fbd97c0be2e9b1154d7f0a1279cf8e5"}, + {file = "rpds_py-0.20.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3d61339e9f84a3f0767b1995adfb171a0d00a1185192718a17af6e124728e0f5"}, + {file = "rpds_py-0.20.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1259c7b3705ac0a0bd38197565a5d603218591d3f6cee6e614e380b6ba61c6f6"}, + {file = "rpds_py-0.20.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5c1dc0f53856b9cc9a0ccca0a7cc61d3d20a7088201c0937f3f4048c1718a209"}, + {file = "rpds_py-0.20.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:7e60cb630f674a31f0368ed32b2a6b4331b8350d67de53c0359992444b116dd3"}, + {file = "rpds_py-0.20.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:dbe982f38565bb50cb7fb061ebf762c2f254ca3d8c20d4006878766e84266272"}, + {file = "rpds_py-0.20.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:514b3293b64187172bc77c8fb0cdae26981618021053b30d8371c3a902d4d5ad"}, + {file = "rpds_py-0.20.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:d0a26ffe9d4dd35e4dfdd1e71f46401cff0181c75ac174711ccff0459135fa58"}, + {file = "rpds_py-0.20.0-cp311-none-win32.whl", hash = "sha256:89c19a494bf3ad08c1da49445cc5d13d8fefc265f48ee7e7556839acdacf69d0"}, + {file = "rpds_py-0.20.0-cp311-none-win_amd64.whl", hash = "sha256:c638144ce971df84650d3ed0096e2ae7af8e62ecbbb7b201c8935c370df00a2c"}, + {file = "rpds_py-0.20.0-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:a84ab91cbe7aab97f7446652d0ed37d35b68a465aeef8fc41932a9d7eee2c1a6"}, + {file = "rpds_py-0.20.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:56e27147a5a4c2c21633ff8475d185734c0e4befd1c989b5b95a5d0db699b21b"}, + {file = "rpds_py-0.20.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2580b0c34583b85efec8c5c5ec9edf2dfe817330cc882ee972ae650e7b5ef739"}, + {file = "rpds_py-0.20.0-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:b80d4a7900cf6b66bb9cee5c352b2d708e29e5a37fe9bf784fa97fc11504bf6c"}, + {file = "rpds_py-0.20.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:50eccbf054e62a7b2209b28dc7a22d6254860209d6753e6b78cfaeb0075d7bee"}, + {file = "rpds_py-0.20.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:49a8063ea4296b3a7e81a5dfb8f7b2d73f0b1c20c2af401fb0cdf22e14711a96"}, + {file = "rpds_py-0.20.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ea438162a9fcbee3ecf36c23e6c68237479f89f962f82dae83dc15feeceb37e4"}, + {file = "rpds_py-0.20.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:18d7585c463087bddcfa74c2ba267339f14f2515158ac4db30b1f9cbdb62c8ef"}, + {file = "rpds_py-0.20.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:d4c7d1a051eeb39f5c9547e82ea27cbcc28338482242e3e0b7768033cb083821"}, + {file = "rpds_py-0.20.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:e4df1e3b3bec320790f699890d41c59d250f6beda159ea3c44c3f5bac1976940"}, + {file = "rpds_py-0.20.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:2cf126d33a91ee6eedc7f3197b53e87a2acdac63602c0f03a02dd69e4b138174"}, + {file = "rpds_py-0.20.0-cp312-none-win32.whl", hash = "sha256:8bc7690f7caee50b04a79bf017a8d020c1f48c2a1077ffe172abec59870f1139"}, + {file = "rpds_py-0.20.0-cp312-none-win_amd64.whl", hash = "sha256:0e13e6952ef264c40587d510ad676a988df19adea20444c2b295e536457bc585"}, + {file = "rpds_py-0.20.0-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:aa9a0521aeca7d4941499a73ad7d4f8ffa3d1affc50b9ea11d992cd7eff18a29"}, + {file = "rpds_py-0.20.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:4a1f1d51eccb7e6c32ae89243cb352389228ea62f89cd80823ea7dd1b98e0b91"}, + {file = "rpds_py-0.20.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8a86a9b96070674fc88b6f9f71a97d2c1d3e5165574615d1f9168ecba4cecb24"}, + {file = "rpds_py-0.20.0-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:6c8ef2ebf76df43f5750b46851ed1cdf8f109d7787ca40035fe19fbdc1acc5a7"}, + {file = "rpds_py-0.20.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b74b25f024b421d5859d156750ea9a65651793d51b76a2e9238c05c9d5f203a9"}, + {file = "rpds_py-0.20.0-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:57eb94a8c16ab08fef6404301c38318e2c5a32216bf5de453e2714c964c125c8"}, + {file = "rpds_py-0.20.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e1940dae14e715e2e02dfd5b0f64a52e8374a517a1e531ad9412319dc3ac7879"}, + {file = "rpds_py-0.20.0-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:d20277fd62e1b992a50c43f13fbe13277a31f8c9f70d59759c88f644d66c619f"}, + {file = "rpds_py-0.20.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:06db23d43f26478303e954c34c75182356ca9aa7797d22c5345b16871ab9c45c"}, + {file = "rpds_py-0.20.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:b2a5db5397d82fa847e4c624b0c98fe59d2d9b7cf0ce6de09e4d2e80f8f5b3f2"}, + {file = "rpds_py-0.20.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:5a35df9f5548fd79cb2f52d27182108c3e6641a4feb0f39067911bf2adaa3e57"}, + {file = "rpds_py-0.20.0-cp313-none-win32.whl", hash = "sha256:fd2d84f40633bc475ef2d5490b9c19543fbf18596dcb1b291e3a12ea5d722f7a"}, + {file = "rpds_py-0.20.0-cp313-none-win_amd64.whl", hash = "sha256:9bc2d153989e3216b0559251b0c260cfd168ec78b1fac33dd485750a228db5a2"}, + {file = "rpds_py-0.20.0-cp38-cp38-macosx_10_12_x86_64.whl", hash = "sha256:f2fbf7db2012d4876fb0d66b5b9ba6591197b0f165db8d99371d976546472a24"}, + {file = "rpds_py-0.20.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:1e5f3cd7397c8f86c8cc72d5a791071431c108edd79872cdd96e00abd8497d29"}, + {file = "rpds_py-0.20.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ce9845054c13696f7af7f2b353e6b4f676dab1b4b215d7fe5e05c6f8bb06f965"}, + {file = "rpds_py-0.20.0-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:c3e130fd0ec56cb76eb49ef52faead8ff09d13f4527e9b0c400307ff72b408e1"}, + {file = "rpds_py-0.20.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4b16aa0107ecb512b568244ef461f27697164d9a68d8b35090e9b0c1c8b27752"}, + {file = "rpds_py-0.20.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:aa7f429242aae2947246587d2964fad750b79e8c233a2367f71b554e9447949c"}, + {file = "rpds_py-0.20.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:af0fc424a5842a11e28956e69395fbbeab2c97c42253169d87e90aac2886d751"}, + {file = "rpds_py-0.20.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:b8c00a3b1e70c1d3891f0db1b05292747f0dbcfb49c43f9244d04c70fbc40eb8"}, + {file = "rpds_py-0.20.0-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:40ce74fc86ee4645d0a225498d091d8bc61f39b709ebef8204cb8b5a464d3c0e"}, + {file = "rpds_py-0.20.0-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:4fe84294c7019456e56d93e8ababdad5a329cd25975be749c3f5f558abb48253"}, + {file = "rpds_py-0.20.0-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:338ca4539aad4ce70a656e5187a3a31c5204f261aef9f6ab50e50bcdffaf050a"}, + {file = "rpds_py-0.20.0-cp38-none-win32.whl", hash = "sha256:54b43a2b07db18314669092bb2de584524d1ef414588780261e31e85846c26a5"}, + {file = "rpds_py-0.20.0-cp38-none-win_amd64.whl", hash = "sha256:a1862d2d7ce1674cffa6d186d53ca95c6e17ed2b06b3f4c476173565c862d232"}, + {file = "rpds_py-0.20.0-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:3fde368e9140312b6e8b6c09fb9f8c8c2f00999d1823403ae90cc00480221b22"}, + {file = "rpds_py-0.20.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:9824fb430c9cf9af743cf7aaf6707bf14323fb51ee74425c380f4c846ea70789"}, + {file = "rpds_py-0.20.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:11ef6ce74616342888b69878d45e9f779b95d4bd48b382a229fe624a409b72c5"}, + {file = "rpds_py-0.20.0-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:c52d3f2f82b763a24ef52f5d24358553e8403ce05f893b5347098014f2d9eff2"}, + {file = "rpds_py-0.20.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9d35cef91e59ebbeaa45214861874bc6f19eb35de96db73e467a8358d701a96c"}, + {file = "rpds_py-0.20.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d72278a30111e5b5525c1dd96120d9e958464316f55adb030433ea905866f4de"}, + {file = "rpds_py-0.20.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b4c29cbbba378759ac5786730d1c3cb4ec6f8ababf5c42a9ce303dc4b3d08cda"}, + {file = "rpds_py-0.20.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:6632f2d04f15d1bd6fe0eedd3b86d9061b836ddca4c03d5cf5c7e9e6b7c14580"}, + {file = "rpds_py-0.20.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:d0b67d87bb45ed1cd020e8fbf2307d449b68abc45402fe1a4ac9e46c3c8b192b"}, + {file = "rpds_py-0.20.0-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:ec31a99ca63bf3cd7f1a5ac9fe95c5e2d060d3c768a09bc1d16e235840861420"}, + {file = "rpds_py-0.20.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:22e6c9976e38f4d8c4a63bd8a8edac5307dffd3ee7e6026d97f3cc3a2dc02a0b"}, + {file = "rpds_py-0.20.0-cp39-none-win32.whl", hash = "sha256:569b3ea770c2717b730b61998b6c54996adee3cef69fc28d444f3e7920313cf7"}, + {file = "rpds_py-0.20.0-cp39-none-win_amd64.whl", hash = "sha256:e6900ecdd50ce0facf703f7a00df12374b74bbc8ad9fe0f6559947fb20f82364"}, + {file = "rpds_py-0.20.0-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:617c7357272c67696fd052811e352ac54ed1d9b49ab370261a80d3b6ce385045"}, + {file = "rpds_py-0.20.0-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:9426133526f69fcaba6e42146b4e12d6bc6c839b8b555097020e2b78ce908dcc"}, + {file = "rpds_py-0.20.0-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:deb62214c42a261cb3eb04d474f7155279c1a8a8c30ac89b7dcb1721d92c3c02"}, + {file = "rpds_py-0.20.0-pp310-pypy310_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:fcaeb7b57f1a1e071ebd748984359fef83ecb026325b9d4ca847c95bc7311c92"}, + {file = "rpds_py-0.20.0-pp310-pypy310_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d454b8749b4bd70dd0a79f428731ee263fa6995f83ccb8bada706e8d1d3ff89d"}, + {file = "rpds_py-0.20.0-pp310-pypy310_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d807dc2051abe041b6649681dce568f8e10668e3c1c6543ebae58f2d7e617855"}, + {file = "rpds_py-0.20.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c3c20f0ddeb6e29126d45f89206b8291352b8c5b44384e78a6499d68b52ae511"}, + {file = "rpds_py-0.20.0-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:b7f19250ceef892adf27f0399b9e5afad019288e9be756d6919cb58892129f51"}, + {file = "rpds_py-0.20.0-pp310-pypy310_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:4f1ed4749a08379555cebf4650453f14452eaa9c43d0a95c49db50c18b7da075"}, + {file = "rpds_py-0.20.0-pp310-pypy310_pp73-musllinux_1_2_i686.whl", hash = "sha256:dcedf0b42bcb4cfff4101d7771a10532415a6106062f005ab97d1d0ab5681c60"}, + {file = "rpds_py-0.20.0-pp310-pypy310_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:39ed0d010457a78f54090fafb5d108501b5aa5604cc22408fc1c0c77eac14344"}, + {file = "rpds_py-0.20.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:bb273176be34a746bdac0b0d7e4e2c467323d13640b736c4c477881a3220a989"}, + {file = "rpds_py-0.20.0-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:f918a1a130a6dfe1d7fe0f105064141342e7dd1611f2e6a21cd2f5c8cb1cfb3e"}, + {file = "rpds_py-0.20.0-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:f60012a73aa396be721558caa3a6fd49b3dd0033d1675c6d59c4502e870fcf0c"}, + {file = "rpds_py-0.20.0-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3d2b1ad682a3dfda2a4e8ad8572f3100f95fad98cb99faf37ff0ddfe9cbf9d03"}, + {file = "rpds_py-0.20.0-pp39-pypy39_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:614fdafe9f5f19c63ea02817fa4861c606a59a604a77c8cdef5aa01d28b97921"}, + {file = "rpds_py-0.20.0-pp39-pypy39_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:fa518bcd7600c584bf42e6617ee8132869e877db2f76bcdc281ec6a4113a53ab"}, + {file = "rpds_py-0.20.0-pp39-pypy39_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f0475242f447cc6cb8a9dd486d68b2ef7fbee84427124c232bff5f63b1fe11e5"}, + {file = "rpds_py-0.20.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f90a4cd061914a60bd51c68bcb4357086991bd0bb93d8aa66a6da7701370708f"}, + {file = "rpds_py-0.20.0-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:def7400461c3a3f26e49078302e1c1b38f6752342c77e3cf72ce91ca69fb1bc1"}, + {file = "rpds_py-0.20.0-pp39-pypy39_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:65794e4048ee837494aea3c21a28ad5fc080994dfba5b036cf84de37f7ad5074"}, + {file = "rpds_py-0.20.0-pp39-pypy39_pp73-musllinux_1_2_i686.whl", hash = "sha256:faefcc78f53a88f3076b7f8be0a8f8d35133a3ecf7f3770895c25f8813460f08"}, + {file = "rpds_py-0.20.0-pp39-pypy39_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:5b4f105deeffa28bbcdff6c49b34e74903139afa690e35d2d9e3c2c2fba18cec"}, + {file = "rpds_py-0.20.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:fdfc3a892927458d98f3d55428ae46b921d1f7543b89382fdb483f5640daaec8"}, + {file = "rpds_py-0.20.0.tar.gz", hash = "sha256:d72a210824facfdaf8768cf2d7ca25a042c30320b3020de2fa04640920d4e121"}, ] [[package]] @@ -3973,36 +4058,36 @@ synapse = ["synapseclient (>=4.0.0,<5.0.0)"] [[package]] name = "scipy" -version = "1.13.0" +version = "1.13.1" description = "Fundamental algorithms for scientific computing in Python" optional = false python-versions = ">=3.9" files = [ - {file = "scipy-1.13.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ba419578ab343a4e0a77c0ef82f088238a93eef141b2b8017e46149776dfad4d"}, - {file = "scipy-1.13.0-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:22789b56a999265431c417d462e5b7f2b487e831ca7bef5edeb56efe4c93f86e"}, - {file = "scipy-1.13.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:05f1432ba070e90d42d7fd836462c50bf98bd08bed0aa616c359eed8a04e3922"}, - {file = "scipy-1.13.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b8434f6f3fa49f631fae84afee424e2483289dfc30a47755b4b4e6b07b2633a4"}, - {file = "scipy-1.13.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:dcbb9ea49b0167de4167c40eeee6e167caeef11effb0670b554d10b1e693a8b9"}, - {file = "scipy-1.13.0-cp310-cp310-win_amd64.whl", hash = "sha256:1d2f7bb14c178f8b13ebae93f67e42b0a6b0fc50eba1cd8021c9b6e08e8fb1cd"}, - {file = "scipy-1.13.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:0fbcf8abaf5aa2dc8d6400566c1a727aed338b5fe880cde64907596a89d576fa"}, - {file = "scipy-1.13.0-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:5e4a756355522eb60fcd61f8372ac2549073c8788f6114449b37e9e8104f15a5"}, - {file = "scipy-1.13.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b5acd8e1dbd8dbe38d0004b1497019b2dbbc3d70691e65d69615f8a7292865d7"}, - {file = "scipy-1.13.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9ff7dad5d24a8045d836671e082a490848e8639cabb3dbdacb29f943a678683d"}, - {file = "scipy-1.13.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:4dca18c3ffee287ddd3bc8f1dabaf45f5305c5afc9f8ab9cbfab855e70b2df5c"}, - {file = "scipy-1.13.0-cp311-cp311-win_amd64.whl", hash = "sha256:a2f471de4d01200718b2b8927f7d76b5d9bde18047ea0fa8bd15c5ba3f26a1d6"}, - {file = "scipy-1.13.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:d0de696f589681c2802f9090fff730c218f7c51ff49bf252b6a97ec4a5d19e8b"}, - {file = "scipy-1.13.0-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:b2a3ff461ec4756b7e8e42e1c681077349a038f0686132d623fa404c0bee2551"}, - {file = "scipy-1.13.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6bf9fe63e7a4bf01d3645b13ff2aa6dea023d38993f42aaac81a18b1bda7a82a"}, - {file = "scipy-1.13.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1e7626dfd91cdea5714f343ce1176b6c4745155d234f1033584154f60ef1ff42"}, - {file = "scipy-1.13.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:109d391d720fcebf2fbe008621952b08e52907cf4c8c7efc7376822151820820"}, - {file = "scipy-1.13.0-cp312-cp312-win_amd64.whl", hash = "sha256:8930ae3ea371d6b91c203b1032b9600d69c568e537b7988a3073dfe4d4774f21"}, - {file = "scipy-1.13.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5407708195cb38d70fd2d6bb04b1b9dd5c92297d86e9f9daae1576bd9e06f602"}, - {file = "scipy-1.13.0-cp39-cp39-macosx_12_0_arm64.whl", hash = "sha256:ac38c4c92951ac0f729c4c48c9e13eb3675d9986cc0c83943784d7390d540c78"}, - {file = "scipy-1.13.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:09c74543c4fbeb67af6ce457f6a6a28e5d3739a87f62412e4a16e46f164f0ae5"}, - {file = "scipy-1.13.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:28e286bf9ac422d6beb559bc61312c348ca9b0f0dae0d7c5afde7f722d6ea13d"}, - {file = "scipy-1.13.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:33fde20efc380bd23a78a4d26d59fc8704e9b5fd9b08841693eb46716ba13d86"}, - {file = "scipy-1.13.0-cp39-cp39-win_amd64.whl", hash = "sha256:45c08bec71d3546d606989ba6e7daa6f0992918171e2a6f7fbedfa7361c2de1e"}, - {file = "scipy-1.13.0.tar.gz", hash = "sha256:58569af537ea29d3f78e5abd18398459f195546bb3be23d16677fb26616cc11e"}, + {file = "scipy-1.13.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:20335853b85e9a49ff7572ab453794298bcf0354d8068c5f6775a0eabf350aca"}, + {file = "scipy-1.13.1-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:d605e9c23906d1994f55ace80e0125c587f96c020037ea6aa98d01b4bd2e222f"}, + {file = "scipy-1.13.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cfa31f1def5c819b19ecc3a8b52d28ffdcc7ed52bb20c9a7589669dd3c250989"}, + {file = "scipy-1.13.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f26264b282b9da0952a024ae34710c2aff7d27480ee91a2e82b7b7073c24722f"}, + {file = "scipy-1.13.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:eccfa1906eacc02de42d70ef4aecea45415f5be17e72b61bafcfd329bdc52e94"}, + {file = "scipy-1.13.1-cp310-cp310-win_amd64.whl", hash = "sha256:2831f0dc9c5ea9edd6e51e6e769b655f08ec6db6e2e10f86ef39bd32eb11da54"}, + {file = "scipy-1.13.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:27e52b09c0d3a1d5b63e1105f24177e544a222b43611aaf5bc44d4a0979e32f9"}, + {file = "scipy-1.13.1-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:54f430b00f0133e2224c3ba42b805bfd0086fe488835effa33fa291561932326"}, + {file = "scipy-1.13.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e89369d27f9e7b0884ae559a3a956e77c02114cc60a6058b4e5011572eea9299"}, + {file = "scipy-1.13.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a78b4b3345f1b6f68a763c6e25c0c9a23a9fd0f39f5f3d200efe8feda560a5fa"}, + {file = "scipy-1.13.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:45484bee6d65633752c490404513b9ef02475b4284c4cfab0ef946def50b3f59"}, + {file = "scipy-1.13.1-cp311-cp311-win_amd64.whl", hash = "sha256:5713f62f781eebd8d597eb3f88b8bf9274e79eeabf63afb4a737abc6c84ad37b"}, + {file = "scipy-1.13.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:5d72782f39716b2b3509cd7c33cdc08c96f2f4d2b06d51e52fb45a19ca0c86a1"}, + {file = "scipy-1.13.1-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:017367484ce5498445aade74b1d5ab377acdc65e27095155e448c88497755a5d"}, + {file = "scipy-1.13.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:949ae67db5fa78a86e8fa644b9a6b07252f449dcf74247108c50e1d20d2b4627"}, + {file = "scipy-1.13.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:de3ade0e53bc1f21358aa74ff4830235d716211d7d077e340c7349bc3542e884"}, + {file = "scipy-1.13.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:2ac65fb503dad64218c228e2dc2d0a0193f7904747db43014645ae139c8fad16"}, + {file = "scipy-1.13.1-cp312-cp312-win_amd64.whl", hash = "sha256:cdd7dacfb95fea358916410ec61bbc20440f7860333aee6d882bb8046264e949"}, + {file = "scipy-1.13.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:436bbb42a94a8aeef855d755ce5a465479c721e9d684de76bf61a62e7c2b81d5"}, + {file = "scipy-1.13.1-cp39-cp39-macosx_12_0_arm64.whl", hash = "sha256:8335549ebbca860c52bf3d02f80784e91a004b71b059e3eea9678ba994796a24"}, + {file = "scipy-1.13.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d533654b7d221a6a97304ab63c41c96473ff04459e404b83275b60aa8f4b7004"}, + {file = "scipy-1.13.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:637e98dcf185ba7f8e663e122ebf908c4702420477ae52a04f9908707456ba4d"}, + {file = "scipy-1.13.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:a014c2b3697bde71724244f63de2476925596c24285c7a637364761f8710891c"}, + {file = "scipy-1.13.1-cp39-cp39-win_amd64.whl", hash = "sha256:392e4ec766654852c25ebad4f64e4e584cf19820b980bc04960bca0b0cd6eaa2"}, + {file = "scipy-1.13.1.tar.gz", hash = "sha256:095a87a0312b08dfd6a6155cbbd310a8c51800fc931b8c0b84003014b874ed3c"}, ] [package.dependencies] @@ -4080,38 +4165,38 @@ files = [ [[package]] name = "soupsieve" -version = "2.5" +version = "2.6" description = "A modern CSS selector implementation for Beautiful Soup." optional = false python-versions = ">=3.8" files = [ - {file = "soupsieve-2.5-py3-none-any.whl", hash = "sha256:eaa337ff55a1579b6549dc679565eac1e3d000563bcb1c8ab0d0fefbc0c2cdc7"}, - {file = "soupsieve-2.5.tar.gz", hash = "sha256:5663d5a7b3bfaeee0bc4372e7fc48f9cff4940b3eec54a6451cc5299f1097690"}, + {file = "soupsieve-2.6-py3-none-any.whl", hash = "sha256:e72c4ff06e4fb6e4b5a9f0f55fe6e81514581fca1515028625d0f299c602ccc9"}, + {file = "soupsieve-2.6.tar.gz", hash = "sha256:e2e68417777af359ec65daac1057404a3c8a5455bb8abc36f1a9866ab1a51abb"}, ] [[package]] name = "sphinx" -version = "7.3.7" +version = "7.4.7" description = "Python documentation generator" optional = false python-versions = ">=3.9" files = [ - {file = "sphinx-7.3.7-py3-none-any.whl", hash = "sha256:413f75440be4cacf328f580b4274ada4565fb2187d696a84970c23f77b64d8c3"}, - {file = "sphinx-7.3.7.tar.gz", hash = "sha256:a4a7db75ed37531c05002d56ed6948d4c42f473a36f46e1382b0bd76ca9627bc"}, + {file = "sphinx-7.4.7-py3-none-any.whl", hash = "sha256:c2419e2135d11f1951cd994d6eb18a1835bd8fdd8429f9ca375dc1f3281bd239"}, + {file = "sphinx-7.4.7.tar.gz", hash = "sha256:242f92a7ea7e6c5b406fdc2615413890ba9f699114a9c09192d7dfead2ee9cfe"}, ] [package.dependencies] alabaster = ">=0.7.14,<0.8.0" -babel = ">=2.9" -colorama = {version = ">=0.4.5", markers = "sys_platform == \"win32\""} -docutils = ">=0.18.1,<0.22" +babel = ">=2.13" +colorama = {version = ">=0.4.6", markers = "sys_platform == \"win32\""} +docutils = ">=0.20,<0.22" imagesize = ">=1.3" -importlib-metadata = {version = ">=4.8", markers = "python_version < \"3.10\""} -Jinja2 = ">=3.0" -packaging = ">=21.0" -Pygments = ">=2.14" -requests = ">=2.25.0" -snowballstemmer = ">=2.0" +importlib-metadata = {version = ">=6.0", markers = "python_version < \"3.10\""} +Jinja2 = ">=3.1" +packaging = ">=23.0" +Pygments = ">=2.17" +requests = ">=2.30.0" +snowballstemmer = ">=2.2" sphinxcontrib-applehelp = "*" sphinxcontrib-devhelp = "*" sphinxcontrib-htmlhelp = ">=2.0.0" @@ -4122,8 +4207,8 @@ tomli = {version = ">=2", markers = "python_version < \"3.11\""} [package.extras] docs = ["sphinxcontrib-websupport"] -lint = ["flake8 (>=3.5.0)", "importlib_metadata", "mypy (==1.9.0)", "pytest (>=6.0)", "ruff (==0.3.7)", "sphinx-lint", "tomli", "types-docutils", "types-requests"] -test = ["cython (>=3.0)", "defusedxml (>=0.7.1)", "pytest (>=6.0)", "setuptools (>=67.0)"] +lint = ["flake8 (>=6.0)", "importlib-metadata (>=6.0)", "mypy (==1.10.1)", "pytest (>=6.0)", "ruff (==0.5.2)", "sphinx-lint (>=0.9)", "tomli (>=2)", "types-docutils (==0.21.0.20240711)", "types-requests (>=2.30.0)"] +test = ["cython (>=3.0)", "defusedxml (>=0.7.1)", "pytest (>=8.0)", "setuptools (>=70.0)", "typing_extensions (>=4.9)"] [[package]] name = "sphinx-click" @@ -4143,49 +4228,49 @@ sphinx = ">=2.0" [[package]] name = "sphinxcontrib-applehelp" -version = "1.0.8" +version = "2.0.0" description = "sphinxcontrib-applehelp is a Sphinx extension which outputs Apple help books" optional = false python-versions = ">=3.9" files = [ - {file = "sphinxcontrib_applehelp-1.0.8-py3-none-any.whl", hash = "sha256:cb61eb0ec1b61f349e5cc36b2028e9e7ca765be05e49641c97241274753067b4"}, - {file = "sphinxcontrib_applehelp-1.0.8.tar.gz", hash = "sha256:c40a4f96f3776c4393d933412053962fac2b84f4c99a7982ba42e09576a70619"}, + {file = "sphinxcontrib_applehelp-2.0.0-py3-none-any.whl", hash = "sha256:4cd3f0ec4ac5dd9c17ec65e9ab272c9b867ea77425228e68ecf08d6b28ddbdb5"}, + {file = "sphinxcontrib_applehelp-2.0.0.tar.gz", hash = "sha256:2f29ef331735ce958efa4734873f084941970894c6090408b079c61b2e1c06d1"}, ] [package.extras] -lint = ["docutils-stubs", "flake8", "mypy"] +lint = ["mypy", "ruff (==0.5.5)", "types-docutils"] standalone = ["Sphinx (>=5)"] test = ["pytest"] [[package]] name = "sphinxcontrib-devhelp" -version = "1.0.6" +version = "2.0.0" description = "sphinxcontrib-devhelp is a sphinx extension which outputs Devhelp documents" optional = false python-versions = ">=3.9" files = [ - {file = "sphinxcontrib_devhelp-1.0.6-py3-none-any.whl", hash = "sha256:6485d09629944511c893fa11355bda18b742b83a2b181f9a009f7e500595c90f"}, - {file = "sphinxcontrib_devhelp-1.0.6.tar.gz", hash = "sha256:9893fd3f90506bc4b97bdb977ceb8fbd823989f4316b28c3841ec128544372d3"}, + {file = "sphinxcontrib_devhelp-2.0.0-py3-none-any.whl", hash = "sha256:aefb8b83854e4b0998877524d1029fd3e6879210422ee3780459e28a1f03a8a2"}, + {file = "sphinxcontrib_devhelp-2.0.0.tar.gz", hash = "sha256:411f5d96d445d1d73bb5d52133377b4248ec79db5c793ce7dbe59e074b4dd1ad"}, ] [package.extras] -lint = ["docutils-stubs", "flake8", "mypy"] +lint = ["mypy", "ruff (==0.5.5)", "types-docutils"] standalone = ["Sphinx (>=5)"] test = ["pytest"] [[package]] name = "sphinxcontrib-htmlhelp" -version = "2.0.5" +version = "2.1.0" description = "sphinxcontrib-htmlhelp is a sphinx extension which renders HTML help files" optional = false python-versions = ">=3.9" files = [ - {file = "sphinxcontrib_htmlhelp-2.0.5-py3-none-any.whl", hash = "sha256:393f04f112b4d2f53d93448d4bce35842f62b307ccdc549ec1585e950bc35e04"}, - {file = "sphinxcontrib_htmlhelp-2.0.5.tar.gz", hash = "sha256:0dc87637d5de53dd5eec3a6a01753b1ccf99494bd756aafecd74b4fa9e729015"}, + {file = "sphinxcontrib_htmlhelp-2.1.0-py3-none-any.whl", hash = "sha256:166759820b47002d22914d64a075ce08f4c46818e17cfc9470a9786b759b19f8"}, + {file = "sphinxcontrib_htmlhelp-2.1.0.tar.gz", hash = "sha256:c9e2916ace8aad64cc13a0d233ee22317f2b9025b9cf3295249fa985cc7082e9"}, ] [package.extras] -lint = ["docutils-stubs", "flake8", "mypy"] +lint = ["mypy", "ruff (==0.5.5)", "types-docutils"] standalone = ["Sphinx (>=5)"] test = ["html5lib", "pytest"] @@ -4205,97 +4290,97 @@ test = ["flake8", "mypy", "pytest"] [[package]] name = "sphinxcontrib-qthelp" -version = "1.0.7" +version = "2.0.0" description = "sphinxcontrib-qthelp is a sphinx extension which outputs QtHelp documents" optional = false python-versions = ">=3.9" files = [ - {file = "sphinxcontrib_qthelp-1.0.7-py3-none-any.whl", hash = "sha256:e2ae3b5c492d58fcbd73281fbd27e34b8393ec34a073c792642cd8e529288182"}, - {file = "sphinxcontrib_qthelp-1.0.7.tar.gz", hash = "sha256:053dedc38823a80a7209a80860b16b722e9e0209e32fea98c90e4e6624588ed6"}, + {file = "sphinxcontrib_qthelp-2.0.0-py3-none-any.whl", hash = "sha256:b18a828cdba941ccd6ee8445dbe72ffa3ef8cbe7505d8cd1fa0d42d3f2d5f3eb"}, + {file = "sphinxcontrib_qthelp-2.0.0.tar.gz", hash = "sha256:4fe7d0ac8fc171045be623aba3e2a8f613f8682731f9153bb2e40ece16b9bbab"}, ] [package.extras] -lint = ["docutils-stubs", "flake8", "mypy"] +lint = ["mypy", "ruff (==0.5.5)", "types-docutils"] standalone = ["Sphinx (>=5)"] -test = ["pytest"] +test = ["defusedxml (>=0.7.1)", "pytest"] [[package]] name = "sphinxcontrib-serializinghtml" -version = "1.1.10" +version = "2.0.0" description = "sphinxcontrib-serializinghtml is a sphinx extension which outputs \"serialized\" HTML files (json and pickle)" optional = false python-versions = ">=3.9" files = [ - {file = "sphinxcontrib_serializinghtml-1.1.10-py3-none-any.whl", hash = "sha256:326369b8df80a7d2d8d7f99aa5ac577f51ea51556ed974e7716cfd4fca3f6cb7"}, - {file = "sphinxcontrib_serializinghtml-1.1.10.tar.gz", hash = "sha256:93f3f5dc458b91b192fe10c397e324f262cf163d79f3282c158e8436a2c4511f"}, + {file = "sphinxcontrib_serializinghtml-2.0.0-py3-none-any.whl", hash = "sha256:6e2cb0eef194e10c27ec0023bfeb25badbbb5868244cf5bc5bdc04e4464bf331"}, + {file = "sphinxcontrib_serializinghtml-2.0.0.tar.gz", hash = "sha256:e9d912827f872c029017a53f0ef2180b327c3f7fd23c87229f7a8e8b70031d4d"}, ] [package.extras] -lint = ["docutils-stubs", "flake8", "mypy"] +lint = ["mypy", "ruff (==0.5.5)", "types-docutils"] standalone = ["Sphinx (>=5)"] test = ["pytest"] [[package]] name = "sqlalchemy" -version = "2.0.24" +version = "2.0.34" description = "Database Abstraction Library" optional = false python-versions = ">=3.7" files = [ - {file = "SQLAlchemy-2.0.24-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:5f801d85ba4753d4ed97181d003e5d3fa330ac7c4587d131f61d7f968f416862"}, - {file = "SQLAlchemy-2.0.24-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:b35c35e3923ade1e7ac44e150dec29f5863513246c8bf85e2d7d313e3832bcfb"}, - {file = "SQLAlchemy-2.0.24-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1d9b3fd5eca3c0b137a5e0e468e24ca544ed8ca4783e0e55341b7ed2807518ee"}, - {file = "SQLAlchemy-2.0.24-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7a6209e689d0ff206c40032b6418e3cfcfc5af044b3f66e381d7f1ae301544b4"}, - {file = "SQLAlchemy-2.0.24-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:37e89d965b52e8b20571b5d44f26e2124b26ab63758bf1b7598a0e38fb2c4005"}, - {file = "SQLAlchemy-2.0.24-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:c6910eb4ea90c0889f363965cd3c8c45a620ad27b526a7899f0054f6c1b9219e"}, - {file = "SQLAlchemy-2.0.24-cp310-cp310-win32.whl", hash = "sha256:d8e7e8a150e7b548e7ecd6ebb9211c37265991bf2504297d9454e01b58530fc6"}, - {file = "SQLAlchemy-2.0.24-cp310-cp310-win_amd64.whl", hash = "sha256:396f05c552f7fa30a129497c41bef5b4d1423f9af8fe4df0c3dcd38f3e3b9a14"}, - {file = "SQLAlchemy-2.0.24-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:adbd67dac4ebf54587198b63cd30c29fd7eafa8c0cab58893d9419414f8efe4b"}, - {file = "SQLAlchemy-2.0.24-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:a0f611b431b84f55779cbb7157257d87b4a2876b067c77c4f36b15e44ced65e2"}, - {file = "SQLAlchemy-2.0.24-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:56a0e90a959e18ac5f18c80d0cad9e90cb09322764f536e8a637426afb1cae2f"}, - {file = "SQLAlchemy-2.0.24-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6db686a1d9f183c639f7e06a2656af25d4ed438eda581de135d15569f16ace33"}, - {file = "SQLAlchemy-2.0.24-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:f0cc0b486a56dff72dddae6b6bfa7ff201b0eeac29d4bc6f0e9725dc3c360d71"}, - {file = "SQLAlchemy-2.0.24-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:4a1d4856861ba9e73bac05030cec5852eabfa9ef4af8e56c19d92de80d46fc34"}, - {file = "SQLAlchemy-2.0.24-cp311-cp311-win32.whl", hash = "sha256:a3c2753bf4f48b7a6024e5e8a394af49b1b12c817d75d06942cae03d14ff87b3"}, - {file = "SQLAlchemy-2.0.24-cp311-cp311-win_amd64.whl", hash = "sha256:38732884eabc64982a09a846bacf085596ff2371e4e41d20c0734f7e50525d01"}, - {file = "SQLAlchemy-2.0.24-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:9f992e0f916201731993eab8502912878f02287d9f765ef843677ff118d0e0b1"}, - {file = "SQLAlchemy-2.0.24-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:2587e108463cc2e5b45a896b2e7cc8659a517038026922a758bde009271aed11"}, - {file = "SQLAlchemy-2.0.24-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0bb7cedcddffca98c40bb0becd3423e293d1fef442b869da40843d751785beb3"}, - {file = "SQLAlchemy-2.0.24-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:83fa6df0e035689df89ff77a46bf8738696785d3156c2c61494acdcddc75c69d"}, - {file = "SQLAlchemy-2.0.24-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:cc889fda484d54d0b31feec409406267616536d048a450fc46943e152700bb79"}, - {file = "SQLAlchemy-2.0.24-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:57ef6f2cb8b09a042d0dbeaa46a30f2df5dd1e1eb889ba258b0d5d7d6011b81c"}, - {file = "SQLAlchemy-2.0.24-cp312-cp312-win32.whl", hash = "sha256:ea490564435b5b204d8154f0e18387b499ea3cedc1e6af3b3a2ab18291d85aa7"}, - {file = "SQLAlchemy-2.0.24-cp312-cp312-win_amd64.whl", hash = "sha256:ccfd336f96d4c9bbab0309f2a565bf15c468c2d8b2d277a32f89c5940f71fcf9"}, - {file = "SQLAlchemy-2.0.24-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:9aaaaa846b10dfbe1bda71079d0e31a7e2cebedda9409fa7dba3dfed1ae803e8"}, - {file = "SQLAlchemy-2.0.24-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:95bae3d38f8808d79072da25d5e5a6095f36fe1f9d6c614dd72c59ca8397c7c0"}, - {file = "SQLAlchemy-2.0.24-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a04191a7c8d77e63f6fc1e8336d6c6e93176c0c010833e74410e647f0284f5a1"}, - {file = "SQLAlchemy-2.0.24-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:acc58b7c2e40235712d857fdfc8f2bda9608f4a850d8d9ac0dd1fc80939ca6ac"}, - {file = "SQLAlchemy-2.0.24-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:00d76fe5d7cdb5d84d625ce002ce29fefba0bfd98e212ae66793fed30af73931"}, - {file = "SQLAlchemy-2.0.24-cp37-cp37m-win32.whl", hash = "sha256:29e51f848f843bbd75d74ae64ab1ab06302cb1dccd4549d1f5afe6b4a946edb2"}, - {file = "SQLAlchemy-2.0.24-cp37-cp37m-win_amd64.whl", hash = "sha256:e9d036e343a604db3f5a6c33354018a84a1d3f6dcae3673358b404286204798c"}, - {file = "SQLAlchemy-2.0.24-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:9bafaa05b19dc07fa191c1966c5e852af516840b0d7b46b7c3303faf1a349bc9"}, - {file = "SQLAlchemy-2.0.24-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:e69290b921b7833c04206f233d6814c60bee1d135b09f5ae5d39229de9b46cd4"}, - {file = "SQLAlchemy-2.0.24-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e8398593ccc4440ce6dffcc4f47d9b2d72b9fe7112ac12ea4a44e7d4de364db1"}, - {file = "SQLAlchemy-2.0.24-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f073321a79c81e1a009218a21089f61d87ee5fa3c9563f6be94f8b41ff181812"}, - {file = "SQLAlchemy-2.0.24-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:9036ebfd934813990c5b9f71f297e77ed4963720db7d7ceec5a3fdb7cd2ef6ce"}, - {file = "SQLAlchemy-2.0.24-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:fcf84fe93397a0f67733aa2a38ed4eab9fc6348189fc950e656e1ea198f45668"}, - {file = "SQLAlchemy-2.0.24-cp38-cp38-win32.whl", hash = "sha256:6f5e75de91c754365c098ac08c13fdb267577ce954fa239dd49228b573ca88d7"}, - {file = "SQLAlchemy-2.0.24-cp38-cp38-win_amd64.whl", hash = "sha256:9f29c7f0f4b42337ec5a779e166946a9f86d7d56d827e771b69ecbdf426124ac"}, - {file = "SQLAlchemy-2.0.24-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:07cc423892f2ceda9ae1daa28c0355757f362ecc7505b1ab1a3d5d8dc1c44ac6"}, - {file = "SQLAlchemy-2.0.24-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:2a479aa1ab199178ff1956b09ca8a0693e70f9c762875d69292d37049ffd0d8f"}, - {file = "SQLAlchemy-2.0.24-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9b8d0e8578e7f853f45f4512b5c920f6a546cd4bed44137460b2a56534644205"}, - {file = "SQLAlchemy-2.0.24-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e17e7e27af178d31b436dda6a596703b02a89ba74a15e2980c35ecd9909eea3a"}, - {file = "SQLAlchemy-2.0.24-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:1ca7903d5e7db791a355b579c690684fac6304478b68efdc7f2ebdcfe770d8d7"}, - {file = "SQLAlchemy-2.0.24-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:db09e424d7bb89b6215a184ca93b4f29d7f00ea261b787918a1af74143b98c06"}, - {file = "SQLAlchemy-2.0.24-cp39-cp39-win32.whl", hash = "sha256:a5cd7d30e47f87b21362beeb3e86f1b5886e7d9b0294b230dde3d3f4a1591375"}, - {file = "SQLAlchemy-2.0.24-cp39-cp39-win_amd64.whl", hash = "sha256:7ae5d44517fe81079ce75cf10f96978284a6db2642c5932a69c82dbae09f009a"}, - {file = "SQLAlchemy-2.0.24-py3-none-any.whl", hash = "sha256:8f358f5cfce04417b6ff738748ca4806fe3d3ae8040fb4e6a0c9a6973ccf9b6e"}, - {file = "SQLAlchemy-2.0.24.tar.gz", hash = "sha256:6db97656fd3fe3f7e5b077f12fa6adb5feb6e0b567a3e99f47ecf5f7ea0a09e3"}, + {file = "SQLAlchemy-2.0.34-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:95d0b2cf8791ab5fb9e3aa3d9a79a0d5d51f55b6357eecf532a120ba3b5524db"}, + {file = "SQLAlchemy-2.0.34-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:243f92596f4fd4c8bd30ab8e8dd5965afe226363d75cab2468f2c707f64cd83b"}, + {file = "SQLAlchemy-2.0.34-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9ea54f7300553af0a2a7235e9b85f4204e1fc21848f917a3213b0e0818de9a24"}, + {file = "SQLAlchemy-2.0.34-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:173f5f122d2e1bff8fbd9f7811b7942bead1f5e9f371cdf9e670b327e6703ebd"}, + {file = "SQLAlchemy-2.0.34-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:196958cde924a00488e3e83ff917be3b73cd4ed8352bbc0f2989333176d1c54d"}, + {file = "SQLAlchemy-2.0.34-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:bd90c221ed4e60ac9d476db967f436cfcecbd4ef744537c0f2d5291439848768"}, + {file = "SQLAlchemy-2.0.34-cp310-cp310-win32.whl", hash = "sha256:3166dfff2d16fe9be3241ee60ece6fcb01cf8e74dd7c5e0b64f8e19fab44911b"}, + {file = "SQLAlchemy-2.0.34-cp310-cp310-win_amd64.whl", hash = "sha256:6831a78bbd3c40f909b3e5233f87341f12d0b34a58f14115c9e94b4cdaf726d3"}, + {file = "SQLAlchemy-2.0.34-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c7db3db284a0edaebe87f8f6642c2b2c27ed85c3e70064b84d1c9e4ec06d5d84"}, + {file = "SQLAlchemy-2.0.34-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:430093fce0efc7941d911d34f75a70084f12f6ca5c15d19595c18753edb7c33b"}, + {file = "SQLAlchemy-2.0.34-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:79cb400c360c7c210097b147c16a9e4c14688a6402445ac848f296ade6283bbc"}, + {file = "SQLAlchemy-2.0.34-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fb1b30f31a36c7f3fee848391ff77eebdd3af5750bf95fbf9b8b5323edfdb4ec"}, + {file = "SQLAlchemy-2.0.34-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:8fddde2368e777ea2a4891a3fb4341e910a056be0bb15303bf1b92f073b80c02"}, + {file = "SQLAlchemy-2.0.34-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:80bd73ea335203b125cf1d8e50fef06be709619eb6ab9e7b891ea34b5baa2287"}, + {file = "SQLAlchemy-2.0.34-cp311-cp311-win32.whl", hash = "sha256:6daeb8382d0df526372abd9cb795c992e18eed25ef2c43afe518c73f8cccb721"}, + {file = "SQLAlchemy-2.0.34-cp311-cp311-win_amd64.whl", hash = "sha256:5bc08e75ed11693ecb648b7a0a4ed80da6d10845e44be0c98c03f2f880b68ff4"}, + {file = "SQLAlchemy-2.0.34-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:53e68b091492c8ed2bd0141e00ad3089bcc6bf0e6ec4142ad6505b4afe64163e"}, + {file = "SQLAlchemy-2.0.34-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:bcd18441a49499bf5528deaa9dee1f5c01ca491fc2791b13604e8f972877f812"}, + {file = "SQLAlchemy-2.0.34-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:165bbe0b376541092bf49542bd9827b048357f4623486096fc9aaa6d4e7c59a2"}, + {file = "SQLAlchemy-2.0.34-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c3330415cd387d2b88600e8e26b510d0370db9b7eaf984354a43e19c40df2e2b"}, + {file = "SQLAlchemy-2.0.34-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:97b850f73f8abbffb66ccbab6e55a195a0eb655e5dc74624d15cff4bfb35bd74"}, + {file = "SQLAlchemy-2.0.34-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:7cee4c6917857fd6121ed84f56d1dc78eb1d0e87f845ab5a568aba73e78adf83"}, + {file = "SQLAlchemy-2.0.34-cp312-cp312-win32.whl", hash = "sha256:fbb034f565ecbe6c530dff948239377ba859420d146d5f62f0271407ffb8c580"}, + {file = "SQLAlchemy-2.0.34-cp312-cp312-win_amd64.whl", hash = "sha256:707c8f44931a4facd4149b52b75b80544a8d824162602b8cd2fe788207307f9a"}, + {file = "SQLAlchemy-2.0.34-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:24af3dc43568f3780b7e1e57c49b41d98b2d940c1fd2e62d65d3928b6f95f021"}, + {file = "SQLAlchemy-2.0.34-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e60ed6ef0a35c6b76b7640fe452d0e47acc832ccbb8475de549a5cc5f90c2c06"}, + {file = "SQLAlchemy-2.0.34-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:413c85cd0177c23e32dee6898c67a5f49296640041d98fddb2c40888fe4daa2e"}, + {file = "SQLAlchemy-2.0.34-cp37-cp37m-musllinux_1_2_aarch64.whl", hash = "sha256:25691f4adfb9d5e796fd48bf1432272f95f4bbe5f89c475a788f31232ea6afba"}, + {file = "SQLAlchemy-2.0.34-cp37-cp37m-musllinux_1_2_x86_64.whl", hash = "sha256:526ce723265643dbc4c7efb54f56648cc30e7abe20f387d763364b3ce7506c82"}, + {file = "SQLAlchemy-2.0.34-cp37-cp37m-win32.whl", hash = "sha256:13be2cc683b76977a700948411a94c67ad8faf542fa7da2a4b167f2244781cf3"}, + {file = "SQLAlchemy-2.0.34-cp37-cp37m-win_amd64.whl", hash = "sha256:e54ef33ea80d464c3dcfe881eb00ad5921b60f8115ea1a30d781653edc2fd6a2"}, + {file = "SQLAlchemy-2.0.34-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:43f28005141165edd11fbbf1541c920bd29e167b8bbc1fb410d4fe2269c1667a"}, + {file = "SQLAlchemy-2.0.34-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:b68094b165a9e930aedef90725a8fcfafe9ef95370cbb54abc0464062dbf808f"}, + {file = "SQLAlchemy-2.0.34-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6a1e03db964e9d32f112bae36f0cc1dcd1988d096cfd75d6a588a3c3def9ab2b"}, + {file = "SQLAlchemy-2.0.34-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:203d46bddeaa7982f9c3cc693e5bc93db476ab5de9d4b4640d5c99ff219bee8c"}, + {file = "SQLAlchemy-2.0.34-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:ae92bebca3b1e6bd203494e5ef919a60fb6dfe4d9a47ed2453211d3bd451b9f5"}, + {file = "SQLAlchemy-2.0.34-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:9661268415f450c95f72f0ac1217cc6f10256f860eed85c2ae32e75b60278ad8"}, + {file = "SQLAlchemy-2.0.34-cp38-cp38-win32.whl", hash = "sha256:895184dfef8708e15f7516bd930bda7e50ead069280d2ce09ba11781b630a434"}, + {file = "SQLAlchemy-2.0.34-cp38-cp38-win_amd64.whl", hash = "sha256:6e7cde3a2221aa89247944cafb1b26616380e30c63e37ed19ff0bba5e968688d"}, + {file = "SQLAlchemy-2.0.34-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:dbcdf987f3aceef9763b6d7b1fd3e4ee210ddd26cac421d78b3c206d07b2700b"}, + {file = "SQLAlchemy-2.0.34-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:ce119fc4ce0d64124d37f66a6f2a584fddc3c5001755f8a49f1ca0a177ef9796"}, + {file = "SQLAlchemy-2.0.34-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a17d8fac6df9835d8e2b4c5523666e7051d0897a93756518a1fe101c7f47f2f0"}, + {file = "SQLAlchemy-2.0.34-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9ebc11c54c6ecdd07bb4efbfa1554538982f5432dfb8456958b6d46b9f834bb7"}, + {file = "SQLAlchemy-2.0.34-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:2e6965346fc1491a566e019a4a1d3dfc081ce7ac1a736536367ca305da6472a8"}, + {file = "SQLAlchemy-2.0.34-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:220574e78ad986aea8e81ac68821e47ea9202b7e44f251b7ed8c66d9ae3f4278"}, + {file = "SQLAlchemy-2.0.34-cp39-cp39-win32.whl", hash = "sha256:b75b00083e7fe6621ce13cfce9d4469c4774e55e8e9d38c305b37f13cf1e874c"}, + {file = "SQLAlchemy-2.0.34-cp39-cp39-win_amd64.whl", hash = "sha256:c29d03e0adf3cc1a8c3ec62d176824972ae29b67a66cbb18daff3062acc6faa8"}, + {file = "SQLAlchemy-2.0.34-py3-none-any.whl", hash = "sha256:7286c353ee6475613d8beff83167374006c6b3e3f0e6491bfe8ca610eb1dec0f"}, + {file = "sqlalchemy-2.0.34.tar.gz", hash = "sha256:10d8f36990dd929690666679b0f42235c159a7051534adb135728ee52828dd22"}, ] [package.dependencies] -greenlet = {version = "!=0.4.17", markers = "platform_machine == \"aarch64\" or platform_machine == \"ppc64le\" or platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"win32\" or platform_machine == \"WIN32\""} -typing-extensions = ">=4.2.0" +greenlet = {version = "!=0.4.17", markers = "python_version < \"3.13\" and (platform_machine == \"aarch64\" or platform_machine == \"ppc64le\" or platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"win32\" or platform_machine == \"WIN32\")"} +typing-extensions = ">=4.6.0" [package.extras] aiomysql = ["aiomysql (>=0.2.0)", "greenlet (!=0.4.17)"] @@ -4385,13 +4470,13 @@ Jinja2 = ">=2.0" [[package]] name = "synapseclient" -version = "4.4.0" +version = "4.4.1" description = "A client for Synapse, a collaborative, open-source research platform that allows teams to share data, track analyses, and collaborate." optional = false python-versions = ">=3.8" files = [ - {file = "synapseclient-4.4.0-py3-none-any.whl", hash = "sha256:efbf8ac46909a5ce50e54aa4c81850e876754bd3c58b26c6a43850fbca96a01b"}, - {file = "synapseclient-4.4.0.tar.gz", hash = "sha256:331d0740a8cebf29a231d5ead35cda164fc74d7d3a8470803e623f2c33e897b5"}, + {file = "synapseclient-4.4.1-py3-none-any.whl", hash = "sha256:fe5716f234184ad0290c930f98383ce87bbf687221365ef477de826831c73994"}, + {file = "synapseclient-4.4.1.tar.gz", hash = "sha256:fc6ec5a0fd49edf2b05ecd7f69316784a4b813dd0fd259785932c0786d480629"}, ] [package.dependencies] @@ -4432,13 +4517,13 @@ widechars = ["wcwidth"] [[package]] name = "tenacity" -version = "8.3.0" +version = "8.5.0" description = "Retry code until it succeeds" optional = false python-versions = ">=3.8" files = [ - {file = "tenacity-8.3.0-py3-none-any.whl", hash = "sha256:3649f6443dbc0d9b01b9d8020a9c4ec7a1ff5f6f3c6c8a036ef371f573fe9185"}, - {file = "tenacity-8.3.0.tar.gz", hash = "sha256:953d4e6ad24357bceffbc9707bc74349aca9d245f68eb65419cf0c249a1949a2"}, + {file = "tenacity-8.5.0-py3-none-any.whl", hash = "sha256:b594c2a5945830c267ce6b79a166228323ed52718f30302c1359836112346687"}, + {file = "tenacity-8.5.0.tar.gz", hash = "sha256:8bc6c0c8a09b31e6cad13c47afbed1a567518250a9a171418582ed8d9c20ca78"}, ] [package.extras] @@ -4540,13 +4625,13 @@ files = [ [[package]] name = "tomlkit" -version = "0.12.5" +version = "0.13.2" description = "Style preserving TOML library" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "tomlkit-0.12.5-py3-none-any.whl", hash = "sha256:af914f5a9c59ed9d0762c7b64d3b5d5df007448eb9cd2edc8a46b1eafead172f"}, - {file = "tomlkit-0.12.5.tar.gz", hash = "sha256:eef34fba39834d4d6b73c9ba7f3e4d1c417a4e56f89a7e96e090dd0d24b8fb3c"}, + {file = "tomlkit-0.13.2-py3-none-any.whl", hash = "sha256:7a974427f6e119197f670fbbbeae7bef749a6c14e793db934baefc1b5f03efde"}, + {file = "tomlkit-0.13.2.tar.gz", hash = "sha256:fff5fe59a87295b278abd31bec92c15d9bc4a06885ab12bcea52c71119392e79"}, ] [[package]] @@ -4562,33 +4647,33 @@ files = [ [[package]] name = "tornado" -version = "6.4" +version = "6.4.1" description = "Tornado is a Python web framework and asynchronous networking library, originally developed at FriendFeed." optional = false -python-versions = ">= 3.8" +python-versions = ">=3.8" files = [ - {file = "tornado-6.4-cp38-abi3-macosx_10_9_universal2.whl", hash = "sha256:02ccefc7d8211e5a7f9e8bc3f9e5b0ad6262ba2fbb683a6443ecc804e5224ce0"}, - {file = "tornado-6.4-cp38-abi3-macosx_10_9_x86_64.whl", hash = "sha256:27787de946a9cffd63ce5814c33f734c627a87072ec7eed71f7fc4417bb16263"}, - {file = "tornado-6.4-cp38-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f7894c581ecdcf91666a0912f18ce5e757213999e183ebfc2c3fdbf4d5bd764e"}, - {file = "tornado-6.4-cp38-abi3-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e43bc2e5370a6a8e413e1e1cd0c91bedc5bd62a74a532371042a18ef19e10579"}, - {file = "tornado-6.4-cp38-abi3-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f0251554cdd50b4b44362f73ad5ba7126fc5b2c2895cc62b14a1c2d7ea32f212"}, - {file = "tornado-6.4-cp38-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:fd03192e287fbd0899dd8f81c6fb9cbbc69194d2074b38f384cb6fa72b80e9c2"}, - {file = "tornado-6.4-cp38-abi3-musllinux_1_1_i686.whl", hash = "sha256:88b84956273fbd73420e6d4b8d5ccbe913c65d31351b4c004ae362eba06e1f78"}, - {file = "tornado-6.4-cp38-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:71ddfc23a0e03ef2df1c1397d859868d158c8276a0603b96cf86892bff58149f"}, - {file = "tornado-6.4-cp38-abi3-win32.whl", hash = "sha256:6f8a6c77900f5ae93d8b4ae1196472d0ccc2775cc1dfdc9e7727889145c45052"}, - {file = "tornado-6.4-cp38-abi3-win_amd64.whl", hash = "sha256:10aeaa8006333433da48dec9fe417877f8bcc21f48dda8d661ae79da357b2a63"}, - {file = "tornado-6.4.tar.gz", hash = "sha256:72291fa6e6bc84e626589f1c29d90a5a6d593ef5ae68052ee2ef000dfd273dee"}, + {file = "tornado-6.4.1-cp38-abi3-macosx_10_9_universal2.whl", hash = "sha256:163b0aafc8e23d8cdc3c9dfb24c5368af84a81e3364745ccb4427669bf84aec8"}, + {file = "tornado-6.4.1-cp38-abi3-macosx_10_9_x86_64.whl", hash = "sha256:6d5ce3437e18a2b66fbadb183c1d3364fb03f2be71299e7d10dbeeb69f4b2a14"}, + {file = "tornado-6.4.1-cp38-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e2e20b9113cd7293f164dc46fffb13535266e713cdb87bd2d15ddb336e96cfc4"}, + {file = "tornado-6.4.1-cp38-abi3-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8ae50a504a740365267b2a8d1a90c9fbc86b780a39170feca9bcc1787ff80842"}, + {file = "tornado-6.4.1-cp38-abi3-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:613bf4ddf5c7a95509218b149b555621497a6cc0d46ac341b30bd9ec19eac7f3"}, + {file = "tornado-6.4.1-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:25486eb223babe3eed4b8aecbac33b37e3dd6d776bc730ca14e1bf93888b979f"}, + {file = "tornado-6.4.1-cp38-abi3-musllinux_1_2_i686.whl", hash = "sha256:454db8a7ecfcf2ff6042dde58404164d969b6f5d58b926da15e6b23817950fc4"}, + {file = "tornado-6.4.1-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:a02a08cc7a9314b006f653ce40483b9b3c12cda222d6a46d4ac63bb6c9057698"}, + {file = "tornado-6.4.1-cp38-abi3-win32.whl", hash = "sha256:d9a566c40b89757c9aa8e6f032bcdb8ca8795d7c1a9762910c722b1635c9de4d"}, + {file = "tornado-6.4.1-cp38-abi3-win_amd64.whl", hash = "sha256:b24b8982ed444378d7f21d563f4180a2de31ced9d8d84443907a0a64da2072e7"}, + {file = "tornado-6.4.1.tar.gz", hash = "sha256:92d3ab53183d8c50f8204a51e6f91d18a15d5ef261e84d452800d4ff6fc504e9"}, ] [[package]] name = "tqdm" -version = "4.66.4" +version = "4.66.5" description = "Fast, Extensible Progress Meter" optional = false python-versions = ">=3.7" files = [ - {file = "tqdm-4.66.4-py3-none-any.whl", hash = "sha256:b75ca56b413b030bc3f00af51fd2c1a1a5eac6a0c1cca83cbb37a5c52abce644"}, - {file = "tqdm-4.66.4.tar.gz", hash = "sha256:e4d936c9de8727928f3be6079590e97d9abfe8d39a590be678eb5919ffc186bb"}, + {file = "tqdm-4.66.5-py3-none-any.whl", hash = "sha256:90279a3770753eafc9194a0364852159802111925aa30eb3f9d85b0e805ac7cd"}, + {file = "tqdm-4.66.5.tar.gz", hash = "sha256:e1020aef2e5096702d8a025ac7d16b1577279c9d63f8375b63083e9a5f0fcbad"}, ] [package.dependencies] @@ -4617,24 +4702,24 @@ test = ["argcomplete (>=3.0.3)", "mypy (>=1.7.0)", "pre-commit", "pytest (>=7.0, [[package]] name = "types-python-dateutil" -version = "2.9.0.20240316" +version = "2.9.0.20240821" description = "Typing stubs for python-dateutil" optional = false python-versions = ">=3.8" files = [ - {file = "types-python-dateutil-2.9.0.20240316.tar.gz", hash = "sha256:5d2f2e240b86905e40944dd787db6da9263f0deabef1076ddaed797351ec0202"}, - {file = "types_python_dateutil-2.9.0.20240316-py3-none-any.whl", hash = "sha256:6b8cb66d960771ce5ff974e9dd45e38facb81718cc1e208b10b1baccbfdbee3b"}, + {file = "types-python-dateutil-2.9.0.20240821.tar.gz", hash = "sha256:9649d1dcb6fef1046fb18bebe9ea2aa0028b160918518c34589a46045f6ebd98"}, + {file = "types_python_dateutil-2.9.0.20240821-py3-none-any.whl", hash = "sha256:f5889fcb4e63ed4aaa379b44f93c32593d50b9a94c9a60a0c854d8cc3511cd57"}, ] [[package]] name = "typing-extensions" -version = "4.5.0" -description = "Backported and Experimental Type Hints for Python 3.7+" +version = "4.12.2" +description = "Backported and Experimental Type Hints for Python 3.8+" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "typing_extensions-4.5.0-py3-none-any.whl", hash = "sha256:fb33085c39dd998ac16d1431ebc293a8b3eedd00fd4a32de0ff79002c19511b4"}, - {file = "typing_extensions-4.5.0.tar.gz", hash = "sha256:5cb5f4a79139d699607b3ef622a1dedafa84e115ab0024e0d9c044a9479ca7cb"}, + {file = "typing_extensions-4.12.2-py3-none-any.whl", hash = "sha256:04e5ca0351e0f3f85c6853954072df659d0d13fac324d0072316b67d7794700d"}, + {file = "typing_extensions-4.12.2.tar.gz", hash = "sha256:1a7ead55c7e559dd4dee8856e3a88b41225abfe1ce8df57b7c13915fe121ffb8"}, ] [[package]] @@ -4707,13 +4792,13 @@ files = [ [[package]] name = "urllib3" -version = "1.26.18" +version = "1.26.20" description = "HTTP library with thread-safe connection pooling, file post, and more." optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7" files = [ - {file = "urllib3-1.26.18-py2.py3-none-any.whl", hash = "sha256:34b97092d7e0a3a8cf7cd10e386f401b3737364026c45e622aa02903dffe0f07"}, - {file = "urllib3-1.26.18.tar.gz", hash = "sha256:f8ecc1bba5667413457c529ab955bf8c67b45db799d159066261719e328580a0"}, + {file = "urllib3-1.26.20-py2.py3-none-any.whl", hash = "sha256:0ed14ccfbf1c30a9072c7ca157e4319b70d65f623e91e7b32fadb2853431016e"}, + {file = "urllib3-1.26.20.tar.gz", hash = "sha256:40c2dc0c681e47eb8f90e7e27bf6ff7df2e677421fd46756da1161c39ca70d32"}, ] [package.extras] @@ -4723,12 +4808,12 @@ socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] [[package]] name = "uwsgi" -version = "2.0.25.1" +version = "2.0.26" description = "The uWSGI server" optional = true python-versions = "*" files = [ - {file = "uwsgi-2.0.25.1.tar.gz", hash = "sha256:d653d2d804c194c8cbe2585fa56efa2650313ae75c686a9d7931374d4dfbfc6e"}, + {file = "uwsgi-2.0.26.tar.gz", hash = "sha256:86e6bfcd4dc20529665f5b7777193cdc48622fb2c59f0a7f1e3dc32b3882e7f9"}, ] [[package]] @@ -4749,13 +4834,13 @@ test = ["flake8 (>=2.4.0)", "isort (>=4.2.2)", "pytest (>=2.2.3)"] [[package]] name = "virtualenv" -version = "20.26.1" +version = "20.26.3" description = "Virtual Python Environment builder" optional = false python-versions = ">=3.7" files = [ - {file = "virtualenv-20.26.1-py3-none-any.whl", hash = "sha256:7aa9982a728ae5892558bff6a2839c00b9ed145523ece2274fad6f414690ae75"}, - {file = "virtualenv-20.26.1.tar.gz", hash = "sha256:604bfdceaeece392802e6ae48e69cec49168b9c5f4a44e483963f9242eb0e78b"}, + {file = "virtualenv-20.26.3-py3-none-any.whl", hash = "sha256:8cc4a31139e796e9a7de2cd5cf2489de1217193116a8fd42328f1bd65f434589"}, + {file = "virtualenv-20.26.3.tar.gz", hash = "sha256:4c43a2a236279d9ea36a0d76f98d84bd6ca94ac4e0f4a3b9d46d05e10fea542a"}, ] [package.dependencies] @@ -4780,18 +4865,18 @@ files = [ [[package]] name = "webcolors" -version = "1.13" +version = "24.8.0" description = "A library for working with the color formats defined by HTML and CSS." optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "webcolors-1.13-py3-none-any.whl", hash = "sha256:29bc7e8752c0a1bd4a1f03c14d6e6a72e93d82193738fa860cbff59d0fcc11bf"}, - {file = "webcolors-1.13.tar.gz", hash = "sha256:c225b674c83fa923be93d235330ce0300373d02885cef23238813b0d5668304a"}, + {file = "webcolors-24.8.0-py3-none-any.whl", hash = "sha256:fc4c3b59358ada164552084a8ebee637c221e4059267d0f8325b3b560f6c7f0a"}, + {file = "webcolors-24.8.0.tar.gz", hash = "sha256:08b07af286a01bcd30d583a7acadf629583d1f79bfef27dd2c2c5c263817277d"}, ] [package.extras] docs = ["furo", "sphinx", "sphinx-copybutton", "sphinx-inline-tabs", "sphinx-notfound-page", "sphinxext-opengraph"] -tests = ["pytest", "pytest-cov"] +tests = ["coverage[toml]"] [[package]] name = "webencodings" @@ -4839,13 +4924,13 @@ watchdog = ["watchdog (>=2.3)"] [[package]] name = "widgetsnbextension" -version = "4.0.10" +version = "4.0.13" description = "Jupyter interactive widgets for Jupyter Notebook" optional = false python-versions = ">=3.7" files = [ - {file = "widgetsnbextension-4.0.10-py3-none-any.whl", hash = "sha256:d37c3724ec32d8c48400a435ecfa7d3e259995201fbefa37163124a9fcb393cc"}, - {file = "widgetsnbextension-4.0.10.tar.gz", hash = "sha256:64196c5ff3b9a9183a8e699a4227fb0b7002f252c814098e66c4d1cd0644688f"}, + {file = "widgetsnbextension-4.0.13-py3-none-any.whl", hash = "sha256:74b2692e8500525cc38c2b877236ba51d34541e6385eeed5aec15a70f88a6c71"}, + {file = "widgetsnbextension-4.0.13.tar.gz", hash = "sha256:ffcb67bc9febd10234a362795f643927f4e0c05d9342c727b65d2384f8feacb6"}, ] [[package]] @@ -4929,18 +5014,22 @@ files = [ [[package]] name = "zipp" -version = "3.18.1" +version = "3.20.1" description = "Backport of pathlib-compatible object wrapper for zip files" optional = false python-versions = ">=3.8" files = [ - {file = "zipp-3.18.1-py3-none-any.whl", hash = "sha256:206f5a15f2af3dbaee80769fb7dc6f249695e940acca08dfb2a4769fe61e538b"}, - {file = "zipp-3.18.1.tar.gz", hash = "sha256:2884ed22e7d8961de1c9a05142eb69a247f120291bc0206a00a7642f09b5b715"}, + {file = "zipp-3.20.1-py3-none-any.whl", hash = "sha256:9960cd8967c8f85a56f920d5d507274e74f9ff813a0ab8889a5b5be2daf44064"}, + {file = "zipp-3.20.1.tar.gz", hash = "sha256:c22b14cc4763c5a5b04134207736c107db42e9d3ef2d9779d465f5f1bcba572b"}, ] [package.extras] -docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] -testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-ignore-flaky", "pytest-mypy", "pytest-ruff (>=0.2.1)"] +check = ["pytest-checkdocs (>=2.4)", "pytest-ruff (>=0.2.1)"] +cover = ["pytest-cov"] +doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] +enabler = ["pytest-enabler (>=2.2)"] +test = ["big-O", "importlib-resources", "jaraco.functools", "jaraco.itertools", "jaraco.test", "more-itertools", "pytest (>=6,!=8.1.*)", "pytest-ignore-flaky"] +type = ["pytest-mypy"] [extras] api = ["Flask", "Flask-Cors", "Jinja2", "connexion", "flask-opentracing", "jaeger-client", "pyopenssl"] @@ -4949,4 +5038,4 @@ aws = ["uWSGI"] [metadata] lock-version = "2.0" python-versions = ">=3.9.0,<3.11" -content-hash = "450ff82615a78cd55b7c526f314898aa38e3186810f2e465d02117ea4fb9d10b" +content-hash = "3d9d2fe8fa5fa8ef6328133f3b510bd5a6ec42632b3972442223093bfeaf59b8" diff --git a/pyproject.toml b/pyproject.toml index 9e601371c..7b988c572 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -54,7 +54,7 @@ pygsheets = "^2.0.4" PyYAML = "^6.0.0" rdflib = "^6.0.0" setuptools = "^66.0.0" -synapseclient = "4.4.0" +synapseclient = "4.4.1" tenacity = "^8.0.1" toml = "^0.10.2" great-expectations = "^0.15.0" @@ -78,6 +78,7 @@ asyncio = "^3.4.3" pytest-asyncio = "^0.23.7" jaeger-client = {version = "^4.8.0", optional = true} flask-opentracing = {version="^2.0.0", optional = true} +PyJWT = "^2.9.0" [tool.poetry.extras] api = ["connexion", "Flask", "Flask-Cors", "Jinja2", "pyopenssl", "jaeger-client", "flask-opentracing"] diff --git a/schematic_api/api/openapi/api.yaml b/schematic_api/api/openapi/api.yaml index 15a3540d1..91a4989ee 100644 --- a/schematic_api/api/openapi/api.yaml +++ b/schematic_api/api/openapi/api.yaml @@ -15,7 +15,7 @@ components: type: http scheme: bearer bearerFormat: JWT - x-bearerInfoFunc: schematic_api.api.security_controller_.info_from_bearerAuth + x-bearerInfoFunc: schematic_api.api.security_controller.info_from_bearer_auth # TO DO: refactor query parameters and remove access_token paths: diff --git a/schematic_api/api/security_controller.py b/schematic_api/api/security_controller.py new file mode 100644 index 000000000..33cc4ad76 --- /dev/null +++ b/schematic_api/api/security_controller.py @@ -0,0 +1,47 @@ +import logging +from typing import Dict, Union + +from jwt import PyJWKClient, decode +from jwt.exceptions import PyJWTError +from synapseclient import Synapse + +from schematic.configuration.configuration import CONFIG + +logger = logging.getLogger(__name__) + +syn = Synapse( + configPath=CONFIG.synapse_configuration_path, + cache_client=False, +) +jwks_client = PyJWKClient( + uri=syn.authEndpoint + "/oauth2/jwks", headers=syn._generate_headers() +) + + +def info_from_bearer_auth(token: str) -> Dict[str, Union[str, int]]: + """ + Authenticate user using bearer token. The token claims are decoded and returned. + + Example from: + + + Args: + token (str): Bearer token. + + Returns: + dict: Decoded token information. + """ + try: + signing_key = jwks_client.get_signing_key_from_jwt(token) + data = decode( + jwt=token, + key=signing_key.key, + algorithms=[signing_key.algorithm_name], + options={"verify_aud": False}, + ) + + return data + except PyJWTError: + logger.exception("Error decoding authentication token") + # When the return type is None the web framework will return a 401 OAuthResponseProblem exception + return None diff --git a/schematic_api/api/security_controller_.py b/schematic_api/api/security_controller_.py deleted file mode 100644 index fbde596bb..000000000 --- a/schematic_api/api/security_controller_.py +++ /dev/null @@ -1,14 +0,0 @@ -from typing import List - - -def info_from_bearerAuth(token): - """ - Check and retrieve authentication information from custom bearer token. - Returned value will be passed in 'token_info' parameter of your operation function, if there is one. - 'sub' or 'uid' will be set in 'user' parameter of your operation function, if there is one. - :param token Token provided by Authorization header - :type token: str - :return: Decoded token information or None if token is invalid - :rtype: dict | None - """ - return {"uid": "user_id"} diff --git a/tests/conftest.py b/tests/conftest.py index 6ce21df5d..f580a08c9 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,4 +1,5 @@ """Fixtures and helpers for use across all tests""" +import configparser import logging import os import shutil @@ -8,7 +9,7 @@ import pytest from dotenv import load_dotenv -from schematic.configuration.configuration import CONFIG +from schematic.configuration.configuration import CONFIG, Configuration from schematic.models.metadata import MetadataModel from schematic.schemas.data_model_graph import DataModelGraph, DataModelGraphExplorer from schematic.schemas.data_model_parser import DataModelParser @@ -152,6 +153,19 @@ def DMGE(helpers: Helpers) -> DataModelGraphExplorer: return dmge +@pytest.fixture(scope="class") +def syn_token(config: Configuration): + synapse_config_path = config.synapse_configuration_path + config_parser = configparser.ConfigParser() + config_parser.read(synapse_config_path) + # try using synapse access token + if "SYNAPSE_ACCESS_TOKEN" in os.environ: + token = os.environ["SYNAPSE_ACCESS_TOKEN"] + else: + token = config_parser["authentication"]["authtoken"] + return token + + def metadata_model(helpers, data_model_labels): metadata_model = MetadataModel( inputMModelLocation=helpers.get_data_path("example.model.jsonld"), diff --git a/tests/integration/test_security_controller.py b/tests/integration/test_security_controller.py new file mode 100644 index 000000000..43a4748a4 --- /dev/null +++ b/tests/integration/test_security_controller.py @@ -0,0 +1,59 @@ +import jwt +from cryptography.hazmat.backends import default_backend +from cryptography.hazmat.primitives.asymmetric import rsa +from pytest import LogCaptureFixture + +from schematic_api.api.security_controller import info_from_bearer_auth + + +class TestSecurityController: + def test_valid_synapse_token(self, syn_token: str) -> None: + # GIVEN a valid synapse token + assert syn_token is not None + + # WHEN the token is decoded + decoded_token = info_from_bearer_auth(syn_token) + + # THEN the decoded claims are a dictionary + assert isinstance(decoded_token, dict) + assert "sub" in decoded_token + assert decoded_token["sub"] is not None + assert "token_type" in decoded_token + assert decoded_token["token_type"] is not None + + def test_invalid_synapse_signing_key(self, caplog: LogCaptureFixture) -> None: + # GIVEN an invalid synapse token + private_key = rsa.generate_private_key( + public_exponent=65537, key_size=2048, backend=default_backend() + ) + + random_token = jwt.encode( + payload={"sub": "random"}, key=private_key, algorithm="RS256" + ) + + # WHEN the token is decoded + decoded_token = info_from_bearer_auth(random_token) + + # THEN nothing is returned + assert decoded_token is None + + # AND an error is logged + assert ( + "jwt.exceptions.PyJWKClientError: Unable to find a signing key that matches:" + in caplog.text + ) + + def test_invalid_synapse_token_not_enough_parts( + self, caplog: LogCaptureFixture + ) -> None: + # GIVEN an invalid synapse token + random_token = "invalid token" + + # WHEN the token is decoded + decoded_token = info_from_bearer_auth(random_token) + + # THEN nothing is returned + assert decoded_token is None + + # AND an error is logged + assert "jwt.exceptions.DecodeError: Not enough segments" in caplog.text diff --git a/tests/test_api.py b/tests/test_api.py index 6ce515622..3b80b3964 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -1,4 +1,3 @@ -import configparser import json import logging import os @@ -100,19 +99,6 @@ def get_MockComponent_attribute() -> Generator[str, None, None]: yield MockComponent_attribute -@pytest.fixture(scope="class") -def syn_token(config: Configuration): - synapse_config_path = config.synapse_configuration_path - config_parser = configparser.ConfigParser() - config_parser.read(synapse_config_path) - # try using synapse access token - if "SYNAPSE_ACCESS_TOKEN" in os.environ: - token = os.environ["SYNAPSE_ACCESS_TOKEN"] - else: - token = config_parser["authentication"]["authtoken"] - return token - - @pytest.fixture def request_headers(syn_token: str) -> Dict[str, str]: headers = {"Authorization": "Bearer " + syn_token} From f6dafd71eaece42b147877caadfe5c3fd8f02b18 Mon Sep 17 00:00:00 2001 From: linglp Date: Fri, 6 Sep 2024 12:39:24 -0400 Subject: [PATCH 191/233] add comments, modify tests --- tests/integration/test_store_synapse.py | 40 +++++++++++++++++-------- 1 file changed, 27 insertions(+), 13 deletions(-) diff --git a/tests/integration/test_store_synapse.py b/tests/integration/test_store_synapse.py index cd087c9ff..05d338945 100644 --- a/tests/integration/test_store_synapse.py +++ b/tests/integration/test_store_synapse.py @@ -4,6 +4,7 @@ import pytest from schematic.schemas.data_model_graph import DataModelGraphExplorer +from schematic.store.synapse import SynapseStorage from schematic.utils.validate_utils import comma_separated_list_regex from tests.conftest import Helpers @@ -38,8 +39,15 @@ class TestStoreSynapse: ids=["display_label", "class_label"], ) def test_process_row_annotations_hide_blanks( - self, dmge, synapse_store, annos, hideBlanks, label_options + self, + dmge: DataModelGraphExplorer, + synapse_store: SynapseStorage, + annos: dict, + hideBlanks: bool, + label_options: str, ): + """ensure that blank values are not added to the annotations dictionary if hideBlanks is True""" + metadata_syn_with_blanks = { "PatientID": "value1", "Sex": "value2", @@ -85,36 +93,42 @@ def test_process_row_annotations_hide_blanks( @pytest.mark.parametrize( "label_options", - ["display_label"], - ids=["display_label"], + ["display_label", "class_label"], + ids=["display_label", "class_label"], ) @pytest.mark.parametrize("hideBlanks", [True, False]) def test_process_row_annotations_get_validation( - self, dmge, synapse_store, hideBlanks, label_options + self, + dmge: DataModelGraphExplorer, + synapse_store: SynapseStorage, + label_options: str, ): + """ensure that get_node_validation_rules is called with the correct arguments""" comma_separated_list = comma_separated_list_regex() - metadata_syn = {"PatientID": "value1", "Sex": "value2"} - annos = {"PatientID": "old_value", "Sex": "old_value"} + metadata_syn = { + "PatientID": "value1", + } + annos = {"PatientID": "old_value"} dmge.get_node_validation_rules = MagicMock() process_row_annos = synapse_store.process_row_annotations( dmge=dmge, metadataSyn=metadata_syn, csv_list_regex=comma_separated_list, - hideBlanks=hideBlanks, + hideBlanks=True, annos=annos, annotation_keys=label_options, ) - # when the label is "display label", make sure that the get_node_validation_rules is called with the display name if label_options == "display_label": - dmge.get_node_validation_rules.assert_called_once_with( + # get_node_validation_rules was called with node_display_name="PatientID" at least once + dmge.get_node_validation_rules.assert_any_call( node_display_name="PatientID" ) - dmge.get_node_validation_rules.assert_called_once_with( - node_display_name="Sex" - ) - # make sure that the get_node_validation_rules is called with the node label + # get_node_validation_rules was called with node_display_name="Sex" at least once + dmge.get_node_validation_rules.assert_any_call(node_display_name="Sex") else: + # get_node_validation_rules was called with node_label="PatientID" at least once dmge.get_node_validation_rules.assert_any_call(node_label="PatientID") + # get_node_validation_rules was called with node_label="Sex" at least once dmge.get_node_validation_rules.assert_any_call(node_label="Sex") From 9365ce850f68b73fe1a18ec330619060f75cd252 Mon Sep 17 00:00:00 2001 From: linglp Date: Fri, 6 Sep 2024 12:41:27 -0400 Subject: [PATCH 192/233] remove unnecessary fixture --- tests/integration/test_store_synapse.py | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/tests/integration/test_store_synapse.py b/tests/integration/test_store_synapse.py index 05d338945..84caaf5bb 100644 --- a/tests/integration/test_store_synapse.py +++ b/tests/integration/test_store_synapse.py @@ -9,21 +9,6 @@ from tests.conftest import Helpers -@pytest.fixture -def metadataSyn(): - return { - "key1": "value1", - "key2": np.nan, - "key3": "val1,val2,val3", # Simulate a CSV-like string - "key4": "another_value", - } - - -@pytest.fixture -def annos(): - return {"key1": "old_value1", "key2": "old_value2", "key3": "old_value3"} - - @pytest.fixture(name="dmge", scope="function") def DMGE(helpers: Helpers) -> DataModelGraphExplorer: """Fixture to instantiate a DataModelGraphExplorer object.""" From ded149b5e79aea532a6fd519e7ea83598f7b5dac Mon Sep 17 00:00:00 2001 From: GiaJordan Date: Fri, 6 Sep 2024 10:31:30 -0700 Subject: [PATCH 193/233] add back deleted test --- tests/test_api.py | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/tests/test_api.py b/tests/test_api.py index c8cf78710..eb6688edf 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -1265,6 +1265,36 @@ def test_submit_manifest_w_file_and_entities( ) assert response_csv.status_code == 200 + @pytest.mark.synapse_credentials_needed + @pytest.mark.submission + def test_submit_manifest_table_and_file_upsert( + self, + client: FlaskClient, + request_headers: Dict[str, str], + test_upsert_manifest_csv: str, + ) -> None: + params = { + "schema_url": DATA_MODEL_JSON_LD, + "data_type": "MockRDB", + "restrict_rules": False, + "manifest_record_type": "table_and_file", + "asset_view": "syn51514557", + "dataset_id": "syn51514551", + "table_manipulation": "upsert", + "data_model_labels": "class_label", + # have to set table_column_names to display_name to ensure upsert feature works + "table_column_names": "display_name", + } + + # test uploading a csv file + response_csv = client.post( + "http://localhost:3001/v1/model/submit", + query_string=params, + data={"file_name": (open(test_upsert_manifest_csv, "rb"), "test.csv")}, + headers=request_headers, + ) + assert response_csv.status_code == 200 + @pytest.mark.synapse_credentials_needed @pytest.mark.submission def test_submit_and_validate_filebased_manifest( From be1751249b80422e766a1c7b0180c03fb1e7c34e Mon Sep 17 00:00:00 2001 From: GiaJordan Date: Fri, 6 Sep 2024 11:22:11 -0700 Subject: [PATCH 194/233] update params and docstring --- schematic/models/validate_attribute.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/schematic/models/validate_attribute.py b/schematic/models/validate_attribute.py index 03f33c5d9..9e43ad10a 100644 --- a/schematic/models/validate_attribute.py +++ b/schematic/models/validate_attribute.py @@ -2013,8 +2013,8 @@ def filename_validation( val_rule: str, manifest: pd.core.frame.DataFrame, access_token: str, + dataset_scope: str, project_scope: Optional[list] = None, - dataset_scope: Optional[str] = None, ): """ Purpose: @@ -2023,6 +2023,7 @@ def filename_validation( val_rule: str, Validation rule for the component manifest: pd.core.frame.DataFrame, manifest access_token: str, Asset Store access token + dataset_scope: str, Dataset with files to validate against project_scope: Optional[list] = None: Projects to limit the scope of cross manifest validation to. Returns: errors: list[str] Error details for further storage. From d467f9ca9de3bbcab5856129e743b0d30da7a0c1 Mon Sep 17 00:00:00 2001 From: GiaJordan Date: Fri, 6 Sep 2024 11:41:05 -0700 Subject: [PATCH 195/233] add type hinting --- schematic/models/commands.py | 45 ++++++++++++++++++------------------ 1 file changed, 23 insertions(+), 22 deletions(-) diff --git a/schematic/models/commands.py b/schematic/models/commands.py index aefc305e7..fbe31db1b 100644 --- a/schematic/models/commands.py +++ b/schematic/models/commands.py @@ -4,7 +4,7 @@ import sys from gc import callbacks from time import perf_counter -from typing import get_args +from typing import Optional, get_args import click import click_log @@ -148,19 +148,19 @@ def model(ctx, config): # use as `schematic model ...` @click.pass_obj def submit_manifest( ctx, - manifest_path, - dataset_id, - validate_component, - manifest_record_type, - hide_blanks, - restrict_rules, - project_scope, - dataset_scope, - table_manipulation, - data_model_labels, - table_column_names, - annotation_keys, - file_annotations_upload: bool, + manifest_path: str, + dataset_id: str, + validate_component: Optional[str], + manifest_record_type: Optional[str], + hide_blanks: Optional[bool], + restrict_rules: Optional[bool], + project_scope: Optional[list[str]], + dataset_scope: Optional[str], + table_manipulation: Optional[str], + data_model_labels: Optional[str], + table_column_names: Optional[str], + annotation_keys: Optional[str], + file_annotations_upload: Optional[bool], ): """ Running CLI with manifest validation (optional) and submission options. @@ -243,19 +243,20 @@ def submit_manifest( @click.option( "--data_model_labels", "-dml", - is_flag=True, + default="class_label", + type=click.Choice(list(get_args(DisplayLabelType)), case_sensitive=True), help=query_dict(model_commands, ("model", "validate", "data_model_labels")), ) @click.pass_obj def validate_manifest( ctx, - manifest_path, - data_type, - json_schema, - restrict_rules, - project_scope, - dataset_scope, - data_model_labels, + manifest_path: str, + data_type: Optional[list[str]], + json_schema: Optional[str], + restrict_rules: Optional[bool], + project_scope: Optional[list[str]], + dataset_scope: Optional[str], + data_model_labels: Optional[str], ): """ Running CLI for manifest validation. From 67f661e68331a33fdf08828f6bbe611f7e73266e Mon Sep 17 00:00:00 2001 From: GiaJordan Date: Fri, 6 Sep 2024 13:41:33 -0700 Subject: [PATCH 196/233] update test manifest --- tests/data/mock_manifests/ValidFilenameManifest.csv | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/data/mock_manifests/ValidFilenameManifest.csv b/tests/data/mock_manifests/ValidFilenameManifest.csv index 9ba58740b..5bf19e1c0 100644 --- a/tests/data/mock_manifests/ValidFilenameManifest.csv +++ b/tests/data/mock_manifests/ValidFilenameManifest.csv @@ -1,5 +1,5 @@ -Component,Filename,entityId -MockFilename,schematic - main/TestSubmitMockFilename/txt1.txt,syn62822369 -MockFilename,schematic - main/TestSubmitMockFilename/txt2.txt,syn62822368 -MockFilename,schematic - main/TestSubmitMockFilename/txt3.txt,syn62822366 -MockFilename,schematic - main/TestSubmitMockFilename/txt4.txt,syn62822364 +Component,Filename,Id,entityId +MockFilename,schematic - main/TestSubmitMockFilename/txt1.txt,3c4c384b-5c49-4a7c-90a6-43f03a5ddbdc,syn62822369 +MockFilename,schematic - main/TestSubmitMockFilename/txt2.txt,3b45f5f3-408f-47ff-945e-9badf0a43195,syn62822368 +MockFilename,schematic - main/TestSubmitMockFilename/txt3.txt,2bbc898f-2651-4af3-834a-10c506de0fbd,syn62822366 +MockFilename,schematic - main/TestSubmitMockFilename/txt4.txt,5a2d3816-436e-458f-9887-cb8355518e23,syn62822364 From 61e00ccc9cff0c616c4aa8a3c5d35663d828b86a Mon Sep 17 00:00:00 2001 From: GiaJordan Date: Fri, 6 Sep 2024 13:47:33 -0700 Subject: [PATCH 197/233] update pos arguments --- schematic/models/validate_manifest.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/schematic/models/validate_manifest.py b/schematic/models/validate_manifest.py index d3f4a34fa..3b85b1414 100644 --- a/schematic/models/validate_manifest.py +++ b/schematic/models/validate_manifest.py @@ -273,7 +273,11 @@ def validate_manifest_rules( ) elif validation_type == "filenameExists": vr_errors, vr_warnings = validation_method( - rule, manifest, access_token, project_scope, dataset_scope + rule, + manifest, + access_token, + dataset_scope, + project_scope, ) else: vr_errors, vr_warnings = validation_method( From 52e11ff8c063f781772d99d760edee99e514d610 Mon Sep 17 00:00:00 2001 From: GiaJordan Date: Fri, 6 Sep 2024 14:00:15 -0700 Subject: [PATCH 198/233] update test condition --- tests/test_validation.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_validation.py b/tests/test_validation.py index 69ade10ca..cf0b70aaa 100644 --- a/tests/test_validation.py +++ b/tests/test_validation.py @@ -697,7 +697,7 @@ def test_filename_manifest(self, helpers, dmge): # Check errors assert ( GenerateError.generate_filename_error( - val_rule="filenameExists syn61682648", + val_rule="filenameExists", attribute_name="Filename", row_num="3", invalid_entry="schematic - main/MockFilenameComponent/txt4.txt", @@ -709,7 +709,7 @@ def test_filename_manifest(self, helpers, dmge): assert ( GenerateError.generate_filename_error( - val_rule="filenameExists syn61682648", + val_rule="filenameExists", attribute_name="Filename", row_num="4", invalid_entry="schematic - main/MockFilenameComponent/txt5.txt", From 9aea09efa88eeeada27ebd3780c8c8277e9033fd Mon Sep 17 00:00:00 2001 From: BryanFauble <17128019+BryanFauble@users.noreply.github.com> Date: Fri, 6 Sep 2024 14:18:52 -0700 Subject: [PATCH 199/233] [fds-2373] Remove the need to do version checking on Synapse creation (#1495) --- schematic_api/api/security_controller.py | 1 + 1 file changed, 1 insertion(+) diff --git a/schematic_api/api/security_controller.py b/schematic_api/api/security_controller.py index 33cc4ad76..5b881576d 100644 --- a/schematic_api/api/security_controller.py +++ b/schematic_api/api/security_controller.py @@ -12,6 +12,7 @@ syn = Synapse( configPath=CONFIG.synapse_configuration_path, cache_client=False, + skip_checks=True, ) jwks_client = PyJWKClient( uri=syn.authEndpoint + "/oauth2/jwks", headers=syn._generate_headers() From 9c603acf9cf57664be5e56fe6ed2d8820d52876b Mon Sep 17 00:00:00 2001 From: linglp Date: Fri, 6 Sep 2024 18:31:22 -0400 Subject: [PATCH 200/233] simplify if else statement and edit test --- schematic/store/synapse.py | 111 ++++++++++++++++++------ tests/integration/test_store_synapse.py | 22 ++--- 2 files changed, 96 insertions(+), 37 deletions(-) diff --git a/schematic/store/synapse.py b/schematic/store/synapse.py index c34c98eec..71c3663a8 100644 --- a/schematic/store/synapse.py +++ b/schematic/store/synapse.py @@ -1550,27 +1550,28 @@ def process_row_annotations( or (isinstance(anno_v, float) and np.isnan(anno_v)) ): annos.pop(anno_k) if anno_k in annos.keys() else annos + # Otherwise save annotation as approrpriate else: - if annotation_keys == "display_label": - node_validation_rules = dmge.get_node_validation_rules( - node_display_name=anno_k - ) - else: - node_validation_rules = dmge.get_node_validation_rules( - node_label=anno_k - ) - if isinstance(anno_v, float) and np.isnan(anno_v): annos[anno_k] = "" - elif ( - isinstance(anno_v, str) - and re.fullmatch(csv_list_regex, anno_v) - and rule_in_rule_list("list", node_validation_rules) - ): - annos[anno_k] = anno_v.split(",") - else: - annos[anno_k] = anno_v + continue + + # Handle strings that match the csv_list_regex and pass the validation rule + if isinstance(anno_v, str) and re.fullmatch(csv_list_regex, anno_v): + # Use a dictionary to dynamically choose the argument + param = ( + {"node_display_name": anno_k} + if annotation_keys == "display_label" + else {"node_label": anno_k} + ) + node_validation_rules = dmge.get_node_validation_rules(**param) + + if rule_in_rule_list("list", node_validation_rules): + annos[anno_k] = anno_v.split(",") + continue + # If none of the conditions are met, assign the original value + annos[anno_k] = anno_v return annos @async_missing_entity_handler @@ -1626,6 +1627,30 @@ async def format_row_annotations( annos = await self.get_async_annotation(entityId) csv_list_regex = comma_separated_list_regex() + # for anno_k, anno_v in metadataSyn.items(): + # # Remove keys with nan or empty string values from dict of annotations to be uploaded + # # if present on current data annotation + # if hideBlanks and ( + # anno_v == "" or (isinstance(anno_v, float) and np.isnan(anno_v)) + # ): + # annos.pop(anno_k) if anno_k in annos.keys() else annos + # # Otherwise save annotation as approrpriate + # else: + # if isinstance(anno_v, float) and np.isnan(anno_v): + # annos[anno_k] = "" + # elif ( + # isinstance(anno_v, str) + # and re.fullmatch(csv_list_regex, anno_v) + # and rule_in_rule_list( + # "list", dmge.get_node_validation_rules(anno_k) + # ) + # ): + # annos[anno_k] = anno_v.split(",") + # else: + # annos[anno_k] = anno_v + + # return annos + annos = self.process_row_annotations( dmge=dmge, metadataSyn=metadataSyn, @@ -1907,6 +1932,34 @@ async def _process_store_annos(self, requests: Set[asyncio.Task]) -> None: except Exception as e: raise RuntimeError(f"failed with { repr(e) }.") from e + def count_entity_id(self, manifest: pd.DataFrame) -> int: + """Check if there are any non-NaN values in the original manifest's entityId column + + Args: + manifest (pd.DataFrame): manifest dataframe + + Returns: + int: The count of non-NaN entityId values. + """ + # Normalize the column names to lowercase + normalized_columns = {col.lower(): col for col in manifest.columns} + + # Check if a case-insensitive 'entityid' column exists + if "entityid" in normalized_columns: + entity_id_column = normalized_columns["entityid"] + entity_id_count = manifest[entity_id_column].notna().sum() + else: + entity_id_count = 0 + return entity_id_count + + def handle_missing_entity_ids( + original_entity_id_count: int, merged_entity_id_count: int + ): + if original_entity_id_count != merged_entity_id_count: + raise LookupError( + "Some entityId values became NaN due to unmatched Filename" + ) + @tracer.start_as_current_span("SynapseStorage::add_annotations_to_entities_files") async def add_annotations_to_entities_files( self, @@ -1943,20 +1996,26 @@ async def add_annotations_to_entities_files( ) file_df = pd.DataFrame(files_and_entityIds) + # count entity ids of the original manifest + original_manifest_entity_id_count = self.count_entity_id(manifest) + # Merge dataframes to add entityIds manifest = manifest.merge( file_df, how="left", on="Filename", suffixes=["_x", None] - ) + ).drop("entityId_x", axis=1) + + # count entity ids after manifest gets merged + merged_manifest_entity_id_count = self.count_entity_id(manifest) # drop the duplicate entity column with NA values - col_to_drop = "entityId_x" - if manifest.entityId.isnull().all(): - col_to_drop = "entityId" - - # If the original entityId column is empty after the merge, drop it and rename the duplicate column - manifest.drop(columns=[col_to_drop], inplace=True) - if col_to_drop == "entityId": - manifest.rename(columns={"entityId_x": "entityId"}, inplace=True) + # col_to_drop = "entityId_x" + # if manifest.entityId.isnull().all(): + # col_to_drop = "entityId" + + # # If the original entityId column is empty after the merge, drop it and rename the duplicate column + # manifest.drop(columns=[col_to_drop], inplace=True) + # if col_to_drop == "entityId": + # manifest.rename(columns={"entityId_x": "entityId"}, inplace=True) # Fill `entityId` for each row if missing and annotate entity as appropriate requests = set() diff --git a/tests/integration/test_store_synapse.py b/tests/integration/test_store_synapse.py index 84caaf5bb..d7a64b9cf 100644 --- a/tests/integration/test_store_synapse.py +++ b/tests/integration/test_store_synapse.py @@ -86,34 +86,34 @@ def test_process_row_annotations_get_validation( self, dmge: DataModelGraphExplorer, synapse_store: SynapseStorage, + hideBlanks: bool, label_options: str, ): """ensure that get_node_validation_rules is called with the correct arguments""" comma_separated_list = comma_separated_list_regex() metadata_syn = { - "PatientID": "value1", + "FamilyHistory": "value1,value2,value3", } - annos = {"PatientID": "old_value"} + annos = {"FamilyHistory": "old_value"} dmge.get_node_validation_rules = MagicMock() process_row_annos = synapse_store.process_row_annotations( dmge=dmge, metadataSyn=metadata_syn, csv_list_regex=comma_separated_list, - hideBlanks=True, + hideBlanks=hideBlanks, annos=annos, annotation_keys=label_options, ) if label_options == "display_label": - # get_node_validation_rules was called with node_display_name="PatientID" at least once + # get_node_validation_rules was called with node_display_name + dmge.get_node_validation_rules.assert_any_call( + node_display_name="FamilyHistory" + ) dmge.get_node_validation_rules.assert_any_call( - node_display_name="PatientID" + node_display_name="FamilyHistory" ) - # get_node_validation_rules was called with node_display_name="Sex" at least once - dmge.get_node_validation_rules.assert_any_call(node_display_name="Sex") else: - # get_node_validation_rules was called with node_label="PatientID" at least once - dmge.get_node_validation_rules.assert_any_call(node_label="PatientID") - # get_node_validation_rules was called with node_label="Sex" at least once - dmge.get_node_validation_rules.assert_any_call(node_label="Sex") + # get_node_validation_rules was called with node_label + dmge.get_node_validation_rules.assert_any_call(node_label="FamilyHistory") From 81a01a5405cd409610172965c23abf6e183eac02 Mon Sep 17 00:00:00 2001 From: linglp Date: Sat, 7 Sep 2024 18:12:24 -0400 Subject: [PATCH 201/233] simplify if else logic and modify test --- schematic/store/synapse.py | 78 +++++++++---------------- tests/integration/test_store_synapse.py | 58 +++++++++--------- 2 files changed, 59 insertions(+), 77 deletions(-) diff --git a/schematic/store/synapse.py b/schematic/store/synapse.py index 71c3663a8..666913044 100644 --- a/schematic/store/synapse.py +++ b/schematic/store/synapse.py @@ -1523,17 +1523,17 @@ async def store_async_annotation(self, annotation_dict: dict) -> Annotations: def process_row_annotations( self, dmge: DataModelGraphExplorer, - metadataSyn: dict, - hideBlanks: bool, + metadata_syn: Dict[str, Any], + hide_blanks: bool, csv_list_regex: str, - annos: dict, + annos: Dict[str, Any], annotation_keys: str, - ): + ) -> Dict[str, Any]: """processes metadata annotations Args: dmge (DataModelGraphExplorer): data model graph explorer - metadataSyn (dict): metadata used for Synapse storage + metadata_syn (dict): metadata used for Synapse storage hideBlanks (bool): if true, does not upload annotation keys with blank values. csv_list_regex (str): annos (dict): @@ -1542,36 +1542,36 @@ def process_row_annotations( Returns: dict: annotations as a dictionary """ - for anno_k, anno_v in metadataSyn.items(): + for anno_k, anno_v in metadata_syn.items(): # Remove keys with nan or empty string values from dict of annotations to be uploaded # if present on current data annotation - if hideBlanks and ( + if hide_blanks and ( (isinstance(anno_v, str) and anno_v.strip() == "") or (isinstance(anno_v, float) and np.isnan(anno_v)) ): annos.pop(anno_k) if anno_k in annos.keys() else annos + continue # Otherwise save annotation as approrpriate - else: - if isinstance(anno_v, float) and np.isnan(anno_v): - annos[anno_k] = "" - continue + if isinstance(anno_v, float) and np.isnan(anno_v): + annos[anno_k] = "" + continue - # Handle strings that match the csv_list_regex and pass the validation rule - if isinstance(anno_v, str) and re.fullmatch(csv_list_regex, anno_v): - # Use a dictionary to dynamically choose the argument - param = ( - {"node_display_name": anno_k} - if annotation_keys == "display_label" - else {"node_label": anno_k} - ) - node_validation_rules = dmge.get_node_validation_rules(**param) + # Handle strings that match the csv_list_regex and pass the validation rule + if isinstance(anno_v, str) and re.fullmatch(csv_list_regex, anno_v): + # Use a dictionary to dynamically choose the argument + param = ( + {"node_display_name": anno_k} + if annotation_keys == "display_label" + else {"node_label": anno_k} + ) + node_validation_rules = dmge.get_node_validation_rules(**param) - if rule_in_rule_list("list", node_validation_rules): - annos[anno_k] = anno_v.split(",") - continue - # If none of the conditions are met, assign the original value - annos[anno_k] = anno_v + if rule_in_rule_list("list", node_validation_rules): + annos[anno_k] = anno_v.split(",") + continue + # default: assign the original value + annos[anno_k] = anno_v return annos @async_missing_entity_handler @@ -1627,34 +1627,10 @@ async def format_row_annotations( annos = await self.get_async_annotation(entityId) csv_list_regex = comma_separated_list_regex() - # for anno_k, anno_v in metadataSyn.items(): - # # Remove keys with nan or empty string values from dict of annotations to be uploaded - # # if present on current data annotation - # if hideBlanks and ( - # anno_v == "" or (isinstance(anno_v, float) and np.isnan(anno_v)) - # ): - # annos.pop(anno_k) if anno_k in annos.keys() else annos - # # Otherwise save annotation as approrpriate - # else: - # if isinstance(anno_v, float) and np.isnan(anno_v): - # annos[anno_k] = "" - # elif ( - # isinstance(anno_v, str) - # and re.fullmatch(csv_list_regex, anno_v) - # and rule_in_rule_list( - # "list", dmge.get_node_validation_rules(anno_k) - # ) - # ): - # annos[anno_k] = anno_v.split(",") - # else: - # annos[anno_k] = anno_v - - # return annos - annos = self.process_row_annotations( dmge=dmge, - metadataSyn=metadataSyn, - hideBlanks=hideBlanks, + metadata_syn=metadataSyn, + hide_blanks=hideBlanks, csv_list_regex=csv_list_regex, annos=annos, annotation_keys=annotation_keys, diff --git a/tests/integration/test_store_synapse.py b/tests/integration/test_store_synapse.py index d7a64b9cf..15f17f5b7 100644 --- a/tests/integration/test_store_synapse.py +++ b/tests/integration/test_store_synapse.py @@ -27,10 +27,9 @@ def test_process_row_annotations_hide_blanks( self, dmge: DataModelGraphExplorer, synapse_store: SynapseStorage, - annos: dict, hideBlanks: bool, label_options: str, - ): + ) -> None: """ensure that blank values are not added to the annotations dictionary if hideBlanks is True""" metadata_syn_with_blanks = { @@ -42,39 +41,40 @@ def test_process_row_annotations_hide_blanks( "CancerType": " ", # Blank value (whitespace string) } annos = { - "PatientID": "value1", - "Sex": "value2", - "Diagnosis": "value3", - "FamilyHistory": "value4", - "YearofBirth": "value5", - "CancerType": "value6", + "PatientID": "old_value1", + "Sex": "old_value2", + "Diagnosis": "old_value3", + "FamilyHistory": "old_value4", + "YearofBirth": "old_value5", + "CancerType": "old_value6", } comma_separated_list = comma_separated_list_regex() - process_row_annos = synapse_store.process_row_annotations( + processed_annos = synapse_store.process_row_annotations( dmge=dmge, - metadataSyn=metadata_syn_with_blanks, + metadata_syn=metadata_syn_with_blanks, csv_list_regex=comma_separated_list, - hideBlanks=hideBlanks, + hide_blanks=hideBlanks, annos=annos, annotation_keys=label_options, ) - # make sure that empty keys are not added if hideBlanks is True + # make sure that empty keys are removed if hideBlanks is True if hideBlanks: assert ( "Diagnosis" and "YearofBirth" - and "CancerType" not in process_row_annos.keys() + and "CancerType" not in processed_annos.keys() ) - assert ( - "Diagnosis" - and "YearofBirth" - and "CancerType" - and "PatientID" - and "Sex" - and "FamilyHistory" in process_row_annos.keys() - ) + else: + # make sure that empty keys are added if hideBlanks is False + # make sure that nan values are converted to empty strings + assert processed_annos["Diagnosis"] == "" + assert processed_annos["YearofBirth"] == "" + assert processed_annos["CancerType"] == " " + # make sure that annotations already in the dictionary are not overwritten - assert "PatientID" and "Sex" in process_row_annos.keys() + assert processed_annos["PatientID"] == "value1" + assert processed_annos["Sex"] == "value2" + assert processed_annos["FamilyHistory"] == 3 @pytest.mark.parametrize( "label_options", @@ -88,7 +88,7 @@ def test_process_row_annotations_get_validation( synapse_store: SynapseStorage, hideBlanks: bool, label_options: str, - ): + ) -> None: """ensure that get_node_validation_rules is called with the correct arguments""" comma_separated_list = comma_separated_list_regex() metadata_syn = { @@ -97,11 +97,15 @@ def test_process_row_annotations_get_validation( annos = {"FamilyHistory": "old_value"} dmge.get_node_validation_rules = MagicMock() - process_row_annos = synapse_store.process_row_annotations( + + # pretend that "FamilyHistory" has a list of validation rules + dmge.get_node_validation_rules.return_value = ["list", "regex"] + + processed_annos = synapse_store.process_row_annotations( dmge=dmge, - metadataSyn=metadata_syn, + metadata_syn=metadata_syn, csv_list_regex=comma_separated_list, - hideBlanks=hideBlanks, + hide_blanks=hideBlanks, annos=annos, annotation_keys=label_options, ) @@ -117,3 +121,5 @@ def test_process_row_annotations_get_validation( else: # get_node_validation_rules was called with node_label dmge.get_node_validation_rules.assert_any_call(node_label="FamilyHistory") + # ensure that the value is split into a list + assert processed_annos["FamilyHistory"] == ["value1", "value2", "value3"] From d132a989c194a463b7f2a3ad8be9df67dfb13b04 Mon Sep 17 00:00:00 2001 From: BryanFauble <17128019+BryanFauble@users.noreply.github.com> Date: Mon, 9 Sep 2024 08:42:58 -0700 Subject: [PATCH 202/233] [FDS-305] Enable table tests and Prevent shared snypase resources during integration tests (#1482) * Prevent shared snypase resources during integration tests --- .github/workflows/test.yml | 16 +- env.example | 9 +- poetry.lock | 114 ++++++++++-- pyproject.toml | 7 +- pytest.ini | 5 +- schematic/store/synapse.py | 37 ++-- schematic_api/api/routes.py | 4 +- tests/conftest.py | 109 +++++++++++- tests/test_store.py | 339 +++++++++++++++++++++--------------- tests/utils.py | 34 ++++ 10 files changed, 494 insertions(+), 180 deletions(-) create mode 100644 tests/utils.py diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2bea3258b..a868a2a62 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -7,7 +7,7 @@ name: Test schematic on: push: - branches: ['main'] + branches: ['main', 'develop'] pull_request: branches: ['*'] workflow_dispatch: # Allow manually triggering the workflow @@ -46,6 +46,18 @@ jobs: with: python-version: ${{ matrix.python-version }} + #---------------------------------------------- + # verify runner environment + #---------------------------------------------- + # - name: Print runner environment information + # run: | + # echo "Running on runner: $RUNNER_NAME" + # echo "Runner OS: $RUNNER_OS" + # echo "Runner OS version: $RUNNER_OS_VERSION" + # echo "Runner architecture: $RUNNER_ARCH" + # echo "Total memory: $(free -h)" + # echo "CPU info: $(lscpu)" + #---------------------------------------------- # install & configure poetry #---------------------------------------------- @@ -122,7 +134,7 @@ jobs: run: > source .venv/bin/activate; pytest --durations=0 --cov-report=term --cov-report=html:htmlcov --cov-report=xml:coverage.xml --cov=schematic/ - -m "not (rule_benchmark or table_operations)" --reruns 2 -n auto + -m "not (rule_benchmark)" --reruns 4 -n 8 - name: Upload pytest test results diff --git a/env.example b/env.example index afeeda77b..176c22c28 100644 --- a/env.example +++ b/env.example @@ -3,4 +3,11 @@ SERVER_PROTOCOL=http:// SERVER_DOMAIN=localhost # port on the host machine USE_LISTEN_PORT=81 -SERVICE_ACCOUNT_CREDS='Provide service account creds' \ No newline at end of file +SERVICE_ACCOUNT_CREDS='Provide service account creds' + +# Integration testing variables (Optional) +# TRACING_EXPORT_FORMAT=otlp +# LOGGING_EXPORT_FORMAT=otlp +# TRACING_SERVICE_NAME=unique-name-testing +# LOGGING_SERVICE_NAME=unique-name-testing +# LOGGING_INSTANCE_NAME=unique-name-testing \ No newline at end of file diff --git a/poetry.lock b/poetry.lock index 954deb011..47d2e1528 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1399,6 +1399,64 @@ files = [ docs = ["Sphinx", "furo"] test = ["objgraph", "psutil"] +[[package]] +name = "grpcio" +version = "1.66.1" +description = "HTTP/2-based RPC framework" +optional = true +python-versions = ">=3.8" +files = [ + {file = "grpcio-1.66.1-cp310-cp310-linux_armv7l.whl", hash = "sha256:4877ba180591acdf127afe21ec1c7ff8a5ecf0fe2600f0d3c50e8c4a1cbc6492"}, + {file = "grpcio-1.66.1-cp310-cp310-macosx_12_0_universal2.whl", hash = "sha256:3750c5a00bd644c75f4507f77a804d0189d97a107eb1481945a0cf3af3e7a5ac"}, + {file = "grpcio-1.66.1-cp310-cp310-manylinux_2_17_aarch64.whl", hash = "sha256:a013c5fbb12bfb5f927444b477a26f1080755a931d5d362e6a9a720ca7dbae60"}, + {file = "grpcio-1.66.1-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b1b24c23d51a1e8790b25514157d43f0a4dce1ac12b3f0b8e9f66a5e2c4c132f"}, + {file = "grpcio-1.66.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b7ffb8ea674d68de4cac6f57d2498fef477cef582f1fa849e9f844863af50083"}, + {file = "grpcio-1.66.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:307b1d538140f19ccbd3aed7a93d8f71103c5d525f3c96f8616111614b14bf2a"}, + {file = "grpcio-1.66.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:1c17ebcec157cfb8dd445890a03e20caf6209a5bd4ac5b040ae9dbc59eef091d"}, + {file = "grpcio-1.66.1-cp310-cp310-win32.whl", hash = "sha256:ef82d361ed5849d34cf09105d00b94b6728d289d6b9235513cb2fcc79f7c432c"}, + {file = "grpcio-1.66.1-cp310-cp310-win_amd64.whl", hash = "sha256:292a846b92cdcd40ecca46e694997dd6b9be6c4c01a94a0dfb3fcb75d20da858"}, + {file = "grpcio-1.66.1-cp311-cp311-linux_armv7l.whl", hash = "sha256:c30aeceeaff11cd5ddbc348f37c58bcb96da8d5aa93fed78ab329de5f37a0d7a"}, + {file = "grpcio-1.66.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:8a1e224ce6f740dbb6b24c58f885422deebd7eb724aff0671a847f8951857c26"}, + {file = "grpcio-1.66.1-cp311-cp311-manylinux_2_17_aarch64.whl", hash = "sha256:a66fe4dc35d2330c185cfbb42959f57ad36f257e0cc4557d11d9f0a3f14311df"}, + {file = "grpcio-1.66.1-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e3ba04659e4fce609de2658fe4dbf7d6ed21987a94460f5f92df7579fd5d0e22"}, + {file = "grpcio-1.66.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4573608e23f7e091acfbe3e84ac2045680b69751d8d67685ffa193a4429fedb1"}, + {file = "grpcio-1.66.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:7e06aa1f764ec8265b19d8f00140b8c4b6ca179a6dc67aa9413867c47e1fb04e"}, + {file = "grpcio-1.66.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:3885f037eb11f1cacc41f207b705f38a44b69478086f40608959bf5ad85826dd"}, + {file = "grpcio-1.66.1-cp311-cp311-win32.whl", hash = "sha256:97ae7edd3f3f91480e48ede5d3e7d431ad6005bfdbd65c1b56913799ec79e791"}, + {file = "grpcio-1.66.1-cp311-cp311-win_amd64.whl", hash = "sha256:cfd349de4158d797db2bd82d2020554a121674e98fbe6b15328456b3bf2495bb"}, + {file = "grpcio-1.66.1-cp312-cp312-linux_armv7l.whl", hash = "sha256:a92c4f58c01c77205df6ff999faa008540475c39b835277fb8883b11cada127a"}, + {file = "grpcio-1.66.1-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:fdb14bad0835914f325349ed34a51940bc2ad965142eb3090081593c6e347be9"}, + {file = "grpcio-1.66.1-cp312-cp312-manylinux_2_17_aarch64.whl", hash = "sha256:f03a5884c56256e08fd9e262e11b5cfacf1af96e2ce78dc095d2c41ccae2c80d"}, + {file = "grpcio-1.66.1-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2ca2559692d8e7e245d456877a85ee41525f3ed425aa97eb7a70fc9a79df91a0"}, + {file = "grpcio-1.66.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:84ca1be089fb4446490dd1135828bd42a7c7f8421e74fa581611f7afdf7ab761"}, + {file = "grpcio-1.66.1-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:d639c939ad7c440c7b2819a28d559179a4508783f7e5b991166f8d7a34b52815"}, + {file = "grpcio-1.66.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:b9feb4e5ec8dc2d15709f4d5fc367794d69277f5d680baf1910fc9915c633524"}, + {file = "grpcio-1.66.1-cp312-cp312-win32.whl", hash = "sha256:7101db1bd4cd9b880294dec41a93fcdce465bdbb602cd8dc5bd2d6362b618759"}, + {file = "grpcio-1.66.1-cp312-cp312-win_amd64.whl", hash = "sha256:b0aa03d240b5539648d996cc60438f128c7f46050989e35b25f5c18286c86734"}, + {file = "grpcio-1.66.1-cp38-cp38-linux_armv7l.whl", hash = "sha256:ecfe735e7a59e5a98208447293ff8580e9db1e890e232b8b292dc8bd15afc0d2"}, + {file = "grpcio-1.66.1-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:4825a3aa5648010842e1c9d35a082187746aa0cdbf1b7a2a930595a94fb10fce"}, + {file = "grpcio-1.66.1-cp38-cp38-manylinux_2_17_aarch64.whl", hash = "sha256:f517fd7259fe823ef3bd21e508b653d5492e706e9f0ef82c16ce3347a8a5620c"}, + {file = "grpcio-1.66.1-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f1fe60d0772831d96d263b53d83fb9a3d050a94b0e94b6d004a5ad111faa5b5b"}, + {file = "grpcio-1.66.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:31a049daa428f928f21090403e5d18ea02670e3d5d172581670be006100db9ef"}, + {file = "grpcio-1.66.1-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:6f914386e52cbdeb5d2a7ce3bf1fdfacbe9d818dd81b6099a05b741aaf3848bb"}, + {file = "grpcio-1.66.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:bff2096bdba686019fb32d2dde45b95981f0d1490e054400f70fc9a8af34b49d"}, + {file = "grpcio-1.66.1-cp38-cp38-win32.whl", hash = "sha256:aa8ba945c96e73de29d25331b26f3e416e0c0f621e984a3ebdb2d0d0b596a3b3"}, + {file = "grpcio-1.66.1-cp38-cp38-win_amd64.whl", hash = "sha256:161d5c535c2bdf61b95080e7f0f017a1dfcb812bf54093e71e5562b16225b4ce"}, + {file = "grpcio-1.66.1-cp39-cp39-linux_armv7l.whl", hash = "sha256:d0cd7050397b3609ea51727b1811e663ffda8bda39c6a5bb69525ef12414b503"}, + {file = "grpcio-1.66.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:0e6c9b42ded5d02b6b1fea3a25f036a2236eeb75d0579bfd43c0018c88bf0a3e"}, + {file = "grpcio-1.66.1-cp39-cp39-manylinux_2_17_aarch64.whl", hash = "sha256:c9f80f9fad93a8cf71c7f161778ba47fd730d13a343a46258065c4deb4b550c0"}, + {file = "grpcio-1.66.1-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5dd67ed9da78e5121efc5c510f0122a972216808d6de70953a740560c572eb44"}, + {file = "grpcio-1.66.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:48b0d92d45ce3be2084b92fb5bae2f64c208fea8ceed7fccf6a7b524d3c4942e"}, + {file = "grpcio-1.66.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:4d813316d1a752be6f5c4360c49f55b06d4fe212d7df03253dfdae90c8a402bb"}, + {file = "grpcio-1.66.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:9c9bebc6627873ec27a70fc800f6083a13c70b23a5564788754b9ee52c5aef6c"}, + {file = "grpcio-1.66.1-cp39-cp39-win32.whl", hash = "sha256:30a1c2cf9390c894c90bbc70147f2372130ad189cffef161f0432d0157973f45"}, + {file = "grpcio-1.66.1-cp39-cp39-win_amd64.whl", hash = "sha256:17663598aadbedc3cacd7bbde432f541c8e07d2496564e22b214b22c7523dac8"}, + {file = "grpcio-1.66.1.tar.gz", hash = "sha256:35334f9c9745add3e357e3372756fd32d925bd52c41da97f4dfdafbde0bf0ee2"}, +] + +[package.extras] +protobuf = ["grpcio-tools (>=1.66.1)"] + [[package]] name = "h11" version = "0.14.0" @@ -2594,6 +2652,30 @@ files = [ backoff = {version = ">=1.10.0,<3.0.0", markers = "python_version >= \"3.7\""} opentelemetry-proto = "1.21.0" +[[package]] +name = "opentelemetry-exporter-otlp-proto-grpc" +version = "1.21.0" +description = "OpenTelemetry Collector Protobuf over gRPC Exporter" +optional = true +python-versions = ">=3.7" +files = [ + {file = "opentelemetry_exporter_otlp_proto_grpc-1.21.0-py3-none-any.whl", hash = "sha256:ab37c63d6cb58d6506f76d71d07018eb1f561d83e642a8f5aa53dddf306087a4"}, + {file = "opentelemetry_exporter_otlp_proto_grpc-1.21.0.tar.gz", hash = "sha256:a497c5611245a2d17d9aa1e1cbb7ab567843d53231dcc844a62cea9f0924ffa7"}, +] + +[package.dependencies] +backoff = {version = ">=1.10.0,<3.0.0", markers = "python_version >= \"3.7\""} +deprecated = ">=1.2.6" +googleapis-common-protos = ">=1.52,<2.0" +grpcio = ">=1.0.0,<2.0.0" +opentelemetry-api = ">=1.15,<2.0" +opentelemetry-exporter-otlp-proto-common = "1.21.0" +opentelemetry-proto = "1.21.0" +opentelemetry-sdk = ">=1.21.0,<1.22.0" + +[package.extras] +test = ["pytest-grpc"] + [[package]] name = "opentelemetry-exporter-otlp-proto-http" version = "1.21.0" @@ -3256,13 +3338,13 @@ diagrams = ["jinja2", "railroad-diagrams"] [[package]] name = "pytest" -version = "7.4.4" +version = "8.3.2" description = "pytest: simple powerful testing with Python" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "pytest-7.4.4-py3-none-any.whl", hash = "sha256:b090cdf5ed60bf4c45261be03239c2c1c22df034fbffe691abe93cd80cea01d8"}, - {file = "pytest-7.4.4.tar.gz", hash = "sha256:2cf0005922c6ace4a3e2ec8b4080eb0d9753fdc93107415332f50ce9e7994280"}, + {file = "pytest-8.3.2-py3-none-any.whl", hash = "sha256:4ba08f9ae7dcf84ded419494d229b48d0903ea6407b030eaec46df5e6a73bba5"}, + {file = "pytest-8.3.2.tar.gz", hash = "sha256:c132345d12ce551242c87269de812483f5bcc87cdbb4722e48487ba194f9fdce"}, ] [package.dependencies] @@ -3270,25 +3352,25 @@ colorama = {version = "*", markers = "sys_platform == \"win32\""} exceptiongroup = {version = ">=1.0.0rc8", markers = "python_version < \"3.11\""} iniconfig = "*" packaging = "*" -pluggy = ">=0.12,<2.0" -tomli = {version = ">=1.0.0", markers = "python_version < \"3.11\""} +pluggy = ">=1.5,<2" +tomli = {version = ">=1", markers = "python_version < \"3.11\""} [package.extras] -testing = ["argcomplete", "attrs (>=19.2.0)", "hypothesis (>=3.56)", "mock", "nose", "pygments (>=2.7.2)", "requests", "setuptools", "xmlschema"] +dev = ["argcomplete", "attrs (>=19.2)", "hypothesis (>=3.56)", "mock", "pygments (>=2.7.2)", "requests", "setuptools", "xmlschema"] [[package]] name = "pytest-asyncio" -version = "0.23.8" +version = "0.24.0" description = "Pytest support for asyncio" optional = false python-versions = ">=3.8" files = [ - {file = "pytest_asyncio-0.23.8-py3-none-any.whl", hash = "sha256:50265d892689a5faefb84df80819d1ecef566eb3549cf915dfb33569359d1ce2"}, - {file = "pytest_asyncio-0.23.8.tar.gz", hash = "sha256:759b10b33a6dc61cce40a8bd5205e302978bbbcc00e279a8b61d9a6a3c82e4d3"}, + {file = "pytest_asyncio-0.24.0-py3-none-any.whl", hash = "sha256:a811296ed596b69bf0b6f3dc40f83bcaf341b155a269052d82efa2b25ac7037b"}, + {file = "pytest_asyncio-0.24.0.tar.gz", hash = "sha256:d081d828e576d85f875399194281e92bf8a68d60d72d1a2faf2feddb6c46b276"}, ] [package.dependencies] -pytest = ">=7.0.0,<9" +pytest = ">=8.2,<9" [package.extras] docs = ["sphinx (>=5.3)", "sphinx-rtd-theme (>=1.0)"] @@ -4702,13 +4784,13 @@ test = ["argcomplete (>=3.0.3)", "mypy (>=1.7.0)", "pre-commit", "pytest (>=7.0, [[package]] name = "types-python-dateutil" -version = "2.9.0.20240821" +version = "2.9.0.20240906" description = "Typing stubs for python-dateutil" optional = false python-versions = ">=3.8" files = [ - {file = "types-python-dateutil-2.9.0.20240821.tar.gz", hash = "sha256:9649d1dcb6fef1046fb18bebe9ea2aa0028b160918518c34589a46045f6ebd98"}, - {file = "types_python_dateutil-2.9.0.20240821-py3-none-any.whl", hash = "sha256:f5889fcb4e63ed4aaa379b44f93c32593d50b9a94c9a60a0c854d8cc3511cd57"}, + {file = "types-python-dateutil-2.9.0.20240906.tar.gz", hash = "sha256:9706c3b68284c25adffc47319ecc7947e5bb86b3773f843c73906fd598bc176e"}, + {file = "types_python_dateutil-2.9.0.20240906-py3-none-any.whl", hash = "sha256:27c8cc2d058ccb14946eebcaaa503088f4f6dbc4fb6093d3d456a49aef2753f6"}, ] [[package]] @@ -5032,10 +5114,10 @@ test = ["big-O", "importlib-resources", "jaraco.functools", "jaraco.itertools", type = ["pytest-mypy"] [extras] -api = ["Flask", "Flask-Cors", "Jinja2", "connexion", "flask-opentracing", "jaeger-client", "pyopenssl"] +api = ["Flask", "Flask-Cors", "Jinja2", "connexion", "flask-opentracing", "jaeger-client", "opentelemetry-exporter-otlp-proto-grpc", "pyopenssl"] aws = ["uWSGI"] [metadata] lock-version = "2.0" python-versions = ">=3.9.0,<3.11" -content-hash = "3d9d2fe8fa5fa8ef6328133f3b510bd5a6ec42632b3972442223093bfeaf59b8" +content-hash = "f814725d68db731c704f4ebcd169ae71e4031f0d939c3ce789145ddcb5f196eb" diff --git a/pyproject.toml b/pyproject.toml index 7b988c572..60c27c1c3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -75,21 +75,22 @@ Flask-Cors = {version = "^3.0.10", optional = true} uWSGI = {version = "^2.0.21", optional = true} Jinja2 = {version = ">2.11.3", optional = true} asyncio = "^3.4.3" -pytest-asyncio = "^0.23.7" jaeger-client = {version = "^4.8.0", optional = true} flask-opentracing = {version="^2.0.0", optional = true} PyJWT = "^2.9.0" +opentelemetry-exporter-otlp-proto-grpc = {version="^1.0.0", optional = true} [tool.poetry.extras] -api = ["connexion", "Flask", "Flask-Cors", "Jinja2", "pyopenssl", "jaeger-client", "flask-opentracing"] +api = ["connexion", "Flask", "Flask-Cors", "Jinja2", "pyopenssl", "jaeger-client", "flask-opentracing", "opentelemetry-exporter-otlp-proto-grpc"] aws = ["uWSGI"] [tool.poetry.group.dev.dependencies] -pytest = "^7.0.0" +pytest = "^8.0.0" pytest-cov = "^4.0.0" pytest-mock = "^3.5.1" pytest-rerunfailures = "^12.0" +pytest-asyncio = "^0.24.0" flake8 = "^6.0.0" python-dotenv = "^0.21.0" black = "^23.7.0" #If the version spec of black is changed here, the version specified in .pre-commit-config.yaml should be updated to match diff --git a/pytest.ini b/pytest.ini index faf59a4ad..982e6ef86 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,3 +1,6 @@ [pytest] python_files = test_*.py -asyncio_mode = auto \ No newline at end of file +asyncio_mode = auto +asyncio_default_fixture_loop_scope = session +log_cli = False +log_cli_level = INFO \ No newline at end of file diff --git a/schematic/store/synapse.py b/schematic/store/synapse.py index 139002a20..45f5b7335 100644 --- a/schematic/store/synapse.py +++ b/schematic/store/synapse.py @@ -189,6 +189,7 @@ class SynapseStorage(BaseStorage): TODO: Need to define the interface and rename and/or refactor some of the methods below. """ + @tracer.start_as_current_span("SynapseStorage::__init__") def __init__( self, token: Optional[str] = None, # optional parameter retrieved from browser cookie @@ -223,6 +224,7 @@ def __init__( if perform_query: self.query_fileview(columns=columns, where_clauses=where_clauses) + @tracer.start_as_current_span("SynapseStorage::_purge_synapse_cache") def _purge_synapse_cache( self, maximum_storage_allowed_cache_gb: int = 1, minute_buffer: int = 15 ) -> None: @@ -1207,6 +1209,7 @@ def move_entities_to_new_project( ) return manifests, manifest_loaded + @tracer.start_as_current_span("SynapseStorage::get_synapse_table") def get_synapse_table(self, synapse_id: str) -> Tuple[pd.DataFrame, CsvFileTable]: """Download synapse table as a pd dataframe; return table schema and etags as results too @@ -1219,6 +1222,7 @@ def get_synapse_table(self, synapse_id: str) -> Tuple[pd.DataFrame, CsvFileTable return df, results + @tracer.start_as_current_span("SynapseStorage::_get_tables") def _get_tables(self, datasetId: str = None, projectId: str = None) -> List[Table]: if projectId: project = projectId @@ -1790,8 +1794,10 @@ def _add_id_columns_to_manifest( def _generate_table_name(self, manifest): """Helper function to generate a table name for upload to synapse. + Args: Manifest loaded as a pd.Dataframe + Returns: table_name (str): Name of the table to load component_name (str): Name of the manifest component (if applicable) @@ -2239,9 +2245,9 @@ def associateMetadataWithFiles( # Upload manifest to synapse based on user input (manifest_record_type) if manifest_record_type == "file_only": manifest_synapse_file_id = self.upload_manifest_as_csv( - dmge, - manifest, - metadataManifestPath, + dmge=dmge, + manifest=manifest, + metadataManifestPath=metadataManifestPath, datasetId=datasetId, restrict=restrict_manifest, hideBlanks=hideBlanks, @@ -2252,9 +2258,9 @@ def associateMetadataWithFiles( ) elif manifest_record_type == "table_and_file": manifest_synapse_file_id = self.upload_manifest_as_table( - dmge, - manifest, - metadataManifestPath, + dmge=dmge, + manifest=manifest, + metadataManifestPath=metadataManifestPath, datasetId=datasetId, table_name=table_name, component_name=component_name, @@ -2268,9 +2274,9 @@ def associateMetadataWithFiles( ) elif manifest_record_type == "file_and_entities": manifest_synapse_file_id = self.upload_manifest_as_csv( - dmge, - manifest, - metadataManifestPath, + dmge=dmge, + manifest=manifest, + metadataManifestPath=metadataManifestPath, datasetId=datasetId, restrict=restrict_manifest, hideBlanks=hideBlanks, @@ -2281,9 +2287,9 @@ def associateMetadataWithFiles( ) elif manifest_record_type == "table_file_and_entities": manifest_synapse_file_id = self.upload_manifest_combo( - dmge, - manifest, - metadataManifestPath, + dmge=dmge, + manifest=manifest, + metadataManifestPath=metadataManifestPath, datasetId=datasetId, table_name=table_name, component_name=component_name, @@ -2465,6 +2471,7 @@ def checkIfinAssetView(self, syn_id) -> str: else: return False + @tracer.start_as_current_span("SynapseStorage::getDatasetProject") @retry( stop=stop_after_attempt(5), wait=wait_chain( @@ -2602,6 +2609,7 @@ def __init__( self.existingTableId = existingTableId self.restrict = restrict + @tracer.start_as_current_span("TableOperations::createTable") def createTable( self, columnTypeDict: dict = None, @@ -2670,6 +2678,7 @@ def createTable( table = self.synStore.syn.store(table, isRestricted=self.restrict) return table.schema.id + @tracer.start_as_current_span("TableOperations::replaceTable") def replaceTable( self, specifySchema: bool = True, @@ -2764,6 +2773,7 @@ def replaceTable( existing_table.drop(columns=["ROW_ID", "ROW_VERSION"], inplace=True) return self.existingTableId + @tracer.start_as_current_span("TableOperations::_get_auth_token") def _get_auth_token( self, ): @@ -2807,6 +2817,7 @@ def _get_auth_token( return authtoken + @tracer.start_as_current_span("TableOperations::upsertTable") def upsertTable(self, dmge: DataModelGraphExplorer): """ Method to upsert rows from a new manifest into an existing table on synapse @@ -2847,6 +2858,7 @@ def upsertTable(self, dmge: DataModelGraphExplorer): return self.existingTableId + @tracer.start_as_current_span("TableOperations::_update_table_uuid_column") def _update_table_uuid_column( self, dmge: DataModelGraphExplorer, @@ -2912,6 +2924,7 @@ def _update_table_uuid_column( return + @tracer.start_as_current_span("TableOperations::updateTable") def updateTable( self, update_col: str = "Id", diff --git a/schematic_api/api/routes.py b/schematic_api/api/routes.py index a286f30bc..e977e480d 100644 --- a/schematic_api/api/routes.py +++ b/schematic_api/api/routes.py @@ -43,8 +43,10 @@ logger = logging.getLogger(__name__) logging.basicConfig(level=logging.DEBUG) +tracing_service_name = os.environ.get("TRACING_SERVICE_NAME", "schematic-api") + trace.set_tracer_provider( - TracerProvider(resource=Resource(attributes={SERVICE_NAME: "schematic-api"})) + TracerProvider(resource=Resource(attributes={SERVICE_NAME: tracing_service_name})) ) diff --git a/tests/conftest.py b/tests/conftest.py index f580a08c9..e6382ed40 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -4,10 +4,21 @@ import os import shutil import sys -from typing import Generator +from typing import Callable, Generator, Set import pytest from dotenv import load_dotenv +from opentelemetry import trace +from opentelemetry._logs import set_logger_provider +from opentelemetry.exporter.otlp.proto.grpc._log_exporter import OTLPLogExporter +from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter +from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler +from opentelemetry.sdk._logs.export import BatchLogRecordProcessor +from opentelemetry.sdk.resources import SERVICE_NAME, Resource +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import BatchSpanProcessor +from opentelemetry.sdk.trace.sampling import ALWAYS_OFF +from pytest_asyncio import is_async_test from schematic.configuration.configuration import CONFIG, Configuration from schematic.models.metadata import MetadataModel @@ -15,6 +26,9 @@ from schematic.schemas.data_model_parser import DataModelParser from schematic.store.synapse import SynapseStorage from schematic.utils.df_utils import load_df +from tests.utils import CleanupAction, CleanupItem + +tracer = trace.get_tracer("Schematic-Tests") load_dotenv() @@ -174,3 +188,96 @@ def metadata_model(helpers, data_model_labels): ) return metadata_model + + +@pytest.fixture(scope="function") +def schedule_for_cleanup( + request, synapse_store: SynapseStorage +) -> Callable[[CleanupItem], None]: + """Returns a closure that takes an item that should be scheduled for cleanup.""" + + items: Set[CleanupItem] = set() + + def _append_cleanup(item: CleanupItem): + print(f"Added {item} to cleanup list") + items.add(item) + + def cleanup_scheduled_items() -> None: + for item in items: + print(f"Cleaning up {item}") + try: + if item.action == CleanupAction.DELETE: + if item.synapse_id: + synapse_store.syn.delete(obj=item.synapse_id) + elif item.name and item.parent_id: + synapse_id = synapse_store.syn.findEntityId( + name=item.name, parent=item.parent_id + ) + if synapse_id: + synapse_store.syn.delete(obj=synapse_id) + else: + logger.error(f"Invalid cleanup item {item}") + else: + logger.error(f"Invalid cleanup action {item.action}") + except Exception as ex: + logger.exception(f"Failed to delete {item}") + + request.addfinalizer(cleanup_scheduled_items) + + return _append_cleanup + + +active_span_processors = [] + + +@pytest.fixture(scope="session", autouse=True) +def set_up_tracing() -> None: + """Set up tracing for the API.""" + tracing_export = os.environ.get("TRACING_EXPORT_FORMAT", None) + tracing_service_name = os.environ.get("TRACING_SERVICE_NAME", "schematic-tests") + if tracing_export == "otlp": + trace.set_tracer_provider( + TracerProvider( + resource=Resource(attributes={SERVICE_NAME: tracing_service_name}) + ) + ) + processor = BatchSpanProcessor(OTLPSpanExporter()) + active_span_processors.append(processor) + trace.get_tracer_provider().add_span_processor(processor) + else: + trace.set_tracer_provider(TracerProvider(sampler=ALWAYS_OFF)) + + +@pytest.fixture(autouse=True, scope="function") +def wrap_with_otel(request): + """Start a new OTEL Span for each test function.""" + with tracer.start_as_current_span(request.node.name): + try: + yield + finally: + for processor in active_span_processors: + processor.force_flush() + + +@pytest.fixture(scope="session", autouse=True) +def set_up_logging() -> None: + """Set up logging to export to OTLP.""" + logging_export = os.environ.get("LOGGING_EXPORT_FORMAT", None) + logging_service_name = os.environ.get("LOGGING_SERVICE_NAME", "schematic-tests") + logging_instance_name = os.environ.get("LOGGING_INSTANCE_NAME", "local") + if logging_export == "otlp": + resource = Resource.create( + { + "service.name": logging_service_name, + "service.instance.id": logging_instance_name, + } + ) + + logger_provider = LoggerProvider(resource=resource) + set_logger_provider(logger_provider=logger_provider) + + # TODO: Add support for secure connections + exporter = OTLPLogExporter(insecure=True) + logger_provider.add_log_record_processor(BatchLogRecordProcessor(exporter)) + handler = LoggingHandler(level=logging.NOTSET, logger_provider=logger_provider) + logging.getLogger().addHandler(handler) diff --git a/tests/test_store.py b/tests/test_store.py index 9ad00a052..f79761b28 100644 --- a/tests/test_store.py +++ b/tests/test_store.py @@ -7,9 +7,10 @@ import math import os import shutil +import tempfile +import uuid from contextlib import nullcontext as does_not_raise -from time import sleep -from typing import Any, Generator +from typing import Any, Callable, Generator from unittest.mock import AsyncMock, MagicMock, patch import pandas as pd @@ -19,6 +20,7 @@ from synapseclient.core.exceptions import SynapseHTTPError from synapseclient.entity import File from synapseclient.models import Annotations +from synapseclient.models import Folder as FolderModel from schematic.configuration.configuration import CONFIG, Configuration from schematic.schemas.data_model_graph import DataModelGraph, DataModelGraphExplorer @@ -27,6 +29,7 @@ from schematic.store.synapse import DatasetFileView, ManifestDownload, SynapseStorage from schematic.utils.general import check_synapse_cache_size from tests.conftest import Helpers +from tests.utils import CleanupItem logging.basicConfig(level=logging.DEBUG) logger = logging.getLogger(__name__) @@ -74,14 +77,18 @@ def projectId(synapse_store, helpers): @pytest.fixture -def datasetId(synapse_store, projectId, helpers): +def datasetId( + synapse_store: SynapseStorage, projectId: str, helpers, schedule_for_cleanup +): dataset = Folder( - name="Table Test Dataset " + helpers.get_python_version(), + name="Table Test Dataset " + + helpers.get_python_version() + + f" integration_test_{str(uuid.uuid4()).replace('-', '_')}", parent=projectId, ) datasetId = synapse_store.syn.store(dataset).id - sleep(5) + schedule_for_cleanup(CleanupItem(synapse_id=datasetId)) yield datasetId @@ -990,51 +997,62 @@ class TestTableOperations: ["display_label", "class_label"], ids=["aks_display_label", "aks_class_label"], ) - def test_createTable( + async def test_create_table( self, - helpers, - synapse_store, - config: Configuration, - projectId, - datasetId, - table_column_names, - annotation_keys, + helpers: Helpers, + synapse_store: SynapseStorage, + projectId: str, + datasetId: str, + table_column_names: str, + annotation_keys: str, dmge: DataModelGraphExplorer, - ): + schedule_for_cleanup: Callable[[CleanupItem], None], + ) -> None: + # GIVEN a table to create table_manipulation = None + table_name = f"followup_synapse_storage_manifest_table_integration_test_{str(uuid.uuid4()).replace('-', '_')}" + schedule_for_cleanup(CleanupItem(name=table_name, parent_id=projectId)) - # Check if FollowUp table exists if so delete - existing_tables = synapse_store.get_table_info(projectId=projectId) - - table_name = "followup_synapse_storage_manifest_table" + # AND a manifest to associate metadata with files + manifest_path = "mock_manifests/table_manifest.csv" - if table_name in existing_tables.keys(): - synapse_store.syn.delete(existing_tables[table_name]) - sleep(10) - # assert no table - assert ( - table_name - not in synapse_store.get_table_info(projectId=projectId).keys() + # AND a copy of all the folders in the manifest. Added to the dataset directory for easy cleanup + manifest = helpers.get_data_frame(manifest_path) + for index, row in manifest.iterrows(): + folder_id = row["entityId"] + folder_copy = FolderModel(id=folder_id).copy( + parent_id=datasetId, synapse_client=synapse_store.syn + ) + schedule_for_cleanup(CleanupItem(synapse_id=folder_copy.id)) + manifest.at[index, "entityId"] = folder_copy.id + + with patch.object( + synapse_store, "_generate_table_name", return_value=(table_name, "followup") + ), patch.object( + synapse_store, "getDatasetProject", return_value=projectId + ), tempfile.NamedTemporaryFile( + delete=True, suffix=".csv" + ) as tmp_file: + # Write the DF to a temporary file to prevent modifying the original + manifest.to_csv(tmp_file.name, index=False) + + # WHEN I associate metadata with files + manifest_id = synapse_store.associateMetadataWithFiles( + dmge=dmge, + metadataManifestPath=tmp_file.name, + datasetId=datasetId, + manifest_record_type="table_and_file", + hideBlanks=True, + restrict_manifest=False, + table_manipulation=table_manipulation, + table_column_names=table_column_names, + annotation_keys=annotation_keys, ) + schedule_for_cleanup(CleanupItem(synapse_id=manifest_id)) - # associate metadata with files - manifest_path = "mock_manifests/table_manifest.csv" - # updating file view on synapse takes a long time - manifestId = synapse_store.associateMetadataWithFiles( - dmge=dmge, - metadataManifestPath=helpers.get_data_path(manifest_path), - datasetId=datasetId, - manifest_record_type="table_and_file", - hideBlanks=True, - restrict_manifest=False, - table_manipulation=table_manipulation, - table_column_names=table_column_names, - annotation_keys=annotation_keys, - ) + # THEN the table should exist existing_tables = synapse_store.get_table_info(projectId=projectId) - # clean Up - synapse_store.syn.delete(manifestId) # assert table exists assert table_name in existing_tables.keys() @@ -1048,142 +1066,166 @@ def test_createTable( ["display_label", "class_label"], ids=["aks_display_label", "aks_class_label"], ) - def test_replaceTable( + async def test_replace_table( self, - helpers, - synapse_store, - config: Configuration, - projectId, - datasetId, - table_column_names, - annotation_keys, + helpers: Helpers, + synapse_store: SynapseStorage, + projectId: str, + datasetId: str, + table_column_names: str, + annotation_keys: str, dmge: DataModelGraphExplorer, - ): + schedule_for_cleanup: Callable[[str], None], + ) -> None: table_manipulation = "replace" - table_name = "followup_synapse_storage_manifest_table" + table_name = f"followup_synapse_storage_manifest_table_integration_test_{str(uuid.uuid4()).replace('-', '_')}" + schedule_for_cleanup(CleanupItem(name=table_name, parent_id=projectId)) manifest_path = "mock_manifests/table_manifest.csv" replacement_manifest_path = "mock_manifests/table_manifest_replacement.csv" column_of_interest = "DaystoFollowUp" + # AND a copy of all the folders in the manifest. Added to the dataset directory for easy cleanup + manifest = helpers.get_data_frame(manifest_path) + replacement_manifest = helpers.get_data_frame(replacement_manifest_path) + for index, row in manifest.iterrows(): + folder_id = row["entityId"] + folder_copy = FolderModel(id=folder_id).copy( + parent_id=datasetId, synapse_client=synapse_store.syn + ) + schedule_for_cleanup(CleanupItem(synapse_id=folder_copy.id)) + manifest.at[index, "entityId"] = folder_copy.id + replacement_manifest.at[index, "entityId"] = folder_copy.id + # Check if FollowUp table exists if so delete existing_tables = synapse_store.get_table_info(projectId=projectId) - if table_name in existing_tables.keys(): - synapse_store.syn.delete(existing_tables[table_name]) - sleep(10) - # assert no table - assert ( - table_name - not in synapse_store.get_table_info(projectId=projectId).keys() + with patch.object( + synapse_store, "_generate_table_name", return_value=(table_name, "followup") + ), patch.object( + synapse_store, "getDatasetProject", return_value=projectId + ), tempfile.NamedTemporaryFile( + delete=True, suffix=".csv" + ) as tmp_file: + # Write the DF to a temporary file to prevent modifying the original + manifest.to_csv(tmp_file.name, index=False) + + # updating file view on synapse takes a long time + manifest_id = synapse_store.associateMetadataWithFiles( + dmge=dmge, + metadataManifestPath=tmp_file.name, + datasetId=datasetId, + manifest_record_type="table_and_file", + hideBlanks=True, + restrict_manifest=False, + table_manipulation=table_manipulation, + table_column_names=table_column_names, + annotation_keys=annotation_keys, ) - - # updating file view on synapse takes a long time - manifestId = synapse_store.associateMetadataWithFiles( - dmge=dmge, - metadataManifestPath=helpers.get_data_path(manifest_path), - datasetId=datasetId, - manifest_record_type="table_and_file", - hideBlanks=True, - restrict_manifest=False, - table_manipulation=table_manipulation, - table_column_names=table_column_names, - annotation_keys=annotation_keys, - ) + schedule_for_cleanup(CleanupItem(synapse_id=manifest_id)) existing_tables = synapse_store.get_table_info(projectId=projectId) # Query table for DaystoFollowUp column - tableId = existing_tables[table_name] - daysToFollowUp = ( - synapse_store.syn.tableQuery(f"SELECT {column_of_interest} FROM {tableId}") + table_id = existing_tables[table_name] + days_to_follow_up = ( + synapse_store.syn.tableQuery(f"SELECT {column_of_interest} FROM {table_id}") .asDataFrame() .squeeze() ) # assert Days to FollowUp == 73 - assert (daysToFollowUp == 73).all() - - # Associate replacement manifest with files - manifestId = synapse_store.associateMetadataWithFiles( - dmge=dmge, - metadataManifestPath=helpers.get_data_path(replacement_manifest_path), - datasetId=datasetId, - manifest_record_type="table_and_file", - hideBlanks=True, - restrict_manifest=False, - table_manipulation=table_manipulation, - table_column_names=table_column_names, - annotation_keys=annotation_keys, - ) + assert (days_to_follow_up == 73).all() + + with patch.object( + synapse_store, "_generate_table_name", return_value=(table_name, "followup") + ), patch.object( + synapse_store, "getDatasetProject", return_value=projectId + ), tempfile.NamedTemporaryFile( + delete=True, suffix=".csv" + ) as tmp_file: + # Write the DF to a temporary file to prevent modifying the original + replacement_manifest.to_csv(tmp_file.name, index=False) + + # Associate replacement manifest with files + manifest_id = synapse_store.associateMetadataWithFiles( + dmge=dmge, + metadataManifestPath=tmp_file.name, + datasetId=datasetId, + manifest_record_type="table_and_file", + hideBlanks=True, + restrict_manifest=False, + table_manipulation=table_manipulation, + table_column_names=table_column_names, + annotation_keys=annotation_keys, + ) + schedule_for_cleanup(CleanupItem(synapse_id=manifest_id)) existing_tables = synapse_store.get_table_info(projectId=projectId) # Query table for DaystoFollowUp column - tableId = existing_tables[table_name] - daysToFollowUp = ( - synapse_store.syn.tableQuery(f"SELECT {column_of_interest} FROM {tableId}") + table_id = existing_tables[table_name] + days_to_follow_up = ( + synapse_store.syn.tableQuery(f"SELECT {column_of_interest} FROM {table_id}") .asDataFrame() .squeeze() ) # assert Days to FollowUp == 89 now and not 73 - assert (daysToFollowUp == 89).all() - # delete table - synapse_store.syn.delete(tableId) + assert (days_to_follow_up == 89).all() @pytest.mark.parametrize( "annotation_keys", ["display_label", "class_label"], ids=["aks_display_label", "aks_class_label"], ) - def test_upsertTable( + async def test_upsert_table( self, - helpers, - synapse_store, - config: Configuration, - projectId, - datasetId, - annotation_keys, + helpers: Helpers, + synapse_store: SynapseStorage, + projectId: str, + datasetId: str, + annotation_keys: str, dmge: DataModelGraphExplorer, + schedule_for_cleanup: Callable[[str], None], ): table_manipulation = "upsert" - table_name = "MockRDB_synapse_storage_manifest_table".lower() + table_name = f"MockRDB_synapse_storage_manifest_table_integration_test_{str(uuid.uuid4()).replace('-', '_')}".lower() + schedule_for_cleanup(CleanupItem(name=table_name, parent_id=projectId)) manifest_path = "mock_manifests/rdb_table_manifest.csv" replacement_manifest_path = "mock_manifests/rdb_table_manifest_upsert.csv" column_of_interest = "MockRDB_id,SourceManifest" - # Check if FollowUp table exists if so delete - existing_tables = synapse_store.get_table_info(projectId=projectId) - - if table_name in existing_tables.keys(): - synapse_store.syn.delete(existing_tables[table_name]) - sleep(10) - # assert no table - assert ( - table_name - not in synapse_store.get_table_info(projectId=projectId).keys() + with patch.object( + synapse_store, "_generate_table_name", return_value=(table_name, "mockrdb") + ), patch.object( + synapse_store, "getDatasetProject", return_value=projectId + ), tempfile.NamedTemporaryFile( + delete=True, suffix=".csv" + ) as tmp_file: + # Copy to a temporary file to prevent modifying the original + shutil.copyfile(helpers.get_data_path(manifest_path), tmp_file.name) + + # updating file view on synapse takes a long time + manifest_id = synapse_store.associateMetadataWithFiles( + dmge=dmge, + metadataManifestPath=tmp_file.name, + datasetId=datasetId, + manifest_record_type="table_and_file", + hideBlanks=True, + restrict_manifest=False, + table_manipulation=table_manipulation, + table_column_names="display_name", + annotation_keys=annotation_keys, ) - - # updating file view on synapse takes a long time - manifestId = synapse_store.associateMetadataWithFiles( - dmge=dmge, - metadataManifestPath=helpers.get_data_path(manifest_path), - datasetId=datasetId, - manifest_record_type="table_and_file", - hideBlanks=True, - restrict_manifest=False, - table_manipulation=table_manipulation, - table_column_names="display_name", - annotation_keys=annotation_keys, - ) + schedule_for_cleanup(CleanupItem(synapse_id=manifest_id)) existing_tables = synapse_store.get_table_info(projectId=projectId) # set primary key annotation for uploaded table - tableId = existing_tables[table_name] + table_id = existing_tables[table_name] # Query table for DaystoFollowUp column table_query = ( - synapse_store.syn.tableQuery(f"SELECT {column_of_interest} FROM {tableId}") + synapse_store.syn.tableQuery(f"SELECT {column_of_interest} FROM {table_id}") .asDataFrame() .squeeze() ) @@ -1193,24 +1235,37 @@ def test_upsertTable( assert table_query.MockRDB_id.size == 4 assert table_query["SourceManifest"][3] == "Manifest1" - # Associate new manifest with files - manifestId = synapse_store.associateMetadataWithFiles( - dmge=dmge, - metadataManifestPath=helpers.get_data_path(replacement_manifest_path), - datasetId=datasetId, - manifest_record_type="table_and_file", - hideBlanks=True, - restrict_manifest=False, - table_manipulation=table_manipulation, - table_column_names="display_name", - annotation_keys=annotation_keys, - ) + with patch.object( + synapse_store, "_generate_table_name", return_value=(table_name, "mockrdb") + ), patch.object( + synapse_store, "getDatasetProject", return_value=projectId + ), tempfile.NamedTemporaryFile( + delete=True, suffix=".csv" + ) as tmp_file: + # Copy to a temporary file to prevent modifying the original + shutil.copyfile( + helpers.get_data_path(replacement_manifest_path), tmp_file.name + ) + + # Associate new manifest with files + manifest_id = synapse_store.associateMetadataWithFiles( + dmge=dmge, + metadataManifestPath=tmp_file.name, + datasetId=datasetId, + manifest_record_type="table_and_file", + hideBlanks=True, + restrict_manifest=False, + table_manipulation=table_manipulation, + table_column_names="display_name", + annotation_keys=annotation_keys, + ) + schedule_for_cleanup(CleanupItem(synapse_id=manifest_id)) existing_tables = synapse_store.get_table_info(projectId=projectId) # Query table for DaystoFollowUp column - tableId = existing_tables[table_name] + table_id = existing_tables[table_name] table_query = ( - synapse_store.syn.tableQuery(f"SELECT {column_of_interest} FROM {tableId}") + synapse_store.syn.tableQuery(f"SELECT {column_of_interest} FROM {table_id}") .asDataFrame() .squeeze() ) @@ -1219,8 +1274,6 @@ def test_upsertTable( assert table_query.MockRDB_id.max() == 8 assert table_query.MockRDB_id.size == 8 assert table_query["SourceManifest"][3] == "Manifest2" - # delete table - synapse_store.syn.delete(tableId) class TestDownloadManifest: diff --git a/tests/utils.py b/tests/utils.py new file mode 100644 index 000000000..ecaaf2450 --- /dev/null +++ b/tests/utils.py @@ -0,0 +1,34 @@ +"""Catch all utility functions and classes used in the tests.""" +from dataclasses import dataclass +from enum import Enum +from typing import Optional + + +class CleanupAction(str, Enum): + """Actions that can be performed on a cleanup item.""" + + DELETE = "delete" + + def __str__(self) -> str: + # See https://peps.python.org/pep-0663/ + return self.value + + +@dataclass(frozen=True) +class CleanupItem: + """Simple class used to create a test finalizer and cleanup resources after test execution. + + synapse_id or (name and parent_id) must be provided. + + Attributes: + synapse_id (str): The Synapse ID of the resource to cleanup. + name (str): The name of the resource to cleanup. + parent_id (str): The parent ID of the resource to cleanup. + action (CleanupAction): The action to perform on the resource. + + """ + + synapse_id: Optional[str] = None + name: Optional[str] = None + parent_id: Optional[str] = None + action: CleanupAction = CleanupAction.DELETE From efaf9a785ddd22662569e1ee483ae59023eaf131 Mon Sep 17 00:00:00 2001 From: BryanFauble <17128019+BryanFauble@users.noreply.github.com> Date: Mon, 9 Sep 2024 12:46:28 -0700 Subject: [PATCH 203/233] [FDS-2383] Update to run integration tests on default runner with python 3.10 (#1498) * Update to run integration tests on default runner with python 3.10 --- .github/workflows/test.yml | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a868a2a62..748640f2c 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -25,7 +25,7 @@ concurrency: cancel-in-progress: true jobs: test: - runs-on: ubuntu-22.04-4core-16GBRAM-150GBSSD + runs-on: ubuntu-latest env: POETRY_VERSION: 1.3.0 strategy: @@ -125,16 +125,29 @@ jobs: poetry run pylint schematic/visualization/* schematic/configuration/*.py schematic/exceptions.py schematic/help.py schematic/loader.py schematic/version.py schematic/utils/*.py schematic/schemas/*.py #---------------------------------------------- - # run test suite + # run unit test suite #---------------------------------------------- - - name: Run tests + - name: Run unit tests env: SYNAPSE_ACCESS_TOKEN: ${{ secrets.SYNAPSE_ACCESS_TOKEN }} SERVICE_ACCOUNT_CREDS: ${{ secrets.SERVICE_ACCOUNT_CREDS }} run: > source .venv/bin/activate; - pytest --durations=0 --cov-report=term --cov-report=html:htmlcov --cov-report=xml:coverage.xml --cov=schematic/ - -m "not (rule_benchmark)" --reruns 4 -n 8 + pytest --durations=0 --cov-append --cov-report=term --cov-report=html:htmlcov + --cov-report=xml:coverage.xml --cov=schematic/ --reruns 4 -n 8 tests/unit + + #---------------------------------------------- + # run integration test suite + #---------------------------------------------- + - name: Run integration tests + if: ${{ contains(fromJSON('["3.10"]'), matrix.python-version) }} + env: + SYNAPSE_ACCESS_TOKEN: ${{ secrets.SYNAPSE_ACCESS_TOKEN }} + SERVICE_ACCOUNT_CREDS: ${{ secrets.SERVICE_ACCOUNT_CREDS }} + run: > + source .venv/bin/activate; + pytest --durations=0 --cov-append --cov-report=term --cov-report=html:htmlcov --cov-report=xml:coverage.xml --cov=schematic/ + -m "not (rule_benchmark)" --reruns 4 -n 8 --ignore=tests/unit - name: Upload pytest test results @@ -143,7 +156,7 @@ jobs: name: pytest-results-${{ matrix.python-version }} path: htmlcov # Use always() to always run this step to publish test results when there are test failures - if: ${{ always() }} + if: ${{ always() && contains(fromJSON('["3.10"]'), matrix.python-version) }} - name: Upload XML coverage report id: upload_coverage_report uses: actions/upload-artifact@v4 From 6e6249fef115d1fbd7f954b351692ae5d2614496 Mon Sep 17 00:00:00 2001 From: linglp Date: Tue, 10 Sep 2024 10:48:14 -0400 Subject: [PATCH 204/233] remove unnecessary part --- schematic/store/synapse.py | 41 -------------------------------------- 1 file changed, 41 deletions(-) diff --git a/schematic/store/synapse.py b/schematic/store/synapse.py index 666913044..54570fe9a 100644 --- a/schematic/store/synapse.py +++ b/schematic/store/synapse.py @@ -1908,34 +1908,6 @@ async def _process_store_annos(self, requests: Set[asyncio.Task]) -> None: except Exception as e: raise RuntimeError(f"failed with { repr(e) }.") from e - def count_entity_id(self, manifest: pd.DataFrame) -> int: - """Check if there are any non-NaN values in the original manifest's entityId column - - Args: - manifest (pd.DataFrame): manifest dataframe - - Returns: - int: The count of non-NaN entityId values. - """ - # Normalize the column names to lowercase - normalized_columns = {col.lower(): col for col in manifest.columns} - - # Check if a case-insensitive 'entityid' column exists - if "entityid" in normalized_columns: - entity_id_column = normalized_columns["entityid"] - entity_id_count = manifest[entity_id_column].notna().sum() - else: - entity_id_count = 0 - return entity_id_count - - def handle_missing_entity_ids( - original_entity_id_count: int, merged_entity_id_count: int - ): - if original_entity_id_count != merged_entity_id_count: - raise LookupError( - "Some entityId values became NaN due to unmatched Filename" - ) - @tracer.start_as_current_span("SynapseStorage::add_annotations_to_entities_files") async def add_annotations_to_entities_files( self, @@ -1980,19 +1952,6 @@ async def add_annotations_to_entities_files( file_df, how="left", on="Filename", suffixes=["_x", None] ).drop("entityId_x", axis=1) - # count entity ids after manifest gets merged - merged_manifest_entity_id_count = self.count_entity_id(manifest) - - # drop the duplicate entity column with NA values - # col_to_drop = "entityId_x" - # if manifest.entityId.isnull().all(): - # col_to_drop = "entityId" - - # # If the original entityId column is empty after the merge, drop it and rename the duplicate column - # manifest.drop(columns=[col_to_drop], inplace=True) - # if col_to_drop == "entityId": - # manifest.rename(columns={"entityId_x": "entityId"}, inplace=True) - # Fill `entityId` for each row if missing and annotate entity as appropriate requests = set() for idx, row in manifest.iterrows(): From 40ca3a075854260ebbbab549c2b2fc137fde5885 Mon Sep 17 00:00:00 2001 From: linglp Date: Tue, 10 Sep 2024 10:58:39 -0400 Subject: [PATCH 205/233] remove set to staging --- schematic/store/synapse.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/schematic/store/synapse.py b/schematic/store/synapse.py index 54570fe9a..f23aa0da0 100644 --- a/schematic/store/synapse.py +++ b/schematic/store/synapse.py @@ -216,9 +216,6 @@ def __init__( Consider necessity of adding "columns" and "where_clauses" params to the constructor. Currently with how `query_fileview` is implemented, these params are not needed at this step but could be useful in the future if the need for more scoped querys expands. """ self.syn = self.login(synapse_cache_path, access_token) - # TODO REMOVE - self.syn.setEndpoints(**synapseclient.client.STAGING_ENDPOINTS) - self.project_scope = project_scope self.storageFileview = CONFIG.synapse_master_fileview_id self.manifest = CONFIG.synapse_manifest_basename From 44757188646282f79157d0ff86527d7610c88527 Mon Sep 17 00:00:00 2001 From: linglp Date: Tue, 10 Sep 2024 10:59:46 -0400 Subject: [PATCH 206/233] remove old code --- schematic/store/synapse.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/schematic/store/synapse.py b/schematic/store/synapse.py index f23aa0da0..95d709fb8 100644 --- a/schematic/store/synapse.py +++ b/schematic/store/synapse.py @@ -1941,9 +1941,6 @@ async def add_annotations_to_entities_files( ) file_df = pd.DataFrame(files_and_entityIds) - # count entity ids of the original manifest - original_manifest_entity_id_count = self.count_entity_id(manifest) - # Merge dataframes to add entityIds manifest = manifest.merge( file_df, how="left", on="Filename", suffixes=["_x", None] From 7188d023a6e8f24a0721f6bf026b1d08f3ea52cb Mon Sep 17 00:00:00 2001 From: Lingling <55448354+linglp@users.noreply.github.com> Date: Tue, 10 Sep 2024 11:12:53 -0400 Subject: [PATCH 207/233] Update schematic/manifest/generator.py Co-authored-by: BryanFauble <17128019+BryanFauble@users.noreply.github.com> --- schematic/manifest/generator.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/schematic/manifest/generator.py b/schematic/manifest/generator.py index c67bd16b3..e821d9f13 100644 --- a/schematic/manifest/generator.py +++ b/schematic/manifest/generator.py @@ -1690,13 +1690,16 @@ def create_manifests( Function->>DataModelGraph: generate graph DataModelGraph-->>Function: return graph data model alt data_types == "all manifests" - Function->>ManifestGenerator: create manifests for all components + loop for each component + Function->>ManifestGenerator: create manifest for component + ManifestGenerator-->>Function: single manifest + end else loop for each data_type Function->>ManifestGenerator: create single manifest + ManifestGenerator-->>Function: single manifest end end - ManifestGenerator-->>Function: return results Function-->>User: return manifests based on output_format ``` """ From 96d941b0a130b7e90d9e593ddf7710f4a9e43e53 Mon Sep 17 00:00:00 2001 From: linglp Date: Tue, 10 Sep 2024 11:24:27 -0400 Subject: [PATCH 208/233] give name to boxes --- schematic/manifest/generator.py | 68 ++++++++++++++++----------------- 1 file changed, 34 insertions(+), 34 deletions(-) diff --git a/schematic/manifest/generator.py b/schematic/manifest/generator.py index e821d9f13..d954506a5 100644 --- a/schematic/manifest/generator.py +++ b/schematic/manifest/generator.py @@ -1831,40 +1831,40 @@ def get_manifest( ```mermaid flowchart TD - A[Start] --> B{Dataset ID provided?} - B -- No --> C[Get Empty Manifest URL] - C --> D{Output Format is 'excel'?} - D -- Yes --> E[Export to Excel] - D -- No --> F[Return Manifest URL] - B -- Yes --> G[Instantiate SynapseStorage] - G --> H[Update Dataset Manifest Files] - H --> I[Get Empty Manifest URL] - I --> J{Manifest Record exists?} - J -- Yes --> K[Update Dataframe] - K --> L[Handle Output Format Logic] - L --> M[Return Result] - J -- No --> AN{Use Annotations?} - - AN -- No --> Q[Create dataframe from empty manifest on Google] - Q --> AJ{Manifest file-based?} - AJ -- Yes --> P[Add entityId and filename to manifest df] - AJ -- No --> R[Use dataframe from an empty manifest] - - P --> L[Handle Output Format Logic] - R --> L[Handle Output Format Logic] - - AN -- Yes --> AM{Manifest file-based?} - AM -- No --> L[Handle Output Format Logic] - AM -- Yes --> AO[Process Annotations] - AO --> AP{Annotations Empty?} - AP -- Yes --> AQ[Create dataframe from an empty manifest on Google] - AQ --> AR[Update dataframe] - AP -- No --> AS[Get Manifest with Annotations] - AS --> AR - AR --> L[Handle Output Format Logic] - M --> T[End] - F --> T - E --> T + Start[Start] --> DatasetIDCheck{Dataset ID provided?} + DatasetIDCheck -- No --> EmptyManifestURL[Get Empty Manifest URL] + EmptyManifestURL --> OutputFormatCheck{Output Format is 'excel'?} + OutputFormatCheck -- Yes --> ExportToExcel[Export to Excel] + OutputFormatCheck -- No --> ReturnManifestURL[Return Manifest URL] + DatasetIDCheck -- Yes --> InstantiateSynapseStorage[Instantiate SynapseStorage] + InstantiateSynapseStorage --> UpdateManifestFiles[Update Dataset Manifest Files] + UpdateManifestFiles --> GetEmptyManifestURL[Get Empty Manifest URL] + GetEmptyManifestURL --> ManifestRecordCheck{Manifest Record exists?} + ManifestRecordCheck -- Yes --> UpdateDataframe[Update Dataframe] + UpdateDataframe --> HandleOutputFormatLogic[Handle Output Format Logic] + HandleOutputFormatLogic --> ReturnResult[Return Result] + ManifestRecordCheck -- No --> UseAnnotationsCheck{Use Annotations?} + + UseAnnotationsCheck -- No --> CreateDataframe[Create dataframe from empty manifest on Google] + CreateDataframe --> ManifestFileBasedCheck1{Manifest file-based?} + ManifestFileBasedCheck1 -- Yes --> AddEntityID[Add entityId and filename to manifest df] + ManifestFileBasedCheck1 -- No --> UseDataframe[Use dataframe from an empty manifest] + + AddEntityID --> HandleOutputFormatLogic + UseDataframe --> HandleOutputFormatLogic + + UseAnnotationsCheck -- Yes --> ManifestFileBasedCheck2{Manifest file-based?} + ManifestFileBasedCheck2 -- No --> HandleOutputFormatLogic + ManifestFileBasedCheck2 -- Yes --> ProcessAnnotations[Process Annotations] + ProcessAnnotations --> AnnotationsEmptyCheck{Annotations Empty?} + AnnotationsEmptyCheck -- Yes --> CreateDataframeFromEmpty[Create dataframe from an empty manifest on Google] + CreateDataframeFromEmpty --> UpdateDataframeWithAnnotations[Update dataframe] + AnnotationsEmptyCheck -- No --> GetManifestWithAnnotations[Get Manifest with Annotations] + GetManifestWithAnnotations --> UpdateDataframeWithAnnotations + UpdateDataframeWithAnnotations --> HandleOutputFormatLogic + ReturnResult --> End[End] + ReturnManifestURL --> End + ExportToExcel --> End ``` """ # Handle case when no dataset ID is provided From c0ae0f65e9b4a3ae2deb1f4ff05727c7ca4c5844 Mon Sep 17 00:00:00 2001 From: linglp Date: Tue, 10 Sep 2024 12:30:31 -0400 Subject: [PATCH 209/233] edit description --- schematic/store/synapse.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/schematic/store/synapse.py b/schematic/store/synapse.py index 95d709fb8..e697a5bc6 100644 --- a/schematic/store/synapse.py +++ b/schematic/store/synapse.py @@ -1532,15 +1532,15 @@ def process_row_annotations( dmge (DataModelGraphExplorer): data model graph explorer metadata_syn (dict): metadata used for Synapse storage hideBlanks (bool): if true, does not upload annotation keys with blank values. - csv_list_regex (str): - annos (dict): + csv_list_regex (str): Regex to match with comma separated list + annos (Dict[str, Any]): dictionary of annotation returned from synapse annotation_keys (str): display_label/class_label Returns: - dict: annotations as a dictionary + Dict[str, Any]: annotations as a dictionary """ for anno_k, anno_v in metadata_syn.items(): - # Remove keys with nan or empty string values from dict of annotations to be uploaded + # Remove keys with nan or empty string values or string that only contains white space from dict of annotations to be uploaded # if present on current data annotation if hide_blanks and ( (isinstance(anno_v, str) and anno_v.strip() == "") From 7c47054d797a7b304cf9824aafd290a23392d045 Mon Sep 17 00:00:00 2001 From: linglp Date: Wed, 11 Sep 2024 13:24:56 -0400 Subject: [PATCH 210/233] update to use v4 version of artifact --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 748640f2c..c17a4981d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -151,7 +151,7 @@ jobs: - name: Upload pytest test results - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 with: name: pytest-results-${{ matrix.python-version }} path: htmlcov From f47d02617eb3f5605eff4424161d19570797c6cd Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Wed, 11 Sep 2024 13:09:09 -0700 Subject: [PATCH 211/233] added codeowners file --- .github/CODEOWNERS | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 .github/CODEOWNERS diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 000000000..482d91a8b --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,2 @@ +# Global owners of whole repo +* @andrewelamb @GiaJordan @linglp \ No newline at end of file From 017256b6f269fa25f6f7a42aaaf6cbe7e067046e Mon Sep 17 00:00:00 2001 From: Brad Macdonald <52762200+BWMac@users.noreply.github.com> Date: Sun, 15 Sep 2024 13:14:19 -0600 Subject: [PATCH 212/233] [FDS-2415] Fixes `pytest` CI Python Versions (#1502) * specify python version in "Load cached venv" steps * bump GH actions versions --- .github/workflows/api_test.yml | 8 ++++---- .github/workflows/docker.yml | 12 ++++++------ .github/workflows/docker_build.yml | 8 ++++---- .github/workflows/pdoc.yml | 12 ++++++------ .github/workflows/publish.yml | 8 ++++---- .github/workflows/scan_repo.yml | 4 ++-- .github/workflows/test.yml | 8 ++++---- 7 files changed, 30 insertions(+), 30 deletions(-) diff --git a/.github/workflows/api_test.yml b/.github/workflows/api_test.yml index 354cbe3ba..31b8e7eff 100644 --- a/.github/workflows/api_test.yml +++ b/.github/workflows/api_test.yml @@ -27,10 +27,10 @@ jobs: # check-out repo and set-up python #---------------------------------------------- - name: Check out repository - uses: actions/checkout@v2 + uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} @@ -48,10 +48,10 @@ jobs: #---------------------------------------------- - name: Load cached venv id: cached-poetry-dependencies - uses: actions/cache@v2 + uses: actions/cache@v4 with: path: .venv - key: venv-${{ runner.os }}-${{ hashFiles('**/poetry.lock') }} + key: venv-${{ runner.os }}-${{ hashFiles('**/poetry.lock') }}-${{ matrix.python-version }} #---------------------------------------------- # install dependencies and root project diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 58f903f76..12b355dc0 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -17,16 +17,16 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Set up QEMU - uses: docker/setup-qemu-action@v2 + uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3 - name: Login to DockerHub - uses: docker/login-action@v2 + uses: docker/login-action@v3 with: username: schematicbot password: ${{ secrets.DOCKER_HUB_TOKEN }} @@ -36,7 +36,7 @@ jobs: run: echo "::set-output name=sha_short::$(git rev-parse --short HEAD)" - name: Build and push (tagged release) - uses: docker/build-push-action@v3 + uses: docker/build-push-action@v6 if: ${{ github.event_name == 'push' }} with: platforms: linux/amd64,linux/arm64 @@ -48,7 +48,7 @@ jobs: ${{ env.DOCKER_ORG }}/${{ env.DOCKER_REPO }}:commit-${{ steps.vars.outputs.sha_short }} - name: Build and push (manual release) - uses: docker/build-push-action@v3 + uses: docker/build-push-action@v6 if: ${{ github.event_name == 'workflow_dispatch' }} with: platforms: linux/amd64,linux/arm64 diff --git a/.github/workflows/docker_build.yml b/.github/workflows/docker_build.yml index 50596b7db..f1beb3489 100644 --- a/.github/workflows/docker_build.yml +++ b/.github/workflows/docker_build.yml @@ -23,13 +23,13 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v2 + uses: actions/checkout@v4 - name: Set env variable for version tag run: echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV - name: Log in to the Container registry - uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9 + uses: docker/login-action@v3 with: registry: ${{ env.REGISTRY }} username: ${{ github.actor }} @@ -37,7 +37,7 @@ jobs: - name: Extract metadata (tags, labels) for Docker id: meta - uses: docker/metadata-action@98669ae865ea3cffbcbaa878cf57c20bbf1c6c38 + uses: docker/metadata-action@v5 with: images: ${{ env.IMAGE_PATH }} tags: | @@ -46,7 +46,7 @@ jobs: type=semver,pattern={{raw}} - name: Build and push Docker image - uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc + uses: docker/build-push-action@v6 with: file: schematic_api/Dockerfile push: true diff --git a/.github/workflows/pdoc.yml b/.github/workflows/pdoc.yml index 51924c4b4..fdf0be220 100644 --- a/.github/workflows/pdoc.yml +++ b/.github/workflows/pdoc.yml @@ -37,10 +37,10 @@ jobs: # check-out repo and set-up python #---------------------------------------------- - name: Check out repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v3 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} @@ -59,10 +59,10 @@ jobs: #---------------------------------------------- - name: Load cached venv id: cached-poetry-dependencies - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: .venv - key: venv-${{ runner.os }}-${{ hashFiles('**/poetry.lock') }} + key: venv-${{ runner.os }}-${{ hashFiles('**/poetry.lock') }}-${{ matrix.python-version }} #---------------------------------------------- # install dependencies and root project @@ -76,7 +76,7 @@ jobs: - run: poetry show pdoc - run: poetry run pdoc --docformat google --mermaid -o docs/schematic schematic/manifest schematic/models schematic/schemas schematic/store schematic/utils schematic/visualization - - uses: actions/upload-pages-artifact@v1 + - uses: actions/upload-pages-artifact@v3 with: path: docs/schematic @@ -93,4 +93,4 @@ jobs: url: ${{ steps.deployment.outputs.page_url }} steps: - id: deployment - uses: actions/deploy-pages@v1 + uses: actions/deploy-pages@v4 diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index eb2ebafcb..c3225661e 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -16,10 +16,10 @@ jobs: # check-out repo and set-up python #---------------------------------------------- - name: Check out repository - uses: actions/checkout@v2 + uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} @@ -38,10 +38,10 @@ jobs: #---------------------------------------------- - name: Load cached venv id: cached-poetry-dependencies - uses: actions/cache@v2 + uses: actions/cache@v4 with: path: .venv - key: venv-${{ runner.os }}-${{ hashFiles('**/poetry.lock') }} + key: venv-${{ runner.os }}-${{ hashFiles('**/poetry.lock') }}-${{ matrix.python-version }} #---------------------------------------------- # install dependencies and root project diff --git a/.github/workflows/scan_repo.yml b/.github/workflows/scan_repo.yml index 6a93beee7..56c7ac35a 100644 --- a/.github/workflows/scan_repo.yml +++ b/.github/workflows/scan_repo.yml @@ -14,7 +14,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout code - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Run Trivy vulnerability scanner in repo mode uses: aquasecurity/trivy-action@master @@ -32,4 +32,4 @@ jobs: uses: github/codeql-action/upload-sarif@v3 with: sarif_file: 'trivy-results.sarif' - category: Git Repository \ No newline at end of file + category: Git Repository diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index c17a4981d..4df1cb2b9 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -39,10 +39,10 @@ jobs: # check-out repo and set-up python #---------------------------------------------- - name: Check out repository - uses: actions/checkout@v2 + uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} @@ -72,10 +72,10 @@ jobs: #---------------------------------------------- - name: Load cached venv id: cached-poetry-dependencies - uses: actions/cache@v2 + uses: actions/cache@v4 with: path: .venv - key: venv-${{ runner.os }}-${{ hashFiles('**/poetry.lock') }} + key: venv-${{ runner.os }}-${{ hashFiles('**/poetry.lock') }}-${{ matrix.python-version }} #---------------------------------------------- # install dependencies and root project From 5721c77d35ce14b45c7bf768af133c6d81eb4abe Mon Sep 17 00:00:00 2001 From: Brad Macdonald <52762200+BWMac@users.noreply.github.com> Date: Mon, 16 Sep 2024 13:12:56 -0600 Subject: [PATCH 213/233] [FDS-2415] Fixes `pdoc` Workflow (#1503) * test python version specification * remove this branch from pdoc branches on push --- .github/workflows/pdoc.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/pdoc.yml b/.github/workflows/pdoc.yml index fdf0be220..c8e09d818 100644 --- a/.github/workflows/pdoc.yml +++ b/.github/workflows/pdoc.yml @@ -79,6 +79,7 @@ jobs: - uses: actions/upload-pages-artifact@v3 with: path: docs/schematic + name: schematic-docs-${{ matrix.python-version }} # Deploy the artifact to GitHub pages. # This is a separate job so that only actions/deploy-pages has the necessary permissions. From 226df334c492bf642c051b2e56821350b534c329 Mon Sep 17 00:00:00 2001 From: linglp Date: Tue, 17 Sep 2024 10:53:28 -0400 Subject: [PATCH 214/233] add comments --- schematic/store/synapse.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/schematic/store/synapse.py b/schematic/store/synapse.py index e697a5bc6..36aed44fc 100644 --- a/schematic/store/synapse.py +++ b/schematic/store/synapse.py @@ -1526,7 +1526,19 @@ def process_row_annotations( annos: Dict[str, Any], annotation_keys: str, ) -> Dict[str, Any]: - """processes metadata annotations + """Processes metadata annotations based on the logic below: + 1. Checks if the hide_blanks flag is True, and if the current annotation value (anno_v) is: + An empty or whitespace-only string. + A NaN value (if the annotation is a float). + if any of the above conditions are met, and hide_blanks is True, the annotation key is not going to be uploaded and skips further processing of that annotation key. + if any of the above conditions are met, and hide_blanks is False, assigns an empty string "" as the annotation value for that key. + + 2. If the value is a string and matches the pattern defined by csv_list_regex, get validation rule based on "node label" or "node display name". + Check if the rule contains "list" as a rule, if it does, split the string by comma and assign the resulting list as the annotation value for that key. + + 3. For any other conditions, assigns the original value of anno_v to the annotation key (anno_k). + + 4. Returns the updated annotations dictionary. Args: dmge (DataModelGraphExplorer): data model graph explorer From e7a705fe7dbb6d02ec9e6958cb1cda201f678a2c Mon Sep 17 00:00:00 2001 From: linglp Date: Tue, 17 Sep 2024 11:28:05 -0400 Subject: [PATCH 215/233] add a mermaid diagram --- schematic/store/synapse.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/schematic/store/synapse.py b/schematic/store/synapse.py index 0fd806ebc..ae47cdccf 100644 --- a/schematic/store/synapse.py +++ b/schematic/store/synapse.py @@ -1554,6 +1554,22 @@ def process_row_annotations( Returns: Dict[str, Any]: annotations as a dictionary + + ```mermaid + flowchart TD + A[Start] --> C{Is anno_v empty, whitespace, or NaN?} + C -- Yes --> D{Is hide_blanks True?} + D -- Yes --> E[Remove this annotation key from the annotation dictionary to be uploaded. Skip further processing] + D -- No --> F[Assign empty string to annotation key] + C -- No --> G{Is anno_v a string?} + G -- No --> H[Assign original value of anno_v to annotation key] + G -- Yes --> I{Does anno_v match csv_list_regex?} + I -- Yes --> J[Get validation rule of anno_k] + J --> K{Does the validation rule contain 'list'} + K -- Yes --> L[Split anno_v by commas and assign as list] + I -- No --> H + K -- No --> H + ``` """ for anno_k, anno_v in metadata_syn.items(): # Remove keys with nan or empty string values or string that only contains white space from dict of annotations to be uploaded From 55fb30c5efd0a06abd3aadc9ad36632fe2565c12 Mon Sep 17 00:00:00 2001 From: linglp Date: Tue, 17 Sep 2024 11:39:14 -0400 Subject: [PATCH 216/233] remove unncessary code --- tests/integration/test_store_synapse.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/tests/integration/test_store_synapse.py b/tests/integration/test_store_synapse.py index 15f17f5b7..61d8ec3cb 100644 --- a/tests/integration/test_store_synapse.py +++ b/tests/integration/test_store_synapse.py @@ -9,13 +9,6 @@ from tests.conftest import Helpers -@pytest.fixture(name="dmge", scope="function") -def DMGE(helpers: Helpers) -> DataModelGraphExplorer: - """Fixture to instantiate a DataModelGraphExplorer object.""" - dmge = helpers.get_data_model_graph_explorer(path="example.model.jsonld") - return dmge - - class TestStoreSynapse: @pytest.mark.parametrize("hideBlanks", [True, False]) @pytest.mark.parametrize( From 83208dec8e2face43f78064662818ae69ed9786f Mon Sep 17 00:00:00 2001 From: Brad Macdonald <52762200+BWMac@users.noreply.github.com> Date: Wed, 18 Sep 2024 13:28:32 -0600 Subject: [PATCH 217/233] Fixes failing `pdoc` deployment (#1504) * test single version * remove python version from name * remove feature branch pdoc run * remove quotes --- .github/workflows/pdoc.yml | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/.github/workflows/pdoc.yml b/.github/workflows/pdoc.yml index c8e09d818..81617424f 100644 --- a/.github/workflows/pdoc.yml +++ b/.github/workflows/pdoc.yml @@ -28,9 +28,7 @@ jobs: runs-on: ubuntu-latest env: POETRY_VERSION: 1.3.0 - strategy: - matrix: - python-version: ["3.9", "3.10"] + PYTHON_VERSION: 3.10 steps: #---------------------------------------------- @@ -39,10 +37,10 @@ jobs: - name: Check out repository uses: actions/checkout@v4 - - name: Set up Python ${{ matrix.python-version }} + - name: Set up Python ${{ env.PYTHON_VERSION }} uses: actions/setup-python@v5 with: - python-version: ${{ matrix.python-version }} + python-version: ${{ env.PYTHON_VERSION }} #---------------------------------------------- # install & configure poetry @@ -62,7 +60,7 @@ jobs: uses: actions/cache@v4 with: path: .venv - key: venv-${{ runner.os }}-${{ hashFiles('**/poetry.lock') }}-${{ matrix.python-version }} + key: venv-${{ runner.os }}-${{ hashFiles('**/poetry.lock') }}-${{ env.PYTHON_VERSION }} #---------------------------------------------- # install dependencies and root project @@ -79,7 +77,7 @@ jobs: - uses: actions/upload-pages-artifact@v3 with: path: docs/schematic - name: schematic-docs-${{ matrix.python-version }} + name: github-pages # Deploy the artifact to GitHub pages. # This is a separate job so that only actions/deploy-pages has the necessary permissions. From 812a00497456c50aa0557238e74bd51b1357eec9 Mon Sep 17 00:00:00 2001 From: linglp Date: Thu, 19 Sep 2024 11:43:23 -0400 Subject: [PATCH 218/233] print out virtual env path --- .github/workflows/test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 4df1cb2b9..82dcae29a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -67,6 +67,7 @@ jobs: | python3 - --version ${{ env.POETRY_VERSION }}; poetry config virtualenvs.create true; poetry config virtualenvs.in-project true; + poetry env info; # print out virtual environment #---------------------------------------------- # load cached venv if cache exists #---------------------------------------------- From 95d9b80d9fad293037ca497a5db13f890ac9e475 Mon Sep 17 00:00:00 2001 From: linglp Date: Thu, 19 Sep 2024 11:46:15 -0400 Subject: [PATCH 219/233] poetry env info path --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 82dcae29a..85c9868e2 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -67,7 +67,7 @@ jobs: | python3 - --version ${{ env.POETRY_VERSION }}; poetry config virtualenvs.create true; poetry config virtualenvs.in-project true; - poetry env info; # print out virtual environment + poetry env info --path; # print out virtual environment #---------------------------------------------- # load cached venv if cache exists #---------------------------------------------- From a40fd567d009f704f75f4dd45951f374d28be9cd Mon Sep 17 00:00:00 2001 From: linglp Date: Thu, 19 Sep 2024 11:48:18 -0400 Subject: [PATCH 220/233] try again --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 85c9868e2..4ed61b730 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -67,7 +67,7 @@ jobs: | python3 - --version ${{ env.POETRY_VERSION }}; poetry config virtualenvs.create true; poetry config virtualenvs.in-project true; - poetry env info --path; # print out virtual environment + poetry env info --path; #---------------------------------------------- # load cached venv if cache exists #---------------------------------------------- From a73f0aaddbe850e31ec02d5fc8f450a2cb3224e9 Mon Sep 17 00:00:00 2001 From: linglp Date: Thu, 19 Sep 2024 11:55:01 -0400 Subject: [PATCH 221/233] print out virutal env info --- .github/workflows/test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 4ed61b730..df4cf42a0 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -67,6 +67,7 @@ jobs: | python3 - --version ${{ env.POETRY_VERSION }}; poetry config virtualenvs.create true; poetry config virtualenvs.in-project true; + poetry env info; poetry env info --path; #---------------------------------------------- # load cached venv if cache exists From 881fe6dd768ef14a7e52c46bec0db7f2ab10a249 Mon Sep 17 00:00:00 2001 From: linglp Date: Thu, 19 Sep 2024 12:02:11 -0400 Subject: [PATCH 222/233] list out config --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index df4cf42a0..40698fcbe 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -68,7 +68,7 @@ jobs: poetry config virtualenvs.create true; poetry config virtualenvs.in-project true; poetry env info; - poetry env info --path; + poetry config --list; #---------------------------------------------- # load cached venv if cache exists #---------------------------------------------- From e75b41c79dbb7c76a3f8467770267d8a47c08215 Mon Sep 17 00:00:00 2001 From: linglp Date: Thu, 19 Sep 2024 12:49:57 -0400 Subject: [PATCH 223/233] enforce to use python 3.10 --- .github/workflows/test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 40698fcbe..b7f358046 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -67,6 +67,7 @@ jobs: | python3 - --version ${{ env.POETRY_VERSION }}; poetry config virtualenvs.create true; poetry config virtualenvs.in-project true; + poetry env use 3.10; poetry env info; poetry config --list; #---------------------------------------------- From 8339da7edc694f4fe75755e799ee1cbe61f119d2 Mon Sep 17 00:00:00 2001 From: linglp Date: Thu, 19 Sep 2024 13:29:20 -0400 Subject: [PATCH 224/233] update workflow --- .github/workflows/test.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b7f358046..9469f813b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -65,9 +65,8 @@ jobs: run: | curl -sSL https://install.python-poetry.org \ | python3 - --version ${{ env.POETRY_VERSION }}; - poetry config virtualenvs.create true; - poetry config virtualenvs.in-project true; - poetry env use 3.10; + poetry config virtualenvs.create true --local; + poetry config virtualenvs.in-project false --local; poetry env info; poetry config --list; #---------------------------------------------- From 7fee9164fe7eaa9cc4fef2cced9677f93b26b17c Mon Sep 17 00:00:00 2001 From: linglp Date: Thu, 19 Sep 2024 13:33:18 -0400 Subject: [PATCH 225/233] update path --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 9469f813b..5a42fec45 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -76,7 +76,7 @@ jobs: id: cached-poetry-dependencies uses: actions/cache@v4 with: - path: .venv + path: ~/.cache/pypoetry/virtualenvs key: venv-${{ runner.os }}-${{ hashFiles('**/poetry.lock') }}-${{ matrix.python-version }} #---------------------------------------------- From 90671c2afbd143823e735fcdbf8da987b19d7904 Mon Sep 17 00:00:00 2001 From: linglp Date: Thu, 19 Sep 2024 13:38:15 -0400 Subject: [PATCH 226/233] run again --- .github/workflows/test.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 5a42fec45..f50af10c9 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -134,9 +134,9 @@ jobs: SYNAPSE_ACCESS_TOKEN: ${{ secrets.SYNAPSE_ACCESS_TOKEN }} SERVICE_ACCOUNT_CREDS: ${{ secrets.SERVICE_ACCOUNT_CREDS }} run: > - source .venv/bin/activate; - pytest --durations=0 --cov-append --cov-report=term --cov-report=html:htmlcov - --cov-report=xml:coverage.xml --cov=schematic/ --reruns 4 -n 8 tests/unit + poetry shell; + poetry run pytest --durations=0 --cov-append --cov-report=term --cov-report=html:htmlcov + --cov-report=xml:coverage.xml --cov=schematic/ --reruns 4 -n 8 tests/unit; #---------------------------------------------- # run integration test suite @@ -147,8 +147,8 @@ jobs: SYNAPSE_ACCESS_TOKEN: ${{ secrets.SYNAPSE_ACCESS_TOKEN }} SERVICE_ACCOUNT_CREDS: ${{ secrets.SERVICE_ACCOUNT_CREDS }} run: > - source .venv/bin/activate; - pytest --durations=0 --cov-append --cov-report=term --cov-report=html:htmlcov --cov-report=xml:coverage.xml --cov=schematic/ + poetry shell; + poetry run pytest --durations=0 --cov-append --cov-report=term --cov-report=html:htmlcov --cov-report=xml:coverage.xml --cov=schematic/ -m "not (rule_benchmark)" --reruns 4 -n 8 --ignore=tests/unit From 242037bdf73955a6a3b1997c0ce56337272785ec Mon Sep 17 00:00:00 2001 From: linglp Date: Thu, 19 Sep 2024 13:43:18 -0400 Subject: [PATCH 227/233] remove poetry shell --- .github/workflows/test.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index f50af10c9..7d2217195 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -134,7 +134,6 @@ jobs: SYNAPSE_ACCESS_TOKEN: ${{ secrets.SYNAPSE_ACCESS_TOKEN }} SERVICE_ACCOUNT_CREDS: ${{ secrets.SERVICE_ACCOUNT_CREDS }} run: > - poetry shell; poetry run pytest --durations=0 --cov-append --cov-report=term --cov-report=html:htmlcov --cov-report=xml:coverage.xml --cov=schematic/ --reruns 4 -n 8 tests/unit; @@ -147,7 +146,6 @@ jobs: SYNAPSE_ACCESS_TOKEN: ${{ secrets.SYNAPSE_ACCESS_TOKEN }} SERVICE_ACCOUNT_CREDS: ${{ secrets.SERVICE_ACCOUNT_CREDS }} run: > - poetry shell; poetry run pytest --durations=0 --cov-append --cov-report=term --cov-report=html:htmlcov --cov-report=xml:coverage.xml --cov=schematic/ -m "not (rule_benchmark)" --reruns 4 -n 8 --ignore=tests/unit From 75520dd19543474c329c3c658791def120c9d645 Mon Sep 17 00:00:00 2001 From: linglp Date: Thu, 19 Sep 2024 15:45:37 -0400 Subject: [PATCH 228/233] remove caching dependencies --- .github/workflows/test.yml | 18 +++--------------- 1 file changed, 3 insertions(+), 15 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 7d2217195..bbf8b9e95 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -65,25 +65,13 @@ jobs: run: | curl -sSL https://install.python-poetry.org \ | python3 - --version ${{ env.POETRY_VERSION }}; - poetry config virtualenvs.create true --local; - poetry config virtualenvs.in-project false --local; - poetry env info; - poetry config --list; - #---------------------------------------------- - # load cached venv if cache exists - #---------------------------------------------- - - name: Load cached venv - id: cached-poetry-dependencies - uses: actions/cache@v4 - with: - path: ~/.cache/pypoetry/virtualenvs - key: venv-${{ runner.os }}-${{ hashFiles('**/poetry.lock') }}-${{ matrix.python-version }} - + poetry config virtualenvs.create true; + poetry config virtualenvs.in-project false; + #---------------------------------------------- # install dependencies and root project #---------------------------------------------- - name: Install dependencies and root project - if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true' run: poetry install --no-interaction --all-extras #---------------------------------------------- From d586da891a86894fb1498e42527a0c6f3a0a5a84 Mon Sep 17 00:00:00 2001 From: linglp Date: Thu, 19 Sep 2024 15:46:44 -0400 Subject: [PATCH 229/233] remove caching dependencies --- .github/workflows/publish.yml | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index c3225661e..5b85c995f 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -33,21 +33,10 @@ jobs: poetry config virtualenvs.create true; poetry config virtualenvs.in-project true; - #---------------------------------------------- - # load cached venv if cache exists - #---------------------------------------------- - - name: Load cached venv - id: cached-poetry-dependencies - uses: actions/cache@v4 - with: - path: .venv - key: venv-${{ runner.os }}-${{ hashFiles('**/poetry.lock') }}-${{ matrix.python-version }} - #---------------------------------------------- # install dependencies and root project #---------------------------------------------- - name: Install dependencies and root project - if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true' run: poetry install --no-interaction --all-extras #---------------------------------------------- From a19c9339a917f0f0b10e50d8bc59df2353d2c036 Mon Sep 17 00:00:00 2001 From: linglp Date: Thu, 19 Sep 2024 15:47:24 -0400 Subject: [PATCH 230/233] remove caching dependencies --- .github/workflows/pdoc.yml | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/.github/workflows/pdoc.yml b/.github/workflows/pdoc.yml index 81617424f..f2d8a4e41 100644 --- a/.github/workflows/pdoc.yml +++ b/.github/workflows/pdoc.yml @@ -52,21 +52,10 @@ jobs: poetry config virtualenvs.create true; poetry config virtualenvs.in-project true; - #---------------------------------------------- - # load cached venv if cache exists - #---------------------------------------------- - - name: Load cached venv - id: cached-poetry-dependencies - uses: actions/cache@v4 - with: - path: .venv - key: venv-${{ runner.os }}-${{ hashFiles('**/poetry.lock') }}-${{ env.PYTHON_VERSION }} - #---------------------------------------------- # install dependencies and root project #---------------------------------------------- - name: Install dependencies and root project - if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true' run: poetry install --no-interaction --all-extras # create documentation From 41b5ebebb11e5fe9aadad9c2f120a1efad7aa308 Mon Sep 17 00:00:00 2001 From: linglp Date: Thu, 19 Sep 2024 15:48:24 -0400 Subject: [PATCH 231/233] remove caching --- .github/workflows/api_test.yml | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/.github/workflows/api_test.yml b/.github/workflows/api_test.yml index 31b8e7eff..0dea21710 100644 --- a/.github/workflows/api_test.yml +++ b/.github/workflows/api_test.yml @@ -43,21 +43,11 @@ jobs: | python3 - --version ${{ env.POETRY_VERSION }}; poetry config virtualenvs.create true; poetry config virtualenvs.in-project true; - #---------------------------------------------- - # load cached venv if cache exists - #---------------------------------------------- - - name: Load cached venv - id: cached-poetry-dependencies - uses: actions/cache@v4 - with: - path: .venv - key: venv-${{ runner.os }}-${{ hashFiles('**/poetry.lock') }}-${{ matrix.python-version }} - + #---------------------------------------------- # install dependencies and root project #---------------------------------------------- - name: Install dependencies and root project - if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true' run: poetry install --no-interaction --all-extras #---------------------------------------------- From 05836c99d7b8d3d2ab2d40cd1f999b708f44e19a Mon Sep 17 00:00:00 2001 From: Brad Macdonald <52762200+BWMac@users.noreply.github.com> Date: Thu, 19 Sep 2024 14:25:08 -0600 Subject: [PATCH 232/233] [FDS-1725] Missing `entityId` handling testing (#1496) * splits up test_filename_manifest * adds unit tests * updates test_validation * removes commented code * fix test order * revert test_validation changes * revert id * updates filename_validation and testing * removes extra comment * updates tests to txt6 * test python version issue * undo last * redo last * fix pipe * run pytest in line * try poetry shell * removes caching * updates unit tests for dataset_scope * try python version setting * split lines * revert ci test * adds +2 comment * accounts for entityId empty strings * use existing DMGE fixture * corrects val_rule * dummy commit * dummy commit * test without caching * remvoe caching * string version * remove caching * remove test caching * try caching path * remove caching key version * try jenny fix for venv corruption --- .github/workflows/pdoc.yml | 2 +- .github/workflows/test.yml | 16 +- schematic/models/validate_attribute.py | 33 ++- .../InvalidFilenameManifest.csv | 5 +- tests/test_validation.py | 39 ++- tests/unit/test_validate_attribute.py | 258 +++++++++++++++++- 6 files changed, 326 insertions(+), 27 deletions(-) diff --git a/.github/workflows/pdoc.yml b/.github/workflows/pdoc.yml index 81617424f..35640c769 100644 --- a/.github/workflows/pdoc.yml +++ b/.github/workflows/pdoc.yml @@ -28,7 +28,7 @@ jobs: runs-on: ubuntu-latest env: POETRY_VERSION: 1.3.0 - PYTHON_VERSION: 3.10 + PYTHON_VERSION: "3.10" steps: #---------------------------------------------- diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 4df1cb2b9..6dc404a84 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -76,12 +76,24 @@ jobs: with: path: .venv key: venv-${{ runner.os }}-${{ hashFiles('**/poetry.lock') }}-${{ matrix.python-version }} - + + #---------------------------------------------- + # validate cached venv if it exists and remove if corrupt + #---------------------------------------------- + - name: Check and remove broken virtual environment (if necessary) + if: steps.cached-poetry-dependencies.outputs.cache-hit == 'true' + run: | + VENV_PATH=$(poetry env info --path) + if [ -d "$VENV_PATH" ]; then + source $VENV_PATH/bin/activate && poetry run black --version || (echo "Removing broken venv"; rm -rf $VENV_PATH; echo "venv-deleted=true" >> $GITHUB_ENV) + else + echo "Virtual environment not found" + fi #---------------------------------------------- # install dependencies and root project #---------------------------------------------- - name: Install dependencies and root project - if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true' + if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true' || env.venv-deleted == 'true' run: poetry install --no-interaction --all-extras #---------------------------------------------- diff --git a/schematic/models/validate_attribute.py b/schematic/models/validate_attribute.py index 5a562e416..e196bbe14 100644 --- a/schematic/models/validate_attribute.py +++ b/schematic/models/validate_attribute.py @@ -11,8 +11,8 @@ import pandas as pd import requests from jsonschema import ValidationError -from synapseclient.core.exceptions import SynapseNoCredentialsError from synapseclient import File +from synapseclient.core.exceptions import SynapseNoCredentialsError from schematic.schemas.data_model_graph import DataModelGraphExplorer from schematic.store.synapse import SynapseStorage @@ -479,10 +479,17 @@ def generate_filename_error( Errors: list[str] Error details for further storage. warnings: list[str] Warning details for further storage. """ - if error_type == "path does not exist": - error_message = f"The file path '{invalid_entry}' on row {row_num} does not exist in the file view." - elif error_type == "mismatched entityId": - error_message = f"The entityId for file path '{invalid_entry}' on row {row_num} does not match the entityId for the file in the file view" + error_messages = { + "mismatched entityId": f"The entityId for file path '{invalid_entry}' on row {row_num}" + " does not match the entityId for the file in the file view.", + "path does not exist": f"The file path '{invalid_entry}' on row {row_num} does not exist in the file view.", + "entityId does not exist": f"The entityId for file path '{invalid_entry}' on row {row_num}" + " does not exist in the file view.", + "missing entityId": f"The entityId is missing for file path '{invalid_entry}' on row {row_num}.", + } + error_message = error_messages.get(error_type, None) + if not error_message: + raise KeyError(f"Unsupported error type provided: '{error_type}'") error_list, warning_list = GenerateError.raise_and_store_message( dmge=dmge, @@ -2075,12 +2082,11 @@ def filename_validation( fileview = self.synStore.storageFileviewTable.reset_index(drop=True) # filename in dataset? files_in_view = manifest["Filename"].isin(fileview["path"]) + entity_ids_in_view = manifest["entityId"].isin(fileview["id"]) # filenames match with entity IDs in dataset joined_df = manifest.merge( - fileview, how="outer", left_on="Filename", right_on="path" + fileview, how="left", left_on="Filename", right_on="path" ) - # cover case where there are more files in dataset than in manifest - joined_df = joined_df.loc[~joined_df["Component"].isna()].reset_index(drop=True) entity_id_match = joined_df["id"] == joined_df["entityId"] @@ -2089,6 +2095,14 @@ def filename_validation( manifest_with_errors["Error"] = pd.NA manifest_with_errors.loc[~entity_id_match, "Error"] = "mismatched entityId" manifest_with_errors.loc[~files_in_view, "Error"] = "path does not exist" + manifest_with_errors.loc[ + ~entity_ids_in_view, "Error" + ] = "entityId does not exist" + manifest_with_errors.loc[ + (manifest_with_errors["entityId"].isna()) + | (manifest_with_errors["entityId"] == ""), + "Error", + ] = "missing entityId" # Generate errors invalid_entries = manifest_with_errors.loc[ @@ -2098,7 +2112,8 @@ def filename_validation( vr_errors, vr_warnings = GenerateError.generate_filename_error( val_rule=val_rule, attribute_name="Filename", - row_num=str(index), + # +2 to make consistent with other validation functions + row_num=str(index + 2), invalid_entry=data["Filename"], error_type=data["Error"], dmge=self.dmge, diff --git a/tests/data/mock_manifests/InvalidFilenameManifest.csv b/tests/data/mock_manifests/InvalidFilenameManifest.csv index 211f2a12c..bc2619e55 100644 --- a/tests/data/mock_manifests/InvalidFilenameManifest.csv +++ b/tests/data/mock_manifests/InvalidFilenameManifest.csv @@ -1,6 +1,7 @@ Component,Filename,entityId MockFilename,schematic - main/MockFilenameComponent/txt1.txt,syn61682653 MockFilename,schematic - main/MockFilenameComponent/txt2.txt,syn61682660 -MockFilename,schematic - main/MockFilenameComponent/txt3.txt,syn61682662 +MockFilename,schematic - main/MockFilenameComponent/txt3.txt,syn61682653 +MockFilename,schematic - main/MockFilenameComponent/this_file_does_not_exist.txt,syn61682653 MockFilename,schematic - main/MockFilenameComponent/txt4.txt,syn6168265 -MockFilename,schematic - main/MockFilenameComponent/txt5.txt, +MockFilename,schematic - main/MockFilenameComponent/txt6.txt, diff --git a/tests/test_validation.py b/tests/test_validation.py index cf0b70aaa..cdd6766c0 100644 --- a/tests/test_validation.py +++ b/tests/test_validation.py @@ -693,14 +693,13 @@ def test_filename_manifest(self, helpers, dmge): project_scope=["syn23643250"], dataset_scope="syn61682648", ) - # Check errors assert ( GenerateError.generate_filename_error( - val_rule="filenameExists", + val_rule="filenameExists syn61682648", attribute_name="Filename", - row_num="3", - invalid_entry="schematic - main/MockFilenameComponent/txt4.txt", + row_num="4", + invalid_entry="schematic - main/MockFilenameComponent/txt3.txt", error_type="mismatched entityId", dmge=dmge, )[0] @@ -709,17 +708,41 @@ def test_filename_manifest(self, helpers, dmge): assert ( GenerateError.generate_filename_error( - val_rule="filenameExists", + val_rule="filenameExists syn61682648", attribute_name="Filename", - row_num="4", - invalid_entry="schematic - main/MockFilenameComponent/txt5.txt", + row_num="5", + invalid_entry="schematic - main/MockFilenameComponent/this_file_does_not_exist.txt", error_type="path does not exist", dmge=dmge, )[0] in errors ) - assert len(errors) == 2 + assert ( + GenerateError.generate_filename_error( + val_rule="filenameExists syn61682648", + attribute_name="Filename", + row_num="6", + invalid_entry="schematic - main/MockFilenameComponent/txt4.txt", + error_type="entityId does not exist", + dmge=dmge, + )[0] + in errors + ) + + assert ( + GenerateError.generate_filename_error( + val_rule="filenameExists syn61682648", + attribute_name="Filename", + row_num="7", + invalid_entry="schematic - main/MockFilenameComponent/txt6.txt", + error_type="missing entityId", + dmge=dmge, + )[0] + in errors + ) + + assert len(errors) == 4 assert len(warnings) == 0 def test_filename_manifest_exception(self, helpers, dmge): diff --git a/tests/unit/test_validate_attribute.py b/tests/unit/test_validate_attribute.py index 26ab9f167..d782fab12 100644 --- a/tests/unit/test_validate_attribute.py +++ b/tests/unit/test_validate_attribute.py @@ -1,15 +1,15 @@ """Unit testing for the ValidateAttribute class""" from typing import Generator -from unittest.mock import patch -import pytest +from unittest.mock import Mock, patch -from pandas import Series, DataFrame, concat import numpy as np +import pytest +from pandas import DataFrame, Series, concat -from schematic.models.validate_attribute import ValidateAttribute -from schematic.schemas.data_model_graph import DataModelGraphExplorer import schematic.models.validate_attribute +from schematic.models.validate_attribute import GenerateError, ValidateAttribute +from schematic.schemas.data_model_graph import DataModelGraphExplorer # pylint: disable=protected-access # pylint: disable=too-many-public-methods @@ -99,6 +99,53 @@ } ) +TEST_DF_FILEVIEW = DataFrame( + { + "id": ["syn1", "syn2", "syn3"], + "path": ["test1.txt", "test2.txt", "test3.txt"], + } +) + +TEST_MANIFEST_GOOD = DataFrame( + { + "Component": ["Mockfilename", "Mockfilename", "Mockfilename"], + "Filename": ["test1.txt", "test2.txt", "test3.txt"], + "entityId": ["syn1", "syn2", "syn3"], + } +) + +TEST_MANIFEST_MISSING_ENTITY_ID = DataFrame( + { + "Component": ["Mockfilename", "Mockfilename", "Mockfilename"], + "Filename": ["test1.txt", "test2.txt", "test3.txt"], + "entityId": ["syn1", "syn2", ""], + } +) + +TEST_MANIFEST_FILENAME_NOT_IN_VIEW = DataFrame( + { + "Component": ["Mockfilename", "Mockfilename", "Mockfilename"], + "Filename": ["test1.txt", "test2.txt", "test_bad.txt"], + "entityId": ["syn1", "syn2", "syn3"], + } +) + +TEST_MANIFEST_ENTITY_ID_NOT_IN_VIEW = DataFrame( + { + "Component": ["Mockfilename", "Mockfilename", "Mockfilename"], + "Filename": ["test1.txt", "test2.txt", "test3.txt"], + "entityId": ["syn1", "syn2", "syn_bad"], + } +) + +TEST_MANIFEST_ENTITY_ID_MISMATCH = DataFrame( + { + "Component": ["Mockfilename", "Mockfilename", "Mockfilename"], + "Filename": ["test1.txt", "test2.txt", "test3.txt"], + "entityId": ["syn1", "syn2", "syn2"], + } +) + @pytest.fixture(name="va_obj") def fixture_va_obj( @@ -171,6 +218,93 @@ def fixture_cross_val_col_names() -> Generator[dict[str, str], None, None]: yield column_names +class TestGenerateError: + """Unit tests for the GenerateError class""" + + val_rule = "filenameExists syn123456" + attribute_name = "Filename" + row_num = "2" + invalid_entry = "test_file.txt" + + @pytest.mark.parametrize( + "error_type, expected_message", + [ + ( + "mismatched entityId", + "The entityId for file path 'test_file.txt' on row 2 does not match the entityId for the file in the file view.", + ), + ( + "path does not exist", + "The file path 'test_file.txt' on row 2 does not exist in the file view.", + ), + ( + "entityId does not exist", + "The entityId for file path 'test_file.txt' on row 2 does not exist in the file view.", + ), + ( + "missing entityId", + "The entityId is missing for file path 'test_file.txt' on row 2.", + ), + ], + ids=[ + "mismatched entityId", + "path does not exist", + "entityId does not exist", + "missing entityId", + ], + ) + def test_generate_filename_error( + self, dmge: DataModelGraphExplorer, error_type: str, expected_message: str + ): + with patch.object( + GenerateError, + "raise_and_store_message", + return_value=( + [ + self.row_num, + self.attribute_name, + expected_message, + self.invalid_entry, + ], + [], + ), + ) as mock_raise_and_store: + error_list, _ = GenerateError.generate_filename_error( + val_rule=self.val_rule, + attribute_name=self.attribute_name, + row_num=self.row_num, + invalid_entry=self.invalid_entry, + error_type=error_type, + dmge=dmge, + ) + mock_raise_and_store.assert_called_once_with( + dmge=dmge, + val_rule=self.val_rule, + error_row=self.row_num, + error_col=self.attribute_name, + error_message=expected_message, + error_val=self.invalid_entry, + ) + + assert len(error_list) == 4 + assert error_list[2] == expected_message + + def test_generate_filename_error_unsupported_error_type( + self, dmge: DataModelGraphExplorer + ): + with pytest.raises( + KeyError, match="Unsupported error type provided: 'unsupported error type'" + ) as exc_info: + GenerateError.generate_filename_error( + dmge=dmge, + val_rule=self.val_rule, + attribute_name=self.attribute_name, + row_num=self.row_num, + invalid_entry=self.invalid_entry, + error_type="unsupported error type", + ) + + class TestValidateAttributeObject: """Testing for ValidateAttribute class with all Synapse calls mocked""" @@ -532,6 +666,120 @@ def test_cross_validation_value_match_none_rules_errors( assert errors == [] assert len(warnings) == 1 + @pytest.mark.parametrize( + "manifest, expected_errors, expected_warnings, generates_error", + [ + (TEST_MANIFEST_GOOD, [], [], False), + ( + TEST_MANIFEST_MISSING_ENTITY_ID, + [ + [ + "4", + "Filename", + "The entityId for file path 'test3.txt' on row 4 does not exist in the file view.", + "test3.txt", + ] + ], + [], + True, + ), + ( + TEST_MANIFEST_FILENAME_NOT_IN_VIEW, + [ + [ + "4", + "Filename", + "The file path 'test_bad.txt' on row 4 does not exist in the file view.", + "test_bad.txt", + ] + ], + [], + True, + ), + ( + TEST_MANIFEST_ENTITY_ID_NOT_IN_VIEW, + [ + [ + "4", + "Filename", + "The entityId for file path 'test3.txt' on row 4 does not exist in the file view.", + "test3.txt", + ] + ], + [], + True, + ), + ( + TEST_MANIFEST_ENTITY_ID_MISMATCH, + [ + [ + "4", + "Filename", + "The entityId for file path 'test3.txt' on row 4 does not match " + "the entityId for the file in the file view.", + "test3.txt", + ] + ], + [], + True, + ), + ], + ids=[ + "valid_manifest", + "missing_entity_id", + "bad_filename", + "bad_entity_id", + "entity_id_mismatch", + ], + ) + def test_filename_validation( + self, + va_obj: ValidateAttribute, + manifest: DataFrame, + expected_errors: list, + expected_warnings: list, + generates_error: bool, + ): + mock_synapse_storage = Mock() + mock_synapse_storage.storageFileviewTable = TEST_DF_FILEVIEW + va_obj.synStore = mock_synapse_storage + with patch.object( + schematic.models.validate_attribute.ValidateAttribute, + "_login", + ), patch.object( + mock_synapse_storage, "reset_index", return_value=TEST_DF_FILEVIEW + ), patch.object( + schematic.models.validate_attribute.GenerateError, + "generate_filename_error", + return_value=( + expected_errors if len(expected_errors) < 1 else expected_errors[0], + expected_warnings, + ), + ) as mock_generate_filename_error: + actual_errors, actual_warnings = va_obj.filename_validation( + val_rule="filenameExists syn61682648", + manifest=manifest, + access_token="test_access_token", + dataset_scope="syn1", + ) + mock_generate_filename_error.assert_called_once() if generates_error else mock_generate_filename_error.assert_not_called() + assert (actual_errors, actual_warnings) == ( + expected_errors, + expected_warnings, + ) + + def test_filename_validation_null_dataset_scope(self, va_obj: ValidateAttribute): + with pytest.raises( + ValueError, + match="A dataset is required to be specified for filename validation", + ): + va_obj.filename_validation( + val_rule="filenameExists syn61682648", + manifest=TEST_MANIFEST_GOOD, + access_token="test_access_token", + dataset_scope=None, + ) + ######################################### # _run_validation_across_target_manifests ######################################### From 21aa2b879eb654692bd5630808bbb0572d535e67 Mon Sep 17 00:00:00 2001 From: Gianna Jordan <61707471+GiaJordan@users.noreply.github.com> Date: Thu, 19 Sep 2024 19:26:53 -0700 Subject: [PATCH 233/233] Update how annotations are accessed (#1485) * update how annotations are accessed * update submission integration test * Remove venv caching --------- Co-authored-by: Jenny Medina Co-authored-by: Jenny V Medina --- .github/workflows/test.yml | 24 -- schematic/store/synapse.py | 17 +- .../mock_manifests/ValidFilenameManifest.csv | 10 +- ...h_submission_test_manifest_sampleidx10.csv | 3 + tests/integration/test_metadata_model.py | 243 +++++++++++++++--- tests/integration/test_store_synapse.py | 26 +- 6 files changed, 248 insertions(+), 75 deletions(-) create mode 100644 tests/data/mock_manifests/filepath_submission_test_manifest_sampleidx10.csv diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 6dc404a84..27f629731 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -68,34 +68,10 @@ jobs: poetry config virtualenvs.create true; poetry config virtualenvs.in-project true; #---------------------------------------------- - # load cached venv if cache exists - #---------------------------------------------- - - name: Load cached venv - id: cached-poetry-dependencies - uses: actions/cache@v4 - with: - path: .venv - key: venv-${{ runner.os }}-${{ hashFiles('**/poetry.lock') }}-${{ matrix.python-version }} - - #---------------------------------------------- - # validate cached venv if it exists and remove if corrupt - #---------------------------------------------- - - name: Check and remove broken virtual environment (if necessary) - if: steps.cached-poetry-dependencies.outputs.cache-hit == 'true' - run: | - VENV_PATH=$(poetry env info --path) - if [ -d "$VENV_PATH" ]; then - source $VENV_PATH/bin/activate && poetry run black --version || (echo "Removing broken venv"; rm -rf $VENV_PATH; echo "venv-deleted=true" >> $GITHUB_ENV) - else - echo "Virtual environment not found" - fi - #---------------------------------------------- # install dependencies and root project #---------------------------------------------- - name: Install dependencies and root project - if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true' || env.venv-deleted == 'true' run: poetry install --no-interaction --all-extras - #---------------------------------------------- # perform linting #---------------------------------------------- diff --git a/schematic/store/synapse.py b/schematic/store/synapse.py index ae47cdccf..7ccb810cf 100644 --- a/schematic/store/synapse.py +++ b/schematic/store/synapse.py @@ -1578,12 +1578,14 @@ def process_row_annotations( (isinstance(anno_v, str) and anno_v.strip() == "") or (isinstance(anno_v, float) and np.isnan(anno_v)) ): - annos.pop(anno_k) if anno_k in annos.keys() else annos + annos["annotations"]["annotations"].pop(anno_k) if anno_k in annos[ + "annotations" + ]["annotations"].keys() else annos["annotations"]["annotations"] continue # Otherwise save annotation as approrpriate if isinstance(anno_v, float) and np.isnan(anno_v): - annos[anno_k] = "" + annos["annotations"]["annotations"][anno_k] = "" continue # Handle strings that match the csv_list_regex and pass the validation rule @@ -1597,10 +1599,11 @@ def process_row_annotations( node_validation_rules = dmge.get_node_validation_rules(**param) if rule_in_rule_list("list", node_validation_rules): - annos[anno_k] = anno_v.split(",") + annos["annotations"]["annotations"][anno_k] = anno_v.split(",") continue # default: assign the original value - annos[anno_k] = anno_v + annos["annotations"]["annotations"][anno_k] = anno_v + return annos @async_missing_entity_handler @@ -1656,6 +1659,7 @@ async def format_row_annotations( annos = await self.get_async_annotation(entityId) csv_list_regex = comma_separated_list_regex() + annos = self.process_row_annotations( dmge=dmge, metadata_syn=metadataSyn, @@ -1926,7 +1930,10 @@ async def _process_store_annos(self, requests: Set[asyncio.Task]) -> None: else: # store annotations if they are not None if annos: - normalized_annos = {k.lower(): v for k, v in annos.items()} + normalized_annos = { + k.lower(): v + for k, v in annos["annotations"]["annotations"].items() + } entity_id = normalized_annos["entityid"] logger.info( f"Obtained and processed annotations for {entity_id} entity" diff --git a/tests/data/mock_manifests/ValidFilenameManifest.csv b/tests/data/mock_manifests/ValidFilenameManifest.csv index 5bf19e1c0..de1b7a448 100644 --- a/tests/data/mock_manifests/ValidFilenameManifest.csv +++ b/tests/data/mock_manifests/ValidFilenameManifest.csv @@ -1,5 +1,5 @@ -Component,Filename,Id,entityId -MockFilename,schematic - main/TestSubmitMockFilename/txt1.txt,3c4c384b-5c49-4a7c-90a6-43f03a5ddbdc,syn62822369 -MockFilename,schematic - main/TestSubmitMockFilename/txt2.txt,3b45f5f3-408f-47ff-945e-9badf0a43195,syn62822368 -MockFilename,schematic - main/TestSubmitMockFilename/txt3.txt,2bbc898f-2651-4af3-834a-10c506de0fbd,syn62822366 -MockFilename,schematic - main/TestSubmitMockFilename/txt4.txt,5a2d3816-436e-458f-9887-cb8355518e23,syn62822364 +Component,Sample ID,Filename,Id,entityId +MockFilename,1.0,schematic - main/TestSubmitMockFilename/txt1.txt,3c4c384b-5c49-4a7c-90a6-43f03a5ddbdc,syn62822369 +MockFilename,2.0,schematic - main/TestSubmitMockFilename/txt2.txt,3b45f5f3-408f-47ff-945e-9badf0a43195,syn62822368 +MockFilename,3.0,schematic - main/TestSubmitMockFilename/txt3.txt,2bbc898f-2651-4af3-834a-10c506de0fbd,syn62822366 +MockFilename,4.0,schematic - main/TestSubmitMockFilename/txt4.txt,5a2d3816-436e-458f-9887-cb8355518e23,syn62822364 diff --git a/tests/data/mock_manifests/filepath_submission_test_manifest_sampleidx10.csv b/tests/data/mock_manifests/filepath_submission_test_manifest_sampleidx10.csv new file mode 100644 index 000000000..082170549 --- /dev/null +++ b/tests/data/mock_manifests/filepath_submission_test_manifest_sampleidx10.csv @@ -0,0 +1,3 @@ +Filename,Sample ID,File Format,Component,Genome Build,Genome FASTA,Id,entityId +schematic - main/Test Filename Upload/txt1.txt,10.0,,BulkRNA-seqAssay,,,01ded8fc-0915-4959-85ab-64e9644c8787,syn62276954 +schematic - main/Test Filename Upload/txt2.txt,20.0,,BulkRNA-seqAssay,,,fd122bb5-3353-4c94-b1f5-0bb93a3e9fc9,syn62276956 diff --git a/tests/integration/test_metadata_model.py b/tests/integration/test_metadata_model.py index ff221d731..6aa15bf18 100644 --- a/tests/integration/test_metadata_model.py +++ b/tests/integration/test_metadata_model.py @@ -1,9 +1,21 @@ +""" +This script contains a test suite for verifying the submission and annotation of +file-based manifests using the `TestMetadataModel` class to communicate with Synapse +and verify the expected behavior of uploading annotation manifest CSVs using the +metadata model. + +It utilizes the `pytest` framework along with `pytest-mock` to mock and spy on methods +of the `SynapseStorage` class, which is responsible for handling file uploads and +annotations in Synapse. +""" + import logging +import pytest +import tempfile + from contextlib import nullcontext as does_not_raise -import pytest from pytest_mock import MockerFixture - from schematic.store.synapse import SynapseStorage from tests.conftest import metadata_model @@ -12,36 +24,152 @@ class TestMetadataModel: + # Define the test cases as a class attribute + test_cases = [ + # Test 1: Check that a valid manifest can be submitted, and corresponding entities annotated from it + ( + "mock_manifests/filepath_submission_test_manifest.csv", + "syn62276880", + None, + "syn62280543", + "syn53011753", + None, + ), + # Test 2: Change the Sample ID annotation from the previous test to ensure the manifest file is getting updated + ( + "mock_manifests/filepath_submission_test_manifest_sampleidx10.csv", + "syn62276880", + None, + "syn62280543", + "syn53011753", + None, + ), + # Test 3: Test manifest file upload with validation based on the MockFilename component and given dataset_scope + ( + "mock_manifests/ValidFilenameManifest.csv", + "syn62822337", + "MockFilename", + "syn62822975", + "syn63192751", + "syn62822337", + ), + ] + + def validate_manifest_annotations( + self, + manifest_annotations, + manifest_entity_type, + expected_entity_id, + manifest_file_contents=None, + ): + """ + Validates that the annotations on a manifest entity (file or table) were correctly updated + by comparing the annotations on the manifest entity with the contents of the manifest file itself, + and ensuring the eTag annotation is not empty. + + This method is wrapped by ``_submit_and_verify_manifest()`` + + Arguments: + manifest_annotations (pd.DataFrame): manifest annotations + manifest_entity_type (str): type of manifest (file or table) + expected_entity_id (str): expected entity ID of the manifest + manifest_file_contents (pd.DataFrame): manifest file contents + + Returns: + None + """ + # Check that the eTag annotation is not empty + assert len(manifest_annotations["eTag"][0]) > 0 + + # Check that entityId is expected + assert manifest_annotations["entityId"][0] == expected_entity_id + + # For manifest files only: Check that all other annotations from the manifest match the annotations in the manifest file itself + if manifest_entity_type.lower() != "file": + return + for annotation in manifest_annotations.keys(): + if annotation in ["eTag", "entityId"]: + continue + else: + assert ( + manifest_annotations[annotation][0] + == manifest_file_contents[annotation].unique() + ) + + @pytest.mark.parametrize( + "manifest_path, dataset_id, validate_component, expected_manifest_id, " + "expected_table_id, dataset_scope", + test_cases, + ) + def test_submit_filebased_manifest_file_and_entities( + self, + helpers, + manifest_path, + dataset_id, + validate_component, + expected_manifest_id, + expected_table_id, + dataset_scope, + mocker: MockerFixture, + synapse_store, + ): + self._submit_and_verify_manifest( + helpers=helpers, + mocker=mocker, + synapse_store=synapse_store, + manifest_path=manifest_path, + dataset_id=dataset_id, + expected_manifest_id=expected_manifest_id, + expected_table_id=expected_table_id, + manifest_record_type="file_and_entities", + validate_component=validate_component, + dataset_scope=dataset_scope, + ) + @pytest.mark.parametrize( - "manifest_path, dataset_id, validate_component, expected_manifest_id, dataset_scope", - [ - ( - "mock_manifests/filepath_submission_test_manifest.csv", - "syn62276880", - None, - "syn62280543", - None, - ), - ( - "mock_manifests/ValidFilenameManifest.csv", - "syn62822337", - "MockFilename", - "syn62822975", - "syn62822337", - ), - ], + "manifest_path, dataset_id, validate_component, expected_manifest_id, " + "expected_table_id, dataset_scope", + test_cases, ) - def test_submit_filebased_manifest( + def test_submit_filebased_manifest_table_and_file( self, helpers, manifest_path, dataset_id, validate_component, expected_manifest_id, + expected_table_id, dataset_scope, mocker: MockerFixture, + synapse_store, + ): + self._submit_and_verify_manifest( + helpers=helpers, + mocker=mocker, + synapse_store=synapse_store, + manifest_path=manifest_path, + dataset_id=dataset_id, + expected_manifest_id=expected_manifest_id, + expected_table_id=expected_table_id, + manifest_record_type="table_and_file", + validate_component=validate_component, + dataset_scope=dataset_scope, + ) + + def _submit_and_verify_manifest( + self, + helpers, + mocker, + synapse_store, + manifest_path, + dataset_id, + expected_manifest_id, + expected_table_id, + manifest_record_type, + validate_component=None, + dataset_scope=None, ): - # spys + # Spies spy_upload_file_as_csv = mocker.spy(SynapseStorage, "upload_manifest_as_csv") spy_upload_file_as_table = mocker.spy( SynapseStorage, "upload_manifest_as_table" @@ -54,16 +182,21 @@ def test_submit_filebased_manifest( # GIVEN a metadata model object using class labels meta_data_model = metadata_model(helpers, "class_label") - # AND a filebased test manifset - manifest_path = helpers.get_data_path(manifest_path) + # AND a filebased test manifest + load_args = {"dtype": "string"} + manifest = helpers.get_data_frame( + manifest_path, preserve_raw_input=True, **load_args + ) + manifest_full_path = helpers.get_data_path(manifest_path) - # WHEN the manifest it submitted + # WHEN the manifest is submitted and files are annotated # THEN submission should complete without error + with does_not_raise(): manifest_id = meta_data_model.submit_metadata_manifest( - manifest_path=manifest_path, + manifest_path=manifest_full_path, dataset_id=dataset_id, - manifest_record_type="file_and_entities", + manifest_record_type=manifest_record_type, restrict_rules=False, file_annotations_upload=True, hide_blanks=False, @@ -71,14 +204,58 @@ def test_submit_filebased_manifest( dataset_scope=dataset_scope, ) - # AND the manifest should be submitted to the correct place - assert manifest_id == expected_manifest_id + # AND the files should be annotated + spy_add_annotations.assert_called_once() - # AND the manifest should be uploaded as a CSV - spy_upload_file_as_csv.assert_called_once() - # AND annotations should be added to the files - spy_add_annotations.assert_called_once() + # AND the annotations on the entities should have the correct metadata + for index, row in manifest.iterrows(): + entityId = row["entityId"] + expected_sample_id = row["Sample ID"] + annos = synapse_store.syn.get_annotations(entityId) + sample_id = annos["SampleID"][0] + assert str(sample_id) == str(expected_sample_id) + + # AND the annotations on the manifest file itself are correct + manifest_file_annotations = synapse_store.syn.get_annotations( + expected_manifest_id + ) + self.validate_manifest_annotations( + manifest_annotations=manifest_file_annotations, + manifest_entity_type="file", + expected_entity_id=expected_manifest_id, + manifest_file_contents=manifest, + ) - # AND the manifest should not be uploaded as a table or combination of table, file, and entities + if manifest_record_type == "table_and_file": + with tempfile.TemporaryDirectory() as download_dir: + manifest_table = synapse_store.syn.tableQuery( + f"select * from {expected_table_id}", downloadLocation=download_dir + ).asDataFrame() + + # AND the columns in the manifest table should reflect the ones in the file + table_columns = manifest_table.columns + manifest_columns = [col.replace(" ", "") for col in manifest.columns] + assert set(table_columns) == set(manifest_columns) + + # AND the annotations on the manifest table itself are correct + manifest_table_annotations = synapse_store.syn.get_annotations( + expected_table_id + ) + self.validate_manifest_annotations( + manifest_annotations=manifest_table_annotations, + manifest_entity_type="table", + expected_entity_id=expected_table_id, + ) + + # AND the manifest should be submitted to the correct place + assert manifest_id == expected_manifest_id + + # AND the correct upload methods were called for the given record type + if manifest_record_type == "file_and_entities": + spy_upload_file_as_csv.assert_called_once() spy_upload_file_as_table.assert_not_called() spy_upload_file_combo.assert_not_called() + elif manifest_record_type == "table_and_file": + spy_upload_file_as_table.assert_called_once() + spy_upload_file_as_csv.assert_not_called() + spy_upload_file_combo.assert_not_called() diff --git a/tests/integration/test_store_synapse.py b/tests/integration/test_store_synapse.py index 61d8ec3cb..0bf23c1b1 100644 --- a/tests/integration/test_store_synapse.py +++ b/tests/integration/test_store_synapse.py @@ -34,12 +34,16 @@ def test_process_row_annotations_hide_blanks( "CancerType": " ", # Blank value (whitespace string) } annos = { - "PatientID": "old_value1", - "Sex": "old_value2", - "Diagnosis": "old_value3", - "FamilyHistory": "old_value4", - "YearofBirth": "old_value5", - "CancerType": "old_value6", + "annotations": { + "annotations": { + "PatientID": "old_value1", + "Sex": "old_value2", + "Diagnosis": "old_value3", + "FamilyHistory": "old_value4", + "YearofBirth": "old_value5", + "CancerType": "old_value6", + } + } } comma_separated_list = comma_separated_list_regex() processed_annos = synapse_store.process_row_annotations( @@ -50,6 +54,8 @@ def test_process_row_annotations_hide_blanks( annos=annos, annotation_keys=label_options, ) + processed_annos = processed_annos["annotations"]["annotations"] + # make sure that empty keys are removed if hideBlanks is True if hideBlanks: assert ( @@ -87,7 +93,7 @@ def test_process_row_annotations_get_validation( metadata_syn = { "FamilyHistory": "value1,value2,value3", } - annos = {"FamilyHistory": "old_value"} + annos = {"annotations": {"annotations": {"FamilyHistory": "old_value"}}} dmge.get_node_validation_rules = MagicMock() @@ -115,4 +121,8 @@ def test_process_row_annotations_get_validation( # get_node_validation_rules was called with node_label dmge.get_node_validation_rules.assert_any_call(node_label="FamilyHistory") # ensure that the value is split into a list - assert processed_annos["FamilyHistory"] == ["value1", "value2", "value3"] + assert processed_annos["annotations"]["annotations"]["FamilyHistory"] == [ + "value1", + "value2", + "value3", + ]