From e84397e68d7a52be7c854e271c29d11e82684ff6 Mon Sep 17 00:00:00 2001 From: Nathan Perkins Date: Tue, 24 Oct 2023 11:26:14 +0100 Subject: [PATCH 1/7] reformatting str for mypy --- darwin/future/core/types/query.py | 45 ++++++++++++++++--- darwin/future/meta/objects/base.py | 44 +----------------- darwin/future/meta/objects/dataset.py | 8 ++++ darwin/future/meta/objects/stage.py | 6 +++ darwin/future/meta/objects/team.py | 7 +++ darwin/future/meta/objects/team_member.py | 7 +++ darwin/future/meta/objects/workflow.py | 7 +++ .../tests/meta/objects/test_datasetmeta.py | 2 +- .../tests/meta/objects/test_stagemeta.py | 2 +- .../tests/meta/objects/test_teammeta.py | 2 +- pyproject.toml | 2 +- 11 files changed, 78 insertions(+), 54 deletions(-) diff --git a/darwin/future/core/types/query.py b/darwin/future/core/types/query.py index 75b278505..0465c0a61 100644 --- a/darwin/future/core/types/query.py +++ b/darwin/future/core/types/query.py @@ -106,12 +106,43 @@ def _from_kwarg(cls, key: str, value: str) -> QueryFilter: class Query(Generic[T], ABC): - """Basic Query object with methods to manage filters + """ + A basic Query object with methods to manage filters. This is an abstract class not + meant to be used directly. Use a subclass instead, like DatasetQuery. + This class will lazy load results and cache them internally, and allows for filtering + of the objects locally by default. To execute the query, call the collect() method, + or iterate over the query object. + + Attributes: + meta_params (dict): A dictionary of metadata parameters. + client (ClientCore): The client used to execute the query. + filters (List[QueryFilter]): A list of QueryFilter objects used to filter the query results. + results (List[T]): A list of query results, cached internally for iterable access. + _changed_since_last (bool): A boolean indicating whether the query has changed since the last execution. + Methods: - filter: adds a filter to the query object, returns a new query object - where: Applies a filter on the query object, returns a new query object - collect: Executes the query on the client and returns the results - _generic_execute_filter: Executes a filter on a list of objects + filter(name: str, param: str, modifier: Optional[Modifier] = None) -> Query[T]: + Adds a filter to the query object and returns a new query object. + where(name: str, param: str, modifier: Optional[Modifier] = None) -> Query[T]: + Applies a filter on the query object and returns a new query object. + first() -> Optional[T]: + Returns the first result of the query. Raises an exception if no results are found. + collect() -> List[T]: + Executes the query on the client and returns the results. Raises an exception if no results are found. + _generic_execute_filter(objects: List[T], filter_: QueryFilter) -> List[T]: + Executes a filter on a list of objects. Locally by default, but can be overwritten by subclasses. + + Examples: + # Create a query object + # DatasetQuery is linked to the object it returns, Dataset, and is iterable + # overwrite the _collect() method to execute insantiate this object + Class DatasetQuery(Query[Dataset]): + ... + + # Intended usage via chaining + # where client.team.datasets returns a DatasetQuery object and can be chained + # further with multiple where calls before collecting + datasets = client.team.datasets.where(...).where(...).collect() """ def __init__( @@ -209,11 +240,11 @@ def collect_one(self) -> T: raise MoreThanOneResultFound("More than one result found") return self.results[0] - def first(self) -> Optional[T]: + def first(self) -> T: if not self.results: self.results = list(self.collect()) if len(self.results) == 0: - return None + raise ResultsNotFound("No results found") return self.results[0] def _generic_execute_filter(self, objects: List[T], filter: QueryFilter) -> List[T]: diff --git a/darwin/future/meta/objects/base.py b/darwin/future/meta/objects/base.py index d86689e3b..ba5bdb884 100644 --- a/darwin/future/meta/objects/base.py +++ b/darwin/future/meta/objects/base.py @@ -1,6 +1,5 @@ from __future__ import annotations -import pprint from typing import Dict, Generic, Optional, TypeVar from darwin.future.core.client import ClientCore @@ -21,46 +20,5 @@ def __init__( self._element = element self.meta_params = meta_params or {} - def __str__(self) -> str: - class_name = self.__class__.__name__ - if class_name == "Team": - return f"Team\n\ -- Team Name: {self._element.name}\n\ -- Team Slug: {self._element.slug}\n\ -- Team ID: {self._element.id}\n\ -- {len(self._element.members if self._element.members else [])} member(s)" - - elif class_name == "TeamMember": - return f"Team Member\n\ -- Name: {self._element.first_name} {self._element.last_name}\n\ -- Role: {self._element.role.value}\n\ -- Email: {self._element.email}\n\ -- User ID: {self._element.user_id}" - - elif class_name == "Dataset": - releases = self._element.releases - return f"Dataset\n\ -- Name: {self._element.name}\n\ -- Dataset Slug: {self._element.slug}\n\ -- Dataset ID: {self._element.id}\n\ -- Dataset Releases: {releases if releases else 'No releases'}" - - elif class_name == "Workflow": - return f"Workflow\n\ -- Workflow Name: {self._element.name}\n\ -- Workflow ID: {self._element.id}\n\ -- Connected Dataset ID: {self._element.dataset.id}\n\ -- Conneted Dataset Name: {self._element.dataset.name}" - - elif class_name == "Stage": - return f"Stage\n\ -- Stage Name: {self._element.name}\n\ -- Stage Type: {self._element.type.value}\n\ -- Stage ID: {self._element.id}" - - else: - return f"Class type '{class_name}' not found in __str__ method:\ -\n{pprint.pformat(self)}" - def __repr__(self) -> str: - return str(self._element) + return str(self) diff --git a/darwin/future/meta/objects/dataset.py b/darwin/future/meta/objects/dataset.py index 8696d41b6..c06554cfb 100644 --- a/darwin/future/meta/objects/dataset.py +++ b/darwin/future/meta/objects/dataset.py @@ -141,3 +141,11 @@ def upload_files( verbose, ) return self + + def __str__(self) -> str: + releases = self._element.releases + return f"Dataset\n\ +- Name: {self._element.name}\n\ +- Dataset Slug: {self._element.slug}\n\ +- Dataset ID: {self._element.id}\n\ +- Dataset Releases: {releases if releases else 'No releases'}" diff --git a/darwin/future/meta/objects/stage.py b/darwin/future/meta/objects/stage.py index e6411fd88..a67ecea62 100644 --- a/darwin/future/meta/objects/stage.py +++ b/darwin/future/meta/objects/stage.py @@ -67,3 +67,9 @@ def type(self) -> str: def edges(self) -> List[WFEdgeCore]: """Edge ID, source stage ID, target stage ID.""" return list(self._element.edges) + + def __str__(self) -> str: + return f"Stage\n\ +- Stage Name: {self._element.name}\n\ +- Stage Type: {self._element.type.value}\n\ +- Stage ID: {self._element.id}" diff --git a/darwin/future/meta/objects/team.py b/darwin/future/meta/objects/team.py index e0c9f144d..9353ba75d 100644 --- a/darwin/future/meta/objects/team.py +++ b/darwin/future/meta/objects/team.py @@ -147,3 +147,10 @@ def _delete_dataset_by_id(client: ClientCore, dataset_id: int) -> int: def create_dataset(self, slug: str) -> Dataset: core = Dataset.create_dataset(self.client, slug) return Dataset(self.client, core, meta_params={"team_slug": self.slug}) + + def __str__(self) -> str: + return f"Team\n\ +- Team Name: {self._element.name}\n\ +- Team Slug: {self._element.slug}\n\ +- Team ID: {self._element.id}\n\ +- {len(self._element.members if self._element.members else [])} member(s)" diff --git a/darwin/future/meta/objects/team_member.py b/darwin/future/meta/objects/team_member.py index 6f87d4326..991fc25cf 100644 --- a/darwin/future/meta/objects/team_member.py +++ b/darwin/future/meta/objects/team_member.py @@ -7,3 +7,10 @@ class TeamMember(MetaBase[TeamMemberCore]): @property def role(self) -> TeamMemberRole: return self._element.role + + def __str__(self) -> str: + return f"Team Member\n\ +- Name: {self._element.first_name} {self._element.last_name}\n\ +- Role: {self._element.role.value}\n\ +- Email: {self._element.email}\n\ +- User ID: {self._element.user_id}" diff --git a/darwin/future/meta/objects/workflow.py b/darwin/future/meta/objects/workflow.py index 3564f6412..d5a1851e8 100644 --- a/darwin/future/meta/objects/workflow.py +++ b/darwin/future/meta/objects/workflow.py @@ -73,3 +73,10 @@ def upload_files( if auto_push: self.push_from_dataset_stage() return self + + def __str__(self) -> str: + return f"Workflow\n\ +- Workflow Name: {self._element.name}\n\ +- Workflow ID: {self._element.id}\n\ +- Connected Dataset ID: {self.datasets[0].id}\n\ +- Conneted Dataset Name: {self.datasets[0].name}" diff --git a/darwin/future/tests/meta/objects/test_datasetmeta.py b/darwin/future/tests/meta/objects/test_datasetmeta.py index 3e32bc335..5f1898607 100644 --- a/darwin/future/tests/meta/objects/test_datasetmeta.py +++ b/darwin/future/tests/meta/objects/test_datasetmeta.py @@ -109,4 +109,4 @@ def test_dataset_str_method(base_meta_dataset: Dataset) -> None: def test_dataset_repr_method(base_meta_dataset: Dataset) -> None: - assert base_meta_dataset.__repr__() == str(base_meta_dataset._element) + assert base_meta_dataset.__repr__() == str(base_meta_dataset) diff --git a/darwin/future/tests/meta/objects/test_stagemeta.py b/darwin/future/tests/meta/objects/test_stagemeta.py index f3958125f..06eeff96d 100644 --- a/darwin/future/tests/meta/objects/test_stagemeta.py +++ b/darwin/future/tests/meta/objects/test_stagemeta.py @@ -150,4 +150,4 @@ def test_stage_str_method(stage_meta: Stage) -> None: def test_stage_repr_method(stage_meta: Stage) -> None: - assert repr(stage_meta) == str(stage_meta._element) + assert repr(stage_meta) == str(stage_meta) diff --git a/darwin/future/tests/meta/objects/test_teammeta.py b/darwin/future/tests/meta/objects/test_teammeta.py index 13dea047b..5f2fc2170 100644 --- a/darwin/future/tests/meta/objects/test_teammeta.py +++ b/darwin/future/tests/meta/objects/test_teammeta.py @@ -193,4 +193,4 @@ def test_team_str_method(base_meta_team: Team) -> None: def test_team_repr_method(base_meta_team: Team) -> None: - assert repr(base_meta_team) == str(base_meta_team._element) + assert repr(base_meta_team) == str(base_meta_team) diff --git a/pyproject.toml b/pyproject.toml index e9e2f36e0..41deba54e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -56,7 +56,7 @@ warn_untyped_fields = true [tool.ruff] select = ["E", "F", "C"] -ignore = ["E203", "E402"] +ignore = ["E203", "E402", "E501"] line-length = 88 [tool.ruff.per-file-ignores] From 4c99c2dda33682a233b026df599a8e68de250c2d Mon Sep 17 00:00:00 2001 From: Nathan Perkins Date: Tue, 24 Oct 2023 15:17:54 +0100 Subject: [PATCH 2/7] comments and dogfooding changes --- darwin/future/data_objects/pydantic_base.py | 3 +- darwin/future/meta/client.py | 28 ++++++++++ darwin/future/meta/objects/base.py | 23 ++++++++ darwin/future/meta/objects/stage.py | 30 ++++++++++- darwin/future/meta/objects/team.py | 54 ++++++++++++++----- darwin/future/meta/objects/team_member.py | 51 ++++++++++++++++-- .../tests/meta/objects/test_teammeta.py | 13 ++--- 7 files changed, 172 insertions(+), 30 deletions(-) diff --git a/darwin/future/data_objects/pydantic_base.py b/darwin/future/data_objects/pydantic_base.py index 9e5ebee24..120d6d2df 100644 --- a/darwin/future/data_objects/pydantic_base.py +++ b/darwin/future/data_objects/pydantic_base.py @@ -2,7 +2,8 @@ class DefaultDarwin(BaseModel): - """Default Darwin-Py pydantic settings for meta information. + """ + Default Darwin-Py pydantic settings for meta information. Default settings include: - auto validating variables on setting/assignment - underscore attributes are private diff --git a/darwin/future/meta/client.py b/darwin/future/meta/client.py index ceb4a182f..011dcea73 100644 --- a/darwin/future/meta/client.py +++ b/darwin/future/meta/client.py @@ -10,6 +10,34 @@ class Client(ClientCore): + """ + The Darwin Client object. Provides access to Darwin's API. + + Args: + ClientCore (Client): Generic ClientCore object expanded by DarwinConfig object + return type + + Returns: + _type_: Client + + Attributes: + _team (Optional[Team]): The team associated with the client. + + Methods: + local(cls) -> Client: Creates a new client object with a local DarwinConfig. + from_api_key(cls, api_key: str, datasets_dir: Optional[Path] = None) -> Client: + Creates a new client object with a DarwinConfig from an API key. + + Example Usage: + # Create a new client object with a local DarwinConfig + client = Client.local() + + # Create a new client object with a DarwinConfig from an API key + client = Client.from_api_key(api_key="my_api_key", datasets_dir="path/to/datasets/dir") + + # Access the team via chaining + team = client.team # returns a Team object which can be chained further + """ def __init__(self, config: DarwinConfig, retries: Optional[Retry] = None) -> None: self._team: Optional[Team] = None super().__init__(config, retries=retries) diff --git a/darwin/future/meta/objects/base.py b/darwin/future/meta/objects/base.py index ba5bdb884..31ec41477 100644 --- a/darwin/future/meta/objects/base.py +++ b/darwin/future/meta/objects/base.py @@ -10,6 +10,29 @@ class MetaBase(Generic[R]): + """ + A base class for metadata objects. This should only ever be inherited from in meta objects. + stores metadata parameters used to access the api that are related to the Meta Objects + but potentially not required for the core object. For example, a dataset object needs + the team slug to access the api which get's passed down from the team object. + + Attributes: + _element (R): The element R to which the object is related. + client (ClientCore): The client used to execute the query. + meta_params (Dict[str, object]): A dictionary of metadata parameters. This is + used in conjuction with the Query object to execute related api calls. + + Methods: + __init__(client: ClientCore, element: R, meta_params: Optional[Param] = None) -> None: + Initializes a new MetaBase object. + __repr__() -> str: + Returns a string representation of the object. + + Examples: + # Create a MetaBase type that manages a TeamCore object from the API + class Team(MetaBase[TeamCore]): + ... + """ _element: R client: ClientCore diff --git a/darwin/future/meta/objects/stage.py b/darwin/future/meta/objects/stage.py index a67ecea62..80e1432ea 100644 --- a/darwin/future/meta/objects/stage.py +++ b/darwin/future/meta/objects/stage.py @@ -9,10 +9,36 @@ class Stage(MetaBase[WFStageCore]): - """_summary_ + """ + Stage Meta object. Facilitates the creation of Query objects, lazy loading of + sub fields Args: - MetaBase (_type_): _description_ + MetaBase (Stage): Generic MetaBase object expanded by WFStageCore object + return type + + Returns: + _type_: Stage + + Attributes: + name (str): The name of the stage. + slug (str): The slug of the stage. + id (UUID): The id of the stage. + item_ids (List[UUID]): A list of item ids attached to the stage. + edges (List[WFEdgeCore]): A list of edges attached to the stage. + + Methods: + move_attached_files_to_stage(new_stage_id: UUID) -> Stage: + Moves all attached files to a new stage. + + Example Usage: + # Get the item ids attached to the stage + stage = client.team.workflows.where(name='test').stages[0] + item_ids = stage.item_ids + + # Move all attached files to a new stage + new_stage = stage.edges[1] + stage.move_attached_files_to_stage(new_stage_id=new_stage.id) """ @property diff --git a/darwin/future/meta/objects/team.py b/darwin/future/meta/objects/team.py index 9353ba75d..c5f713f1d 100644 --- a/darwin/future/meta/objects/team.py +++ b/darwin/future/meta/objects/team.py @@ -29,6 +29,41 @@ class Team(MetaBase[TeamCore]): Returns: Team: Team object + + Attributes: + name (str): The name of the team. + slug (str): The slug of the team. + id (int): The id of the team. + members (List[TeamMember]): A list of team members associated with the team. + datasets (List[Dataset]): A list of datasets associated with the team. + workflows (List[Workflow]): A list of workflows associated with the team. + + Methods: + create_dataset(slug: str) -> Dataset: + Creates a new dataset with the given name and slug. + delete_dataset(client: Client, id: int) -> None: + Removes the dataset with the given slug. + + + Example Usage: + # Get the team object + client = Client.local() + team = client.team[0] + + # Get a dataset object associated with the team + dataset = team.datasets.where(name="my_dataset_name").collect_one() + + # Create a new dataset associated with the team + new_dataset = team.create_dataset(name="new_dataset", slug="new_dataset_slug") + + # Remove a dataset associated with the team + team.remove_dataset(slug="my_dataset_slug") + + # Get a workflow object associated with the team + workflow = team.workflows.where(name="my_workflow_name").collect_one() + + # Get a team member object associated with the team + team_member = team.members.where(email="...") """ def __init__(self, client: ClientCore, team: Optional[TeamCore] = None) -> None: @@ -63,7 +98,7 @@ def workflows(self) -> WorkflowQuery: @classmethod def delete_dataset( cls, client: ClientCore, dataset_id: Union[int, str] - ) -> Tuple[Optional[List[Exception]], int]: + ) -> int: """ Deletes a dataset by id or slug @@ -77,19 +112,12 @@ def delete_dataset( Tuple[Optional[List[Exception]], int] A tuple containing a list of exceptions and the number of datasets deleted """ - exceptions = [] - dataset_deleted = -1 - - try: - if isinstance(dataset_id, str): - dataset_deleted = cls._delete_dataset_by_slug(client, dataset_id) - else: - dataset_deleted = cls._delete_dataset_by_id(client, dataset_id) - - except Exception as e: - exceptions.append(e) + if isinstance(dataset_id, str): + dataset_deleted = cls._delete_dataset_by_slug(client, dataset_id) + else: + dataset_deleted = cls._delete_dataset_by_id(client, dataset_id) - return exceptions or None, dataset_deleted + return dataset_deleted @staticmethod def _delete_dataset_by_slug(client: ClientCore, slug: str) -> int: diff --git a/darwin/future/meta/objects/team_member.py b/darwin/future/meta/objects/team_member.py index 991fc25cf..a68830de7 100644 --- a/darwin/future/meta/objects/team_member.py +++ b/darwin/future/meta/objects/team_member.py @@ -4,13 +4,54 @@ class TeamMember(MetaBase[TeamMemberCore]): + """ + Team Member Meta object. Facilitates the creation of Query objects, lazy loading of + sub fields + + Args: + MetaBase (TeamMember): Generic MetaBase object expanded by TeamMemberCore object + return type + + Returns: + _type_: TeamMember + + Attributes: + first_name (str): The first name of the team member. + last_name (str): The last name of the team member. + email (str): The email of the team member. + user_id (int): The user id of the team member. + role (TeamMemberRole): The role of the team member. + + Methods: + None + + Example Usage: + # Get the role of the team member + team_member = client.team.members + .where(first_name='John', last_name='Doe') + .collect_one() + + role = team_member.role + """ @property def role(self) -> TeamMemberRole: return self._element.role - + @property + def first_name(self) -> str: + return self._element.first_name + @property + def last_name(self) -> str: + return self._element.last_name + @property + def email(self) -> str: + return self._element.email + @property + def user_id(self) -> int: + return self._element.user_id + def __str__(self) -> str: return f"Team Member\n\ -- Name: {self._element.first_name} {self._element.last_name}\n\ -- Role: {self._element.role.value}\n\ -- Email: {self._element.email}\n\ -- User ID: {self._element.user_id}" +- Name: {self.first_name} {self.last_name}\n\ +- Role: {self.role.value}\n\ +- Email: {self.email}\n\ +- User ID: {self.user_id}" diff --git a/darwin/future/tests/meta/objects/test_teammeta.py b/darwin/future/tests/meta/objects/test_teammeta.py index 5f2fc2170..25a73473c 100644 --- a/darwin/future/tests/meta/objects/test_teammeta.py +++ b/darwin/future/tests/meta/objects/test_teammeta.py @@ -47,12 +47,9 @@ def test_delete_dataset_returns_exceptions_thrown( _delete_by_slug_mock.side_effect = Exception("test exception") valid_client = Client(base_config) + with raises(Exception): + _ = Team.delete_dataset(valid_client, "test_dataset") - exceptions, dataset_deleted = Team.delete_dataset(valid_client, "test_dataset") - - assert exceptions is not None - assert str(exceptions[0]) == "test exception" - assert dataset_deleted == -1 assert _delete_by_slug_mock.call_count == 1 assert _delete_by_id_mock.call_count == 0 @@ -63,9 +60,8 @@ def test_delete_dataset_calls_delete_by_slug_as_appropriate( ) -> None: valid_client = Client(base_config) - exceptions, _ = Team.delete_dataset(valid_client, "test_dataset") + _ = Team.delete_dataset(valid_client, "test_dataset") - assert exceptions is None assert _delete_by_slug_mock.call_count == 1 assert _delete_by_id_mock.call_count == 0 @@ -75,9 +71,8 @@ def test_delete_dataset_calls_delete_by_id_as_appropriate( ) -> None: valid_client = Client(base_config) - exceptions, _ = Team.delete_dataset(valid_client, 1) + _ = Team.delete_dataset(valid_client, 1) - assert exceptions is None assert _delete_by_slug_mock.call_count == 0 assert _delete_by_id_mock.call_count == 1 From e75cb753a3e6e454d65f8518b44a2816e5576107 Mon Sep 17 00:00:00 2001 From: Nathan Perkins Date: Tue, 24 Oct 2023 15:32:55 +0100 Subject: [PATCH 3/7] comments and dogfooding changes --- darwin/future/core/team/get_raw.py | 10 +++++- darwin/future/core/team/get_team.py | 30 ++++++++++++++++- darwin/future/meta/client.py | 3 +- darwin/future/meta/objects/base.py | 5 +-- darwin/future/meta/objects/team.py | 12 +++---- darwin/future/meta/objects/team_member.py | 9 +++-- darwin/future/meta/objects/workflow.py | 33 +++++++++++++++++++ .../tests/meta/objects/test_teammeta.py | 1 - 8 files changed, 88 insertions(+), 15 deletions(-) diff --git a/darwin/future/core/team/get_raw.py b/darwin/future/core/team/get_raw.py index 87555bdd4..4d7481fd6 100644 --- a/darwin/future/core/team/get_raw.py +++ b/darwin/future/core/team/get_raw.py @@ -4,7 +4,15 @@ def get_team_raw(session: Session, url: str) -> JSONType: - """Returns the team with the given slug in raw JSON format""" + """ Gets the raw JSON response from a team endpoint + + Parameters: + session (Session): Requests session to use + url (str): URL to get + + Returns: + JSONType: JSON response from the endpoint + """ response = session.get(url) response.raise_for_status() return response.json() diff --git a/darwin/future/core/team/get_team.py b/darwin/future/core/team/get_team.py index 5570099ef..d4520915d 100644 --- a/darwin/future/core/team/get_team.py +++ b/darwin/future/core/team/get_team.py @@ -5,7 +5,20 @@ def get_team(client: ClientCore, team_slug: Optional[str] = None) -> TeamCore: - """Returns the team with the given slug""" + """ + Returns a TeamCore object for the specified team slug. + + Parameters: + client (ClientCore): The client to use for the request. + team_slug (Optional[str]): The slug of the team to get. If not specified, the + default team from the client's config will be used. + + Returns: + TeamCore: The TeamCore object for the specified team slug. + + Raises: + HTTPError: If the response status code is not in the 200-299 range. + """ if not team_slug: team_slug = client.config.default_team response = client.get(f"/teams/{team_slug}/") @@ -15,6 +28,21 @@ def get_team(client: ClientCore, team_slug: Optional[str] = None) -> TeamCore: def get_team_members( client: ClientCore, ) -> Tuple[List[TeamMemberCore], List[Exception]]: + """ + Returns a tuple containing a list of TeamMemberCore objects and a list of exceptions + that occurred while parsing the response. + + Parameters: + client (ClientCore): The client to use for the request. + + Returns: + Tuple[List[TeamMemberCore], List[Exception]]: A tuple containing a list of + TeamMemberCore objects and a list of exceptions that occurred while parsing + the response. + + Raises: + HTTPError: If the response status code is not in the 200-299 range. + """ response = client.get("/memberships") members = [] errors = [] diff --git a/darwin/future/meta/client.py b/darwin/future/meta/client.py index 011dcea73..cf023061c 100644 --- a/darwin/future/meta/client.py +++ b/darwin/future/meta/client.py @@ -34,10 +34,11 @@ class Client(ClientCore): # Create a new client object with a DarwinConfig from an API key client = Client.from_api_key(api_key="my_api_key", datasets_dir="path/to/datasets/dir") - + # Access the team via chaining team = client.team # returns a Team object which can be chained further """ + def __init__(self, config: DarwinConfig, retries: Optional[Retry] = None) -> None: self._team: Optional[Team] = None super().__init__(config, retries=retries) diff --git a/darwin/future/meta/objects/base.py b/darwin/future/meta/objects/base.py index 31ec41477..3015dbc8b 100644 --- a/darwin/future/meta/objects/base.py +++ b/darwin/future/meta/objects/base.py @@ -14,12 +14,12 @@ class MetaBase(Generic[R]): A base class for metadata objects. This should only ever be inherited from in meta objects. stores metadata parameters used to access the api that are related to the Meta Objects but potentially not required for the core object. For example, a dataset object needs - the team slug to access the api which get's passed down from the team object. + the team slug to access the api which get's passed down from the team object. Attributes: _element (R): The element R to which the object is related. client (ClientCore): The client used to execute the query. - meta_params (Dict[str, object]): A dictionary of metadata parameters. This is + meta_params (Dict[str, object]): A dictionary of metadata parameters. This is used in conjuction with the Query object to execute related api calls. Methods: @@ -33,6 +33,7 @@ class MetaBase(Generic[R]): class Team(MetaBase[TeamCore]): ... """ + _element: R client: ClientCore diff --git a/darwin/future/meta/objects/team.py b/darwin/future/meta/objects/team.py index c5f713f1d..275e0c686 100644 --- a/darwin/future/meta/objects/team.py +++ b/darwin/future/meta/objects/team.py @@ -1,4 +1,4 @@ -from typing import List, Optional, Tuple, Union +from typing import Optional, Union from darwin.future.core.client import ClientCore from darwin.future.core.datasets import get_dataset, remove_dataset @@ -34,9 +34,9 @@ class Team(MetaBase[TeamCore]): name (str): The name of the team. slug (str): The slug of the team. id (int): The id of the team. - members (List[TeamMember]): A list of team members associated with the team. - datasets (List[Dataset]): A list of datasets associated with the team. - workflows (List[Workflow]): A list of workflows associated with the team. + members (TeamMemberQuery): A query of team members associated with the team. + datasets (DatasetQuery): A query of datasets associated with the team. + workflows (WorkflowQuery): A query of workflows associated with the team. Methods: create_dataset(slug: str) -> Dataset: @@ -96,9 +96,7 @@ def workflows(self) -> WorkflowQuery: return WorkflowQuery(self.client, meta_params={"team_slug": self.slug}) @classmethod - def delete_dataset( - cls, client: ClientCore, dataset_id: Union[int, str] - ) -> int: + def delete_dataset(cls, client: ClientCore, dataset_id: Union[int, str]) -> int: """ Deletes a dataset by id or slug diff --git a/darwin/future/meta/objects/team_member.py b/darwin/future/meta/objects/team_member.py index a68830de7..ffe7fa942 100644 --- a/darwin/future/meta/objects/team_member.py +++ b/darwin/future/meta/objects/team_member.py @@ -30,25 +30,30 @@ class TeamMember(MetaBase[TeamMemberCore]): team_member = client.team.members .where(first_name='John', last_name='Doe') .collect_one() - + role = team_member.role """ + @property def role(self) -> TeamMemberRole: return self._element.role + @property def first_name(self) -> str: return self._element.first_name + @property def last_name(self) -> str: return self._element.last_name + @property def email(self) -> str: return self._element.email + @property def user_id(self) -> int: return self._element.user_id - + def __str__(self) -> str: return f"Team Member\n\ - Name: {self.first_name} {self.last_name}\n\ diff --git a/darwin/future/meta/objects/workflow.py b/darwin/future/meta/objects/workflow.py index d5a1851e8..1ca77d95b 100644 --- a/darwin/future/meta/objects/workflow.py +++ b/darwin/future/meta/objects/workflow.py @@ -12,6 +12,39 @@ class Workflow(MetaBase[WorkflowCore]): + """ + Workflow Meta object. Facilitates the creation of Query objects, lazy loading of + sub fields + + Args: + MetaBase (Workflow): Generic MetaBase object expanded by Workflow core object + return type + + Returns: + _type_: Workflow + + Attributes: + name (str): The name of the workflow. + id (UUID): The id of the workflow + datasets (List[Dataset]): A list of datasets associated with the workflow. + stages (StageQuery): Queries stages associated with the workflow. + + Methods: + push_from_dataset_stage() -> Workflow: + moves all items associated with the dataset stage to the next connected stage + upload_files(...): -> Workflow: + Uploads files to the dataset stage of the workflow + + Example Usage: + # Get the workflow object + workflow = client.team.workflows.where(name='test').collect_one() + + # Get the stages associated with the workflow + stages = workflow.stages + + # Get the datasets associated with the workflow + datasets = workflow.datasets + """ @property def stages(self) -> StageQuery: meta_params = self.meta_params.copy() diff --git a/darwin/future/tests/meta/objects/test_teammeta.py b/darwin/future/tests/meta/objects/test_teammeta.py index 25a73473c..742af948b 100644 --- a/darwin/future/tests/meta/objects/test_teammeta.py +++ b/darwin/future/tests/meta/objects/test_teammeta.py @@ -50,7 +50,6 @@ def test_delete_dataset_returns_exceptions_thrown( with raises(Exception): _ = Team.delete_dataset(valid_client, "test_dataset") - assert _delete_by_slug_mock.call_count == 1 assert _delete_by_id_mock.call_count == 0 From a5ff0bfa5b530f951155b9b5b0d1bad585f19cfe Mon Sep 17 00:00:00 2001 From: Nathan Perkins Date: Tue, 24 Oct 2023 16:16:45 +0100 Subject: [PATCH 4/7] refactor exceptions --- darwin/future/core/team/get_raw.py | 2 +- darwin/future/core/types/common.py | 38 ++++++++++++++++- darwin/future/core/workflows/get_workflow.py | 41 ++++++++++++++----- darwin/future/meta/objects/workflow.py | 1 + darwin/future/meta/queries/stage.py | 2 +- .../tests/core/workflows/test_get_workflow.py | 27 ++++-------- 6 files changed, 77 insertions(+), 34 deletions(-) diff --git a/darwin/future/core/team/get_raw.py b/darwin/future/core/team/get_raw.py index 4d7481fd6..aae0a0ace 100644 --- a/darwin/future/core/team/get_raw.py +++ b/darwin/future/core/team/get_raw.py @@ -4,7 +4,7 @@ def get_team_raw(session: Session, url: str) -> JSONType: - """ Gets the raw JSON response from a team endpoint + """Gets the raw JSON response from a team endpoint Parameters: session (Session): Requests session to use diff --git a/darwin/future/core/types/common.py b/darwin/future/core/types/common.py index 6ca2450a1..58fe67ea0 100644 --- a/darwin/future/core/types/common.py +++ b/darwin/future/core/types/common.py @@ -7,8 +7,25 @@ class TeamSlug(str): - """Team slug type""" + """ + Represents a team slug, which is a string identifier for a team. + Attributes: + ----------- + min_length : int + The minimum length of a valid team slug. + max_length : int + The maximum length of a valid team slug. + + Methods: + -------- + __get_validators__() -> generator + Returns a generator that yields the validator function for this model. + validate(v: str) -> TeamSlug + Validates the input string and returns a new TeamSlug object. + __repr__() -> str + Returns a string representation of the TeamSlug object. + """ min_length = 1 max_length = 256 @@ -34,7 +51,24 @@ def __repr__(self) -> str: class QueryString: - """Query string type""" + """ + Represents a query string, which is a dictionary of string key-value pairs. + + Attributes: + ----------- + value : Dict[str, str] + The dictionary of key-value pairs that make up the query string. + + Methods: + -------- + dict_check(value: Any) -> Dict[str, str] + Validates that the input value is a dictionary of string key-value pairs. + Returns the validated dictionary. + __init__(value: Dict[str, str]) -> None + Initializes a new QueryString object with the given dictionary of key-value pairs. + __str__() -> str + Returns a string representation of the QueryString object, in the format "?key1=value1&key2=value2". + """ value: Dict[str, str] diff --git a/darwin/future/core/workflows/get_workflow.py b/darwin/future/core/workflows/get_workflow.py index 0afca7047..8f4f22038 100644 --- a/darwin/future/core/workflows/get_workflow.py +++ b/darwin/future/core/workflows/get_workflow.py @@ -1,4 +1,4 @@ -from typing import List, Optional, Tuple +from typing import List, Optional from pydantic import parse_obj_as @@ -8,16 +8,35 @@ def get_workflow( client: ClientCore, workflow_id: str, team_slug: Optional[str] = None -) -> Tuple[Optional[WorkflowCore], List[Exception]]: - workflow: Optional[WorkflowCore] = None - exceptions: List[Exception] = [] +) -> WorkflowCore: + """ + Retrieves a workflow by ID from the Darwin API. - try: - team_slug = team_slug or client.config.default_team - response = client.get(f"/v2/teams/{team_slug}/workflows/{workflow_id}") + Parameters: + ----------- + client : ClientCore + The Darwin API client to use for the request. + workflow_id : str + The ID of the workflow to retrieve. + team_slug : Optional[str] + The slug of the team that owns the workflow. If not provided, the default team from the client's configuration + will be used. - workflow = parse_obj_as(WorkflowCore, response) - except Exception as e: - exceptions.append(e) + Returns: + -------- + WorkflowCore + The retrieved workflow, as a WorkflowCore object. - return workflow, exceptions + Raises: + ------- + HTTPError + If the API returns an error response. + ValidationError + If the API response does not match the expected schema. + """ + team_slug = team_slug or client.config.default_team + response = client.get(f"/v2/teams/{team_slug}/workflows/{workflow_id}") + + workflow = parse_obj_as(WorkflowCore, response) + + return workflow diff --git a/darwin/future/meta/objects/workflow.py b/darwin/future/meta/objects/workflow.py index 1ca77d95b..7e931e1da 100644 --- a/darwin/future/meta/objects/workflow.py +++ b/darwin/future/meta/objects/workflow.py @@ -45,6 +45,7 @@ class Workflow(MetaBase[WorkflowCore]): # Get the datasets associated with the workflow datasets = workflow.datasets """ + @property def stages(self) -> StageQuery: meta_params = self.meta_params.copy() diff --git a/darwin/future/meta/queries/stage.py b/darwin/future/meta/queries/stage.py index 7211bb5d5..3a18bc0c1 100644 --- a/darwin/future/meta/queries/stage.py +++ b/darwin/future/meta/queries/stage.py @@ -14,7 +14,7 @@ def _collect(self) -> List[Stage]: raise ValueError("Must specify workflow_id to query stages") workflow_id: UUID = self.meta_params["workflow_id"] meta_params = self.meta_params - workflow, exceptions = get_workflow(self.client, str(workflow_id)) + workflow = get_workflow(self.client, str(workflow_id)) assert workflow is not None stages = [ Stage(self.client, s, meta_params=meta_params) for s in workflow.stages diff --git a/darwin/future/tests/core/workflows/test_get_workflow.py b/darwin/future/tests/core/workflows/test_get_workflow.py index f63119092..29878f75e 100644 --- a/darwin/future/tests/core/workflows/test_get_workflow.py +++ b/darwin/future/tests/core/workflows/test_get_workflow.py @@ -1,5 +1,6 @@ import responses from pydantic import ValidationError +from pytest import raises from requests import HTTPError from darwin.future.core.client import ClientCore @@ -24,12 +25,11 @@ def test_get_workflow( ) # Call the function being tested - workflow, exceptions = get_workflow(base_client, workflow_id) + workflow = get_workflow(base_client, workflow_id) # Assertions assert isinstance(workflow, WorkflowCore) - assert not exceptions - + @responses.activate def test_get_workflow_with_team_slug( @@ -48,12 +48,10 @@ def test_get_workflow_with_team_slug( ) # Call the function being tested - workflow, exceptions = get_workflow(base_client, workflow_id, team_slug) + workflow = get_workflow(base_client, workflow_id, team_slug) # Assertions assert isinstance(workflow, WorkflowCore) - assert not exceptions - @responses.activate def test_get_workflows_with_invalid_response(base_client: ClientCore) -> None: @@ -69,12 +67,8 @@ def test_get_workflows_with_invalid_response(base_client: ClientCore) -> None: # fmt: on # Call the function being tested - workflow, exceptions = get_workflow(base_client, NON_EXISTENT_ID) - - assert not workflow - assert exceptions - assert len(exceptions) == 1 - assert isinstance(exceptions[0], ValidationError) + with raises(ValidationError): + get_workflow(base_client, NON_EXISTENT_ID) @responses.activate @@ -89,10 +83,5 @@ def test_get_workflows_with_error(base_client: ClientCore) -> None: status=400 ) # fmt: on - - workflow, exceptions = get_workflow(base_client, NON_EXISTENT_ID) - - assert not workflow - assert exceptions - assert len(exceptions) == 1 - assert isinstance(exceptions[0], HTTPError) + with raises(HTTPError): + get_workflow(base_client, NON_EXISTENT_ID) From a74890d948d622126102b2efbb639a52dfa8948c Mon Sep 17 00:00:00 2001 From: Nathan Perkins Date: Tue, 24 Oct 2023 17:28:39 +0100 Subject: [PATCH 5/7] list exception unification --- darwin/future/core/items/get.py | 32 ++++++++++--- darwin/future/core/types/common.py | 1 + darwin/future/core/workflows/get_workflow.py | 2 +- darwin/future/exceptions.py | 32 +++++++++++-- darwin/future/meta/objects/dataset.py | 27 +++++++++-- .../future/tests/core/items/test_get_items.py | 46 ++++++++++++++++++- .../tests/core/workflows/test_get_workflow.py | 3 +- 7 files changed, 126 insertions(+), 17 deletions(-) diff --git a/darwin/future/core/items/get.py b/darwin/future/core/items/get.py index f2b67f2f2..e4d6bdf5f 100644 --- a/darwin/future/core/items/get.py +++ b/darwin/future/core/items/get.py @@ -1,7 +1,7 @@ -from typing import List, Union +from typing import List, Tuple, Union from uuid import UUID -from pydantic import parse_obj_as +from pydantic import ValidationError, parse_obj_as from darwin.future.core.client import ClientCore from darwin.future.core.types.common import QueryString @@ -112,7 +112,7 @@ def list_items( api_client: ClientCore, team_slug: str, params: QueryString, -) -> List[Item]: +) -> Tuple[List[Item], List[ValidationError]]: """ Returns a list of items for the dataset @@ -129,18 +129,28 @@ def list_items( ------- List[Item] A list of items + List[ValidationError] + A list of validation errors """ assert "dataset_ids" in params.value, "dataset_ids must be provided" response = api_client.get(f"/v2/teams/{team_slug}/items", params) assert isinstance(response, dict) - return parse_obj_as(List[Item], response["items"]) + items: List[Item] = [] + exceptions: List[ValidationError] = [] + for item in response["items"]: + assert isinstance(item, dict) + try: + items.append(parse_obj_as(Item, item)) + except ValidationError as e: + exceptions.append(e) + return items, exceptions def list_folders( api_client: ClientCore, team_slug: str, params: QueryString, -) -> List[Folder]: +) -> Tuple[List[Folder], List[ValidationError]]: """ Returns a list of folders for the team and dataset @@ -157,9 +167,19 @@ def list_folders( ------- List[Folder] The folders + List[ValidationError] + A list of validation errors """ assert "dataset_ids" in params.value, "dataset_ids must be provided" response = api_client.get(f"/v2/teams/{team_slug}/items/folders", params) assert isinstance(response, dict) assert "folders" in response - return parse_obj_as(List[Folder], response["folders"]) + exceptions: List[ValidationError] = [] + folders: List[Folder] = [] + for item in response["folders"]: + assert isinstance(item, dict) + try: + folders.append(parse_obj_as(Folder, item)) + except ValidationError as e: + exceptions.append(e) + return folders, exceptions diff --git a/darwin/future/core/types/common.py b/darwin/future/core/types/common.py index 58fe67ea0..561ec5eaa 100644 --- a/darwin/future/core/types/common.py +++ b/darwin/future/core/types/common.py @@ -26,6 +26,7 @@ class TeamSlug(str): __repr__() -> str Returns a string representation of the TeamSlug object. """ + min_length = 1 max_length = 256 diff --git a/darwin/future/core/workflows/get_workflow.py b/darwin/future/core/workflows/get_workflow.py index 8f4f22038..96779bd77 100644 --- a/darwin/future/core/workflows/get_workflow.py +++ b/darwin/future/core/workflows/get_workflow.py @@ -1,4 +1,4 @@ -from typing import List, Optional +from typing import Optional from pydantic import parse_obj_as diff --git a/darwin/future/exceptions.py b/darwin/future/exceptions.py index 72f5e760d..e73bb8f73 100644 --- a/darwin/future/exceptions.py +++ b/darwin/future/exceptions.py @@ -1,4 +1,6 @@ -from typing import List, Optional +from __future__ import annotations + +from typing import Optional, Sequence from darwin.future.data_objects.typing import KeyValuePairDict, UnknownType @@ -18,13 +20,13 @@ class DarwinException(Exception): """ parent_exception: Optional[Exception] = None - combined_exceptions: Optional[List[Exception]] = None + combined_exceptions: Optional[Sequence[Exception]] = None def __init__(self, *args: UnknownType, **kwargs: KeyValuePairDict) -> None: super().__init__(*args, **kwargs) @classmethod - def from_exception(cls, exc: Exception) -> "DarwinException": + def from_exception(cls, exc: Exception) -> DarwinException: """ Creates a new exception from an existing exception. @@ -43,6 +45,30 @@ def from_exception(cls, exc: Exception) -> "DarwinException": return instance + @classmethod + def from_multiple_exceptions( + cls, exceptions: Sequence[Exception] + ) -> DarwinException: + """ + Creates a new exception from a list of exceptions. + + Parameters + ---------- + exceptions: List[Exception] + The list of exceptions. + + Returns + ------- + DarwinException + The new exception. + """ + instance = cls( + f"Multiple errors occurred while exporting: {', '.join([str(e) for e in exceptions])}", + ) + instance.combined_exceptions = exceptions + + return instance + def __str__(self) -> str: output_string = f"{self.__class__.__name__}: {super().__str__()}\n" if self.parent_exception: diff --git a/darwin/future/meta/objects/dataset.py b/darwin/future/meta/objects/dataset.py index c06554cfb..714d8bc2f 100644 --- a/darwin/future/meta/objects/dataset.py +++ b/darwin/future/meta/objects/dataset.py @@ -16,15 +16,34 @@ class Dataset(MetaBase[DatasetCore]): """ - Dataset Meta object. Facilitates the creation of Query objects, lazy loading of - sub fields + Dataset Meta object. Facilitates the management of a dataset, querying of items, + uploading data, and other dataset related operations. Args: - MetaBase (Dataset): Generic MetaBase object expanded by Dataset core object - return type + MetaBase (Dataset): Generic MetaBase object that manages a DatasetCore object Returns: _type_: DatasetMeta + + Attributes: + name (str): The name of the dataset. + slug (str): The slug of the dataset. + id (int): The id of the dataset. + item_ids (List[UUID]): A list of item ids for the dataset. + + Example Usage: + # Create a new dataset + dataset = Dataset.create(client, slug="my_dataset_slug") + + # Upload data to the dataset + local_file = LocalFile("path/to/local/file") + upload_data(dataset, [local_file]) + + # Get the item ids for the dataset + item_ids = dataset.item_ids + + # Remove the dataset + dataset.remove() """ @property diff --git a/darwin/future/tests/core/items/test_get_items.py b/darwin/future/tests/core/items/test_get_items.py index 96c0fcf97..ed808cd4b 100644 --- a/darwin/future/tests/core/items/test_get_items.py +++ b/darwin/future/tests/core/items/test_get_items.py @@ -2,6 +2,7 @@ from uuid import UUID, uuid4 import responses +from pydantic import ValidationError from darwin.future.core.client import ClientCore from darwin.future.core.items import get_item_ids, get_item_ids_stage @@ -69,13 +70,34 @@ def test_list_items( json={"items": base_items_json}, status=200, ) - items = list_items( + items, _ = list_items( base_client, "default-team", QueryString({"dataset_ids": "1337"}) ) for item, comparator in zip(items, base_items): assert item == comparator +def test_list_items_breaks( + base_items_json: List[dict], base_client: ClientCore +) -> None: + malformed = base_items_json.copy() + del malformed[0]["name"] + del malformed[0]["dataset_id"] + with responses.RequestsMock() as rsps: + rsps.add( + rsps.GET, + base_client.config.api_endpoint + + "v2/teams/default-team/items?dataset_ids=1337", + json={"items": base_items_json}, + status=200, + ) + items, exceptions = list_items( + base_client, "default-team", QueryString({"dataset_ids": "1337"}) + ) + assert len(exceptions) == 1 + assert isinstance(exceptions[0], ValidationError) + + def test_list_folders( base_folders_json: List[dict], base_folders: List[Folder], base_client: ClientCore ) -> None: @@ -87,8 +109,28 @@ def test_list_folders( json={"folders": base_folders_json}, status=200, ) - folders = list_folders( + folders, _ = list_folders( base_client, "default-team", QueryString({"dataset_ids": "1337"}) ) for folder, comparator in zip(folders, base_folders): assert folder == comparator + + +def test_list_folders_breaks( + base_folders_json: List[dict], base_client: ClientCore +) -> None: + malformed = base_folders_json.copy() + del malformed[0]["dataset_id"] + with responses.RequestsMock() as rsps: + rsps.add( + rsps.GET, + base_client.config.api_endpoint + + "v2/teams/default-team/items/folders?dataset_ids=1337", + json={"folders": malformed}, + status=200, + ) + folders, exceptions = list_folders( + base_client, "default-team", QueryString({"dataset_ids": "1337"}) + ) + assert len(exceptions) == 1 + assert isinstance(exceptions[0], ValidationError) diff --git a/darwin/future/tests/core/workflows/test_get_workflow.py b/darwin/future/tests/core/workflows/test_get_workflow.py index 29878f75e..b073c0002 100644 --- a/darwin/future/tests/core/workflows/test_get_workflow.py +++ b/darwin/future/tests/core/workflows/test_get_workflow.py @@ -29,7 +29,7 @@ def test_get_workflow( # Assertions assert isinstance(workflow, WorkflowCore) - + @responses.activate def test_get_workflow_with_team_slug( @@ -53,6 +53,7 @@ def test_get_workflow_with_team_slug( # Assertions assert isinstance(workflow, WorkflowCore) + @responses.activate def test_get_workflows_with_invalid_response(base_client: ClientCore) -> None: # Mocking the response using responses library From 1b1900a5f6114206da65205f2c7a04756e6267b3 Mon Sep 17 00:00:00 2001 From: Nathan Perkins Date: Tue, 24 Oct 2023 17:55:14 +0100 Subject: [PATCH 6/7] cleanup on exceptions --- darwin/future/core/datasets/list_datasets.py | 17 +++++++++++------ darwin/future/core/datasets/remove_dataset.py | 5 ++++- darwin/future/core/items/get.py | 9 ++++----- darwin/future/core/team/get_team.py | 15 +++++++++------ .../tests/core/datasets/test_list_datasets.py | 12 ++++-------- 5 files changed, 32 insertions(+), 26 deletions(-) diff --git a/darwin/future/core/datasets/list_datasets.py b/darwin/future/core/datasets/list_datasets.py index 0b214bc78..824ef0d2f 100644 --- a/darwin/future/core/datasets/list_datasets.py +++ b/darwin/future/core/datasets/list_datasets.py @@ -1,12 +1,14 @@ from typing import List, Tuple -from pydantic import parse_obj_as +from pydantic import ValidationError, parse_obj_as from darwin.future.core.client import ClientCore from darwin.future.data_objects.dataset import DatasetCore -def list_datasets(api_client: ClientCore) -> Tuple[List[DatasetCore], List[Exception]]: +def list_datasets( + api_client: ClientCore, +) -> Tuple[List[DatasetCore], List[ValidationError]]: """ Returns a list of datasets for the given team @@ -19,16 +21,19 @@ def list_datasets(api_client: ClientCore) -> Tuple[List[DatasetCore], List[Excep Returns ------- - Tuple[DatasetList, List[Exception]] + List[DatasetList]: + A list of datasets + List[ValidationError] + A list of Validation errors on failed objects """ datasets: List[DatasetCore] = [] - errors: List[Exception] = [] + errors: List[ValidationError] = [] + response = api_client.get("/datasets") try: - response = api_client.get("/datasets") for item in response: datasets.append(parse_obj_as(DatasetCore, item)) - except Exception as e: + except ValidationError as e: errors.append(e) return datasets, errors diff --git a/darwin/future/core/datasets/remove_dataset.py b/darwin/future/core/datasets/remove_dataset.py index 86e21de1f..d8d2d65a9 100644 --- a/darwin/future/core/datasets/remove_dataset.py +++ b/darwin/future/core/datasets/remove_dataset.py @@ -16,10 +16,13 @@ def remove_dataset( The client to use to make the request id : int The name of the dataset to create + team_slug : str + The slug of the team to create the dataset in Returns ------- - JSONType + int + The dataset deleted id """ if not team_slug: team_slug = api_client.config.default_team diff --git a/darwin/future/core/items/get.py b/darwin/future/core/items/get.py index e4d6bdf5f..b0e74785f 100644 --- a/darwin/future/core/items/get.py +++ b/darwin/future/core/items/get.py @@ -100,8 +100,8 @@ def get_item( Returns ------- - dict - The item + Item + An item object """ response = api_client.get(f"/v2/teams/{team_slug}/items/{item_id}", params) assert isinstance(response, dict) @@ -130,7 +130,7 @@ def list_items( List[Item] A list of items List[ValidationError] - A list of validation errors + A list of ValidationError on failed objects """ assert "dataset_ids" in params.value, "dataset_ids must be provided" response = api_client.get(f"/v2/teams/{team_slug}/items", params) @@ -168,7 +168,7 @@ def list_folders( List[Folder] The folders List[ValidationError] - A list of validation errors + A list of ValidationError on failed objects """ assert "dataset_ids" in params.value, "dataset_ids must be provided" response = api_client.get(f"/v2/teams/{team_slug}/items/folders", params) @@ -177,7 +177,6 @@ def list_folders( exceptions: List[ValidationError] = [] folders: List[Folder] = [] for item in response["folders"]: - assert isinstance(item, dict) try: folders.append(parse_obj_as(Folder, item)) except ValidationError as e: diff --git a/darwin/future/core/team/get_team.py b/darwin/future/core/team/get_team.py index d4520915d..b14c4ed69 100644 --- a/darwin/future/core/team/get_team.py +++ b/darwin/future/core/team/get_team.py @@ -1,5 +1,7 @@ from typing import List, Optional, Tuple +from pydantic import ValidationError + from darwin.future.core.client import ClientCore from darwin.future.data_objects.team import TeamCore, TeamMemberCore @@ -27,7 +29,7 @@ def get_team(client: ClientCore, team_slug: Optional[str] = None) -> TeamCore: def get_team_members( client: ClientCore, -) -> Tuple[List[TeamMemberCore], List[Exception]]: +) -> Tuple[List[TeamMemberCore], List[ValidationError]]: """ Returns a tuple containing a list of TeamMemberCore objects and a list of exceptions that occurred while parsing the response. @@ -36,9 +38,10 @@ def get_team_members( client (ClientCore): The client to use for the request. Returns: - Tuple[List[TeamMemberCore], List[Exception]]: A tuple containing a list of - TeamMemberCore objects and a list of exceptions that occurred while parsing - the response. + List[TeamMemberCore]: + List of TeamMembers + List[ValidationError]: + List of ValidationError on failed objects Raises: HTTPError: If the response status code is not in the 200-299 range. @@ -49,6 +52,6 @@ def get_team_members( for item in response: try: members.append(TeamMemberCore.parse_obj(item)) - except Exception as e: + except ValidationError as e: errors.append(e) - return (members, errors) + return members, errors diff --git a/darwin/future/tests/core/datasets/test_list_datasets.py b/darwin/future/tests/core/datasets/test_list_datasets.py index f6ce5bdbe..8f84a463e 100644 --- a/darwin/future/tests/core/datasets/test_list_datasets.py +++ b/darwin/future/tests/core/datasets/test_list_datasets.py @@ -41,11 +41,7 @@ def test_it_returns_an_error_if_the_client_returns_an_http_error( json={}, status=400, ) - - response, errors = list_datasets(base_client) - - assert len(errors) == 1 - assert isinstance(error := errors[0], HTTPError) - assert error.response is not None - assert error.response.status_code == 400 - assert not response + with pytest.raises(HTTPError) as execinfo: + list_datasets(base_client) + + assert execinfo.value.response.status_code == 400 # type: ignore From d83c541dc7800ebc7b0864906c817c4948e89f12 Mon Sep 17 00:00:00 2001 From: Nathan Perkins Date: Tue, 24 Oct 2023 17:56:39 +0100 Subject: [PATCH 7/7] pytest fix --- darwin/future/tests/core/datasets/test_list_datasets.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/darwin/future/tests/core/datasets/test_list_datasets.py b/darwin/future/tests/core/datasets/test_list_datasets.py index 8f84a463e..dcc84530a 100644 --- a/darwin/future/tests/core/datasets/test_list_datasets.py +++ b/darwin/future/tests/core/datasets/test_list_datasets.py @@ -1,5 +1,6 @@ from typing import List +import pytest import responses from requests.exceptions import HTTPError @@ -43,5 +44,5 @@ def test_it_returns_an_error_if_the_client_returns_an_http_error( ) with pytest.raises(HTTPError) as execinfo: list_datasets(base_client) - - assert execinfo.value.response.status_code == 400 # type: ignore + + assert execinfo.value.response.status_code == 400 # type: ignore