[IO-1461][IO-1820] Code Documentation + exception refactoring + mypy …

…checking (#701)

* reformatting str for mypy

* comments and dogfooding changes

* comments and dogfooding changes

* refactor exceptions

* list exception unification

* cleanup on exceptions

* pytest fix

darwin/future/core/datasets/list_datasets.py

@@ -1,12 +1,14 @@
 from typing import List, Tuple
 
-from pydantic import parse_obj_as
+from pydantic import ValidationError, parse_obj_as
 
 from darwin.future.core.client import ClientCore
 from darwin.future.data_objects.dataset import DatasetCore
 
 
-def list_datasets(api_client: ClientCore) -> Tuple[List[DatasetCore], List[Exception]]:
+def list_datasets(
+    api_client: ClientCore,
+) -> Tuple[List[DatasetCore], List[ValidationError]]:
     """
     Returns a list of datasets for the given team
 
@@ -19,16 +21,19 @@ def list_datasets(api_client: ClientCore) -> Tuple[List[DatasetCore], List[Excep
 
     Returns
     -------
-    Tuple[DatasetList, List[Exception]]
+    List[DatasetList]:
+        A list of datasets
+    List[ValidationError]
+        A list of Validation errors on failed objects
     """
     datasets: List[DatasetCore] = []
-    errors: List[Exception] = []
+    errors: List[ValidationError] = []
 
+    response = api_client.get("/datasets")
     try:
-        response = api_client.get("/datasets")
         for item in response:
             datasets.append(parse_obj_as(DatasetCore, item))
-    except Exception as e:
+    except ValidationError as e:
         errors.append(e)
 
     return datasets, errors

darwin/future/core/datasets/remove_dataset.py

@@ -16,10 +16,13 @@ def remove_dataset(
         The client to use to make the request
     id : int
         The name of the dataset to create
+    team_slug : str
+        The slug of the team to create the dataset in
 
     Returns
     -------
-    JSONType
+    int
+        The dataset deleted id
     """
     if not team_slug:
         team_slug = api_client.config.default_team

darwin/future/core/items/get.py

@@ -1,7 +1,7 @@
-from typing import List, Union
+from typing import List, Tuple, Union
 from uuid import UUID
 
-from pydantic import parse_obj_as
+from pydantic import ValidationError, parse_obj_as
 
 from darwin.future.core.client import ClientCore
 from darwin.future.core.types.common import QueryString
@@ -100,8 +100,8 @@ def get_item(
 
     Returns
     -------
-    dict
-        The item
+    Item
+        An item object
     """
     response = api_client.get(f"/v2/teams/{team_slug}/items/{item_id}", params)
     assert isinstance(response, dict)
@@ -112,7 +112,7 @@ def list_items(
     api_client: ClientCore,
     team_slug: str,
     params: QueryString,
-) -> List[Item]:
+) -> Tuple[List[Item], List[ValidationError]]:
     """
     Returns a list of items for the dataset
 
@@ -129,18 +129,28 @@ def list_items(
     -------
     List[Item]
         A list of items
+    List[ValidationError]
+        A list of ValidationError on failed objects
     """
     assert "dataset_ids" in params.value, "dataset_ids must be provided"
     response = api_client.get(f"/v2/teams/{team_slug}/items", params)
     assert isinstance(response, dict)
-    return parse_obj_as(List[Item], response["items"])
+    items: List[Item] = []
+    exceptions: List[ValidationError] = []
+    for item in response["items"]:
+        assert isinstance(item, dict)
+        try:
+            items.append(parse_obj_as(Item, item))
+        except ValidationError as e:
+            exceptions.append(e)
+    return items, exceptions
 
 
 def list_folders(
     api_client: ClientCore,
     team_slug: str,
     params: QueryString,
-) -> List[Folder]:
+) -> Tuple[List[Folder], List[ValidationError]]:
     """
     Returns a list of folders for the team and dataset
 
@@ -157,9 +167,18 @@ def list_folders(
     -------
     List[Folder]
         The folders
+    List[ValidationError]
+        A list of ValidationError on failed objects
     """
     assert "dataset_ids" in params.value, "dataset_ids must be provided"
     response = api_client.get(f"/v2/teams/{team_slug}/items/folders", params)
     assert isinstance(response, dict)
     assert "folders" in response
-    return parse_obj_as(List[Folder], response["folders"])
+    exceptions: List[ValidationError] = []
+    folders: List[Folder] = []
+    for item in response["folders"]:
+        try:
+            folders.append(parse_obj_as(Folder, item))
+        except ValidationError as e:
+            exceptions.append(e)
+    return folders, exceptions

darwin/future/core/team/get_raw.py

@@ -4,7 +4,15 @@
 
 
 def get_team_raw(session: Session, url: str) -> JSONType:
-    """Returns the team with the given slug in raw JSON format"""
+    """Gets the raw JSON response from a team endpoint
+
+    Parameters:
+        session (Session): Requests session to use
+        url (str): URL to get
+
+    Returns:
+        JSONType: JSON response from the endpoint
+    """
     response = session.get(url)
     response.raise_for_status()
     return response.json()

darwin/future/core/team/get_team.py

@@ -1,11 +1,26 @@
 from typing import List, Optional, Tuple
 
+from pydantic import ValidationError
+
 from darwin.future.core.client import ClientCore
 from darwin.future.data_objects.team import TeamCore, TeamMemberCore
 
 
 def get_team(client: ClientCore, team_slug: Optional[str] = None) -> TeamCore:
-    """Returns the team with the given slug"""
+    """
+    Returns a TeamCore object for the specified team slug.
+
+    Parameters:
+        client (ClientCore): The client to use for the request.
+        team_slug (Optional[str]): The slug of the team to get. If not specified, the
+            default team from the client's config will be used.
+
+    Returns:
+        TeamCore: The TeamCore object for the specified team slug.
+
+    Raises:
+        HTTPError: If the response status code is not in the 200-299 range.
+    """
     if not team_slug:
         team_slug = client.config.default_team
     response = client.get(f"/teams/{team_slug}/")
@@ -14,13 +29,29 @@ def get_team(client: ClientCore, team_slug: Optional[str] = None) -> TeamCore:
 
 def get_team_members(
     client: ClientCore,
-) -> Tuple[List[TeamMemberCore], List[Exception]]:
+) -> Tuple[List[TeamMemberCore], List[ValidationError]]:
+    """
+    Returns a tuple containing a list of TeamMemberCore objects and a list of exceptions
+    that occurred while parsing the response.
+
+    Parameters:
+        client (ClientCore): The client to use for the request.
+
+    Returns:
+        List[TeamMemberCore]:
+            List of TeamMembers
+        List[ValidationError]:
+            List of ValidationError on failed objects
+
+    Raises:
+        HTTPError: If the response status code is not in the 200-299 range.
+    """
     response = client.get("/memberships")
     members = []
     errors = []
     for item in response:
         try:
             members.append(TeamMemberCore.parse_obj(item))
-        except Exception as e:
+        except ValidationError as e:
             errors.append(e)
-    return (members, errors)
+    return members, errors

darwin/future/core/types/common.py

@@ -7,7 +7,25 @@
 
 
 class TeamSlug(str):
-    """Team slug type"""
+    """
+    Represents a team slug, which is a string identifier for a team.
+
+    Attributes:
+    -----------
+    min_length : int
+        The minimum length of a valid team slug.
+    max_length : int
+        The maximum length of a valid team slug.
+
+    Methods:
+    --------
+    __get_validators__() -> generator
+        Returns a generator that yields the validator function for this model.
+    validate(v: str) -> TeamSlug
+        Validates the input string and returns a new TeamSlug object.
+    __repr__() -> str
+        Returns a string representation of the TeamSlug object.
+    """
 
     min_length = 1
     max_length = 256
@@ -34,7 +52,24 @@ def __repr__(self) -> str:
 
 
 class QueryString:
-    """Query string type"""
+    """
+    Represents a query string, which is a dictionary of string key-value pairs.
+
+    Attributes:
+    -----------
+    value : Dict[str, str]
+        The dictionary of key-value pairs that make up the query string.
+
+    Methods:
+    --------
+    dict_check(value: Any) -> Dict[str, str]
+        Validates that the input value is a dictionary of string key-value pairs.
+        Returns the validated dictionary.
+    __init__(value: Dict[str, str]) -> None
+        Initializes a new QueryString object with the given dictionary of key-value pairs.
+    __str__() -> str
+        Returns a string representation of the QueryString object, in the format "?key1=value1&key2=value2".
+    """
 
     value: Dict[str, str]
 

darwin/future/core/types/query.py

@@ -106,12 +106,43 @@ def _from_kwarg(cls, key: str, value: str) -> QueryFilter:
 
 
 class Query(Generic[T], ABC):
-    """Basic Query object with methods to manage filters
+    """
+    A basic Query object with methods to manage filters. This is an abstract class not
+    meant to be used directly. Use a subclass instead, like DatasetQuery.
+    This class will lazy load results and cache them internally, and allows for filtering
+    of the objects locally by default. To execute the query, call the collect() method,
+    or iterate over the query object.
+
+    Attributes:
+        meta_params (dict): A dictionary of metadata parameters.
+        client (ClientCore): The client used to execute the query.
+        filters (List[QueryFilter]): A list of QueryFilter objects used to filter the query results.
+        results (List[T]): A list of query results, cached internally for iterable access.
+        _changed_since_last (bool): A boolean indicating whether the query has changed since the last execution.
+
     Methods:
-        filter: adds a filter to the query object, returns a new query object
-        where: Applies a filter on the query object, returns a new query object
-        collect: Executes the query on the client and returns the results
-        _generic_execute_filter: Executes a filter on a list of objects
+        filter(name: str, param: str, modifier: Optional[Modifier] = None) -> Query[T]:
+            Adds a filter to the query object and returns a new query object.
+        where(name: str, param: str, modifier: Optional[Modifier] = None) -> Query[T]:
+            Applies a filter on the query object and returns a new query object.
+        first() -> Optional[T]:
+            Returns the first result of the query. Raises an exception if no results are found.
+        collect() -> List[T]:
+            Executes the query on the client and returns the results. Raises an exception if no results are found.
+        _generic_execute_filter(objects: List[T], filter_: QueryFilter) -> List[T]:
+            Executes a filter on a list of objects. Locally by default, but can be overwritten by subclasses.
+
+    Examples:
+        # Create a query object
+        # DatasetQuery is linked to the object it returns, Dataset, and is iterable
+        # overwrite the _collect() method to execute insantiate this object
+        Class DatasetQuery(Query[Dataset]):
+            ...
+
+        # Intended usage via chaining
+        # where client.team.datasets returns a DatasetQuery object and can be chained
+        # further with multiple where calls before collecting
+        datasets = client.team.datasets.where(...).where(...).collect()
     """
 
     def __init__(
@@ -209,11 +240,11 @@ def collect_one(self) -> T:
             raise MoreThanOneResultFound("More than one result found")
         return self.results[0]
 
-    def first(self) -> Optional[T]:
+    def first(self) -> T:
         if not self.results:
             self.results = list(self.collect())
         if len(self.results) == 0:
-            return None
+            raise ResultsNotFound("No results found")
         return self.results[0]
 
     def _generic_execute_filter(self, objects: List[T], filter: QueryFilter) -> List[T]:

darwin/future/core/workflows/get_workflow.py

@@ -1,4 +1,4 @@
-from typing import List, Optional, Tuple
+from typing import Optional
 
 from pydantic import parse_obj_as
 
@@ -8,16 +8,35 @@
 
 def get_workflow(
     client: ClientCore, workflow_id: str, team_slug: Optional[str] = None
-) -> Tuple[Optional[WorkflowCore], List[Exception]]:
-    workflow: Optional[WorkflowCore] = None
-    exceptions: List[Exception] = []
+) -> WorkflowCore:
+    """
+    Retrieves a workflow by ID from the Darwin API.
 
-    try:
-        team_slug = team_slug or client.config.default_team
-        response = client.get(f"/v2/teams/{team_slug}/workflows/{workflow_id}")
+    Parameters:
+    -----------
+    client : ClientCore
+        The Darwin API client to use for the request.
+    workflow_id : str
+        The ID of the workflow to retrieve.
+    team_slug : Optional[str]
+        The slug of the team that owns the workflow. If not provided, the default team from the client's configuration
+        will be used.
 
-        workflow = parse_obj_as(WorkflowCore, response)
-    except Exception as e:
-        exceptions.append(e)
+    Returns:
+    --------
+    WorkflowCore
+        The retrieved workflow, as a WorkflowCore object.
 
-    return workflow, exceptions
+    Raises:
+    -------
+    HTTPError
+        If the API returns an error response.
+    ValidationError
+        If the API response does not match the expected schema.
+    """
+    team_slug = team_slug or client.config.default_team
+    response = client.get(f"/v2/teams/{team_slug}/workflows/{workflow_id}")
+
+    workflow = parse_obj_as(WorkflowCore, response)
+
+    return workflow

darwin/future/data_objects/pydantic_base.py

@@ -2,7 +2,8 @@
 
 
 class DefaultDarwin(BaseModel):
-    """Default Darwin-Py pydantic settings for meta information.
+    """
+    Default Darwin-Py pydantic settings for meta information.
     Default settings include:
         - auto validating variables on setting/assignment
         - underscore attributes are private