From ff6da32bb51bc6c84cc9559c3e39474adb3b56f1 Mon Sep 17 00:00:00 2001 From: Tyler Romero Date: Wed, 18 Oct 2023 15:52:35 -0700 Subject: [PATCH] Improve the plumbing and documentation for some of the ask* methods (#125) Address a some documentation papercuts for the new ask_* methods. Plus, improve plumbing of arguments to submit_image_query and to wait_for_confident_result. --------- Co-authored-by: Auto-format Bot Co-authored-by: Sunil Kumar --- .../building-applications/5-async-queries.md | 7 +- pyproject.toml | 2 +- src/groundlight/client.py | 79 ++++++++++++++----- 3 files changed, 65 insertions(+), 23 deletions(-) diff --git a/docs/docs/building-applications/5-async-queries.md b/docs/docs/building-applications/5-async-queries.md index d14704c2..3507f66e 100644 --- a/docs/docs/building-applications/5-async-queries.md +++ b/docs/docs/building-applications/5-async-queries.md @@ -16,7 +16,7 @@ from time import sleep detector = gl.get_or_create_detector(name="your_detector_name", query="your_query") -cam = cv2.VideoCapture(0) # Initialize camera (0 is the default index) +cam = cv2.VideoCapture(0) # Initialize camera (0 is the default index) while True: _, image = cam.read() # Capture one frame from the camera @@ -35,7 +35,7 @@ from groundlight import Groundlight detector = gl.get_or_create_detector(name="your_detector_name", query="your_query") -image_query_id = db.get_next_image_query_id() +image_query_id = db.get_next_image_query_id() while image_query_id is not None: image_query = gl.get_image_query(id=image_query_id) # retrieve the image query from Groundlight @@ -67,4 +67,7 @@ result = image_query.result # This will always be 'None' as you asked asynchron image_query = gl.get_image_query(id=image_query.id) # Immediately retrieve the image query from Groundlight result = image_query.result # This will likely be 'UNCLEAR' as Groundlight is still processing your query + +image_query = gl.wait_for_confident_result(id=image_query.id) # Poll for a confident result from Groundlight +result = image_query.result ``` \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 5ac44064..ebecead3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,7 +9,7 @@ packages = [ {include = "**/*.py", from = "src"}, ] readme = "README.md" -version = "0.12.0" +version = "0.12.1" [tool.poetry.dependencies] # For certifi, use ">=" instead of "^" since it upgrades its "major version" every year, not really following semver diff --git a/src/groundlight/client.py b/src/groundlight/client.py index c2c3b508..081b2f71 100644 --- a/src/groundlight/client.py +++ b/src/groundlight/client.py @@ -1,6 +1,7 @@ import logging import os import time +from functools import partial from io import BufferedReader, BytesIO from typing import Callable, Optional, Union @@ -304,9 +305,20 @@ def submit_image_query( # noqa: PLR0913 # pylint: disable=too-many-arguments, t Any pixel format will get converted to JPEG at high quality before sending to service. :type image: str or bytes or Image.Image or BytesIO or BufferedReader or np.ndarray - :param wait: How long to wait (in seconds) for a confident answer. + :param wait: How long to poll (in seconds) for a confident answer. This is a client-side timeout. :type wait: float + :param patience_time: How long to wait (in seconds) for a confident answer for this image query. + The longer the patience_time, the more likely Groundlight will arrive at a confident answer. + Within patience_time, Groundlight will update ML predictions based on stronger findings, + and, additionally, Groundlight will prioritize human review of the image query if necessary. + This is a soft server-side timeout. If not set, use the detector's patience_time. + :type patience_time: float + + :param confidence_threshold: The confidence threshold to wait for. + If not set, use the detector's confidence threshold. + :type confidence_threshold: float + :param human_review: If `None` or `DEFAULT`, send the image query for human review only if the ML prediction is not confident. If set to `ALWAYS`, always send the image query for human review. @@ -375,8 +387,10 @@ def ask_confident( confidence_threshold: Optional[float] = None, wait: Optional[float] = None, ) -> ImageQuery: - """Evaluates an image with Groundlight waiting until an answer above the confidence threshold - of the detector is reached or the wait period has passed. + """ + Evaluates an image with Groundlight waiting until an answer above the confidence threshold + of the detector is reached or the wait period has passed. + :param detector: the Detector object, or string id of a detector like `det_12345` :type detector: Detector or str @@ -405,6 +419,8 @@ def ask_confident( image, confidence_threshold=confidence_threshold, wait=wait, + patience_time=wait, + human_review=None, ) def ask_ml( @@ -413,7 +429,9 @@ def ask_ml( image: Union[str, bytes, Image.Image, BytesIO, BufferedReader, np.ndarray], wait: Optional[float] = None, ) -> ImageQuery: - """Evaluates an image with Groundlight, getting the first answer Groundlight can provide. + """ + Evaluates an image with Groundlight, getting the first answer Groundlight can provide. + :param detector: the Detector object, or string id of a detector like `det_12345` :type detector: Detector or str @@ -443,12 +461,13 @@ def ask_ml( wait = self.DEFAULT_WAIT if wait is None else wait return self.wait_for_ml_result(iq, timeout_sec=wait) - def ask_async( + def ask_async( # noqa: PLR0913 # pylint: disable=too-many-arguments self, detector: Union[Detector, str], image: Union[str, bytes, Image.Image, BytesIO, BufferedReader, np.ndarray], + patience_time: Optional[float] = None, + confidence_threshold: Optional[float] = None, human_review: Optional[str] = None, - inspection_id: Optional[str] = None, ) -> ImageQuery: """ Convenience method for submitting an `ImageQuery` asynchronously. This is equivalent to calling @@ -469,6 +488,17 @@ def ask_async( :type image: str or bytes or Image.Image or BytesIO or BufferedReader or np.ndarray + :param patience_time: How long to wait (in seconds) for a confident answer for this image query. + The longer the patience_time, the more likely Groundlight will arrive at a confident answer. + Within patience_time, Groundlight will update ML predictions based on stronger findings, + and, additionally, Groundlight will prioritize human review of the image query if necessary. + This is a soft server-side timeout. If not set, use the detector's patience_time. + :type patience_time: float + + :param confidence_threshold: The confidence threshold to wait for. + If not set, use the detector's confidence threshold. + :type confidence_threshold: float + :param human_review: If `None` or `DEFAULT`, send the image query for human review only if the ML prediction is not confident. If set to `ALWAYS`, always send the image query for human review. @@ -500,26 +530,34 @@ def ask_async( assert image_query.id is not None # Do not attempt to access the result of this query as the result for all async queries - # will be None. Your result is being computed asynchronously and will be available - # later + # will be None. Your result is being computed asynchronously and will be available later assert image_query.result is None - # retrieve the result later or on another machine by calling gl.get_image_query() - # with the id of the image_query above - image_query = gl.get_image_query(image_query.id) + # retrieve the result later or on another machine by calling gl.wait_for_confident_result() + # with the id of the image_query above. This will block until the result is available. + image_query = gl.wait_for_confident_result(image_query.id) # now the result will be available for your use assert image_query.result is not None + # alternatively, you can check if the result is available (without blocking) by calling + # gl.get_image_query() with the id of the image_query above. + image_query = gl.get_image_query(image_query.id) """ return self.submit_image_query( - detector, image, wait=0, human_review=human_review, want_async=True, inspection_id=inspection_id + detector, + image, + wait=0, + patience_time=patience_time, + confidence_threshold=confidence_threshold, + human_review=human_review, + want_async=True, ) def wait_for_confident_result( self, image_query: Union[ImageQuery, str], - confidence_threshold: float, + confidence_threshold: Optional[float] = None, timeout_sec: float = 30.0, ) -> ImageQuery: """ @@ -529,7 +567,8 @@ def wait_for_confident_result( :param image_query: An ImageQuery object to poll :type image_query: ImageQuery or str - :param confidence_threshold: The minimum confidence level required to return before the timeout. + :param confidence_threshold: The confidence threshold to wait for. + If not set, use the detector's confidence threshold. :type confidence_threshold: float :param timeout_sec: The maximum number of seconds to wait. @@ -538,10 +577,12 @@ def wait_for_confident_result( :return: ImageQuery :rtype: ImageQuery """ + if confidence_threshold is None: + if isinstance(image_query, str): + image_query = self.get_image_query(image_query) + confidence_threshold = self.get_detector(image_query.detector_id).confidence_threshold - def confidence_above_thresh(iq): - return iq_is_confident(iq, confidence_threshold=confidence_threshold) - + confidence_above_thresh = partial(iq_is_confident, confidence_threshold=confidence_threshold) return self._wait_for_result(image_query, condition=confidence_above_thresh, timeout_sec=timeout_sec) def wait_for_ml_result(self, image_query: Union[ImageQuery, str], timeout_sec: float = 30.0) -> ImageQuery: @@ -551,9 +592,6 @@ def wait_for_ml_result(self, image_query: Union[ImageQuery, str], timeout_sec: f :param image_query: An ImageQuery object to poll :type image_query: ImageQuery or str - :param confidence_threshold: The minimum confidence level required to return before the timeout. - :type confidence_threshold: float - :param timeout_sec: The maximum number of seconds to wait. :type timeout_sec: float @@ -623,6 +661,7 @@ def add_label(self, image_query: Union[ImageQuery, str], label: Union[Label, str else: image_query_id = str(image_query) # Some old imagequery id's started with "chk_" + # TODO: handle iqe_ for image_queries returned from edge endpoints if not image_query_id.startswith(("chk_", "iq_")): raise ValueError(f"Invalid image query id {image_query_id}") api_label = convert_display_label_to_internal(image_query_id, label)