diff --git a/.github/workflows/metrics.yml b/.github/workflows/metrics.yml deleted file mode 100644 index 1f3d1720..00000000 --- a/.github/workflows/metrics.yml +++ /dev/null @@ -1,23 +0,0 @@ -name: Generate Metrics -on: - # Schedule daily updates - schedule: [{cron: "0 0 * * *"}] - workflow_dispatch: - push: {branches: ["master", "main"]} -jobs: - github-metrics: - runs-on: ubuntu-latest - steps: - - uses: lowlighter/metrics@latest - with: - template: repository - filename: metrics.repository.svg - token: ${{ secrets.METRICS_TOKEN }} - user: UWARG - repo: computer-vision-python - plugin_lines: yes - plugin_followup: yes - plugin_activity: yes - plugin_activity_limit: 10 - plugin_activity_filter: pr, comment, push, review - plugin_contributors: yes diff --git a/.github/workflows/python-app.yml b/.github/workflows/python-app.yml deleted file mode 100644 index bb96fc6e..00000000 --- a/.github/workflows/python-app.yml +++ /dev/null @@ -1,43 +0,0 @@ -# This workflow will install Python dependencies, run tests and lint with a single version of Python -# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions - -name: Python application - -on: - push: - branches: [ main ] - pull_request: - branches: [ main ] - -jobs: - build: - - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v2 - - name: Set up Python 3.8 - uses: actions/setup-python@v2 - with: - python-version: 3.8 - - name: Setup Submodules and their dependancies - run: | - git submodule update --init --recursive - cd 2022/modules/targetAcquisition/Yolov5_DeepSort_Pytorch - pip install -r requirements.txt - - name: Install dependencies - run: | - sudo apt install git-lfs - python -m pip install --upgrade pip - pip install flake8 pytest - if [ -f requirements.txt ]; then pip install -r requirements.txt; fi - - name: Test with unittest - run: | - bash scripts/test.sh - # Restore linting once lint errors fixed -# - name: Lint with flake8 -# run: | - # stop the build if there are Python syntax errors or undefined names -# flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics - # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide -# flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml new file mode 100644 index 00000000..aca9d927 --- /dev/null +++ b/.github/workflows/run-tests.yml @@ -0,0 +1,41 @@ +# This workflow will install Python dependencies and run tests with PyTest using Python 3.8 +# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions + +name: Run tests + +on: + push: + branches: [main] + pull_request: + branches: [main] + +jobs: + build: + runs-on: ubuntu-latest + + steps: + # Checkout repository + - uses: actions/checkout@v3 + + # Set Python version + - name: Set up Python 3.8 + uses: actions/setup-python@v4 + with: + python-version: 3.8 + + # Set up submodules and submodule dependencies + - name: Set up submodule and submodule dependencies + run: | + git submodule update --init --recursive --remote + pip install -r ./modules/common/requirements.txt + + # Install computer-vision-python dependencies + - name: Install project dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + pip install -r requirements-pytorch.txt + + # Run tests with PyTest + - name: Run tests + run: pytest -vv diff --git a/main_2023.py b/main_2023.py index fe1d40a6..9f0f5193 100644 --- a/main_2023.py +++ b/main_2023.py @@ -40,6 +40,7 @@ def main() -> int: # Parse whether or not to force cpu from command line parser = argparse.ArgumentParser() parser.add_argument("--cpu", action="store_true", help="option to force cpu") + parser.add_argument("--full", action="store_true", help="option to force full precision") args = parser.parse_args() # Set constants @@ -53,6 +54,7 @@ def main() -> int: DETECT_TARGET_WORKER_COUNT = config["detect_target"]["worker_count"] DETECT_TARGET_DEVICE = "cpu" if args.cpu else config["detect_target"]["device"] DETECT_TARGET_MODEL_PATH = config["detect_target"]["model_path"] + DETECT_TARGET_OVERRIDE_FULL_PRECISION = args.full DETECT_TARGET_SAVE_PREFIX = config["detect_target"]["save_prefix"] except KeyError: print("Config key(s) not found") @@ -91,6 +93,7 @@ def main() -> int: ( DETECT_TARGET_DEVICE, DETECT_TARGET_MODEL_PATH, + DETECT_TARGET_OVERRIDE_FULL_PRECISION, DETECT_TARGET_SAVE_PREFIX, video_input_to_detect_target_queue, detect_target_to_main_queue, diff --git a/main_2024.py b/main_2024.py index 3cd6c253..0bc56e4b 100644 --- a/main_2024.py +++ b/main_2024.py @@ -42,6 +42,7 @@ def main() -> int: # Parse whether or not to force cpu from command line parser = argparse.ArgumentParser() parser.add_argument("--cpu", action="store_true", help="option to force cpu") + parser.add_argument("--full", action="store_true", help="option to force full precision") args = parser.parse_args() # Set constants @@ -55,6 +56,7 @@ def main() -> int: DETECT_TARGET_WORKER_COUNT = config["detect_target"]["worker_count"] DETECT_TARGET_DEVICE = "cpu" if args.cpu else config["detect_target"]["device"] DETECT_TARGET_MODEL_PATH = config["detect_target"]["model_path"] + DETECT_TARGET_OVERRIDE_FULL_PRECISION = args.full DETECT_TARGET_SAVE_PREFIX = config["detect_target"]["save_prefix"] FLIGHT_INTERFACE_ADDRESS = config["flight_interface"]["address"] @@ -100,6 +102,7 @@ def main() -> int: ( DETECT_TARGET_DEVICE, DETECT_TARGET_MODEL_PATH, + DETECT_TARGET_OVERRIDE_FULL_PRECISION, DETECT_TARGET_SAVE_PREFIX, video_input_to_detect_target_queue, detect_target_to_main_queue, diff --git a/modules/detect_target/detect_target.py b/modules/detect_target/detect_target.py index 6205fb58..41b28d2a 100644 --- a/modules/detect_target/detect_target.py +++ b/modules/detect_target/detect_target.py @@ -17,10 +17,19 @@ class DetectTarget: """ Contains the YOLOv8 model for prediction. """ - def __init__(self, device: "str | int", model_path: str, save_name: str = ""): + def __init__(self, device: "str | int", model_path: str, override_full: bool, save_name: str = ""): + """ + device: name of target device to run inference on (i.e. "cpu" or cuda device 0, 1, 2, 3). + model_path: path to the YOLOv8 model. + override_full: Force full precision floating point calculations. + save_name: filename prefix for logging detections and annotated images. + """ self.__device = device self.__model = ultralytics.YOLO(model_path) self.__counter = 0 + self.__enable_half_precision = False if self.__device == "cpu" else True + if override_full: + self.__enable_half_precision = False self.__filename_prefix = "" if save_name != "": self.__filename_prefix = save_name + "_" + str(int(time.time())) + "_" @@ -33,7 +42,7 @@ def run(self, data: image_and_time.ImageAndTime) -> "tuple[bool, np.ndarray | No image = data.image predictions = self.__model.predict( source=image, - half=True, + half=self.__enable_half_precision, device=self.__device, stream=False, ) diff --git a/modules/detect_target/detect_target_worker.py b/modules/detect_target/detect_target_worker.py index 3d8c3e39..6d4fc8fa 100644 --- a/modules/detect_target/detect_target_worker.py +++ b/modules/detect_target/detect_target_worker.py @@ -11,6 +11,7 @@ # pylint: disable=too-many-arguments def detect_target_worker(device: "str | int", model_path: str, + override_full: bool, save_name: str, input_queue: queue_proxy_wrapper.QueueProxyWrapper, output_queue: queue_proxy_wrapper.QueueProxyWrapper, @@ -18,11 +19,11 @@ def detect_target_worker(device: "str | int", """ Worker process. - model_path and save_name are initial settings. + device, model_path, override_full, and save_name are initial settings. input_queue and output_queue are data queues. controller is how the main process communicates to this worker process. """ - detector = detect_target.DetectTarget(device, model_path, save_name) + detector = detect_target.DetectTarget(device, model_path, override_full, save_name) while not controller.is_exit_requested(): controller.check_pause() diff --git a/tests/test_detect_target.py b/tests/test_detect_target.py index 8b1c3a0f..f1a74356 100644 --- a/tests/test_detect_target.py +++ b/tests/test_detect_target.py @@ -14,6 +14,7 @@ DEVICE = 0 if torch.cuda.is_available() else "cpu" MODEL_PATH = "tests/model_example/yolov8s_ultralytics_pretrained_default.pt" +OVERRIDE_FULL = False # Tests are able to handle both full and half precision. IMAGE_BUS_PATH = "tests/model_example/bus.jpg" IMAGE_BUS_ANNOTATED_PATH = "tests/model_example/bus_annotated.png" IMAGE_ZIDANE_PATH = "tests/model_example/zidane.jpg" @@ -25,9 +26,10 @@ def detector(): """ Construct DetectTarget. """ - detection = detect_target.DetectTarget(DEVICE, MODEL_PATH) + detection = detect_target.DetectTarget(DEVICE, MODEL_PATH, OVERRIDE_FULL) yield detection + @pytest.fixture() def image_bus(): """ @@ -39,6 +41,7 @@ def image_bus(): assert bus_image is not None yield bus_image + @pytest.fixture() def image_zidane(): """ @@ -51,11 +54,39 @@ def image_zidane(): yield zidane_image +def rmse(actual: np.ndarray, + expected: np.ndarray) -> float: + """ + Helper function to compute root mean squared error. + """ + mean_squared_error = np.square(actual - expected).mean() + + return np.sqrt(mean_squared_error) + + +def test_rmse(): + """ + Root mean squared error. + """ + # Setup + sample_actual = np.array([1, 2, 3, 4, 5]) + sample_expected = np.array([1.6, 2.5, 2.9, 3, 4.1]) + EXPECTED_ERROR = np.sqrt(0.486) + + # Run + actual_error = rmse(sample_actual, sample_expected) + + # Test + np.testing.assert_almost_equal(actual_error, EXPECTED_ERROR) + + class TestDetector: """ Tests `DetectTarget.run()` . """ + __IMAGE_DIFFERENCE_TOLERANCE = 1 + def test_single_bus_image(self, detector: detect_target.DetectTarget, image_bus: image_and_time.ImageAndTime): @@ -72,7 +103,9 @@ def test_single_bus_image(self, # Test assert result assert actual is not None - np.testing.assert_array_equal(actual, expected) + + error = rmse(actual, expected) + assert error < self.__IMAGE_DIFFERENCE_TOLERANCE def test_single_zidane_image(self, detector: detect_target.DetectTarget, @@ -90,7 +123,9 @@ def test_single_zidane_image(self, # Test assert result assert actual is not None - np.testing.assert_array_equal(actual, expected) + + error = rmse(actual, expected) + assert error < self.__IMAGE_DIFFERENCE_TOLERANCE def test_multiple_zidane_image(self, detector: detect_target.DetectTarget, @@ -121,4 +156,6 @@ def test_multiple_zidane_image(self, result, actual = output assert result assert actual is not None - np.testing.assert_array_equal(actual, expected) + + error = rmse(actual, expected) + assert error < self.__IMAGE_DIFFERENCE_TOLERANCE diff --git a/tests/test_detect_target_worker.py b/tests/test_detect_target_worker.py index 0c729032..0c855d03 100644 --- a/tests/test_detect_target_worker.py +++ b/tests/test_detect_target_worker.py @@ -21,6 +21,7 @@ IMAGE_ZIDANE_PATH = "tests/model_example/zidane.jpg" WORK_COUNT = 3 +OVERRIDE_FULL = False def simulate_previous_worker(image_path: str, in_queue: queue_proxy_wrapper.QueueProxyWrapper): @@ -46,7 +47,7 @@ def simulate_previous_worker(image_path: str, in_queue: queue_proxy_wrapper.Queu worker = mp.Process( target=detect_target_worker.detect_target_worker, - args=(device, MODEL_PATH, "", image_in_queue, image_out_queue, controller), + args=(device, MODEL_PATH, OVERRIDE_FULL, "", image_in_queue, image_out_queue, controller), ) # Run