automatic ruff --fix changes (#723)

* automatic ruff --fix changes

* black changes

* revert of client

darwin/cli.py

@@ -2,10 +2,7 @@
 
 import getpass
 import os
-import platform
 from argparse import ArgumentParser, Namespace
-from datetime import datetime
-from json import dumps
 
 import requests.exceptions
 from rich.console import Console

darwin/cli_functions.py

@@ -8,7 +8,7 @@
 from glob import glob
 from itertools import tee
 from pathlib import Path
-from typing import Any, Dict, Iterator, List, NoReturn, Optional, Set, Union
+from typing import Dict, Iterator, List, NoReturn, Optional, Set, Union
 
 import humanize
 from rich.console import Console
@@ -1065,7 +1065,7 @@ def delete_files(
             console.print("Cancelled.")
             return
 
-        found_filenames: Set[str] = set([item.filename for item in items_2])
+        found_filenames: Set[str] = {item.filename for item in items_2}
         not_found_filenames: Set[str] = set(files) - found_filenames
         for filename in not_found_filenames:
             console.print(f"File not found: {filename}", style="warning")

darwin/dataset/download_manager.py

@@ -93,9 +93,7 @@ def download_all_images_from_annotations(
         raise ValueError(f"Annotation format {annotation_format} not supported")
 
     # Verify that there is not already image in the images folder
-    unfiltered_files = (
-        images_path.rglob(f"*") if use_folders else images_path.glob(f"*")
-    )
+    unfiltered_files = images_path.rglob("*") if use_folders else images_path.glob("*")
     existing_images = {
         image for image in unfiltered_files if is_image_extension_allowed(image.suffix)
     }
@@ -666,9 +664,9 @@ def _extract_frames_from_segment(path: Path, manifest: dt.SegmentManifest) -> No
     cap = VideoCapture(str(path))
 
     # Read and save frames. Iterates over every frame because frame seeking in OCV is not reliable or guaranteed.
-    frames_to_extract = dict(
-        [(item.frame, item.visible_frame) for item in manifest.items if item.visibility]
-    )
+    frames_to_extract = {
+        item.frame: item.visible_frame for item in manifest.items if item.visibility
+    }
     frame_index = 0
     while cap.isOpened():
         success, frame = cap.read()

darwin/dataset/remote_dataset.py

@@ -286,7 +286,7 @@ def pull(
                         continue
 
                     if video_frames and any(
-                        [not slot.frame_urls for slot in annotation.slots]
+                        not slot.frame_urls for slot in annotation.slots
                     ):
                         # will raise if not installed via pip install darwin-py[ocv]
                         try:
@@ -632,7 +632,7 @@ def fetch_remote_classes(self, team_wide=False) -> List[Dict[str, Any]]:
         classes_to_return = []
         for cls in all_classes:
             belongs_to_current_dataset = any(
-                [dataset["id"] == self.dataset_id for dataset in cls["datasets"]]
+                dataset["id"] == self.dataset_id for dataset in cls["datasets"]
             )
             cls["available"] = belongs_to_current_dataset
             if team_wide or belongs_to_current_dataset:

darwin/dataset/remote_dataset_v1.py

@@ -1,6 +1,5 @@
 import itertools
 from typing import TYPE_CHECKING, Any, Dict, Iterator, List, Optional, Sequence, Union
-from xml.dom import ValidationErr
 
 from requests.models import Response
 
@@ -346,9 +345,10 @@ def complete(self, items: Iterator[DatasetItem]) -> None:
         items : Iterator[DatasetItem]
             The ``DatasetItem``\\s to be completed.
         """
-        wf_template_id_mapper = lambda item: item.current_workflow[
-            "workflow_template_id"
-        ]
+
+        def wf_template_id_mapper(item):
+            return item.current_workflow["workflow_template_id"]
+
         input_items: List[DatasetItem] = list(items)
 
         # We split into items with and without workflow

darwin/dataset/remote_dataset_v2.py

@@ -430,7 +430,7 @@ def export(
             format = "darwin_json_2"
         elif str_version == "1.0":
             format = "json"
-        elif version == None:
+        elif version is None:
             format = None
         else:
             raise UnknownExportVersion(version)

darwin/dataset/upload_manager.py

@@ -76,7 +76,7 @@ def __init__(
     @staticmethod
     def parse_v2(payload):
         if len(payload["slots"]) > 1:
-            raise NotImplemented("multiple files support not yet implemented")
+            raise NotImplementedError("multiple files support not yet implemented")
         slot = payload["slots"][0]
         return ItemPayload(
             dataset_item_id=payload.get("id", None),
@@ -552,7 +552,7 @@ def upload_function(
         file_lookup = {file.full_path: file for file in self.local_files}
         for item in self.pending_items:
             if len(item.slots) != 1:
-                raise NotImplemented("Multi file upload is not supported")
+                raise NotImplementedError("Multi file upload is not supported")
             upload_id = item.slots[0]["upload_id"]
             file = file_lookup.get(item.full_path)
             if not file:

darwin/datatypes.py

@@ -1,5 +1,4 @@
 from dataclasses import dataclass, field
-from email.policy import default
 from enum import Enum, auto
 from pathlib import Path
 from typing import (

darwin/exceptions.py

@@ -1,4 +1,3 @@
-from cmath import exp
 from pathlib import Path
 from pprint import pprint
 from textwrap import dedent

darwin/exporter/exporter.py

@@ -60,7 +60,7 @@ def export_annotations(
     output_directory : PathLike
         Where the parsed files will be placed after the operation is complete.
     """
-    print(f"Converting annotations...")
+    print("Converting annotations...")
     exporter(
         darwin_to_dt_gen(file_paths, split_sequences=split_sequences),
         Path(output_directory),

darwin/exporter/formats/coco.py

@@ -9,7 +9,6 @@
 from upolygon import draw_polygon, rle_encode
 
 import darwin.datatypes as dt
-from darwin.exporter.formats.numpy_encoder import NumpyEncoder
 from darwin.utils import convert_polygons_to_sequences
 from darwin.version import __version__
 

darwin/exporter/formats/dataloop.py

@@ -5,7 +5,6 @@
 import orjson as json
 
 import darwin.datatypes as dt
-from darwin.exporter.formats.numpy_encoder import NumpyEncoder
 from darwin.version import __version__
 
 DEPRECATION_MESSAGE = """

darwin/exporter/formats/mask.py

@@ -2,9 +2,8 @@
 import math
 import os
 from csv import writer as csv_writer
-from functools import reduce
 from pathlib import Path
-from typing import Dict, Iterable, List, Literal, Optional, Set, Tuple, get_args
+from typing import Dict, Iterable, List, Optional, Set, Tuple, get_args
 
 import numpy as np
 
@@ -17,7 +16,7 @@
 
 import darwin.datatypes as dt
 from darwin.exceptions import DarwinException
-from darwin.utils import convert_polygons_to_sequences, ispolygon
+from darwin.utils import convert_polygons_to_sequences
 
 
 def get_palette(mode: dt.MaskTypes.Mode, categories: List[str]) -> dt.MaskTypes.Palette:
@@ -37,7 +36,7 @@ def get_palette(mode: dt.MaskTypes.Mode, categories: List[str]) -> dt.MaskTypes.
         A dict of categories and their corresponding palette value.
     """
 
-    if not mode in get_args(dt.MaskTypes.Mode):
+    if mode not in get_args(dt.MaskTypes.Mode):
         raise ValueError(f"Unknown mode {mode}.") from DarwinException
 
     if not isinstance(categories, list) or not categories:
@@ -68,7 +67,7 @@ def get_palette(mode: dt.MaskTypes.Mode, categories: List[str]) -> dt.MaskTypes.
 
     if not palette:
         raise ValueError(
-            f"Failed to generate a palette.", mode, categories
+            "Failed to generate a palette.", mode, categories
         ) from DarwinException
 
     return palette
@@ -101,14 +100,12 @@ def get_rgb_colours(
         (x / num_categories, SATURATION_OF_COLOUR, VALUE_OF_COLOUR)
         for x in range(num_categories - 1)
     ]
-    rgb_colour_list: dt.MaskTypes.RgbColorList = list(
-        map(lambda x: [int(e * 255) for e in colorsys.hsv_to_rgb(*x)], hsv_colours)
-    )
+    rgb_colour_list: dt.MaskTypes.RgbColorList = [
+        [int(e * 255) for e in colorsys.hsv_to_rgb(*x)] for x in hsv_colours
+    ]
     # Now we add BG class with [0 0 0] RGB value
     rgb_colour_list.insert(0, [0, 0, 0])
-    palette_rgb: dt.MaskTypes.RgbPalette = {
-        c: rgb for c, rgb in zip(categories, rgb_colour_list)
-    }
+    palette_rgb: dt.MaskTypes.RgbPalette = dict(zip(categories, rgb_colour_list))
     rgb_colours: dt.MaskTypes.RgbColors = [c for e in rgb_colour_list for c in e]
 
     return rgb_colours, palette_rgb
@@ -195,7 +192,7 @@ def colours_in_rle(
                 f"Could not find mask with uuid {uuid} in mask lookup table."
             )
 
-        if not mask.name in colours:
+        if mask.name not in colours:
             colours[mask.name] = colour_value
 
     return colours  # Returns same item as the outset, technically not needed, but best practice.
@@ -235,7 +232,7 @@ def get_or_generate_colour(cat_name: str, colours: dt.MaskTypes.ColoursDict) ->
     -------
     int - the integer for the colour name.  These will later be reassigned to a wider spread across the colour spectrum.
     """
-    if not cat_name in colours:
+    if cat_name not in colours:
         colours[cat_name] = len(colours) + 1
 
     return colours[cat_name]
@@ -293,7 +290,7 @@ def render_polygons(
     for a in filtered_annotations:
         try:
             cat = a.annotation_class.name
-            if not cat in categories:
+            if cat not in categories:
                 categories.append(cat)
 
             if a.annotation_class.annotation_type == "polygon":
@@ -368,7 +365,7 @@ def render_raster(
     mask_annotations: List[dt.AnnotationMask] = []
     raster_layer: Optional[dt.RasterLayer] = None
 
-    mask_lookup: Dict[str, dt.AnnotationMask] = dict()
+    mask_lookup: Dict[str, dt.AnnotationMask] = {}
 
     for a in annotations:
         if isinstance(a, dt.VideoAnnotation):
@@ -390,11 +387,11 @@ def render_raster(
 
             mask_annotations.append(new_mask)
 
-            if not new_mask.id in mask_lookup:
+            if new_mask.id not in mask_lookup:
                 mask_lookup[new_mask.id] = new_mask
 
             # Add the category to the list of categories
-            if not new_mask.name in categories:
+            if new_mask.name not in categories:
                 categories.append(new_mask.name)
 
         if a.annotation_class.annotation_type == "raster_layer" and (rl := data):
@@ -415,11 +412,11 @@ def render_raster(
             raster_layer = new_rl
 
     if not raster_layer:
-        errors.append(ValueError(f"Annotation has no raster layer"))
+        errors.append(ValueError("Annotation has no raster layer"))
         return errors, mask, categories, colours
 
     if not mask_annotations:
-        errors.append(ValueError(f"Annotation has no masks"))
+        errors.append(ValueError("Annotation has no masks"))
         return errors, mask, categories, colours
 
     try:
@@ -447,19 +444,15 @@ def export(
     if len(all_classes_sets) > 0:
         all_classes: Set[dt.AnnotationClass] = set.union(*all_classes_sets)
         categories: List[str] = ["__background__"] + sorted(
-            list(
-                set(
-                    [c.name for c in all_classes if c.annotation_type in accepted_types]
-                )
-            ),
+            {c.name for c in all_classes if c.annotation_type in accepted_types},
             key=lambda x: x.lower(),
         )
         palette = get_palette(mode, categories)
     else:
         categories = ["__background__"]
-        palette = dict()
+        palette = {}
 
-    colours: dt.MaskTypes.ColoursDict = dict()
+    colours: dt.MaskTypes.ColoursDict = {}
 
     for annotation_file in annotation_files:
         image_rel_path = os.path.splitext(annotation_file.full_path)[0].lstrip("/")

darwin/exporter/formats/nifti.py

@@ -1,6 +1,5 @@
 import ast
 import json as native_json
-from asyncore import loop
 from dataclasses import dataclass
 from pathlib import Path
 from typing import Dict, Iterable, List, Optional, Tuple, Union
@@ -19,11 +18,9 @@
     console.print(import_fail_string)
     exit()
 import numpy as np
-import orjson as json
-from PIL import Image
 
 import darwin.datatypes as dt
-from darwin.utils import convert_polygons_to_mask, get_progress_bar
+from darwin.utils import convert_polygons_to_mask
 
 
 @dataclass
@@ -141,7 +138,6 @@ def check_for_error_and_return_imageid(
 
     """
 
-    output_volumes = None
     filename = Path(video_annotation.filename)
     try:
         suffixes = filename.suffixes[-2:]
@@ -277,7 +273,7 @@ def populate_output_volumes(
             )
         else:
             continue
-        class_name = frames[frame_idx].annotation_class.name
+        frames[frame_idx].annotation_class.name
         im_mask = convert_polygons_to_mask(polygons, height=height, width=width)
         volume = output_volumes[series_instance_uid]
         if view_idx == 0:

darwin/exporter/formats/pascalvoc.py

@@ -3,7 +3,6 @@
 from xml.etree.ElementTree import Element, SubElement, tostring
 
 import deprecation
-import orjson as json
 
 import darwin.datatypes as dt
 from darwin.utils import attempt_decode

darwin/exporter/formats/yolo_segmented.py

@@ -1,11 +1,10 @@
 from collections import namedtuple
 from enum import Enum, auto
 from logging import getLogger
-from multiprocessing.pool import CLOSE
 from pathlib import Path
 from typing import Iterable, List
 
-from darwin.datatypes import Annotation, AnnotationFile, JSONType, VideoAnnotation
+from darwin.datatypes import Annotation, AnnotationFile, VideoAnnotation
 from darwin.exceptions import DarwinException
 from darwin.exporter.formats.helpers.yolo_class_builder import (
     ClassIndex,
@@ -208,7 +207,7 @@ def _handle_polygon(
         )
         return False
 
-    except Exception as exc:
+    except Exception:
         logger.error(
             f"An unexpected error occured while exporting annotation at index {annotation_index}."
         )

darwin/importer/formats/coco.py

@@ -97,9 +97,7 @@ def parse_json(
     for image_id in image_annotations.keys():
         image = image_lookup_table[int(image_id)]
         annotations = list(filter(None, image_annotations[image_id]))
-        annotation_classes = set(
-            [annotation.annotation_class for annotation in annotations]
-        )
+        annotation_classes = {annotation.annotation_class for annotation in annotations}
         remote_path, filename = deconstruct_full_path(image["file_name"])
         yield dt.AnnotationFile(
             path, filename, annotation_classes, annotations, remote_path=remote_path

darwin/importer/formats/csv_tags.py

@@ -28,13 +28,13 @@ def parse_path(path: Path) -> Optional[List[dt.AnnotationFile]]:
     with path.open() as f:
         reader = csv.reader(f)
         for row in reader:
-            filename, *tags = map(lambda s: s.strip(), row)
+            filename, *tags = (s.strip() for s in row)
             if filename == "":
                 continue
             annotations = [dt.make_tag(tag) for tag in tags if len(tag) > 0]
-            annotation_classes = set(
-                [annotation.annotation_class for annotation in annotations]
-            )
+            annotation_classes = {
+                annotation.annotation_class for annotation in annotations
+            }
             remote_path, filename = deconstruct_full_path(filename)
             files.append(
                 dt.AnnotationFile(