From fed98378e9c7ce637bcfe9bddef09724ccf19f9b Mon Sep 17 00:00:00 2001 From: ktro2828 Date: Fri, 22 Nov 2024 19:41:58 +0900 Subject: [PATCH 01/13] feat: add T4 updater from FastLabel annotations Signed-off-by: ktro2828 --- config/update_t4_with_fastlabel_sample.yaml | 24 ++++++ perception_dataset/convert.py | 33 ++++++++ .../fastlabel_2d_to_t4_converter.py | 75 +++++++++++++---- .../fastlabel_2d_to_t4_updater.py | 80 +++++++++++++++++++ .../t4_dataset/annotation_files_updater.py | 38 +++++++++ .../t4_dataset/classes/abstract_class.py | 26 ++++++ 6 files changed, 262 insertions(+), 14 deletions(-) create mode 100644 config/update_t4_with_fastlabel_sample.yaml create mode 100644 perception_dataset/fastlabel_to_t4/fastlabel_2d_to_t4_updater.py create mode 100644 perception_dataset/t4_dataset/annotation_files_updater.py diff --git a/config/update_t4_with_fastlabel_sample.yaml b/config/update_t4_with_fastlabel_sample.yaml new file mode 100644 index 00000000..635c2ca4 --- /dev/null +++ b/config/update_t4_with_fastlabel_sample.yaml @@ -0,0 +1,24 @@ +task: update_t4_with_fastlabel +description: + visibility: + full: "No occlusion of the object." + most: "Object is occluded, but by less than 50%." + partial: "The object is occluded by more than 50% (but not completely)." + none: "The object is 90-100% occluded and no points/pixels are visible in the label." + camera_index: + CAM_FRONT: 0 + CAM_FRONT_RIGHT: 1 + CAM_BACK_RIGHT: 2 + CAM_BACK: 3 + CAM_BACK_LEFT: 4 + CAM_FRONT_LEFT: 5 + +conversion: + input_base: ./data/input/t4_format_2d_annotated # could be non_annotated_t4_format or t4_format_3d_annotated + input_anno_base: ./data/fastlabel + input_bag_base: ./data/rosbag2 + output_base: ./data/output/t4_format_2d_annotated # this only includes the 2D annotations + topic_list: ./config/topic_list_sample.yaml + dataset_corresponding: + # input t4dataset_name: FastLabel json file name + DBv2.0-2-4: 2-4.json diff --git a/perception_dataset/convert.py b/perception_dataset/convert.py index c1dd0887..23c3b969 100644 --- a/perception_dataset/convert.py +++ b/perception_dataset/convert.py @@ -370,6 +370,39 @@ def main(): converter.convert() logger.info(f"[END] Converting Fastlabel data ({input_base}) to T4 data ({output_base})") + elif task == "update_t4_with_fastlabel": + from perception_dataset.fastlabel_to_t4.fastlabel_2d_to_t4_updater import ( + FastLabel2dToUpdater, + ) + + input_base = config_dict["conversion"]["input_base"] + output_base = config_dict["conversion"]["output_base"] + input_anno_base = config_dict["conversion"]["input_anno_base"] + dataset_corresponding = config_dict["conversion"]["dataset_corresponding"] + description = config_dict["description"] + input_bag_base = config_dict["conversion"]["input_bag_base"] + topic_list_yaml_path = config_dict["conversion"]["topic_list"] + with open(topic_list_yaml_path) as f: + topic_list_yaml = yaml.safe_load(f) + + converter = FastLabel2dToUpdater( + input_base=input_base, + output_base=output_base, + input_anno_base=input_anno_base, + dataset_corresponding=dataset_corresponding, + overwrite_mode=args.overwrite, + description=description, + input_bag_base=input_bag_base, + topic_list=topic_list_yaml, + ) + logger.info( + f"[BEGIN] Updating T4 dataset ({input_base}) with FastLabel {input_anno_base} into T4 data ({output_base})" + ) + converter.convert() + logger.info( + f"[DONE] Updating T4 dataset ({input_base}) with FastLabel {input_anno_base} into T4 data ({output_base})" + ) + elif task == "merge_2d_t4dataset_to_3d": from perception_dataset.t4_dataset.t4_dataset_2d3d_merger import T4dataset2D3DMerger diff --git a/perception_dataset/fastlabel_to_t4/fastlabel_2d_to_t4_converter.py b/perception_dataset/fastlabel_to_t4/fastlabel_2d_to_t4_converter.py index 1f352e2f..2e83be99 100644 --- a/perception_dataset/fastlabel_to_t4/fastlabel_2d_to_t4_converter.py +++ b/perception_dataset/fastlabel_to_t4/fastlabel_2d_to_t4_converter.py @@ -1,3 +1,4 @@ +import base64 from collections import defaultdict import json import os.path as osp @@ -5,12 +6,16 @@ import shutil from typing import Any, Dict, List, Optional, Union +import pycocotools.mask as cocomask + from perception_dataset.deepen.deepen_to_t4_converter import DeepenToT4Converter from perception_dataset.t4_dataset.annotation_files_generator import AnnotationFilesGenerator from perception_dataset.utils.logger import configure_logger logger = configure_logger(modname=__name__) +Points2DLike = list[list[list[float]]] + class FastLabel2dToT4Converter(DeepenToT4Converter): def __init__( @@ -146,18 +151,14 @@ def _format_fastlabel_annotation(self, annotations: Dict[str, List[Dict[str, Any ] } ], - "annotations": [ - { - "points": [ + "points": [ 1221.25, 488.44, 1275.38, 570.47 - ], - "rotation": 0, - "autogenerated": false - } - ] + ], + "rotation": 0, + "autogenerated": false }, }, .... @@ -177,6 +178,9 @@ def _format_fastlabel_annotation(self, annotations: Dict[str, List[Dict[str, Any if dataset_name not in fl_annotations: fl_annotations[dataset_name] = defaultdict(list) + width: int = ann["width"] + height: int = ann["height"] + for a in ann["annotations"]: occlusion_state: str = "occlusion_state.none" visibility: str = "Not available" @@ -199,12 +203,55 @@ def _format_fastlabel_annotation(self, annotations: Dict[str, List[Dict[str, Any "attribute_names": [occlusion_state], "visibility_name": visibility, } - label_t4_dict.update( - { - "two_d_box": a["annotations"][0]["points"], - "sensor_id": self._camera2idx[camera], - } - ) + if a["type"] == "bbox": + label_t4_dict.update( + { + "two_d_box": a["points"], + "sensor_id": self._camera2idx[camera], + } + ) + elif a["type"] == "segmentation": + label_t4_dict.update( + { + "two_d_segmentation": _rle_from_points(a["points"], width, height), + "sensor_id": self._camera2idx[camera], + } + ) fl_annotations[dataset_name][file_id].append(label_t4_dict) return fl_annotations + + +def _rle_from_points(points: Points2DLike, width: int, height: int) -> Dict[str, Any]: + """Encode points to RLE format mask. + + Points format of 2D segmentation in FastLabel: + ``` + "points": [ + [ + [...], // outer points (1) + [...], // hollowed out points (1) + ], + [ + [...], // outer points (2) + [...], // hollowed out points (2) + ] + ], + ``` + + Args: + points (Points2DLike): 2D points, such as `[[[o1, o1, o2, o2, ...], [ho1, ho1, ho2, ho2, ...], ...]]`. + width (int): Image width. + height (int): Image height. + + Returns: + Dict[str, Any]: RLE format mask. + """ + flattened = [[coord for p in point for coord in p] for point in points] + + rle_objects = cocomask.frPyObjects(flattened, height, width) + rle = cocomask.merge(rle_objects) + + rle["counts"] = base64.b64encode(rle["counts"]).decode("ascii") + + return rle diff --git a/perception_dataset/fastlabel_to_t4/fastlabel_2d_to_t4_updater.py b/perception_dataset/fastlabel_to_t4/fastlabel_2d_to_t4_updater.py new file mode 100644 index 00000000..0df1fb85 --- /dev/null +++ b/perception_dataset/fastlabel_to_t4/fastlabel_2d_to_t4_updater.py @@ -0,0 +1,80 @@ +from __future__ import annotations + +import os.path as osp +from pathlib import Path +import shutil +from typing import Dict, List + +from perception_dataset.fastlabel_to_t4.fastlabel_2d_to_t4_converter import ( + FastLabel2dToT4Converter, +) +from perception_dataset.t4_dataset.annotation_files_updater import AnnotationFilesUpdater +from perception_dataset.utils.logger import configure_logger + +logger = configure_logger(modname=__name__) + + +class FastLabel2dToUpdater(FastLabel2dToT4Converter): + def __init__( + self, + input_base: str, + output_base: str, + input_anno_base: str, + dataset_corresponding: Dict[str, int], + overwrite_mode: bool, + description: Dict[str, Dict[str, str]], + input_bag_base: str | None, + topic_list: Dict[str, List[str]] | List[str], + ): + super().__init__( + input_base, + output_base, + input_anno_base, + dataset_corresponding, + overwrite_mode, + description, + input_bag_base, + topic_list, + ) + + def convert(self) -> None: + anno_jsons_dict = self._load_annotation_jsons() + fl_annotations = self._format_deepen_annotation(anno_jsons_dict) + + for t4dataset_name in self._t4dataset_name_to_merge: + # Check if input directory exists + input_dir = self._input_base / t4dataset_name + input_annotation_dir = input_dir / "annotation" + if not osp.exists(input_annotation_dir): + logger.warning(f"input_dir {input_dir} not exists.") + continue + + # Check if output directory already exists + output_dir = self._output_base / t4dataset_name / "t4_dataset" + if self._input_bag_base is not None: + input_bag_dir = Path(self._input_bag_base) / t4dataset_name + if osp.exists(output_dir): + logger.error(f"{output_dir} already exists.") + is_dir_exist = True + if self._overwrite_mode or not is_dir_exist: + # Remove existing output directory + shutil.rmtree(output_dir, ignore_errors=True) + # Copy input data to output directory + self._copy_data(input_dir, output_dir) + # Make rosbag + if self._input_bag_base is not None and not osp.exists( + osp.join(output_dir, "input_bag") + ): + self._find_start_end_time(input_dir) + self._make_rosbag(str(input_bag_dir), str(output_dir)) + else: + raise ValueError("If you want to overwrite files, use --overwrite option.") + + # Start updating annotations + annotation_files_updater = AnnotationFilesUpdater(description=self._description) + annotation_files_updater.convert_one_scene( + input_dir=input_dir, + output_dir=output_dir, + scene_anno_dict=fl_annotations[t4dataset_name], + dataset_name=t4dataset_name, + ) diff --git a/perception_dataset/t4_dataset/annotation_files_updater.py b/perception_dataset/t4_dataset/annotation_files_updater.py new file mode 100644 index 00000000..02f82d9f --- /dev/null +++ b/perception_dataset/t4_dataset/annotation_files_updater.py @@ -0,0 +1,38 @@ +import os.path as osp +from typing import Any + +from perception_dataset.t4_dataset.annotation_files_generator import AnnotationFilesGenerator + + +class AnnotationFilesUpdater(AnnotationFilesGenerator): + def convert_one_scene( + self, + input_dir: str, + output_dir: str, + scene_anno_dict: dict[str, list[dict[str, Any]]], + dataset_name: str, + ) -> None: + anno_dir = osp.join(input_dir, "annotation") + if not osp.exists(anno_dir): + raise ValueError(f"Annotations files doesn't exist in {anno_dir}") + + # Load existence annotation files + self._attribute_table.insert_from_json(osp.join(anno_dir, self._attribute_table.FILENAME)) + self._category_table.insert_from_json(osp.join(anno_dir, self._category_table.FILENAME)) + self._instance_table.insert_from_json(osp.join(anno_dir, self._instance_table.FILENAME)) + self._sample_annotation_table.insert_from_json( + osp.join(anno_dir, self._sample_annotation_table.FILENAME) + ) + self._object_ann_table.insert_from_json( + osp.join(anno_dir, self._object_ann_table.FILENAME) + ) + self._surface_ann_table.insert_from_json( + osp.join(anno_dir, self._surface_ann_table.FILENAME) + ) + + super().convert_one_scene( + input_dir=input_dir, + output_dir=output_dir, + scene_anno_dict=scene_anno_dict, + dataset_name=dataset_name, + ) diff --git a/perception_dataset/t4_dataset/classes/abstract_class.py b/perception_dataset/t4_dataset/classes/abstract_class.py index 25e2793d..563d3863 100644 --- a/perception_dataset/t4_dataset/classes/abstract_class.py +++ b/perception_dataset/t4_dataset/classes/abstract_class.py @@ -1,3 +1,5 @@ +from __future__ import annotations + from abc import ABCMeta, abstractmethod import json import os.path as osp @@ -14,10 +16,17 @@ def __init__(self): def token(self) -> str: return self._token + @token.setter + def token(self, token: str): + self._token = token + @abstractmethod def to_dict(self) -> Dict[str, Any]: raise NotImplementedError() + def __eq__(self, value: T) -> bool: + return self.__dict__ == value.__dict__ + T = TypeVar("T", bound=AbstractRecord) @@ -37,6 +46,10 @@ def _to_record(self, **kwargs) -> T: raise NotImplementedError() def set_record_to_table(self, record: T): + same_tokens = [token for token, v in self._token_to_record.items() if v == record] + assert len(same_tokens) in (0, 1) + if len(same_tokens) == 1: + record.token = same_tokens[0] # overwrite record token with the existing one self._token_to_record[record.token] = record def insert_into_table(self, **kwargs) -> str: @@ -47,6 +60,19 @@ def insert_into_table(self, **kwargs) -> str: self.set_record_to_table(record) return record.token + def insert_from_json(self, filepath: str): + with open(filepath, "r") as f: + table_data: List[Dict[str, Any]] = json.load(f) + + for data in table_data: + token: str = data.pop("token") + record = self._to_record(**data) + record.token = token + assert isinstance( + record, AbstractRecord + ), "_to_record function must return the instance of RecordClass" + self.set_record_to_table(record) + def select_record_from_token(self, token: str) -> T: assert ( token in self._token_to_record From 4cf260a333a97151c81a7a792fd101b4909f7096 Mon Sep 17 00:00:00 2001 From: ktro2828 Date: Thu, 28 Nov 2024 07:31:34 +0900 Subject: [PATCH 02/13] fix: update conversion Signed-off-by: ktro2828 --- config/update_t4_with_fastlabel_sample.yaml | 8 +- perception_dataset/convert.py | 6 -- .../fastlabel_2d_to_t4_updater.py | 28 +++++-- .../t4_dataset/annotation_files_updater.py | 82 ++++++++++++++++--- .../t4_dataset/classes/abstract_class.py | 25 ++---- .../t4_dataset/classes/attribute.py | 22 +++++ .../t4_dataset/classes/calibrated_sensor.py | 31 +++++++ .../t4_dataset/classes/category.py | 21 +++++ .../t4_dataset/classes/ego_pose.py | 59 +++++++++++++ .../t4_dataset/classes/instance.py | 19 +++++ perception_dataset/t4_dataset/classes/log.py | 21 +++++ perception_dataset/t4_dataset/classes/map.py | 20 +++++ .../t4_dataset/classes/object_ann.py | 23 ++++++ .../t4_dataset/classes/sample.py | 22 +++++ .../t4_dataset/classes/sample_annotation.py | 57 +++++++++++++ .../t4_dataset/classes/sample_data.py | 30 +++++++ .../t4_dataset/classes/scene.py | 23 ++++++ .../t4_dataset/classes/sensor.py | 16 ++++ .../t4_dataset/classes/surface_ann.py | 20 +++++ .../t4_dataset/classes/vehicle_state.py | 24 ++++++ .../t4_dataset/classes/visibility.py | 21 +++++ 21 files changed, 528 insertions(+), 50 deletions(-) diff --git a/config/update_t4_with_fastlabel_sample.yaml b/config/update_t4_with_fastlabel_sample.yaml index 635c2ca4..af0a7c2a 100644 --- a/config/update_t4_with_fastlabel_sample.yaml +++ b/config/update_t4_with_fastlabel_sample.yaml @@ -14,11 +14,9 @@ description: CAM_FRONT_LEFT: 5 conversion: - input_base: ./data/input/t4_format_2d_annotated # could be non_annotated_t4_format or t4_format_3d_annotated + input_base: ./data/input_t4_format # could be non_annotated_t4_format or t4_format_3d_annotated input_anno_base: ./data/fastlabel - input_bag_base: ./data/rosbag2 - output_base: ./data/output/t4_format_2d_annotated # this only includes the 2D annotations - topic_list: ./config/topic_list_sample.yaml + output_base: ./data/output_t4_format # currently, this only includes the 2D annotations dataset_corresponding: # input t4dataset_name: FastLabel json file name - DBv2.0-2-4: 2-4.json + T4DatasetName: FastLabelAnnotationFile diff --git a/perception_dataset/convert.py b/perception_dataset/convert.py index 23c3b969..670fd409 100644 --- a/perception_dataset/convert.py +++ b/perception_dataset/convert.py @@ -380,10 +380,6 @@ def main(): input_anno_base = config_dict["conversion"]["input_anno_base"] dataset_corresponding = config_dict["conversion"]["dataset_corresponding"] description = config_dict["description"] - input_bag_base = config_dict["conversion"]["input_bag_base"] - topic_list_yaml_path = config_dict["conversion"]["topic_list"] - with open(topic_list_yaml_path) as f: - topic_list_yaml = yaml.safe_load(f) converter = FastLabel2dToUpdater( input_base=input_base, @@ -392,8 +388,6 @@ def main(): dataset_corresponding=dataset_corresponding, overwrite_mode=args.overwrite, description=description, - input_bag_base=input_bag_base, - topic_list=topic_list_yaml, ) logger.info( f"[BEGIN] Updating T4 dataset ({input_base}) with FastLabel {input_anno_base} into T4 data ({output_base})" diff --git a/perception_dataset/fastlabel_to_t4/fastlabel_2d_to_t4_updater.py b/perception_dataset/fastlabel_to_t4/fastlabel_2d_to_t4_updater.py index 0df1fb85..c92601d8 100644 --- a/perception_dataset/fastlabel_to_t4/fastlabel_2d_to_t4_updater.py +++ b/perception_dataset/fastlabel_to_t4/fastlabel_2d_to_t4_updater.py @@ -1,9 +1,10 @@ from __future__ import annotations +import json import os.path as osp from pathlib import Path import shutil -from typing import Dict, List +from typing import Dict from perception_dataset.fastlabel_to_t4.fastlabel_2d_to_t4_converter import ( FastLabel2dToT4Converter, @@ -23,8 +24,6 @@ def __init__( dataset_corresponding: Dict[str, int], overwrite_mode: bool, description: Dict[str, Dict[str, str]], - input_bag_base: str | None, - topic_list: Dict[str, List[str]] | List[str], ): super().__init__( input_base, @@ -33,13 +32,13 @@ def __init__( dataset_corresponding, overwrite_mode, description, - input_bag_base, - topic_list, + input_bag_base=None, + topic_list=None, ) def convert(self) -> None: anno_jsons_dict = self._load_annotation_jsons() - fl_annotations = self._format_deepen_annotation(anno_jsons_dict) + fl_annotations = self._format_fastlabel_annotation(anno_jsons_dict) for t4dataset_name in self._t4dataset_name_to_merge: # Check if input directory exists @@ -53,9 +52,13 @@ def convert(self) -> None: output_dir = self._output_base / t4dataset_name / "t4_dataset" if self._input_bag_base is not None: input_bag_dir = Path(self._input_bag_base) / t4dataset_name + if osp.exists(output_dir): logger.error(f"{output_dir} already exists.") is_dir_exist = True + else: + is_dir_exist = False + if self._overwrite_mode or not is_dir_exist: # Remove existing output directory shutil.rmtree(output_dir, ignore_errors=True) @@ -78,3 +81,16 @@ def convert(self) -> None: scene_anno_dict=fl_annotations[t4dataset_name], dataset_name=t4dataset_name, ) + + def _load_annotation_jsons(self): + anno_dict = {} + for file in self._input_anno_files: + t4_dataset_name = None + for name, ann_filename in self._t4dataset_name_to_merge.items(): + if ann_filename == file.name: + t4_dataset_name = name + + assert t4_dataset_name is not None + with open(file) as f: + anno_dict[t4_dataset_name] = json.load(f) + return anno_dict diff --git a/perception_dataset/t4_dataset/annotation_files_updater.py b/perception_dataset/t4_dataset/annotation_files_updater.py index 02f82d9f..62d7f4f7 100644 --- a/perception_dataset/t4_dataset/annotation_files_updater.py +++ b/perception_dataset/t4_dataset/annotation_files_updater.py @@ -1,10 +1,30 @@ +import json import os.path as osp -from typing import Any +from typing import Any, Dict from perception_dataset.t4_dataset.annotation_files_generator import AnnotationFilesGenerator +from perception_dataset.t4_dataset.classes import ( + AttributeTable, + CategoryTable, + InstanceTable, + ObjectAnnTable, + SampleAnnotationTable, + SurfaceAnnTable, + VisibilityTable, +) + + +def _load_json(filepath: str) -> Any: + with open(filepath) as f: + data = json.load(f) + return data class AnnotationFilesUpdater(AnnotationFilesGenerator): + def __init__(self, with_camera: bool = True, description: Dict[str, Dict[str, str]] = ...): + super().__init__(with_camera, description) + self.description = description + def convert_one_scene( self, input_dir: str, @@ -17,18 +37,7 @@ def convert_one_scene( raise ValueError(f"Annotations files doesn't exist in {anno_dir}") # Load existence annotation files - self._attribute_table.insert_from_json(osp.join(anno_dir, self._attribute_table.FILENAME)) - self._category_table.insert_from_json(osp.join(anno_dir, self._category_table.FILENAME)) - self._instance_table.insert_from_json(osp.join(anno_dir, self._instance_table.FILENAME)) - self._sample_annotation_table.insert_from_json( - osp.join(anno_dir, self._sample_annotation_table.FILENAME) - ) - self._object_ann_table.insert_from_json( - osp.join(anno_dir, self._object_ann_table.FILENAME) - ) - self._surface_ann_table.insert_from_json( - osp.join(anno_dir, self._surface_ann_table.FILENAME) - ) + self._init_table_from_json(anno_dir=anno_dir) super().convert_one_scene( input_dir=input_dir, @@ -36,3 +45,50 @@ def convert_one_scene( scene_anno_dict=scene_anno_dict, dataset_name=dataset_name, ) + + def _init_table_from_json(self, anno_dir: str) -> None: + self._attribute_table = AttributeTable.from_json( + filepath=osp.join(anno_dir, AttributeTable.FILENAME), + name_to_description={}, + default_value="", + ) + + self._category_table = CategoryTable.from_json( + filepath=osp.join(anno_dir, CategoryTable.FILENAME), + name_to_description={}, + default_value="", + ) + + self._instance_table = InstanceTable.from_json( + filepath=osp.join(anno_dir, InstanceTable.FILENAME) + ) + + self._visibility_table = VisibilityTable.from_json( + filepath=osp.join(anno_dir, VisibilityTable.FILENAME), + level_to_description=self.description.get( + "visibility", + { + "v0-40": "visibility of whole object is between 0 and 40%", + "v40-60": "visibility of whole object is between 40 and 60%", + "v60-80": "visibility of whole object is between 60 and 80%", + "v80-100": "visibility of whole object is between 80 and 100%", + "none": "visibility isn't available", + }, + ), + default_value="", + ) + + if osp.exists(osp.join(anno_dir, SampleAnnotationTable.FILENAME)): + self._sample_annotation_table = SampleAnnotationTable.from_json( + osp.join(anno_dir, SampleAnnotationTable.FILENAME) + ) + + if osp.exists(osp.join(anno_dir, ObjectAnnTable.FILENAME)): + self._object_ann_table = ObjectAnnTable.from_json( + osp.join(anno_dir, ObjectAnnTable.FILENAME) + ) + + if osp.exists(osp.join(anno_dir, SurfaceAnnTable.FILENAME)): + self._surface_ann_table = SurfaceAnnTable.from_json( + osp.join(anno_dir, SurfaceAnnTable.FILENAME) + ) diff --git a/perception_dataset/t4_dataset/classes/abstract_class.py b/perception_dataset/t4_dataset/classes/abstract_class.py index 563d3863..6eb762f4 100644 --- a/perception_dataset/t4_dataset/classes/abstract_class.py +++ b/perception_dataset/t4_dataset/classes/abstract_class.py @@ -24,9 +24,6 @@ def token(self, token: str): def to_dict(self) -> Dict[str, Any]: raise NotImplementedError() - def __eq__(self, value: T) -> bool: - return self.__dict__ == value.__dict__ - T = TypeVar("T", bound=AbstractRecord) @@ -46,10 +43,6 @@ def _to_record(self, **kwargs) -> T: raise NotImplementedError() def set_record_to_table(self, record: T): - same_tokens = [token for token, v in self._token_to_record.items() if v == record] - assert len(same_tokens) in (0, 1) - if len(same_tokens) == 1: - record.token = same_tokens[0] # overwrite record token with the existing one self._token_to_record[record.token] = record def insert_into_table(self, **kwargs) -> str: @@ -60,19 +53,6 @@ def insert_into_table(self, **kwargs) -> str: self.set_record_to_table(record) return record.token - def insert_from_json(self, filepath: str): - with open(filepath, "r") as f: - table_data: List[Dict[str, Any]] = json.load(f) - - for data in table_data: - token: str = data.pop("token") - record = self._to_record(**data) - record.token = token - assert isinstance( - record, AbstractRecord - ), "_to_record function must return the instance of RecordClass" - self.set_record_to_table(record) - def select_record_from_token(self, token: str) -> T: assert ( token in self._token_to_record @@ -97,3 +77,8 @@ def save_json(self, output_dir: str): table_data = self.to_data() with open(osp.join(output_dir, self.FILENAME), "w") as f: json.dump(table_data, f, indent=4) + + @classmethod + @abstractmethod + def from_json(cls, filepath: str): + raise NotImplementedError diff --git a/perception_dataset/t4_dataset/classes/attribute.py b/perception_dataset/t4_dataset/classes/attribute.py index 2b63cfcb..74561a02 100644 --- a/perception_dataset/t4_dataset/classes/attribute.py +++ b/perception_dataset/t4_dataset/classes/attribute.py @@ -1,3 +1,6 @@ +from __future__ import annotations + +import json from typing import Dict from perception_dataset.constants import EXTENSION_ENUM @@ -46,3 +49,22 @@ def get_token_from_name(self, name: str) -> str: self._name_to_token[name] = token return token + + @classmethod + def from_json( + cls, + filepath: str, + name_to_description: Dict[str, str], + default_value: str, + ) -> AttributeTable: + with open(filepath) as f: + items = json.load(f) + + table = cls(name_to_description=name_to_description, default_value=default_value) + + for item in items: + record = AttributeRecord(name=item["name"], description=item["description"]) + record.token = item["token"] + table.set_record_to_table(record) + + return table diff --git a/perception_dataset/t4_dataset/classes/calibrated_sensor.py b/perception_dataset/t4_dataset/classes/calibrated_sensor.py index d1574496..3d76fc4d 100644 --- a/perception_dataset/t4_dataset/classes/calibrated_sensor.py +++ b/perception_dataset/t4_dataset/classes/calibrated_sensor.py @@ -1,3 +1,6 @@ +from __future__ import annotations + +import json from typing import Any, Dict, List import numpy as np @@ -58,3 +61,31 @@ def __init__(self): def _to_record(self, **kwargs) -> CalibratedSensorRecord: return CalibratedSensorRecord(**kwargs) + + @classmethod + def from_json(cls, filepath: str) -> CalibratedSensorTable: + with open(filepath) as f: + items = json.load(f) + + table = cls() + for item in items: + record = CalibratedSensorRecord( + sensor_token=item["sensor_token"], + translation={ + "x": item["translation"][0], + "y": item["translation"][1], + "z": item["translation"][2], + }, + rotation={ + "w": item["rotation"][0], + "x": item["rotation"][1], + "y": item["rotation"][2], + "z": item["rotation"][3], + }, + camera_intrinsic=item["camera_intrinsic"], + camera_distortion=item["camera_distortion"], + ) + record.token = item["token"] + table.set_record_to_table(record) + + return table diff --git a/perception_dataset/t4_dataset/classes/category.py b/perception_dataset/t4_dataset/classes/category.py index 655638da..89fdd38a 100644 --- a/perception_dataset/t4_dataset/classes/category.py +++ b/perception_dataset/t4_dataset/classes/category.py @@ -1,3 +1,6 @@ +from __future__ import annotations + +import json from typing import Dict from perception_dataset.constants import EXTENSION_ENUM @@ -46,3 +49,21 @@ def get_token_from_name(self, name: str) -> str: self._name_to_token[name] = token return token + + @classmethod + def from_json( + cls, + filepath: str, + name_to_description: Dict[str, str], + default_value: str, + ) -> CategoryTable: + with open(filepath) as f: + items = json.load(f) + + table = cls(name_to_description=name_to_description, default_value=default_value) + for item in items: + record = CategoryRecord(name=item["name"], description=item["description"]) + record.token = item["token"] + table.set_record_to_table(record) + + return table diff --git a/perception_dataset/t4_dataset/classes/ego_pose.py b/perception_dataset/t4_dataset/classes/ego_pose.py index 878703ff..22cd45ee 100644 --- a/perception_dataset/t4_dataset/classes/ego_pose.py +++ b/perception_dataset/t4_dataset/classes/ego_pose.py @@ -1,3 +1,6 @@ +from __future__ import annotations + +import json from typing import Any, Dict, Optional from perception_dataset.constants import EXTENSION_ENUM @@ -92,3 +95,59 @@ def __init__(self): def _to_record(self, **kwargs) -> EgoPoseRecord: return EgoPoseRecord(**kwargs) + + @classmethod + def from_json(cls, filepath: str) -> EgoPoseTable: + with open(filepath) as f: + items = json.load(f) + + table = cls() + for item in items: + record = EgoPoseRecord( + translation={ + "x": item["translation"][0], + "y": item["translation"][1], + "z": item["translation"][2], + }, + rotation={ + "w": item["rotation"][0], + "x": item["rotation"][1], + "y": item["rotation"][2], + "z": item["rotation"][3], + }, + timestamp=item["timestamp"], + twist=( + { + "vx": item["twist"][0], + "vy": item["twist"][1], + "vz": item["twist"][2], + "yaw_rate": item["twist"][3], + "pitch_rate": item["twist"][4], + "roll_rate": item["twist"][5], + } + if item.get("twist") is not None + else None + ), + acceleration=( + { + "ax": item["acceleration"][0], + "ay": item["acceleration"][1], + "az": item["acceleration"][2], + } + if item.get("acceleration") is not None + else None + ), + geocoordinate=( + { + "latitude": item["geocoordinate"][0], + "longitude": item["geocoordinate"][1], + "altitude": item["geocoordinate"][2], + } + if item.get("geocoordinate") is not None + else None + ), + ) + record.token = item["token"] + table.set_record_to_table(record) + + return table diff --git a/perception_dataset/t4_dataset/classes/instance.py b/perception_dataset/t4_dataset/classes/instance.py index 2420d40a..1e6d0135 100644 --- a/perception_dataset/t4_dataset/classes/instance.py +++ b/perception_dataset/t4_dataset/classes/instance.py @@ -1,3 +1,6 @@ +from __future__ import annotations + +import json from typing import Any, Dict from perception_dataset.constants import EXTENSION_ENUM @@ -59,3 +62,19 @@ def get_token_from_id(self, instance_id: str, category_token: str, dataset_name: self._id_to_token[instance_id] = token return token + + @classmethod + def from_json(cls, filepath: str) -> InstanceTable: + with open(filepath) as f: + items = json.load(f) + + table = cls() + for item in items: + record = InstanceRecord( + category_token=item["category_token"], + instance_name=item.get("instance_name", ""), + ) + record.token = item["token"] + table.set_record_to_table(record) + + return table diff --git a/perception_dataset/t4_dataset/classes/log.py b/perception_dataset/t4_dataset/classes/log.py index f409c4b3..da241899 100644 --- a/perception_dataset/t4_dataset/classes/log.py +++ b/perception_dataset/t4_dataset/classes/log.py @@ -1,3 +1,6 @@ +from __future__ import annotations + +import json from typing import Any, Dict from perception_dataset.constants import EXTENSION_ENUM @@ -40,3 +43,21 @@ def __init__(self): def _to_record(self, **kwargs) -> LogRecord: return LogRecord(**kwargs) + + @classmethod + def from_json(cls, filepath: str) -> LogTable: + with open(filepath) as f: + items = json.load(f) + + table = cls() + for item in items: + record = LogRecord( + logfile=item["logfile"], + vehicle=item["vehicle"], + data_captured=item["data_captured"], + location=item["location"], + ) + record.token = item["token"] + table.set_record_to_table(record) + + return table diff --git a/perception_dataset/t4_dataset/classes/map.py b/perception_dataset/t4_dataset/classes/map.py index a0a5c725..613946d0 100644 --- a/perception_dataset/t4_dataset/classes/map.py +++ b/perception_dataset/t4_dataset/classes/map.py @@ -1,3 +1,6 @@ +from __future__ import annotations + +import json from typing import Any, Dict, List from perception_dataset.constants import EXTENSION_ENUM @@ -37,3 +40,20 @@ def __init__(self): def _to_record(self, **kwargs) -> MapRecord: return MapRecord(**kwargs) + + @classmethod + def from_json(cls, filepath: str) -> MapTable: + with open(filepath) as f: + items = json.load(f) + + table = cls() + for item in items: + record = MapRecord( + log_tokens=item["log_tokens"], + category=item["category"], + filename=item["filename"], + ) + record.token = item["token"] + table.set_record_to_table(record) + + return table diff --git a/perception_dataset/t4_dataset/classes/object_ann.py b/perception_dataset/t4_dataset/classes/object_ann.py index 3b587a8f..c4022947 100644 --- a/perception_dataset/t4_dataset/classes/object_ann.py +++ b/perception_dataset/t4_dataset/classes/object_ann.py @@ -1,3 +1,6 @@ +from __future__ import annotations + +import json from typing import Dict, List from perception_dataset.constants import EXTENSION_ENUM @@ -69,3 +72,23 @@ def _to_record( mask=mask, ) return record + + @classmethod + def from_json(cls, filepath: str) -> ObjectAnnTable: + with open(filepath) as f: + items = json.load(f) + + table = cls() + for item in items: + record = ObjectAnnRecord( + sample_data_token=item["sample_data_token"], + instance_token=item["instance_token"], + category_token=item["category_token"], + attribute_tokens=item["attribute_tokens"], + bbox=item["bbox"], + mask=item["mask"], + ) + record.token = item["token"] + table.set_record_to_table(record) + + return table diff --git a/perception_dataset/t4_dataset/classes/sample.py b/perception_dataset/t4_dataset/classes/sample.py index aa70ec23..86ca58f9 100644 --- a/perception_dataset/t4_dataset/classes/sample.py +++ b/perception_dataset/t4_dataset/classes/sample.py @@ -1,3 +1,7 @@ +from __future__ import annotations + +import json + from perception_dataset.constants import EXTENSION_ENUM from perception_dataset.t4_dataset.classes.abstract_class import AbstractRecord, AbstractTable @@ -38,3 +42,21 @@ def __init__(self): def _to_record(self, **kwargs) -> SampleRecord: return SampleRecord(**kwargs) + + @classmethod + def from_json(cls, filepath: str) -> SampleTable: + with open(filepath) as f: + items = json.load(f) + + table = cls() + for item in items: + record = SampleRecord( + timestamp=item["timestamp"], + scene_token=item["scene_token"], + next_token=item["next"], + prev_token=item["prev"], + ) + record.token = item["token"] + table.set_record_to_table(record) + + return table diff --git a/perception_dataset/t4_dataset/classes/sample_annotation.py b/perception_dataset/t4_dataset/classes/sample_annotation.py index ba723f46..05dc948f 100644 --- a/perception_dataset/t4_dataset/classes/sample_annotation.py +++ b/perception_dataset/t4_dataset/classes/sample_annotation.py @@ -1,3 +1,6 @@ +from __future__ import annotations + +import json from typing import Dict, List, Optional from perception_dataset.constants import EXTENSION_ENUM @@ -144,3 +147,57 @@ def _to_record( num_radar_pts=num_radar_pts, ) return record + + @classmethod + def from_json(cls, filepath: str) -> SampleAnnotationTable: + with open(filepath) as f: + items = json.load(f) + + table = cls() + for item in items: + record = SampleAnnotationRecord( + sample_token=item["sample_token"], + instance_token=item["instance_token"], + attribute_tokens=item["attribute_tokens"], + visibility_token=item["visibility_token"], + translation={ + "x": item["translation"][0], + "y": item["translation"][1], + "z": item["translation"][2], + }, + velocity=( + { + "x": item["velocity"][0], + "y": item["velocity"][1], + "z": item["velocity"][2], + } + if item.get("velocity") is not None + else None + ), + acceleration=( + { + "x": item["acceleration"][0], + "y": item["acceleration"][1], + "z": item["acceleration"][2], + } + if item.get("acceleration") is not None + else None + ), + size={ + "width": item["size"][0], + "length": item["size"][1], + "height": item["size"][2], + }, + rotation={ + "w": item["rotation"][0], + "x": item["rotation"][1], + "y": item["rotation"][2], + "z": item["rotation"][3], + }, + num_lidar_pts=item["num_lidar_pts"], + num_radar_pts=item["num_radar_pts"], + ) + record.token = item["token"] + table.set_record_to_table(record) + + return table diff --git a/perception_dataset/t4_dataset/classes/sample_data.py b/perception_dataset/t4_dataset/classes/sample_data.py index 316aed7e..2327ba64 100644 --- a/perception_dataset/t4_dataset/classes/sample_data.py +++ b/perception_dataset/t4_dataset/classes/sample_data.py @@ -1,3 +1,7 @@ +from __future__ import annotations + +import json + from perception_dataset.constants import EXTENSION_ENUM from perception_dataset.t4_dataset.classes.abstract_class import AbstractRecord, AbstractTable @@ -62,3 +66,29 @@ def __init__(self): def _to_record(self, **kwargs) -> SampleDataRecord: return SampleDataRecord(**kwargs) + + @classmethod + def from_json(cls, filepath: str) -> SampleDataTable: + with open(filepath) as f: + items = json.load(f) + + table = cls() + for item in items: + record = SampleDataRecord( + sample_token=item["sample_token"], + ego_pose_token=item["ego_pose_token"], + calibrated_sensor_token=item["calibrated_sensor_token"], + filename=item["filename"], + fileformat=item["fileformat"], + timestamp=item["timestamp"], + is_key_frame=item["is_key_frame"], + width=item["width"], + height=item["height"], + next_token=item["next"], + prev_token=item["prev"], + is_valid=item["is_valid"], + ) + record.token = item["token"] + table.set_record_to_table(record) + + return table diff --git a/perception_dataset/t4_dataset/classes/scene.py b/perception_dataset/t4_dataset/classes/scene.py index 412216c4..1c0c186e 100644 --- a/perception_dataset/t4_dataset/classes/scene.py +++ b/perception_dataset/t4_dataset/classes/scene.py @@ -1,3 +1,6 @@ +from __future__ import annotations + +import json from typing import Any, Dict from perception_dataset.constants import EXTENSION_ENUM @@ -46,3 +49,23 @@ def __init__(self): def _to_record(self, **kwargs) -> SceneRecord: return SceneRecord(**kwargs) + + @classmethod + def from_json(cls, filepath: str) -> SceneTable: + with open(filepath) as f: + items = json.load(f) + + table = cls() + for item in items: + record = SceneRecord( + name=item["name"], + description=item["description"], + log_token=item["log_token"], + nbr_samples=item["nbr_samples"], + first_sample_token=item["first_sample_token"], + last_sample_token=item["last_sample_token"], + ) + record.token = item["token"] + table.set_record_to_table(record) + + return table diff --git a/perception_dataset/t4_dataset/classes/sensor.py b/perception_dataset/t4_dataset/classes/sensor.py index aa7e7829..3c985b67 100644 --- a/perception_dataset/t4_dataset/classes/sensor.py +++ b/perception_dataset/t4_dataset/classes/sensor.py @@ -1,3 +1,6 @@ +from __future__ import annotations + +import json from typing import Dict from perception_dataset.constants import EXTENSION_ENUM @@ -46,3 +49,16 @@ def get_token_from_channel(self, channel: str): self._channel_to_token[channel] = token return token + + @classmethod + def from_json(cls, filepath: str, channel_to_modality: Dict[str, str]) -> SensorTable: + with open(filepath) as f: + items = json.load(f) + + table = cls(channel_to_modality=channel_to_modality) + for item in items: + record = SensorRecord(channel=item["channel"], modality=item["modality"]) + record.token = item["token"] + table.set_record_to_table(record) + + return table diff --git a/perception_dataset/t4_dataset/classes/surface_ann.py b/perception_dataset/t4_dataset/classes/surface_ann.py index d3ec94af..943a4c6e 100644 --- a/perception_dataset/t4_dataset/classes/surface_ann.py +++ b/perception_dataset/t4_dataset/classes/surface_ann.py @@ -1,3 +1,6 @@ +from __future__ import annotations + +import json from typing import Dict from perception_dataset.constants import EXTENSION_ENUM @@ -47,3 +50,20 @@ def _to_record( sample_data_token=sample_data_token, ) return record + + @classmethod + def from_json(cls, filepath: str) -> SurfaceAnnTable: + with open(filepath) as f: + items = json.load(f) + + table = cls() + for item in items: + record = SurfaceAnnRecord( + category_token=item["category_token"], + mask=item["mask"], + sample_data_token=item["sample_data_token"], + ) + record.token = item["token"] + table.select_record_from_token(record) + + return table diff --git a/perception_dataset/t4_dataset/classes/vehicle_state.py b/perception_dataset/t4_dataset/classes/vehicle_state.py index 07c9ae66..61cf4317 100644 --- a/perception_dataset/t4_dataset/classes/vehicle_state.py +++ b/perception_dataset/t4_dataset/classes/vehicle_state.py @@ -1,5 +1,6 @@ from __future__ import annotations +import json from typing import Any, Dict, Optional from perception_dataset.constants import EXTENSION_ENUM @@ -73,3 +74,26 @@ def __init__(self): def _to_record(self, **kwargs) -> VehicleStateRecord: return VehicleStateRecord(**kwargs) + + @classmethod + def from_json(cls, filepath: str) -> VehicleStateTable: + with open(filepath) as f: + items = json.load(f) + + table = cls() + for item in items: + record = VehicleStateRecord( + timestamp=item["timestamp"], + accel_pedal=item.get("accel_pedal"), + brake_pedal=item.get("brake_pedal"), + steer_pedal=item.get("steer_pedal"), + steering_tire_angle=item.get("steering_tire_angle"), + steering_wheel_angle=item.get("steering_wheel_angle"), + shift_state=item.get("shift_state"), + indicators=item.get("indicators"), + additional_info=items.get("additional_info"), + ) + record.token = item["token"] + table.set_record_to_table(record) + + return table diff --git a/perception_dataset/t4_dataset/classes/visibility.py b/perception_dataset/t4_dataset/classes/visibility.py index 42e3410f..b7b405e6 100644 --- a/perception_dataset/t4_dataset/classes/visibility.py +++ b/perception_dataset/t4_dataset/classes/visibility.py @@ -1,3 +1,6 @@ +from __future__ import annotations + +import json from typing import Dict from perception_dataset.constants import EXTENSION_ENUM @@ -46,3 +49,21 @@ def get_token_from_level(self, level: str) -> str: self._level_to_token[level] = token return token + + @classmethod + def from_json( + cls, + filepath: str, + level_to_description: Dict[str, str], + default_value: str, + ) -> VisibilityTable: + with open(filepath) as f: + items = json.load(f) + + table = cls(level_to_description=level_to_description, default_value=default_value) + for item in items: + record = VisibilityRecord(level=item["level"], description=item["description"]) + record.token = item["token"] + table.set_record_to_table(record) + + return table From bc5c3fef13cc7cec0f1b5279971edc692765672f Mon Sep 17 00:00:00 2001 From: ktro2828 Date: Thu, 28 Nov 2024 08:05:49 +0900 Subject: [PATCH 03/13] test: update tests Signed-off-by: ktro2828 --- perception_dataset/convert.py | 4 ++-- .../fastlabel_to_t4/fastlabel_2d_to_t4_updater.py | 3 ++- perception_dataset/t4_dataset/classes/abstract_class.py | 2 +- tests/t4_dataset/classes/test_abstract_class.py | 4 ++++ 4 files changed, 9 insertions(+), 4 deletions(-) diff --git a/perception_dataset/convert.py b/perception_dataset/convert.py index 670fd409..dc3c7146 100644 --- a/perception_dataset/convert.py +++ b/perception_dataset/convert.py @@ -372,7 +372,7 @@ def main(): elif task == "update_t4_with_fastlabel": from perception_dataset.fastlabel_to_t4.fastlabel_2d_to_t4_updater import ( - FastLabel2dToUpdater, + FastLabel2dToT4Updater, ) input_base = config_dict["conversion"]["input_base"] @@ -381,7 +381,7 @@ def main(): dataset_corresponding = config_dict["conversion"]["dataset_corresponding"] description = config_dict["description"] - converter = FastLabel2dToUpdater( + converter = FastLabel2dToT4Updater( input_base=input_base, output_base=output_base, input_anno_base=input_anno_base, diff --git a/perception_dataset/fastlabel_to_t4/fastlabel_2d_to_t4_updater.py b/perception_dataset/fastlabel_to_t4/fastlabel_2d_to_t4_updater.py index c92601d8..33acd5a6 100644 --- a/perception_dataset/fastlabel_to_t4/fastlabel_2d_to_t4_updater.py +++ b/perception_dataset/fastlabel_to_t4/fastlabel_2d_to_t4_updater.py @@ -15,7 +15,8 @@ logger = configure_logger(modname=__name__) -class FastLabel2dToUpdater(FastLabel2dToT4Converter): +# TODO: add support of 3D annotation format +class FastLabel2dToT4Updater(FastLabel2dToT4Converter): def __init__( self, input_base: str, diff --git a/perception_dataset/t4_dataset/classes/abstract_class.py b/perception_dataset/t4_dataset/classes/abstract_class.py index 6eb762f4..5cd2b44e 100644 --- a/perception_dataset/t4_dataset/classes/abstract_class.py +++ b/perception_dataset/t4_dataset/classes/abstract_class.py @@ -81,4 +81,4 @@ def save_json(self, output_dir: str): @classmethod @abstractmethod def from_json(cls, filepath: str): - raise NotImplementedError + raise NotImplementedError() diff --git a/tests/t4_dataset/classes/test_abstract_class.py b/tests/t4_dataset/classes/test_abstract_class.py index 10d58aa9..8721285b 100644 --- a/tests/t4_dataset/classes/test_abstract_class.py +++ b/tests/t4_dataset/classes/test_abstract_class.py @@ -26,6 +26,10 @@ def __init__(self): def _to_record(self, **kwargs) -> str: return AbstractRecordForTest() + @classmethod + def from_json(cls, filepath: str): + return AbstractTableForTest() + class TestAbstractRecord: def test_token(self): From c77c682ec80a151b5a772ba0ccdfc0f87932c51b Mon Sep 17 00:00:00 2001 From: Shunsuke Miura <37187849+miursh@users.noreply.github.com> Date: Wed, 18 Dec 2024 02:14:40 +0900 Subject: [PATCH 04/13] feat: update FastLabel merger (#178) * update configs Signed-off-by: Shunsuke Miura * add label converter to FastLabel2dToT4Converter Signed-off-by: Shunsuke Miura * fix bug in from_json of surface_ann Signed-off-by: Shunsuke Miura * add automatic_annotation key to sample_annotation/object_ann/surface_ann Signed-off-by: Shunsuke Miura * accelerate default object_mask generation Signed-off-by: Shunsuke Miura * feat: improve T4 updater functionality and directory handling Signed-off-by: Shunsuke Miura * feat: support polygon-to-bbox conversion and improve polygon-to-RLE conversion * update object config Signed-off-by: Shunsuke Miura * feat: update annotation loader * feat: only convert to bbox for objects labels Signed-off-by: Shunsuke Miura * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * fix: handling automatic_annotation entry Signed-off-by: Shunsuke Miura * fix: bug fix in rle_from_points Signed-off-by: Shunsuke Miura * fix: add default value for automatic_annotation Signed-off-by: Shunsuke Miura --------- Signed-off-by: Shunsuke Miura Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- config/label/object.yaml | 38 ++++++++++++++ config/label/surface.yaml | 2 + config/update_t4_with_fastlabel_sample.yaml | 19 +++---- perception_dataset/convert.py | 4 +- .../fastlabel_2d_to_t4_converter.py | 52 +++++++++++++++++-- .../fastlabel_2d_to_t4_updater.py | 38 ++++++++------ .../t4_dataset/annotation_files_generator.py | 28 ++++++---- .../t4_dataset/classes/object_ann.py | 24 ++++++--- .../t4_dataset/classes/sample_annotation.py | 6 +++ .../t4_dataset/classes/surface_ann.py | 8 ++- perception_dataset/utils/label_converter.py | 3 ++ 11 files changed, 173 insertions(+), 49 deletions(-) diff --git a/config/label/object.yaml b/config/label/object.yaml index 8a66d36c..fb15833d 100644 --- a/config/label/object.yaml +++ b/config/label/object.yaml @@ -4,6 +4,7 @@ bus: [bus, BUS, vehicle.bus, vehicle.bus (bendy & rigid)] car: [ car, + cars, CAR, vehicle.car, vehicle.construction, @@ -53,3 +54,40 @@ stroller: [stroller, STROLLER, pedestrian.stroller] police_officer: [police_officer, POLICE_OFFICER, pedestrian.police_officer] wheelchair: [wheelchair, WHEELCHAIR, pedestrian.wheelchair] forklift: [forklift, FORKLIFT] +other_vehicle: [other_vehicle] +other_pedestrian: [other_pedestrian] +train: [train, TRAIN] +cone: [cone] +curb: [curb] +gate: [gate] +guide_post: [guide_post] +construction: [constructions, construction] +traffic_sign: [traffic_sign] +road_debris: [road_debris] +other_obstacle: [other_obstacle] +obstacle_others: [obstacle_others] +laneline_solid_white: [laneline_solid_white] +laneline_dash_white: [laneline_dash_white] +laneline_solid_yellow: [laneline_solid_yellow] +laneline_dash_yellow: [laneline_dash_yellow] +laneline_solid_green: [laneline_solid_green] +laneline_solid_red: [laneline_solid_red] +deceleration_line: [deceleration_line] +dashed_lane_marking: [dashed_lane_markings, dash_lane_markings, dashed_lane_marking, dash_white_merge, dash_white_branch] +stopline: [stopline] +crosswalk: [crosswalk] +marking_character: [marking_character] +marking_arrow: [marking_arrow] +striped_road_marking: [striped_road_markings, striped_road_marking] +parking_lot: [parking_lot] +marking_other: [marking_other] +road: [road] +road_paint_lane_solid_white: [road_paint_lane_solid_white] +road_paint_lane_dash_white: [road_paint_lane_dash_white] +sidewalk: [sidewalk] +building: [building, buildling] # typo +wall/fence: [wall_fence, wall/fence] +pole: [pole] +vegetation/terrain: [vegetation_terrain, vegetation/terrain] +sky: [sky] +traffic_light: [traffic_light] diff --git a/config/label/surface.yaml b/config/label/surface.yaml index f90b37f1..5af24e64 100644 --- a/config/label/surface.yaml +++ b/config/label/surface.yaml @@ -4,8 +4,10 @@ - sidewalk - building - wall_fence +- wall/fence - pole - vegetation_terrain +- vegetation/terrain - sky - road_paint_lane_solid_white - road_paint_lane_dash_white diff --git a/config/update_t4_with_fastlabel_sample.yaml b/config/update_t4_with_fastlabel_sample.yaml index af0a7c2a..ca281c96 100644 --- a/config/update_t4_with_fastlabel_sample.yaml +++ b/config/update_t4_with_fastlabel_sample.yaml @@ -6,17 +6,18 @@ description: partial: "The object is occluded by more than 50% (but not completely)." none: "The object is 90-100% occluded and no points/pixels are visible in the label." camera_index: - CAM_FRONT: 0 - CAM_FRONT_RIGHT: 1 - CAM_BACK_RIGHT: 2 - CAM_BACK: 3 - CAM_BACK_LEFT: 4 - CAM_FRONT_LEFT: 5 + CAM_FRONT_NARROW: 0 + CAM_FRONT_WIDE: 1 + CAM_FRONT_RIGHT: 2 + CAM_BACK_RIGHT: 3 + CAM_BACK_NARROW: 4 + CAM_BACK_WIDE: 5 + CAM_BACK_LEFT: 6 + CAM_FRONT_LEFT: 7 + surface_categories: ./config/label/surface.yaml conversion: + make_t4_dataset_dir: false # If true, the output directory includes t4_dataset directory (such as "scene_dir"/t4_dataset/data|annotation). If false, "scene_dir"/data|annotation. input_base: ./data/input_t4_format # could be non_annotated_t4_format or t4_format_3d_annotated input_anno_base: ./data/fastlabel output_base: ./data/output_t4_format # currently, this only includes the 2D annotations - dataset_corresponding: - # input t4dataset_name: FastLabel json file name - T4DatasetName: FastLabelAnnotationFile diff --git a/perception_dataset/convert.py b/perception_dataset/convert.py index dc3c7146..6c84548c 100644 --- a/perception_dataset/convert.py +++ b/perception_dataset/convert.py @@ -378,16 +378,16 @@ def main(): input_base = config_dict["conversion"]["input_base"] output_base = config_dict["conversion"]["output_base"] input_anno_base = config_dict["conversion"]["input_anno_base"] - dataset_corresponding = config_dict["conversion"]["dataset_corresponding"] description = config_dict["description"] + make_t4_dataset_dir = config_dict["conversion"]["make_t4_dataset_dir"] converter = FastLabel2dToT4Updater( input_base=input_base, output_base=output_base, input_anno_base=input_anno_base, - dataset_corresponding=dataset_corresponding, overwrite_mode=args.overwrite, description=description, + make_t4_dataset_dir=make_t4_dataset_dir, ) logger.info( f"[BEGIN] Updating T4 dataset ({input_base}) with FastLabel {input_anno_base} into T4 data ({output_base})" diff --git a/perception_dataset/fastlabel_to_t4/fastlabel_2d_to_t4_converter.py b/perception_dataset/fastlabel_to_t4/fastlabel_2d_to_t4_converter.py index 2e83be99..3ebbcf8d 100644 --- a/perception_dataset/fastlabel_to_t4/fastlabel_2d_to_t4_converter.py +++ b/perception_dataset/fastlabel_to_t4/fastlabel_2d_to_t4_converter.py @@ -6,10 +6,13 @@ import shutil from typing import Any, Dict, List, Optional, Union +import numpy as np import pycocotools.mask as cocomask +from perception_dataset.constants import LABEL_PATH_ENUM from perception_dataset.deepen.deepen_to_t4_converter import DeepenToT4Converter from perception_dataset.t4_dataset.annotation_files_generator import AnnotationFilesGenerator +from perception_dataset.utils.label_converter import LabelConverter from perception_dataset.utils.logger import configure_logger logger = configure_logger(modname=__name__) @@ -48,6 +51,10 @@ def __init__( self._input_anno_files: List[Path] = [] for f in Path(input_anno_base).rglob("*.json"): self._input_anno_files.append(f) + self._label_converter = LabelConverter( + label_path=LABEL_PATH_ENUM.OBJECT_LABEL, + attribute_path=LABEL_PATH_ENUM.ATTRIBUTE, + ) def convert(self): # Load and format Fastlabel annotations @@ -184,6 +191,7 @@ def _format_fastlabel_annotation(self, annotations: Dict[str, List[Dict[str, Any for a in ann["annotations"]: occlusion_state: str = "occlusion_state.none" visibility: str = "Not available" + instance_id = "" for att in a["attributes"]: if att["key"] == "id": instance_id = att["value"] @@ -197,8 +205,9 @@ def _format_fastlabel_annotation(self, annotations: Dict[str, List[Dict[str, Any att["key"].split("_")[-1] ) break + category_label = self._label_converter.convert_label(a["title"]) label_t4_dict: Dict[str, Any] = { - "category_name": a["title"], + "category_name": category_label, "instance_id": instance_id, "attribute_names": [occlusion_state], "visibility_name": visibility, @@ -217,6 +226,10 @@ def _format_fastlabel_annotation(self, annotations: Dict[str, List[Dict[str, Any "sensor_id": self._camera2idx[camera], } ) + if self._label_converter.is_object_label(category_label): + label_t4_dict["two_d_box"] = _convert_polygon_to_bbox( + a["points"][0][0] + ) fl_annotations[dataset_name][file_id].append(label_t4_dict) return fl_annotations @@ -247,11 +260,40 @@ def _rle_from_points(points: Points2DLike, width: int, height: int) -> Dict[str, Returns: Dict[str, Any]: RLE format mask. """ - flattened = [[coord for p in point for coord in p] for point in points] + final_mask = np.zeros((height, width, 1), dtype=np.uint8) - rle_objects = cocomask.frPyObjects(flattened, height, width) - rle = cocomask.merge(rle_objects) + for polygon in points: + outer_polygon = polygon[0] # outer points + outer_rle = cocomask.frPyObjects([outer_polygon], height, width) + outer_mask = cocomask.decode(outer_rle) + combined_mask = outer_mask + for i in range(1, len(polygon)): + hollow_polygon = polygon[i] # hollowed out points + hollow_rle = cocomask.frPyObjects([hollow_polygon], height, width) + hollow_mask = cocomask.decode(hollow_rle) + combined_mask = combined_mask - hollow_mask + final_mask = np.maximum(final_mask, combined_mask) + # encode RLE + rle = cocomask.encode(np.asfortranarray(np.squeeze(final_mask))) rle["counts"] = base64.b64encode(rle["counts"]).decode("ascii") - return rle + + +def _convert_polygon_to_bbox(polygon: List[int]) -> List[float]: + """Convert polygon points to bounding box. + + Args: + polygon: 2D points, such as `[x1, y1, x2, y2 ....]`. + + Returns: + List[float]: Bounding box in [x1, y1, x2, y2] format. + """ + x_coords = polygon[0::2] + y_coords = polygon[1::2] + + xmin = min(x_coords) + xmax = max(x_coords) + ymin = min(y_coords) + ymax = max(y_coords) + return [xmin, ymin, xmax, ymax] diff --git a/perception_dataset/fastlabel_to_t4/fastlabel_2d_to_t4_updater.py b/perception_dataset/fastlabel_to_t4/fastlabel_2d_to_t4_updater.py index 33acd5a6..71fff24a 100644 --- a/perception_dataset/fastlabel_to_t4/fastlabel_2d_to_t4_updater.py +++ b/perception_dataset/fastlabel_to_t4/fastlabel_2d_to_t4_updater.py @@ -22,26 +22,28 @@ def __init__( input_base: str, output_base: str, input_anno_base: str, - dataset_corresponding: Dict[str, int], overwrite_mode: bool, description: Dict[str, Dict[str, str]], + make_t4_dataset_dir: bool = True, ): super().__init__( input_base, output_base, input_anno_base, - dataset_corresponding, - overwrite_mode, - description, + dataset_corresponding=None, + overwrite_mode=overwrite_mode, + description=description, input_bag_base=None, topic_list=None, ) + self._make_t4_dataset_dir = make_t4_dataset_dir def convert(self) -> None: - anno_jsons_dict = self._load_annotation_jsons() + t4_datasets = sorted([d.name for d in self._input_base.iterdir() if d.is_dir()]) + anno_jsons_dict = self._load_annotation_jsons(t4_datasets) fl_annotations = self._format_fastlabel_annotation(anno_jsons_dict) - for t4dataset_name in self._t4dataset_name_to_merge: + for t4dataset_name in t4_datasets: # Check if input directory exists input_dir = self._input_base / t4dataset_name input_annotation_dir = input_dir / "annotation" @@ -50,7 +52,9 @@ def convert(self) -> None: continue # Check if output directory already exists - output_dir = self._output_base / t4dataset_name / "t4_dataset" + output_dir = self._output_base / t4dataset_name + if self._make_t4_dataset_dir: + output_dir = output_dir / "t4_dataset" if self._input_bag_base is not None: input_bag_dir = Path(self._input_bag_base) / t4dataset_name @@ -74,6 +78,10 @@ def convert(self) -> None: else: raise ValueError("If you want to overwrite files, use --overwrite option.") + if t4dataset_name not in fl_annotations.keys(): + logger.warning(f"No annotation for {t4dataset_name}") + continue + # Start updating annotations annotation_files_updater = AnnotationFilesUpdater(description=self._description) annotation_files_updater.convert_one_scene( @@ -83,15 +91,15 @@ def convert(self) -> None: dataset_name=t4dataset_name, ) - def _load_annotation_jsons(self): + def _load_annotation_jsons(self, t4_datasets: list[str]) -> dict[str, list[dict[str, any]]]: anno_dict = {} for file in self._input_anno_files: - t4_dataset_name = None - for name, ann_filename in self._t4dataset_name_to_merge.items(): - if ann_filename == file.name: - t4_dataset_name = name - - assert t4_dataset_name is not None + t4_dataset_name = file.name.split("_CAM")[0] + if t4_dataset_name not in t4_datasets: + continue with open(file) as f: - anno_dict[t4_dataset_name] = json.load(f) + one_label = json.load(f) + if t4_dataset_name not in anno_dict.keys(): + anno_dict[t4_dataset_name] = [] + anno_dict[t4_dataset_name].extend(one_label) return anno_dict diff --git a/perception_dataset/t4_dataset/annotation_files_generator.py b/perception_dataset/t4_dataset/annotation_files_generator.py index 01665202..ffbaa381 100644 --- a/perception_dataset/t4_dataset/annotation_files_generator.py +++ b/perception_dataset/t4_dataset/annotation_files_generator.py @@ -1,7 +1,7 @@ import base64 from collections import defaultdict import os.path as osp -from typing import Any, Dict, List, Optional +from typing import Any, Dict, List, Optional, Tuple from nptyping import NDArray from nuimages import NuImages @@ -113,6 +113,8 @@ def convert_one_scene( break if has_2d_annotation: + object_mask: NDArray = np.zeros((0, 0), dtype=np.uint8) + prev_wid_hgt: Tuple = (0, 0) # NOTE: num_cameras is always 6, because it is hard coded above. for frame_index_nuim, sample_nuim in enumerate(nuim.sample_data): if ( @@ -126,13 +128,14 @@ def convert_one_scene( {frame_index: sample_nuim["token"]} ) - width: int = sample_nuim["width"] - height: int = sample_nuim["height"] - object_mask: NDArray = np.array( - [[0 for _ in range(height)] for __ in range(width)], dtype=np.uint8 - ) - object_mask = cocomask.encode(np.asfortranarray(object_mask)) - object_mask["counts"] = repr(base64.b64encode(object_mask["counts"]))[2:] + wid_hgt = (sample_nuim["width"], sample_nuim["height"]) + if wid_hgt != prev_wid_hgt: + prev_wid_hgt = wid_hgt + object_mask = np.zeros(wid_hgt, dtype=np.uint8) + object_mask = cocomask.encode(np.asfortranarray(object_mask)) + object_mask["counts"] = base64.b64encode(object_mask["counts"]).decode( + "ascii" + ) mask[cam_idx].update({frame_index: object_mask}) self.convert_annotations( @@ -287,6 +290,7 @@ def _convert_to_t4_format( rotation=anno_three_d_bbox["rotation"], num_lidar_pts=anno["num_lidar_pts"], num_radar_pts=anno["num_radar_pts"], + automatic_annotation=False, ) self._instance_token_to_annotation_token_list[instance_token].append( sample_annotation_token @@ -299,8 +303,10 @@ def _convert_to_t4_format( sensor_id: int = int(anno["sensor_id"]) if frame_index not in frame_index_to_sample_data_token[sensor_id]: continue - anno_two_d_box: List[float] = self._clip_bbox( - anno["two_d_box"], mask[sensor_id][frame_index] + anno_two_d_box: List[float] = ( + self._clip_bbox(anno["two_d_box"], mask[sensor_id][frame_index]) + if "two_d_box" in anno.keys() + else None ) self._object_ann_table.insert_into_table( sample_data_token=frame_index_to_sample_data_token[sensor_id][frame_index], @@ -313,6 +319,7 @@ def _convert_to_t4_format( if "two_d_segmentation" in anno.keys() else mask[sensor_id][frame_index] ), + automatic_annotation=False, ) # Surface Annotation @@ -327,6 +334,7 @@ def _convert_to_t4_format( category_token=category_token, mask=anno["two_d_segmentation"], sample_data_token=frame_index_to_sample_data_token[sensor_id][frame_index], + automatic_annotation=False, ) def _clip_bbox(self, bbox: List[float], mask: Dict[str, Any]) -> List[float]: diff --git a/perception_dataset/t4_dataset/classes/object_ann.py b/perception_dataset/t4_dataset/classes/object_ann.py index c4022947..b1cc7cbb 100644 --- a/perception_dataset/t4_dataset/classes/object_ann.py +++ b/perception_dataset/t4_dataset/classes/object_ann.py @@ -16,10 +16,11 @@ def __init__( attribute_tokens: str, bbox: List[float], mask: Dict[str, any], + automatic_annotation: bool = False, ): super().__init__() - assert len(bbox) == 4 + assert bbox is None or len(bbox) == 4 self._sample_data_token: str = sample_data_token self._instance_token: str = instance_token @@ -27,6 +28,7 @@ def __init__( self._attribute_tokens: List[str] = attribute_tokens self._bbox: List[float] = bbox self._mask: Dict[str, any] = mask + self._automatic_annotation: bool = automatic_annotation def to_dict(self): d = { @@ -35,13 +37,18 @@ def to_dict(self): "instance_token": self._instance_token, "category_token": self._category_token, "attribute_tokens": self._attribute_tokens, - "bbox": [ - self._bbox[0], - self._bbox[1], - self._bbox[2], - self._bbox[3], - ], + "bbox": ( + [ + self._bbox[0], + self._bbox[1], + self._bbox[2], + self._bbox[3], + ] + if self._bbox is not None + else None + ), "mask": self._mask, + "automatic_annotation": self._automatic_annotation, } return d @@ -62,6 +69,7 @@ def _to_record( attribute_tokens: str, bbox: List[float], mask: Dict[str, any], + automatic_annotation: bool = False, ): record = ObjectAnnRecord( sample_data_token=sample_data_token, @@ -70,6 +78,7 @@ def _to_record( attribute_tokens=attribute_tokens, bbox=bbox, mask=mask, + automatic_annotation=automatic_annotation, ) return record @@ -87,6 +96,7 @@ def from_json(cls, filepath: str) -> ObjectAnnTable: attribute_tokens=item["attribute_tokens"], bbox=item["bbox"], mask=item["mask"], + automatic_annotation=item["automatic_annotation"], ) record.token = item["token"] table.set_record_to_table(record) diff --git a/perception_dataset/t4_dataset/classes/sample_annotation.py b/perception_dataset/t4_dataset/classes/sample_annotation.py index 05dc948f..4f9d90e2 100644 --- a/perception_dataset/t4_dataset/classes/sample_annotation.py +++ b/perception_dataset/t4_dataset/classes/sample_annotation.py @@ -21,6 +21,7 @@ def __init__( rotation: Dict[str, float], num_lidar_pts: int, num_radar_pts: int, + automatic_annotation: bool = False, ): super().__init__() @@ -43,6 +44,7 @@ def __init__( self._rotation: Dict[str, float] = rotation self._num_lidar_pts: int = num_lidar_pts self._num_radar_pts: int = num_radar_pts + self._automatic_annotation: bool = automatic_annotation self._next: str = "" self._prev: str = "" @@ -105,6 +107,7 @@ def to_dict(self): ], "num_lidar_pts": self._num_lidar_pts, "num_radar_pts": self._num_radar_pts, + "automatic_annotation": self._automatic_annotation, "next": self._next, "prev": self._prev, } @@ -132,6 +135,7 @@ def _to_record( rotation: Dict[str, float], num_lidar_pts: int, num_radar_pts: int, + automatic_annotation: bool = False, ): record = SampleAnnotationRecord( sample_token=sample_token, @@ -145,6 +149,7 @@ def _to_record( rotation=rotation, num_lidar_pts=num_lidar_pts, num_radar_pts=num_radar_pts, + automatic_annotation=automatic_annotation, ) return record @@ -196,6 +201,7 @@ def from_json(cls, filepath: str) -> SampleAnnotationTable: }, num_lidar_pts=item["num_lidar_pts"], num_radar_pts=item["num_radar_pts"], + automatic_annotation=item["automatic_annotation"], ) record.token = item["token"] table.set_record_to_table(record) diff --git a/perception_dataset/t4_dataset/classes/surface_ann.py b/perception_dataset/t4_dataset/classes/surface_ann.py index 943a4c6e..63ad658a 100644 --- a/perception_dataset/t4_dataset/classes/surface_ann.py +++ b/perception_dataset/t4_dataset/classes/surface_ann.py @@ -13,12 +13,14 @@ def __init__( category_token: str, mask: Dict[str, any], sample_data_token: str, + automatic_annotation: bool = False, ): super().__init__() self._category_token: str = category_token self._mask: Dict[str, any] = mask self._sample_data_token: str = sample_data_token + self._automatic_annotation: bool = automatic_annotation def to_dict(self): d = { @@ -26,6 +28,7 @@ def to_dict(self): "category_token": self._category_token, "mask": self._mask, "sample_data_token": self._sample_data_token, + "automatic_annotation": self._automatic_annotation, } return d @@ -43,11 +46,13 @@ def _to_record( category_token: str, mask: Dict[str, any], sample_data_token: str, + automatic_annotation: bool = False, ): record = SurfaceAnnRecord( category_token=category_token, mask=mask, sample_data_token=sample_data_token, + automatic_annotation=automatic_annotation, ) return record @@ -62,8 +67,9 @@ def from_json(cls, filepath: str) -> SurfaceAnnTable: category_token=item["category_token"], mask=item["mask"], sample_data_token=item["sample_data_token"], + automatic_annotation=item["automatic_annotation"], ) record.token = item["token"] - table.select_record_from_token(record) + table.set_record_to_table(record) return table diff --git a/perception_dataset/utils/label_converter.py b/perception_dataset/utils/label_converter.py index 207f23fc..33a45028 100644 --- a/perception_dataset/utils/label_converter.py +++ b/perception_dataset/utils/label_converter.py @@ -52,6 +52,9 @@ def convert_attribute( return_attribute: str = self.attribute_map[attribute] return return_attribute + def is_object_label(self, label: str) -> bool: + return label in self.label_map + class TrafficLightLabelConverter(BaseConverter): def __init__( From c1ff9eaaf2c9f7b70b0427ddd7bc622af39c00b1 Mon Sep 17 00:00:00 2001 From: Shunsuke Miura <37187849+miursh@users.noreply.github.com> Date: Thu, 19 Dec 2024 09:51:28 +0900 Subject: [PATCH 05/13] feat: enhance surface category handling in AnnotationFilesGenerator and DeepenToT4Converter (#180) * feat: enhance surface category handling in AnnotationFilesGenerator and DeepenToT4Converter Signed-off-by: Shunsuke Miura * delete unnecessary import Signed-off-by: Shunsuke Miura * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --------- Signed-off-by: Shunsuke Miura Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .../deepen/deepen_to_t4_converter.py | 11 ++++++++++- .../fastlabel_2d_to_t4_converter.py | 5 ++++- .../t4_dataset/annotation_files_generator.py | 15 +++++++-------- 3 files changed, 21 insertions(+), 10 deletions(-) diff --git a/perception_dataset/deepen/deepen_to_t4_converter.py b/perception_dataset/deepen/deepen_to_t4_converter.py index 6e9adf28..5290cdd1 100644 --- a/perception_dataset/deepen/deepen_to_t4_converter.py +++ b/perception_dataset/deepen/deepen_to_t4_converter.py @@ -6,6 +6,7 @@ from typing import Any, Dict, List, Optional, Union from nuscenes.nuscenes import NuScenes +import yaml from perception_dataset.abstract_converter import AbstractConverter from perception_dataset.deepen.deepen_annotation import ( @@ -57,6 +58,11 @@ def __init__( self._label_info: Optional[LabelInfo] = label_info self._topic_list_yaml: Union[List, Dict] = topic_list + if description.get("surface_categories"): + with open(description["surface_categories"], "r") as f: + self._surface_categories: List[str] = yaml.safe_load(f) + else: + self._surface_categories = [] def convert(self): camera_index: Dict[str, int] = self._description["camera_index"] @@ -111,7 +117,10 @@ def convert(self): for t4data_name, dataset_id in self._t4data_name_to_deepen_dataset_id.items(): output_dir = osp.join(self._output_base, t4data_name, self._t4_dataset_dir_name) input_dir = osp.join(self._input_base, t4data_name) - annotation_files_generator = AnnotationFilesGenerator(description=self._description) + + annotation_files_generator = AnnotationFilesGenerator( + description=self._description, surface_categories=self._surface_categories + ) annotation_files_generator.convert_one_scene( input_dir=input_dir, output_dir=output_dir, diff --git a/perception_dataset/fastlabel_to_t4/fastlabel_2d_to_t4_converter.py b/perception_dataset/fastlabel_to_t4/fastlabel_2d_to_t4_converter.py index 3ebbcf8d..76b33393 100644 --- a/perception_dataset/fastlabel_to_t4/fastlabel_2d_to_t4_converter.py +++ b/perception_dataset/fastlabel_to_t4/fastlabel_2d_to_t4_converter.py @@ -226,7 +226,10 @@ def _format_fastlabel_annotation(self, annotations: Dict[str, List[Dict[str, Any "sensor_id": self._camera2idx[camera], } ) - if self._label_converter.is_object_label(category_label): + if ( + self._label_converter.is_object_label(category_label) + and category_label not in self._surface_categories + ): label_t4_dict["two_d_box"] = _convert_polygon_to_bbox( a["points"][0][0] ) diff --git a/perception_dataset/t4_dataset/annotation_files_generator.py b/perception_dataset/t4_dataset/annotation_files_generator.py index ffbaa381..524d6a4f 100644 --- a/perception_dataset/t4_dataset/annotation_files_generator.py +++ b/perception_dataset/t4_dataset/annotation_files_generator.py @@ -8,7 +8,6 @@ import numpy as np from nuscenes.nuscenes import NuScenes from pycocotools import mask as cocomask -import yaml from perception_dataset.constants import SENSOR_ENUM from perception_dataset.t4_dataset.classes.abstract_class import AbstractTable @@ -26,7 +25,12 @@ class AnnotationFilesGenerator: - def __init__(self, with_camera: bool = True, description: Dict[str, Dict[str, str]] = {}): + def __init__( + self, + with_camera: bool = True, + description: Dict[str, Dict[str, str]] = {}, + surface_categories: List[str] = [], + ): # TODO(yukke42): remove the hard coded attribute description self._attribute_table = AttributeTable( name_to_description={}, @@ -62,12 +66,7 @@ def __init__(self, with_camera: bool = True, description: Dict[str, Dict[str, st else: self._camera2idx = None self._with_lidar = description.get("with_lidar", True) - - if description.get("surface_categories"): - with open(description["surface_categories"], "r") as f: - self._surface_categories: List[str] = yaml.safe_load(f) - else: - self._surface_categories = [] + self._surface_categories: List[str] = surface_categories def save_tables(self, anno_dir: str): for cls_attr in self.__dict__.values(): From 3872c996b4bc37c5b4f1f53d7e0c79cb8987c9d1 Mon Sep 17 00:00:00 2001 From: Shunsuke Miura <37187849+miursh@users.noreply.github.com> Date: Mon, 30 Dec 2024 10:55:56 +0900 Subject: [PATCH 06/13] refactor: parallelizate fl annotation processing (#182) * feat: enhance surface category handling in AnnotationFilesGenerator and DeepenToT4Converter Signed-off-by: Shunsuke Miura # Conflicts: # perception_dataset/deepen/deepen_to_t4_converter.py # perception_dataset/fastlabel_to_t4/fastlabel_2d_to_t4_converter.py # perception_dataset/t4_dataset/annotation_files_generator.py * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * refactor: parallelizate fl annotation processing Signed-off-by: Shunsuke Miura * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --------- Signed-off-by: Shunsuke Miura Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .../fastlabel_2d_to_t4_converter.py | 140 +++++++++++------- 1 file changed, 85 insertions(+), 55 deletions(-) diff --git a/perception_dataset/fastlabel_to_t4/fastlabel_2d_to_t4_converter.py b/perception_dataset/fastlabel_to_t4/fastlabel_2d_to_t4_converter.py index 76b33393..f969dd02 100644 --- a/perception_dataset/fastlabel_to_t4/fastlabel_2d_to_t4_converter.py +++ b/perception_dataset/fastlabel_to_t4/fastlabel_2d_to_t4_converter.py @@ -1,5 +1,6 @@ import base64 from collections import defaultdict +from concurrent.futures import ProcessPoolExecutor, as_completed import json import os.path as osp from pathlib import Path @@ -175,67 +176,96 @@ def _format_fastlabel_annotation(self, annotations: Dict[str, List[Dict[str, Any fl_annotations: Dict[str, Dict[int, List[Dict[str, Any]]]] = {} for filename, ann_list in sorted(annotations.items()): - dataset_name: str = Path(filename).stem - for ann in ann_list: - filename: str = ann["name"].split("/")[-1] - file_id: int = int(filename.split(".")[0]) - frame_no: int = file_id + 1 - camera: str = ann["name"].split("/")[-2] + dataset_name = Path(filename).stem + # Process annotations for each file in parallel + file_annotations = self._process_annotations_for_file(ann_list) - if dataset_name not in fl_annotations: - fl_annotations[dataset_name] = defaultdict(list) + # Combine results + if dataset_name not in fl_annotations: + fl_annotations[dataset_name] = defaultdict(list) - width: int = ann["width"] - height: int = ann["height"] + for file_id, ann_list in file_annotations.items(): + fl_annotations[dataset_name][file_id].extend(ann_list) - for a in ann["annotations"]: - occlusion_state: str = "occlusion_state.none" - visibility: str = "Not available" - instance_id = "" - for att in a["attributes"]: - if att["key"] == "id": - instance_id = att["value"] - if "occlusion_state" in att["key"]: - for v in att["value"]: - if frame_no in range(v[0], v[1]): - occlusion_state = ( - "occlusion_state." + att["key"].split("_")[-1] - ) - visibility = self._convert_occlusion_to_visibility( - att["key"].split("_")[-1] - ) - break - category_label = self._label_converter.convert_label(a["title"]) - label_t4_dict: Dict[str, Any] = { - "category_name": category_label, - "instance_id": instance_id, - "attribute_names": [occlusion_state], - "visibility_name": visibility, - } - if a["type"] == "bbox": - label_t4_dict.update( - { - "two_d_box": a["points"], - "sensor_id": self._camera2idx[camera], - } + return fl_annotations + + def process_annotation(self, a, frame_no, camera, width, height): + occlusion_state = "occlusion_state.none" + visibility = "Not available" + instance_id = "" + + # process attributes + for att in a["attributes"]: + if att["key"] == "id": + instance_id = att["value"] + if "occlusion_state" in att["key"]: + for v in att["value"]: + if frame_no in range(v[0], v[1]): + occlusion_state = "occlusion_state." + att["key"].split("_")[-1] + visibility = self._convert_occlusion_to_visibility( + att["key"].split("_")[-1] ) - elif a["type"] == "segmentation": - label_t4_dict.update( - { - "two_d_segmentation": _rle_from_points(a["points"], width, height), - "sensor_id": self._camera2idx[camera], - } + break + + # category and label generation + category_label = self._label_converter.convert_label(a["title"]) + label_t4_dict = { + "category_name": category_label, + "instance_id": instance_id, + "attribute_names": [occlusion_state], + "visibility_name": visibility, + } + + # bbox or segmentation type processing + if a["type"] == "bbox": + label_t4_dict.update( + { + "two_d_box": a["points"], + "sensor_id": self._camera2idx[camera], + } + ) + elif a["type"] == "segmentation": + label_t4_dict.update( + { + "two_d_segmentation": _rle_from_points(a["points"], width, height), + "sensor_id": self._camera2idx[camera], + } + ) + if ( + self._label_converter.is_object_label(category_label) + and category_label not in self._surface_categories + ): + label_t4_dict["two_d_box"] = _convert_polygon_to_bbox(a["points"][0][0]) + print(f"Converted polygon to bbox for {category_label}") + + return label_t4_dict + + def _process_annotations_for_file(self, ann_list): + file_annotations = defaultdict(list) + + # parallelize for each annotation in the file + with ProcessPoolExecutor() as executor: + futures = [] + for ann in ann_list: + filename = ann["name"].split("/")[-1] + file_id = int(filename.split(".")[0]) + frame_no = file_id + 1 + camera = ann["name"].split("/")[-2] + width = ann["width"] + height = ann["height"] + + for a in ann["annotations"]: + futures.append( + executor.submit( + self.process_annotation, a, frame_no, camera, width, height ) - if ( - self._label_converter.is_object_label(category_label) - and category_label not in self._surface_categories - ): - label_t4_dict["two_d_box"] = _convert_polygon_to_bbox( - a["points"][0][0] - ) - fl_annotations[dataset_name][file_id].append(label_t4_dict) + ) - return fl_annotations + for future in as_completed(futures): + label_t4_dict = future.result() + file_annotations[file_id].append(label_t4_dict) + + return file_annotations def _rle_from_points(points: Points2DLike, width: int, height: int) -> Dict[str, Any]: From 53e0a3e67281fe2600d7dd257e9ed84b66b84c19 Mon Sep 17 00:00:00 2001 From: Shunsuke Miura Date: Mon, 30 Dec 2024 12:47:16 +0900 Subject: [PATCH 07/13] remove duplicate key in object.yaml Signed-off-by: Shunsuke Miura --- config/label/object.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/config/label/object.yaml b/config/label/object.yaml index 68bdb567..85b9146c 100644 --- a/config/label/object.yaml +++ b/config/label/object.yaml @@ -56,8 +56,6 @@ stroller: [stroller, STROLLER, pedestrian.stroller] police_officer: [police_officer, POLICE_OFFICER, pedestrian.police_officer] wheelchair: [wheelchair, WHEELCHAIR, pedestrian.wheelchair] forklift: [forklift, FORKLIFT] -other_vehicle: [other_vehicle] -other_pedestrian: [other_pedestrian] train: [train, TRAIN] cone: [cone] curb: [curb] From c241d936898b96b819401118c681d9ec7a159a6e Mon Sep 17 00:00:00 2001 From: Shunsuke Miura <37187849+miursh@users.noreply.github.com> Date: Mon, 30 Dec 2024 13:00:35 +0900 Subject: [PATCH 08/13] feat(fastalbel_to_t4dataset): fix keyframe handling (#187) * merge _load_annotation_jsons function to one Signed-off-by: Shunsuke Miura * add default value for automatic_annotation Signed-off-by: Shunsuke Miura * change to non-keyframe for no-annotation frame Signed-off-by: Shunsuke Miura * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * remove unused import Signed-off-by: Shunsuke Miura --------- Signed-off-by: Shunsuke Miura Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .../deepen/deepen_to_t4_converter.py | 4 ++- .../fastlabel_2d_to_t4_updater.py | 29 ++++++------------- .../t4_dataset/annotation_files_generator.py | 11 ++++--- .../t4_dataset/annotation_files_updater.py | 18 ++++++++++-- .../t4_dataset/classes/object_ann.py | 2 +- .../t4_dataset/classes/sample_annotation.py | 2 +- .../t4_dataset/classes/surface_ann.py | 2 +- .../keyframe_consistency_resolver.py | 0 8 files changed, 35 insertions(+), 33 deletions(-) rename perception_dataset/t4_dataset/{ => resolver}/keyframe_consistency_resolver.py (100%) diff --git a/perception_dataset/deepen/deepen_to_t4_converter.py b/perception_dataset/deepen/deepen_to_t4_converter.py index 5290cdd1..328e660c 100644 --- a/perception_dataset/deepen/deepen_to_t4_converter.py +++ b/perception_dataset/deepen/deepen_to_t4_converter.py @@ -21,7 +21,9 @@ ) from perception_dataset.rosbag2.rosbag2_converter import Rosbag2Converter from perception_dataset.t4_dataset.annotation_files_generator import AnnotationFilesGenerator -from perception_dataset.t4_dataset.keyframe_consistency_resolver import KeyFrameConsistencyResolver +from perception_dataset.t4_dataset.resolver.keyframe_consistency_resolver import ( + KeyFrameConsistencyResolver, +) from perception_dataset.utils.logger import configure_logger import perception_dataset.utils.misc as misc_utils diff --git a/perception_dataset/fastlabel_to_t4/fastlabel_2d_to_t4_updater.py b/perception_dataset/fastlabel_to_t4/fastlabel_2d_to_t4_updater.py index 71fff24a..7a099b49 100644 --- a/perception_dataset/fastlabel_to_t4/fastlabel_2d_to_t4_updater.py +++ b/perception_dataset/fastlabel_to_t4/fastlabel_2d_to_t4_updater.py @@ -1,6 +1,5 @@ from __future__ import annotations -import json import os.path as osp from pathlib import Path import shutil @@ -40,10 +39,14 @@ def __init__( def convert(self) -> None: t4_datasets = sorted([d.name for d in self._input_base.iterdir() if d.is_dir()]) - anno_jsons_dict = self._load_annotation_jsons(t4_datasets) + anno_jsons_dict = self._load_annotation_jsons(t4_datasets, "_CAM") fl_annotations = self._format_fastlabel_annotation(anno_jsons_dict) for t4dataset_name in t4_datasets: + # Check if annotation exists + if t4dataset_name not in fl_annotations.keys(): + continue + # Check if input directory exists input_dir = self._input_base / t4dataset_name input_annotation_dir = input_dir / "annotation" @@ -78,28 +81,14 @@ def convert(self) -> None: else: raise ValueError("If you want to overwrite files, use --overwrite option.") - if t4dataset_name not in fl_annotations.keys(): - logger.warning(f"No annotation for {t4dataset_name}") - continue - # Start updating annotations - annotation_files_updater = AnnotationFilesUpdater(description=self._description) + annotation_files_updater = AnnotationFilesUpdater( + description=self._description, surface_categories=self._surface_categories + ) annotation_files_updater.convert_one_scene( input_dir=input_dir, output_dir=output_dir, scene_anno_dict=fl_annotations[t4dataset_name], dataset_name=t4dataset_name, ) - - def _load_annotation_jsons(self, t4_datasets: list[str]) -> dict[str, list[dict[str, any]]]: - anno_dict = {} - for file in self._input_anno_files: - t4_dataset_name = file.name.split("_CAM")[0] - if t4_dataset_name not in t4_datasets: - continue - with open(file) as f: - one_label = json.load(f) - if t4_dataset_name not in anno_dict.keys(): - anno_dict[t4_dataset_name] = [] - anno_dict[t4_dataset_name].extend(one_label) - return anno_dict + logger.info(f"Finished updating annotations for {t4dataset_name}") diff --git a/perception_dataset/t4_dataset/annotation_files_generator.py b/perception_dataset/t4_dataset/annotation_files_generator.py index 524d6a4f..4d6ad554 100644 --- a/perception_dataset/t4_dataset/annotation_files_generator.py +++ b/perception_dataset/t4_dataset/annotation_files_generator.py @@ -87,10 +87,11 @@ def convert_one_scene( nusc = NuScenes(version="annotation", dataroot=input_dir, verbose=False) frame_index_to_sample_token: Dict[int, str] = {} - for frame_index, sample in enumerate(nusc.sample): - frame_index_to_sample_token[frame_index] = sample["token"] + for sample_data in nusc.sample_data: + frame_index = int((sample_data["filename"].split("/")[2]).split(".")[0]) + frame_index_to_sample_token[frame_index] = sample_data["sample_token"] try: - if "LIDAR_TOP" in sample["data"]: + if "LIDAR_TOP" in nusc.sample[0]["data"]: lidar_sensor_channel = SENSOR_ENUM.LIDAR_TOP.value["channel"] else: lidar_sensor_channel = SENSOR_ENUM.LIDAR_CONCAT.value["channel"] @@ -116,9 +117,7 @@ def convert_one_scene( prev_wid_hgt: Tuple = (0, 0) # NOTE: num_cameras is always 6, because it is hard coded above. for frame_index_nuim, sample_nuim in enumerate(nuim.sample_data): - if ( - sample_nuim["fileformat"] == "png" or sample_nuim["fileformat"] == "jpg" - ) and sample_nuim["is_key_frame"]: + if sample_nuim["fileformat"] == "png" or sample_nuim["fileformat"] == "jpg": cam = sample_nuim["filename"].split("/")[1] cam_idx = self._camera2idx[cam] diff --git a/perception_dataset/t4_dataset/annotation_files_updater.py b/perception_dataset/t4_dataset/annotation_files_updater.py index 62d7f4f7..2753c968 100644 --- a/perception_dataset/t4_dataset/annotation_files_updater.py +++ b/perception_dataset/t4_dataset/annotation_files_updater.py @@ -1,6 +1,7 @@ import json import os.path as osp -from typing import Any, Dict +from pathlib import Path +from typing import Any, Dict, List from perception_dataset.t4_dataset.annotation_files_generator import AnnotationFilesGenerator from perception_dataset.t4_dataset.classes import ( @@ -12,6 +13,9 @@ SurfaceAnnTable, VisibilityTable, ) +from perception_dataset.t4_dataset.resolver.keyframe_consistency_resolver import ( + KeyFrameConsistencyResolver, +) def _load_json(filepath: str) -> Any: @@ -21,8 +25,13 @@ def _load_json(filepath: str) -> Any: class AnnotationFilesUpdater(AnnotationFilesGenerator): - def __init__(self, with_camera: bool = True, description: Dict[str, Dict[str, str]] = ...): - super().__init__(with_camera, description) + def __init__( + self, + with_camera: bool = True, + description: Dict[str, Dict[str, str]] = ..., + surface_categories: List[str] = [], + ): + super().__init__(with_camera, description, surface_categories) self.description = description def convert_one_scene( @@ -46,6 +55,9 @@ def convert_one_scene( dataset_name=dataset_name, ) + modifier = KeyFrameConsistencyResolver() + modifier.inspect_and_fix_t4_segment(Path(output_dir)) + def _init_table_from_json(self, anno_dir: str) -> None: self._attribute_table = AttributeTable.from_json( filepath=osp.join(anno_dir, AttributeTable.FILENAME), diff --git a/perception_dataset/t4_dataset/classes/object_ann.py b/perception_dataset/t4_dataset/classes/object_ann.py index 677065f7..ea4cbed8 100644 --- a/perception_dataset/t4_dataset/classes/object_ann.py +++ b/perception_dataset/t4_dataset/classes/object_ann.py @@ -96,7 +96,7 @@ def from_json(cls, filepath: str) -> ObjectAnnTable: attribute_tokens=item["attribute_tokens"], bbox=item["bbox"], mask=item["mask"], - automatic_annotation=item["automatic_annotation"], + automatic_annotation=item.get("automatic_annotation", False), ) record.token = item["token"] table.set_record_to_table(record) diff --git a/perception_dataset/t4_dataset/classes/sample_annotation.py b/perception_dataset/t4_dataset/classes/sample_annotation.py index 4f9d90e2..0e7465b3 100644 --- a/perception_dataset/t4_dataset/classes/sample_annotation.py +++ b/perception_dataset/t4_dataset/classes/sample_annotation.py @@ -201,7 +201,7 @@ def from_json(cls, filepath: str) -> SampleAnnotationTable: }, num_lidar_pts=item["num_lidar_pts"], num_radar_pts=item["num_radar_pts"], - automatic_annotation=item["automatic_annotation"], + automatic_annotation=item.get("automatic_annotation", False), ) record.token = item["token"] table.set_record_to_table(record) diff --git a/perception_dataset/t4_dataset/classes/surface_ann.py b/perception_dataset/t4_dataset/classes/surface_ann.py index cf25188b..9e0fe690 100644 --- a/perception_dataset/t4_dataset/classes/surface_ann.py +++ b/perception_dataset/t4_dataset/classes/surface_ann.py @@ -67,7 +67,7 @@ def from_json(cls, filepath: str) -> SurfaceAnnTable: category_token=item["category_token"], mask=item["mask"], sample_data_token=item["sample_data_token"], - automatic_annotation=item["automatic_annotation"], + automatic_annotation=item.get("automatic_annotation", False), ) record.token = item["token"] table.set_record_to_table(record) diff --git a/perception_dataset/t4_dataset/keyframe_consistency_resolver.py b/perception_dataset/t4_dataset/resolver/keyframe_consistency_resolver.py similarity index 100% rename from perception_dataset/t4_dataset/keyframe_consistency_resolver.py rename to perception_dataset/t4_dataset/resolver/keyframe_consistency_resolver.py From 1335cb40d2807a4d61ff48d9b19e2b06a562799c Mon Sep 17 00:00:00 2001 From: Shunsuke Miura Date: Mon, 30 Dec 2024 14:01:44 +0900 Subject: [PATCH 09/13] change import path Signed-off-by: Shunsuke Miura --- .../fastlabel_to_t4/fastlabel_to_t4_converter.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/perception_dataset/fastlabel_to_t4/fastlabel_to_t4_converter.py b/perception_dataset/fastlabel_to_t4/fastlabel_to_t4_converter.py index 690e6b07..c6907e8f 100644 --- a/perception_dataset/fastlabel_to_t4/fastlabel_to_t4_converter.py +++ b/perception_dataset/fastlabel_to_t4/fastlabel_to_t4_converter.py @@ -9,7 +9,9 @@ FastLabel2dToT4Converter, ) from perception_dataset.t4_dataset.annotation_files_generator import AnnotationFilesGenerator -from perception_dataset.t4_dataset.keyframe_consistency_resolver import KeyFrameConsistencyResolver +from perception_dataset.t4_dataset.resolver.keyframe_consistency_resolver import ( + KeyFrameConsistencyResolver, +) from perception_dataset.utils.label_converter import LabelConverter from perception_dataset.utils.logger import configure_logger from perception_dataset.utils.transform import rotation_to_quaternion From 37366d3d1fa193761a08949ac1fda66f0f2e2064 Mon Sep 17 00:00:00 2001 From: Shunsuke Miura Date: Wed, 1 Jan 2025 19:53:03 +0900 Subject: [PATCH 10/13] Revert "refactor: parallelizate fl annotation processing (#182)" This reverts commit 3872c996b4bc37c5b4f1f53d7e0c79cb8987c9d1. --- .../fastlabel_2d_to_t4_converter.py | 140 +++++++----------- 1 file changed, 55 insertions(+), 85 deletions(-) diff --git a/perception_dataset/fastlabel_to_t4/fastlabel_2d_to_t4_converter.py b/perception_dataset/fastlabel_to_t4/fastlabel_2d_to_t4_converter.py index 1bd88fc0..0c98425a 100644 --- a/perception_dataset/fastlabel_to_t4/fastlabel_2d_to_t4_converter.py +++ b/perception_dataset/fastlabel_to_t4/fastlabel_2d_to_t4_converter.py @@ -1,6 +1,5 @@ import base64 from collections import defaultdict -from concurrent.futures import ProcessPoolExecutor, as_completed import json import os.path as osp from pathlib import Path @@ -191,96 +190,67 @@ def _format_fastlabel_annotation(self, annotations: Dict[str, List[Dict[str, Any fl_annotations: Dict[str, Dict[int, List[Dict[str, Any]]]] = {} for filename, ann_list in sorted(annotations.items()): - dataset_name = Path(filename).stem - # Process annotations for each file in parallel - file_annotations = self._process_annotations_for_file(ann_list) - - # Combine results - if dataset_name not in fl_annotations: - fl_annotations[dataset_name] = defaultdict(list) - - for file_id, ann_list in file_annotations.items(): - fl_annotations[dataset_name][file_id].extend(ann_list) - - return fl_annotations - - def process_annotation(self, a, frame_no, camera, width, height): - occlusion_state = "occlusion_state.none" - visibility = "Not available" - instance_id = "" - - # process attributes - for att in a["attributes"]: - if att["key"] == "id": - instance_id = att["value"] - if "occlusion_state" in att["key"]: - for v in att["value"]: - if frame_no in range(v[0], v[1]): - occlusion_state = "occlusion_state." + att["key"].split("_")[-1] - visibility = self._convert_occlusion_to_visibility( - att["key"].split("_")[-1] - ) - break - - # category and label generation - category_label = self._label_converter.convert_label(a["title"]) - label_t4_dict = { - "category_name": category_label, - "instance_id": instance_id, - "attribute_names": [occlusion_state], - "visibility_name": visibility, - } - - # bbox or segmentation type processing - if a["type"] == "bbox": - label_t4_dict.update( - { - "two_d_box": a["points"], - "sensor_id": self._camera2idx[camera], - } - ) - elif a["type"] == "segmentation": - label_t4_dict.update( - { - "two_d_segmentation": _rle_from_points(a["points"], width, height), - "sensor_id": self._camera2idx[camera], - } - ) - if ( - self._label_converter.is_object_label(category_label) - and category_label not in self._surface_categories - ): - label_t4_dict["two_d_box"] = _convert_polygon_to_bbox(a["points"][0][0]) - print(f"Converted polygon to bbox for {category_label}") - - return label_t4_dict + dataset_name: str = Path(filename).stem + for ann in ann_list: + filename: str = ann["name"].split("/")[-1] + file_id: int = int(filename.split(".")[0]) + frame_no: int = file_id + 1 + camera: str = ann["name"].split("/")[-2] - def _process_annotations_for_file(self, ann_list): - file_annotations = defaultdict(list) + if dataset_name not in fl_annotations: + fl_annotations[dataset_name] = defaultdict(list) - # parallelize for each annotation in the file - with ProcessPoolExecutor() as executor: - futures = [] - for ann in ann_list: - filename = ann["name"].split("/")[-1] - file_id = int(filename.split(".")[0]) - frame_no = file_id + 1 - camera = ann["name"].split("/")[-2] - width = ann["width"] - height = ann["height"] + width: int = ann["width"] + height: int = ann["height"] for a in ann["annotations"]: - futures.append( - executor.submit( - self.process_annotation, a, frame_no, camera, width, height + occlusion_state: str = "occlusion_state.none" + visibility: str = "Not available" + instance_id = "" + for att in a["attributes"]: + if att["key"] == "id": + instance_id = att["value"] + if "occlusion_state" in att["key"]: + for v in att["value"]: + if frame_no in range(v[0], v[1]): + occlusion_state = ( + "occlusion_state." + att["key"].split("_")[-1] + ) + visibility = self._convert_occlusion_to_visibility( + att["key"].split("_")[-1] + ) + break + category_label = self._label_converter.convert_label(a["title"]) + label_t4_dict: Dict[str, Any] = { + "category_name": category_label, + "instance_id": instance_id, + "attribute_names": [occlusion_state], + "visibility_name": visibility, + } + if a["type"] == "bbox": + label_t4_dict.update( + { + "two_d_box": a["points"], + "sensor_id": self._camera2idx[camera], + } ) - ) - - for future in as_completed(futures): - label_t4_dict = future.result() - file_annotations[file_id].append(label_t4_dict) + elif a["type"] == "segmentation": + label_t4_dict.update( + { + "two_d_segmentation": _rle_from_points(a["points"], width, height), + "sensor_id": self._camera2idx[camera], + } + ) + if ( + self._label_converter.is_object_label(category_label) + and category_label not in self._surface_categories + ): + label_t4_dict["two_d_box"] = _convert_polygon_to_bbox( + a["points"][0][0] + ) + fl_annotations[dataset_name][file_id].append(label_t4_dict) - return file_annotations + return fl_annotations def _rle_from_points(points: Points2DLike, width: int, height: int) -> Dict[str, Any]: From 0d2d409913947a674da711d9d7a5bf5993f99c5b Mon Sep 17 00:00:00 2001 From: Shunsuke Miura Date: Thu, 2 Jan 2025 02:06:37 +0900 Subject: [PATCH 11/13] refactor: FastLabel annotation processing for parallel execution Signed-off-by: Shunsuke Miura --- .../fastlabel_2d_to_t4_converter.py | 139 ++++++++++-------- 1 file changed, 79 insertions(+), 60 deletions(-) diff --git a/perception_dataset/fastlabel_to_t4/fastlabel_2d_to_t4_converter.py b/perception_dataset/fastlabel_to_t4/fastlabel_2d_to_t4_converter.py index 0c98425a..b6957aa6 100644 --- a/perception_dataset/fastlabel_to_t4/fastlabel_2d_to_t4_converter.py +++ b/perception_dataset/fastlabel_to_t4/fastlabel_2d_to_t4_converter.py @@ -1,10 +1,12 @@ import base64 +from concurrent.futures import ProcessPoolExecutor, as_completed from collections import defaultdict import json import os.path as osp from pathlib import Path import shutil from typing import Any, Dict, List, Optional, Union +from tqdm import tqdm import numpy as np import pycocotools.mask as cocomask @@ -104,6 +106,7 @@ def _load_annotation_jsons( self, t4_datasets: Optional[List[str]] = None, delimiter: Optional[str] = None ) -> Dict[str, List[dict[str, Any]]]: """Load annotations from all JSON files in the input directory and return as a dictionary.""" + logger.info("Loading annotation JSON files") anno_dict = defaultdict(list) if t4_datasets is None: for file in self._input_anno_files: @@ -111,7 +114,9 @@ def _load_annotation_jsons( anno_dict[file.name] = json.load(f) return anno_dict + pbar = tqdm(total=len(self._input_anno_files), desc="Loading annotation files") for file in self._input_anno_files: + pbar.update(1) t4_dataset_name = file.name.split(delimiter)[0] for dataset in t4_datasets: if dataset in t4_dataset_name: @@ -121,8 +126,67 @@ def _load_annotation_jsons( with open(file) as f: one_label = json.load(f) anno_dict[dataset].extend(one_label) + pbar.close() return anno_dict + def _process_annotation(self, dataset_name, annotation): + filename: str = annotation["name"].split("/")[-1] + file_id: int = int(filename.split(".")[0]) + frame_no: int = file_id + 1 + camera = annotation["name"].split("/")[-2] + + width = annotation["width"] + height = annotation["height"] + + labels = [] + for a in annotation["annotations"]: + occlusion_state = "occlusion_state.none" + visibility = "Not available" + instance_id = "" + + for att in a["attributes"]: + if att["key"] == "id": + instance_id = att["value"] + if "occlusion_state" in att["key"]: + for v in att["value"]: + if frame_no in range(v[0], v[1]): + occlusion_state = "occlusion_state." + att["key"].split("_")[-1] + visibility = self._convert_occlusion_to_visibility( + att["key"].split("_")[-1] + ) + break + + category_label = self._label_converter.convert_label(a["title"]) + label_t4_dict = { + "category_name": category_label, + "instance_id": instance_id, + "attribute_names": [occlusion_state], + "visibility_name": visibility, + } + + if a["type"] == "bbox": + label_t4_dict.update( + { + "two_d_box": a["points"], + "sensor_id": self._camera2idx[camera], + } + ) + elif a["type"] == "segmentation": + label_t4_dict.update( + { + "two_d_segmentation": _rle_from_points(a["points"], width, height), + "sensor_id": self._camera2idx[camera], + } + ) + if ( + self._label_converter.is_object_label(category_label) + and category_label not in self._surface_categories + ): + label_t4_dict["two_d_box"] = _convert_polygon_to_bbox(a["points"][0][0]) + labels.append(label_t4_dict) + + return dataset_name, file_id, labels + def _format_fastlabel_annotation(self, annotations: Dict[str, List[Dict[str, Any]]]): """ e.g. of input_anno_file(fastlabel): @@ -189,66 +253,21 @@ def _format_fastlabel_annotation(self, annotations: Dict[str, List[Dict[str, Any """ fl_annotations: Dict[str, Dict[int, List[Dict[str, Any]]]] = {} - for filename, ann_list in sorted(annotations.items()): - dataset_name: str = Path(filename).stem - for ann in ann_list: - filename: str = ann["name"].split("/")[-1] - file_id: int = int(filename.split(".")[0]) - frame_no: int = file_id + 1 - camera: str = ann["name"].split("/")[-2] - - if dataset_name not in fl_annotations: - fl_annotations[dataset_name] = defaultdict(list) - - width: int = ann["width"] - height: int = ann["height"] - - for a in ann["annotations"]: - occlusion_state: str = "occlusion_state.none" - visibility: str = "Not available" - instance_id = "" - for att in a["attributes"]: - if att["key"] == "id": - instance_id = att["value"] - if "occlusion_state" in att["key"]: - for v in att["value"]: - if frame_no in range(v[0], v[1]): - occlusion_state = ( - "occlusion_state." + att["key"].split("_")[-1] - ) - visibility = self._convert_occlusion_to_visibility( - att["key"].split("_")[-1] - ) - break - category_label = self._label_converter.convert_label(a["title"]) - label_t4_dict: Dict[str, Any] = { - "category_name": category_label, - "instance_id": instance_id, - "attribute_names": [occlusion_state], - "visibility_name": visibility, - } - if a["type"] == "bbox": - label_t4_dict.update( - { - "two_d_box": a["points"], - "sensor_id": self._camera2idx[camera], - } - ) - elif a["type"] == "segmentation": - label_t4_dict.update( - { - "two_d_segmentation": _rle_from_points(a["points"], width, height), - "sensor_id": self._camera2idx[camera], - } - ) - if ( - self._label_converter.is_object_label(category_label) - and category_label not in self._surface_categories - ): - label_t4_dict["two_d_box"] = _convert_polygon_to_bbox( - a["points"][0][0] - ) - fl_annotations[dataset_name][file_id].append(label_t4_dict) + with ProcessPoolExecutor() as executor: + futures = [] + for filename, ann_list in sorted(annotations.items()): + dataset_name: str = Path(filename).stem + for ann in ann_list: + futures.append( + executor.submit(self._process_annotation, dataset_name, ann) + ) + + for future in tqdm(as_completed(futures), total=len(futures), desc=f"Processing {dataset_name} labels"): + dataset_name, file_id, labels = future.result() + if dataset_name not in fl_annotations: + fl_annotations[dataset_name] = defaultdict(list) + for label_t4_dict in labels: + fl_annotations[dataset_name][file_id].append(label_t4_dict) return fl_annotations From 8dd2d91a46dffabe7b934d43a26a7da72e2d5556 Mon Sep 17 00:00:00 2001 From: Shunsuke Miura Date: Thu, 2 Jan 2025 02:07:30 +0900 Subject: [PATCH 12/13] supress no-3D annotation warning Signed-off-by: Shunsuke Miura --- perception_dataset/utils/calculate_num_points.py | 1 - 1 file changed, 1 deletion(-) diff --git a/perception_dataset/utils/calculate_num_points.py b/perception_dataset/utils/calculate_num_points.py index ff753d68..10fb27ac 100644 --- a/perception_dataset/utils/calculate_num_points.py +++ b/perception_dataset/utils/calculate_num_points.py @@ -60,7 +60,6 @@ def calculate_num_points( # connect next/prev tokens for instance in nusc.instance: if instance["nbr_annotations"] == 0: - logger.warning(f"instance:{instance['token']} has no 3D annotation") continue try: prev_sample_data: str = annotation_table._token_to_record[ From 36f3525fe05fae963c7148bd1fca59517b8668f7 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 1 Jan 2025 17:24:33 +0000 Subject: [PATCH 13/13] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../fastlabel_2d_to_t4_converter.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/perception_dataset/fastlabel_to_t4/fastlabel_2d_to_t4_converter.py b/perception_dataset/fastlabel_to_t4/fastlabel_2d_to_t4_converter.py index b6957aa6..78ad5494 100644 --- a/perception_dataset/fastlabel_to_t4/fastlabel_2d_to_t4_converter.py +++ b/perception_dataset/fastlabel_to_t4/fastlabel_2d_to_t4_converter.py @@ -1,15 +1,15 @@ import base64 -from concurrent.futures import ProcessPoolExecutor, as_completed from collections import defaultdict +from concurrent.futures import ProcessPoolExecutor, as_completed import json import os.path as osp from pathlib import Path import shutil from typing import Any, Dict, List, Optional, Union -from tqdm import tqdm import numpy as np import pycocotools.mask as cocomask +from tqdm import tqdm from perception_dataset.constants import LABEL_PATH_ENUM from perception_dataset.deepen.deepen_to_t4_converter import DeepenToT4Converter @@ -258,11 +258,13 @@ def _format_fastlabel_annotation(self, annotations: Dict[str, List[Dict[str, Any for filename, ann_list in sorted(annotations.items()): dataset_name: str = Path(filename).stem for ann in ann_list: - futures.append( - executor.submit(self._process_annotation, dataset_name, ann) - ) + futures.append(executor.submit(self._process_annotation, dataset_name, ann)) - for future in tqdm(as_completed(futures), total=len(futures), desc=f"Processing {dataset_name} labels"): + for future in tqdm( + as_completed(futures), + total=len(futures), + desc=f"Processing {dataset_name} labels", + ): dataset_name, file_id, labels = future.result() if dataset_name not in fl_annotations: fl_annotations[dataset_name] = defaultdict(list)