From 8d9061c05f5f95170a5020ca1658ca59db659c8d Mon Sep 17 00:00:00 2001 From: Kotaro Uetake <60615504+ktro2828@users.noreply.github.com> Date: Mon, 9 Dec 2024 15:47:35 +0900 Subject: [PATCH] docs: update documents of schema (#67) * docs: update documents of scheme Signed-off-by: ktro2828 * chore: add warning Signed-off-by: ktro2828 --------- Signed-off-by: ktro2828 --- docs/apis/schema.md | 54 ---------------- docs/apis/schema/index.md | 7 +++ docs/apis/schema/name.md | 21 +++++++ docs/apis/schema/other.md | 65 ++++++++++++++++++++ docs/apis/schema/registry.md | 12 ++++ docs/apis/schema/serialize.md | 7 +++ docs/apis/schema/table.md | 11 ++++ mkdocs.yaml | 8 ++- t4_devkit/schema/tables/log.py | 1 + t4_devkit/schema/tables/object_ann.py | 1 + t4_devkit/schema/tables/sample.py | 1 + t4_devkit/schema/tables/sample_annotation.py | 1 + t4_devkit/schema/tables/sample_data.py | 1 + t4_devkit/schema/tables/sensor.py | 1 + t4_devkit/schema/tables/surface_ann.py | 4 ++ t4_devkit/tier4.py | 2 + 16 files changed, 142 insertions(+), 55 deletions(-) delete mode 100644 docs/apis/schema.md create mode 100644 docs/apis/schema/index.md create mode 100644 docs/apis/schema/name.md create mode 100644 docs/apis/schema/other.md create mode 100644 docs/apis/schema/registry.md create mode 100644 docs/apis/schema/serialize.md create mode 100644 docs/apis/schema/table.md diff --git a/docs/apis/schema.md b/docs/apis/schema.md deleted file mode 100644 index 2428502..0000000 --- a/docs/apis/schema.md +++ /dev/null @@ -1,54 +0,0 @@ -# `schema` - - -## Schema names - ---- - -::: t4_devkit.schema.name - options: - show_docstring_attributes: true - show_bases: false - -## Schema tables - ---- - -::: t4_devkit.schema.tables - options: - filters: ["!SchemaBase", "!FileFormat", "!SensorModality", "!VisibilityLevel", "!RLEMask"] - show_root_toc_entry: false - merge_init_into_class: false - show_signature_annotations: false - show_docstring_attributes: true - -### Other items constructing schema table - ---- - -::: t4_devkit.schema.tables - options: - members: ["FileFormat", "SensorModality", "VisibilityLevel", "RLEMask"] - show_root_toc_entry: false - merge_init_into_class: false - show_signature_annotations: false - show_docstring_attributes: true - -## Schema registry - ---- - -::: t4_devkit.schema.builder - -::: t4_devkit.schema.tables.registry - options: - members: ["SCHEMAS", "SchemaRegistry"] - show_root_toc_entry: false - -## Schema serialization - ---- - -::: t4_devkit.schema.serialize - - diff --git a/docs/apis/schema/index.md b/docs/apis/schema/index.md new file mode 100644 index 0000000..c58e7f9 --- /dev/null +++ b/docs/apis/schema/index.md @@ -0,0 +1,7 @@ +# `schema` + +- [Schema Names](./name.md) +- [Schema Tables](./table.md) +- [Other Items for Schema Tables](./table.md) +- [Schema Registry](./registry.md) +- [Serialize Schema](./serialize.md) diff --git a/docs/apis/schema/name.md b/docs/apis/schema/name.md new file mode 100644 index 0000000..a0b5d1d --- /dev/null +++ b/docs/apis/schema/name.md @@ -0,0 +1,21 @@ +Under the hood, `t4-devkit` declares an enum called `SchemaName`. +This enum includes names of each schema table that should be contained in the T4 dataset as `.json` file. + +Note that some schema tables are not mandatory, such as `object_ann.json` and `surface_ann.json`. +For these tables, the method called `is_optional()` returns `True` and it is OK that these corresponding `.json` files are not contained in T4 dataset: + +```python +from t4_devkit.schema import SchemaName + +>>> SchemaName.OBJECT_ANN.is_optional() +True +``` + + + +::: t4_devkit.schema.name + options: + show_docstring_attributes: true + show_bases: false + + diff --git a/docs/apis/schema/other.md b/docs/apis/schema/other.md new file mode 100644 index 0000000..e2b5135 --- /dev/null +++ b/docs/apis/schema/other.md @@ -0,0 +1,65 @@ +Following classes are sub items composed of each schema tables. + + + +## `SampleData` + +--- + +::: t4_devkit.schema.tables + options: + members: ["FileFormat", "SensorModality", "VisibilityLevel", "RLEMask", "ShiftState", "IndicatorState", "Indicators", "AdditionalInfo"] + show_root_toc_entry: false + merge_init_into_class: false + show_signature_annotations: false + show_docstring_attributes: true + +## `Sensor` + +--- + +::: t4_devkit.schema.tables + options: + members: ["SensorModality"] + show_root_toc_entry: false + merge_init_into_class: false + show_signature_annotations: false + show_docstring_attributes: true + +## `ObjectAnn`/`SurfaceAnn` + +--- + +::: t4_devkit.schema.tables + options: + members: ["RLEMask"] + show_root_toc_entry: false + merge_init_into_class: false + show_signature_annotations: false + show_docstring_attributes: true + +## `Visibility` + +--- + +::: t4_devkit.schema.tables + options: + members: ["VisibilityLevel"] + show_root_toc_entry: false + merge_init_into_class: false + show_signature_annotations: false + show_docstring_attributes: true + +## `VehicleState` + +--- + +::: t4_devkit.schema.tables + options: + members: ["ShiftState", "IndicatorState", "Indicators", "AdditionalInfo"] + show_root_toc_entry: false + merge_init_into_class: false + show_signature_annotations: false + show_docstring_attributes: true + + diff --git a/docs/apis/schema/registry.md b/docs/apis/schema/registry.md new file mode 100644 index 0000000..ea3f932 --- /dev/null +++ b/docs/apis/schema/registry.md @@ -0,0 +1,12 @@ +Each schema table is registered in `SchemaRegistry` dynamically at the runtime. + + + +::: t4_devkit.schema.builder + +::: t4_devkit.schema.tables.registry + options: + members: ["SCHEMAS", "SchemaRegistry"] + show_root_toc_entry: false + + diff --git a/docs/apis/schema/serialize.md b/docs/apis/schema/serialize.md new file mode 100644 index 0000000..f27b601 --- /dev/null +++ b/docs/apis/schema/serialize.md @@ -0,0 +1,7 @@ +You can serialize each schema table into `dict` using following functions. + + + +::: t4_devkit.schema.serialize + + diff --git a/docs/apis/schema/table.md b/docs/apis/schema/table.md new file mode 100644 index 0000000..8189f99 --- /dev/null +++ b/docs/apis/schema/table.md @@ -0,0 +1,11 @@ + + +::: t4_devkit.schema.tables + options: + filters: ["!SchemaBase", "!FileFormat", "!SensorModality", "!VisibilityLevel", "!RLEMask", "!ShiftState", "!IndicatorState", "!Indicators", "!AdditionalInfo"] + show_root_toc_entry: false + merge_init_into_class: false + show_signature_annotations: false + show_docstring_attributes: true + + diff --git a/mkdocs.yaml b/mkdocs.yaml index eada842..ef56c77 100644 --- a/mkdocs.yaml +++ b/mkdocs.yaml @@ -10,7 +10,13 @@ nav: - Customization: tutorials/customize.md - API: - TIER IV: apis/tier4.md - - Schema: apis/schema.md + - Schema: + - Home: apis/schema/index.md + - Schema Names: apis/schema/name.md + - Schema Tables: apis/schema/table.md + - Other Items for Tables: apis/schema/other.md + - Schema Registry: apis/schema/registry.md + - Serialize Schema: apis/schema/serialize.md - DataClass: apis/dataclass.md - Filtering: apis/filtering.md - Viewer: apis/viewer.md diff --git a/t4_devkit/schema/tables/log.py b/t4_devkit/schema/tables/log.py index aaf2f6a..deed631 100644 --- a/t4_devkit/schema/tables/log.py +++ b/t4_devkit/schema/tables/log.py @@ -22,6 +22,7 @@ class Log(SchemaBase): location (str): Area where log was captured. Shortcuts: + --------- map_token (str): Foreign key pointing to the map record. This should be set after instantiated. """ diff --git a/t4_devkit/schema/tables/object_ann.py b/t4_devkit/schema/tables/object_ann.py index 795b458..ff5dbf0 100644 --- a/t4_devkit/schema/tables/object_ann.py +++ b/t4_devkit/schema/tables/object_ann.py @@ -62,6 +62,7 @@ class ObjectAnn(SchemaBase): mask (RLEMask): Instance mask using the COCO format compressed by RLE. Shortcuts: + --------- category_name (str): Category name. This should be set after instantiated. """ diff --git a/t4_devkit/schema/tables/sample.py b/t4_devkit/schema/tables/sample.py index 2dd3a35..ca83564 100644 --- a/t4_devkit/schema/tables/sample.py +++ b/t4_devkit/schema/tables/sample.py @@ -22,6 +22,7 @@ class Sample(SchemaBase): prev (str): Foreign key pointing the sample that precedes this in time. Empty if start of scene. Shortcuts: + --------- data (dict[str, str]): Sensor channel and its token. This should be set after instantiated. ann_3ds (list[str]): List of foreign keys pointing the sample annotations. diff --git a/t4_devkit/schema/tables/sample_annotation.py b/t4_devkit/schema/tables/sample_annotation.py index 04956f6..7fc7926 100644 --- a/t4_devkit/schema/tables/sample_annotation.py +++ b/t4_devkit/schema/tables/sample_annotation.py @@ -50,6 +50,7 @@ class SampleAnnotation(SchemaBase): given as [ax, ay, av] in [m/s^2]. Shortcuts: + --------- category_name (str): Category name. This should be set after instantiated. """ diff --git a/t4_devkit/schema/tables/sample_data.py b/t4_devkit/schema/tables/sample_data.py index 6d5a970..41994a6 100644 --- a/t4_devkit/schema/tables/sample_data.py +++ b/t4_devkit/schema/tables/sample_data.py @@ -85,6 +85,7 @@ class SampleData(SchemaBase): is_valid (bool): True if this data is valid, else False. Invalid data should be ignored. Shortcuts: + --------- modality (SensorModality): Sensor modality. This should be set after instantiated. channel (str): Sensor channel. This should be set after instantiated. """ diff --git a/t4_devkit/schema/tables/sensor.py b/t4_devkit/schema/tables/sensor.py index cb96c99..19fe895 100644 --- a/t4_devkit/schema/tables/sensor.py +++ b/t4_devkit/schema/tables/sensor.py @@ -36,6 +36,7 @@ class Sensor(SchemaBase): modality (SensorModality): Sensor modality. Shortcuts: + --------- first_sd_token (str): The first sample data token corresponding to its sensor channel. """ diff --git a/t4_devkit/schema/tables/surface_ann.py b/t4_devkit/schema/tables/surface_ann.py index e29461f..f032ce3 100644 --- a/t4_devkit/schema/tables/surface_ann.py +++ b/t4_devkit/schema/tables/surface_ann.py @@ -26,6 +26,10 @@ class SurfaceAnn(SchemaBase): sample_data_token (str): Foreign key pointing to the sample data, which must be a keyframe image. category_token (str): Foreign key pointing to the surface category. mask (RLEMask): Segmentation mask using the COCO format compressed by RLE. + + Shortcuts: + --------- + category_name (str): Category name. This should be set after instantiated. """ sample_data_token: str diff --git a/t4_devkit/tier4.py b/t4_devkit/tier4.py index eba797f..df0df2c 100644 --- a/t4_devkit/tier4.py +++ b/t4_devkit/tier4.py @@ -2,6 +2,7 @@ import os.path as osp import time +import warnings from typing import TYPE_CHECKING import numpy as np @@ -191,6 +192,7 @@ def __make_reverse_index__(self, verbose: bool) -> None: for record in self.surface_ann: if record.category_token == "": # NOTE: Some database contains this case + warnings.warn(f"Category token is empty for surface ann: {record.token}") continue category: Category = self.get("category", record.category_token) record.category_name = category.name