From ae9d8bd8eb187e3802b24e967ab45a94f5bb4573 Mon Sep 17 00:00:00 2001 From: Hayato Mizushima Date: Wed, 21 Feb 2024 22:47:43 +0900 Subject: [PATCH 1/9] chore: update ruff setting --- pyproject.toml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index cedac69..a02859b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,15 +1,17 @@ [tool.black] -line_length = 100 +line-length = 100 [tool.ruff] line-length = 100 -show-source = true +output-format = "full" + +[tool.ruff.lint] # https://beta.ruff.rs/docs/rules/ select = ["A", "B", "C", "D", "E", "F", "G", "I", "N", "Q", "S", "T", "W", "ANN", "ARG", "BLE", "COM", "DJ", "DTZ", "EM", "ERA", "EXE", "FBT", "ICN", "ISC", "NPY", "PD", "PGH", "PIE", "PL", "PT", "PTH", "PYI", "RET", "RSE", "RUF", "SIM", "SLF", "TCH", "TID", "TRY", "UP", "YTT"] ignore = ["Q000", "ANN101", "ANN102", "ANN401", "PGH004", "E501", "PLR0913", "S101", "S301", "S603", "SIM115", "D100", "D101", "D102", "D103", "D104", "D105", "D106", "D107", "D203", "D212", "D404", "D417", "PD011", "PD002", "PD901"] -fixable=["D", "I", "ANN", "COM", "EXE", "PIE"] +fixable = ["D", "I", "ANN", "COM", "EXE", "PIE"] -[tool.ruff.isort] +[tool.ruff.lint.isort] # https://pycqa.github.io/isort/docs/configuration/options.html#default-section force-sort-within-sections = true known-third-party = ["launch", "yaml"] From 354f6752127b74dfff25e9e7949213709667e380 Mon Sep 17 00:00:00 2001 From: Hayato Mizushima Date: Wed, 21 Feb 2024 22:48:19 +0900 Subject: [PATCH 2/9] feat: search mcap bag file directory --- autoware_msg_bag_converter/main.py | 2 +- test/resource/dummy_mcap/dummy.mcap | 1 + test/test_converter.py | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) create mode 100644 test/resource/dummy_mcap/dummy.mcap diff --git a/autoware_msg_bag_converter/main.py b/autoware_msg_bag_converter/main.py index 7947d13..f17eac4 100644 --- a/autoware_msg_bag_converter/main.py +++ b/autoware_msg_bag_converter/main.py @@ -23,7 +23,7 @@ def convert_bag_in_directory(input_dir: str, output_dir: str) -> None: input_root = Path(input_dir) output_root = Path(output_dir) - bag_paths = input_root.glob("**/*.db3") # Will mcap conversion be supported? + bag_paths = input_root.glob("**/*.db3|**/*.mcap") for db3_path in bag_paths: input_bag_dir = db3_path.parent rel_path = input_bag_dir.relative_to(input_root) diff --git a/test/resource/dummy_mcap/dummy.mcap b/test/resource/dummy_mcap/dummy.mcap new file mode 100644 index 0000000..7871b11 --- /dev/null +++ b/test/resource/dummy_mcap/dummy.mcap @@ -0,0 +1 @@ +dummy mcap file diff --git a/test/test_converter.py b/test/test_converter.py index f008b8f..7c5857e 100644 --- a/test/test_converter.py +++ b/test/test_converter.py @@ -35,7 +35,7 @@ def test_get_rosbag_path() -> None: # Test to confirm bag path acquisition in directory mode input_root = Path(__file__).resolve().parent.joinpath("resource") output_root = Path(__file__).resolve().parent.joinpath("converted") - bag_paths = input_root.glob("**/*.db3") + bag_paths = input_root.glob("**/*.db3|**/*.mcap") for db3_path in bag_paths: input_bag_dir = db3_path.parent rel_path = input_bag_dir.relative_to(input_root) From 98cbc4aa2ca0efc33f57688ea4f137f9d6269722 Mon Sep 17 00:00:00 2001 From: Hayato Mizushima Date: Wed, 21 Feb 2024 23:10:40 +0900 Subject: [PATCH 3/9] fix: test --- test/test_converter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_converter.py b/test/test_converter.py index 7c5857e..07589ff 100644 --- a/test/test_converter.py +++ b/test/test_converter.py @@ -35,7 +35,7 @@ def test_get_rosbag_path() -> None: # Test to confirm bag path acquisition in directory mode input_root = Path(__file__).resolve().parent.joinpath("resource") output_root = Path(__file__).resolve().parent.joinpath("converted") - bag_paths = input_root.glob("**/*.db3|**/*.mcap") + bag_paths = input_root.glob("**/*.db3|*.mcap") for db3_path in bag_paths: input_bag_dir = db3_path.parent rel_path = input_bag_dir.relative_to(input_root) From c5ca3fe9fb1e2f33eec6dde3a3d3bc55004cb2da Mon Sep 17 00:00:00 2001 From: Hayato Mizushima Date: Wed, 21 Feb 2024 23:35:28 +0900 Subject: [PATCH 4/9] WIP --- test/test_converter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_converter.py b/test/test_converter.py index 07589ff..02cd898 100644 --- a/test/test_converter.py +++ b/test/test_converter.py @@ -35,7 +35,7 @@ def test_get_rosbag_path() -> None: # Test to confirm bag path acquisition in directory mode input_root = Path(__file__).resolve().parent.joinpath("resource") output_root = Path(__file__).resolve().parent.joinpath("converted") - bag_paths = input_root.glob("**/*.db3|*.mcap") + bag_paths = input_root.glob("**/*.db3|*.mcap") # not working for db3_path in bag_paths: input_bag_dir = db3_path.parent rel_path = input_bag_dir.relative_to(input_root) From d0dd8a4dded375c22cb5e01ce1c8a0e25da527ce Mon Sep 17 00:00:00 2001 From: Hayato Mizushima Date: Mon, 29 Jul 2024 15:43:34 +0900 Subject: [PATCH 5/9] feat: support mcap file --- autoware_msg_bag_converter/bag.py | 12 ++++++------ autoware_msg_bag_converter/converter.py | 10 +++++----- autoware_msg_bag_converter/main.py | 11 +++++++---- test/test_converter.py | 5 ++++- 4 files changed, 22 insertions(+), 16 deletions(-) diff --git a/autoware_msg_bag_converter/bag.py b/autoware_msg_bag_converter/bag.py index cb8a233..2122fa4 100644 --- a/autoware_msg_bag_converter/bag.py +++ b/autoware_msg_bag_converter/bag.py @@ -25,15 +25,15 @@ def get_default_converter_options() -> ConverterOptions: ) -def get_default_storage_options(uri: str) -> StorageOptions: +def get_storage_options(uri: str, storage_type: str) -> StorageOptions: return StorageOptions( uri=uri, - storage_id="sqlite3", + storage_id=storage_type, ) -def create_reader(bag_dir: str) -> SequentialReader: - storage_options = get_default_storage_options(bag_dir) +def create_reader(bag_dir: str, storage_type: str) -> SequentialReader: + storage_options = get_storage_options(bag_dir, storage_type) converter_options = get_default_converter_options() reader = SequentialReader() @@ -41,8 +41,8 @@ def create_reader(bag_dir: str) -> SequentialReader: return reader -def create_writer(bag_dir: str) -> SequentialWriter: - storage_options = get_default_storage_options(bag_dir) +def create_writer(bag_dir: str, storage_type: str) -> SequentialReader: + storage_options = get_storage_options(bag_dir, storage_type) converter_options = get_default_converter_options() writer = SequentialWriter() diff --git a/autoware_msg_bag_converter/converter.py b/autoware_msg_bag_converter/converter.py index 14be529..16f57b2 100644 --- a/autoware_msg_bag_converter/converter.py +++ b/autoware_msg_bag_converter/converter.py @@ -21,7 +21,7 @@ from autoware_msg_bag_converter.bag import create_reader from autoware_msg_bag_converter.bag import create_writer -from autoware_msg_bag_converter.bag import get_default_storage_options +from autoware_msg_bag_converter.bag import get_storage_options def change_topic_type(old_type: TopicMetadata) -> TopicMetadata: @@ -33,11 +33,11 @@ def change_topic_type(old_type: TopicMetadata) -> TopicMetadata: ) -def convert_bag(input_bag_path: str, output_bag_path: str) -> None: +def convert_bag(input_bag_path: str, output_bag_path: str, storage_type: str) -> None: # open reader - reader = create_reader(input_bag_path) + reader = create_reader(input_bag_path, storage_type) # open writer - writer = create_writer(output_bag_path) + writer = create_writer(output_bag_path, storage_type) # create topic type_map = {} @@ -55,4 +55,4 @@ def convert_bag(input_bag_path: str, output_bag_path: str) -> None: # reindex to update metadata.yaml del writer - Reindexer().reindex(get_default_storage_options(output_bag_path)) + Reindexer().reindex(get_storage_options(output_bag_path, storage_type)) diff --git a/autoware_msg_bag_converter/main.py b/autoware_msg_bag_converter/main.py index f17eac4..75b194a 100644 --- a/autoware_msg_bag_converter/main.py +++ b/autoware_msg_bag_converter/main.py @@ -15,6 +15,7 @@ import argparse from os.path import expandvars from pathlib import Path +import re from autoware_msg_bag_converter.converter import convert_bag @@ -23,12 +24,14 @@ def convert_bag_in_directory(input_dir: str, output_dir: str) -> None: input_root = Path(input_dir) output_root = Path(output_dir) - bag_paths = input_root.glob("**/*.db3|**/*.mcap") - for db3_path in bag_paths: - input_bag_dir = db3_path.parent + pattern = re.compile(r".*\.(db3|mcap)$") + bag_paths = [p for p in input_root.rglob("*") if pattern.match(str(p))] + for db3_or_mcap_path in bag_paths: + storage_type = "mcap" if db3_or_mcap_path.suffix == "mcap" else "sqlite3" + input_bag_dir = db3_or_mcap_path.parent rel_path = input_bag_dir.relative_to(input_root) output_bag_dir = output_root.joinpath(rel_path) - convert_bag(input_bag_dir.as_posix(), output_bag_dir.as_posix()) + convert_bag(input_bag_dir.as_posix(), output_bag_dir.as_posix(), storage_type) def main() -> None: diff --git a/test/test_converter.py b/test/test_converter.py index 02cd898..3d8cab2 100644 --- a/test/test_converter.py +++ b/test/test_converter.py @@ -13,6 +13,7 @@ # limitations under the License. from pathlib import Path +import re from rosbag2_py import TopicMetadata @@ -35,7 +36,9 @@ def test_get_rosbag_path() -> None: # Test to confirm bag path acquisition in directory mode input_root = Path(__file__).resolve().parent.joinpath("resource") output_root = Path(__file__).resolve().parent.joinpath("converted") - bag_paths = input_root.glob("**/*.db3|*.mcap") # not working + pattern = re.compile(r".*\.(db3|mcap)$") + bag_paths = [p for p in input_root.rglob("*") if pattern.match(str(p))] + assert len(bag_paths) == 3 # noqa for db3_path in bag_paths: input_bag_dir = db3_path.parent rel_path = input_bag_dir.relative_to(input_root) From a16e40af09f0274364bbefec4e63754074e0350a Mon Sep 17 00:00:00 2001 From: Hayato Mizushima Date: Mon, 29 Jul 2024 16:11:37 +0900 Subject: [PATCH 6/9] fix: match suffix --- autoware_msg_bag_converter/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/autoware_msg_bag_converter/main.py b/autoware_msg_bag_converter/main.py index 75b194a..8e796d2 100644 --- a/autoware_msg_bag_converter/main.py +++ b/autoware_msg_bag_converter/main.py @@ -27,7 +27,7 @@ def convert_bag_in_directory(input_dir: str, output_dir: str) -> None: pattern = re.compile(r".*\.(db3|mcap)$") bag_paths = [p for p in input_root.rglob("*") if pattern.match(str(p))] for db3_or_mcap_path in bag_paths: - storage_type = "mcap" if db3_or_mcap_path.suffix == "mcap" else "sqlite3" + storage_type = "mcap" if db3_or_mcap_path.suffix == ".mcap" else "sqlite3" input_bag_dir = db3_or_mcap_path.parent rel_path = input_bag_dir.relative_to(input_root) output_bag_dir = output_root.joinpath(rel_path) From cd3ec25b9182840367560ee844f57d6f33a08734 Mon Sep 17 00:00:00 2001 From: Hayato Mizushima Date: Mon, 29 Jul 2024 16:35:27 +0900 Subject: [PATCH 7/9] fix: set storage_type --- autoware_msg_bag_converter/converter.py | 9 ++++++++- autoware_msg_bag_converter/main.py | 3 +-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/autoware_msg_bag_converter/converter.py b/autoware_msg_bag_converter/converter.py index 16f57b2..aa690fc 100644 --- a/autoware_msg_bag_converter/converter.py +++ b/autoware_msg_bag_converter/converter.py @@ -16,6 +16,8 @@ # https://github.com/ros2/rosbag2/blob/rolling/rosbag2_py/test/test_sequential_writer.py # https://github.com/ros2/rosbag2/blob/rolling/rosbag2_py/test/test_reindexer.py +from pathlib import Path + from rosbag2_py import Reindexer from rosbag2_py import TopicMetadata @@ -33,7 +35,12 @@ def change_topic_type(old_type: TopicMetadata) -> TopicMetadata: ) -def convert_bag(input_bag_path: str, output_bag_path: str, storage_type: str) -> None: +def convert_bag(input_bag_path: str, output_bag_path: str) -> None: + p_input = Path(input_bag_path) + storage_type = "mcap" + for _ in p_input.glob("*.db3"): + storage_type = "sqlite3" + break # open reader reader = create_reader(input_bag_path, storage_type) # open writer diff --git a/autoware_msg_bag_converter/main.py b/autoware_msg_bag_converter/main.py index 8e796d2..d8ace1b 100644 --- a/autoware_msg_bag_converter/main.py +++ b/autoware_msg_bag_converter/main.py @@ -27,11 +27,10 @@ def convert_bag_in_directory(input_dir: str, output_dir: str) -> None: pattern = re.compile(r".*\.(db3|mcap)$") bag_paths = [p for p in input_root.rglob("*") if pattern.match(str(p))] for db3_or_mcap_path in bag_paths: - storage_type = "mcap" if db3_or_mcap_path.suffix == ".mcap" else "sqlite3" input_bag_dir = db3_or_mcap_path.parent rel_path = input_bag_dir.relative_to(input_root) output_bag_dir = output_root.joinpath(rel_path) - convert_bag(input_bag_dir.as_posix(), output_bag_dir.as_posix(), storage_type) + convert_bag(input_bag_dir.as_posix(), output_bag_dir.as_posix()) def main() -> None: From 424368e4adfde7481dc801caea3eb02b94a0ee26 Mon Sep 17 00:00:00 2001 From: Hayato Mizushima Date: Mon, 29 Jul 2024 16:44:35 +0900 Subject: [PATCH 8/9] fix: toml unknown field `output-format` --- pyproject.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index a02859b..97d6c36 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,6 @@ line-length = 100 [tool.ruff] line-length = 100 -output-format = "full" [tool.ruff.lint] # https://beta.ruff.rs/docs/rules/ From 9b95f85618be2a1735462421c3a2f3b327754cbb Mon Sep 17 00:00:00 2001 From: Hayato Mizushima Date: Mon, 29 Jul 2024 16:58:25 +0900 Subject: [PATCH 9/9] fix: update pre-commit tool version --- .pre-commit-config.yaml | 15 +++++++-------- pyproject.toml | 1 + 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 9468d1c..115dbb0 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -3,7 +3,7 @@ ci: repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.4.0 + rev: v4.5.0 hooks: - id: check-json exclude: \.vscode/ @@ -18,9 +18,8 @@ repos: - id: trailing-whitespace args: [--markdown-linebreak-ext=md] - # disable because cannot ignore CONTRIBUTING.md - repo: https://github.com/igorshubovych/markdownlint-cli - rev: v0.38.0 + rev: v0.39.0 hooks: - id: markdownlint args: [-c, .markdownlint.yaml, --fix, -i, CONTRIBUTING.md] @@ -31,7 +30,7 @@ repos: - id: prettier - repo: https://github.com/adrienverge/yamllint - rev: v1.30.0 + rev: v1.34.0 hooks: - id: yamllint @@ -45,24 +44,24 @@ repos: - id: sort-package-xml - repo: https://github.com/shellcheck-py/shellcheck-py - rev: v0.9.0.2 + rev: v0.9.0.6 hooks: - id: shellcheck - repo: https://github.com/scop/pre-commit-shfmt - rev: v3.6.0-2 + rev: v3.7.0-4 hooks: - id: shfmt args: [-w, -s, -i=4] - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.0.284 + rev: v0.2.1 hooks: - id: ruff args: [--fix] - repo: https://github.com/psf/black - rev: 23.3.0 + rev: 24.1.1 hooks: - id: black args: [--line-length=100] diff --git a/pyproject.toml b/pyproject.toml index 97d6c36..a02859b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,6 +3,7 @@ line-length = 100 [tool.ruff] line-length = 100 +output-format = "full" [tool.ruff.lint] # https://beta.ruff.rs/docs/rules/