perf(can): migrate from pandas to polars (#22082)

hongbo-miao · Dec 28, 2024 · 9907f9e · 9907f9e
1 parent 233128d
commit 9907f9e
Show file tree

Hide file tree

Showing 3 changed files with 24 additions and 100 deletions.
diff --git a/embedded/decode-can-data/pyproject.toml b/embedded/decode-can-data/pyproject.toml
@@ -4,7 +4,7 @@ version = "1.0.0"
 requires-python = "~=3.13.0"
 dependencies = [
   "cantools==40.0.0",
-  "pandas==2.2.3",
+  "polars==1.18.0",
   "python-can==4.5.0",
 ]
 

diff --git a/embedded/decode-can-data/src/main.py b/embedded/decode-can-data/src/main.py
@@ -5,7 +5,7 @@
 
 import can
 import cantools
-import pandas as pd
+import polars as pl
 
 
 def load_dbc_dict(dbc_path_dict: dict[str, Path]) -> dict[str, cantools.db.Database]:
@@ -100,5 +100,9 @@ def decode_can_data(
 
     for unit_type_and_unit_id, data in unit_type_and_unit_id_dict.items():
         parquet_path = data_dir_path / Path(f"{unit_type_and_unit_id}.parquet")
-        df = pd.DataFrame(data)
-        df.to_parquet(parquet_path, engine="pyarrow")
+        df = pl.DataFrame(data)
+        df.write_parquet(
+            parquet_path,
+            compression="zstd",
+            compression_level=19,
+        )
diff --git a/embedded/decode-can-data/uv.lock b/embedded/decode-can-data/uv.lock