Black formatting

.gitignore

@@ -1,6 +1,6 @@
 /saved
 /logs
-/data
+/data/semantic_kitti
 /.vscode
 .python-version
 __pycache__/

README.md

@@ -25,8 +25,58 @@ Mask4D is a transformer-based model for 4D Panoptic Segmentation, achieving a ne
 
 * **2023-09-28**: Paper on arXiv
 
-## Running the code
-Soon!
+### Dependencies
+The main dependencies of the project are the following:
+```yaml
+python: 3.8
+cuda: 11.7
+```
+You can set up a conda environment as follows
+```
+conda create --name mask4d python=3.8
+conda activate mask4d
+pip install -r requirements.txt
+
+pip install torch==1.13.0+cu117 torchvision==0.14.0+cu117 torchaudio==0.13.0 --extra-index-url https://download.pytorch.org/whl/cu117
+
+pip install torch-scatter -f https://data.pyg.org/whl/torch-1.13.0+cu117.html
+
+pip install 'git+https://github.com/facebookresearch/detectron2.git@710e7795d0eeadf9def0e7ef957eea13532e34cf' --no-deps
+
+cd third_party/pointnet2 && python setup.py install
+
+cd ..
+git clone https://github.com/NVIDIA/MinkowskiEngine.git
+cd MinkowskiEngine
+python setup.py install
+cd ../..
+```
+
+### Data preprocessing
+After installing the dependencies, we preprocess the SemanticKITTI dataset.
+
+```
+python -m datasets.preprocessing.semantic_kitti_preprocessing preprocess \
+--data_dir "PATH_TO_RAW_SEMKITTI_DATASET" \
+--save_dir "data/semantic_kitti"
+
+python -m datasets.preprocessing.semantic_kitti_preprocessing make_instance_database \
+--data_dir "PATH_TO_RAW_SEMKITTI_DATASET" \
+--save_dir "data/semantic_kitti"
+```
+
+### Training and testing
+Train Mask4D:
+```bash
+python main_panoptic.py
+```
+Please refer to the [config scripts](https://github.com/YilmazKadir/Mask4D/tree/main/scripts) for detailed instructions how to reproduce our results.
+In the simplest case the inference command looks as follows:
+```bash
+python main_panoptic.py \
+general.mode="validate" \
+general.ckpt_path='PATH_TO_CHECKPOINT.ckpt'
+```
 
 ## BibTeX
 ```

conf/config_panoptic_4d.yaml

@@ -6,7 +6,6 @@ general:
   workspace: kadiryilmaz
   instance_population: 10
   dbscan_eps: null
-  data_percent: 1.0
   experiment_name: ${now:%Y-%m-%d_%H%M%S}
   save_dir: saved/${general.experiment_name}
   gpus: 1

conf/data/datasets/semantic_kitti.yaml

@@ -6,7 +6,6 @@ train_dataset:
   add_distance: ${data.add_distance}
   sweep: ${data.sweep}
   instance_population: ${data.instance_population}
-  data_percent: ${data.data_percent}
   ignore_label: ${data.ignore_label}
   volume_augmentations_path: conf/augmentation/volumentations_aug.yaml
 
@@ -17,7 +16,6 @@ validation_dataset:
   add_distance: ${data.add_distance}
   sweep: ${data.sweep}
   instance_population: 0
-  data_percent: 1.0
   ignore_label: ${data.ignore_label}
   volume_augmentations_path: null
 
@@ -28,6 +26,5 @@ test_dataset:
   add_distance: ${data.add_distance}
   sweep: ${data.sweep}
   instance_population: 0
-  data_percent: 1.0
   ignore_label: ${data.ignore_label}
   volume_augmentations_path: null

conf/data/kitti.yaml

@@ -14,7 +14,6 @@ add_distance: true
 in_channels: 2
 num_labels: 19
 instance_population:  ${general.instance_population}
-data_percent: ${general.data_percent}
 sweep: 2
 min_stuff_cls_id: 9
 min_points: 50

datasets/lidar.py

@@ -13,7 +13,6 @@ def __init__(
         data_dir: Optional[str] = "data/processed/semantic_kitti",
         mode: Optional[str] = "train",
         add_distance: Optional[bool] = False,
-        data_percent: Optional[float] = 1.0,
         ignore_label: Optional[Union[int, List[int]]] = 255,
         volume_augmentations_path: Optional[str] = None,
         instance_population: Optional[int] = 0,
@@ -26,21 +25,19 @@ def __init__(
         self.instance_population = instance_population
         self.sweep = sweep
         self.config = self._load_yaml("conf/semantic-kitti.yaml")
-        
+
         # loading database file
         database_path = Path(self.data_dir)
         if not (database_path / f"{mode}_database.yaml").exists():
             print(f"generate {database_path}/{mode}_database.yaml first")
             exit()
         self.data = self._load_yaml(database_path / f"{mode}_database.yaml")
-        
+
         self.label_info = self._select_correct_labels(self.config["learning_ignore"])
         # augmentations
         self.volume_augmentations = V.NoOp()
         if volume_augmentations_path is not None:
-            self.volume_augmentations = V.load(
-                volume_augmentations_path, data_format="yaml"
-            )
+            self.volume_augmentations = V.load(volume_augmentations_path, data_format="yaml")
         # reformulating in sweeps
         data = [[]]
         last_scene = self.data[0]["scene"]
@@ -53,22 +50,19 @@ def __init__(
         for i in range(len(data)):
             data[i] = list(self.chunks(data[i], sweep))
         self.data = [val for sublist in data for val in sublist]
-        
+
         if instance_population > 0:
             self.instance_data = self._load_yaml(database_path / f"{mode}_instances_database.yaml")
-
-        if data_percent < 1.0:
-            self.data = self.data[: int(len(self.data) * data_percent)]
 
     def chunks(self, lst, n):
-        if "train" in self.mode or n==1:
+        if "train" in self.mode or n == 1:
             for i in range(len(lst) - n + 1):
                 yield lst[i : i + n]
         else:
-            for i in range(0, len(lst) - n + 1, n-1):
+            for i in range(0, len(lst) - n + 1, n - 1):
                 yield lst[i : i + n]
             if i != len(lst) - n:
-                yield lst[i + n - 1: ]
+                yield lst[i + n - 1 :]
 
     def __len__(self):
         return len(self.data)
@@ -99,51 +93,48 @@ def __getitem__(self, idx: int):
                 semantic_label = np.vectorize(self.config["learning_map"].__getitem__)(semantic_label)
                 labels = np.hstack((semantic_label[:, None], panoptic_label[:, None]))
                 labels_list.append(labels)
-        
+
         coordinates = np.vstack(coordinates_list)
         features = np.vstack(features_list)
         labels = np.vstack(labels_list)
-        
+
         if "train" in self.mode and self.instance_population > 0:
             max_instance_id = np.amax(labels[:, 1])
             pc_center = coordinates.mean(axis=0)
             instance_c, instance_f, instance_l = self.populate_instances(
-                max_instance_id, pc_center, self.instance_population)
+                max_instance_id, pc_center, self.instance_population
+            )
             coordinates = np.vstack((coordinates, instance_c))
             features = np.vstack((features, instance_f))
             labels = np.vstack((labels, instance_l))
-            
+
         if self.add_distance:
             center_coordinate = coordinates.mean(0)
             features = np.hstack(
                 (
                     features,
-                    np.linalg.norm(coordinates - center_coordinate, axis=1)[
-                        :, np.newaxis
-                    ],
+                    np.linalg.norm(coordinates - center_coordinate, axis=1)[:, np.newaxis],
                 )
             )
 
         # volume and image augmentations for train
         if "train" in self.mode:
             coordinates -= coordinates.mean(0)
             if 0.5 > random():
-                coordinates += (
-                    np.random.uniform(coordinates.min(0), coordinates.max(0)) / 2
-                )
+                coordinates += np.random.uniform(coordinates.min(0), coordinates.max(0)) / 2
             aug = self.volume_augmentations(points=coordinates)
             coordinates = aug["points"]
-            
+
         features = np.hstack((coordinates, features))
-        
+
         labels[:, 0] = np.vectorize(self.label_info.__getitem__)(labels[:, 0])
-        
+
         return {
             "num_points": acc_num_points,
             "coordinates": coordinates,
             "features": features,
             "labels": labels,
-            "sequence": scan["scene"]
+            "sequence": scan["scene"],
         }
 
     @staticmethod
@@ -155,7 +146,7 @@ def _load_yaml(filepath):
     def _select_correct_labels(self, learning_ignore):
         count = 0
         label_info = dict()
-        for k, v, in learning_ignore.items():
+        for k, v in learning_ignore.items():
             if v:
                 label_info[k] = self.ignore_label
             else:
@@ -181,7 +172,7 @@ def populate_instances(self, max_instance_id, pc_center, instance_population):
                     filepath = instance_dict["filepaths"][idx]
                     instance = np.load(filepath)
                     time_array = np.ones((instance.shape[0], 1)) * time
-                    instance = np.hstack((instance[:, :3], time_array , instance[:, 3:4]))
+                    instance = np.hstack((instance[:, :3], time_array, instance[:, 3:4]))
                     instance_list.append(instance)
                     idx = idx + 1
             instances = np.vstack(instance_list)