From 5de3384eab37f0495b88631f16ee1a66809dcfd2 Mon Sep 17 00:00:00 2001 From: Julian Gaal Date: Fri, 19 May 2023 10:38:42 +0200 Subject: [PATCH 1/4] support semantics only visualization --- auxiliary/laserscanvis.py | 174 ++++++++++++++++++++++++++++---------- visualize.py | 31 ++++--- 2 files changed, 147 insertions(+), 58 deletions(-) diff --git a/auxiliary/laserscanvis.py b/auxiliary/laserscanvis.py index 4404c5a..ac4e288 100644 --- a/auxiliary/laserscanvis.py +++ b/auxiliary/laserscanvis.py @@ -12,13 +12,14 @@ class LaserScanVis: """Class that creates and handles a visualizer for a pointcloud""" def __init__(self, scan, scan_names, label_names, offset=0, - semantics=True, instances=False, images=True, link=False): + semantics=True, semantics_only=False, instances=False, images=True, link=False): self.scan = scan self.scan_names = scan_names self.label_names = label_names self.offset = offset self.total = len(self.scan_names) self.semantics = semantics + self.semantics_only = semantics_only self.instances = instances self.images = images self.link = link @@ -30,23 +31,12 @@ def __init__(self, scan, scan_names, label_names, offset=0, self.reset() self.update_scan() - def reset(self): - """ Reset. """ - # last key press (it should have a mutex, but visualization is not - # safety critical, so let's do things wrong) - self.action = "no" # no, next, back, quit are the possibilities - - # new canvas prepared for visualizing data - self.canvas = SceneCanvas(keys='interactive', show=True) - # interface (n next, b back, q quit, very simple) - self.canvas.events.key_press.connect(self.key_press) - self.canvas.events.draw.connect(self.draw) - # grid - self.grid = self.canvas.central_widget.add_grid() - - # laserscan part + def default_reset(self): + """ + Default reset with raw pointcloud, colored point cloud and/or instances + """ self.scan_view = vispy.scene.widgets.ViewBox( - border_color='white', parent=self.canvas.scene) + border_color='white', parent=self.canvas.scene) self.grid.add_widget(self.scan_view, 0, 0) self.scan_vis = visuals.Markers() self.scan_view.camera = 'turntable' @@ -56,7 +46,7 @@ def reset(self): if self.semantics: print("Using semantics in visualizer") self.sem_view = vispy.scene.widgets.ViewBox( - border_color='white', parent=self.canvas.scene) + border_color='white', parent=self.canvas.scene) self.grid.add_widget(self.sem_view, 0, 1) self.sem_vis = visuals.Markers() self.sem_view.camera = 'turntable' @@ -68,7 +58,7 @@ def reset(self): if self.instances: print("Using instances in visualizer") self.inst_view = vispy.scene.widgets.ViewBox( - border_color='white', parent=self.canvas.scene) + border_color='white', parent=self.canvas.scene) self.grid.add_widget(self.inst_view, 0, 2) self.inst_vis = visuals.Markers() self.inst_view.camera = 'turntable' @@ -97,7 +87,7 @@ def reset(self): self.img_canvas.events.key_press.connect(self.key_press) self.img_canvas.events.draw.connect(self.draw) self.img_view = vispy.scene.widgets.ViewBox( - border_color='white', parent=self.img_canvas.scene) + border_color='white', parent=self.img_canvas.scene) self.img_grid.add_widget(self.img_view, 0, 0) self.img_vis = visuals.Image(cmap='viridis') self.img_view.add(self.img_vis) @@ -105,21 +95,109 @@ def reset(self): # add image semantics if self.semantics: self.sem_img_view = vispy.scene.widgets.ViewBox( - border_color='white', parent=self.img_canvas.scene) + border_color='white', parent=self.img_canvas.scene) self.img_grid.add_widget(self.sem_img_view, 1, 0) self.sem_img_vis = visuals.Image(cmap='viridis') self.sem_img_view.add(self.sem_img_vis) # add instances - if self.instances: + if self.instances and self.images: self.inst_img_view = vispy.scene.widgets.ViewBox( - border_color='white', parent=self.img_canvas.scene) + border_color='white', parent=self.img_canvas.scene) self.img_grid.add_widget(self.inst_img_view, 2, 0) self.inst_img_vis = visuals.Image(cmap='viridis') self.inst_img_view.add(self.inst_img_vis) if self.link: self.inst_view.camera.link(self.scan_view.camera) + def semantics_only_reset(self): + """ + Reset without raw pointcloud, colored point cloud and/or instances + """ + # add semantics + print("Using semantics in visualizer") + self.sem_view = vispy.scene.widgets.ViewBox( + border_color='white', parent=self.canvas.scene) + self.grid.add_widget(self.sem_view, 0, 0) + self.sem_vis = visuals.Markers() + self.sem_view.camera = 'turntable' + self.sem_view.add(self.sem_vis) + visuals.XYZAxis(parent=self.sem_view.scene) + if self.link and not self.semantics_only: + self.sem_view.camera.link(self.scan_view.camera) + + if self.instances: + print("Using instances in visualizer") + self.inst_view = vispy.scene.widgets.ViewBox( + border_color='white', parent=self.canvas.scene) + self.grid.add_widget(self.inst_view, 0, 1) + self.inst_vis = visuals.Markers() + self.inst_view.camera = 'turntable' + self.inst_view.add(self.inst_vis) + visuals.XYZAxis(parent=self.inst_view.scene) + if self.link and not self.semantics_only: + self.inst_view.camera.link(self.scan_view.camera) + else: + self.inst_view.camera.link(self.sem_view.camera) + + # add a view for the depth + if self.images: + # img canvas size + self.multiplier = 1 + self.canvas_W = 1024 + self.canvas_H = 64 + self.multiplier += 1 # for semantics, which are definitely wanted in this case, because semantics_only=True + if self.instances: + self.multiplier += 1 + + # new canvas for img + self.img_canvas = SceneCanvas(keys='interactive', show=True, + size=(self.canvas_W, self.canvas_H * self.multiplier)) + # grid + self.img_grid = self.img_canvas.central_widget.add_grid() + # interface (n next, b back, q quit, very simple) + self.img_canvas.events.key_press.connect(self.key_press) + self.img_canvas.events.draw.connect(self.draw) + + # add image semantics + if self.semantics: + self.sem_img_view = vispy.scene.widgets.ViewBox( + border_color='white', parent=self.img_canvas.scene) + self.img_grid.add_widget(self.sem_img_view, 0, 0) + self.sem_img_vis = visuals.Image(cmap='viridis') + self.sem_img_view.add(self.sem_img_vis) + + # add instances + if self.instances and self.images: + self.inst_img_view = vispy.scene.widgets.ViewBox( + border_color='white', parent=self.img_canvas.scene) + self.img_grid.add_widget(self.inst_img_view, 1, 0) + self.inst_img_vis = visuals.Image(cmap='viridis') + self.inst_img_view.add(self.inst_img_vis) + if self.link: + self.inst_view.camera.link(self.sem_view.camera) + + def reset(self): + """ Reset. """ + # last key press (it should have a mutex, but visualization is not + # safety critical, so let's do things wrong) + self.action = "no" # no, next, back, quit are the possibilities + + # new canvas prepared for visualizing data + self.canvas = SceneCanvas(keys='interactive', show=True) + # interface (n next, b back, q quit, very simple) + self.canvas.events.key_press.connect(self.key_press) + self.canvas.events.draw.connect(self.draw) + # grid + self.grid = self.canvas.central_widget.add_grid() + + # laserscan part + + if not self.semantics_only: + self.default_reset() + else: + self.semantics_only_reset() + def get_mpl_colormap(self, cmap_name): cmap = plt.get_cmap(cmap_name) @@ -130,6 +208,7 @@ def get_mpl_colormap(self, cmap_name): color_range = sm.to_rgba(np.linspace(0, 1, 256), bytes=True)[:, 2::-1] return color_range.reshape(256, 3).astype(np.float32) / 255.0 + def update_scan(self): # first open data self.scan.open_scan(self.scan_names[self.offset]) @@ -147,20 +226,20 @@ def update_scan(self): # plot scan power = 16 - # print() - range_data = np.copy(self.scan.unproj_range) - # print(range_data.max(), range_data.min()) - range_data = range_data**(1 / power) - # print(range_data.max(), range_data.min()) - viridis_range = ((range_data - range_data.min()) / - (range_data.max() - range_data.min()) * - 255).astype(np.uint8) - viridis_map = self.get_mpl_colormap("viridis") - viridis_colors = viridis_map[viridis_range] - self.scan_vis.set_data(self.scan.points, - face_color=viridis_colors[..., ::-1], - edge_color=viridis_colors[..., ::-1], - size=1) + if not self.semantics_only: + range_data = np.copy(self.scan.unproj_range) + # print(range_data.max(), range_data.min()) + range_data = range_data ** (1 / power) + # print(range_data.max(), range_data.min()) + viridis_range = ((range_data - range_data.min()) / + (range_data.max() - range_data.min()) * + 255).astype(np.uint8) + viridis_map = self.get_mpl_colormap("viridis") + viridis_colors = viridis_map[viridis_range] + self.scan_vis.set_data(self.scan.points, + face_color=viridis_colors[..., ::-1], + edge_color=viridis_colors[..., ::-1], + size=1) # plot semantics if self.semantics: @@ -179,16 +258,17 @@ def update_scan(self): if self.images: # now do all the range image stuff # plot range image - data = np.copy(self.scan.proj_range) - # print(data[data > 0].max(), data[data > 0].min()) - data[data > 0] = data[data > 0]**(1 / power) - data[data < 0] = data[data > 0].min() - # print(data.max(), data.min()) - data = (data - data[data > 0].min()) / \ - (data.max() - data[data > 0].min()) - # print(data.max(), data.min()) - self.img_vis.set_data(data) - self.img_vis.update() + if not self.semantics_only: + data = np.copy(self.scan.proj_range) + # print(data[data > 0].max(), data[data > 0].min()) + data[data > 0] = data[data > 0] ** (1 / power) + data[data < 0] = data[data > 0].min() + # print(data.max(), data.min()) + data = (data - data[data > 0].min()) / \ + (data.max() - data[data > 0].min()) + # print(data.max(), data.min()) + self.img_vis.set_data(data) + self.img_vis.update() if self.semantics: self.sem_img_vis.set_data(self.scan.proj_sem_color[..., ::-1]) diff --git a/visualize.py b/visualize.py index cba28ba..a3bf74e 100755 --- a/visualize.py +++ b/visualize.py @@ -47,6 +47,14 @@ help='Ignore semantics. Visualizes uncolored pointclouds.' 'Defaults to %(default)s', ) + parser.add_argument( + '--semantics_only', + dest='semantics_only', + default=False, + action='store_true', + help='Only visualize semantics.' + 'Defaults to %(default)s', + ) parser.add_argument( '--do_instances', '-o', dest='do_instances', @@ -89,14 +97,6 @@ ' that safety.' 'Defaults to %(default)s', ) - parser.add_argument( - '--color_learning_map', - dest='color_learning_map', - default=False, - required=False, - action='store_true', - help='Apply learning map to color map: visualize only classes that were trained on', - ) FLAGS, unparsed = parser.parse_known_args() # print summary of what we will do @@ -107,11 +107,11 @@ print("Sequence", FLAGS.sequence) print("Predictions", FLAGS.predictions) print("ignore_semantics", FLAGS.ignore_semantics) + print("semantics_only", FLAGS.semantics_only) print("do_instances", FLAGS.do_instances) print("ignore_images", FLAGS.ignore_images) print("link", FLAGS.link) print("ignore_safety", FLAGS.ignore_safety) - print("color_learning_map", FLAGS.color_learning_map) print("offset", FLAGS.offset) print("*" * 80) @@ -124,6 +124,12 @@ print("Error opening yaml file.") quit() + # if custom config is given as parameter, assume that coloring must be exactly according to learning map + color_learning_map = FLAGS.config is not None + + assert not (FLAGS.ignore_semantics and FLAGS.semantics_only),\ + "Can't ignore semantics and show semantics only at the same time!" + # fix sequence name FLAGS.sequence = '{0:02d}'.format(int(FLAGS.sequence)) @@ -169,7 +175,7 @@ scan = LaserScan(project=True) # project all opened scans to spheric proj else: color_dict = CFG["color_map"] - if FLAGS.color_learning_map: + if color_learning_map: learning_map_inv = CFG["learning_map_inv"] learning_map = CFG["learning_map"] color_dict = {key:color_dict[learning_map_inv[learning_map[key]]] for key, value in color_dict.items()} @@ -178,6 +184,7 @@ # create a visualizer semantics = not FLAGS.ignore_semantics + semantics_only = FLAGS.semantics_only instances = FLAGS.do_instances images = not FLAGS.ignore_images if not semantics: @@ -186,7 +193,9 @@ scan_names=scan_names, label_names=label_names, offset=FLAGS.offset, - semantics=semantics, instances=instances and semantics, images=images, link=FLAGS.link) + semantics=semantics, + semantics_only=semantics_only, + instances=instances and semantics, images=images, link=FLAGS.link) # print instructions print("To navigate:") From 6a29b8876e2281af883324ea76bd9807c7943fdb Mon Sep 17 00:00:00 2001 From: Julian Gaal Date: Fri, 19 May 2023 10:47:48 +0200 Subject: [PATCH 2/4] fix learning map coloring behavior --- visualize.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/visualize.py b/visualize.py index a3bf74e..89672c5 100755 --- a/visualize.py +++ b/visualize.py @@ -7,6 +7,8 @@ from auxiliary.laserscan import LaserScan, SemLaserScan from auxiliary.laserscanvis import LaserScanVis +DEFAULT_CONFIG="config/semantic-kitti.yaml" + if __name__ == '__main__': parser = argparse.ArgumentParser("./visualize.py") parser.add_argument( @@ -19,7 +21,7 @@ '--config', '-c', type=str, required=False, - default="config/semantic-kitti.yaml", + default=DEFAULT_CONFIG, help='Dataset config file. Defaults to %(default)s', ) parser.add_argument( @@ -175,7 +177,7 @@ scan = LaserScan(project=True) # project all opened scans to spheric proj else: color_dict = CFG["color_map"] - if color_learning_map: + if FLAGS.config != DEFAULT_CONFIG: learning_map_inv = CFG["learning_map_inv"] learning_map = CFG["learning_map"] color_dict = {key:color_dict[learning_map_inv[learning_map[key]]] for key, value in color_dict.items()} From 4b243db2fbb9c587910f771b8aa2d53efa1306c2 Mon Sep 17 00:00:00 2001 From: Julian Gaal Date: Fri, 19 May 2023 10:49:05 +0200 Subject: [PATCH 3/4] adapt mos visualization to semantics only parameter --- visualize_mos.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/visualize_mos.py b/visualize_mos.py index f242661..0b67ff2 100755 --- a/visualize_mos.py +++ b/visualize_mos.py @@ -166,7 +166,9 @@ scan_names=scan_names, label_names=label_names, offset=FLAGS.offset, - semantics=semantics, instances=instances and semantics) + semantics=semantics, + semantics_only=False, + instances=instances and semantics) # print instructions print("To navigate:") From 9a24f97a420d9b2089abc024646c82642f205607 Mon Sep 17 00:00:00 2001 From: Julian Gaal Date: Fri, 19 May 2023 10:54:02 +0200 Subject: [PATCH 4/4] update compare viewer instructions --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 7925d0b..ee46c24 100644 --- a/README.md +++ b/README.md @@ -154,13 +154,13 @@ To directly compare two sets of data, use the `compare.py` script. It will open opengl visualization of the pointcloud labels. ```sh -$ ./compare.py --sequence 00 --dataset_a /path/to/dataset_a/ --dataset_a /path/to/kitti/dataset_b/ +$ ./compare.py --sequence 00 --dataset /path/to/dataset/ --labels "labels" "predictions" ``` where: - `sequence` is the sequence to be accessed. -- `dataset_a` is the path to a dataset in KITTI format where the `sequences` directory is. -- `dataset_b` is the path to another dataset in KITTI format where the `sequences` directory is. +- `dataset` is the path to a dataset in KITTI format where the `sequences` directory is. +- `labels` describes two folders with different sets of labels in the sequence folder. For example: ground truth ("labels") and predictions from e.g. RangeNet in folder "predictions" Navigation: - `n` is next scan,