diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index bbacff716..cb3d217e9 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -178,6 +178,54 @@ jobs: cd metadrive/ pytest --cov=./ --cov-config=.coveragerc --cov-report=xml -sv tests/test_export_record_scenario + test_sensor_pipeline: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Set up Python 3.9 + uses: actions/setup-python@v2 + with: + python-version: 3.9 + - name: Prepare OpenGL + run: | + sudo apt-get -y install xvfb + sudo /usr/bin/Xvfb :0 -screen 0 1280x1024x24 & + - name: Blackbox tests + run: | + pip install cython + pip install numpy + pip install -e . + python -m metadrive.pull_asset + pip install pytest + pip install pytest-cov + pip install ray + cd metadrive/ + pytest --cov=./ --cov-config=.coveragerc --cov-report=xml -sv tests/test_sensors + + test_examples: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Set up Python 3.9 + uses: actions/setup-python@v2 + with: + python-version: 3.9 + - name: Prepare OpenGL + run: | + sudo apt-get -y install xvfb + sudo /usr/bin/Xvfb :0 -screen 0 1280x1024x24 & + - name: Blackbox tests + run: | + pip install cython + pip install numpy + pip install -e . + python -m metadrive.pull_asset + pip install pytest + pip install pytest-cov + pip install ray + cd metadrive/ + pytest --cov=./ --cov-config=.coveragerc --cov-report=xml -sv tests/test_examples + test_ipynb: runs-on: ubuntu-latest steps: @@ -203,8 +251,6 @@ jobs: test_ros: runs-on: ubuntu-22.04 -# container: -# image: ubuntu:jammy steps: - name: Set up ROS2 humble uses: ros-tooling/setup-ros@v0.7 diff --git a/metadrive/component/sensors/__init__.py b/metadrive/component/sensors/__init__.py index 441480f69..6bed1b241 100644 --- a/metadrive/component/sensors/__init__.py +++ b/metadrive/component/sensors/__init__.py @@ -1,3 +1,7 @@ -class BaseSensor: - def perceive(self, *args, **kwargs): - raise NotImplementedError +# from metadrive.component.sensors.depth_camera import DepthCamera +# from metadrive.component.sensors.rgb_camera import RGBCamera +# from metadrive.component.sensors.semantic_camera import SemanticCamera +# from metadrive.component.sensors.mini_map import MiniMap +# from metadrive.component.sensors.lidar import Lidar +# from metadrive.component.sensors.distance_detector import DistanceDetector, SideDetector, LaneLineDetector +# from metadrive.component.sensors.dashboard import DashBoard diff --git a/metadrive/component/sensors/base_camera.py b/metadrive/component/sensors/base_camera.py index f39a3c86d..051bd9026 100644 --- a/metadrive/component/sensors/base_camera.py +++ b/metadrive/component/sensors/base_camera.py @@ -1,6 +1,6 @@ import numpy as np import cv2 -from metadrive.component.sensors import BaseSensor +from metadrive.component.sensors.base_sensor import BaseSensor from metadrive.utils.cuda import check_cudart_err _cuda_enable = True @@ -19,25 +19,26 @@ class BaseCamera(ImageBuffer, BaseSensor): """ + This class wrapping the ImageBuffer and BaseSensor to implement perceive() function to capture images in the virtual + world. It also extends a support for cuda, so the rendered images can be retained on GPU and converted to torch + tensor directly. The sensor is shared and thus can be set at any position in the world for any objects' use. To enable the image observation, set image_observation to True. - Every objects share the same camera, to boost the efficiency and save memory. - Camera configuration is read from the global config automatically. """ # shape(dim_1, dim_2) BUFFER_W = 84 # dim 1 BUFFER_H = 84 # dim 2 CAM_MASK = None - display_region_size = [1 / 3, 2 / 3, 0.8, 1.0] attached_object = None - def __init__(self, engine, setup_pbr=False, need_cuda=False, frame_buffer_property=None): + num_channels = 3 + + def __init__(self, engine, need_cuda=False, frame_buffer_property=None): self._enable_cuda = need_cuda super(BaseCamera, self).__init__( self.BUFFER_W, self.BUFFER_H, Vec3(0., 0.8, 1.5), self.BKG_COLOR, - setup_pbr=setup_pbr, engine=engine, frame_buffer_property=frame_buffer_property ) @@ -47,7 +48,7 @@ def __init__(self, engine, setup_pbr=False, need_cuda=False, frame_buffer_proper if (width > 100 or height > 100) and not self.enable_cuda: # Too large height or width will cause corruption in Mac. self.logger.warning( - "You may using too large buffer! The height is {}, and width is {}. " + "You are using too large buffer! The height is {}, and width is {}. " "It may lower the sample efficiency! Consider reducing buffer size or use cuda image by" " set [image_on_cuda=True].".format(height, width) ) @@ -110,7 +111,7 @@ def perceive(self, base_object, clip=True) -> np.ndarray: self.track(base_object) if self.enable_cuda: assert self.cuda_rendered_result is not None - ret = self.cuda_rendered_result[..., :-1][..., ::-1][::-1] + ret = self.cuda_rendered_result[..., :-1][..., ::-1][::-1][..., :self.num_channels] else: ret = self.get_rgb_array_cpu() if self.engine.global_config["rgb_to_grayscale"]: diff --git a/metadrive/component/sensors/base_sensor.py b/metadrive/component/sensors/base_sensor.py new file mode 100644 index 000000000..8eea8acf1 --- /dev/null +++ b/metadrive/component/sensors/base_sensor.py @@ -0,0 +1,15 @@ +class BaseSensor: + """ + This is the base class of all sensors + """ + def perceive(self, *args, **kwargs): + """ + All sensors have to implement this API as the interface for accessing the sensor output + Args: + *args: varies according to sensor type + **kwargs: varies according to sensor type + + Returns: sensor output. It could be matrices like images or other data structures + + """ + raise NotImplementedError diff --git a/metadrive/component/sensors/vehicle_panel.py b/metadrive/component/sensors/dashboard.py similarity index 91% rename from metadrive/component/sensors/vehicle_panel.py rename to metadrive/component/sensors/dashboard.py index bdf3fc48c..dc02b397a 100644 --- a/metadrive/component/sensors/vehicle_panel.py +++ b/metadrive/component/sensors/dashboard.py @@ -1,11 +1,14 @@ from panda3d.core import NodePath, PGTop, TextNode, CardMaker, Vec3 -from metadrive.component.sensors import BaseSensor +from metadrive.component.sensors.base_sensor import BaseSensor from metadrive.constants import CamMask from metadrive.engine.core.image_buffer import ImageBuffer -class VehiclePanel(ImageBuffer, BaseSensor): +class DashBoard(ImageBuffer, BaseSensor): + """ + Dashboard for showing the speed and brake/throttle/steering + """ PARA_VIS_LENGTH = 12 PARA_VIS_HEIGHT = 1 MAX_SPEED = 120 @@ -14,7 +17,6 @@ class VehiclePanel(ImageBuffer, BaseSensor): CAM_MASK = CamMask.PARA_VIS GAP = 4.1 TASK_NAME = "update panel" - display_region_size = [2 / 3, 1, 0.8, 1.0] def __init__(self, engine, *, cuda): if engine.win is None: @@ -74,7 +76,7 @@ def __init__(self, engine, *, cuda): card.setPos(0.2 + self.PARA_VIS_LENGTH / 2, 0, 0.22) self.para_vis_np[name] = card - super(VehiclePanel, self).__init__( + super(DashBoard, self).__init__( self.BUFFER_W, self.BUFFER_H, Vec3(-0.9, -1.01, 0.78), @@ -82,7 +84,6 @@ def __init__(self, engine, *, cuda): parent_node=self.aspect2d_np, engine=engine ) - # self.add_display_region(self.display_region_size) self._node_path_list.extend(tmp_node_path_list) def update_vehicle_state(self, vehicle): @@ -112,15 +113,15 @@ def update_vehicle_state(self, vehicle): def remove_display_region(self): self.buffer.set_active(False) - super(VehiclePanel, self).remove_display_region() + super(DashBoard, self).remove_display_region() def add_display_region(self, display_region): - super(VehiclePanel, self).add_display_region(display_region) + super(DashBoard, self).add_display_region(display_region) self.buffer.set_active(True) self.origin.reparentTo(self.aspect2d_np) def destroy(self): - super(VehiclePanel, self).destroy() + super(DashBoard, self).destroy() for para in self.para_vis_np.values(): para.removeNode() self.aspect2d_np.removeNode() diff --git a/metadrive/component/sensors/depth_camera.py b/metadrive/component/sensors/depth_camera.py index 922617979..072ef9171 100644 --- a/metadrive/component/sensors/depth_camera.py +++ b/metadrive/component/sensors/depth_camera.py @@ -1,12 +1,10 @@ -import cv2 -from panda3d.core import Shader, RenderState, ShaderAttrib, GeoMipTerrain, PNMImage, Texture, LightAttrib, \ +from panda3d.core import Shader, RenderState, ShaderAttrib, GeoMipTerrain, PNMImage, LightAttrib, \ TextureAttrib, ColorAttrib from metadrive.component.sensors.base_camera import BaseCamera from metadrive.constants import CamMask from metadrive.constants import RENDER_MODE_NONE from metadrive.engine.asset_loader import AssetLoader -from panda3d.core import FrameBufferProperties class DepthCamera(BaseCamera): @@ -14,17 +12,16 @@ class DepthCamera(BaseCamera): CAM_MASK = CamMask.DepthCam GROUND_HEIGHT = -0.5 - VIEW_GROUND = False + VIEW_GROUND = True GROUND = None GROUND_MODEL = None + num_channels = 1 + shader_name = "depth_cam" + def __init__(self, width, height, engine, *, cuda=False): self.BUFFER_W, self.BUFFER_H = width, height - self.VIEW_GROUND = True # default true - frame_buffer_property = FrameBufferProperties() - frame_buffer_property.set_rgba_bits(8, 8, 8, 0) # disable alpha for RGB camera - # TODO It can be made more efficient by only using one channel - super(DepthCamera, self).__init__(engine, False, cuda) + super(DepthCamera, self).__init__(engine, cuda) cam = self.get_cam() lens = self.get_lens() @@ -40,21 +37,6 @@ def __init__(self, width, height, engine, *, cuda=False): # vert_path = AssetLoader.file_path("shaders", "depth_cam_gles.vert.glsl") # frag_path = AssetLoader.file_path("shaders", "depth_cam_gles.frag.glsl") # else: - from metadrive.utils import is_mac - if is_mac(): - vert_path = AssetLoader.file_path("shaders", "depth_cam_mac.vert.glsl") - frag_path = AssetLoader.file_path("shaders", "depth_cam_mac.frag.glsl") - else: - vert_path = AssetLoader.file_path("shaders", "depth_cam.vert.glsl") - frag_path = AssetLoader.file_path("shaders", "depth_cam.frag.glsl") - custom_shader = Shader.load(Shader.SL_GLSL, vertex=vert_path, fragment=frag_path) - cam.node().setInitialState( - RenderState.make( - LightAttrib.makeAllOff(), TextureAttrib.makeOff(), ColorAttrib.makeOff(), - ShaderAttrib.make(custom_shader, 1) - ) - ) - if self.VIEW_GROUND: ground = PNMImage(513, 513, 4) ground.fill(1., 1., 1.) @@ -80,3 +62,24 @@ def track(self, base_object): # self.GROUND_MODEL.setP(-base_object.origin.getR()) # self.GROUND_MODEL.setR(-base_object.origin.getR()) return super(DepthCamera, self).track(base_object) + + def _setup_effect(self): + """ + Setup Camera Effect enabling depth calculation + + Returns: None + """ + from metadrive.utils import is_mac + if is_mac(): + vert_path = AssetLoader.file_path("shaders", "{}_mac.vert.glsl".format(self.shader_name)) + frag_path = AssetLoader.file_path("shaders", "{}_mac.frag.glsl".format(self.shader_name)) + else: + vert_path = AssetLoader.file_path("shaders", "{}.vert.glsl".format(self.shader_name)) + frag_path = AssetLoader.file_path("shaders", "{}.frag.glsl".format(self.shader_name)) + custom_shader = Shader.load(Shader.SL_GLSL, vertex=vert_path, fragment=frag_path) + self.get_cam().node().setInitialState( + RenderState.make( + LightAttrib.makeAllOff(), TextureAttrib.makeOff(), ColorAttrib.makeOff(), + ShaderAttrib.make(custom_shader, 1) + ) + ) diff --git a/metadrive/component/sensors/distance_detector.py b/metadrive/component/sensors/distance_detector.py index db77d17eb..0d4c653ba 100644 --- a/metadrive/component/sensors/distance_detector.py +++ b/metadrive/component/sensors/distance_detector.py @@ -4,7 +4,7 @@ import numpy as np from panda3d.core import NodePath, LVecBase4 -from metadrive.component.sensors import BaseSensor +from metadrive.component.sensors.base_sensor import BaseSensor from metadrive.constants import CamMask, CollisionGroup from metadrive.engine.asset_loader import AssetLoader from metadrive.engine.logger import get_logger diff --git a/metadrive/component/sensors/mini_map.py b/metadrive/component/sensors/mini_map.py index 6e9caaee6..275fc8504 100644 --- a/metadrive/component/sensors/mini_map.py +++ b/metadrive/component/sensors/mini_map.py @@ -7,13 +7,10 @@ class MiniMap(BaseCamera): CAM_MASK = CamMask.MiniMap - display_region_size = [0., 1 / 3, 0.8, 1.0] def __init__(self, width, height, z_pos, engine, *, cuda=False): self.BUFFER_W, self.BUFFER_H, height = width, height, z_pos - frame_buffer_property = FrameBufferProperties() - frame_buffer_property.set_rgba_bits(8, 8, 8, 0) # disable alpha for RGB camera - super(MiniMap, self).__init__(engine=engine, need_cuda=cuda, frame_buffer_property=frame_buffer_property) + super(MiniMap, self).__init__(engine=engine, need_cuda=cuda) cam = self.get_cam() lens = self.get_lens() diff --git a/metadrive/component/sensors/rgb_camera.py b/metadrive/component/sensors/rgb_camera.py index b1b163da4..ab9131aa8 100644 --- a/metadrive/component/sensors/rgb_camera.py +++ b/metadrive/component/sensors/rgb_camera.py @@ -1,8 +1,9 @@ +import panda3d.core as p3d +from direct.filter.FilterManager import FilterManager +from simplepbr import _load_shader_str + from metadrive.component.sensors.base_camera import BaseCamera from metadrive.constants import CamMask -from metadrive.engine.engine_utils import engine_initialized, get_global_config -from direct.filter.CommonFilters import CommonFilters -from panda3d.core import FrameBufferProperties class RGBCamera(BaseCamera): @@ -14,13 +15,40 @@ class RGBCamera(BaseCamera): def __init__(self, width, height, engine, *, cuda=False): self.BUFFER_W, self.BUFFER_H = width, height - frame_buffer_property = FrameBufferProperties() - frame_buffer_property.set_rgba_bits(8, 8, 8, 0) # disable alpha for RGB camera - super(RGBCamera, self).__init__(engine, True, cuda, frame_buffer_property=frame_buffer_property) + super(RGBCamera, self).__init__(engine, cuda) cam = self.get_cam() lens = self.get_lens() # cam.lookAt(0, 2.4, 1.3) cam.lookAt(0, 10.4, 1.6) - lens.setFov(60) - # lens.setAspectRatio(2.0) + + def _setup_effect(self): + """ + Setup simple PBR effect + Returns: None + + """ + self.scene_tex = None + self.manager = FilterManager(self.buffer, self.cam) + fbprops = p3d.FrameBufferProperties() + fbprops.float_color = True + fbprops.set_rgba_bits(16, 16, 16, 16) + fbprops.set_depth_bits(24) + fbprops.set_multisamples(self.engine.pbrpipe.msaa_samples) + self.scene_tex = p3d.Texture() + self.scene_tex.set_format(p3d.Texture.F_rgba16) + self.scene_tex.set_component_type(p3d.Texture.T_float) + self.tonemap_quad = self.manager.render_scene_into(colortex=self.scene_tex, fbprops=fbprops) + # + defines = {} + # + post_vert_str = _load_shader_str('post.vert', defines) + post_frag_str = _load_shader_str('tonemap.frag', defines) + tonemap_shader = p3d.Shader.make( + p3d.Shader.SL_GLSL, + vertex=post_vert_str, + fragment=post_frag_str, + ) + self.tonemap_quad.set_shader(tonemap_shader) + self.tonemap_quad.set_shader_input('tex', self.scene_tex) + self.tonemap_quad.set_shader_input('exposure', 1.0) diff --git a/metadrive/component/sensors/rgb_depth_camera.py b/metadrive/component/sensors/rgb_depth_camera.py new file mode 100644 index 000000000..a9a58456d --- /dev/null +++ b/metadrive/component/sensors/rgb_depth_camera.py @@ -0,0 +1,10 @@ +from metadrive.component.sensors.depth_camera import DepthCamera + + +class RGBDepthCamera(DepthCamera): + """ + (Deprecated) Same as RGBCamera, while the forth channel is for storing depth information + """ + raise DeprecationWarning("This one won't work currently") + shader_name = "rgb_depth_cam" + VIEW_GROUND = False diff --git a/metadrive/component/sensors/semantic_camera.py b/metadrive/component/sensors/semantic_camera.py index f44812490..e32d12c97 100644 --- a/metadrive/component/sensors/semantic_camera.py +++ b/metadrive/component/sensors/semantic_camera.py @@ -22,8 +22,7 @@ class SemanticCamera(BaseCamera): def __init__(self, width, height, engine, *, cuda=False): self.BUFFER_W, self.BUFFER_H = width, height self.VIEW_GROUND = True # default true - # The framebuffer can not be 3 channel like RGB Camera... - super(SemanticCamera, self).__init__(engine, False, cuda) + super(SemanticCamera, self).__init__(engine, cuda) cam = self.get_cam() lens = self.get_lens() @@ -35,19 +34,6 @@ def __init__(self, width, height, engine, *, cuda=False): if self.engine.mode == RENDER_MODE_NONE or not AssetLoader.initialized(): return - # setup camera - cam = cam.node() - cam.setInitialState( - RenderState.make( - ShaderAttrib.makeOff(), LightAttrib.makeAllOff(), TextureAttrib.makeOff(), - ColorAttrib.makeFlat((0, 0, 1, 1)), 1 - ) - ) - cam.setTagStateKey("type") - for t in [v for v, m in vars(Semantics).items() if not (v.startswith('_') or callable(m))]: - label, c = getattr(Semantics, t) - cam.setTagState(label, RenderState.make(ColorAttrib.makeFlat((c[0] / 255, c[1] / 255, c[2] / 255, 1)), 1)) - if self.VIEW_GROUND: ground = PNMImage(513, 513, 4) ground.fill(1., 1., 1.) @@ -74,3 +60,35 @@ def track(self, base_object): # self.GROUND_MODEL.setP(-base_object.origin.getR()) # self.GROUND_MODEL.setR(-base_object.origin.getR()) return super(SemanticCamera, self).track(base_object) + + def _setup_effect(self): + """ + Use tag to apply color to different object class + Returns: None + + """ + # setup camera + cam = self.get_cam().node() + cam.setInitialState( + RenderState.make( + ShaderAttrib.makeOff(), LightAttrib.makeAllOff(), TextureAttrib.makeOff(), + ColorAttrib.makeFlat((0, 0, 1, 1)), 1 + ) + ) + cam.setTagStateKey("type") + for t in [v for v, m in vars(Semantics).items() if not (v.startswith('_') or callable(m))]: + label, c = getattr(Semantics, t) + cam.setTagState(label, RenderState.make(ColorAttrib.makeFlat((c[0] / 255, c[1] / 255, c[2] / 255, 1)), 1)) + + def _create_buffer(self, width, height, frame_buffer_property): + """ + The buffer should be created without frame_buffer_property + Args: + width: Image width + height: Image height + frame_buffer_property: disabled in Semantic Camera + + Returns: Buffer object + + """ + return self.engine.win.makeTextureBuffer("camera", width, height) diff --git a/metadrive/engine/core/engine_core.py b/metadrive/engine/core/engine_core.py index fef5534e5..514ec9975 100644 --- a/metadrive/engine/core/engine_core.py +++ b/metadrive/engine/core/engine_core.py @@ -10,7 +10,7 @@ from panda3d.bullet import BulletDebugNode from panda3d.core import AntialiasAttrib, loadPrcFileData, LineSegs, PythonCallbackObject, Vec3, NodePath, LVecBase4 -from metadrive.component.sensors import BaseSensor +from metadrive.component.sensors.base_sensor import BaseSensor from metadrive.constants import RENDER_MODE_OFFSCREEN, RENDER_MODE_NONE, RENDER_MODE_ONSCREEN, EDITION, CamMask, \ BKG_COLOR from metadrive.engine.asset_loader import initialize_asset_loader, close_asset_loader, randomize_cover, get_logo_file diff --git a/metadrive/engine/core/image_buffer.py b/metadrive/engine/core/image_buffer.py index 2a38ad5a7..5a3fd12de 100644 --- a/metadrive/engine/core/image_buffer.py +++ b/metadrive/engine/core/image_buffer.py @@ -3,7 +3,7 @@ import panda3d.core as p3d from simplepbr import _load_shader_str from typing import Union, List - +from panda3d.core import FrameBufferProperties import numpy as np from panda3d.core import NodePath, Vec3, Vec4, Camera, PNMImage, Shader, RenderState, ShaderAttrib @@ -11,6 +11,12 @@ class ImageBuffer: + """ + This is a wrapper for FrameBuffer, associated with a camera. The camera scene in the camera view will be rendered + into the buffer. Thus, we can access the image in the buffer and can apply effect to the image to implement + DepthCamera, SemanticCamera and So on. It also allows opening a display region on the main window to show sensor + output. + """ LINE_FRAME_COLOR = (0.8, 0.8, 0.8, 0) CAM_MASK = None BUFFER_W = 84 # left to right @@ -19,8 +25,8 @@ class ImageBuffer: # display_bottom = 0.8 # display_top = 1 display_region = None - display_region_size = [1 / 3, 2 / 3, 0.8, 1.0] line_borders = [] + num_channels = 3 def __init__( self, @@ -30,7 +36,6 @@ def __init__( bkg_color: Union[Vec4, Vec3], parent_node: NodePath = None, frame_buffer_property=None, - setup_pbr=False, engine=None ): self.logger = get_logger() @@ -53,12 +58,7 @@ def __init__( return # self.texture = Texture() - if frame_buffer_property is None: - self.buffer = self.engine.win.makeTextureBuffer("camera", width, height) - else: - self.buffer = self.engine.win.makeTextureBuffer("camera", width, height, fbp=frame_buffer_property) - # now we have to setup a new scene graph to make this scene - + self.buffer = self._create_buffer(width, height, frame_buffer_property) self.origin = NodePath("new render") # this takes care of setting up their camera properly @@ -71,43 +71,39 @@ def __init__( self.cam.node().setCameraMask(self.CAM_MASK) if parent_node is not None: self.origin.reparentTo(parent_node) - self.scene_tex = None - if setup_pbr: - self.manager = FilterManager(self.buffer, self.cam) - fbprops = p3d.FrameBufferProperties() - fbprops.float_color = True - fbprops.set_rgba_bits(16, 16, 16, 16) - fbprops.set_depth_bits(24) - fbprops.set_multisamples(self.engine.pbrpipe.msaa_samples) - self.scene_tex = p3d.Texture() - self.scene_tex.set_format(p3d.Texture.F_rgba16) - self.scene_tex.set_component_type(p3d.Texture.T_float) - self.tonemap_quad = self.manager.render_scene_into(colortex=self.scene_tex, fbprops=fbprops) - # - defines = {} - # - post_vert_str = _load_shader_str('post.vert', defines) - post_frag_str = _load_shader_str('tonemap.frag', defines) - tonemap_shader = p3d.Shader.make( - p3d.Shader.SL_GLSL, - vertex=post_vert_str, - fragment=post_frag_str, - ) - self.tonemap_quad.set_shader(tonemap_shader) - self.tonemap_quad.set_shader_input('tex', self.scene_tex) - self.tonemap_quad.set_shader_input('exposure', 1.0) - + self._setup_effect() self.logger.debug("Load Image Buffer: {}".format(self.__class__.__name__)) + def _create_buffer(self, width, height, frame_buffer_property): + """ + Create the buffer object to render the scene into it + Args: + width: image width + height: image height + frame_buffer_property: panda3d.core.FrameBufferProperties + + Returns: buffer object + + """ + if frame_buffer_property is None: + frame_buffer_property = FrameBufferProperties() + frame_buffer_property.set_rgba_bits(8, 8, 8, 0) # disable alpha for RGB camera + return self.engine.win.makeTextureBuffer("camera", width, height, fbp=frame_buffer_property) + + def _setup_effect(self): + """ + Apply effect to the render the scene. Usually setup shader here + Returns: None + + """ + pass + def get_rgb_array_cpu(self): origin_img = self.buffer.getDisplayRegion(1).getScreenshot() img = np.frombuffer(origin_img.getRamImage().getData(), dtype=np.uint8) img = img.reshape((origin_img.getYSize(), origin_img.getXSize(), -1)) - # img = np.swapaxes(img, 1, 0) img = img[::-1] - if img.shape[-1] == 4: - # To 3 channel - img = img[..., :-1] + img = img[..., :self.num_channels] return img @staticmethod @@ -175,7 +171,3 @@ def destroy(self): def __del__(self): self.logger.debug("{} is destroyed".format(self.__class__.__name__)) - - @classmethod - def update_display_region_size(cls, display_region_size): - cls.display_region_size = display_region_size diff --git a/metadrive/engine/core/main_camera.py b/metadrive/engine/core/main_camera.py index e99208212..f6e022301 100644 --- a/metadrive/engine/core/main_camera.py +++ b/metadrive/engine/core/main_camera.py @@ -2,7 +2,6 @@ import queue from collections import deque from typing import Tuple - import numpy as np from direct.controls.InputState import InputState from panda3d.core import Vec3, Point3, PNMImage @@ -22,12 +21,13 @@ from panda3d.core import GraphicsOutput, Texture, GraphicsStateGuardianBase, DisplayRegionDrawCallbackData except ImportError: _cuda_enable = False -from metadrive.component.sensors import BaseSensor +from metadrive.component.sensors.base_sensor import BaseSensor class MainCamera(BaseSensor): """ - Only chase vehicle now + It is a third-person perspective camera for chasing the vehicle. The view in this camera will be rendered into the + main image buffer (main window). It is also a sensor, so perceive() can be called to access the rendered image. """ queue_length = 3 @@ -41,6 +41,8 @@ class MainCamera(BaseSensor): MOUSE_MOVE_INTO_LATENCY = 2 MOUSE_SPEED_MULTIPLIER = 1 + num_channels = 3 + def __init__(self, engine, camera_height: float, camera_dist: float): self._origin_height = camera_height # self.engine = engine @@ -423,10 +425,10 @@ def perceive(self, vehicle, clip): img = np.frombuffer(origin_img.getRamImage().getData(), dtype=np.uint8) img = img.reshape((origin_img.getYSize(), origin_img.getXSize(), 4)) img = img[::-1] - img = img[..., :-1] + img = img[..., :self.num_channels] if not clip: - return img.astype(np.uint8) + return img.astype(np.uint8, copy=False, order="C") else: return img / 255 diff --git a/metadrive/engine/interface.py b/metadrive/engine/interface.py index 033683b34..26e7fed49 100644 --- a/metadrive/engine/interface.py +++ b/metadrive/engine/interface.py @@ -25,7 +25,7 @@ class Interface: def __init__(self, base_engine): self._node_path_list = [] # self.engine = base_engine - self.vehicle_panel = None + self.dashboard = None self.right_panel = None self.mid_panel = None self.left_panel = None @@ -46,8 +46,8 @@ def __init__(self, base_engine): def after_step(self): if self.engine.current_track_vehicle is not None and self.need_interface and self.engine.mode != RENDER_MODE_NONE: track_v = self.engine.current_track_vehicle - if self.vehicle_panel is not None: - self.vehicle_panel.update_vehicle_state(track_v) + if self.dashboard is not None: + self.dashboard.update_vehicle_state(track_v) self._render_contact_result(track_v.contact_results) if hasattr(track_v, "navigation") and track_v.navigation is not None: self._update_navi_arrow(track_v.navigation.navi_arrow_dir) @@ -70,7 +70,7 @@ def init_interface(self): else: raise ValueError("Can not add > 3 panels!") if panel_name == "dashboard": - self.vehicle_panel = self.engine.get_sensor(panel_name) + self.dashboard = self.engine.get_sensor(panel_name) self.arrow = self.engine.aspect2d.attachNewNode("arrow") self._node_path_list.append(self.arrow) diff --git a/metadrive/envs/base_env.py b/metadrive/envs/base_env.py index d4ecd4e5e..406e910c5 100644 --- a/metadrive/envs/base_env.py +++ b/metadrive/envs/base_env.py @@ -10,7 +10,7 @@ from metadrive.component.sensors.base_camera import BaseCamera from metadrive.component.sensors.distance_detector import LaneLineDetector, SideDetector from metadrive.component.sensors.lidar import Lidar -from metadrive.component.sensors.vehicle_panel import VehiclePanel +from metadrive.component.sensors.dashboard import DashBoard from metadrive.constants import RENDER_MODE_NONE, DEFAULT_AGENT from metadrive.constants import RENDER_MODE_ONSCREEN, RENDER_MODE_OFFSCREEN from metadrive.constants import TerminationState @@ -291,12 +291,12 @@ def _post_process_config(self, config): config["sensors"] = filtered config["interface_panel"] = [] - # Merge vehicle_panel config with sensors + # Merge dashboard config with sensors to_use = [] if not config["render_pipeline"]: for panel in config["interface_panel"]: if panel == "dashboard": - config["sensors"]["dashboard"] = (VehiclePanel, ) + config["sensors"]["dashboard"] = (DashBoard, ) if panel not in config["sensors"]: self.logger.warning( "Fail to add sensor: {} to the interface. Remove it from panel list!".format(panel) @@ -331,7 +331,7 @@ def _post_process_config(self, config): else: config["_render_mode"] = RENDER_MODE_NONE for sensor in config["sensors"].values(): - if sensor[0] == "MainCamera" or (issubclass(BaseCamera, sensor[0]) and sensor[0] != VehiclePanel): + if sensor[0] == "MainCamera" or (issubclass(BaseCamera, sensor[0]) and sensor[0] != DashBoard): config["_render_mode"] = RENDER_MODE_OFFSCREEN break self.logger.info("Render Mode: {}".format(config["_render_mode"])) diff --git a/metadrive/envs/marl_envs/multi_agent_metadrive.py b/metadrive/envs/marl_envs/multi_agent_metadrive.py index ae50367e5..39f834bef 100644 --- a/metadrive/envs/marl_envs/multi_agent_metadrive.py +++ b/metadrive/envs/marl_envs/multi_agent_metadrive.py @@ -1,5 +1,4 @@ import copy -from metadrive.component.sensors.vehicle_panel import VehiclePanel import logging from typing import Dict, Any diff --git a/metadrive/envs/real_data_envs/nuplan_env.py b/metadrive/envs/real_data_envs/nuplan_env.py index 401c93a65..bcec5ecd9 100644 --- a/metadrive/envs/real_data_envs/nuplan_env.py +++ b/metadrive/envs/real_data_envs/nuplan_env.py @@ -3,7 +3,7 @@ import numpy as np -from metadrive.component.sensors.vehicle_panel import VehiclePanel +from metadrive.component.sensors.dashboard import DashBoard from metadrive.component.vehicle_navigation_module.trajectory_navigation import NuPlanTrajectoryNavigation from metadrive.constants import TerminationState from metadrive.envs.base_env import BaseEnv diff --git a/metadrive/envs/scenario_env.py b/metadrive/envs/scenario_env.py index 84ed9ef50..72d3add22 100644 --- a/metadrive/envs/scenario_env.py +++ b/metadrive/envs/scenario_env.py @@ -3,7 +3,6 @@ """ import numpy as np -from metadrive.component.sensors.vehicle_panel import VehiclePanel from metadrive.component.vehicle_navigation_module.trajectory_navigation import TrajectoryNavigation from metadrive.constants import TerminationState from metadrive.engine.asset_loader import AssetLoader diff --git a/metadrive/obs/image_obs.py b/metadrive/obs/image_obs.py index cce8d85f3..bd611359c 100644 --- a/metadrive/obs/image_obs.py +++ b/metadrive/obs/image_obs.py @@ -1,4 +1,5 @@ import gymnasium as gym +from metadrive.component.sensors.base_camera import BaseCamera import numpy as np from metadrive.component.vehicle.base_vehicle import BaseVehicle @@ -52,14 +53,17 @@ def __init__(self, config, image_source: str, clip_rgb: bool): self.image_source = image_source super(ImageObservation, self).__init__(config) self.rgb_clip = clip_rgb - self.state = np.zeros(self.observation_space.shape, dtype=np.float32) + self.state = np.zeros(self.observation_space.shape, dtype=np.float32 if self.rgb_clip else np.uint8) if self.enable_cuda: self.state = cp.asarray(self.state) @property def observation_space(self): + sensor_cls = self.config["sensors"][self.image_source][0] + assert sensor_cls == "MainCamera" or issubclass(sensor_cls, BaseCamera), "Sensor should be BaseCamera" + channel = sensor_cls.num_channels if sensor_cls != "MainCamera" else 3 shape = (self.config["sensors"][self.image_source][2], self.config["sensors"][self.image_source][1] - ) + ((self.STACK_SIZE, ) if self.config["rgb_to_grayscale"] else (3, self.STACK_SIZE)) + ) + ((self.STACK_SIZE, ) if self.config["rgb_to_grayscale"] else (channel, self.STACK_SIZE)) if self.rgb_clip: return gym.spaces.Box(-0.0, 1.0, shape=shape, dtype=np.float32) else: diff --git a/metadrive/tests/scripts/capture_obs.py b/metadrive/tests/scripts/capture_obs.py index 1c7bded94..964215c8b 100644 --- a/metadrive/tests/scripts/capture_obs.py +++ b/metadrive/tests/scripts/capture_obs.py @@ -54,7 +54,6 @@ # for sensor in env.vehicle.image_sensors.values(): # sensor.remove_display_region(env.engine) - # env.vehicle.vehicle_panel.remove_display_region(env.engine) # env.vehicle.contact_result_render.detachNode() # env.vehicle.navigation._right_arrow.detachNode() diff --git a/metadrive/tests/test_examples/test_examples.py b/metadrive/tests/test_examples/test_examples.py new file mode 100644 index 000000000..c5f1a758a --- /dev/null +++ b/metadrive/tests/test_examples/test_examples.py @@ -0,0 +1,40 @@ +import os.path +import subprocess +from metadrive import MetaDrive_PACKAGE_DIR +import time +import pytest + +examples = [ + "draw_maps.py", "drive_in_multi_agent_env.py --top_down", "drive_in_waymo_env.py --top_down", + "drive_in_waymo_env.py --reactive_traffic", "drive_in_safe_metadrive_env.py", "drive_in_single_agent_env.py", + "procedural_generation.py", "profile_metadrive.py", "profile_metadrive_marl.py", "top_down_metadrive.py" +] +examples_dir_path = os.path.join(MetaDrive_PACKAGE_DIR, "examples") +scripts = [os.path.join(examples_dir_path, exp) for exp in examples] + + +@pytest.mark.parametrize("script", scripts, ids=examples) +def test_script(script, timeout=60): + """ + Run script in a subprocess and check its running time. + Args: + script: the path to the script + timeout: script that can run over `timeout` seconds can pass the test + + Returns: None + + """ + start_time = time.time() + + # Run your script using subprocess + process = subprocess.Popen(['python', script]) + + # Wait for the script to finish or timeout after 60 seconds + try: + process.wait(timeout=timeout) + except subprocess.TimeoutExpired: + # If the script is still running after 60 seconds, terminate it and pass the test + process.kill() + finally: + runtime = time.time() - start_time + assert runtime >= 0, "Script terminated unexpectedly" diff --git a/metadrive/tests/test_installation.py b/metadrive/tests/test_installation.py index cc657fc96..b1348a9a8 100644 --- a/metadrive/tests/test_installation.py +++ b/metadrive/tests/test_installation.py @@ -6,7 +6,7 @@ from metadrive import MetaDrive_PACKAGE_DIR from metadrive.component.sensors.mini_map import MiniMap from metadrive.component.sensors.rgb_camera import RGBCamera -from metadrive.component.sensors.vehicle_panel import VehiclePanel +from metadrive.component.sensors.dashboard import DashBoard from metadrive.envs.metadrive_env import MetaDriveEnv diff --git a/metadrive/tests/test_sensors/test_depth_cam.py b/metadrive/tests/test_sensors/test_depth_cam.py new file mode 100644 index 000000000..f65ca0f4a --- /dev/null +++ b/metadrive/tests/test_sensors/test_depth_cam.py @@ -0,0 +1,61 @@ +import pytest + +from metadrive.component.sensors.depth_camera import DepthCamera +from metadrive.envs.metadrive_env import MetaDriveEnv + +blackbox_test_configs = dict( + # standard=dict(stack_size=3, width=256, height=128, rgb_clip=True), + small=dict(stack_size=1, width=64, height=32, rgb_clip=False), +) + + +@pytest.mark.parametrize("config", list(blackbox_test_configs.values()), ids=list(blackbox_test_configs.keys())) +def test_depth_cam(config, render=False): + """ + Test the output shape of Depth camera. This can not make sure the correctness of rendered image but only for + checking the shape of image output and image retrieve pipeline + Args: + config: test parameter + render: render with cv2 + + Returns: None + + """ + env = MetaDriveEnv( + { + "num_scenarios": 1, + "traffic_density": 0.1, + "map": "S", + "start_seed": 4, + "stack_size": config["stack_size"], + "vehicle_config": dict(image_source="camera"), + "sensors": { + "camera": (DepthCamera, config["width"], config["height"]) + }, + "interface_panel": ["dashboard", "camera"], + "image_observation": True, # it is a switch telling metadrive to use rgb as observation + "rgb_clip": config["rgb_clip"], # clip rgb to range(0,1) instead of (0, 255) + } + ) + env.reset() + try: + import cv2 + import time + start = time.time() + for i in range(1, 10): + o, r, tm, tc, info = env.step([0, 1]) + assert env.observation_space.contains(o) + # Reverse + assert o["image"].shape == ( + config["height"], config["width"], DepthCamera.num_channels, config["stack_size"] + ) + if render: + cv2.imshow('img', o["image"][..., -1]) + cv2.waitKey(1) + print("FPS:", 10 / (time.time() - start)) + finally: + env.close() + + +if __name__ == '__main__': + test_depth_cam(config=blackbox_test_configs["small"], render=True) diff --git a/metadrive/tests/test_sensors/test_main_camera.py b/metadrive/tests/test_sensors/test_main_camera.py new file mode 100644 index 000000000..709f36ff4 --- /dev/null +++ b/metadrive/tests/test_sensors/test_main_camera.py @@ -0,0 +1,56 @@ +import pytest + +from metadrive.envs.metadrive_env import MetaDriveEnv + +blackbox_test_configs = dict( + standard=dict(stack_size=3, width=256, height=128, rgb_clip=True), + small=dict(stack_size=1, width=64, height=32, rgb_clip=False), +) + + +@pytest.mark.parametrize("config", list(blackbox_test_configs.values()), ids=list(blackbox_test_configs.keys())) +def test_main_camera(config, render=False): + """ + Test the output shape of main camera. This can not make sure the correctness of rendered image but only for + checking the shape of image output and image retrieve pipeline + Args: + config: test parameter + render: render with cv2 + + Returns: None + + """ + env = MetaDriveEnv( + { + "num_scenarios": 1, + "traffic_density": 0.1, + "map": "S", + "start_seed": 4, + "window_size": (config["width"], config["height"]), + "stack_size": config["stack_size"], + "vehicle_config": dict(image_source="main_camera"), + "interface_panel": [], + "image_observation": True, # it is a switch telling metadrive to use rgb as observation + "rgb_clip": config["rgb_clip"], # clip rgb to range(0,1) instead of (0, 255) + } + ) + env.reset() + try: + import cv2 + import time + start = time.time() + for i in range(1, 10): + o, r, tm, tc, info = env.step([0, 1]) + assert env.observation_space.contains(o) + # Reverse + assert o["image"].shape == (config["height"], config["width"], 3, config["stack_size"]) + if render: + cv2.imshow('img', o["image"][..., -1]) + cv2.waitKey(1) + print("FPS:", 10 / (time.time() - start)) + finally: + env.close() + + +if __name__ == '__main__': + test_main_camera(config=blackbox_test_configs["small"], render=True) diff --git a/metadrive/tests/test_sensors/test_rgb_cam.py b/metadrive/tests/test_sensors/test_rgb_cam.py new file mode 100644 index 000000000..ab6322ac8 --- /dev/null +++ b/metadrive/tests/test_sensors/test_rgb_cam.py @@ -0,0 +1,59 @@ +import pytest + +from metadrive.component.sensors.rgb_camera import RGBCamera +from metadrive.envs.metadrive_env import MetaDriveEnv + +blackbox_test_configs = dict( + standard=dict(stack_size=3, width=256, height=128, rgb_clip=True), + # small=dict(stack_size=1, width=64, height=32, rgb_clip=False), +) + + +@pytest.mark.parametrize("config", list(blackbox_test_configs.values()), ids=list(blackbox_test_configs.keys())) +def test_rgb_cam(config, render=False): + """ + Test the output shape of rgb camera. This can not make sure the correctness of rendered image but only for + checking the shape of image output and image retrieve pipeline + Args: + config: test parameter + render: render with cv2 + + Returns: None + + """ + env = MetaDriveEnv( + { + "num_scenarios": 1, + "traffic_density": 0.1, + "map": "S", + "start_seed": 4, + "stack_size": config["stack_size"], + "vehicle_config": dict(image_source="rgb_camera"), + "sensors": { + "rgb_camera": (RGBCamera, config["width"], config["height"]) + }, + "interface_panel": ["dashboard", "rgb_camera"], + "image_observation": True, # it is a switch telling metadrive to use rgb as observation + "rgb_clip": config["rgb_clip"], # clip rgb to range(0,1) instead of (0, 255) + } + ) + env.reset() + try: + import cv2 + import time + start = time.time() + for i in range(1, 10): + o, r, tm, tc, info = env.step([0, 1]) + assert env.observation_space.contains(o) + # Reverse + assert o["image"].shape == (config["height"], config["width"], 3, config["stack_size"]) + if render: + cv2.imshow('img', o["image"][..., -1]) + cv2.waitKey(1) + print("FPS:", 10 / (time.time() - start)) + finally: + env.close() + + +if __name__ == '__main__': + test_rgb_cam(config=blackbox_test_configs["small"], render=True) diff --git a/metadrive/tests/test_sensors/test_semantic_cam.py b/metadrive/tests/test_sensors/test_semantic_cam.py new file mode 100644 index 000000000..31969443b --- /dev/null +++ b/metadrive/tests/test_sensors/test_semantic_cam.py @@ -0,0 +1,61 @@ +import pytest + +from metadrive.component.sensors.semantic_camera import SemanticCamera +from metadrive.envs.metadrive_env import MetaDriveEnv + +blackbox_test_configs = dict( + # standard=dict(stack_size=3, width=256, height=128, rgb_clip=True), + small=dict(stack_size=1, width=64, height=32, rgb_clip=False), +) + + +@pytest.mark.parametrize("config", list(blackbox_test_configs.values()), ids=list(blackbox_test_configs.keys())) +def test_semantic_cam(config, render=False): + """ + Test the output shape of Semantic camera. This can NOT make sure the correctness of rendered image but only for + checking the shape of image output and image retrieve pipeline + Args: + config: test parameter + render: render with cv2 + + Returns: None + + """ + env = MetaDriveEnv( + { + "num_scenarios": 1, + "traffic_density": 0.1, + "map": "S", + "start_seed": 4, + "stack_size": config["stack_size"], + "vehicle_config": dict(image_source="camera"), + "sensors": { + "camera": (SemanticCamera, config["width"], config["height"]) + }, + "interface_panel": ["dashboard", "camera"], + "image_observation": True, # it is a switch telling metadrive to use rgb as observation + "rgb_clip": config["rgb_clip"], # clip rgb to range(0,1) instead of (0, 255) + } + ) + env.reset() + try: + import cv2 + import time + start = time.time() + for i in range(1, 10): + o, r, tm, tc, info = env.step([0, 1]) + assert env.observation_space.contains(o) + # Reverse + assert o["image"].shape == ( + config["height"], config["width"], SemanticCamera.num_channels, config["stack_size"] + ) + if render: + cv2.imshow('img', o["image"][..., -1]) + cv2.waitKey(1) + print("FPS:", 10 / (time.time() - start)) + finally: + env.close() + + +if __name__ == '__main__': + test_semantic_cam(config=blackbox_test_configs["small"], render=True) diff --git a/metadrive/tests/tools/adjust_collision_model.py b/metadrive/tests/tools/adjust_collision_model.py index 19b187a0b..72d6260f6 100644 --- a/metadrive/tests/tools/adjust_collision_model.py +++ b/metadrive/tests/tools/adjust_collision_model.py @@ -2,7 +2,7 @@ from metadrive.component.vehicle.base_vehicle import BaseVehicle from metadrive.component.sensors.mini_map import MiniMap from metadrive.component.sensors.rgb_camera import RGBCamera -from metadrive.component.sensors.vehicle_panel import VehiclePanel +from metadrive.component.sensors.dashboard import DashBoard from metadrive.envs.metadrive_env import MetaDriveEnv from metadrive.utils import setup_logger @@ -20,7 +20,7 @@ "manual_control": True, "use_render": True, "decision_repeat": 5, - "interface_panel": [MiniMap, VehiclePanel, RGBCamera], + "interface_panel": [MiniMap, DashBoard, RGBCamera], "need_inverse_traffic": False, "rgb_clip": True, "map": "SSS", diff --git a/metadrive/tests/vis_block/vis_block_base.py b/metadrive/tests/vis_block/vis_block_base.py index 64a7c8df0..d1b3ec529 100644 --- a/metadrive/tests/vis_block/vis_block_base.py +++ b/metadrive/tests/vis_block/vis_block_base.py @@ -15,6 +15,7 @@ class TestBlock(ShowBase.ShowBase): def __init__(self, debug=False, window_type="onscreen"): self.debug = debug super(TestBlock, self).__init__(windowType=window_type) + self.mode = "onscreen" self.setBackgroundColor(BKG_COLOR) if window_type != "none": self.setFrameRateMeter(True) diff --git a/metadrive/tests/vis_functionality/vis_depth_cam.py b/metadrive/tests/vis_functionality/vis_depth_cam.py new file mode 100644 index 000000000..960f628ea --- /dev/null +++ b/metadrive/tests/vis_functionality/vis_depth_cam.py @@ -0,0 +1,55 @@ +from metadrive.component.sensors.depth_camera import DepthCamera +from metadrive.envs.safe_metadrive_env import SafeMetaDriveEnv + +if __name__ == "__main__": + env = SafeMetaDriveEnv( + { + "num_scenarios": 1, + "traffic_density": 0., + "accident_prob": 1., + "start_seed": 4, + "map": "SSSSS", + "manual_control": False, + "use_render": False, + "image_observation": True, + "rgb_clip": True, + "interface_panel": ["depth_camera"], + "sensors": dict(depth_camera=(DepthCamera, 800, 600)), + "vehicle_config": dict(image_source="depth_camera"), + # "map_config": { + # BaseMap.GENERATE_TYPE: MapGenerateMethod.BIG_BLOCK_NUM, + # BaseMap.GENERATE_CONFIG: 12, + # BaseMap.LANE_WIDTH: 3.5, + # BaseMap.LANE_NUM: 3, + # } + } + ) + env.reset() + + def get_image(env): + depth_cam = env.vehicle.get_camera(env.vehicle.config["image_source"]) + rgb_cam = env.vehicle.get_camera("rgb_camera") + for h in range(-180, 180, 20): + env.engine.graphicsEngine.renderFrame() + depth_cam.get_cam().setH(h) + rgb_cam.get_cam().setH(h) + depth_cam.save_image(env.vehicle, "depth_{}.jpg".format(h)) + rgb_cam.save_image(env.vehicle, "rgb_{}.jpg".format(h)) + env.engine.screenshot() + + env.engine.accept("m", get_image, extraArgs=[env]) + import cv2 + + for i in range(1, 100000): + o, r, tm, tc, info = env.step([0, 1]) + assert env.observation_space.contains(o) + cv2.imshow('img', o["image"][..., -1]) + cv2.waitKey(1) + if env.config["use_render"]: + # for i in range(ImageObservation.STACK_SIZE): + # ObservationType.show_gray_scale_array(o["image"][:, :, i]) + env.render() + # if tm or tc: + # # print("Reset") + # env.reset() + env.close() diff --git a/metadrive/tests/vis_functionality/vis_depth_cam_no_ground.py b/metadrive/tests/vis_functionality/vis_depth_cam_no_ground.py deleted file mode 100644 index a9aa698ec..000000000 --- a/metadrive/tests/vis_functionality/vis_depth_cam_no_ground.py +++ /dev/null @@ -1,38 +0,0 @@ -from metadrive.component.map.base_map import BaseMap -from metadrive.component.map.pg_map import MapGenerateMethod -from metadrive.envs.metadrive_env import MetaDriveEnv - -if __name__ == "__main__": - env = MetaDriveEnv( - { - "num_scenarios": 1, - "traffic_density": 0.1, - "start_seed": 4, - "manual_control": True, - "use_render": True, - "image_observation": True, - "rgb_clip": True, - "vehicle_config": dict(depth_camera=(200, 88), image_source="depth_camera"), - "map_config": { - BaseMap.GENERATE_TYPE: MapGenerateMethod.BIG_BLOCK_NUM, - BaseMap.GENERATE_CONFIG: 12, - BaseMap.LANE_WIDTH: 3.5, - BaseMap.LANE_NUM: 3, - } - } - ) - env.reset() - env.engine.accept("m", env.vehicle.get_camera("depth_camera").save_image, extraArgs=[env.vehicle, "debug.jpg"]) - - for i in range(1, 100000): - o, r, tm, tc, info = env.step([0, 1]) - assert env.observation_space.contains(o) - if env.config["use_render"]: - # for i in range(ImageObservation.STACK_SIZE): - # ObservationType.show_gray_scale_array(o["image"][:, :, i]) - env.render(text={"can you see me": i}) - if tm or tc: - env.vehicle.get_camera("rgb_camera").save_image(env.vehicle) - # print("Reset") - env.reset() - env.close() diff --git a/metadrive/tests/vis_functionality/vis_grayscale_cam.py b/metadrive/tests/vis_functionality/vis_grayscale_cam.py index 1b7372050..4325d19c2 100644 --- a/metadrive/tests/vis_functionality/vis_grayscale_cam.py +++ b/metadrive/tests/vis_functionality/vis_grayscale_cam.py @@ -31,7 +31,7 @@ assert env.observation_space.contains(o) # save rgb_cam = env.vehicle.get_camera(env.vehicle.config["image_source"]) - rgb_cam.save_image(env.vehicle, name="{}.png".format(i)) + # rgb_cam.save_image(env.vehicle, name="{}.png".format(i)) cv2.imshow('img', o["image"][..., -1] / 255) cv2.waitKey(0) diff --git a/metadrive/tests/vis_functionality/vis_rgb_cam.py b/metadrive/tests/vis_functionality/vis_rgb_cam.py index 4887f32a8..f192a199e 100644 --- a/metadrive/tests/vis_functionality/vis_rgb_cam.py +++ b/metadrive/tests/vis_functionality/vis_rgb_cam.py @@ -34,7 +34,7 @@ assert env.observation_space.contains(o) # save rgb_cam = env.engine.get_sensor(env.vehicle.config["image_source"]) - rgb_cam.save_image(env.vehicle, name="{}.png".format(i)) + # rgb_cam.save_image(env.vehicle, name="{}.png".format(i)) cv2.imshow('img', o["image"][..., -1]) cv2.waitKey(1) diff --git a/metadrive/tests/vis_functionality/vis_depth_cam_ground.py b/metadrive/tests/vis_functionality/vis_rgb_depth_cam.py similarity index 90% rename from metadrive/tests/vis_functionality/vis_depth_cam_ground.py rename to metadrive/tests/vis_functionality/vis_rgb_depth_cam.py index 1086ecea4..e0f590234 100644 --- a/metadrive/tests/vis_functionality/vis_depth_cam_ground.py +++ b/metadrive/tests/vis_functionality/vis_rgb_depth_cam.py @@ -1,7 +1,8 @@ from metadrive.component.sensors.mini_map import MiniMap from metadrive.component.sensors.rgb_camera import RGBCamera from metadrive.component.sensors.depth_camera import DepthCamera -from metadrive.component.sensors.vehicle_panel import VehiclePanel +from metadrive.component.sensors.rgb_depth_camera import RGBDepthCamera +from metadrive.component.sensors.dashboard import DashBoard from metadrive.envs.safe_metadrive_env import SafeMetaDriveEnv if __name__ == "__main__": @@ -29,7 +30,7 @@ def get_image(env): "image_observation": True, "rgb_clip": True, "interface_panel": ["depth_camera"], - "sensors": dict(depth_camera=(DepthCamera, 800, 600)), + "sensors": dict(depth_camera=(RGBDepthCamera, 800, 600)), "vehicle_config": dict(image_source="depth_camera"), # "map_config": { # BaseMap.GENERATE_TYPE: MapGenerateMethod.BIG_BLOCK_NUM, diff --git a/metadrive/tests/vis_functionality/vis_semantic_cam.py b/metadrive/tests/vis_functionality/vis_semantic_cam.py index b2e17d941..7487a81f9 100644 --- a/metadrive/tests/vis_functionality/vis_semantic_cam.py +++ b/metadrive/tests/vis_functionality/vis_semantic_cam.py @@ -17,27 +17,33 @@ def get_image(env): env = NuScenesEnv( { - "use_render": True, + "use_render": False, "image_observation": True, "rgb_clip": True, - "show_interface": False, + "show_interface": True, "agent_policy": ReplayEgoCarPolicy, "interface_panel": ["semantic_camera"], "sensors": dict(semantic_camera=(SemanticCamera, 800, 600)), "vehicle_config": dict(image_source="semantic_camera"), - "data_directory": AssetLoader.file_path("nuscenes", return_raw_style=False), + "data_directory": AssetLoader.file_path("waymo", return_raw_style=False), } ) - env.reset() + env.reset(seed=1) env.engine.accept("m", get_image, extraArgs=[env]) - + import cv2 for i in range(1, 100000): o, r, tm, tc, info = env.step([0, 1]) assert env.observation_space.contains(o) - if env.config["use_render"]: - # for i in range(ImageObservation.STACK_SIZE): - # ObservationType.show_gray_scale_array(o["image"][:, :, i]) - env.render() + # save + rgb_cam = env.engine.get_sensor(env.vehicle.config["image_source"]) + # rgb_cam.save_image(env.vehicle, name="{}.png".format(i)) + cv2.imshow('img', o["image"][..., -1]) + cv2.waitKey(1) + + # if env.config["use_render"]: + # for i in range(ImageObservation.STACK_SIZE): + # ObservationType.show_gray_scale_array(o["image"][:, :, i]) + # env.render() # if tm or tc: # # print("Reset") # env.reset()