From 9631a3c8166abc48d8de35f8db7ee5b77764ad70 Mon Sep 17 00:00:00 2001 From: Pascal Roth Date: Wed, 11 Dec 2024 23:07:47 +0100 Subject: [PATCH 1/4] add rgb to data collection --- .../omni/viplanner/config/carla_cfg.py | 2 +- .../omni/viplanner/config/matterport_cfg.py | 19 ++++++++++++++----- .../omni/viplanner/config/warehouse_cfg.py | 2 +- omniverse/standalone/data_collect.py | 7 +++++++ 4 files changed, 23 insertions(+), 7 deletions(-) diff --git a/omniverse/extension/omni.viplanner/omni/viplanner/config/carla_cfg.py b/omniverse/extension/omni.viplanner/omni/viplanner/config/carla_cfg.py index bf32be5..0c87d0c 100644 --- a/omniverse/extension/omni.viplanner/omni/viplanner/config/carla_cfg.py +++ b/omniverse/extension/omni.viplanner/omni/viplanner/config/carla_cfg.py @@ -91,7 +91,7 @@ class TerrainSceneCfg(InteractiveSceneCfg): ), width=1280, height=720, - data_types=["semantic_segmentation"], + data_types=["semantic_segmentation", "rgb"], colorize_semantic_segmentation=False, ) diff --git a/omniverse/extension/omni.viplanner/omni/viplanner/config/matterport_cfg.py b/omniverse/extension/omni.viplanner/omni/viplanner/config/matterport_cfg.py index d886263..c4a0306 100644 --- a/omniverse/extension/omni.viplanner/omni/viplanner/config/matterport_cfg.py +++ b/omniverse/extension/omni.viplanner/omni/viplanner/config/matterport_cfg.py @@ -11,7 +11,7 @@ from omni.isaac.lab.managers import ObservationTermCfg as ObsTerm from omni.isaac.lab.managers import SceneEntityCfg from omni.isaac.lab.scene import InteractiveSceneCfg -from omni.isaac.lab.sensors import ContactSensorCfg, patterns +from omni.isaac.lab.sensors import ContactSensorCfg, patterns, CameraCfg from omni.isaac.lab.utils import configclass from omni.isaac.matterport.config import MatterportImporterCfg from omni.isaac.matterport.domains import MatterportRayCasterCfg @@ -45,7 +45,7 @@ class TerrainSceneCfg(InteractiveSceneCfg): dynamic_friction=1.0, ), # NOTE: this path should be absolute to load the textures correctly - obj_filepath="${USER_PATH_TO_USD}/matterport.usd", + obj_filepath="/home/pascal/Downloads/matterport/matterport.usd", groundplane=True, ) # robots @@ -60,7 +60,7 @@ class TerrainSceneCfg(InteractiveSceneCfg): attach_yaw_only=True, pattern_cfg=patterns.GridPatternCfg(resolution=0.1, size=[1.6, 1.0]), debug_vis=True, - mesh_prim_paths=["${USER_PATH_TO_USD}/matterport.ply"], + mesh_prim_paths=["/home/pascal/Downloads/matterport/matterport.ply"], ) contact_forces = ContactSensorCfg(prim_path="{ENV_REGEX_NS}/Robot/.*", history_length=3, debug_vis=False) # lights @@ -110,7 +110,7 @@ class TerrainSceneCfg(InteractiveSceneCfg): pattern_cfg=patterns.PinholeCameraPatternCfg(width=848, height=480), debug_vis=False, max_distance=10, - mesh_prim_paths=["${USER_PATH_TO_USD}/matterport.ply"], + mesh_prim_paths=["/home/pascal/Downloads/matterport/matterport.ply"], data_types=["distance_to_image_plane"], ) semantic_camera = VIPlannerMatterportRayCasterCameraCfg( @@ -119,7 +119,16 @@ class TerrainSceneCfg(InteractiveSceneCfg): pattern_cfg=patterns.PinholeCameraPatternCfg(width=1280, height=720), data_types=["semantic_segmentation"], debug_vis=False, - mesh_prim_paths=["${USER_PATH_TO_USD}/matterport.ply"], + mesh_prim_paths=["/home/pascal/Downloads/matterport/matterport.ply"], + ) + rgb_camera = CameraCfg( + prim_path="{ENV_REGEX_NS}/Robot/base/rgb_camera", + offset=CameraCfg.OffsetCfg(pos=(0.510, 0.0, 0.015), rot=(-0.5, 0.5, -0.5, 0.5)), + spawn=sim_utils.PinholeCameraCfg(), + width=1280, + height=720, + data_types=["rgb"], + colorize_semantic_segmentation=False, ) diff --git a/omniverse/extension/omni.viplanner/omni/viplanner/config/warehouse_cfg.py b/omniverse/extension/omni.viplanner/omni/viplanner/config/warehouse_cfg.py index 633617b..f015fc7 100644 --- a/omniverse/extension/omni.viplanner/omni/viplanner/config/warehouse_cfg.py +++ b/omniverse/extension/omni.viplanner/omni/viplanner/config/warehouse_cfg.py @@ -86,7 +86,7 @@ class TerrainSceneCfg(InteractiveSceneCfg): spawn=sim_utils.PinholeCameraCfg(), width=1280, height=720, - data_types=["semantic_segmentation"], + data_types=["semantic_segmentation", "rgb"], colorize_semantic_segmentation=False, ) diff --git a/omniverse/standalone/data_collect.py b/omniverse/standalone/data_collect.py index 8d484b5..16fa152 100644 --- a/omniverse/standalone/data_collect.py +++ b/omniverse/standalone/data_collect.py @@ -97,6 +97,13 @@ def main(): "semantic_camera": "semantic_segmentation", } + # if also RGB images should be rendered + if args_cli.scene == "matterport": + scene_cfg.rgb_camera.prim_path = "/World/rgb_camera" + cfg.cameras["rgb_camera"] = "rgb" + else: + cfg.cameras["semantic_camera"] = "rgb" + # Load kit helper sim_cfg = sim_utils.SimulationCfg() sim = SimulationContext(sim_cfg) From 13b5eed8b3e32bbbbea0f0fd01b0afdecf47a066 Mon Sep 17 00:00:00 2001 From: Takahiro Date: Wed, 18 Dec 2024 20:39:43 +0100 Subject: [PATCH 2/4] Fixed bug in the rgb saving. Also changed the folder name to annotator name. --- .../collectors/viewpoint_sampling.py | 28 ++++++++++--------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/omniverse/extension/omni.viplanner/omni/viplanner/collectors/viewpoint_sampling.py b/omniverse/extension/omni.viplanner/omni/viplanner/collectors/viewpoint_sampling.py index b2b72ab..625d6ba 100644 --- a/omniverse/extension/omni.viplanner/omni/viplanner/collectors/viewpoint_sampling.py +++ b/omniverse/extension/omni.viplanner/omni/viplanner/collectors/viewpoint_sampling.py @@ -81,9 +81,9 @@ def sample_viewpoints(self, nbr_viewpoints: int, seed: int = 1) -> torch.Tensor: sample_idx_select = torch.randperm(sample_idx.sum())[ : min(nbr_samples_per_point, nbr_viewpoints - sample_locations_count) ] - sample_locations[ - sample_locations_count : sample_locations_count + sample_idx_select.shape[0] - ] = self.terrain_analyser.samples[sample_idx][sample_idx_select, :2] + sample_locations[sample_locations_count : sample_locations_count + sample_idx_select.shape[0]] = ( + self.terrain_analyser.samples[sample_idx][sample_idx_select, :2] + ) sample_locations_count += sample_idx_select.shape[0] curr_point_idx += 1 # reset point index if all points are sampled @@ -154,9 +154,9 @@ def render_viewpoints(self, samples: torch.Tensor): # save poses filedir = self.cfg.save_path if self.cfg.save_path else self._get_save_filedir() - # create directories - os.makedirs(os.path.join(filedir, "semantics"), exist_ok=True) - os.makedirs(os.path.join(filedir, "depth"), exist_ok=True) + # # create directories + for cam_idx, (cam, annotator) in enumerate(self.cfg.cameras.items()): + os.makedirs(os.path.join(filedir, annotator), exist_ok=True) # save camera configurations print(f"[INFO] Saving camera configurations to {filedir}.") @@ -206,16 +206,18 @@ def render_viewpoints(self, samples: torch.Tensor): # save images for idx in range(samples_idx.shape[0]): # semantic segmentation - if annotator == "semantic_segmentation": + if annotator == "semantic_segmentation" or annotator == "rgb": if image_data_np.shape[-1] == 1: # get info data info = self.scene.sensors[cam].data.info[idx][annotator]["idToLabels"] # assign each key a color from the VIPlanner color space info = { - int(k): self.viplanner_sem_meta.class_color["static"] - if v["class"] in ("BACKGROUND", "UNLABELLED") - else self.viplanner_sem_meta.class_color[v["class"]] + int(k): ( + self.viplanner_sem_meta.class_color["static"] + if v["class"] in ("BACKGROUND", "UNLABELLED") + else self.viplanner_sem_meta.class_color[v["class"]] + ) for k, v in info.items() } @@ -232,18 +234,18 @@ def render_viewpoints(self, samples: torch.Tensor): output = image_data_np[idx] assert cv2.imwrite( - os.path.join(filedir, "semantics", f"{image_idx[cam_idx]}".zfill(4) + ".png"), + os.path.join(filedir, annotator, f"{image_idx[cam_idx]}".zfill(4) + ".png"), cv2.cvtColor(output.astype(np.uint8), cv2.COLOR_RGB2BGR), ) # depth else: assert cv2.imwrite( - os.path.join(filedir, "depth", f"{image_idx[cam_idx]}".zfill(4) + ".png"), + os.path.join(filedir, annotator, f"{image_idx[cam_idx]}".zfill(4) + ".png"), np.uint16(image_data_np[idx] * self.cfg.depth_scale), ) # save as npy np.save( - os.path.join(filedir, "depth", f"{image_idx[cam_idx]}".zfill(4) + ".npy"), + os.path.join(filedir, annotator, f"{image_idx[cam_idx]}".zfill(4) + ".npy"), image_data_np[idx] * self.cfg.depth_scale, ) From b8873eebdc52f89e4a1a9af851048d15dba8b6ed Mon Sep 17 00:00:00 2001 From: Pascal Roth Date: Thu, 2 Jan 2025 21:41:50 +0100 Subject: [PATCH 3/4] remove annotator naming due to names in other parts of the code, remove hard coded paths --- .../viplanner/collectors/viewpoint_sampling.py | 18 ++++++++++++------ .../omni/viplanner/config/carla_cfg.py | 2 ++ .../omni/viplanner/config/matterport_cfg.py | 10 ++++++---- .../omni/viplanner/config/warehouse_cfg.py | 2 ++ omniverse/standalone/data_collect.py | 6 +++--- 5 files changed, 25 insertions(+), 13 deletions(-) diff --git a/omniverse/extension/omni.viplanner/omni/viplanner/collectors/viewpoint_sampling.py b/omniverse/extension/omni.viplanner/omni/viplanner/collectors/viewpoint_sampling.py index 625d6ba..896e74e 100644 --- a/omniverse/extension/omni.viplanner/omni/viplanner/collectors/viewpoint_sampling.py +++ b/omniverse/extension/omni.viplanner/omni/viplanner/collectors/viewpoint_sampling.py @@ -154,9 +154,11 @@ def render_viewpoints(self, samples: torch.Tensor): # save poses filedir = self.cfg.save_path if self.cfg.save_path else self._get_save_filedir() - # # create directories - for cam_idx, (cam, annotator) in enumerate(self.cfg.cameras.items()): - os.makedirs(os.path.join(filedir, annotator), exist_ok=True) + # create directories + os.makedirs(os.path.join(filedir, "semantics"), exist_ok=True) + os.makedirs(os.path.join(filedir, "depth"), exist_ok=True) + if "rgb" in self.cfg.cameras.values(): + os.makedirs(os.path.join(filedir, "rgb"), exist_ok=True) # save camera configurations print(f"[INFO] Saving camera configurations to {filedir}.") @@ -234,18 +236,22 @@ def render_viewpoints(self, samples: torch.Tensor): output = image_data_np[idx] assert cv2.imwrite( - os.path.join(filedir, annotator, f"{image_idx[cam_idx]}".zfill(4) + ".png"), + os.path.join( + filedir, + "semantics" if annotator == "semantic_segmentation" else "rgb", + f"{image_idx[cam_idx]}".zfill(4) + ".png" + ), cv2.cvtColor(output.astype(np.uint8), cv2.COLOR_RGB2BGR), ) # depth else: assert cv2.imwrite( - os.path.join(filedir, annotator, f"{image_idx[cam_idx]}".zfill(4) + ".png"), + os.path.join(filedir, "depth", f"{image_idx[cam_idx]}".zfill(4) + ".png"), np.uint16(image_data_np[idx] * self.cfg.depth_scale), ) # save as npy np.save( - os.path.join(filedir, annotator, f"{image_idx[cam_idx]}".zfill(4) + ".npy"), + os.path.join(filedir, "depth", f"{image_idx[cam_idx]}".zfill(4) + ".npy"), image_data_np[idx] * self.cfg.depth_scale, ) diff --git a/omniverse/extension/omni.viplanner/omni/viplanner/config/carla_cfg.py b/omniverse/extension/omni.viplanner/omni/viplanner/config/carla_cfg.py index 0c87d0c..9b62cb7 100644 --- a/omniverse/extension/omni.viplanner/omni/viplanner/config/carla_cfg.py +++ b/omniverse/extension/omni.viplanner/omni/viplanner/config/carla_cfg.py @@ -82,6 +82,8 @@ class TerrainSceneCfg(InteractiveSceneCfg): height=480, data_types=["distance_to_image_plane"], ) + + # NOTE: remove "rgb" from the data_types to only render the semantic segmentation semantic_camera = CameraCfg( prim_path="{ENV_REGEX_NS}/Robot/base/semantic_camera", offset=CameraCfg.OffsetCfg(pos=(0.510, 0.0, 0.015), rot=(-0.5, 0.5, -0.5, 0.5)), diff --git a/omniverse/extension/omni.viplanner/omni/viplanner/config/matterport_cfg.py b/omniverse/extension/omni.viplanner/omni/viplanner/config/matterport_cfg.py index c4a0306..b132f92 100644 --- a/omniverse/extension/omni.viplanner/omni/viplanner/config/matterport_cfg.py +++ b/omniverse/extension/omni.viplanner/omni/viplanner/config/matterport_cfg.py @@ -45,7 +45,7 @@ class TerrainSceneCfg(InteractiveSceneCfg): dynamic_friction=1.0, ), # NOTE: this path should be absolute to load the textures correctly - obj_filepath="/home/pascal/Downloads/matterport/matterport.usd", + obj_filepath="${USER_PATH_TO_USD}/matterport.usd", groundplane=True, ) # robots @@ -60,7 +60,7 @@ class TerrainSceneCfg(InteractiveSceneCfg): attach_yaw_only=True, pattern_cfg=patterns.GridPatternCfg(resolution=0.1, size=[1.6, 1.0]), debug_vis=True, - mesh_prim_paths=["/home/pascal/Downloads/matterport/matterport.ply"], + mesh_prim_paths=["${USER_PATH_TO_USD}/matterport.ply"], ) contact_forces = ContactSensorCfg(prim_path="{ENV_REGEX_NS}/Robot/.*", history_length=3, debug_vis=False) # lights @@ -110,7 +110,7 @@ class TerrainSceneCfg(InteractiveSceneCfg): pattern_cfg=patterns.PinholeCameraPatternCfg(width=848, height=480), debug_vis=False, max_distance=10, - mesh_prim_paths=["/home/pascal/Downloads/matterport/matterport.ply"], + mesh_prim_paths=["${USER_PATH_TO_USD}/matterport.ply"], data_types=["distance_to_image_plane"], ) semantic_camera = VIPlannerMatterportRayCasterCameraCfg( @@ -119,8 +119,10 @@ class TerrainSceneCfg(InteractiveSceneCfg): pattern_cfg=patterns.PinholeCameraPatternCfg(width=1280, height=720), data_types=["semantic_segmentation"], debug_vis=False, - mesh_prim_paths=["/home/pascal/Downloads/matterport/matterport.ply"], + mesh_prim_paths=["${USER_PATH_TO_USD}/matterport.ply"], ) + + # NOTE: comment the following lines to only render the semantic segmentation and depth images rgb_camera = CameraCfg( prim_path="{ENV_REGEX_NS}/Robot/base/rgb_camera", offset=CameraCfg.OffsetCfg(pos=(0.510, 0.0, 0.015), rot=(-0.5, 0.5, -0.5, 0.5)), diff --git a/omniverse/extension/omni.viplanner/omni/viplanner/config/warehouse_cfg.py b/omniverse/extension/omni.viplanner/omni/viplanner/config/warehouse_cfg.py index f015fc7..17a7683 100644 --- a/omniverse/extension/omni.viplanner/omni/viplanner/config/warehouse_cfg.py +++ b/omniverse/extension/omni.viplanner/omni/viplanner/config/warehouse_cfg.py @@ -80,6 +80,8 @@ class TerrainSceneCfg(InteractiveSceneCfg): height=480, data_types=["distance_to_image_plane"], ) + + # NOTE: remove "rgb" from the data_types to only render the semantic segmentation semantic_camera = CameraCfg( prim_path="{ENV_REGEX_NS}/Robot/base/semantic_camera", offset=CameraCfg.OffsetCfg(pos=(0.510, 0.0, 0.015), rot=(-0.5, 0.5, -0.5, 0.5)), diff --git a/omniverse/standalone/data_collect.py b/omniverse/standalone/data_collect.py index 16fa152..8befbcf 100644 --- a/omniverse/standalone/data_collect.py +++ b/omniverse/standalone/data_collect.py @@ -97,11 +97,11 @@ def main(): "semantic_camera": "semantic_segmentation", } - # if also RGB images should be rendered - if args_cli.scene == "matterport": + # adustments if also RGB images should be rendered + if args_cli.scene == "matterport" and hasattr(scene_cfg, "rgb_camera"): scene_cfg.rgb_camera.prim_path = "/World/rgb_camera" cfg.cameras["rgb_camera"] = "rgb" - else: + elif "rgb" in scene_cfg.semantic_camera.data_types: cfg.cameras["semantic_camera"] = "rgb" # Load kit helper From ab403a223be9be0f4dd16b222cd8004924548d9e Mon Sep 17 00:00:00 2001 From: Pascal Roth Date: Thu, 2 Jan 2025 21:44:09 +0100 Subject: [PATCH 4/4] update --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 832a21d..83a7937 100644 --- a/README.md +++ b/README.md @@ -124,7 +124,7 @@ The latest model is available to download: [[checkpoint](https://drive.google.co This code belongs to Robotic Systems Lab, ETH Zurich. All right reserved -**Authors: [Pascal Roth](https://github.com/pascal-roth), [Julian Nubert](https://juliannubert.com/), [Fan Yang](https://github.com/MichaelFYang), [Mayank Mittal](https://mayankm96.github.io/), [Ziqi Fan](https://github.com/fan-ziqi), and [Marco Hutter](https://rsl.ethz.ch/the-lab/people/person-detail.MTIxOTEx.TGlzdC8yNDQxLC0xNDI1MTk1NzM1.html)
+**Authors: [Pascal Roth](https://github.com/pascal-roth), [Julian Nubert](https://juliannubert.com/), [Fan Yang](https://github.com/MichaelFYang), [Mayank Mittal](https://mayankm96.github.io/), and [Marco Hutter](https://rsl.ethz.ch/the-lab/people/person-detail.MTIxOTEx.TGlzdC8yNDQxLC0xNDI1MTk1NzM1.html)
Maintainer: Pascal Roth, rothpa@ethz.ch** The ViPlanner package has been tested under ROS Noetic on Ubuntu 20.04.