From a91cbf55adae34e1acd9794685f260f3782c5029 Mon Sep 17 00:00:00 2001 From: Matias Mattamala Date: Fri, 22 Mar 2024 10:08:47 +0000 Subject: [PATCH 01/15] Rename jackal package, add postprocess scripts --- .gitignore | 1 + docker/Dockerfile | 4 +- docker/docker-compose-gui-nvidia.yaml | 2 +- docker/docker-compose-nvidia.yaml | 2 +- docker/docker-compose.yaml | 1 - .../CMakeLists.txt | 2 +- .../Media/models/OliveTree1.dae | 0 .../Media/models/OliveTree2.dae | 0 .../Media/models/PineTree.dae | 0 .../Media/models/heightmap.png | Bin .../Media/models/scenario.dae | 0 .../textures/Vegetation_Grass1 (copy).jpg | Bin .../models/textures/Vegetation_Grass1.jpg | Bin .../Media/models/textures/mtt.png | Bin .../Media/models/textures/texture0.jpg | Bin .../Media/models/textures/texture0.png | Bin .../README.md | 0 .../launch/sim.launch | 0 .../package.xml | 4 +- .../worlds/outdoor.world | 0 .../worlds/scenario.world | 0 .../plot_learning_curves_step.py | 88 ++++++++++++ .../postprocessing/postprocess_logger.py | 136 ++++++++++++++++++ 23 files changed, 232 insertions(+), 8 deletions(-) rename {wild_visual_navigation_sim => wild_visual_navigation_jackal}/CMakeLists.txt (94%) rename {wild_visual_navigation_sim => wild_visual_navigation_jackal}/Media/models/OliveTree1.dae (100%) rename {wild_visual_navigation_sim => wild_visual_navigation_jackal}/Media/models/OliveTree2.dae (100%) rename {wild_visual_navigation_sim => wild_visual_navigation_jackal}/Media/models/PineTree.dae (100%) rename {wild_visual_navigation_sim => wild_visual_navigation_jackal}/Media/models/heightmap.png (100%) rename {wild_visual_navigation_sim => wild_visual_navigation_jackal}/Media/models/scenario.dae (100%) rename {wild_visual_navigation_sim => wild_visual_navigation_jackal}/Media/models/textures/Vegetation_Grass1 (copy).jpg (100%) rename {wild_visual_navigation_sim => wild_visual_navigation_jackal}/Media/models/textures/Vegetation_Grass1.jpg (100%) rename {wild_visual_navigation_sim => wild_visual_navigation_jackal}/Media/models/textures/mtt.png (100%) rename {wild_visual_navigation_sim => wild_visual_navigation_jackal}/Media/models/textures/texture0.jpg (100%) rename {wild_visual_navigation_sim => wild_visual_navigation_jackal}/Media/models/textures/texture0.png (100%) rename {wild_visual_navigation_sim => wild_visual_navigation_jackal}/README.md (100%) rename {wild_visual_navigation_sim => wild_visual_navigation_jackal}/launch/sim.launch (100%) rename {wild_visual_navigation_sim => wild_visual_navigation_jackal}/package.xml (85%) rename {wild_visual_navigation_sim => wild_visual_navigation_jackal}/worlds/outdoor.world (100%) rename {wild_visual_navigation_sim => wild_visual_navigation_jackal}/worlds/scenario.world (100%) create mode 100644 wild_visual_navigation_ros/scripts/postprocessing/plot_learning_curves_step.py create mode 100644 wild_visual_navigation_ros/scripts/postprocessing/postprocess_logger.py diff --git a/.gitignore b/.gitignore index 88488fd1..61e069a0 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ .neptune* wild_visual_navigation/.neptune* wild_visual_navigation/notebooks* +wild_visual_navigation_ros/output* results/* assets/dino/* assets/stego/cocostuff27_vit_base_5.ckpt diff --git a/docker/Dockerfile b/docker/Dockerfile index a78b68ec..8446fb17 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,7 +1,7 @@ ##################### # Base image # ##################### -FROM nvidia/cuda:11.8.0-runtime-ubuntu20.04 as base +FROM nvidia/cuda:12.3.2-runtime-ubuntu20.04 as base # Labels LABEL maintainer="Matias Mattamala;Jonas Frey" @@ -80,7 +80,7 @@ RUN cd /root \ termcolor \ pydensecrf@git+https://github.com/lucasb-eyer/pydensecrf.git \ liegroups@git+https://github.com/mmattamala/liegroups \ - opencv-python>=4.6 + opencv-python # == # Remove cache and extra files diff --git a/docker/docker-compose-gui-nvidia.yaml b/docker/docker-compose-gui-nvidia.yaml index 23414ae0..4d2fadc8 100644 --- a/docker/docker-compose-gui-nvidia.yaml +++ b/docker/docker-compose-gui-nvidia.yaml @@ -1,6 +1,6 @@ version: "3.9" services: - wvn: + wvn_nvidia: extends: file: docker-compose-gui.yaml service: wvn diff --git a/docker/docker-compose-nvidia.yaml b/docker/docker-compose-nvidia.yaml index 4b82f8df..1bd55c43 100644 --- a/docker/docker-compose-nvidia.yaml +++ b/docker/docker-compose-nvidia.yaml @@ -1,6 +1,6 @@ version: "3.9" services: - wvn: + wvn_nvidia: extends: file: docker-compose.yaml service: wvn diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index 4a1058e0..56f48cc2 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -11,7 +11,6 @@ services: volumes: - ../../wild_visual_navigation:/root/catkin_ws/src/wild_visual_navigation - ../../self_supervised_segmentation:/root/catkin_ws/src/self_supervised_segmentation - - ../../segment-anything:/root/catkin_ws/src/segment-anything environment: - ROS_IP=127.0.0.1 command: "/bin/bash" diff --git a/wild_visual_navigation_sim/CMakeLists.txt b/wild_visual_navigation_jackal/CMakeLists.txt similarity index 94% rename from wild_visual_navigation_sim/CMakeLists.txt rename to wild_visual_navigation_jackal/CMakeLists.txt index d451ee18..03515b49 100644 --- a/wild_visual_navigation_sim/CMakeLists.txt +++ b/wild_visual_navigation_jackal/CMakeLists.txt @@ -1,5 +1,5 @@ cmake_minimum_required(VERSION 3.0.2) -project(wild_visual_navigation_sim) +project(wild_visual_navigation_jackal) find_package(catkin REQUIRED COMPONENTS rospy diff --git a/wild_visual_navigation_sim/Media/models/OliveTree1.dae b/wild_visual_navigation_jackal/Media/models/OliveTree1.dae similarity index 100% rename from wild_visual_navigation_sim/Media/models/OliveTree1.dae rename to wild_visual_navigation_jackal/Media/models/OliveTree1.dae diff --git a/wild_visual_navigation_sim/Media/models/OliveTree2.dae b/wild_visual_navigation_jackal/Media/models/OliveTree2.dae similarity index 100% rename from wild_visual_navigation_sim/Media/models/OliveTree2.dae rename to wild_visual_navigation_jackal/Media/models/OliveTree2.dae diff --git a/wild_visual_navigation_sim/Media/models/PineTree.dae b/wild_visual_navigation_jackal/Media/models/PineTree.dae similarity index 100% rename from wild_visual_navigation_sim/Media/models/PineTree.dae rename to wild_visual_navigation_jackal/Media/models/PineTree.dae diff --git a/wild_visual_navigation_sim/Media/models/heightmap.png b/wild_visual_navigation_jackal/Media/models/heightmap.png similarity index 100% rename from wild_visual_navigation_sim/Media/models/heightmap.png rename to wild_visual_navigation_jackal/Media/models/heightmap.png diff --git a/wild_visual_navigation_sim/Media/models/scenario.dae b/wild_visual_navigation_jackal/Media/models/scenario.dae similarity index 100% rename from wild_visual_navigation_sim/Media/models/scenario.dae rename to wild_visual_navigation_jackal/Media/models/scenario.dae diff --git a/wild_visual_navigation_sim/Media/models/textures/Vegetation_Grass1 (copy).jpg b/wild_visual_navigation_jackal/Media/models/textures/Vegetation_Grass1 (copy).jpg similarity index 100% rename from wild_visual_navigation_sim/Media/models/textures/Vegetation_Grass1 (copy).jpg rename to wild_visual_navigation_jackal/Media/models/textures/Vegetation_Grass1 (copy).jpg diff --git a/wild_visual_navigation_sim/Media/models/textures/Vegetation_Grass1.jpg b/wild_visual_navigation_jackal/Media/models/textures/Vegetation_Grass1.jpg similarity index 100% rename from wild_visual_navigation_sim/Media/models/textures/Vegetation_Grass1.jpg rename to wild_visual_navigation_jackal/Media/models/textures/Vegetation_Grass1.jpg diff --git a/wild_visual_navigation_sim/Media/models/textures/mtt.png b/wild_visual_navigation_jackal/Media/models/textures/mtt.png similarity index 100% rename from wild_visual_navigation_sim/Media/models/textures/mtt.png rename to wild_visual_navigation_jackal/Media/models/textures/mtt.png diff --git a/wild_visual_navigation_sim/Media/models/textures/texture0.jpg b/wild_visual_navigation_jackal/Media/models/textures/texture0.jpg similarity index 100% rename from wild_visual_navigation_sim/Media/models/textures/texture0.jpg rename to wild_visual_navigation_jackal/Media/models/textures/texture0.jpg diff --git a/wild_visual_navigation_sim/Media/models/textures/texture0.png b/wild_visual_navigation_jackal/Media/models/textures/texture0.png similarity index 100% rename from wild_visual_navigation_sim/Media/models/textures/texture0.png rename to wild_visual_navigation_jackal/Media/models/textures/texture0.png diff --git a/wild_visual_navigation_sim/README.md b/wild_visual_navigation_jackal/README.md similarity index 100% rename from wild_visual_navigation_sim/README.md rename to wild_visual_navigation_jackal/README.md diff --git a/wild_visual_navigation_sim/launch/sim.launch b/wild_visual_navigation_jackal/launch/sim.launch similarity index 100% rename from wild_visual_navigation_sim/launch/sim.launch rename to wild_visual_navigation_jackal/launch/sim.launch diff --git a/wild_visual_navigation_sim/package.xml b/wild_visual_navigation_jackal/package.xml similarity index 85% rename from wild_visual_navigation_sim/package.xml rename to wild_visual_navigation_jackal/package.xml index 921c777c..014af6b6 100644 --- a/wild_visual_navigation_sim/package.xml +++ b/wild_visual_navigation_jackal/package.xml @@ -1,8 +1,8 @@ - wild_visual_navigation_sim + wild_visual_navigation_jackal 0.0.1 - Simulation environment for wild_visual_navigation + Jackal simulation environment for wild_visual_navigation Matias Mattamala Jonas Frey Matias Mattamala diff --git a/wild_visual_navigation_sim/worlds/outdoor.world b/wild_visual_navigation_jackal/worlds/outdoor.world similarity index 100% rename from wild_visual_navigation_sim/worlds/outdoor.world rename to wild_visual_navigation_jackal/worlds/outdoor.world diff --git a/wild_visual_navigation_sim/worlds/scenario.world b/wild_visual_navigation_jackal/worlds/scenario.world similarity index 100% rename from wild_visual_navigation_sim/worlds/scenario.world rename to wild_visual_navigation_jackal/worlds/scenario.world diff --git a/wild_visual_navigation_ros/scripts/postprocessing/plot_learning_curves_step.py b/wild_visual_navigation_ros/scripts/postprocessing/plot_learning_curves_step.py new file mode 100644 index 00000000..71ceed54 --- /dev/null +++ b/wild_visual_navigation_ros/scripts/postprocessing/plot_learning_curves_step.py @@ -0,0 +1,88 @@ +import matplotlib.pyplot as plt +from wild_visual_navigation.visu import paper_colors as pc +from pathlib import Path +import pandas as pd + +from wild_visual_navigation import WVN_ROOT_DIR + +PALETTE = [[r, g, b] for r, g, b in pc.paper_colors_rgb_f.values()] + +experiment_folder = f"{WVN_ROOT_DIR}/wild_visual_navigation_ros/output" +cases = {} + +run_dirs = sorted(list(Path(experiment_folder).iterdir())) +for d in run_dirs: + if not d.is_dir(): + continue + features, sampling, run = d.name.split("_") + label = f"{features.upper()}-{sampling.upper()}" + try: + cases[label].append(d) + except Exception: + cases[label] = [d] + + +# Matplotlib config +cm = 1 / 2.54 +plot_width = 8.89 * cm +plot_height = 4 * cm +plt.rcParams["font.size"] = 8 +n_colors = 10 + +fig, ax = plt.subplots(1, 1, figsize=(plot_width, plot_height), constrained_layout=False, dpi=300) +# Axes +ax.set_axisbelow(True) +# ax.set_aspect("equal") +ax.grid(which="major", color=(0.8, 0.8, 0.8), linewidth=0.7) +ax.grid(which="minor", color=(0.9, 0.9, 0.9), linestyle=":", linewidth=0.5) +ax.minorticks_on() + +# Plotting +for i, c in enumerate(cases): + print(c, len(cases[c])) + case_df = pd.DataFrame() + dfs = [] + + for path in cases[c]: + state_file = list(path.glob("*.csv"))[0] + df = pd.read_csv(state_file).drop_duplicates(subset="stamp") + df["stamp"] = df["stamp"] - df["stamp"][0] + + valid_idx = (df["stamp"] < 110) & (df["loss_total"] >= 0.0) + df = df[valid_idx] + # df = df.set_index(df["stamp"]) + df = df.set_index(df["step"]) + # joined_indices = case_df.index.union(df.index).drop_duplicates() + # case_df = case_df.reindex(index=joined_indices) + # case_df = case_df.interpolate(method="index") + dfs.append(df) + + for j, df in enumerate(dfs): + # df_j = df.reindex(index=case_df.index) + # df_j = df_j.interpolate(method="index") + # case_df[j] = df_j["loss_total"] + case_df[j] = df["loss_total"] + + # Compute runs + # case_df = case_df.dropna() + stamp = case_df.index + loss_mean = case_df.mean(axis=1) + loss_std = case_df.std(axis=1) + + # Plot + ax.plot(stamp, loss_mean, label=c, color=PALETTE[i]) + plt.fill_between( + stamp, + loss_mean - 2 * loss_std, + loss_mean + 2 * loss_std, + alpha=0.3, + # label="Confidence bounds (1$\sigma$)", + color=PALETTE[i], + ) +ax.set_xlabel("Training step") +ax.set_ylabel("Loss") +# ax.margins(x=0.01, y=0.01) +plt.legend() + +fig.set_tight_layout(True) +fig.savefig(f"{experiment_folder}/empirical_learning_curves.pdf") diff --git a/wild_visual_navigation_ros/scripts/postprocessing/postprocess_logger.py b/wild_visual_navigation_ros/scripts/postprocessing/postprocess_logger.py new file mode 100644 index 00000000..efb74363 --- /dev/null +++ b/wild_visual_navigation_ros/scripts/postprocessing/postprocess_logger.py @@ -0,0 +1,136 @@ +#!/bin/env python +# This node implements a simple data logging pipeline +# that subscribes to important topics generated by WVN +# for offline analysis, such as images and learning curves + +from sensor_msgs.msg import Image +from wild_visual_navigation_msgs.msg import SystemState +from cv_bridge import CvBridge + +import csv +import cv2 +import os +import rospy +import rospkg +import yaml + + +def secs_to_str(secs): + SECONDS_DIGITS = 10 + secs_str = str(secs) + return secs_str.zfill(SECONDS_DIGITS) + + +def nsecs_to_str(nsecs): + NSECONDS_DIGITS = 9 + nsecs_str = str(nsecs) + return nsecs_str.zfill(NSECONDS_DIGITS) + + +class OfflineLogger: + def __init__(self): + # Read parameters + self._read_params() + + # Initialize variables + self._bridge = CvBridge() + + package_path = rospkg.RosPack().get_path("wild_visual_navigation_ros") + base_output_path = os.path.join(package_path, "output") + + # Remove old checkpoint + state_dict = os.path.join(package_path, "../.tmp_state_dict.pt") + if os.path.exists(state_dict): + os.remove(state_dict) + + # Initialize log folder + run = 0 + while True: + mission_name = f"{self._mission_name}_{str(run).zfill(2)}" + self._output_path = os.path.join(base_output_path, mission_name) + + if os.path.exists(self._output_path): + run += 1 + else: + break + + # Make folder + mission_name = f"{self._mission_name}_{str(run).zfill(2)}" + self._output_path = os.path.join(base_output_path, mission_name) + os.makedirs(self._output_path, exist_ok=False) + + # Initialize CSV writer + csv_file = open(f"{self._output_path}/wvn_state.csv", "w") + self._csv_writer = csv.writer( + csv_file, + delimiter=",", + quotechar="|", + quoting=csv.QUOTE_MINIMAL, + ) + # write header + self._csv_writer.writerow( + [ + "stamp", + "mode", + "mission_graph_node_num_valid_node", + "loss_total", + "loss_trav", + "loss_reco", + "step", + "pause_learning", + "scale_traversability_threshold", + ] + ) + + # Dump parameters + with open(f"{self._output_path}/wvn_feature_extractor_params.yaml", "w") as outfile: + yaml.dump(rospy.get_param("/wvn_feature_extractor_node"), outfile, default_flow_style=False) + + # Initialize subscribers + self._image_sub = rospy.Subscriber( + self._image_topic, Image, self._image_callback, callback_args="image", queue_size=1 + ) + self._trav_sub = rospy.Subscriber( + self._trav_topic, Image, self._image_callback, callback_args="trav", queue_size=1 + ) + self._state_sub = rospy.Subscriber(self._state_topic, SystemState, self._state_callback, queue_size=1) + + def _read_params(self): + self._mission_name = rospy.get_param("~mission_name", "default") + self._image_topic = rospy.get_param("~image_topic", "/wild_visual_navigation_node/front/image_input") + self._trav_topic = rospy.get_param( + "~trav_topic", "/wild_visual_navigation_visu_traversability/traversability_overlayed" + ) + self._state_topic = rospy.get_param("~state_topic", "/wild_visual_navigation_node/system_state") + + rospy.loginfo(f"Logging mission [{self._mission_name}]") + + def _image_callback(self, image_msg, source): + cv_img = self._bridge.imgmsg_to_cv2(image_msg, desired_encoding="bgr8") + secs = image_msg.header.stamp.secs + nsecs = image_msg.header.stamp.nsecs + + filename = f"{self._output_path}/{source}_{secs_to_str(secs)}_{nsecs_to_str(nsecs)}.png" + cv2.imwrite(filename, cv_img) + + def _state_callback(self, system_state_msg): + stamp = rospy.get_rostime() + self._csv_writer.writerow( + [ + f"{secs_to_str(stamp.secs)}.{nsecs_to_str(stamp.nsecs)}", + system_state_msg.mode, + system_state_msg.mission_graph_num_valid_node, + system_state_msg.loss_total, + system_state_msg.loss_trav, + system_state_msg.loss_reco, + system_state_msg.step, + system_state_msg.pause_learning, + system_state_msg.scale_traversability_threshold, + ] + ) + + +if __name__ == "__main__": + rospy.init_node("wild_visual_navigation_postprocess_logger") + wvn = OfflineLogger() + rospy.spin() From 38b617da7ce3258e78809b680af8c79bf197cde1 Mon Sep 17 00:00:00 2001 From: Matias Mattamala Date: Sat, 23 Mar 2024 16:09:07 +0000 Subject: [PATCH 02/15] Add depth camera to jackal sim --- .../launch/sim.launch | 30 +++--- .../urdf/depth_camera.xacro | 96 +++++++++++++++++++ .../urdf/extras.xacro | 10 ++ .../urdf/load_xacro.sh | 4 + .../worlds/outdoor.world | 2 +- 5 files changed, 125 insertions(+), 17 deletions(-) create mode 100644 wild_visual_navigation_jackal/urdf/depth_camera.xacro create mode 100644 wild_visual_navigation_jackal/urdf/extras.xacro create mode 100755 wild_visual_navigation_jackal/urdf/load_xacro.sh diff --git a/wild_visual_navigation_jackal/launch/sim.launch b/wild_visual_navigation_jackal/launch/sim.launch index 68e83fa8..ae44791d 100644 --- a/wild_visual_navigation_jackal/launch/sim.launch +++ b/wild_visual_navigation_jackal/launch/sim.launch @@ -1,15 +1,15 @@ - - - - - + + + + + + @@ -20,15 +20,13 @@ - - - - - - - - - + + + + + - + diff --git a/wild_visual_navigation_jackal/urdf/depth_camera.xacro b/wild_visual_navigation_jackal/urdf/depth_camera.xacro new file mode 100644 index 00000000..474952ae --- /dev/null +++ b/wild_visual_navigation_jackal/urdf/depth_camera.xacro @@ -0,0 +1,96 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + true + + 30 + + + 1.518 + 1.012 + + 640 + 480 + R8G8B8 + + + + 0.01 + 50.0 + + + + / + 0.2 + true + 0.0 + ${name} + /${name}/color/image_raw + /${name}/color/camera_info + /${name}/depth/image_rect_raw + /${name}/depth/camera_info + /${name}/depth/color/points + ${name}_link + 0.105 + 8.0 + 0.00000001 + 0.00000001 + 0.00000001 + 0.00000001 + 0.00000001 + 0 + 0 + 0 + 0 + 0 + + + + + \ No newline at end of file diff --git a/wild_visual_navigation_jackal/urdf/extras.xacro b/wild_visual_navigation_jackal/urdf/extras.xacro new file mode 100644 index 00000000..ab18448d --- /dev/null +++ b/wild_visual_navigation_jackal/urdf/extras.xacro @@ -0,0 +1,10 @@ + + + + + diff --git a/wild_visual_navigation_jackal/urdf/load_xacro.sh b/wild_visual_navigation_jackal/urdf/load_xacro.sh new file mode 100755 index 00000000..e5e1ac67 --- /dev/null +++ b/wild_visual_navigation_jackal/urdf/load_xacro.sh @@ -0,0 +1,4 @@ +#!/bin/bash +# This is a workaround to set the JACKAL_URDF_EXTRAS path +export JACKAL_URDF_EXTRAS="$(rospack find wild_visual_navigation_jackal)/urdf/extras.xacro" +xacro $(rospack find jackal_description)/urdf/jackal.urdf.xacro \ No newline at end of file diff --git a/wild_visual_navigation_jackal/worlds/outdoor.world b/wild_visual_navigation_jackal/worlds/outdoor.world index a0bb69ba..209372f8 100644 --- a/wild_visual_navigation_jackal/worlds/outdoor.world +++ b/wild_visual_navigation_jackal/worlds/outdoor.world @@ -266,7 +266,7 @@ true - 9 -4 -0.5 0 0 0.5 + 9 -4 -1.0 0 0 0.5 From 7845d7fd50c630c6ef99fc0a3d099f9d3194fbe8 Mon Sep 17 00:00:00 2001 From: Matias Mattamala Date: Mon, 25 Mar 2024 18:39:35 +0000 Subject: [PATCH 03/15] Complete jackal example running on Docker container --- docker/Dockerfile | 4 +- docker/README.md | 54 +- wild_visual_navigation_jackal/CMakeLists.txt | 8 +- .../config/rviz/open_source.rviz | 882 ++++++++++++++++++ .../config/wild_visual_navigation/camera.yaml | 12 + .../config/wild_visual_navigation/jackal.yaml | 57 ++ .../launch/sim.launch | 10 +- .../launch/teleop.launch | 7 + .../launch/view.launch | 7 + .../launch/wild_visual_navigation.launch | 17 + .../scripts/jackal_state_converter_node.py | 81 ++ .../urdf/depth_camera.xacro | 12 +- .../launch/wild_visual_navigation.launch | 6 +- .../scripts/wvn_learning_node.py | 8 +- 14 files changed, 1143 insertions(+), 22 deletions(-) create mode 100644 wild_visual_navigation_jackal/config/rviz/open_source.rviz create mode 100644 wild_visual_navigation_jackal/config/wild_visual_navigation/camera.yaml create mode 100644 wild_visual_navigation_jackal/config/wild_visual_navigation/jackal.yaml create mode 100644 wild_visual_navigation_jackal/launch/teleop.launch create mode 100644 wild_visual_navigation_jackal/launch/view.launch create mode 100644 wild_visual_navigation_jackal/launch/wild_visual_navigation.launch create mode 100644 wild_visual_navigation_jackal/scripts/jackal_state_converter_node.py diff --git a/docker/Dockerfile b/docker/Dockerfile index 8446fb17..a82fb53b 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -97,7 +97,7 @@ ARG DEBIAN_FRONTEND=dialog # Run bash # == CMD ["/bin/bash"] -WORKDIR /root +WORKDIR /root/catkin_ws ############################################################### @@ -109,7 +109,7 @@ FROM base as dev ENV DEBIAN_FRONTEND=noninteractive -RUN mkdir -p catkin_ws/src \ +RUN mkdir -p /root/catkin_ws/src \ && source /opt/ros/noetic/setup.bash \ && source "/root/.bashrc" \ && cd /root/catkin_ws && catkin build \ diff --git a/docker/README.md b/docker/README.md index 4723f414..1ed75a0d 100644 --- a/docker/README.md +++ b/docker/README.md @@ -1,17 +1,24 @@ -# Docker files +# Jackal Simulation Demo -## Build and run containers +We provide an example package to demonstrate Wild Visual Navigation on a simulated environment with a Clearpath Jackal robot. + +This example should be self-contained and it should run on a Docker container. This was tested on Ubuntu machines, we do not expect the GUI to run on Windows or Mac computers (due to X11 support). + + +## Build the image To build the container: ```sh -docker compose -f docker-compose.yaml build +docker compose -f docker-compose-gui-nvidia.yaml build ``` +## Run the container + To run the container (terminal-based): ```sh -docker compose -f docker-compose.yaml up -d +docker compose -f docker-compose-gui-nvidia.yaml up -d ``` To launch bash on the container: @@ -20,15 +27,48 @@ To launch bash on the container: docker compose exec wvn bash ``` +## Stop the container + To stop the container: ```sh docker compose -f docker-compose.yaml stop ``` -To run the GUI-enabled version and check the Gazebo environment: +## Running Wild Visual Navigation + +You can either run the following commands in 4 terminals that initialize a bash terminal in the container, or you can use VS Code with the Docker extension to instantiate terminal in the container directly. + +### Launch Jackal sim + +```sh +roslaunch wild_visual_navigation_jackal sim.launch +``` + +### Launch WVN + +```sh +roslaunch wild_visual_navigation_jackal wild_visual_navigation.launch +``` + +### Launch Teleop node + +```sh +roslaunch wild_visual_navigation_jackal teleop.launch +``` + +### Launch RViz window + +```sh +roslaunch wild_visual_navigation_jackal view.launch +``` + + +## Troubleshooting + +If the GUI doesn't work, you might need to allow the X Server to connect before running the container: ```sh -docker compose -f docker-compose-gui.yaml up -d -docker compose exec wvn_gui bash +xhost +Local:* +xhost ``` diff --git a/wild_visual_navigation_jackal/CMakeLists.txt b/wild_visual_navigation_jackal/CMakeLists.txt index 03515b49..b3d3d189 100644 --- a/wild_visual_navigation_jackal/CMakeLists.txt +++ b/wild_visual_navigation_jackal/CMakeLists.txt @@ -24,4 +24,10 @@ catkin_package( message(${CATKIN_PACKAGE_SHARE_DESTINATION}) install(DIRECTORY launch Media worlds DESTINATION ${CATKIN_PACKAGE_SHARE_DESTINATION} -) \ No newline at end of file +) + +############ +## Python ## +############ +catkin_install_python(PROGRAMS scripts/jackal_state_converter_node.py + DESTINATION ${CATKIN_PACKAGE_BIN_DESTINATION}) \ No newline at end of file diff --git a/wild_visual_navigation_jackal/config/rviz/open_source.rviz b/wild_visual_navigation_jackal/config/rviz/open_source.rviz new file mode 100644 index 00000000..e5a291f2 --- /dev/null +++ b/wild_visual_navigation_jackal/config/rviz/open_source.rviz @@ -0,0 +1,882 @@ +Panels: + - Class: rviz/Displays + Help Height: 0 + Name: Displays + Property Tree Widget: + Expanded: + - /Global Options1 + - /TF1/Tree1 + - /Robot Info1 + - /Wild Visual Navigation1 + - /Wild Visual Navigation1/Self Supervision1 + - /Wild Visual Navigation1/Cameras Resized1 + - /Wild Visual Navigation1/Prediction1 + - /Wild Visual Navigation1/Prediction1/R Traversability Overlay1 + Splitter Ratio: 0.6411483287811279 + Tree Height: 536 + - Class: rviz/Selection + Name: Selection + - Class: rviz/Tool Properties + Expanded: + - /2D Pose Estimate1 + - /2D Nav Goal1 + - /Publish Point1 + Name: Tool Properties + Splitter Ratio: 0.5886790156364441 + - Class: rviz/Views + Expanded: + - /Current View1 + Name: Views + Splitter Ratio: 0.5 + - Class: rviz/Time + Name: Time + SyncMode: 0 + SyncSource: F Traversability Overlay +Preferences: + PromptSaveOnExit: true +Toolbars: + toolButtonStyle: 2 +Visualization Manager: + Class: "" + Displays: + - Alpha: 0.5 + Cell Size: 1 + Class: rviz/Grid + Color: 160; 160; 164 + Enabled: false + Line Style: + Line Width: 0.029999999329447746 + Value: Lines + Name: Grid + Normal Cell Count: 0 + Offset: + X: 0 + Y: 0 + Z: 0 + Plane: XY + Plane Cell Count: 20 + Reference Frame: + Value: false + - Class: rviz/TF + Enabled: true + Filter (blacklist): "" + Filter (whitelist): "" + Frame Timeout: 1000 + Frames: + All Enabled: false + base_link: + Value: true + chassis_link: + Value: true + depth_camera_link: + Value: true + front_fender_link: + Value: true + front_left_wheel_link: + Value: true + front_mount: + Value: true + front_right_wheel_link: + Value: true + imu_link: + Value: true + mid_mount: + Value: true + navsat_link: + Value: true + odom: + Value: true + optical_depth_camera_link: + Value: true + rear_fender_link: + Value: true + rear_left_wheel_link: + Value: true + rear_mount: + Value: true + rear_right_wheel_link: + Value: true + Marker Alpha: 1 + Marker Scale: 0.5 + Name: TF + Show Arrows: true + Show Axes: true + Show Names: true + Tree: + odom: + base_link: + chassis_link: + front_fender_link: + {} + front_left_wheel_link: + {} + front_right_wheel_link: + {} + imu_link: + {} + mid_mount: + front_mount: + depth_camera_link: + optical_depth_camera_link: + {} + rear_mount: + {} + navsat_link: + {} + rear_fender_link: + {} + rear_left_wheel_link: + {} + rear_right_wheel_link: + {} + Update Interval: 0 + Value: true + - Class: rviz/Group + Displays: + - Angle Tolerance: 0.10000000149011612 + Class: rviz/Odometry + Covariance: + Orientation: + Alpha: 0.5 + Color: 255; 255; 127 + Color Style: Unique + Frame: Local + Offset: 1 + Scale: 1 + Value: true + Position: + Alpha: 0.30000001192092896 + Color: 204; 51; 204 + Scale: 1 + Value: true + Value: true + Enabled: false + Keep: 200 + Name: Odometry + Position Tolerance: 1 + Queue Size: 10 + Shape: + Alpha: 1 + Axes Length: 1 + Axes Radius: 0.10000000149011612 + Color: 255; 25; 0 + Head Length: 0.30000001192092896 + Head Radius: 0.10000000149011612 + Shaft Length: 1 + Shaft Radius: 0.05000000074505806 + Value: Arrow + Topic: /loam/odometry + Unreliable: false + Value: false + - Alpha: 1 + Class: rviz/RobotModel + Collision Enabled: false + Enabled: true + Links: + All Links Enabled: true + Expand Joint Details: false + Expand Link Details: false + Expand Tree: false + Link Tree Style: Links in Alphabetic Order + base_link: + Alpha: 1 + Show Axes: false + Show Trail: false + chassis_link: + Alpha: 1 + Show Axes: false + Show Trail: false + Value: true + depth_camera_link: + Alpha: 1 + Show Axes: false + Show Trail: false + Value: true + front_fender_link: + Alpha: 1 + Show Axes: false + Show Trail: false + Value: true + front_left_wheel_link: + Alpha: 1 + Show Axes: false + Show Trail: false + Value: true + front_mount: + Alpha: 1 + Show Axes: false + Show Trail: false + front_right_wheel_link: + Alpha: 1 + Show Axes: false + Show Trail: false + Value: true + imu_link: + Alpha: 1 + Show Axes: false + Show Trail: false + mid_mount: + Alpha: 1 + Show Axes: false + Show Trail: false + navsat_link: + Alpha: 1 + Show Axes: false + Show Trail: false + Value: true + optical_depth_camera_link: + Alpha: 1 + Show Axes: false + Show Trail: false + rear_fender_link: + Alpha: 1 + Show Axes: false + Show Trail: false + Value: true + rear_left_wheel_link: + Alpha: 1 + Show Axes: false + Show Trail: false + Value: true + rear_mount: + Alpha: 1 + Show Axes: false + Show Trail: false + rear_right_wheel_link: + Alpha: 1 + Show Axes: false + Show Trail: false + Value: true + Name: RobotModel + Robot Description: robot_description + TF Prefix: "" + Update Interval: 0 + Value: true + Visual Enabled: true + Enabled: true + Name: Robot Info + - Class: rviz/Group + Displays: + - Class: rviz/Group + Displays: + - Class: rviz/Marker + Enabled: true + Marker Topic: /wild_visual_navigation_node/graph_footprints + Name: Footprint + Namespaces: + footprints: true + Queue Size: 100 + Value: true + - Alpha: 1 + Buffer Length: 1 + Class: rviz/Path + Color: 67; 110; 176 + Enabled: true + Head Diameter: 0.30000001192092896 + Head Length: 0.20000000298023224 + Length: 0.30000001192092896 + Line Style: Billboards + Line Width: 0.029999999329447746 + Name: Supervision Graph + Offset: + X: 0 + Y: 0 + Z: 0 + Pose Color: 255; 85; 255 + Pose Style: None + Queue Size: 10 + Radius: 0.029999999329447746 + Shaft Diameter: 0.10000000149011612 + Shaft Length: 0.10000000149011612 + Topic: /wild_visual_navigation_node/supervision_graph + Unreliable: false + Value: true + - Alpha: 1 + Buffer Length: 1 + Class: rviz/Path + Color: 25; 255; 0 + Enabled: true + Head Diameter: 0.30000001192092896 + Head Length: 0.20000000298023224 + Length: 0.30000001192092896 + Line Style: Billboards + Line Width: 0.029999999329447746 + Name: Mission Graph + Offset: + X: 0 + Y: 0 + Z: 0 + Pose Color: 255; 85; 255 + Pose Style: Axes + Queue Size: 10 + Radius: 0.029999999329447746 + Shaft Diameter: 0.10000000149011612 + Shaft Length: 0.10000000149011612 + Topic: /wild_visual_navigation_node/mission_graph + Unreliable: false + Value: true + Enabled: true + Name: Self Supervision + - Class: rviz/Group + Displays: + - Class: rviz/Image + Enabled: true + Image Topic: /wild_visual_navigation_node/front/debug/last_image_traversability + Max Value: 1 + Median window: 5 + Min Value: 0 + Name: Traversability + Normalize Range: true + Queue Size: 2 + Transport Hint: raw + Unreliable: false + Value: true + - Class: rviz/Image + Enabled: true + Image Topic: /wild_visual_navigation_node/front/debug/last_image_confidence + Max Value: 1 + Median window: 5 + Min Value: 0 + Name: Uncertainty + Normalize Range: true + Queue Size: 2 + Transport Hint: raw + Unreliable: false + Value: true + - Alpha: 1 + Buffer Length: 1 + Class: rviz/Path + Color: 136; 138; 133 + Enabled: true + Head Diameter: 0.30000001192092896 + Head Length: 0.20000000298023224 + Length: 0.10000000149011612 + Line Style: Billboards + Line Width: 0.05000000074505806 + Name: Mission Graph + Offset: + X: 0 + Y: 0 + Z: 0 + Pose Color: 255; 85; 255 + Pose Style: Axes + Queue Size: 10 + Radius: 0.009999999776482582 + Shaft Diameter: 0.10000000149011612 + Shaft Length: 0.10000000149011612 + Topic: /wild_visual_navigation_node/mission_graph + Unreliable: false + Value: true + - Class: rviz/Image + Enabled: true + Image Topic: /wild_visual_navigation_node/front/debug/last_node_image_mask + Max Value: 1 + Median window: 5 + Min Value: 0 + Name: Reprojected Path + Normalize Range: true + Queue Size: 2 + Transport Hint: raw + Unreliable: false + Value: true + - Class: rviz/Image + Enabled: true + Image Topic: /wild_visual_navigation_node/front/debug/last_node_image_labeled + Max Value: 1 + Median window: 5 + Min Value: 0 + Name: Supervision Signal + Normalize Range: true + Queue Size: 2 + Transport Hint: raw + Unreliable: false + Value: true + Enabled: false + Name: Self Supervision (Debug) + - Class: rviz/Group + Displays: + - Class: rviz/Image + Enabled: false + Image Topic: /wide_angle_camera_front/image_color_rect_resize + Max Value: 1 + Median window: 5 + Min Value: 0 + Name: AB Wide Angle Front + Normalize Range: true + Queue Size: 2 + Transport Hint: raw + Unreliable: false + Value: false + - Class: rviz/Image + Enabled: false + Image Topic: /wide_angle_camera_rear/image_color_rect_resize + Max Value: 1 + Median window: 5 + Min Value: 0 + Name: AB Wide Rear Front + Normalize Range: true + Queue Size: 2 + Transport Hint: raw + Unreliable: false + Value: false + - Class: rviz/Image + Enabled: false + Image Topic: /hdr_camera_resize/image_raw + Max Value: 1 + Median window: 5 + Min Value: 0 + Name: HDR + Normalize Range: true + Queue Size: 2 + Transport Hint: raw + Unreliable: false + Value: false + Enabled: true + Name: Cameras Resized + - Class: rviz/Group + Displays: + - Alpha: 1 + Autocompute Intensity Bounds: true + Autocompute Value Bounds: + Max Value: 21.55472183227539 + Min Value: -7.581119537353516 + Value: true + Axis: Z + Channel Name: intensity + Class: rviz/PointCloud2 + Color: 239; 41; 41 + Color Transformer: FlatColor + Decay Time: 0 + Enabled: true + Invert Rainbow: false + Max Color: 255; 255; 255 + Min Color: 0; 0; 0 + Name: realsense-front + Position Transformer: XYZ + Queue Size: 10 + Selectable: true + Size (Pixels): 3 + Size (m): 0.009999999776482582 + Style: Squares + Topic: /depth_camera_front_upper/point_cloud_self_filtered + Unreliable: false + Use Fixed Frame: true + Use rainbow: true + Value: true + - Alpha: 1 + Autocompute Intensity Bounds: true + Autocompute Value Bounds: + Max Value: 21.55472183227539 + Min Value: -7.581119537353516 + Value: true + Axis: Z + Channel Name: intensity + Class: rviz/PointCloud2 + Color: 239; 41; 41 + Color Transformer: FlatColor + Decay Time: 0 + Enabled: true + Invert Rainbow: false + Max Color: 255; 255; 255 + Min Color: 0; 0; 0 + Name: realsense-rear + Position Transformer: XYZ + Queue Size: 10 + Selectable: true + Size (Pixels): 3 + Size (m): 0.009999999776482582 + Style: Squares + Topic: /depth_camera_rear_upper/point_cloud_self_filtered + Unreliable: false + Use Fixed Frame: true + Use rainbow: true + Value: true + - Alpha: 1 + Autocompute Intensity Bounds: true + Autocompute Value Bounds: + Max Value: 21.55472183227539 + Min Value: -7.581119537353516 + Value: true + Axis: Z + Channel Name: intensity + Class: rviz/PointCloud2 + Color: 239; 41; 41 + Color Transformer: FlatColor + Decay Time: 0 + Enabled: true + Invert Rainbow: false + Max Color: 255; 255; 255 + Min Color: 0; 0; 0 + Name: realsense-left + Position Transformer: XYZ + Queue Size: 10 + Selectable: true + Size (Pixels): 3 + Size (m): 0.009999999776482582 + Style: Squares + Topic: /depth_camera_left/point_cloud_self_filtered + Unreliable: false + Use Fixed Frame: true + Use rainbow: true + Value: true + - Alpha: 1 + Autocompute Intensity Bounds: true + Autocompute Value Bounds: + Max Value: 21.55472183227539 + Min Value: -7.581119537353516 + Value: true + Axis: Z + Channel Name: intensity + Class: rviz/PointCloud2 + Color: 239; 41; 41 + Color Transformer: FlatColor + Decay Time: 0 + Enabled: true + Invert Rainbow: false + Max Color: 255; 255; 255 + Min Color: 0; 0; 0 + Name: realsense-right + Position Transformer: XYZ + Queue Size: 10 + Selectable: true + Size (Pixels): 3 + Size (m): 0.009999999776482582 + Style: Squares + Topic: /depth_camera_right/point_cloud_self_filtered + Unreliable: false + Use Fixed Frame: true + Use rainbow: true + Value: true + - Alpha: 1 + Autocompute Intensity Bounds: true + Autocompute Value Bounds: + Max Value: 4.33540153503418 + Min Value: -0.6894725561141968 + Value: true + Axis: Z + Channel Name: intensity + Class: rviz/PointCloud2 + Color: 239; 41; 41 + Color Transformer: AxisColor + Decay Time: 0 + Enabled: false + Invert Rainbow: false + Max Color: 255; 255; 255 + Min Color: 0; 0; 0 + Name: velodyne + Position Transformer: XYZ + Queue Size: 10 + Selectable: true + Size (Pixels): 3 + Size (m): 0.019999999552965164 + Style: Squares + Topic: /point_cloud_filter/lidar/point_cloud_filtered + Unreliable: false + Use Fixed Frame: true + Use rainbow: true + Value: false + Enabled: false + Name: Depth Sensors + - Class: rviz/Group + Displays: + - Class: rviz/Image + Enabled: true + Image Topic: /wild_visual_navigation_node/front/image_input + Max Value: 1 + Median window: 5 + Min Value: 0 + Name: Input Image + Normalize Range: true + Queue Size: 2 + Transport Hint: raw + Unreliable: false + Value: true + - Class: rviz/Image + Enabled: false + Image Topic: /wild_visual_navigation_node/front/traversability + Max Value: 1 + Median window: 5 + Min Value: 0 + Name: Traversability Raw + Normalize Range: true + Queue Size: 2 + Transport Hint: raw + Unreliable: false + Value: false + - Class: rviz/Image + Enabled: false + Image Topic: /wild_visual_navigation_node/front/debug/last_node_image_overlay + Max Value: 1 + Median window: 5 + Min Value: 0 + Name: Learning Mask + Normalize Range: true + Queue Size: 2 + Transport Hint: raw + Unreliable: false + Value: false + - Class: rviz/Image + Enabled: true + Image Topic: /wild_visual_navigation_visu_traversability_front/traversability_overlayed + Max Value: 1 + Median window: 5 + Min Value: 0 + Name: F Traversability Overlay + Normalize Range: true + Queue Size: 2 + Transport Hint: raw + Unreliable: false + Value: true + - Class: rviz/Image + Enabled: false + Image Topic: /wild_visual_navigation_visu_traversability_rear/traversability_overlayed + Max Value: 1 + Median window: 5 + Min Value: 0 + Name: R Traversability Overlay + Normalize Range: true + Queue Size: 2 + Transport Hint: raw + Unreliable: false + Value: false + - Class: rviz/Image + Enabled: true + Image Topic: /wild_visual_navigation_visu_confidence_front/confidence_overlayed + Max Value: 1 + Median window: 5 + Min Value: 0 + Name: F Confidence Overlay + Normalize Range: true + Queue Size: 2 + Transport Hint: raw + Unreliable: false + Value: true + - Class: rviz/Image + Enabled: false + Image Topic: /wild_visual_navigation_visu_confidence_rear/confidence_overlayed + Max Value: 1 + Median window: 5 + Min Value: 0 + Name: R Confidence Overlay + Normalize Range: true + Queue Size: 2 + Transport Hint: raw + Unreliable: false + Value: false + Enabled: true + Name: Prediction + - Class: rviz/Group + Displays: + - Class: rviz/Marker + Enabled: false + Marker Topic: /field_local_planner/real_carrot + Name: Goal (carrot) + Namespaces: + {} + Queue Size: 100 + Value: false + - Alpha: 1 + Axes Length: 1 + Axes Radius: 0.10000000149011612 + Class: rviz/PoseWithCovariance + Color: 255; 25; 0 + Covariance: + Orientation: + Alpha: 0.5 + Color: 255; 255; 127 + Color Style: Unique + Frame: Local + Offset: 1 + Scale: 1 + Value: true + Position: + Alpha: 0.30000001192092896 + Color: 204; 51; 204 + Scale: 1 + Value: true + Value: true + Enabled: false + Head Length: 0.30000001192092896 + Head Radius: 0.10000000149011612 + Name: Goal (pose) + Queue Size: 10 + Shaft Length: 1 + Shaft Radius: 0.05000000074505806 + Shape: Arrow + Topic: /field_local_planner/current_goal + Unreliable: false + Value: false + Enabled: true + Name: Local planner + Enabled: true + Name: Wild Visual Navigation + - Class: rviz/Group + Displays: + - Alpha: 1 + Autocompute Intensity Bounds: true + Class: grid_map_rviz_plugin/GridMap + Color: 200; 200; 200 + Color Layer: elevation + Color Transformer: GridMapLayer + ColorMap: terrain + Enabled: false + Grid Cell Decimation: 1 + Grid Line Thickness: 0.10000000149011612 + Height Layer: elevation + Height Transformer: GridMapLayer + History Length: 1 + Invert ColorMap: false + Max Color: 255; 255; 255 + Max Intensity: 10 + Min Color: 0; 0; 0 + Min Intensity: 0 + Name: Elevation + Show Grid Lines: true + Topic: /elevation_mapping/semantic_map + Unreliable: false + Use ColorMap: true + Value: false + - Alpha: 1 + Autocompute Intensity Bounds: false + Class: grid_map_rviz_plugin/GridMap + Color: 200; 200; 200 + Color Layer: visual_traversability + Color Transformer: "" + ColorMap: coolwarm + Enabled: false + Grid Cell Decimation: 1 + Grid Line Thickness: 0.10000000149011612 + Height Layer: elevation_with_semantics + Height Transformer: GridMapLayer + History Length: 1 + Invert ColorMap: true + Max Color: 255; 255; 255 + Max Intensity: 1 + Min Color: 0; 0; 0 + Min Intensity: 0 + Name: Visual Traversability + Show Grid Lines: false + Topic: /elevation_mapping/semantic_map + Unreliable: false + Use ColorMap: true + Value: false + - Alpha: 1 + Autocompute Intensity Bounds: false + Class: grid_map_rviz_plugin/GridMap + Color: 200; 200; 200 + Color Layer: traversability + Color Transformer: "" + ColorMap: coolwarm + Enabled: false + Grid Cell Decimation: 1 + Grid Line Thickness: 0.10000000149011612 + Height Layer: elevation + Height Transformer: GridMapLayer + History Length: 1 + Invert ColorMap: true + Max Color: 255; 255; 255 + Max Intensity: 1 + Min Color: 0; 0; 0 + Min Intensity: 0 + Name: Geom Traversability + Show Grid Lines: true + Topic: /elevation_mapping/semantic_map + Unreliable: false + Use ColorMap: true + Value: false + Enabled: true + Name: GridMaps + Enabled: true + Global Options: + Background Color: 255; 255; 255 + Default Light: true + Fixed Frame: odom + Frame Rate: 30 + Name: root + Tools: + - Class: rviz/Interact + Hide Inactive Objects: true + - Class: rviz/MoveCamera + - Class: rviz/Select + - Class: rviz/FocusCamera + - Class: rviz/Measure + - Class: rviz/SetInitialPose + Theta std deviation: 0.2617993950843811 + Topic: /initialpose + X std deviation: 0.5 + Y std deviation: 0.5 + - Class: rviz/SetGoal + Topic: /move_base_simple/goal + - Class: rviz/PublishPoint + Single click: true + Topic: /clicked_point + Value: true + Views: + Current: + Class: rviz/ThirdPersonFollower + Distance: 3.611910343170166 + Enable Stereo Rendering: + Stereo Eye Separation: 0.05999999865889549 + Stereo Focal Distance: 1 + Swap Stereo Eyes: false + Value: false + Field of View: 0.7853981852531433 + Focal Point: + X: 2.143528938293457 + Y: -0.917067289352417 + Z: -2.384185791015625e-07 + Focal Shape Fixed Size: false + Focal Shape Size: 0.05000000074505806 + Invert Z Axis: false + Name: Current View + Near Clip Distance: 0.009999999776482582 + Pitch: 0.7603982090950012 + Target Frame: base + Yaw: 0.42539846897125244 + Saved: ~ +Window Geometry: + AB Wide Angle Front: + collapsed: false + AB Wide Rear Front: + collapsed: false + Displays: + collapsed: false + F Confidence Overlay: + collapsed: false + F Traversability Overlay: + collapsed: false + HDR: + collapsed: false + Height: 1043 + Hide Left Dock: false + Hide Right Dock: false + Input Image: + collapsed: false + Learning Mask: + collapsed: false + QMainWindow State: 000000ff00000000fd0000000400000000000001a400000253fc0200000012fb0000001200530065006c0065006300740069006f006e00000001e10000009b0000005c00fffffffb0000001e0054006f006f006c002000500072006f007000650072007400690065007302000005fb000001df00000185000000a3fb000000120056006900650077007300200054006f006f02000001df000002110000018500000122fb000000200054006f006f006c002000500072006f0070006500720074006900650073003203000002880000011d000002210000017afb000000100044006900730070006c00610079007301000001a500000253000000c700fffffffb0000002000730065006c0065006300740069006f006e00200062007500660066006500720200000138000000aa0000023a00000294fb00000014005700690064006500530074006500720065006f02000000e6000000d2000003ee0000030bfb0000000c004b0069006e0065006300740200000186000001060000030c00000261fb0000000c00430061006d00650072006100000002ea000000c80000000000000000fb00000036005300750070006500720076006900730069006f006e0020002d00200049006d0061006700650020004c006100620065006c006500640000000259000000520000000000000000fb00000024005300750070006500720076006900730069006f006e0020002d0020004d00610073006b000000027e000000b80000000000000000fb0000000a0049006d00610067006500000002eb000001280000000000000000fb000000140043006f006e0066006900640065006e0063006500000003b3000000c60000000000000000fb0000003e004600200054007200610076006500720073006100620069006c0069007400790020004f007600650072006c006100790020005200650070006c0061007900000004a40000009f0000000000000000fb00000028004600200054007200610076006500720073006100620069006c00690074007900200052006100770000000482000000c10000000000000000fb00000028004600200054007200610076006500720073006100620069006c00690074007900200052006100770000000482000000c10000000000000000fb0000000a0049006d0061006700650000000249000000210000000000000000fb0000000a0049006d0061006700650000000270000000200000000000000000000000010000010f00000253fc0200000003fb0000001e0054006f006f006c002000500072006f00700065007200740069006500730100000041000000780000000000000000fb0000000a0056006900650077007301000001a500000253000000a000fffffffb0000001200530065006c0065006300740069006f006e010000025a000000b20000000000000000000000020000078000000164fc0100000015fb0000001c0049006d00610067006500200046006500610074007500720065007300000000bf000003690000000000000000fb000000060048004400520000000000000000950000004f00fffffffb00000024004100420020005700690064006500200052006500610072002000460072006f006e0074000000000000000117000000aa00fffffffc0000012a0000010c0000000000fffffffa000000000200000003fb0000000a0049006d0061006700650000000000ffffffff0000000000000000fb0000002400460072006f006e0074002000280063006f006d007000720065007300730065006400290000000000ffffffff0000000000000000fb0000001e00410053002000460072006f006e00740020002800630061006d00340029000000003d0000010c0000000000000000fc000002570000019b0000000000fffffffa000000000200000001fb0000000a0049006d0061006700650000000000ffffffff0000000000000000fc000000000000025a0000007d00fffffffa000000000200000003fb000000160049006e00700075007400200049006d006100670065010000003d000001ac0000001600fffffffb0000001a005200200049006e00700075007400200049006d0061006700650000000000ffffffff0000000000000000fb0000001a004c00200049006e00700075007400200049006d0061006700650000000000ffffffff0000000000000000fb00000028005200200043006f006e0066006900640065006e006300650020004f007600650072006c0061007900000000000000013f000000b600fffffffb00000030005200200054007200610076006500720073006100620069006c0069007400790020004f007600650072006c00610079000000000000000162000000c500fffffffb00000026004100420020005700690064006500200041006e0067006c0065002000460072006f006e0074000000012500000282000000b000fffffffb00000028004600200043006f006e0066006900640065006e006300650020004f007600650072006c00610079010000026000000242000000b500fffffffb00000030004600200054007200610076006500720073006100620069006c0069007400790020004f007600650072006c0061007901000004a8000002d8000000c400fffffffb00000028004600200043006f006e0066006900640065006e006300650020004f007600650072006c00610079010000080e000001aa0000000000000000fb0000000a0049006d00610067006501000003240000018a0000000000000000fb0000002c0054007200610076006500720073006100620069006c0069007400790020004f007600650072006c0061007900000004ae0000010c0000000000000000fc00000846000000bd0000000000fffffffa000000000200000006fb000000160046002000540072006100760020004f0076006500720000000000ffffffff0000000000000000fb0000003e004600200054007200610076006500720073006100620069006c0069007400790020004f007600650072006c006100790020005200650070006c006100790100000000ffffffff0000000000000000fb000000240054007200610076006500720073006100620069006c0069007400790020005200610077000000003d0000012b0000001600fffffffb00000028005200200054007200610076006500720073006100620069006c00690074007900200052006100770000000000ffffffff0000000000000000fb00000028004c00200054007200610076006500720073006100620069006c00690074007900200052006100770000000000ffffffff0000000000000000fb00000030004600200054007200610076006500720073006100620069006c0069007400790020004f007600650072006c006100790000000000ffffffff0000000000000000fc00000909000000af0000000000fffffffa000000000200000005fb00000036004600200043006f006e0066006900640065006e006300650020004f007600650072006c006100790020005200650070006c006100790100000000ffffffff0000000000000000fb0000001a004c006500610072006e0069006e00670020004d00610073006b000000003d0000012b0000001600fffffffb00000020005200200043006f006e0066006900640065006e0063006500200052006100770000000000ffffffff0000000000000000fb00000020004c00200043006f006e0066006900640065006e0063006500200052006100770000000000ffffffff0000000000000000fb00000028004600200043006f006e0066006900640065006e006300650020004f007600650072006c006100790000000000ffffffff0000000000000000fb000000160050006c0061006300650068006f006c00640065007200000000000000074a0000000000000000fb000000160049006e00700075007400200049006d006100670065010000031e000001790000000000000000fb00000036005300750070006500720076006900730069006f006e0020002d00200049006d0061006700650020004c006100620065006c006500640100000659000001af0000000000000000fb00000024005300750070006500720076006900730069006f006e0020002d0020004d00610073006b010000080e000001aa0000000000000000fb0000000a00560069006500770073030000004e00000080000002e100000197000000030000074a00000042fc010000000dfb000000200052006500700072006f006a00650063007400650064002000500061007400680000000000000000ab0000009c00fffffffb00000024005300750070006500720076006900730069006f006e0020005300690067006e0061006c00000000b10000056e000000a300fffffffb0000001c0054007200610076006500720073006100620069006c00690074007900000006250000009a0000008600fffffffb000000160055006e006300650072007400610069006e0074007900000006c5000000850000007b00fffffffb000000160050006c0061006300650068006f006c006400650072000000000000000eb80000000000000000fb0000001c0054007200610076006500720073006100620069006c006900740079010000031f0000017d0000000000000000fb000000160055006e006300650072007400610069006e0074007901000004a2000002960000000000000000fb000000140043006f006e0066006900640065006e006300650000000000000003d40000000000000000fb0000001c0054007200610076006500720073006100620069006c0069007400790000000231000001f20000000000000000fb0000001c004100530020004c0065006600740020002800630061006d003300290000000000000007380000000000000000fb0000001e004100530020005200690067006800740020002800630061006d0035002900000003f0000003480000000000000000fb0000000800540069006d00650000000000000007800000035c00fffffffb0000000800540069006d00650100000000000004500000000000000000000004c10000025300000004000000040000000800000008fc0000000100000002000000010000000a0054006f006f006c00730100000000ffffffff0000000000000000 + R Confidence Overlay: + collapsed: false + R Traversability Overlay: + collapsed: false + Reprojected Path: + collapsed: false + Selection: + collapsed: false + Supervision Signal: + collapsed: false + Time: + collapsed: false + Tool Properties: + collapsed: false + Traversability: + collapsed: false + Traversability Raw: + collapsed: false + Uncertainty: + collapsed: false + Views: + collapsed: false + Width: 1920 + X: 0 + Y: 136 diff --git a/wild_visual_navigation_jackal/config/wild_visual_navigation/camera.yaml b/wild_visual_navigation_jackal/config/wild_visual_navigation/camera.yaml new file mode 100644 index 00000000..be7c3262 --- /dev/null +++ b/wild_visual_navigation_jackal/config/wild_visual_navigation/camera.yaml @@ -0,0 +1,12 @@ +camera_topics: + front: + image_topic: "/depth_camera/color/image_raw" + info_topic: "/depth_camera/color/camera_info" + use_for_training: true + publish_confidence: true + publish_input_image: true + scheduler_weight: 1 + +# Provides 1080 (height) x 1920 (width) images +network_input_image_height: 224 # 448 +network_input_image_width: 224 # 448 \ No newline at end of file diff --git a/wild_visual_navigation_jackal/config/wild_visual_navigation/jackal.yaml b/wild_visual_navigation_jackal/config/wild_visual_navigation/jackal.yaml new file mode 100644 index 00000000..8ebd9c7a --- /dev/null +++ b/wild_visual_navigation_jackal/config/wild_visual_navigation/jackal.yaml @@ -0,0 +1,57 @@ +# Input topics +robot_state_topic: "/wild_visual_navigation_node/robot_state" +desired_twist_topic: "/wild_visual_navigation_node/reference_twist" + +# Relevant frames +fixed_frame: odom +base_frame: base_link +footprint_frame: base_link + +# Robot size used to project the footprint +robot_length: 0.5 +robot_width: 0.3 +robot_height: 0.3 + + +# Traversability estimation parameters +traversability_radius: 3.0 # Maximum distance in meters of the supervision graph +image_graph_dist_thr: 0.2 # Minimum distance between mission nodes being added +supervision_graph_dist_thr: 0.1 # Minimum distance between supervision nodes being added +network_input_image_height: 224 # Unput image height +network_input_image_width: 224 # Input image width +segmentation_type: "stego" # Options: slic, grid, random, stego +feature_type: "stego" # Options: dino, dinov2, stego +dino_patch_size: 8 # Options: 8, 16; We found 8 is sufficient and faster +dino_backbone: vit_small # Options: vit_small +slic_num_components: 100 # Number of segments for slic +confidence_std_factor: 1.0 # Tuning parameter to change the confidence computation / explained in confidence_generator.py +min_samples_for_training: 5 # Minimum number of mission nodes with successfull reprojection before start training +prediction_per_pixel: True # If true, trained network is inferenced for each pixel, otherwise per segment +vis_node_index: 10 # Defines node which is used for visualization, can help to debug the projected footprint + # 1 -> indicates latest added mission node; 10 -> indicates the node added 10 steps in the past + +# Supervision Generator +untraversable_thr: 0.0 # Threshold for untraversable area -> this threshold is used to spawn special "collision walls" -> not used in the paper +mission_name: "test" # Name of the mission_folder used for logging +mission_timestamp: True # If true current timestamp is used as a prefix for the mission_folder + +# Threads +image_callback_rate: 10 # Maximum rate at which images are processed in hertz +supervision_callback_rate: 10 # Maximum rate at which supervision_signals are processed in hertz +learning_thread_rate: 10 # Gradient steps per second in hertz +logging_thread_rate: 2 # Logging of learning_node in hertz +status_thread_rate: 0.5 # Status of feature_extractor_node in hertz +load_save_checkpoint_rate: 1.0 # Rate at which checkpoints are saved / loaded between learning and feature extraction + + +# Runtime options +device: "cuda" +mode: "online" # Options: "online", "debug", "extract_labels"; Sets operation mode of both nodes +colormap: "RdYlBu" + +print_image_callback_time: false +print_supervision_callback_time: false +log_time: false +log_confidence: false +verbose: true +extraction_store_folder: "nan" \ No newline at end of file diff --git a/wild_visual_navigation_jackal/launch/sim.launch b/wild_visual_navigation_jackal/launch/sim.launch index ae44791d..717b71bf 100644 --- a/wild_visual_navigation_jackal/launch/sim.launch +++ b/wild_visual_navigation_jackal/launch/sim.launch @@ -2,13 +2,13 @@ - + - - - + + + @@ -28,5 +28,5 @@ + args="-urdf -model jackal -param robot_description -x $(arg x) -y $(arg y) -z $(arg z) -R 0 -P 0 -Y $(arg yaw)" /> diff --git a/wild_visual_navigation_jackal/launch/teleop.launch b/wild_visual_navigation_jackal/launch/teleop.launch new file mode 100644 index 00000000..3ffe082f --- /dev/null +++ b/wild_visual_navigation_jackal/launch/teleop.launch @@ -0,0 +1,7 @@ + + + + + + + \ No newline at end of file diff --git a/wild_visual_navigation_jackal/launch/view.launch b/wild_visual_navigation_jackal/launch/view.launch new file mode 100644 index 00000000..444798f5 --- /dev/null +++ b/wild_visual_navigation_jackal/launch/view.launch @@ -0,0 +1,7 @@ + + + + + + + diff --git a/wild_visual_navigation_jackal/launch/wild_visual_navigation.launch b/wild_visual_navigation_jackal/launch/wild_visual_navigation.launch new file mode 100644 index 00000000..0d9eed37 --- /dev/null +++ b/wild_visual_navigation_jackal/launch/wild_visual_navigation.launch @@ -0,0 +1,17 @@ + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/wild_visual_navigation_jackal/scripts/jackal_state_converter_node.py b/wild_visual_navigation_jackal/scripts/jackal_state_converter_node.py new file mode 100644 index 00000000..a6087d44 --- /dev/null +++ b/wild_visual_navigation_jackal/scripts/jackal_state_converter_node.py @@ -0,0 +1,81 @@ +#!/usr/bin/python3 +# +# Copyright (c) 2022-2024, ETH Zurich, Matias Mattamala, Jonas Frey. +# All rights reserved. Licensed under the MIT license. +# See LICENSE file in the project root for details. +# +from geometry_msgs.msg import Twist, TwistStamped +from nav_msgs.msg import Odometry +from wild_visual_navigation_msgs.msg import RobotState, CustomState +import rospy + + +def jackal_msg_callback(jackal_state, return_msg=False): + robot_state_msg = RobotState() + + # For RobotState msg + robot_state_msg.header = jackal_state.header + + # Extract pose + robot_state_msg.pose.header = jackal_state.header + robot_state_msg.pose.pose = jackal_state.pose.pose + + # Extract twist + robot_state_msg.twist.header = jackal_state.header + robot_state_msg.twist.header.frame_id = jackal_state.child_frame_id + robot_state_msg.twist.twist = jackal_state.twist.twist + + vector_state = CustomState() + vector_state.name = "vector_state" + vector_state.dim = 7 + 6 # + 4 * 12 + vector_state.values = [0] * vector_state.dim + vector_state.labels = [""] * vector_state.dim + vector_state.values = [0] * vector_state.dim + robot_state_msg.states.append(vector_state) + + robot_state_msg.states[0].values[0] = robot_state_msg.pose.pose.position.x + robot_state_msg.states[0].values[1] = robot_state_msg.pose.pose.position.y + robot_state_msg.states[0].values[2] = robot_state_msg.pose.pose.position.z + robot_state_msg.states[0].values[3] = robot_state_msg.pose.pose.orientation.x + robot_state_msg.states[0].values[4] = robot_state_msg.pose.pose.orientation.y + robot_state_msg.states[0].values[5] = robot_state_msg.pose.pose.orientation.z + robot_state_msg.states[0].values[6] = robot_state_msg.pose.pose.orientation.w + robot_state_msg.states[0].values[7] = robot_state_msg.twist.twist.linear.x + robot_state_msg.states[0].values[8] = robot_state_msg.twist.twist.linear.y + robot_state_msg.states[0].values[9] = robot_state_msg.twist.twist.linear.z + robot_state_msg.states[0].values[10] = robot_state_msg.twist.twist.angular.x + robot_state_msg.states[0].values[11] = robot_state_msg.twist.twist.angular.y + robot_state_msg.states[0].values[12] = robot_state_msg.twist.twist.angular.z + + for i, x in enumerate(["tx", "ty", "tz", "qx", "qy", "qz", "qw", "vx", "vy", "vz", "wx", "wy", "wz"]): + robot_state_msg.states[0].labels[i] = x + + if return_msg: + return robot_state_msg + # Publish + robot_state_pub.publish(robot_state_msg) + + +def twist_msg_callback(msg): + ts = rospy.Time.now() + out_msg = TwistStamped() + out_msg.header.stamp = ts + out_msg.header.frame_id = "base_link" + out_msg.twist = msg + + ref_twiststamped_pub.publish(out_msg) + + +if __name__ == "__main__": + rospy.init_node("jackal_state_converter_node") + + # We subscribe the odometry topic (state) + jackal_state_sub = rospy.Subscriber("/odometry/filtered", Odometry, jackal_msg_callback, queue_size=20) + robot_state_pub = rospy.Publisher("/wild_visual_navigation_node/robot_state", RobotState, queue_size=20) + + # And also the twist command from teleoperation + ref_twist_sub = rospy.Subscriber("/jackal_velocity_controller/cmd_vel", Twist, twist_msg_callback, queue_size=20) + ref_twiststamped_pub = rospy.Publisher("/wild_visual_navigation_node/reference_twist", TwistStamped, queue_size=20) + + rospy.loginfo("[jackal_state_converter_node] ready") + rospy.spin() diff --git a/wild_visual_navigation_jackal/urdf/depth_camera.xacro b/wild_visual_navigation_jackal/urdf/depth_camera.xacro index 474952ae..c885a038 100644 --- a/wild_visual_navigation_jackal/urdf/depth_camera.xacro +++ b/wild_visual_navigation_jackal/urdf/depth_camera.xacro @@ -18,7 +18,7 @@ - + @@ -45,6 +45,14 @@ + + + + + + + + true @@ -76,7 +84,7 @@ /${name}/depth/image_rect_raw /${name}/depth/camera_info /${name}/depth/color/points - ${name}_link + optical_${name}_link 0.105 8.0 0.00000001 diff --git a/wild_visual_navigation_ros/launch/wild_visual_navigation.launch b/wild_visual_navigation_ros/launch/wild_visual_navigation.launch index 0e7f8042..0e75c4b5 100644 --- a/wild_visual_navigation_ros/launch/wild_visual_navigation.launch +++ b/wild_visual_navigation_ros/launch/wild_visual_navigation.launch @@ -1,6 +1,6 @@ - + @@ -10,9 +10,9 @@ - + - + diff --git a/wild_visual_navigation_ros/scripts/wvn_learning_node.py b/wild_visual_navigation_ros/scripts/wvn_learning_node.py index 9bcb3a55..67882da1 100644 --- a/wild_visual_navigation_ros/scripts/wvn_learning_node.py +++ b/wild_visual_navigation_ros/scripts/wvn_learning_node.py @@ -42,7 +42,6 @@ import traceback import signal import sys -import time def time_func(): @@ -296,10 +295,16 @@ def setup_ros(self, setup_fully=True): # Wait for features message to determine the input size of the model cam = list(self._ros_params.camera_topics.keys())[0] + exists_camera_used_for_training = False for cam in self._ros_params.camera_topics: rospy.loginfo(f"[{self._node_name}] Waiting for feat topic {cam}...") if self._ros_params.camera_topics[cam]["use_for_training"]: feat_msg = rospy.wait_for_message(f"/wild_visual_navigation_node/{cam}/feat", ImageFeatures) + exists_camera_used_for_training = True + + if not exists_camera_used_for_training: + rospy.logerror("No camera selected for training") + sys.exit(-1) feature_dim = int(feat_msg.features.layout.dim[1].size) # Modify the parameters @@ -379,7 +384,6 @@ def learning_thread_loop(self): # Check the rate ts = time_func() if abs(ts - self._last_checkpoint_ts) > 1.0 / self._ros_params.load_save_checkpoint_rate: - cg = self._traversability_estimator._traversability_loss._confidence_generator new_model_state_dict["confidence_generator"] = cg.get_dict() From 7aeffb6bd1d7ce649cf1a9607c530f37a53d2164 Mon Sep 17 00:00:00 2001 From: Jonas Frey Date: Tue, 26 Mar 2024 11:54:49 +0100 Subject: [PATCH 04/15] updated installation instructions --- wild_visual_navigation_jackal/README.md | 25 ++++++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/wild_visual_navigation_jackal/README.md b/wild_visual_navigation_jackal/README.md index bb92611e..50a45ad5 100644 --- a/wild_visual_navigation_jackal/README.md +++ b/wild_visual_navigation_jackal/README.md @@ -4,9 +4,16 @@ Simulation environment to test Wild Visual Navigation (WVN). We use a modified C ## Requirements + +```sh +wget https://packages.clearpathrobotics.com/public.key -O - | sudo apt-key add - +sudo sh -c 'echo "deb https://packages.clearpathrobotics.com/stable/ubuntu $(lsb_release -cs) main" > /etc/apt/sources.list.d/clearpath-latest.list' +sudo apt-get update +``` + ```sh -$ sudo apt update -$ sudo apt install -y \ +sudo apt update +sudo apt install -y \ ros-noetic-jackal-simulator \ ros-noetic-jackal-desktop \ ros-noetic-teleop-twist-keyboard \ @@ -16,5 +23,17 @@ $ sudo apt install -y \ ## Running ```sh -$ roslaunch wild_visual_navigation_sim sim.launch +roslaunch wild_visual_navigation_jackal sim.launch +``` + +```sh +roslaunch wild_visual_navigation_jackal teleop.launch +``` + +```sh +roslaunch wild_visual_navigation_jackal view.launch +``` + +```sh +roslaunch wild_visual_navigation_jackal wild_visual_navigation.launch ``` From b788284fb5ee8b1b8d0b68bb5906b2b598ff488f Mon Sep 17 00:00:00 2001 From: Jonas Frey Date: Tue, 26 Mar 2024 12:01:25 +0100 Subject: [PATCH 05/15] Updated Readme --- README.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/README.md b/README.md index 4bf9e289..8c23ed42 100644 --- a/README.md +++ b/README.md @@ -200,6 +200,13 @@ python wild_visual_navigation_ros/scripts/wvn_learning_node.py ## Development +### Install pre-commit +```shell +pip3 install pre-commit +cd wild_visual_navigation && python3 -m pre_commit install +cd wild_visual_navigation && python3 -m pre_commit run +``` + ### Code formatting ```shell # for formatting From 0eb349e0a12e2676f63f5a3c603ad90f53797518 Mon Sep 17 00:00:00 2001 From: Jonas Frey Date: Tue, 26 Mar 2024 22:24:11 +0100 Subject: [PATCH 06/15] started implementing quick start --- quick_start.py | 189 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 189 insertions(+) create mode 100644 quick_start.py diff --git a/quick_start.py b/quick_start.py new file mode 100644 index 00000000..2ed5524c --- /dev/null +++ b/quick_start.py @@ -0,0 +1,189 @@ +# +# Copyright (c) 2022-2024, ETH Zurich, Matias Mattamala, Jonas Frey. +# All rights reserved. Licensed under the MIT license. +# See LICENSE file in the project root for details. +# +from wild_visual_navigation import WVN_ROOT_DIR +from wild_visual_navigation.feature_extractor import FeatureExtractor +from wild_visual_navigation.cfg import ExperimentParams +from wild_visual_navigation.image_projector import ImageProjector +from wild_visual_navigation.model import get_model +from wild_visual_navigation.utils import ConfidenceGenerator +from wild_visual_navigation.utils import AnomalyLoss +from PIL import Image +import torch +import numpy as np +import torch.nn.functional as F +from omegaconf import OmegaConf +from wild_visual_navigation.utils import Data +from os.path import join +import os +from argparse import ArgumentParser +from wild_visual_navigation.model import get_model +from pathlib import Path +from wild_visual_navigation.visu import LearningVisualizer + + +# Function to handle folder creation +def parse_folders(args): + input_image_folder = args.input_image_folder + output_folder = args.output_folder_name + + # Check if input folder is global or local + if not os.path.isabs(input_image_folder): + input_image_folder = os.path.join(WVN_ROOT_DIR, "assets", input_image_folder) + + # Check if output folder is global or local + if not os.path.isabs(output_folder): + output_folder = os.path.join(WVN_ROOT_DIR, "results", output_folder) + + # Create input folder if it doesn't exist + if not os.path.exists(input_image_folder): + raise ValueError(f"Input folder '{input_image_folder}' does not exist.") + + # Create output folder if it doesn't exist + if not os.path.exists(output_folder): + os.makedirs(output_folder) + return input_image_folder, output_folder + + +if __name__ == "__main__": + parser = ArgumentParser() + + # Define command line arguments + parser.add_argument("--prediction_per_pixel", default=True, help="Description of prediction per pixel argument") + parser.add_argument("--model_name", default="indoor_mpi", help="Description of model name argument") + parser.add_argument( + "--input_image_folder", + default="demo_data", + help="If not gloabl will search for the folde name within the assests folder", + ) + parser.add_argument( + "--output_folder_name", + default="demo_data", + help="If not global will create the folder within the results folder", + ) + + # Fixed values + parser.add_argument("--network_input_image_height", type=int, default=224) + parser.add_argument("--network_input_image_width", type=int, default=224) + parser.add_argument( + "--segmentation_type", + default="stego", + choices=["slic", "grid", "random", "stego"], + help="Options: slic, grid, random, stego", + ) + parser.add_argument( + "--feature_type", default="stego", choices=["dino", "dinov2", "stego"], help="Options: dino, dinov2, stego" + ) + parser.add_argument("--dino_patch_size", type=int, default=8, choices=[8, 16], help="Options: 8, 16") + parser.add_argument("--dino_backbone", default="vit_small", choices=["vit_small"], help="Options: vit_small") + parser.add_argument( + "--slic_num_components", type=int, default=100, help="Number of components for SLIC segmentation" + ) + + # Parse the command line arguments + args = parser.parse_args() + + input_image_folder, output_folder = parse_folders(args) + + params = OmegaConf.structured(ExperimentParams) + anomaly_detection = False + + # Update model from file if possible + device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + + visualizer = LearningVisualizer() + + if anomaly_detection: + confidence_generator = ConfidenceGenerator( + method=params.loss_anomaly.method, std_factor=params.loss_anomaly.confidence_std_factor + ) + else: + confidence_generator = ConfidenceGenerator( + method=params.loss.method, std_factor=params.loss.confidence_std_factor + ) + + # Load feature and segment extractor + feature_extractor = FeatureExtractor( + device=device, + segmentation_type=args.segmentation_type, + feature_type=args.feature_type, + patch_size=args.dino_patch_size, + backbone_type=args.dino_backbone, + input_size=args.network_input_image_height, + slic_num_components=args.slic_num_components, + ) + + # Sorry for that 💩 + params.model.simple_mlp_cfg.input_size = feature_extractor.feature_dim + params.model.double_mlp_cfg.input_size = feature_extractor.feature_dim + params.model.simple_gcn_cfg.input_size = feature_extractor.feature_dim + params.model.linear_rnvp_cfg.input_size = feature_extractor.feature_dim + + # Load traversability model + model = get_model(params.model).to(device) + model.eval() + p = join(WVN_ROOT_DIR, "assets", "checkpoints", f"{args.model_name}.pt") + model_state_dict = torch.load(p) + model.load_state_dict(model_state_dict, strict=False) + print(f"Model {args.model_name} successfully loaded!") + + cg = model_state_dict["confidence_generator"] + confidence_generator.var = cg["var"] + confidence_generator.mean = cg["mean"] + confidence_generator.std = cg["std"] + + images = [str(s) for s in Path(input_image_folder).rglob("*.png" or "*.jpg")] + print(f"Found {len(images)} images in the folder!") + + for i, img_p in enumerate(images): + torch_image = torch.from_numpy(np.array(Image.open(img_p))).to(device).permute(2, 0, 1).float() / 255.0 + C, H, W = torch_image.shape + # K can be ignored given that no reprojection is performed + image_projector = ImageProjector( + K=torch.eye(4, device=device)[None], + h=H, + w=W, + new_h=args.network_input_image_height, + new_w=args.network_input_image_width, + ) + torch_image = image_projector.resize_image(torch_image) + print(torch_image.shape, "post") + # Extract features + _, feat, seg, center, dense_feat = feature_extractor.extract( + img=torch_image[None], + return_centers=False, + return_dense_features=True, + n_random_pixels=100, + ) + + # Forward pass to predict traversability + if args.prediction_per_pixel: + # Pixel-wise traversability prediction using the dense features + data = Data(x=dense_feat[0].permute(1, 2, 0).reshape(-1, dense_feat.shape[1])) + else: + # input_feat = dense_feat[0].permute(1, 2, 0).reshape(-1, dense_feat.shape[1]) + # Segment-wise traversability prediction using the average feature per segment + input_feat = feat[seg.reshape(-1)] + data = Data(x=input_feat) + + # Predict traversability per feature + prediction = model.forward(data) + + if not anomaly_detection: + out_trav = prediction.reshape(H, W, -1)[:, :, 0] + else: + losses = prediction["logprob"].sum(1) + prediction["log_det"] + confidence = confidence_generator.inference_without_update(x=-losses) + trav = confidence + out_trav = trav.reshape(H, W, -1)[:, :, 0] + + # Publish traversability + out_trav.cpu().numpy() + + # Store confidence + loss_reco = F.mse_loss(prediction[:, 1:], data.x, reduction="none").mean(dim=1) + confidence = confidence_generator.inference_without_update(x=loss_reco) + out_confidence = confidence.reshape(H, W) + out_confidence.cpu().numpy() From ddf2c631b8e6ea45b84d5890d05629ddcd16c4a2 Mon Sep 17 00:00:00 2001 From: Jonas Frey Date: Mon, 1 Apr 2024 11:47:25 +0200 Subject: [PATCH 07/15] moved data and finished quick_start --- README.md | 2 +- .../bahnhofstrasse_raw.png | Bin .../mountain_bike_trail_raw.png | Bin .../{images => demo_data}/mpi_indoor_raw.png | Bin .../{images => demo_data}/mpi_outdoor_raw.png | Bin quick_start.py | 70 +++++++++++++----- .../feature_extractor/feature_extractor.py | 11 ++- .../utils/confidence_generator.py | 2 +- 8 files changed, 61 insertions(+), 24 deletions(-) rename assets/{images => demo_data}/bahnhofstrasse_raw.png (100%) rename assets/{images => demo_data}/mountain_bike_trail_raw.png (100%) rename assets/{images => demo_data}/mpi_indoor_raw.png (100%) rename assets/{images => demo_data}/mpi_outdoor_raw.png (100%) diff --git a/README.md b/README.md index 8c23ed42..719b5d65 100644 --- a/README.md +++ b/README.md @@ -121,7 +121,7 @@ What we provide: | MPI Outdoor | MPI Indoor | Bahnhofstrasse | Bike Trail | |----------------|------------|-------------|---------------------| | MPI Outdoor | MPI Indoor | Bahnhofstrasse | Mountain Bike | -| MPI Outdoor | MPI Indoor | Bahnhofstrasse | Mountain Bike | +| MPI Outdoor | MPI Indoor | Bahnhofstrasse | Mountain Bike | #### Setup diff --git a/assets/images/bahnhofstrasse_raw.png b/assets/demo_data/bahnhofstrasse_raw.png similarity index 100% rename from assets/images/bahnhofstrasse_raw.png rename to assets/demo_data/bahnhofstrasse_raw.png diff --git a/assets/images/mountain_bike_trail_raw.png b/assets/demo_data/mountain_bike_trail_raw.png similarity index 100% rename from assets/images/mountain_bike_trail_raw.png rename to assets/demo_data/mountain_bike_trail_raw.png diff --git a/assets/images/mpi_indoor_raw.png b/assets/demo_data/mpi_indoor_raw.png similarity index 100% rename from assets/images/mpi_indoor_raw.png rename to assets/demo_data/mpi_indoor_raw.png diff --git a/assets/images/mpi_outdoor_raw.png b/assets/demo_data/mpi_outdoor_raw.png similarity index 100% rename from assets/images/mpi_outdoor_raw.png rename to assets/demo_data/mpi_outdoor_raw.png diff --git a/quick_start.py b/quick_start.py index 2ed5524c..a6b0a50b 100644 --- a/quick_start.py +++ b/quick_start.py @@ -51,7 +51,7 @@ def parse_folders(args): parser = ArgumentParser() # Define command line arguments - parser.add_argument("--prediction_per_pixel", default=True, help="Description of prediction per pixel argument") + parser.add_argument("--model_name", default="indoor_mpi", help="Description of model name argument") parser.add_argument( "--input_image_folder", @@ -82,6 +82,18 @@ def parse_folders(args): "--slic_num_components", type=int, default=100, help="Number of components for SLIC segmentation" ) + parser.add_argument( + "--compute_confidence", action="store_true", help="Compute confidence for the traversability prediction" + ) + parser.add_argument("--no-compute_confidence", dest="compute_confidence", action="store_false") + parser.set_defaults(compute_confidence=True) + + parser.add_argument( + "--prediction_per_pixel", action="store_true", help="Description of prediction per pixel argument" + ) + parser.add_argument("--no-prediction_per_pixel", dest="prediction_per_pixel", action="store_false") + parser.set_defaults(prediction_per_pixel=True) + # Parse the command line arguments args = parser.parse_args() @@ -93,7 +105,7 @@ def parse_folders(args): # Update model from file if possible device = torch.device("cuda" if torch.cuda.is_available() else "cpu") - visualizer = LearningVisualizer() + visualizer = LearningVisualizer(p_visu=output_folder, store=True) if anomaly_detection: confidence_generator = ConfidenceGenerator( @@ -124,32 +136,42 @@ def parse_folders(args): # Load traversability model model = get_model(params.model).to(device) model.eval() + torch.set_grad_enabled(False) + p = join(WVN_ROOT_DIR, "assets", "checkpoints", f"{args.model_name}.pt") model_state_dict = torch.load(p) model.load_state_dict(model_state_dict, strict=False) - print(f"Model {args.model_name} successfully loaded!") + print(f"\nLoaded model `{args.model_name}` successfully!") cg = model_state_dict["confidence_generator"] + # Only mean and std are needed confidence_generator.var = cg["var"] confidence_generator.mean = cg["mean"] confidence_generator.std = cg["std"] images = [str(s) for s in Path(input_image_folder).rglob("*.png" or "*.jpg")] - print(f"Found {len(images)} images in the folder!") + print(f"Found {len(images)} images in the folder! \n") + H, W = args.network_input_image_height, args.network_input_image_width for i, img_p in enumerate(images): - torch_image = torch.from_numpy(np.array(Image.open(img_p))).to(device).permute(2, 0, 1).float() / 255.0 - C, H, W = torch_image.shape + print(f"Processing image {i+1}/{len(images)}: {img_p}") + img = Image.open(img_p) + img = img.convert("RGB") + torch_image = torch.from_numpy(np.array(img)) + torch_image = torch_image.to(device).permute(2, 0, 1).float() / 255.0 + + C, H_in, W_in = torch_image.shape + # K can be ignored given that no reprojection is performed image_projector = ImageProjector( K=torch.eye(4, device=device)[None], - h=H, - w=W, - new_h=args.network_input_image_height, - new_w=args.network_input_image_width, + h=H_in, + w=W_in, + new_h=H, + new_w=W, ) + torch_image = image_projector.resize_image(torch_image) - print(torch_image.shape, "post") # Extract features _, feat, seg, center, dense_feat = feature_extractor.extract( img=torch_image[None], @@ -168,9 +190,10 @@ def parse_folders(args): input_feat = feat[seg.reshape(-1)] data = Data(x=input_feat) - # Predict traversability per feature + # Inference model prediction = model.forward(data) + # Calculate traversability if not anomaly_detection: out_trav = prediction.reshape(H, W, -1)[:, :, 0] else: @@ -179,11 +202,20 @@ def parse_folders(args): trav = confidence out_trav = trav.reshape(H, W, -1)[:, :, 0] - # Publish traversability - out_trav.cpu().numpy() + original_img = visualizer.plot_image(torch_image, store=False) + img_ls = [original_img] + + if args.compute_confidence: + # Calculate confidence + loss_reco = F.mse_loss(prediction[:, 1:], data.x, reduction="none").mean(dim=1) + confidence = confidence_generator.inference_without_update(x=loss_reco) + out_confidence = confidence.reshape(H, W) + conf_img = visualizer.plot_detectron_classification(torch_image, out_confidence, store=False) + img_ls.append(conf_img) + + name = img_p.split("/")[-1].split(".")[0] + trav_img = visualizer.plot_detectron_classification(torch_image, out_trav, store=False) + print(out_trav.sum(), out_trav.max(), torch_image.sum(), data.x.sum(), dense_feat.sum(), torch_image.sum()) - # Store confidence - loss_reco = F.mse_loss(prediction[:, 1:], data.x, reduction="none").mean(dim=1) - confidence = confidence_generator.inference_without_update(x=loss_reco) - out_confidence = confidence.reshape(H, W) - out_confidence.cpu().numpy() + img_ls.append(trav_img) + visualizer.plot_list(img_ls, tag=f"{name}_original_conf_trav", store=True) diff --git a/wild_visual_navigation/feature_extractor/feature_extractor.py b/wild_visual_navigation/feature_extractor/feature_extractor.py index 053a2e82..88e26173 100644 --- a/wild_visual_navigation/feature_extractor/feature_extractor.py +++ b/wild_visual_navigation/feature_extractor/feature_extractor.py @@ -36,6 +36,7 @@ def __init__( self._segmentation_type = segmentation_type self._feature_type = feature_type self._input_size = input_size + self._stego_features_already_computed_in_segmentation = False # Prepare segment extractor self.segment_extractor = SegmentExtractor().to(self._device) @@ -243,6 +244,8 @@ def segment_stego(self, img, **kwargs): # Change the segment indices by numbers from 0 to N for i, k in enumerate(seg.unique()): seg[seg == k.item()] = i + + self._stego_features_already_computed_in_segmentation = True return seg def compute_features(self, img: torch.tensor, seg: torch.tensor, center: torch.tensor, **kwargs): @@ -296,10 +299,12 @@ def compute_torchvision(self, img: torch.tensor, seg: torch.tensor, center: torc @torch.no_grad() def compute_stego(self, img: torch.tensor, seg: torch.tensor, center: torch.tensor, **kwargs): - try: + if self._stego_features_already_computed_in_segmentation: + self._stego_features_already_computed_in_segmentation = False return self._extractor.features - except Exception: - self.segment_stego(img, **kwargs) + else: + img_internal = img.clone() + self._extractor.inference(img_internal) return self._extractor.features def sparsify_features(self, dense_features: torch.tensor, seg: torch.tensor, cumsum_trick=False): diff --git a/wild_visual_navigation/utils/confidence_generator.py b/wild_visual_navigation/utils/confidence_generator.py index 1d920c5b..234ed71b 100644 --- a/wild_visual_navigation/utils/confidence_generator.py +++ b/wild_visual_navigation/utils/confidence_generator.py @@ -185,7 +185,7 @@ def inference_without_update(self, x: torch.tensor): return torch.zeros_like(x) shifted_mean = self.mean + self.std * self.std_factor std_fac = 1 - interval_min = max(shifted_mean - std_fac * self.std, 0) + interval_min = max(shifted_mean - std_fac * self.std, torch.zeros_like(self.std)) interval_max = shifted_mean + std_fac * self.std x = torch.clip(x, interval_min, interval_max) confidence = 1 - ((x - interval_min) / (interval_max - interval_min)) From e6cd7c2eac64f8c518dc30c086c71645e0fce06a Mon Sep 17 00:00:00 2001 From: JonasFrey96 <56234378+JonasFrey96@users.noreply.github.com> Date: Mon, 1 Apr 2024 13:23:50 +0200 Subject: [PATCH 08/15] Update README.md first draft --- README.md | 134 ++++++++++++++++++++++++++++++------------------------ 1 file changed, 74 insertions(+), 60 deletions(-) diff --git a/README.md b/README.md index 719b5d65..884d117b 100644 --- a/README.md +++ b/README.md @@ -1,98 +1,73 @@

Fast Traversability Estimation for Wild Visual Navigation

- Citation • Installation • Overview • Experiments • - Development + Development • + Citation ![Formatting](https://github.com/leggedrobotics/wild_visual_navigation/actions/workflows/formatting.yml/badge.svg)

![Overview](./assets/drawings/header.jpg) -Dino - -## Citation -``` -@INPROCEEDINGS{frey23fast, - AUTHOR = {Jonas, Frey and Matias, Mattamala and Nived, Chebrolu and Cesar, Cadena and Maurice, Fallon and Marco, Hutter}, - TITLE = {\href{https://arxiv.org/abs/2305.08510}{Fast Traversability Estimation for Wild Visual Navigation}}, - BOOKTITLE = {Proceedings of Robotics: Science and Systems}, - YEAR = {2023}, - ADDRESS = {Daegu, Republic of Korea}, - MONTH = {July}, - DOI = {10.15607/RSS.2023.XIX.054} -} -``` - -If you are also building up on the STEGO integration or using the pre-trained models for a comparision please cite: -``` -@INPROCEEDINGS{mattamala24wild, - AUTHOR = {Mattamala, Matias and Jonas, Frey and Piotr Libera and Chebrolu, Nived and Georg Martius and Cadena, Cesar and Hutter, Marco and Fallon, Maurice}, - TITLE = {{Wild Visual Navigation: Fast Traversability Learning via Pre-Trained Models and Online Self-Supervision}}, - BOOKTITLE = {under review for Autonomous Robots}, - YEAR = {2024} -} -``` - -If you are using the elevation_mapping integration -``` -@INPROCEEDINGS{erni23mem, - AUTHOR={Erni, Gian and Frey, Jonas and Miki, Takahiro and Mattamala, Matias and Hutter, Marco}, - TITLE={\href{https://arxiv.org/abs/2309.16818}{MEM: Multi-Modal Elevation Mapping for Robotics and Learning}}, - BOOKTITLE={2023 IEEE/RSJ International Conference on Intelligent Robots and Systems (IROS)}, - YEAR={2023}, - PAGES={11011-11018}, - DOI={10.1109/IROS55552.2023.10342108} -} -``` - -Checkout out also our other works. Dino ## Installation -1. Clone the WVN and our STEGO reimplementation. +### Minimal +Clone the WVN and our STEGO reimplementation. ```shell mkdir ~/git && cd ~/git git clone git@github.com:leggedrobotics/wild_visual_navigation.git git clone git@github.com:leggedrobotics/self_supervised_segmentation.git ``` -2. Install the virtual environment. +(Recommended) Install the virtual environment. ```shell -cd ~/git/wild_visual_navigation -# TODO +mkdir ~/.venv +python -m venv ~/venv/wvn +source ~/venv/wvn/bin/activate ``` -3. Install the wild_visual_navigation package. +Install the wild_visual_navigation package. ```shell cd ~/git pip3 install -e ./wild_visual_navigation ``` -4. [Optionally] Configure custom paths -Set your custom global paths by defining the ENV_WORKSTATION_NAME and exporting the variable in your `~/.bashrc`. - - ```shell - export ENV_WORKSTATION_NAME=your_workstation_name - ``` -The paths can be specified by modifying `wild_visual_navigation/wild_visual_navigation/cfg/gloabl_params.py`. -Add your desired global paths. -Per default, all results are stored in `wild_visual_navigation/results`. - - - Dino ## Overview -![Overview](./assets/drawings/software_overview.jpg) -What we provide: + +### Repository Structure +``` +📦wild_visual_navigation + ┣ 📂assets + ┣ 📂demo_data # Example images + ┣ 🖼 example_images.png + ┗ .... + ┗ 📂checkpoints # Pre-trained model checkpoints + ┣ 📜 mountain_bike_trail_v2.pt + ┗ .... + ┣ 📂docker # Quick start docker container + ┣ 📂results + ┣ 📂test + ┣ 📂wild_visual_navigation # Core implementation of WVN + ┣ 📂wild_visual_navigation_anymal # ROS1 ANYmal helper package + ┣ 📂wild_visual_navigation_jackal # ROS1 Jackal simulation example + ┣ 📂wild_visual_navigation_msgs # ROS1 message definitions + ┣ 📂wild_visual_navigation_ros # ROS1 nodes for running WVN + ┗ 📂scripts + ┗ 📜 wvn_feature_extractor_node.py + ┗ 📜 wvn_learning_node.py + ┗ 📜 quick_start.py # Inferencing demo_data from pre-trained checkpoints +``` +### Features - Learning and Feature Extraction Nodes integrated in ROS1 -- Gazebo Test Simulation Envrionment +- Gazebo Test Simulation Environment - Example ROSbags - Pre-trained models with minimalistic inference script (can be used as a easy baseline) - Integration into elevation_mapping_cupy @@ -246,3 +221,42 @@ rosrun procman_ros sheriff -l ~/git/wild_visual_navigation/wild_visual_navigatio ```shell rosbag_play --tf --sem --flp --wvn mission/*.bag ``` + +Dino + +## Citation +``` +@INPROCEEDINGS{frey23fast, + AUTHOR = {Jonas, Frey and Matias, Mattamala and Nived, Chebrolu and Cesar, Cadena and Maurice, Fallon and Marco, Hutter}, + TITLE = {\href{https://arxiv.org/abs/2305.08510}{Fast Traversability Estimation for Wild Visual Navigation}}, + BOOKTITLE = {Proceedings of Robotics: Science and Systems}, + YEAR = {2023}, + ADDRESS = {Daegu, Republic of Korea}, + MONTH = {July}, + DOI = {10.15607/RSS.2023.XIX.054} +} +``` + +If you are also building up on the STEGO integration or using the pre-trained models for a comparision please cite: +``` +@INPROCEEDINGS{mattamala24wild, + AUTHOR = {Mattamala, Matias and Jonas, Frey and Piotr Libera and Chebrolu, Nived and Georg Martius and Cadena, Cesar and Hutter, Marco and Fallon, Maurice}, + TITLE = {{Wild Visual Navigation: Fast Traversability Learning via Pre-Trained Models and Online Self-Supervision}}, + BOOKTITLE = {under review for Autonomous Robots}, + YEAR = {2024} +} +``` + +If you are using the elevation_mapping integration +``` +@INPROCEEDINGS{erni23mem, + AUTHOR={Erni, Gian and Frey, Jonas and Miki, Takahiro and Mattamala, Matias and Hutter, Marco}, + TITLE={\href{https://arxiv.org/abs/2309.16818}{MEM: Multi-Modal Elevation Mapping for Robotics and Learning}}, + BOOKTITLE={2023 IEEE/RSJ International Conference on Intelligent Robots and Systems (IROS)}, + YEAR={2023}, + PAGES={11011-11018}, + DOI={10.1109/IROS55552.2023.10342108} +} +``` + + From b0b3c66ff0c6d4e0ab5e13854645a0595f5396ae Mon Sep 17 00:00:00 2001 From: JonasFrey96 <56234378+JonasFrey96@users.noreply.github.com> Date: Mon, 1 Apr 2024 14:49:44 +0200 Subject: [PATCH 09/15] Update README.md --- README.md | 69 +++++++++++++++++++++++++++++-------------------------- 1 file changed, 37 insertions(+), 32 deletions(-) diff --git a/README.md b/README.md index 884d117b..23e3968d 100644 --- a/README.md +++ b/README.md @@ -66,10 +66,10 @@ pip3 install -e ./wild_visual_navigation ┗ 📜 quick_start.py # Inferencing demo_data from pre-trained checkpoints ``` ### Features -- Learning and Feature Extraction Nodes integrated in ROS1 -- Gazebo Test Simulation Environment -- Example ROSbags -- Pre-trained models with minimalistic inference script (can be used as a easy baseline) +- quick_start script for inference using pre-trained models (can be used as an easy baseline) +- ROS1 integration for online deployment +- Jackal Gazebo simulation demo integration +- Docker container for easy installation - Integration into elevation_mapping_cupy @@ -77,43 +77,52 @@ pip3 install -e ./wild_visual_navigation ## Experiments -### Inference pre-trained model +### Inference of pre-trained model +Script to inference traversability of images within input folder (`assets/demo_data/*.png`), given a pre-trained model checkpoint (`assets/checkpoints/model_name.pt`). The script stores the result in the provided output folder (`results/demo_data/*.png`). +```python +python3 quick_start.py -### Online adaptation [Simulation] +# python3 quick_start.py --help for more CLI information +# usage: quick_start.py [-h] [--model_name MODEL_NAME] [--input_image_folder INPUT_IMAGE_FOLDER] +# [--output_folder_name OUTPUT_FOLDER_NAME] [--network_input_image_height NETWORK_INPUT_IMAGE_HEIGHT] +# [--network_input_image_width NETWORK_INPUT_IMAGE_WIDTH] [--segmentation_type {slic,grid,random,stego}] +# [--feature_type {dino,dinov2,stego}] [--dino_patch_size {8,16}] [--dino_backbone {vit_small}] +# [--slic_num_components SLIC_NUM_COMPONENTS] [--compute_confidence] [--no-compute_confidence] +# [--prediction_per_pixel] [--no-prediction_per_pixel] +``` +### Online adaptation [Simulation] +Instructions can be found within [wild_visual_navigation_jackal/README.md](wild_visual_navigation_jackal/README.md). ### Online adaptation [Rosbag] +#### Download Rosbags: +To quickly test out online training and adaption we provide some [example rosbags](some/wild/GDRIVELink/wild_visual_navigation_jackal/README.md), collected with our ANYmal D robot. -

- - - - -

- +#### Example Result: +
| MPI Outdoor | MPI Indoor | Bahnhofstrasse | Bike Trail | |----------------|------------|-------------|---------------------| | MPI Outdoor | MPI Indoor | Bahnhofstrasse | Mountain Bike | | MPI Outdoor | MPI Indoor | Bahnhofstrasse | Mountain Bike | +
-#### Setup -Let`s set up a new catkin_ws: +#### ROS-Setup: ```shell -# Create Workspace +# Create new catkin workspace source /opt/ros/noetic/setup.bash mkdir -r ~/catkin_ws/src && cd ~/catkin_ws/src catkin init catkin config --extend /opt/ros/noetic catkin config --cmake-args -DCMAKE_BUILD_TYPE=RelWithDebInfo -# Clone Repos +# Clone repos git clone git@github.com:ANYbotics/anymal_d_simple_description.git git clone git@github.com:ori-drs/procman_ros.git -# Symlink WVN +# Symlink WVN-repository ln -s ~/git/wild_visual_navigation ~/catkin_ws/src # Dependencies @@ -122,7 +131,6 @@ rosdep install -ryi --from-paths . --ignore-src # Build cd ~/catkin_ws catkin build anymal_d_simple_description -catkin build procman_ros catkin build wild_visual_navigation_ros # Source @@ -130,8 +138,8 @@ source /opt/ros/noetic/setup.bash source ~/catkin_ws/devel/setup.bash ``` -After successfully building the ros workspace you can run the full pipeline by either using the launch file (this requires all packages to be installed into your system python installation), or by running the nodes from the virtual environment as plain python scripts. - +After successfully building the ros workspace, you can run the entire pipeline by either using the launch file or by running the nodes individually. +Open multiple terminals and run the following commands: - Run wild_visual_navigation ```shell @@ -153,8 +161,7 @@ robag play --clock path_to_mission/*.bag roslaunch wild_visual_navigation_ros view.launch ``` - -Degugging (sometimes it is desirable to run the nodes seperate): +- Debugging (sometimes it is desirable to run the two nodes separately): ```shell python wild_visual_navigation_ros/scripts/wvn_feature_extractor_node.py ``` @@ -162,13 +169,11 @@ python wild_visual_navigation_ros/scripts/wvn_feature_extractor_node.py python wild_visual_navigation_ros/scripts/wvn_learning_node.py ``` - - The general configuration files can be found under: `wild_visual_navigation/cfg/experiment_params.py` - This configuration is used in the `offline-model-training` and in the `online-ros` mode. -- When running the `online-ros` mode additional configurations for the individual nodes are defined in `wild_visual_navigation/cfg/ros_params.py`. +- When running the `online-ros` mode, additional configurations for the individual nodes are defined in `wild_visual_navigation/cfg/ros_params.py`. - These configuration file is filled based on the rosparameter-server during runtime. - The default values for this configuration can be found under `wild_visual_navigation/wild_visual_navigation_ros/config/wild_visual_navigation`. -- We set an environment variable to automatically load the correct global paths and trigger some special behavior e.g. when training on a cluster. Dino @@ -227,8 +232,8 @@ rosbag_play --tf --sem --flp --wvn mission/*.bag ## Citation ``` @INPROCEEDINGS{frey23fast, - AUTHOR = {Jonas, Frey and Matias, Mattamala and Nived, Chebrolu and Cesar, Cadena and Maurice, Fallon and Marco, Hutter}, - TITLE = {\href{https://arxiv.org/abs/2305.08510}{Fast Traversability Estimation for Wild Visual Navigation}}, + AUTHOR = {Jonas Frey AND Matias Mattamala AND Nived Chebrolu AND Cesar Cadena AND Maurice Fallon AND Marco Hutter}, + TITLE = {{Fast Traversability Estimation for Wild Visual Navigation}}, BOOKTITLE = {Proceedings of Robotics: Science and Systems}, YEAR = {2023}, ADDRESS = {Daegu, Republic of Korea}, @@ -237,21 +242,21 @@ rosbag_play --tf --sem --flp --wvn mission/*.bag } ``` -If you are also building up on the STEGO integration or using the pre-trained models for a comparision please cite: +If you are also building up on the STEGO integration or using the pre-trained models for comparison, please cite: ``` @INPROCEEDINGS{mattamala24wild, - AUTHOR = {Mattamala, Matias and Jonas, Frey and Piotr Libera and Chebrolu, Nived and Georg Martius and Cadena, Cesar and Hutter, Marco and Fallon, Maurice}, + AUTHOR = {Jonas Frey AND Matias Mattamala AND Libera Piotr AND Nived Chebrolu AND Cesar Cadena AND Georg Martius AND Marco Hutter AND Maurice Fallon}, TITLE = {{Wild Visual Navigation: Fast Traversability Learning via Pre-Trained Models and Online Self-Supervision}}, BOOKTITLE = {under review for Autonomous Robots}, YEAR = {2024} } ``` -If you are using the elevation_mapping integration +If you are using the elevation_mapping integration: ``` @INPROCEEDINGS{erni23mem, AUTHOR={Erni, Gian and Frey, Jonas and Miki, Takahiro and Mattamala, Matias and Hutter, Marco}, - TITLE={\href{https://arxiv.org/abs/2309.16818}{MEM: Multi-Modal Elevation Mapping for Robotics and Learning}}, + TITLE={{MEM: Multi-Modal Elevation Mapping for Robotics and Learning}}, BOOKTITLE={2023 IEEE/RSJ International Conference on Intelligent Robots and Systems (IROS)}, YEAR={2023}, PAGES={11011-11018}, From f7e0fee807111535ddc7a51ad6ff7ffb21a4a32c Mon Sep 17 00:00:00 2001 From: JonasFrey96 <56234378+JonasFrey96@users.noreply.github.com> Date: Mon, 1 Apr 2024 14:54:39 +0200 Subject: [PATCH 10/15] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 23e3968d..425be1c5 100644 --- a/README.md +++ b/README.md @@ -97,7 +97,7 @@ Instructions can be found within [wild_visual_navigation_jackal/README.md](wild_ ### Online adaptation [Rosbag] #### Download Rosbags: -To quickly test out online training and adaption we provide some [example rosbags](some/wild/GDRIVELink/wild_visual_navigation_jackal/README.md), collected with our ANYmal D robot. +To quickly test out online training and adaption we provide some example rosbags ( ![GDrive](https://drive.google.com/drive/folders/1Rf2TRPT6auFxOpnV9-ZfVMjmsvdsrSD3?usp=sharing) ), collected with our ANYmal D robot. #### Example Result:
From 4046e4b8b43828a53ece524e5ed5fdf55e92e056 Mon Sep 17 00:00:00 2001 From: JonasFrey96 <56234378+JonasFrey96@users.noreply.github.com> Date: Mon, 1 Apr 2024 14:55:40 +0200 Subject: [PATCH 11/15] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 425be1c5..6d5d0987 100644 --- a/README.md +++ b/README.md @@ -97,7 +97,7 @@ Instructions can be found within [wild_visual_navigation_jackal/README.md](wild_ ### Online adaptation [Rosbag] #### Download Rosbags: -To quickly test out online training and adaption we provide some example rosbags ( ![GDrive](https://drive.google.com/drive/folders/1Rf2TRPT6auFxOpnV9-ZfVMjmsvdsrSD3?usp=sharing) ), collected with our ANYmal D robot. +To quickly test out online training and adaption we provide some example rosbags ( [GDrive](https://drive.google.com/drive/folders/1Rf2TRPT6auFxOpnV9-ZfVMjmsvdsrSD3?usp=sharing) ), collected with our ANYmal D robot. #### Example Result:
From 24c853ecc6c6fd61e345e973e5037076ab2a6985 Mon Sep 17 00:00:00 2001 From: JonasFrey96 <56234378+JonasFrey96@users.noreply.github.com> Date: Mon, 1 Apr 2024 18:52:38 +0200 Subject: [PATCH 12/15] Update README.md --- README.md | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 6d5d0987..65a5ce0c 100644 --- a/README.md +++ b/README.md @@ -1,15 +1,26 @@ -

Fast Traversability Estimation for Wild Visual Navigation

+

+
+ Wild Visual Navigation +
+

- Installation • + Installation • Overview • Experiments • Development • Citation - - ![Formatting](https://github.com/leggedrobotics/wild_visual_navigation/actions/workflows/formatting.yml/badge.svg)

+ + + +

+ MIT License + formatting +

+ + ![Overview](./assets/drawings/header.jpg) From 1bef9eade78a5106ca0d630c9ea29aab718b7a66 Mon Sep 17 00:00:00 2001 From: JonasFrey96 <56234378+JonasFrey96@users.noreply.github.com> Date: Mon, 1 Apr 2024 18:55:03 +0200 Subject: [PATCH 13/15] Update README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 65a5ce0c..7bde1758 100644 --- a/README.md +++ b/README.md @@ -47,6 +47,7 @@ Install the wild_visual_navigation package. ```shell cd ~/git pip3 install -e ./wild_visual_navigation +pip3 install -e ./self_supervised_segmentation ``` Dino From 2601fdd57cdb6f16f81623582be18ff0758c7a94 Mon Sep 17 00:00:00 2001 From: Jonas Frey Date: Mon, 1 Apr 2024 20:35:32 +0200 Subject: [PATCH 14/15] imporved CLI for quick visu - polished details --- quick_start.py | 10 +- setup.py | 2 + .../config/rviz/open_source.rviz | 656 ++++++++++-------- 3 files changed, 368 insertions(+), 300 deletions(-) diff --git a/quick_start.py b/quick_start.py index a6b0a50b..d9e63fb8 100644 --- a/quick_start.py +++ b/quick_start.py @@ -56,17 +56,17 @@ def parse_folders(args): parser.add_argument( "--input_image_folder", default="demo_data", - help="If not gloabl will search for the folde name within the assests folder", + help="Gloabl path or folder name within the assests directory", ) parser.add_argument( "--output_folder_name", default="demo_data", - help="If not global will create the folder within the results folder", + help="Gloabl path or folder name within the results directory", ) # Fixed values - parser.add_argument("--network_input_image_height", type=int, default=224) - parser.add_argument("--network_input_image_width", type=int, default=224) + parser.add_argument("--network_input_image_height", type=int, default=224, help="Height of the input image") + parser.add_argument("--network_input_image_width", type=int, default=224, help="Width of the input image") parser.add_argument( "--segmentation_type", default="stego", @@ -89,7 +89,7 @@ def parse_folders(args): parser.set_defaults(compute_confidence=True) parser.add_argument( - "--prediction_per_pixel", action="store_true", help="Description of prediction per pixel argument" + "--prediction_per_pixel", action="store_true", help="Inference traversability per-pixel or per-segment" ) parser.add_argument("--no-prediction_per_pixel", dest="prediction_per_pixel", action="store_false") parser.set_defaults(prediction_per_pixel=True) diff --git a/setup.py b/setup.py index 6d0d5d64..682e7ce4 100644 --- a/setup.py +++ b/setup.py @@ -34,6 +34,8 @@ "liegroups@git+https://github.com/mmattamala/liegroups", "wget", "rospkg", + "wandb", + "opencv-python==4.2.0.34", ] setup( name="wild_visual_navigation", diff --git a/wild_visual_navigation_ros/config/rviz/open_source.rviz b/wild_visual_navigation_ros/config/rviz/open_source.rviz index baafb702..7614da9a 100644 --- a/wild_visual_navigation_ros/config/rviz/open_source.rviz +++ b/wild_visual_navigation_ros/config/rviz/open_source.rviz @@ -12,8 +12,9 @@ Panels: - /Wild Visual Navigation1/Cameras Resized1 - /Wild Visual Navigation1/Depth Sensors1 - /Wild Visual Navigation1/Prediction1 + - /GridMaps1 Splitter Ratio: 0.6411483287811279 - Tree Height: 863 + Tree Height: 856 - Class: rviz/Selection Name: Selection - Class: rviz/Tool Properties @@ -58,178 +59,10 @@ Visualization Manager: Reference Frame: Value: false - Class: rviz/TF - Enabled: true + Enabled: false Frame Timeout: 1000 Frames: All Enabled: false - LF_FOOT: - Value: true - LF_HAA_drive: - Value: true - LF_HFE_drive: - Value: true - LF_HFE_output: - Value: true - LF_HIP: - Value: true - LF_KFE_drive: - Value: true - LF_SHANK: - Value: true - LF_THIGH: - Value: true - LF_hip_fixed: - Value: true - LF_shank_fixed: - Value: true - LF_thigh_fixed: - Value: true - LH_FOOT: - Value: true - LH_HAA_drive: - Value: true - LH_HFE_drive: - Value: true - LH_HFE_output: - Value: true - LH_HIP: - Value: true - LH_KFE_drive: - Value: true - LH_SHANK: - Value: true - LH_THIGH: - Value: true - LH_hip_fixed: - Value: true - LH_shank_fixed: - Value: true - LH_thigh_fixed: - Value: true - RF_FOOT: - Value: true - RF_HAA_drive: - Value: true - RF_HFE_drive: - Value: true - RF_HFE_output: - Value: true - RF_HIP: - Value: true - RF_KFE_drive: - Value: true - RF_SHANK: - Value: true - RF_THIGH: - Value: true - RF_hip_fixed: - Value: true - RF_shank_fixed: - Value: true - RF_thigh_fixed: - Value: true - RH_FOOT: - Value: true - RH_HAA_drive: - Value: true - RH_HFE_drive: - Value: true - RH_HFE_output: - Value: true - RH_HIP: - Value: true - RH_KFE_drive: - Value: true - RH_SHANK: - Value: true - RH_THIGH: - Value: true - RH_hip_fixed: - Value: true - RH_shank_fixed: - Value: true - RH_thigh_fixed: - Value: true - base: - Value: true - base_inertia: - Value: true - battery: - Value: true - body_top: - Value: true - bottom_shell: - Value: true - camera_init: - Value: true - camera_init_CORRECTED: - Value: true - depth_camera_front_lower_camera: - Value: true - depth_camera_front_lower_camera_parent: - Value: true - depth_camera_front_upper_camera: - Value: true - depth_camera_front_upper_camera_parent: - Value: true - depth_camera_left_camera: - Value: true - depth_camera_left_camera_parent: - Value: true - depth_camera_rear_lower_camera: - Value: true - depth_camera_rear_lower_camera_parent: - Value: true - depth_camera_rear_upper_camera: - Value: true - depth_camera_rear_upper_camera_parent: - Value: true - depth_camera_right_camera: - Value: true - depth_camera_right_camera_parent: - Value: true - docking_socket: - Value: true - face_front: - Value: true - face_rear: - Value: true - face_shell_front: - Value: true - face_shell_rear: - Value: true - feetcenter: - Value: true - footprint: - Value: true - hatch_shell: - Value: true - hbc_receiver: - Value: true - imu_link: - Value: true - lidar: - Value: true - lidar_parent: - Value: true - map: - Value: true - map_o3d_localization_manager: - Value: true - msf_body_imu_map: - Value: true - odom: - Value: true - top_shell: - Value: true - wide_angle_camera_front_camera: - Value: true - wide_angle_camera_front_camera_parent: - Value: true - wide_angle_camera_rear_camera: - Value: true - wide_angle_camera_rear_camera_parent: - Value: true Marker Alpha: 1 Marker Scale: 0.5 Name: TF @@ -237,117 +70,9 @@ Visualization Manager: Show Axes: true Show Names: true Tree: - odom: - base: - LF_HAA_drive: - LF_HIP: - LF_hip_fixed: - LF_HFE_output: - LF_THIGH: - LF_HFE_drive: - LF_thigh_fixed: - LF_KFE_drive: - LF_SHANK: - LF_shank_fixed: - LF_FOOT: - {} - LH_HAA_drive: - LH_HIP: - LH_hip_fixed: - LH_HFE_output: - LH_THIGH: - LH_HFE_drive: - LH_thigh_fixed: - LH_KFE_drive: - LH_SHANK: - LH_shank_fixed: - LH_FOOT: - {} - RF_HAA_drive: - RF_HIP: - RF_hip_fixed: - RF_HFE_output: - RF_THIGH: - RF_HFE_drive: - RF_thigh_fixed: - RF_KFE_drive: - RF_SHANK: - RF_shank_fixed: - RF_FOOT: - {} - RH_HAA_drive: - RH_HIP: - RH_hip_fixed: - RH_HFE_output: - RH_THIGH: - RH_HFE_drive: - RH_thigh_fixed: - RH_KFE_drive: - RH_SHANK: - RH_shank_fixed: - RH_FOOT: - {} - base_inertia: - {} - battery: - {} - body_top: - {} - bottom_shell: - {} - depth_camera_left_camera: - depth_camera_left_camera_parent: - {} - depth_camera_right_camera: - depth_camera_right_camera_parent: - {} - docking_socket: - {} - face_front: - depth_camera_front_lower_camera: - depth_camera_front_lower_camera_parent: - {} - depth_camera_front_upper_camera: - depth_camera_front_upper_camera_parent: - {} - face_shell_front: - {} - wide_angle_camera_front_camera: - wide_angle_camera_front_camera_parent: - {} - face_rear: - depth_camera_rear_lower_camera: - depth_camera_rear_lower_camera_parent: - {} - depth_camera_rear_upper_camera: - depth_camera_rear_upper_camera_parent: - {} - face_shell_rear: - {} - wide_angle_camera_rear_camera: - wide_angle_camera_rear_camera_parent: - {} - hatch_shell: - {} - hbc_receiver: - {} - imu_link: - {} - lidar_parent: - lidar: - {} - top_shell: - {} - feetcenter: - {} - footprint: - {} - map_o3d_localization_manager: - {} - msf_body_imu_map: - {} + {} Update Interval: 0 - Value: true + Value: false - Class: rviz/Group Displays: - Angle Tolerance: 0.10000000149011612 @@ -394,7 +119,348 @@ Visualization Manager: Expand Joint Details: false Expand Link Details: false Expand Tree: false + LF_FOOT: + Alpha: 1 + Show Axes: false + Show Trail: false + Value: true + LF_HAA_drive: + Alpha: 1 + Show Axes: false + Show Trail: false + Value: true + LF_HFE_drive: + Alpha: 1 + Show Axes: false + Show Trail: false + Value: true + LF_HFE_output: + Alpha: 1 + Show Axes: false + Show Trail: false + LF_HIP: + Alpha: 1 + Show Axes: false + Show Trail: false + LF_KFE_drive: + Alpha: 1 + Show Axes: false + Show Trail: false + Value: true + LF_SHANK: + Alpha: 1 + Show Axes: false + Show Trail: false + LF_THIGH: + Alpha: 1 + Show Axes: false + Show Trail: false + LF_hip_fixed: + Alpha: 1 + Show Axes: false + Show Trail: false + Value: true + LF_shank_fixed: + Alpha: 1 + Show Axes: false + Show Trail: false + Value: true + LF_thigh_fixed: + Alpha: 1 + Show Axes: false + Show Trail: false + Value: true + LH_FOOT: + Alpha: 1 + Show Axes: false + Show Trail: false + Value: true + LH_HAA_drive: + Alpha: 1 + Show Axes: false + Show Trail: false + Value: true + LH_HFE_drive: + Alpha: 1 + Show Axes: false + Show Trail: false + Value: true + LH_HFE_output: + Alpha: 1 + Show Axes: false + Show Trail: false + LH_HIP: + Alpha: 1 + Show Axes: false + Show Trail: false + LH_KFE_drive: + Alpha: 1 + Show Axes: false + Show Trail: false + Value: true + LH_SHANK: + Alpha: 1 + Show Axes: false + Show Trail: false + LH_THIGH: + Alpha: 1 + Show Axes: false + Show Trail: false + LH_hip_fixed: + Alpha: 1 + Show Axes: false + Show Trail: false + Value: true + LH_shank_fixed: + Alpha: 1 + Show Axes: false + Show Trail: false + Value: true + LH_thigh_fixed: + Alpha: 1 + Show Axes: false + Show Trail: false + Value: true Link Tree Style: Links in Alphabetic Order + RF_FOOT: + Alpha: 1 + Show Axes: false + Show Trail: false + Value: true + RF_HAA_drive: + Alpha: 1 + Show Axes: false + Show Trail: false + Value: true + RF_HFE_drive: + Alpha: 1 + Show Axes: false + Show Trail: false + Value: true + RF_HFE_output: + Alpha: 1 + Show Axes: false + Show Trail: false + RF_HIP: + Alpha: 1 + Show Axes: false + Show Trail: false + RF_KFE_drive: + Alpha: 1 + Show Axes: false + Show Trail: false + Value: true + RF_SHANK: + Alpha: 1 + Show Axes: false + Show Trail: false + RF_THIGH: + Alpha: 1 + Show Axes: false + Show Trail: false + RF_hip_fixed: + Alpha: 1 + Show Axes: false + Show Trail: false + Value: true + RF_shank_fixed: + Alpha: 1 + Show Axes: false + Show Trail: false + Value: true + RF_thigh_fixed: + Alpha: 1 + Show Axes: false + Show Trail: false + Value: true + RH_FOOT: + Alpha: 1 + Show Axes: false + Show Trail: false + Value: true + RH_HAA_drive: + Alpha: 1 + Show Axes: false + Show Trail: false + Value: true + RH_HFE_drive: + Alpha: 1 + Show Axes: false + Show Trail: false + Value: true + RH_HFE_output: + Alpha: 1 + Show Axes: false + Show Trail: false + RH_HIP: + Alpha: 1 + Show Axes: false + Show Trail: false + RH_KFE_drive: + Alpha: 1 + Show Axes: false + Show Trail: false + Value: true + RH_SHANK: + Alpha: 1 + Show Axes: false + Show Trail: false + RH_THIGH: + Alpha: 1 + Show Axes: false + Show Trail: false + RH_hip_fixed: + Alpha: 1 + Show Axes: false + Show Trail: false + Value: true + RH_shank_fixed: + Alpha: 1 + Show Axes: false + Show Trail: false + Value: true + RH_thigh_fixed: + Alpha: 1 + Show Axes: false + Show Trail: false + Value: true + base: + Alpha: 1 + Show Axes: false + Show Trail: false + Value: true + base_inertia: + Alpha: 1 + Show Axes: false + Show Trail: false + battery: + Alpha: 1 + Show Axes: false + Show Trail: false + Value: true + body_top: + Alpha: 1 + Show Axes: false + Show Trail: false + bottom_shell: + Alpha: 1 + Show Axes: false + Show Trail: false + Value: true + depth_camera_front_lower_camera: + Alpha: 1 + Show Axes: false + Show Trail: false + depth_camera_front_lower_camera_parent: + Alpha: 1 + Show Axes: false + Show Trail: false + depth_camera_front_upper_camera: + Alpha: 1 + Show Axes: false + Show Trail: false + depth_camera_front_upper_camera_parent: + Alpha: 1 + Show Axes: false + Show Trail: false + depth_camera_left_camera: + Alpha: 1 + Show Axes: false + Show Trail: false + depth_camera_left_camera_parent: + Alpha: 1 + Show Axes: false + Show Trail: false + depth_camera_rear_lower_camera: + Alpha: 1 + Show Axes: false + Show Trail: false + depth_camera_rear_lower_camera_parent: + Alpha: 1 + Show Axes: false + Show Trail: false + depth_camera_rear_upper_camera: + Alpha: 1 + Show Axes: false + Show Trail: false + depth_camera_rear_upper_camera_parent: + Alpha: 1 + Show Axes: false + Show Trail: false + depth_camera_right_camera: + Alpha: 1 + Show Axes: false + Show Trail: false + depth_camera_right_camera_parent: + Alpha: 1 + Show Axes: false + Show Trail: false + docking_socket: + Alpha: 1 + Show Axes: false + Show Trail: false + Value: true + face_front: + Alpha: 1 + Show Axes: false + Show Trail: false + face_rear: + Alpha: 1 + Show Axes: false + Show Trail: false + face_shell_front: + Alpha: 1 + Show Axes: false + Show Trail: false + Value: true + face_shell_rear: + Alpha: 1 + Show Axes: false + Show Trail: false + Value: true + hatch_shell: + Alpha: 1 + Show Axes: false + Show Trail: false + Value: true + hbc_receiver: + Alpha: 1 + Show Axes: false + Show Trail: false + imu_link: + Alpha: 1 + Show Axes: false + Show Trail: false + lidar: + Alpha: 1 + Show Axes: false + Show Trail: false + Value: true + lidar_parent: + Alpha: 1 + Show Axes: false + Show Trail: false + top_shell: + Alpha: 1 + Show Axes: false + Show Trail: false + Value: true + wide_angle_camera_front_camera: + Alpha: 1 + Show Axes: false + Show Trail: false + wide_angle_camera_front_camera_parent: + Alpha: 1 + Show Axes: false + Show Trail: false + wide_angle_camera_rear_camera: + Alpha: 1 + Show Axes: false + Show Trail: false + wide_angle_camera_rear_camera_parent: + Alpha: 1 + Show Axes: false + Show Trail: false Name: RobotModel Robot Description: anymal_description TF Prefix: "" @@ -412,7 +478,7 @@ Visualization Manager: Marker Topic: /wild_visual_navigation_node/graph_footprints Name: Footprint Namespaces: - footprints: true + {} Queue Size: 100 Value: true - Alpha: 1 @@ -891,7 +957,7 @@ Visualization Manager: Color Layer: visual_traversability Color Transformer: "" ColorMap: coolwarm - Enabled: false + Enabled: true Grid Cell Decimation: 1 Grid Line Thickness: 0.10000000149011612 Height Layer: elevation_with_semantics @@ -907,7 +973,7 @@ Visualization Manager: Topic: /elevation_mapping/semantic_map Unreliable: false Use ColorMap: true - Value: false + Value: true - Alpha: 1 Autocompute Intensity Bounds: false Class: grid_map_rviz_plugin/GridMap @@ -962,7 +1028,7 @@ Visualization Manager: Views: Current: Class: rviz/ThirdPersonFollower - Distance: 13.404160499572754 + Distance: 8.189547538757324 Enable Stereo Rendering: Stereo Eye Separation: 0.05999999865889549 Stereo Focal Distance: 1 @@ -970,17 +1036,17 @@ Visualization Manager: Value: false Field of View: 0.7853981852531433 Focal Point: - X: 2.1661949157714844 - Y: -0.48903560638427734 + X: 2.6289563179016113 + Y: 0.7574070692062378 Z: 0 Focal Shape Fixed Size: true Focal Shape Size: 0.05000000074505806 Invert Z Axis: false Name: Current View Near Clip Distance: 0.009999999776482582 - Pitch: 0.4953981041908264 + Pitch: 0.5653980374336243 Target Frame: base - Yaw: 0.4853982925415039 + Yaw: 0.46539831161499023 Saved: ~ Window Geometry: AB Wide Angle Front: @@ -995,14 +1061,14 @@ Window Geometry: collapsed: false HDR: collapsed: false - Height: 1376 + Height: 1403 Hide Left Dock: false - Hide Right Dock: false + Hide Right Dock: true Input Image: collapsed: false Learning Mask: collapsed: false - QMainWindow State: 000000ff00000000fd0000000400000000000001a40000039cfc0200000012fb0000001200530065006c0065006300740069006f006e00000001e10000009b0000005c00fffffffb0000001e0054006f006f006c002000500072006f007000650072007400690065007302000005fb000001df00000185000000a3fb000000120056006900650077007300200054006f006f02000001df000002110000018500000122fb000000200054006f006f006c002000500072006f0070006500720074006900650073003203000002880000011d000002210000017afb000000100044006900730070006c00610079007301000001a70000039c000000c900fffffffb0000002000730065006c0065006300740069006f006e00200062007500660066006500720200000138000000aa0000023a00000294fb00000014005700690064006500530074006500720065006f02000000e6000000d2000003ee0000030bfb0000000c004b0069006e0065006300740200000186000001060000030c00000261fb0000000c00430061006d00650072006100000002ea000000c80000000000000000fb00000036005300750070006500720076006900730069006f006e0020002d00200049006d0061006700650020004c006100620065006c006500640000000259000000520000000000000000fb00000024005300750070006500720076006900730069006f006e0020002d0020004d00610073006b000000027e000000b80000000000000000fb0000000a0049006d00610067006500000002eb000001280000000000000000fb000000140043006f006e0066006900640065006e0063006500000003b3000000c60000000000000000fb0000003e004600200054007200610076006500720073006100620069006c0069007400790020004f007600650072006c006100790020005200650070006c0061007900000004a40000009f0000000000000000fb00000028004600200054007200610076006500720073006100620069006c00690074007900200052006100770000000482000000c10000000000000000fb00000028004600200054007200610076006500720073006100620069006c00690074007900200052006100770000000482000000c10000000000000000fb0000000a0049006d0061006700650000000249000000210000000000000000fb0000000a0049006d0061006700650000000270000000200000000000000000000000010000010f0000039cfc0200000003fb0000001e0054006f006f006c002000500072006f00700065007200740069006500730100000041000000780000000000000000fb0000000a0056006900650077007301000001a70000039c000000a400fffffffb0000001200530065006c0065006300740069006f006e010000025a000000b2000000000000000000000002000009b800000164fc0100000015fb0000001c0049006d00610067006500200046006500610074007500720065007300000000bf000003690000000000000000fb000000060048004400520000000000000000950000005400fffffffb00000024004100420020005700690064006500200052006500610072002000460072006f006e00740100000000000001d6000000ba00fffffffc0000012a0000010c0000000000fffffffa000000000200000003fb0000000a0049006d0061006700650000000000ffffffff0000000000000000fb0000002400460072006f006e0074002000280063006f006d007000720065007300730065006400290000000000ffffffff0000000000000000fb0000001e00410053002000460072006f006e00740020002800630061006d00340029000000003d0000010c0000000000000000fc000002570000019b0000000000fffffffa000000000200000001fb0000000a0049006d0061006700650000000000ffffffff0000000000000000fc0000036f000001530000000000fffffffa000000010200000003fb000000160049006e00700075007400200049006d006100670065000000003d000001ac0000001600fffffffb0000001a005200200049006e00700075007400200049006d0061006700650000000000ffffffff0000000000000000fb0000001a004c00200049006e00700075007400200049006d0061006700650000000000ffffffff0000000000000000fb00000028005200200043006f006e0066006900640065006e006300650020004f007600650072006c0061007901000001dc000001ae000000c700fffffffb00000030005200200054007200610076006500720073006100620069006c0069007400790020004f007600650072006c0061007901000003900000016e000000d500fffffffb00000026004100420020005700690064006500200041006e0067006c0065002000460072006f006e00740100000504000001b7000000c200fffffffb00000028004600200043006f006e0066006900640065006e006300650020004f007600650072006c006100790100000709000001540000000000000000fb00000030004600200054007200610076006500720073006100620069006c0069007400790020004f007600650072006c0061007901000006c100000147000000d400fffffffb00000028004600200043006f006e0066006900640065006e006300650020004f007600650072006c00610079010000080e000001aa000000c600fffffffb0000000a0049006d00610067006501000003240000018a0000000000000000fb0000002c0054007200610076006500720073006100620069006c0069007400790020004f007600650072006c0061007900000004ae0000010c0000000000000000fc00000846000000bd0000000000fffffffa000000000200000006fb000000160046002000540072006100760020004f0076006500720000000000ffffffff0000000000000000fb0000003e004600200054007200610076006500720073006100620069006c0069007400790020004f007600650072006c006100790020005200650070006c006100790100000000ffffffff0000000000000000fb000000240054007200610076006500720073006100620069006c0069007400790020005200610077000000003d0000012b0000001600fffffffb00000028005200200054007200610076006500720073006100620069006c00690074007900200052006100770000000000ffffffff0000000000000000fb00000028004c00200054007200610076006500720073006100620069006c00690074007900200052006100770000000000ffffffff0000000000000000fb00000030004600200054007200610076006500720073006100620069006c0069007400790020004f007600650072006c006100790000000000ffffffff0000000000000000fc00000909000000af0000000000fffffffa000000000200000005fb00000036004600200043006f006e0066006900640065006e006300650020004f007600650072006c006100790020005200650070006c006100790100000000ffffffff0000000000000000fb0000001a004c006500610072006e0069006e00670020004d00610073006b000000003d0000012b0000001600fffffffb00000020005200200043006f006e0066006900640065006e0063006500200052006100770000000000ffffffff0000000000000000fb00000020004c00200043006f006e0066006900640065006e0063006500200052006100770000000000ffffffff0000000000000000fb00000028004600200043006f006e0066006900640065006e006300650020004f007600650072006c006100790000000000ffffffff0000000000000000fb000000160050006c0061006300650068006f006c00640065007200000000000000074a0000000000000000fb000000160049006e00700075007400200049006d006100670065010000031e000001790000000000000000fb00000036005300750070006500720076006900730069006f006e0020002d00200049006d0061006700650020004c006100620065006c006500640100000659000001af0000000000000000fb00000024005300750070006500720076006900730069006f006e0020002d0020004d00610073006b010000080e000001aa0000000000000000fb0000000a00560069006500770073030000004e00000080000002e100000197000000030000074a00000042fc010000000dfb000000200052006500700072006f006a00650063007400650064002000500061007400680000000000000000ab000000ab00fffffffb00000024005300750070006500720076006900730069006f006e0020005300690067006e0061006c00000000b10000056e000000b000fffffffb0000001c0054007200610076006500720073006100620069006c00690074007900000006250000009a0000009100fffffffb000000160055006e006300650072007400610069006e0074007900000006c5000000850000008500fffffffb000000160050006c0061006300650068006f006c006400650072000000000000000eb80000000000000000fb0000001c0054007200610076006500720073006100620069006c006900740079010000031f0000017d0000000000000000fb000000160055006e006300650072007400610069006e0074007901000004a2000002960000000000000000fb000000140043006f006e0066006900640065006e006300650000000000000003d40000000000000000fb0000001c0054007200610076006500720073006100620069006c0069007400790000000231000001f20000000000000000fb0000001c004100530020004c0065006600740020002800630061006d003300290000000000000007380000000000000000fb0000001e004100530020005200690067006800740020002800630061006d0035002900000003f0000003480000000000000000fb0000000800540069006d00650000000000000007800000041800fffffffb0000000800540069006d00650100000000000004500000000000000000000006f90000039c00000004000000040000000800000008fc0000000100000002000000010000000a0054006f006f006c00730100000000ffffffff0000000000000000 + QMainWindow State: 000000ff00000000fd00000004000000000000016700000395fc0200000012fb0000001200530065006c0065006300740069006f006e00000001e10000009b0000005c00fffffffb0000001e0054006f006f006c002000500072006f007000650072007400690065007302000005fb000001df00000185000000a3fb000000120056006900650077007300200054006f006f02000001df000002110000018500000122fb000000200054006f006f006c002000500072006f0070006500720074006900650073003203000002880000011d000002210000017afb000000100044006900730070006c00610079007301000001c900000395000000c900fffffffb0000002000730065006c0065006300740069006f006e00200062007500660066006500720200000138000000aa0000023a00000294fb00000014005700690064006500530074006500720065006f02000000e6000000d2000003ee0000030bfb0000000c004b0069006e0065006300740200000186000001060000030c00000261fb0000000c00430061006d00650072006100000002ea000000c80000000000000000fb00000036005300750070006500720076006900730069006f006e0020002d00200049006d0061006700650020004c006100620065006c006500640000000259000000520000000000000000fb00000024005300750070006500720076006900730069006f006e0020002d0020004d00610073006b000000027e000000b80000000000000000fb0000000a0049006d00610067006500000002eb000001280000000000000000fb000000140043006f006e0066006900640065006e0063006500000003b3000000c60000000000000000fb0000003e004600200054007200610076006500720073006100620069006c0069007400790020004f007600650072006c006100790020005200650070006c0061007900000004a40000009f0000000000000000fb00000028004600200054007200610076006500720073006100620069006c00690074007900200052006100770000000482000000c10000000000000000fb00000028004600200054007200610076006500720073006100620069006c00690074007900200052006100770000000482000000c10000000000000000fb0000000a0049006d0061006700650000000249000000210000000000000000fb0000000a0049006d0061006700650000000270000000200000000000000000000000010000010f000003b7fc0200000003fb0000001e0054006f006f006c002000500072006f00700065007200740069006500730100000041000000780000000000000000fb0000000a0056006900650077007300000001a7000003b7000000a400fffffffb0000001200530065006c0065006300740069006f006e010000025a000000b200000000000000000000000200000a0000000186fc0100000015fb0000001c0049006d00610067006500200046006500610074007500720065007300000000bf000003690000000000000000fb000000060048004400520000000000000000950000005400fffffffb00000024004100420020005700690064006500200052006500610072002000460072006f006e0074010000000000000210000000ba00fffffffc0000012a0000010c0000000000fffffffa000000000200000003fb0000000a0049006d0061006700650000000000ffffffff0000000000000000fb0000002400460072006f006e0074002000280063006f006d007000720065007300730065006400290000000000ffffffff0000000000000000fb0000001e00410053002000460072006f006e00740020002800630061006d00340029000000003d0000010c0000000000000000fc000002570000019b0000000000fffffffa000000000200000001fb0000000a0049006d0061006700650000000000ffffffff0000000000000000fc0000036f000001530000000000fffffffa000000010200000003fb000000160049006e00700075007400200049006d006100670065000000003d000001ac0000001600fffffffb0000001a005200200049006e00700075007400200049006d0061006700650000000000ffffffff0000000000000000fb0000001a004c00200049006e00700075007400200049006d0061006700650000000000ffffffff0000000000000000fb00000028005200200043006f006e0066006900640065006e006300650020004f007600650072006c00610079010000021600000177000000c700fffffffb00000030005200200054007200610076006500720073006100620069006c0069007400790020004f007600650072006c00610079010000039300000175000000d500fffffffb00000026004100420020005700690064006500200041006e0067006c0065002000460072006f006e0074010000050e0000020c000000c200fffffffb00000028004600200043006f006e0066006900640065006e006300650020004f007600650072006c0061007901000007200000016b000000c600fffffffb00000030004600200054007200610076006500720073006100620069006c0069007400790020004f007600650072006c0061007901000008910000016f000000d400fffffffb00000028004600200043006f006e0066006900640065006e006300650020004f007600650072006c00610079010000080e000001aa0000000000000000fb0000000a0049006d00610067006501000003240000018a0000000000000000fb0000002c0054007200610076006500720073006100620069006c0069007400790020004f007600650072006c0061007900000004ae0000010c0000000000000000fc00000846000000bd0000000000fffffffa000000000200000006fb000000160046002000540072006100760020004f0076006500720000000000ffffffff0000000000000000fb0000003e004600200054007200610076006500720073006100620069006c0069007400790020004f007600650072006c006100790020005200650070006c006100790100000000ffffffff0000000000000000fb000000240054007200610076006500720073006100620069006c0069007400790020005200610077000000003d0000012b0000001600fffffffb00000028005200200054007200610076006500720073006100620069006c00690074007900200052006100770000000000ffffffff0000000000000000fb00000028004c00200054007200610076006500720073006100620069006c00690074007900200052006100770000000000ffffffff0000000000000000fb00000030004600200054007200610076006500720073006100620069006c0069007400790020004f007600650072006c006100790000000000ffffffff0000000000000000fc00000909000000af0000000000fffffffa000000000200000005fb00000036004600200043006f006e0066006900640065006e006300650020004f007600650072006c006100790020005200650070006c006100790100000000ffffffff0000000000000000fb0000001a004c006500610072006e0069006e00670020004d00610073006b000000003d0000012b0000001600fffffffb00000020005200200043006f006e0066006900640065006e0063006500200052006100770000000000ffffffff0000000000000000fb00000020004c00200043006f006e0066006900640065006e0063006500200052006100770000000000ffffffff0000000000000000fb00000028004600200043006f006e0066006900640065006e006300650020004f007600650072006c006100790000000000ffffffff0000000000000000fb000000160050006c0061006300650068006f006c00640065007200000000000000074a0000000000000000fb000000160049006e00700075007400200049006d006100670065010000031e000001790000000000000000fb00000036005300750070006500720076006900730069006f006e0020002d00200049006d0061006700650020004c006100620065006c006500640100000659000001af0000000000000000fb00000024005300750070006500720076006900730069006f006e0020002d0020004d00610073006b010000080e000001aa0000000000000000fb0000000a00560069006500770073030000004e00000080000002e100000197000000030000074a00000042fc010000000dfb000000200052006500700072006f006a00650063007400650064002000500061007400680000000000000000ab000000ab00fffffffb00000024005300750070006500720076006900730069006f006e0020005300690067006e0061006c00000000b10000056e000000b000fffffffb0000001c0054007200610076006500720073006100620069006c00690074007900000006250000009a0000009100fffffffb000000160055006e006300650072007400610069006e0074007900000006c5000000850000008500fffffffb000000160050006c0061006300650068006f006c006400650072000000000000000eb80000000000000000fb0000001c0054007200610076006500720073006100620069006c006900740079010000031f0000017d0000000000000000fb000000160055006e006300650072007400610069006e0074007901000004a2000002960000000000000000fb000000140043006f006e0066006900640065006e006300650000000000000003d40000000000000000fb0000001c0054007200610076006500720073006100620069006c0069007400790000000231000001f20000000000000000fb0000001c004100530020004c0065006600740020002800630061006d003300290000000000000007380000000000000000fb0000001e004100530020005200690067006800740020002800630061006d0035002900000003f0000003480000000000000000fb0000000800540069006d00650000000000000007800000041800fffffffb0000000800540069006d00650100000000000004500000000000000000000008930000039500000004000000040000000800000008fc0000000100000002000000010000000a0054006f006f006c00730100000000ffffffff0000000000000000 R Confidence Overlay: collapsed: false R Traversability Overlay: @@ -1024,7 +1090,7 @@ Window Geometry: Uncertainty: collapsed: false Views: - collapsed: false - Width: 2488 - X: 72 - Y: 27 + collapsed: true + Width: 2560 + X: 0 + Y: 0 From 457ac7cee634d9de8f3e1a81f48535650a23cc91 Mon Sep 17 00:00:00 2001 From: JonasFrey96 <56234378+JonasFrey96@users.noreply.github.com> Date: Mon, 1 Apr 2024 20:37:46 +0200 Subject: [PATCH 15/15] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 7bde1758..b07c0549 100644 --- a/README.md +++ b/README.md @@ -36,7 +36,7 @@ git clone git@github.com:leggedrobotics/wild_visual_navigation.git git clone git@github.com:leggedrobotics/self_supervised_segmentation.git ``` -(Recommended) Install the virtual environment. +(Recommended) Create new virtual environment. ```shell mkdir ~/.venv python -m venv ~/venv/wvn