Skip to content

Commit

Permalink
Merge branch 'develop'
Browse files Browse the repository at this point in the history
  • Loading branch information
@btalb
btalb committed Apr 20, 2022
2 parents 77a214e + b356b6e commit 7617e2a
Show file tree
Hide file tree
Showing 2 changed files with 40 additions and 91 deletions.
16 changes: 5 additions & 11 deletions bin/.helpers
View file
Original file line number Diff line number Diff line change
Expand Up @@ -161,18 +161,12 @@ function kill_benchbot() {

# Kill specific processes within the container instead if it exists
if [ -n "$(docker ps -q -f name=$HOSTNAME_ROBOT)" ]; then
kill_targets="$(docker exec benchbot_robot /bin/bash -c \
'ps aux | grep benchbot_robot_controller' | grep -v 'grep' | \
awk '{ for (i=11; i<=NF; i++) { printf $i " " } print "(pid="$2")" }' \
|| true)"
# Use the supervisor to ask the simulator to stop
printf "\n${colour_blue}%s${colour_nc}\n" \
"Killing following targets in '$HOSTNAME_ROBOT' container:"
echo "$kill_targets"
if [ -n "$kill_targets" ]; then
echo "$kill_targets" | sed 's/.*pid=\(.*\)).*/\1/' | while read pid ; \
do docker exec $HOSTNAME_ROBOT /bin/bash -c \
'kill -TERM '$pid'; while kill -0 '$pid' 2>/dev/null; do sleep 1; done'; done
fi
"Sending stop request to running controller:"
curl -sS -L "$HOSTNAME_ROBOT:$PORT_ROBOT/stop"

# TODO some wait / success checking logic?
fi
fi
if [ -n "$targets" ]; then
Expand Down
115 changes: 35 additions & 80 deletions bin/benchbot_run
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,13 +14,7 @@ source .helpers
########################### Script Specific Settings ###########################
################################################################################

SIM_OMNI_PERSISTENT_CHK=$(printf "%s%s" \
'docker logs benchbot_robot 2>/dev/null | tail -n 100 | ' \
'grep -q "Omniverse System Monitor.*started"')

SIM_OMNI_PERSISTENT_CMD=(
'/isaac-sim/start_nucleus.sh && sleep infinity'
)
# None

################################################################################
######################## Helper functions for commands #########################
Expand Down Expand Up @@ -71,6 +65,10 @@ OPTION DETAILS:
novel work will focus on the latest software stack. You should only
use this flag when it is inconvenient to update immediately.
-k, --kill-controller
Run a kill command that stops everything BenchBot currently
running, including the persistent robot controller.
--list-envs, --list-environments
Search for & list all installed environments. The listed
environment names are in the format needed for the '--env' option.
Expand Down Expand Up @@ -172,40 +170,28 @@ log below for a dump of the crash output:"
exit_code=
kill_persist=
function exit_gracefully() {
if [ -z "$simulator_required" ]; then
if [ "$simulator_required" -ne 0 ]; then
printf "\n\n${colour_blue}%s${colour_nc}\n" \
"Re-closing network openings used for real robot:"
close_network $network_forwarding $network_policy
fi
kill_benchbot "" $kill_persist
xhost -local:root > /dev/null
trap '' SIGINT SIGQUIT SIGKILL SIGTERM EXIT
exit ${exit_code:-0}
}

function persist_chk() {
# $1 run type
if [ "$type" = "sim_omni" ]; then
echo "$SIM_OMNI_PERSISTENT_CHK"
fi
}

function persist_cmd() {
# $1 run type
if [ "$type" = "sim_omni" ]; then
echo "$SIM_OMNI_PERSISTENT_CMD"
fi
}


################################################################################
#################### Parse & handle command line arguments #####################
################################################################################

# Safely parse options input
_args="help,env:,environment:,force-updateless,list-envs,list-environments,\
list-formats,list-robots,list-tasks,robot:,show-env:,show-environment:,\
show-format:,show-robot:,show-task:,task:,updates-check,version"
parse_out=$(getopt -o he:t:r:fuv --long $_args -n "$(basename "$abs_path")" \
_args="help,env:,environment:,force-updateless,kill-controller,list-envs,\
list-environments,list-formats,list-robots,list-tasks,robot:,show-env:,\
show-environment:,show-format:,show-robot:,show-task:,task:,updates-check,\
version"
parse_out=$(getopt -o he:t:r:fuvk --long $_args -n "$(basename "$abs_path")" \
-- "$@")
if [ $? != 0 ]; then exit 1; fi
eval set -- "$parse_out"
Expand All @@ -222,6 +208,8 @@ while true; do
environment="$2"; shift 2 ;;
-f|--force-updateless)
updates_skip=1 ; shift ;;
-k|--kill-controller)
kill_persist=0; simulator_required=0; exit_gracefully ;;
--list-envs|--list-environments)
list_environments "$_list_environments_pre" "an"; exit $? ;;
--list-formats)
Expand Down Expand Up @@ -263,10 +251,10 @@ if [ -z "$updates_exit" ]; then
type="$(run_manager_cmd 'exists("robots", [("name", "'$robot'")]) and print(\
get_value_by_name("robots", "'$robot'", "type"))')"
simulator_required=1
if [[ "$type" != "sim_"* ]]; then simulator_required=0; fi
if [[ "$type" == "sim_"* ]]; then simulator_required=0; fi

# TODO this is logic is flaky as all hell... (see 'TODO DANGER' notice below)
[ -z "$(persist_cmd "$type")" ] && kill_persist=0 || kill_persist=1
# TODO add an option for managing the persistent container
kill_persist=1

# Bail if any of the requested configurations are invalid
validate_run_args "$robot" "$task" "$type" "$environment" "${environments[@]}"
Expand Down Expand Up @@ -314,16 +302,16 @@ for i in "${!environments[@]}"; do
"'${environments[$i]}'", "map_path"))')"
done
printf " %-22s" "Simulator required:"
printf "%s (%s)\n" $([ -z "$simulator_required" ] && echo "No" || echo "Yes") \
"$type"
printf "%s (%s)\n" \
$([ "$simulator_required" -ne 0 ] && echo "No" || echo "Yes") "$type"
echo ""

# Create the network for BenchBot software stack
echo -e "${colour_blue}Creating shared network '$DOCKER_NETWORK':${colour_nc}"
docker network inspect "$DOCKER_NETWORK" >/dev/null 2>&1 || \
docker network create "$DOCKER_NETWORK" --subnet="$URL_DOCKER_SUBNET" \
--ip-range="$URL_DOCKER_SUBNET" --gateway="$URL_DOCKER_GATEWAY"
if [ -z "$simulator_required" ]; then
if [ "$simulator_required" -ne 0 ]; then
printf "\n${colour_blue}%s${colour_nc}\n" \
"Opening network to facilitate communications with real robot:"
network_forwarding=$(cat /proc/sys/net/ipv4/conf/all/forwarding)
Expand Down Expand Up @@ -352,61 +340,28 @@ cmd="${docker_run//'$name'/$ros_master_host}"
${cmd// /$'\t'} --ip "$URL_ROS" -d $DOCKER_TAG_BACKEND /bin/bash -c \
"$cmd_prefix"'roscore'

if [ -n "$simulator_required" ]; then
if [ "$simulator_required" -eq 0 ]; then
# TODO would be nice to have a less stupid way to do this, but bash's lack of
# nested arrays (and difficulties passing arrays in general from functions)
# makes it hard...
args=()
if [ "$type" = "sim_omni" ]; then
args=("${SIM_OMNI_ARGS[@]}")
fi
persist_cmd="$(persist_cmd "$type")"
persist_chk="$(persist_chk "$type")"

if [ -n "$persist_cmd" ]; then
# TODO DANGER: there are LOTS of ways this can go wrong. Need to do this
# more robustly if we ever expand outside Omniverse-only focus. Examples:
# - if next run uses a different simulator type, what do we do?
# - the persist variables are vary ad-hoc / hacky
# - it's really unclear to the user what is still running in background
# - our checks for the status of things are weak (how do we know if an exec
# has failed now???)

# Run a persistent container and execute within it if our simulator has
# caching utilities (e.g. Cache in Omniverse)
printf "\n${colour_blue}%s${colour_nc}\n" \
"Starting persistent container for BenchBot Robot Controller ($type):"
eval $persist_chk && loaded=0 || loaded=1
if [ $loaded -eq 0 ]; then
printf "Skipping (already running)\n"
else
cmd="${docker_run//'$name'/benchbot_robot}"
${cmd// /$'\t'} --ip "$URL_ROBOT" -d "${args[@]}" \
-t $DOCKER_TAG_SIM_PREFIX$type /bin/bash -c $persist_cmd
fi

printf "\n${colour_blue}%s${colour_nc}\n" \
"Executing BenchBot Robot Controller in persistent container:"
while [ $loaded -eq 1 ]; do
eval $persist_chk && loaded=0 || loaded=1
sleep 1
done

docker exec -d benchbot_robot /bin/bash -c \
"$cmd_prefix"'rosrun benchbot_robot_controller benchbot_robot_controller'

controller_pid=
while [ -z "$controller_pid" ]; do
controller_pid="$(docker container top benchbot_robot | \
grep 'benchbot_robot_controller' | grep -v 'grep' | awk '{print $2}' | \
tr '\n' ',' | sed 's/,$//')"
sleep 1
done
printf "Started in persistent container (pids = $controller_pid)\n"
# TODO DANGER: there are LOTS of ways this can go wrong. Need to do this
# more robustly if we ever expand outside Omniverse-only focus. Examples:
# - if next run uses a different simulator type, what do we do?
# - it's really unclear to the user what is still running in background

# Run a persistent container and execute within it if our simulator has
# caching utilities (e.g. Cache in Omniverse)
printf "\n${colour_blue}%s${colour_nc}\n" \
"Starting persistent container for BenchBot Robot Controller ($type):"
if [ "$(docker container inspect -f '{{.State.Running}}' \
benchbot_robot 2> /dev/null)" == "true" ]; then
printf "Skipping (already running)\n"
else
# Run a new container each time when there's no persistence requirement
printf "\n${colour_blue}%s${colour_nc}\n" \
"Starting container for BenchBot Robot Controller ($type):"
cmd="${docker_run//'$name'/benchbot_robot}"
${cmd// /$'\t'} --ip "$URL_ROBOT" -d "${args[@]}" \
-t $DOCKER_TAG_SIM_PREFIX$type /bin/bash -c \
Expand All @@ -431,13 +386,13 @@ header_block "BENCHBOT IS RUNNING (Ctrl^C to exit) ..." ${colour_green}
docker logs --follow benchbot_supervisor &

while [ -n "$(docker ps -q -f 'name=benchbot_supervisor')" ] && \
([ -z "$simulator_required" ] || \
([ "$simulator_required" -ne 0 ] || \
[ -n "$(docker ps -q -f 'name=benchbot_robot')" ]); do
sleep 1
done
# sleep infinity

if [ -n "$simulator_required" ] && \
if [ "$simulator_required" -eq 0 ] && \
[ -z "$(docker ps -q -f 'name=benchbot_robot')" ]; then
header_block "BENCHBOT ROBOT CONTROLLER ERROR" ${colour_red}
echo -e "\n${colour_red}$_robot_err${colour_nc}\n"
Expand Down

0 comments on commit 7617e2a

Please sign in to comment.