From 4d51189066c7110b81cfd59d579963edc028d8e1 Mon Sep 17 00:00:00 2001 From: Chris Paxton Date: Mon, 8 Apr 2024 16:54:20 -0400 Subject: [PATCH 01/17] update stuff --- imitation-in-homes/configs/env_vars/env_vars.yaml.sample | 4 ++-- imitation-in-homes/configs/finetune.yaml | 2 +- imitation-in-homes/requirements.txt | 3 ++- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/imitation-in-homes/configs/env_vars/env_vars.yaml.sample b/imitation-in-homes/configs/env_vars/env_vars.yaml.sample index 29a40f3..4c9d321 100644 --- a/imitation-in-homes/configs/env_vars/env_vars.yaml.sample +++ b/imitation-in-homes/configs/env_vars/env_vars.yaml.sample @@ -1,8 +1,8 @@ home_ssl_data_root: /home/shared/data/iphone_data # root to home ssl data folder -finetune_task_data_root: /home/shared/data/finetuning_data_extracted # root to the exported finetuning data folder +finetune_task_data_root: /home/cpaxton/data/finetune_directory # root to the exported finetuning data folder home_ssl_data_original_root: /path/to/directory # path included in the r3d_files.txt file of dataset -finetune_task_data_original_root: /path/to/finetune_directory # path included in r3d_files.txt of dataset (may be the same as finetune_task_data_root if data has not been moved nor transferred across machines) +finetune_task_data_original_root: /home/cpaxton/finetune_directory # path included in r3d_files.txt of dataset (may be the same as finetune_task_data_root if data has not been moved nor transferred across machines) project_root: /home/shared/code/imitation-in-homes # path to this repo wandb: diff --git a/imitation-in-homes/configs/finetune.yaml b/imitation-in-homes/configs/finetune.yaml index 16055e6..0d2c2eb 100644 --- a/imitation-in-homes/configs/finetune.yaml +++ b/imitation-in-homes/configs/finetune.yaml @@ -50,4 +50,4 @@ wandb: id: null save_code: true -seed: 42 \ No newline at end of file +seed: 42 diff --git a/imitation-in-homes/requirements.txt b/imitation-in-homes/requirements.txt index 9bb08a6..97dcfe5 100644 --- a/imitation-in-homes/requirements.txt +++ b/imitation-in-homes/requirements.txt @@ -11,4 +11,5 @@ tqdm==4.65.0 accelerate timm==0.9.2 wandb==0.15.8 -vit_pytorch==1.6.4 \ No newline at end of file +vit_pytorch==1.6.4 +decord From 2cad19b0974b4afb06a25605a65bb03f10d30b23 Mon Sep 17 00:00:00 2001 From: Chris Paxton Date: Mon, 8 Apr 2024 17:04:08 -0400 Subject: [PATCH 02/17] cleanup --- imitation-in-homes/dataloaders/utils.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/imitation-in-homes/dataloaders/utils.py b/imitation-in-homes/dataloaders/utils.py index 89ffa35..ede6d90 100644 --- a/imitation-in-homes/dataloaders/utils.py +++ b/imitation-in-homes/dataloaders/utils.py @@ -19,8 +19,12 @@ def load_trajectory_roots( original_root: Optional[str] = "/path/to/directory", new_root: Optional[str] = "/path/to/new/dataset_root", ) -> Iterable[Path]: - with open(trajectory_root_path, "r") as f: - trajectory_roots = json.load(f) + try: + with open(trajectory_root_path, "r") as f: + trajectory_roots = json.load(f) + except FileNotFoundError as e: + print(f"Could not find file {trajectory_root_path}") + raise e # Trim the .zip from the names before returning. if original_root is not None and new_root is not None: trajectory_roots = [ From bcf8bce538abffc5f6c7775ae76ca47473f035d9 Mon Sep 17 00:00:00 2001 From: Chris Paxton Date: Mon, 8 Apr 2024 17:04:46 -0400 Subject: [PATCH 03/17] update --- .gitignore | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index b5bcf67..0fd4305 100644 --- a/.gitignore +++ b/.gitignore @@ -523,4 +523,10 @@ test_* *.out # Hardware gitignore -.DS_Store \ No newline at end of file +.DS_Store + +# Vim stuff +*.swp +*.swo +*.swn +*.swm From 8da7fb7a5e1d0a7b1c0d124725f3dcafab9ff921 Mon Sep 17 00:00:00 2001 From: Chris Paxton Date: Tue, 7 May 2024 13:19:37 -0400 Subject: [PATCH 04/17] moving files around for easier use with stretchpy codebase --- .../{ => dobbe}/configs/dataset/finetune_task_data.yaml | 0 imitation-in-homes/{ => dobbe}/configs/dataset/hony.yaml | 0 .../{ => dobbe}/configs/dataset/offline_test.yaml | 0 .../{ => dobbe}/configs/dataset/open_loop_traj.yaml | 0 .../{ => dobbe}/configs/dataset/vinn_deploy_dataset.yaml | 0 imitation-in-homes/{ => dobbe}/configs/dataset/vinn_eval.yaml | 0 .../{ => dobbe}/configs/env_vars/env_vars.yaml.sample | 0 imitation-in-homes/{ => dobbe}/configs/finetune.yaml | 0 imitation-in-homes/{ => dobbe}/configs/loss_fn/bc_loss.yaml | 0 imitation-in-homes/{ => dobbe}/configs/loss_fn/byol_loss.yaml | 0 .../{ => dobbe}/configs/loss_fn/identity_loss_fn.yaml | 0 imitation-in-homes/{ => dobbe}/configs/loss_fn/mae_loss.yaml | 0 imitation-in-homes/{ => dobbe}/configs/loss_fn/moco_loss.yaml | 0 .../{ => dobbe}/configs/loss_fn/vinn_loss_fn.yaml | 0 imitation-in-homes/{ => dobbe}/configs/model/byol.yaml | 0 .../{ => dobbe}/configs/model/image_bc_depth.yaml | 0 .../{ => dobbe}/configs/model/image_bc_depth_timm.yaml | 0 .../{ => dobbe}/configs/model/image_bc_depth_vc1.yaml | 0 imitation-in-homes/{ => dobbe}/configs/model/image_vinn.yaml | 0 imitation-in-homes/{ => dobbe}/configs/model/mae.yaml | 0 imitation-in-homes/{ => dobbe}/configs/model/moco.yaml | 0 imitation-in-homes/{ => dobbe}/configs/model/open_loop.yaml | 0 imitation-in-homes/{ => dobbe}/configs/run.yaml | 0 imitation-in-homes/{ => dobbe}/configs/run_open_loop.yaml | 0 imitation-in-homes/{ => dobbe}/configs/run_vinn.yaml | 0 imitation-in-homes/{ => dobbe}/configs/train_byol.yaml | 0 imitation-in-homes/{ => dobbe}/configs/train_mae.yaml | 0 imitation-in-homes/{ => dobbe}/configs/train_moco.yaml | 0 imitation-in-homes/{ => dobbe}/configs/train_vinn.yaml | 0 imitation-in-homes/{ => dobbe}/loss_fns/__init__.py | 0 imitation-in-homes/{ => dobbe}/loss_fns/abstract_loss_fn.py | 0 imitation-in-homes/{ => dobbe}/loss_fns/bc_loss_fn.py | 0 imitation-in-homes/{ => dobbe}/loss_fns/identity_loss_fn.py | 0 imitation-in-homes/{ => dobbe}/loss_fns/ssl/__init__.py | 0 imitation-in-homes/{ => dobbe}/loss_fns/ssl/byol_loss_fn.py | 0 imitation-in-homes/{ => dobbe}/loss_fns/ssl/mae_loss_fn.py | 0 imitation-in-homes/{ => dobbe}/loss_fns/ssl/moco_loss_fn.py | 0 imitation-in-homes/{ => dobbe}/loss_fns/vinn_loss_fn.py | 0 imitation-in-homes/{ => dobbe}/models/__init__.py | 0 imitation-in-homes/{ => dobbe}/models/encoders/__init__.py | 0 .../{ => dobbe}/models/encoders/abstract_base_encoder.py | 0 .../{ => dobbe}/models/encoders/baseline_enc/mvp.py | 0 .../{ => dobbe}/models/encoders/baseline_enc/r3m.py | 0 .../{ => dobbe}/models/encoders/baseline_enc/utils.py | 0 .../{ => dobbe}/models/encoders/baseline_enc/vc1.py | 0 imitation-in-homes/{ => dobbe}/models/encoders/timm_encoders.py | 0 .../{ => dobbe}/models/encoders/torch_encoders.py | 0 imitation-in-homes/{ => dobbe}/models/policies/__init__.py | 0 imitation-in-homes/{ => dobbe}/models/policies/bc.py | 0 imitation-in-homes/{ => dobbe}/models/policies/depth_net.py | 0 .../{ => dobbe}/models/policies/normalize_actions.py | 0 imitation-in-homes/{ => dobbe}/models/policies/open_loop.py | 0 imitation-in-homes/{ => dobbe}/models/policies/vinn.py | 0 imitation-in-homes/{ => dobbe}/models/ssl/__init__.py | 0 imitation-in-homes/{ => dobbe}/models/ssl/mae.py | 0 imitation-in-homes/{ => dobbe}/models/ssl/moco/__init__.py | 0 imitation-in-homes/{ => dobbe}/models/ssl/moco/builder.py | 0 imitation-in-homes/{ => dobbe}/models/ssl/moco/loader.py | 0 imitation-in-homes/{ => dobbe}/models/ssl/moco/optimizer.py | 0 imitation-in-homes/{ => dobbe}/robot/controller.py | 0 imitation-in-homes/{ => dobbe}/robot/publisher.py | 0 imitation-in-homes/{ => dobbe}/robot/subscriber.py | 0 imitation-in-homes/{ => dobbe}/robot/utils.py | 0 imitation-in-homes/{ => dobbe}/run.py | 0 imitation-in-homes/{ => dobbe}/train.py | 2 +- imitation-in-homes/{ => dobbe}/utils/__init__.py | 0 imitation-in-homes/{ => dobbe}/utils/action_transforms.py | 0 imitation-in-homes/{ => dobbe}/utils/decord_transforms.py | 0 imitation-in-homes/{ => dobbe}/utils/image_plots.py | 0 imitation-in-homes/{ => dobbe}/utils/metrics.py | 0 imitation-in-homes/{ => dobbe}/utils/schedulers.py | 0 imitation-in-homes/{ => dobbe}/utils/ssl_transfroms.py | 0 imitation-in-homes/{ => dobbe}/utils/trajectory_vis.py | 0 73 files changed, 1 insertion(+), 1 deletion(-) rename imitation-in-homes/{ => dobbe}/configs/dataset/finetune_task_data.yaml (100%) rename imitation-in-homes/{ => dobbe}/configs/dataset/hony.yaml (100%) rename imitation-in-homes/{ => dobbe}/configs/dataset/offline_test.yaml (100%) rename imitation-in-homes/{ => dobbe}/configs/dataset/open_loop_traj.yaml (100%) rename imitation-in-homes/{ => dobbe}/configs/dataset/vinn_deploy_dataset.yaml (100%) rename imitation-in-homes/{ => dobbe}/configs/dataset/vinn_eval.yaml (100%) rename imitation-in-homes/{ => dobbe}/configs/env_vars/env_vars.yaml.sample (100%) rename imitation-in-homes/{ => dobbe}/configs/finetune.yaml (100%) rename imitation-in-homes/{ => dobbe}/configs/loss_fn/bc_loss.yaml (100%) rename imitation-in-homes/{ => dobbe}/configs/loss_fn/byol_loss.yaml (100%) rename imitation-in-homes/{ => dobbe}/configs/loss_fn/identity_loss_fn.yaml (100%) rename imitation-in-homes/{ => dobbe}/configs/loss_fn/mae_loss.yaml (100%) rename imitation-in-homes/{ => dobbe}/configs/loss_fn/moco_loss.yaml (100%) rename imitation-in-homes/{ => dobbe}/configs/loss_fn/vinn_loss_fn.yaml (100%) rename imitation-in-homes/{ => dobbe}/configs/model/byol.yaml (100%) rename imitation-in-homes/{ => dobbe}/configs/model/image_bc_depth.yaml (100%) rename imitation-in-homes/{ => dobbe}/configs/model/image_bc_depth_timm.yaml (100%) rename imitation-in-homes/{ => dobbe}/configs/model/image_bc_depth_vc1.yaml (100%) rename imitation-in-homes/{ => dobbe}/configs/model/image_vinn.yaml (100%) rename imitation-in-homes/{ => dobbe}/configs/model/mae.yaml (100%) rename imitation-in-homes/{ => dobbe}/configs/model/moco.yaml (100%) rename imitation-in-homes/{ => dobbe}/configs/model/open_loop.yaml (100%) rename imitation-in-homes/{ => dobbe}/configs/run.yaml (100%) rename imitation-in-homes/{ => dobbe}/configs/run_open_loop.yaml (100%) rename imitation-in-homes/{ => dobbe}/configs/run_vinn.yaml (100%) rename imitation-in-homes/{ => dobbe}/configs/train_byol.yaml (100%) rename imitation-in-homes/{ => dobbe}/configs/train_mae.yaml (100%) rename imitation-in-homes/{ => dobbe}/configs/train_moco.yaml (100%) rename imitation-in-homes/{ => dobbe}/configs/train_vinn.yaml (100%) rename imitation-in-homes/{ => dobbe}/loss_fns/__init__.py (100%) rename imitation-in-homes/{ => dobbe}/loss_fns/abstract_loss_fn.py (100%) rename imitation-in-homes/{ => dobbe}/loss_fns/bc_loss_fn.py (100%) rename imitation-in-homes/{ => dobbe}/loss_fns/identity_loss_fn.py (100%) rename imitation-in-homes/{ => dobbe}/loss_fns/ssl/__init__.py (100%) rename imitation-in-homes/{ => dobbe}/loss_fns/ssl/byol_loss_fn.py (100%) rename imitation-in-homes/{ => dobbe}/loss_fns/ssl/mae_loss_fn.py (100%) rename imitation-in-homes/{ => dobbe}/loss_fns/ssl/moco_loss_fn.py (100%) rename imitation-in-homes/{ => dobbe}/loss_fns/vinn_loss_fn.py (100%) rename imitation-in-homes/{ => dobbe}/models/__init__.py (100%) rename imitation-in-homes/{ => dobbe}/models/encoders/__init__.py (100%) rename imitation-in-homes/{ => dobbe}/models/encoders/abstract_base_encoder.py (100%) rename imitation-in-homes/{ => dobbe}/models/encoders/baseline_enc/mvp.py (100%) rename imitation-in-homes/{ => dobbe}/models/encoders/baseline_enc/r3m.py (100%) rename imitation-in-homes/{ => dobbe}/models/encoders/baseline_enc/utils.py (100%) rename imitation-in-homes/{ => dobbe}/models/encoders/baseline_enc/vc1.py (100%) rename imitation-in-homes/{ => dobbe}/models/encoders/timm_encoders.py (100%) rename imitation-in-homes/{ => dobbe}/models/encoders/torch_encoders.py (100%) rename imitation-in-homes/{ => dobbe}/models/policies/__init__.py (100%) rename imitation-in-homes/{ => dobbe}/models/policies/bc.py (100%) rename imitation-in-homes/{ => dobbe}/models/policies/depth_net.py (100%) rename imitation-in-homes/{ => dobbe}/models/policies/normalize_actions.py (100%) rename imitation-in-homes/{ => dobbe}/models/policies/open_loop.py (100%) rename imitation-in-homes/{ => dobbe}/models/policies/vinn.py (100%) rename imitation-in-homes/{ => dobbe}/models/ssl/__init__.py (100%) rename imitation-in-homes/{ => dobbe}/models/ssl/mae.py (100%) rename imitation-in-homes/{ => dobbe}/models/ssl/moco/__init__.py (100%) rename imitation-in-homes/{ => dobbe}/models/ssl/moco/builder.py (100%) rename imitation-in-homes/{ => dobbe}/models/ssl/moco/loader.py (100%) rename imitation-in-homes/{ => dobbe}/models/ssl/moco/optimizer.py (100%) rename imitation-in-homes/{ => dobbe}/robot/controller.py (100%) rename imitation-in-homes/{ => dobbe}/robot/publisher.py (100%) rename imitation-in-homes/{ => dobbe}/robot/subscriber.py (100%) rename imitation-in-homes/{ => dobbe}/robot/utils.py (100%) rename imitation-in-homes/{ => dobbe}/run.py (100%) rename imitation-in-homes/{ => dobbe}/train.py (99%) rename imitation-in-homes/{ => dobbe}/utils/__init__.py (100%) rename imitation-in-homes/{ => dobbe}/utils/action_transforms.py (100%) rename imitation-in-homes/{ => dobbe}/utils/decord_transforms.py (100%) rename imitation-in-homes/{ => dobbe}/utils/image_plots.py (100%) rename imitation-in-homes/{ => dobbe}/utils/metrics.py (100%) rename imitation-in-homes/{ => dobbe}/utils/schedulers.py (100%) rename imitation-in-homes/{ => dobbe}/utils/ssl_transfroms.py (100%) rename imitation-in-homes/{ => dobbe}/utils/trajectory_vis.py (100%) diff --git a/imitation-in-homes/configs/dataset/finetune_task_data.yaml b/imitation-in-homes/dobbe/configs/dataset/finetune_task_data.yaml similarity index 100% rename from imitation-in-homes/configs/dataset/finetune_task_data.yaml rename to imitation-in-homes/dobbe/configs/dataset/finetune_task_data.yaml diff --git a/imitation-in-homes/configs/dataset/hony.yaml b/imitation-in-homes/dobbe/configs/dataset/hony.yaml similarity index 100% rename from imitation-in-homes/configs/dataset/hony.yaml rename to imitation-in-homes/dobbe/configs/dataset/hony.yaml diff --git a/imitation-in-homes/configs/dataset/offline_test.yaml b/imitation-in-homes/dobbe/configs/dataset/offline_test.yaml similarity index 100% rename from imitation-in-homes/configs/dataset/offline_test.yaml rename to imitation-in-homes/dobbe/configs/dataset/offline_test.yaml diff --git a/imitation-in-homes/configs/dataset/open_loop_traj.yaml b/imitation-in-homes/dobbe/configs/dataset/open_loop_traj.yaml similarity index 100% rename from imitation-in-homes/configs/dataset/open_loop_traj.yaml rename to imitation-in-homes/dobbe/configs/dataset/open_loop_traj.yaml diff --git a/imitation-in-homes/configs/dataset/vinn_deploy_dataset.yaml b/imitation-in-homes/dobbe/configs/dataset/vinn_deploy_dataset.yaml similarity index 100% rename from imitation-in-homes/configs/dataset/vinn_deploy_dataset.yaml rename to imitation-in-homes/dobbe/configs/dataset/vinn_deploy_dataset.yaml diff --git a/imitation-in-homes/configs/dataset/vinn_eval.yaml b/imitation-in-homes/dobbe/configs/dataset/vinn_eval.yaml similarity index 100% rename from imitation-in-homes/configs/dataset/vinn_eval.yaml rename to imitation-in-homes/dobbe/configs/dataset/vinn_eval.yaml diff --git a/imitation-in-homes/configs/env_vars/env_vars.yaml.sample b/imitation-in-homes/dobbe/configs/env_vars/env_vars.yaml.sample similarity index 100% rename from imitation-in-homes/configs/env_vars/env_vars.yaml.sample rename to imitation-in-homes/dobbe/configs/env_vars/env_vars.yaml.sample diff --git a/imitation-in-homes/configs/finetune.yaml b/imitation-in-homes/dobbe/configs/finetune.yaml similarity index 100% rename from imitation-in-homes/configs/finetune.yaml rename to imitation-in-homes/dobbe/configs/finetune.yaml diff --git a/imitation-in-homes/configs/loss_fn/bc_loss.yaml b/imitation-in-homes/dobbe/configs/loss_fn/bc_loss.yaml similarity index 100% rename from imitation-in-homes/configs/loss_fn/bc_loss.yaml rename to imitation-in-homes/dobbe/configs/loss_fn/bc_loss.yaml diff --git a/imitation-in-homes/configs/loss_fn/byol_loss.yaml b/imitation-in-homes/dobbe/configs/loss_fn/byol_loss.yaml similarity index 100% rename from imitation-in-homes/configs/loss_fn/byol_loss.yaml rename to imitation-in-homes/dobbe/configs/loss_fn/byol_loss.yaml diff --git a/imitation-in-homes/configs/loss_fn/identity_loss_fn.yaml b/imitation-in-homes/dobbe/configs/loss_fn/identity_loss_fn.yaml similarity index 100% rename from imitation-in-homes/configs/loss_fn/identity_loss_fn.yaml rename to imitation-in-homes/dobbe/configs/loss_fn/identity_loss_fn.yaml diff --git a/imitation-in-homes/configs/loss_fn/mae_loss.yaml b/imitation-in-homes/dobbe/configs/loss_fn/mae_loss.yaml similarity index 100% rename from imitation-in-homes/configs/loss_fn/mae_loss.yaml rename to imitation-in-homes/dobbe/configs/loss_fn/mae_loss.yaml diff --git a/imitation-in-homes/configs/loss_fn/moco_loss.yaml b/imitation-in-homes/dobbe/configs/loss_fn/moco_loss.yaml similarity index 100% rename from imitation-in-homes/configs/loss_fn/moco_loss.yaml rename to imitation-in-homes/dobbe/configs/loss_fn/moco_loss.yaml diff --git a/imitation-in-homes/configs/loss_fn/vinn_loss_fn.yaml b/imitation-in-homes/dobbe/configs/loss_fn/vinn_loss_fn.yaml similarity index 100% rename from imitation-in-homes/configs/loss_fn/vinn_loss_fn.yaml rename to imitation-in-homes/dobbe/configs/loss_fn/vinn_loss_fn.yaml diff --git a/imitation-in-homes/configs/model/byol.yaml b/imitation-in-homes/dobbe/configs/model/byol.yaml similarity index 100% rename from imitation-in-homes/configs/model/byol.yaml rename to imitation-in-homes/dobbe/configs/model/byol.yaml diff --git a/imitation-in-homes/configs/model/image_bc_depth.yaml b/imitation-in-homes/dobbe/configs/model/image_bc_depth.yaml similarity index 100% rename from imitation-in-homes/configs/model/image_bc_depth.yaml rename to imitation-in-homes/dobbe/configs/model/image_bc_depth.yaml diff --git a/imitation-in-homes/configs/model/image_bc_depth_timm.yaml b/imitation-in-homes/dobbe/configs/model/image_bc_depth_timm.yaml similarity index 100% rename from imitation-in-homes/configs/model/image_bc_depth_timm.yaml rename to imitation-in-homes/dobbe/configs/model/image_bc_depth_timm.yaml diff --git a/imitation-in-homes/configs/model/image_bc_depth_vc1.yaml b/imitation-in-homes/dobbe/configs/model/image_bc_depth_vc1.yaml similarity index 100% rename from imitation-in-homes/configs/model/image_bc_depth_vc1.yaml rename to imitation-in-homes/dobbe/configs/model/image_bc_depth_vc1.yaml diff --git a/imitation-in-homes/configs/model/image_vinn.yaml b/imitation-in-homes/dobbe/configs/model/image_vinn.yaml similarity index 100% rename from imitation-in-homes/configs/model/image_vinn.yaml rename to imitation-in-homes/dobbe/configs/model/image_vinn.yaml diff --git a/imitation-in-homes/configs/model/mae.yaml b/imitation-in-homes/dobbe/configs/model/mae.yaml similarity index 100% rename from imitation-in-homes/configs/model/mae.yaml rename to imitation-in-homes/dobbe/configs/model/mae.yaml diff --git a/imitation-in-homes/configs/model/moco.yaml b/imitation-in-homes/dobbe/configs/model/moco.yaml similarity index 100% rename from imitation-in-homes/configs/model/moco.yaml rename to imitation-in-homes/dobbe/configs/model/moco.yaml diff --git a/imitation-in-homes/configs/model/open_loop.yaml b/imitation-in-homes/dobbe/configs/model/open_loop.yaml similarity index 100% rename from imitation-in-homes/configs/model/open_loop.yaml rename to imitation-in-homes/dobbe/configs/model/open_loop.yaml diff --git a/imitation-in-homes/configs/run.yaml b/imitation-in-homes/dobbe/configs/run.yaml similarity index 100% rename from imitation-in-homes/configs/run.yaml rename to imitation-in-homes/dobbe/configs/run.yaml diff --git a/imitation-in-homes/configs/run_open_loop.yaml b/imitation-in-homes/dobbe/configs/run_open_loop.yaml similarity index 100% rename from imitation-in-homes/configs/run_open_loop.yaml rename to imitation-in-homes/dobbe/configs/run_open_loop.yaml diff --git a/imitation-in-homes/configs/run_vinn.yaml b/imitation-in-homes/dobbe/configs/run_vinn.yaml similarity index 100% rename from imitation-in-homes/configs/run_vinn.yaml rename to imitation-in-homes/dobbe/configs/run_vinn.yaml diff --git a/imitation-in-homes/configs/train_byol.yaml b/imitation-in-homes/dobbe/configs/train_byol.yaml similarity index 100% rename from imitation-in-homes/configs/train_byol.yaml rename to imitation-in-homes/dobbe/configs/train_byol.yaml diff --git a/imitation-in-homes/configs/train_mae.yaml b/imitation-in-homes/dobbe/configs/train_mae.yaml similarity index 100% rename from imitation-in-homes/configs/train_mae.yaml rename to imitation-in-homes/dobbe/configs/train_mae.yaml diff --git a/imitation-in-homes/configs/train_moco.yaml b/imitation-in-homes/dobbe/configs/train_moco.yaml similarity index 100% rename from imitation-in-homes/configs/train_moco.yaml rename to imitation-in-homes/dobbe/configs/train_moco.yaml diff --git a/imitation-in-homes/configs/train_vinn.yaml b/imitation-in-homes/dobbe/configs/train_vinn.yaml similarity index 100% rename from imitation-in-homes/configs/train_vinn.yaml rename to imitation-in-homes/dobbe/configs/train_vinn.yaml diff --git a/imitation-in-homes/loss_fns/__init__.py b/imitation-in-homes/dobbe/loss_fns/__init__.py similarity index 100% rename from imitation-in-homes/loss_fns/__init__.py rename to imitation-in-homes/dobbe/loss_fns/__init__.py diff --git a/imitation-in-homes/loss_fns/abstract_loss_fn.py b/imitation-in-homes/dobbe/loss_fns/abstract_loss_fn.py similarity index 100% rename from imitation-in-homes/loss_fns/abstract_loss_fn.py rename to imitation-in-homes/dobbe/loss_fns/abstract_loss_fn.py diff --git a/imitation-in-homes/loss_fns/bc_loss_fn.py b/imitation-in-homes/dobbe/loss_fns/bc_loss_fn.py similarity index 100% rename from imitation-in-homes/loss_fns/bc_loss_fn.py rename to imitation-in-homes/dobbe/loss_fns/bc_loss_fn.py diff --git a/imitation-in-homes/loss_fns/identity_loss_fn.py b/imitation-in-homes/dobbe/loss_fns/identity_loss_fn.py similarity index 100% rename from imitation-in-homes/loss_fns/identity_loss_fn.py rename to imitation-in-homes/dobbe/loss_fns/identity_loss_fn.py diff --git a/imitation-in-homes/loss_fns/ssl/__init__.py b/imitation-in-homes/dobbe/loss_fns/ssl/__init__.py similarity index 100% rename from imitation-in-homes/loss_fns/ssl/__init__.py rename to imitation-in-homes/dobbe/loss_fns/ssl/__init__.py diff --git a/imitation-in-homes/loss_fns/ssl/byol_loss_fn.py b/imitation-in-homes/dobbe/loss_fns/ssl/byol_loss_fn.py similarity index 100% rename from imitation-in-homes/loss_fns/ssl/byol_loss_fn.py rename to imitation-in-homes/dobbe/loss_fns/ssl/byol_loss_fn.py diff --git a/imitation-in-homes/loss_fns/ssl/mae_loss_fn.py b/imitation-in-homes/dobbe/loss_fns/ssl/mae_loss_fn.py similarity index 100% rename from imitation-in-homes/loss_fns/ssl/mae_loss_fn.py rename to imitation-in-homes/dobbe/loss_fns/ssl/mae_loss_fn.py diff --git a/imitation-in-homes/loss_fns/ssl/moco_loss_fn.py b/imitation-in-homes/dobbe/loss_fns/ssl/moco_loss_fn.py similarity index 100% rename from imitation-in-homes/loss_fns/ssl/moco_loss_fn.py rename to imitation-in-homes/dobbe/loss_fns/ssl/moco_loss_fn.py diff --git a/imitation-in-homes/loss_fns/vinn_loss_fn.py b/imitation-in-homes/dobbe/loss_fns/vinn_loss_fn.py similarity index 100% rename from imitation-in-homes/loss_fns/vinn_loss_fn.py rename to imitation-in-homes/dobbe/loss_fns/vinn_loss_fn.py diff --git a/imitation-in-homes/models/__init__.py b/imitation-in-homes/dobbe/models/__init__.py similarity index 100% rename from imitation-in-homes/models/__init__.py rename to imitation-in-homes/dobbe/models/__init__.py diff --git a/imitation-in-homes/models/encoders/__init__.py b/imitation-in-homes/dobbe/models/encoders/__init__.py similarity index 100% rename from imitation-in-homes/models/encoders/__init__.py rename to imitation-in-homes/dobbe/models/encoders/__init__.py diff --git a/imitation-in-homes/models/encoders/abstract_base_encoder.py b/imitation-in-homes/dobbe/models/encoders/abstract_base_encoder.py similarity index 100% rename from imitation-in-homes/models/encoders/abstract_base_encoder.py rename to imitation-in-homes/dobbe/models/encoders/abstract_base_encoder.py diff --git a/imitation-in-homes/models/encoders/baseline_enc/mvp.py b/imitation-in-homes/dobbe/models/encoders/baseline_enc/mvp.py similarity index 100% rename from imitation-in-homes/models/encoders/baseline_enc/mvp.py rename to imitation-in-homes/dobbe/models/encoders/baseline_enc/mvp.py diff --git a/imitation-in-homes/models/encoders/baseline_enc/r3m.py b/imitation-in-homes/dobbe/models/encoders/baseline_enc/r3m.py similarity index 100% rename from imitation-in-homes/models/encoders/baseline_enc/r3m.py rename to imitation-in-homes/dobbe/models/encoders/baseline_enc/r3m.py diff --git a/imitation-in-homes/models/encoders/baseline_enc/utils.py b/imitation-in-homes/dobbe/models/encoders/baseline_enc/utils.py similarity index 100% rename from imitation-in-homes/models/encoders/baseline_enc/utils.py rename to imitation-in-homes/dobbe/models/encoders/baseline_enc/utils.py diff --git a/imitation-in-homes/models/encoders/baseline_enc/vc1.py b/imitation-in-homes/dobbe/models/encoders/baseline_enc/vc1.py similarity index 100% rename from imitation-in-homes/models/encoders/baseline_enc/vc1.py rename to imitation-in-homes/dobbe/models/encoders/baseline_enc/vc1.py diff --git a/imitation-in-homes/models/encoders/timm_encoders.py b/imitation-in-homes/dobbe/models/encoders/timm_encoders.py similarity index 100% rename from imitation-in-homes/models/encoders/timm_encoders.py rename to imitation-in-homes/dobbe/models/encoders/timm_encoders.py diff --git a/imitation-in-homes/models/encoders/torch_encoders.py b/imitation-in-homes/dobbe/models/encoders/torch_encoders.py similarity index 100% rename from imitation-in-homes/models/encoders/torch_encoders.py rename to imitation-in-homes/dobbe/models/encoders/torch_encoders.py diff --git a/imitation-in-homes/models/policies/__init__.py b/imitation-in-homes/dobbe/models/policies/__init__.py similarity index 100% rename from imitation-in-homes/models/policies/__init__.py rename to imitation-in-homes/dobbe/models/policies/__init__.py diff --git a/imitation-in-homes/models/policies/bc.py b/imitation-in-homes/dobbe/models/policies/bc.py similarity index 100% rename from imitation-in-homes/models/policies/bc.py rename to imitation-in-homes/dobbe/models/policies/bc.py diff --git a/imitation-in-homes/models/policies/depth_net.py b/imitation-in-homes/dobbe/models/policies/depth_net.py similarity index 100% rename from imitation-in-homes/models/policies/depth_net.py rename to imitation-in-homes/dobbe/models/policies/depth_net.py diff --git a/imitation-in-homes/models/policies/normalize_actions.py b/imitation-in-homes/dobbe/models/policies/normalize_actions.py similarity index 100% rename from imitation-in-homes/models/policies/normalize_actions.py rename to imitation-in-homes/dobbe/models/policies/normalize_actions.py diff --git a/imitation-in-homes/models/policies/open_loop.py b/imitation-in-homes/dobbe/models/policies/open_loop.py similarity index 100% rename from imitation-in-homes/models/policies/open_loop.py rename to imitation-in-homes/dobbe/models/policies/open_loop.py diff --git a/imitation-in-homes/models/policies/vinn.py b/imitation-in-homes/dobbe/models/policies/vinn.py similarity index 100% rename from imitation-in-homes/models/policies/vinn.py rename to imitation-in-homes/dobbe/models/policies/vinn.py diff --git a/imitation-in-homes/models/ssl/__init__.py b/imitation-in-homes/dobbe/models/ssl/__init__.py similarity index 100% rename from imitation-in-homes/models/ssl/__init__.py rename to imitation-in-homes/dobbe/models/ssl/__init__.py diff --git a/imitation-in-homes/models/ssl/mae.py b/imitation-in-homes/dobbe/models/ssl/mae.py similarity index 100% rename from imitation-in-homes/models/ssl/mae.py rename to imitation-in-homes/dobbe/models/ssl/mae.py diff --git a/imitation-in-homes/models/ssl/moco/__init__.py b/imitation-in-homes/dobbe/models/ssl/moco/__init__.py similarity index 100% rename from imitation-in-homes/models/ssl/moco/__init__.py rename to imitation-in-homes/dobbe/models/ssl/moco/__init__.py diff --git a/imitation-in-homes/models/ssl/moco/builder.py b/imitation-in-homes/dobbe/models/ssl/moco/builder.py similarity index 100% rename from imitation-in-homes/models/ssl/moco/builder.py rename to imitation-in-homes/dobbe/models/ssl/moco/builder.py diff --git a/imitation-in-homes/models/ssl/moco/loader.py b/imitation-in-homes/dobbe/models/ssl/moco/loader.py similarity index 100% rename from imitation-in-homes/models/ssl/moco/loader.py rename to imitation-in-homes/dobbe/models/ssl/moco/loader.py diff --git a/imitation-in-homes/models/ssl/moco/optimizer.py b/imitation-in-homes/dobbe/models/ssl/moco/optimizer.py similarity index 100% rename from imitation-in-homes/models/ssl/moco/optimizer.py rename to imitation-in-homes/dobbe/models/ssl/moco/optimizer.py diff --git a/imitation-in-homes/robot/controller.py b/imitation-in-homes/dobbe/robot/controller.py similarity index 100% rename from imitation-in-homes/robot/controller.py rename to imitation-in-homes/dobbe/robot/controller.py diff --git a/imitation-in-homes/robot/publisher.py b/imitation-in-homes/dobbe/robot/publisher.py similarity index 100% rename from imitation-in-homes/robot/publisher.py rename to imitation-in-homes/dobbe/robot/publisher.py diff --git a/imitation-in-homes/robot/subscriber.py b/imitation-in-homes/dobbe/robot/subscriber.py similarity index 100% rename from imitation-in-homes/robot/subscriber.py rename to imitation-in-homes/dobbe/robot/subscriber.py diff --git a/imitation-in-homes/robot/utils.py b/imitation-in-homes/dobbe/robot/utils.py similarity index 100% rename from imitation-in-homes/robot/utils.py rename to imitation-in-homes/dobbe/robot/utils.py diff --git a/imitation-in-homes/run.py b/imitation-in-homes/dobbe/run.py similarity index 100% rename from imitation-in-homes/run.py rename to imitation-in-homes/dobbe/run.py diff --git a/imitation-in-homes/train.py b/imitation-in-homes/dobbe/train.py similarity index 99% rename from imitation-in-homes/train.py rename to imitation-in-homes/dobbe/train.py index 3ef8097..e6baf9b 100644 --- a/imitation-in-homes/train.py +++ b/imitation-in-homes/dobbe/train.py @@ -17,7 +17,7 @@ from omegaconf import OmegaConf from torch.utils.data import DataLoader -from utils import AveragingDict, Callbacks, set_seed_everywhere +from dobbe.utils import AveragingDict, Callbacks, set_seed_everywhere logger = logging.getLogger(__name__) if environ.get("ACCEL_DEBUG", False): diff --git a/imitation-in-homes/utils/__init__.py b/imitation-in-homes/dobbe/utils/__init__.py similarity index 100% rename from imitation-in-homes/utils/__init__.py rename to imitation-in-homes/dobbe/utils/__init__.py diff --git a/imitation-in-homes/utils/action_transforms.py b/imitation-in-homes/dobbe/utils/action_transforms.py similarity index 100% rename from imitation-in-homes/utils/action_transforms.py rename to imitation-in-homes/dobbe/utils/action_transforms.py diff --git a/imitation-in-homes/utils/decord_transforms.py b/imitation-in-homes/dobbe/utils/decord_transforms.py similarity index 100% rename from imitation-in-homes/utils/decord_transforms.py rename to imitation-in-homes/dobbe/utils/decord_transforms.py diff --git a/imitation-in-homes/utils/image_plots.py b/imitation-in-homes/dobbe/utils/image_plots.py similarity index 100% rename from imitation-in-homes/utils/image_plots.py rename to imitation-in-homes/dobbe/utils/image_plots.py diff --git a/imitation-in-homes/utils/metrics.py b/imitation-in-homes/dobbe/utils/metrics.py similarity index 100% rename from imitation-in-homes/utils/metrics.py rename to imitation-in-homes/dobbe/utils/metrics.py diff --git a/imitation-in-homes/utils/schedulers.py b/imitation-in-homes/dobbe/utils/schedulers.py similarity index 100% rename from imitation-in-homes/utils/schedulers.py rename to imitation-in-homes/dobbe/utils/schedulers.py diff --git a/imitation-in-homes/utils/ssl_transfroms.py b/imitation-in-homes/dobbe/utils/ssl_transfroms.py similarity index 100% rename from imitation-in-homes/utils/ssl_transfroms.py rename to imitation-in-homes/dobbe/utils/ssl_transfroms.py diff --git a/imitation-in-homes/utils/trajectory_vis.py b/imitation-in-homes/dobbe/utils/trajectory_vis.py similarity index 100% rename from imitation-in-homes/utils/trajectory_vis.py rename to imitation-in-homes/dobbe/utils/trajectory_vis.py From 65470d3e60985b99e2e78412eb23629886a56aa9 Mon Sep 17 00:00:00 2001 From: Chris Paxton Date: Tue, 7 May 2024 13:22:03 -0400 Subject: [PATCH 05/17] update code a bit --- imitation-in-homes/dobbe/configs/finetune.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/imitation-in-homes/dobbe/configs/finetune.yaml b/imitation-in-homes/dobbe/configs/finetune.yaml index 0d2c2eb..054d8e8 100644 --- a/imitation-in-homes/dobbe/configs/finetune.yaml +++ b/imitation-in-homes/dobbe/configs/finetune.yaml @@ -17,11 +17,11 @@ include_env: Env1 sampler: null batch_sampler: - _target_: dataloaders.decord_dataset.DecordBatchSampler + _target_: dobbe.dataloaders.decord_dataset.DecordBatchSampler batch_size: ${batch_size} dataloader: - _target_: dataloaders.decord_dataloader.DecordVideoLoader + _target_: dobbe.dataloaders.decord_dataloader.DecordVideoLoader device: cpu num_workers: 8 @@ -32,7 +32,7 @@ optimizer: weight_decay: 1.5e-6 scheduler: - _target_: models.ssl.moco.optimizer.MoCoScheduler + _target_: dobbe.models.ssl.moco.optimizer.MoCoScheduler warmup_epochs: 10 start_lr: 2e-6 num_epochs: ${num_epochs} From b3fd81aff6da11868c5dfeafc9ab7e8653ee39ae Mon Sep 17 00:00:00 2001 From: Chris Paxton Date: Tue, 7 May 2024 13:23:37 -0400 Subject: [PATCH 06/17] update and move --- imitation-in-homes/{ => dobbe}/dataloaders/__init__.py | 0 imitation-in-homes/{ => dobbe}/dataloaders/abstract_dataloader.py | 0 imitation-in-homes/{ => dobbe}/dataloaders/decord_dataloader.py | 0 imitation-in-homes/{ => dobbe}/dataloaders/decord_dataset.py | 0 imitation-in-homes/{ => dobbe}/dataloaders/depth_data.py | 0 imitation-in-homes/{ => dobbe}/dataloaders/file_dataloader.py | 0 imitation-in-homes/{ => dobbe}/dataloaders/pose_data.py | 0 imitation-in-homes/{ => dobbe}/dataloaders/utils.py | 0 8 files changed, 0 insertions(+), 0 deletions(-) rename imitation-in-homes/{ => dobbe}/dataloaders/__init__.py (100%) rename imitation-in-homes/{ => dobbe}/dataloaders/abstract_dataloader.py (100%) rename imitation-in-homes/{ => dobbe}/dataloaders/decord_dataloader.py (100%) rename imitation-in-homes/{ => dobbe}/dataloaders/decord_dataset.py (100%) rename imitation-in-homes/{ => dobbe}/dataloaders/depth_data.py (100%) rename imitation-in-homes/{ => dobbe}/dataloaders/file_dataloader.py (100%) rename imitation-in-homes/{ => dobbe}/dataloaders/pose_data.py (100%) rename imitation-in-homes/{ => dobbe}/dataloaders/utils.py (100%) diff --git a/imitation-in-homes/dataloaders/__init__.py b/imitation-in-homes/dobbe/dataloaders/__init__.py similarity index 100% rename from imitation-in-homes/dataloaders/__init__.py rename to imitation-in-homes/dobbe/dataloaders/__init__.py diff --git a/imitation-in-homes/dataloaders/abstract_dataloader.py b/imitation-in-homes/dobbe/dataloaders/abstract_dataloader.py similarity index 100% rename from imitation-in-homes/dataloaders/abstract_dataloader.py rename to imitation-in-homes/dobbe/dataloaders/abstract_dataloader.py diff --git a/imitation-in-homes/dataloaders/decord_dataloader.py b/imitation-in-homes/dobbe/dataloaders/decord_dataloader.py similarity index 100% rename from imitation-in-homes/dataloaders/decord_dataloader.py rename to imitation-in-homes/dobbe/dataloaders/decord_dataloader.py diff --git a/imitation-in-homes/dataloaders/decord_dataset.py b/imitation-in-homes/dobbe/dataloaders/decord_dataset.py similarity index 100% rename from imitation-in-homes/dataloaders/decord_dataset.py rename to imitation-in-homes/dobbe/dataloaders/decord_dataset.py diff --git a/imitation-in-homes/dataloaders/depth_data.py b/imitation-in-homes/dobbe/dataloaders/depth_data.py similarity index 100% rename from imitation-in-homes/dataloaders/depth_data.py rename to imitation-in-homes/dobbe/dataloaders/depth_data.py diff --git a/imitation-in-homes/dataloaders/file_dataloader.py b/imitation-in-homes/dobbe/dataloaders/file_dataloader.py similarity index 100% rename from imitation-in-homes/dataloaders/file_dataloader.py rename to imitation-in-homes/dobbe/dataloaders/file_dataloader.py diff --git a/imitation-in-homes/dataloaders/pose_data.py b/imitation-in-homes/dobbe/dataloaders/pose_data.py similarity index 100% rename from imitation-in-homes/dataloaders/pose_data.py rename to imitation-in-homes/dobbe/dataloaders/pose_data.py diff --git a/imitation-in-homes/dataloaders/utils.py b/imitation-in-homes/dobbe/dataloaders/utils.py similarity index 100% rename from imitation-in-homes/dataloaders/utils.py rename to imitation-in-homes/dobbe/dataloaders/utils.py From f1c3b4d1a50e3f942c800ee2eb04ea3189d5f4a0 Mon Sep 17 00:00:00 2001 From: Chris Paxton Date: Tue, 7 May 2024 13:23:43 -0400 Subject: [PATCH 07/17] update setup --- imitation-in-homes/setup.py | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 imitation-in-homes/setup.py diff --git a/imitation-in-homes/setup.py b/imitation-in-homes/setup.py new file mode 100644 index 0000000..3346c84 --- /dev/null +++ b/imitation-in-homes/setup.py @@ -0,0 +1,33 @@ +from setuptools import setup, find_packages + +setup( + name='dobbe', + version='0.1.0', + description='A brief description of my package', + long_description=open('README.md').read(), + long_description_content_type='text/markdown', + author='Your Name', + author_email='your_email@example.com', + url='https://github.com/cpaxton/dobbe', + packages=find_packages(), + install_requires=[ + #'dependency1>=1.0.0', + #'dependency2>=2.0.0', + ], + classifiers=[ + #'Development Status :: 3 - Alpha', + #'Intended Audience :: Developers', + #'License :: OSI Approved :: MIT License', + #'Programming Language :: Python :: 3', + #'Programming Language :: Python :: 3.6', + #'Programming Language :: Python :: 3.7', + #'Programming Language :: Python :: 3.8', + #'Programming Language :: Python :: 3.9', + #'Programming Language :: Python :: 3.10', + ], + entry_points={ + #'console_scripts': [ + # 'my-script = my_package.main:main', + #], + }, +) From 1e9eeb8ded235ab01904f02249a6e80927278660 Mon Sep 17 00:00:00 2001 From: Chris Paxton Date: Tue, 7 May 2024 13:25:19 -0400 Subject: [PATCH 08/17] add utils to load --- imitation-in-homes/dobbe/dataloaders/utils.py | 38 +++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/imitation-in-homes/dobbe/dataloaders/utils.py b/imitation-in-homes/dobbe/dataloaders/utils.py index ede6d90..c89463c 100644 --- a/imitation-in-homes/dobbe/dataloaders/utils.py +++ b/imitation-in-homes/dobbe/dataloaders/utils.py @@ -1,3 +1,4 @@ +import glob import json import os import warnings @@ -34,6 +35,43 @@ def load_trajectory_roots( return trajectory_roots +def create_trajectory_roots( + trajectory_root_path: Union[str, Path], + original_root: Optional[str] = "/path/to/directory", + new_root: Optional[str] = "/path/to/new/dataset_root", +) -> Iterable[Path]: + """Create trajectory roots; computed from existing directories.""" + print("-" * 50) + print(f"Creating trajectory roots from {trajectory_root_path}") + print(f"Original root: {original_root}") + print(f"New root: {new_root}") + paths = glob.glob(str(trajectory_root_path) + "/*/*/*/*") + if original_root is not None and new_root is not None: + paths = [str(root).replace(original_root, new_root) for root in paths] + + # paths = [p.split('/')[-4:] for p in paths] + # breakpoint() + print(f"Trajectory roots: {paths}") + print("-" * 50) + return [Path(path) for path in paths] + + +def flatten_nested_lists(nested_list, level=0, max_level=-1): + if max_level > 0 and level >= max_level: + return nested_list + flattened_list = [] + for sublist in nested_list: + if isinstance(sublist, list): + flattened_list.extend( + flatten_nested_lists(sublist, level=level + 1, max_level=max_level) + ) + else: + flattened_list.append(sublist) + return flattened_list + + + + def flatten_nested_lists(nested_list, level=0, max_level=-1): if max_level > 0 and level >= max_level: return nested_list From 135a4219c3949a768a71aa0c3280e518279d941c Mon Sep 17 00:00:00 2001 From: Chris Paxton Date: Tue, 7 May 2024 13:26:24 -0400 Subject: [PATCH 09/17] update --- imitation-in-homes/dobbe/dataloaders/decord_dataloader.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/imitation-in-homes/dobbe/dataloaders/decord_dataloader.py b/imitation-in-homes/dobbe/dataloaders/decord_dataloader.py index 4d3cd35..92c3b5e 100644 --- a/imitation-in-homes/dobbe/dataloaders/decord_dataloader.py +++ b/imitation-in-homes/dobbe/dataloaders/decord_dataloader.py @@ -25,6 +25,9 @@ def __init__( # Check if the dataset is a DecordDataset and the device is GPU if isinstance(dataset, DecordDataset): self._is_decord_dataset = True + else: + print(dataset) + self._is_decord_dataset = False if num_workers > 0 and self._is_decord_dataset: if device == "gpu": warnings.warn( From 2701751adafacb21fecf69a30730f773a0a61e94 Mon Sep 17 00:00:00 2001 From: Chris Paxton Date: Tue, 7 May 2024 13:27:55 -0400 Subject: [PATCH 10/17] we can change theese bug should we? --- imitation-in-homes/dobbe/dataloaders/decord_dataloader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/imitation-in-homes/dobbe/dataloaders/decord_dataloader.py b/imitation-in-homes/dobbe/dataloaders/decord_dataloader.py index 92c3b5e..44584d7 100644 --- a/imitation-in-homes/dobbe/dataloaders/decord_dataloader.py +++ b/imitation-in-homes/dobbe/dataloaders/decord_dataloader.py @@ -5,7 +5,7 @@ import decord from torch.utils.data import DataLoader, Dataset, Sampler -from dataloaders.decord_dataset import DecordDataset +from dobbe.dataloaders.decord_dataset import DecordDataset class DecordVideoLoader(DataLoader): From 7418cec3f1f50e33d872e529fa5af41f57b90bc6 Mon Sep 17 00:00:00 2001 From: Chris Paxton Date: Tue, 7 May 2024 13:47:13 -0400 Subject: [PATCH 11/17] configs modified --- .../dobbe/configs/dataset/debug.yaml | 57 +++++++++++++++++++ imitation-in-homes/dobbe/configs/debug.yaml | 53 +++++++++++++++++ imitation-in-homes/dobbe/configs/run.yaml | 2 +- imitation-in-homes/dobbe/run.py | 2 +- 4 files changed, 112 insertions(+), 2 deletions(-) create mode 100644 imitation-in-homes/dobbe/configs/dataset/debug.yaml create mode 100644 imitation-in-homes/dobbe/configs/debug.yaml diff --git a/imitation-in-homes/dobbe/configs/dataset/debug.yaml b/imitation-in-homes/dobbe/configs/dataset/debug.yaml new file mode 100644 index 0000000..ea35a29 --- /dev/null +++ b/imitation-in-homes/dobbe/configs/dataset/debug.yaml @@ -0,0 +1,57 @@ +train: + _target_: dobbe.dataloaders.decord_dataset.DecordDataset + config: + _target_: dobbe.dataloaders.utils.DataLoaderConfig + dataset_root: ${env_vars.debug_data_root} + trajectory_roots: + _target_: dobbe.dataloaders.utils.create_trajectory_roots + trajectory_root_path: ${dataset.train.config.dataset_root} + original_root: ${env_vars.debug_data_original_root} + new_root: ${dataset.train.config.dataset_root} + include_tasks: + - ${include_task} + exclude_homes: [] + include_envs: + - ${include_env} + sequence_length: 1 + trajectory_subsample_fraction: 1.0 + control_timeskip: 7 + fps_subsample: 1 + relative_gripper: False + use_depth: ${use_depth} + binarize_gripper: false + binarize_gripper_threshold: 0.8 + binarize_gripper_upper_value: 1.0 + binarize_gripper_lower_value: 0.35 + max_videoreaders_in_memory: 128 + shuffle_mode: RANDOM_ORDER + device: cpu + + +test: + _target_: dobbe.dataloaders.decord_dataset.DecordDataset + config: + _target_: dobbe.dataloaders.utils.DataLoaderConfig + dataset_root: ${env_vars.debug_data_root} + trajectory_roots: + _target_: dobbe.dataloaders.utils.create_trajectory_roots + trajectory_root_path: ${dataset.test.config.dataset_root} + original_root: ${env_vars.debug_data_original_root} + new_root: ${dataset.test.config.dataset_root} + include_tasks: + - ${include_task} + exclude_homes: [] + include_envs: + - ${include_env}_val + sequence_length: ${dataset.train.config.sequence_length} + control_timeskip: ${dataset.train.config.control_timeskip} + fps_subsample: ${dataset.train.config.fps_subsample} + relative_gripper: ${dataset.train.config.relative_gripper} + use_depth: ${use_depth} + binarize_gripper: ${dataset.train.config.binarize_gripper} + binarize_gripper_threshold: ${dataset.train.config.binarize_gripper_threshold} + binarize_gripper_upper_value: ${dataset.train.config.binarize_gripper_upper_value} + binarize_gripper_lower_value: ${dataset.train.config.binarize_gripper_lower_value} + max_videoreaders_in_memory: 128 + shuffle_mode: SEQUENTIAL + device: cpu diff --git a/imitation-in-homes/dobbe/configs/debug.yaml b/imitation-in-homes/dobbe/configs/debug.yaml new file mode 100644 index 0000000..9639f29 --- /dev/null +++ b/imitation-in-homes/dobbe/configs/debug.yaml @@ -0,0 +1,53 @@ +defaults: + - env_vars: env_vars + - dataset: debug + - model: image_bc_depth_timm + - loss_fn: bc_loss + - _self_ + +num_epochs: 50 +eval_every: 1 +device: cuda +batch_size: 256 +use_depth: true + +include_task: cactus +include_env: default_env + +sampler: null + +batch_sampler: + _target_: dobbe.dataloaders.decord_dataset.DecordBatchSampler + batch_size: ${batch_size} + +dataloader: + _target_: dobbe.dataloaders.decord_dataloader.DecordVideoLoader + device: cpu + num_workers: 8 + + +optimizer: + _target_: torch.optim.Adam + lr: 0.00003 + weight_decay: 1.5e-6 + +scheduler: + _target_: dobbe.models.ssl.moco.optimizer.MoCoScheduler + warmup_epochs: 10 + start_lr: 2e-6 + num_epochs: ${num_epochs} + + +model_path: checkpoints/${now:%Y-%m-%d}/${include_task}-${include_env}-moco-${now:%H-%M-%S} +resume_training: false +save_every: 10 + +wandb: + entity: ${env_vars.wandb.entity} + project: dobbe-${include_task} + name: ${include_task}-${include_env}-moco-4x4 + # id starts out null, hoawever, if you resume training, it will be set to the original id. + id: null + save_code: true + +seed: 42 diff --git a/imitation-in-homes/dobbe/configs/run.yaml b/imitation-in-homes/dobbe/configs/run.yaml index ea1e48b..d43be02 100644 --- a/imitation-in-homes/dobbe/configs/run.yaml +++ b/imitation-in-homes/dobbe/configs/run.yaml @@ -11,7 +11,7 @@ device: cpu include_task: Random include_env: Env1 -model_weight_pth: TODO_FINETUNED_MODEL_PATH +model_weight_pth: ${env_vars.project_root}/checkpoints/2024-05-07/cactus-default_env-movo-13-27-02/checkpoint_40.pt experiment: bc_with_hpr_encoder batch_size: 32 diff --git a/imitation-in-homes/dobbe/run.py b/imitation-in-homes/dobbe/run.py index fc6346d..44565ee 100644 --- a/imitation-in-homes/dobbe/run.py +++ b/imitation-in-homes/dobbe/run.py @@ -8,7 +8,7 @@ from torch.utils.data import DataLoader import wandb -from utils.trajectory_vis import visualize_trajectory +from dobbe.utils.trajectory_vis import visualize_trajectory class WrapperPolicy(nn.Module): From 947e1f4e23ca987113262201554aedd49216cd9e Mon Sep 17 00:00:00 2001 From: Chris Paxton Date: Tue, 7 May 2024 13:54:35 -0400 Subject: [PATCH 12/17] saving --- imitation-in-homes/dobbe/configs/model/image_bc_depth.yaml | 4 ++-- .../dobbe/configs/model/image_bc_depth_timm.yaml | 4 ++-- imitation-in-homes/dobbe/configs/run.yaml | 2 +- imitation-in-homes/dobbe/models/encoders/timm_encoders.py | 4 ++-- imitation-in-homes/dobbe/models/encoders/torch_encoders.py | 4 ++-- imitation-in-homes/dobbe/models/policies/bc.py | 4 ++-- 6 files changed, 11 insertions(+), 11 deletions(-) diff --git a/imitation-in-homes/dobbe/configs/model/image_bc_depth.yaml b/imitation-in-homes/dobbe/configs/model/image_bc_depth.yaml index 4ab1ac3..0462d1f 100644 --- a/imitation-in-homes/dobbe/configs/model/image_bc_depth.yaml +++ b/imitation-in-homes/dobbe/configs/model/image_bc_depth.yaml @@ -1,6 +1,6 @@ -_target_: models.policies.bc.BCDepthModel +_target_: dobbe.models.policies.bc.BCDepthModel encoder: - _target_: models.encoders.torch_encoders.TorchSSL + _target_: dobbe.models.encoders.torch_encoders.TorchSSL model_name: resnet34 override_aug_kwargs: hflip: 0.0 diff --git a/imitation-in-homes/dobbe/configs/model/image_bc_depth_timm.yaml b/imitation-in-homes/dobbe/configs/model/image_bc_depth_timm.yaml index f36d243..5697603 100644 --- a/imitation-in-homes/dobbe/configs/model/image_bc_depth_timm.yaml +++ b/imitation-in-homes/dobbe/configs/model/image_bc_depth_timm.yaml @@ -1,6 +1,6 @@ -_target_: models.policies.bc.BCDepthModel +_target_: dobbe.models.policies.bc.BCDepthModel encoder: - _target_: models.encoders.timm_encoders.TimmSSL + _target_: dobbe.models.encoders.timm_encoders.TimmSSL model_name: hf-hub:notmahi/dobb-e override_aug_kwargs: hflip: 0.0 diff --git a/imitation-in-homes/dobbe/configs/run.yaml b/imitation-in-homes/dobbe/configs/run.yaml index d43be02..9157355 100644 --- a/imitation-in-homes/dobbe/configs/run.yaml +++ b/imitation-in-homes/dobbe/configs/run.yaml @@ -1,6 +1,6 @@ defaults: - env_vars: env_vars - - model: image_bc_depth + - model: image_bc_depth_timm - dataset: finetune_task_data - _self_ diff --git a/imitation-in-homes/dobbe/models/encoders/timm_encoders.py b/imitation-in-homes/dobbe/models/encoders/timm_encoders.py index 160c0a5..e6a2916 100644 --- a/imitation-in-homes/dobbe/models/encoders/timm_encoders.py +++ b/imitation-in-homes/dobbe/models/encoders/timm_encoders.py @@ -7,8 +7,8 @@ import einops import timm -from models.encoders.abstract_base_encoder import AbstractEncoder -from utils.decord_transforms import create_transform +from dobbe.models.encoders.abstract_base_encoder import AbstractEncoder +from dobbe.utils.decord_transforms import create_transform class TimmModel(AbstractEncoder): diff --git a/imitation-in-homes/dobbe/models/encoders/torch_encoders.py b/imitation-in-homes/dobbe/models/encoders/torch_encoders.py index f9c1bbb..084c799 100644 --- a/imitation-in-homes/dobbe/models/encoders/torch_encoders.py +++ b/imitation-in-homes/dobbe/models/encoders/torch_encoders.py @@ -6,8 +6,8 @@ import torch.nn as nn import torchvision.models as models -from models.encoders.abstract_base_encoder import AbstractEncoder -from utils.decord_transforms import create_transform +from dobbe.models.encoders.abstract_base_encoder import AbstractEncoder +from dobbe.utils.decord_transforms import create_transform class TorchModel(AbstractEncoder): diff --git a/imitation-in-homes/dobbe/models/policies/bc.py b/imitation-in-homes/dobbe/models/policies/bc.py index f0335ea..d42d087 100644 --- a/imitation-in-homes/dobbe/models/policies/bc.py +++ b/imitation-in-homes/dobbe/models/policies/bc.py @@ -7,8 +7,8 @@ import torch.nn.functional as F from torchvision import transforms as T -from models.policies.depth_net import DepthBC -from models.policies.normalize_actions import NormalizeActions +from dobbe.models.policies.depth_net import DepthBC +from dobbe.models.policies.normalize_actions import NormalizeActions ENCODER_LOADING_ERROR_MSG = ( "Could not load encoder weights: trying to load as a BC model. " From 9d51a60e19c53d2fc0d2e47352299462aede4f90 Mon Sep 17 00:00:00 2001 From: Chris Paxton Date: Tue, 7 May 2024 13:59:05 -0400 Subject: [PATCH 13/17] various pathing fixes --- .../dobbe/configs/dataset/finetune_task_data.yaml | 12 ++++++------ imitation-in-homes/dobbe/configs/run.yaml | 4 ++-- .../dobbe/dataloaders/abstract_dataloader.py | 6 +++--- .../dobbe/dataloaders/decord_dataset.py | 4 ++-- imitation-in-homes/dobbe/dataloaders/depth_data.py | 2 +- imitation-in-homes/dobbe/dataloaders/pose_data.py | 2 +- imitation-in-homes/dobbe/run.py | 2 +- 7 files changed, 16 insertions(+), 16 deletions(-) diff --git a/imitation-in-homes/dobbe/configs/dataset/finetune_task_data.yaml b/imitation-in-homes/dobbe/configs/dataset/finetune_task_data.yaml index 3170735..9a09d60 100644 --- a/imitation-in-homes/dobbe/configs/dataset/finetune_task_data.yaml +++ b/imitation-in-homes/dobbe/configs/dataset/finetune_task_data.yaml @@ -1,10 +1,10 @@ train: - _target_: dataloaders.decord_dataset.DecordDataset + _target_: dobbe.dataloaders.decord_dataset.DecordDataset config: - _target_: dataloaders.utils.DataLoaderConfig + _target_: dobbe.dataloaders.utils.DataLoaderConfig dataset_root: ${env_vars.finetune_task_data_root} trajectory_roots: - _target_: dataloaders.utils.load_trajectory_roots + _target_: dobbe.dataloaders.utils.load_trajectory_roots trajectory_root_path: ${dataset.train.config.dataset_root}/r3d_files.txt original_root: ${env_vars.finetune_task_data_original_root} new_root: ${dataset.train.config.dataset_root} @@ -29,12 +29,12 @@ train: test: - _target_: dataloaders.decord_dataset.DecordDataset + _target_: dobbe.dataloaders.decord_dataset.DecordDataset config: - _target_: dataloaders.utils.DataLoaderConfig + _target_: dobbe.dataloaders.utils.DataLoaderConfig dataset_root: ${env_vars.finetune_task_data_root} trajectory_roots: - _target_: dataloaders.utils.load_trajectory_roots + _target_: dobbe.dataloaders.utils.load_trajectory_roots trajectory_root_path: ${dataset.test.config.dataset_root}/r3d_files.txt original_root: ${env_vars.finetune_task_data_original_root} new_root: ${dataset.test.config.dataset_root} diff --git a/imitation-in-homes/dobbe/configs/run.yaml b/imitation-in-homes/dobbe/configs/run.yaml index 9157355..a587fea 100644 --- a/imitation-in-homes/dobbe/configs/run.yaml +++ b/imitation-in-homes/dobbe/configs/run.yaml @@ -11,12 +11,12 @@ device: cpu include_task: Random include_env: Env1 -model_weight_pth: ${env_vars.project_root}/checkpoints/2024-05-07/cactus-default_env-movo-13-27-02/checkpoint_40.pt +model_weight_pth: ${env_vars.project_root}/checkpoints/2024-05-07/cactus-default_env-moco-13-27-02/checkpoint_40.pt experiment: bc_with_hpr_encoder batch_size: 32 sampler: null -run_offline: false +run_offline: true use_depth: true robot_params: diff --git a/imitation-in-homes/dobbe/dataloaders/abstract_dataloader.py b/imitation-in-homes/dobbe/dataloaders/abstract_dataloader.py index 0e0b77c..73283ce 100644 --- a/imitation-in-homes/dobbe/dataloaders/abstract_dataloader.py +++ b/imitation-in-homes/dobbe/dataloaders/abstract_dataloader.py @@ -4,9 +4,9 @@ from torch.utils.data import Dataset -from dataloaders.depth_data import DepthDataLoader -from dataloaders.pose_data import PoseDataLoader -from dataloaders.utils import DataLoaderConfig, calculate_start_end_indices +from dobbe.dataloaders.depth_data import DepthDataLoader +from dobbe.dataloaders.pose_data import PoseDataLoader +from dobbe.dataloaders.utils import DataLoaderConfig, calculate_start_end_indices class AbstractVideoDataset(ABC, Dataset): diff --git a/imitation-in-homes/dobbe/dataloaders/decord_dataset.py b/imitation-in-homes/dobbe/dataloaders/decord_dataset.py index 39487fb..eaceff6 100644 --- a/imitation-in-homes/dobbe/dataloaders/decord_dataset.py +++ b/imitation-in-homes/dobbe/dataloaders/decord_dataset.py @@ -20,8 +20,8 @@ import torch import tqdm -from dataloaders.abstract_dataloader import AbstractVideoDataset -from dataloaders.utils import DataLoaderConfig, TrajectorySlice, flatten_nested_lists +from dobbe.dataloaders.abstract_dataloader import AbstractVideoDataset +from dobbe.dataloaders.utils import DataLoaderConfig, TrajectorySlice, flatten_nested_lists # Create a dataclass for shuffle modes. diff --git a/imitation-in-homes/dobbe/dataloaders/depth_data.py b/imitation-in-homes/dobbe/dataloaders/depth_data.py index 0dc7dba..f262332 100644 --- a/imitation-in-homes/dobbe/dataloaders/depth_data.py +++ b/imitation-in-homes/dobbe/dataloaders/depth_data.py @@ -6,7 +6,7 @@ import numpy as np import torch -from dataloaders.utils import TrajectorySlice +from dobbe.dataloaders.utils import TrajectorySlice DEPTH_FOLDER = "compressed_depths" DEPTH_FILENAME = "compressed_np_depth_float32.bin" diff --git a/imitation-in-homes/dobbe/dataloaders/pose_data.py b/imitation-in-homes/dobbe/dataloaders/pose_data.py index 6bb7ab4..92ee394 100644 --- a/imitation-in-homes/dobbe/dataloaders/pose_data.py +++ b/imitation-in-homes/dobbe/dataloaders/pose_data.py @@ -5,7 +5,7 @@ import numpy as np from scipy.spatial.transform import Rotation as R -from dataloaders.utils import TrajectorySlice +from dobbe.dataloaders.utils import TrajectorySlice LABEL_FILENAME = "labels.json" PICKLE_FILENAME = "relative_poses.pkl" diff --git a/imitation-in-homes/dobbe/run.py b/imitation-in-homes/dobbe/run.py index 44565ee..d05554d 100644 --- a/imitation-in-homes/dobbe/run.py +++ b/imitation-in-homes/dobbe/run.py @@ -108,7 +108,7 @@ def run(cfg: OmegaConf, init_model=_init_model): else: # Lazy loading so we can run offline eval without the robot set up. - from robot.controller import Controller + from dobbe.robot.controller import Controller dict_cfg = _init_run(cfg) controller = Controller(cfg=dict_cfg) From 5585d4197bcc35328696b10df99586429dfa9767 Mon Sep 17 00:00:00 2001 From: Chris Paxton Date: Tue, 7 May 2024 14:37:08 -0400 Subject: [PATCH 14/17] make a bunch of changes to fix things --- imitation-in-homes/dobbe/configs/dataset/debug.yaml | 2 +- imitation-in-homes/dobbe/configs/loss_fn/bc_loss.yaml | 2 +- imitation-in-homes/dobbe/configs/run.yaml | 6 +++--- imitation-in-homes/dobbe/dataloaders/utils.py | 1 + imitation-in-homes/dobbe/loss_fns/bc_loss_fn.py | 2 +- imitation-in-homes/dobbe/loss_fns/ssl/mae_loss_fn.py | 8 ++++---- imitation-in-homes/dobbe/models/policies/bc.py | 2 ++ imitation-in-homes/dobbe/models/ssl/mae.py | 2 +- imitation-in-homes/dobbe/models/ssl/moco/builder.py | 4 ++-- imitation-in-homes/dobbe/utils/trajectory_vis.py | 4 ++++ 10 files changed, 20 insertions(+), 13 deletions(-) diff --git a/imitation-in-homes/dobbe/configs/dataset/debug.yaml b/imitation-in-homes/dobbe/configs/dataset/debug.yaml index ea35a29..5c0020d 100644 --- a/imitation-in-homes/dobbe/configs/dataset/debug.yaml +++ b/imitation-in-homes/dobbe/configs/dataset/debug.yaml @@ -42,7 +42,7 @@ test: - ${include_task} exclude_homes: [] include_envs: - - ${include_env}_val + - validation sequence_length: ${dataset.train.config.sequence_length} control_timeskip: ${dataset.train.config.control_timeskip} fps_subsample: ${dataset.train.config.fps_subsample} diff --git a/imitation-in-homes/dobbe/configs/loss_fn/bc_loss.yaml b/imitation-in-homes/dobbe/configs/loss_fn/bc_loss.yaml index cfd81f1..3b4c6f4 100644 --- a/imitation-in-homes/dobbe/configs/loss_fn/bc_loss.yaml +++ b/imitation-in-homes/dobbe/configs/loss_fn/bc_loss.yaml @@ -1 +1 @@ -_target_: loss_fns.bc_loss_fn.BCPolicyLossFn \ No newline at end of file +_target_: dobbe.loss_fns.bc_loss_fn.BCPolicyLossFn diff --git a/imitation-in-homes/dobbe/configs/run.yaml b/imitation-in-homes/dobbe/configs/run.yaml index a587fea..3e3af0b 100644 --- a/imitation-in-homes/dobbe/configs/run.yaml +++ b/imitation-in-homes/dobbe/configs/run.yaml @@ -1,15 +1,15 @@ defaults: - env_vars: env_vars - model: image_bc_depth_timm - - dataset: finetune_task_data + - dataset: debug - _self_ image_save_dir: ${env_vars.project_root}/robot_images image_buffer_size: 1 device: cpu -include_task: Random -include_env: Env1 +include_task: cactus +include_env: default_env model_weight_pth: ${env_vars.project_root}/checkpoints/2024-05-07/cactus-default_env-moco-13-27-02/checkpoint_40.pt diff --git a/imitation-in-homes/dobbe/dataloaders/utils.py b/imitation-in-homes/dobbe/dataloaders/utils.py index c89463c..eedc8ac 100644 --- a/imitation-in-homes/dobbe/dataloaders/utils.py +++ b/imitation-in-homes/dobbe/dataloaders/utils.py @@ -392,6 +392,7 @@ def _build_trajectories(self): continue self.trajectories.append(root) self.trajectory_lengths.append(len(poses)) + # Subsample trajectories. if self._do_subsample: self._subsample_trajectories() diff --git a/imitation-in-homes/dobbe/loss_fns/bc_loss_fn.py b/imitation-in-homes/dobbe/loss_fns/bc_loss_fn.py index 72e95fb..2d9889e 100644 --- a/imitation-in-homes/dobbe/loss_fns/bc_loss_fn.py +++ b/imitation-in-homes/dobbe/loss_fns/bc_loss_fn.py @@ -4,7 +4,7 @@ from einops import rearrange from torchvision.ops import MLP -from loss_fns.abstract_loss_fn import AbstractLossFn +from dobbe.loss_fns.abstract_loss_fn import AbstractLossFn class BCLossFn(AbstractLossFn): diff --git a/imitation-in-homes/dobbe/loss_fns/ssl/mae_loss_fn.py b/imitation-in-homes/dobbe/loss_fns/ssl/mae_loss_fn.py index c952072..5f57f4e 100644 --- a/imitation-in-homes/dobbe/loss_fns/ssl/mae_loss_fn.py +++ b/imitation-in-homes/dobbe/loss_fns/ssl/mae_loss_fn.py @@ -2,10 +2,10 @@ import torch -from loss_fns.abstract_loss_fn import AbstractLossFn -from models.ssl.mae import MAE -from utils import log_wandb_image -from utils.ssl_transfroms import IMAGENET_MEAN, IMAGENET_STD +from dobbe.loss_fns.abstract_loss_fn import AbstractLossFn +from dobbe.models.ssl.mae import MAE +from dobbe.utils import log_wandb_image +from dobbe.utils.ssl_transfroms import IMAGENET_MEAN, IMAGENET_STD class MAELossFn(AbstractLossFn): diff --git a/imitation-in-homes/dobbe/models/policies/bc.py b/imitation-in-homes/dobbe/models/policies/bc.py index d42d087..199c593 100644 --- a/imitation-in-homes/dobbe/models/policies/bc.py +++ b/imitation-in-homes/dobbe/models/policies/bc.py @@ -184,6 +184,8 @@ def step(self, input, *args, **kwargs): self.eval() normalized_image = self.img_transform(input[0].squeeze(1)) if self._use_depth: + if len(model_input) < 2: + raise RuntimeError('did not include all necessary inputs for depth model! Something is misconfigured.') norm_action, _ = self((normalized_image.unsqueeze(1), input[1], input[2])) else: norm_action, _ = self((normalized_image.unsqueeze(1), input[1])) diff --git a/imitation-in-homes/dobbe/models/ssl/mae.py b/imitation-in-homes/dobbe/models/ssl/mae.py index a93f848..1d5a523 100644 --- a/imitation-in-homes/dobbe/models/ssl/mae.py +++ b/imitation-in-homes/dobbe/models/ssl/mae.py @@ -15,7 +15,7 @@ ) from vit_pytorch.vit import Transformer -from utils.ssl_transfroms import IMAGENET_MEAN, IMAGENET_STD +from dobbe.utils.ssl_transfroms import IMAGENET_MEAN, IMAGENET_STD def mask_image_using_indices(image, indices, num_patches=64): diff --git a/imitation-in-homes/dobbe/models/ssl/moco/builder.py b/imitation-in-homes/dobbe/models/ssl/moco/builder.py index 26af19f..f7d8993 100644 --- a/imitation-in-homes/dobbe/models/ssl/moco/builder.py +++ b/imitation-in-homes/dobbe/models/ssl/moco/builder.py @@ -13,8 +13,8 @@ import torch.nn as nn from einops import rearrange -from models.ssl.moco import loader as moco_loader -from utils.ssl_transfroms import IMAGENET_MEAN, IMAGENET_STD, get_moco_transforms +from dobbe.models.ssl.moco import loader as moco_loader +from dobbe.utils.ssl_transfroms import IMAGENET_MEAN, IMAGENET_STD, get_moco_transforms class MoCo(nn.Module): diff --git a/imitation-in-homes/dobbe/utils/trajectory_vis.py b/imitation-in-homes/dobbe/utils/trajectory_vis.py index 17b3b3d..4787060 100644 --- a/imitation-in-homes/dobbe/utils/trajectory_vis.py +++ b/imitation-in-homes/dobbe/utils/trajectory_vis.py @@ -75,6 +75,9 @@ def visualize_trajectory( image_buffer = collections.deque(maxlen=buffer_size) test_dataset.set_include_trajectory_end(True) + print("Visualizing trajectories...") + print("# frames in trajectory =", len(test_dataset)) + i = 0 done_visualizing = 0 while (done_visualizing < n_visualized_trajectories) and (i < len(test_dataset)): @@ -89,6 +92,7 @@ def visualize_trajectory( torch.tensor(gt_actions).unsqueeze(0).to(device), ) + breakpoint() out, _ = model.step(model_input) action_preds.append(out.squeeze().cpu().detach().numpy()) From ea0ca499d4b915eda295e46e591e4c34a1a3d972 Mon Sep 17 00:00:00 2001 From: Chris Paxton Date: Tue, 7 May 2024 14:45:27 -0400 Subject: [PATCH 15/17] make sure models run fine on the depth data --- .../dobbe/models/policies/bc.py | 2 +- .../dobbe/utils/trajectory_vis.py | 31 ++++++++++++++----- 2 files changed, 25 insertions(+), 8 deletions(-) diff --git a/imitation-in-homes/dobbe/models/policies/bc.py b/imitation-in-homes/dobbe/models/policies/bc.py index 199c593..eb4c66c 100644 --- a/imitation-in-homes/dobbe/models/policies/bc.py +++ b/imitation-in-homes/dobbe/models/policies/bc.py @@ -184,7 +184,7 @@ def step(self, input, *args, **kwargs): self.eval() normalized_image = self.img_transform(input[0].squeeze(1)) if self._use_depth: - if len(model_input) < 2: + if len(input) < 2: raise RuntimeError('did not include all necessary inputs for depth model! Something is misconfigured.') norm_action, _ = self((normalized_image.unsqueeze(1), input[1], input[2])) else: diff --git a/imitation-in-homes/dobbe/utils/trajectory_vis.py b/imitation-in-homes/dobbe/utils/trajectory_vis.py index 4787060..10d2cd5 100644 --- a/imitation-in-homes/dobbe/utils/trajectory_vis.py +++ b/imitation-in-homes/dobbe/utils/trajectory_vis.py @@ -73,6 +73,7 @@ def visualize_trajectory( ground_truth = [] images = [] image_buffer = collections.deque(maxlen=buffer_size) + depth_buffer = collections.deque(maxlen=buffer_size) test_dataset.set_include_trajectory_end(True) print("Visualizing trajectories...") @@ -81,18 +82,34 @@ def visualize_trajectory( i = 0 done_visualizing = 0 while (done_visualizing < n_visualized_trajectories) and (i < len(test_dataset)): - (input_images, terminate), *_, gt_actions = test_dataset[i] + (input_images, terminate), input_depth, gt_actions = test_dataset[i] + + # Convert images input_images = input_images.float() / 255.0 image_buffer.append(input_images[-1]) img = input_images[-1] + + # For visualization images.append(einops.rearrange(img, "c h w -> h w c").cpu().detach().numpy()) + + # Add gt actions ground_truth.append(gt_actions[-1]) - model_input = ( - torch.stack(tuple(image_buffer), dim=0).unsqueeze(0).to(device), - torch.tensor(gt_actions).unsqueeze(0).to(device), - ) - breakpoint() + # Add depth + depth_buffer.append(input_depth[-1]) + + if model._use_depth: + model_input = ( + torch.stack(tuple(image_buffer), dim=0).unsqueeze(0).to(device), + torch.stack(tuple(depth_buffer), dim=0).unsqueeze(0).to(device), + torch.tensor(gt_actions).unsqueeze(0).to(device), + ) + else: + model_input = ( + torch.stack(tuple(image_buffer), dim=0).unsqueeze(0).to(device), + torch.tensor(gt_actions).unsqueeze(0).to(device), + ) + out, _ = model.step(model_input) action_preds.append(out.squeeze().cpu().detach().numpy()) @@ -101,7 +118,7 @@ def visualize_trajectory( ground_truth = np.array(ground_truth) images = np.array(images) - print(action_preds.shape, ground_truth.shape, images.shape) + print("shapes:", action_preds.shape, ground_truth.shape, images.shape) generate_plots( ground_truth, action_preds, images, traj_index=done_visualizing From da4a065a805ee828dd615523c7a4f6b5efda1edb Mon Sep 17 00:00:00 2001 From: Chris Paxton Date: Wed, 8 May 2024 12:28:27 -0400 Subject: [PATCH 16/17] update some things --- imitation-in-homes/dobbe/dataloaders/depth_data.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/imitation-in-homes/dobbe/dataloaders/depth_data.py b/imitation-in-homes/dobbe/dataloaders/depth_data.py index f262332..aaf7e89 100644 --- a/imitation-in-homes/dobbe/dataloaders/depth_data.py +++ b/imitation-in-homes/dobbe/dataloaders/depth_data.py @@ -13,6 +13,10 @@ class DepthDataLoader: + + max_depth = 5.0 + clamp_depth = True + def __init__( self, bin_root_path: Optional[Union[str, Path]] = None, @@ -60,6 +64,8 @@ def load_depth_zip(self, filepath: Union[str, Path]): depth_tensor = torch.from_numpy(depth_img) if self._binarize: depth_tensor = self.bin_pixels(depth_tensor) + print("-" * 80) + print(depth_tensor) return depth_tensor # 1,192,256 def set_depth_data_path(self, path: Path): @@ -85,6 +91,10 @@ def get_batch_bin(self, indices: Union[np.ndarray, TrajectorySlice]): depth_tensor = torch.from_numpy(self._depth_data[indices]) if self._binarize: depth_tensor = self.bin_pixels(depth_tensor) + if self.clamp_depth: + # Clamp to a reasonable max depth + depth_tensor.clamp_(0, self.max_depth) + # print(np.unique(depth_tensor.cpu().numpy())) return depth_tensor.unsqueeze_(dim=1) # B, 1, 192, 256 def get_batch(self, indices: Union[np.ndarray, TrajectorySlice]): From 64a0497a6cc12384c5f6c31c9c7dbaee9f611f53 Mon Sep 17 00:00:00 2001 From: Chris Paxton Date: Wed, 8 May 2024 12:49:48 -0400 Subject: [PATCH 17/17] some config fixes --- .../dobbe/configs/loss_fn/mae_loss.yaml | 2 +- .../dobbe/configs/loss_fn/moco_loss.yaml | 2 +- .../dobbe/configs/model/moco.yaml | 4 ++-- .../dobbe/configs/train_moco.yaml | 17 +++++++++++------ .../dobbe/loss_fns/ssl/byol_loss_fn.py | 4 ++-- .../dobbe/loss_fns/ssl/moco_loss_fn.py | 4 ++-- 6 files changed, 19 insertions(+), 14 deletions(-) diff --git a/imitation-in-homes/dobbe/configs/loss_fn/mae_loss.yaml b/imitation-in-homes/dobbe/configs/loss_fn/mae_loss.yaml index bef3642..c1ca931 100644 --- a/imitation-in-homes/dobbe/configs/loss_fn/mae_loss.yaml +++ b/imitation-in-homes/dobbe/configs/loss_fn/mae_loss.yaml @@ -1,4 +1,4 @@ -_target_: loss_fns.ssl.mae_loss_fn.MAELossFn +_target_: dobbe.loss_fns.ssl.mae_loss_fn.MAELossFn decoder_dim: 512 masking_ratio: 0.75 decoder_depth: 6 diff --git a/imitation-in-homes/dobbe/configs/loss_fn/moco_loss.yaml b/imitation-in-homes/dobbe/configs/loss_fn/moco_loss.yaml index 40e6260..4b8488d 100644 --- a/imitation-in-homes/dobbe/configs/loss_fn/moco_loss.yaml +++ b/imitation-in-homes/dobbe/configs/loss_fn/moco_loss.yaml @@ -1 +1 @@ -_target_: loss_fns.ssl.moco_loss_fn.MocoLossFn \ No newline at end of file +_target_: dobbe.loss_fns.ssl.moco_loss_fn.MocoLossFn diff --git a/imitation-in-homes/dobbe/configs/model/moco.yaml b/imitation-in-homes/dobbe/configs/model/moco.yaml index 086e57d..0b58f68 100644 --- a/imitation-in-homes/dobbe/configs/model/moco.yaml +++ b/imitation-in-homes/dobbe/configs/model/moco.yaml @@ -1,3 +1,3 @@ -_target_: models.encoders.timm_encoders.TimmSSLByolMoco +_target_: dobbe.models.encoders.timm_encoders.TimmSSLByolMoco model_name: resnet18 -pretrained: True \ No newline at end of file +pretrained: True diff --git a/imitation-in-homes/dobbe/configs/train_moco.yaml b/imitation-in-homes/dobbe/configs/train_moco.yaml index ce82de6..205d8f6 100644 --- a/imitation-in-homes/dobbe/configs/train_moco.yaml +++ b/imitation-in-homes/dobbe/configs/train_moco.yaml @@ -1,6 +1,6 @@ defaults: - env_vars: env_vars - - dataset: hony + - dataset: debug - model: moco - loss_fn: moco_loss - _self_ @@ -8,16 +8,21 @@ defaults: num_epochs: 60 eval_every: 5 device: cuda -batch_size: 800 +# batch_size: 800 +batch_size: 256 +use_depth: true + +include_task: cactus +include_env: default_env sampler: null batch_sampler: - _target_: dataloaders.decord_dataset.DecordBatchSampler + _target_: dobbe.dataloaders.decord_dataset.DecordBatchSampler batch_size: ${batch_size} dataloader: - _target_: dataloaders.decord_dataloader.DecordVideoLoader + _target_: dobbe.dataloaders.decord_dataloader.DecordVideoLoader device: cpu num_workers: 10 @@ -28,7 +33,7 @@ optimizer: weight_decay: 1.0e-6 scheduler: - _target_: models.ssl.moco.optimizer.MoCoScheduler + _target_: dobbe.models.ssl.moco.optimizer.MoCoScheduler warmup_epochs: 70 num_epochs: ${num_epochs} @@ -44,4 +49,4 @@ wandb: id: null save_code: true -seed: 42 \ No newline at end of file +seed: 42 diff --git a/imitation-in-homes/dobbe/loss_fns/ssl/byol_loss_fn.py b/imitation-in-homes/dobbe/loss_fns/ssl/byol_loss_fn.py index 0bb30b6..41e0fb4 100644 --- a/imitation-in-homes/dobbe/loss_fns/ssl/byol_loss_fn.py +++ b/imitation-in-homes/dobbe/loss_fns/ssl/byol_loss_fn.py @@ -4,8 +4,8 @@ from byol_pytorch import BYOL from einops import rearrange -from loss_fns.abstract_loss_fn import AbstractLossFn -from utils.ssl_transfroms import IMAGENET_MEAN, IMAGENET_STD, get_byol_transforms +from dobbe.loss_fns.abstract_loss_fn import AbstractLossFn +from dobbe.utils.ssl_transfroms import IMAGENET_MEAN, IMAGENET_STD, get_byol_transforms class ByolLossFn(AbstractLossFn): diff --git a/imitation-in-homes/dobbe/loss_fns/ssl/moco_loss_fn.py b/imitation-in-homes/dobbe/loss_fns/ssl/moco_loss_fn.py index 6ad4862..e361e06 100644 --- a/imitation-in-homes/dobbe/loss_fns/ssl/moco_loss_fn.py +++ b/imitation-in-homes/dobbe/loss_fns/ssl/moco_loss_fn.py @@ -1,7 +1,7 @@ import torch -from loss_fns.abstract_loss_fn import AbstractLossFn -from models.ssl.moco.builder import MoCo +from dobbe.loss_fns.abstract_loss_fn import AbstractLossFn +from dobbe.models.ssl.moco.builder import MoCo class MocoLossFn(AbstractLossFn):