[BugFix,Refactor] Dreamer refactor

examples/dreamer/config.yaml

@@ -1,39 +1,55 @@
-env_name: cheetah
-env_task: run
-env_library: dm_control
-catframes: 1
-async_collection: True
-record_video: 0
-frame_skip: 2
-batch_size: 50
-batch_length: 50
-total_frames: 5000000
-world_model_lr: 6e-4
-actor_value_lr: 8e-5
-from_pixels: True
-# we want 50 frames / traj in the replay buffer. Given the frame_skip=2 this makes each traj 100 steps long
-env_per_collector: 8
-num_workers: 8
-collector_device: cuda:1
-model_device: cuda:0
-frames_per_batch: 800
-optim_steps_per_batch: 80
-record_interval: 30
-max_frames_per_traj: 1000
-record_frames: 1000
-batch_transform: 1
-state_dim: 30
-rssm_hidden_dim: 200
-grad_clip: 100
-grayscale: False
-image_size : 64
-buffer_size: 20000
-init_env_steps: 1000
-init_random_frames: 5000
-logger: csv
-offline_logging: False
-project_name: torchrl_example_dreamer
-normalize_rewards_online: True
-normalize_rewards_online_scale: 5.0
-normalize_rewards_online_decay: 0.99999
-reward_scaling: 1.0
+env:
+  name: cheetah
+  task: run
+  seed: 0
+  backend: dm_control
+  frame_skip: 2
+  from_pixels: True
+  grayscale: False
+  image_size : 64
+
+collector:
+  total_frames: 5_000_000
+  init_random_frames: 1000
+  frames_per_batch: 1000
+  max_frames_per_traj: 1000
+  device: cpu
+
+optimization:
+  train_every: 1000
+  grad_clip: 100
+  batch_size: 50
+  batch_length: 50
+
+  world_model_lr: 6e-4
+  actor_lr: 8e-5
+  value_lr: 8e-5
+  kl_scale: 1.0
+  free_nats: 3.0
+  optim_steps_per_batch: 80
+  gamma: 0.99
+  lambda: 0.95
+  imagination_horizon: 15
+
+networks:
+  exploration_noise: 0.3
+  device: cuda:0
+  state_dim: 30
+  rssm_hidden_dim: 200
+  hidden_dim: 400
+  activation: "elu"
+
+
+replay_buffer: 
+  uint8_casting: True
+  buffer_size: 20000
+  batch_size: 50
+  scratch_dir: ${logger.exp_name}_${env.seed}
+
+logger:
+  backend: wandb
+  project: dreamer-v1
+  exp_name: ${env.name}-${env.task}-${env.seed}
+  mode: online
+  eval_iter: 1000
+  eval_rollout_steps: 1000