multiprocess

agi-brain · Nov 2, 2023 · b42bb20 · b42bb20
1 parent cd563a7
commit b42bb20
Show file tree

Hide file tree

Showing 52 changed files with 300 additions and 98 deletions.
diff --git a/xuance/common/memory_tools_marl.py b/xuance/common/memory_tools_marl.py
@@ -131,10 +131,6 @@ def clear_episodes(self):
             'obs': np.zeros((self.n_envs, self.n_agents, self.max_eps_len + 1) + self.obs_space, dtype=np.float32),
             'actions': np.zeros((self.n_envs, self.n_agents, self.max_eps_len) + self.act_space, dtype=np.float32),
             'rewards': np.zeros((self.n_envs, self.n_agents, self.max_eps_len) + self.rew_space, dtype=np.float32),
-            'returns': np.zeros((self.n_envs, self.n_agents, self.max_eps_len) + self.rew_space, np.float32),
-            'values': np.zeros((self.n_envs, self.n_agents, self.max_eps_len) + self.rew_space, np.float32),
-            'advantages': np.zeros((self.n_envs, self.n_agents, self.max_eps_len) + self.rew_space, np.float32),
-            'log_pi_old': np.zeros((self.n_envs, self.n_agents, self.max_eps_len,), np.float32),
             'terminals': np.zeros((self.n_envs, self.max_eps_len) + self.done_space, dtype=np.bool),
             'avail_actions': np.ones((self.n_envs, self.n_agents, self.max_eps_len + 1, self.dim_act), dtype=np.bool),
             'filled': np.zeros((self.n_envs, self.max_eps_len, 1), dtype=np.bool),
@@ -149,8 +145,6 @@ def store_transitions(self, t_envs, *transition_data):
         self.episode_data['obs'][:, :, t_envs] = obs_n
         self.episode_data['actions'][:, :, t_envs] = actions_dict['actions_n']
         self.episode_data['rewards'][:, :, t_envs] = rewards
-        self.episode_data['values'][:, :, t_envs] = actions_dict['values']
-        self.episode_data['log_pi_old'][:, :, t_envs] = actions_dict['log_pi']
         self.episode_data['terminals'][:, t_envs] = terminated
         self.episode_data['avail_actions'][:, :, t_envs] = avail_actions
         if self.state_space is not None:
@@ -159,7 +153,7 @@ def store_transitions(self, t_envs, *transition_data):
     def store_episodes(self):
         for i_env in range(self.n_envs):
             for k in self.keys:
-                self.data[k][self.ptr] = self.episode_data[k][i_env]
+                self.data[k][self.ptr] = self.episode_data[k][i_env].copy()
             self.ptr = (self.ptr + 1) % self.buffer_size
             self.size = np.min([self.size + 1, self.buffer_size])
         self.clear_episodes()

diff --git a/xuance/configs/dcg/sc2/1c3s5z.yaml b/xuance/configs/dcg/sc2/1c3s5z.yaml
@@ -4,7 +4,7 @@ env_id: "1c3s5z"
 fps: 15
 policy: "DCG_policy"
 representation: "Basic_RNN"
-vectorize: "Subproc_StarCraft2
+vectorize: "Subproc_StarCraft2"
 runner: "StarCraft2_Runner"
 on_policy: False
 

diff --git a/xuance/configs/dcg/sc2/25m.yaml b/xuance/configs/dcg/sc2/25m.yaml
@@ -4,7 +4,7 @@ env_id: "25m"
 fps: 15
 policy: "DCG_policy"
 representation: "Basic_RNN"
-vectorize: "Subproc_StarCraft2
+vectorize: "Subproc_StarCraft2"
 runner: "StarCraft2_Runner"
 on_policy: False
 

diff --git a/xuance/configs/dcg/sc2/2m_vs_1z.yaml b/xuance/configs/dcg/sc2/2m_vs_1z.yaml
@@ -4,7 +4,7 @@ env_id: "2m_vs_1z"
 fps: 15
 policy: "DCG_policy"
 representation: "Basic_RNN"
-vectorize: "Subproc_StarCraft2
+vectorize: "Subproc_StarCraft2"
 runner: "StarCraft2_Runner"
 on_policy: False
 

diff --git a/xuance/configs/dcg/sc2/2s3z.yaml b/xuance/configs/dcg/sc2/2s3z.yaml
@@ -4,7 +4,7 @@ env_id: "2s3z"
 fps: 15
 policy: "DCG_policy"
 representation: "Basic_RNN"
-vectorize: "Subproc_StarCraft2
+vectorize: "Subproc_StarCraft2"
 runner: "StarCraft2_Runner"
 on_policy: False
 

diff --git a/xuance/configs/dcg/sc2/3m.yaml b/xuance/configs/dcg/sc2/3m.yaml
@@ -4,7 +4,7 @@ env_id: "3m"
 fps: 15
 policy: "DCG_policy"
 representation: "Basic_RNN"
-vectorize: "Subproc_StarCraft2
+vectorize: "Subproc_StarCraft2"
 runner: "StarCraft2_Runner"
 on_policy: False
 

diff --git a/xuance/configs/dcg/sc2/5m_vs_6m.yaml b/xuance/configs/dcg/sc2/5m_vs_6m.yaml
@@ -4,7 +4,7 @@ env_id: "5m_vs_6m"
 fps: 15
 policy: "DCG_policy"
 representation: "Basic_RNN"
-vectorize: "Subproc_StarCraft2
+vectorize: "Subproc_StarCraft2"
 runner: "StarCraft2_Runner"
 on_policy: False
 

diff --git a/xuance/configs/dcg/sc2/8m.yaml b/xuance/configs/dcg/sc2/8m.yaml
@@ -4,7 +4,7 @@ env_id: "8m"
 fps: 15
 policy: "DCG_policy"
 representation: "Basic_RNN"
-vectorize: "Subproc_StarCraft2
+vectorize: "Subproc_StarCraft2"
 runner: "StarCraft2_Runner"
 on_policy: False
 

diff --git a/xuance/configs/dcg/sc2/8m_vs_9m.yaml b/xuance/configs/dcg/sc2/8m_vs_9m.yaml
@@ -4,7 +4,7 @@ env_id: "8m_vs_9m"
 fps: 15
 policy: "DCG_policy"
 representation: "Basic_RNN"
-vectorize: "Subproc_StarCraft2
+vectorize: "Subproc_StarCraft2"
 runner: "StarCraft2_Runner"
 on_policy: False
 

diff --git a/xuance/configs/dcg/sc2/MMM2.yaml b/xuance/configs/dcg/sc2/MMM2.yaml
@@ -4,7 +4,7 @@ env_id: "MMM2"
 fps: 15
 policy: "DCG_policy"
 representation: "Basic_RNN"
-vectorize: "Subproc_StarCraft2
+vectorize: "Subproc_StarCraft2"
 runner: "StarCraft2_Runner"
 on_policy: False
 

diff --git a/xuance/configs/dcg/sc2/corridor.yaml b/xuance/configs/dcg/sc2/corridor.yaml
@@ -4,7 +4,7 @@ env_id: "corridor"
 fps: 15
 policy: "DCG_policy"
 representation: "Basic_RNN"
-vectorize: "Subproc_StarCraft2
+vectorize: "Subproc_StarCraft2"
 runner: "StarCraft2_Runner"
 on_policy: False
 

diff --git a/xuance/configs/iql/sc2/1c3s5z.yaml b/xuance/configs/iql/sc2/1c3s5z.yaml
@@ -4,7 +4,7 @@ env_id: "1c3s5z"
 fps: 15
 policy: "Basic_Q_network_marl"
 representation: "Basic_RNN"
-vectorize: "Subproc_StarCraft2
+vectorize: "Subproc_StarCraft2"
 runner: "StarCraft2_Runner"
 on_policy: False
 

diff --git a/xuance/configs/iql/sc2/25m.yaml b/xuance/configs/iql/sc2/25m.yaml
@@ -4,7 +4,7 @@ env_id: "25m"
 fps: 15
 policy: "Basic_Q_network_marl"
 representation: "Basic_RNN"
-vectorize: "Subproc_StarCraft2
+vectorize: "Subproc_StarCraft2"
 runner: "StarCraft2_Runner"
 on_policy: False
 

diff --git a/xuance/configs/iql/sc2/2m_vs_1z.yaml b/xuance/configs/iql/sc2/2m_vs_1z.yaml
@@ -5,7 +5,7 @@ env_id: "2m_vs_1z"
 fps: 15
 policy: "Basic_Q_network_marl"
 representation: "Basic_RNN"
-vectorize: "Subproc_StarCraft2
+vectorize: "Subproc_StarCraft2"
 runner: "StarCraft2_Runner"
 on_policy: False
 

diff --git a/xuance/configs/iql/sc2/2s3z.yaml b/xuance/configs/iql/sc2/2s3z.yaml
@@ -4,7 +4,7 @@ env_id: "2s3z"
 fps: 15
 policy: "Basic_Q_network_marl"
 representation: "Basic_RNN"
-vectorize: "Subproc_StarCraft2
+vectorize: "Subproc_StarCraft2"
 runner: "StarCraft2_Runner"
 on_policy: False
 

diff --git a/xuance/configs/iql/sc2/3m.yaml b/xuance/configs/iql/sc2/3m.yaml
@@ -4,7 +4,7 @@ env_id: "3m"
 fps: 15
 policy: "Basic_Q_network_marl"
 representation: "Basic_RNN"
-vectorize: "Subproc_StarCraft2
+vectorize: "Subproc_StarCraft2"
 runner: "StarCraft2_Runner"
 on_policy: False
 

diff --git a/xuance/configs/iql/sc2/5m_vs_6m.yaml b/xuance/configs/iql/sc2/5m_vs_6m.yaml
@@ -4,7 +4,7 @@ env_id: "5m_vs_6m"
 fps: 15
 policy: "Basic_Q_network_marl"
 representation: "Basic_RNN"
-vectorize: "Subproc_StarCraft2
+vectorize: "Subproc_StarCraft2"
 runner: "StarCraft2_Runner"
 on_policy: False
 

diff --git a/xuance/configs/iql/sc2/8m.yaml b/xuance/configs/iql/sc2/8m.yaml
@@ -4,7 +4,7 @@ env_id: "8m"
 fps: 15
 policy: "Basic_Q_network_marl"
 representation: "Basic_RNN"
-vectorize: "Subproc_StarCraft2
+vectorize: "Subproc_StarCraft2"
 runner: "StarCraft2_Runner"
 on_policy: False
 

diff --git a/xuance/configs/iql/sc2/8m_vs_9m.yaml b/xuance/configs/iql/sc2/8m_vs_9m.yaml
@@ -4,7 +4,7 @@ env_id: "8m_vs_9m"
 fps: 15
 policy: "Basic_Q_network_marl"
 representation: "Basic_RNN"
-vectorize: "Subproc_StarCraft2
+vectorize: "Subproc_StarCraft2"
 runner: "StarCraft2_Runner"
 on_policy: False
 

diff --git a/xuance/configs/iql/sc2/MMM2.yaml b/xuance/configs/iql/sc2/MMM2.yaml
@@ -4,7 +4,7 @@ env_id: "MMM2"
 fps: 15
 policy: "Basic_Q_network_marl"
 representation: "Basic_RNN"
-vectorize: "Subproc_StarCraft2
+vectorize: "Subproc_StarCraft2"
 runner: "StarCraft2_Runner"
 on_policy: False
 

diff --git a/xuance/configs/iql/sc2/corridor.yaml b/xuance/configs/iql/sc2/corridor.yaml
@@ -4,7 +4,7 @@ env_id: "corridor"
 fps: 15
 policy: "Basic_Q_network_marl"
 representation: "Basic_RNN"
-vectorize: "Subproc_StarCraft2
+vectorize: "Subproc_StarCraft2"
 runner: "StarCraft2_Runner"
 on_policy: False
 

diff --git a/xuance/configs/qmix/sc2/1c3s5z.yaml b/xuance/configs/qmix/sc2/1c3s5z.yaml
@@ -5,7 +5,7 @@ env_id: "1c3s5z"
 fps: 15
 policy: "Mixing_Q_network"
 representation: "Basic_RNN"
-vectorize: "Subproc_StarCraft2
+vectorize: "Subproc_StarCraft2"
 runner: "StarCraft2_Runner"
 on_policy: False
 

diff --git a/xuance/configs/qmix/sc2/25m.yaml b/xuance/configs/qmix/sc2/25m.yaml
@@ -5,7 +5,7 @@ env_id: "25m"
 fps: 15
 policy: "Mixing_Q_network"
 representation: "Basic_RNN"
-vectorize: "Subproc_StarCraft2
+vectorize: "Subproc_StarCraft2"
 runner: "StarCraft2_Runner"
 on_policy: False
 

diff --git a/xuance/configs/qmix/sc2/2m_vs_1z.yaml b/xuance/configs/qmix/sc2/2m_vs_1z.yaml
@@ -5,7 +5,7 @@ env_id: "2m_vs_1z"
 fps: 15
 policy: "Mixing_Q_network"
 representation: "Basic_RNN"
-vectorize: "Subproc_StarCraft2
+vectorize: "Subproc_StarCraft2"
 runner: "StarCraft2_Runner"
 on_policy: False
 

diff --git a/xuance/configs/qmix/sc2/2s3z.yaml b/xuance/configs/qmix/sc2/2s3z.yaml
@@ -5,7 +5,7 @@ env_id: "2s3z"
 fps: 15
 policy: "Mixing_Q_network"
 representation: "Basic_RNN"
-vectorize: "Subproc_StarCraft2
+vectorize: "Subproc_StarCraft2"
 runner: "StarCraft2_Runner"
 on_policy: False
 

diff --git a/xuance/configs/qmix/sc2/5m_vs_6m.yaml b/xuance/configs/qmix/sc2/5m_vs_6m.yaml
@@ -5,7 +5,7 @@ env_id: "5m_vs_6m"
 fps: 15
 policy: "Mixing_Q_network"
 representation: "Basic_RNN"
-vectorize: "Subproc_StarCraft2
+vectorize: "Subproc_StarCraft2"
 runner: "StarCraft2_Runner"
 on_policy: False
 

diff --git a/xuance/configs/qmix/sc2/8m.yaml b/xuance/configs/qmix/sc2/8m.yaml
@@ -5,7 +5,7 @@ env_id: "8m"
 fps: 15
 policy: "Mixing_Q_network"
 representation: "Basic_RNN"
-vectorize: "Subproc_StarCraft2
+vectorize: "Subproc_StarCraft2"
 runner: "StarCraft2_Runner"
 on_policy: False
 

diff --git a/xuance/configs/qmix/sc2/8m_vs_9m.yaml b/xuance/configs/qmix/sc2/8m_vs_9m.yaml
@@ -5,7 +5,7 @@ env_id: "8m_vs_9m"
 fps: 15
 policy: "Mixing_Q_network"
 representation: "Basic_RNN"
-vectorize: "Subproc_StarCraft2
+vectorize: "Subproc_StarCraft2"
 runner: "StarCraft2_Runner"
 on_policy: False
 

diff --git a/xuance/configs/qmix/sc2/MMM2.yaml b/xuance/configs/qmix/sc2/MMM2.yaml
@@ -5,7 +5,7 @@ env_id: "MMM2"
 fps: 15
 policy: "Mixing_Q_network"
 representation: "Basic_RNN"
-vectorize: "Subproc_StarCraft2
+vectorize: "Subproc_StarCraft2"
 runner: "StarCraft2_Runner"
 on_policy: False
 

diff --git a/xuance/configs/qmix/sc2/corridor.yaml b/xuance/configs/qmix/sc2/corridor.yaml
@@ -5,7 +5,7 @@ env_id: "corridor"
 fps: 15
 policy: "Mixing_Q_network"
 representation: "Basic_RNN"
-vectorize: "Subproc_StarCraft2
+vectorize: "Subproc_StarCraft2"
 runner: "StarCraft2_Runner"
 on_policy: False
 

diff --git a/xuance/configs/vdn/sc2/1c3s5z.yaml b/xuance/configs/vdn/sc2/1c3s5z.yaml
@@ -5,7 +5,7 @@ env_id: "1c3s5z"
 fps: 15
 policy: "Mixing_Q_network"
 representation: "Basic_RNN"
-vectorize: "Subproc_StarCraft2
+vectorize: "Subproc_StarCraft2"
 runner: "StarCraft2_Runner"
 on_policy: False
 

diff --git a/xuance/configs/vdn/sc2/25m.yaml b/xuance/configs/vdn/sc2/25m.yaml
@@ -5,7 +5,7 @@ env_id: "25m"
 fps: 15
 policy: "Mixing_Q_network"
 representation: "Basic_RNN"
-vectorize: "Subproc_StarCraft2
+vectorize: "Subproc_StarCraft2"
 runner: "StarCraft2_Runner"
 on_policy: False
 

diff --git a/xuance/configs/vdn/sc2/2m_vs_1z.yaml b/xuance/configs/vdn/sc2/2m_vs_1z.yaml
@@ -5,7 +5,7 @@ env_id: "2m_vs_1z"
 fps: 15
 policy: "Mixing_Q_network"
 representation: "Basic_RNN"
-vectorize: "Subproc_StarCraft2
+vectorize: "Subproc_StarCraft2"
 runner: "StarCraft2_Runner"
 on_policy: False
 

diff --git a/xuance/configs/vdn/sc2/2s3z.yaml b/xuance/configs/vdn/sc2/2s3z.yaml
@@ -5,7 +5,7 @@ env_id: "2s3z"
 fps: 15
 policy: "Mixing_Q_network"
 representation: "Basic_RNN"
-vectorize: "Subproc_StarCraft2
+vectorize: "Subproc_StarCraft2"
 runner: "StarCraft2_Runner"
 on_policy: False
 

diff --git a/xuance/configs/vdn/sc2/3m.yaml b/xuance/configs/vdn/sc2/3m.yaml
@@ -5,7 +5,7 @@ env_id: "3m"
 fps: 15
 policy: "Mixing_Q_network"
 representation: "Basic_RNN"
-vectorize: "Subproc_StarCraft2
+vectorize: "Subproc_StarCraft2"
 runner: "StarCraft2_Runner"
 on_policy: False
 

diff --git a/xuance/configs/vdn/sc2/5m_vs_6m.yaml b/xuance/configs/vdn/sc2/5m_vs_6m.yaml
@@ -5,7 +5,7 @@ env_id: "5m_vs_6m"
 fps: 15
 policy: "Mixing_Q_network"
 representation: "Basic_RNN"
-vectorize: "Subproc_StarCraft2
+vectorize: "Subproc_StarCraft2"
 runner: "StarCraft2_Runner"
 on_policy: False
 

diff --git a/xuance/configs/vdn/sc2/8m.yaml b/xuance/configs/vdn/sc2/8m.yaml
@@ -5,7 +5,7 @@ env_id: "8m"
 fps: 15
 policy: "Mixing_Q_network"
 representation: "Basic_RNN"
-vectorize: "Subproc_StarCraft2
+vectorize: "Subproc_StarCraft2"
 runner: "StarCraft2_Runner"
 on_policy: False
 

diff --git a/xuance/configs/vdn/sc2/8m_vs_9m.yaml b/xuance/configs/vdn/sc2/8m_vs_9m.yaml
@@ -5,7 +5,7 @@ env_id: "8m_vs_9m"
 fps: 15
 policy: "Mixing_Q_network"
 representation: "Basic_RNN"
-vectorize: "Subproc_StarCraft2
+vectorize: "Subproc_StarCraft2"
 runner: "StarCraft2_Runner"
 on_policy: False
 

diff --git a/xuance/configs/vdn/sc2/MMM2.yaml b/xuance/configs/vdn/sc2/MMM2.yaml
@@ -5,7 +5,7 @@ env_id: "MMM2"
 fps: 15
 policy: "Mixing_Q_network"
 representation: "Basic_RNN"
-vectorize: "Subproc_StarCraft2
+vectorize: "Subproc_StarCraft2"
 runner: "StarCraft2_Runner"
 on_policy: False
 

diff --git a/xuance/configs/vdn/sc2/corridor.yaml b/xuance/configs/vdn/sc2/corridor.yaml
@@ -5,7 +5,7 @@ env_id: "corridor"
 fps: 15
 policy: "Mixing_Q_network"
 representation: "Basic_RNN"
-vectorize: "Subproc_StarCraft2
+vectorize: "Subproc_StarCraft2"
 runner: "StarCraft2_Runner"
 on_policy: False
 

diff --git a/xuance/configs/wqmix/sc2/1c3s5z.yaml b/xuance/configs/wqmix/sc2/1c3s5z.yaml
@@ -4,7 +4,7 @@ env_id: "1c3s5z"
 fps: 15
 policy: "Weighted_Mixing_Q_network"
 representation: "Basic_RNN"
-vectorize: "Subproc_StarCraft2
+vectorize: "Subproc_StarCraft2"
 runner: "StarCraft2_Runner"
 on_policy: False
 

diff --git a/xuance/configs/wqmix/sc2/25m.yaml b/xuance/configs/wqmix/sc2/25m.yaml
@@ -4,7 +4,7 @@ env_id: "25m"
 fps: 15
 policy: "Weighted_Mixing_Q_network"
 representation: "Basic_RNN"
-vectorize: "Subproc_StarCraft2
+vectorize: "Subproc_StarCraft2"
 runner: "StarCraft2_Runner"
 on_policy: False