16.A3C算法实现（连续）.ipynb

{"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"name":"16.A3C算法实现（连续）","provenance":[],"collapsed_sections":[],"authorship_tag":"ABX9TyMazhOQGtpaw0w/0xLuu/Fe"},"kernelspec":{"name":"python3","display_name":"Python 3"},"language_info":{"name":"python"}},"cells":[{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"Fiinj39UR4Kt","executionInfo":{"status":"ok","timestamp":1617967730739,"user_tz":-480,"elapsed":164566,"user":{"displayName":"白梦伟","photoUrl":"","userId":"14318251260292756104"}},"outputId":"792a2b70-9bd5-42ad-ea60-95fb6aeee44f"},"source":["import torch\n","import gym\n","import numpy as np\n","import os\n","\n","import multiprocessing as mp\n","from multiprocessing import Manager\n","from multiprocessing import Pool\n","\n","torch.set_default_dtype(torch.float64)\n","\n","class Stack_queue:\n","    def __init__(self):\n","        self.list = []\n","    \n","    def get(self):\n","        o = self.list[-1]\n","        del self.list[-1]\n","        return o\n","    \n","    def put(self, o):\n","        self.list.append(o)\n","    \n","    def qsize(self):\n","        return len(self.list)\n","\n","class memory:\n","    def __init__(self, env, gamma=0.9):\n","        self.env = env\n","        self.gamma = 0.9\n","\n","        self.state_dimension = self.env.observation_space.shape[0]\n","        try:self.action_dimension = self.env.action_space.shape[0] \n","        except:self.action_dimension = 1\n","        self.data = Stack_queue()\n","        \n","    def put(self, state, action, reward, next_state):\n","        self.data.put(np.hstack((state, action, reward, next_state)))\n","        \n","    def get(self):\n","        return self.data.get()\n","\n","    def preprocess(self):\n","        length_real = self.data.qsize()\n","        self.state = np.zeros((length_real, self.state_dimension))\n","        self.action = np.zeros((length_real, self.action_dimension), dtype=np.float32)\n","        self.reward = np.zeros((length_real, 1))\n","        self.next_state = np.zeros((length_real, self.state_dimension))\n","        value_last = 0\n","        for i in range(length_real)[::-1]:\n","            transaction = self.data.get()\n","            state = transaction[:self.state_dimension]\n","            action = transaction[self.state_dimension:\n","                                 self.state_dimension+self.action_dimension]\n","            value = transaction[self.state_dimension+self.action_dimension:\n","                                                     self.state_dimension+self.action_dimension+1]\n","            next_state = transaction[-self.state_dimension:]\n","\n","            value_last = value + self.gamma * value_last\n","            self.state[i] = state\n","            self.reward[i] = value_last\n","            self.action[i] = action\n","            self.next_state[i] = next_state\n","\n","class actor:\n","    def __init__(self, env, hidden_dimension=200, learning_rate=1e-3, delta=-0.2):\n","        self.env = env\n","        self.hidden_dimension = hidden_dimension\n","        self.learning_rate = learning_rate\n","        self.delta = delta\n","\n","        self.state_dimension = self.env.observation_space.shape[0]\n","        try:self.action_dimension = self.env.action_space.shape[0]\n","        except:self.action_dimension = self.env.action_space.n\n","\n","        self.model = self.__create_network()\n","\n","        self.optimizer = torch.optim.Adam(self.model.parameters(), lr=self.learning_rate)\n","        self.entropy = torch.nn.CrossEntropyLoss(reduction='none')\n","\n","    def learn(self, state, action, advantage):\n","        state = torch.from_numpy(state)\n","        action = torch.tensor(action)\n","        advantage = torch.tensor(advantage)\n","\n","        mean, std = self.model(state)\n","        distribution = torch.distributions.Normal(mean, std)\n","        action_probablity_log = distribution.log_prob(action)\n","        loss = -action_probablity_log * advantage\n","    \n","        # entropy = torch.sum(-torch.log(action_prob) * action_prob, axis=1)\n","        # loss += self.delta * (-entropy)\n","\n","        loss = torch.mean(loss)\n","        self.optimizer.zero_grad()\n","        loss.backward()\n","        # self.optimizer.step()\n","        return loss.item(), self.model.parameters()\n","\n","    def output_action(self, state):\n","        state = torch.from_numpy(state)\n","        mean, std = self.model(state)\n","        distribution = torch.distributions.normal.Normal(mean, std)\n","        action = distribution.sample()\n","        action = np.array(torch.clip(action, -2, 2))\n","        return action\n","        \n","    class actor_net(torch.nn.Module):\n","        def __init__(self, dim_state, dim_action, dim_hidden):\n","            super().__init__()\n","            self.linear1 = torch.nn.Linear(dim_state, dim_hidden)\n","            self.activate1 = torch.nn.ReLU6()\n","            self.linear2 = torch.nn.Linear(dim_hidden, dim_action)\n","            self.mean = torch.nn.Tanh()\n","            self.linear3 = torch.nn.Linear(dim_hidden, dim_action)\n","            self.std = torch.nn.Softplus()\n","            # essential \n","            for layer in [self.linear1, self.linear2, self.linear3]:\n","                torch.nn.init.normal_(layer.weight, mean=0.0, std=0.1)\n","                torch.nn.init.constant_(layer.bias, 0.0)\n","        \n","        def forward(self, state):\n","            hidden = self.activate1(self.linear1(state))\n","            mean_value = self.linear2(hidden)\n","            mean = self.mean(mean_value) * 2\n","            std_value = self.linear3(hidden)\n","            std = self.std(std_value)\n","            return mean, std\n","        \n","    def __create_network(self):\n","        return self.actor_net(self.state_dimension,\n","                        self.action_dimension,\n","                        self.hidden_dimension)\n","\n","\n","class critic:\n","    def __init__(self, env, hidden_dimension=200, learning_rate=1e-3, gamma=0.9):\n","        self.env = env\n","        self.hidden_dimension = hidden_dimension\n","        self.learning_rate = learning_rate\n","        self.gamma = gamma\n","\n","        self.state_dimension = self.env.observation_space.shape[0]\n","        try:self.action_dimension = self.env.action_space.shape[0]\n","        except:self.action_dimension = 1\n","\n","        self.model = self.__create_network()\n","        self.optimizer = torch.optim.Adam(self.model.parameters(), lr=self.learning_rate)\n","\n","    def learn(self, state, reward, next_state):\n","        state = torch.from_numpy(state)\n","        reward = torch.tensor(reward)\n","        next_state = torch.from_numpy(next_state)\n","\n","        # value_next = self.model(next_state).detach()\n","        value = self.model(state)\n","        advantage = reward - value\n","\n","        loss = torch.square(advantage)\n","        loss = torch.mean(loss)\n","\n","        self.optimizer.zero_grad()\n","        loss.backward()\n","        # self.optimizer.step()\n","        return advantage.tolist(), self.model.parameters()\n","        \n","    def __create_network(self):\n","        return torch.nn.Sequential(torch.nn.Linear(self.state_dimension, self.hidden_dimension),\n","                                                 torch.nn.ReLU6(),\n","                                                 torch.nn.Linear(self.hidden_dimension,\n","                                                                  self.action_dimension)\n","                                                 )\n","\n","class agent_actor_critic(mp.Process):\n","    def __init__(self, env_name, epoch, q_in, q_out):\n","        mp.Process.__init__(self)\n","        self.env_name = env_name\n","        self.epoch = epoch\n","        self.q_in = q_in\n","        self.q_out = q_out\n","        \n","        self.env = gym.make(self.env_name)\n","        self.critic_network = critic(self.env)\n","        self.actor_network = actor(self.env)\n","        self.memory = memory(self.env)\n","        \n","        self.id = os.getpid()\n","        self.display = 30\n","\n","    def run(self):\n","        self.learn()\n","\n","    def learn(self):\n","        print('task start, pid: %d'%self.id)\n","        reward_collect = []\n","        \n","        for i in range(self.epoch):\n","            self.reset_param()\n","            state = self.env.reset()\n","            reward_total = 0\n","\n","            while True:\n","                action = self.actor_network.output_action(state)\n","                next_state, reward, done, info = self.env.step(action)\n","                \n","                reward_total += reward\n","                # essential for mdp, reward: -1~1\n","                reward = reward/8 + 1\n","                \n","                if reward_total < -800:\n","                    done = True\n","                    # reward = -1\n","                \n","                self.memory.put(state, action ,reward, next_state)\n","                state = next_state\n","                \n","                if done:\n","                    self.memory.preprocess()\n","                    #self.memory.next_state is NOT used here\n","                    advantage, critic_param = self.critic_network.learn(self.memory.state,\n","                                                                        self.memory.reward,\n","                                                                        self.memory.next_state)\n","                    \n","                    loss, actor_param = self.actor_network.learn(self.memory.state,\n","                                                                 self.memory.action, advantage)\n","                    self.update_param()\n","                    break\n","            # reward_collect.append(reward_total)\n","            # if (i+1) % 30 == 0:\n","                # print('epoch %5d, reward is %.5f'%(i, np.array(reward_collect).mean()))\n","                # reward_collect = []\n","\n","    \n","    def reset_param(self):\n","        critic_net, actor_net = self.q_in.get()\n","        self.critic_network.model.load_state_dict(\n","                critic_net.state_dict())\n","        self.actor_network.model.load_state_dict(\n","                actor_net.state_dict())\n","\n","    def update_param(self):\n","        critic_grad = []\n","        actor_grad = []\n","        for each in self.critic_network.model.parameters():\n","            critic_grad.append(each.grad)\n","        for each in self.actor_network.model.parameters():\n","            actor_grad.append(each.grad)\n","        self.q_out.put((critic_grad,\n","                        actor_grad))\n","\n","class agent_master(mp.Process):\n","    def __init__(self, env_name, epoch, work_cnt, q_in, q_out):\n","        mp.Process.__init__(self)\n","        self.env = gym.make(env_name)\n","        self.epoch = epoch\n","        self.work_cnt = work_cnt\n","        self.q_in = q_in\n","        self.q_out = q_out\n","        \n","        self.critic_network = critic(self.env)\n","        self.actor_network = actor(self.env)\n","    \n","    def run(self):\n","        print('master start, pid: %d'%os.getpid())\n","\n","        for i in range(self.epoch):\n","            for _ in range(self.work_cnt):\n","                self.put()\n","            \n","            # for _ in range(self.work_cnt):\n","                # self.get()\n","            # self.critic_network.optimizer.step()\n","            # self.actor_network.optimizer.step()\n","            #\n","            for _ in range(self.work_cnt):\n","                self.get_rand_grad()\n","\n","            if (i+1) % 30 == 0:\n","                self.perform_test(i)\n","  \n","    def perform_test(self, i):\n","        reward_total = 0\n","        state = self.env.reset()\n","        while True:\n","            # self.env.render()\n","            action = self.actor_network.output_action(state)\n","            next_state, reward, done, info = self.env.step(action)\n","            state = next_state\n","            reward_total += reward\n","            if done :\n","                break\n","        print('%d, performance : reward is %.6f'%(i, reward_total))\n","\n","    def put(self):\n","        self.q_in.put((self.critic_network.model,\n","                       self.actor_network.model))\n","\n","    def get(self):\n","        critic_grad, actor_grad = self.q_out.get()\n","        \n","        self.critic_network.optimizer.zero_grad()\n","        self.actor_network.optimizer.zero_grad()\n","\n","        for lo_grad, glo_model in zip(critic_grad, \n","                                      self.critic_network.model.parameters()):\n","            if glo_model._grad is None:\n","                glo_model._grad = lo_grad\n","            else:\n","                glo_model._grad += lo_grad\n","        for lo_grad, glo_model in zip(actor_grad, \n","                                      self.actor_network.model.parameters()):\n","            if glo_model._grad is None:\n","                glo_model._grad = lo_grad\n","            else:\n","                glo_model._grad += lo_grad\n","                \n","    def get_rand_grad(self):\n","        critic_grad, actor_grad = self.q_out.get()\n","        \n","        self.critic_network.optimizer.zero_grad()\n","        self.actor_network.optimizer.zero_grad()\n","\n","        for lo_grad, glo_model in zip(critic_grad, \n","                                      self.critic_network.model.parameters()):\n","            if glo_model._grad is None:\n","                glo_model._grad = lo_grad\n","            else:\n","                glo_model._grad += lo_grad\n","        for lo_grad, glo_model in zip(actor_grad, \n","                                      self.actor_network.model.parameters()):\n","            if glo_model._grad is None:\n","                glo_model._grad = lo_grad\n","            else:\n","                glo_model._grad += lo_grad\n","        self.critic_network.optimizer.step()\n","        self.actor_network.optimizer.step()\n","\n","\n","class algorithm_a3c:\n","    #Pendulum-v0\n","    #BipedalWalker-v3\n","    def __init__(self, env_name='Pendulum-v0', worker_cnt=4, epoch=1000):\n","        self.env_name = env_name\n","        self.worker_cnt = worker_cnt\n","        self.epoch = epoch\n","        \n","    def start_task(self):\n","        slaver = agent_actor_critic(self.env_name, self.epoch,\n","                                    self.q_in, self.q_out)\n","        slaver.run()\n","    \n","    def start_master(self):\n","        master = agent_master(self.env_name, self.epoch,\n","                              self.worker_cnt,\n","                              self.q_in, self.q_out)\n","        master.run()\n","\n","    def start_execute(self):\n","        self.q_in = Manager().Queue()\n","        self.q_out = Manager().Queue()\n","        pool = Pool(self.worker_cnt+1)\n","        for i in range(self.worker_cnt):\n","            pool.apply_async(self.start_task,\n","                             args=( ),\n","                             )\n","        pool.apply_async(self.start_master,\n","                         args=( ),\n","                         )\n","        pool.close()\n","        pool.join()\n","        print('this is the end of the program.')\n","\n","if __name__ == '__main__':\n","        torch.set_default_dtype(torch.float64)\n","        \n","        algorithm_test = algorithm_a3c()\n","        algorithm_test.start_execute()\n","\n","\n"],"execution_count":2,"outputs":[{"output_type":"stream","text":["task start, pid: 558\n","master start, pid: 567\n","task start, pid: 561\n","task start, pid: 564\n","task start, pid: 555\n","29, performance : reward is -1258.907978\n","59, performance : reward is -1430.007360\n","89, performance : reward is -1296.537709\n","119, performance : reward is -1271.910916\n","149, performance : reward is -1354.917787\n","179, performance : reward is -1164.562282\n","209, performance : reward is -637.655306\n","239, performance : reward is -774.797724\n","269, performance : reward is -981.036776\n","299, performance : reward is -519.476278\n","329, performance : reward is -524.067668\n","359, performance : reward is -646.630057\n","389, performance : reward is -1.772456\n","419, performance : reward is -1120.670156\n","449, performance : reward is -133.174627\n","479, performance : reward is -632.995571\n","509, performance : reward is -269.226796\n","539, performance : reward is -553.798997\n","569, performance : reward is -1142.327053\n","599, performance : reward is -1334.586567\n","629, performance : reward is -1254.408174\n","659, performance : reward is -1461.320112\n","689, performance : reward is -1456.516588\n","719, performance : reward is -1354.368779\n","749, performance : reward is -141.652859\n","779, performance : reward is -1528.043162\n","809, performance : reward is -1405.849616\n","839, performance : reward is -5.683831\n","869, performance : reward is -9.478524\n","899, performance : reward is -1259.140256\n","929, performance : reward is -1.852834\n","959, performance : reward is -1387.292451\n","989, performance : reward is -244.725813\n","this is the end of the program.\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"76W8qK6LR9vx","executionInfo":{"status":"aborted","timestamp":1617967121908,"user_tz":-480,"elapsed":23,"user":{"displayName":"白梦伟","photoUrl":"","userId":"14318251260292756104"}}},"source":[""],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"Gs0f7BE9R9tB","executionInfo":{"status":"aborted","timestamp":1617967121909,"user_tz":-480,"elapsed":20,"user":{"displayName":"白梦伟","photoUrl":"","userId":"14318251260292756104"}}},"source":[""],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"GLX-SDoMR9qa","executionInfo":{"status":"aborted","timestamp":1617967121910,"user_tz":-480,"elapsed":17,"user":{"displayName":"白梦伟","photoUrl":"","userId":"14318251260292756104"}}},"source":[""],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"Wbyd_II5R9nz","executionInfo":{"status":"aborted","timestamp":1617967121910,"user_tz":-480,"elapsed":13,"user":{"displayName":"白梦伟","photoUrl":"","userId":"14318251260292756104"}}},"source":[""],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"KQ6R0TOAR9ky","executionInfo":{"status":"aborted","timestamp":1617967121911,"user_tz":-480,"elapsed":10,"user":{"displayName":"白梦伟","photoUrl":"","userId":"14318251260292756104"}}},"source":[""],"execution_count":null,"outputs":[]}]}