q_and_sarsa_for_Taxi_problem_with_pso.ipynb

{"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"name":"q_and_sarsa_for_Taxi_problem_with_pso","provenance":[],"collapsed_sections":[],"toc_visible":true,"authorship_tag":"ABX9TyMiLk9YvaAAcNP8VKYZ0PDh"},"kernelspec":{"name":"python3","display_name":"Python 3"}},"cells":[{"cell_type":"markdown","metadata":{"id":"O6Bju80kh_Bf"},"source":["# 导入模块"]},{"cell_type":"code","metadata":{"id":"si6_YTGTh3Wa"},"source":["import gym\n","import random\n","from tqdm import tqdm\n","import numpy as np\n","import matplotlib.pyplot as plt"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"8gxgggTAjLqt"},"source":["# Q-learning"]},{"cell_type":"markdown","metadata":{"id":"9jijJjX6iLti"},"source":["## Q-learning 算法"]},{"cell_type":"code","metadata":{"id":"DL-vReyPiVGl"},"source":["class agent_q_learning():\n","  def __init__(self, env, q_param):\n","    self.env = env\n","    if q_param is None:\n","      self.alpha = 0.5 \n","      self.gamma = 0.9\n","    else:\n","      self.alpha = q_param['alpha']\n","      self.gamma =  q_param['gamma']\n","    self.epsilon = 0.1\n","\n","    self.q={}\n","    self.__init_q_table()\n","    pass\n","\n","  def output_action(self, state, is_greedy = True):\n","    if random.uniform(0,1) < self.epsilon and is_greedy:\n","      return self.env.action_space.sample()\n","    else:\n","      return max(list(range(self.env.action_space.n)) ,\n","                 key=lambda x: self.q[(state,x)])\n","    pass\n","\n","  def __init_q_table(self):\n","    for s in range(self.env.observation_space.n):\n","      for a in range(self.env.action_space.n):\n","        self.q[(s,a)]=0.0\n","\n","  def update_q_table(self, state, action, next_state, next_maxaction, reward):\n","    self.q[(state,action)] += self.alpha * \\\n","                            (reward + \\\n","                             self.gamma * self.q[(next_state,next_maxaction)] - \\\n","                             self.q[(state,action)] \\\n","                             )\n"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"556X48DBid1E"},"source":[""],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"4TvBu8XsiaHd"},"source":["## Q-learning 执行和评估类"]},{"cell_type":"code","metadata":{"id":"QmGYf4ZHh3Rn"},"source":["class evaluate_alg_q():\n","  def __init__(self, env, cnt=50, episodes=2000, q_param=None):\n","    #cnt 用作多次求平均值\n","    self.env = env\n","    self.cnt = cnt\n","    self.episodes = episodes\n","    self.q_param = q_param\n","\n","    self.reward_array = np.zeros(episodes)\n","    \n","    pass\n","\n","  def start_execute(self):\n","    for _ in tqdm(range(self.cnt)):\n","      self.agent = agent_q_learning(self.env, self.q_param)\n","      for i in range(self.episodes):\n","        r = 0\n","        state = self.env.reset()\n","        while True:\n","          action = self.agent.output_action(state)\n","\n","          next_state, reward, done , _ = self.env.step(action)\n","\n","          next_maxaction = self.agent.output_action(state, is_greedy=False)\n","\n","          self.agent.update_q_table(state, action, next_state, next_maxaction, reward)\n","\n","          state = next_state\n","          r += reward\n","\n","          if done:\n","              break\n","        self.reward_array[i] += r\n","      self.reward_array[i] /= self.cnt\n","\n","  def evaluate_plot_reward(self):\n","    plt.plot(self.reward_array,'-')\n","  \n","  def evaluate_plot_reward_dev(self, dev_num=10,style='-'):\n","    size_row = int(len(self.reward_array)/dev_num)\n","    reward_array_dev = self.reward_array.reshape((size_row, dev_num)).mean(axis = 1)\n","    envaluate_plt = plt.plot(reward_array_dev, style)\n","    return envaluate_plt"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"mCaDQ2-2h3T4"},"source":[""],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"0i0pfblymSMI"},"source":["## Q-learning 创建实例和运行"]},{"cell_type":"code","metadata":{"id":"kb-07vw6mR_0"},"source":["env=gym.make('Taxi-v3')\n","algorithm_q = evaluate_alg_q(env, cnt=5, episodes=2000)\n","algorithm_q.start_execute()\n","# algorithm_q.evaluate_plot_reward()\n","env.close()"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"5JMU9RTkmPpt"},"source":["algorithm_q.evaluate_plot_reward_dev(dev_num=20)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"M3JyCFF3eLkp"},"source":[""],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"rOxH2Pv4i6V8"},"source":["# Sarsa"]},{"cell_type":"markdown","metadata":{"id":"vdvPcTcGi-Wd"},"source":["## Sarsa 算法"]},{"cell_type":"code","metadata":{"id":"LJGUuTzfj9xO"},"source":["class agent_sarsa():\n","  def __init__(self, env, sarsa_param):\n","    self.env = env\n","    if sarsa_param is None:\n","      self.alpha = 0.5 \n","      self.gamma = 0.9\n","    else:\n","      self.alpha = sarsa_param['alpha']\n","      self.gamma =  sarsa_param['gamma']\n","\n","    self.epsilon = 0.1\n","    self.q = {}\n","    self.__q_table_init()\n","    pass\n","\n","  def output_action(self, state):\n","    random_value = np.random.random()\n","    if random_value < self.epsilon:\n","      return env.action_space.sample()\n","    else:\n","      action_list = list(range(self.env.action_space.n))\n","      return max(action_list, key = lambda x: self.q[(state, x)])\n","    pass\n","\n","  def update_q_table(self,state, action, next_state, next_action, reward):\n","    self.q[(state,action)] += self.alpha*(reward+self.gamma*self.q[(next_state,next_action)]-self.q[(state,action)])\n","\n","  def __q_table_init(self):\n","    for s in range(self.env.observation_space.n):\n","      for a in range(self.env.action_space.n):\n","        self.q[(s,a)] = 0.0\n"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"3rj4Nf1WjblO"},"source":["## Sarsa 执行和评估类"]},{"cell_type":"code","metadata":{"id":"lsLbLR7HmJCY"},"source":["class evaluate_alg_sarsa():\n","  def __init__(self, env, cnt=50, episodes=2000, sarsa_param=None):\n","    #cnt 用作多次求平均值\n","    self.env = env\n","    self.cnt = cnt\n","    self.episodes = episodes\n","    self.sarsa_param = sarsa_param\n","\n","    self.reward_array = np.zeros(episodes)\n","    pass\n","\n","  def start_execute(self):\n","    for _ in tqdm(range(self.cnt)):\n","      self.agent = agent_sarsa(self.env, self.sarsa_param)\n","      for i in range(self.episodes):\n","        r = 0\n","        state = self.env.reset()\n","        action = self.agent.output_action(state)\n","        while True:\n","          next_state, reward, done , _ = self.env.step(action)\n","          next_action = self.agent.output_action(state)\n","\n","          self.agent.update_q_table(state, action, next_state, next_action, reward)\n","\n","          state = next_state\n","          action = next_action\n","          r += reward\n","          if done:\n","            break\n","        self.reward_array[i] += r\n","      self.reward_array[i] /= self.cnt\n","\n","  def evaluate_plot_reward(self):\n","    plt.plot(self.reward_array)\n","  \n","  def evaluate_plot_reward_dev(self, dev_num=10, style='-.'):\n","    size_row = int(len(self.reward_array)/dev_num)\n","    reward_array_dev = self.reward_array.reshape((size_row, dev_num)).mean(axis = 1)\n","    envaluate_plt = plt.plot(reward_array_dev,style)\n","    return envaluate_plt"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"piuX1Yp_ji0u"},"source":["## Sarsa创建实例和运行"]},{"cell_type":"code","metadata":{"id":"ONOODMo6m5vV"},"source":["env=gym.make('Taxi-v3')\n","algorithm_sarsa = evaluate_alg_sarsa(env, cnt=5, episodes=2000)\n","algorithm_sarsa.start_execute()\n","# algorithm_sarsa.evaluate_plot_reward()\n","env.close()"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"rKlLE_CHuBE2"},"source":["algorithm_sarsa.evaluate_plot_reward_dev(20)"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"aOWHBdVgu406"},"source":["# 期望Sarsa"]},{"cell_type":"markdown","metadata":{"id":"qFGYzlznvAmY"},"source":["## 期望Sarsa算法"]},{"cell_type":"code","metadata":{"id":"Rtiy91Tau82Y"},"source":["class agent_sarsa_exp():\n","  def __init__(self, env, sarsa_param):\n","    self.env = env\n","    if sarsa_param is None:\n","      self.alpha = 0.5 \n","      self.gamma = 0.9\n","    else:\n","      self.alpha = sarsa_param['alpha']\n","      self.gamma =  sarsa_param['gamma']\n","\n","    self.epsilon = 0.1\n","    self.q = {}\n","    self.__q_table_init()\n","    pass\n","\n","  def output_action(self, state, isgreedy=True):\n","    random_value = np.random.random()\n","    if random_value < self.epsilon and isgreedy==True:\n","      return env.action_space.sample()\n","    else:\n","      action_list = list(range(self.env.action_space.n))\n","      return max(action_list, key = lambda x: self.q[(state, x)])\n","    pass\n","\n","  def update_q_table(self,state, action, next_state, next_maxaction, reward):\n","    target_value = []\n","    for each_action in range(self.env.action_space.n):\n","      target_value.append(1/self.env.action_space.n * self.epsilon \\\n","                          * self.q[(next_state, each_action)]) \n","      if each_action == next_maxaction:\n","        target_value.append((1 - self.epsilon) * self.q[(next_state, each_action)])\n","\n","    self.q[(state,action)] += self.alpha*(reward+self.gamma*(np.array(target_value)).sum()-self.q[(state,action)])\n","\n","  def __q_table_init(self):\n","    for s in range(self.env.observation_space.n):\n","      for a in range(self.env.action_space.n):\n","        self.q[(s,a)] = 0.0\n"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"ud4VtYGQu-aR"},"source":[""],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"OyGA91DDvEba"},"source":["## 期望Sarsa执行和评估类"]},{"cell_type":"code","metadata":{"id":"MsqBiYA32nhp"},"source":["class evaluate_alg_sarsa_exp():\n","  def __init__(self, env, cnt=50, episodes=2000, sarsa_exp_param=None):\n","    #cnt 用作多次求平均值\n","    self.env = env\n","    self.cnt = cnt\n","    self.episodes = episodes\n","    self.sarsa_exp_param = sarsa_exp_param\n","\n","    self.reward_array = np.zeros(episodes)\n","    pass\n","\n","  def start_execute(self):\n","    for _ in tqdm(range(self.cnt)):\n","      self.agent = agent_sarsa_exp(self.env, self.sarsa_exp_param)\n","      for i in range(self.episodes):\n","        r = 0\n","        state = self.env.reset()\n","        action = self.agent.output_action(state)\n","        while True:\n","          next_state, reward, done , _ = self.env.step(action)\n","          next_action = self.agent.output_action(state)\n","          next_maxaction = self.agent.output_action(state, isgreedy=False)\n","\n","          self.agent.update_q_table(state, action, next_state, next_maxaction, reward)\n","\n","          state = next_state\n","          action = next_action\n","          r += reward\n","          if done:\n","            break\n","        self.reward_array[i] += r\n","      self.reward_array[i] /= self.cnt\n","\n","  def evaluate_plot_reward(self):\n","    plt.plot(self.reward_array,'--')\n","  \n","  def evaluate_plot_reward_dev(self, dev_num=10, style='--'):\n","    size_row = int(len(self.reward_array)/dev_num)\n","    reward_array_dev = self.reward_array.reshape((size_row, dev_num)).mean(axis = 1)\n","    envaluate_plt = plt.plot(reward_array_dev, style)\n","    return envaluate_plt"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"27A7Nc8xu-Xl"},"source":[""],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"-OEXqpLhvL5R"},"source":["## 期望Sarsa创建实例和运行"]},{"cell_type":"code","metadata":{"id":"8stdlxwrmcjy"},"source":["env=gym.make('Taxi-v3')\n","algorithm_sarsa_exp = evaluate_alg_sarsa_exp(env, cnt=5, episodes=2000)\n","algorithm_sarsa_exp.start_execute()\n","# algorithm_sarsa.evaluate_plot_reward()\n","env.close()\n","algorithm_sarsa_exp.evaluate_plot_reward_dev(20)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"rFwL5HqCmgvc"},"source":[""],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"Ky8AeckEmufY"},"source":["# 双Q学习"]},{"cell_type":"markdown","metadata":{"id":"nziazv6Bm5dm"},"source":["## 双Q学习算法"]},{"cell_type":"code","metadata":{"id":"h61AX4IXncjw"},"source":["class agent_double_q_learning():\n","  def __init__(self, env, q_param):\n","    self.env = env\n","    if q_param is None:\n","      self.alpha = 0.5 \n","      self.gamma = 0.9\n","    else:\n","      self.alpha = q_param['alpha']\n","      self.gamma =  q_param['gamma']\n","    self.epsilon = 0.1\n","\n","    self.q={}\n","    self.__init_q_table()\n","    pass\n","\n","  def output_action(self, state, q1, q2=None, is_greedy = True):\n","    if random.uniform(0,1) < self.epsilon and is_greedy:\n","      return self.env.action_space.sample()\n","    elif q2 is not None:\n","      return max(list(range(self.env.action_space.n)) ,\n","                 key=lambda x: q1[(state,x)]+q2[(state,x)] )\n","    else:\n","      return max(list(range(self.env.action_space.n)) ,\n","                 key=lambda x: q1[(state,x)] )\n","    pass\n","\n","  def __init_q_table(self):\n","    for s in range(self.env.observation_space.n):\n","      for a in range(self.env.action_space.n):\n","        self.q[(s,a)] = 0.0\n","\n","  def update_q_table(self, state, action, next_state, next_maxaction, reward, q):\n","    self.q[(state,action)] += self.alpha * \\\n","                            (reward + \\\n","                             self.gamma * q[(next_state,next_maxaction)] - \\\n","                             self.q[(state,action)] \\\n","                             )\n"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"AkVjwQjWpp7Q"},"source":["## 双Q学习执行和评估类"]},{"cell_type":"code","metadata":{"id":"fB4BGaDencF5"},"source":["class evaluate_alg_double_q():\n","  def __init__(self, env, cnt=50, episodes=2000, q_param=None):\n","    #cnt 用作多次求平均值\n","    self.env = env\n","    self.cnt = cnt\n","    self.episodes = episodes\n","    self.q_param = q_param\n","    self.reward_array = np.zeros(episodes)\n","    pass\n","\n","  def start_execute(self):\n","    for _ in tqdm(range(self.cnt)):\n","      self.agent_q1 = agent_double_q_learning(self.env, self.q_param)\n","      self.agent_q2 = agent_double_q_learning(self.env, self.q_param)\n","      for i in range(self.episodes):\n","        r = 0\n","        state = self.env.reset()\n","        while True:\n","          action = self.agent_q1.output_action(state, self.agent_q1.q,\\\n","                                            self.agent_q2.q)\n","          next_state, reward, done , _ = self.env.step(action)\n","          if np.random.random()< 0.5:\n","            next_maxaction = self.agent_q1.output_action(state,\\\n","                                                         self.agent_q1.q,\\\n","                                                         is_greedy=False)\n","            self.agent_q1.update_q_table(state, action, next_state,\\\n","                                         next_maxaction, reward,\\\n","                                         self.agent_q2.q)\n","          else:\n","            next_maxaction = self.agent_q2.output_action(state,\\\n","                                              self.agent_q2.q,\\\n","                                              is_greedy=False)\n","            self.agent_q2.update_q_table(state, action, next_state,\\\n","                                         next_maxaction, reward,\\\n","                                         self.agent_q1.q)\n","          state = next_state\n","          r += reward\n","          if done:\n","              break\n","        self.reward_array[i] += r\n","      self.reward_array[i] /= self.cnt\n","\n","  def evaluate_plot_reward(self):\n","    plt.plot(self.reward_array,':')\n","  \n","  def evaluate_plot_reward_dev(self, dev_num=10,style=':'):\n","    size_row = int(len(self.reward_array)/dev_num)\n","    reward_array_dev = self.reward_array.reshape((size_row, dev_num)).mean(axis = 1)\n","    envaluate_plt = plt.plot(reward_array_dev, style)\n","    return envaluate_plt"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"1NLWJ84vncDE"},"source":[""],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"xowfHYiqwWPn"},"source":["## 创建实例和运行"]},{"cell_type":"code","metadata":{"id":"4o4YcncxwZVi"},"source":["env=gym.make('Taxi-v3')\n","algorithm_double_q = evaluate_alg_double_q(env, cnt=5, episodes=2000)\n","algorithm_double_q.start_execute()\n","# algorithm_double_q.evaluate_plot_reward()\n","env.close()"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"IEu0sHzKwbR-"},"source":["algorithm_double_q.evaluate_plot_reward_dev(dev_num=20)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"HzNmQdv2wbPB"},"source":[""],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"80FXwYR3mfxE"},"source":["# 算法联合对比测试"]},{"cell_type":"code","metadata":{"id":"1cA0vVcHNF9G"},"source":["algorithm_sarsa_exp.evaluate_plot_reward()\n","algorithm_sarsa.evaluate_plot_reward()\n","algorithm_q.evaluate_plot_reward()\n","algorithm_double_q.evaluate_plot_reward()\n","plt.legend(['exp','sarsa','q', 'double_q'])\n","plt.xlabel('episode')\n","plt.ylabel('average reward')"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"aWO0T_iPu-PX"},"source":["algorithm_q.evaluate_plot_reward_dev(dev_num=20)\n","algorithm_sarsa.evaluate_plot_reward_dev(20)\n","algorithm_double_q.evaluate_plot_reward_dev(dev_num=20)\n","algorithm_sarsa_exp.evaluate_plot_reward_dev(20)\n","plt.legend(['q learning', 'sarsa','double q learning','expect sarsa'])\n","plt.xlabel('episode')\n","plt.ylabel('average reward')"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"bCMyjszvjmR9"},"source":["# 工具和测试代码"]},{"cell_type":"markdown","metadata":{"id":"dQl87QrF8cmQ"},"source":["## 效果对比"]},{"cell_type":"code","metadata":{"id":"MIDpyly_jlzN"},"source":["algorithm_q.evaluate_plot_reward_dev(20)\n","algorithm_sarsa.evaluate_plot_reward_dev(20)\n","plt.legend(['q-learning', 'sarsa'])"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"Rr3hVFCYR9p9"},"source":["for i in range(5):\n","  print(algorithm_q.reward_array[i*400:(i+1)*400].mean())"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"EvUBdW5wCGXF"},"source":["for i in range(5):\n","  print(algorithm_sarsa.reward_array[i*400:(i+1)*400].mean())"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"802_Lv1ySMEi"},"source":["for i in range(5):\n","  print(algorithm_double_q.reward_array[i*400:(i+1)*400].mean())"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"GYUSGmN-SMBU"},"source":["for i in range(5):\n","  print(algorithm_sarsa_exp.reward_array[i*400:(i+1)*400].mean())"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"fP3avlyNSL3M"},"source":[""],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"hODtRJeBCb-M"},"source":["algorithm_sarsa.reward_array[:-500].mean()"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"56FUk1EK858A"},"source":["## Sarsa粒子群算法，超参数选取"]},{"cell_type":"markdown","metadata":{"id":"z0zo6TCwEGJB"},"source":["网址：https://zh.wikipedia.org/wiki/%E7%B2%92%E5%AD%90%E7%BE%A4%E4%BC%98%E5%8C%96"]},{"cell_type":"code","metadata":{"id":"gTcBeFB9h3M3"},"source":["class pso_evaluate():\n","  def __init__(self, pso_cnt, episode):\n","    #第0个存储全局信息\n","    self.pso_cnt = pso_cnt+1\n","    self.episode = episode\n","    self.pso_param_head = ['alpha', 'gamma']\n","    self.pso_param_limit = [(0.2, 1), (0.6, 1.2)]\n","\n","    self.pso_agent = {}\n","    self.__pso_init_param()\n","\n","  def pso_execute(self):\n","    for _ in range(self.episode):\n","      self.__pso_fitness_call1()\n","      self.__pso_fitness_call2()\n","      self.__pso_update_param()\n","\n","  def pso_print_ans(self):\n","    print('ans is :',self.pso_agent[(0,'optim_param')])\n","    print('check it!')\n","    print('all msg:',self.pso_agent)\n","\n","  def __pso_update_param(self):\n","    for i in range(self.pso_cnt-1):\n","      pso_v = np.array(list( self.pso_agent[(i+1, 'pso_v')].values() ))\n","      pso_optim_param = np.array(list( self.pso_agent[i+1, 'optim_param'].values() ))\n","      pso_now_param = np.array(list( self.pso_agent[i+1, 'now_param'].values() ))\n","      pso_best_param = np.array(list( self.pso_agent[0, 'optim_param'].values() ))\n","\n","      pso_v = 0.5*pso_v + \\\n","              2*np.random.random()*(pso_best_param - pso_now_param) + \\\n","              2*np.random.random()*(pso_optim_param - pso_now_param)\n","      pso_now_param += pso_v\n","\n","      self.pso_agent[(i+1, 'pso_v')] = { k:v for k,v in zip(self.pso_param_head, pso_v.tolist()) }\n","      self.pso_agent[(i+1,'now_param')] = { k:v for k,v in zip(self.pso_param_head, pso_now_param.tolist()) }\n","      self.__pso_modify_param(i+1)\n","\n","  def __pso_modify_param(self, pso_agent_index):\n","    for i, title in enumerate(self.pso_param_head):\n","      if self.pso_agent[(pso_agent_index,'now_param')][title] < self.pso_param_limit[i][0]:\n","        self.pso_agent[(pso_agent_index,'now_param')][title] = self.pso_param_limit[i][0]\n","      if self.pso_agent[(pso_agent_index,'now_param')][title] > self.pso_param_limit[i][1]:\n","        self.pso_agent[(pso_agent_index,'now_param')][title] = self.pso_param_limit[i][1]\n","\n","  def __pso_fitness_call1(self):\n","    fit_value_array = []\n","    for i in range(self.pso_cnt-1):\n","      fit_value = self.__fitness_evaluate(i+1)\n","      fit_value_array.append(fit_value)\n","      self.pso_agent[(i+1,'fit_value')]['fit_now'] = fit_value\n","      if fit_value > self.pso_agent[(i+1,'fit_value')]['fit_best']:\n","        self.pso_agent[(i+1,'fit_value')]['fit_best'] = fit_value\n","        self.pso_agent[(i+1, \"optim_param\")] = self.pso_agent[(i+1, 'now_param')]\n","    print(fit_value_array)\n","        \n","  def __fitness_evaluate(self,pso_agent_index):\n","    env=gym.make('Taxi-v3')\n","    sarsa_param = self.pso_agent[(pso_agent_index, 'now_param')]\n","    algorithm_sarsa = evaluate_alg_sarsa(env, cnt=1, episodes=2000, sarsa_param=sarsa_param)\n","    algorithm_sarsa.start_execute()\n","    env.close()\n","    fit_value = algorithm_sarsa.reward_array[:-500].mean()\n","    return fit_value\n","  \n","  def __pso_fitness_call2(self):\n","    for i in range(self.pso_cnt-1):\n","      if self.pso_agent[(i+1,'fit_value')]['fit_best'] > self.pso_agent[(0,'fit_value')]['fit_best']:\n","        self.pso_agent[(0,'fit_value')]['fit_best'] = self.pso_agent[(i+1,'fit_value')]['fit_best']\n","        self.pso_agent[(0, \"optim_param\")] = self.pso_agent[(i+1, 'now_param')]\n","\n","  def __pso_init_param(self):\n","    for i in range(self.pso_cnt):\n","      self.pso_agent[(i,'optim_param')] = { k:v for k,v in zip(self.pso_param_head, np.zeros(len(self.pso_param_head)).tolist()) }\n","      self.pso_agent[(i,'now_param')] = { k:np.random.uniform(v[0],v[1]) \\\n","                                         for k,v in zip(self.pso_param_head, self.pso_param_limit) }\n","      self.pso_agent[(i,'pso_v')] = { k:np.random.random()*(v[1]-v[0])*0.5\n","                                         for k,v in zip(self.pso_param_head, self.pso_param_limit) }\n","      self.pso_agent[(i,'fit_value')] = {'fit_best':float('-inf'), 'fit_now':float('-inf')}\n"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"QCmc9kg5NP2A"},"source":["## Sarsa粒子群算法执行"]},{"cell_type":"code","metadata":{"id":"Dz_CNDPsE7XD"},"source":["pst_find  = pso_evaluate(pso_cnt=5, episode=20)\n","pst_find.pso_execute()\n","pst_find.pso_print_ans()"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"puXDwPyxE7H5"},"source":["pst_find.pso_print_ans()"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"DMqsPtIh7U_J"},"source":["pst_find.pso_print_ans()"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"B6SKlprB7Vmv"},"source":["## Sarsa超参数效果测试"]},{"cell_type":"code","metadata":{"id":"Yg85k8mYw74q"},"source":["env=gym.make('Taxi-v3')\n","sarsa_param = {'alpha': 0.5317706294307902, 'gamma': 0.99545498514414}\n","algorithm_sarsa1 = evaluate_alg_sarsa(env, cnt=5, episodes=2000, sarsa_param=sarsa_param)\n","algorithm_sarsa1.start_execute()\n","env.close()"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"1Ef-bP7lw72a"},"source":["algorithm_sarsa.evaluate_plot_reward_dev(20)\n","algorithm_sarsa1.evaluate_plot_reward_dev(20,style='x-.')\n","plt.xlabel('episode')\n","plt.ylabel('average reward')\n","plt.legend(['sarsa', 'pso sarsa'], loc='lower right')"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"wMlQ2YSU-rF8"},"source":["## Q-learning粒子群算法参数选取"]},{"cell_type":"code","metadata":{"id":"ZJ4N8BXc8ku9"},"source":["class pso_evaluate_q(pso_evaluate):\n","  def __fitness_evaluate(self,pso_agent_index):\n","    env=gym.make('Taxi-v3')\n","    q_param = self.pso_agent[(pso_agent_index, 'now_param')]\n","    algorithm_q = evaluate_alg_q(env, cnt=1, episodes=2000, q_param=q_param)\n","    algorithm_q.start_execute()\n","    env.close()\n","    fit_value = algorithm_q.reward_array[:-500].mean()\n","    return fit_value\n"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"1QpY8FS6w70D"},"source":["pst_find = pso_evaluate_q(pso_cnt=5, episode=20)\n","pst_find.pso_execute()"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"5dRnOhNsqfV9"},"source":["pst_find.pso_print_ans()"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"81sl8_wo76gX"},"source":["pst_find.pso_print_ans()\\"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"YKQujFIpCbBs"},"source":["## Q-learning超参数效果测试"]},{"cell_type":"code","metadata":{"id":"1b_wZEfY1D3N"},"source":["env=gym.make('Taxi-v3')\n","q_param = {'alpha': 0.5374978944586886, 'gamma': 1.0119420190638535}\n","algorithm_q1 = evaluate_alg_q(env, cnt=5, episodes=2000, q_param=q_param)\n","algorithm_q1.start_execute()\n","env.close()"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"xjq8UkPT1Dy9"},"source":["algorithm_q.evaluate_plot_reward_dev(20)\n","algorithm_q1.evaluate_plot_reward_dev(20,style='x-')\n","plt.legend(['q learning', 'pso q learning'])\n","plt.xlabel('episode')\n","plt.ylabel('average reward')"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"FLSRJJy6b-XY"},"source":["## 期望Sarsa粒子群算法参数选取"]},{"cell_type":"code","metadata":{"id":"FVLvydhY1Dv0"},"source":["class pso_evaluate_sarsa_exp():\n","  def __init__(self, pso_cnt, episode):\n","    #第0个存储全局信息\n","    self.pso_cnt = pso_cnt+1\n","    self.episode = episode\n","    self.pso_param_head = ['alpha', 'gamma']\n","    self.pso_param_limit = [(0.2, 1), (0.6, 1.2)]\n","\n","    self.pso_agent = {}\n","    self.__pso_init_param()\n","\n","  def pso_execute(self):\n","    for _ in range(self.episode):\n","      self.__pso_fitness_call1()\n","      self.__pso_fitness_call2()\n","      self.__pso_update_param()\n","\n","  def pso_print_ans(self):\n","    print('ans is :',self.pso_agent[(0,'optim_param')])\n","    print('check it!')\n","    print('all msg:',self.pso_agent)\n","\n","  def __pso_update_param(self):\n","    for i in range(self.pso_cnt-1):\n","      pso_v = np.array(list( self.pso_agent[(i+1, 'pso_v')].values() ))\n","      pso_optim_param = np.array(list( self.pso_agent[i+1, 'optim_param'].values() ))\n","      pso_now_param = np.array(list( self.pso_agent[i+1, 'now_param'].values() ))\n","      pso_best_param = np.array(list( self.pso_agent[0, 'optim_param'].values() ))\n","\n","      pso_v = 0.5*pso_v + \\\n","              2*np.random.random()*(pso_best_param - pso_now_param) + \\\n","              2*np.random.random()*(pso_optim_param - pso_now_param)\n","      pso_now_param += pso_v\n","\n","      self.pso_agent[(i+1, 'pso_v')] = { k:v for k,v in zip(self.pso_param_head, pso_v.tolist()) }\n","      self.pso_agent[(i+1,'now_param')] = { k:v for k,v in zip(self.pso_param_head, pso_now_param.tolist()) }\n","      self.__pso_modify_param(i+1)\n","\n","  def __pso_modify_param(self, pso_agent_index):\n","    for i, title in enumerate(self.pso_param_head):\n","      if self.pso_agent[(pso_agent_index,'now_param')][title] < self.pso_param_limit[i][0]:\n","        self.pso_agent[(pso_agent_index,'now_param')][title] = self.pso_param_limit[i][0]\n","      if self.pso_agent[(pso_agent_index,'now_param')][title] > self.pso_param_limit[i][1]:\n","        self.pso_agent[(pso_agent_index,'now_param')][title] = self.pso_param_limit[i][1]\n","\n","  def __pso_fitness_call1(self):\n","    fit_value_array = []\n","    for i in range(self.pso_cnt-1):\n","      fit_value = self.__fitness_evaluate(i+1)\n","      fit_value_array.append(fit_value)\n","      self.pso_agent[(i+1,'fit_value')]['fit_now'] = fit_value\n","      if fit_value > self.pso_agent[(i+1,'fit_value')]['fit_best']:\n","        self.pso_agent[(i+1,'fit_value')]['fit_best'] = fit_value\n","        self.pso_agent[(i+1, \"optim_param\")] = self.pso_agent[(i+1, 'now_param')]\n","    print(fit_value_array)\n","        \n","  def __fitness_evaluate(self,pso_agent_index):\n","    env=gym.make('Taxi-v3')\n","    sarsa_exp_param = self.pso_agent[(pso_agent_index, 'now_param')]\n","    algorithm_sarsa = evaluate_alg_sarsa_exp(env, cnt=1, episodes=2000, sarsa_exp_param=sarsa_exp_param)\n","    algorithm_sarsa.start_execute()\n","    env.close()\n","    fit_value = algorithm_sarsa.reward_array[:-500].mean()\n","    return fit_value\n","  \n","  def __pso_fitness_call2(self):\n","    for i in range(self.pso_cnt-1):\n","      if self.pso_agent[(i+1,'fit_value')]['fit_best'] > self.pso_agent[(0,'fit_value')]['fit_best']:\n","        self.pso_agent[(0,'fit_value')]['fit_best'] = self.pso_agent[(i+1,'fit_value')]['fit_best']\n","        self.pso_agent[(0, \"optim_param\")] = self.pso_agent[(i+1, 'now_param')]\n","\n","  def __pso_init_param(self):\n","    for i in range(self.pso_cnt):\n","      self.pso_agent[(i,'optim_param')] = { k:v for k,v in zip(self.pso_param_head, np.zeros(len(self.pso_param_head)).tolist()) }\n","      self.pso_agent[(i,'now_param')] = { k:np.random.uniform(v[0],v[1]) \\\n","                                         for k,v in zip(self.pso_param_head, self.pso_param_limit) }\n","      self.pso_agent[(i,'pso_v')] = { k:np.random.random()*(v[1]-v[0])*0.5\n","                                         for k,v in zip(self.pso_param_head, self.pso_param_limit) }\n","      self.pso_agent[(i,'fit_value')] = {'fit_best':float('-inf'), 'fit_now':float('-inf')}\n"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"EcSLLOa01DtG"},"source":[""],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"Ar-6nETkcWKl"},"source":["## 期望Sarsa粒子群算法执行"]},{"cell_type":"code","metadata":{"id":"SUbJZRbwcWAV"},"source":["pst_find  = pso_evaluate_sarsa_exp(pso_cnt=5, episode=20)\n","pst_find.pso_execute()\n","pst_find.pso_print_ans()"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"ZPQkqnIEcV9k"},"source":[""],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"phMR5Xw5cZfM"},"source":["## 期望Sarsa粒子群算法测试"]},{"cell_type":"code","metadata":{"id":"yBSaqT6bcV7N"},"source":["env=gym.make('Taxi-v3')\n","sarsa_exp_param = {'alpha': 0.8793776792728718, 'gamma': 0.9964040263575571}\n","algorithm_sarsa_exp1 = evaluate_alg_sarsa_exp(env, cnt=5, episodes=2000, sarsa_exp_param=sarsa_exp_param)\n","algorithm_sarsa_exp1.start_execute()\n","env.close()"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"PntSzNRhcV4e"},"source":["algorithm_sarsa_exp.evaluate_plot_reward_dev(20)\n","algorithm_sarsa_exp1.evaluate_plot_reward_dev(20, style='x--')\n","plt.legend(['expect sarsa', 'pso expect sarsa'])"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"TIUQWa3DcVxV"},"source":[""],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"tpochRme1DqM"},"source":[""],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"mLKTSZnuh_jV"},"source":["## Double Q-learning粒子群算法参数选取"]},{"cell_type":"code","metadata":{"id":"vK0MgnWsiHll"},"source":["class pso_evaluate_double_q(pso_evaluate):\n","  def __fitness_evaluate(self,pso_agent_index):\n","    env=gym.make('Taxi-v3')\n","    q_param = self.pso_agent[(pso_agent_index, 'now_param')]\n","    algorithm_q = evaluate_alg_double_q(env, cnt=1, episodes=2000, q_param=q_param)\n","    algorithm_q.start_execute()\n","    env.close()\n","    fit_value = algorithm_q.reward_array[:-500].mean()\n","    return fit_value"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"qgPPUaJyiHjH"},"source":["pst_find = pso_evaluate_double_q(pso_cnt=5, episode=20)\n","pst_find.pso_execute()"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"sneWl0DViHgw"},"source":["pst_find.pso_print_ans()"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"--pm5XqqlF8O"},"source":["env=gym.make('Taxi-v3')\n","q_param = {'alpha': 0.5265945680168334, 'gamma': 0.9764647562876259}\n","algorithm_double_q1 = evaluate_alg_double_q(env, cnt=5, episodes=2000, q_param=q_param)\n","algorithm_double_q1.start_execute()\n","env.close()"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"czEB_3baiINs"},"source":["## Double Q-learning粒子群算法效果测试"]},{"cell_type":"code","metadata":{"id":"ZqzYDqv_iHeM"},"source":[""],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"m5jaLCwPiHbx"},"source":[""],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"KVHR2IWbiHY7"},"source":[""],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"Rbh8gLOjiHWo"},"source":["algorithm_double_q.evaluate_plot_reward_dev(20)\n","algorithm_double_q1.evaluate_plot_reward_dev(20, style='x:')\n","plt.legend(['double q learning', 'pso double q learning'])\n","plt.xlabel('episode')\n","plt.ylabel('average reward')"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"ovH81mCliHUN"},"source":[""],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"7bueAXGSiHSN"},"source":[""],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"a4tjjfgfiHPY"},"source":[""],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"T5bbm69ywRdk"},"source":["# 综合效果"]},{"cell_type":"code","metadata":{"id":"0zowUZfziHNK"},"source":["\n","algorithm_q1.evaluate_plot_reward_dev(20)\n","algorithm_sarsa1.evaluate_plot_reward_dev(20)\n","algorithm_double_q1.evaluate_plot_reward_dev(20)\n","algorithm_sarsa_exp1.evaluate_plot_reward_dev(20)\n","\n","\n","plt.xlabel('episode')\n","plt.ylabel('average reward')\n","\n","plt.legend(['pso q learning','pso sarsa','pso double q learning', 'pso expect sarsa'])"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"O06lXoAyiHKl"},"source":["for i in range(5):\n","  print(algorithm_q1.reward_array[i*400:(i+1)*400].mean())\n","print('')\n","for i in range(5):\n","  print(algorithm_sarsa1.reward_array[i*400:(i+1)*400].mean())\n","print('')\n","for i in range(5):\n","  print(algorithm_double_q1.reward_array[i*400:(i+1)*400].mean())\n","print('')\n","for i in range(5):\n","  print(algorithm_sarsa_exp1.reward_array[i*400:(i+1)*400].mean())\n","print('')"],"execution_count":null,"outputs":[]}]}