QFPIS-2.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "'''\n",
    "@Author: Yitao Qiu\n",
    "'''\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim\n",
    "import random\n",
    "import math\n",
    "import json\n",
    "from torch.autograd import Variable\n",
    "from torch.distributions import Categorical\n",
    "from utils.qf_data import normalize,load_observations\n",
    "from environment.QF_env_2 import envs\n",
    "from tools.ddpg.replay_buffer import ReplayBuffer\n",
    "from tools.ddpg.ornstein_uhlenbeck import OrnsteinUhlenbeckActionNoise\n",
    "from tensorboardX import SummaryWriter"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## GPU Setting"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"0\"\n",
    "C_CUDA = torch.cuda.is_available()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model Setting"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define actor network--CNN\n",
    "class Actor(nn.Module):\n",
    "    def __init__(self,product_num, win_size):\n",
    "        super(Actor, self).__init__()\n",
    "        self.conv1 = nn.Conv2d(\n",
    "            in_channels =  1,\n",
    "            out_channels = 32,\n",
    "            kernel_size = (1,3),\n",
    "            #stride = (1,3)\n",
    "        )\n",
    "        self.conv2 = nn.Conv2d(\n",
    "            in_channels = 32,\n",
    "            out_channels = 32,\n",
    "            kernel_size = (1, win_size-2),\n",
    "            #stride = (1, win_size-2)\n",
    "        )\n",
    "        self.linear1 = nn.Linear((product_num + 1)*1*32, 64)\n",
    "        self.linear2 = nn.Linear(64, 64)\n",
    "        self.linear3 = nn.Linear(64,product_num + 1)\n",
    "    \n",
    "    def reset_parameters(self):\n",
    "        self.linear1.weight.data.uniform_(*hidden_init(self.linear1))\n",
    "        self.linear2.weight.data.uniform_(*hidden_init(self.linear2))\n",
    "        self.linear3.weight.data.uniform_(-3e-3, 3e-3)\n",
    "    \n",
    "    def forward(self, state):\n",
    "        conv1_out = self.conv1(state)\n",
    "        conv1_out = F.relu(conv1_out)\n",
    "        conv2_out = self.conv2(conv1_out)\n",
    "        conv2_out = F.relu(conv2_out)\n",
    "        # Flatten\n",
    "        conv2_out = conv2_out.view(conv2_out.size(0), -1)\n",
    "        fc1_out = self.linear1(conv2_out)\n",
    "        fc1_out = F.relu(fc1_out)\n",
    "        fc2_out = self.linear2(fc1_out)\n",
    "        fc2_out = F.relu(fc2_out)\n",
    "        fc3_out = self.linear3(fc2_out)\n",
    "        fc3_out = F.softmax(fc3_out,dim=1)\n",
    "        \n",
    "        return fc3_out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define policy gradient actor network--LSTM\n",
    "class Policy(nn.Module):\n",
    "    def __init__(self,product_num, win_size,action_size):\n",
    "        super(Policy, self).__init__()\n",
    "        \n",
    "        self.lstm = nn.LSTM(win_size,32,1)\n",
    "        \n",
    "        self.linear1 = nn.Linear((product_num+1)*1*32, 64)\n",
    "        self.linear2 = nn.Linear(64, 64)\n",
    "        self.linear3 = nn.Linear(64,action_size)\n",
    "        \n",
    "        # Define the  vars for recording log prob and reawrd\n",
    "        self.saved_log_probs = []\n",
    "        self.rewards = []\n",
    "        \n",
    "    \n",
    "    def reset_parameters(self):\n",
    "        self.linear1.weight.data.uniform_(*hidden_init(self.linear1))\n",
    "        self.linear2.weight.data.uniform_(*hidden_init(self.linear2))\n",
    "        self.linear3.weight.data.uniform_(-3e-3, 3e-3)\n",
    "    \n",
    "    def forward(self, state):\n",
    "        state = torch.reshape(state, (-1, 1, 3))\n",
    "        lstm_out, _ = self.lstm(state)\n",
    "        batch_n,win_s,hidden_s = lstm_out.shape\n",
    "        lstm_out = lstm_out.view(batch_n, win_s*hidden_s)\n",
    "        lstm_out = torch.reshape(lstm_out, (-1, product_num+1, 32))\n",
    "        lstm_out = lstm_out.view(lstm_out.size(0), -1)\n",
    "        fc1_out = self.linear1(lstm_out)\n",
    "        #fc1_out = F.relu(fc1_out)\n",
    "        fc2_out = self.linear2(fc1_out)\n",
    "        #fc2_out = F.relu(fc2_out)\n",
    "        fc3_out = self.linear3(fc2_out)\n",
    "        fc3_out = F.softmax(fc3_out,dim=1)\n",
    "        \n",
    "        return fc3_out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define Critic network--CNN\n",
    "class Critic(nn.Module):\n",
    "    def __init__(self, product_num, win_size):\n",
    "        super(Critic, self).__init__()\n",
    "        self.conv1 = nn.Conv2d(\n",
    "            in_channels =  1,\n",
    "            out_channels = 32,\n",
    "            kernel_size = (1,3),\n",
    "            #stride = (1,3)\n",
    "        )\n",
    "        self.conv2 = nn.Conv2d(\n",
    "            in_channels = 32,\n",
    "            out_channels = 32,\n",
    "            kernel_size = (1, win_size-2),\n",
    "            #stride = (1, win_size-2)\n",
    "        )\n",
    "        self.linear1 = nn.Linear((product_num + 1)*1*32, 64)\n",
    "        self.linear2 = nn.Linear((product_num + 1), 64)\n",
    "        self.linear3 = nn.Linear(64, 1)\n",
    "    \n",
    "    def reset_parameters(self):\n",
    "        self.linear1.weight.data.uniform_(*hidden_init(self.linear1))\n",
    "        self.linear2.weight.data.uniform_(*hidden_init(self.linear2))\n",
    "        self.linear3.weight.data.uniform_(-3e-3, 3e-3)\n",
    "    \n",
    "    def forward(self, state, action):\n",
    "        # Observation channel\n",
    "        conv1_out = self.conv1(state)\n",
    "        conv1_out = F.relu(conv1_out)\n",
    "        conv2_out = self.conv2(conv1_out)\n",
    "        conv2_out = F.relu(conv2_out)\n",
    "        # Flatten\n",
    "        conv2_out = conv2_out.view(conv2_out.size(0), -1)\n",
    "        fc1_out = self.linear1(conv2_out)\n",
    "        # Action channel\n",
    "        fc2_out = self.linear2(action)\n",
    "        obs_plus_ac = torch.add(fc1_out,fc2_out)\n",
    "        obs_plus_ac = F.relu(obs_plus_ac)\n",
    "        fc3_out = self.linear3(obs_plus_ac)\n",
    "        \n",
    "        \n",
    "        return fc3_out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def obs_normalizer(observation):\n",
    "    # Normalize the observation into close/open ratio\n",
    "    if isinstance(observation, tuple):\n",
    "        observation = observation[0]\n",
    "    \n",
    "    observation = observation[:, :, 3:4] / observation[:, :, 0:1]\n",
    "    observation = normalize(observation)\n",
    "    return observation\n",
    "\n",
    "def hidden_init(layer):\n",
    "    # Initialize the parameter of hidden layer\n",
    "    fan_in = layer.weight.data.size()[0]\n",
    "    lim = 1. / np.sqrt(fan_in)\n",
    "    return (-lim, lim)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Agent Setting"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "class QFPIS(object):\n",
    "    def __init__(self, env, product_num, win_size, action_size, actor_noise, config_file = 'config/config.json'):\n",
    "        \n",
    "        with open(config_file) as f:\n",
    "            self.config = json.load(f)\n",
    "        assert self.config != None, \"Can't load config file\"\n",
    "        \n",
    "        self.env = env\n",
    "        self.actor_noise = actor_noise\n",
    "        self.summary_path ='results/ddpg/'\n",
    "        if C_CUDA:\n",
    "            self.actor = Actor(product_num,win_size).cuda()\n",
    "            self.actor_target = Actor(product_num,win_size).cuda()\n",
    "            self.critic = Critic(product_num,win_size).cuda()\n",
    "            self.critic_target = Critic(product_num,win_size).cuda()\n",
    "        else:\n",
    "            self.actor = Actor(product_num,win_size)\n",
    "            self.actor_target = Actor(product_num,win_size)\n",
    "            self.critic = Critic(product_num,win_size)\n",
    "            self.critic_target = Critic(product_num,win_size)\n",
    "        \n",
    "        self.actor.reset_parameters()\n",
    "        self.actor_target.reset_parameters()\n",
    "        self.critic_target.reset_parameters()\n",
    "        self.actor.reset_parameters()\n",
    "        \n",
    "        self.actor_optim = optim.Adam(self.actor.parameters(), lr = self.config['actor learning rate'])\n",
    "        self.critic_optim = optim.Adam(self.critic.parameters(), lr = self.config['critic learning rate'])\n",
    "        \n",
    "        self.actor_target.load_state_dict(self.actor.state_dict())\n",
    "        self.critic_target.load_state_dict(self.critic.state_dict())\n",
    "        \n",
    "        # Here is the code for the policy-gradeint\n",
    "        if C_CUDA:\n",
    "            self.policy = Policy(product_num, win_size, action_size).cuda()\n",
    "        else:\n",
    "            self.policy = Policy(product_num, win_size, action_size)\n",
    "        self.policy_optim = optim.Adam(self.policy.parameters(), lr=1e-4)\n",
    "    \n",
    "    def act(self, state):\n",
    "        if C_CUDA:\n",
    "            state = torch.tensor(state, dtype=torch.float).unsqueeze(0).cuda()\n",
    "        else:\n",
    "            state = torch.tensor(state, dtype=torch.float).unsqueeze(0)\n",
    "        \n",
    "        action = self.actor(state).squeeze(0).cpu().detach().numpy()+ self.actor_noise()\n",
    "        return action\n",
    "    \n",
    "    def critic_learn(self, state, action, predicted_q_value):\n",
    "        actual_q = self.critic(state, action)\n",
    "        if C_CUDA:\n",
    "            target_Q = torch.tensor(predicted_q_value, dtype=torch.float).cuda()\n",
    "        else:\n",
    "            target_Q = torch.tensor(predicted_q_value, dtype=torch.float)\n",
    "        target_Q=Variable(target_Q,requires_grad=True)\n",
    "        \n",
    "        td_error  = F.mse_loss(actual_q, target_Q)\n",
    "        \n",
    "        self.critic_optim.zero_grad()\n",
    "        td_error.backward()\n",
    "        self.critic_optim.step()\n",
    "        return predicted_q_value,td_error\n",
    "    \n",
    "    def actor_learn(self, state):\n",
    "\n",
    "        loss = -self.critic(state, self.actor(state)).mean()\n",
    "        \n",
    "        self.actor_optim.zero_grad()\n",
    "        loss.backward()\n",
    "        self.actor_optim.step()\n",
    "        return loss\n",
    "        \n",
    "\n",
    "    \n",
    "    def soft_update(self, net_target, net, tau):\n",
    "        for target_param, param  in zip(net_target.parameters(), net.parameters()):\n",
    "            target_param.data.copy_(target_param.data * (1.0 - tau) + param.data * tau)\n",
    "    \n",
    "    # Here is the code for the policy gradient actor\n",
    "    def select_action(self, state):\n",
    "        if C_CUDA:\n",
    "            state = torch.tensor(state, dtype=torch.float).unsqueeze(0).cuda()\n",
    "        else:\n",
    "            state = torch.tensor(state, dtype=torch.float).unsqueeze(0)\n",
    "        # Get the probability distribution\n",
    "        probs = self.policy(state)\n",
    "        m = Categorical(probs)\n",
    "        # Sample action from the distribution\n",
    "        action = m.sample()\n",
    "        self.policy.saved_log_probs.append(m.log_prob(action))\n",
    "        return action.item()\n",
    "    \n",
    "    def policy_learn(self, eps):\n",
    "        R = 0\n",
    "        policy_loss = []\n",
    "        returns = []\n",
    "        \n",
    "        # Reversed Traversal and calculate cumulative rewards for t to T\n",
    "        for r in self.policy.rewards[::-1]:\n",
    "            R = r + 0.95 * R # R: culumative rewards for t to T\n",
    "            returns.insert(0, R) # Evaluate the R and keep original order\n",
    "        if C_CUDA:\n",
    "            returns = torch.tensor(returns).cuda()\n",
    "        else:\n",
    "            returns = torch.tensor(returns)\n",
    "        # Normalized returns\n",
    "        returns = (returns - returns.mean()) / (returns.std() + eps)\n",
    "        \n",
    "        # After one episode, update once\n",
    "        for log_prob, R in zip(self.policy.saved_log_probs, returns):\n",
    "            # Actual loss definition:\n",
    "            policy_loss.append(-log_prob * R)\n",
    "        self.policy_optim.zero_grad()\n",
    "        policy_loss = torch.cat(policy_loss).sum()\n",
    "        policy_loss.backward()\n",
    "        self.policy_optim.step()\n",
    "        \n",
    "        del self.policy.rewards[:]\n",
    "        del self.policy.saved_log_probs[:]\n",
    "        \n",
    "        return policy_loss\n",
    "        \n",
    "    \n",
    "    def train(self):\n",
    "        num_episode = self.config['episode']\n",
    "        batch_size = self.config['batch size']\n",
    "        gamma = self.config['gamma']\n",
    "        tau = self.config['tau']\n",
    "        eps = np.finfo(np.float32).eps.item()\n",
    "        self.buffer = ReplayBuffer(self.config['buffer size'])\n",
    "        total_step = 0\n",
    "        writer = SummaryWriter(self.summary_path)\n",
    "        # Main training loop\n",
    "        for i in range(100):\n",
    "            previous_observation = self.env.reset()\n",
    "            #print(previous_observation)\n",
    "            # Normalization\n",
    "            previous_observation = obs_normalizer(previous_observation)\n",
    "            \n",
    "            previous_observation = previous_observation.transpose(2, 0, 1)\n",
    "            ep_reward = 0\n",
    "            ep_ave_max_q = 0\n",
    "            \n",
    "            # Keep sampling until done\n",
    "            for j in range (self.config['max step']):\n",
    "                # ================================================\n",
    "        \t\t# 1. Given state st, take action at based on actor\n",
    "        \t\t# ================================================\n",
    "                \n",
    "                action = self.act(previous_observation)\n",
    "                action_policy = self.select_action(previous_observation)\n",
    "                # ================================================\n",
    "        \t\t# 2. Obtain reward rt and reach new state st+1\n",
    "                # ================================================\n",
    "                observation_origin, reward, policy_reward, done, _ = self.env.step(action,action_policy)\n",
    "                \n",
    "                # ================================================\n",
    "                # For Policy Gradient, append reward\n",
    "                # ================================================\n",
    "                self.policy.rewards.append(policy_reward)\n",
    "                # ================================================\n",
    "                # For Policy Gradient, Update network parameter\n",
    "                # ================================================\n",
    "                \n",
    "                \n",
    "                observation = obs_normalizer(observation_origin)\n",
    "                # Reshape\n",
    "                observation = observation.transpose(2, 0, 1)\n",
    "                # ================================================\n",
    "        \t\t# 3. Store (st, at, rt, st+1)\n",
    "        \t\t# ================================================\n",
    "                self.buffer.add(previous_observation, action, reward, done, observation)\n",
    "                if self.buffer.size() >= batch_size:\n",
    "                    # ==========================================\n",
    "        \t\t\t# 4. Sample (si,ai,ri,si+1) from the buffer\n",
    "        \t\t\t# ==========================================\n",
    "                    s_batch, a_batch, r_batch, t_batch, s2_batch = self.buffer.sample_batch(batch_size)\n",
    "                    # Convert to torch tensor\n",
    "                    if C_CUDA:\n",
    "                        s_batch = torch.tensor(s_batch, dtype=torch.float).cuda()\n",
    "                        a_batch = torch.tensor(a_batch, dtype=torch.float).cuda()\n",
    "                        r_batch = torch.tensor(r_batch, dtype=torch.float).cuda()#.view(batch_size,-1)\n",
    "                        t_batch = torch.tensor(t_batch, dtype=torch.float).cuda()\n",
    "                        s2_batch = torch.tensor(s2_batch, dtype=torch.float).cuda()\n",
    "                        target_q = self.critic_target(s2_batch,self.actor_target(s2_batch)).cpu().detach()\n",
    "                    else:\n",
    "                        s_batch = torch.tensor(s_batch, dtype=torch.float)\n",
    "                        a_batch = torch.tensor(a_batch, dtype=torch.float)\n",
    "                        r_batch = torch.tensor(r_batch, dtype=torch.float)\n",
    "                        t_batch = torch.tensor(t_batch, dtype=torch.float)\n",
    "                        s2_batch = torch.tensor(s2_batch, dtype=torch.float)\n",
    "                        target_q = self.critic_target(s2_batch,self.actor_target(s2_batch)).detach()\n",
    "                    y_i = []\n",
    "                    for k in range(batch_size):\n",
    "                        if t_batch[k]:\n",
    "                            y_i.append(r_batch[k])\n",
    "                        else:\n",
    "                            y_i.append(r_batch[k].cpu().numpy() + gamma * target_q[k].numpy())\n",
    "                    #y_i = r_batch + gamma * target_q\n",
    "                    # =========================================================\n",
    "        \t\t\t# 6. Update the parameters of Q to make Q(si,ai) close to y\n",
    "        \t\t\t# =========================================================\n",
    "                    predicted_q_value,td_error = self.critic_learn(s_batch, a_batch,np.reshape(y_i, (batch_size, 1)))\n",
    "                    writer.add_scalar('TD error', td_error, global_step=total_step)\n",
    "                    ep_ave_max_q += np.amax(predicted_q_value)\n",
    "                    \n",
    "                    # ================================================================\n",
    "        \t\t\t# 7. Update the parameters of of actor to maximize Q(si,actor(si))\n",
    "        \t\t\t# ================================================================\n",
    "                    actor_loss = self.actor_learn(s_batch)\n",
    "                    writer.add_scalar('Actor loss', actor_loss, global_step=total_step)\n",
    "                    # ===============================================\n",
    "        \t\t\t# 8. Every C steps reset Q^ = Q, actor^ = actor\n",
    "        \t\t\t# ================================================\n",
    "                    self.soft_update(self.critic_target, self.critic, tau)\n",
    "                    self.soft_update(self.actor_target, self.actor, tau)\n",
    "                \n",
    "                ep_reward += reward\n",
    "                previous_observation =  observation\n",
    "                total_step = total_step+1\n",
    "                if done or j == self.config['max step'] - 1:\n",
    "                    writer.add_scalar('Q-max', ep_ave_max_q / float(j), global_step=i)\n",
    "                    writer.add_scalar('Reward', ep_reward, global_step=i)\n",
    "                    \n",
    "                    print('Episode: {:d}, Reward: {:.2f}, Qmax: {:.4f}'.format(i, ep_reward, (ep_ave_max_q / float(j))))\n",
    "                    #print(action)\n",
    "                    break\n",
    "            policy_loss = self.policy_learn(eps)\n",
    "            writer.add_scalar('Policy Loss', policy_loss, global_step=i)\n",
    "        print('Finish.')\n",
    "        torch.save(self.actor.state_dict(), model_add+model_name)\n",
    "        torch.save(self.policy.state_dict(), model_add+pg_model_name)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Shape for Train observations -- T:  (1637, 8, 9, 1)\n",
      "Episode: 0, Reward: 1.02, Qmax: 0.2271\n",
      "Episode: 1, Reward: 0.64, Qmax: 0.2562\n",
      "Episode: 2, Reward: 0.97, Qmax: 0.2681\n",
      "Episode: 3, Reward: 1.07, Qmax: 0.2678\n",
      "Episode: 4, Reward: 1.00, Qmax: 0.2649\n",
      "Episode: 5, Reward: 0.90, Qmax: 0.2674\n",
      "Episode: 6, Reward: 0.91, Qmax: 0.2677\n",
      "Episode: 7, Reward: 0.48, Qmax: 0.2704\n",
      "Episode: 8, Reward: 1.00, Qmax: 0.2742\n",
      "Episode: 9, Reward: 0.81, Qmax: 0.2765\n",
      "Episode: 10, Reward: 1.21, Qmax: 0.2816\n",
      "Episode: 11, Reward: 0.62, Qmax: 0.2835\n",
      "Episode: 12, Reward: 0.81, Qmax: 0.2835\n",
      "Episode: 13, Reward: 1.21, Qmax: 0.2844\n",
      "Episode: 14, Reward: 1.39, Qmax: 0.2866\n",
      "Episode: 15, Reward: 1.76, Qmax: 0.2882\n",
      "Episode: 16, Reward: 2.25, Qmax: 0.2925\n",
      "Episode: 17, Reward: 1.53, Qmax: 0.2940\n",
      "Episode: 18, Reward: 1.53, Qmax: 0.2963\n",
      "Episode: 19, Reward: 2.33, Qmax: 0.2988\n",
      "Episode: 20, Reward: 2.00, Qmax: 0.3002\n",
      "Episode: 21, Reward: 1.45, Qmax: 0.3022\n",
      "Episode: 22, Reward: 1.80, Qmax: 0.3039\n",
      "Episode: 23, Reward: 1.32, Qmax: 0.3035\n",
      "Episode: 24, Reward: 1.90, Qmax: 0.3038\n",
      "Episode: 25, Reward: 2.83, Qmax: 0.3066\n",
      "Episode: 26, Reward: 1.34, Qmax: 0.3064\n",
      "Episode: 27, Reward: 2.09, Qmax: 0.3066\n",
      "Episode: 28, Reward: 2.65, Qmax: 0.3081\n",
      "Episode: 29, Reward: 2.25, Qmax: 0.3074\n",
      "Episode: 30, Reward: 2.42, Qmax: 0.3084\n",
      "Episode: 31, Reward: 1.38, Qmax: 0.3094\n",
      "Episode: 32, Reward: 1.39, Qmax: 0.3086\n",
      "Episode: 33, Reward: 1.60, Qmax: 0.3083\n",
      "Episode: 34, Reward: 2.39, Qmax: 0.3091\n",
      "Episode: 35, Reward: 1.74, Qmax: 0.3089\n",
      "Episode: 36, Reward: 2.22, Qmax: 0.3099\n",
      "Episode: 37, Reward: 2.78, Qmax: 0.3089\n",
      "Episode: 38, Reward: 1.61, Qmax: 0.3084\n",
      "Episode: 39, Reward: 2.14, Qmax: 0.3076\n",
      "Episode: 40, Reward: 2.32, Qmax: 0.3101\n",
      "Episode: 41, Reward: 1.89, Qmax: 0.3087\n",
      "Episode: 42, Reward: 2.10, Qmax: 0.3089\n",
      "Episode: 43, Reward: 2.28, Qmax: 0.3099\n",
      "Episode: 44, Reward: 2.41, Qmax: 0.3120\n",
      "Episode: 45, Reward: 2.10, Qmax: 0.3111\n",
      "Episode: 46, Reward: 1.93, Qmax: 0.3117\n",
      "Episode: 47, Reward: 2.48, Qmax: 0.3130\n",
      "Episode: 48, Reward: 2.92, Qmax: 0.3143\n",
      "Episode: 49, Reward: 2.93, Qmax: 0.3174\n",
      "Episode: 50, Reward: 2.92, Qmax: 0.3179\n",
      "Episode: 51, Reward: 2.04, Qmax: 0.3214\n",
      "Episode: 52, Reward: 2.31, Qmax: 0.3231\n",
      "Episode: 53, Reward: 1.86, Qmax: 0.3256\n",
      "Episode: 54, Reward: 2.14, Qmax: 0.3272\n",
      "Episode: 55, Reward: 2.03, Qmax: 0.3285\n",
      "Episode: 56, Reward: 2.24, Qmax: 0.3308\n",
      "Episode: 57, Reward: 2.40, Qmax: 0.3311\n",
      "Episode: 58, Reward: 2.13, Qmax: 0.3309\n",
      "Episode: 59, Reward: 2.67, Qmax: 0.3353\n",
      "Episode: 60, Reward: 2.38, Qmax: 0.3376\n",
      "Episode: 61, Reward: 1.60, Qmax: 0.3374\n",
      "Episode: 62, Reward: 2.69, Qmax: 0.3390\n",
      "Episode: 63, Reward: 2.69, Qmax: 0.3384\n",
      "Episode: 64, Reward: 1.91, Qmax: 0.3400\n",
      "Episode: 65, Reward: 1.62, Qmax: 0.3432\n",
      "Episode: 66, Reward: 2.01, Qmax: 0.3431\n",
      "Episode: 67, Reward: 2.05, Qmax: 0.3427\n",
      "Episode: 68, Reward: 2.11, Qmax: 0.3470\n",
      "Episode: 69, Reward: 1.41, Qmax: 0.3441\n",
      "Episode: 70, Reward: 2.12, Qmax: 0.3470\n",
      "Episode: 71, Reward: 2.33, Qmax: 0.3449\n",
      "Episode: 72, Reward: 1.67, Qmax: 0.3464\n",
      "Episode: 73, Reward: 2.27, Qmax: 0.3472\n",
      "Episode: 74, Reward: 2.34, Qmax: 0.3487\n",
      "Episode: 75, Reward: 2.47, Qmax: 0.3499\n",
      "Episode: 76, Reward: 2.73, Qmax: 0.3473\n",
      "Episode: 77, Reward: 2.08, Qmax: 0.3473\n",
      "Episode: 78, Reward: 2.10, Qmax: 0.3498\n",
      "Episode: 79, Reward: 3.72, Qmax: 0.3498\n",
      "Episode: 80, Reward: 2.01, Qmax: 0.3512\n",
      "Episode: 81, Reward: 2.09, Qmax: 0.3514\n",
      "Episode: 82, Reward: 1.48, Qmax: 0.3532\n",
      "Episode: 83, Reward: 2.54, Qmax: 0.3521\n",
      "Episode: 84, Reward: 1.41, Qmax: 0.3538\n",
      "Episode: 85, Reward: 1.45, Qmax: 0.3567\n",
      "Episode: 86, Reward: 2.37, Qmax: 0.3535\n",
      "Episode: 87, Reward: 3.08, Qmax: 0.3547\n"
     ]
    }
   ],
   "source": [
    "if __name__ == '__main__':\n",
    "    # Parameter Settings\n",
    "    model_add = 'models/'\n",
    "    model_name = 'QFPIS_DDPG_test'\n",
    "    pg_model_name = 'QFPIS_PG_test'\n",
    "    mode = \"Train\"\n",
    "    steps = 1000\n",
    "    product_num = 9\n",
    "    window_length = 3\n",
    "    action_dim = [10]\n",
    "    train_ratio = 0.8\n",
    "    window_size = 1\n",
    "    feature_num = 8\n",
    "    action_size = 3\n",
    "    market_feature = ['Open','High','Low','Close','QPL1','QPL-1','QPL2','QPL-2']\n",
    "    product_list = [\"AUDCAD\",\"AUDUSD\",\"EURAUD\",\"EURCAD\",\"EURUSD\",\"GBPUSD\",\"NZDCHF\",\"NZDUSD\",\"USDCHF\"]\n",
    "    \n",
    "    actor_noise = OrnsteinUhlenbeckActionNoise(mu=np.zeros(action_dim))\n",
    "    \n",
    "    env = envs(product_list,market_feature,feature_num,steps,window_length,mode)\n",
    "    qf_system = QFPIS(env,product_num ,window_length, action_size,actor_noise ,config_file='config/config.json')\n",
    "\n",
    "    # Start Training\n",
    "    qf_system.train()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_actor():\n",
    "    actor = Actor(product_num =9,win_size = 3).cuda()\n",
    "    actor.load_state_dict(torch.load(model_add+model_name))\n",
    "    return actor\n",
    "\n",
    "def load_policy():\n",
    "    test = Policy(product_num = 9, win_size = 3, action_size = 3).cuda()\n",
    "    test.load_state_dict(torch.load(model_add+pg_model_name))\n",
    "    return test\n",
    "    \n",
    "def test_model(env, actor, policy):\n",
    "    eps = 1e-8\n",
    "    actions = []\n",
    "    weights = []\n",
    "    observation, info = env.reset()\n",
    "    observation = obs_normalizer(observation)\n",
    "    observation = observation.transpose(2, 0, 1)\n",
    "    done = False\n",
    "    ep_reward = 0\n",
    "    while not done:\n",
    "        observation = torch.tensor(observation, dtype=torch.float).unsqueeze(0).cuda()\n",
    "        action = actor(observation).squeeze(0).cpu().detach().numpy()\n",
    "        # Here is the code for the policy gradient\n",
    "        actions_prob = policy(observation)\n",
    "        m = Categorical(actions_prob)\n",
    "        # Selection action by sampling the action prob\n",
    "        action_policy = m.sample()\n",
    "        actions.append(action_policy.cpu().numpy())\n",
    "        w1 = np.clip(action, 0, 1)  # np.array([cash_bias] + list(action))  # [w0, w1...]\n",
    "        w1 /= (w1.sum() + eps)\n",
    "        weights.append(w1)\n",
    "        observation, reward,policy_reward, done, _ = env.step(action,action_policy)\n",
    "        ep_reward += reward\n",
    "        observation = obs_normalizer(observation)\n",
    "        observation = observation.transpose(2, 0, 1)\n",
    "    #print(ep_reward)\n",
    "    env.render()\n",
    "    return actions, weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Testing\n",
    "\n",
    "data_add ='Data/'\n",
    "train_ratio = 0.8\n",
    "window_size = 1\n",
    "window_length = 3\n",
    "market_feature = ['Open','High','Low','Close','QPL1','QPL-1','QPL2','QPL-2']\n",
    "feature_num = 8\n",
    "product_list = [\"AUDCAD\",\"AUDUSD\",\"EURAUD\",\"EURCAD\",\"EURUSD\",\"GBPUSD\",\"NZDCHF\",\"NZDUSD\",\"USDCHF\"]\n",
    "\n",
    "observations,ts_d_len = load_observations(window_size,market_feature,feature_num,product_list)\n",
    "train_size = int(train_ratio*ts_d_len)\n",
    "\n",
    "test_observations = observations[int(train_ratio * observations.shape[0]):]\n",
    "test_observations = np.squeeze(test_observations)\n",
    "test_observations = test_observations.transpose(2, 0, 1)\n",
    "\n",
    "mode = \"Test\"\n",
    "steps = 405\n",
    "env = envs(product_list,market_feature,feature_num,steps,window_length,mode,start_index=train_size+282,start_date='2019-6-25')\n",
    "actor = load_actor()\n",
    "policy = load_policy()\n",
    "test_actions, test_weight = test_model(env,actor,policy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}