Labeling.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "2u0CKIzY1C6-"
   },
   "source": [
    "# Labeling"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "V102f4er1C7E"
   },
   "source": [
    "This notebook includes a script for labeling candlestick charts, which are randomly chosen from daily NASDAQ data."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "VDKqJUt5aztc"
   },
   "source": [
    "Load necessary libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "pIZcOjbpa5fX"
   },
   "outputs": [],
   "source": [
    "!pip install mpl_finance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "OxcK2-Lc1C7G",
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import pickle\n",
    "from mpl_finance import candlestick2_ohlc\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "u5PkP9Mra77n"
   },
   "source": [
    "Load stock data from Yahoo Finance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "sJZgM3XWaav5"
   },
   "outputs": [],
   "source": [
    "# Date and only valid symbols\n",
    "date = pd.read_csv('Data_stock_yahoo/date.csv',header=None)\n",
    "aux = date.iloc[:,0].values\n",
    "validSymbols = pd.read_csv('Data_stock_yahoo/selectedSymbols.csv', header=None)\n",
    "validCols = validSymbols.iloc[0,:].values - 1\n",
    "# OHLC\n",
    "close_quotes = pd.read_csv('Data_stock_yahoo/close.csv', header=None, usecols=validCols)\n",
    "open_quotes = pd.read_csv('Data_stock_yahoo/open.csv',header=None, usecols=validCols)\n",
    "low_quotes = pd.read_csv('Data_stock_yahoo/low.csv',header=None, usecols=validCols)\n",
    "high_quotes = pd.read_csv('Data_stock_yahoo/high.csv',header=None, usecols=validCols)\n",
    "volume_quotes = pd.read_csv('Data_stock_yahoo/volume.csv',header=None, usecols=validCols)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "3Ion0PBD1C7J"
   },
   "outputs": [],
   "source": [
    "# Rename df\n",
    "# Col names --> stock ticker names\n",
    "ticker = pd.read_csv('Data_stock_yahoo/ticker.csv', header=None)\n",
    "valid_stock_tickers = ticker.loc[validCols, 0].values\n",
    "close_quotes.columns = valid_stock_tickers\n",
    "open_quotes.columns = valid_stock_tickers\n",
    "high_quotes.columns = valid_stock_tickers\n",
    "low_quotes.columns = valid_stock_tickers\n",
    "volume_quotes.columns = valid_stock_tickers\n",
    "# Row index --> date index\n",
    "close_quotes.index = aux\n",
    "open_quotes.index = aux\n",
    "high_quotes.index = aux\n",
    "low_quotes.index = aux\n",
    "volume_quotes.index = aux"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "2GXU5S3Waav6",
    "outputId": "9987f1b2-3132-4dd0-ad48-d5e125c5314c"
   },
   "outputs": [],
   "source": [
    "# Check\n",
    "volume_quotes.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "aqIm_l9j1C7K"
   },
   "source": [
    "#### Labeling loop\n",
    "The following code is a loop that:\n",
    "1. Picks a random position in the series to create a 40-day window, as well as a particular action.\n",
    "2. Draws the candlestick chart \n",
    "3. Asks the user to enter a number to label the chart.\n",
    "\n",
    "First choose the size of the window and the number of examples you want to generate.  \n",
    "You have the option to discard an example if you do not want to include it (e.g. possibly ambiguous).\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "eEKhmcZcbwpQ"
   },
   "source": [
    "Parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "l4bU3LZN1C7L"
   },
   "outputs": [],
   "source": [
    "window = 40 \n",
    "n = 300 # 300.000"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "Bw_mxNyHbyAm"
   },
   "source": [
    "For-loop"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 649
    },
    "id": "rpr__wv61C7M",
    "outputId": "359bca8f-3819-42b3-b294-c8ea3b61d3ff",
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "etiquetas = np.zeros(n)\n",
    "idfechas = np.zeros(n)\n",
    "idstocks = np.zeros(n)\n",
    "i = 0\n",
    "while i<n:\n",
    "    i = i + 1\n",
    "    idx_stock = np.random.randint(0, close_quotes.shape[1])\n",
    "    idx = np.random.randint(0, close_quotes.shape[0] - window)\n",
    "    windata_close = close_quotes.iloc[idx:idx+window, idx_stock]\n",
    "    if np.sum(np.isnan(windata_close))>0:\n",
    "        i = i - 1 # repeat iteration if there is a NaN\n",
    "    else:\n",
    "        # Save case info\n",
    "        idfechas[i-1] = idx\n",
    "        idstocks[i-1] = idx_stock\n",
    "        windata_open = open_quotes.iloc[idx:idx+window, idx_stock]\n",
    "        windata_high = high_quotes.iloc[idx:idx+window, idx_stock]\n",
    "        windata_low = low_quotes.iloc[idx:idx+window, idx_stock]\n",
    "        fig, ax = plt.subplots(figsize=(6,4))\n",
    "        _ = candlestick2_ohlc(ax, windata_open, windata_high,\n",
    "                             windata_low , windata_close,\n",
    "                             colorup='g', colordown='r', width=0.66, alpha=0.8)\n",
    "        plt.show()\n",
    "        print(f'Ejemplo {i} de {n}')\n",
    "        print(\"Etiquete la siguiete grafica como:\")\n",
    "        print(\" [1] Doble suelo, [0] Sin doble suelo, [2] Descartar ejemplo\")\n",
    "        opcion_usuario = input()\n",
    "        if opcion_usuario >= '0' and opcion_usuario <= '2':\n",
    "            etiquetas[i-1] = opcion_usuario   \n",
    "        else:\n",
    "            print('Error en la opción introducida. Ejemplo descartado')\n",
    "            etiquetas[i-1] = 2\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "hTW7jlcqb3my"
   },
   "source": [
    "Save info"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "dhqGgP3KaawB"
   },
   "outputs": [],
   "source": [
    "# Save info\n",
    "doble_suelo_labels = pd.DataFrame(np.zeros((n, 4)))\n",
    "doble_suelo_labels.columns = ['Date', 'DateIndex','Ticker', 'Label']\n",
    "doble_suelo_labels\n",
    "doble_suelo_labels.Date = aux[idfechas.astype(np.int)]\n",
    "doble_suelo_labels.DateIndex = idfechas.astype(np.int)\n",
    "doble_suelo_labels.Ticker = idstocks.astype(np.int)    \n",
    "doble_suelo_labels.Label =  etiquetas                             "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "1UhbEleZaawB",
    "outputId": "f4f00bb1-5f36-48bf-ab97-7a25ae68c2d5"
   },
   "outputs": [],
   "source": [
    "doble_suelo_labels"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "BAFcDrji1C7O"
   },
   "source": [
    "Labels discarted are not considered"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "zErLgvBO1C7P"
   },
   "outputs": [],
   "source": [
    "doble_suelo_labels = doble_suelo_labels[doble_suelo_labels.Label!=2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "iizAAW2R1C7Q",
    "outputId": "61223785-88c2-4cc1-c907-945ca2ebcacf"
   },
   "outputs": [],
   "source": [
    "doble_suelo_labels.Label.value_counts() # to treat unbalanced data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "iSD73go_1C7Q"
   },
   "source": [
    "Save the CSV file for further processing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "VnfjcNuV1C7R"
   },
   "outputs": [],
   "source": [
    "doble_suelo_labels.to_csv('Data_labeled/ejemplo25_doble_suelo.csv')"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [],
   "name": "Labeling.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}