lda_sklearn.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 0. Basic Packages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "#Set copy warning to off\n",
    "pd.set_option('mode.chained_assignment', None)\n",
    "import numpy as np\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### We import all the functions used in the previous notebooks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from nlp_preprocess import *"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### We add accents to stop_words and exclude the words we want"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from nltk.corpus import stopwords\n",
    "\n",
    "#we select the spanish stop words (by default all the words with accents have them)\n",
    "stop_words = stopwords.words('spanish')\n",
    "\n",
    "#now we have the words with accents and the same ones without accents\n",
    "stop_words = add_non_accents(stop_words)\n",
    "\n",
    "#list to exclude in stop_words (excluded based in \"count_stop_words\" fn)\n",
    "stop_w_list = [\"no\", \"mucho\", \"muchos\", \"tiene\", \"tienen\", \"otras\", \"sin\", \"nada\", \"algo\"\n",
    "               \"tanto\", \"alguna\", \"estamos\", \"tengan\", \"tenemos\", \"nuestros\", \"fuera\", \"algunos\", \"hasta\"]\n",
    "\n",
    "#exclude previous list of words from stop_words\n",
    "stop_words.difference_update(stop_w_list)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. Data Wrangling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(584, 4)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ID</th>\n",
       "      <th>Area</th>\n",
       "      <th>Comment</th>\n",
       "      <th>Initial_Classification</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>Dept 1</td>\n",
       "      <td>Cerrar la brecha entre los que deciden el trab...</td>\n",
       "      <td>Negativo</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>3</td>\n",
       "      <td>Dept 1</td>\n",
       "      <td>Sobretodo, que se miren las cargas de trabajo ...</td>\n",
       "      <td>Negativo</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>4</td>\n",
       "      <td>Dept 1</td>\n",
       "      <td>1. Dar coaching a algunos Gerentes (que no sab...</td>\n",
       "      <td>Negativo</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>19</td>\n",
       "      <td>Dept 1</td>\n",
       "      <td>s</td>\n",
       "      <td>Negativo</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>29</td>\n",
       "      <td>Dept 1</td>\n",
       "      <td>Mas oportunidades de crecimiento y major salar...</td>\n",
       "      <td>Negativo</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   ID    Area                                            Comment  \\\n",
       "0   1  Dept 1  Cerrar la brecha entre los que deciden el trab...   \n",
       "1   3  Dept 1  Sobretodo, que se miren las cargas de trabajo ...   \n",
       "2   4  Dept 1  1. Dar coaching a algunos Gerentes (que no sab...   \n",
       "3  19  Dept 1                                                  s   \n",
       "4  29  Dept 1  Mas oportunidades de crecimiento y major salar...   \n",
       "\n",
       "  Initial_Classification  \n",
       "0               Negativo  \n",
       "1               Negativo  \n",
       "2               Negativo  \n",
       "3               Negativo  \n",
       "4               Negativo  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "f1 = pd.read_excel(\"comments.xlsx\", [0])[0]\n",
    "f2 = pd.read_excel(\"comments.xlsx\", [1])[1]\n",
    "\n",
    "df = pd.concat([f1, f2]) \n",
    "df.rename(columns={\"Initial Classification\":\"Initial_Classification\"}, inplace=True)\n",
    "print(df.shape)\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Checking for nulls"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "ID                         0\n",
       "Area                       0\n",
       "Comment                   11\n",
       "Initial_Classification     0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#cecking for nulls\n",
    "df.isna().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 584 entries, 0 to 291\n",
      "Data columns (total 4 columns):\n",
      " #   Column                  Non-Null Count  Dtype \n",
      "---  ------                  --------------  ----- \n",
      " 0   ID                      584 non-null    int64 \n",
      " 1   Area                    584 non-null    object\n",
      " 2   Comment                 573 non-null    object\n",
      " 3   Initial_Classification  584 non-null    object\n",
      "dtypes: int64(1), object(3)\n",
      "memory usage: 22.8+ KB\n"
     ]
    }
   ],
   "source": [
    "#checking data type\n",
    "df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ID</th>\n",
       "      <th>Area</th>\n",
       "      <th>Comment</th>\n",
       "      <th>Initial_Classification</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>120</td>\n",
       "      <td>Dept 1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Negativo</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48</th>\n",
       "      <td>561</td>\n",
       "      <td>Dept 4</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Negativo</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>109</th>\n",
       "      <td>1280</td>\n",
       "      <td>Dept 7</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Negativo</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>129</th>\n",
       "      <td>1475</td>\n",
       "      <td>Dept 9</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Negativo</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>158</th>\n",
       "      <td>1768</td>\n",
       "      <td>Dept 9</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Negativo</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>227</th>\n",
       "      <td>2481</td>\n",
       "      <td>Dept 9</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Negativo</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>240</th>\n",
       "      <td>2587</td>\n",
       "      <td>Dept 9</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Negativo</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>274</th>\n",
       "      <td>2850</td>\n",
       "      <td>Dept 10</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Negativo</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>277</th>\n",
       "      <td>2887</td>\n",
       "      <td>Dept 10</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Negativo</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>227</th>\n",
       "      <td>2481</td>\n",
       "      <td>Dept 9</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Positivo</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>277</th>\n",
       "      <td>2887</td>\n",
       "      <td>Dept 10</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Positivo</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       ID     Area Comment Initial_Classification\n",
       "12    120   Dept 1     NaN               Negativo\n",
       "48    561   Dept 4     NaN               Negativo\n",
       "109  1280   Dept 7     NaN               Negativo\n",
       "129  1475   Dept 9     NaN               Negativo\n",
       "158  1768   Dept 9     NaN               Negativo\n",
       "227  2481   Dept 9     NaN               Negativo\n",
       "240  2587   Dept 9     NaN               Negativo\n",
       "274  2850  Dept 10     NaN               Negativo\n",
       "277  2887  Dept 10     NaN               Negativo\n",
       "227  2481   Dept 9     NaN               Positivo\n",
       "277  2887  Dept 10     NaN               Positivo"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#cheking the null comments\n",
    "df[df.Comment.isna()]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Updating nulls with NA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "#we will replace with the term \"NA\"\n",
    "df.Comment[df.Comment.isna()] = \"NA\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Lowering comments"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.Comment = df.Comment.str.lower()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2. Preprocess"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Detecting language"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "#we create a new column with the detected lang\n",
    "#choose between langdetect or cld3. By default cld3, which works bettter (for es)\n",
    "df[\"language\"] = dect_lang(df.Comment, detector=\"langdetect\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Sentence: s ----> Lang: sv\n",
      "Sentence: prefiero no comentar ----> Lang: it\n",
      "Sentence: na ----> Lang: tl\n",
      "Sentence: more alignment in priorities to focus more and accomplish better results. we try to do too much at the same time, limiting the chances of success.  ----> Lang: en\n",
      "Sentence: reduce legal and compliance bureaucracy. there is a pervasive view that lawyers and risk managers can manage the business from afar. client facing employees are handcuffed and more worrisome, at times prefer not to look for better alternatives for clients afraid that they might run afoul of one of the many new rules. also, innovation is stifled by all the roadblocks and permissions needed to launch a product or service.  ----> Lang: en\n",
      "Sentence: no cambiaria nada. ----> Lang: pt\n",
      "Sentence: ninguna. ----> Lang: tl\n",
      "Sentence: los salarios justos  ----> Lang: lt\n",
      "Sentence: nada ----> Lang: so\n",
      "Sentence: mejorar la paga ----> Lang: id\n",
      "Sentence: na ----> Lang: tl\n",
      "Sentence: no cambiaria nada. ----> Lang: pt\n",
      "Sentence: continuar innovando en technologia ----> Lang: it\n",
      "Sentence: tienen que mejorar e incentivar el work-life balance o work-life integration en vez de incentivar el burnout.  ----> Lang: en\n",
      "Sentence: nothing specific ----> Lang: en\n",
      "Sentence: nada ----> Lang: so\n",
      "Sentence: hasta el momento no observo ninguna área a mencionar. ----> Lang: pt\n",
      "Sentence: salario ----> Lang: id\n",
      "Sentence: nada ----> Lang: so\n",
      "Sentence: todo me gusta ----> Lang: pt\n",
      "Sentence: - ----> Lang: NO LANG\n",
      "Sentence: nada ----> Lang: so\n",
      "Sentence: nada ----> Lang: so\n",
      "Sentence: nada. ----> Lang: so\n",
      "Sentence: nada, por lo danaria ----> Lang: pt\n",
      "Sentence: na ----> Lang: tl\n",
      "Sentence: some businesses within la empresa should be evaluated/treated differently than the rest of the company... because they are vey different in nature. employees / businesses should not be punished for situations that occurred more than 10 years ago under very different industry/market conditions.  ----> Lang: en\n",
      "Sentence: nada ----> Lang: so\n",
      "Sentence: darle importancia al work life balance. no solo decirlo. ----> Lang: it\n",
      "Sentence: most groups are spread too thin and need to hire additional persons.  ----> Lang: en\n",
      "Sentence: * ----> Lang: NO LANG\n",
      "Sentence: na ----> Lang: tl\n",
      "Sentence: no cambiaria nada. ----> Lang: pt\n",
      "Sentence: na ----> Lang: tl\n",
      "Sentence: na ----> Lang: tl\n",
      "Sentence: salario ----> Lang: id\n",
      "Sentence: nada ----> Lang: so\n",
      "Sentence: no cambiaria nada ----> Lang: pt\n",
      "Sentence: nada ----> Lang: so\n",
      "Sentence:  no cambiaria nada. ----> Lang: pt\n",
      "Sentence: igualdad salarial ----> Lang: id\n",
      "Sentence: nada ----> Lang: so\n",
      "Sentence: nada ----> Lang: so\n",
      "Sentence: . ----> Lang: NO LANG\n",
      "Sentence: salario ----> Lang: id\n",
      "Sentence: n-a ----> Lang: cy\n",
      "Sentence: nada ----> Lang: so\n",
      "Sentence: ** ----> Lang: NO LANG\n",
      "Sentence: . ----> Lang: NO LANG\n",
      "Sentence: * ----> Lang: NO LANG\n",
      "Sentence: nada ----> Lang: so\n",
      "Sentence: na ----> Lang: tl\n",
      "Sentence: hasta ahora todo bien ----> Lang: pt\n",
      "Sentence: la re numeracion economica  ----> Lang: it\n",
      "Sentence: na ----> Lang: tl\n",
      "Sentence: nada ----> Lang: so\n",
      "Sentence: no se  ----> Lang: pt\n",
      "Sentence: no haria cambios. ----> Lang: pt\n",
      "Sentence: n.a ----> Lang: af\n",
      "Sentence: ensure all divisions are help to the same high standards of service and to be more decisive on investing in businesses with the best longer term opportunities. ----> Lang: en\n",
      "Sentence: yearly pay increases ----> Lang: en\n",
      "Sentence: technology and favoritism ----> Lang: en\n",
      "Sentence: na ----> Lang: tl\n",
      "Sentence: na ----> Lang: tl\n",
      "Sentence: mejorar la paga. ----> Lang: tl\n",
      "Sentence: - ----> Lang: NO LANG\n",
      "Sentence: nada  ----> Lang: so\n",
      "Sentence: nada para mi es excelente ----> Lang: pt\n",
      "Sentence: s ----> Lang: sv\n",
      "Sentence: does what’s right for the company, the employees, customers and community. has a great impact in puerto rico.  ----> Lang: en\n",
      "Sentence: strong sense of responsibility for the well being of clients, employees and society in general ( non for profit, culture, sports, economic growth)  ----> Lang: en\n",
      "Sentence: ambiente professional, tiempo para compartir con tu familia ----> Lang: it\n",
      "Sentence: excelente patrono. ----> Lang: pt\n",
      "Sentence: valores ----> Lang: et\n",
      "Sentence: the team always take a proactive approach. ----> Lang: en\n",
      "Sentence: grandiosa. excelente. distinguida. buenisima. ----> Lang: pt\n",
      "Sentence: no. ----> Lang: pt\n",
      "Sentence: na  ----> Lang: tl\n",
      "Sentence: - ----> Lang: NO LANG\n",
      "Sentence: excelencia ----> Lang: pt\n",
      "Sentence: good culture... that is why it is important to get back to the office... culture cannot be transmitted remotely.  ----> Lang: en\n",
      "Sentence: no ----> Lang: tl\n",
      "Sentence: i believe their is mutual respect for everyone throughout the organization, no matter what your role or title is.  ----> Lang: en\n",
      "Sentence: 1 ----> Lang: NO LANG\n",
      "Sentence: no ----> Lang: tl\n",
      "Sentence: n ----> Lang: tl\n",
      "Sentence: na ----> Lang: tl\n",
      "Sentence: respeto ----> Lang: it\n",
      "Sentence: seguridad ----> Lang: pt\n",
      "Sentence: - ----> Lang: NO LANG\n",
      "Sentence: si ----> Lang: fi\n",
      "Sentence: no ----> Lang: tl\n",
      "Sentence: me ciento segura ----> Lang: it\n",
      "Sentence: prestigio, solidez ----> Lang: it\n",
      "Sentence: beneficio ----> Lang: it\n",
      "Sentence: . ----> Lang: NO LANG\n",
      "Sentence: seguridad ----> Lang: pt\n",
      "Sentence: n-a ----> Lang: af\n",
      "Sentence: * ----> Lang: NO LANG\n",
      "Sentence: na ----> Lang: tl\n",
      "Sentence: marca  ----> Lang: pt\n",
      "Sentence: empleo bastante seguro ----> Lang: pt\n",
      "Sentence: su gente ----> Lang: nl\n",
      "Sentence: family atmosphere. recognize and celebrate individuality. leadership is very approachable. ----> Lang: en\n",
      "Sentence: everyone is so kind ----> Lang: en\n",
      "Sentence: job stability, the bank takes care about their employees ----> Lang: en\n",
      "Sentence: na ----> Lang: tl\n",
      "Sentence: - ----> Lang: NO LANG\n",
      "Sentence: no ----> Lang: tl\n"
     ]
    }
   ],
   "source": [
    "#we check the non-snpanish ones\n",
    "for c, l in zip(df.Comment[df.language!=\"es\"], df.language[df.language!=\"es\"]):\n",
    "    print(\"Sentence:\", c, \"---->\", \"Lang:\", l)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Once we have checked the previous output we can see the english comments are well identified, and the rest are either, errors / sentences with no meaning, or spanish sentences identified as catalan, italian or other language\n",
    "\n",
    "Therefore it is fair to consider, non-english comments as spanish comments in this case"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "#we replace non-english comments for \"es\" term\n",
    "df.language[df.language!=\"en\"] = \"es\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Sentence: more alignment in priorities to focus more and accomplish better results. we try to do too much at the same time, limiting the chances of success.  ----> Lang: en\n",
      "Sentence: reduce legal and compliance bureaucracy. there is a pervasive view that lawyers and risk managers can manage the business from afar. client facing employees are handcuffed and more worrisome, at times prefer not to look for better alternatives for clients afraid that they might run afoul of one of the many new rules. also, innovation is stifled by all the roadblocks and permissions needed to launch a product or service.  ----> Lang: en\n",
      "Sentence: tienen que mejorar e incentivar el work-life balance o work-life integration en vez de incentivar el burnout.  ----> Lang: en\n",
      "Sentence: nothing specific ----> Lang: en\n",
      "Sentence: some businesses within la empresa should be evaluated/treated differently than the rest of the company... because they are vey different in nature. employees / businesses should not be punished for situations that occurred more than 10 years ago under very different industry/market conditions.  ----> Lang: en\n",
      "Sentence: most groups are spread too thin and need to hire additional persons.  ----> Lang: en\n",
      "Sentence: ensure all divisions are help to the same high standards of service and to be more decisive on investing in businesses with the best longer term opportunities. ----> Lang: en\n",
      "Sentence: yearly pay increases ----> Lang: en\n",
      "Sentence: technology and favoritism ----> Lang: en\n",
      "Sentence: does what’s right for the company, the employees, customers and community. has a great impact in puerto rico.  ----> Lang: en\n",
      "Sentence: strong sense of responsibility for the well being of clients, employees and society in general ( non for profit, culture, sports, economic growth)  ----> Lang: en\n",
      "Sentence: the team always take a proactive approach. ----> Lang: en\n",
      "Sentence: good culture... that is why it is important to get back to the office... culture cannot be transmitted remotely.  ----> Lang: en\n",
      "Sentence: i believe their is mutual respect for everyone throughout the organization, no matter what your role or title is.  ----> Lang: en\n",
      "Sentence: family atmosphere. recognize and celebrate individuality. leadership is very approachable. ----> Lang: en\n",
      "Sentence: everyone is so kind ----> Lang: en\n",
      "Sentence: job stability, the bank takes care about their employees ----> Lang: en\n"
     ]
    }
   ],
   "source": [
    "#we re-check the list again\n",
    "for c, l in zip(df.Comment[df.language!=\"es\"], df.language[df.language!=\"es\"]):\n",
    "    print(\"Sentence:\", c, \"---->\", \"Lang:\", l)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Creating Dataframe for each Language"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_es = df[df.language==\"es\"]\n",
    "df_en = df[df.language==\"en\"]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Positive vs Negative Comments, Preprocess & Clean"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "#separate comments by type\n",
    "neg_comments_es = df_es.Comment.values[df_es.Initial_Classification==\"Negativo\"]\n",
    "pos_comments_es = df_es.Comment.values[df_es.Initial_Classification==\"Positivo\"]\n",
    "\n",
    "#clean comments\n",
    "n_clean_comments_es = clean_comments(neg_comments_es)\n",
    "p_clean_comments_es = clean_comments(pos_comments_es)\n",
    "\n",
    "\n",
    "#1. set root to \"no\" if you don't want the root (lemma) of the word, but the original world (default=\"yes\")\n",
    "#2. set stop_w to \"yes\" if you want to include stop_words (default=\"no\")\n",
    "n_pre_comments_es = preprocess(n_clean_comments_es, root=\"no\", stop_w=\"no\", stop_words=stop_words)\n",
    "p_pre_comments_es = preprocess(p_clean_comments_es, root=\"no\", stop_w=\"no\", stop_words=stop_words)\n",
    "\n",
    "#flatten comments in a list\n",
    "n_flatten_comments_es = [\" \".join(comments) for comments in n_pre_comments_es]\n",
    "p_flatten_comments_es = [\" \".join(comments) for comments in p_pre_comments_es]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Matrix to fit sklearn LDA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def make_matrix(vectorizer=str, ngrams_range=(int, int), arr=np.ndarray):\n",
    "    if vectorizer == \"CountVectorizer\":\n",
    "        vectorizer = CountVectorizer\n",
    "    elif vectorizer == \"TfidfVectorizer\":\n",
    "        vectorizer = TfidfVectorizer\n",
    "    else:\n",
    "        vectorizer = None\n",
    "    \n",
    "    #the fn raises an error if the vectorizer is not count or tf-idf\n",
    "    if vectorizer != CountVectorizer and vectorizer != TfidfVectorizer:\n",
    "        raise ValueError(\"Please select between CountVectorizer / TfidfVectorizer\")\n",
    "    \n",
    "    #setting params\n",
    "    vectorize = vectorizer(ngram_range=ngrams_range) \n",
    "    #getting matrix of counts / tf-idf\n",
    "    matrix = vectorize.fit_transform(arr)\n",
    "    \n",
    "    return matrix, vectorize"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Select between the group of comments\n",
      "'n_flatten_comments_es' (negative)\n",
      "'p_flatten_comments_es' (positive)\n",
      "n_flatten_comments_es\n"
     ]
    }
   ],
   "source": [
    "tokens = input(\"Select between the group of comments\\n'n_flatten_comments_es' (negative)\\n'p_flatten_comments_es' (positive)\\n\")\n",
    "\n",
    "while tokens != \"n_flatten_comments_es\" and tokens != \"n_flatten_comments_es\":\n",
    "    tokens = input(\"Select between the one of them\")\n",
    "    \n",
    "if tokens == \"n_flatten_comments_es\":\n",
    "    tokens = n_flatten_comments_es\n",
    "elif tokens == \"p_flatten_comments_es\":\n",
    "    tokens = p_flatten_comments_es\n",
    "    \n",
    "\n",
    "vectors, vectorizer = make_matrix(vectorizer=\"CountVectorizer\", ngrams_range=(3, 3), arr=tokens)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3. Topic Modeling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best Model's Params:  {'learning_decay': 0.5, 'n_components': 1}\n",
      "Best Log Likelihood Score:  -6271.5745606376495\n",
      "Model Perplexity:  1155.7975069308372\n"
     ]
    }
   ],
   "source": [
    "from sklearn.decomposition import LatentDirichletAllocation\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "\n",
    "#lda model\n",
    "lda = LatentDirichletAllocation(learning_method=\"batch\", max_iter=30, n_jobs=-1)\n",
    "\n",
    "#params to test in grid search\n",
    "search_params = {\"n_components\":list(range(1, 21)), \"learning_decay\":[.5, .7, .9]}\n",
    "\n",
    "topic_model = GridSearchCV(lda, param_grid=search_params, n_jobs=-1)\n",
    "\n",
    "topic_model.fit(vectors)\n",
    "\n",
    "#best model\n",
    "best_lda_model = topic_model.best_estimator_\n",
    "\n",
    "#model parameters\n",
    "print(\"Best Model's Params: \", topic_model.best_params_)\n",
    "\n",
    "#log likelihood Score\n",
    "print(\"Best Log Likelihood Score: \", topic_model.best_score_)\n",
    "\n",
    "# Perplexity\n",
    "print(\"Model Perplexity: \", best_lda_model.perplexity(vectors))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "ename": "ValueError",
     "evalue": "Found array with 1 sample(s) (shape=(1, 1)) while a minimum of 2 is required.",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-25-0c2be9a4512e>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0mpyLDAvis\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0menable_notebook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 5\u001b[0;31m \u001b[0mpanel\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpyLDAvis\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msklearn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mprepare\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbest_lda_model\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvectors\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvectorizer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmds\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'tsne'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/pyLDAvis/sklearn.py\u001b[0m in \u001b[0;36mprepare\u001b[0;34m(lda_model, dtm, vectorizer, **kwargs)\u001b[0m\n\u001b[1;32m     93\u001b[0m     \"\"\"\n\u001b[1;32m     94\u001b[0m     \u001b[0mopts\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmerge\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m_extract_data\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlda_model\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdtm\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvectorizer\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 95\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0mpyLDAvis\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mprepare\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m**\u001b[0m\u001b[0mopts\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/pyLDAvis/_prepare.py\u001b[0m in \u001b[0;36mprepare\u001b[0;34m(topic_term_dists, doc_topic_dists, doc_lengths, vocab, term_frequency, R, lambda_step, mds, n_jobs, plot_opts, sort_topics)\u001b[0m\n\u001b[1;32m    398\u001b[0m    \u001b[0mtopic_info\u001b[0m         \u001b[0;34m=\u001b[0m \u001b[0m_topic_info\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtopic_term_dists\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtopic_proportion\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mterm_frequency\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mterm_topic_freq\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvocab\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlambda_step\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mR\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mn_jobs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    399\u001b[0m    \u001b[0mtoken_table\u001b[0m        \u001b[0;34m=\u001b[0m \u001b[0m_token_table\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtopic_info\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mterm_topic_freq\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvocab\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mterm_frequency\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 400\u001b[0;31m    \u001b[0mtopic_coordinates\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_topic_coordinates\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmds\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtopic_term_dists\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtopic_proportion\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    401\u001b[0m    \u001b[0mclient_topic_order\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mx\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;36m1\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mx\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mtopic_order\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    402\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/pyLDAvis/_prepare.py\u001b[0m in \u001b[0;36m_topic_coordinates\u001b[0;34m(mds, topic_term_dists, topic_proportion)\u001b[0m\n\u001b[1;32m    181\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m_topic_coordinates\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmds\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtopic_term_dists\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtopic_proportion\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    182\u001b[0m    \u001b[0mK\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtopic_term_dists\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 183\u001b[0;31m    \u001b[0mmds_res\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmds\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtopic_term_dists\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    184\u001b[0m    \u001b[0;32massert\u001b[0m \u001b[0mmds_res\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mK\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    185\u001b[0m    mds_df = pd.DataFrame({'x': mds_res[:,0], 'y': mds_res[:,1], 'topics': range(1, K + 1), \\\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/pyLDAvis/_prepare.py\u001b[0m in \u001b[0;36mjs_TSNE\u001b[0;34m(distributions, **kwargs)\u001b[0m\n\u001b[1;32m    156\u001b[0m     \u001b[0mdist_matrix\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msquareform\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpdist\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdistributions\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmetric\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0m_jensen_shannon\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    157\u001b[0m     \u001b[0mmodel\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mTSNE\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mn_components\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrandom_state\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmetric\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'precomputed'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 158\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit_transform\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdist_matrix\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    159\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    160\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/sklearn/manifold/_t_sne.py\u001b[0m in \u001b[0;36mfit_transform\u001b[0;34m(self, X, y)\u001b[0m\n\u001b[1;32m    889\u001b[0m             \u001b[0mEmbedding\u001b[0m \u001b[0mof\u001b[0m \u001b[0mthe\u001b[0m \u001b[0mtraining\u001b[0m \u001b[0mdata\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mlow\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0mdimensional\u001b[0m \u001b[0mspace\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    890\u001b[0m         \"\"\"\n\u001b[0;32m--> 891\u001b[0;31m         \u001b[0membedding\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_fit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    892\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0membedding_\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0membedding\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    893\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0membedding_\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/sklearn/manifold/_t_sne.py\u001b[0m in \u001b[0;36m_fit\u001b[0;34m(self, X, skip_num_points)\u001b[0m\n\u001b[1;32m    669\u001b[0m             X = self._validate_data(X, accept_sparse=['csr'],\n\u001b[1;32m    670\u001b[0m                                     \u001b[0mensure_min_samples\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 671\u001b[0;31m                                     dtype=[np.float32, np.float64])\n\u001b[0m\u001b[1;32m    672\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    673\u001b[0m             X = self._validate_data(X, accept_sparse=['csr', 'csc', 'coo'],\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/sklearn/base.py\u001b[0m in \u001b[0;36m_validate_data\u001b[0;34m(self, X, y, reset, validate_separately, **check_params)\u001b[0m\n\u001b[1;32m    418\u001b[0m                     \u001b[0;34mf\"requires y to be passed, but the target y is None.\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    419\u001b[0m                 )\n\u001b[0;32m--> 420\u001b[0;31m             \u001b[0mX\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcheck_array\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mcheck_params\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    421\u001b[0m             \u001b[0mout\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mX\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    422\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/sklearn/utils/validation.py\u001b[0m in \u001b[0;36minner_f\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m     70\u001b[0m                           FutureWarning)\n\u001b[1;32m     71\u001b[0m         \u001b[0mkwargs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mupdate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m{\u001b[0m\u001b[0mk\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0marg\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mk\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0marg\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mzip\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msig\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mparameters\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 72\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     73\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0minner_f\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     74\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/sklearn/utils/validation.py\u001b[0m in \u001b[0;36mcheck_array\u001b[0;34m(array, accept_sparse, accept_large_sparse, dtype, order, copy, force_all_finite, ensure_2d, allow_nd, ensure_min_samples, ensure_min_features, estimator)\u001b[0m\n\u001b[1;32m    651\u001b[0m                              \u001b[0;34m\" minimum of %d is required%s.\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    652\u001b[0m                              % (n_samples, array.shape, ensure_min_samples,\n\u001b[0;32m--> 653\u001b[0;31m                                 context))\n\u001b[0m\u001b[1;32m    654\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    655\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mensure_min_features\u001b[0m \u001b[0;34m>\u001b[0m \u001b[0;36m0\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0marray\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mndim\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m2\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mValueError\u001b[0m: Found array with 1 sample(s) (shape=(1, 1)) while a minimum of 2 is required."
     ]
    }
   ],
   "source": [
    "import pyLDAvis\n",
    "import pyLDAvis.sklearn\n",
    "\n",
    "pyLDAvis.enable_notebook()\n",
    "panel = pyLDAvis.sklearn.prepare(best_lda_model, vectors, vectorizer, mds='tsne')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}