app-desc-analysis.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import nltk"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import string"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "desc = pandas.read_csv('appleStore_description.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>track_name</th>\n",
       "      <th>size_bytes</th>\n",
       "      <th>app_desc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>281656475</td>\n",
       "      <td>PAC-MAN Premium</td>\n",
       "      <td>100788224</td>\n",
       "      <td>SAVE 20%, now only $3.99 for a limited time!\\n...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>281796108</td>\n",
       "      <td>Evernote - stay organized</td>\n",
       "      <td>158578688</td>\n",
       "      <td>Let Evernote change the way you organize your ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>281940292</td>\n",
       "      <td>WeatherBug - Local Weather, Radar, Maps, Alerts</td>\n",
       "      <td>100524032</td>\n",
       "      <td>Download the most popular free weather app pow...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>282614216</td>\n",
       "      <td>eBay: Best App to Buy, Sell, Save! Online Shop...</td>\n",
       "      <td>128512000</td>\n",
       "      <td>The eBay app is the best way to find anything ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>282935706</td>\n",
       "      <td>Bible</td>\n",
       "      <td>92774400</td>\n",
       "      <td>On more than 250 million devices around the wo...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          id                                         track_name  size_bytes  \\\n",
       "0  281656475                                    PAC-MAN Premium   100788224   \n",
       "1  281796108                          Evernote - stay organized   158578688   \n",
       "2  281940292    WeatherBug - Local Weather, Radar, Maps, Alerts   100524032   \n",
       "3  282614216  eBay: Best App to Buy, Sell, Save! Online Shop...   128512000   \n",
       "4  282935706                                              Bible    92774400   \n",
       "\n",
       "                                            app_desc  \n",
       "0  SAVE 20%, now only $3.99 for a limited time!\\n...  \n",
       "1  Let Evernote change the way you organize your ...  \n",
       "2  Download the most popular free weather app pow...  \n",
       "3  The eBay app is the best way to find anything ...  \n",
       "4  On more than 250 million devices around the wo...  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "desc.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. Count the number of words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# test for the first line\n",
    "mac_man_desc = desc.app_desc[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['save', 'now', 'only', 'for', 'a', 'limited', 'time', 'one', 'of', 'the', 'most', 'popular', 'video', 'games', 'in', 'arcade', 'history', 'world', 'video', 'game', 'hall', 'of', 'fame', 'inductee', 'who', 'can', 'forget', 'the', 'countless', 'hours', 'and', 'quarters', 'spent', 'outrunning', 'pesky', 'ghosts', 'and', 'chompin', 'on', 'dots', 'now', 'you', 'can', 'have', 'the', 'same', 'arcade', 'excitement', 'on', 'your', 'mobile', 'devices', 'guide', 'pac', 'man', 'through', 'the', 'mazes', 'with', 'easy', 'swipe', 'controls', 'a', 'mfi', 'controller', 'or', 'kick', 'it', 'old', 'school', 'with', 'the', 'onscreen', 'joystick', 'eat', 'all', 'of', 'the', 'dots', 'to', 'advance', 'to', 'the', 'next', 'stage', 'go', 'for', 'high', 'scores', 'and', 'higher', 'levels', 'gain', 'an', 'extra', 'life', 'at', 'points', 'gobble', 'power']\n"
     ]
    }
   ],
   "source": [
    "# split by whitespace and remove punctuation, then converted into lower cases\n",
    "mac_man_desc_words = [str.lower(word) for word in re.split(r'\\W+', mac_man_desc) if word.isalpha()]\n",
    "print(mac_man_desc_words[:100])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "262"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# check the number of words for first entry \n",
    "len(mac_man_desc_words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# define a function to convert a string of sentense into list of words\n",
    "def to_words_list(sentense):\n",
    "    return [str.lower(word) for word in re.split(r'\\W+', sentense) if word.isalpha()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['save', 'now', 'only', 'for', 'a', 'limited', 'time', 'one', 'of', 'the', 'most', 'popular', 'video', 'games', 'in', 'arcade', 'history', 'world', 'video', 'game', 'hall', 'of', 'fame', 'inductee', 'who', 'can', 'forget', 'the', 'countless', 'hours', 'and', 'quarters', 'spent', 'outrunning', 'pesky', 'ghosts', 'and', 'chompin', 'on', 'dots', 'now', 'you', 'can', 'have', 'the', 'same', 'arcade', 'excitement', 'on', 'your', 'mobile', 'devices', 'guide', 'pac', 'man', 'through', 'the', 'mazes', 'with', 'easy', 'swipe', 'controls', 'a', 'mfi', 'controller', 'or', 'kick', 'it', 'old', 'school', 'with', 'the', 'onscreen', 'joystick', 'eat', 'all', 'of', 'the', 'dots', 'to', 'advance', 'to', 'the', 'next', 'stage', 'go', 'for', 'high', 'scores', 'and', 'higher', 'levels', 'gain', 'an', 'extra', 'life', 'at', 'points', 'gobble', 'power', 'pellets', 'to', 'weaken', 'ghosts', 'temporarily', 'and', 'eat', 'them', 'up', 'before', 'they', 'change', 'back', 'avoid', 'blinky', 'the', 'leader', 'of', 'the', 'ghosts', 'and', 'his', 'fellow', 'ghosts', 'pinky', 'inky', 'and', 'clyde', 'or', 'you', 'will', 'lose', 'a', 'life', 'it', 's', 'game', 'over', 'when', 'you', 'lose', 'all', 'your', 'lives', 'new', 'mazes', 'included', 'the', 'game', 'includes', 'new', 'mazes', 'in', 'addition', 'to', 'the', 'pixel', 'for', 'pixel', 'recreation', 'of', 'the', 'classic', 'original', 'maze', 'challenge', 'your', 'skill', 'to', 'beat', 'them', 'all', 'we', 'are', 'constantly', 'updating', 'the', 'game', 'with', 'new', 'maze', 'packs', 'that', 'you', 'can', 'buy', 'to', 'complete', 'your', 'pac', 'man', 'collection', 'hints', 'and', 'tips', 'insider', 'pro', 'tips', 'and', 'hints', 'are', 'being', 'made', 'available', 'for', 'the', 'first', 'time', 'in', 'game', 'use', 'these', 'to', 'help', 'you', 'become', 'a', 'pac', 'man', 'champion', 'features', 'new', 'tournaments', 'new', 'visual', 'hints', 'and', 'pro', 'tips', 'new', 'mazes', 'for', 'all', 'new', 'challenges', 'play', 'an', 'arcade', 'perfect', 'port', 'of', 'classic', 'pac', 'man', 'two', 'different', 'control', 'modes', 'three', 'game', 'difficulties', 'including', 'the', 'original', 'arcade', 'game', 'retina', 'display', 'support', 'mfi', 'controller', 'support']\n"
     ]
    }
   ],
   "source": [
    "# test the function\n",
    "print(to_words_list(desc.app_desc[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# apply function to entire data \n",
    "desc['words_list'] = desc.app_desc.apply(lambda x: to_words_list(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# count the number of words in each list\n",
    "desc['word_count'] = desc.words_list.apply(lambda x: len(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>app_desc</th>\n",
       "      <th>words_list</th>\n",
       "      <th>word_count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>SAVE 20%, now only $3.99 for a limited time!\\n...</td>\n",
       "      <td>[save, now, only, for, a, limited, time, one, ...</td>\n",
       "      <td>262</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Let Evernote change the way you organize your ...</td>\n",
       "      <td>[let, evernote, change, the, way, you, organiz...</td>\n",
       "      <td>600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Download the most popular free weather app pow...</td>\n",
       "      <td>[download, the, most, popular, free, weather, ...</td>\n",
       "      <td>297</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>The eBay app is the best way to find anything ...</td>\n",
       "      <td>[the, ebay, app, is, the, best, way, to, find,...</td>\n",
       "      <td>662</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>On more than 250 million devices around the wo...</td>\n",
       "      <td>[on, more, than, million, devices, around, the...</td>\n",
       "      <td>476</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                            app_desc  \\\n",
       "0  SAVE 20%, now only $3.99 for a limited time!\\n...   \n",
       "1  Let Evernote change the way you organize your ...   \n",
       "2  Download the most popular free weather app pow...   \n",
       "3  The eBay app is the best way to find anything ...   \n",
       "4  On more than 250 million devices around the wo...   \n",
       "\n",
       "                                          words_list  word_count  \n",
       "0  [save, now, only, for, a, limited, time, one, ...         262  \n",
       "1  [let, evernote, change, the, way, you, organiz...         600  \n",
       "2  [download, the, most, popular, free, weather, ...         297  \n",
       "3  [the, ebay, app, is, the, best, way, to, find,...         662  \n",
       "4  [on, more, than, million, devices, around, the...         476  "
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# check the result \n",
    "desc[['app_desc','words_list','word_count']].head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count    7197.000000\n",
       "mean      240.636932\n",
       "std       164.968363\n",
       "min         2.000000\n",
       "25%       105.000000\n",
       "50%       213.000000\n",
       "75%       346.000000\n",
       "max       728.000000\n",
       "Name: word_count, dtype: float64"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# app description words statistics \n",
    "desc.word_count.describe()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1.1 Count number of digits  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "7"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# test for the first line\n",
    "matched = re.findall(r'\\d+\\.?\\d*', mac_man_desc)\n",
    "len(matched)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# define a function\n",
    "def count_digits(sentense):\n",
    "    return len(re.findall(r'\\d+\\.?\\d*', sentense))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "7"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "count_digits(mac_man_desc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "desc['digit_count'] = desc.app_desc.apply(lambda x: count_digits(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>app_desc</th>\n",
       "      <th>words_list</th>\n",
       "      <th>word_count</th>\n",
       "      <th>digit_count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>SAVE 20%, now only $3.99 for a limited time!\\n...</td>\n",
       "      <td>[save, now, only, for, a, limited, time, one, ...</td>\n",
       "      <td>262</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Let Evernote change the way you organize your ...</td>\n",
       "      <td>[let, evernote, change, the, way, you, organiz...</td>\n",
       "      <td>600</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Download the most popular free weather app pow...</td>\n",
       "      <td>[download, the, most, popular, free, weather, ...</td>\n",
       "      <td>297</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>The eBay app is the best way to find anything ...</td>\n",
       "      <td>[the, ebay, app, is, the, best, way, to, find,...</td>\n",
       "      <td>662</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>On more than 250 million devices around the wo...</td>\n",
       "      <td>[on, more, than, million, devices, around, the...</td>\n",
       "      <td>476</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                            app_desc  \\\n",
       "0  SAVE 20%, now only $3.99 for a limited time!\\n...   \n",
       "1  Let Evernote change the way you organize your ...   \n",
       "2  Download the most popular free weather app pow...   \n",
       "3  The eBay app is the best way to find anything ...   \n",
       "4  On more than 250 million devices around the wo...   \n",
       "\n",
       "                                          words_list  word_count  digit_count  \n",
       "0  [save, now, only, for, a, limited, time, one, ...         262            7  \n",
       "1  [let, evernote, change, the, way, you, organiz...         600            8  \n",
       "2  [download, the, most, popular, free, weather, ...         297            8  \n",
       "3  [the, ebay, app, is, the, best, way, to, find,...         662            0  \n",
       "4  [on, more, than, million, devices, around, the...         476            8  "
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "desc[['app_desc','words_list','word_count','digit_count']].head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2. Filter stopwords "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "from nltk.tokenize import word_tokenize\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "stop_words = nltk.corpus.stopwords.words('english')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "161"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# test for first line\n",
    "mac_man_filtered_words = [w for w in mac_man_desc_words if not w in stop_words]\n",
    "len(mac_man_filtered_words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "# define a function to filter stop words in word list\n",
    "def filter_stop_words(word_list):\n",
    "    return [w for w in word_list if not w in stop_words]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "desc['filtered_word_list'] = desc.words_list.apply(lambda x: filter_stop_words(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>app_desc</th>\n",
       "      <th>words_list</th>\n",
       "      <th>filtered_word_list</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>SAVE 20%, now only $3.99 for a limited time!\\n...</td>\n",
       "      <td>[save, now, only, for, a, limited, time, one, ...</td>\n",
       "      <td>[save, limited, time, one, popular, video, gam...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Let Evernote change the way you organize your ...</td>\n",
       "      <td>[let, evernote, change, the, way, you, organiz...</td>\n",
       "      <td>[let, evernote, change, way, organize, persona...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Download the most popular free weather app pow...</td>\n",
       "      <td>[download, the, most, popular, free, weather, ...</td>\n",
       "      <td>[download, popular, free, weather, app, powere...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>The eBay app is the best way to find anything ...</td>\n",
       "      <td>[the, ebay, app, is, the, best, way, to, find,...</td>\n",
       "      <td>[ebay, app, best, way, find, anything, need, d...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>On more than 250 million devices around the wo...</td>\n",
       "      <td>[on, more, than, million, devices, around, the...</td>\n",
       "      <td>[million, devices, around, world, people, read...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                            app_desc  \\\n",
       "0  SAVE 20%, now only $3.99 for a limited time!\\n...   \n",
       "1  Let Evernote change the way you organize your ...   \n",
       "2  Download the most popular free weather app pow...   \n",
       "3  The eBay app is the best way to find anything ...   \n",
       "4  On more than 250 million devices around the wo...   \n",
       "\n",
       "                                          words_list  \\\n",
       "0  [save, now, only, for, a, limited, time, one, ...   \n",
       "1  [let, evernote, change, the, way, you, organiz...   \n",
       "2  [download, the, most, popular, free, weather, ...   \n",
       "3  [the, ebay, app, is, the, best, way, to, find,...   \n",
       "4  [on, more, than, million, devices, around, the...   \n",
       "\n",
       "                                  filtered_word_list  \n",
       "0  [save, limited, time, one, popular, video, gam...  \n",
       "1  [let, evernote, change, way, organize, persona...  \n",
       "2  [download, popular, free, weather, app, powere...  \n",
       "3  [ebay, app, best, way, find, anything, need, d...  \n",
       "4  [million, devices, around, world, people, read...  "
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "desc[['app_desc','words_list','filtered_word_list']].head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "desc['filtered_word_count'] = desc.filtered_word_list.apply(lambda x: len(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>app_desc</th>\n",
       "      <th>word_count</th>\n",
       "      <th>filtered_word_count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>SAVE 20%, now only $3.99 for a limited time!\\n...</td>\n",
       "      <td>262</td>\n",
       "      <td>161</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Let Evernote change the way you organize your ...</td>\n",
       "      <td>600</td>\n",
       "      <td>396</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Download the most popular free weather app pow...</td>\n",
       "      <td>297</td>\n",
       "      <td>223</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>The eBay app is the best way to find anything ...</td>\n",
       "      <td>662</td>\n",
       "      <td>415</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>On more than 250 million devices around the wo...</td>\n",
       "      <td>476</td>\n",
       "      <td>300</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                            app_desc  word_count  \\\n",
       "0  SAVE 20%, now only $3.99 for a limited time!\\n...         262   \n",
       "1  Let Evernote change the way you organize your ...         600   \n",
       "2  Download the most popular free weather app pow...         297   \n",
       "3  The eBay app is the best way to find anything ...         662   \n",
       "4  On more than 250 million devices around the wo...         476   \n",
       "\n",
       "   filtered_word_count  \n",
       "0                  161  \n",
       "1                  396  \n",
       "2                  223  \n",
       "3                  415  \n",
       "4                  300  "
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "desc[['app_desc','word_count','filtered_word_count']].head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3. Normalize by lemmatization"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Lemmatization is a more effective option than stemming because it converts the word into its root word, rather than just stripping the suffices. It makes use of the vocabulary and does a morphological analysis to obtain the root word. Therefore, we usually prefer using lemmatization over stemming."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "lemmer = nltk.stem.WordNetLemmatizer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "def LemTokens(tokens):\n",
    "     return [lemmer.lemmatize(token) for token in tokens]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "# test for the first line\n",
    "mac_man_lem_words= LemTokens(mac_man_filtered_words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "desc['lem_words'] = desc.filtered_word_list.apply(lambda x: LemTokens(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>app_desc</th>\n",
       "      <th>lem_words</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>SAVE 20%, now only $3.99 for a limited time!\\n...</td>\n",
       "      <td>[save, limited, time, one, popular, video, gam...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Let Evernote change the way you organize your ...</td>\n",
       "      <td>[let, evernote, change, way, organize, persona...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Download the most popular free weather app pow...</td>\n",
       "      <td>[download, popular, free, weather, app, powere...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>The eBay app is the best way to find anything ...</td>\n",
       "      <td>[ebay, app, best, way, find, anything, need, d...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>On more than 250 million devices around the wo...</td>\n",
       "      <td>[million, device, around, world, people, readi...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                            app_desc  \\\n",
       "0  SAVE 20%, now only $3.99 for a limited time!\\n...   \n",
       "1  Let Evernote change the way you organize your ...   \n",
       "2  Download the most popular free weather app pow...   \n",
       "3  The eBay app is the best way to find anything ...   \n",
       "4  On more than 250 million devices around the wo...   \n",
       "\n",
       "                                           lem_words  \n",
       "0  [save, limited, time, one, popular, video, gam...  \n",
       "1  [let, evernote, change, way, organize, persona...  \n",
       "2  [download, popular, free, weather, app, powere...  \n",
       "3  [ebay, app, best, way, find, anything, need, d...  \n",
       "4  [million, device, around, world, people, readi...  "
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "desc[['app_desc','lem_words']].head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 4. Turn text into vectors of term frequency"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 4.1 What are the most common words for the first entry?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('game', 8),\n",
       " ('new', 7),\n",
       " ('maze', 6),\n",
       " ('arcade', 4),\n",
       " ('ghost', 4),\n",
       " ('pac', 4),\n",
       " ('man', 4),\n",
       " ('life', 3),\n",
       " ('hint', 3),\n",
       " ('tip', 3),\n",
       " ('time', 2),\n",
       " ('video', 2),\n",
       " ('dot', 2),\n",
       " ('control', 2),\n",
       " ('mfi', 2),\n",
       " ('controller', 2),\n",
       " ('eat', 2),\n",
       " ('lose', 2),\n",
       " ('pixel', 2),\n",
       " ('classic', 2)]"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text = nltk.Text(mac_man_lem_words)\n",
    "# Calculate Frequency distribution\n",
    "freq = nltk.FreqDist(text)\n",
    "\n",
    "# Print and plot most common words\n",
    "freq.most_common(20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "freq.plot(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "nltk.probability.FreqDist"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(freq)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 4.2 Calculate the term frequency for each text in sorted order"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "def cal_freq(word_list):\n",
    "    text = nltk.Text(word_list)\n",
    "    freq = nltk.FreqDist(text)\n",
    "    return freq.most_common()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "desc['freq'] = desc.lem_words.apply(lambda x: cal_freq(x))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    [(game, 8), (new, 7), (maze, 6), (arcade, 4), ...\n",
       "1    [(evernote, 21), (note, 14), (device, 6), (wor...\n",
       "2    [(weather, 17), (alert, 7), (network, 6), (for...\n",
       "3    [(item, 16), (ebay, 15), (find, 10), (new, 10)...\n",
       "4    [(bible, 28), (app, 8), (version, 6), (share, ...\n",
       "Name: freq, dtype: object"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "desc.freq.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}