NLP1.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# NLTK Download"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "Nltk is python library for natural language processing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import nltk"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "showing info https://raw.githubusercontent.com/nltk/nltk_data/gh-pages/index.xml\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Downloading nltk library\n",
    "nltk.download() "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'3.2.1'"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nltk.__version__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Corpus - Body of text, singular. Corpora is the plural of this. Example: A collection of medical journals.\n",
    "#Lexicon - Words and their meanings.\n",
    "#Token - Each \"entity\" that is a part of whatever was split up based on rules."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Tokenizing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Tokenizing is nothing but a splitting a text into sentences or words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from nltk.tokenize import sent_tokenize,word_tokenize"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "EXAMPLE_TEXT = '''Hello Mr. Smith, how are you doing today? \n",
    "The weather is great, and Python is awesome. \n",
    "The sky is pinkish-blue. You shouldn't eat cardboard.'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['Hello Mr. Smith, how are you doing today?', 'The weather is great, and Python is awesome.', 'The sky is pinkish-blue.', \"You shouldn't eat cardboard.\"]\n"
     ]
    }
   ],
   "source": [
    "print(sent_tokenize(EXAMPLE_TEXT))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['Hello', 'Mr.', 'Smith', ',', 'how', 'are', 'you', 'doing', 'today', '?', 'The', 'weather', 'is', 'great', ',', 'and', 'Python', 'is', 'awesome', '.', 'The', 'sky', 'is', 'pinkish-blue', '.', 'You', 'should', \"n't\", 'eat', 'cardboard', '.']\n"
     ]
    }
   ],
   "source": [
    "print(word_tokenize(EXAMPLE_TEXT))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# StopWords"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Stopwords are nothing but a most repeatable words in a particular language(these words repeats in every sentences)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from nltk.corpus import stopwords"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'a',\n",
       " 'about',\n",
       " 'above',\n",
       " 'after',\n",
       " 'again',\n",
       " 'against',\n",
       " 'ain',\n",
       " 'all',\n",
       " 'am',\n",
       " 'an',\n",
       " 'and',\n",
       " 'any',\n",
       " 'are',\n",
       " 'aren',\n",
       " 'as',\n",
       " 'at',\n",
       " 'be',\n",
       " 'because',\n",
       " 'been',\n",
       " 'before',\n",
       " 'being',\n",
       " 'below',\n",
       " 'between',\n",
       " 'both',\n",
       " 'but',\n",
       " 'by',\n",
       " 'can',\n",
       " 'couldn',\n",
       " 'd',\n",
       " 'did',\n",
       " 'didn',\n",
       " 'do',\n",
       " 'does',\n",
       " 'doesn',\n",
       " 'doing',\n",
       " 'don',\n",
       " 'down',\n",
       " 'during',\n",
       " 'each',\n",
       " 'few',\n",
       " 'for',\n",
       " 'from',\n",
       " 'further',\n",
       " 'had',\n",
       " 'hadn',\n",
       " 'has',\n",
       " 'hasn',\n",
       " 'have',\n",
       " 'haven',\n",
       " 'having',\n",
       " 'he',\n",
       " 'her',\n",
       " 'here',\n",
       " 'hers',\n",
       " 'herself',\n",
       " 'him',\n",
       " 'himself',\n",
       " 'his',\n",
       " 'how',\n",
       " 'i',\n",
       " 'if',\n",
       " 'in',\n",
       " 'into',\n",
       " 'is',\n",
       " 'isn',\n",
       " 'it',\n",
       " 'its',\n",
       " 'itself',\n",
       " 'just',\n",
       " 'll',\n",
       " 'm',\n",
       " 'ma',\n",
       " 'me',\n",
       " 'mightn',\n",
       " 'more',\n",
       " 'most',\n",
       " 'mustn',\n",
       " 'my',\n",
       " 'myself',\n",
       " 'needn',\n",
       " 'no',\n",
       " 'nor',\n",
       " 'not',\n",
       " 'now',\n",
       " 'o',\n",
       " 'of',\n",
       " 'off',\n",
       " 'on',\n",
       " 'once',\n",
       " 'only',\n",
       " 'or',\n",
       " 'other',\n",
       " 'our',\n",
       " 'ours',\n",
       " 'ourselves',\n",
       " 'out',\n",
       " 'over',\n",
       " 'own',\n",
       " 're',\n",
       " 's',\n",
       " 'same',\n",
       " 'shan',\n",
       " 'she',\n",
       " 'should',\n",
       " 'shouldn',\n",
       " 'so',\n",
       " 'some',\n",
       " 'such',\n",
       " 't',\n",
       " 'than',\n",
       " 'that',\n",
       " 'the',\n",
       " 'their',\n",
       " 'theirs',\n",
       " 'them',\n",
       " 'themselves',\n",
       " 'then',\n",
       " 'there',\n",
       " 'these',\n",
       " 'they',\n",
       " 'this',\n",
       " 'those',\n",
       " 'through',\n",
       " 'to',\n",
       " 'too',\n",
       " 'under',\n",
       " 'until',\n",
       " 'up',\n",
       " 've',\n",
       " 'very',\n",
       " 'was',\n",
       " 'wasn',\n",
       " 'we',\n",
       " 'were',\n",
       " 'weren',\n",
       " 'what',\n",
       " 'when',\n",
       " 'where',\n",
       " 'which',\n",
       " 'while',\n",
       " 'who',\n",
       " 'whom',\n",
       " 'why',\n",
       " 'will',\n",
       " 'with',\n",
       " 'won',\n",
       " 'wouldn',\n",
       " 'y',\n",
       " 'you',\n",
       " 'your',\n",
       " 'yours',\n",
       " 'yourself',\n",
       " 'yourselves'}"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "set(stopwords.words(\"english\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "EXAMPLE_TEXT = '''Hello Mr. Smith, how are you doing today? \n",
    "The weather is great, and Python is awesome. \n",
    "The sky is pinkish-blue. You shouldn't eat cardboard.'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "tokens= word_tokenize(EXAMPLE_TEXT)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "particular_tokens= [w for w in tokens if not w in set(stopwords.words(\"english\"))]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "31"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(tokens)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "22"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(particular_tokens)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Stemming"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#stemming is a process of reducing a each word into base form (playing ----> play)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from nltk.stem import PorterStemmer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from nltk.tokenize import sent_tokenize, word_tokenize"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "ps= PorterStemmer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "example_words = [\"python\",\"pythoner\",\"pythoning\",\"pythoned\",\"pythonly\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "python\n",
      "python\n",
      "python\n",
      "python\n",
      "pythonli\n"
     ]
    }
   ],
   "source": [
    "for w in example_words:\n",
    "    print(ps.stem(w))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "new_text = \"It is important to by very pythonly while you are pythoning with python. All pythoners have pythoned poorly at least once.\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "words = word_tokenize(new_text)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "It\n",
      "is\n",
      "import\n",
      "to\n",
      "by\n",
      "veri\n",
      "pythonli\n",
      "while\n",
      "you\n",
      "are\n",
      "python\n",
      "with\n",
      "python\n",
      ".\n",
      "All\n",
      "python\n",
      "have\n",
      "python\n",
      "poorli\n",
      "at\n",
      "least\n",
      "onc\n",
      ".\n"
     ]
    }
   ],
   "source": [
    "for w in words:\n",
    "    print(ps.stem(w))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Parts of speech"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Labeling words in a sentence as nouns, adjectives, verbs...etc. "
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "POS tag list:\n",
    "\n",
    "CC\tcoordinating conjunction\n",
    "CD\tcardinal digit\n",
    "DT\tdeterminer\n",
    "EX\texistential there (like: \"there is\" ... think of it like \"there exists\")\n",
    "FW\tforeign word\n",
    "IN\tpreposition/subordinating conjunction\n",
    "JJ\tadjective\t'big'\n",
    "JJR\tadjective, comparative\t'bigger'\n",
    "JJS\tadjective, superlative\t'biggest'\n",
    "LS\tlist marker\t1)\n",
    "MD\tmodal\tcould, will\n",
    "NN\tnoun, singular 'desk'\n",
    "NNS\tnoun plural\t'desks'\n",
    "NNP\tproper noun, singular\t'Harrison'\n",
    "NNPS\tproper noun, plural\t'Americans'\n",
    "PDT\tpredeterminer\t'all the kids'\n",
    "POS\tpossessive ending\tparent's\n",
    "PRP\tpersonal pronoun\tI, he, she\n",
    "PRP$\tpossessive pronoun\tmy, his, hers\n",
    "RB\tadverb\tvery, silently,\n",
    "RBR\tadverb, comparative\tbetter\n",
    "RBS\tadverb, superlative\tbest\n",
    "RP\tparticle\tgive up\n",
    "TO\tto\tgo 'to' the store.\n",
    "UH\tinterjection\terrrrrrrrm\n",
    "VB\tverb, base form\ttake\n",
    "VBD\tverb, past tense\ttook\n",
    "VBG\tverb, gerund/present participle\ttaking\n",
    "VBN\tverb, past participle\ttaken\n",
    "VBP\tverb, sing. present, non-3d\ttake\n",
    "VBZ\tverb, 3rd person sing. present\ttakes\n",
    "WDT\twh-determiner\twhich\n",
    "WP\twh-pronoun\twho, what\n",
    "WP$\tpossessive wh-pronoun\twhose\n",
    "WRB\twh-abverb\twhere, when"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from nltk.corpus import state_union"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from nltk.tokenize import PunktSentenceTokenizer"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "A punkt sentence tokenizer which uses an unsupervised algorithm to build\n",
    "a model for abbreviation words, collocations, and words that start\n",
    "sentences; and then uses that model to find sentence boundaries.\n",
    "This approach has been shown to work well for many European\n",
    "languages."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['It',\n",
       " 'is',\n",
       " 'important',\n",
       " 'to',\n",
       " 'by',\n",
       " 'very',\n",
       " 'pythonly',\n",
       " 'while',\n",
       " 'you',\n",
       " 'are',\n",
       " 'pythoning',\n",
       " 'with',\n",
       " 'python',\n",
       " '.',\n",
       " 'All',\n",
       " 'pythoners',\n",
       " 'have',\n",
       " 'pythoned',\n",
       " 'poorly',\n",
       " 'at',\n",
       " 'least',\n",
       " 'once',\n",
       " '.']"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('It', 'PRP'),\n",
       " ('is', 'VBZ'),\n",
       " ('important', 'JJ'),\n",
       " ('to', 'TO'),\n",
       " ('by', 'IN'),\n",
       " ('very', 'RB'),\n",
       " ('pythonly', 'RB'),\n",
       " ('while', 'IN'),\n",
       " ('you', 'PRP'),\n",
       " ('are', 'VBP'),\n",
       " ('pythoning', 'VBG'),\n",
       " ('with', 'IN'),\n",
       " ('python', 'NN'),\n",
       " ('.', '.'),\n",
       " ('All', 'DT'),\n",
       " ('pythoners', 'NNS'),\n",
       " ('have', 'VBP'),\n",
       " ('pythoned', 'VBN'),\n",
       " ('poorly', 'RB'),\n",
       " ('at', 'IN'),\n",
       " ('least', 'JJS'),\n",
       " ('once', 'RB'),\n",
       " ('.', '.')]"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nltk.pos_tag(words)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Lemmating"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "A very similar operation to stemming is called lemmatizing. \n",
    "The major difference between these is, as you saw earlier, \n",
    "stemming can often create non-existent words, whereas lemmas are actual words."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from nltk.stem import WordNetLemmatizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "lemmatizer= WordNetLemmatizer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['It',\n",
       " 'is',\n",
       " 'important',\n",
       " 'to',\n",
       " 'by',\n",
       " 'very',\n",
       " 'pythonly',\n",
       " 'while',\n",
       " 'you',\n",
       " 'are',\n",
       " 'pythoning',\n",
       " 'with',\n",
       " 'python',\n",
       " '.',\n",
       " 'All',\n",
       " 'pythoners',\n",
       " 'have',\n",
       " 'pythoned',\n",
       " 'poorly',\n",
       " 'at',\n",
       " 'least',\n",
       " 'once',\n",
       " '.']"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "It\n",
      "is\n",
      "important\n",
      "to\n",
      "by\n",
      "very\n",
      "pythonly\n",
      "while\n",
      "you\n",
      "are\n",
      "pythoning\n",
      "with\n",
      "python\n",
      ".\n",
      "All\n",
      "pythoners\n",
      "have\n",
      "pythoned\n",
      "poorly\n",
      "at\n",
      "least\n",
      "once\n",
      ".\n"
     ]
    }
   ],
   "source": [
    "for w in words:\n",
    "    print(lemmatizer.lemmatize(w,pos='a'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [conda root]",
   "language": "python",
   "name": "conda-root-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}