Exercise 2 - Advanced Analytics NLP-Copy1.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Exercise 2: Advanced Analytics NLP"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-12-04T16:28:12.472305Z",
     "start_time": "2018-12-04T16:27:10.886061Z"
    }
   },
   "outputs": [],
   "source": [
    "!pip install spark-nlp==1.7.3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-12-04T16:28:14.748410Z",
     "start_time": "2018-12-04T16:28:14.342555Z"
    }
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "pd.set_option('max_colwidth', 800)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Create a spark context that includes a 3rd party jar for NLP"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-12-04T16:28:35.841869Z",
     "start_time": "2018-12-04T16:28:31.934985Z"
    }
   },
   "outputs": [],
   "source": [
    "#jarPath = \"spark-nlp-assembly-1.7.3.jar\"\n",
    "\n",
    "from pyspark.sql import SparkSession\n",
    "spark = SparkSession.builder \\\n",
    "    .config(\"spark.jars.packages\", \"JohnSnowLabs:spark-nlp:1.8.2\") \\\n",
    "    .getOrCreate()\n",
    "spark"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Read multiple files in a dir as one Dataframe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-12-04T16:28:40.343540Z",
     "start_time": "2018-12-04T16:28:35.844308Z"
    }
   },
   "outputs": [],
   "source": [
    "dataPath = \"./data/reddit/*.json\"\n",
    "df = spark.read.json(dataPath)\n",
    "print(df.count())\n",
    "df.printSchema()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Deal with Struct type to query subfields "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-12-04T16:28:40.819564Z",
     "start_time": "2018-12-04T16:28:40.345335Z"
    }
   },
   "outputs": [],
   "source": [
    "title = \"data.title\"\n",
    "author = \"data.author\"\n",
    "# Todo"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Try to implement the equivalent of flatMap in dataframes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-12-04T16:28:43.068755Z",
     "start_time": "2018-12-04T16:28:40.826537Z"
    }
   },
   "outputs": [],
   "source": [
    "import pyspark.sql.functions as F\n",
    "\n",
    "# Todo"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Use an NLP libary to do Part-of-Speech Tagging"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-12-04T16:29:10.967990Z",
     "start_time": "2018-12-04T16:28:43.072151Z"
    }
   },
   "outputs": [],
   "source": [
    "from com.johnsnowlabs.nlp.pretrained.pipeline.en import BasicPipeline as bp\n",
    "# Todo"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Deal with Map type to query subfields"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-12-04T16:29:11.430140Z",
     "start_time": "2018-12-04T16:29:10.973865Z"
    }
   },
   "outputs": [],
   "source": [
    "dfPos = dfAnnotated.select(\"text\", \"pos.metadata\", \"pos.result\")\n",
    "dfPos.limit(5).toPandas()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-12-04T16:29:12.012202Z",
     "start_time": "2018-12-04T16:29:11.432322Z"
    }
   },
   "outputs": [],
   "source": [
    "dfPos= dfAnnotated.select(F.explode(\"pos\").alias(\"pos\"))\n",
    "dfPos.printSchema()\n",
    "dfPos.toPandas()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Keep only proper nouns NNP or NNPS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-12-04T16:29:12.551881Z",
     "start_time": "2018-12-04T16:29:12.014196Z"
    }
   },
   "outputs": [],
   "source": [
    "nnpFilter = \"pos.result = 'NNP' or pos.result = 'NNPS' \"\n",
    "dfNNP = dfPos.where(nnpFilter)\n",
    "dfNNP.limit(10).toPandas()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Extract columns form a map in a col"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-12-04T16:29:12.811100Z",
     "start_time": "2018-12-04T16:29:12.556429Z"
    }
   },
   "outputs": [],
   "source": [
    "dfWordTag = dfNNP.selectExpr(\"pos.metadata['word'] as word\", \"pos.result as tag\")\n",
    "dfWordTag.limit(10).toPandas()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyspark.sql.functions import desc\n",
    "dfWordTag.groupBy(\"word\").count().orderBy(desc(\"count\")).show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {
    "height": "84px",
    "width": "160px"
   },
   "number_sections": false,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {
    "height": "calc(100% - 180px)",
    "left": "10px",
    "top": "150px",
    "width": "236px"
   },
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}