Step3_MergeData.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Merge data\n",
    "* Data format is following Hyesoo's format\n",
    "* All data is exported to a csv file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import os"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Import Jinmei's data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### import fake news"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "path = os.path.join('data', 'fakenews_jz.csv')\n",
    "df_fakenews = pd.read_csv(path,usecols=[1,2,3,4,5])\n",
    "df_fakenews['authenticity'] = 1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### import real news"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "path = os.path.join('data', 'realnews_jz.csv')\n",
    "df_realnews = pd.read_csv(path,usecols=[0,1,2,3,4])\n",
    "df_realnews['authenticity'] = 0"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### merge fake and real news"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>url</th>\n",
       "      <th>source</th>\n",
       "      <th>title</th>\n",
       "      <th>author</th>\n",
       "      <th>text</th>\n",
       "      <th>authenticity</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1271</th>\n",
       "      <td>http://www.newyorker.com/news/news-desk</td>\n",
       "      <td>newyorker</td>\n",
       "      <td>News Desk: Breaking News, Reporting, and Polit...</td>\n",
       "      <td>[]</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1272</th>\n",
       "      <td>http://www.newyorker.com/cartoon/dernavich-201...</td>\n",
       "      <td>newyorker</td>\n",
       "      <td>A Cartoon from The New Yorker</td>\n",
       "      <td>[]</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1273</th>\n",
       "      <td>http://tunein.com/radio/New-Yorker-Poetry-p803...</td>\n",
       "      <td>newyorker</td>\n",
       "      <td>New Yorker Poetry</td>\n",
       "      <td>[]</td>\n",
       "      <td>Yusef Komunyakaa reads a poem by Marilyn Hacke...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1274</th>\n",
       "      <td>http://video.newyorker.com/watch/shorts-murmur...</td>\n",
       "      <td>newyorker</td>\n",
       "      <td>The Startup to End All Startups</td>\n",
       "      <td>[]</td>\n",
       "      <td>The Startup to End All Startups\\n\\nMeet uBox, ...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1275</th>\n",
       "      <td>http://www.newyorker.com/humor/borowitz-report...</td>\n",
       "      <td>newyorker</td>\n",
       "      <td>Cruz: “The Dream of Keeping Poor People from S...</td>\n",
       "      <td>[]</td>\n",
       "      <td>WASHINGTON ( The Borowitz Report )—Acknowledgi...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                    url     source  \\\n",
       "1271            http://www.newyorker.com/news/news-desk  newyorker   \n",
       "1272  http://www.newyorker.com/cartoon/dernavich-201...  newyorker   \n",
       "1273  http://tunein.com/radio/New-Yorker-Poetry-p803...  newyorker   \n",
       "1274  http://video.newyorker.com/watch/shorts-murmur...  newyorker   \n",
       "1275  http://www.newyorker.com/humor/borowitz-report...  newyorker   \n",
       "\n",
       "                                                  title author  \\\n",
       "1271  News Desk: Breaking News, Reporting, and Polit...     []   \n",
       "1272                      A Cartoon from The New Yorker     []   \n",
       "1273                                  New Yorker Poetry     []   \n",
       "1274                    The Startup to End All Startups     []   \n",
       "1275  Cruz: “The Dream of Keeping Poor People from S...     []   \n",
       "\n",
       "                                                   text  authenticity  \n",
       "1271                                                NaN             0  \n",
       "1272                                                NaN             0  \n",
       "1273  Yusef Komunyakaa reads a poem by Marilyn Hacke...             0  \n",
       "1274  The Startup to End All Startups\\n\\nMeet uBox, ...             0  \n",
       "1275  WASHINGTON ( The Borowitz Report )—Acknowledgi...             0  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_jinmei = pd.concat([df_fakenews, df_realnews], ignore_index=True)\n",
    "df_jinmei.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1276, 6)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_jinmei.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### clean data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>url</th>\n",
       "      <th>source</th>\n",
       "      <th>title</th>\n",
       "      <th>author</th>\n",
       "      <th>text</th>\n",
       "      <th>authenticity</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1268</th>\n",
       "      <td>http://www.newyorker.com/humor/borowitz-report...</td>\n",
       "      <td>newyorker</td>\n",
       "      <td>Stephen Hawking Angers Trump Supporters with B...</td>\n",
       "      <td>[]</td>\n",
       "      <td>LONDON ( The Borowitz Report )—The theoretical...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1269</th>\n",
       "      <td>http://tunein.com/radio/New-Yorker-Radio-Hour-...</td>\n",
       "      <td>newyorker</td>\n",
       "      <td>The New Yorker Radio Hour</td>\n",
       "      <td>[]</td>\n",
       "      <td>Description:\\n\\nDavid Remnick is joined by The...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1273</th>\n",
       "      <td>http://tunein.com/radio/New-Yorker-Poetry-p803...</td>\n",
       "      <td>newyorker</td>\n",
       "      <td>New Yorker Poetry</td>\n",
       "      <td>[]</td>\n",
       "      <td>Yusef Komunyakaa reads a poem by Marilyn Hacke...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1274</th>\n",
       "      <td>http://video.newyorker.com/watch/shorts-murmur...</td>\n",
       "      <td>newyorker</td>\n",
       "      <td>The Startup to End All Startups</td>\n",
       "      <td>[]</td>\n",
       "      <td>The Startup to End All Startups\\n\\nMeet uBox, ...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1275</th>\n",
       "      <td>http://www.newyorker.com/humor/borowitz-report...</td>\n",
       "      <td>newyorker</td>\n",
       "      <td>Cruz: “The Dream of Keeping Poor People from S...</td>\n",
       "      <td>[]</td>\n",
       "      <td>WASHINGTON ( The Borowitz Report )—Acknowledgi...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                    url     source  \\\n",
       "1268  http://www.newyorker.com/humor/borowitz-report...  newyorker   \n",
       "1269  http://tunein.com/radio/New-Yorker-Radio-Hour-...  newyorker   \n",
       "1273  http://tunein.com/radio/New-Yorker-Poetry-p803...  newyorker   \n",
       "1274  http://video.newyorker.com/watch/shorts-murmur...  newyorker   \n",
       "1275  http://www.newyorker.com/humor/borowitz-report...  newyorker   \n",
       "\n",
       "                                                  title author  \\\n",
       "1268  Stephen Hawking Angers Trump Supporters with B...     []   \n",
       "1269                          The New Yorker Radio Hour     []   \n",
       "1273                                  New Yorker Poetry     []   \n",
       "1274                    The Startup to End All Startups     []   \n",
       "1275  Cruz: “The Dream of Keeping Poor People from S...     []   \n",
       "\n",
       "                                                   text  authenticity  \n",
       "1268  LONDON ( The Borowitz Report )—The theoretical...             0  \n",
       "1269  Description:\\n\\nDavid Remnick is joined by The...             0  \n",
       "1273  Yusef Komunyakaa reads a poem by Marilyn Hacke...             0  \n",
       "1274  The Startup to End All Startups\\n\\nMeet uBox, ...             0  \n",
       "1275  WASHINGTON ( The Borowitz Report )—Acknowledgi...             0  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# drop rows where text contains NaN\n",
    "df_jinmei = df_jinmei.dropna(axis=0,subset=['text'], how='any') \n",
    "df_jinmei.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1213, 6)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_jinmei.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(822, 6)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# drop rows where the length of characters in text is less than 500\n",
    "df_jinmei = df_jinmei[df_jinmei.text.map(len) > 500]\n",
    "df_jinmei.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Import Hyesoo's data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>url</th>\n",
       "      <th>source</th>\n",
       "      <th>title</th>\n",
       "      <th>author</th>\n",
       "      <th>text</th>\n",
       "      <th>authenticity</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1736</th>\n",
       "      <td>https://www.ice.gov/news/releases/operation-ma...</td>\n",
       "      <td>politico</td>\n",
       "      <td>Operation Matador nets 39 MS-13 arrests in las...</td>\n",
       "      <td>[]</td>\n",
       "      <td>NEW YORK – U.S. Immigration and Customs Enforc...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1737</th>\n",
       "      <td>http://www.politico.com/story/2017/07/27/obama...</td>\n",
       "      <td>politico</td>\n",
       "      <td>Senate Republicans prepare to pass Obamacare r...</td>\n",
       "      <td>['John Bresnahan', 'Burgess Everett', 'Jennife...</td>\n",
       "      <td>Senate Republicans are closing in on passage o...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1738</th>\n",
       "      <td>https://www.nytimes.com/2017/07/26/technology/...</td>\n",
       "      <td>nytimes</td>\n",
       "      <td>Google’s New Parental Control App Has a Flaw: ...</td>\n",
       "      <td>['Brian X. Chen', 'Tech Fix']</td>\n",
       "      <td>“The fact that the kid can graduate themselves...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1739</th>\n",
       "      <td>http://www.foxnews.com/entertainment/2017/07/2...</td>\n",
       "      <td>foxnews</td>\n",
       "      <td>Hulu resurrects TGIF lineup with acquisition o...</td>\n",
       "      <td>['Tyler Mccarthy']</td>\n",
       "      <td>Hulu is hoping to make itself the go-to stream...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1740</th>\n",
       "      <td>http://www.npr.org/2017/07/27/539559582/5-unan...</td>\n",
       "      <td>npr</td>\n",
       "      <td>5 Unanswered Questions About Trump's 'Ban' On ...</td>\n",
       "      <td>['Philip Ewing']</td>\n",
       "      <td>5 Unanswered Questions About Trump's 'Ban' On ...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                    url    source  \\\n",
       "1736  https://www.ice.gov/news/releases/operation-ma...  politico   \n",
       "1737  http://www.politico.com/story/2017/07/27/obama...  politico   \n",
       "1738  https://www.nytimes.com/2017/07/26/technology/...   nytimes   \n",
       "1739  http://www.foxnews.com/entertainment/2017/07/2...   foxnews   \n",
       "1740  http://www.npr.org/2017/07/27/539559582/5-unan...       npr   \n",
       "\n",
       "                                                  title  \\\n",
       "1736  Operation Matador nets 39 MS-13 arrests in las...   \n",
       "1737  Senate Republicans prepare to pass Obamacare r...   \n",
       "1738  Google’s New Parental Control App Has a Flaw: ...   \n",
       "1739  Hulu resurrects TGIF lineup with acquisition o...   \n",
       "1740  5 Unanswered Questions About Trump's 'Ban' On ...   \n",
       "\n",
       "                                                 author  \\\n",
       "1736                                                 []   \n",
       "1737  ['John Bresnahan', 'Burgess Everett', 'Jennife...   \n",
       "1738                      ['Brian X. Chen', 'Tech Fix']   \n",
       "1739                                 ['Tyler Mccarthy']   \n",
       "1740                                   ['Philip Ewing']   \n",
       "\n",
       "                                                   text  authenticity  \n",
       "1736  NEW YORK – U.S. Immigration and Customs Enforc...             0  \n",
       "1737  Senate Republicans are closing in on passage o...             0  \n",
       "1738  “The fact that the kid can graduate themselves...             0  \n",
       "1739  Hulu is hoping to make itself the go-to stream...             0  \n",
       "1740  5 Unanswered Questions About Trump's 'Ban' On ...             0  "
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "path = os.path.join('data', 'hyesoo_df.csv')\n",
    "df_hyesoo = pd.read_csv(path,usecols=[1,2,3,4,5,6]) \n",
    "swap = {1:0, 0:1}\n",
    "df_hyesoo.authenticity = df_hyesoo.authenticity.map(swap)\n",
    "df_hyesoo.tail()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Import additional fake data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1    476\n",
       "Name: authenticity, dtype: int64"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "path = os.path.join('data', 'Additional_fake_df.csv')\n",
    "df_add = pd.read_csv(path,usecols=[1,2,3,4,5,6]) \n",
    "df_add.drop(df_add.index[474], inplace = True)\n",
    "# df_add.tail()\n",
    "df_add.authenticity.value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Merge all data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>url</th>\n",
       "      <th>source</th>\n",
       "      <th>title</th>\n",
       "      <th>author</th>\n",
       "      <th>text</th>\n",
       "      <th>authenticity</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>3034</th>\n",
       "      <td>http://americanflavor.news/2017/04/30/trump-ad...</td>\n",
       "      <td>AmericanFlavor</td>\n",
       "      <td>After Trump Administration Negotiations China ...</td>\n",
       "      <td>[]</td>\n",
       "      <td>A Houston woman held in custody by China in 20...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3035</th>\n",
       "      <td>https://sputniknews.com/columnists/20170713105...</td>\n",
       "      <td>sputniknews</td>\n",
       "      <td>Russia 'Collusion' Smoking Gun?</td>\n",
       "      <td>[]</td>\n",
       "      <td>This is the proof, we are told, that the Trump...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3036</th>\n",
       "      <td>http://pamelageller.com/2017/07/state-departme...</td>\n",
       "      <td>pamelageller</td>\n",
       "      <td>State Department Removes Word ‘Genocide’ From ...</td>\n",
       "      <td>['Geller Report Staff', 'Mark Steiner', 'David...</td>\n",
       "      <td>The U.S. State Department under President Dona...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3037</th>\n",
       "      <td>https://sputniknews.com/science/20170801105605...</td>\n",
       "      <td>sputniknews</td>\n",
       "      <td>To the Stars We Return: Start-Ups Offering Spa...</td>\n",
       "      <td>[]</td>\n",
       "      <td>A Houston-based startup will send your cremate...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3038</th>\n",
       "      <td>https://sputniknews.com/radio_trendstorm/20170...</td>\n",
       "      <td>sputniknews</td>\n",
       "      <td>China &amp; India: Walking the Himalayan Tightrope</td>\n",
       "      <td>[]</td>\n",
       "      <td>China &amp; India: Walking the Himalayan Tightrope...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                    url          source  \\\n",
       "3034  http://americanflavor.news/2017/04/30/trump-ad...  AmericanFlavor   \n",
       "3035  https://sputniknews.com/columnists/20170713105...     sputniknews   \n",
       "3036  http://pamelageller.com/2017/07/state-departme...    pamelageller   \n",
       "3037  https://sputniknews.com/science/20170801105605...     sputniknews   \n",
       "3038  https://sputniknews.com/radio_trendstorm/20170...     sputniknews   \n",
       "\n",
       "                                                  title  \\\n",
       "3034  After Trump Administration Negotiations China ...   \n",
       "3035                    Russia 'Collusion' Smoking Gun?   \n",
       "3036  State Department Removes Word ‘Genocide’ From ...   \n",
       "3037  To the Stars We Return: Start-Ups Offering Spa...   \n",
       "3038     China & India: Walking the Himalayan Tightrope   \n",
       "\n",
       "                                                 author  \\\n",
       "3034                                                 []   \n",
       "3035                                                 []   \n",
       "3036  ['Geller Report Staff', 'Mark Steiner', 'David...   \n",
       "3037                                                 []   \n",
       "3038                                                 []   \n",
       "\n",
       "                                                   text  authenticity  \n",
       "3034  A Houston woman held in custody by China in 20...             1  \n",
       "3035  This is the proof, we are told, that the Trump...             1  \n",
       "3036  The U.S. State Department under President Dona...             1  \n",
       "3037  A Houston-based startup will send your cremate...             1  \n",
       "3038  China & India: Walking the Himalayan Tightrope...             1  "
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_merge = pd.concat([df_jinmei, df_hyesoo, df_add], ignore_index=True)\n",
    "df_merge.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    1566\n",
       "1    1473\n",
       "Name: authenticity, dtype: int64"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_merge.authenticity.value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Export data to csv file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "path = os.path.join('data','merged_data.csv')\n",
    "df_merge.to_csv(path)  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9406130268199234"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "A = 1473/1566\n",
    "A"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "## Balance data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### sample real news\n",
    "number of real news ~= number of fake news"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1473, 6)"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_balanced_realnews = df_merge[df_merge.authenticity==0]\n",
    "data_balanced_realnews = data_balanced_realnews.sample(frac=0.9406)\n",
    "data_balanced_realnews.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### get all fake news"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1473, 6)"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_balanced_fakenews = df_merge[df_merge.authenticity==1]\n",
    "data_balanced_fakenews.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### combine real and fake news"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "data_balanced = pd.concat([data_balanced_realnews, data_balanced_fakenews], ignore_index=True)\n",
    "data_balanced.authenticity.value_counts()\n",
    "data_balanced = data_balanced.sample(frac=1).reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### export data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "path = os.path.join('data','balanced_data.csv')\n",
    "data_balanced.to_csv(path)  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>url</th>\n",
       "      <th>source</th>\n",
       "      <th>title</th>\n",
       "      <th>author</th>\n",
       "      <th>text</th>\n",
       "      <th>authenticity</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>http://politicalo.com/reince-priebus-older-peo...</td>\n",
       "      <td>politicalo</td>\n",
       "      <td>Reince Priebus: “Older People Should Pay 5 Tim...</td>\n",
       "      <td>['Lea Vat Kens']</td>\n",
       "      <td>Appearing on Fox News Sunday, White House Chie...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>http://www.reuters.com/article/us-usa-trump-sc...</td>\n",
       "      <td>reuters</td>\n",
       "      <td>Factbox: Scaramucci's financial potpourri - ba...</td>\n",
       "      <td>['Lawrence Delevingne', 'Svea Herbst-Bayliss']</td>\n",
       "      <td>White House Communications Director Anthony Sc...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>http://www.cnn.com/travel/article/vermont-whis...</td>\n",
       "      <td>cnn</td>\n",
       "      <td>Is this the world's best whiskey?</td>\n",
       "      <td>[]</td>\n",
       "      <td>Jen Rose Smith, CNN • Updated 27th July 2017\\n...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>http://www.npr.org/2017/07/27/539907467/senate...</td>\n",
       "      <td>npr</td>\n",
       "      <td>Senate Careens Toward High-Drama, Late Night H...</td>\n",
       "      <td>['Susan Davis']</td>\n",
       "      <td>Senate Careens Toward High-Drama, Late Night H...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>https://www.interestingdailynews.com/rachel-ma...</td>\n",
       "      <td>interestingdailynews</td>\n",
       "      <td>Rachel Maddow Exposes Mike Pence’s 4 Biggest L...</td>\n",
       "      <td>['Interesting Daily News']</td>\n",
       "      <td>Rachel Maddow demonstrated why Vice President ...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>https://www.interestingdailynews.com/man-rapes...</td>\n",
       "      <td>interestingdailynews</td>\n",
       "      <td>Man Rapes Infant, Leaves Her With Something To...</td>\n",
       "      <td>['Interesting Daily News']</td>\n",
       "      <td>Man Rapes Infant, Leaves Her With Something To...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>http://www.cnn.com/2017/07/27/europe/charlie-g...</td>\n",
       "      <td>cnn</td>\n",
       "      <td>Charlie Gard will be moved to hospice for fina...</td>\n",
       "      <td>['Lauren Said-Moorhouse', 'Richard Allen Greene']</td>\n",
       "      <td>London (CNN) Charlie Gard, the terminally ill ...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>http://www.newyorker.com/news/sporting-scene/k...</td>\n",
       "      <td>newyorker</td>\n",
       "      <td>Kyrie Irving’s Anxiety of Influence</td>\n",
       "      <td>[]</td>\n",
       "      <td>When it was reported, last week, that Kyrie Ir...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>http://www.newyorker.com/magazine/2017/07/31/p...</td>\n",
       "      <td>newyorker</td>\n",
       "      <td>Panorama’s Contemporary Scope</td>\n",
       "      <td>[]</td>\n",
       "      <td>Modern music festivals affirm the contemporary...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>http://beforeitsnews.com/sports/2017/07/titans...</td>\n",
       "      <td>beforeitsnews</td>\n",
       "      <td>Titans Ink Former Colts OLB Erik Walden</td>\n",
       "      <td>['Sports Gab Network']</td>\n",
       "      <td>(Before It's News)\\n\\nThe Titans have agreed t...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>http://24wpn.com/index.php/2017/06/13/breaking...</td>\n",
       "      <td>24wpn</td>\n",
       "      <td>Breaking: Obama Administration Caught Using “B...</td>\n",
       "      <td>[]</td>\n",
       "      <td>Share Facebook\\n\\nTwitter\\n\\nStumbleupon\\n\\nWh...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>http://www.reuters.com/article/us-rosenbaum-he...</td>\n",
       "      <td>reuters</td>\n",
       "      <td>Commentary: Want Congress to fast track health...</td>\n",
       "      <td>['Michael Rosenbaum']</td>\n",
       "      <td>With John McCain’s support, and a tie-breaking...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>http://www.nephef.com/donald-trump-attacks-fbi...</td>\n",
       "      <td>nephef</td>\n",
       "      <td>Donald Trump attacks FBI director James Comey ...</td>\n",
       "      <td>[]</td>\n",
       "      <td>Donald Trump has attacked the director of the ...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>http://www.cnn.com/2016/11/04/health/drug-resi...</td>\n",
       "      <td>cnn</td>\n",
       "      <td>First US cases of drug-resistant fungal infection</td>\n",
       "      <td>['Susan Scutti']</td>\n",
       "      <td>Story highlights Candida auris is a sometimes ...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>http://www.politico.com/story/2017/07/27/tax-r...</td>\n",
       "      <td>politico</td>\n",
       "      <td>Tax reform: Is that all there is?</td>\n",
       "      <td>['Nancy Cook', 'Aaron Lorenzo', 'Rachael Bade']</td>\n",
       "      <td>The White House, the Treasury Department and c...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                  url                source  \\\n",
       "0   http://politicalo.com/reince-priebus-older-peo...            politicalo   \n",
       "1   http://www.reuters.com/article/us-usa-trump-sc...               reuters   \n",
       "2   http://www.cnn.com/travel/article/vermont-whis...                   cnn   \n",
       "3   http://www.npr.org/2017/07/27/539907467/senate...                   npr   \n",
       "4   https://www.interestingdailynews.com/rachel-ma...  interestingdailynews   \n",
       "5   https://www.interestingdailynews.com/man-rapes...  interestingdailynews   \n",
       "6   http://www.cnn.com/2017/07/27/europe/charlie-g...                   cnn   \n",
       "7   http://www.newyorker.com/news/sporting-scene/k...             newyorker   \n",
       "8   http://www.newyorker.com/magazine/2017/07/31/p...             newyorker   \n",
       "9   http://beforeitsnews.com/sports/2017/07/titans...         beforeitsnews   \n",
       "10  http://24wpn.com/index.php/2017/06/13/breaking...                 24wpn   \n",
       "11  http://www.reuters.com/article/us-rosenbaum-he...               reuters   \n",
       "12  http://www.nephef.com/donald-trump-attacks-fbi...                nephef   \n",
       "13  http://www.cnn.com/2016/11/04/health/drug-resi...                   cnn   \n",
       "14  http://www.politico.com/story/2017/07/27/tax-r...              politico   \n",
       "\n",
       "                                                title  \\\n",
       "0   Reince Priebus: “Older People Should Pay 5 Tim...   \n",
       "1   Factbox: Scaramucci's financial potpourri - ba...   \n",
       "2                   Is this the world's best whiskey?   \n",
       "3   Senate Careens Toward High-Drama, Late Night H...   \n",
       "4   Rachel Maddow Exposes Mike Pence’s 4 Biggest L...   \n",
       "5   Man Rapes Infant, Leaves Her With Something To...   \n",
       "6   Charlie Gard will be moved to hospice for fina...   \n",
       "7                 Kyrie Irving’s Anxiety of Influence   \n",
       "8                       Panorama’s Contemporary Scope   \n",
       "9             Titans Ink Former Colts OLB Erik Walden   \n",
       "10  Breaking: Obama Administration Caught Using “B...   \n",
       "11  Commentary: Want Congress to fast track health...   \n",
       "12  Donald Trump attacks FBI director James Comey ...   \n",
       "13  First US cases of drug-resistant fungal infection   \n",
       "14                  Tax reform: Is that all there is?   \n",
       "\n",
       "                                               author  \\\n",
       "0                                    ['Lea Vat Kens']   \n",
       "1      ['Lawrence Delevingne', 'Svea Herbst-Bayliss']   \n",
       "2                                                  []   \n",
       "3                                     ['Susan Davis']   \n",
       "4                          ['Interesting Daily News']   \n",
       "5                          ['Interesting Daily News']   \n",
       "6   ['Lauren Said-Moorhouse', 'Richard Allen Greene']   \n",
       "7                                                  []   \n",
       "8                                                  []   \n",
       "9                              ['Sports Gab Network']   \n",
       "10                                                 []   \n",
       "11                              ['Michael Rosenbaum']   \n",
       "12                                                 []   \n",
       "13                                   ['Susan Scutti']   \n",
       "14    ['Nancy Cook', 'Aaron Lorenzo', 'Rachael Bade']   \n",
       "\n",
       "                                                 text  authenticity  \n",
       "0   Appearing on Fox News Sunday, White House Chie...             1  \n",
       "1   White House Communications Director Anthony Sc...             0  \n",
       "2   Jen Rose Smith, CNN • Updated 27th July 2017\\n...             0  \n",
       "3   Senate Careens Toward High-Drama, Late Night H...             0  \n",
       "4   Rachel Maddow demonstrated why Vice President ...             1  \n",
       "5   Man Rapes Infant, Leaves Her With Something To...             1  \n",
       "6   London (CNN) Charlie Gard, the terminally ill ...             0  \n",
       "7   When it was reported, last week, that Kyrie Ir...             0  \n",
       "8   Modern music festivals affirm the contemporary...             0  \n",
       "9   (Before It's News)\\n\\nThe Titans have agreed t...             1  \n",
       "10  Share Facebook\\n\\nTwitter\\n\\nStumbleupon\\n\\nWh...             1  \n",
       "11  With John McCain’s support, and a tie-breaking...             0  \n",
       "12  Donald Trump has attacked the director of the ...             1  \n",
       "13  Story highlights Candida auris is a sometimes ...             0  \n",
       "14  The White House, the Treasury Department and c...             0  "
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_balanced.head(n=15)"
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}