Clean.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import os\n",
    "import bz2\n",
    "import io\n",
    "from bz2 import BZ2File\n",
    "import pandas as pd\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Read dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "input_dir = \"C:\\\\...\\data\" # create folder 'data' containing .jsonl.bz2 file and update with your path "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# a helper function to get the lines from am archive\n",
    "def read_jsonlines(bz2_file):\n",
    "    text = bz2_file.read().decode('utf-8')\n",
    "    for line in text.split('\\n'):\n",
    "        if line != '':\n",
    "            yield line"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "ename": "FileNotFoundError",
     "evalue": "[WinError 3] Das System kann den angegebenen Pfad nicht finden: 'C:\\\\...\\\\data'",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mFileNotFoundError\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-4-d2a0e6d905c9>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0marchives\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mos\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mlistdir\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0minput_dir\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;36m4\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m+\u001b[0m \u001b[0mos\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mlistdir\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0minput_dir\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m5\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;31mFileNotFoundError\u001b[0m: [WinError 3] Das System kann den angegebenen Pfad nicht finden: 'C:\\\\...\\\\data'"
     ]
    }
   ],
   "source": [
    "archives = os.listdir(input_dir)[:4] + os.listdir(input_dir)[5:]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Extract articles"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "id_, journal_, date_, page_, text_ = [], [], [], [], []\n",
    "\n",
    "for archive in archives:\n",
    "    \n",
    "    # take only the transformed archives\n",
    "    # open the archive\n",
    "    f = BZ2File(os.path.join(input_dir, archive), 'r')\n",
    "        \n",
    "    # get the list of articles it contains (= a json object on each line)\n",
    "    articles = list(read_jsonlines(f))\n",
    "    \n",
    "    print(archive, ':', len(articles), 'articles à extraire')\n",
    "        \n",
    "    # load the first 100 articles as json and access their attributes    \n",
    "    for a in articles:\n",
    "         \n",
    "        # decode the json string into an object (dict)\n",
    "        json_article = json.loads(a)\n",
    "        # print(json_article)\n",
    "        if 'ft' in json_article:\n",
    "            journal_.append(str(json_article[\"id\"])[:3])\n",
    "            date_.append(str(json_article[\"id\"])[4:14])\n",
    "            page_.append(str(json_article[\"pp\"])[1:-1])\n",
    "            text_.append(str(json_article[\"ft\"]))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Create a pandas DataFrame"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame.from_dict(\n",
    "    {\n",
    "        'journal': journal_,\n",
    "        'date': date_,\n",
    "        'page': page_,\n",
    "        'text': text_\n",
    "    })"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(df)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Clean and enrich the DataFrame "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def measure_articles(df):\n",
    "    \n",
    "    lengths = []\n",
    "    \n",
    "    for ind, row in df.iterrows():\n",
    "        lengths.append(len(row['text']))\n",
    "        \n",
    "    return lengths\n",
    "\n",
    "df['length'] = measure_articles(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "def handle_multiple_pages(df):\n",
    "    \n",
    "    page, ppage = [], []\n",
    "    \n",
    "    for ind, row in df.iterrows():\n",
    "        \n",
    "        found = re.findall('([0-9]+)', row['page'])\n",
    "        if len(found) > 1:\n",
    "            page.append(found[0])\n",
    "            ppage.append(found[1])\n",
    "        else:\n",
    "            page.append(row['page'])\n",
    "            ppage.append(np.nan)\n",
    "        \n",
    "    return page, ppage\n",
    "    \n",
    "page, ppage = handle_multiple_pages(df)\n",
    "df['page'] = page\n",
    "df['ppage'] = ppage"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Jeter les articles vides ou ne contenant que quelques caractères (p.ex titre des rubriques)\n",
    "df = df[df['length'] > 50]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Sauvegarder l'index\n",
    "df['id'] = df.index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Formater les types\n",
    "df['date'] = pd.to_datetime(df['date'])\n",
    "df['journal'] = df['journal'].astype('category')\n",
    "df['page'] = df['page'].astype('int')\n",
    "df['ppage'] = df['ppage'].astype('float')\n",
    "df['text'] = df['text'].astype('str')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df[['id', 'journal', 'date', 'page', 'ppage', 'text', 'length']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_json('cleaned.json.bz2', compression = 'bz2')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_lengths = df['length'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Creating txt file\n",
    "txt = open('data.txt', 'w+')\n",
    "for ind, row in df.iterrows():\n",
    "    date = str(row['date'])[0:10]\n",
    "    txt.write('**** *'), txt.write(str(ind))\n",
    "    txt.write(' *'), txt.write(str(row['page']))\n",
    "    txt.write(' *'), txt.write(row['journal'])\n",
    "    txt.write(' *'), txt.write(date), txt.write('\\n')\n",
    "    txt.write(row['text']), txt.write('\\n')\n",
    "\n",
    "txt.close()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}