tweet_disaster_final.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "private_outputs": true,
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "source": [
        "# Import des bibliothèques"
      ],
      "metadata": {
        "id": "-CFL6vTl-crJ"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "i7ZVkA8HOX7h"
      },
      "outputs": [],
      "source": [
        "# Classique\n",
        "import pandas as pd\n",
        "import numpy as np\n",
        "import matplotlib.pyplot as plt\n",
        "import math\n",
        "import datetime\n",
        "import os\n",
        "import seaborn as sns\n",
        "import tqdm\n",
        "import sys\n",
        "import string\n",
        "!{sys.executable} -m pip install spacy\n",
        "\n",
        "# NLP\n",
        "!pip install -U spacy\n",
        "!pip install -U spacy-lookups-data\n",
        "import spacy\n",
        "import re\n",
        "import gensim\n",
        "!pip install emoji\n",
        "import emoji\n",
        "!{sys.executable} -m spacy download en_core_web_sm\n",
        "nlp = spacy.load(\"en_core_web_sm\")\n",
        "\n",
        "!pip install nltk\n",
        "import nltk\n",
        "nltk.download('all')\n",
        "from nltk.stem import PorterStemmer\n",
        "\n",
        "!pip install lazypredict\n",
        "\n",
        "# Dataviz textuelle\n",
        "from wordcloud import WordCloud\n",
        "from PIL import Image\n",
        "!pip install geotext\n",
        "from geotext import GeoText\n",
        "!pip install better_profanity\n",
        "from better_profanity import profanity\n",
        "from textblob import TextBlob \n",
        "\n",
        "# ML\n",
        "from sklearn.model_selection import train_test_split\n",
        "from sklearn.feature_extraction.text import TfidfVectorizer\n",
        "from sklearn.feature_extraction.text import TfidfTransformer\n",
        "from sklearn.feature_selection import SelectKBest\n",
        "from sklearn.feature_selection import chi2\n",
        "\n",
        "from sklearn.preprocessing import StandardScaler\n",
        "from sklearn.pipeline import Pipeline\n",
        "\n",
        "# Exploration de modèles\n",
        "from sklearn.model_selection import GridSearchCV\n",
        "from sklearn.linear_model import LogisticRegression\n",
        "from sklearn.svm import SVC\n",
        "from sklearn.neural_network import MLPClassifier\n",
        "from sklearn.metrics import f1_score\n",
        "from sklearn.metrics import accuracy_score \n",
        "\n",
        "# Enregistrement du modèle\n",
        "!pip install joblib\n",
        "import joblib\n",
        "\n",
        "# Masquer les messages d'alerte\n",
        "import warnings\n",
        "warnings.filterwarnings('ignore')"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Chargement des données"
      ],
      "metadata": {
        "id": "cRL6iG4--gYg"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# le dataset est stocké dans un repo github afin d'avoir un lien dur sur la base\n",
        "url1 = 'https://raw.githubusercontent.com/Francois-lenne/Big-data-SIAD/main/data/train.csv'\n",
        "url2 = 'https://raw.githubusercontent.com/Francois-lenne/Big-data-SIAD/main/data/test.csv'\n",
        "\n",
        "# lecture (depuis github)\n",
        "train = pd.read_csv('train.csv', sep=',', encoding='utf-8')\n",
        "test = pd.read_csv('test.csv', sep=',', encoding='utf-8')\n",
        "\n",
        "# lecture (si stocké dans l'environnement de travail)\n",
        "train = pd.read_csv(url1, sep=',', encoding='utf-8')\n",
        "test = pd.read_csv(url2, sep=',', encoding='utf-8')"
      ],
      "metadata": {
        "id": "Q3QJt5kgOcX5"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Extraction d'informations"
      ],
      "metadata": {
        "id": "GJHLfmZNQ8F_"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Nombre de mots\n",
        "def countWords(text):\n",
        "  nb_words = len(text.split())\n",
        "  return nb_words\n",
        "\n",
        "# Taille moyenne d'un mot\n",
        "def avgWordLength(text):\n",
        "  avg = np.mean([len(word) for word in text.split()])\n",
        "  return avg\n",
        "\n",
        "# Création de variables binaires indiquant le type de lieu mentionné\n",
        "def precision(text):\n",
        "    rep = '0'\n",
        "    reg1 = r'(,)'\n",
        "    p = re.compile(reg1)\n",
        "    check = re.search(reg1,text)\n",
        "    if check is not None:\n",
        "      rep = '1'\n",
        "    return rep\n",
        "\n",
        "def city(tweet):\n",
        "    tweet.capitalize()\n",
        "    places = GeoText(tweet)\n",
        "    if len(places.cities) == 0:\n",
        "        cities = 0\n",
        "    else:\n",
        "        cities = 1\n",
        "    return cities\n",
        "\n",
        "def country(tweet):\n",
        "    tweet.capitalize()\n",
        "    places = GeoText(tweet)\n",
        "    if len(places.countries) == 0:\n",
        "        countries = 0\n",
        "    else:\n",
        "        countries = 1\n",
        "    return countries\n",
        "\n",
        "# Détecteur de vulgarités\n",
        "def isProfanity(text): \n",
        "  from better_profanity import profanity \n",
        "  check = profanity.contains_profanity(text)\n",
        "  if check == \"false\":\n",
        "      profanity = 0\n",
        "  else:\n",
        "      profanity = 1\n",
        "  return(profanity)\n",
        "\n",
        "# Récupération des hashtag\n",
        "def recupHashtag(text):\n",
        "    \"\"\"\n",
        "    Fonction prenant en entrée une chaîne de caractère et retournant une chaîne contenant les hashtags séparés par un espace (s'ils existent).\n",
        "    \"\"\"\n",
        "    reg = r\"#[^# ]\"\n",
        "    p = re.compile(reg)\n",
        "    check = re.search(reg,text)\n",
        "    if check is not None:\n",
        "        ge = p.findall(text)\n",
        "        ge_join = ' '.join(ge)\n",
        "        return ge_join\n",
        "\n",
        "# Variable binaire : présence ou non d'un hashtag\n",
        "def recupHashtagBinaire(text):\n",
        "    \"\"\"\n",
        "    Fonction prenant en entrée une chaîne de caractère et retournant \"0\" si aucun hashtag n'est présent et \"1\" si un hashtag est présent.\n",
        "    \"\"\"\n",
        "    rep = '0'\n",
        "    reg = r\"#[^# ]*\"\n",
        "    p = re.compile(reg)\n",
        "    check = re.search(reg,text)\n",
        "    if check is not None:\n",
        "      rep = '1'\n",
        "    return rep\n",
        "\n",
        "# Décompte des hashtags\n",
        "def countHashtag(text):\n",
        "    \"\"\"\n",
        "    Fonction prenant en entrée une chaîne de caractère et retournant retournant le nombre de hashtag # dans le tweet.\n",
        "    \"\"\"\n",
        "    rep = 0\n",
        "    reg = '#[^# ]*'\n",
        "    if re.search(reg,text) is not None:\n",
        "      rep = len(re.findall(reg,text))\n",
        "    return rep\n",
        "\n",
        "# Détecte la présence de mentions\n",
        "def recupNameBinaire(text):\n",
        "    rep = '0'\n",
        "    reg = r\"@[^@ ]*\"\n",
        "    p = re.compile(reg)\n",
        "    check = re.search(reg,text)\n",
        "    if check is not None:\n",
        "        rep = '1'\n",
        "    return rep\n",
        "\n",
        "# Récupération des mentions (@)\n",
        "def recupName(text):\n",
        "    \"\"\"\n",
        "    Fonction prenant en entrée une chaîne de caractère et retournant une chaîne contenant les mentions séparées par un espace (si elles existent).\n",
        "    \"\"\"\n",
        "    reg = r\"@[^@ ]*\"\n",
        "    p = re.compile(reg)\n",
        "    check = re.search(reg,text)\n",
        "    if check is not None:\n",
        "        ge = p.findall(text)\n",
        "        ge_join = ' '.join(ge)\n",
        "        return ge_join\n",
        "\n",
        "# Compte le nombre de mentions (@)\n",
        "def countName(text):\n",
        "    \"\"\"\n",
        "    Fonction prenant en entrée une chaîne de caractère et retournant le nombre de mentions @ dans le tweet.\n",
        "    \"\"\"\n",
        "    rep = 0\n",
        "    reg='@[^@ ]*'\n",
        "    if re.search('@[^@ ]*',text) is not None:\n",
        "      rep = len(re.findall(reg,text))\n",
        "    return rep\n",
        "\n",
        "# Récupération des dates\n",
        "def recupDate(text):\n",
        "    \"\"\"\n",
        "    Fonction prenant en entrée une chaîne de caractère et retournant une chaîne contenant une date (si elle existe).\n",
        "    \"\"\"\n",
        "    reg = r\"([A-Za-z]{3})\\s(\\d{1,2}),\\s(\\d{4})\"\n",
        "    p = re.compile(reg)\n",
        "    check = re.search(reg,text)\n",
        "    if check is not None:\n",
        "        ge = p.findall(text)\n",
        "        return ge\n",
        "\n",
        "# Variable binaire : présence d'une date\n",
        "def recupDateBinaire(text):\n",
        "    \"\"\"\n",
        "    Fonction prenant en entrée une chaîne de caractère et retournant \"0\" si aucune date n'est présente et \"1\" si une date est présente.\n",
        "    \"\"\"\n",
        "    rep = '0'\n",
        "    reg = r\"([A-Za-z]{3})\\s(\\d{1,2}),\\s(\\d{4})\"\n",
        "    p = re.compile(reg)\n",
        "    check = re.search(reg,text)\n",
        "    if check is not None:\n",
        "      rep = '1'\n",
        "    return rep\n",
        "\n",
        "# Récupération des liens\n",
        "def getChemin(text):\n",
        "    \"\"\"\n",
        "    Fonction prenant en entrée une chaîne de caractère et retournant une chaîne contenant le chemin du lien (s'il existe).\n",
        "    \"\"\"\n",
        "    reg1 = r'(https?:\\/\\/[^\\s]+)'\n",
        "    reg2 = r'(https?)://([^:/]+)(?::(\\d+))?(/[^?]*)?(\\?[^#]*)?(#.*)?'\n",
        "    p = re.compile(reg1)\n",
        "    check = re.search(reg1,text)\n",
        "    if check is not None:\n",
        "        ge = p.findall(text)\n",
        "        for val in ge:\n",
        "            match = re.search(reg2,val)\n",
        "            if match:\n",
        "                rt = match.group(4)\n",
        "            return rt\n",
        "\n",
        "# Variable binaire : présence ou non d'un lien\n",
        "def getCheminBinaire(text):\n",
        "    \"\"\"\n",
        "    Fonction prenant en entrée une chaîne de caractère et retournant \"0\" si aucun hashtag n'est présent et \"1\" si un hashtag est présent.\n",
        "    \"\"\"\n",
        "    rep = '0'\n",
        "    reg1 = r'(https?:\\/\\/[^\\s]+)'\n",
        "    p = re.compile(reg1)\n",
        "    check = re.search(reg1,text)\n",
        "    if check is not None:\n",
        "      rep = '1'\n",
        "    return rep\n",
        "\n",
        "# Variable binaire : présence ou non d'un lien\n",
        "def countChemin(text):\n",
        "    \"\"\"\n",
        "    Fonction prenant en entrée une chaîne de caractère et retournant \"0\" si aucun hashtag n'est présent et \"1\" si un hashtag est présent.\n",
        "    \"\"\"\n",
        "    rep = 0\n",
        "    reg = '(https?:\\/\\/[^\\s]+)'\n",
        "    if re.search(reg,text) is not None:\n",
        "      rep = len(re.findall(reg,text))\n",
        "    return rep\n",
        "\n",
        "# Récupération des lieux cités\n",
        "def getLocation(text):\n",
        "    \"\"\"\n",
        "    Fonction prenant en entrée une chaîne de caractère et retournant une liste de lieu(x) (s'il(s) existe(nt)).\n",
        "    \"\"\"\n",
        "    global nlp\n",
        "    save = []\n",
        "    doc = nlp(text)\n",
        "    for ent in doc.ents:\n",
        "        if ent.label_ == \"GPE\":\n",
        "            save.append(ent.text)\n",
        "    return save\n",
        "\n",
        "# Variable binaire : présence ou non d'un lieu\n",
        "def getLocationBinaire(text):\n",
        "    \"\"\"\n",
        "    Fonction prenant en entrée une chaîne de caractère et retournant \"0\" si aucun hashtag n'est présent et \"1\" si un hashtag est présent.\n",
        "    \"\"\"\n",
        "    rep = \"0\"\n",
        "    global nlp\n",
        "    doc = nlp(text)\n",
        "    for ent in doc.ents:\n",
        "        if ent.label_ == \"GPE\":\n",
        "            rep = \"1\"\n",
        "    return rep\n",
        "\n",
        "def getSubjectivity(text):\n",
        "    subj = TextBlob(text).sentiment.subjectivity\n",
        "    if subj < 0:\n",
        "        score = 2\n",
        "    elif subj == 0:\n",
        "        score = 0\n",
        "    else:\n",
        "        score = 1\n",
        "    return score\n",
        "    \n",
        "def getPolarity(text):\n",
        "    polar = TextBlob(text).sentiment.polarity\n",
        "    if polar < 0:\n",
        "        score = 2\n",
        "    elif polar == 0:\n",
        "        score = 0\n",
        "    else:\n",
        "        score = 1\n",
        "    return score\n",
        "\n",
        "def removeElements(text):\n",
        "  text = re.sub('([A-Za-z]{3})\\s(\\d{1,2}),\\s(\\d{4})','',text)\n",
        "  text = re.sub('@[^@ ]*','',text)\n",
        "  text = re.sub('#[^# ]','',text)\n",
        "  text = re.sub('(https?:\\/\\/[^\\s]+)','',text)\n",
        "  text = re.sub('(https?)://([^:/]+)(?::(\\d+))?(/[^?]*)?(\\?[^#]*)?(#.*)?','',text)\n",
        "  return text\n",
        "\n",
        "def countEmojis(text):\n",
        "  rep = 0\n",
        "  reg = '\\U0001F600-\\U0001F64F|\\U0001F300-\\U0001F5FF|\\U0001F680-\\U0001F6FF|\\U0001F1E0-\\U0001F1FF|\\U00002702-\\U000027B0|\\U000024C2-\\U0001F251'\n",
        "  if re.search(reg,text) is not None:\n",
        "    rep = len(re.findall(reg,text))\n",
        "  return rep"
      ],
      "metadata": {
        "id": "92xVH0aZQ23D"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Conversion en chaîne de caractère - certaines valeurs ne sont en effet pas reconnues comme telles.\n",
        "train['location'] = train['location'].fillna('')\n",
        "train['keyword'] = train['keyword'].fillna('')\n",
        "\n",
        "test['location'] = test['location'].fillna('').replace()\n",
        "test['keyword'] = test['keyword'].fillna('')\n",
        "\n",
        "# Nombre de caractères du tweet, espaces inclus\n",
        "train['text_length'] = [len(text) for text in train['text']]\n",
        "test['text_length'] = [len(text) for text in test['text']]\n",
        "\n",
        "# Catégorie de taille des tweets\n",
        "train['tweet_length_cut'] = pd.cut(train['text_length'], [0,100,140], labels=['<100','>100'])\n",
        "test['tweet_length_cut'] = pd.cut(test['text_length'], [0,100,140], labels=['<100','>100'])\n",
        "\n",
        "# Nombre de mots\n",
        "train['nb_mots'] = train['text'].apply(countWords)\n",
        "test['nb_mots'] = test['text'].apply(countWords)\n",
        "\n",
        "# Longueur des mots\n",
        "train['longueur_mots'] = train['text'].apply(avgWordLength)\n",
        "test['longueur_mots'] = test['text'].apply(avgWordLength)\n",
        "\n",
        "\n",
        "# Présence ou non d'une ville\n",
        "train['city'] = train['location'].apply(city)\n",
        "test['city'] = test['location'].apply(city)\n",
        "\n",
        "# Présence ou non d'un pays\n",
        "train['country'] = train['location'].apply(country)\n",
        "test['country'] = test['location'].apply(country)\n",
        "\n",
        "# présence d'un lieu ou non\n",
        "train['precision'] = train['location'].apply(precision)\n",
        "test['precision'] = test['location'].apply(precision)\n",
        "\n",
        "# Présence ou non d'une vulgarité\n",
        "# train['profanity'] = train['text'].apply(isProfanity)\n",
        "# test['profanity'] = test['text'].apply(isProfanity)\n",
        "\n",
        "# Compte des emojis\n",
        "train['nb_emojis'] = train['text'].apply(countEmojis)\n",
        "test['nb_emojis'] = test['text'].apply(countEmojis)\n",
        "\n",
        "# Extraction des hashtags\n",
        "train['hashtags'] = train['text'].apply(recupHashtag)\n",
        "test['hashtags'] = test['text'].apply(recupHashtag)\n",
        "\n",
        "# Présence d'un hashtag\n",
        "train['hashtags_b'] = train['text'].apply(recupHashtagBinaire)\n",
        "test['hashtags_b'] = test['text'].apply(recupHashtagBinaire)\n",
        "\n",
        "# Décompte des hashtags\n",
        "train['nb_hashtags'] = train['text'].apply(countHashtag)\n",
        "test['nb_hashtags'] = test['text'].apply(countHashtag)\n",
        "\n",
        "# Extraction des mentions\n",
        "train['mentions'] = train['text'].apply(recupName)\n",
        "test['mentions'] = test['text'].apply(recupName)\n",
        "\n",
        "# Présence d'une mention\n",
        "train['mentions_b'] = train['text'].apply(recupNameBinaire)\n",
        "test['mentions_b'] = test['text'].apply(recupNameBinaire)\n",
        "\n",
        "# Décompte des mentions\n",
        "train['nb_mentions'] = train['text'].apply(countName)\n",
        "test['nb_mentions'] = test['text'].apply(countName)\n",
        "\n",
        "# Extraction des dates\n",
        "train['dates'] = train['text'].apply(recupDate)\n",
        "test['dates'] = test['text'].apply(recupDate)\n",
        "\n",
        "# Présence d'une date\n",
        "train['dates_b'] = train['text'].apply(recupDateBinaire)\n",
        "test['dates_b'] = test['text'].apply(recupDateBinaire)\n",
        "\n",
        "# Extraction des liens\n",
        "train['rt_path'] = train['text'].apply(getChemin)\n",
        "test['rt_path'] = test['text'].apply(getChemin)\n",
        "\n",
        "# Présence d'un lien\n",
        "train['rt_path_b'] = train['text'].apply(getCheminBinaire)\n",
        "test['rt_path_b'] = test['text'].apply(getCheminBinaire)\n",
        "\n",
        "# Décompte des liens\n",
        "train['nb_liens'] = train['text'].apply(countChemin)\n",
        "test['nb_liens'] = test['text'].apply(countChemin)\n",
        "\n",
        "# Analyse de sentiment\n",
        "## Subjectivité\n",
        "train['Subjectivity'] = train['text'].apply(getSubjectivity)\n",
        "test['Subjectivity'] = test['text'].apply(getSubjectivity)\n",
        "\n",
        "## Polarité\n",
        "train['Polarity'] = train['text'].apply(getPolarity)\n",
        "test['Polarity'] = test['text'].apply(getPolarity)\n",
        "\n",
        "# Présence ou non d'une ville\n",
        "train['text'] = train['text'].apply(removeElements)\n",
        "test['text'] = test['text'].apply(removeElements)"
      ],
      "metadata": {
        "id": "7pgCpShbRVGA"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Nettoyage"
      ],
      "metadata": {
        "id": "tgS6ldaBRHA_"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def preprocessing(text):\n",
        "    \"\"\"\n",
        "    Fonction prenant en entrée une chaîne de caractères et retournant cette chaîne de caractères\n",
        "    après avoir appliqué l'ensemble des modifications décrites ci-dessous.\n",
        "    \"\"\"\n",
        "    text = str(text)\n",
        "\n",
        "    # Harmonisation de la casse : mise en minuscule\n",
        "    text = text.lower()\n",
        "\n",
        "    # retrait espaces entre les nombres\n",
        "    text = re.sub(r'(?<=\\\\d) +(?=\\\\d)', '', text) \n",
        "\n",
        "    # Gestion des accents et ponctuations\n",
        "    text = re.sub(r'%20',' ', text) # remplacement %20 par un espace\n",
        "    text = re.sub(r'&amp|& amp', '&', text) # remplacement &amp par &\n",
        "    # text = re.sub(\"\\d+\", \" \", text) # retrait nombres\n",
        "    text = re.sub('[éèê]', \"e\", text) # retrait accents sur le e\n",
        "    text = re.sub(\"[ôöóò]\", \"o\", text) # retrait accents sur le o\n",
        "    text = re.sub(\"[üùû]\", \"u\", text) # retrait accents sur le u\n",
        "    text = re.sub(\"[ïiî]\", \"i\", text) # retrait accents sur le i\n",
        "    text = re.sub(\"[âàäå]\", \"a\", text) # retrait accents sur le a\n",
        "    text = re.sub(\"[_.,;:!?]\", \" \", text) # retrait ponctuation\n",
        "    text = re.sub(\"[|{}()«»/]\", \" \", text) # retrait parenthèses, guillemets, slashs...\n",
        "    text = re.sub(\"[“”]\", \" \", text) # retrait guillemets (autre forme)\n",
        "    text = re.sub(\"'\", \" \", text) # retrait apostrophes\n",
        "    text = re.sub(\"’\", \" \", text) # retrait apostrophes (autre forme)\n",
        "    text = re.sub('\"', \" \", text) # retrait quotes\n",
        "    text = re.sub('[+-]', \" \", text) # retrait + et -\n",
        "    text = re.sub('[=*/]', \" \", text) # retrait opérateurs\n",
        "    text = re.sub(\"°\", \"\", text) # retrait symbole °\n",
        "\n",
        "    # Gestion des symboles\n",
        "    text = re.sub(\"[€%$£]\", \"\", text) # retrait symboles devises\n",
        "\n",
        "    # Gestions des retours à la ligne ou fin de lignes (caractères non-imprimables)\n",
        "    text = re.sub('\\r\\n', \" \", text) # retrait retour charriot/retour à la ligne\n",
        "    text = re.sub('\\n', \" \", text) # retrait retour à la ligne\n",
        "\n",
        "    # Gestion des espaces\n",
        "    text = re.sub('\\s+', \" \", text) # retrait espaces en trop\n",
        "    text = text.rstrip(\" \") # retrait espaces à droite\n",
        "    text = text.lstrip(\" \") # retrait espaces à gauche\n",
        "\n",
        "    # Si un nombre est collé à un texte, les sépare avec un espace\n",
        "    text = re.sub(r'(?<=[a-zA-Z])(?=\\d)|(?<=\\d)(?=[a-zA-Z])', ' ',text)\n",
        "\n",
        "    # Gestion des emojis\n",
        "    emoji_pattern = re.compile(\n",
        "    '['\n",
        "    u'\\U0001F600-\\U0001F64F'  # emoticons\n",
        "    u'\\U0001F300-\\U0001F5FF'  # symbols & pictographs\n",
        "    u'\\U0001F680-\\U0001F6FF'  # transport & map symbols\n",
        "    u'\\U0001F1E0-\\U0001F1FF'  # flags (iOS)\n",
        "    u'\\U00002702-\\U000027B0'\n",
        "    u'\\U000024C2-\\U0001F251'\n",
        "    ']+',\n",
        "    flags=re.UNICODE)\n",
        "    emoji_pattern.sub(r'', text)\n",
        "\n",
        "    return text"
      ],
      "metadata": {
        "id": "k42QENOVRHg4"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Application de la fonction de nettoyage\n",
        "train['text_CLEAN'] = train['text'].apply(preprocessing)\n",
        "train['keyword'] = train['keyword'].apply(preprocessing)\n",
        "\n",
        "test['text_CLEAN'] = test['text'].apply(preprocessing)\n",
        "test['keyword'] = test['keyword'].apply(preprocessing)"
      ],
      "metadata": {
        "id": "lFdSf25HRjSO"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Analyse exploratoire"
      ],
      "metadata": {
        "id": "1qo13azIRBdP"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# df = train.groupby(['target'])['nb_liens'].mean().reset_index()\n",
        "# sns.barplot(data=df, x=\"target\", y=\"nb_liens\").set(title = 'Nombre de liens', xticklabels = ['fake','non-fake'])"
      ],
      "metadata": {
        "id": "D82h7FLN1u0s"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Diagramme en barres\n",
        "# pd.crosstab(train['target'],train['country'],normalize='index').plot(kind='bar').set(title='mention d\\'un pays',xticklabels = ['fake','non-fake'])"
      ],
      "metadata": {
        "id": "-vNf4nVtPL3z"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# # Diagramme en barres\n",
        "# graph = sns.countplot(y ='location',\n",
        "#                       hue = 'target',\n",
        "#                       data=train,\n",
        "#                       order=train.location.value_counts().iloc[1:16].index)\n",
        "# graph.set(title = 'Top 15 localisations')\n",
        "# graph.legend(labels=['fake','non-fake'],\n",
        "#            loc='upper center',\n",
        "#            bbox_to_anchor=(1.2, 1))"
      ],
      "metadata": {
        "id": "FZBSjRTKPpI3"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Traitements divers NLP"
      ],
      "metadata": {
        "id": "cE6XkHD0S4Rb"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "#########################################\n",
        "# LEMMISATION OU STEMMISATION + STOPWORDS\n",
        "#########################################\n",
        "\n",
        "nlp = spacy.load('en_core_web_sm')\n",
        "stopwords = nltk.corpus.stopwords.words('english')\n",
        "\n",
        "def Lemmatization(train,texts):\n",
        "  pbar = tqdm.tqdm(total=train.shape[0])\n",
        "  nlp = spacy.load(\"en_core_web_sm\", disable=[\"parser\", \"ner\"])\n",
        "  texts_out = []\n",
        "  for text in texts:\n",
        "      doc = nlp(text)\n",
        "      new_text = []\n",
        "      for token in doc: \n",
        "          new_text.append(token.lemma_)\n",
        "      final = \" \".join(new_text)\n",
        "      texts_out.append(final)\n",
        "      pbar.update(1) \n",
        "  return (texts_out)\n",
        "  pbar.close()\n",
        "\n",
        "train['text_CLEAN_LMT'] = Lemmatization(train,train['text_CLEAN'])\n",
        "test['text_CLEAN_LMT'] = Lemmatization(test,test['text_CLEAN'])\n",
        "\n",
        "def Remove_stopwords(text):\n",
        "  text = [word for word in text.split() if word not in stopwords and len(word)>2]\n",
        "  return text\n",
        "\n",
        "train['text_CLEAN_LMT_TOKEN_WSW'] = train['text_CLEAN_LMT'].apply(Remove_stopwords)\n",
        "test['text_CLEAN_LMT_TOKEN_WSW'] = test['text_CLEAN_LMT'].apply(Remove_stopwords)\n",
        "\n",
        "########################\n",
        "# BIGRAMMES - TRIGRAMMES\n",
        "########################\n",
        "\n",
        "# Fonctions utilitaires\n",
        "def make_bigrams(texts):\n",
        "    return [bigram_mod[text] for text in texts]\n",
        "def make_trigrams(texts):\n",
        "    return [trigram_mod[bigram_mod[text]] for text in texts]\n",
        "\n",
        "# Train\n",
        "data_words_train = train['text_CLEAN_LMT_TOKEN_WSW'].values.tolist()\n",
        "data_words_train\n",
        "\n",
        "bigram = gensim.models.Phrases(data_words_train, threshold = 1, scoring = 'npmi')\n",
        "trigram = gensim.models.Phrases(bigram[data_words_train], threshold = 1, scoring = 'npmi')\n",
        "\n",
        "bigram_mod = gensim.models.phrases.Phraser(bigram)\n",
        "trigram_mod = gensim.models.phrases.Phraser(trigram)\n",
        "\n",
        "data_words_train_bigrams = make_bigrams(data_words_train)\n",
        "train['text_CLEAN_LMT_TOKEN_WSW_BIGRAMS'] = data_words_train_bigrams\n",
        "\n",
        "# Test\n",
        "data_words_test = test['text_CLEAN_LMT_TOKEN_WSW'].values.tolist()\n",
        "data_words_test\n",
        "\n",
        "bigram = gensim.models.Phrases(data_words_test, min_count=5, threshold=1, scoring = 'npmi')\n",
        "trigram = gensim.models.Phrases(bigram[data_words_test], threshold=1, scoring = 'npmi')\n",
        "\n",
        "bigram_mod = gensim.models.phrases.Phraser(bigram)\n",
        "trigram_mod = gensim.models.phrases.Phraser(trigram)\n",
        "\n",
        "data_words_test_bigrams = make_bigrams(data_words_test)\n",
        "test['text_CLEAN_LMT_TOKEN_WSW_BIGRAMS'] = data_words_test_bigrams\n",
        "\n",
        "#################\n",
        "# DETOKENIZATION\n",
        "#################\n",
        "\n",
        "def Detokenize(txt):\n",
        "    txt = ' '.join([word for word in txt])\n",
        "    return txt\n",
        "\n",
        "train['text_CLEAN_LMT_WSW_BIGRAMS'] = train['text_CLEAN_LMT_TOKEN_WSW_BIGRAMS'].apply(lambda x: Detokenize(x))\n",
        "test['text_CLEAN_LMT_WSW_BIGRAMS'] = test['text_CLEAN_LMT_TOKEN_WSW_BIGRAMS'].apply(lambda x: Detokenize(x))"
      ],
      "metadata": {
        "id": "Ag6nsMb2S3U_"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Nuages de mots FAKE vs NOT FAKE"
      ],
      "metadata": {
        "id": "Dl5DgkQAp7Yx"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "twitter_mask = np.array(Image.open('twitter_logo.jpg'))\n",
        "\n",
        "# Création d'une table dont les colonnes sont les mots et les données sont les valeurs TF-Itrain\n",
        "vectorizer = TfidfVectorizer()\n",
        "vectors = vectorizer.fit_transform(train['text_CLEAN_LMT_WSW_BIGRAMS'].loc[train['target']==0].astype('U'))\n",
        "feature_names = vectorizer.get_feature_names_out()\n",
        "dense = vectors.todense()\n",
        "df = pd.DataFrame(dense, columns=feature_names)\n",
        "\n",
        "wordcloud = WordCloud(background_color='white', mask = twitter_mask, colormap = 'winter')\n",
        "wordcloud.generate_from_frequencies(df.T.sum(axis=1))\n",
        "\n",
        "plt.figure(figsize=(15,15))\n",
        "plt.imshow(wordcloud, interpolation='bilinear')\n",
        "plt.axis(\"off\")\n",
        "plt.show()"
      ],
      "metadata": {
        "id": "C9wuJ8NTo-gG"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Création d'une table dont les colonnes sont les mots et les données sont les valeurs TF-Itrain\n",
        "vectors = vectorizer.fit_transform(train['text_CLEAN_LMT_WSW_BIGRAMS'].loc[train['target']==1].astype('U'))\n",
        "feature_names = vectorizer.get_feature_names_out()\n",
        "dense = vectors.todense()\n",
        "df = pd.DataFrame(dense, columns=feature_names)\n",
        "\n",
        "wordcloud = WordCloud(background_color='white', mask = twitter_mask, colormap = 'winter')\n",
        "wordcloud.generate_from_frequencies(df.T.sum(axis=1))\n",
        "\n",
        "plt.figure(figsize=(15,15))\n",
        "plt.imshow(wordcloud, interpolation='bilinear')\n",
        "plt.axis(\"off\")\n",
        "plt.show()"
      ],
      "metadata": {
        "id": "ZTior7L4QhEQ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Transformations des variables"
      ],
      "metadata": {
        "id": "rCWNftxay-Q5"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Vectorisation\n",
        "\n",
        "print('Vectorisation des tweets d\\'apprentissage en cours ...')\n",
        "# Création d'une table dont les colonnes sont les mots et les données sont les valeurs TF-Itrain\n",
        "text_vectorizer = TfidfVectorizer()\n",
        "# Entraîne le modèle sur les données d'entraînement (on s'assure que les données textuelles sont bien encodée en unicode)\n",
        "text_vectorizer_fitted = text_vectorizer.fit(train['text_CLEAN_LMT_WSW_BIGRAMS'].astype('U'))\n",
        "# On enregistre ce modèle pour le réutiliser dans l'API (pour transformer les tweets passés en entrée)\n",
        "joblib.dump(text_vectorizer_fitted,'tweet_vectorizer.gz')\n",
        "# Applique la transformation\n",
        "text_vectors = text_vectorizer_fitted.transform(train['text_CLEAN_LMT_WSW_BIGRAMS'].astype('U'))\n",
        "# Récupère la liste des mots distincts qui deviennent des noms de features\n",
        "text_feature_names = text_vectorizer_fitted.get_feature_names_out()\n",
        "# Structure les données sous la forme d'un dataframe où 1 colonne = 1 mot de la liste de tous les mots présents dans les tweets\n",
        "text_dense = text_vectors.todense()\n",
        "train_tfidf = pd.DataFrame(text_dense, columns=text_feature_names)\n",
        "print('Vectorisation des tweets d\\'apprentissage terminée.')\n",
        "\n",
        "\n",
        "print('Vectorisation des tweets de test en cours ...')\n",
        "# Applique la même transformation sur les données de test que sur les données d'apprentissage.\n",
        "# Cela permet d'obtenir une table structurée avec les mêmes mots-colonnes.\n",
        "text_vectors = text_vectorizer_fitted.transform(test['text_CLEAN_LMT_WSW_BIGRAMS'].astype('U'))\n",
        "# Récupère la liste des mots distincts qui deviennent des noms de features\n",
        "text_feature_names = text_vectorizer_fitted.get_feature_names_out()\n",
        "# Structure les données sous la forme d'un dataframe où 1 colonne = 1 mot de la liste de tous les mots présents dans les tweets\n",
        "text_dense = text_vectors.todense()\n",
        "test_tfidf = pd.DataFrame(text_dense, columns=text_feature_names)\n",
        "print('Vectorisation des tweets de test terminée.')"
      ],
      "metadata": {
        "id": "1gNdu_80f5yR"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "train_tfidf.sample(5)"
      ],
      "metadata": {
        "id": "WZa19UqMrBoS"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Sélection des variables"
      ],
      "metadata": {
        "id": "66Spj9May3C_"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "print('Sélection des variables en cours ...')\n",
        "\n",
        "# Données d'entraînement\n",
        "X = train_tfidf # mots (devenus des variables)\n",
        "y = train['target'] # cible\n",
        "\n",
        "# Fonction initialisant un modèle de sélection des variables basée sur le chi2 entre chaque variable explicatives et la cible 2 à 2.\n",
        "# Ici on exclut les X% des variables les moins associées à la cible, donc les X% les moins discriminantes.\n",
        "# Cette sélection permet de réduire la dimensionnalité du modèle et par conséquent, de limiter le surapprentissage.\n",
        "chi2_features = SelectKBest(chi2, k = int(X.shape[1]*0.9))\n",
        "# Ajustement du modèle sur les données d'apprentissage. On crée ainsi la liste des mots à conserver dans les tweets.\n",
        "chi2_features_selector = chi2_features.fit(X,y)\n",
        "# On enregistre ce modèle pour le réutiliser dans l'API (pour transformer les tweets passés en entrée)\n",
        "joblib.dump(chi2_features_selector,'features_selector.gz')\n",
        "# On applique le modèle sur les tweets d'apprentissage \n",
        "features_selected = chi2_features_selector.transform(X)\n",
        "features_names = chi2_features_selector.get_feature_names_out()\n",
        "# On stocke la sortie dans une table\n",
        "train_features = pd.DataFrame(features_selected,columns=features_names)\n",
        "\n",
        "# Données de test\n",
        "X = test_tfidf \n",
        "# On applique le modèle (précédemment ajusté sur les données d'entraînements) sur les tweets d'apprentissage \n",
        "features_selected = chi2_features_selector.transform(X)\n",
        "features_names = chi2_features_selector.get_feature_names_out()\n",
        "# On stocke la sortie dans une table\n",
        "test_features = pd.DataFrame(features_selected,columns=features_names)\n",
        "\n",
        "print('Sélection des variables terminée.')"
      ],
      "metadata": {
        "id": "2A3ZifwZy2Sy"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "joblib.dump(chi2_features_selector,'features_selector.gz')"
      ],
      "metadata": {
        "id": "ir4hiFUM_d0B"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Modélisation"
      ],
      "metadata": {
        "id": "DONiwFy9f-fg"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "train_variables_quali = pd.get_dummies(train[['keyword','text_length']])\n",
        "train_variables_quanti = train[['nb_mots','nb_hashtags','nb_mentions','nb_liens','longueur_mots','city','country','precision']]\n",
        "\n",
        "final_train = pd.concat([train_features, # variables-mots créées à l'aide des tweets (features TF IDF)\n",
        "                         train_variables_quanti, # variables quantitatives créées via l'extraction de données dans les tweets\n",
        "                         train_variables_quali], # variables qualitatives créées via l'extraction de données dans les tweets\n",
        "                         axis = 'columns')\n",
        "\n",
        "X_train = final_train.reset_index(drop=True)\n",
        "y_train = train['target']"
      ],
      "metadata": {
        "id": "CRfKq1yrgf8d"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Exploration des modèles"
      ],
      "metadata": {
        "id": "wIJaRiolyS4x"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# # Echantillons de 30% des jeux de données\n",
        "# X_train_sample = X_train.sample(frac=0.3)\n",
        "# y_train_sample = y_train.sample(frac=0.3)\n",
        "\n",
        "# pipe = Pipeline([\n",
        "#     (\"scaler\", StandardScaler()), # mean = True ne fonctionne pas avec une sparse matrix (TD-IDF)\n",
        "#     # Modèle testé en premier (n'importe lequel)\n",
        "#     (\"classifier\", LogisticRegression(random_state=0))\n",
        "# ])\n",
        "\n",
        "# param_grid = [\n",
        "\n",
        "#     # Régression Logistique (one vs rest)\n",
        "#     # {'classifier': [LogisticRegression(random_state=0)],\n",
        "#     #  'classifier__penalty':['l1','l2'],\n",
        "#     #  },\n",
        "\n",
        "#     # Support Vector Machine\n",
        "#     {'classifier': [SVC(kernel='rbf', random_state=0, probability = True)],\n",
        "#     #  'classifier__kernel' : ['rbf','linear','poly'],\n",
        "#      'classifier__C':[0.01,0.1,1]\n",
        "#      },\n",
        "    \n",
        "#     # Réseau de neurones\n",
        "#     # {'classifier': [MLPClassifier(hidden_layer_sizes=(100,))],\n",
        "#     #  },\n",
        "# ]\n",
        "\n",
        "# # Appel de la fonction d'exploration de modèles\n",
        "# grid = GridSearchCV(pipe, param_grid, verbose = 2, cv = 3) # verbose pour l'affichage du temps de traitement\n",
        "# # Ajustement des modèles\n",
        "\n",
        "# grid.fit(X_train_sample, y_train_sample)\n",
        "\n",
        "# # Visualisation des résultats\n",
        "# print('best model : ', grid.best_params_)\n",
        "# print('best cv score : ', grid.best_score_)"
      ],
      "metadata": {
        "id": "C0iOcnK3yRSp"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Entrainement et validation"
      ],
      "metadata": {
        "id": "K0bTmotnSwfX"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Echantillonnage\n",
        "X_train_t, X_train_v, y_train_t, y_train_v = train_test_split(X_train,\n",
        "                                                              y_train,\n",
        "                                                              test_size=0.2)\n",
        "\n",
        "# Transformation séparée pour éviter la fuite des données\n",
        "scaler = StandardScaler()\n",
        "X_train_t = scaler.fit_transform(X_train_t)\n",
        "X_train_v = scaler.fit_transform(X_train_v)\n",
        "\n",
        "# Entraînement\n",
        "SVC_model = SVC(kernel='rbf',probability = True, random_state=0, C = 1)\n",
        "SVC_model.fit(X_train_t,y_train_t)\n",
        "\n",
        "# Predictions\n",
        "y_pred = SVC_model.predict(X_train_v)\n",
        "\n",
        "# Validation\n",
        "print('f1 score : ', f1_score(y_train_v,y_pred))\n",
        "print('accuracy score : ', accuracy_score(y_train_v,y_pred))"
      ],
      "metadata": {
        "id": "tVYFjeMBd27b"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Entraînement (sur tout l'échantillon) et enregistrement"
      ],
      "metadata": {
        "id": "0_1YM1Zj8eGo"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# SVM\n",
        "scaler = StandardScaler()\n",
        "X_train = scaler.fit_transform(X_train)\n",
        "\n",
        "# Chargement du modèle retenu\n",
        "SVC_model = SVC(kernel='rbf',probability = True, random_state=0, C = 1)\n",
        "# Ajustement du modèle sur l'échantillon total\n",
        "SVC_model.fit(X_train,y_train)\n",
        "\n",
        "# Enregistrement du modèle\n",
        "joblib.dump(SVC_model, 'Model.gz')"
      ],
      "metadata": {
        "id": "_i7gVZpR7tf4"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Prédictions sur données de test (pour la soumission)"
      ],
      "metadata": {
        "id": "3uOT9xzetEOI"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Concaténation de l'ensemble des features du jeu de données de test\n",
        "test_variables_quali = pd.get_dummies(test[['keyword','text_length']])\n",
        "test_variables_quanti = test[['nb_mots','nb_hashtags','nb_mentions','nb_liens','longueur_mots','city','country','precision']]\n",
        "\n",
        "final_test = pd.concat([test_features, # variables-mots créées à l'aide des tweets (features TF IDF)\n",
        "                        test_variables_quanti, # variables quantitatives créées via l'extraction de données dans les tweets\n",
        "                        test_variables_quali], # variables qualitatives créées via l'extraction de données dans les tweets\n",
        "                         axis='columns')\n",
        "\n",
        "X_test = final_test.reset_index(drop=True)\n",
        "\n",
        "X_test = scaler.fit_transform(X_test)\n",
        "\n",
        "test_predictions_tab = pd.DataFrame(SVC_model.predict(X_test)) \\\n",
        "                            .reset_index(drop=True).rename(columns={0 : 'target_prédite'})\n",
        "\n",
        "test_probas_tab = pd.DataFrame(SVC_model.predict_proba(X_test), columns=[SVC_model.classes_])\n",
        "\n",
        "test_results = pd.concat([test[['id','text']],\n",
        "                          test_predictions_tab,\n",
        "                          test_probas_tab],\n",
        "                          axis=1).dropna() # probas et prédictions\n",
        "\n",
        "test_results.dropna().to_csv('test_results.csv',sep=';',encoding='utf-16')\n",
        "\n",
        "submission = test_results.rename(columns = {'target_prédite' : 'target'})\n",
        "submission = submission[['id','target']].set_index('id')\n",
        "submission.to_csv('submission_SVC.csv')"
      ],
      "metadata": {
        "id": "tr0x__MTS52C"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}