Trigram_Model_and_Top_Words_Extraction.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/malabyte/Text-Mining/blob/main/Trigram_Model_and_Top_Words_Extraction.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import re\n",
        "from collections import defaultdict"
      ],
      "metadata": {
        "id": "lOw-zyWPKakw"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def preprocess_text(text):\n",
        "    stop_words = ['A', 'An', 'for', 'the', 'at', 'and']\n",
        "    text = re.sub(r'[^\\w\\s]', '', text)\n",
        "    text = text.lower()\n",
        "    words = text.split()\n",
        "    words = [word for word in words if word not in stop_words]\n",
        "    return words"
      ],
      "metadata": {
        "id": "BKCMWcFWKfhW"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def build_trigram_model(texts, top_n):\n",
        "    trigram_model = defaultdict(lambda: defaultdict(int))\n",
        "\n",
        "def build_trigram_model(texts, top_n):\n",
        "    trigram_model = defaultdict(lambda: defaultdict(int))\n",
        "\n",
        "    for text in texts:\n",
        "        words = preprocess_text(text)\n",
        "        for i in range(len(words)-2):\n",
        "            w1, w2, w3 = words[i], words[i+1], words[i+2]\n",
        "            trigram_model[(w1, w2)][w3] += 1\n",
        "\n",
        "    top_n_words = []\n",
        "    for key in trigram_model.keys():\n",
        "        top_words = sorted(trigram_model[key], key=trigram_model[key].get, reverse=True)[:top_n]\n",
        "        top_n_words.extend(top_words)\n",
        "\n",
        "    return trigram_model, list(set(top_n_words))"
      ],
      "metadata": {
        "id": "SYc97jQzKi_o"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "texts = [\n",
        "    \"I can no longer fly around the globe for 45 min talks/seminars, and at the same time explain my sons how their future planet will look like.\",\n",
        "    \"You owe some people your absence.\",\n",
        "    \"I've heard it all now, it's NOT hypocrisy to fly around the globe telling ordinary folk to use their cars less & walk to work if you're a climate change advocate.\"\n",
        "]\n",
        "\n",
        "top_n = 5\n",
        "trigram_model, top_n_words = build_trigram_model(texts, top_n)\n",
        "\n",
        "print(\"Trigram Model:\")\n",
        "for key in trigram_model.keys():\n",
        "    print(key, \"->\", dict(trigram_model[key]))\n",
        "\n",
        "print(\"\\nTop\", top_n, \"Frequent Words:\")\n",
        "print(top_n_words)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "pkgbqX88KaiP",
        "outputId": "cf50f7fc-7f4c-46dc-a3ec-50a447471784"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Trigram Model:\n",
            "('i', 'can') -> {'no': 1}\n",
            "('can', 'no') -> {'longer': 1}\n",
            "('no', 'longer') -> {'fly': 1}\n",
            "('longer', 'fly') -> {'around': 1}\n",
            "('fly', 'around') -> {'globe': 2}\n",
            "('around', 'globe') -> {'45': 1, 'telling': 1}\n",
            "('globe', '45') -> {'min': 1}\n",
            "('45', 'min') -> {'talksseminars': 1}\n",
            "('min', 'talksseminars') -> {'same': 1}\n",
            "('talksseminars', 'same') -> {'time': 1}\n",
            "('same', 'time') -> {'explain': 1}\n",
            "('time', 'explain') -> {'my': 1}\n",
            "('explain', 'my') -> {'sons': 1}\n",
            "('my', 'sons') -> {'how': 1}\n",
            "('sons', 'how') -> {'their': 1}\n",
            "('how', 'their') -> {'future': 1}\n",
            "('their', 'future') -> {'planet': 1}\n",
            "('future', 'planet') -> {'will': 1}\n",
            "('planet', 'will') -> {'look': 1}\n",
            "('will', 'look') -> {'like': 1}\n",
            "('you', 'owe') -> {'some': 1}\n",
            "('owe', 'some') -> {'people': 1}\n",
            "('some', 'people') -> {'your': 1}\n",
            "('people', 'your') -> {'absence': 1}\n",
            "('ive', 'heard') -> {'it': 1}\n",
            "('heard', 'it') -> {'all': 1}\n",
            "('it', 'all') -> {'now': 1}\n",
            "('all', 'now') -> {'its': 1}\n",
            "('now', 'its') -> {'not': 1}\n",
            "('its', 'not') -> {'hypocrisy': 1}\n",
            "('not', 'hypocrisy') -> {'to': 1}\n",
            "('hypocrisy', 'to') -> {'fly': 1}\n",
            "('to', 'fly') -> {'around': 1}\n",
            "('globe', 'telling') -> {'ordinary': 1}\n",
            "('telling', 'ordinary') -> {'folk': 1}\n",
            "('ordinary', 'folk') -> {'to': 1}\n",
            "('folk', 'to') -> {'use': 1}\n",
            "('to', 'use') -> {'their': 1}\n",
            "('use', 'their') -> {'cars': 1}\n",
            "('their', 'cars') -> {'less': 1}\n",
            "('cars', 'less') -> {'walk': 1}\n",
            "('less', 'walk') -> {'to': 1}\n",
            "('walk', 'to') -> {'work': 1}\n",
            "('to', 'work') -> {'if': 1}\n",
            "('work', 'if') -> {'youre': 1}\n",
            "('if', 'youre') -> {'a': 1}\n",
            "('youre', 'a') -> {'climate': 1}\n",
            "('a', 'climate') -> {'change': 1}\n",
            "('climate', 'change') -> {'advocate': 1}\n",
            "\n",
            "Top 5 Frequent Words:\n",
            "['its', 'how', 'absence', 'folk', 'less', 'longer', 'walk', 'cars', 'like', 'globe', 'telling', 'youre', 'not', 'your', 'people', 'min', 'explain', 'hypocrisy', 'work', 'now', 'sons', 'change', 'no', 'time', 'will', 'use', 'climate', 'look', 'some', 'advocate', 'my', 'ordinary', 'same', 'a', 'their', 'planet', 'it', 'around', 'talksseminars', 'if', 'fly', 'all', 'to', 'future', '45']\n"
          ]
        }
      ]
    }
  ]
}