PJN4.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 105,
   "id": "3faac36b-1e2a-4b21-ab38-5c901712237c",
   "metadata": {},
   "outputs": [],
   "source": [
    "from datasets import load_dataset\n",
    "import pandas as pd\n",
    "import spacy\n",
    "\n",
    "dataset = load_dataset(\"clarin-knext/fiqa-pl\", 'corpus')['corpus'].to_pandas()\n",
    "df = pd.DataFrame(dataset)\n",
    "data = []\n",
    "for index, row in df.iterrows():\n",
    "    data.append(row.text.lower())\n",
    "    #zgodnie z treścią zadania, będziemy zapisywać bigramy lowercase."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "4a16cb38-32c3-4abc-b627-75f553d58784",
   "metadata": {},
   "outputs": [],
   "source": [
    "from spacy.tokenizer import Tokenizer\n",
    "from spacy.lang.pl import Polish\n",
    "\n",
    "nlp = Polish()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7ac32a46-76d3-4729-b5fa-79bbcfde0805",
   "metadata": {},
   "source": [
    "### Zadanie 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "7a912083-6515-43e3-aa81-0f25c1f2674c",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Obliczam częstość występowania dla każdego bigramu\n",
    "bigram_dict = {}\n",
    "all_bigrams = 0\n",
    "for doc in data:\n",
    "    tokens = nlp(doc)\n",
    "    for i in range(0, len(tokens) -1):\n",
    "        bigram = tokens[i].text + \" \" + tokens[i+1].text\n",
    "        if bigram in bigram_dict:\n",
    "            bigram_dict[bigram] += 1\n",
    "        else:\n",
    "            bigram_dict[bigram] = 1\n",
    "        all_bigrams += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "248e55bb-30df-4940-907d-2226331deee8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['(;  ', '  powodzenia', 'naprawdę  ', '  .', ')  ', '  what', 'nhập  ', '  01', '.  ', '  \"', 'หนังโป๊  ', '  ไม่ใช่ว่าในประเทศ', 'นั่นเพื่อการดำรงอยู่ของมวลมนุษยชาติ  ', '  การจะมองว่าเป็นเรื่องที่อนาจาร', 'มากกว่าจะใช้เพื่อกระตุ้นทางเพศเท่านั้นเอง  ', '  ใช่ว่าเรื่องเหล่านี้จะถูกยอมรับไปเสียหมด', ',  ', '  ankara', 'fiyatları  ', '  iş', '  argon', '  sakarya', '  adapazarı', '  puset', '  4190', '  cerrahi', '  deri', '  eldiven', '  eldivenler', '  invertür', '  itfaici', '  kesilmez', '  kevlar', '  reusch', '  selex', '  welder', '  göz', '  kaynakçı', '  kaynakçılar', '  kimyasallara', '  toptan', '  yağmurluk', '  yağmurlukçular', '  18001', '  mevcut', '  temel', '  güvenlik', '  acil', '  yangın', '  yasal', '  işyeri', '  sabit', '  poliklinik', 'chwilę  ', '  loppujen', '  asiakkaiden', '  tiedot', '  )', ': \\xa0 ', '\\xa0  zobacz', '] \\xa0 ', '\\xa0  ten', '\\xa0  specjaliści', 'wiele  ', 'możesz \\xa0 ', '\\xa0  twój', '  na', '. \\xa0 ', '\\xa0  [', 'https://finance.yahoo.com/chart/efx#eyjjdxn0b21syw5nzuvuzci6mtq5mtaxote5oswiy3vzdg9tumfuz2vtdgfydci6mtq4nzy1mziwmcwibxvsdgldb2xvckxpbmuiomzhbhnllcjib2xsaw5nzxjvchblcknvbg9yijoii2uymda4msisimjvbgxpbmdlckxvd2vyq29sb3iioiijotu1mmzmiiwibwzptgluzunvbg9yijoiizq1ztnmziisim1hy2reaxzlcmdlbmnlq29sb3iioiijzmy3yjeyiiwibwfjze1hy2rdb2xvcii6iim3oddkodiilcjtywnku2lnbmfsq29sb3iioiijmdawmdawiiwicnnptgluzunvbg9yijoii2zmyjcwmcisinn0b2nos0xpbmvdb2xvcii6iinmzmi3mdailcjzdg9jaermaw5lq29sb3iioiijndvlm2zmiiwicmfuz2uioiixesj9  ', '  this', '  the', '  you', '  while', 'for   ', '   finally', \"  it's\", '  good', '  finally', 'alternatywa \\xa0\\xa0\\xa0\\xa0 ', '\\xa0\\xa0\\xa0\\xa0  kontakt', '00   ', '   1993', '   1994', '   1995', '   1996', '   1997', '   1998', '   1999', '   2000', '   2001', '   2002', '   2003', '   2004', '00  ', '  2005', '   2006', '  2007', '   2008', '   2009', '   2010', '   2011', '   2012', '.oh  ', '...  ', '  i', '\\xa0  potrzeby', ':)  ', '  o']\n"
     ]
    }
   ],
   "source": [
    "keys = []\n",
    "for key in bigram_dict.keys():\n",
    "    if len(key.split(\" \")) > 2:\n",
    "        keys.append(key)\n",
    "print(keys)\n",
    "# odrzucę w następnym kroku bigramy, które spełniają warunek: len(key.split(\" \")) > 2, jak widać poniżej i tak nie spełniają one warunków."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d88e14e7-261a-4c6f-9773-007fe4896392",
   "metadata": {},
   "source": [
    "### Zadanie 3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "id": "ec9c0a9c-2a06-4261-8c3a-689ec6b5f06e",
   "metadata": {},
   "outputs": [],
   "source": [
    "selected_bigram_dict = {}\n",
    "for key in bigram_dict.keys():\n",
    "    if len(key.split(\" \")) != 2:\n",
    "        continue\n",
    "    [key1, key2] = key.split(\" \")\n",
    "    if key1.isalpha() and key2.isalpha():\n",
    "        selected_bigram_dict[key] = bigram_dict[key]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6e2a469b-9262-4573-b833-6d6ac2532059",
   "metadata": {},
   "source": [
    "### Zadanie 4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "id": "d21c5548-c4ec-45be-8afd-609cdfe19193",
   "metadata": {},
   "outputs": [],
   "source": [
    "single_word_dict = {}\n",
    "nlp = Polish()\n",
    "all_single_words = 0\n",
    "for doc in data:\n",
    "    for token in nlp(doc):\n",
    "        if token.text in single_word_dict:\n",
    "            single_word_dict[token.text] += 1\n",
    "        else:\n",
    "            single_word_dict[token.text] = 1\n",
    "        all_single_words += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "id": "32ed7d16-ac1b-4201-a04f-3472b83ec126",
   "metadata": {},
   "outputs": [],
   "source": [
    "import math\n",
    "def count_pointwise_mutual_information(p1p2, p1, p2):\n",
    "    return math.log(p1p2/(p1*p2), 2)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6c0a97eb-af36-4310-b453-edd9e435c604",
   "metadata": {},
   "source": [
    "### Zadanie 5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "id": "82720d85-414f-4405-b690-c568554989ff",
   "metadata": {},
   "outputs": [],
   "source": [
    "dict_with_computed_pointwise_mutual_information = {}\n",
    "for key in selected_bigram_dict.keys():\n",
    "    [key1, key2] = key.split(\" \")\n",
    "    p1p2 = selected_bigram_dict[key]/all_bigrams\n",
    "    p1 = single_word_dict[key1]/all_single_words\n",
    "    p2 = single_word_dict[key2]/all_single_words\n",
    "    dict_with_computed_pointwise_mutual_information[key] = count_pointwise_mutual_information(p1p2, p1, p2)\n",
    "\n",
    "bigram_quantity_list = [[word, dict_with_computed_pointwise_mutual_information[word]] for word in dict_with_computed_pointwise_mutual_information.keys()]\n",
    "bigram_quantity_list.sort(key=lambda x: x[1], reverse=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "id": "6b86fee2-b127-4d96-a7ba-3e51c18495e6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[['jankesem skrzeczącym', 22.947362831472844], ['wizach panamskich', 22.947362831472844], ['devrait éviter', 22.947362831472844], ['facturer fortement', 22.947362831472844], ['peletki chmielowe', 22.947362831472844], ['opalaniem wzmiankowane', 22.947362831472844], ['wielowęzłowy sekwencer', 22.947362831472844], ['sonarowe zamontowane', 22.947362831472844], ['napompowanymi gpas', 22.947362831472844], ['непрекъсната памет', 22.947362831472844]]\n"
     ]
    }
   ],
   "source": [
    "print(bigram_quantity_list[:10])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3ac7867f-ea44-4fef-b238-8f9d015f9de1",
   "metadata": {},
   "source": [
    "### Zadanie 6"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "id": "9b798b45-10b1-4483-8b30-7bdb6978abc6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[['klęska żywiołowa', 20.625434736585483], ['bert hellinger', 20.625434736585483], ['królicza nora', 20.625434736585483], ['инарные опционы', 20.625434736585483], ['опционы олимп', 20.625434736585483], ['олимп трейд', 20.625434736585483], ['мою команду', 20.625434736585483], ['моя группа', 20.625434736585483], ['stucco veneziano', 20.625434736585483], ['остались вопросы', 20.625434736585483]]\n"
     ]
    }
   ],
   "source": [
    "filtered_bigram_quantity_list = [[word, count] for [word, count] in bigram_quantity_list if selected_bigram_dict[word] >= 5]\n",
    "filtered_bigram_quantity_list.sort(key=lambda x: x[1], reverse=True)\n",
    "print(filtered_bigram_quantity_list[:10])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "id": "aec93043-dfd6-4539-a1e4-d01fad5d2734",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[['sharon fussell', 18.859899990222505], ['fine jewelry', 18.625434736585483], ['orzeszki ziemne', 18.61197247677892], ['lợi nhuận', 18.538970645836017], ['adriana fine', 18.537971895335144], ['dolina krzemowa', 18.423800875415832], ['gazów cieplarnianych', 18.423800875415832], ['suze orman', 18.40304231524904], ['ayn rand', 18.13358164025581], ['stwardnienie rozsiane', 18.089381836345275]]\n"
     ]
    }
   ],
   "source": [
    "filtered_bigram_quantity_list2 = [[word, count] for [word, count] in bigram_quantity_list if selected_bigram_dict[word] >= 15]\n",
    "filtered_bigram_quantity_list2.sort(key=lambda x: x[1], reverse=True)\n",
    "print(filtered_bigram_quantity_list2[:10])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f5691662-58a7-4f3a-86ed-ff76c12b05ed",
   "metadata": {},
   "source": [
    "### Zadanie 7, 8, 9"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 194,
   "id": "8e9a152e-7370-4ce0-b755-acb25f08b03e",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing: 100%|██████████| 57638/57638 [32:55<00:00, 29.17it/s]   \n"
     ]
    }
   ],
   "source": [
    "bigram_dict2 = {}\n",
    "from tqdm import tqdm\n",
    "import spacy\n",
    "all_bigrams2 = 0\n",
    "nlp = spacy.load('pl_core_news_sm')\n",
    "iterations = len(data)\n",
    "#iterations = 1000\n",
    "for doc_id in tqdm(range(iterations), desc=\"Processing\"):\n",
    "    doc = data[doc_id]\n",
    "    tokens = nlp(doc)\n",
    "    for i in range(0, len(tokens) -1):\n",
    "        bigram = tokens[i].lemma_ + \":\" + tokens[i].tag_ + \" \" + tokens[i+1].lemma_ + \":\" + tokens[i+1].tag_\n",
    "        if bigram in bigram_dict2:\n",
    "            bigram_dict2[bigram][0] += 1\n",
    "        else:\n",
    "            bigram_dict2[bigram] = [1, tokens[i].lemma_, tokens[i+1].lemma_, tokens[i].tag_, tokens[i+1].tag_]\n",
    "        all_bigrams2 += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 195,
   "id": "5042d881-166d-4c69-97ac-f67e7b4d7a7a",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing: 100%|██████████| 57638/57638 [25:44<00:00, 37.31it/s]  \n"
     ]
    }
   ],
   "source": [
    "iterations = len(data)\n",
    "#iterations = 1000\n",
    "single_word_dict2 = {}\n",
    "all_single_words2 = 0\n",
    "for doc_id in tqdm(range(iterations), desc=\"Processing\"):\n",
    "    doc = data[doc_id]\n",
    "    tokens = nlp(doc)\n",
    "    for token in tokens:\n",
    "        token_key = token.lemma_ + \":\" + token.tag_\n",
    "        if token_key in single_word_dict2:\n",
    "            single_word_dict2[token_key] += 1\n",
    "        else:\n",
    "            single_word_dict2[token_key] = 1\n",
    "    all_single_words2 += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 196,
   "id": "caa35cec-4482-497c-99ed-fa3ac9d0ba7a",
   "metadata": {},
   "outputs": [],
   "source": [
    "selected_bigram_dict2 = {}\n",
    "for key in bigram_dict2.keys():\n",
    "    [_, lemma1, lemma2, _, _] = bigram_dict2[key]\n",
    "    if lemma1.isalpha() and lemma2.isalpha():\n",
    "        selected_bigram_dict2[key] = bigram_dict2[key]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ed484a6e-f27e-41a6-ae4f-a0abf09b2b15",
   "metadata": {},
   "source": [
    "### Zadanie 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 197,
   "id": "b0a18247-cf18-4890-b35a-8fceb2e7c0c4",
   "metadata": {},
   "outputs": [],
   "source": [
    "PMI2_dict = {}\n",
    "for key in selected_bigram_dict2.keys():\n",
    "    [p1p2, lemma1, lemma2, tag1, tag2] = selected_bigram_dict2[key]\n",
    "    p1 = single_word_dict2[lemma1 + \":\" + tag1]/all_single_words2\n",
    "    p2 = single_word_dict2[lemma2 + \":\" + tag2]/all_single_words2\n",
    "    p1p2 = p1p2/all_bigrams2\n",
    "    PMI2_dict[key] = [count_pointwise_mutual_information(p1p2, p1, p2), lemma1, lemma2, tag1, tag2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 198,
   "id": "095ba0b3-ff33-443b-a1a8-1d2a20173c41",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[['bert:SUBST hellinger:SUBST', 6.380946044923653], ['инарные:SUBST опционы:SUBST', 6.380946044923653], ['опционы:SUBST олимп:SUBST', 6.380946044923653], ['олимп:SUBST трейд:SUBST', 6.380946044923653], ['моюa:SUBST команду:SUBST', 6.380946044923653], ['моя:SUBST группа:SUBST', 6.380946044923653], ['остались:SUBST вопросы:SUBST', 6.380946044923653], ['gone:PPAS fishin:SUBST', 6.1179116390898605], ['sameer:SUBST thakar:SUBST', 6.1179116390898605], ['deming:XXX electro:SUBST', 6.1179116390898605]]\n"
     ]
    }
   ],
   "source": [
    "def sort_list_and_return(list):\n",
    "    list.sort(key = lambda x: x[1], reverse = True)\n",
    "    return list\n",
    "\n",
    "print(sort_list_and_return([[word, PMI2_dict[word][0]] for word in PMI2_dict.keys() if selected_bigram_dict2[word][0] >= 5])[:10])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "557a1592-7921-4124-b9fc-001a7445b29d",
   "metadata": {},
   "source": [
    "### Zadanie 11"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 199,
   "id": "79957f70-2daf-4231-9787-b941c0373910",
   "metadata": {},
   "outputs": [],
   "source": [
    "groupped_bigrams = {}\n",
    "\n",
    "for key in PMI2_dict.keys():\n",
    "    [_, _, _, tag1, tag2] = PMI2_dict[key]\n",
    "    group = tag1 + \" \" + tag2\n",
    "    if group in groupped_bigrams:\n",
    "        groupped_bigrams[group].append([key, PMI2_dict[key]])\n",
    "    else:\n",
    "        groupped_bigrams[group] = [[key, PMI2_dict[key]]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 200,
   "id": "157b5677-cf0e-4e99-b2e4-b008c83fd121",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['QUB FIN', 'COMP QUB', 'FIN PPRON12', 'PPRON12 QUB', 'QUB QUB', 'QUB SUBST', 'SUBST GER', 'GER PREP', 'PREP SUBST', 'SUBST SUBST']\n"
     ]
    }
   ],
   "source": [
    "print(list(groupped_bigrams.keys())[:10])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bb8dc648-dacc-4864-9cc8-63e9ad752cc0",
   "metadata": {},
   "source": [
    "### Zadanie 12"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 201,
   "id": "eb404ffd-7674-4038-9084-5a3ff987fbb1",
   "metadata": {},
   "outputs": [],
   "source": [
    "def print_group(group, group_name):\n",
    "    group.sort(key = lambda x: x[1][0], reverse = True)\n",
    "    print(\"\\nGroup name:\", group_name, \"\\n\")\n",
    "    for i in range(5):\n",
    "        print(f\"  {i}:\",group[i][0])\n",
    "    print(\"-------------------------------\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 202,
   "id": "dbc005ca-ff87-4838-a1d4-ea750086197f",
   "metadata": {},
   "outputs": [],
   "source": [
    "groupped_bigrams_list = sort_list_and_return([[key, len(groupped_bigrams[key]), groupped_bigrams[key]] for key in groupped_bigrams.keys()])[0:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 203,
   "id": "24da19d0-b81f-4890-b0b5-c7e4c5928ad6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Group name: SUBST SUBST \n",
      "\n",
      "  0: devrait:SUBST éviter:SUBST\n",
      "  1: assignment:SUBST databaseo:SUBST\n",
      "  2: непрекъсната:SUBST памет:SUBST\n",
      "  3: наречени:SUBST блокове:SUBST\n",
      "  4: запазва:SUBST записите:SUBST\n",
      "-------------------------------\n",
      "\n",
      "Group name: ADJ SUBST \n",
      "\n",
      "  0: wielowęzłowy:ADJ sekwencer:SUBST\n",
      "  1: napompowanymi:ADJ gpas:SUBST\n",
      "  2: продължителен:ADJ период:SUBST\n",
      "  3: съхранение:ADJ записва:SUBST\n",
      "  4: elektronikos:ADJ parduotuv휊:SUBST\n",
      "-------------------------------\n",
      "\n",
      "Group name: SUBST ADJ \n",
      "\n",
      "  0: peletka:SUBST chmielowy:ADJ\n",
      "  1: която:SUBST изтрива:ADJ\n",
      "  2: бъдат:SUBST записание:ADJ\n",
      "  3: резервно:SUBST копие:ADJ\n",
      "  4: куп:SUBST случаи:ADJ\n",
      "-------------------------------\n",
      "\n",
      "Group name: SUBST FIN \n",
      "\n",
      "  0: u탑:SUBST prieinam훳:FIN\n",
      "  1: gautum휊te:SUBST daugia:FIN\n",
      "  2: stevej:SUBST popra:FIN\n",
      "  3: розничного:SUBST торгового:FIN\n",
      "  4: ولكنها:SUBST ليستa:FIN\n",
      "-------------------------------\n",
      "\n",
      "Group name: PREP SUBST \n",
      "\n",
      "  0: gwola:PREP ścisłość:SUBST\n",
      "  1: nahi:PREP duzun:SUBST\n",
      "  2: at:PREP produksjonsvekst:SUBST\n",
      "  3: nhảnmmmmmmmm:PREP mớnmć:SUBST\n",
      "  4: mớnmmmmm:PREP mớnmm:SUBST\n",
      "-------------------------------\n",
      "\n",
      "Group name: FIN SUBST \n",
      "\n",
      "  0: prieinam훳:FIN kain훳:SUBST\n",
      "  1: daugia:FIN informacijosa:SUBST\n",
      "  2: synonić:FIN ubijak:SUBST\n",
      "  3: торгового:FIN оборота:SUBST\n",
      "  4: zapobiegnieć:FIN problemom:SUBST\n",
      "-------------------------------\n",
      "\n",
      "Group name: SUBST PREP \n",
      "\n",
      "  0: ulert:SUBST nahi:PREP\n",
      "  1: haber:SUBST dejado:PREP\n",
      "  2: mớnmć:SUBST mớnmmmmm:PREP\n",
      "  3: temprać:SUBST pued:PREP\n",
      "  4: califica:SUBST în:PREP\n",
      "-------------------------------\n",
      "\n",
      "Group name: INF SUBST \n",
      "\n",
      "  0: tilbyr:INF insentiver:SUBST\n",
      "  1: płatać:INF figle:SUBST\n",
      "  2: لخلق:INF القيمة:SUBST\n",
      "  3: implied:INF volatility:SUBST\n",
      "  4: saya:INF sarankan:SUBST\n",
      "-------------------------------\n",
      "\n",
      "Group name: CONJ SUBST \n",
      "\n",
      "  0: yra:CONJ specializuota:SUBST\n",
      "  1: eskakizunak:CONJ betetzen:SUBST\n",
      "  2: tọa:CONJ lạc:SUBST\n",
      "  3: itulah:CONJ sebabnya:SUBST\n",
      "  4: scheint:CONJ offensichtli:SUBST\n",
      "-------------------------------\n",
      "\n",
      "Group name: GER SUBST \n",
      "\n",
      "  0: industrien:GER akselerert:SUBST\n",
      "  1: opryskiwanie:GER szkodnika:SUBST\n",
      "  2: wulkać:GER arenal:SUBST\n",
      "  3: odparzenie:GER pieluszkowy:SUBST\n",
      "  4: wyprać:GER ściereczka:SUBST\n",
      "-------------------------------\n"
     ]
    }
   ],
   "source": [
    "for group in groupped_bigrams_list:\n",
    "    print_group(group[2], group[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 204,
   "id": "2fb417fc-22b7-4e68-a239-b452fbae4031",
   "metadata": {},
   "outputs": [],
   "source": [
    "most_popular1 = sort_list_and_return([[word, count] for [word, count] in bigram_quantity_list if selected_bigram_dict[word] >= 15])[:10]\n",
    "most_popular2 = sort_list_and_return([[word, PMI2_dict[word][0]] for word in PMI2_dict.keys() if selected_bigram_dict2[word][0] >= 15])[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 205,
   "id": "d79a9ff1-f5d1-4dcd-8e33-f98849023379",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Porównanie top 10 elementów\n",
      "  0: sharon fussell | ława:SUBST przysięgły:SUBST\n",
      "  1: fine jewelry | navi:ADJ mumbai:SUBST\n",
      "  2: orzeszki ziemne | Canadian:SUBST Couch:SUBST\n",
      "  3: lợi nhuận | ctcp:ADJ ck:QUB\n",
      "  4: adriana fine | stwardnienie:SUBST rozsiać:ADJ\n",
      "  5: dolina krzemowa | bear:SUBST stearns:SUBST\n",
      "  6: gazów cieplarnianych | Couch:SUBST potato:SUBST\n",
      "  7: suze orman | shri:SUBST vinayak:SUBST\n",
      "  8: ayn rand | monte:SUBST carlo:SUBST\n",
      "  9: stwardnienie rozsiane | straż:SUBST pożarny:ADJ\n"
     ]
    }
   ],
   "source": [
    "print(\"Porównanie top 10 elementów\")\n",
    "for i in range(len(most_popular1)):\n",
    "    print(f\"  {i}:\", most_popular1[i][0], \"|\", most_popular2[i][0])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bce2f1bf-82ab-4b69-9189-5d7af807cfc0",
   "metadata": {},
   "source": [
    "## Why do we have to filter the bigrams, rather than the token sequence?\n",
    "\n",
    "Filtrujemy bigramy, a nie sekwencję tokentów, gdyż po odfiltrowaniu tokenów z sekwencji otrzymalibyśmy bigramy, które w rzeczywistości nie występują w tekście. Chodzi mi o sytuację: Słowo1 Słowo2 Słowo3 Słowo4. Gdybyśmy odfiltrowali z sekwencji Słowo2, w bigramach otrzymalibyśmy między innymi bigram: \"Słowo1 Słowo3\", co byłoby błędem\n",
    "\n",
    "## What types of expressions are discovered by the methods.\n",
    "\n",
    "W trakcie wykonywania zadania, filtrowaliśmy elementy o najwyższym współczynniku PMI. Wysoką wartość tę metryki posiadają bigramy, które składają się ze słów, które występują jedynie w konkretnym bigramie i nie są używane samodzielnie, lub w innym bigramie. Przykładem takiego słowa może być \"cieplarnianych\", które występuje praktycznie jedynie w bigramie \"gazy cieplarniane\". \n",
    "\n",
    "## Can you devise a different type of filtering that would yield better results?\n",
    "\n",
    "W zadaniu polecone było odfiltrowanie bigramów, występujących mniej niż 5 razy. Wynikowe bigramy były jednak dość dziwne, tak więc zdecydowałem się na użycie innego filtrowania, do przygotowania wniosków z tego laboratorium. Zastosowałem filtrowanie, w którym usuwam bigramy występujące mniej niż 15 razy. Warto dostosować tą liczbę do wielkości tekstu, tak aby usunąć \"dziwne\" bigramy, których słowa składowe są zbyt nietypowe, aby występować samodzielnie. \n",
    "\n",
    "Inną metodą filtracji mogłoby być odfiltrowanie bigramów, których słowa składowe występują zbyt małą liczbę razy. Przykładowo, możnaby odfiltrować bigramy, dla których liczba występień pierwszego słowa zsumowana z liczbą wystąpień drugiego słowa jest mniejsza niż 30."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fd6cd23c-9c32-4c17-a295-0aedd7c0314f",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}