Ignatiev_kw.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json, os\n",
    "import pandas as pd\n",
    "from nltk.corpus import stopwords\n",
    "import numpy as np\n",
    "from pymorphy2 import MorphAnalyzer\n",
    "from collections import Counter\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "morph = MorphAnalyzer()\n",
    "stops = set(stopwords.words('russian'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.set_option('display.max_colwidth', 1000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "PATH_TO_DATA = './'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "files = [os.path.join(PATH_TO_DATA, file) for file in os.listdir(PATH_TO_DATA) if file.endswith('jsonlines')]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = pd.concat([pd.read_json(file, lines=True) for file in files][:5], axis=0, ignore_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1987, 5)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "def evaluate(true_kws, predicted_kws):\n",
    "    assert len(true_kws) == len(predicted_kws)\n",
    "    \n",
    "    precisions = []\n",
    "    recalls = []\n",
    "    f1s = []\n",
    "    jaccards = []\n",
    "    \n",
    "    for i in range(len(true_kws)):\n",
    "        \n",
    "        true_kw = set(true_kws[i])\n",
    "        predicted_kw = set(predicted_kws[i])\n",
    "        \n",
    "        tp = len(true_kw & predicted_kw)\n",
    "        union = len(true_kw | predicted_kw)\n",
    "        fp = len(predicted_kw - true_kw)\n",
    "        fn = len(true_kw - predicted_kw)\n",
    "        \n",
    "        if (tp+fp) == 0:\n",
    "            prec = 0\n",
    "        else:\n",
    "            prec = tp / (tp + fp)\n",
    "        \n",
    "        if (tp+fn) == 0:\n",
    "            rec = 0\n",
    "        else:\n",
    "            rec = tp / (tp + fn)\n",
    "        if (prec+rec) == 0:\n",
    "            f1 = 0\n",
    "        else:\n",
    "            f1 = (2*(prec*rec))/(prec+rec)\n",
    "            \n",
    "        jac = tp / union\n",
    "        \n",
    "        precisions.append(prec)\n",
    "        recalls.append(rec)\n",
    "        f1s.append(f1)\n",
    "        jaccards.append(jac)\n",
    "    print('Precision - ', round(np.mean(precisions), 2))\n",
    "    print('Recall - ', round(np.mean(recalls), 2))\n",
    "    print('F1 - ', round(np.mean(f1s), 2))\n",
    "    print('Jaccard - ', round(np.mean(jaccards), 2))\n",
    "    \n",
    "    \n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "from string import punctuation\n",
    "from nltk.corpus import stopwords\n",
    "punct = punctuation+'«»—…“”*№–'\n",
    "stops = set(stopwords.words('russian'))\n",
    "\n",
    "def normalize(text):\n",
    "    \n",
    "    words = [word.strip(punct) for word in text.lower().split()]\n",
    "    words = [morph.parse(word)[0].normal_form for word in words if word and word not in stops]\n",
    "\n",
    "    return words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "data['content_norm'] = data['content'].apply(normalize)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "data['title_norm'] = data['title'].apply(normalize)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def normalize(text):\n",
    "    \n",
    "    words = [word.strip(punct) for word in text.lower().split()]\n",
    "    words = [morph.parse(word)[0] for word in words if word and word not in stops]\n",
    "    words = [word.normal_form for word in words if word.tag.POS == 'NOUN']\n",
    "\n",
    "    return words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "data['content_norm'] = data['content'].apply(normalize)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "data['content_norm_str'] = data['content_norm'].apply(' '.join)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# можно заодно сделать нграммы\n",
    "tfidf = TfidfVectorizer(ngram_range=(1,2), min_df=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "TfidfVectorizer(min_df=2, ngram_range=(1, 2))"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tfidf.fit(data['content_norm_str'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "id2word = {i:word for i,word in enumerate(tfidf.get_feature_names())}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "texts_vectors = tfidf.transform(data['content_norm_str'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## BASELINE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "## так как матрица в tfidf в спарс формате,  ее нельзя просто так отсортировать\n",
    "## перевести ее в обычный формат для всех данных тоже не получится - не хватит памяти\n",
    "## поэтому пройдем по строчкам, переведем строчку в обычный array и отсортируем ее\n",
    "keywords = []\n",
    "\n",
    "for row in range(texts_vectors.shape[0]):\n",
    "    row_data = texts_vectors.getrow(row)\n",
    "    top_inds = row_data.toarray().argsort()[0,:-11:-1]\n",
    "    keywords.append([id2word[w] for w in top_inds])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Precision -  0.13\n",
      "Recall -  0.25\n",
      "F1 -  0.16\n",
      "Jaccard -  0.09\n"
     ]
    }
   ],
   "source": [
    "evaluate(data['keywords'], keywords)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Precision -  0.13\n",
    "Recall -  0.25\n",
    "F1 -  0.16\n",
    "Jaccard -  0.09"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## ОПЫТ 1: добавление ЛДА"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "import gensim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "dictionary = gensim.corpora.Dictionary(data['content_norm'])\n",
    "dictionary.filter_extremes(no_above=0.075, no_below=25)\n",
    "dictionary.compactify()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "corpus = data[\"content_norm\"].apply(\n",
    "    lambda x: dictionary.doc2bow(x)\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "lda = gensim.models.LdaMulticore(corpus, 200, alpha=0.11, eta=0.66, id2word=dictionary, eval_every=0, passes=15)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_topic_tokens(model, corpus):\n",
    "    tokens = []\n",
    "    for index in range(len(corpus)):\n",
    "        probs = model[corpus[index]]\n",
    "        if len(probs) == 0: \n",
    "            tokens.append(list())\n",
    "            continue\n",
    "        relevant_key = max(probs, key = lambda x: x[1])[0]\n",
    "        topic_tokens = [i[0] for i in model.show_topic(relevant_key)]\n",
    "        tokens.append(topic_tokens)\n",
    "    return tokens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "t_t = get_topic_tokens(lda, corpus)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [],
   "source": [
    "frequent = data['content_norm'].apply(lambda x: [x[0] for x in Counter(x).most_common(20)])\n",
    "filtered = []\n",
    "for idx in range(len(frequent)):\n",
    "    row_data = texts_vectors.getrow(idx)\n",
    "    top_inds = row_data.toarray().argsort()[0,:-21:-1]\n",
    "    top_words = [id2word[w] for w in top_inds]\n",
    "    if len(t_t[idx]) > 0:\n",
    "        common = [i for i in frequent[idx] if (i in t_t[idx] or i in top_words)]\n",
    "        filtered.append(common)\n",
    "    else:\n",
    "        filtered.append([i for i in frequent[idx] if i in top_words])\n",
    "       "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Выше реколл и эф-1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Precision -  0.14\n",
      "Recall -  0.29\n",
      "F1 -  0.18\n",
      "Jaccard -  0.1\n"
     ]
    }
   ],
   "source": [
    "evaluate(data['keywords'], filtered)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## ОПЫТ 2: добавление эмбеддингов с русвекторес"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "fasttext_model = gensim.models.KeyedVectors.load(\"fasttext/model.model\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics.pairwise import cosine_similarity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "def embed_list(lst:list, model:object):\n",
    "    arr = np.array([model.get_vector(n) for n in lst])\n",
    "    return np.average(arr, axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "averaged_embeddings = [embed_list(item, fasttext_model) for item in data[\"content_norm\"]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "most_common = data['content_norm'].apply(lambda x: [x[0] for x in Counter(x).most_common(30)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "relevant = [\n",
    "    sorted(most_common[idx],\n",
    "           reverse=True,\n",
    "           key=lambda x: cosine_similarity(\n",
    "               np.atleast_2d(fasttext_model[x]),\n",
    "               np.atleast_2d(averaged_embeddings[idx]))[0])[:15] \n",
    "    for idx \n",
    "    in range(len(most_common))\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "frequent = data['content_norm'].apply(lambda x: [x[0] for x in Counter(x).most_common(30)])\n",
    "filtered = []\n",
    "for idx in range(len(frequent)):\n",
    "    row_data = texts_vectors.getrow(idx)\n",
    "    top_inds = row_data.toarray().argsort()[0,:-21:-1]\n",
    "    top_words = [id2word[w] for w in top_inds]\n",
    "    common = [i for i in frequent[idx] if (i in relevant[idx] and i in top_words)]\n",
    "    filtered.append(common)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Выше пресижн и ф-1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 146,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Precision -  0.16\n",
      "Recall -  0.24\n",
      "F1 -  0.18\n",
      "Jaccard -  0.11\n"
     ]
    }
   ],
   "source": [
    "evaluate(data['keywords'], filtered)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## ОПЫТ 2.5: комбинация"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "filtered = []\n",
    "for idx in range(len(frequent)):\n",
    "    row_data = texts_vectors.getrow(idx)\n",
    "    top_inds = row_data.toarray().argsort()[0,:-21:-1]\n",
    "    top_words = [id2word[w] for w in top_inds]\n",
    "    if len(t_t[idx]) > 0:\n",
    "        common = [i for i in frequent[idx] if (i in t_t[idx] or (i in relevant[idx] and i in top_words))]\n",
    "        filtered.append(common)\n",
    "    else:\n",
    "        filtered.append([i for i in frequent[idx] if (i in relevant[idx] and i in top_words)])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Тот же пресижн, больше реколл"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 152,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Precision -  0.16\n",
      "Recall -  0.26\n",
      "F1 -  0.18\n",
      "Jaccard -  0.11\n"
     ]
    }
   ],
   "source": [
    "evaluate(data['keywords'], filtered)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## ОПЫТ 3: то же + другая метрика для графа"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "from itertools import combinations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\tqdm\\std.py:668: FutureWarning: The Panel class is removed from pandas. Accessing it from the top-level namespace will also be removed in the next version\n",
      "  from pandas import Panel\n"
     ]
    }
   ],
   "source": [
    "from tqdm import tqdm\n",
    "tqdm.pandas()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Попбруем теперь важность считать с помощью какой-нибудь метрики из networkx."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "import networkx as nx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "def build_matrix(text, window_size=5):\n",
    "    vocab = set(text)\n",
    "    word2id = {w:i for i, w in enumerate(vocab)}\n",
    "    id2word = {i:w for i, w in enumerate(vocab)}\n",
    "    # преобразуем слова в индексы для удобства\n",
    "    ids = [word2id[word] for word in text]\n",
    "\n",
    "    # создадим матрицу совстречаемости\n",
    "    m = np.zeros((len(vocab), len(vocab)))\n",
    "\n",
    "    # пройдемся окном по всему тексту\n",
    "    for i in range(0, len(ids), window_size):\n",
    "        window = ids[i:i+window_size]\n",
    "        # добавим единичку всем парам слов в этом окне\n",
    "        for j, k in combinations(window, 2):\n",
    "            # чтобы граф был ненаправленный \n",
    "            m[j][k] += 1\n",
    "            m[k][j] += 1\n",
    "    \n",
    "    return m, id2word\n",
    "\n",
    "def some_centrality_measure(text, window_size=5, topn=5):\n",
    "    \n",
    "    matrix, id2word = build_matrix(text, window_size)\n",
    "    G = nx.from_numpy_array(matrix)\n",
    "    # тут можно поставить любую метрику\n",
    "    # менять тут \n",
    "    node2measure = dict(nx.betweenness_centrality(G)) \n",
    "    \n",
    "    return [id2word[index] for index,measure in sorted(node2measure.items(), key=lambda x: -x[1])[:topn]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 8min 52s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "keyword_nx = data['content_norm'].apply(lambda x: some_centrality_measure(x, 10, 20))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "keyword_nx_filtered = []\n",
    "for idx in range(len(keyword_nx)):\n",
    "    keyword_nx_filtered.append(\n",
    "    [item for item in keyword_nx[idx] if item in filtered[idx]]\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Выше пресижн, эф-1 та же"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Precision -  0.17\n",
      "Recall -  0.23\n",
      "F1 -  0.18\n",
      "Jaccard -  0.11\n"
     ]
    }
   ],
   "source": [
    "evaluate(data['keywords'], keyword_nx_filtered)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}