data analysis/7.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "811d1fcc-2a1a-449e-9834-b988ed4af55f",
   "metadata": {},
   "source": [
    "# Натянуть сову на линейное пространство\n",
    "\n",
    "Создайте эмбеддинги слов и визуализируйте векторные операции над ними: сложение, вычитание, взятие ближайшего, дальнейшего и прочее. Сравните качество представлений gensim и BERT с точки зрения операций над словами, докажите примерами.\n",
    "\n",
    "Для создания эмбеддингов с gensim обучите модель на нормализованных текстовых данных. Данные найдите на kaggle или выберите один из предложенных датасетов. Для создания эмбеддингов с BERT используйте предобученные модели.\n",
    "\n",
    "Предлагаемые датасеты:\n",
    " - [sentiment твитов про ковид](https://www.kaggle.com/datatattle/covid-19-nlp-text-classification)\n",
    " - [Amazon product reviews](https://www.kaggle.com/kashnitsky/hierarchical-text-classification) - этот\n",
    " - [Отзывы интернет-магазина](https://www.kaggle.com/shymammoth/shopee-reviews)\n",
    " - [Тексты статей конференции NIPS](https://www.kaggle.com/rowhitswami/nips-papers-1987-2019-updated?select=papers.csv)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "9cd28820",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Title</th>\n",
       "      <th>Text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Proraso Proraso Liquid Cream After-Shave 3.4oz</td>\n",
       "      <td>I do not know what they were thinking with all...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>PondCare 169J Algae Fix, 2-1/2 Gallon</td>\n",
       "      <td>I have used Algae Fix once a week in my pond a...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Philips Norelco Bodygroom Shaver</td>\n",
       "      <td>Norelco got it right with their new Bodygroom....</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>CHAMPION Neoprene Knee Support with Open Patella</td>\n",
       "      <td>very comfortable support, at least compared to...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Melissa &amp; Doug Baby Zoo Animals Stamp Set</td>\n",
       "      <td>The stamps are a little smaller than I thought...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                              Title  \\\n",
       "0    Proraso Proraso Liquid Cream After-Shave 3.4oz   \n",
       "1             PondCare 169J Algae Fix, 2-1/2 Gallon   \n",
       "2                  Philips Norelco Bodygroom Shaver   \n",
       "3  CHAMPION Neoprene Knee Support with Open Patella   \n",
       "4         Melissa & Doug Baby Zoo Animals Stamp Set   \n",
       "\n",
       "                                                Text  \n",
       "0  I do not know what they were thinking with all...  \n",
       "1  I have used Algae Fix once a week in my pond a...  \n",
       "2  Norelco got it right with their new Bodygroom....  \n",
       "3  very comfortable support, at least compared to...  \n",
       "4  The stamps are a little smaller than I thought...  "
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "\n",
    "sns.set_theme()\n",
    "\n",
    "df = pd.read_csv('unlabeled_150k.csv')\n",
    "df.head(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8bf2bcf8",
   "metadata": {},
   "source": [
    "## Нормализация"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "06c8849e",
   "metadata": {},
   "outputs": [],
   "source": [
    "from nltk.corpus import stopwords\n",
    "import nltk"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "6e827b20",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package omw-1.4 to /home/jovvik/nltk_data...\n",
      "[nltk_data]   Package omw-1.4 is already up-to-date!\n",
      "[nltk_data] Downloading package wordnet to /home/jovvik/nltk_data...\n",
      "[nltk_data]   Package wordnet is already up-to-date!\n"
     ]
    }
   ],
   "source": [
    "from nltk.stem import WordNetLemmatizer, PorterStemmer\n",
    "from nltk.corpus import wordnet\n",
    "\n",
    "nltk.download('omw-1.4')\n",
    "nltk.download('wordnet')\n",
    "\n",
    "lemmatizer = WordNetLemmatizer()\n",
    "stemmer = PorterStemmer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "ac900a75",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package stopwords to /home/jovvik/nltk_data...\n",
      "[nltk_data]   Package stopwords is already up-to-date!\n"
     ]
    }
   ],
   "source": [
    "nltk.download('stopwords')\n",
    "eng_stopwords = set(stopwords.words('english'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "11d69288",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 150000/150000 [00:18<00:00, 8067.31it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0    know thinking reformulations excellent product...\n",
       "1    used algae fix week pond kept algae check unli...\n",
       "2    norelco got right new bodygroom shaver promise...\n",
       "3    comfortable support least compared elastic one...\n",
       "4    stamp little smaller thought would others stat...\n",
       "Name: Text, dtype: object"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def preprocess_text(text: str) -> str:\n",
    "    text = text.lower()\n",
    "    def is_allowed_char(c): return c.isalpha() or c == ' '\n",
    "    text = ''.join(list(filter(is_allowed_char, text)))\n",
    "\n",
    "    def is_stopword(word): return word not in eng_stopwords\n",
    "    text = ' '.join(list(filter(is_stopword, text.split())))\n",
    "\n",
    "    text = ' '.join(list(map(lemmatizer.lemmatize, text.split())))\n",
    "    return text\n",
    "\n",
    "from tqdm import tqdm\n",
    "tqdm.pandas()\n",
    "df = df.Text.progress_apply(preprocess_text)\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "da902c69",
   "metadata": {},
   "source": [
    "## Gensim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "10492d14",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/jovvik/anaconda3/envs/ad7/lib/python3.9/site-packages/gensim/similarities/__init__.py:15: UserWarning: The gensim.similarities.levenshtein submodule is disabled, because the optional Levenshtein package <https://pypi.org/project/python-Levenshtein/> is unavailable. Install Levenhstein (e.g. `pip install python-Levenshtein`) to suppress this warning.\n",
      "  warnings.warn(msg)\n"
     ]
    }
   ],
   "source": [
    "from gensim.models import Word2Vec"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "cc91491c",
   "metadata": {},
   "outputs": [],
   "source": [
    "reviews = [i.split() for i in df]\n",
    "word_count = {}\n",
    "for sentence in reviews:\n",
    "    for word in sentence:\n",
    "        word_count[word] = (word_count.get(word) or 0) + 1\n",
    "words_by_freq = [i[0] for i in sorted(word_count.items(), key=lambda x: x[1])][::-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "a05d6a18",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "размерность векторов слов в модели: (154349, 100)\n"
     ]
    }
   ],
   "source": [
    "gmodel = Word2Vec(sentences=reviews,\n",
    "                  sg=False,  # cbow model\n",
    "                  vector_size=100,\n",
    "                  window=5,\n",
    "                  seed=0,\n",
    "                  epochs=15,\n",
    "                  min_count=1,\n",
    "                  workers=8)\n",
    "print('размерность векторов слов в модели:', gmodel.wv.vectors.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "67617732",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Самые похожие\n",
      "ship ['shipped', 'shipping', 'shipper', 'send', 'arrive', 'fee', 'mailed', 'sent', 'sh', 'sending']\n",
      "man ['guy', 'lady', 'woman', 'gentleman', 'gay', 'dude', 'men', 'ya', 'supergirl', 'bos']\n",
      "order ['reorder', 'ordering', 'purchase', 'buy', 'ordered', 'shipment', 'return', 'iparcel', 'cancel', 'stock']\n",
      "Самые непохожие\n",
      "ship ['commandreward', 'detailfirst', 'jawpropping', 'exfoliating', 'struggled', 'someithing', 'breastmilkformula', 'oilbased', 'washcloth', 'productsfelt']\n",
      "man ['visco', 'woil', 'eyesbut', 'achiveve', 'ruinedmy', 'provided', 'bridgetunnel', 'factmy', 'blanketeven', 'minimal']\n",
      "order ['withstands', 'kidspros', 'ribbit', 'realisticlooking', 'musclesin', 'deetz', 'prmotes', 'sratchng', 'resilience', 'glorifies']\n",
      "Сумма/произведение\n",
      "ship + man ['ship']\n",
      "ship * man ['oldests']\n",
      "ship + order ['order']\n",
      "ship * order ['ethic']\n",
      "man + ship ['ship']\n",
      "man * ship ['oldests']\n",
      "man + order ['order']\n",
      "man * order ['releaved']\n",
      "order + ship ['order']\n",
      "order * ship ['ethic']\n",
      "order + man ['order']\n",
      "order * man ['releaved']\n",
      "['child', 'kid', 'man', 'sibling', 'boy', 'toddler', 'preschooler', 'girl', 'woman', 'parent']\n",
      "['son', 'daughter', 'grandson', 'granddaughter', 'toddler', 'baby', 'crawl', 'child', 'anywayand', 'jumperoo']\n",
      "['fantastic', 'amazing', 'great', 'fabulous', 'wonderful', 'awsome', 'cool', 'incredible', 'good', 'terrific']\n"
     ]
    }
   ],
   "source": [
    "sample_words = [\"ship\", \"man\", \"order\"]\n",
    "first = lambda l : [x[0] for x in l]\n",
    "print(\"Самые похожие\")\n",
    "for word in sample_words:\n",
    "    print(word, first(gmodel.wv.similar_by_word(word)))\n",
    "print(\"Самые непохожие\")\n",
    "for word in sample_words:\n",
    "    print(word, first(gmodel.wv.similar_by_vector(-gmodel.wv[word])))\n",
    "print(\"Сумма/произведение\")\n",
    "from itertools import product\n",
    "for w1, w2 in product(sample_words, sample_words):\n",
    "    if w1 == w2:\n",
    "        continue\n",
    "    print(f\"{w1} + {w2}\",\n",
    "          first(gmodel.wv.similar_by_vector(gmodel.wv[w1] + gmodel.wv[w2], topn=1)))\n",
    "    print(f\"{w1} * {w2}\",\n",
    "          first(gmodel.wv.similar_by_vector(gmodel.wv[w1] * gmodel.wv[w2], topn=1)))\n",
    "print(first(gmodel.wv.similar_by_vector(\n",
    "    gmodel.wv[\"child\"] + gmodel.wv[\"man\"])))\n",
    "print(first(gmodel.wv.similar_by_vector(\n",
    "    gmodel.wv[\"son\"] - gmodel.wv[\"man\"])))\n",
    "print(first(gmodel.wv.similar_by_word(\"awesome\")))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "3d058c45",
   "metadata": {},
   "outputs": [],
   "source": [
    "top_words = words_by_freq[:3000]\n",
    "from tensorboardX import SummaryWriter\n",
    "with SummaryWriter() as writer:\n",
    "    writer.add_embedding(gmodel.wv[top_words], metadata=top_words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "da73b91c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(3000, 100)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gmodel.wv[top_words].shape"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8c20021a",
   "metadata": {},
   "source": [
    "## Bert\n",
    "https://mccormickml.com/2019/05/14/BERT-word-embeddings-tutorial/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "36fcded2",
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "from transformers import BertTokenizer, BertModel"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "3d488fea",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at bert-base-uncased were not used when initializing BertModel: ['cls.predictions.transform.dense.weight', 'cls.predictions.decoder.weight', 'cls.seq_relationship.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.transform.dense.bias', 'cls.predictions.transform.LayerNorm.bias', 'cls.seq_relationship.weight', 'cls.predictions.bias']\n",
      "- This IS expected if you are initializing BertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing BertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n"
     ]
    }
   ],
   "source": [
    "bert = BertModel.from_pretrained('bert-base-uncased', output_hidden_states=True)\n",
    "bert.eval()\n",
    "bertTokenizer = BertTokenizer.from_pretrained('bert-base-uncased')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "f7a4f690",
   "metadata": {},
   "outputs": [],
   "source": [
    "def mostSimilar(vec):\n",
    "    similarities = [(i, torch.cosine_similarity(vec, i[1], 0).item())\n",
    "                    for i in precomp_top]\n",
    "    res = sorted(similarities, key=lambda x: x[1])[::-1]\n",
    "    return [i[0][0] for i in res[:10]]\n",
    "\n",
    "def toVec(word):\n",
    "    toks = bertTokenizer.tokenize(\"[CLS] \" + word + \" [SEP]\")\n",
    "    itoks = bertTokenizer.convert_tokens_to_ids(toks)\n",
    "    toks_tensor = torch.tensor([itoks])\n",
    "    segments_ids = [1] * len(itoks)\n",
    "    segments_tensor = torch.tensor([segments_ids])\n",
    "    with torch.no_grad():\n",
    "        outputs = bert(toks_tensor, segments_tensor)\n",
    "        embedding = outputs[2]\n",
    "        tvecs = embedding[-2][0]\n",
    "        return torch.mean(tvecs, dim=0)\n",
    "\n",
    "def mostSimilarByWord(word):\n",
    "    return mostSimilar(toVec(word))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "4a636375",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 5000/5000 [01:41<00:00, 49.42it/s]\n"
     ]
    }
   ],
   "source": [
    "TOP_SIZE = 5000\n",
    "top = words_by_freq[:TOP_SIZE]\n",
    "precomp_top = list(tqdm(zip(top, map(toVec, top)), total=TOP_SIZE))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "6f3590c3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Самые похожие\n",
      "ship ['ship', 'planet', 'vehicle', 'gift', 'glass', 'movement', 'crate', 'pirate', 'elephant', 'spirit']\n",
      "man ['man', 'woman', 'guy', 'monster', 'alpha', 'scratch', 'suit', 'collar', 'earl', 'thicker']\n",
      "order ['order', 'ordering', 'ordered', 'pattern', 'count', 'charge', 'suggestion', 'demand', 'movement', 'shipment']\n",
      "Самые непохожие\n",
      "ship ['articulation', 'underarms', 'assist', 'moisturizer', 'hadnt', 'oatmeal', 'draw', 'shouldnt', 'darn', 'ovulation']\n",
      "man ['articulation', 'oatmeal', 'chamomile', 'moisturizer', 'melatonin', 'cholesterol', 'assist', 'peppermint', 'underarms', 'ovulation']\n",
      "order ['articulation', 'chamomile', 'oatmeal', 'moisturizer', 'peppermint', 'underarms', 'diabetic', 'antioxidant', 'assist', 'dishwasher']\n",
      "Сумма/произведение\n",
      "ship + man man\n",
      "ship * man articulation\n",
      "ship + order ship\n",
      "ship * order articulation\n",
      "man + ship man\n",
      "man * ship articulation\n",
      "man + order man\n",
      "man * order articulation\n",
      "order + ship ship\n",
      "order * ship articulation\n",
      "order + man man\n",
      "order * man articulation\n",
      "['man', 'child', 'woman', 'kid', 'son', 'giant', 'person', 'teenager', 'baby', 'hero']\n",
      "['grandchild', 'son', 'grandparent', 'grandkids', 'child', 'pediatrician', 'infant', 'melatonin', 'birth', 'fingernail']\n",
      "['awesome', 'fabulous', 'amazing', 'incredible', 'fantastic', 'great', 'brilliant', 'unbelievable', 'weird', 'delicious']\n"
     ]
    }
   ],
   "source": [
    "sample_words = [\"ship\", \"man\", \"order\"]\n",
    "def first(l): return [x[0] for x in l]\n",
    "def second(l): return [x[1] for x in l]\n",
    "from itertools import product\n",
    "\n",
    "print(\"Самые похожие\")\n",
    "for word in sample_words:\n",
    "    print(word, mostSimilarByWord(word))\n",
    "print(\"Самые непохожие\")\n",
    "for word in sample_words:\n",
    "    print(word, mostSimilar(toVec(word) * (-1)))\n",
    "print(\"Сумма/произведение\")\n",
    "for w1, w2 in product(sample_words, sample_words):\n",
    "    if w1 == w2:\n",
    "        continue\n",
    "    print(f\"{w1} + {w2}\", mostSimilar(toVec(w1) + toVec(w2))[0])\n",
    "    print(f\"{w1} * {w2}\", mostSimilar(toVec(w1) * toVec(w2))[0])\n",
    "print(mostSimilar(toVec(\"child\") + toVec(\"man\")))\n",
    "print(mostSimilar(toVec(\"son\") - toVec(\"man\")))\n",
    "print(mostSimilarByWord(\"awesome\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "70d595df",
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensorboardX import SummaryWriter\n",
    "import numpy as np\n",
    "\n",
    "with SummaryWriter() as writer:\n",
    "    writer.add_embedding(\n",
    "        np.stack(second(precomp_top)),\n",
    "        metadata=np.stack(first(precomp_top))\n",
    "    )"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}