키워드_다중_분류_모델링.ipynb

{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": 1,
      "id": "50btTucDm8vf",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "50btTucDm8vf",
        "outputId": "0cb168a4-0fe2-414b-da44-56156318475b"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Mounted at /content/drive\n"
          ]
        }
      ],
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "id": "1d5ced40",
      "metadata": {
        "id": "1d5ced40"
      },
      "outputs": [],
      "source": [
        "import pandas as pd\n",
        "path = '/content/drive/MyDrive/aida/dataset/'\n",
        "df = pd.read_csv(path+'기술과학분야_train_set.csv',index_col=0)\n",
        "df_val = pd.read_csv(path+'기술과학분야_valid_set.csv',index_col=0)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "id": "2d441625",
      "metadata": {
        "id": "2d441625"
      },
      "outputs": [],
      "source": [
        "df = df.drop(index=df[df['domain'] == '기후'].index)\n",
        "df = df.reset_index(drop=True)\n",
        "df_val = df_val.drop(index=df_val[df_val['domain'] == '기후'].index)\n",
        "df_val = df_val.reset_index(drop=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "id": "a5e06856",
      "metadata": {
        "id": "a5e06856"
      },
      "outputs": [],
      "source": [
        "train = pd.concat([df.loc[df['domain']=='기술과학'].sample(150000),df.loc[df['domain']=='세계'].sample(150000),df.loc[df['domain']=='정치'].sample(150000),df.loc[df['domain']=='경제'].sample(150000)]).sample(600000).reset_index(drop=True)\n",
        "valid = df_val\n",
        "\n",
        "train['labels'] = train['domain'].replace(['세계', '경제', '기술과학','정치'],[0, 1, 2, 3])\n",
        "valid['labels'] = df_val['domain'].replace(['세계', '경제', '기술과학','정치'],[0, 1, 2, 3])"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "rGRc9zLaGRqd",
      "metadata": {
        "id": "rGRc9zLaGRqd"
      },
      "source": [
        "# 모델링"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "id": "SlHT-hQhnhye",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "SlHT-hQhnhye",
        "outputId": "d45873e0-9a7a-4f02-c8d9-a0f41c4aa6ca"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting transformers\n",
            "  Downloading transformers-4.24.0-py3-none-any.whl (5.5 MB)\n",
            "\u001b[K     |████████████████████████████████| 5.5 MB 3.7 MB/s \n",
            "\u001b[?25hCollecting tokenizers!=0.11.3,<0.14,>=0.11.1\n",
            "  Downloading tokenizers-0.13.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (7.6 MB)\n",
            "\u001b[K     |████████████████████████████████| 7.6 MB 36.6 MB/s \n",
            "\u001b[?25hRequirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.7/dist-packages (from transformers) (4.64.1)\n",
            "Collecting huggingface-hub<1.0,>=0.10.0\n",
            "  Downloading huggingface_hub-0.10.1-py3-none-any.whl (163 kB)\n",
            "\u001b[K     |████████████████████████████████| 163 kB 64.9 MB/s \n",
            "\u001b[?25hRequirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (1.21.6)\n",
            "Requirement already satisfied: importlib-metadata in /usr/local/lib/python3.7/dist-packages (from transformers) (4.13.0)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.7/dist-packages (from transformers) (6.0)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (2022.6.2)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.7/dist-packages (from transformers) (21.3)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from transformers) (3.8.0)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from transformers) (2.23.0)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.7/dist-packages (from huggingface-hub<1.0,>=0.10.0->transformers) (4.1.1)\n",
            "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging>=20.0->transformers) (3.0.9)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata->transformers) (3.10.0)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (3.0.4)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2.10)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2022.9.24)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (1.24.3)\n",
            "Installing collected packages: tokenizers, huggingface-hub, transformers\n",
            "Successfully installed huggingface-hub-0.10.1 tokenizers-0.13.2 transformers-4.24.0\n"
          ]
        }
      ],
      "source": [
        "! pip install transformers"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "id": "24701fd4",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 368
        },
        "id": "24701fd4",
        "outputId": "952a6298-9dd6-4158-daba-7e97c2128356"
      },
      "outputs": [
        {
          "ename": "ModuleNotFoundError",
          "evalue": "ignored",
          "output_type": "error",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mModuleNotFoundError\u001b[0m                       Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-3-0b08cf1ac271>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     12\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mutils\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdata\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mDataset\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mDataLoader\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     13\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 14\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0mtransformers\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mAutoModel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mAutoTokenizer\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mBertForSequenceClassification\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     15\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     16\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mModuleNotFoundError\u001b[0m: No module named 'transformers'",
            "",
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0;32m\nNOTE: If your import is failing due to a missing package, you can\nmanually install dependencies using either !pip or !apt.\n\nTo view examples of installing some common dependencies, click the\n\"Open Examples\" button below.\n\u001b[0;31m---------------------------------------------------------------------------\u001b[0m\n"
          ]
        }
      ],
      "source": [
        "from tqdm import tqdm\n",
        "import json\n",
        "\n",
        "import pandas as pd\n",
        "import numpy as np\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "import torch.nn.functional as F\n",
        "import torch.optim as optim\n",
        "from torch.utils.data import Dataset, DataLoader\n",
        "\n",
        "from transformers import AutoModel, AutoTokenizer,BertForSequenceClassification\n",
        "\n",
        "\n",
        "model = BertForSequenceClassification.from_pretrained(\"klue/bert-base\",num_labels=4)\n",
        "tokenizer = AutoTokenizer.from_pretrained(\"klue/bert-base\")\n",
        "# \n",
        "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
        "model.to(device)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 7,
      "id": "97746eff",
      "metadata": {
        "id": "97746eff"
      },
      "outputs": [],
      "source": [
        "class CustomDataset(Dataset):\n",
        "\n",
        "    def __init__(self,text, label,tokenizer, max_len):\n",
        "        self.tokenizer = tokenizer\n",
        "        self.text = text\n",
        "        self.labels = label\n",
        "        self.max_len = max_len\n",
        "\n",
        "    def __len__(self):\n",
        "        return len(self.labels)\n",
        "\n",
        "    def __getitem__(self, index):\n",
        "        text = str(self.text[index])\n",
        "        inputs = self.tokenizer.encode_plus(\n",
        "            text,\n",
        "            None,\n",
        "            add_special_tokens=True,\n",
        "            max_length=self.max_len,\n",
        "            pad_to_max_length=True,\n",
        "            return_token_type_ids=True\n",
        "        )\n",
        "        ids = inputs['input_ids']\n",
        "        mask = inputs['attention_mask']\n",
        "        token_type_ids = inputs[\"token_type_ids\"]\n",
        "\n",
        "\n",
        "        return {\n",
        "            'input_ids': torch.tensor(ids, dtype=torch.long),\n",
        "            'attention_mask': torch.tensor(mask, dtype=torch.long),\n",
        "            'token_type_ids': torch.tensor(token_type_ids, dtype=torch.long),\n",
        "            'labels': torch.tensor(self.labels[index], dtype=torch.float),\n",
        "            }\n",
        "    \n",
        "\n",
        "\n",
        "training_set = CustomDataset(train['ko'],train['labels'], tokenizer, 128)\n",
        "testing_set = CustomDataset(valid['ko'], valid['labels'],tokenizer, 128)\n",
        "\n",
        "train_params = {'batch_size': 32,\n",
        "                'shuffle': True,\n",
        "                'num_workers': 0\n",
        "                }\n",
        "\n",
        "test_params = {'batch_size': 32,\n",
        "                'shuffle': True,\n",
        "                'num_workers': 0\n",
        "                }\n",
        "\n",
        "training_loader = DataLoader(training_set, **train_params)\n",
        "testing_loader = DataLoader(testing_set, **test_params)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 8,
      "id": "524cb2dc",
      "metadata": {
        "id": "524cb2dc"
      },
      "outputs": [],
      "source": [
        "loss_fn = nn.CrossEntropyLoss() \n",
        "\n",
        "LEARNING_RATE = 5e-5\n",
        "optimizer = torch.optim.AdamW(params =  model.parameters(), lr=LEARNING_RATE)\n",
        "\n",
        "\n",
        "train_steps = len(training_loader.dataset) // train_params['batch_size']\n",
        "val_steps = len(testing_loader.dataset) // test_params['batch_size']\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "f03b57db",
      "metadata": {
        "id": "f03b57db",
        "outputId": "1d4b5f6b-883b-4b46-ede5-4439d2e22468"
      },
      "outputs": [
        {
          "ename": "",
          "evalue": "",
          "output_type": "error",
          "traceback": [
            "\u001b[1;31m'Python 3.10.6 64-bit'(으)로 셀을 실행하려면 ipykernel 패키지가 필요합니다.\n",
            "\u001b[1;31m다음 명령어를 실행하여 Python 환경에 'ipykernel'을(를) 설치합니다. \n",
            "\u001b[1;31m 명령: 'c:/Python310/python.exe -m pip install ipykernel -U --user --force-reinstall'"
          ]
        }
      ],
      "source": [
        "from tqdm import tqdm\n",
        "epochs= 5\n",
        "PATH = \"C:/Users/withus/Desktop/sabin/Trend-analysis/Keyphrase_classification/model_output/\"\n",
        "\n",
        "for epoch in (range(epochs)):\n",
        "    model.train()\n",
        "    num_correct = 0\n",
        "    trainning_loss = 0\n",
        "    for i, data in enumerate(tqdm(training_loader)):\n",
        "        \n",
        "        ids = data['input_ids'].to(device, dtype = torch.long)\n",
        "        mask = data['attention_mask'].to(device, dtype = torch.long)\n",
        "        token_type_ids = data['token_type_ids'].to(device, dtype = torch.long)\n",
        "        label = data['labels'].to(device, dtype = torch.long)\n",
        "\n",
        "        optimizer.zero_grad()\n",
        "        \n",
        "        output = model(ids, mask, token_type_ids)\n",
        "        print(output)\n",
        "        loss = loss_fn(output[0], label)\n",
        "        trainning_loss += loss\n",
        "        loss.backward()\n",
        "        optimizer.step()\n",
        "        if (i+1) % 1000 == 0:\n",
        "          print(\"Training Losses: {}\".format(trainning_loss/i+1))\n",
        "          print(\"Trainong Accuracy:{}\".format(num_correct/i+1))\n",
        "    \n",
        "        pred = output[0].argmax(dim=1)\n",
        "        num_correct += torch.eq(pred,label).sum().float().item()\n",
        "        \n",
        "\n",
        "\n",
        "    print('EPOCH ', epoch+1)\n",
        "    print(\"Training Losses: {}\".format(trainning_loss/len(training_loader.dataset)))\n",
        "    print(\"Trainong Accuracy:{}\".format(num_correct/len(training_loader.dataset)))\n",
        "    \n",
        "    \n",
        "    \n",
        "    if epoch % 5 == 0 or epoch == epochs-1:\n",
        "        torch.save({\n",
        "                    'epoch': epoch,\n",
        "                    'model_state_dict': model.state_dict(),\n",
        "                    'optimizer_state_dict': optimizer.state_dict(),\n",
        "                    'loss': trainning_loss,\n",
        "                    }, PATH+f\"model600000data{epoch}.pt\")\n",
        "  "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 9,
      "id": "84b10226",
      "metadata": {
        "id": "84b10226"
      },
      "outputs": [],
      "source": [
        "checkpoint = torch.load('/content/drive/MyDrive/aida/model600000data4.pt')\n",
        "model.load_state_dict(checkpoint['model_state_dict'])\n",
        "optimizer.load_state_dict(checkpoint['optimizer_state_dict'])\n",
        "epoch = checkpoint['epoch']\n",
        "loss = checkpoint['loss']"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 10,
      "id": "0c0f4100",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "0c0f4100",
        "outputId": "6dc8d95c-8434-4dde-d8da-c997a1e5f118"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\r  0%|          | 0/4377 [00:00<?, ?it/s]Truncation was not explicitly activated but `max_length` is provided a specific value, please use `truncation=True` to explicitly truncate examples to max length. Defaulting to 'longest_first' truncation strategy. If you encode pairs of sequences (GLUE-style) with the tokenizer you can select this strategy more precisely by providing a specific strategy to `truncation`.\n",
            "/usr/local/lib/python3.7/dist-packages/transformers/tokenization_utils_base.py:2310: FutureWarning: The `pad_to_max_length` argument is deprecated and will be removed in a future version, use `padding=True` or `padding='longest'` to pad to the longest sequence in the batch, or use `padding='max_length'` to pad to a max length. In this case, you can give a specific length with `max_length` (e.g. `max_length=45`) or leave max_length to None to pad to the maximal input size of the model (e.g. 512 for Bert).\n",
            "  FutureWarning,\n",
            "100%|██████████| 4377/4377 [18:33<00:00,  3.93it/s]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "EPOCH  5\n",
            "validation Losses: 0.012366868555545807\n",
            "validation Accuracy:0.9405813347422834\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\n"
          ]
        }
      ],
      "source": [
        "model.eval()\n",
        "num_correct = 0\n",
        "\n",
        "for i, data in enumerate(tqdm(testing_loader)):\n",
        "\n",
        "      ids = data['input_ids'].to(device, dtype = torch.long)\n",
        "      mask = data['attention_mask'].to(device, dtype = torch.long)\n",
        "      token_type_ids = data['token_type_ids'].to(device, dtype = torch.long)\n",
        "      label = data['labels'].to(device, dtype = torch.long)\n",
        "\n",
        "\n",
        "      optimizer.zero_grad()\n",
        "\n",
        "      output = model(ids, mask, token_type_ids)\n",
        "      loss = loss_fn(output[0], label)\n",
        "      optimizer.step()\n",
        "\n",
        "      pred = output[0].argmax(dim=1)\n",
        "      num_correct += torch.eq(pred, label).sum().float().item()\n",
        "\n",
        "\n",
        "print('EPOCH ', epoch+1)\n",
        "print(\"validation Losses: {}\".format(loss))\n",
        "print(\"validation Accuracy:{}\".format(num_correct/len(testing_loader.dataset)))\n"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "6_1Y9XXoGNTE",
      "metadata": {
        "id": "6_1Y9XXoGNTE"
      },
      "source": [
        "# keyphrase classification(키워드 뽑은거 돌려 보는 부분 부분)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 16,
      "id": "ggCi_VQ38t12",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ggCi_VQ38t12",
        "outputId": "be466c84-b88f-4982-8435-202706464f75"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "  0%|          | 0/17492 [00:00<?, ?it/s]/usr/local/lib/python3.7/dist-packages/transformers/tokenization_utils_base.py:2310: FutureWarning: The `pad_to_max_length` argument is deprecated and will be removed in a future version, use `padding=True` or `padding='longest'` to pad to the longest sequence in the batch, or use `padding='max_length'` to pad to a max length. In this case, you can give a specific length with `max_length` (e.g. `max_length=45`) or leave max_length to None to pad to the maximal input size of the model (e.g. 512 for Bert).\n",
            "  FutureWarning,\n",
            "100%|██████████| 17492/17492 [11:53<00:00, 24.53it/s]\n",
            "100%|██████████| 21068/21068 [14:17<00:00, 24.58it/s]\n",
            "100%|██████████| 7204/7204 [04:53<00:00, 24.57it/s]\n",
            "100%|██████████| 29995/29995 [20:20<00:00, 24.58it/s]\n"
          ]
        }
      ],
      "source": [
        "for i in range(1,5):\n",
        "  file_path = \"/content/drive/MyDrive/aida\"\n",
        "  if(i == 1):\n",
        "    file_path += \"/rent.json\"\n",
        "  if(i == 2):\n",
        "    file_path += \"/accident.json\"\n",
        "  if(i == 3):\n",
        "    file_path += \"/discrimination.json\"\n",
        "  if(i == 4):\n",
        "    file_path += \"/neutral.json\"\n",
        "\n",
        "  save_path = \"/content/drive/MyDrive/aida\"\n",
        "  if(i == 1):\n",
        "    save_path += \"/rent_multi_content.json\"\n",
        "  if(i == 2):\n",
        "    save_path += \"/accident_multi_content.json\"\n",
        "  if(i == 3):\n",
        "    save_path += \"/discrimination_multi_content.json\"\n",
        "  if(i == 4):\n",
        "    save_path += \"/neutral_multi_content.json\"\n",
        "\n",
        "\n",
        "  with open(file_path, 'r') as file:\n",
        "      keywords = json.load(file)\n",
        "\n",
        "  category = ['사회', '경제', '기술과학','정치']\n",
        "  keyword_params = {'batch_size': 5,\n",
        "                  'shuffle': True,\n",
        "                  'num_workers': 0\n",
        "                  }\n",
        "  key_and_category = []\n",
        "  for keyword in tqdm(keywords):\n",
        "    keyword_5 = list(keyword.values())[0]\n",
        "    keyword_set = CustomDataset(keyword_5,[-1]*len(keyword_5), tokenizer, 128)\n",
        "    keyword_loader = DataLoader(keyword_set, **keyword_params)\n",
        "    for i, data in enumerate((keyword_loader)):\n",
        "\n",
        "      ids = data['input_ids'].to(device, dtype = torch.long)\n",
        "      mask = data['attention_mask'].to(device, dtype = torch.long)\n",
        "      token_type_ids = data['token_type_ids'].to(device, dtype = torch.long)\n",
        "      label = data['labels'].to(device, dtype = torch.long)\n",
        "\n",
        "      output = model(ids, mask, token_type_ids)\n",
        "      \n",
        "      pred = pd.Series(output[0].argmax(dim=1).cpu().numpy()).replace([0,1,2,3],category)\n",
        "      [key_and_category.append([keyword_5[i],pred[i]]) for i in range(5)]\n",
        "\n",
        "  with open(save_path, 'w', encoding='utf-8') as f:\n",
        "          json.dump(key_and_category, f, ensure_ascii=False, indent=4)\n",
        "      "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "N4wVfQI-9b6X",
      "metadata": {
        "id": "N4wVfQI-9b6X"
      },
      "outputs": [],
      "source": []
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "collapsed_sections": [],
      "machine_shape": "hm",
      "provenance": []
    },
    "gpuClass": "standard",
    "kernelspec": {
      "display_name": "Python 3.9.13 ('ML')",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.9.13"
    },
    "vscode": {
      "interpreter": {
        "hash": "4aba9991dbdcd39a1f8cd48e6a221aa6b158ae7b01b040b5ca58c457126bb894"
      }
    }
  },
  "nbformat": 4,
  "nbformat_minor": 5
}