kcelectra_multi_label.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "c08686b8",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "from tqdm import tqdm\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim\n",
    "from torch.utils.data import Dataset, DataLoader\n",
    "\n",
    "from transformers import AutoTokenizer, AutoModelForSequenceClassification, TrainingArguments, Trainer , AutoModel\n",
    "from sklearn.metrics import precision_recall_fscore_support, accuracy_score\n",
    "\n",
    "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
    "\n",
    "train_data = pd.read_csv(\"./Dataset/unsmile_train_v1.0.tsv\", sep=\"\\t\")\n",
    "valid_data = pd.read_csv(\"./Dataset/unsmile_valid_v1.0.tsv\", sep=\"\\t\")\n",
    "\n",
    "cols = train_data.columns.tolist()\n",
    "cols.remove(\"문장\")\n",
    "cols.remove(\"clean\")\n",
    "\n",
    "MODEL_NAME = \"beomi/KcELECTRA-base\"\n",
    "tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME)\n",
    "\n",
    "MAX_LEN = 128\n",
    "tokenized_train = tokenizer(\n",
    "                    list(train_data[\"문장\"]), \n",
    "                    return_tensors=\"pt\", \n",
    "                    padding=True, \n",
    "                    truncation=True, \n",
    "                    max_length=MAX_LEN, \n",
    "                    add_special_tokens=True\n",
    "                )\n",
    "\n",
    "tokenized_valid = tokenizer(\n",
    "                    list(valid_data[\"문장\"]), \n",
    "                    return_tensors=\"pt\", \n",
    "                    padding=True, \n",
    "                    truncation=True, \n",
    "                    max_length=MAX_LEN, \n",
    "                    add_special_tokens=True\n",
    "                )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "52b33f2e",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "class MultiLabelDataset(Dataset):\n",
    "\n",
    "    def __init__(self, dataframe, tokenizer, max_len):\n",
    "        self.tokenizer = tokenizer\n",
    "        self.data = dataframe\n",
    "        self.text = dataframe[\"문장\"]\n",
    "        self.targets = self.data[cols].values\n",
    "        self.max_len = max_len\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.text)\n",
    "\n",
    "    def __getitem__(self, index):\n",
    "        text = str(self.text[index])\n",
    "        text = \" \".join(text.split())\n",
    "\n",
    "        inputs = self.tokenizer.encode_plus(\n",
    "            text,\n",
    "            None,\n",
    "            add_special_tokens=True,\n",
    "            max_length=self.max_len,\n",
    "            pad_to_max_length=True,\n",
    "            return_token_type_ids=True\n",
    "        )\n",
    "        ids = inputs['input_ids']\n",
    "        mask = inputs['attention_mask']\n",
    "        token_type_ids = inputs[\"token_type_ids\"]\n",
    "\n",
    "\n",
    "        return {\n",
    "            'input_ids': torch.tensor(ids, dtype=torch.long),\n",
    "            'attention_mask': torch.tensor(mask, dtype=torch.long),\n",
    "            'token_type_ids': torch.tensor(token_type_ids, dtype=torch.long),\n",
    "            'labels': torch.tensor(self.targets[index], dtype=torch.float)\n",
    "        }\n",
    "    \n",
    "\n",
    "\n",
    "training_set = MultiLabelDataset(train_data, tokenizer, MAX_LEN)\n",
    "testing_set = MultiLabelDataset(valid_data, tokenizer, MAX_LEN)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "3df3515d",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_params = {'batch_size': 8,\n",
    "                'shuffle': True,\n",
    "                'num_workers': 0\n",
    "                }\n",
    "\n",
    "test_params = {'batch_size': 16,\n",
    "                'shuffle': True,\n",
    "                'num_workers': 0\n",
    "                }\n",
    "\n",
    "training_loader = DataLoader(training_set, **train_params)\n",
    "testing_loader = DataLoader(testing_set, **test_params)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "302ce39c",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at beomi/KcELECTRA-base were not used when initializing ElectraForSequenceClassification: ['discriminator_predictions.dense_prediction.bias', 'discriminator_predictions.dense.bias', 'discriminator_predictions.dense_prediction.weight', 'discriminator_predictions.dense.weight']\n",
      "- This IS expected if you are initializing ElectraForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing ElectraForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "Some weights of ElectraForSequenceClassification were not initialized from the model checkpoint at beomi/KcELECTRA-base and are newly initialized: ['classifier.out_proj.weight', 'classifier.out_proj.bias', 'classifier.dense.weight', 'classifier.dense.bias']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "BERTClass(\n",
       "  (l1): ElectraForSequenceClassification(\n",
       "    (electra): ElectraModel(\n",
       "      (embeddings): ElectraEmbeddings(\n",
       "        (word_embeddings): Embedding(50135, 768, padding_idx=0)\n",
       "        (position_embeddings): Embedding(512, 768)\n",
       "        (token_type_embeddings): Embedding(2, 768)\n",
       "        (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "        (dropout): Dropout(p=0.1, inplace=False)\n",
       "      )\n",
       "      (encoder): ElectraEncoder(\n",
       "        (layer): ModuleList(\n",
       "          (0): ElectraLayer(\n",
       "            (attention): ElectraAttention(\n",
       "              (self): ElectraSelfAttention(\n",
       "                (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (dropout): Dropout(p=0.1, inplace=False)\n",
       "              )\n",
       "              (output): ElectraSelfOutput(\n",
       "                (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "                (dropout): Dropout(p=0.1, inplace=False)\n",
       "              )\n",
       "            )\n",
       "            (intermediate): ElectraIntermediate(\n",
       "              (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "              (intermediate_act_fn): GELUActivation()\n",
       "            )\n",
       "            (output): ElectraOutput(\n",
       "              (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (1): ElectraLayer(\n",
       "            (attention): ElectraAttention(\n",
       "              (self): ElectraSelfAttention(\n",
       "                (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (dropout): Dropout(p=0.1, inplace=False)\n",
       "              )\n",
       "              (output): ElectraSelfOutput(\n",
       "                (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "                (dropout): Dropout(p=0.1, inplace=False)\n",
       "              )\n",
       "            )\n",
       "            (intermediate): ElectraIntermediate(\n",
       "              (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "              (intermediate_act_fn): GELUActivation()\n",
       "            )\n",
       "            (output): ElectraOutput(\n",
       "              (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (2): ElectraLayer(\n",
       "            (attention): ElectraAttention(\n",
       "              (self): ElectraSelfAttention(\n",
       "                (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (dropout): Dropout(p=0.1, inplace=False)\n",
       "              )\n",
       "              (output): ElectraSelfOutput(\n",
       "                (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "                (dropout): Dropout(p=0.1, inplace=False)\n",
       "              )\n",
       "            )\n",
       "            (intermediate): ElectraIntermediate(\n",
       "              (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "              (intermediate_act_fn): GELUActivation()\n",
       "            )\n",
       "            (output): ElectraOutput(\n",
       "              (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (3): ElectraLayer(\n",
       "            (attention): ElectraAttention(\n",
       "              (self): ElectraSelfAttention(\n",
       "                (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (dropout): Dropout(p=0.1, inplace=False)\n",
       "              )\n",
       "              (output): ElectraSelfOutput(\n",
       "                (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "                (dropout): Dropout(p=0.1, inplace=False)\n",
       "              )\n",
       "            )\n",
       "            (intermediate): ElectraIntermediate(\n",
       "              (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "              (intermediate_act_fn): GELUActivation()\n",
       "            )\n",
       "            (output): ElectraOutput(\n",
       "              (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (4): ElectraLayer(\n",
       "            (attention): ElectraAttention(\n",
       "              (self): ElectraSelfAttention(\n",
       "                (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (dropout): Dropout(p=0.1, inplace=False)\n",
       "              )\n",
       "              (output): ElectraSelfOutput(\n",
       "                (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "                (dropout): Dropout(p=0.1, inplace=False)\n",
       "              )\n",
       "            )\n",
       "            (intermediate): ElectraIntermediate(\n",
       "              (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "              (intermediate_act_fn): GELUActivation()\n",
       "            )\n",
       "            (output): ElectraOutput(\n",
       "              (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (5): ElectraLayer(\n",
       "            (attention): ElectraAttention(\n",
       "              (self): ElectraSelfAttention(\n",
       "                (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (dropout): Dropout(p=0.1, inplace=False)\n",
       "              )\n",
       "              (output): ElectraSelfOutput(\n",
       "                (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "                (dropout): Dropout(p=0.1, inplace=False)\n",
       "              )\n",
       "            )\n",
       "            (intermediate): ElectraIntermediate(\n",
       "              (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "              (intermediate_act_fn): GELUActivation()\n",
       "            )\n",
       "            (output): ElectraOutput(\n",
       "              (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (6): ElectraLayer(\n",
       "            (attention): ElectraAttention(\n",
       "              (self): ElectraSelfAttention(\n",
       "                (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (dropout): Dropout(p=0.1, inplace=False)\n",
       "              )\n",
       "              (output): ElectraSelfOutput(\n",
       "                (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "                (dropout): Dropout(p=0.1, inplace=False)\n",
       "              )\n",
       "            )\n",
       "            (intermediate): ElectraIntermediate(\n",
       "              (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "              (intermediate_act_fn): GELUActivation()\n",
       "            )\n",
       "            (output): ElectraOutput(\n",
       "              (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (7): ElectraLayer(\n",
       "            (attention): ElectraAttention(\n",
       "              (self): ElectraSelfAttention(\n",
       "                (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (dropout): Dropout(p=0.1, inplace=False)\n",
       "              )\n",
       "              (output): ElectraSelfOutput(\n",
       "                (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "                (dropout): Dropout(p=0.1, inplace=False)\n",
       "              )\n",
       "            )\n",
       "            (intermediate): ElectraIntermediate(\n",
       "              (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "              (intermediate_act_fn): GELUActivation()\n",
       "            )\n",
       "            (output): ElectraOutput(\n",
       "              (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (8): ElectraLayer(\n",
       "            (attention): ElectraAttention(\n",
       "              (self): ElectraSelfAttention(\n",
       "                (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (dropout): Dropout(p=0.1, inplace=False)\n",
       "              )\n",
       "              (output): ElectraSelfOutput(\n",
       "                (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "                (dropout): Dropout(p=0.1, inplace=False)\n",
       "              )\n",
       "            )\n",
       "            (intermediate): ElectraIntermediate(\n",
       "              (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "              (intermediate_act_fn): GELUActivation()\n",
       "            )\n",
       "            (output): ElectraOutput(\n",
       "              (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (9): ElectraLayer(\n",
       "            (attention): ElectraAttention(\n",
       "              (self): ElectraSelfAttention(\n",
       "                (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (dropout): Dropout(p=0.1, inplace=False)\n",
       "              )\n",
       "              (output): ElectraSelfOutput(\n",
       "                (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "                (dropout): Dropout(p=0.1, inplace=False)\n",
       "              )\n",
       "            )\n",
       "            (intermediate): ElectraIntermediate(\n",
       "              (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "              (intermediate_act_fn): GELUActivation()\n",
       "            )\n",
       "            (output): ElectraOutput(\n",
       "              (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (10): ElectraLayer(\n",
       "            (attention): ElectraAttention(\n",
       "              (self): ElectraSelfAttention(\n",
       "                (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (dropout): Dropout(p=0.1, inplace=False)\n",
       "              )\n",
       "              (output): ElectraSelfOutput(\n",
       "                (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "                (dropout): Dropout(p=0.1, inplace=False)\n",
       "              )\n",
       "            )\n",
       "            (intermediate): ElectraIntermediate(\n",
       "              (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "              (intermediate_act_fn): GELUActivation()\n",
       "            )\n",
       "            (output): ElectraOutput(\n",
       "              (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (11): ElectraLayer(\n",
       "            (attention): ElectraAttention(\n",
       "              (self): ElectraSelfAttention(\n",
       "                (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (dropout): Dropout(p=0.1, inplace=False)\n",
       "              )\n",
       "              (output): ElectraSelfOutput(\n",
       "                (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "                (dropout): Dropout(p=0.1, inplace=False)\n",
       "              )\n",
       "            )\n",
       "            (intermediate): ElectraIntermediate(\n",
       "              (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "              (intermediate_act_fn): GELUActivation()\n",
       "            )\n",
       "            (output): ElectraOutput(\n",
       "              (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "    )\n",
       "    (classifier): ElectraClassificationHead(\n",
       "      (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "      (dropout): Dropout(p=0.1, inplace=False)\n",
       "      (out_proj): Linear(in_features=768, out_features=10, bias=True)\n",
       "    )\n",
       "  )\n",
       ")"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "class BERTClass(torch.nn.Module):\n",
    "    def __init__(self):\n",
    "        super(BERTClass, self).__init__()\n",
    "        self.l1 = AutoModelForSequenceClassification.from_pretrained(\"beomi/KcELECTRA-base\",num_labels=10)\n",
    "        \n",
    "\n",
    "    def forward(self, input_ids, attention_mask,token_type_ids):\n",
    "        output = self.l1(input_ids=input_ids, attention_mask=attention_mask,token_type_ids=token_type_ids)\n",
    "        return output\n",
    "\n",
    "model = BERTClass()\n",
    "model.to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "73551b32",
   "metadata": {},
   "outputs": [],
   "source": [
    "training_args = TrainingArguments(\n",
    "    output_dir=\"./output_1\",\n",
    "    num_train_epochs=10,\n",
    "    per_device_eval_batch_size=16,\n",
    "    per_device_train_batch_size=8,\n",
    "    logging_dir=\"./log\",\n",
    "    save_steps=1000,\n",
    "    save_total_limit=2,\n",
    ")\n",
    "\n",
    "def compute_metrics(preds):\n",
    "    labels = preds.label_ids\n",
    "    preds = preds.predictions.argmax(-1)\n",
    "    precision, recall, f1, _ = precision_recall_fscore_support(labels, preds, average=\"weighted\")\n",
    "    acc = accuracy_score(labels, preds)\n",
    "    return {\n",
    "        \"acc\": acc, \n",
    "        \"precision\": precision, \n",
    "        \"recall\": recall,\n",
    "        \"f1\": f1,\n",
    "    }\n",
    "\n",
    "trainer = Trainer(\n",
    "    model=model,\n",
    "    args=training_args,\n",
    "    train_dataset=training_set,\n",
    "    eval_dataset=testing_set,\n",
    "    \n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "54381e45",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "def loss_fn(outputs, targets):\n",
    "    return torch.nn.BCEWithLogitsLoss()(outputs, targets)\n",
    "\n",
    "LEARNING_RATE = 5e-5\n",
    "optimizer = torch.optim.Adam(params =  model.parameters(), lr=LEARNING_RATE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "54a086f4",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0it [00:00, ?it/s]Truncation was not explicitly activated but `max_length` is provided a specific value, please use `truncation=True` to explicitly truncate examples to max length. Defaulting to 'longest_first' truncation strategy. If you encode pairs of sequences (GLUE-style) with the tokenizer you can select this strategy more precisely by providing a specific strategy to `truncation`.\n",
      "/opt/conda/lib/python3.8/site-packages/transformers/tokenization_utils_base.py:2271: FutureWarning: The `pad_to_max_length` argument is deprecated and will be removed in a future version, use `padding=True` or `padding='longest'` to pad to the longest sequence in the batch, or use `padding='max_length'` to pad to a max length. In this case, you can give a specific length with `max_length` (e.g. `max_length=45`) or leave max_length to None to pad to the maximal input size of the model (e.g. 512 for Bert).\n",
      "  warnings.warn(\n",
      "1it [00:00,  1.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 0, Loss:  0.6956549882888794\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "1876it [02:14, 13.92it/s]\n",
      "2it [00:00, 14.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 1, Loss:  0.12894748151302338\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "1876it [02:14, 13.96it/s]\n",
      "2it [00:00, 14.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 2, Loss:  0.025834811851382256\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "1876it [02:14, 13.95it/s]\n",
      "2it [00:00, 14.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 3, Loss:  0.035010017454624176\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "1876it [02:14, 13.94it/s]\n",
      "2it [00:00, 14.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 4, Loss:  0.023657796904444695\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "1876it [02:14, 13.94it/s]\n"
     ]
    }
   ],
   "source": [
    "def train(epoch):\n",
    "    model.train()\n",
    "    for _,data in tqdm(enumerate(training_loader, 0)):\n",
    "        ids = data['input_ids'].to(device, dtype = torch.long)\n",
    "        mask = data['attention_mask'].to(device, dtype = torch.long)\n",
    "        token_type_ids = data['token_type_ids'].to(device, dtype = torch.long)\n",
    "        targets = data['labels'].to(device, dtype = torch.float)\n",
    "\n",
    "        outputs = model(ids, mask, token_type_ids)\n",
    "\n",
    "        optimizer.zero_grad()\n",
    "        loss = loss_fn(outputs['logits'], targets)\n",
    "        if _%5000==0:\n",
    "            print(f'Epoch: {epoch}, Loss:  {loss.item()}')\n",
    "        \n",
    "        optimizer.zero_grad()\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        \n",
    "for epoch in range(5):\n",
    "    train(epoch)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "ec126240",
   "metadata": {},
   "outputs": [],
   "source": [
    "# trainer.train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "0d790b72",
   "metadata": {},
   "outputs": [],
   "source": [
    "def validation(testing_loader):\n",
    "    model.eval()\n",
    "    fin_targets=[]\n",
    "    fin_outputs=[]\n",
    "    with torch.no_grad():\n",
    "        for _, data in tqdm(enumerate(testing_loader, 0)):\n",
    "            ids = data['input_ids'].to(device, dtype = torch.long)\n",
    "            mask = data['attention_mask'].to(device, dtype = torch.long)\n",
    "            token_type_ids = data['token_type_ids'].to(device, dtype = torch.long)\n",
    "            targets = data['labels'].to(device, dtype = torch.float)\n",
    "            outputs = model(ids, mask, token_type_ids)\n",
    "            fin_targets.extend(targets.cpu().detach().numpy().tolist())\n",
    "            fin_outputs.extend(torch.sigmoid(outputs[\"logits\"]).cpu().detach().numpy().tolist())\n",
    "    return fin_outputs, fin_targets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "30ba6668",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "234it [00:08, 26.76it/s]\n"
     ]
    }
   ],
   "source": [
    "outputs, targets = validation(testing_loader)\n",
    "final_outputs = np.array(outputs) >=0.5\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "915fb2f3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Hamming Score = 0.7619302470787619\n",
      "Hamming Loss = 0.0356970832218357\n"
     ]
    }
   ],
   "source": [
    "from sklearn import metrics\n",
    "\n",
    "def hamming_score(y_true, y_pred, normalize=True, sample_weight=None):\n",
    "    acc_list = []\n",
    "    for i in range(y_true.shape[0]):\n",
    "        set_true = set( np.where(y_true[i])[0] )\n",
    "        set_pred = set( np.where(y_pred[i])[0] )\n",
    "        tmp_a = None\n",
    "        if len(set_true) == 0 and len(set_pred) == 0:\n",
    "            tmp_a = 1\n",
    "        else:\n",
    "            tmp_a = len(set_true.intersection(set_pred))/\\\n",
    "                    float( len(set_true.union(set_pred)) )\n",
    "        acc_list.append(tmp_a)\n",
    "    return np.mean(acc_list)\n",
    "\n",
    "\n",
    "val_hamming_loss = metrics.hamming_loss(targets, final_outputs)\n",
    "val_hamming_score = hamming_score(np.array(targets), np.array(final_outputs))\n",
    "\n",
    "print(f\"Hamming Score = {val_hamming_score}\")\n",
    "print(f\"Hamming Loss = {val_hamming_loss}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "d1415265",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>여성/가족</th>\n",
       "      <th>남성</th>\n",
       "      <th>성소수자</th>\n",
       "      <th>인종/국적</th>\n",
       "      <th>연령</th>\n",
       "      <th>지역</th>\n",
       "      <th>종교</th>\n",
       "      <th>기타 혐오</th>\n",
       "      <th>악플/욕설</th>\n",
       "      <th>개인지칭</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   여성/가족     남성   성소수자  인종/국적     연령     지역     종교  기타 혐오  악플/욕설   개인지칭\n",
       "0  False  False  False  False  False  False  False  False  False  False"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def sentence_classification(sentence):\n",
    "    \n",
    "    tokenized_sent = tokenizer(\n",
    "        sentence,\n",
    "        return_tensors=\"pt\",\n",
    "        padding=True,\n",
    "        truncation=True,\n",
    "        max_length=MAX_LEN,\n",
    "        add_special_tokens=True,\n",
    "    )\n",
    "    \n",
    "    tokenized_sent.to(device)\n",
    "    \n",
    "    model.eval()\n",
    "    with torch.no_grad():\n",
    "        output = model(**tokenized_sent)\n",
    "        \n",
    "    outputs = output['logits'].detach().cpu().numpy()\n",
    "    final_outputs = np.array(outputs) >=0.5\n",
    "    \n",
    "    return (pd.DataFrame(final_outputs,columns=cols))\n",
    "    \n",
    "sentence_classification(\"그만하쟈\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "7d940a3c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>문장</th>\n",
       "      <th>여성/가족</th>\n",
       "      <th>남성</th>\n",
       "      <th>성소수자</th>\n",
       "      <th>인종/국적</th>\n",
       "      <th>연령</th>\n",
       "      <th>지역</th>\n",
       "      <th>종교</th>\n",
       "      <th>기타 혐오</th>\n",
       "      <th>악플/욕설</th>\n",
       "      <th>clean</th>\n",
       "      <th>개인지칭</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>56</th>\n",
       "      <td>조국 수사는 이제 막바지. 기껏해봐야 와이프가 표장장관련 의혹이 전부. 국민을 위한...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>860</th>\n",
       "      <td>천조국같았으면 바로 총알받이이넫 아숩 ㅋㅋㅋㅋㅋㅋㅋㅋㅋㅋㅋㅋㅋ무슬림시발세키덜</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>953</th>\n",
       "      <td>조팔 저게 18년도 선진국에서 일어나는 일이라니 믿을수가없노 내조국 좆되기전에 무슬...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1387</th>\n",
       "      <td>도대체 무슨 겉멋이 들어서 다문화 우대 개지랄들을 하는지 모르겠다 특히 십짱개새끼들...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3529</th>\n",
       "      <td>갓조국 미투운동 응원합니다 충성충성^^7 한녀는 제외</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3552</th>\n",
       "      <td>ㅋㅋㅋㅋ 한국은 대놓고 성차별하네 ㅋㅋㅋㅋㅋㅋㅋㅋㅋㅋㅋㅋㅋㅋㅋㅋㅋㅋㅋㅋㅋ천조국 게이...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4393</th>\n",
       "      <td>내가 이래사 갓조국으로 이민온거임. 이새끼들은 진짜 불합리한거보면 절대 가만히 안 ...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4758</th>\n",
       "      <td>탈북자 새끼들 다 북한으로 돌려보내야한다 얘들은 자기 조국을 배신한 변절자다참고로 ...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5710</th>\n",
       "      <td>ㅋㅋㅋ없는증거도 만들어내겠군 조국때처럼</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5760</th>\n",
       "      <td>인권팔이하는 애들아~~조국 아들딸때문에 피해본 학생들의 인권은 뭐라 이야기할래?</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7077</th>\n",
       "      <td>너련=나련 조국에서 잘먹고 잘살자 이니야</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7078</th>\n",
       "      <td>천조국 대통령이 성경책에 손얹고 선서하는 거 보면 존나 멋있고 감동적인데김치맨들이 ...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8586</th>\n",
       "      <td>한인이민충 특징: 현지에서는 세탁소 똥송칭챙총이면서 일베만 오면 천조국게이,천조국응...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8816</th>\n",
       "      <td>조국비리, 북한목선, 북한청년들강제북송, 태양광, 탈원전, 노조불법행위묵인,,,,게...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9905</th>\n",
       "      <td>조국사태때 제발 지금 어투로 말해보질그랬냐</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10098</th>\n",
       "      <td>이번 정권은 진짜 쓰레기 주변 자영업자들 다죽는소리한다.검찰개혁필요하지만 그전에 조...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10241</th>\n",
       "      <td>서초동200백만 조국사수 검찰개혁 그러는 니넨 1천명 어따. 대구. 에서. 뭐해~</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11168</th>\n",
       "      <td>어디 천조국만 그러겠냐세계 어딜가든 마찬가지지조셴징이 해외 나가서 제일 조심해야되는...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11574</th>\n",
       "      <td>장애남만 가득한 나라라 저게 정상이 되노 내조국이었음 저렇게 애새끼 기형적이게 키우...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12775</th>\n",
       "      <td>버닝썬 조사 다시하고 조국은 특검</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13823</th>\n",
       "      <td>나는 한심한 게 저러고 사는 인생들이 지들도 조국이라 염병을 하냐고저것들은 시궁창 ...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                      문장  여성/가족  남성  성소수자  \\\n",
       "56     조국 수사는 이제 막바지. 기껏해봐야 와이프가 표장장관련 의혹이 전부. 국민을 위한...      0   0     0   \n",
       "860           천조국같았으면 바로 총알받이이넫 아숩 ㅋㅋㅋㅋㅋㅋㅋㅋㅋㅋㅋㅋㅋ무슬림시발세키덜      0   0     0   \n",
       "953    조팔 저게 18년도 선진국에서 일어나는 일이라니 믿을수가없노 내조국 좆되기전에 무슬...      0   1     0   \n",
       "1387   도대체 무슨 겉멋이 들어서 다문화 우대 개지랄들을 하는지 모르겠다 특히 십짱개새끼들...      0   0     0   \n",
       "3529                       갓조국 미투운동 응원합니다 충성충성^^7 한녀는 제외      1   0     0   \n",
       "3552   ㅋㅋㅋㅋ 한국은 대놓고 성차별하네 ㅋㅋㅋㅋㅋㅋㅋㅋㅋㅋㅋㅋㅋㅋㅋㅋㅋㅋㅋㅋㅋ천조국 게이...      0   0     0   \n",
       "4393   내가 이래사 갓조국으로 이민온거임. 이새끼들은 진짜 불합리한거보면 절대 가만히 안 ...      0   0     0   \n",
       "4758   탈북자 새끼들 다 북한으로 돌려보내야한다 얘들은 자기 조국을 배신한 변절자다참고로 ...      0   0     0   \n",
       "5710                               ㅋㅋㅋ없는증거도 만들어내겠군 조국때처럼      0   0     0   \n",
       "5760        인권팔이하는 애들아~~조국 아들딸때문에 피해본 학생들의 인권은 뭐라 이야기할래?      0   0     0   \n",
       "7077                              너련=나련 조국에서 잘먹고 잘살자 이니야      0   0     0   \n",
       "7078   천조국 대통령이 성경책에 손얹고 선서하는 거 보면 존나 멋있고 감동적인데김치맨들이 ...      0   0     0   \n",
       "8586   한인이민충 특징: 현지에서는 세탁소 똥송칭챙총이면서 일베만 오면 천조국게이,천조국응...      0   0     0   \n",
       "8816   조국비리, 북한목선, 북한청년들강제북송, 태양광, 탈원전, 노조불법행위묵인,,,,게...      0   0     0   \n",
       "9905                             조국사태때 제발 지금 어투로 말해보질그랬냐      0   0     0   \n",
       "10098  이번 정권은 진짜 쓰레기 주변 자영업자들 다죽는소리한다.검찰개혁필요하지만 그전에 조...      0   0     0   \n",
       "10241      서초동200백만 조국사수 검찰개혁 그러는 니넨 1천명 어따. 대구. 에서. 뭐해~      0   0     0   \n",
       "11168  어디 천조국만 그러겠냐세계 어딜가든 마찬가지지조셴징이 해외 나가서 제일 조심해야되는...      0   0     0   \n",
       "11574  장애남만 가득한 나라라 저게 정상이 되노 내조국이었음 저렇게 애새끼 기형적이게 키우...      0   1     0   \n",
       "12775                                 버닝썬 조사 다시하고 조국은 특검      0   0     0   \n",
       "13823  나는 한심한 게 저러고 사는 인생들이 지들도 조국이라 염병을 하냐고저것들은 시궁창 ...      0   0     0   \n",
       "\n",
       "       인종/국적  연령  지역  종교  기타 혐오  악플/욕설  clean  개인지칭  \n",
       "56         0   0   0   0      1      0      0     1  \n",
       "860        0   0   0   1      0      0      0     1  \n",
       "953        1   0   0   0      0      0      0     1  \n",
       "1387       1   0   0   0      0      0      0     1  \n",
       "3529       0   0   0   0      0      0      0     1  \n",
       "3552       1   0   0   0      0      0      0     1  \n",
       "4393       1   0   0   0      0      0      0     1  \n",
       "4758       1   0   0   0      0      0      0     1  \n",
       "5710       0   0   0   0      0      1      0     0  \n",
       "5760       0   0   0   0      1      0      0     1  \n",
       "7077       0   0   0   0      0      0      1     0  \n",
       "7078       1   0   0   0      0      0      0     1  \n",
       "8586       1   0   0   0      0      0      0     1  \n",
       "8816       0   0   0   0      0      1      0     1  \n",
       "9905       0   0   0   0      0      1      0     0  \n",
       "10098      0   0   0   0      1      0      0     1  \n",
       "10241      0   0   1   0      0      0      0     1  \n",
       "11168      1   0   0   0      0      0      0     1  \n",
       "11574      0   0   0   0      0      0      0     1  \n",
       "12775      0   0   0   0      0      0      1     0  \n",
       "13823      0   0   0   0      0      1      0     0  "
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_data[train_data[\"문장\"].str.contains(\"조국\")]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d1d0a3b0",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "PyTorch 1.10 (NGC 21.11/Python 3.8 Conda) on Backend.AI",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}