sinkaf.ipynb

{
 "metadata": {
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.1-final"
  },
  "orig_nbformat": 2,
  "kernelspec": {
   "name": "python3",
   "display_name": "Python 3.9.1 64-bit",
   "metadata": {
    "interpreter": {
     "hash": "aee8b7b246df8f9039afb4144a1f6fd8d2ca17a180786b69acc140d282b71a49"
    }
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2,
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "#: Try different models\n",
    "\n",
    "import pandas as pd\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "from sklearn.metrics import classification_report\n",
    "from sinkaf.utils import Preprocessor\n",
    "\n",
    "stop_words_tr = pd.read_csv(\"https://raw.githubusercontent.com/ahmetax/trstop/master/dosyalar/turkce-stop-words\", header=None)\n",
    "stop_words_tr = stop_words_tr[0].to_numpy()\n",
    "\n",
    "def score_model(X, y, X_test = None, y_test = None, estimator = None, **kwargs):\n",
    "    \"\"\"\n",
    "    Test various estimators.\n",
    "    \"\"\"\n",
    "    # y = LabelEncoder().fit_transform(y)\n",
    "\n",
    "    model = Pipeline([\n",
    "        ('preprocess', Preprocessor()),\n",
    "        ('counts', CountVectorizer(min_df=5, max_df=0.99, stop_words = frozenset(stop_words_tr))),\n",
    "        ('tf_idf', TfidfTransformer()),\n",
    "        ('estimator', estimator)\n",
    "    ])\n",
    "\n",
    "    # Instantiate the classification model and visualizer\n",
    "    model.fit(X, y, **kwargs)\n",
    "\n",
    "    # if not used for training\n",
    "    if (X_test is None):\n",
    "        return model\n",
    "\n",
    "    expected  = y\n",
    "    predicted = model.predict(X)\n",
    "\n",
    "    expected_test = LabelEncoder().fit_transform(y_test)\n",
    "    predicted_test = model.predict(X_test)\n",
    "\n",
    "    # Compute and return F1 (harmonic mean of precision and recall)\n",
    "    print(\"\\n- {}:\\n\\nTrain:\\n{}\\nTest:\\n{}\".format(estimator.__class__.__name__, \\\n",
    "        classification_report(expected, predicted), classification_report(expected_test, predicted_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "                   id      timestamp  \\\n",
       "0  973568937593065472  1520952977415   \n",
       "1  973568937723035648  1520952977446   \n",
       "2  973568937911873536  1520952977491   \n",
       "3  973568939925090304  1520952977971   \n",
       "4  973568940667539457  1520952978148   \n",
       "\n",
       "                                                text label  \n",
       "0  @USER06095 Hırsız demişken Tuncay sizin şu 1.2...   grp  \n",
       "1  Ne bileyim sen hastayım deyince bende veterine...   ind  \n",
       "2  Akşam eve gittiğimizde yorgunluğuma iyi gelece...   grp  \n",
       "3  Kook’un sesini 18381 kez dinledikten sonra eğe...  prof  \n",
       "4  @USER05270 @USER04816 o macta adam 6 7 tane ne...   grp  "
      ],
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>id</th>\n      <th>timestamp</th>\n      <th>text</th>\n      <th>label</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>973568937593065472</td>\n      <td>1520952977415</td>\n      <td>@USER06095 Hırsız demişken Tuncay sizin şu 1.2...</td>\n      <td>grp</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>973568937723035648</td>\n      <td>1520952977446</td>\n      <td>Ne bileyim sen hastayım deyince bende veterine...</td>\n      <td>ind</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>973568937911873536</td>\n      <td>1520952977491</td>\n      <td>Akşam eve gittiğimizde yorgunluğuma iyi gelece...</td>\n      <td>grp</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>973568939925090304</td>\n      <td>1520952977971</td>\n      <td>Kook’un sesini 18381 kez dinledikten sonra eğe...</td>\n      <td>prof</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>973568940667539457</td>\n      <td>1520952978148</td>\n      <td>@USER05270 @USER04816 o macta adam 6 7 tane ne...</td>\n      <td>grp</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "metadata": {},
     "execution_count": 10
    }
   ],
   "source": [
    "#: load data\n",
    "data = pd.read_csv(\"sinkaf/data/troff-v1.0.tsv\",  sep='\\t')\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "False    28439\n",
       "True      6845\n",
       "Name: label_sinkaf, dtype: int64"
      ]
     },
     "metadata": {},
     "execution_count": 11
    }
   ],
   "source": [
    "#: create labels\n",
    "data['label_sinkaf'] = data['label'] != 'non'\n",
    "\n",
    "data['label_sinkaf'].value_counts()\n",
    "# 6.8K offensive - kufurlu yorum\n",
    "# 28K  non-offensive - kufursuz yorum"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "- SVC:\n",
      "\n",
      "Train:\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.81      1.00      0.89     21340\n",
      "           1       0.00      0.00      0.00      5123\n",
      "\n",
      "    accuracy                           0.81     26463\n",
      "   macro avg       0.40      0.50      0.45     26463\n",
      "weighted avg       0.65      0.81      0.72     26463\n",
      "\n",
      "Test:\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.80      1.00      0.89      7099\n",
      "           1       0.00      0.00      0.00      1722\n",
      "\n",
      "    accuracy                           0.80      8821\n",
      "   macro avg       0.40      0.50      0.45      8821\n",
      "weighted avg       0.65      0.80      0.72      8821\n",
      "\n",
      "\n",
      "- LinearSVC:\n",
      "\n",
      "Train:\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.92      0.99      0.96     21340\n",
      "           1       0.94      0.66      0.78      5123\n",
      "\n",
      "    accuracy                           0.93     26463\n",
      "   macro avg       0.93      0.82      0.87     26463\n",
      "weighted avg       0.93      0.93      0.92     26463\n",
      "\n",
      "Test:\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.87      0.95      0.91      7099\n",
      "           1       0.66      0.40      0.50      1722\n",
      "\n",
      "    accuracy                           0.84      8821\n",
      "   macro avg       0.76      0.67      0.70      8821\n",
      "weighted avg       0.83      0.84      0.83      8821\n",
      "\n",
      "\n",
      "- SGDClassifier:\n",
      "\n",
      "Train:\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.84      1.00      0.91     21340\n",
      "           1       0.95      0.22      0.36      5123\n",
      "\n",
      "    accuracy                           0.85     26463\n",
      "   macro avg       0.90      0.61      0.64     26463\n",
      "weighted avg       0.86      0.85      0.81     26463\n",
      "\n",
      "Test:\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.83      1.00      0.91      7099\n",
      "           1       0.90      0.17      0.29      1722\n",
      "\n",
      "    accuracy                           0.83      8821\n",
      "   macro avg       0.87      0.58      0.60      8821\n",
      "weighted avg       0.85      0.83      0.79      8821\n",
      "\n",
      "\n",
      "- KNeighborsClassifier:\n",
      "\n",
      "Train:\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.83      0.99      0.90     21340\n",
      "           1       0.78      0.13      0.23      5123\n",
      "\n",
      "    accuracy                           0.83     26463\n",
      "   macro avg       0.80      0.56      0.56     26463\n",
      "weighted avg       0.82      0.83      0.77     26463\n",
      "\n",
      "Test:\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.81      0.98      0.89      7099\n",
      "           1       0.33      0.04      0.07      1722\n",
      "\n",
      "    accuracy                           0.80      8821\n",
      "   macro avg       0.57      0.51      0.48      8821\n",
      "weighted avg       0.71      0.80      0.73      8821\n",
      "\n",
      "\n",
      "- LogisticRegression:\n",
      "\n",
      "Train:\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.85      1.00      0.92     21340\n",
      "           1       0.95      0.29      0.44      5123\n",
      "\n",
      "    accuracy                           0.86     26463\n",
      "   macro avg       0.90      0.64      0.68     26463\n",
      "weighted avg       0.87      0.86      0.83     26463\n",
      "\n",
      "Test:\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.84      0.99      0.91      7099\n",
      "           1       0.85      0.21      0.34      1722\n",
      "\n",
      "    accuracy                           0.84      8821\n",
      "   macro avg       0.84      0.60      0.62      8821\n",
      "weighted avg       0.84      0.84      0.80      8821\n",
      "\n",
      "\n",
      "- LogisticRegressionCV:\n",
      "\n",
      "Train:\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.89      0.99      0.94     21340\n",
      "           1       0.94      0.47      0.63      5123\n",
      "\n",
      "    accuracy                           0.89     26463\n",
      "   macro avg       0.91      0.73      0.78     26463\n",
      "weighted avg       0.90      0.89      0.88     26463\n",
      "\n",
      "Test:\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.85      0.98      0.91      7099\n",
      "           1       0.78      0.31      0.45      1722\n",
      "\n",
      "    accuracy                           0.85      8821\n",
      "   macro avg       0.82      0.65      0.68      8821\n",
      "weighted avg       0.84      0.85      0.82      8821\n",
      "\n",
      "\n",
      "- BaggingClassifier:\n",
      "\n",
      "Train:\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.98      1.00      0.99     21340\n",
      "           1       0.99      0.90      0.94      5123\n",
      "\n",
      "    accuracy                           0.98     26463\n",
      "   macro avg       0.98      0.95      0.97     26463\n",
      "weighted avg       0.98      0.98      0.98     26463\n",
      "\n",
      "Test:\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.86      0.94      0.90      7099\n",
      "           1       0.58      0.35      0.44      1722\n",
      "\n",
      "    accuracy                           0.82      8821\n",
      "   macro avg       0.72      0.65      0.67      8821\n",
      "weighted avg       0.80      0.82      0.81      8821\n",
      "\n",
      "\n",
      "- ExtraTreesClassifier:\n",
      "\n",
      "Train:\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      1.00      1.00     21340\n",
      "           1       1.00      0.99      0.99      5123\n",
      "\n",
      "    accuracy                           1.00     26463\n",
      "   macro avg       1.00      0.99      1.00     26463\n",
      "weighted avg       1.00      1.00      1.00     26463\n",
      "\n",
      "Test:\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.86      0.96      0.90      7099\n",
      "           1       0.66      0.34      0.45      1722\n",
      "\n",
      "    accuracy                           0.84      8821\n",
      "   macro avg       0.76      0.65      0.67      8821\n",
      "weighted avg       0.82      0.84      0.81      8821\n",
      "\n",
      "\n",
      "- RandomForestClassifier:\n",
      "\n",
      "Train:\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      1.00      1.00     21340\n",
      "           1       1.00      0.99      0.99      5123\n",
      "\n",
      "    accuracy                           1.00     26463\n",
      "   macro avg       1.00      0.99      1.00     26463\n",
      "weighted avg       1.00      1.00      1.00     26463\n",
      "\n",
      "Test:\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.85      0.97      0.91      7099\n",
      "           1       0.70      0.31      0.43      1722\n",
      "\n",
      "    accuracy                           0.84      8821\n",
      "   macro avg       0.78      0.64      0.67      8821\n",
      "weighted avg       0.82      0.84      0.81      8821\n",
      "\n",
      "\n",
      "- MultinomialNB:\n",
      "\n",
      "Train:\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.85      1.00      0.92     21340\n",
      "           1       0.97      0.26      0.41      5123\n",
      "\n",
      "    accuracy                           0.86     26463\n",
      "   macro avg       0.91      0.63      0.66     26463\n",
      "weighted avg       0.87      0.86      0.82     26463\n",
      "\n",
      "Test:\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.83      1.00      0.90      7099\n",
      "           1       0.92      0.15      0.26      1722\n",
      "\n",
      "    accuracy                           0.83      8821\n",
      "   macro avg       0.87      0.57      0.58      8821\n",
      "weighted avg       0.85      0.83      0.78      8821\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#: split the data & find best model\n",
    "from sklearn.model_selection import train_test_split\n",
    "X_train, X_test, y_train, y_test = train_test_split(data['text'], data['label_sinkaf'])\n",
    " \n",
    "from sklearn.svm import LinearSVC, NuSVC, SVC\n",
    "from sklearn.metrics import f1_score, classification_report\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.ensemble import BaggingClassifier, ExtraTreesClassifier, RandomForestClassifier\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.naive_bayes import MultinomialNB\n",
    "from sklearn.linear_model import LogisticRegressionCV, LogisticRegression, SGDClassifier\n",
    "from sklearn.feature_extraction.text import CountVectorizer, TfidfTransformer\n",
    "\n",
    "\n",
    "models = [\n",
    "    SVC(gamma='auto'), LinearSVC(),\n",
    "    SGDClassifier(max_iter=100, tol=1e-3), KNeighborsClassifier(),\n",
    "    LogisticRegression(solver='lbfgs'), LogisticRegressionCV(cv=3),\n",
    "    BaggingClassifier(), ExtraTreesClassifier(n_estimators=300),\n",
    "    RandomForestClassifier(n_estimators=300), MultinomialNB()\n",
    "]\n",
    "\n",
    "for model in models:\n",
    "    score_model(X_train, y_train, X_test, y_test, model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.svm import LinearSVC, NuSVC, SVC\n",
    "from sklearn.metrics import f1_score, classification_report\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.ensemble import BaggingClassifier, ExtraTreesClassifier, RandomForestClassifier\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.naive_bayes import MultinomialNB\n",
    "from sklearn.linear_model import LogisticRegressionCV, LogisticRegression, SGDClassifier\n",
    "from sklearn.feature_extraction.text import CountVectorizer, TfidfTransformer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "#: Use best model (LinearSVC)\n",
    "\n",
    "# Tum veriyi kullanma\n",
    "X = data['text']\n",
    "y = data['label_sinkaf']\n",
    "\n",
    "# LinearSVC does not have predict_proba function\n",
    "from sklearn.calibration import CalibratedClassifierCV\n",
    "clf = CalibratedClassifierCV(LinearSVC()) \n",
    "\n",
    "model = score_model(X, y, estimator = clf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "[False False  True  True False]\n[0.08525992 0.44037578 0.98655027 0.77686361 0.12219944]\n"
     ]
    }
   ],
   "source": [
    "# Offensive? - Kufur mu?\n",
    "test = [\"cok iyi\", \n",
    "        \"bi git\", \n",
    "        \"bi siktir git\", \n",
    "        \"bi defol\",\n",
    "        \"mukemmel bir insansin\"]\n",
    "\n",
    "print(model.predict(test))\n",
    "print(model.predict_proba(test)[:,1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "['sinkaf/data/model_linearSVC.joblib']"
      ]
     },
     "metadata": {},
     "execution_count": 18
    }
   ],
   "source": [
    "import joblib\n",
    "\n",
    "#dump it (higher), dump it (higher!)\n",
    "joblib.dump(model, \"sinkaf/data/model_linearSVC.joblib\")"
   ]
  }
 ]
}