model_evaluation.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "e1f98a1d-f224-4b07-90d9-fe38c7f5c93a",
   "metadata": {},
   "source": [
    "# Dependency management"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c18a12b7-c8bf-4e2f-81d7-e17b00567a2a",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import os\n",
    "import mlflow\n",
    "import json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c42b68e5-596b-4324-a089-723b0a4c8a75",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer, TfidfTransformer\n",
    "from sklearn.metrics import (accuracy_score, mean_absolute_error, f1_score,\n",
    "                             precision_score, recall_score, matthews_corrcoef)\n",
    "from sklearn.compose import ColumnTransformer\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.naive_bayes import MultinomialNB"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a10f3bc7-4a97-4b13-a532-cfa27b443025",
   "metadata": {},
   "source": [
    "# Minio Bucket setup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ca1d5a82-aa18-4d13-891a-5c9d97f06205",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "MLFLOW_EXP_NAME = os.environ.get('MLFLOW_EXP_NAME', '20news_clf')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a6dacbcc-43bb-42f0-a3a3-de9dacc9397e",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "bucket_name = 'datasets'\n",
    "filepath = f'{bucket_name}/20news/test.csv'"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "964602a7-f5dc-4dce-bdf7-cafcb93168b3",
   "metadata": {},
   "source": [
    "# Load pandas dataframe from S3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a8ab206c-e9d6-4742-896e-7078e296aafb",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "df = pd.read_csv(\n",
    "    f's3://{filepath}',\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "61a07fd3-a6c7-4993-8eb4-2abe12901c87",
   "metadata": {},
   "source": [
    "# Start your code here"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "80c4be5d-98ac-404c-8375-88d517f3ee71",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "from mlflow import MlflowClient\n",
    "\n",
    "# Initialize an MLflow Client\n",
    "client = MlflowClient()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "914571b0-525e-4bd9-930d-a4c0d720e068",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "best_run = None\n",
    "best_score = -9999\n",
    "\n",
    "experiment_id = client.get_experiment_by_name(MLFLOW_EXP_NAME).experiment_id\n",
    "\n",
    "\n",
    "for run in client.search_runs(experiment_id, filter_string=\"tag.`evaluated` = 'False'\"):\n",
    "    artifact_path = json.loads(run.data.tags['mlflow.log-model.history'])[0]['artifact_path']\n",
    "    clf = mlflow.sklearn.load_model(model_uri=f'runs:/{run.info.run_id}/{artifact_path}')\n",
    "    y_pred = clf.predict(df['text'])\n",
    "    score = f1_score(df['target'], y_pred, average='macro')\n",
    "    if score > best_score:\n",
    "        best_score = score\n",
    "        best_run = run\n",
    "    client.set_tag(run.info.run_id,'evaluated', True)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "146b29a8-bf33-4962-a9d0-b48c6ea1fd55",
   "metadata": {},
   "source": [
    "## Register highest performance model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "13b541ae-8aa1-470c-a908-8459cf294459",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "try:\n",
    "    client.create_registered_model(MLFLOW_EXP_NAME)\n",
    "except:\n",
    "    pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8ae27236-51b5-4cfd-8307-0f5b11266c43",
   "metadata": {},
   "outputs": [],
   "source": [
    "if best_run:\n",
    "    client.create_model_version(\n",
    "        MLFLOW_EXP_NAME, \n",
    "        f'runs:/{best_run.info.run_id}/{artifact_path}', \n",
    "        best_run.info.run_id, \n",
    "        description=\"Selected by model evaluation step\"\n",
    "    )\n",
    "else:\n",
    "    print(\"No run selected\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c92e4e34-00f5-413e-88e2-376100d7f8d3",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}