mlm_experimentation_pipeline/Experimentation-Pipeline.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1. Settings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "#####################################\n",
    "##########  DEPENDECIES ############\n",
    "#####################################\n",
    "\n",
    "import os\n",
    "import pickle\n",
    "import sys\n",
    "sys.path.append(\"../\")\n",
    "\n",
    "from tqdm import tqdm # type: ignore\n",
    "from datetime import date\n",
    "\n",
    "import evaluate\n",
    "\n",
    "from transformers import AutoModelForSeq2SeqLM, AutoTokenizer\n",
    "from utils.sampling import create_splits, prep_cv_validation\n",
    "from utils.training import cv_cluster_set, cv_training_epochs_sets, test_cluster_set\n",
    "from utils.training import results_dict_todf, cv_step_2, full_step_2, test_training_epochs_sets\n",
    "from utils.inference import meta_predict, create_ensemble_map, ensemble_compute\n",
    "\n",
    "tqdm.pandas()\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "\n",
    "os.environ[\"TOKENIZERS_PARALLELISM\"] = \"false\"\n",
    "os.environ[\"WANDB_DISABLED\"] = \"true\"\n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"0\"\n",
    "\n",
    "import torch\n",
    "\n",
    "#####################################\n",
    "############  CONSTANTS #############\n",
    "#####################################\n",
    "\n",
    "RS = 42\n",
    "BATCH_SIZE = 16\n",
    "DECODER_LENGTH = 30\n",
    "ENCODER_LENGTH = 30\n",
    "MODEL_NAME = \"Salesforce/codet5-base-multi-sum\"\n",
    "\n",
    "FULL_TRAIN_ARGS = {\n",
    "    \"BATCH_SIZE\": BATCH_SIZE,\n",
    "    \"DECODER_LENGTH\": DECODER_LENGTH,\n",
    "    \"ENCODER_LENGTH\": ENCODER_LENGTH,\n",
    "    \"SEQ_TRAINER_ARGS\": {\n",
    "        \"overwrite_output_dir\": True,\n",
    "        \"num_train_epochs\": [0, 1, 2, 5, 10],\n",
    "        \"do_train\": True,\n",
    "        \"do_eval\": True,\n",
    "        \"per_device_train_batch_size\": 4,\n",
    "        \"per_device_eval_batch_size\": 4,\n",
    "        \"learning_rate\": 1e-5,\n",
    "        \"warmup_steps\": 500,\n",
    "        \"weight_decay\": 0.1,\n",
    "        \"label_smoothing_factor\": 0.1,\n",
    "        \"predict_with_generate\": True,\n",
    "        \"logging_steps\": 100,\n",
    "        \"save_total_limit\": 1,\n",
    "        \"save_strategy\": \"no\",\n",
    "        \"logging_strategy\": \"epoch\",\n",
    "        \"evaluation_strategy\": \"epoch\",\n",
    "        \"load_best_model_at_end\": False,\n",
    "        \"output_dir\" : 'reports/results',\n",
    "        \"logging_dir\" : \"reports/logs\",\n",
    "    },\n",
    "}\n",
    "\n",
    "experiment_config = {\n",
    "    \"DATA_STR\" : \"20240908\",\n",
    "    \"RS\" : RS,\n",
    "    \"DRIFT_TYPE\" : \"drift\",\n",
    "    \"NFOLD\" : 3,\n",
    "    \"FULL_TRAIN_ARGS\" : FULL_TRAIN_ARGS,\n",
    "    \"MODEL_NAME\" : MODEL_NAME,\n",
    "    \"CLUSTER_EPOCHS\" : 3,\n",
    "    \"CLUSTER_SET_ID\" : [0, 3, [0, 3,]],\n",
    "    \"TRAIN_SIZE\" : 7000,\n",
    "    \"TEST_SIZE\" : 2500,\n",
    "}\n",
    "experiment_config[\"ANALYSIS_POSTFIX\"] = f\"mined_{experiment_config['DRIFT_TYPE']}_{str(date.today())}\"\n",
    "experiment_config[\"FULL_TRAIN_ARGS\"][\"SEQ_TRAINER_ARGS\"][\"output_dir\"] += \"/\" + experiment_config[\"ANALYSIS_POSTFIX\"] \n",
    "experiment_config[\"FULL_TRAIN_ARGS\"][\"SEQ_TRAINER_ARGS\"][\"logging_dir\"] += \"/\" + experiment_config[\"ANALYSIS_POSTFIX\"] \n",
    "\n",
    "if not os.path.exists(experiment_config[\"FULL_TRAIN_ARGS\"][\"SEQ_TRAINER_ARGS\"][\"logging_dir\"]):\n",
    "    os.mkdir(experiment_config[\"FULL_TRAIN_ARGS\"][\"SEQ_TRAINER_ARGS\"][\"logging_dir\"])\n",
    "\n",
    "if not os.path.exists(experiment_config[\"FULL_TRAIN_ARGS\"][\"SEQ_TRAINER_ARGS\"][\"output_dir\"]):\n",
    "    os.mkdir(experiment_config[\"FULL_TRAIN_ARGS\"][\"SEQ_TRAINER_ARGS\"][\"output_dir\"])\n",
    "\n",
    "tokenizer = AutoTokenizer.from_pretrained(experiment_config[\"MODEL_NAME\"], skip_special_tokens=False)\n",
    "model = AutoModelForSeq2SeqLM.from_pretrained(experiment_config[\"MODEL_NAME\"])\n",
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "rouge = evaluate.load('rouge')\n",
    "\n",
    "experiment_config[\"ANALYSIS_POSTFIX\"] = f\"mined_{experiment_config['DRIFT_TYPE']}_2024-09-09\"\n",
    "RUN_BASE_TRAINING = False\n",
    "t_models=[\"catboost\"]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2. Conala data. Preprocessing. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "sampling_dict = create_splits(experiment_config=experiment_config, tokenizer=tokenizer, train_size=experiment_config[\"TRAIN_SIZE\"], test_size=experiment_config[\"TEST_SIZE\"], cluster_id=4)\n",
    "train_dataset, test_data, test_df, train_df = sampling_dict[\"train_data\"], sampling_dict[\"test_data\"], sampling_dict[\"test_df\"], sampling_dict[\"train_df\"]\n",
    "\n",
    "splits, questions_list = prep_cv_validation(train_dataset=train_dataset, \n",
    "                            experiment_config=experiment_config)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "if RUN_BASE_TRAINING:\n",
    "    fold_results = cv_training_epochs_sets(experiment_config=experiment_config,\n",
    "                                splits=splits,\n",
    "                                questions_list=questions_list,\n",
    "                                train_dataset=train_dataset,\n",
    "                                tokenizer=tokenizer)\n",
    "\n",
    "    with open(f'reports/results/{experiment_config[\"ANALYSIS_POSTFIX\"]}/cv_fold_epoch_set.pickle', 'wb') as handle:\n",
    "        pickle.dump(fold_results, handle, protocol=pickle.HIGHEST_PROTOCOL)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "if RUN_BASE_TRAINING:\n",
    "    with open(f'reports/results/{experiment_config[\"ANALYSIS_POSTFIX\"]}/cv_fold_epoch_set.pickle', 'rb') as handle:\n",
    "        fold_results = pickle.load(handle)\n",
    "    \n",
    "    for cluster_idx in experiment_config[\"CLUSTER_SET_ID\"]:\n",
    "        fold_results = cv_cluster_set(experiment_config=experiment_config,\n",
    "                                                splits=splits,\n",
    "                                                questions_list=questions_list,\n",
    "                                                train_dataset=train_dataset,\n",
    "                                                tokenizer=tokenizer,\n",
    "                                                fold_results=fold_results,\n",
    "                                                cluster_id=cluster_idx)\n",
    "\n",
    "    cv_df = results_dict_todf(fold_results)\n",
    "\n",
    "    ########## SAVE THE FILE\n",
    "\n",
    "    with open(f'reports/results/{experiment_config[\"ANALYSIS_POSTFIX\"]}/cv_step1.pickle', 'wb') as handle:\n",
    "        pickle.dump(cv_df, handle, protocol=pickle.HIGHEST_PROTOCOL)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "with open(f'reports/results/{experiment_config[\"ANALYSIS_POSTFIX\"]}/cv_step1.pickle', 'rb') as handle:\n",
    "    cv_df = pickle.load(handle)\n",
    "\n",
    "print(\"Mean\")\n",
    "print(cv_df.groupby([\"model_set\"])[\"rouge\"].mean())\n",
    "\n",
    "print(\"STD\")\n",
    "print(cv_df.groupby(\"model_set\")[\"rouge\"].std())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Step 2. Learn performance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "########## LOAD CV RESULTS\n",
    "\n",
    "import pickle\n",
    "with open(f'reports/results/{experiment_config[\"ANALYSIS_POSTFIX\"]}/cv_step1.pickle', 'rb') as handle:\n",
    "    cv_df = pickle.load(handle)\n",
    "\n",
    "########## RUN STEP 2 ON CV\n",
    "\n",
    "cv_df, model_results = cv_step_2(experiment_config=experiment_config, cv_df=cv_df, t_models=t_models)\n",
    "\n",
    "with open(f'reports/results/{experiment_config[\"ANALYSIS_POSTFIX\"]}/s2_model_results.pickle', 'wb') as handle:\n",
    "    pickle.dump(model_results, handle, protocol=pickle.HIGHEST_PROTOCOL)\n",
    "\n",
    "with open(f'reports/results/{experiment_config[\"ANALYSIS_POSTFIX\"]}/cv_results.pickle', 'wb') as handle:\n",
    "    pickle.dump(cv_df, handle, protocol=pickle.HIGHEST_PROTOCOL)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "### TO SAVE THE VECTORIZER AND STEP 2 MODELS\n",
    "\n",
    "with open(f'reports/results/{experiment_config[\"ANALYSIS_POSTFIX\"]}/cv_results.pickle', 'rb') as handle:\n",
    "    cv_df = pickle.load(handle)\n",
    "\n",
    "\n",
    "print(\"Mean\")\n",
    "print(cv_df.groupby([\"model_set\"])[\"catboost_perf_hat\"].mean())\n",
    "\n",
    "print(\"STD\")\n",
    "print(cv_df.groupby(\"model_set\")[\"catboost_perf_hat\"].std())\n",
    "\n",
    "\n",
    "full_step_2(cv_df=cv_df, \n",
    "            experiment_config=experiment_config, t_models=t_models)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# TEST"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(f\"reports/results/{experiment_config['ANALYSIS_POSTFIX']}/cv_results.pickle\", \"rb\") as handle:\n",
    "    cv_resutls = pickle.load(handle)\n",
    "\n",
    "base_models_list = list(cv_resutls.model_set.unique())\n",
    "base_models_list.pop(-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "jupyter": {
     "source_hidden": true
    }
   },
   "outputs": [],
   "source": [
    "meta_preds_df = meta_predict(experiment_config=experiment_config, \n",
    "                    test_df=test_df,\n",
    "                    base_models_names=base_models_list,\n",
    "                    t_models=t_models)\n",
    "\n",
    "########## SAVE THE FILE\n",
    "\n",
    "with open(f'reports/results/{experiment_config[\"ANALYSIS_POSTFIX\"]}/test_step2.pickle', 'wb') as handle:\n",
    "    pickle.dump(meta_preds_df, handle, protocol=pickle.HIGHEST_PROTOCOL)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "with open(f'reports/results/{experiment_config[\"ANALYSIS_POSTFIX\"]}/test_step2.pickle', 'rb') as handle:\n",
    "    meta_preds_df = pickle.load(handle)\n",
    "    \n",
    "meta_preds_df.groupby(\"model_set\").catboost_preds.mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "meta_preds_df.groupby(\"model_set\").catboost_preds.std()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "optimal_ensemble_map, ensemble_val_estim = create_ensemble_map(meta_preds_df=meta_preds_df, \n",
    "                                                                t_model_name=\"catboost\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ensemble_val_estim.catboost_preds.mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ensemble_val_estim.catboost_preds.std()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "if RUN_BASE_TRAINING:\n",
    "    test_result_df = test_training_epochs_sets(experiment_config=experiment_config,\n",
    "                                test_df=test_df,\n",
    "                                test_data=test_data,\n",
    "                                train_data=train_dataset,\n",
    "                                tokenizer=tokenizer)\n",
    "\n",
    "    with open(f'reports/results/{experiment_config[\"ANALYSIS_POSTFIX\"]}/test_epoch_set.pickle', 'wb') as handle:\n",
    "        pickle.dump(test_result_df, handle, protocol=pickle.HIGHEST_PROTOCOL)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_df.cluster.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "if RUN_BASE_TRAINING:\n",
    "    with open(f'reports/results/{experiment_config[\"ANALYSIS_POSTFIX\"]}/test_epoch_set.pickle', 'rb') as handle:\n",
    "        test_result_df = pickle.load(handle)\n",
    "\n",
    "    test_result_df = test_result_df.rename(columns={\"epoch_set\": \"model_set\"})\n",
    "\n",
    "    for cluster_idx in experiment_config[\"CLUSTER_SET_ID\"]:\n",
    "        test_result_df = test_cluster_set(experiment_config=experiment_config,\n",
    "                                        test_df=test_df,\n",
    "                                        test_data=test_data,\n",
    "                                        tokenizer=tokenizer,\n",
    "                                        results_df=test_result_df,\n",
    "                                        cluster_id=cluster_idx,\n",
    "                                        train_df=train_df)\n",
    "\n",
    "    ########## SAVE THE FILE\n",
    "\n",
    "    with open(f'reports/results/{experiment_config[\"ANALYSIS_POSTFIX\"]}/test_results.pickle', 'wb') as handle:\n",
    "        pickle.dump(test_result_df, handle, protocol=pickle.HIGHEST_PROTOCOL)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(f'reports/results/{experiment_config[\"ANALYSIS_POSTFIX\"]}/test_results.pickle', 'rb') as handle:\n",
    "        test_result_df = pickle.load(handle)\n",
    "\n",
    "### ENSEMBLE COMPUTE\n",
    "test_result_df = ensemble_compute(test_result_df=test_result_df,\n",
    "                                  optimal_ensemble_map=optimal_ensemble_map)\n",
    "\n",
    "########## ROUGE PER SETTING\n",
    "\n",
    "print(\"Mean\")\n",
    "print(test_result_df.groupby(\"model_set\")[\"rouge\"].mean())\n",
    "\n",
    "print(\"STD\")\n",
    "print(test_result_df.groupby(\"model_set\")[\"rouge\"].std())\n",
    "\n",
    "print(test_result_df.loc[test_result_df[\"model_set\"] ==test_result_df[\"opt_es_id\"]].opt_es_id.value_counts())\n",
    "\n",
    "with open(f'reports/results/{experiment_config[\"ANALYSIS_POSTFIX\"]}/test_results_full.pickle', 'wb') as handle:\n",
    "    pickle.dump(test_result_df, handle, protocol=pickle.HIGHEST_PROTOCOL)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(test_result_df.loc[test_result_df[\"model_set\"] ==test_result_df[\"opt_es_id\"]].opt_es_id.value_counts())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "ensemble",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}