08 - ML Trial 3.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "3a25e778",
   "metadata": {},
   "outputs": [],
   "source": [
    "# MSM VM config prep\n",
    "import findspark\n",
    "findspark.init('/home/mitch/spark-3.3.0-bin-hadoop2')\n",
    "import pyspark\n",
    " \n",
    "from pyspark.sql import SparkSession\n",
    "spark = SparkSession.builder.appName('BApredsV2').getOrCreate()\n",
    "\n",
    "# --- suppress future spark warnings/error/etc output ---\n",
    "spark.sparkContext.setLogLevel(\"OFF\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "c63f08a6",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "def load_data_and_merge():\n",
    "    labels_and_calcs = spark.read.csv(\"bioavailability_data_final.csv\",inferSchema=True,sep=',',header=True)\n",
    "    df1 = labels_and_calcs.toPandas()\n",
    "    \n",
    "    df3 = pd.read_pickle('bioavailabilityData_w_Frags__final.pkl')\n",
    "    df3 = df3.drop(columns=['ba_pct'])\n",
    "    \n",
    "    #temp1 = pd.merge(df1,df2,how='left',left_on='_c0',right_on=df2.index)\n",
    "    \n",
    "    temp1 = labels_and_calcs\n",
    "    #temp1 = spark.createDataFrame(temp1)\n",
    "    \n",
    "    temp2 = spark.createDataFrame(df3)\n",
    "    \n",
    "    data = temp2.join(temp1,(temp2.drug_smiles==temp1.smile),\"left\")\n",
    "    \n",
    "    return data\n",
    "    \n",
    "\n",
    "data = load_data_and_merge()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "ef0f0351",
   "metadata": {},
   "outputs": [],
   "source": [
    "''' \n",
    "# INTIAL LABELS:\n",
    "# --- Data has 1 continuous label column, and 4 categorical label columns (discretized variants of continuous label).\n",
    "# ------ categorical labels applied by dividing the continuous label values into 3-5 categories \n",
    "# ------ the value range associated with each group were selected based on histogram dist./mean/stdev\n",
    "# --- We'll add one more discretization variant,  using Spark's built-in QuantileDiscretizer\n",
    "'''\n",
    "# -- Add QuantileDiscretizer labels\n",
    "from pyspark.ml.feature import QuantileDiscretizer\n",
    "import pandas as pd\n",
    "qd5 = QuantileDiscretizer(numBuckets=5,inputCol='BA_pct',outputCol='label_QD5')\n",
    "\n",
    "data_wLabels = qd5.fit(data).transform(data)\n",
    "\n",
    "# -- INDEX / ENCODE LABELS\n",
    "from pyspark.ml.feature import (StringIndexer,OneHotEncoder)\n",
    "\n",
    "label_quant0 = 'BA_pct'\n",
    "#label_cat0_vector = OneHotEncoder(inputCol='label_QD5',outputCol='label_cat0_vector')\n",
    "\n",
    "label_cat1_index = StringIndexer(inputCol='label1',outputCol='label_cat1_index')\n",
    "#label_cat1_vector = OneHotEncoder(inputCol='label_cat1_index',outputCol='label_cat1_vector')\n",
    "\n",
    "label_cat2_index = StringIndexer(inputCol='label2',outputCol='label_cat2_index')\n",
    "#label_cat2_vector = OneHotEncoder(inputCol='label_cat2_index',outputCol='label_cat2_vector')\n",
    "\n",
    "label_cat3_index = StringIndexer(inputCol='label3a',outputCol='label_cat3_index')\n",
    "#label_cat3_vector = OneHotEncoder(inputCol='label_cat3_index',outputCol='label_cat3_vector')\n",
    "\n",
    "label_cat4_index = StringIndexer(inputCol='label3b',outputCol='label_cat4_index')\n",
    "#label_cat4_vector = OneHotEncoder(inputCol='label_cat4_index',outputCol='label_cat4_vector')\n",
    "\n",
    "from pyspark.ml import Pipeline\n",
    "label_pipeline = Pipeline(stages=[#label_cat0_vector,\n",
    "                                 label_cat1_index,#label_cat1_vector,\n",
    "                                 label_cat2_index,#label_cat2_vector,\n",
    "                                 label_cat3_index,#label_cat3_vector,\n",
    "                                 label_cat4_index])#,label_cat4_vector])\n",
    "data_wLabels = label_pipeline.fit(data_wLabels).transform(data_wLabels)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0391d85b",
   "metadata": {},
   "source": [
    "## Test predictions by fragment data alone"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "44a4af93",
   "metadata": {},
   "outputs": [],
   "source": [
    "test_subset = data_wLabels.select(['Name','BA_pct',\n",
    "                                   'label1','label2','label3a','label3b',\n",
    "                                   'label_QD5',\n",
    "                                   'label_cat1_index','label_cat2_index','label_cat3_index','label_cat4_index',\n",
    "                                   'frags_all','frags_subset','frags_subset2','frags_efgs','frags_brics'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "e2d9d01c",
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyspark.ml.feature import NGram,Word2Vec,CountVectorizer,HashingTF,IDF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "ba91a8b3",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    }
   ],
   "source": [
    "''' # Fragments NLP processing  - NEWER -\n",
    "'''\n",
    "\n",
    "from pyspark.ml.feature import NGram,Word2Vec,CountVectorizer,IDF\n",
    "\n",
    "fragment_types = ['frags_all','frags_subset','frags_subset2','frags_efgs','frags_brics']\n",
    "                  \n",
    "fragment_shortname = ['f_all','f_subset','f_subset2','f_efgs','f_brics']\n",
    "\n",
    "test_subset_nlp = test_subset\n",
    "for i,frag_type in enumerate(fragment_types):\n",
    "    \n",
    "    frag_type_short = fragment_shortname[i]\n",
    "    \n",
    "    cv2 = CountVectorizer(inputCol=frag_type, outputCol=f\"{frag_type_short}_cv2\", minDF=2.0)\n",
    "    cv2_idf = IDF(inputCol=f\"{frag_type_short}_cv2\", outputCol=f\"{frag_type_short}_cv2_idf\")\n",
    "    \n",
    "    cv5 = CountVectorizer(inputCol=frag_type, outputCol=f\"{frag_type_short}_cv5\")\n",
    "    \n",
    "    w2v = Word2Vec(inputCol=frag_type, outputCol=f\"{frag_type_short}_w2v\")\n",
    "    \n",
    "    n2gram = NGram(n=2, inputCol=frag_type, outputCol=f\"{frag_type_short}_n2g\")\n",
    "    n2gram_cv2 = CountVectorizer(inputCol=f\"{frag_type_short}_n2g\", outputCol=f\"{frag_type_short}_n2g_cv2\", minDF=2.0)\n",
    "    \n",
    "    n2gram_cv5 = CountVectorizer(inputCol=f\"{frag_type_short}_n2g\", outputCol=f\"{frag_type_short}_n2g_cv5\")\n",
    "    \n",
    "    nlp_pipeline = Pipeline(stages=[cv2, cv2_idf, \n",
    "                                    cv5, w2v,\n",
    "                                    n2gram, n2gram_cv2, n2gram_cv5])\n",
    "    \n",
    "    test_subset_nlp = nlp_pipeline.fit(test_subset_nlp).transform(test_subset_nlp)\n",
    "    \n",
    "    column_to_drop = f\"{frag_type_short}_n2g\"\n",
    "    test_subset_nlp = test_subset_nlp.drop(column_to_drop)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "3b02f0ab",
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Name',\n",
       " 'BA_pct',\n",
       " 'label1',\n",
       " 'label2',\n",
       " 'label3a',\n",
       " 'label3b',\n",
       " 'label_QD5',\n",
       " 'label_cat1_index',\n",
       " 'label_cat2_index',\n",
       " 'label_cat3_index',\n",
       " 'label_cat4_index',\n",
       " 'frags_all',\n",
       " 'frags_subset',\n",
       " 'frags_subset2',\n",
       " 'frags_efgs',\n",
       " 'frags_brics',\n",
       " 'f_all_cv2',\n",
       " 'f_all_cv2_idf',\n",
       " 'f_all_cv5',\n",
       " 'f_all_w2v',\n",
       " 'f_all_n2g_cv2',\n",
       " 'f_all_n2g_cv5',\n",
       " 'f_subset_cv2',\n",
       " 'f_subset_cv2_idf',\n",
       " 'f_subset_cv5',\n",
       " 'f_subset_w2v',\n",
       " 'f_subset_n2g_cv2',\n",
       " 'f_subset_n2g_cv5',\n",
       " 'f_subset2_cv2',\n",
       " 'f_subset2_cv2_idf',\n",
       " 'f_subset2_cv5',\n",
       " 'f_subset2_w2v',\n",
       " 'f_subset2_n2g_cv2',\n",
       " 'f_subset2_n2g_cv5',\n",
       " 'f_efgs_cv2',\n",
       " 'f_efgs_cv2_idf',\n",
       " 'f_efgs_cv5',\n",
       " 'f_efgs_w2v',\n",
       " 'f_efgs_n2g_cv2',\n",
       " 'f_efgs_n2g_cv5',\n",
       " 'f_brics_cv2',\n",
       " 'f_brics_cv2_idf',\n",
       " 'f_brics_cv5',\n",
       " 'f_brics_w2v',\n",
       " 'f_brics_n2g_cv2',\n",
       " 'f_brics_n2g_cv5']"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_subset_nlp.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "456efdda",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "'FEAT_f_all_cv2', 'FEAT_f_all_cv2_idf', 'FEAT_f_all_cv5', 'FEAT_f_all_w2v', 'FEAT_f_all_n2g_cv2', 'FEAT_f_all_n2g_cv5', 'FEAT_f_subset_cv2', 'FEAT_f_subset_cv2_idf', 'FEAT_f_subset_cv5', 'FEAT_f_subset_w2v', 'FEAT_f_subset_n2g_cv2', 'FEAT_f_subset_n2g_cv5', 'FEAT_f_subset2_cv2', 'FEAT_f_subset2_cv2_idf', 'FEAT_f_subset2_cv5', 'FEAT_f_subset2_w2v', 'FEAT_f_subset2_n2g_cv2', 'FEAT_f_subset2_n2g_cv5', 'FEAT_f_efgs_cv2', 'FEAT_f_efgs_cv2_idf', 'FEAT_f_efgs_cv5', 'FEAT_f_efgs_w2v', 'FEAT_f_efgs_n2g_cv2', 'FEAT_f_efgs_n2g_cv5', 'FEAT_f_brics_cv2', 'FEAT_f_brics_cv2_idf', 'FEAT_f_brics_cv5', 'FEAT_f_brics_w2v', 'FEAT_f_brics_n2g_cv2', 'FEAT_f_brics_n2g_cv5'\n"
     ]
    }
   ],
   "source": [
    "test_subset_nlp = test_subset_nlp.select(['Name','BA_pct',\n",
    "                                          'label_cat1_index','label_cat2_index','label_cat3_index','label_cat4_index','label_QD5',\n",
    "                                         \n",
    "                                          'f_all_cv2','f_all_cv2_idf','f_all_cv5','f_all_w2v',\n",
    "                                          'f_all_n2g_cv2','f_all_n2g_cv5',\n",
    "                                          \n",
    "                                          'f_subset_cv2','f_subset_cv2_idf','f_subset_cv5','f_subset_w2v',\n",
    "                                          'f_subset_n2g_cv2','f_subset_n2g_cv5',\n",
    "                                          \n",
    "                                          'f_subset2_cv2','f_subset2_cv2_idf','f_subset2_cv5','f_subset2_w2v',\n",
    "                                          'f_subset2_n2g_cv2','f_subset2_n2g_cv5',\n",
    "                                          \n",
    "                                          'f_efgs_cv2','f_efgs_cv2_idf','f_efgs_cv5','f_efgs_w2v',\n",
    "                                          'f_efgs_n2g_cv2','f_efgs_n2g_cv5',\n",
    "                                          \n",
    "                                          'f_brics_cv2','f_brics_cv2_idf','f_brics_cv5','f_brics_w2v',\n",
    "                                          'f_brics_n2g_cv2','f_brics_n2g_cv5'])\n",
    "\n",
    "from pyspark.ml.linalg import Vector\n",
    "from pyspark.ml.feature import (VectorAssembler,VectorIndexer)\n",
    "\n",
    "vector_assemblers = []\n",
    "\n",
    "alternative_features = ['f_all_cv2','f_all_cv2_idf','f_all_cv5','f_all_w2v',\n",
    "                        'f_all_n2g_cv2','f_all_n2g_cv5',\n",
    "                        \n",
    "                        'f_subset_cv2','f_subset_cv2_idf','f_subset_cv5','f_subset_w2v',\n",
    "                        'f_subset_n2g_cv2','f_subset_n2g_cv5',\n",
    "                        \n",
    "                        'f_subset2_cv2','f_subset2_cv2_idf','f_subset2_cv5','f_subset2_w2v',\n",
    "                        'f_subset2_n2g_cv2','f_subset2_n2g_cv5',\n",
    "                        \n",
    "                        'f_efgs_cv2','f_efgs_cv2_idf','f_efgs_cv5','f_efgs_w2v',\n",
    "                        'f_efgs_n2g_cv2','f_efgs_n2g_cv5',\n",
    "                        \n",
    "                        'f_brics_cv2','f_brics_cv2_idf','f_brics_cv5','f_brics_w2v',\n",
    "                        'f_brics_n2g_cv2','f_brics_n2g_cv5']\n",
    "output_features = \"\"\n",
    "for feats in alternative_features:\n",
    "    #feats_input = ['MolWt','MolLogP','TPSA',feats]\n",
    "    feats_input = [feats]\n",
    "    feats_output = f\"FEAT_{feats}\"\n",
    "    \n",
    "    vec_assembler = VectorAssembler(inputCols=feats_input, outputCol=feats_output)\n",
    "    \n",
    "    vector_assemblers.append(vec_assembler)\n",
    "    \n",
    "    output_features += \"'\"+feats_output+\"'\"+\", \"\n",
    "output_features = output_features[0:len(output_features)-2]\n",
    "print(output_features)\n",
    "\n",
    "from pyspark.ml import Pipeline\n",
    "feature_pipeline = Pipeline(stages=[x for x in vector_assemblers])\n",
    "\n",
    "test_subset_nlpFeatures = feature_pipeline.fit(test_subset_nlp).transform(test_subset_nlp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "816a2d91",
   "metadata": {},
   "outputs": [],
   "source": [
    "test_subset_final = test_subset_nlpFeatures.select(['Name','BA_pct',\n",
    "                                                   'FEAT_f_all_cv2', 'FEAT_f_all_cv2_idf', 'FEAT_f_all_cv5', \n",
    "                                                    'FEAT_f_all_w2v', 'FEAT_f_all_n2g_cv2', 'FEAT_f_all_n2g_cv5',\n",
    "                                                    'FEAT_f_subset_cv2', 'FEAT_f_subset_cv2_idf', 'FEAT_f_subset_cv5',\n",
    "                                                    'FEAT_f_subset_w2v', 'FEAT_f_subset_n2g_cv2', 'FEAT_f_subset_n2g_cv5',\n",
    "                                                    'FEAT_f_subset2_cv2', 'FEAT_f_subset2_cv2_idf', 'FEAT_f_subset2_cv5',\n",
    "                                                    'FEAT_f_subset2_w2v', 'FEAT_f_subset2_n2g_cv2', 'FEAT_f_subset2_n2g_cv5',\n",
    "                                                    'FEAT_f_efgs_cv2', 'FEAT_f_efgs_cv2_idf', 'FEAT_f_efgs_cv5', \n",
    "                                                    'FEAT_f_efgs_w2v', 'FEAT_f_efgs_n2g_cv2', 'FEAT_f_efgs_n2g_cv5', \n",
    "                                                    'FEAT_f_brics_cv2', 'FEAT_f_brics_cv2_idf', 'FEAT_f_brics_cv5', \n",
    "                                                    'FEAT_f_brics_w2v', 'FEAT_f_brics_n2g_cv2', 'FEAT_f_brics_n2g_cv5'])\n",
    "\n",
    "(training,testing) = test_subset_final.randomSplit([0.7,0.3])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "472d310c",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING: An illegal reflective access operation has occurred                    \n",
      "WARNING: Illegal reflective access by org.apache.spark.util.SizeEstimator$ (file:/home/mitch/spark-3.3.0-bin-hadoop2/jars/spark-core_2.12-3.3.0.jar) to field java.nio.charset.Charset.name\n",
      "WARNING: Please consider reporting this to the maintainers of org.apache.spark.util.SizeEstimator$\n",
      "WARNING: Use --illegal-access=warn to enable warnings of further illegal reflective access operations\n",
      "WARNING: All illegal access operations will be denied in a future release\n",
      "                                                                                \r"
     ]
    }
   ],
   "source": [
    "allFeatures =  ['FEAT_f_all_cv2', 'FEAT_f_all_cv2_idf', 'FEAT_f_all_cv5', \n",
    "                'FEAT_f_all_w2v', 'FEAT_f_all_n2g_cv2', 'FEAT_f_all_n2g_cv5',\n",
    "                'FEAT_f_subset_cv2', 'FEAT_f_subset_cv2_idf', 'FEAT_f_subset_cv5',\n",
    "                'FEAT_f_subset_w2v', 'FEAT_f_subset_n2g_cv2', 'FEAT_f_subset_n2g_cv5',\n",
    "                'FEAT_f_subset2_cv2', 'FEAT_f_subset2_cv2_idf', 'FEAT_f_subset2_cv5',\n",
    "                'FEAT_f_subset2_w2v', 'FEAT_f_subset2_n2g_cv2', 'FEAT_f_subset2_n2g_cv5',\n",
    "                'FEAT_f_efgs_cv2', 'FEAT_f_efgs_cv2_idf', 'FEAT_f_efgs_cv5', \n",
    "                'FEAT_f_efgs_w2v', 'FEAT_f_efgs_n2g_cv2', 'FEAT_f_efgs_n2g_cv5', \n",
    "                'FEAT_f_brics_cv2', 'FEAT_f_brics_cv2_idf', 'FEAT_f_brics_cv5', \n",
    "                'FEAT_f_brics_w2v', 'FEAT_f_brics_n2g_cv2', 'FEAT_f_brics_n2g_cv5']\n",
    "featuresName = '' # temp value, redefined below\n",
    "\n",
    "eval_df = pd.DataFrame() # already exists from prior run\n",
    "labelName = 'BA_pct'  # SPECIFY\n",
    "\n",
    "modelname_short = 'rfr'\n",
    "iteration = 0\n",
    "\n",
    "\n",
    "for index,features in enumerate(allFeatures):\n",
    "    featuresName = features\n",
    "    \n",
    "    '''# SPECIFY MODEL \n",
    "    '''\n",
    "    from pyspark.ml.regression import RandomForestRegressor\n",
    "    rfr = RandomForestRegressor(featuresCol=features,labelCol='BA_pct')\n",
    "    modeltype = rfr  # SPECIFY (lr,dtr,rfr,gbtr,glr,ir)\n",
    "    \n",
    "    \n",
    "    modelname = f\"rfr{iteration}_{allFeatures[index]}\"\n",
    "    modelname = modelname.replace('FEAT_f_','')\n",
    "    \n",
    "    #evaluation_history[modelname_short][iteration][features] = {}\n",
    "    \n",
    "    # FIT/TRAIN MODEL & TRANSFORM DATA\n",
    "    mymodel = modeltype.fit(training)\n",
    "    myresults = mymodel.transform(testing)\n",
    "    \n",
    "    # CALCULATE KEY EVALS\n",
    "    from pyspark.ml.evaluation import RegressionEvaluator\n",
    "    regEvaluator = RegressionEvaluator(labelCol=labelName,predictionCol='prediction')\n",
    "\n",
    "    evaluator = regEvaluator\n",
    "    evalMetrics = {regEvaluator:['rmse','mse','mae','r2','var']}\n",
    "    \n",
    "    evaluation = []\n",
    "    \n",
    "    for each_metric in evalMetrics[evaluator]:        \n",
    "        metric = each_metric\n",
    "\n",
    "        result = evaluator.evaluate(myresults, {evaluator.metricName: metric})\n",
    "\n",
    "        evaluation.append((metric,result))\n",
    "        \n",
    "        #evaluation_history[modelname_short][iteration][features][metric] = result\n",
    "        \n",
    "    #r2_adj = mymodel.summary.r2adj\n",
    "    #evaluation.append(('r2_adj(Training)',r2_adj))\n",
    "    column0 = [x for x,y in evaluation]\n",
    "    column1 = [y for x,y in evaluation]\n",
    "    eval_df['metric'] = column0\n",
    "    eval_df[modelname] = column1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "be2b32d4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>metric</th>\n",
       "      <th>rfr0_all_cv2</th>\n",
       "      <th>rfr0_all_cv2_idf</th>\n",
       "      <th>rfr0_all_cv5</th>\n",
       "      <th>rfr0_all_w2v</th>\n",
       "      <th>rfr0_all_n2g_cv2</th>\n",
       "      <th>rfr0_all_n2g_cv5</th>\n",
       "      <th>rfr0_subset_cv2</th>\n",
       "      <th>rfr0_subset_cv2_idf</th>\n",
       "      <th>rfr0_subset_cv5</th>\n",
       "      <th>rfr0_subset_w2v</th>\n",
       "      <th>rfr0_subset_n2g_cv2</th>\n",
       "      <th>rfr0_subset_n2g_cv5</th>\n",
       "      <th>rfr0_subset2_cv2</th>\n",
       "      <th>rfr0_subset2_cv2_idf</th>\n",
       "      <th>rfr0_subset2_cv5</th>\n",
       "      <th>rfr0_subset2_w2v</th>\n",
       "      <th>rfr0_subset2_n2g_cv2</th>\n",
       "      <th>rfr0_subset2_n2g_cv5</th>\n",
       "      <th>rfr0_efgs_cv2</th>\n",
       "      <th>rfr0_efgs_cv2_idf</th>\n",
       "      <th>rfr0_efgs_cv5</th>\n",
       "      <th>rfr0_efgs_w2v</th>\n",
       "      <th>rfr0_efgs_n2g_cv2</th>\n",
       "      <th>rfr0_efgs_n2g_cv5</th>\n",
       "      <th>rfr0_brics_cv2</th>\n",
       "      <th>rfr0_brics_cv2_idf</th>\n",
       "      <th>rfr0_brics_cv5</th>\n",
       "      <th>rfr0_brics_w2v</th>\n",
       "      <th>rfr0_brics_n2g_cv2</th>\n",
       "      <th>rfr0_brics_n2g_cv5</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>rmse</td>\n",
       "      <td>30.320921</td>\n",
       "      <td>30.320921</td>\n",
       "      <td>30.391067</td>\n",
       "      <td>31.083407</td>\n",
       "      <td>30.280319</td>\n",
       "      <td>30.601199</td>\n",
       "      <td>30.213488</td>\n",
       "      <td>30.213488</td>\n",
       "      <td>30.245852</td>\n",
       "      <td>31.439973</td>\n",
       "      <td>30.354182</td>\n",
       "      <td>30.531797</td>\n",
       "      <td>30.460687</td>\n",
       "      <td>30.460687</td>\n",
       "      <td>30.173174</td>\n",
       "      <td>31.525836</td>\n",
       "      <td>30.497086</td>\n",
       "      <td>30.629666</td>\n",
       "      <td>30.738370</td>\n",
       "      <td>30.738370</td>\n",
       "      <td>30.788090</td>\n",
       "      <td>30.606959</td>\n",
       "      <td>31.620247</td>\n",
       "      <td>31.521389</td>\n",
       "      <td>31.396090</td>\n",
       "      <td>31.396090</td>\n",
       "      <td>31.594195</td>\n",
       "      <td>31.553144</td>\n",
       "      <td>32.313564</td>\n",
       "      <td>32.368579</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>mse</td>\n",
       "      <td>919.358221</td>\n",
       "      <td>919.358221</td>\n",
       "      <td>923.616958</td>\n",
       "      <td>966.178188</td>\n",
       "      <td>916.897711</td>\n",
       "      <td>936.433398</td>\n",
       "      <td>912.854853</td>\n",
       "      <td>912.854853</td>\n",
       "      <td>914.811545</td>\n",
       "      <td>988.471889</td>\n",
       "      <td>921.376373</td>\n",
       "      <td>932.190626</td>\n",
       "      <td>927.853449</td>\n",
       "      <td>927.853449</td>\n",
       "      <td>910.420429</td>\n",
       "      <td>993.878353</td>\n",
       "      <td>930.072284</td>\n",
       "      <td>938.176411</td>\n",
       "      <td>944.847368</td>\n",
       "      <td>944.847368</td>\n",
       "      <td>947.906484</td>\n",
       "      <td>936.785948</td>\n",
       "      <td>999.840037</td>\n",
       "      <td>993.597935</td>\n",
       "      <td>985.714457</td>\n",
       "      <td>985.714457</td>\n",
       "      <td>998.193148</td>\n",
       "      <td>995.600903</td>\n",
       "      <td>1044.166434</td>\n",
       "      <td>1047.724920</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>mae</td>\n",
       "      <td>26.106168</td>\n",
       "      <td>26.106168</td>\n",
       "      <td>26.208338</td>\n",
       "      <td>26.839702</td>\n",
       "      <td>26.104657</td>\n",
       "      <td>26.335899</td>\n",
       "      <td>25.951259</td>\n",
       "      <td>25.951259</td>\n",
       "      <td>26.078349</td>\n",
       "      <td>27.047467</td>\n",
       "      <td>26.076333</td>\n",
       "      <td>26.279762</td>\n",
       "      <td>26.253976</td>\n",
       "      <td>26.253976</td>\n",
       "      <td>26.015198</td>\n",
       "      <td>27.372333</td>\n",
       "      <td>26.207888</td>\n",
       "      <td>26.478834</td>\n",
       "      <td>26.461278</td>\n",
       "      <td>26.461278</td>\n",
       "      <td>26.571943</td>\n",
       "      <td>26.573573</td>\n",
       "      <td>27.274667</td>\n",
       "      <td>27.283648</td>\n",
       "      <td>27.244932</td>\n",
       "      <td>27.244932</td>\n",
       "      <td>27.335514</td>\n",
       "      <td>26.949939</td>\n",
       "      <td>27.803498</td>\n",
       "      <td>27.889929</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>r2</td>\n",
       "      <td>0.140859</td>\n",
       "      <td>0.140859</td>\n",
       "      <td>0.136880</td>\n",
       "      <td>0.097106</td>\n",
       "      <td>0.143159</td>\n",
       "      <td>0.124903</td>\n",
       "      <td>0.146937</td>\n",
       "      <td>0.146937</td>\n",
       "      <td>0.145108</td>\n",
       "      <td>0.076273</td>\n",
       "      <td>0.138973</td>\n",
       "      <td>0.128868</td>\n",
       "      <td>0.132921</td>\n",
       "      <td>0.132921</td>\n",
       "      <td>0.149212</td>\n",
       "      <td>0.071220</td>\n",
       "      <td>0.130847</td>\n",
       "      <td>0.123274</td>\n",
       "      <td>0.117040</td>\n",
       "      <td>0.117040</td>\n",
       "      <td>0.114181</td>\n",
       "      <td>0.124573</td>\n",
       "      <td>0.065649</td>\n",
       "      <td>0.071482</td>\n",
       "      <td>0.078850</td>\n",
       "      <td>0.078850</td>\n",
       "      <td>0.067188</td>\n",
       "      <td>0.069611</td>\n",
       "      <td>0.024226</td>\n",
       "      <td>0.020901</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>var</td>\n",
       "      <td>124.318748</td>\n",
       "      <td>124.318748</td>\n",
       "      <td>119.974605</td>\n",
       "      <td>85.785854</td>\n",
       "      <td>116.108132</td>\n",
       "      <td>105.615612</td>\n",
       "      <td>125.334503</td>\n",
       "      <td>125.334503</td>\n",
       "      <td>118.208406</td>\n",
       "      <td>95.195071</td>\n",
       "      <td>121.775725</td>\n",
       "      <td>119.702673</td>\n",
       "      <td>117.628682</td>\n",
       "      <td>117.628682</td>\n",
       "      <td>121.584871</td>\n",
       "      <td>102.727707</td>\n",
       "      <td>128.229044</td>\n",
       "      <td>107.647532</td>\n",
       "      <td>57.294891</td>\n",
       "      <td>57.294891</td>\n",
       "      <td>55.560586</td>\n",
       "      <td>103.232881</td>\n",
       "      <td>27.579935</td>\n",
       "      <td>28.728255</td>\n",
       "      <td>37.872567</td>\n",
       "      <td>37.872567</td>\n",
       "      <td>33.557996</td>\n",
       "      <td>78.885398</td>\n",
       "      <td>24.904375</td>\n",
       "      <td>22.465992</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  metric  rfr0_all_cv2  rfr0_all_cv2_idf  rfr0_all_cv5  rfr0_all_w2v  \\\n",
       "0   rmse     30.320921         30.320921     30.391067     31.083407   \n",
       "1    mse    919.358221        919.358221    923.616958    966.178188   \n",
       "2    mae     26.106168         26.106168     26.208338     26.839702   \n",
       "3     r2      0.140859          0.140859      0.136880      0.097106   \n",
       "4    var    124.318748        124.318748    119.974605     85.785854   \n",
       "\n",
       "   rfr0_all_n2g_cv2  rfr0_all_n2g_cv5  rfr0_subset_cv2  rfr0_subset_cv2_idf  \\\n",
       "0         30.280319         30.601199        30.213488            30.213488   \n",
       "1        916.897711        936.433398       912.854853           912.854853   \n",
       "2         26.104657         26.335899        25.951259            25.951259   \n",
       "3          0.143159          0.124903         0.146937             0.146937   \n",
       "4        116.108132        105.615612       125.334503           125.334503   \n",
       "\n",
       "   rfr0_subset_cv5  rfr0_subset_w2v  rfr0_subset_n2g_cv2  rfr0_subset_n2g_cv5  \\\n",
       "0        30.245852        31.439973            30.354182            30.531797   \n",
       "1       914.811545       988.471889           921.376373           932.190626   \n",
       "2        26.078349        27.047467            26.076333            26.279762   \n",
       "3         0.145108         0.076273             0.138973             0.128868   \n",
       "4       118.208406        95.195071           121.775725           119.702673   \n",
       "\n",
       "   rfr0_subset2_cv2  rfr0_subset2_cv2_idf  rfr0_subset2_cv5  rfr0_subset2_w2v  \\\n",
       "0         30.460687             30.460687         30.173174         31.525836   \n",
       "1        927.853449            927.853449        910.420429        993.878353   \n",
       "2         26.253976             26.253976         26.015198         27.372333   \n",
       "3          0.132921              0.132921          0.149212          0.071220   \n",
       "4        117.628682            117.628682        121.584871        102.727707   \n",
       "\n",
       "   rfr0_subset2_n2g_cv2  rfr0_subset2_n2g_cv5  rfr0_efgs_cv2  \\\n",
       "0             30.497086             30.629666      30.738370   \n",
       "1            930.072284            938.176411     944.847368   \n",
       "2             26.207888             26.478834      26.461278   \n",
       "3              0.130847              0.123274       0.117040   \n",
       "4            128.229044            107.647532      57.294891   \n",
       "\n",
       "   rfr0_efgs_cv2_idf  rfr0_efgs_cv5  rfr0_efgs_w2v  rfr0_efgs_n2g_cv2  \\\n",
       "0          30.738370      30.788090      30.606959          31.620247   \n",
       "1         944.847368     947.906484     936.785948         999.840037   \n",
       "2          26.461278      26.571943      26.573573          27.274667   \n",
       "3           0.117040       0.114181       0.124573           0.065649   \n",
       "4          57.294891      55.560586     103.232881          27.579935   \n",
       "\n",
       "   rfr0_efgs_n2g_cv5  rfr0_brics_cv2  rfr0_brics_cv2_idf  rfr0_brics_cv5  \\\n",
       "0          31.521389       31.396090           31.396090       31.594195   \n",
       "1         993.597935      985.714457          985.714457      998.193148   \n",
       "2          27.283648       27.244932           27.244932       27.335514   \n",
       "3           0.071482        0.078850            0.078850        0.067188   \n",
       "4          28.728255       37.872567           37.872567       33.557996   \n",
       "\n",
       "   rfr0_brics_w2v  rfr0_brics_n2g_cv2  rfr0_brics_n2g_cv5  \n",
       "0       31.553144           32.313564           32.368579  \n",
       "1      995.600903         1044.166434         1047.724920  \n",
       "2       26.949939           27.803498           27.889929  \n",
       "3        0.069611            0.024226            0.020901  \n",
       "4       78.885398           24.904375           22.465992  "
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# iteration 4\n",
    "pd.set_option('display.max_columns',None)\n",
    "eval_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "0e2b09fd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>metric</th>\n",
       "      <th>index</th>\n",
       "      <th>rmse</th>\n",
       "      <th>mse</th>\n",
       "      <th>mae</th>\n",
       "      <th>r2</th>\n",
       "      <th>var</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>rfr0_subset2_cv5</td>\n",
       "      <td>30.173174</td>\n",
       "      <td>910.420429</td>\n",
       "      <td>26.015198</td>\n",
       "      <td>0.149212</td>\n",
       "      <td>121.584871</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>rfr0_subset_cv2_idf</td>\n",
       "      <td>30.213488</td>\n",
       "      <td>912.854853</td>\n",
       "      <td>25.951259</td>\n",
       "      <td>0.146937</td>\n",
       "      <td>125.334503</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>rfr0_subset_cv2</td>\n",
       "      <td>30.213488</td>\n",
       "      <td>912.854853</td>\n",
       "      <td>25.951259</td>\n",
       "      <td>0.146937</td>\n",
       "      <td>125.334503</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>rfr0_subset_cv5</td>\n",
       "      <td>30.245852</td>\n",
       "      <td>914.811545</td>\n",
       "      <td>26.078349</td>\n",
       "      <td>0.145108</td>\n",
       "      <td>118.208406</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>rfr0_all_n2g_cv2</td>\n",
       "      <td>30.280319</td>\n",
       "      <td>916.897711</td>\n",
       "      <td>26.104657</td>\n",
       "      <td>0.143159</td>\n",
       "      <td>116.108132</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>rfr0_all_cv2</td>\n",
       "      <td>30.320921</td>\n",
       "      <td>919.358221</td>\n",
       "      <td>26.106168</td>\n",
       "      <td>0.140859</td>\n",
       "      <td>124.318748</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>rfr0_all_cv2_idf</td>\n",
       "      <td>30.320921</td>\n",
       "      <td>919.358221</td>\n",
       "      <td>26.106168</td>\n",
       "      <td>0.140859</td>\n",
       "      <td>124.318748</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>rfr0_subset_n2g_cv2</td>\n",
       "      <td>30.354182</td>\n",
       "      <td>921.376373</td>\n",
       "      <td>26.076333</td>\n",
       "      <td>0.138973</td>\n",
       "      <td>121.775725</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>rfr0_all_cv5</td>\n",
       "      <td>30.391067</td>\n",
       "      <td>923.616958</td>\n",
       "      <td>26.208338</td>\n",
       "      <td>0.136880</td>\n",
       "      <td>119.974605</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>rfr0_subset2_cv2_idf</td>\n",
       "      <td>30.460687</td>\n",
       "      <td>927.853449</td>\n",
       "      <td>26.253976</td>\n",
       "      <td>0.132921</td>\n",
       "      <td>117.628682</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>rfr0_subset2_cv2</td>\n",
       "      <td>30.460687</td>\n",
       "      <td>927.853449</td>\n",
       "      <td>26.253976</td>\n",
       "      <td>0.132921</td>\n",
       "      <td>117.628682</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>rfr0_subset2_n2g_cv2</td>\n",
       "      <td>30.497086</td>\n",
       "      <td>930.072284</td>\n",
       "      <td>26.207888</td>\n",
       "      <td>0.130847</td>\n",
       "      <td>128.229044</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>rfr0_subset_n2g_cv5</td>\n",
       "      <td>30.531797</td>\n",
       "      <td>932.190626</td>\n",
       "      <td>26.279762</td>\n",
       "      <td>0.128868</td>\n",
       "      <td>119.702673</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>rfr0_all_n2g_cv5</td>\n",
       "      <td>30.601199</td>\n",
       "      <td>936.433398</td>\n",
       "      <td>26.335899</td>\n",
       "      <td>0.124903</td>\n",
       "      <td>105.615612</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>rfr0_efgs_w2v</td>\n",
       "      <td>30.606959</td>\n",
       "      <td>936.785948</td>\n",
       "      <td>26.573573</td>\n",
       "      <td>0.124573</td>\n",
       "      <td>103.232881</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>rfr0_subset2_n2g_cv5</td>\n",
       "      <td>30.629666</td>\n",
       "      <td>938.176411</td>\n",
       "      <td>26.478834</td>\n",
       "      <td>0.123274</td>\n",
       "      <td>107.647532</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>rfr0_efgs_cv2_idf</td>\n",
       "      <td>30.738370</td>\n",
       "      <td>944.847368</td>\n",
       "      <td>26.461278</td>\n",
       "      <td>0.117040</td>\n",
       "      <td>57.294891</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>rfr0_efgs_cv2</td>\n",
       "      <td>30.738370</td>\n",
       "      <td>944.847368</td>\n",
       "      <td>26.461278</td>\n",
       "      <td>0.117040</td>\n",
       "      <td>57.294891</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>rfr0_efgs_cv5</td>\n",
       "      <td>30.788090</td>\n",
       "      <td>947.906484</td>\n",
       "      <td>26.571943</td>\n",
       "      <td>0.114181</td>\n",
       "      <td>55.560586</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>rfr0_all_w2v</td>\n",
       "      <td>31.083407</td>\n",
       "      <td>966.178188</td>\n",
       "      <td>26.839702</td>\n",
       "      <td>0.097106</td>\n",
       "      <td>85.785854</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>rfr0_brics_cv2</td>\n",
       "      <td>31.396090</td>\n",
       "      <td>985.714457</td>\n",
       "      <td>27.244932</td>\n",
       "      <td>0.078850</td>\n",
       "      <td>37.872567</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>rfr0_brics_cv2_idf</td>\n",
       "      <td>31.396090</td>\n",
       "      <td>985.714457</td>\n",
       "      <td>27.244932</td>\n",
       "      <td>0.078850</td>\n",
       "      <td>37.872567</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>rfr0_subset_w2v</td>\n",
       "      <td>31.439973</td>\n",
       "      <td>988.471889</td>\n",
       "      <td>27.047467</td>\n",
       "      <td>0.076273</td>\n",
       "      <td>95.195071</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>rfr0_efgs_n2g_cv5</td>\n",
       "      <td>31.521389</td>\n",
       "      <td>993.597935</td>\n",
       "      <td>27.283648</td>\n",
       "      <td>0.071482</td>\n",
       "      <td>28.728255</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>rfr0_subset2_w2v</td>\n",
       "      <td>31.525836</td>\n",
       "      <td>993.878353</td>\n",
       "      <td>27.372333</td>\n",
       "      <td>0.071220</td>\n",
       "      <td>102.727707</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>rfr0_brics_w2v</td>\n",
       "      <td>31.553144</td>\n",
       "      <td>995.600903</td>\n",
       "      <td>26.949939</td>\n",
       "      <td>0.069611</td>\n",
       "      <td>78.885398</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>rfr0_brics_cv5</td>\n",
       "      <td>31.594195</td>\n",
       "      <td>998.193148</td>\n",
       "      <td>27.335514</td>\n",
       "      <td>0.067188</td>\n",
       "      <td>33.557996</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>rfr0_efgs_n2g_cv2</td>\n",
       "      <td>31.620247</td>\n",
       "      <td>999.840037</td>\n",
       "      <td>27.274667</td>\n",
       "      <td>0.065649</td>\n",
       "      <td>27.579935</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>rfr0_brics_n2g_cv2</td>\n",
       "      <td>32.313564</td>\n",
       "      <td>1044.166434</td>\n",
       "      <td>27.803498</td>\n",
       "      <td>0.024226</td>\n",
       "      <td>24.904375</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>rfr0_brics_n2g_cv5</td>\n",
       "      <td>32.368579</td>\n",
       "      <td>1047.724920</td>\n",
       "      <td>27.889929</td>\n",
       "      <td>0.020901</td>\n",
       "      <td>22.465992</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "metric                 index       rmse          mse        mae        r2  \\\n",
       "0           rfr0_subset2_cv5  30.173174   910.420429  26.015198  0.149212   \n",
       "1        rfr0_subset_cv2_idf  30.213488   912.854853  25.951259  0.146937   \n",
       "2            rfr0_subset_cv2  30.213488   912.854853  25.951259  0.146937   \n",
       "3            rfr0_subset_cv5  30.245852   914.811545  26.078349  0.145108   \n",
       "4           rfr0_all_n2g_cv2  30.280319   916.897711  26.104657  0.143159   \n",
       "5               rfr0_all_cv2  30.320921   919.358221  26.106168  0.140859   \n",
       "6           rfr0_all_cv2_idf  30.320921   919.358221  26.106168  0.140859   \n",
       "7        rfr0_subset_n2g_cv2  30.354182   921.376373  26.076333  0.138973   \n",
       "8               rfr0_all_cv5  30.391067   923.616958  26.208338  0.136880   \n",
       "9       rfr0_subset2_cv2_idf  30.460687   927.853449  26.253976  0.132921   \n",
       "10          rfr0_subset2_cv2  30.460687   927.853449  26.253976  0.132921   \n",
       "11      rfr0_subset2_n2g_cv2  30.497086   930.072284  26.207888  0.130847   \n",
       "12       rfr0_subset_n2g_cv5  30.531797   932.190626  26.279762  0.128868   \n",
       "13          rfr0_all_n2g_cv5  30.601199   936.433398  26.335899  0.124903   \n",
       "14             rfr0_efgs_w2v  30.606959   936.785948  26.573573  0.124573   \n",
       "15      rfr0_subset2_n2g_cv5  30.629666   938.176411  26.478834  0.123274   \n",
       "16         rfr0_efgs_cv2_idf  30.738370   944.847368  26.461278  0.117040   \n",
       "17             rfr0_efgs_cv2  30.738370   944.847368  26.461278  0.117040   \n",
       "18             rfr0_efgs_cv5  30.788090   947.906484  26.571943  0.114181   \n",
       "19              rfr0_all_w2v  31.083407   966.178188  26.839702  0.097106   \n",
       "20            rfr0_brics_cv2  31.396090   985.714457  27.244932  0.078850   \n",
       "21        rfr0_brics_cv2_idf  31.396090   985.714457  27.244932  0.078850   \n",
       "22           rfr0_subset_w2v  31.439973   988.471889  27.047467  0.076273   \n",
       "23         rfr0_efgs_n2g_cv5  31.521389   993.597935  27.283648  0.071482   \n",
       "24          rfr0_subset2_w2v  31.525836   993.878353  27.372333  0.071220   \n",
       "25            rfr0_brics_w2v  31.553144   995.600903  26.949939  0.069611   \n",
       "26            rfr0_brics_cv5  31.594195   998.193148  27.335514  0.067188   \n",
       "27         rfr0_efgs_n2g_cv2  31.620247   999.840037  27.274667  0.065649   \n",
       "28        rfr0_brics_n2g_cv2  32.313564  1044.166434  27.803498  0.024226   \n",
       "29        rfr0_brics_n2g_cv5  32.368579  1047.724920  27.889929  0.020901   \n",
       "\n",
       "metric         var  \n",
       "0       121.584871  \n",
       "1       125.334503  \n",
       "2       125.334503  \n",
       "3       118.208406  \n",
       "4       116.108132  \n",
       "5       124.318748  \n",
       "6       124.318748  \n",
       "7       121.775725  \n",
       "8       119.974605  \n",
       "9       117.628682  \n",
       "10      117.628682  \n",
       "11      128.229044  \n",
       "12      119.702673  \n",
       "13      105.615612  \n",
       "14      103.232881  \n",
       "15      107.647532  \n",
       "16       57.294891  \n",
       "17       57.294891  \n",
       "18       55.560586  \n",
       "19       85.785854  \n",
       "20       37.872567  \n",
       "21       37.872567  \n",
       "22       95.195071  \n",
       "23       28.728255  \n",
       "24      102.727707  \n",
       "25       78.885398  \n",
       "26       33.557996  \n",
       "27       27.579935  \n",
       "28       24.904375  \n",
       "29       22.465992  "
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# best features in terms of r2: \n",
    "testx = eval_df.set_index('metric')\n",
    "testx = testx.transpose()\n",
    "#testx.sort_values(by=['r2'],ascending=False)\n",
    "testx.sort_values(by=['r2'],ascending=False).reset_index()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "67c13dff",
   "metadata": {},
   "source": [
    "### observations\n",
    "* The top 5 positions were:\n",
    "  1. subset2 .. **cv5**\n",
    "  2. subset ... **cv2** / **cv2-idf**\n",
    "  3. subset ... **cv5** \n",
    "  4. all  ......... **n2g-cv2**\n",
    "  5. all  ......... **cv2** / **cv2-idf**\n",
    "\n",
    "* this shows that we can ignore **brics** and **efgs** fragment sets.\n",
    "* this also shows that we can disregard **w2v** and **n2g-cv5** NLP processing.\n",
    "\n",
    "* <font color='red'> Still unknown, however, is the effect of the train/test split. </font>\n",
    "  * we should average the _r2_ values over several train/test splits and find the top-performing set & NLP model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "05a14044",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'rmse'"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "eval_df.loc[0][0]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "680a6b9e",
   "metadata": {},
   "source": [
    "#### let's construct a dictionary of the previous trial's evaluation results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "7963a62c",
   "metadata": {},
   "outputs": [],
   "source": [
    "allFeatures =  ['FEAT_f_all_cv2', 'FEAT_f_all_cv5', 'FEAT_f_all_n2g_cv2', \n",
    "                'FEAT_f_subset_cv2', 'FEAT_f_subset_cv5','FEAT_f_subset_n2g_cv2',\n",
    "                'FEAT_f_subset2_cv2', 'FEAT_f_subset2_cv5', 'FEAT_f_subset2_n2g_cv2']\n",
    "\n",
    "metric_averages = {}\n",
    "\n",
    "for index,features in enumerate(allFeatures):\n",
    "\n",
    "    modelname = f\"rfr0_{allFeatures[index]}\"\n",
    "    modelname = modelname.replace('FEAT_f_','')\n",
    "\n",
    "    metric_averages[modelname] = {}\n",
    "    \n",
    "    for (metric,N) in [('rmse',0),('r2',3),('var',4)]:\n",
    "        metric_averages[modelname]['num_trials'] = 1\n",
    "        metric_averages[modelname][metric] = eval_df.loc[N][modelname]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "id": "fb57f515",
   "metadata": {},
   "outputs": [],
   "source": [
    "# manual entry of the metric_averages after the first trial run\n",
    "\n",
    "metric_averages = {'rfr0_all_cv2': {'num_trials': 1,\n",
    "                  'rmse': 30.32092051137974,\n",
    "                  'r2': 0.14085944821279428,\n",
    "                  'var': 124.31874808234403},\n",
    "                 'rfr0_all_cv5': {'num_trials': 1,\n",
    "                  'rmse': 30.391067075572746,\n",
    "                  'r2': 0.13687965680905978,\n",
    "                  'var': 119.97460462038228},\n",
    "                 'rfr0_all_n2g_cv2': {'num_trials': 1,\n",
    "                  'rmse': 30.280318872939436,\n",
    "                  'r2': 0.14315879523222386,\n",
    "                  'var': 116.10813226263414},\n",
    "                 'rfr0_subset_cv2': {'num_trials': 1,\n",
    "                  'rmse': 30.213487937176403,\n",
    "                  'r2': 0.14693684706417076,\n",
    "                  'var': 125.33450341337398},\n",
    "                 'rfr0_subset_cv5': {'num_trials': 1,\n",
    "                  'rmse': 30.245851692871423,\n",
    "                  'r2': 0.1451083184199059,\n",
    "                  'var': 118.20840619976119},\n",
    "                 'rfr0_subset_n2g_cv2': {'num_trials': 1,\n",
    "                  'rmse': 30.354182128097218,\n",
    "                  'r2': 0.1389734845143883,\n",
    "                  'var': 121.77572510571312},\n",
    "                 'rfr0_subset2_cv2': {'num_trials': 1,\n",
    "                  'rmse': 30.4606869491698,\n",
    "                  'r2': 0.13292065421646182,\n",
    "                  'var': 117.62868221829186},\n",
    "                 'rfr0_subset2_cv5': {'num_trials': 1,\n",
    "                  'rmse': 30.17317399527205,\n",
    "                  'r2': 0.14921181742820566,\n",
    "                  'var': 121.58487100619823},\n",
    "                 'rfr0_subset2_n2g_cv2': {'num_trials': 1,\n",
    "                  'rmse': 30.497086481216126,\n",
    "                  'r2': 0.13084715273304437,\n",
    "                  'var': 128.22904356766273}}"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "eaba13b1",
   "metadata": {},
   "source": [
    "#### now let's perform additional trials, _each with a new train/test split_, & update the average evaluation result each time\n",
    "#### <font color='red'> Each time the below cell runs, it performs another trial and updates the average value </font>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "id": "948b2ff6",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    }
   ],
   "source": [
    "allFeatures =  ['FEAT_f_all_cv2', 'FEAT_f_all_cv5', 'FEAT_f_all_n2g_cv2', \n",
    "                'FEAT_f_subset_cv2', 'FEAT_f_subset_cv5','FEAT_f_subset_n2g_cv2',\n",
    "                'FEAT_f_subset2_cv2', 'FEAT_f_subset2_cv5', 'FEAT_f_subset2_n2g_cv2']\n",
    "featuresName = '' # temp value, redefined below\n",
    "\n",
    "eval_df = pd.DataFrame()   # already exists from prior run\n",
    "labelName = 'BA_pct'  # SPECIFY\n",
    "\n",
    "modelname_short = 'rfr'\n",
    "iteration = 0\n",
    "\n",
    "#for i in [1,2,3]:\n",
    "(training,testing) = test_subset_final.randomSplit([0.7,0.3])\n",
    "#evaluation = []\n",
    "\n",
    "for index,features in enumerate(allFeatures):\n",
    "    featuresName = features\n",
    "\n",
    "    '''# SPECIFY MODEL \n",
    "    '''\n",
    "    from pyspark.ml.regression import RandomForestRegressor\n",
    "    rfr = RandomForestRegressor(featuresCol=features,labelCol='BA_pct')\n",
    "    modeltype = rfr  # SPECIFY (lr,dtr,rfr,gbtr,glr,ir)\n",
    "\n",
    "\n",
    "    modelname = f\"rfr{iteration}_{allFeatures[index]}\"\n",
    "    modelname = modelname.replace('FEAT_f_','')\n",
    "\n",
    "    #evaluation_history[modelname_short][iteration][features] = {}\n",
    "\n",
    "    # FIT/TRAIN MODEL & TRANSFORM DATA\n",
    "    mymodel = modeltype.fit(training)\n",
    "    myresults = mymodel.transform(testing)\n",
    "\n",
    "    # CALCULATE KEY EVALS\n",
    "    from pyspark.ml.evaluation import RegressionEvaluator\n",
    "    regEvaluator = RegressionEvaluator(labelCol=labelName,predictionCol='prediction')\n",
    "\n",
    "    evaluator = regEvaluator\n",
    "    evalMetrics = {regEvaluator:['rmse','r2','var']}\n",
    "    evaluation = []\n",
    "\n",
    "    for each_metric in evalMetrics[evaluator]:        \n",
    "        metric = each_metric\n",
    "\n",
    "        number_of_trials = metric_averages[modelname]['num_trials']\n",
    "        previous_average = metric_averages[modelname][metric]\n",
    "\n",
    "        result = evaluator.evaluate(myresults, {evaluator.metricName: metric})\n",
    "\n",
    "        new_average = ((number_of_trials*previous_average) + result)/(number_of_trials+1)\n",
    "\n",
    "        evaluation.append((metric,new_average))\n",
    "\n",
    "        metric_averages[modelname][metric] = new_average\n",
    "    \n",
    "    column0 = [x for x,y in evaluation]\n",
    "    column1 = [y for x,y in evaluation]\n",
    "    #column2 = [z for x,y,z in evaluation]\n",
    "    eval_df['metric'] = column0\n",
    "    eval_df[modelname] = column1\n",
    "    #eval_df['num_trials'] = column2\n",
    "\n",
    "metric_averages[modelname]['num_trials'] = (number_of_trials+1)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "990fec9e",
   "metadata": {},
   "source": [
    "#### finally, display the ranking of _average r2_ for each Feature/NLP combo after 4 trials"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "id": "b4700a6e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Updated ranking of average r2 after 3 trials\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>metric</th>\n",
       "      <th>index</th>\n",
       "      <th>rmse</th>\n",
       "      <th>r2</th>\n",
       "      <th>var</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>rfr0_all_n2g_cv2</td>\n",
       "      <td>29.859975</td>\n",
       "      <td>0.154954</td>\n",
       "      <td>107.332755</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>rfr0_all_cv2</td>\n",
       "      <td>29.964635</td>\n",
       "      <td>0.148975</td>\n",
       "      <td>115.257930</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>rfr0_subset2_cv2</td>\n",
       "      <td>30.003700</td>\n",
       "      <td>0.146811</td>\n",
       "      <td>110.744062</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>rfr0_subset2_cv5</td>\n",
       "      <td>30.007720</td>\n",
       "      <td>0.146590</td>\n",
       "      <td>108.869176</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>rfr0_subset_n2g_cv2</td>\n",
       "      <td>30.025197</td>\n",
       "      <td>0.145618</td>\n",
       "      <td>105.443823</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>rfr0_subset_cv2</td>\n",
       "      <td>30.047092</td>\n",
       "      <td>0.144318</td>\n",
       "      <td>115.071874</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>rfr0_subset2_n2g_cv2</td>\n",
       "      <td>30.148189</td>\n",
       "      <td>0.142212</td>\n",
       "      <td>110.596470</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>rfr0_all_cv5</td>\n",
       "      <td>30.103133</td>\n",
       "      <td>0.141111</td>\n",
       "      <td>114.482404</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>rfr0_subset_cv5</td>\n",
       "      <td>30.139479</td>\n",
       "      <td>0.138993</td>\n",
       "      <td>114.135159</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "metric                 index       rmse        r2         var\n",
       "0           rfr0_all_n2g_cv2  29.859975  0.154954  107.332755\n",
       "1               rfr0_all_cv2  29.964635  0.148975  115.257930\n",
       "2           rfr0_subset2_cv2  30.003700  0.146811  110.744062\n",
       "3           rfr0_subset2_cv5  30.007720  0.146590  108.869176\n",
       "4        rfr0_subset_n2g_cv2  30.025197  0.145618  105.443823\n",
       "5            rfr0_subset_cv2  30.047092  0.144318  115.071874\n",
       "6       rfr0_subset2_n2g_cv2  30.148189  0.142212  110.596470\n",
       "7               rfr0_all_cv5  30.103133  0.141111  114.482404\n",
       "8            rfr0_subset_cv5  30.139479  0.138993  114.135159"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# best features in terms of r2: \n",
    "testx = eval_df.set_index('metric')\n",
    "testx = testx.transpose()\n",
    "\n",
    "num_trials = metric_averages[modelname]['num_trials']\n",
    "print(f\"Updated ranking of average r2 after {num_trials} trials\")\n",
    "\n",
    "testx.sort_values(by=['r2'],ascending=False).reset_index()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "85a320ac",
   "metadata": {},
   "source": [
    "### Results:\n",
    "**Findings:**\n",
    "1. the most effective fragment set is **`frags_all`**\n",
    "2. the most effective NLP representation is **`n2g_cv2`**"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}