09 - Merge data - Property features and Structure features.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "26e1e950",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "22/09/28 13:20:47 WARN Utils: Your hostname, mitch-VirtualBox resolves to a loopback address: 127.0.1.1; using 10.0.2.15 instead (on interface enp0s3)\n",
      "22/09/28 13:20:47 WARN Utils: Set SPARK_LOCAL_IP if you need to bind to another address\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Setting default log level to \"WARN\".\n",
      "To adjust logging level use sc.setLogLevel(newLevel). For SparkR, use setLogLevel(newLevel).\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "22/09/28 13:20:47 WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable\n"
     ]
    }
   ],
   "source": [
    "# MSM VM config prep\n",
    "import findspark\n",
    "findspark.init('/home/mitch/spark-3.3.0-bin-hadoop2')\n",
    "import pyspark\n",
    " \n",
    "from pyspark.sql import SparkSession\n",
    "spark = SparkSession.builder.appName('BApredsV3').getOrCreate()\n",
    "\n",
    "# --- suppress future spark warnings/error/etc output ---\n",
    "spark.sparkContext.setLogLevel(\"OFF\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "dec3c272",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "def load_data_and_merge():\n",
    "    labels_and_calcs = spark.read.csv(\"data/bioavailability_data_final.csv\",inferSchema=True,sep=',',header=True)\n",
    "    df1 = labels_and_calcs.toPandas()\n",
    "    \n",
    "    df3 = pd.read_pickle('data/bioavailabilityData_w_Frags__final.pkl')\n",
    "    df3 = df3.drop(columns=['ba_pct'])\n",
    "    \n",
    "    #temp1 = pd.merge(df1,df2,how='left',left_on='_c0',right_on=df2.index)\n",
    "    \n",
    "    temp1 = labels_and_calcs\n",
    "    #temp1 = spark.createDataFrame(temp1)\n",
    "    \n",
    "    temp2 = spark.createDataFrame(df3)\n",
    "    \n",
    "    data = temp2.join(temp1,(temp2.drug_smiles==temp1.smile),\"left\")\n",
    "    \n",
    "    return data\n",
    "    \n",
    "\n",
    "data = load_data_and_merge()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b49eb87e",
   "metadata": {},
   "source": [
    "* Label data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "d04fabab",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    }
   ],
   "source": [
    "''' \n",
    "# INTIAL LABELS:\n",
    "# --- Data has 1 continuous label column, and 4 categorical label columns (discretized variants of continuous label).\n",
    "# ------ categorical labels applied by dividing the continuous label values into 3-5 categories \n",
    "# ------ the value range associated with each group were selected based on histogram dist./mean/stdev\n",
    "# --- We'll add one more discretization variant,  using Spark's built-in QuantileDiscretizer\n",
    "'''\n",
    "# -- Add QuantileDiscretizer labels\n",
    "from pyspark.ml.feature import QuantileDiscretizer\n",
    "import pandas as pd\n",
    "qd5 = QuantileDiscretizer(numBuckets=5,inputCol='BA_pct',outputCol='label_QD5')\n",
    "\n",
    "data_wLabels = qd5.fit(data).transform(data)\n",
    "\n",
    "# -- INDEX LABELS\n",
    "from pyspark.ml.feature import (StringIndexer,OneHotEncoder)\n",
    "\n",
    "label_quant0 = 'BA_pct'\n",
    "\n",
    "label_cat1_index = StringIndexer(inputCol='label1',outputCol='label_cat1_index')\n",
    "\n",
    "label_cat2_index = StringIndexer(inputCol='label2',outputCol='label_cat2_index')\n",
    "\n",
    "label_cat3_index = StringIndexer(inputCol='label3a',outputCol='label_cat3_index')\n",
    "\n",
    "label_cat4_index = StringIndexer(inputCol='label3b',outputCol='label_cat4_index')\n",
    "\n",
    "from pyspark.ml import Pipeline\n",
    "label_pipeline = Pipeline(stages=[label_cat1_index,label_cat2_index,label_cat3_index,label_cat4_index])\n",
    "\n",
    "data_wLabels = label_pipeline.fit(data_wLabels).transform(data_wLabels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "80e2f8e9",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    }
   ],
   "source": [
    "''' # Fragments NLP processing  - NEWER -\n",
    "'''\n",
    "from pyspark.ml.feature import NGram,Word2Vec,CountVectorizer,IDF\n",
    "\n",
    "fragment_types = ['frags_all','frags_subset','frags_subset2','frags_efgs','frags_brics']\n",
    "                  \n",
    "fragment_shortname = ['f_all','f_subset','f_subset2','f_efgs','f_brics']\n",
    "\n",
    "data_wLabels_NLP = data_wLabels\n",
    "for i,frag_type in enumerate(fragment_types):\n",
    "    \n",
    "    frag_type_short = fragment_shortname[i]\n",
    "    \n",
    "    cv2 = CountVectorizer(inputCol=frag_type, outputCol=f\"{frag_type_short}_cv2\", minDF=2.0)\n",
    "    cv2_idf = IDF(inputCol=f\"{frag_type_short}_cv2\", outputCol=f\"{frag_type_short}_cv2_idf\")\n",
    "    \n",
    "    cv5 = CountVectorizer(inputCol=frag_type, outputCol=f\"{frag_type_short}_cv5\")\n",
    "    \n",
    "    w2v = Word2Vec(inputCol=frag_type, outputCol=f\"{frag_type_short}_w2v\")\n",
    "    \n",
    "    n2gram = NGram(n=2, inputCol=frag_type, outputCol=f\"{frag_type_short}_n2g\")\n",
    "    n2gram_cv2 = CountVectorizer(inputCol=f\"{frag_type_short}_n2g\", outputCol=f\"{frag_type_short}_n2g_cv2\", minDF=2.0)\n",
    "    \n",
    "    n2gram_cv5 = CountVectorizer(inputCol=f\"{frag_type_short}_n2g\", outputCol=f\"{frag_type_short}_n2g_cv5\")\n",
    "    \n",
    "    nlp_pipeline = Pipeline(stages=[cv2, cv2_idf, \n",
    "                                    cv5, w2v,\n",
    "                                    n2gram, n2gram_cv2, n2gram_cv5])\n",
    "    \n",
    "    data_wLabels_NLP = nlp_pipeline.fit(data_wLabels_NLP).transform(data_wLabels_NLP)\n",
    "    \n",
    "    column_to_drop = f\"{frag_type_short}_n2g\"\n",
    "    data_wLabels_NLP = data_wLabels_NLP.drop(column_to_drop)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8236b294",
   "metadata": {},
   "source": [
    "* make fragment NLP feature vectors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "2a42ed9a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "'FEAT_f_all_cv2', 'FEAT_f_all_cv2_idf', 'FEAT_f_all_cv5', 'FEAT_f_all_w2v', 'FEAT_f_all_n2g_cv2', 'FEAT_f_all_n2g_cv5', 'FEAT_f_subset_cv2', 'FEAT_f_subset_cv2_idf', 'FEAT_f_subset_cv5', 'FEAT_f_subset_w2v', 'FEAT_f_subset_n2g_cv2', 'FEAT_f_subset_n2g_cv5', 'FEAT_f_subset2_cv2', 'FEAT_f_subset2_cv2_idf', 'FEAT_f_subset2_cv5', 'FEAT_f_subset2_w2v', 'FEAT_f_subset2_n2g_cv2', 'FEAT_f_subset2_n2g_cv5', 'FEAT_f_efgs_cv2', 'FEAT_f_efgs_cv2_idf', 'FEAT_f_efgs_cv5', 'FEAT_f_efgs_w2v', 'FEAT_f_efgs_n2g_cv2', 'FEAT_f_efgs_n2g_cv5', 'FEAT_f_brics_cv2', 'FEAT_f_brics_cv2_idf', 'FEAT_f_brics_cv5', 'FEAT_f_brics_w2v', 'FEAT_f_brics_n2g_cv2', 'FEAT_f_brics_n2g_cv5'\n"
     ]
    }
   ],
   "source": [
    "from pyspark.ml.linalg import Vector\n",
    "from pyspark.ml.feature import (VectorAssembler,VectorIndexer)\n",
    "\n",
    "vector_assemblers = []\n",
    "\n",
    "alternative_features = ['f_all_cv2','f_all_cv2_idf','f_all_cv5','f_all_w2v',\n",
    "                        'f_all_n2g_cv2','f_all_n2g_cv5',\n",
    "                        \n",
    "                        'f_subset_cv2','f_subset_cv2_idf','f_subset_cv5','f_subset_w2v',\n",
    "                        'f_subset_n2g_cv2','f_subset_n2g_cv5',\n",
    "                        \n",
    "                        'f_subset2_cv2','f_subset2_cv2_idf','f_subset2_cv5','f_subset2_w2v',\n",
    "                        'f_subset2_n2g_cv2','f_subset2_n2g_cv5',\n",
    "                        \n",
    "                        'f_efgs_cv2','f_efgs_cv2_idf','f_efgs_cv5','f_efgs_w2v',\n",
    "                        'f_efgs_n2g_cv2','f_efgs_n2g_cv5',\n",
    "                        \n",
    "                        'f_brics_cv2','f_brics_cv2_idf','f_brics_cv5','f_brics_w2v',\n",
    "                        'f_brics_n2g_cv2','f_brics_n2g_cv5']\n",
    "output_features = \"\"\n",
    "for feats in alternative_features:\n",
    "    #feats_input = ['MolWt','MolLogP','TPSA',feats]\n",
    "    feats_input = [feats]\n",
    "    feats_output = f\"FEAT_{feats}\"\n",
    "    \n",
    "    vec_assembler = VectorAssembler(inputCols=feats_input, outputCol=feats_output)\n",
    "    \n",
    "    vector_assemblers.append(vec_assembler)\n",
    "    \n",
    "    output_features += \"'\"+feats_output+\"'\"+\", \"\n",
    "output_features = output_features[0:len(output_features)-2]\n",
    "print(output_features)\n",
    "\n",
    "from pyspark.ml import Pipeline\n",
    "feature_pipeline = Pipeline(stages=[x for x in vector_assemblers])\n",
    "\n",
    "data_wLabels_NLPFeatures = feature_pipeline.fit(data_wLabels_NLP).transform(data_wLabels_NLP)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "04847a22",
   "metadata": {},
   "source": [
    "* Prepare Vector Features for RDKit calculations "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "1eb0ca53",
   "metadata": {},
   "outputs": [],
   "source": [
    "''' # RDKit \"1D\" FEATURE SELECTION:\n",
    "'''\n",
    "# to load the Features Information, use the command:\n",
    "featuresDF = pd.read_parquet('featuresCatalogDF.parquet')\n",
    "feature_set1b = featuresDF.loc[1,'features']\n",
    "feature_set2b = featuresDF.loc[3,'features']\n",
    "F1bANOVA = featuresDF.loc[7,'features']\n",
    "F2bANOVA = featuresDF.loc[8,'features']\n",
    "\n",
    "# VECTOR ASSEMBLY - feature sets \n",
    "from pyspark.ml.linalg import Vector\n",
    "from pyspark.ml.feature import (VectorAssembler,VectorIndexer)\n",
    "\n",
    "vec_assembler1b = VectorAssembler(inputCols = feature_set1b, outputCol='FEAT_rdkit_1b')\n",
    "vec_assembler2b = VectorAssembler(inputCols = feature_set2b, outputCol='FEAT_rdkit_2b')\n",
    "vec_assembler1bANOVA = VectorAssembler(inputCols = F1bANOVA, outputCol='FEAT_rdkit_1bANOVA')\n",
    "vec_assembler2bANOVA = VectorAssembler(inputCols = F2bANOVA, outputCol='FEAT_rdkit_2bANOVA')\n",
    "\n",
    "from pyspark.ml import Pipeline\n",
    "feature_pipeline = Pipeline(stages=[vec_assembler1b,\n",
    "                                    vec_assembler2b,\n",
    "                                    vec_assembler1bANOVA,\n",
    "                                    vec_assembler2bANOVA])\n",
    "\n",
    "data_allFeaturesAndLabels = feature_pipeline.fit(data_wLabels_NLPFeatures).transform(data_wLabels_NLPFeatures)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c9272770",
   "metadata": {},
   "source": [
    "* clean up the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "2eb0f5e8",
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['drug_name',\n",
       " 'drug_smiles',\n",
       " 'frags_all',\n",
       " 'num_frags_all',\n",
       " 'frags_subset',\n",
       " 'num_frags_subset',\n",
       " 'frags_subset2',\n",
       " 'frags_efgs',\n",
       " 'frags_brics',\n",
       " '_c0',\n",
       " 'Name',\n",
       " 'smile',\n",
       " 'BA_pct',\n",
       " 'MolWt',\n",
       " 'ExactMolWt',\n",
       " 'qed',\n",
       " 'MolLogP',\n",
       " 'MolMR',\n",
       " 'VSA_total',\n",
       " 'LabuteASA',\n",
       " 'TPSA',\n",
       " 'MaxPartialCharge',\n",
       " 'MinPartialCharge',\n",
       " 'MaxAbsPartialCharge',\n",
       " 'MinAbsPartialCharge',\n",
       " 'NumHAcceptors',\n",
       " 'NumHDonors',\n",
       " 'HeavyAtomCount',\n",
       " 'NumHeteroatoms',\n",
       " 'NumRotatableBonds',\n",
       " 'NHOHCount',\n",
       " 'NOCount',\n",
       " 'FractionCSP3',\n",
       " 'RingCount',\n",
       " 'NumAliphaticRings',\n",
       " 'NumAromaticRings',\n",
       " 'NumAliphaticHeterocycles',\n",
       " 'NumAromaticHeterocycles',\n",
       " 'NumSaturatedHeterocycles',\n",
       " 'NumSaturatedRings',\n",
       " 'BalabanJ',\n",
       " 'BertzCT',\n",
       " 'HallKierAlpha',\n",
       " 'PEOE_VSA1',\n",
       " 'PEOE_VSA2',\n",
       " 'PEOE_VSA3',\n",
       " 'PEOE_VSA4',\n",
       " 'PEOE_VSA5',\n",
       " 'PEOE_VSA6',\n",
       " 'PEOE_VSA7',\n",
       " 'PEOE_VSA8',\n",
       " 'PEOE_VSA9',\n",
       " 'PEOE_VSA10',\n",
       " 'PEOE_VSA11',\n",
       " 'PEOE_VSA12',\n",
       " 'PEOE_VSA13',\n",
       " 'PEOE_VSA14',\n",
       " 'SMR_VSA1',\n",
       " 'SMR_VSA2',\n",
       " 'SMR_VSA3',\n",
       " 'SMR_VSA4',\n",
       " 'SMR_VSA5',\n",
       " 'SMR_VSA6',\n",
       " 'SMR_VSA7',\n",
       " 'SMR_VSA8',\n",
       " 'SMR_VSA9',\n",
       " 'SMR_VSA10',\n",
       " 'SlogP_VSA1',\n",
       " 'SlogP_VSA2',\n",
       " 'SlogP_VSA3',\n",
       " 'SlogP_VSA4',\n",
       " 'SlogP_VSA5',\n",
       " 'SlogP_VSA6',\n",
       " 'SlogP_VSA7',\n",
       " 'SlogP_VSA8',\n",
       " 'SlogP_VSA9',\n",
       " 'SlogP_VSA10',\n",
       " 'SlogP_VSA11',\n",
       " 'SlogP_VSA12',\n",
       " 'PEOE_VSA1.1',\n",
       " 'PEOE_VSA2.1',\n",
       " 'PEOE_VSA3.1',\n",
       " 'PEOE_VSA4.1',\n",
       " 'PEOE_VSA5.1',\n",
       " 'PEOE_VSA6.1',\n",
       " 'PEOE_VSA7.1',\n",
       " 'PEOE_VSA8.1',\n",
       " 'PEOE_VSA9.1',\n",
       " 'PEOE_VSA10.1',\n",
       " 'PEOE_VSA11.1',\n",
       " 'PEOE_VSA12.1',\n",
       " 'PEOE_VSA13.1',\n",
       " 'PEOE_VSA14.1',\n",
       " 'SMR_VSA1.1',\n",
       " 'SMR_VSA2.1',\n",
       " 'SMR_VSA3.1',\n",
       " 'SMR_VSA4.1',\n",
       " 'SMR_VSA5.1',\n",
       " 'SMR_VSA6.1',\n",
       " 'SMR_VSA7.1',\n",
       " 'SMR_VSA8.1',\n",
       " 'SMR_VSA9.1',\n",
       " 'SMR_VSA10.1',\n",
       " 'SlogP_VSA1.1',\n",
       " 'SlogP_VSA2.1',\n",
       " 'SlogP_VSA3.1',\n",
       " 'SlogP_VSA4.1',\n",
       " 'SlogP_VSA5.1',\n",
       " 'SlogP_VSA6.1',\n",
       " 'SlogP_VSA7.1',\n",
       " 'SlogP_VSA8.1',\n",
       " 'SlogP_VSA9.1',\n",
       " 'SlogP_VSA10.1',\n",
       " 'SlogP_VSA11.1',\n",
       " 'SlogP_VSA12.1',\n",
       " 'fracVSA_PEOE01',\n",
       " 'fracVSA_PEOE02',\n",
       " 'fracVSA_PEOE03',\n",
       " 'fracVSA_PEOE04',\n",
       " 'fracVSA_PEOE05',\n",
       " 'fracVSA_PEOE06',\n",
       " 'fracVSA_PEOE07',\n",
       " 'fracVSA_PEOE08',\n",
       " 'fracVSA_PEOE09',\n",
       " 'fracVSA_PEOE10',\n",
       " 'fracVSA_PEOE11',\n",
       " 'fracVSA_PEOE12',\n",
       " 'fracVSA_PEOE13',\n",
       " 'fracVSA_PEOE14',\n",
       " 'fracVSA_SMR01',\n",
       " 'fracVSA_SMR02',\n",
       " 'fracVSA_SMR03',\n",
       " 'fracVSA_SMR04',\n",
       " 'fracVSA_SMR05',\n",
       " 'fracVSA_SMR06',\n",
       " 'fracVSA_SMR07',\n",
       " 'fracVSA_SMR08',\n",
       " 'fracVSA_SMR09',\n",
       " 'fracVSA_SMR10',\n",
       " 'fracVSA_SlogP01',\n",
       " 'fracVSA_SlogP02',\n",
       " 'fracVSA_SlogP03',\n",
       " 'fracVSA_SlogP04',\n",
       " 'fracVSA_SlogP05',\n",
       " 'fracVSA_SlogP06',\n",
       " 'fracVSA_SlogP07',\n",
       " 'fracVSA_SlogP08',\n",
       " 'fracVSA_SlogP09',\n",
       " 'fracVSA_SlogP10',\n",
       " 'fracVSA_SlogP11',\n",
       " 'fracVSA_SlogP12',\n",
       " 'label1',\n",
       " 'label2',\n",
       " 'label3a',\n",
       " 'label3b',\n",
       " 'label_QD5',\n",
       " 'label_cat1_index',\n",
       " 'label_cat2_index',\n",
       " 'label_cat3_index',\n",
       " 'label_cat4_index',\n",
       " 'f_all_cv2',\n",
       " 'f_all_cv2_idf',\n",
       " 'f_all_cv5',\n",
       " 'f_all_w2v',\n",
       " 'f_all_n2g_cv2',\n",
       " 'f_all_n2g_cv5',\n",
       " 'f_subset_cv2',\n",
       " 'f_subset_cv2_idf',\n",
       " 'f_subset_cv5',\n",
       " 'f_subset_w2v',\n",
       " 'f_subset_n2g_cv2',\n",
       " 'f_subset_n2g_cv5',\n",
       " 'f_subset2_cv2',\n",
       " 'f_subset2_cv2_idf',\n",
       " 'f_subset2_cv5',\n",
       " 'f_subset2_w2v',\n",
       " 'f_subset2_n2g_cv2',\n",
       " 'f_subset2_n2g_cv5',\n",
       " 'f_efgs_cv2',\n",
       " 'f_efgs_cv2_idf',\n",
       " 'f_efgs_cv5',\n",
       " 'f_efgs_w2v',\n",
       " 'f_efgs_n2g_cv2',\n",
       " 'f_efgs_n2g_cv5',\n",
       " 'f_brics_cv2',\n",
       " 'f_brics_cv2_idf',\n",
       " 'f_brics_cv5',\n",
       " 'f_brics_w2v',\n",
       " 'f_brics_n2g_cv2',\n",
       " 'f_brics_n2g_cv5',\n",
       " 'FEAT_f_all_cv2',\n",
       " 'FEAT_f_all_cv2_idf',\n",
       " 'FEAT_f_all_cv5',\n",
       " 'FEAT_f_all_w2v',\n",
       " 'FEAT_f_all_n2g_cv2',\n",
       " 'FEAT_f_all_n2g_cv5',\n",
       " 'FEAT_f_subset_cv2',\n",
       " 'FEAT_f_subset_cv2_idf',\n",
       " 'FEAT_f_subset_cv5',\n",
       " 'FEAT_f_subset_w2v',\n",
       " 'FEAT_f_subset_n2g_cv2',\n",
       " 'FEAT_f_subset_n2g_cv5',\n",
       " 'FEAT_f_subset2_cv2',\n",
       " 'FEAT_f_subset2_cv2_idf',\n",
       " 'FEAT_f_subset2_cv5',\n",
       " 'FEAT_f_subset2_w2v',\n",
       " 'FEAT_f_subset2_n2g_cv2',\n",
       " 'FEAT_f_subset2_n2g_cv5',\n",
       " 'FEAT_f_efgs_cv2',\n",
       " 'FEAT_f_efgs_cv2_idf',\n",
       " 'FEAT_f_efgs_cv5',\n",
       " 'FEAT_f_efgs_w2v',\n",
       " 'FEAT_f_efgs_n2g_cv2',\n",
       " 'FEAT_f_efgs_n2g_cv5',\n",
       " 'FEAT_f_brics_cv2',\n",
       " 'FEAT_f_brics_cv2_idf',\n",
       " 'FEAT_f_brics_cv5',\n",
       " 'FEAT_f_brics_w2v',\n",
       " 'FEAT_f_brics_n2g_cv2',\n",
       " 'FEAT_f_brics_n2g_cv5',\n",
       " 'FEAT_rdkit_1b',\n",
       " 'FEAT_rdkit_2b',\n",
       " 'FEAT_rdkit_1bANOVA',\n",
       " 'FEAT_rdkit_2bANOVA']"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_allFeaturesAndLabels.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "6c7a2a7e",
   "metadata": {},
   "outputs": [],
   "source": [
    "data_allFeaturesAndLabels = data_allFeaturesAndLabels.drop('label1')\n",
    "data_allFeaturesAndLabels = data_allFeaturesAndLabels.drop('label2')\n",
    "data_allFeaturesAndLabels = data_allFeaturesAndLabels.drop('label3a')\n",
    "data_allFeaturesAndLabels = data_allFeaturesAndLabels.drop('label3b')\n",
    "\n",
    "data_allFeaturesAndLabels = data_allFeaturesAndLabels.withColumnRenamed('BA_pct','label_q0')\n",
    "data_allFeaturesAndLabels = data_allFeaturesAndLabels.withColumnRenamed('label_QD5','label_cat0')\n",
    "data_allFeaturesAndLabels = data_allFeaturesAndLabels.withColumnRenamed('label_cat1_index','label_cat1')\n",
    "data_allFeaturesAndLabels = data_allFeaturesAndLabels.withColumnRenamed('label_cat2_index','label_cat2')\n",
    "data_allFeaturesAndLabels = data_allFeaturesAndLabels.withColumnRenamed('label_cat3_index','label_cat3')\n",
    "data_allFeaturesAndLabels = data_allFeaturesAndLabels.withColumnRenamed('label_cat4_index','label_cat4')\n",
    "\n",
    "features_to_drop = [\n",
    "    'PEOE_VSA1','PEOE_VSA2','PEOE_VSA3','PEOE_VSA4','PEOE_VSA5','PEOE_VSA6','PEOE_VSA7','PEOE_VSA8',\n",
    "    'PEOE_VSA9','PEOE_VSA10','PEOE_VSA11','PEOE_VSA12','PEOE_VSA13','PEOE_VSA14','SMR_VSA1','SMR_VSA2',\n",
    "    'SMR_VSA3','SMR_VSA4','SMR_VSA5','SMR_VSA6','SMR_VSA7','SMR_VSA8','SMR_VSA9','SMR_VSA10','SlogP_VSA1',\n",
    "    'SlogP_VSA2','SlogP_VSA3','SlogP_VSA4','SlogP_VSA5','SlogP_VSA6','SlogP_VSA7','SlogP_VSA8','SlogP_VSA9',\n",
    "    'SlogP_VSA10','SlogP_VSA11','SlogP_VSA12','PEOE_VSA1.1','PEOE_VSA2.1','PEOE_VSA3.1','PEOE_VSA4.1',\n",
    "    'PEOE_VSA5.1','PEOE_VSA6.1','PEOE_VSA7.1','PEOE_VSA8.1','PEOE_VSA9.1','PEOE_VSA10.1','PEOE_VSA11.1',\n",
    "    'PEOE_VSA12.1','PEOE_VSA13.1','PEOE_VSA14.1','SMR_VSA1.1','SMR_VSA2.1','SMR_VSA3.1','SMR_VSA4.1',\n",
    "    'SMR_VSA5.1','SMR_VSA6.1','SMR_VSA7.1','SMR_VSA8.1','SMR_VSA9.1','SMR_VSA10.1','SlogP_VSA1.1',\n",
    "    'SlogP_VSA2.1','SlogP_VSA3.1','SlogP_VSA4.1','SlogP_VSA5.1','SlogP_VSA6.1','SlogP_VSA7.1','SlogP_VSA8.1',\n",
    "    'SlogP_VSA9.1','SlogP_VSA10.1','SlogP_VSA11.1','SlogP_VSA12.1']\n",
    "\n",
    "for x in features_to_drop:\n",
    "    data_allFeaturesAndLabels = data_allFeaturesAndLabels.drop(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "b9a356df",
   "metadata": {},
   "outputs": [],
   "source": [
    "final_column_order = ['_c0','Name','smile','drug_name','drug_smiles',                    \n",
    "    # labels\n",
    "    'label_q0','label_cat0','label_cat1','label_cat2','label_cat3','label_cat4',\n",
    "                      \n",
    "    # fragment features\n",
    "    #'FEAT_f_all_cv2', 'FEAT_f_all_cv2_idf', 'FEAT_f_all_cv5', 'FEAT_f_all_w2v', 'FEAT_f_all_n2g_cv2', 'FEAT_f_all_n2g_cv5',\n",
    "    'FEAT_f_all_cv2', 'FEAT_f_all_cv2_idf', 'FEAT_f_all_cv5','FEAT_f_all_n2g_cv2', 'FEAT_f_all_n2g_cv5', \n",
    "    #'FEAT_f_subset_cv2', 'FEAT_f_subset_cv2_idf', 'FEAT_f_subset_cv5', 'FEAT_f_subset_w2v', 'FEAT_f_subset_n2g_cv2', 'FEAT_f_subset_n2g_cv5',\n",
    "    'FEAT_f_subset_cv2', 'FEAT_f_subset_cv2_idf', 'FEAT_f_subset_cv5', 'FEAT_f_subset_n2g_cv2', 'FEAT_f_subset_n2g_cv5', \n",
    "    #'FEAT_f_subset2_cv2', 'FEAT_f_subset2_cv2_idf', 'FEAT_f_subset2_cv5', 'FEAT_f_subset2_w2v', 'FEAT_f_subset2_n2g_cv2', 'FEAT_f_subset2_n2g_cv5', \n",
    "    'FEAT_f_subset2_cv2', 'FEAT_f_subset2_cv2_idf', 'FEAT_f_subset2_cv5', 'FEAT_f_subset2_n2g_cv2', 'FEAT_f_subset2_n2g_cv5',\n",
    "    #'FEAT_f_efgs_cv2', 'FEAT_f_efgs_cv2_idf', 'FEAT_f_efgs_cv5', 'FEAT_f_efgs_w2v', 'FEAT_f_efgs_n2g_cv2', 'FEAT_f_efgs_n2g_cv5', \n",
    "    #'FEAT_f_brics_cv2', 'FEAT_f_brics_cv2_idf', 'FEAT_f_brics_cv5', 'FEAT_f_brics_w2v', 'FEAT_f_brics_n2g_cv2', 'FEAT_f_brics_n2g_cv5',\n",
    "                      \n",
    "    # rdkit features\n",
    "    'FEAT_rdkit_1b','FEAT_rdkit_2b','FEAT_rdkit_1bANOVA','FEAT_rdkit_2bANOVA',\n",
    "                      \n",
    "    # fragment data\n",
    "    #'frags_all','frags_subset','frags_subset2','f_all_n2g_cv2','f_subset2_cv5','f_subset_n2g_cv2',\n",
    "                      \n",
    "    # rdkit data\n",
    "    'MolWt','ExactMolWt','qed','MolLogP','MolMR','VSA_total','LabuteASA',\n",
    "    'TPSA','MaxPartialCharge','MinPartialCharge','MaxAbsPartialCharge','MinAbsPartialCharge','NumHAcceptors',\n",
    "    'NumHDonors','HeavyAtomCount','NumHeteroatoms','NumRotatableBonds','NHOHCount','NOCount','FractionCSP3',\n",
    "    'RingCount','NumAliphaticRings','NumAromaticRings','NumAliphaticHeterocycles','NumAromaticHeterocycles',\n",
    "    'NumSaturatedHeterocycles','NumSaturatedRings','BalabanJ','BertzCT','HallKierAlpha','fracVSA_PEOE01',\n",
    "    'fracVSA_PEOE02','fracVSA_PEOE03','fracVSA_PEOE04','fracVSA_PEOE05','fracVSA_PEOE06','fracVSA_PEOE07',\n",
    "    'fracVSA_PEOE08','fracVSA_PEOE09','fracVSA_PEOE10','fracVSA_PEOE11','fracVSA_PEOE12','fracVSA_PEOE13',\n",
    "    'fracVSA_PEOE14','fracVSA_SMR01','fracVSA_SMR02','fracVSA_SMR03','fracVSA_SMR04','fracVSA_SMR05',\n",
    "    'fracVSA_SMR06','fracVSA_SMR07','fracVSA_SMR08','fracVSA_SMR09','fracVSA_SMR10','fracVSA_SlogP01',\n",
    "    'fracVSA_SlogP02','fracVSA_SlogP03','fracVSA_SlogP04','fracVSA_SlogP05','fracVSA_SlogP06','fracVSA_SlogP07',\n",
    "    'fracVSA_SlogP08','fracVSA_SlogP09','fracVSA_SlogP10','fracVSA_SlogP11','fracVSA_SlogP12']\n",
    "\n",
    "data_masterFinal_clean = data_allFeaturesAndLabels.select(final_column_order)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c850f035",
   "metadata": {},
   "source": [
    "* export cleaned data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "5e52c68a",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    }
   ],
   "source": [
    "data_masterFinal_clean.toPandas().to_pickle(\"data/bioavailability_data_masterFinal2.pkl\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}