Algorithm Execution Container-Jupyter Version.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {
    "hideCode": true,
    "hidePrompt": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<button type=\"button\" class=\"btn btn-primary btn-md\" onclick=\"initAEC();\">Initialise</button>\n",
       "<script>\n",
       "$.ajaxSetup({\n",
       "    headers: { \"Authorization\": localStorage.getItem(\"token\") }\n",
       "});\n",
       "function initAEC(){\n",
       "    Jupyter.notebook.execute_cells([1]);\n",
       "    Jupyter.notebook.execute_cells([2]);\n",
       "    Jupyter.notebook.execute_cells([3]);\n",
       "}\n",
       "</script>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "%%html\n",
    "<button type=\"button\" class=\"btn btn-primary btn-md\" onclick=\"initAEC();\">Initialise</button>\n",
    "<script>\n",
    "$.ajaxSetup({\n",
    "    headers: { \"Authorization\": localStorage.getItem(\"token\") }\n",
    "});\n",
    "function initAEC(){\n",
    "    Jupyter.notebook.execute_cells([1]);\n",
    "    Jupyter.notebook.execute_cells([2]);\n",
    "    Jupyter.notebook.execute_cells([3]);\n",
    "}\n",
    "</script>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "metadata": {
    "code_folding": [
     362,
     385,
     420,
     462,
     505,
     547,
     589,
     630,
     666,
     701,
     774,
     797,
     813,
     828,
     846,
     878,
     885,
     895,
     909
    ],
    "hideCode": true
   },
   "outputs": [],
   "source": [
    "import json\n",
    "import requests\n",
    "from hops import hdfs\n",
    "import pydoop.hdfs as phdfs\n",
    "from  pyspark.sql.functions import year, ltrim, rtrim, coalesce, lit, udf\n",
    "from pyspark.sql.types import StringType, ArrayType\n",
    "from pyspark.ml.classification import OneVsRest, NaiveBayes, MultilayerPerceptronClassifier, LinearSVC, LogisticRegression, DecisionTreeClassifier, RandomForestClassifier, GBTClassifier\n",
    "from pyspark.ml.regression import DecisionTreeRegressor, RandomForestRegressor, LinearRegression, GBTRegressor, GeneralizedLinearRegression\n",
    "from pyspark.ml.clustering import KMeans, GaussianMixture\n",
    "from pyspark.ml import Pipeline\n",
    "from pyspark.ml.tuning import ParamGridBuilder, TrainValidationSplit, CrossValidator\n",
    "from pyspark.ml.feature import HashingTF, IDF, Tokenizer, NGram, RegexTokenizer, PCA, ChiSqSelector, StringIndexer, VectorIndexer, IndexToString, VectorAssembler, OneHotEncoderEstimator\n",
    "from pyspark.ml.evaluation import MulticlassClassificationEvaluator, BinaryClassificationEvaluator, RegressionEvaluator, ClusteringEvaluator\n",
    "from pyspark.ml.linalg import Vectors\n",
    "from pyspark.ml.stat import Correlation, ChiSquareTest\n",
    "import numpy as np\n",
    "from pyspark.ml.recommendation import ALS\n",
    "from pyspark.ml.fpm import FPGrowth\n",
    "\n",
    "myhdfs = phdfs.hdfs(user = hdfs.project_user())\n",
    "\n",
    "modelDF = None\n",
    "tempDF = None\n",
    "masterDF = None\n",
    "hdfspath = hdfs.project_path()\n",
    "g=None\n",
    "model = None\n",
    "\n",
    "# helper function to transform input column, used by FPGrowth\n",
    "def colToArray(s):\n",
    "    input = [str(int(x)) for x in s[1:-1].split(\",\")]\n",
    "    return input\n",
    "toarray_udf = udf(colToArray,ArrayType(StringType()))\n",
    "\n",
    "\n",
    "PARAMETERS = {\n",
    "    \"maxiter\":{\n",
    "        \"grid\":\"Y\",\n",
    "        \"type\":\"number\",\n",
    "        \"min\":10,\n",
    "        \"max\":1000,\n",
    "        \"placeholder\":\"10-1000\",\n",
    "        \"label\":\"Maximum Iterations\",\n",
    "        \"title\":\"Maximum number of iterations the algorithm will perform.\"\n",
    "    },\n",
    "    \"regParam\":{\n",
    "        \"grid\":\"Y\",\n",
    "        \"type\":\"number\",\n",
    "        \"min\":0,\n",
    "        \"max\":10,\n",
    "        \"step\":0.01,\n",
    "        \"placeholder\":\"e.g. 0.01\",\n",
    "        \"label\":\"Regularisation Parameter\",\n",
    "        \"title\":\"Configuring this parameter correctly is important to avoid over- and under-fitting.\"\n",
    "    },\n",
    "    \"elasticnet\":{\n",
    "        \"grid\":\"Y\",\n",
    "        \"type\":\"number\",\n",
    "        \"min\":0.0,\n",
    "        \"step\":\"0.01\",\n",
    "        \"max\":1.0,\n",
    "        \"placeholder\":\"0\",\n",
    "        \"label\":\"Elastic Net Parameter\",\n",
    "        \"title\":\"The elastic net linearly combines the L1 and L2 penalties of the lasso and ridge methods.\"\n",
    "    },\n",
    "    \"max_depth\":{\n",
    "        \"grid\":\"Y\",\n",
    "        \"type\":\"number\",\n",
    "        \"min\":1,\n",
    "        \"max\":100,\n",
    "        \"placeholder\":\"1-100\",\n",
    "        \"label\":\"Maximum Depth\",\n",
    "        \"title\":\"The maximum tree depth.\"\n",
    "    },\n",
    "    \"max_bins\":{\n",
    "        \"grid\":\"Y\",\n",
    "        \"type\":\"number\",\n",
    "        \"min\":1,\n",
    "        \"max\":64,\n",
    "        \"placeholder\":\"e.g. 32\",\n",
    "        \"label\":\"Maximum Bins\",\n",
    "        \"title\":\"Note that the number of bins cannot be greater than the number of instances N (a rare scenario since the default maxBins value is 32). The tree algorithm automatically reduces the number of bins if the condition is not satisfied\"\n",
    "    },\n",
    "    \"min_instances_per_node\":{\n",
    "        \"grid\":\"Y\",\n",
    "        \"type\":\"number\",\n",
    "        \"min\":1,\n",
    "        \"max\":10,\n",
    "        \"placeholder\":\"1-100\",\n",
    "        \"label\":\"Minimum Instances per Node\",\n",
    "        \"title\":\"Selecting a very small number here may result to major overfitting.\"\n",
    "    },\n",
    "    \"min_info_gain\":{\n",
    "        \"grid\":\"Y\",\n",
    "        \"type\":\"number\",\n",
    "        \"min\":0,\n",
    "        \"max\":10,\n",
    "        \"step\":0.01,\n",
    "        \"placeholder\":\"e.g. 0.01\",\n",
    "        \"label\":\"Minimum Info Gain\",\n",
    "        \"title\":\"Configuring this parameter correctly is important to avoid over- and under-fitting.\"\n",
    "    },\n",
    "    \"step\":{\n",
    "        \"grid\":\"N\",\n",
    "        \"type\":\"number\",\n",
    "        \"min\":0,\n",
    "        \"max\":10,\n",
    "        \"step\":0.01,\n",
    "        \"placeholder\":\"e.g. 0.01\",\n",
    "        \"label\":\"Step\",\n",
    "        \"title\":\"Step\"\n",
    "    },\n",
    "    \"stepSize\":{\n",
    "        \"grid\":\"Y\",\n",
    "        \"type\":\"number\",\n",
    "        \"min\":0,\n",
    "        \"max\":10,\n",
    "        \"step\":0.01,\n",
    "        \"placeholder\":\"e.g. 0.01\",\n",
    "        \"label\":\"Step Size\",\n",
    "        \"title\":\"Step Size\"\n",
    "    },\n",
    "    \"convergence_tolerance\":{\n",
    "        \"grid\":\"N\",\n",
    "        \"type\":\"number\",\n",
    "        \"min\":0,\n",
    "        \"max\":10,\n",
    "        \"step\":0.01,\n",
    "        \"placeholder\":\"e.g. 0.01\",\n",
    "        \"label\":\"Convergence Tolerance\"\n",
    "    },\n",
    "    \"nodes_l1\":{\n",
    "        \"grid\":\"N\",\n",
    "        \"type\":\"number\",\n",
    "        \"min\":1,\n",
    "        \"max\":10,\n",
    "        \"placeholder\":\"e.g. 4\",\n",
    "        \"label\":\"Nodes in 1st hidden layer\"\n",
    "    },\n",
    "    \"nodes_l2\":{\n",
    "        \"grid\":\"N\",\n",
    "        \"type\":\"number\",\n",
    "        \"min\":1,\n",
    "        \"max\":10,\n",
    "        \"placeholder\":\"e.g. 4\",\n",
    "        \"label\":\"Nodes in 2nd hidden layer\"\n",
    "    },\n",
    "    \"seed\":{\n",
    "        \"grid\":\"N\",\n",
    "        \"type\":\"number\",\n",
    "        \"min\":0,\n",
    "        \"max\":10000,\n",
    "        \"placeholder\":\"e.g. 0.01\",\n",
    "        \"label\":\"Seed\"\n",
    "    },\n",
    "    \"smoothing\":{\n",
    "        \"grid\":\"Y\",\n",
    "        \"type\":\"number\",\n",
    "        \"min\":0,\n",
    "        \"max\":10,\n",
    "        \"step\":0.01,\n",
    "        \"placeholder\":\"e.g. 0.01\",\n",
    "        \"label\":\"Smoothing\"\n",
    "    },\n",
    "    \"num_trees\":{\n",
    "        \"grid\":\"Y\",\n",
    "        \"type\":\"number\",\n",
    "        \"min\":0,\n",
    "        \"max\":10,\n",
    "        \"placeholder\":\"e.g. 4\",\n",
    "        \"label\":\"Number of Trees\"\n",
    "    },\n",
    "    \"distribution_family\":{\n",
    "        \"grid\":\"N\",\n",
    "        \"type\":\"select\",\n",
    "        \"options\":[\"Gaussian\",\"Binomial\",\"Poisson\",\"Gamma\",\"Tweedie\"],\n",
    "        \"label\":\"Distribution Family\"\n",
    "    },\n",
    "    \"k\":{\n",
    "        \"grid\":\"N\",\n",
    "        \"type\":\"number\",\n",
    "        \"min\":2,\n",
    "        \"max\":10,\n",
    "        \"placeholder\":\"e.g. 4\",\n",
    "        \"label\":\"Number of Clusters (k)\"\n",
    "    },\n",
    "    \"pca_k\":{\n",
    "        \"grid\":\"N\",\n",
    "        \"type\":\"number\",\n",
    "        \"min\":1,\n",
    "        \"max\":20,\n",
    "        \"placeholder\":\"e.g. 4\",\n",
    "        \"label\":\"Number of Principal Components (k)\"\n",
    "    },\n",
    "    \"numTopFeatures\":{\n",
    "        \"grid\":\"N\",\n",
    "        \"type\":\"number\",\n",
    "        \"min\":1,\n",
    "        \"max\":20,\n",
    "        \"placeholder\":\"e.g. 4\",\n",
    "        \"label\":\"Number of Top Features to Select\"\n",
    "    },\n",
    "    \"ngrams_n\":{\n",
    "        \"grid\":\"N\",\n",
    "        \"type\":\"number\",\n",
    "        \"min\":2,\n",
    "        \"max\":5,\n",
    "        \"placeholder\":\"e.g. 2\",\n",
    "        \"label\":\"N\"\n",
    "    },\n",
    "    \"tfidf_n\":{\n",
    "        \"grid\":\"N\",\n",
    "        \"type\":\"number\",\n",
    "        \"min\":2,\n",
    "        \"max\":30,\n",
    "        \"placeholder\":\"e.g. 2\",\n",
    "        \"label\":\"Number of TF-IDF Features\"\n",
    "    },\n",
    "    \"minSupport\":{\n",
    "        \"grid\":\"N\",\n",
    "        \"type\":\"number\",\n",
    "        \"min\":0,\n",
    "        \"max\":1,\n",
    "        \"step\":0.001,\n",
    "        \"placeholder\":\"e.g. 0.01\",\n",
    "        \"label\":\"Minimal Support\"\n",
    "    },\n",
    "    \"minConfidence\":{\n",
    "        \"grid\":\"N\",\n",
    "        \"type\":\"number\",\n",
    "        \"min\":0,\n",
    "        \"max\":1,\n",
    "        \"step\":0.001,\n",
    "        \"placeholder\":\"e.g. 0.01\",\n",
    "        \"label\":\"Minimal Confidence\"\n",
    "    }\n",
    "}\n",
    "\n",
    "ALGORITHM_PARAMETERS = {\n",
    "    \"OLS\":[\"maxiter\",\"regParam\",\"elasticnet\"],\n",
    "    \"LOGRE\":[\"maxiter\",\"regParam\",\"elasticnet\"],\n",
    "    \"DTR\":[\"max_depth\",\"max_bins\",\"min_instances_per_node\",\"min_info_gain\"],\n",
    "    \"DTC\":[\"max_depth\",\"max_bins\",\"min_instances_per_node\",\"min_info_gain\"],\n",
    "    \"MLP\":[\"stepSize\",\"seed\",\"maxiter\",\"nodes_l2\",\"nodes_l1\"], #\"convergence_tolerance\" removed for now\n",
    "    \"NB\":[\"smoothing\"],\n",
    "    \"GLM\":[\"maxiter\",\"regParam\",\"distribution_family\"],\n",
    "    \"RFR\":[\"max_depth\",\"max_bins\",\"min_instances_per_node\",\"min_info_gain\",\"num_trees\"],\n",
    "    \"RFC\":[\"max_depth\",\"max_bins\",\"min_instances_per_node\",\"min_info_gain\",\"num_trees\"],\n",
    "    \"GBTR\":[\"max_depth\",\"max_bins\",\"min_instances_per_node\",\"min_info_gain\",\"maxiter\"],\n",
    "    \"GBTC\":[\"max_depth\",\"max_bins\",\"min_instances_per_node\",\"min_info_gain\",\"maxiter\"],\n",
    "    \"KMEANS\":[\"k\",\"seed\"],\n",
    "    \"GAUSSMIX\":[\"k\",\"seed\"],\n",
    "    \"PCA\":[\"pca_k\"],\n",
    "    \"ChiSquared\":[\"numTopFeatures\"],\n",
    "    \"ALS\":[\"maxiter\",\"regParam\"],\n",
    "    \"TOKENIZER\":[],\n",
    "    \"N-GRAMS\":[\"ngrams_n\"],\n",
    "    \"TF-IDF\":[\"tfidf_n\"],\n",
    "    \"FPGrowth\":[\"minConfidence\",\"minSupport\"]\n",
    "}\n",
    "\n",
    "#TODO check if possible to have the algorithm definitions only in Python - currently also in JS\n",
    "ALGORITHMS = {\n",
    "    \"DIMRE_FE\":[\"PCA\",\"ChiSquared\"],\n",
    "    \"NLP\":[\"TOKENIZER\",\"N-GRAMS\",\"TF-IDF\"],\n",
    "    \"RECOM\":[\"ALS\"],\n",
    "    \"CLUST\":[\"KMEANS\",\"GAUSSMIX\"],\n",
    "    \"CL_REG\":[\"OLS\",\"DTR\",\"DTC\",\"MLP\",\"NB\",\"GLM\",\"RFR\",\"GBTR\",\"RFC\",\"GBTC\",\"LOGRE\"],\n",
    "    \"FREQPM\":[\"FPGrowth\"]\n",
    "}\n",
    "\n",
    "\n",
    "ALGORITHM_DESCRIPTIONS = {\n",
    "    \n",
    "}       \n",
    "\n",
    "# ----------- Input & Output -----------\n",
    "\n",
    "def read_dataset(selectedFile,separator):\n",
    "    global tempDF\n",
    "    global g\n",
    "    try:\n",
    "        tempDF = spark.read.csv(selectedFile,header=True, inferSchema=True, sep=separator)\n",
    "    except Exception as e:\n",
    "        g=e\n",
    "    return tempDF.columns\n",
    "\n",
    "def getTempColumns():\n",
    "    global tempDF\n",
    "    return tempDF.columns\n",
    "\n",
    "# used to retrieve available Datasets also\n",
    "def find_current_projectID(pDict):\n",
    "    projectsDict = json.loads(pDict)\n",
    "    pname = hdfs.project_name()\n",
    "    return projectsDict[pname] #KeyError should not be possible here, so no try-except...\n",
    "\n",
    "#used to find csv files in selected dataset\n",
    "def walk_dataset(dataset_path):\n",
    "    fileSet = set()\n",
    "    hdfs_start_path = \"hdfs://\"+myhdfs.host+\":\"+str(myhdfs.port)\n",
    "    for f in myhdfs.walk(hdfs_start_path + dataset_path):\n",
    "        fileSet.add(f['name'])\n",
    "    available_files = [x for x in list(fileSet) if x.endswith(\".csv\")]\n",
    "    return available_files\n",
    "\n",
    "#used for preview\n",
    "def get_tempdf_10first_lines_new():\n",
    "    result_temp = tempDF   \n",
    "    result_temp = result_temp.head(10)\n",
    "    result_final = []\n",
    "    for i in result_temp:\n",
    "        k = [i.asDict()[x] for x in tempDF.columns]\n",
    "        result_final.append(k)\n",
    "    return result_final\n",
    "\n",
    "\n",
    "def saveDataframe(savepath, df):\n",
    "    global g\n",
    "    hdfs_start_path = \"hdfs://\"+myhdfs.host+\":\"+str(myhdfs.port)\n",
    "    completepath = hdfs_start_path + savepath\n",
    "    try:\n",
    "        df.write.save(path=savepath, mode='append', format=\"parquet\")\n",
    "        return 200\n",
    "    except Exception as e:\n",
    "        g=e\n",
    "\n",
    "# helper function to transform input column, used by FPGrowth\n",
    "def colToArray(s):\n",
    "    input = [str(int(x)) for x in s[1:-1].split(\",\")]\n",
    "    return input\n",
    "\n",
    "# ----------- Algorithms -----------\n",
    "\n",
    "# Algo Select and Configure\n",
    "\n",
    "def get_algorithm_parameters(algorithm):\n",
    "    pdict = {}\n",
    "    if algorithm in ALGORITHM_PARAMETERS:\n",
    "        for p in ALGORITHM_PARAMETERS[algorithm]:\n",
    "            pdict[p] = PARAMETERS[p]\n",
    "    return {algorithm:pdict}\n",
    "\n",
    "\n",
    "\n",
    "def get_algos_in_family(family):\n",
    "    return ALGORITHMS[family]\n",
    "\n",
    "# Algo execution\n",
    "\n",
    "def get_validator(apply_train_test_split,pipeline,paramGrid,evaluator,evalVal):\n",
    "    if apply_train_test_split:\n",
    "        val = TrainValidationSplit(estimator=pipeline,\n",
    "                          estimatorParamMaps=paramGrid,\n",
    "                          evaluator=evaluator,\n",
    "                          trainRatio=float(evalVal)/100) \n",
    "    else:\n",
    "        val = CrossValidator(estimator=pipeline,\n",
    "                          estimatorParamMaps=paramGrid,\n",
    "                          evaluator=evaluator,\n",
    "                          numFolds=int(evalVal))\n",
    "    return val\n",
    "\n",
    "#TODO add modelType param\n",
    "def run_naive_bayes(algo_configuration):\n",
    "    global model\n",
    "    training = tempDF\n",
    "    smoothing_min = float(algo_configuration[\"Smoothing MinValue\"])\n",
    "    smoothing_max = float(algo_configuration[\"Smoothing MaxValue\"])\n",
    "    apply_train_test_split = True if algo_configuration[\"eval\"] == 'train-test' else False\n",
    "    evalVal = algo_configuration[\"evalVal\"]\n",
    "    nb = NaiveBayes(modelType=\"multinomial\")    \n",
    "    pipeline = Pipeline(stages=[nb])\n",
    "    paramGrid = ParamGridBuilder() \\\n",
    "    .addGrid(nb.smoothing, [smoothing_min, smoothing_max]) \\\n",
    "    .build()\n",
    "    evaluator=MulticlassClassificationEvaluator()\n",
    "    val = get_validator(apply_train_test_split,pipeline,paramGrid,evaluator,evalVal)\n",
    "    assembler = VectorAssembler(inputCols=algo_configuration[\"features\"],outputCol=\"features\")\n",
    "    output = assembler.transform(training)\n",
    "    output = output.withColumnRenamed(algo_configuration[\"label\"],\"label\").select(\"label\",\"features\")\n",
    "    cvModel = val.fit(output)\n",
    "    final_model = cvModel.bestModel\n",
    "    final_model.write().overwrite().save(algo_configuration[\"output_path\"])\n",
    "    model = final_model\n",
    "    resDict = {\"Theta\":str(final_model.stages[0].theta).encode(\"utf-8\"),\n",
    "              \"Output\":\"The created model has been saved in the selected output folder.\"}\n",
    "    return {\"results\":resDict}\n",
    "\n",
    "def run_decision_tree_classifier(algo_configuration):\n",
    "    global model\n",
    "    training = tempDF\n",
    "    minInfoGain_min = float(algo_configuration[\"Minimum Info Gain MinValue\"])\n",
    "    minInfoGain_max = float(algo_configuration[\"Minimum Info Gain MaxValue\"])\n",
    "    maxDepth_min = int(algo_configuration[\"Maximum Depth MinValue\"])\n",
    "    maxDepth_max = int(algo_configuration[\"Maximum Depth MaxValue\"])\n",
    "    maxBins_min = int(algo_configuration[\"Maximum Bins MinValue\"])\n",
    "    maxBins_max = int(algo_configuration[\"Maximum Bins MaxValue\"])\n",
    "    minInstancesPerNode_min = int(algo_configuration[\"Minimum Instances per Node MinValue\"])\n",
    "    minInstancesPerNode_max = int(algo_configuration[\"Minimum Instances per Node MaxValue\"])\n",
    "    apply_train_test_split = True if algo_configuration[\"eval\"] == 'train-test' else False\n",
    "    evalVal = algo_configuration[\"evalVal\"]\n",
    "    assembler = VectorAssembler(inputCols=algo_configuration[\"features\"],outputCol=\"features\")\n",
    "    output = assembler.transform(training)\n",
    "    output = output.withColumnRenamed(algo_configuration[\"label\"],\"label\").select(\"label\",\"features\")\n",
    "    labelIndexer = StringIndexer(inputCol=\"label\", outputCol=\"indexedLabel\").fit(output)\n",
    "    featureIndexer =\\\n",
    "    VectorIndexer(inputCol=\"features\", outputCol=\"indexedFeatures\", maxCategories=4).fit(output)\n",
    "    dt = DecisionTreeClassifier(labelCol=\"indexedLabel\", featuresCol=\"indexedFeatures\")\n",
    "    pipeline = Pipeline(stages=[labelIndexer, featureIndexer, dt])\n",
    "    paramGrid = ParamGridBuilder() \\\n",
    "    .addGrid(dt.minInfoGain, [minInfoGain_min, minInfoGain_max]) \\\n",
    "    .addGrid(dt.maxDepth, [maxDepth_min, maxDepth_max]) \\\n",
    "    .addGrid(dt.maxBins, [maxBins_min, maxBins_max]) \\\n",
    "    .addGrid(dt.minInstancesPerNode, [minInstancesPerNode_min, minInstancesPerNode_max]) \\\n",
    "    .build()\n",
    "    evaluator=MulticlassClassificationEvaluator()\n",
    "    val = get_validator(apply_train_test_split,pipeline,paramGrid,evaluator,evalVal)\n",
    "    cvModel = val.fit(output)\n",
    "    final_model = cvModel.bestModel\n",
    "    final_model.write().overwrite().save(algo_configuration[\"output_path\"])\n",
    "    treeDict = {\"Tree\":final_model.stages[2].toDebugString.encode(\"utf-8\"),\n",
    "               \"Output\":\"The created model has been saved in the selected output folder.\"}\n",
    "    model = final_model\n",
    "    return {\"results\":treeDict}\n",
    "\n",
    "def run_decision_tree_regressor(algo_configuration):\n",
    "    global g\n",
    "    global model\n",
    "    g = algo_configuration\n",
    "    training = tempDF\n",
    "    minInfoGain_min = float(algo_configuration[\"Minimum Info Gain MinValue\"])\n",
    "    minInfoGain_max = float(algo_configuration[\"Minimum Info Gain MaxValue\"])\n",
    "    maxDepth_min = int(algo_configuration[\"Maximum Depth MinValue\"])\n",
    "    maxDepth_max = int(algo_configuration[\"Maximum Depth MaxValue\"])\n",
    "    maxBins_min = int(algo_configuration[\"Maximum Bins MinValue\"])\n",
    "    maxBins_max = int(algo_configuration[\"Maximum Bins MaxValue\"])\n",
    "    minInstancesPerNode_min = int(algo_configuration[\"Minimum Instances per Node MinValue\"])\n",
    "    minInstancesPerNode_max = int(algo_configuration[\"Minimum Instances per Node MaxValue\"])\n",
    "    apply_train_test_split = True if algo_configuration[\"eval\"] == 'train-test' else False\n",
    "    evalVal = algo_configuration[\"evalVal\"]\n",
    "    assembler = VectorAssembler(inputCols=algo_configuration[\"features\"],outputCol=\"features\")\n",
    "    output = assembler.transform(training)\n",
    "    output = output.withColumnRenamed(algo_configuration[\"label\"],\"label\").select(\"label\",\"features\")\n",
    "    labelIndexer = StringIndexer(inputCol=\"label\", outputCol=\"indexedLabel\").fit(output)\n",
    "    featureIndexer =\\\n",
    "    VectorIndexer(inputCol=\"features\", outputCol=\"indexedFeatures\", maxCategories=4).fit(output)\n",
    "    dt = DecisionTreeRegressor(labelCol=\"indexedLabel\", featuresCol=\"indexedFeatures\")\n",
    "    pipeline = Pipeline(stages=[labelIndexer, featureIndexer, dt])\n",
    "    paramGrid = ParamGridBuilder() \\\n",
    "    .addGrid(dt.minInfoGain, [minInfoGain_min, minInfoGain_max]) \\\n",
    "    .addGrid(dt.maxDepth, [maxDepth_min, maxDepth_max]) \\\n",
    "    .addGrid(dt.maxBins, [maxBins_min, maxBins_max]) \\\n",
    "    .addGrid(dt.minInstancesPerNode, [minInstancesPerNode_min, minInstancesPerNode_max]) \\\n",
    "    .build()\n",
    "    evaluator=RegressionEvaluator()\n",
    "#     val = TrainValidationSplit(estimator=pipeline,\n",
    "#                            estimatorParamMaps=paramGrid,\n",
    "#                            evaluator=evaluator,\n",
    "#                            trainRatio=0.8) \n",
    "    val = get_validator(apply_train_test_split,pipeline,paramGrid,evaluator,evalVal)\n",
    "    cvModel = val.fit(output)\n",
    "    final_model = cvModel.bestModel\n",
    "    final_model.write().overwrite().save(algo_configuration[\"output_path\"])\n",
    "    treeDict = {\"Tree\":final_model.stages[2].toDebugString.encode(\"utf-8\"),\n",
    "               \"Output\":\"The created model has been saved in the selected output folder.\"}\n",
    "    model = final_model\n",
    "    return {\"results\":treeDict}\n",
    "\n",
    "#TODO add the remaining parameters & give a \"prettier\" result\n",
    "def run_random_forest_classifier(algo_configuration):\n",
    "    global model\n",
    "    training = tempDF\n",
    "#     elasticNetParam = float(algo_configuration[\"Elastic Net Parameter\"])\n",
    "#     regParam = float(algo_configuration[\"Regularisation Parameter\"])\n",
    "    maxDepth_min = int(algo_configuration[\"Maximum Depth MinValue\"])\n",
    "    maxDepth_max = int(algo_configuration[\"Maximum Depth MaxValue\"])\n",
    "    maxBins_min = int(algo_configuration[\"Maximum Bins MinValue\"])\n",
    "    maxBins_max = int(algo_configuration[\"Maximum Bins MaxValue\"])\n",
    "    minInstancesPerNode_min = int(algo_configuration[\"Minimum Instances per Node MinValue\"])\n",
    "    minInstancesPerNode_max = int(algo_configuration[\"Minimum Instances per Node MaxValue\"])\n",
    "    numTrees_min = int(algo_configuration[\"Number of Trees MinValue\"])\n",
    "    numTrees_max = int(algo_configuration[\"Number of Trees MaxValue\"])\n",
    "    minInfoGain_min = float(algo_configuration[\"Minimum Info Gain MinValue\"])\n",
    "    minInfoGain_max = float(algo_configuration[\"Minimum Info Gain MaxValue\"])\n",
    "    apply_train_test_split = True if algo_configuration[\"eval\"] == 'train-test' else False\n",
    "    evalVal = algo_configuration[\"evalVal\"]\n",
    "    \n",
    "    assembler = VectorAssembler(inputCols=algo_configuration[\"features\"],outputCol=\"features\")\n",
    "    output = assembler.transform(training)\n",
    "    output = output.withColumnRenamed(algo_configuration[\"label\"],\"label\").select(\"label\",\"features\")\n",
    "    labelIndexer = StringIndexer(inputCol=\"label\", outputCol=\"indexedLabel\").fit(output)\n",
    "    featureIndexer =\\\n",
    "    VectorIndexer(inputCol=\"features\", outputCol=\"indexedFeatures\", maxCategories=4).fit(output)\n",
    "    \n",
    "    rf = RandomForestClassifier(labelCol=\"indexedLabel\", featuresCol=\"indexedFeatures\")\n",
    "    pipeline = Pipeline(stages=[labelIndexer, featureIndexer, rf])\n",
    "    \n",
    "    paramGrid = ParamGridBuilder() \\\n",
    "    .addGrid(rf.numTrees, [numTrees_min, numTrees_max]) \\\n",
    "    .addGrid(rf.maxDepth, [maxDepth_min, maxDepth_max]) \\\n",
    "    .addGrid(rf.maxBins, [maxBins_min, maxBins_max]) \\\n",
    "    .addGrid(rf.minInfoGain, [minInfoGain_min, minInfoGain_max]) \\\n",
    "    .addGrid(rf.minInstancesPerNode, [minInstancesPerNode_min, minInstancesPerNode_max]) \\\n",
    "    .build()\n",
    "    evaluator=MulticlassClassificationEvaluator()\n",
    "    val = get_validator(apply_train_test_split,pipeline,paramGrid,evaluator,evalVal)\n",
    "    cvModel = val.fit(output)\n",
    "    final_model = cvModel.bestModel\n",
    "    final_model.write().overwrite().save(algo_configuration[\"output_path\"])\n",
    "    treeDict = {\"Tree\":final_model.stages[2].toDebugString.encode(\"utf-8\"),\n",
    "               \"Output\":\"The created model has been saved in the selected output folder.\"}\n",
    "    model = final_model\n",
    "    return {\"results\":treeDict}\n",
    "\n",
    "def run_random_forest_regressor(algo_configuration):\n",
    "    global model\n",
    "    training = tempDF\n",
    "    maxDepth_min = int(algo_configuration[\"Maximum Depth MinValue\"])\n",
    "    maxDepth_max = int(algo_configuration[\"Maximum Depth MaxValue\"])\n",
    "    maxBins_min = int(algo_configuration[\"Maximum Bins MinValue\"])\n",
    "    maxBins_max = int(algo_configuration[\"Maximum Bins MaxValue\"])\n",
    "    minInstancesPerNode_min = int(algo_configuration[\"Minimum Instances per Node MinValue\"])\n",
    "    minInstancesPerNode_max = int(algo_configuration[\"Minimum Instances per Node MaxValue\"])\n",
    "    numTrees_min = int(algo_configuration[\"Number of Trees MinValue\"])\n",
    "    numTrees_max = int(algo_configuration[\"Number of Trees MaxValue\"])\n",
    "    minInfoGain_min = float(algo_configuration[\"Minimum Info Gain MinValue\"])\n",
    "    minInfoGain_max = float(algo_configuration[\"Minimum Info Gain MaxValue\"])\n",
    "    apply_train_test_split = True if algo_configuration[\"eval\"] == 'train-test' else False\n",
    "    evalVal = algo_configuration[\"evalVal\"]\n",
    "    \n",
    "    assembler = VectorAssembler(inputCols=algo_configuration[\"features\"],outputCol=\"features\")\n",
    "    output = assembler.transform(training)\n",
    "    output = output.withColumnRenamed(algo_configuration[\"label\"],\"label\").select(\"label\",\"features\")\n",
    "    labelIndexer = StringIndexer(inputCol=\"label\", outputCol=\"indexedLabel\").fit(output)\n",
    "    featureIndexer =\\\n",
    "    VectorIndexer(inputCol=\"features\", outputCol=\"indexedFeatures\", maxCategories=4).fit(output)\n",
    "    \n",
    "    rf = RandomForestRegressor(labelCol=\"indexedLabel\", featuresCol=\"indexedFeatures\")\n",
    "    pipeline = Pipeline(stages=[labelIndexer, featureIndexer, rf])\n",
    "    \n",
    "    paramGrid = ParamGridBuilder() \\\n",
    "    .addGrid(rf.numTrees, [numTrees_min, numTrees_max]) \\\n",
    "    .addGrid(rf.maxDepth, [maxDepth_min, maxDepth_max]) \\\n",
    "    .addGrid(rf.maxBins, [maxBins_min, maxBins_max]) \\\n",
    "    .addGrid(rf.minInfoGain, [minInfoGain_min, minInfoGain_max]) \\\n",
    "    .addGrid(rf.minInstancesPerNode, [minInstancesPerNode_min, minInstancesPerNode_max]) \\\n",
    "    .build()\n",
    "    evaluator=RegressionEvaluator()\n",
    "    val = get_validator(apply_train_test_split,pipeline,paramGrid,evaluator,evalVal)\n",
    "    cvModel = val.fit(output)\n",
    "    final_model = cvModel.bestModel\n",
    "    final_model.write().overwrite().save(algo_configuration[\"output_path\"])\n",
    "    treeDict = {\"Tree\":final_model.stages[2].toDebugString.encode(\"utf-8\"),\n",
    "               \"Output\":\"The created model has been saved in the selected output folder.\"}\n",
    "    model = final_model\n",
    "    return {\"results\":treeDict}\n",
    "\n",
    "# TODO: add note that it currently only supports binary classification\n",
    "def run_gradient_boosted_tree_classifier(algo_configuration):\n",
    "    global model\n",
    "    training = tempDF\n",
    "    maxDepth_min = int(algo_configuration[\"Maximum Depth MinValue\"])\n",
    "    maxDepth_max = int(algo_configuration[\"Maximum Depth MaxValue\"])\n",
    "    maxBins_min = int(algo_configuration[\"Maximum Bins MinValue\"])\n",
    "    maxBins_max = int(algo_configuration[\"Maximum Bins MaxValue\"])\n",
    "    minInstancesPerNode_min = int(algo_configuration[\"Minimum Instances per Node MinValue\"])\n",
    "    minInstancesPerNode_max = int(algo_configuration[\"Minimum Instances per Node MaxValue\"])\n",
    "    maxIter_min = int(algo_configuration[\"Maximum Iterations MinValue\"])\n",
    "    maxIter_max = int(algo_configuration[\"Maximum Iterations MaxValue\"])\n",
    "    minInfoGain_min = float(algo_configuration[\"Minimum Info Gain MinValue\"])\n",
    "    minInfoGain_max = float(algo_configuration[\"Minimum Info Gain MaxValue\"])\n",
    "    apply_train_test_split = True if algo_configuration[\"eval\"] == 'train-test' else False\n",
    "    evalVal = algo_configuration[\"evalVal\"]\n",
    "    \n",
    "    assembler = VectorAssembler(inputCols=algo_configuration[\"features\"],outputCol=\"features\")\n",
    "    output = assembler.transform(training)\n",
    "    output = output.withColumnRenamed(algo_configuration[\"label\"],\"label\").select(\"label\",\"features\")\n",
    "    labelIndexer = StringIndexer(inputCol=\"label\", outputCol=\"indexedLabel\").fit(output)\n",
    "    featureIndexer =\\\n",
    "    VectorIndexer(inputCol=\"features\", outputCol=\"indexedFeatures\", maxCategories=4).fit(output)\n",
    "    \n",
    "    gbt = GBTClassifier(labelCol=\"indexedLabel\", featuresCol=\"indexedFeatures\")\n",
    "    pipeline = Pipeline(stages=[labelIndexer, featureIndexer, gbt])\n",
    "    \n",
    "    paramGrid = ParamGridBuilder() \\\n",
    "    .addGrid(gbt.maxIter, [maxIter_min, maxIter_max]) \\\n",
    "    .addGrid(gbt.maxDepth, [maxDepth_min, maxDepth_max]) \\\n",
    "    .addGrid(gbt.maxBins, [maxBins_min, maxBins_max]) \\\n",
    "    .addGrid(gbt.minInfoGain, [minInfoGain_min, minInfoGain_max]) \\\n",
    "    .addGrid(gbt.minInstancesPerNode, [minInstancesPerNode_min, minInstancesPerNode_max]) \\\n",
    "    .build()\n",
    "    evaluator=BinaryClassificationEvaluator()\n",
    "    val = get_validator(apply_train_test_split,pipeline,paramGrid,evaluator,evalVal)\n",
    "    cvModel = val.fit(output)\n",
    "    final_model = cvModel.bestModel\n",
    "    final_model.write().overwrite().save(algo_configuration[\"output_path\"])\n",
    "    treeDict = {\"Tree\":final_model.stages[2].toDebugString.encode(\"utf-8\"),\n",
    "               \"Output\":\"The created model has been saved in the selected output folder.\"}\n",
    "    model = final_model\n",
    "    return {\"results\":treeDict}\n",
    "\n",
    "#TODO note that this is very demanding in time (and resources?)\n",
    "def run_gradient_boosted_tree_regressor(algo_configuration):\n",
    "    global model\n",
    "    training = tempDF\n",
    "    maxDepth_min = int(algo_configuration[\"Maximum Depth MinValue\"])\n",
    "    maxDepth_max = int(algo_configuration[\"Maximum Depth MaxValue\"])\n",
    "    maxBins_min = int(algo_configuration[\"Maximum Bins MinValue\"])\n",
    "    maxBins_max = int(algo_configuration[\"Maximum Bins MaxValue\"])\n",
    "    minInstancesPerNode_min = int(algo_configuration[\"Minimum Instances per Node MinValue\"])\n",
    "    minInstancesPerNode_max = int(algo_configuration[\"Minimum Instances per Node MaxValue\"])\n",
    "    maxIter_min = int(algo_configuration[\"Maximum Iterations MinValue\"])\n",
    "    maxIter_max = int(algo_configuration[\"Maximum Iterations MaxValue\"])\n",
    "    minInfoGain_min = float(algo_configuration[\"Minimum Info Gain MinValue\"])\n",
    "    minInfoGain_max = float(algo_configuration[\"Minimum Info Gain MaxValue\"])\n",
    "    apply_train_test_split = True if algo_configuration[\"eval\"] == 'train-test' else False\n",
    "    evalVal = algo_configuration[\"evalVal\"]\n",
    "    \n",
    "    assembler = VectorAssembler(inputCols=algo_configuration[\"features\"],outputCol=\"features\")\n",
    "    output = assembler.transform(training)\n",
    "    output = output.withColumnRenamed(algo_configuration[\"label\"],\"label\").select(\"label\",\"features\")\n",
    "    labelIndexer = StringIndexer(inputCol=\"label\", outputCol=\"indexedLabel\").fit(output)\n",
    "    featureIndexer =\\\n",
    "    VectorIndexer(inputCol=\"features\", outputCol=\"indexedFeatures\", maxCategories=4).fit(output)\n",
    "    \n",
    "    gbt = GBTRegressor(labelCol=\"indexedLabel\", featuresCol=\"indexedFeatures\")\n",
    "    pipeline = Pipeline(stages=[labelIndexer, featureIndexer, gbt])\n",
    "    \n",
    "    paramGrid = ParamGridBuilder() \\\n",
    "    .addGrid(gbt.maxIter, [maxIter_min, maxIter_max]) \\\n",
    "    .addGrid(gbt.maxDepth, [maxDepth_min, maxDepth_max]) \\\n",
    "    .addGrid(gbt.maxBins, [maxBins_min, maxBins_max]) \\\n",
    "    .addGrid(gbt.minInfoGain, [minInfoGain_min, minInfoGain_max]) \\\n",
    "    .addGrid(gbt.minInstancesPerNode, [minInstancesPerNode_min, minInstancesPerNode_max]) \\\n",
    "    .build()\n",
    "    evaluator=RegressionEvaluator()\n",
    "    val = get_validator(apply_train_test_split,pipeline,paramGrid,evaluator,evalVal)\n",
    "    cvModel = val.fit(output)\n",
    "    final_model = cvModel.bestModel\n",
    "    final_model.write().overwrite().save(algo_configuration[\"output_path\"])\n",
    "    treeDict = {\"Tree\":final_model.stages[2].toDebugString.encode(\"utf-8\"),\n",
    "               \"Output\":\"The created model has been saved in the selected output folder.\"}\n",
    "    model = final_model\n",
    "    return {\"results\":treeDict}\n",
    "\n",
    "def run_logistic_regression(algo_configuration):\n",
    "    global model\n",
    "    training = tempDF\n",
    "    elasticNetParam_min = float(algo_configuration[\"Elastic Net Parameter MinValue\"])\n",
    "    elasticNetParam_max = float(algo_configuration[\"Elastic Net Parameter MaxValue\"])\n",
    "    regParam_min = float(algo_configuration[\"Regularisation Parameter MinValue\"])\n",
    "    regParam_max = float(algo_configuration[\"Regularisation Parameter MaxValue\"])\n",
    "    maxIter_min = int(algo_configuration[\"Maximum Iterations MinValue\"])\n",
    "    maxIter_max = int(algo_configuration[\"Maximum Iterations MaxValue\"])\n",
    "    apply_train_test_split = True if algo_configuration[\"eval\"] == 'train-test' else False\n",
    "    evalVal = algo_configuration[\"evalVal\"]\n",
    "    lr = LogisticRegression()\n",
    "    pipeline = Pipeline(stages=[lr])\n",
    "    paramGrid = ParamGridBuilder() \\\n",
    "    .addGrid(lr.regParam, [regParam_min, regParam_max]) \\\n",
    "    .addGrid(lr.elasticNetParam, [elasticNetParam_min, elasticNetParam_max]) \\\n",
    "    .addGrid(lr.maxIter, [maxIter_min, maxIter_max]) \\\n",
    "    .build()\n",
    "    evaluator=MulticlassClassificationEvaluator()\n",
    "    val = get_validator(apply_train_test_split,pipeline,paramGrid,evaluator,evalVal)\n",
    "    assembler = VectorAssembler(inputCols=algo_configuration[\"features\"],outputCol=\"features\")\n",
    "    output = assembler.transform(training)\n",
    "    output = output.withColumnRenamed(algo_configuration[\"label\"],\"label\").select(\"label\",\"features\")\n",
    "    cvModel = val.fit(output)\n",
    "    final_model = cvModel.bestModel\n",
    "    final_model.write().overwrite().save(algo_configuration[\"output_path\"])\n",
    "    trainingSummary = final_model.stages[0].summary\n",
    "    summaryDict = {}\n",
    "    summaryDict[\"accuracy\"] = trainingSummary.accuracy\n",
    "    summaryDict[\"falsePositiveRate\"] = trainingSummary.weightedFalsePositiveRate\n",
    "    summaryDict[\"truePositiveRate\"] = trainingSummary.weightedTruePositiveRate\n",
    "    summaryDict[\"fMeasure\"] = trainingSummary.weightedFMeasure()\n",
    "    summaryDict[\"precision\"] = trainingSummary.weightedPrecision\n",
    "    summaryDict[\"recall\"] = trainingSummary.weightedRecall\n",
    "    summaryDict[\"Output\"] = \"The created model has been saved in the selected output folder.\"\n",
    "    model = final_model\n",
    "    return {\"results\":summaryDict}\n",
    "\n",
    "def run_linear_regression(algo_configuration):\n",
    "    global model\n",
    "    training = tempDF\n",
    "    elasticNetParam_min = float(algo_configuration[\"Elastic Net Parameter MinValue\"])\n",
    "    elasticNetParam_max = float(algo_configuration[\"Elastic Net Parameter MaxValue\"])\n",
    "    regParam_min = float(algo_configuration[\"Regularisation Parameter MinValue\"])\n",
    "    regParam_max = float(algo_configuration[\"Regularisation Parameter MaxValue\"])\n",
    "    maxIter_min = int(algo_configuration[\"Maximum Iterations MinValue\"])\n",
    "    maxIter_max = int(algo_configuration[\"Maximum Iterations MaxValue\"])\n",
    "    apply_train_test_split = True if algo_configuration[\"eval\"] == 'train-test' else False\n",
    "    evalVal = algo_configuration[\"evalVal\"]\n",
    "    lr = LinearRegression()\n",
    "    pipeline = Pipeline(stages=[lr])\n",
    "    paramGrid = ParamGridBuilder() \\\n",
    "    .addGrid(lr.regParam, [regParam_min, regParam_max]) \\\n",
    "    .addGrid(lr.elasticNetParam, [elasticNetParam_min, elasticNetParam_max]) \\\n",
    "    .addGrid(lr.maxIter, [maxIter_min, maxIter_max]) \\\n",
    "    .build()\n",
    "    evaluator=RegressionEvaluator()\n",
    "    val = get_validator(apply_train_test_split,pipeline,paramGrid,evaluator,evalVal)\n",
    "    assembler = VectorAssembler(inputCols=algo_configuration[\"features\"],outputCol=\"features\")\n",
    "    output = assembler.transform(training)\n",
    "    output = output.withColumnRenamed(algo_configuration[\"label\"],\"label\").select(\"label\",\"features\")\n",
    "    cvModel = val.fit(output)\n",
    "    final_model = cvModel.bestModel\n",
    "    final_model.write().overwrite().save(algo_configuration[\"output_path\"])\n",
    "    trainingSummary = final_model.stages[0].summary\n",
    "    summaryDict = {}\n",
    "    summaryDict[\"Number of Iterations\"] = trainingSummary.totalIterations\n",
    "    summaryDict[\"Objective History\"] = str(trainingSummary.objectiveHistory)\n",
    "    summaryDict[\"RMSE\"] = str(trainingSummary.rootMeanSquaredError)\n",
    "    summaryDict[\"r2\"] = str(trainingSummary.r2)\n",
    "    summaryDict[\"Deviance Residuals (first 10)\"] = str(np.array(trainingSummary.residuals.head(10)))\n",
    "    summaryDict[\"Output\"] = \"The created model has been saved in the selected output folder.\"\n",
    "    model = final_model\n",
    "    return {\"results\":summaryDict}\n",
    "\n",
    "def run_generalized_linear_regression(algo_configuration):\n",
    "    global model\n",
    "    training = tempDF\n",
    "    family = algo_configuration[\"Distribution Family\"].lower()\n",
    "    regParam_min = float(algo_configuration[\"Regularisation Parameter MinValue\"])\n",
    "    regParam_max = float(algo_configuration[\"Regularisation Parameter MaxValue\"])\n",
    "    maxIter_min = int(algo_configuration[\"Maximum Iterations MinValue\"])\n",
    "    maxIter_max = int(algo_configuration[\"Maximum Iterations MaxValue\"])\n",
    "    apply_train_test_split = True if algo_configuration[\"eval\"] == 'train-test' else False\n",
    "    evalVal = algo_configuration[\"evalVal\"]\n",
    "    gr = GeneralizedLinearRegression(family=family)\n",
    "    pipeline = Pipeline(stages=[gr])\n",
    "    paramGrid = ParamGridBuilder() \\\n",
    "    .addGrid(gr.regParam, [regParam_min, regParam_max]) \\\n",
    "    .addGrid(gr.maxIter, [maxIter_min, maxIter_max]) \\\n",
    "    .build()\n",
    "    evaluator=RegressionEvaluator()\n",
    "    val = get_validator(apply_train_test_split,pipeline,paramGrid,evaluator,evalVal)\n",
    "    assembler = VectorAssembler(inputCols=algo_configuration[\"features\"],outputCol=\"features\")\n",
    "    output = assembler.transform(training)\n",
    "    output = output.withColumnRenamed(algo_configuration[\"label\"],\"label\").select(\"label\",\"features\")\n",
    "    cvModel = val.fit(output)\n",
    "    final_model = cvModel.bestModel\n",
    "    final_model.write().overwrite().save(algo_configuration[\"output_path\"])\n",
    "    trainingSummary = final_model.stages[0].summary\n",
    "    summaryDict = {}\n",
    "    summaryDict[\"Coefficient Standard Errors\"] = trainingSummary.coefficientStandardErrors\n",
    "    summaryDict[\"T Values\"] = trainingSummary.tValues\n",
    "    summaryDict[\"P Values\"] = trainingSummary.pValues\n",
    "    summaryDict[\"Dispersion\"] = trainingSummary.dispersion\n",
    "    summaryDict[\"Null Deviance\"] = trainingSummary.nullDeviance\n",
    "    summaryDict[\"Residual Degree Of Freedom Null\"] = str(trainingSummary.residualDegreeOfFreedomNull)\n",
    "    summaryDict[\"Deviance\"] = str(trainingSummary.deviance)\n",
    "    summaryDict[\"Residual Degree Of Freedom\"] = str(trainingSummary.residualDegreeOfFreedom)\n",
    "    summaryDict[\"AIC\"] = trainingSummary.aic\n",
    "    summaryDict[\"Deviance Residuals (first 10)\"] = str(np.array(trainingSummary.residuals().head(10)))\n",
    "    summaryDict[\"Output\"] = \"The created model has been saved in the selected output folder.\"\n",
    "    model = final_model\n",
    "    return {\"results\":summaryDict}\n",
    "\n",
    "#TODO explain a bit what happens with hidden layers & notify about class labeling issue\n",
    "def run_multilayer_perceptron_classifier(algo_configuration):\n",
    "    global model\n",
    "    training = tempDF\n",
    "    stepSize_min = float(algo_configuration[\"Step Size MinValue\"])\n",
    "    stepSize_max = float(algo_configuration[\"Step Size MaxValue\"])\n",
    "    seed = int(algo_configuration[\"Seed\"])\n",
    "    maxIter_min = int(algo_configuration[\"Maximum Iterations MinValue\"])\n",
    "    maxIter_max = int(algo_configuration[\"Maximum Iterations MaxValue\"])\n",
    "    apply_train_test_split = True if algo_configuration[\"eval\"] == 'train-test' else False\n",
    "    evalVal = algo_configuration[\"evalVal\"]\n",
    "    \n",
    "    assembler = VectorAssembler(inputCols=algo_configuration[\"features\"],outputCol=\"features\")\n",
    "    output = assembler.transform(training)\n",
    "    output = output.withColumnRenamed(algo_configuration[\"label\"],\"label\").select(\"label\",\"features\")\n",
    "    \n",
    "    lastLayerNodes = output.select(\"label\").distinct().count()\n",
    "    firstLayerNodes = len(algo_configuration[\"features\"])\n",
    "    layers = [firstLayerNodes, 5, 4, lastLayerNodes]\n",
    "    \n",
    "    trainer = MultilayerPerceptronClassifier(layers=layers,blockSize=128, seed=1234)\n",
    "    pipeline = Pipeline(stages=[trainer])\n",
    "    paramGrid = ParamGridBuilder() \\\n",
    "    .addGrid(trainer.maxIter, [maxIter_min, maxIter_max]) \\\n",
    "    .addGrid(trainer.seed, [seed, seed]) \\\n",
    "    .addGrid(trainer.stepSize, [stepSize_min, stepSize_max]) \\\n",
    "    .build()\n",
    "    evaluator=MulticlassClassificationEvaluator()\n",
    "    val = get_validator(apply_train_test_split,pipeline,paramGrid,evaluator,evalVal)\n",
    "    cvModel = val.fit(output)\n",
    "    final_model = cvModel.bestModel\n",
    "    final_model.write().overwrite().save(algo_configuration[\"output_path\"])\n",
    "    resDict = {\"Weights\":str(final_model.stages[0].weights).encode(\"utf-8\"),\n",
    "              \"Output\":\"The created model has been saved in the selected output folder.\"}\n",
    "    model = final_model\n",
    "    return {\"results\":resDict}\n",
    "\n",
    "def run_kmeans_clustering(algo_configuration):\n",
    "    training = tempDF\n",
    "    k = float(algo_configuration[\"Number of Clusters (k)\"])\n",
    "    seed = int(algo_configuration[\"Seed\"])\n",
    "    assembler = VectorAssembler(inputCols=algo_configuration[\"features\"],outputCol=\"features\")\n",
    "    output = assembler.transform(training)\n",
    "    kmeans = KMeans().setK(k).setSeed(seed)\n",
    "    model = kmeans.fit(output)\n",
    "    model.write().overwrite().save(algo_configuration[\"output_path\"])\n",
    "    predictions = model.transform(output)\n",
    "    saveDataframe(algo_configuration[\"output_path\"]+\"/kmeans_predictions\",predictions)\n",
    "    evaluator = ClusteringEvaluator()\n",
    "    silhouette = evaluator.evaluate(predictions)\n",
    "    resultDict = {}\n",
    "    resultDict[\"Silhouette with squared euclidean distance\"] = str(silhouette)\n",
    "    centers = model.clusterCenters()\n",
    "    centers_string = \"\"\n",
    "    for center in centers:\n",
    "        centers_string+=str(center)\n",
    "    resultDict[\"Centers\"] = centers_string\n",
    "    resultDict[\"Output\"] = \"The cluster predictions and the created model have been saved in the selected output folder.\"\n",
    "    return {\"results\":resultDict}\n",
    "\n",
    "def run_gaussian_mixtures(algo_configuration):\n",
    "    training = tempDF\n",
    "    k = float(algo_configuration[\"Number of Clusters (k)\"])\n",
    "    seed = int(algo_configuration[\"Seed\"])\n",
    "    assembler = VectorAssembler(inputCols=algo_configuration[\"features\"],outputCol=\"features\")\n",
    "    output = assembler.transform(training)\n",
    "    gmm = GaussianMixture().setK(k).setSeed(seed)\n",
    "    model = gmm.fit(output)\n",
    "    model.write().overwrite().save(algo_configuration[\"output_path\"])\n",
    "    gaussiansDF = np.array(model.gaussiansDF.select('mean').collect())\n",
    "    saveDataframe(algo_configuration[\"output_path\"]+\"/gaussian_mixtures\",gaussiansDF)\n",
    "    resultDict = {}\n",
    "    resultDict[\"Gaussians Mean\"] = str(gaussiansDF)\n",
    "    resultDict[\"Output\"] = \"The created model and the Gaussians have been saved in the selected output folder.\"\n",
    "    return {\"results\":resultDict}\n",
    "\n",
    "def run_pca(algo_configuration):\n",
    "    training = tempDF\n",
    "    k = float(algo_configuration[\"Number of Principal Components (k)\"])\n",
    "    assembler = VectorAssembler(inputCols=algo_configuration[\"features\"],outputCol=\"features\")\n",
    "    output = assembler.transform(training)\n",
    "    pca = PCA(k=k, inputCol=\"features\", outputCol=\"pcaFeatures\")\n",
    "    model = pca.fit(output)\n",
    "    result = model.transform(output).select(\"pcaFeatures\")\n",
    "    model.write().overwrite().save(algo_configuration[\"output_path\"])\n",
    "    saveDataframe(algo_configuration[\"output_path\"]+\"/pca_features\",result)\n",
    "    resultDict = {}\n",
    "    resultDict[\"PCA Features (first 10 rows)\"] = str(np.array(result.head(20)))\n",
    "    resultDict[\"Output\"] = \"The created model and the PCA features have been saved in the selected output folder.\"\n",
    "    return {\"results\":resultDict}\n",
    "\n",
    "def run_chisquare(algo_configuration):\n",
    "    training = tempDF\n",
    "    numTopFeatures = float(algo_configuration[\"Number of Top Features to Select\"])\n",
    "    assembler = VectorAssembler(inputCols=algo_configuration[\"features\"],outputCol=\"features\")\n",
    "    output = assembler.transform(training)\n",
    "    selector = ChiSqSelector(numTopFeatures=numTopFeatures, featuresCol=\"features\",\n",
    "                         outputCol=\"selectedFeatures\", labelCol=algo_configuration[\"label\"])\n",
    "    model = selector.fit(output)\n",
    "    feature_names = [output.columns[x] for x in model.selectedFeatures]\n",
    "    result = model.transform(output)\n",
    "    model.write().overwrite().save(algo_configuration[\"output_path\"])\n",
    "    saveDataframe(algo_configuration[\"output_path\"]+\"/chisquare_result\",result)\n",
    "    resultDict = {}\n",
    "    resultDict[\"Selected Features\"] = feature_names\n",
    "    resultDict[\"Output\"] = \"The created model and the Dataframe containing current result features have been saved in the selected output folder.\"\n",
    "    return {\"results\":resultDict}\n",
    "\n",
    "# TODO: remove grid from als\n",
    "def run_als(algo_configuration):\n",
    "    ratings = tempDF\n",
    "    (training, test) = ratings.randomSplit([0.8, 0.2])\n",
    "    regParam = float(algo_configuration[\"Regularisation Parameter MinValue\"])\n",
    "    # regParam_max = float(algo_configuration[\"Regularisation Parameter MaxValue\"])\n",
    "    maxIter = int(algo_configuration[\"Maximum Iterations MaxValue\"])\n",
    "    # maxIter_max = int(algo_configuration[\"Maximum Iterations MaxValue\"])\n",
    "    userCol = algo_configuration[\"user\"]\n",
    "    itemCol = algo_configuration[\"item\"]\n",
    "    ratingCol = algo_configuration[\"rating\"]\n",
    "    \n",
    "    # set cold start strategy to 'drop' to ensure we don't get NaN evaluation metrics\n",
    "    als = ALS(maxIter=maxIter, regParam=regParam, userCol=userCol, itemCol=itemCol, ratingCol=ratingCol,\n",
    "          coldStartStrategy=\"drop\")\n",
    "    model = als.fit(training)\n",
    "    model.write().overwrite().save(algo_configuration[\"output_path\"])\n",
    "    # Evaluate the model by computing the RMSE on the test data\n",
    "    predictions = model.transform(test)\n",
    "    evaluator = RegressionEvaluator(metricName=\"rmse\", labelCol=ratingCol,\n",
    "                                predictionCol=\"prediction\")\n",
    "    rmse = evaluator.evaluate(predictions)\n",
    "    # Generate top 10 movie recommendations for each user\n",
    "    userRecs = model.recommendForAllUsers(10)\n",
    "    # Generate top 10 user recommendations for each item\n",
    "    itemRecs = model.recommendForAllItems(10)\n",
    "    saveDataframe(algo_configuration[\"output_path\"]+\"/als_recommendations_for_users\",userRecs)\n",
    "    saveDataframe(algo_configuration[\"output_path\"]+\"/als_recommendations_for_items\",itemRecs)\n",
    "    resultDict = {}\n",
    "    resultDict[\"RMSE (from 80%-20% train-test split)\"] = rmse\n",
    "    resultDict[\"Output\"] = \"The created model has been saved in the selected output folder. To facilitate its evaluation, two dataframes containing generated recommendations have been created and saved as parquet files: one containing the top 10 recommended items for all users and one containing the top 10 recommnded users for each item.\"\n",
    "    return {\"results\":resultDict}\n",
    "\n",
    "def run_tokenizer(algo_configuration):\n",
    "    sentenceDF = tempDF\n",
    "    regexTokenizer = RegexTokenizer(inputCol=algo_configuration[\"textcol\"], outputCol=\"result-words\", pattern=\"\\\\W\")\n",
    "    regexTokenized = regexTokenizer.transform(sentenceDF)\n",
    "    saveDataframe(algo_configuration[\"output_path\"],regexTokenized)\n",
    "    return {\"results\":{\"Result\":\"The tokenized version of the input file has been saved in the specified folder.\"}}\n",
    "\n",
    "def run_ngrams(algo_configuration):\n",
    "    sentenceDF = tempDF\n",
    "    regexTokenizer = RegexTokenizer(inputCol=algo_configuration[\"textcol\"], outputCol=\"result-words\", pattern=\"\\\\W\")\n",
    "    regexTokenized = regexTokenizer.transform(sentenceDF)\n",
    "    n = int(algo_configuration[\"N\"])\n",
    "    ngram = NGram(n=n, inputCol=\"result-words\", outputCol=\"result-ngrams\")\n",
    "    ngramDataFrame = ngram.transform(regexTokenized).drop(\"result-words\") #TODO save this\n",
    "    saveDataframe(algo_configuration[\"output_path\"],ngramDataFrame)\n",
    "    return {\"results\":{\"Result\":\"The result file with the n-grams has been saved in the specified folder.\"}}\n",
    "\n",
    "def run_tfidf(algo_configuration):\n",
    "    sentenceDF = tempDF\n",
    "    regexTokenizer = RegexTokenizer(inputCol=algo_configuration[\"textcol\"], outputCol=\"temp-words\", pattern=\"\\\\W\")\n",
    "    regexTokenized = regexTokenizer.transform(sentenceDF)\n",
    "    n = int(algo_configuration[\"Number of TF-IDF Features\"])\n",
    "    hashingTF = HashingTF(inputCol=\"temp-words\", outputCol=\"rawFeatures\", numFeatures=n)\n",
    "    featurizedData = hashingTF.transform(regexTokenized)\n",
    "    idf = IDF(inputCol=\"rawFeatures\", outputCol=\"features\")\n",
    "    idfModel = idf.fit(featurizedData)\n",
    "    idfModel.write().overwrite().save(algo_configuration[\"output_path\"])\n",
    "    rescaledData = idfModel.transform(featurizedData).drop(\"words\",\"rawFeatures\")\n",
    "    saveDataframe(algo_configuration[\"output_path\"]+\"/tfidf_features\",rescaledData)\n",
    "    return {\"results\":{\"Result\":\"The created model and the result file with the TF-IDF features have been saved in the specified folder.\"}}\n",
    "\n",
    "def run_fpgrowth(algo_configuration):\n",
    "    df= tempDF\n",
    "    df = df.withColumn(\"fpinputcol\",toarray_udf(df.items))\n",
    "    minSup = float(algo_configuration[\"Minimal Support\"])\n",
    "    minConf = float(algo_configuration[\"Minimal Confidence\"])\n",
    "    fpGrowth = FPGrowth(itemsCol=\"fpinputcol\", minSupport=minSup, minConfidence=minConf)\n",
    "    model = fpGrowth.fit(df)\n",
    "    #frequent itemsets\n",
    "    freqitemsetsDF = model.freqItemsets.orderBy(\"freq\", ascending = [0])\n",
    "    #generated association rules.\n",
    "    associationRulesDF = model.associationRules\n",
    "    # consequents as prediction\n",
    "    predictionDF = model.transform(df)\n",
    "    model.write().overwrite().save(algo_configuration[\"output_path\"])\n",
    "    saveDataframe(algo_configuration[\"output_path\"]+\"/fpgrowth_associationRules\",associationRulesDF)\n",
    "    saveDataframe(algo_configuration[\"output_path\"]+\"/fpgrowth_predictions\",predictionDF)\n",
    "    return {\"results\":{\"Result\":\"The model has been saved in the specified output folder. Also, two created Dataframes have been saved in the same folder as parquet files: one contains the generated association rules and the other contains item predictions.\"}}\n",
    "\n",
    "#TODO: differentiate binary with multiclass classification!!!\n",
    "#TODO: SOS classification needs classes from 2..#classes-1\n",
    "def run_algo(algo,data):\n",
    "    global g\n",
    "    try:\n",
    "        algo_configuration = json.loads(data)\n",
    "        result = {\"results\":{\"result\":\"No result\"}}\n",
    "        if algo == \"LOGRE\":\n",
    "            result = run_logistic_regression(algo_configuration)\n",
    "        elif algo == \"DTC\":\n",
    "            result = run_decision_tree_classifier(algo_configuration)\n",
    "        elif algo == \"NB\":\n",
    "            result = run_naive_bayes(algo_configuration)\n",
    "        elif algo == \"RFC\":\n",
    "            result = run_random_forest_classifier(algo_configuration)\n",
    "        elif algo == \"GBTC\":\n",
    "            result = run_gradient_boosted_tree_classifier(algo_configuration)\n",
    "        elif algo == \"MLP\":\n",
    "            result = run_multilayer_perceptron_classifier(algo_configuration)\n",
    "        elif algo == \"DTR\":\n",
    "            result = run_decision_tree_regressor(algo_configuration)\n",
    "        elif algo == \"RFR\":\n",
    "            result = run_random_forest_regressor(algo_configuration)\n",
    "        elif algo == \"GBTR\":\n",
    "            result = run_gradient_boosted_tree_regressor(algo_configuration)\n",
    "        elif algo == \"OLS\":\n",
    "            result = run_linear_regression(algo_configuration)\n",
    "        elif algo == \"GLM\":\n",
    "            result = run_generalized_linear_regression(algo_configuration)\n",
    "        elif algo == \"KMEANS\":\n",
    "            result = run_kmeans_clustering(algo_configuration)\n",
    "        elif algo == \"GAUSSMIX\":\n",
    "            result = run_gaussian_mixtures(algo_configuration)\n",
    "        elif algo == \"PCA\":\n",
    "            result = run_pca(algo_configuration)\n",
    "        elif algo == \"ChiSquared\":\n",
    "            result = run_chisquare(algo_configuration)\n",
    "        elif algo == \"ALS\":\n",
    "            result = run_als(algo_configuration)\n",
    "        elif algo == \"TOKENIZER\":\n",
    "            result = run_tokenizer(algo_configuration)\n",
    "        elif algo == \"N-GRAMS\":\n",
    "            result = run_ngrams(algo_configuration)\n",
    "        elif algo == \"FPGrowth\":\n",
    "            result =run_fpgrowth(algo_configuration)\n",
    "    except Exception as e:\n",
    "        g=e\n",
    "        result = {\"results\":{\"ERROR\":str(e).replace(\"'\",\"\").replace(\"\\\"\",\"\").encode(\"utf-8\")}}\n",
    "    return result\n",
    "    \n",
    "\n",
    "\n",
    "# ----------- Error Handling -----------\n",
    "\n",
    "# TODO for dubugging, show in UI\n",
    "def sanity_check():\n",
    "    for a in ALGORITHMS:\n",
    "        for algo in ALGORITHMS[a]:\n",
    "            if algo not in ALGORITHM_PARAMETERS:\n",
    "                error = \"INIT ERROR: No parameters defined for {}\".format(algo)\n",
    "                return error\n",
    "    for p in ALGORITHM_PARAMETERS:\n",
    "        for parameter in ALGORITHM_PARAMETERS[p]:\n",
    "            if parameter not in PARAMETERS:\n",
    "                error = \"INIT ERROR: Parameter {} not specified in parameters dict\".format(parameter)\n",
    "                return error\n",
    "\n",
    "\n",
    "# ----------- Model Apply -----------\n",
    "\n",
    "def apply_model(selectedFile,separator,resultFolder):\n",
    "    global modelDF\n",
    "    global model\n",
    "    global g\n",
    "    try:\n",
    "        modelDF = spark.read.csv(selectedFile,header=True, inferSchema=True, sep=separator)\n",
    "        if not model is None:\n",
    "            assembler_m = VectorAssembler(inputCols=modelDF.columns,outputCol=\"features\")\n",
    "            output_m = assembler_m.transform(modelDF)\n",
    "            prediction = model.transform(output_m)\n",
    "            colsKeep = modelDF.columns\n",
    "            colsKeep.append(\"prediction\")\n",
    "            saveDataframe(resultFolder,prediction)\n",
    "            result_final = []\n",
    "            for i in prediction[colsKeep].head(5):\n",
    "                result_final.append(i.asDict())\n",
    "            res = {\"Result preview\":result_final,\n",
    "               \"Output\":\"The result of the model application has been saved in the specified folder.\"}\n",
    "            return {\"results\":res}\n",
    "    except Exception as e:\n",
    "        g=e\n",
    "        result = {\"results\":{\"ERROR\":str(e).replace(\"'\",\"\").replace(\"\\\"\",\"\").encode(\"utf-8\")}} \n",
    "        return result\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {
    "code_folding": [
     92,
     138,
     151,
     157,
     178,
     191,
     217,
     243,
     301
    ],
    "hideCode": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "<style>\n",
       ".tagclass {\n",
       "    background:grey;\n",
       "    border:none;\n",
       "}\n",
       ".fieldcell{\n",
       "    white-space:pre-wrap;\n",
       "}\n",
       "td{\n",
       "  text-align:left;\n",
       "}\n",
       "\n",
       "table{\n",
       "    table-layout: auto !important;\n",
       "}\n",
       "\n",
       "td{\n",
       "    word-wrap:break-word\n",
       "}\n",
       "\n",
       "\n",
       ".paragraph .form-horizontal .form-group {\n",
       "    margin-right: 0px !important;\n",
       "    margin-left: 0px !important;\n",
       "}\n",
       "\n",
       "#list2 { width:320px; }\n",
       "#list2 ol { font-style:italic; font-family:Georgia, Times, serif; font-size:18px; color:#337ab7;  }\n",
       "#list2 ol li { }\n",
       "#list2 ol li p { padding:8px; font-style:normal; font-family:Arial; font-size:13px; color:black; border-left: 1px solid #999; }\n",
       "#list2 ol li.applied p em { color:lightgray;  }\n",
       "#list2 ol li p em { display:block; }\n",
       "\n",
       "\n",
       ".qblabel {\n",
       "    font-style:italic; font-family:Georgia, Times, serif; font-size:18px; color:#337ab7; \n",
       "}\n",
       "\n",
       ".aec-btn {\n",
       "    background-color: Transparent;\n",
       "    background-repeat:no-repeat;\n",
       "    border: none;\n",
       "    cursor:pointer;\n",
       "    overflow: hidden;\n",
       "    outline:none;\n",
       "    font-style:italic; \n",
       "    font-family:Georgia, Times, serif; \n",
       "    font-size:20px; \n",
       "    color:#435066;\n",
       "    padding: 10px 12px 14px 12px;\n",
       "}\n",
       "\n",
       ".aec-btn:hover {\n",
       "    background-color: #ddd;\n",
       "}\n",
       "\n",
       "/* Create an active/current tablink class */\n",
       ".aec-btn.active {\n",
       "    background-color: #ccc;\n",
       "}\n",
       "\n",
       "#aec div{\n",
       "    padding: 2px 4px 2px 16px;\n",
       "}\n",
       "\n",
       "</style>\n",
       "\n",
       "\n",
       "<button type=\"button\" class=\"btn aec-btn\" onclick=\"openTab('aec-file-selector',this);\">Input File</button>\n",
       "<button type=\"button\" class=\"btn aec-btn\" onclick=\"openTab('aec-algo-selector',this);\">Algorithm Selection & Configuration</button>\n",
       "<button type=\"button\" class=\"btn aec-btn\" onclick=\"openTab('aec-save-conf',this);\">Output File</button>\n",
       "<button type=\"button\" class=\"btn aec-btn\" onclick=\"updateOverview();openTab('aec-overview',this);\">Overview</button>\n",
       "<button type=\"button\" class=\"btn aec-btn\" onclick=\"openTab('aec-apply-model',this);\">Apply Model</button>\n",
       "<div id=\"aec\">\n",
       "   <div id=\"aec-overview\" style=\"display:none;\">\n",
       "      <p>\n",
       "         <label style=\"display: inline-block;width: 20%;\" class=\"qblabel\">Selected Input File:</label>\n",
       "      <div id=\"overview-input\" style=\"display: inline-block;width: 70%;\"></div>\n",
       "      </p>\n",
       "      <p>\n",
       "         <label style=\"width: 20%;\" class=\"qblabel\">Selected Algorithm:</label>\n",
       "      <div id=\"overview-algorithm\" style=\"display: inline-block;width: 70%;\"></div>\n",
       "      </p>\n",
       "      <p>\n",
       "         <label style=\"width: 20%;\" class=\"qblabel\">Selected Output File:</label>\n",
       "      <div id=\"overview-output\" style=\"display: inline-block;width: 70%;\"></div>\n",
       "      </p>\n",
       "      <button class=\"btn btn-primary\" id=\"filters-done\" onclick=\"executeAlgo();\"> Execute </button>\n",
       "      <div id=\"algo-results\"></div>\n",
       "   </div>\n",
       "   <div id=\"aec-file-selector\" style=\"display:none;\">\n",
       "      <div class=\"form-group\">\n",
       "         <label style=\"width: 20%;\" class=\"qblabel\">Available Datasets</label>\n",
       "         <div style=\"display: inline-block;width: 60%;\">\n",
       "            <select style=\"margin-left:4px !important;\" id=\"datasetSelect\" class=\"form-control\">\n",
       "            </select>\n",
       "         </div>\n",
       "         <button type=\"button\" class=\"btn btn-primary btn-md\" onclick=\"selectDataset()\"> Apply </button>\n",
       "         <button type=\"button\" class=\"btn btn-primary btn-md\" onclick=\"updateBrowser();\"> Refresh </button>\n",
       "      </div>\n",
       "      <div class=\"form-group\">\n",
       "         <label style=\"width: 20%;\" class=\"qblabel\">Available Files</label>\n",
       "         <div style=\"display: inline-block;width: 60%;\">\n",
       "            <select style=\"margin-left:4px !important;\" id=\"fileSelect\" class=\"form-control\">\n",
       "            </select>\n",
       "         </div>\n",
       "      </div>\n",
       "      <div class=\"form-group\">\n",
       "         <label style=\"width: 20%;\" class=\"qblabel\">CSV Separator</label>\n",
       "         <div style=\"display: inline-block;width: 60%;\">\n",
       "            <select style=\"margin-left:4px !important;\" id=\"sepSelect\" class=\"form-control\">\n",
       "               <option value=\",\">,</option>\n",
       "               <option value=\";\">;</option>\n",
       "            </select>\n",
       "         </div>\n",
       "         <button type=\"button\" class=\"btn btn-primary btn-md\" onclick=\"selectDatafile()\"> Apply </button>\n",
       "         <button type=\"button\" class=\"btn btn-primary btn-md\" onclick=\"getTempDFpreview();\">Refresh Preview</button>\n",
       "      </div>\n",
       "      <div id=\"data-preview\" style=\"display:none\">\n",
       "         <div class=\"panel-body\">\n",
       "            <table class=\"table\">\n",
       "               <tbody id=\"preview-table\">\n",
       "               </tbody>\n",
       "            </table>\n",
       "         </div>\n",
       "      </div>\n",
       "      <div class=\"form-group\">\n",
       "         <button type=\"button\" class=\"btn btn-primary btn-md\" onclick=\"openTab('aec-algo-selector',this);\"> Next </button>\n",
       "      </div>\n",
       "   </div>\n",
       "   <hr>\n",
       "   <hr>\n",
       "   <div class=\"form-horizontal\" id=\"aec-algo-selector\" style=\"display:none;\">\n",
       "      <div class=\"form-group\">\n",
       "         <label style=\"width: 20%;text-align:left;\" class=\"control-label col-sm-2 qblabel\">Algorithm Family</label>\n",
       "         <div class=\"col-sm-10\" style=\"display: inline-block;width: 70%;\">\n",
       "            <select style=\"margin-left:4px !important;\" class=\"form-control\" id=\"algoFamilySelect\" onchange=\"loadAlgorithms();$('.algo_configuration').hide();\">\n",
       "               <option value=\"\" disabled selected></option>\n",
       "               <option value=\"DIMRE_FE\">DIMENSIONALITY REDUCTION/FEATURE EXTRACTION/SELECTION</option>\n",
       "               <option value=\"NLP\">NLP FUNCTIONS</option>\n",
       "               <option value=\"RECOM\">RECOMMENDERS</option>\n",
       "               <option value=\"CLUST\">CLUSTERING</option>\n",
       "               <option value=\"CL_REG\">CLASSIFICATION/REGRESSION</option>\n",
       "               <option value=\"FREQPM\">FREQUENT PATTERN MINING</option>\n",
       "            </select>\n",
       "         </div>\n",
       "      </div>\n",
       "      <div class=\"form-group\">\n",
       "         <label style=\"width: 20%;text-align:left;\" class=\"control-label col-sm-2 qblabel\">Algorithm</label>\n",
       "         <div class=\"col-sm-10\" style=\"display: inline-block;width: 70%;\">\n",
       "            <select style=\"margin-left:4px !important;\" class=\"form-control\" id=\"algoSelect\" onchange=\"populate_algo_conf_form()\">\n",
       "               <option value=\"\" disabled selected>-- select --</option>\n",
       "            </select>\n",
       "         </div>\n",
       "      </div>\n",
       "      <div class=\"form-horizontal algo_configuration \" id=\"algocommon_DIMRE_FE\" style=\"display:none\">\n",
       "         <p class=\"algoDescription\"></p>\n",
       "         <h3>Configuration</h3>\n",
       "         <div class=\"form-group\">\n",
       "            <label class=\"control-label col-sm-2\">Feature Columns</label>\n",
       "            <div class=\"col-sm-10\">\n",
       "               <select multiple class=\"form-control multiple-column-selector df-col\">\n",
       "               </select>\n",
       "            </div>\n",
       "         </div>\n",
       "         <div class=\"form-horizontal algo_configuration\" id=\"algo_PCA\" style=\"display:none\"></div>\n",
       "         <div class=\"form-horizontal algo_configuration\" id=\"algo_ChiSquared\" style=\"display:none\">\n",
       "            <div class=\"form-group\">\n",
       "               <label class=\"control-label col-sm-2\">Label Column</label>\n",
       "               <div class=\"col-sm-10\">\n",
       "                  <select class=\"form-control single-column-selector df-col\">\n",
       "                  </select>\n",
       "               </div>\n",
       "            </div>\n",
       "         </div>\n",
       "      </div>\n",
       "      <div class=\"form-horizontal algo_configuration \" id=\"algocommon_NLP\" style=\"display:none\">\n",
       "         <p class=\"algoDescription\"></p>\n",
       "         <div class=\"form-group\">\n",
       "            <label class=\"control-label col-sm-2\">Text Column</label>\n",
       "            <div class=\"col-sm-10\">\n",
       "               <select class=\"form-control single-column-selector df-col\" id=\"nlp-text-col\">\n",
       "               </select>\n",
       "            </div>\n",
       "         </div>\n",
       "         <div class=\"form-horizontal algo_configuration\" id=\"algo_TOKENIZER\" style=\"display:none\"></div>\n",
       "         <div class=\"form-horizontal algo_configuration\" id=\"algo_N-GRAMS\" style=\"display:none\"></div>\n",
       "         <div class=\"form-horizontal algo_configuration\" id=\"algo_TF-IDF\" style=\"display:none\">TF-IDF</div>\n",
       "      </div>\n",
       "      <div class=\"form-horizontal algo_configuration \" id=\"algocommon_RECOM\" style=\"display:none\">\n",
       "         <p class=\"algoDescription\"> </p>\n",
       "         <div class=\"form-horizontal algo_configuration\" id=\"algo_ALS\" style=\"display:none\">\n",
       "            <div class=\"form-group\">\n",
       "               <label class=\"control-label col-sm-2\">User Column</label>\n",
       "               <div class=\"col-sm-10\">\n",
       "                  <select class=\"form-control single-column-selector df-col\" id=\"als-user-id-col\">\n",
       "                  </select>\n",
       "               </div>\n",
       "            </div>\n",
       "            <div class=\"form-group\">\n",
       "               <label class=\"control-label col-sm-2\">Item Column</label>\n",
       "               <div class=\"col-sm-10\">\n",
       "                  <select class=\"form-control single-column-selector df-col\" id=\"als-item-id-col\">\n",
       "                  </select>\n",
       "               </div>\n",
       "            </div>\n",
       "            <div class=\"form-group\">\n",
       "               <label class=\"control-label col-sm-2\">Rating Column</label>\n",
       "               <div class=\"col-sm-10\">\n",
       "                  <select class=\"form-control single-column-selector df-col\" id=\"als-rating-col\">\n",
       "                  </select>\n",
       "               </div>\n",
       "            </div>\n",
       "         </div>\n",
       "      </div>\n",
       "      <div class=\"form-horizontal algo_configuration \" id=\"algocommon_FREQPM\" style=\"display:none\">\n",
       "         <p class=\"algoDescription\"> </p>\n",
       "         <h3>Configuration</h3>\n",
       "         <div class=\"form-horizontal algo_configuration\" id=\"algo_FPGrowth\" style=\"display:none\">\n",
       "            <div class=\"form-group\">\n",
       "               <label class=\"control-label col-sm-2\">Items Column</label>\n",
       "               <div class=\"col-sm-10\">\n",
       "                  <select class=\"form-control single-column-selector df-col\" id=\"fpginput-col\">\n",
       "                  </select>\n",
       "               </div>\n",
       "            </div>\n",
       "         </div>\n",
       "      </div>\n",
       "      <div class=\"form-horizontal algo_configuration \" id=\"algocommon_CLUST\" style=\"display:none\">\n",
       "         <p class=\"algoDescription\"></p>\n",
       "         <h3>Configuration</h3>\n",
       "         <div class=\"form-group\">\n",
       "            <label class=\"control-label col-sm-2\">Feature Columns</label>\n",
       "            <div class=\"col-sm-10\">\n",
       "               <select multiple class=\"form-control multiple-column-selector df-col\">\n",
       "               </select>\n",
       "            </div>\n",
       "         </div>\n",
       "         <div class=\"form-horizontal algo_configuration\" id=\"algo_KMEANS\" style=\"display:none\"></div>\n",
       "         <div class=\"form-horizontal algo_configuration\" id=\"algo_GAUSSMIX\" style=\"display:none\"></div>\n",
       "      </div>\n",
       "      <div class=\"form-horizontal algo_configuration \" id=\"algocommon_CL_REG\" style=\"display:none\">\n",
       "         <p> Note that for the classification algorithms, the label column should contain integers from 0 to N-1, where N the number of classes. </p>\n",
       "         <p class=\"algoDescription\"></p>\n",
       "         <h3>Configuration</h3>\n",
       "         <div class=\"form-group\">\n",
       "            <label class=\"control-label col-sm-2\">Label Column</label>\n",
       "            <div class=\"col-sm-10\">\n",
       "               <select class=\"form-control single-column-selector df-col\">\n",
       "               </select>\n",
       "            </div>\n",
       "         </div>\n",
       "         <div class=\"form-group\">\n",
       "            <label class=\"control-label col-sm-2\">Feature Columns</label>\n",
       "            <div class=\"col-sm-10\">\n",
       "               <select multiple class=\"form-control multiple-column-selector df-col\">\n",
       "               </select>\n",
       "            </div>\n",
       "         </div>\n",
       "         <div class=\"form-horizontal algo_configuration\" id=\"algo_OLS\" style=\"display:none\"></div>\n",
       "         <div class=\"form-horizontal algo_configuration\" id=\"algo_DTR\" style=\"display:none\"></div>\n",
       "         <div class=\"form-horizontal algo_configuration\" id=\"algo_DTC\" style=\"display:none\"></div>\n",
       "         <div class=\"form-horizontal algo_configuration\" id=\"algo_MLP\" style=\"display:none\"> </div>\n",
       "         <div class=\"form-horizontal algo_configuration\" id=\"algo_NB\" style=\"display:none\"></div>\n",
       "         <div class=\"form-horizontal algo_configuration\" id=\"algo_GLM\" style=\"display:none\"></div>\n",
       "         <div class=\"form-horizontal algo_configuration\" id=\"algo_RFR\" style=\"display:none\"></div>\n",
       "         <div class=\"form-horizontal algo_configuration\" id=\"algo_RFC\" style=\"display:none\"></div>\n",
       "         <div class=\"form-horizontal algo_configuration\" id=\"algo_GBTR\" style=\"display:none\"></div>\n",
       "         <div class=\"form-horizontal algo_configuration\" id=\"algo_GBTC\" style=\"display:none\"></div>\n",
       "         <div class=\"form-horizontal algo_configuration\" id=\"algo_LOGRE\" style=\"display:none\"></div>\n",
       "         <div class=\"form-horizontal \" id=\"CL_REG_EVAL\">\n",
       "            <div class=\"form-group\">\n",
       "               <div class=\"btn-group btn-group-toggle\" data-toggle=\"buttons\">\n",
       "                  <label class=\"btn btn-secondary eval-label\" onclick=\"selectEvalMethod('train-test-form',this);\" style=\"font-size:16px\">\n",
       "                  <input type=\"radio\" name=\"validation-method\" id=\"train-test\"> Train-Test Split\n",
       "                  </label>\n",
       "                  <label class=\"btn btn-secondary eval-label\" onclick=\"selectEvalMethod('k-fold-form',this);\" style=\"font-size:16px\">\n",
       "                  <input type=\"radio\" name=\"validation-method\"> K-Fold Cross Validation\n",
       "                  </label>\n",
       "               </div>\n",
       "               <div class=\"form-group eval-form\" id=\"k-fold-form\" style=\"display:none\">\n",
       "                  <label class=\"control-label col-sm-2\">Number of Folds for Cross-Validation</label>\n",
       "                  <div class=\"col-sm-10\">\n",
       "                     <input type=\"number\" min=\"2\" max=\"10\" step=\"1\" value=\"2\"></input>\n",
       "                  </div>\n",
       "               </div>\n",
       "               <div class=\"form-group eval-form\" id=\"train-test-form\" style=\"display:none\">\n",
       "                  <label class=\"control-label col-sm-2\">Percentage % of input data to be used for training</label>\n",
       "                  <div class=\"col-sm-10\">\n",
       "                     <input type=\"number\" min=\"10\" max=\"100\" step=\"5\" value=\"80\"></input>\n",
       "                  </div>\n",
       "               </div>\n",
       "            </div>\n",
       "         </div>\n",
       "      </div>\n",
       "      <div class=\"form-group\">\n",
       "         <button type=\"button\" class=\"btn btn-primary btn-md\" onclick=\"openTab('aec-save-conf',this);\"> Next </button>\n",
       "      </div>\n",
       "   </div>\n",
       "   <div style=\"display:none;\" id=\"aec-save-conf\">\n",
       "      <div class=\"form-group\">\n",
       "         <label style=\"width: 20%;\" class=\"qblabel\">Select Dataset</label>\n",
       "         <div style=\"display: inline-block;width: 60%;\">\n",
       "            <select style=\"margin-left:4px !important;\" id=\"datasetSaveSelect\" class=\"form-control\">\n",
       "            </select>\n",
       "         </div>\n",
       "         <button type=\"button\" class=\"btn btn-primary btn-md\" onclick=\"updateBrowser();\"> Update Browser </button>\n",
       "      </div>\n",
       "      <div class=\"form-group\">\n",
       "         <label style=\"width: 20%;\" class=\"qblabel\">Result Folder Name</label>\n",
       "         <div style=\"display: inline-block;width: 60%;\">\n",
       "            <input style=\"margin-left:4px !important;\" id=\"folderNameSelect\" class=\"form-control\">\n",
       "            </input>\n",
       "         </div>\n",
       "      </div>\n",
       "      <!--\n",
       "         <div class=\"form-group\">\n",
       "           <label style=\"width: 20%;\" class=\"qblabel\">Include Header</label>\n",
       "           <div style=\"display: inline-block;width: 60%;\">\n",
       "         <input type=\"checkbox\" id=\"saveHeaderCheckbox\" value=\"with\" checked>                   \n",
       "             </input> \n",
       "           </div>\n",
       "         </div>\n",
       "         -->\n",
       "      <div class=\"form-group\">\n",
       "         <p>If the folder you specified exists, <b> it will be overwritten </b> with the results of the last algorithm execution.</p>\n",
       "         <button type=\"button\" class=\"btn btn-primary btn-md\" onclick=\"updateOverview();openTab('aec-overview',this);\"> Done </button>\n",
       "      </div>\n",
       "   </div>\n",
       "   <div style=\"display:none;\" id=\"aec-apply-model\">\n",
       "      <div class=\"form-group\">\n",
       "         <p>This is only possible for classification and regression models. <b> Make sure that the input file you provide has the same \n",
       "            structure as the one used for training the model,i.e. that its columns correspond to the features used in the training -\n",
       "            no extra columns and no label column should be present in the file. </b>  </p>\n",
       "            <div class=\"form-group\">\n",
       "         <label style=\"width: 20%;\" class=\"qblabel\">Available Datasets</label>\n",
       "         <div style=\"display: inline-block;width: 60%;\">\n",
       "            <select style=\"margin-left:4px !important;\" id=\"datasetModelApplySelect\" class=\"form-control\">\n",
       "            </select>\n",
       "         </div>\n",
       "         <button type=\"button\" class=\"btn btn-primary btn-md\" onclick=\"selectModelApplyDataset()\"> Apply </button>\n",
       "         <button type=\"button\" class=\"btn btn-primary btn-md\" onclick=\"updateBrowser();\"> Refresh </button>\n",
       "      </div>\n",
       "      <div class=\"form-group\">\n",
       "         <label style=\"width: 20%;\" class=\"qblabel\">Available Files</label>\n",
       "         <div style=\"display: inline-block;width: 60%;\">\n",
       "            <select style=\"margin-left:4px !important;\" id=\"fileModelApplySelect\" class=\"form-control\">\n",
       "            </select>\n",
       "         </div>\n",
       "      </div>\n",
       "      <div class=\"form-group\">\n",
       "         <label style=\"width: 20%;\" class=\"qblabel\">CSV Separator</label>\n",
       "         <div style=\"display: inline-block;width: 60%;\">\n",
       "            <select style=\"margin-left:4px !important;\" id=\"sepModelApplySelect\" class=\"form-control\">\n",
       "               <option value=\",\">,</option>\n",
       "               <option value=\";\">;</option>\n",
       "            </select>\n",
       "         </div>\n",
       "         \n",
       "      </div>\n",
       "    <div class=\"form-group\">\n",
       "         <label style=\"width: 20%;\" class=\"qblabel\">Result Folder Name</label>\n",
       "         <div style=\"display: inline-block;width: 60%;\">\n",
       "            <input style=\"margin-left:4px !important;\" id=\"folderModelSelect\" class=\"form-control\">\n",
       "            </input>\n",
       "         </div>\n",
       "        <button type=\"button\" class=\"btn btn-primary btn-md\" onclick=\"selectDataModelApplyfile()\"> Apply Model</button>\n",
       "      </div>\n",
       "    <div id=\"model-results\"></div>\n",
       "      </div>\n",
       "   </div>\n",
       "</div>\n",
       "\n",
       "\n",
       "\n",
       "<script>\n",
       "\n",
       "var tempDF_columns = []\n",
       "var datafile_options = []\n",
       "var datafile_options_ma = []\n",
       "\n",
       "var currentDFpath = \"\"\n",
       "var lines\n",
       "\n",
       "var algorithm_list = {\n",
       "    \"PCA\":\"Principal Component Analysis (PCA)\",\n",
       "    \"ChiSquared\":\"Chi-squared test\",\n",
       "    \"TOKENIZER\":\"Tokenizer\",\n",
       "    \"N-GRAMS\":\"N-Grams\",\n",
       "    \"TF-IDF\":\"TF-IDF\",\n",
       "    \"ALS\":\"Collaborative Filtering (ALS)\",\n",
       "    \"KMEANS\":\"K-MEANS\",\n",
       "    \"GAUSSMIX\":\"Gaussian Mixtures\",\n",
       "    \"OLS\":\"Linear Regression\",\n",
       "    \"DTR\":\"Decision Trees Regression\",\n",
       "    \"DTC\":\"Decision Trees Classifier\",\n",
       "    \"MLP\":\"Multi-Layer Perceptron\",\n",
       "    \"NB\":\"Naive Bayes\",\n",
       "    \"GLM\":\"Generalized Linear Models (GLM)\",\n",
       "    \"RFR\":\"Random Forest Regressor (RFR)\",\n",
       "    \"GBTR\":\"Gradient-boosted tree Regression (GBTR)\",\n",
       "    \"RFC\":\"Random Forest Classifier (RFC)\",\n",
       "    \"GBTC\":\"Gradient-boosted tree Classifier (GBTC)\",\n",
       "    \"LOGRE\":\"Logistic Regression\",\n",
       "    \"FPGrowth\":\"FP Growth\"\n",
       "}\n",
       "\n",
       "var algorithm_descriptions = {\n",
       "    \"PCA\":\"Principal Component Analysis (PCA) is a statistical procedure that uses an orthogonal transformation to \\\n",
       "        convert a set of observations of possibly correlated variables into a set of values of linearly uncorrelated variables\\\n",
       "        called principal components. A PCA class trains a model to project vectors to a low-dimensional space using PCA. \\\n",
       "        <a href=\\\"https://spark.apache.org/docs/2.3.2/ml-features.html#pca\\\" target=\\\"_blank\\\"> More info...</a>\",\n",
       "    \"ChiSquared\":\"Chi-squared stands for Chi-Squared feature selection. It operates on labeled data with categorical \\\n",
       "        features. ChiSqSelector uses the Chi-Squared test of independence to decide which features to choose.test\\\n",
       "        <a href=\\\"https://spark.apache.org/docs/2.3.2/ml-features.html#chisqselector\\\" target=\\\"_blank\\\">More info...</a>\",\n",
       "    \"TOKENIZER\":\"Tokenization is the process of taking text (such as a sentence) and breaking it into individual terms \\\n",
       "        (usually words). Here we use RegexTokenizer that converts the input string to lowercase, \\\n",
       "        removes stopwords and then splits it by white spaces. \\\n",
       "        <a href=\\\"https://spark.apache.org/docs/2.3.2/ml-features.html#tokenizer\\\" target=\\\"_blank\\\"> More info...</a>\",\n",
       "    \"N-GRAMS\":\"An n-gram is a sequence of n tokens (typically words) for some integer n. The NGram class can be used to \\\n",
       "        transform input features into n-grams. The parameter n is used to determine the number of terms in each n-gram. \\\n",
       "        The output will consist of a sequence of n-grams where each n-gram is represented by a space-delimited string \\\n",
       "        of n consecutive words. If the input sequence contains fewer than n strings, no output is produced. \\\n",
       "        <a href=\\\"https://spark.apache.org/docs/2.3.2/ml-features.html#n-gram\\\" target=\\\"_blank\\\"> More info...</a>\",\n",
       "    \"TF-IDF\":\"Term frequency-inverse document frequency (TF-IDF) is a feature vectorization method widely used in \\\n",
       "        text mining to reflect the importance of a term to a document in the corpus. \\\n",
       "        <a href=\\\"https://spark.apache.org/docs/2.3.2/ml-features.html#tf-idf\\\" target=\\\"_blank\\\"> More info...</a>\",\n",
       "    \"ALS\":\"Collaborative filtering is commonly used for recommender systems. These techniques aim to fill in the \\\n",
       "        missing entries of a user-item association matrix. spark.ml currently supports model-based collaborative filtering,\\\n",
       "        in which users and products are described by a small set of latent factors that can be used to predict missing \\\n",
       "        entries. spark.ml uses the alternating least squares (ALS) algorithm to learn these latent factors. \\\n",
       "        <a href=\\\"https://spark.apache.org/docs/2.3.2/ml-collaborative-filtering.html\\\" target=\\\"_blank\\\"> More info...</a>\",\n",
       "    \"KMEANS\":\"K-means is one of the most commonly used clustering algorithms that clusters the data points into a \\\n",
       "        predefined number of clusters (K). \\\n",
       "        <a href=\\\"https://spark.apache.org/docs/2.3.2/ml-clustering.html#k-means\\\" target=\\\"_blank\\\"> More info...</a>\",\n",
       "    \"GAUSSMIX\":\"A Gaussian Mixture Model represents a composite distribution whereby points are drawn from one of \\\n",
       "        k Gaussian sub-distributions, each with its own probability. The spark.ml implementation uses the \\\n",
       "        expectation-maximization algorithm to induce the maximum-likelihood model given a set of samples. \\\n",
       "        <a href=\\\"https://spark.apache.org/docs/2.3.2/ml-clustering.html#gaussian-mixture-model-gmm\\\" target=\\\"_blank\\\"> More info...</a>\",\n",
       "    \"OLS\":\"Ordinary Least squares (OLS) is the simplest and most common linear regressor. The learning objective of OLS \\\n",
       "        is to minimize the sum of squared residuals, in order to estimate the coefficients of the linear regression \\\n",
       "        expression. \\\n",
       "        <a href=\\\"https://spark.apache.org/docs/2.3.2/ml-classification-regression.html#linear-regression\\\" target=\\\"_blank\\\">More info...</a>\",\n",
       "    \"DTR\":\"Decision trees are a popular family of classification and regression methods. \\\n",
       "        More information about the spark.ml implementation can be found \\\n",
       "        <a href=\\\"https://spark.apache.org/docs/2.3.2/ml-classification-regression.html#decision-trees\\\" target=\\\"_blank\\\"> More info...</a>\",\n",
       "    \"DTC\":\"Decision trees are a popular family of classification and regression methods. \\\n",
       "        <a href=\\\"https://spark.apache.org/docs/2.3.2/ml-classification-regression.html#decision-trees\\\" target=\\\"_blank\\\"> More info...</a>\",\n",
       "    \"MLP\":\"Multilayer perceptron classifier (MLPC) is a classifier based on the feedforward artificial neural network. \\\n",
       "        MLPC consists of multiple layers of nodes. Each layer is fully connected to the next layer in the network. \\\n",
       "        Nodes in the input layer represent the input data. The number of nodes N in the output layer corresponds to the \\\n",
       "        number of classes. Here we use two inner layers for which the number of nodes can be selected by the user.\\\n",
       "        <a href=\\\"https://spark.apache.org/docs/2.3.2/ml-classification-regression.html#multilayer-perceptron-classifier\\\" target=\\\"_blank\\\"> More info...</a>\",\n",
       "    \"NB\":\"Naive Bayes classifiers are a family of simple probabilistic classifiers based on applying Bayes’ theorem \\\n",
       "        with strong (naive) independence assumptions between the features.  \\\n",
       "        <a href=\\\"https://spark.apache.org/docs/2.3.2/ml-classification-regression.html#naive-bayes\\\" target=\\\"_blank\\\"> More info...</a>\",\n",
       "    \"GLM\":\"Contrasted with linear regression where the output is assumed to follow a Gaussian distribution, generalized linear models (GLM) are specifications of \\\n",
       "        linear models where the response variable follows some distribution from the exponential family of distributions. \\\n",
       "        <a href=\\\"https://spark.apache.org/docs/2.3.2/ml-classification-regression.html#generalized-linear-regression\\\" target=\\\"_blank\\\"> More info...</a>\",\n",
       "    \"RFR\":\"Random forests are a popular family of classification and regression methods. \\\n",
       "        <a href=\\\"https://spark.apache.org/docs/2.3.2/ml-classification-regression.html#random-forests\\\" target=\\\"_blank\\\"> More info...</a>\",\n",
       "    \"GBTR\":\"Gradient-boosted trees (GBTs) are a popular regression method using ensembles of decision trees. \\\n",
       "        Parameter configuration should be performed with caution, as parameters significantly impact the algorithm's execution time.\\\n",
       "        <a href=\\\"https://spark.apache.org/docs/2.3.2/ml-classification-regression.html#gradient-boosted-tree-regression\\\" target=\\\"_blank\\\"> More info...</a>\",\n",
       "    \"RFC\":\"Random forests are a popular family of classification and regression methods. \\\n",
       "        <a href=\\\"https://spark.apache.org/docs/2.3.2/ml-classification-regression.html#random-forests\\\" target=\\\"_blank\\\"> More info...</a>\",\n",
       "    \"GBTC\":\"Gradient-boosted trees (GBTs) are a popular classification and regression method using ensembles \\\n",
       "        of decision trees. <b> The current implementation supports only binary classification. </b>\\\n",
       "        <a href=\\\"https://spark.apache.org/docs/2.3.2/ml-classification-regression.html#gradient-boosted-tree-classifier\\\" target=\\\"_blank\\\"> More info...</a>\",\n",
       "    \"LOGRE\":\"<p><b>Logistic regression</b> is a popular method to predict a categorical response. \\\n",
       "        It is a special case of Generalized Linear models that predicts the probability of the outcomes. \\\n",
       "        In spark.ml logistic regression can be used to predict a binary outcome by using binomial logistic regression, \\\n",
       "        or it can be used to predict a multiclass outcome by using multinomial logistic regression. \\\n",
       "        <a href=\\\"https://spark.apache.org/docs/2.3.2/ml-classification-regression.html#logistic-regression\\\" target=\\\"_blank\\\"> More info...</a></p>\",\n",
       "    \"FPGrowth\":\"Mining frequent items, itemsets, subsequences, or other substructures is usually among the first steps to analyze a large-scale dataset, \\\n",
       "        which has been an active research topic in data mining for years. The <b>FP Growth</b> algorithm  is described in \\\n",
       "        the paper Han et al., Mining frequent patterns without candidate generation, where “FP” stands for frequent pattern.\\\n",
       "        Given a dataset of transactions, the first step of FP-growth is to calculate item frequencies and identify frequent \\\n",
       "        items. Different from Apriori-like algorithms designed for the same purpose, the second step of FP-growth uses \\\n",
       "        a suffix tree (FP-tree) structure to encode transactions without generating candidate sets explicitly, which are \\\n",
       "        usually expensive to generate. After the second step, the frequent itemsets can be extracted from the FP-tree. \\\n",
       "        <a href=\\\"https://spark.apache.org/docs/2.3.2/ml-frequent-pattern-mining.html#fp-growth\\\" target=\\\"_blank\\\"> More info...</a>\"\n",
       "}\n",
       "\n",
       "var parameters = []\n",
       "var selected_algorithm;\n",
       "var selected_eval = \"train-test\"\n",
       "\n",
       "<!-- Helpers -->\n",
       "\n",
       "<!-- https://www.jstips.co/en/javascript/picking-and-rejecting-object-properties/  -->\n",
       "function reject(obj, keys) {\n",
       "    return Object.keys(obj)\n",
       "        .filter(k => !keys.includes(k))\n",
       "        .map(k => Object.assign({}, {[k]: obj[k]}))\n",
       "        .reduce((res, o) => Object.assign(res, o), {});\n",
       "}\n",
       "\n",
       "function openTab(tab,btn){\n",
       "    $(\".aec-btn\").not(btn).removeClass('active');\n",
       "    $(btn).toggleClass('active');\n",
       "    $(\"#aec\").children().not(\"#\"+tab).hide()\n",
       "    $(\"#\"+tab).slideToggle(\"fast\");\n",
       "}\n",
       "\n",
       "<!-- helper: adds options to help populate algo option dropdowns -->\n",
       "function algoOptionGenerator(instance,paramObj){  \n",
       "    instance\n",
       "    .append($(\"<option></option>\")\n",
       "            .attr(\"value\",paramObj)\n",
       "            .text(algorithm_list[paramObj]));\n",
       "};\n",
       "                                           \n",
       "<!-- helper: adds options to help populate column dropdowns -->\n",
       "function columnOptionGenerator(instance,paramObj){  \n",
       "    instance\n",
       "    .append($(\"<option></option>\")\n",
       "            .attr(\"value\",paramObj)\n",
       "            .text(paramObj));\n",
       "};\n",
       "\n",
       "\n",
       "<!-- Part 1 - Input -->\n",
       "\n",
       "<!-- call python to read file from hdfs, reset filter dropdown and populate column dropdowns -->\n",
       "<!-- aec required -->  \n",
       "function selectDatafile(){\n",
       "    var kernel = IPython.notebook.kernel;\n",
       "    var callbacks = {\n",
       "        iopub : {\n",
       "             output : handle_output,\n",
       "        }\n",
       "    }\n",
       "    dataset = $('#fileSelect').val();\n",
       "    currentDFpath = $('#fileSelect').val();\n",
       "    separator = $('#sepSelect').val();\n",
       "    var msg_id = kernel.execute('read_dataset(\"'+dataset+'\",\"'+separator+'\")', callbacks);\n",
       "}\n",
       "\n",
       "<!-- aec required -->  \n",
       "function selectDataset(){\n",
       "    var kernel = IPython.notebook.kernel;\n",
       "    var callbacks = {\n",
       "        iopub : {\n",
       "             output : getCSVfilesFromFS,\n",
       "        }\n",
       "    }\n",
       "    dataset = $('#datasetSelect').val();\n",
       "    var msg_id = kernel.execute('walk_dataset(\"'+dataset+'\")', callbacks);\n",
       "}\n",
       "\n",
       "<!-- aec required -->  \n",
       "function getCSVfilesFromFS(data){\n",
       "    datafile_options = data.content.text;\n",
       "    datafilesObtained(JSON.parse(datafile_options.replace(/u\\'/g,'\"').replace(/\\'/g,'\"')))\n",
       "}\n",
       "                                                          \n",
       "<!-- aec required -->                                                              \n",
       "function populateAvailableDatasetsList(data){\n",
       "    $.get(\"/hopsworks-api/api/project/\"+data.content.text+\"/dataset/getContent\",  function(data,status){\n",
       "        var user_datasets = []\n",
       "        skipthem = [\"Jupyter\",\"Logs\",\"Models\",\"notebook\",\"Resources\"]\n",
       "        data.forEach(function(element){\n",
       "            if(skipthem.includes(element.name) || element.name.endsWith(\".db\")){\n",
       "                return;\n",
       "            } \n",
       "            user_datasets.push(element.path)\n",
       "        });\n",
       "        var dropl = document.getElementById(\"datasetSelect\");\n",
       "        dropl.options.length = 0;\n",
       "        for (var i = 0; i < user_datasets.length; i++) { \n",
       "            dropl.options[dropl.options.length] = new Option(user_datasets[i].substring(user_datasets[i].lastIndexOf(\"/\")+1),user_datasets[i]);\n",
       "        }\n",
       "        var droplsave = document.getElementById(\"datasetSaveSelect\");\n",
       "        droplsave.options.length = 0;\n",
       "        for (var i = 0; i < user_datasets.length; i++) { \n",
       "            droplsave.options[droplsave.options.length] = new Option(user_datasets[i].substring(user_datasets[i].lastIndexOf(\"/\")+1),user_datasets[i]);\n",
       "        }\n",
       "        var droplmodelapply = document.getElementById(\"datasetModelApplySelect\");\n",
       "        droplmodelapply.options.length = 0;\n",
       "        for (var i = 0; i < user_datasets.length; i++) { \n",
       "            droplmodelapply.options[droplmodelapply.options.length] = new Option(user_datasets[i].substring(user_datasets[i].lastIndexOf(\"/\")+1),user_datasets[i]);\n",
       "        }\n",
       "    })\n",
       "}\n",
       "                                                             \n",
       "<!-- aec required -->                                                               \n",
       "function updateBrowser() {\n",
       "        $.get(\"/hopsworks-api/api/project\", function(data,status){\n",
       "            var pdict = {}\n",
       "            data.forEach(function(e){\n",
       "                pdict[e.project.name]=e.project.id;\n",
       "            });\n",
       "            var kernel = IPython.notebook.kernel;\n",
       "            var callbacks = {\n",
       "                iopub : {\n",
       "                    output : populateAvailableDatasetsList,\n",
       "                }\n",
       "            }\n",
       "            var pdict_parameter = (JSON.stringify(pdict))\n",
       "            var msg_id = kernel.execute('find_current_projectID(\\''+pdict_parameter+'\\')', callbacks);\n",
       "        });    \n",
       "} \n",
       "                                                             \n",
       "function datafilesObtained(options){\n",
       "    var dropl = document.getElementById(\"fileSelect\");\n",
       "    dropl.options.length = 0;\n",
       "    for (opt in options) {\n",
       "        dropl.options[dropl.options.length] = new Option(options[opt]);\n",
       "    }\n",
       "}                                                             \n",
       "\n",
       "                                                          \n",
       "<!-- Part 1.1 Preview -->\n",
       "function getTempDFpreview(){\n",
       "    var kernel = IPython.notebook.kernel;\n",
       "    var callbacks = {\n",
       "        iopub : {\n",
       "             output : show_preview_new,\n",
       "        }\n",
       "    }\n",
       "    var msg_id = kernel.execute('get_tempdf_10first_lines_new()', callbacks); \n",
       "}\n",
       "\n",
       "function show_preview_new(data){\n",
       "    lines = data.content.text.slice(1,-1).replace(/ u\\'/g,'\"').replace(/\\'/g,'\"').replace(/\\[/g,'').split('],');\n",
       "    $('#preview-table').empty();\n",
       "    newline=\"<tr>\"\n",
       "    $.each( JSON.parse(tempDF_columns.replace(/\\'/g,'\"')), function( key, val ) {\n",
       "        newline = newline.concat(\"<th>\"+ val +\"</th>\")\n",
       "    });\n",
       "    newline = newline.concat(\"</tr>\")\n",
       "    $(\"#preview-table\").append(newline)\n",
       "    $.each( lines, function( key, val ) {\n",
       "        newline = \"<tr>\"\n",
       "        valarray = val.split(',');\n",
       "        $.each( valarray, function( k, v ) {\n",
       "            newline = newline.concat(\"<td>\"+ v +\"</td>\")\n",
       "        });\n",
       "        newline = newline.concat(\"</tr>\")\n",
       "        $(\"#preview-table\").append(newline)\n",
       "    });\n",
       "    \n",
       "    \n",
       "    if ($('#data-preview').is(\":hidden\")){$('#data-preview').show()}\n",
       "}\n",
       " \n",
       "\n",
       "<!-- Part 2 - Algo Population & Selection -->\n",
       "\n",
       "<!-- Part 3 - Algo Configuration -->\n",
       "\n",
       "                       \n",
       "function add_conf_forms(data){\n",
       "    dataDict = JSON.parse(data.content.text.replace(/'/g,'\"'));\n",
       "    var temp_algo = Object.keys(dataDict)[0];\n",
       "    params = dataDict[temp_algo];\n",
       "    for(var p in params){\n",
       "        var keepLabel = params[p][\"label\"];\n",
       "        var attributes = reject(params[p],[\"label\"]);\n",
       "        if(attributes[\"type\"]!==\"select\"){\n",
       "            if(attributes[\"grid\"]===\"Y\"){\n",
       "                var $label = $(\"<label>\",{\"class\":\"control-label col-sm-2\",text:keepLabel.concat(\" MinValue\")});\n",
       "                \n",
       "                var attrDict = {};\n",
       "                for(attr in attributes){attrDict[attr]=attributes[attr]}\n",
       "                attrDict[\"grid_\"]=\"min\"\n",
       "                $(\"#algo_\".concat(temp_algo)).append(\n",
       "                    $(\"<div/>\",{class:\"form-group\"}).append(\n",
       "                        $label,\n",
       "                        $(\"<div/>\",{\"class\":\"col-sm-10\"}).append(\n",
       "                            $(\"<input/>\",attrDict\n",
       "                             )\n",
       "                        )\n",
       "                    )\n",
       "                );\n",
       "                var $label = $(\"<label>\",{\"class\":\"control-label col-sm-2\",text:keepLabel.concat(\" MaxValue\")});\n",
       "                attrDict[\"grid_\"]=\"max\"\n",
       "                $(\"#algo_\".concat(temp_algo)).append(\n",
       "                    $(\"<div/>\",{class:\"form-group\"}).append(\n",
       "                        $label,\n",
       "                        $(\"<div/>\",{\"class\":\"col-sm-10\"}).append(\n",
       "                            $(\"<input/>\",attrDict\n",
       "                             )\n",
       "                        )\n",
       "                    )\n",
       "                );\n",
       "            }\n",
       "            else{\n",
       "                var $label = $(\"<label>\",{\"class\":\"control-label col-sm-2\",text:keepLabel});\n",
       "                var attrDict = {};\n",
       "               for(attr in attributes){attrDict[attr]=attributes[attr]}\n",
       "                $(\"#algo_\".concat(temp_algo)).append(\n",
       "                    $(\"<div/>\",{class:\"form-group\"}).append(\n",
       "                        $label,\n",
       "                        $(\"<div/>\",{\"class\":\"col-sm-10\"}).append(\n",
       "                            $(\"<input/>\",attrDict\n",
       "                             )\n",
       "                        )\n",
       "                    )\n",
       "                ); \n",
       "            }\n",
       "        }\n",
       "        else{\n",
       "            var $label = $(\"<label>\",{\"class\":\"control-label col-sm-2\",text:keepLabel});\n",
       "            var attrDict = {};\n",
       "            var newselect = $('<select>')\n",
       "            $(attributes.options).each(function() { newselect.append($(\"<option>\").attr('value',this).text(this));});\n",
       "            var select_div = $($(\"#algo_\".concat(temp_algo)).append(\n",
       "            $(\"<div/>\",{class:\"form-group\"}).append(\n",
       "                $label,\n",
       "                $(\"<div/>\",{\"class\":\"col-sm-10\"}).append(\n",
       "                    $(newselect)\n",
       "                )\n",
       "            )\n",
       "        ));\n",
       "            \n",
       "        }\n",
       "    };\n",
       "    \n",
       "}\n",
       "\n",
       "                                                    \n",
       "<!-- add tempDF columns to the dropdown selectors -->\n",
       "<!-- aec required -->  \n",
       "function handle_output(data){\n",
       "    tempDF_columns = data.content.text;\n",
       "    addColumnFields(tempDF_columns);\n",
       "}\n",
       "\n",
       "\n",
       "function initAlgoConfigurations(){\n",
       "    for(var k in algorithm_list){\n",
       "        var kernel = IPython.notebook.kernel;\n",
       "        var callbacks = {\n",
       "        iopub : {\n",
       "             output : add_conf_forms,\n",
       "        }\n",
       "    }\n",
       "    var msg_id = kernel.execute('get_algorithm_parameters(\"'+k+'\")', callbacks);\n",
       "    }\n",
       "}\n",
       "                                                   \n",
       "function loadAlgorithms(){\n",
       "    var kernel = IPython.notebook.kernel;\n",
       "    var callbacks = {\n",
       "        iopub : {\n",
       "             output : showAlgorithmList,\n",
       "        }\n",
       "    }\n",
       "    family = $('#algoFamilySelect').val();\n",
       "    var msg_id = kernel.execute('get_algos_in_family(\"'+family+'\")', callbacks);\n",
       "}\n",
       "                                                          \n",
       "                                                          \n",
       "function showAlgorithmList(data){\n",
       "    cols = JSON.parse(data.content.text.replace(/\\'/g,'\"'))\n",
       "    $(\"#algoSelect\").children('option:not(:first)').remove();\n",
       "    for(var i = 0; i < cols.length; i++){\n",
       "        algoOptionGenerator($(\"#algoSelect\"),cols[i].trim());\n",
       "    }  \n",
       "    $(\"#algoSelect\")[0].selectedIndex = 0;\n",
       "}\n",
       "                                                \n",
       "<!-- populate column dropdowns -->\n",
       "function addColumnFields(columns){\n",
       "    arraycols = JSON.parse(columns.replace(/\\'/g,'\"'))\n",
       "    $('.df-col').each(function(index){\n",
       "        if($(this).hasClass(\"empty-option-allowed\")){\n",
       "            $(this).children('option:not(:first)').remove();\n",
       "        }\n",
       "        else{\n",
       "            $(this).children('option').remove();\n",
       "        }\n",
       "        \n",
       "        for(var i = 0; i < arraycols.length; i++){\n",
       "            columnOptionGenerator($(this),arraycols[i].trim());\n",
       "        }\n",
       "        \n",
       "    });\n",
       "    \n",
       "}\n",
       "                                           \n",
       "function populate_algo_conf_form(){\n",
       "    selected_algorithm = $(\"#algoSelect\").val();\n",
       "    selected_algorithm_family = $(\"#algoFamilySelect\").val();\n",
       "    algoDiv = \"#algo_\".concat(selected_algorithm);\n",
       "    familyDiv = \"#algocommon_\".concat(selected_algorithm_family);\n",
       "    $(\".algo_configuration\").hide();\n",
       "    $(familyDiv.concat(\" .algoDescription\")).html(algorithm_descriptions[selected_algorithm])\n",
       "    $(familyDiv).show();\n",
       "    $(algoDiv).show();\n",
       "}\n",
       "                                           \n",
       "function selectEvalMethod(form,label){\n",
       "    $(\".eval-form\").not(\"#\"+form).hide()\n",
       "    $(\"#\"+form).show(); \n",
       "    selected_eval = form.replace(\"-form\",\"\");\n",
       "}\n",
       "\n",
       "\n",
       "<!-- Part 4 - Algo Execution -->\n",
       "\n",
       "\n",
       "function executeAlgo(){\n",
       "    \n",
       "    var algoParamDict = {}\n",
       "    \n",
       "    $(\"#algo-results\").html(\"-- pending --\")\n",
       "    \n",
       "    selected_algorithm = $(\"#algoSelect\").val();\n",
       "    selected_algorithm_family = $(\"#algoFamilySelect\").val();\n",
       "    algoDiv = \"#algo_\".concat(selected_algorithm);\n",
       "    familyDiv = \"#algocommon_\".concat(selected_algorithm_family);\n",
       "    \n",
       "    if(selected_algorithm === \"ALS\"){\n",
       "        user_col = $(\"#als-user-id-col\").val()\n",
       "        algoParamDict.user = user_col\n",
       "        item_col = $(\"#als-item-id-col\").val()\n",
       "        algoParamDict.item = item_col\n",
       "        rating_col = $(\"#als-rating-col\").val()\n",
       "        algoParamDict.rating = rating_col\n",
       "        \n",
       "    }\n",
       "    \n",
       "    else if(selected_algorithm_family===\"NLP\"){\n",
       "        text_col = $(\"#nlp-text-col\").val()\n",
       "        algoParamDict.textcol = text_col\n",
       "    }\n",
       "    \n",
       "    else if(selected_algorithm===\"FPGrowth\"){\n",
       "        items_col = $(\"#fpginput-col\").val()\n",
       "        algoParamDict.itemscol = items_col\n",
       "    }\n",
       "    \n",
       "    else{\n",
       "    \n",
       "    if(selected_algorithm_family===\"CL_REG\"){\n",
       "        label_col = $(familyDiv+\" .single-column-selector.df-col\").val()\n",
       "        algoParamDict.label = label_col\n",
       "        if(selected_eval===\"k-fold\"){\n",
       "            algoParamDict.eval = \"k-fold\"\n",
       "            algoParamDict.evalVal = ($(\"#k-fold-form div input\")[0].value) \n",
       "        }\n",
       "        else{\n",
       "            algoParamDict.eval = \"train-test\"\n",
       "            algoParamDict.evalVal = ($(\"#train-test-form div input\")[0].value) \n",
       "        }\n",
       "    }\n",
       "    \n",
       "    else if(selected_algorithm===\"ChiSquared\"){\n",
       "        label_col = $(algoDiv+\" .single-column-selector.df-col\").val()\n",
       "        algoParamDict.label = label_col\n",
       "    }\n",
       "    \n",
       "    feature_cols = $(familyDiv+\" .multiple-column-selector.df-col\").val()\n",
       "    algoParamDict.features=feature_cols\n",
       "    }\n",
       "    \n",
       "    $(algoDiv+\" .form-group\").each(function( index ) {\n",
       "        var label = $(this).children(\"label\")[0];\n",
       "        var value = $($(this).children(\"div\")[0]).children()[0];\n",
       "        algoParamDict[label.innerText]=value.value\n",
       "    });\n",
       "    \n",
       "    \n",
       "    out_dataset = $(\"#datasetSaveSelect\").val()\n",
       "    out_folder = ($(\"#folderNameSelect\").val() !== \"\" ? $(\"#folderNameSelect\").val() : \"unknown_folder_AEC\")\n",
       "    out_path = out_dataset.concat(\"/\",out_folder)\n",
       "    algoParamDict[\"output_path\"]=out_path\n",
       "    lines = algoParamDict\n",
       "    agh = (JSON.stringify(algoParamDict))\n",
       "    \n",
       "    var kernel = IPython.notebook.kernel;\n",
       "    var callbacks = {\n",
       "        iopub : {\n",
       "             output : algo_run_show,\n",
       "        }\n",
       "    }\n",
       "    var msg_id = kernel.execute('run_algo(\"'+selected_algorithm+'\",\\''+agh+'\\')', callbacks); \n",
       "}\n",
       "\n",
       "function algo_run_show(data){\n",
       "    lines = data;\n",
       "    if(data.content.name === \"stderr\"){\n",
       "        $(\"#algo-results\").text(data.content.text)\n",
       "    }\n",
       "    else{\n",
       "        try{\n",
       "            results = JSON.parse(data.content.text.replace(/'/g,'\"'))[\"results\"];\n",
       "            var newcontent = '<br/><label style=\"width: 20%;\" class=\"qblabel\">Results Summary</label>'\n",
       "            $.each( results, function( key, val ) {\n",
       "                newcontent = newcontent.concat(\"<p><b>\",key,\":</b>\",val,\"</p>\")\n",
       "            });\n",
       "            $(\"#algo-results\").html(newcontent);\n",
       "        }\n",
       "        catch(err) {\n",
       "            $(\"#algo-results\").text(\"Oups! Something went wrong: \".concat(data.content.text))\n",
       "        }  \n",
       "    }\n",
       "    \n",
       "}\n",
       "\n",
       "<!-- Part 5 - Output -->\n",
       "\n",
       "<!-- Part 6 - Overview -->\n",
       "function updateOverview(){\n",
       "    out_dataset = ($(\"#datasetSaveSelect\").val() !== \"\" ? $(\"#datasetSaveSelect\").val() : \"--- missing ---\")\n",
       "    out_folder = ($(\"#folderNameSelect\").val() !== \"\" ? $(\"#folderNameSelect\").val() : \"--- missing ---\")\n",
       "    in_file = (currentDFpath !== \"\" ? currentDFpath : \"---missing---\")\n",
       "    $(\"#overview-algorithm\").text($(\"#algoSelect :selected\").text());\n",
       "    $(\"#overview-input\").text(in_file);\n",
       "    $(\"#overview-output\").html(\"Dataset: \".concat(out_dataset,\" </br>Folder: \",out_folder));\n",
       "};\n",
       "\n",
       "                                                   \n",
       "                                                   \n",
       "<!-- Part 7 - Errors -->\n",
       "\n",
       "\n",
       "<!-- Part 8 - Apply Model -->\n",
       "<!-- aec required -->  \n",
       "function selectModelApplyDataset(){\n",
       "    var kernel = IPython.notebook.kernel;\n",
       "    var callbacks = {\n",
       "        iopub : {\n",
       "             output : getCSVfilesFromFS_MA,\n",
       "        }\n",
       "    }\n",
       "    dataset = $('#datasetModelApplySelect').val();\n",
       "    var msg_id = kernel.execute('walk_dataset(\"'+dataset+'\")', callbacks);\n",
       "}\n",
       "                                                           \n",
       "function getCSVfilesFromFS_MA(data){\n",
       "    datafile_options_ma = data.content.text;\n",
       "    datafilesObtainedModelApply(JSON.parse(datafile_options_ma.replace(/u\\'/g,'\"').replace(/\\'/g,'\"')))\n",
       "}\n",
       "                                                           \n",
       "function datafilesObtainedModelApply(options){\n",
       "    var dropl = document.getElementById(\"fileModelApplySelect\");\n",
       "    dropl.options.length = 0;\n",
       "    for (opt in options) {\n",
       "        dropl.options[dropl.options.length] = new Option(options[opt]);\n",
       "    }\n",
       "}  \n",
       "                                                                       \n",
       "function selectDataModelApplyfile(){\n",
       "    var kernel = IPython.notebook.kernel;\n",
       "    var callbacks = {\n",
       "        iopub : {\n",
       "             output : handle_output_model,\n",
       "        }\n",
       "    }\n",
       "    dataset = $('#fileModelApplySelect').val();\n",
       "    separator = $('#sepModelApplySelect').val();\n",
       "    resultFolder = $('#datasetModelApplySelect').val().concat(\"/\").concat($('#folderModelSelect').val());\n",
       "    var msg_id = kernel.execute('apply_model(\"'+dataset+'\",\"'+separator+'\",\"'+resultFolder+'\")', callbacks);\n",
       "}\n",
       "                                                                       \n",
       "function handle_output_model(data){\n",
       "    lines = data;\n",
       "    if(data.content.name === \"stderr\"){\n",
       "        $(\"#model-results\").text(data.content.text)\n",
       "    }\n",
       "    else{\n",
       "        try{\n",
       "            results = JSON.parse(data.content.text.replace(/'/g,'\"'))[\"results\"];\n",
       "            preview_result = results[\"Result preview\"];\n",
       "            var newcontent = '<br/><label style=\"width: 20%;\" class=\"qblabel\">Preview</label>'\n",
       "            $.each( preview_result, function( key, val ) {\n",
       "                newcontent = newcontent.concat(\"<p>\")\n",
       "                $.each( val, function( k, v ) {\n",
       "                newcontent = newcontent.concat(\"<b>\",k,\":</b>\",v,\"  \")\n",
       "            })\n",
       "            newcontent = newcontent.concat(\"</p>\")});\n",
       "            newcontent = newcontent.concat(\"<p>\",results[\"Output\"],\"</p>\")                                               \n",
       "            $(\"#model-results\").html(newcontent);\n",
       "        }\n",
       "        catch(err) {\n",
       "            $(\"#model-results\").text(data.content.text)\n",
       "        }  \n",
       "    }\n",
       "}\n",
       "                                                      \n",
       "</script>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "%%html\n",
    "\n",
    "<style>\n",
    ".tagclass {\n",
    "    background:grey;\n",
    "    border:none;\n",
    "}\n",
    ".fieldcell{\n",
    "    white-space:pre-wrap;\n",
    "}\n",
    "td{\n",
    "  text-align:left;\n",
    "}\n",
    "\n",
    "table{\n",
    "    table-layout: auto !important;\n",
    "}\n",
    "\n",
    "td{\n",
    "    word-wrap:break-word\n",
    "}\n",
    "\n",
    "\n",
    ".paragraph .form-horizontal .form-group {\n",
    "    margin-right: 0px !important;\n",
    "    margin-left: 0px !important;\n",
    "}\n",
    "\n",
    "#list2 { width:320px; }\n",
    "#list2 ol { font-style:italic; font-family:Georgia, Times, serif; font-size:18px; color:#337ab7;  }\n",
    "#list2 ol li { }\n",
    "#list2 ol li p { padding:8px; font-style:normal; font-family:Arial; font-size:13px; color:black; border-left: 1px solid #999; }\n",
    "#list2 ol li.applied p em { color:lightgray;  }\n",
    "#list2 ol li p em { display:block; }\n",
    "\n",
    "\n",
    ".qblabel {\n",
    "    font-style:italic; font-family:Georgia, Times, serif; font-size:18px; color:#337ab7; \n",
    "}\n",
    "\n",
    ".aec-btn {\n",
    "    background-color: Transparent;\n",
    "    background-repeat:no-repeat;\n",
    "    border: none;\n",
    "    cursor:pointer;\n",
    "    overflow: hidden;\n",
    "    outline:none;\n",
    "    font-style:italic; \n",
    "    font-family:Georgia, Times, serif; \n",
    "    font-size:20px; \n",
    "    color:#435066;\n",
    "    padding: 10px 12px 14px 12px;\n",
    "}\n",
    "\n",
    ".aec-btn:hover {\n",
    "    background-color: #ddd;\n",
    "}\n",
    "\n",
    "/* Create an active/current tablink class */\n",
    ".aec-btn.active {\n",
    "    background-color: #ccc;\n",
    "}\n",
    "\n",
    "#aec div{\n",
    "    padding: 2px 4px 2px 16px;\n",
    "}\n",
    "\n",
    "</style>\n",
    "\n",
    "\n",
    "<button type=\"button\" class=\"btn aec-btn\" onclick=\"openTab('aec-file-selector',this);\">Input File</button>\n",
    "<button type=\"button\" class=\"btn aec-btn\" onclick=\"openTab('aec-algo-selector',this);\">Algorithm Selection & Configuration</button>\n",
    "<button type=\"button\" class=\"btn aec-btn\" onclick=\"openTab('aec-save-conf',this);\">Output File</button>\n",
    "<button type=\"button\" class=\"btn aec-btn\" onclick=\"updateOverview();openTab('aec-overview',this);\">Overview</button>\n",
    "<button type=\"button\" class=\"btn aec-btn\" onclick=\"openTab('aec-apply-model',this);\">Apply Model</button>\n",
    "<div id=\"aec\">\n",
    "   <div id=\"aec-overview\" style=\"display:none;\">\n",
    "      <p>\n",
    "         <label style=\"display: inline-block;width: 20%;\" class=\"qblabel\">Selected Input File:</label>\n",
    "      <div id=\"overview-input\" style=\"display: inline-block;width: 70%;\"></div>\n",
    "      </p>\n",
    "      <p>\n",
    "         <label style=\"width: 20%;\" class=\"qblabel\">Selected Algorithm:</label>\n",
    "      <div id=\"overview-algorithm\" style=\"display: inline-block;width: 70%;\"></div>\n",
    "      </p>\n",
    "      <p>\n",
    "         <label style=\"width: 20%;\" class=\"qblabel\">Selected Output File:</label>\n",
    "      <div id=\"overview-output\" style=\"display: inline-block;width: 70%;\"></div>\n",
    "      </p>\n",
    "      <button class=\"btn btn-primary\" id=\"filters-done\" onclick=\"executeAlgo();\"> Execute </button>\n",
    "      <div id=\"algo-results\"></div>\n",
    "   </div>\n",
    "   <div id=\"aec-file-selector\" style=\"display:none;\">\n",
    "      <div class=\"form-group\">\n",
    "         <label style=\"width: 20%;\" class=\"qblabel\">Available Datasets</label>\n",
    "         <div style=\"display: inline-block;width: 60%;\">\n",
    "            <select style=\"margin-left:4px !important;\" id=\"datasetSelect\" class=\"form-control\">\n",
    "            </select>\n",
    "         </div>\n",
    "         <button type=\"button\" class=\"btn btn-primary btn-md\" onclick=\"selectDataset()\"> Apply </button>\n",
    "         <button type=\"button\" class=\"btn btn-primary btn-md\" onclick=\"updateBrowser();\"> Refresh </button>\n",
    "      </div>\n",
    "      <div class=\"form-group\">\n",
    "         <label style=\"width: 20%;\" class=\"qblabel\">Available Files</label>\n",
    "         <div style=\"display: inline-block;width: 60%;\">\n",
    "            <select style=\"margin-left:4px !important;\" id=\"fileSelect\" class=\"form-control\">\n",
    "            </select>\n",
    "         </div>\n",
    "      </div>\n",
    "      <div class=\"form-group\">\n",
    "         <label style=\"width: 20%;\" class=\"qblabel\">CSV Separator</label>\n",
    "         <div style=\"display: inline-block;width: 60%;\">\n",
    "            <select style=\"margin-left:4px !important;\" id=\"sepSelect\" class=\"form-control\">\n",
    "               <option value=\",\">,</option>\n",
    "               <option value=\";\">;</option>\n",
    "            </select>\n",
    "         </div>\n",
    "         <button type=\"button\" class=\"btn btn-primary btn-md\" onclick=\"selectDatafile()\"> Apply </button>\n",
    "         <button type=\"button\" class=\"btn btn-primary btn-md\" onclick=\"getTempDFpreview();\">Refresh Preview</button>\n",
    "      </div>\n",
    "      <div id=\"data-preview\" style=\"display:none\">\n",
    "         <div class=\"panel-body\">\n",
    "            <table class=\"table\">\n",
    "               <tbody id=\"preview-table\">\n",
    "               </tbody>\n",
    "            </table>\n",
    "         </div>\n",
    "      </div>\n",
    "      <div class=\"form-group\">\n",
    "         <button type=\"button\" class=\"btn btn-primary btn-md\" onclick=\"openTab('aec-algo-selector',this);\"> Next </button>\n",
    "      </div>\n",
    "   </div>\n",
    "   <hr>\n",
    "   <hr>\n",
    "   <div class=\"form-horizontal\" id=\"aec-algo-selector\" style=\"display:none;\">\n",
    "      <div class=\"form-group\">\n",
    "         <label style=\"width: 20%;text-align:left;\" class=\"control-label col-sm-2 qblabel\">Algorithm Family</label>\n",
    "         <div class=\"col-sm-10\" style=\"display: inline-block;width: 70%;\">\n",
    "            <select style=\"margin-left:4px !important;\" class=\"form-control\" id=\"algoFamilySelect\" onchange=\"loadAlgorithms();$('.algo_configuration').hide();\">\n",
    "               <option value=\"\" disabled selected></option>\n",
    "               <option value=\"DIMRE_FE\">DIMENSIONALITY REDUCTION/FEATURE EXTRACTION/SELECTION</option>\n",
    "               <option value=\"NLP\">NLP FUNCTIONS</option>\n",
    "               <option value=\"RECOM\">RECOMMENDERS</option>\n",
    "               <option value=\"CLUST\">CLUSTERING</option>\n",
    "               <option value=\"CL_REG\">CLASSIFICATION/REGRESSION</option>\n",
    "               <option value=\"FREQPM\">FREQUENT PATTERN MINING</option>\n",
    "            </select>\n",
    "         </div>\n",
    "      </div>\n",
    "      <div class=\"form-group\">\n",
    "         <label style=\"width: 20%;text-align:left;\" class=\"control-label col-sm-2 qblabel\">Algorithm</label>\n",
    "         <div class=\"col-sm-10\" style=\"display: inline-block;width: 70%;\">\n",
    "            <select style=\"margin-left:4px !important;\" class=\"form-control\" id=\"algoSelect\" onchange=\"populate_algo_conf_form()\">\n",
    "               <option value=\"\" disabled selected>-- select --</option>\n",
    "            </select>\n",
    "         </div>\n",
    "      </div>\n",
    "      <div class=\"form-horizontal algo_configuration \" id=\"algocommon_DIMRE_FE\" style=\"display:none\">\n",
    "         <p class=\"algoDescription\"></p>\n",
    "         <h3>Configuration</h3>\n",
    "         <div class=\"form-group\">\n",
    "            <label class=\"control-label col-sm-2\">Feature Columns</label>\n",
    "            <div class=\"col-sm-10\">\n",
    "               <select multiple class=\"form-control multiple-column-selector df-col\">\n",
    "               </select>\n",
    "            </div>\n",
    "         </div>\n",
    "         <div class=\"form-horizontal algo_configuration\" id=\"algo_PCA\" style=\"display:none\"></div>\n",
    "         <div class=\"form-horizontal algo_configuration\" id=\"algo_ChiSquared\" style=\"display:none\">\n",
    "            <div class=\"form-group\">\n",
    "               <label class=\"control-label col-sm-2\">Label Column</label>\n",
    "               <div class=\"col-sm-10\">\n",
    "                  <select class=\"form-control single-column-selector df-col\">\n",
    "                  </select>\n",
    "               </div>\n",
    "            </div>\n",
    "         </div>\n",
    "      </div>\n",
    "      <div class=\"form-horizontal algo_configuration \" id=\"algocommon_NLP\" style=\"display:none\">\n",
    "         <p class=\"algoDescription\"></p>\n",
    "         <div class=\"form-group\">\n",
    "            <label class=\"control-label col-sm-2\">Text Column</label>\n",
    "            <div class=\"col-sm-10\">\n",
    "               <select class=\"form-control single-column-selector df-col\" id=\"nlp-text-col\">\n",
    "               </select>\n",
    "            </div>\n",
    "         </div>\n",
    "         <div class=\"form-horizontal algo_configuration\" id=\"algo_TOKENIZER\" style=\"display:none\"></div>\n",
    "         <div class=\"form-horizontal algo_configuration\" id=\"algo_N-GRAMS\" style=\"display:none\"></div>\n",
    "         <div class=\"form-horizontal algo_configuration\" id=\"algo_TF-IDF\" style=\"display:none\">TF-IDF</div>\n",
    "      </div>\n",
    "      <div class=\"form-horizontal algo_configuration \" id=\"algocommon_RECOM\" style=\"display:none\">\n",
    "         <p class=\"algoDescription\"> </p>\n",
    "         <div class=\"form-horizontal algo_configuration\" id=\"algo_ALS\" style=\"display:none\">\n",
    "            <div class=\"form-group\">\n",
    "               <label class=\"control-label col-sm-2\">User Column</label>\n",
    "               <div class=\"col-sm-10\">\n",
    "                  <select class=\"form-control single-column-selector df-col\" id=\"als-user-id-col\">\n",
    "                  </select>\n",
    "               </div>\n",
    "            </div>\n",
    "            <div class=\"form-group\">\n",
    "               <label class=\"control-label col-sm-2\">Item Column</label>\n",
    "               <div class=\"col-sm-10\">\n",
    "                  <select class=\"form-control single-column-selector df-col\" id=\"als-item-id-col\">\n",
    "                  </select>\n",
    "               </div>\n",
    "            </div>\n",
    "            <div class=\"form-group\">\n",
    "               <label class=\"control-label col-sm-2\">Rating Column</label>\n",
    "               <div class=\"col-sm-10\">\n",
    "                  <select class=\"form-control single-column-selector df-col\" id=\"als-rating-col\">\n",
    "                  </select>\n",
    "               </div>\n",
    "            </div>\n",
    "         </div>\n",
    "      </div>\n",
    "      <div class=\"form-horizontal algo_configuration \" id=\"algocommon_FREQPM\" style=\"display:none\">\n",
    "         <p class=\"algoDescription\"> </p>\n",
    "         <h3>Configuration</h3>\n",
    "         <div class=\"form-horizontal algo_configuration\" id=\"algo_FPGrowth\" style=\"display:none\">\n",
    "            <div class=\"form-group\">\n",
    "               <label class=\"control-label col-sm-2\">Items Column</label>\n",
    "               <div class=\"col-sm-10\">\n",
    "                  <select class=\"form-control single-column-selector df-col\" id=\"fpginput-col\">\n",
    "                  </select>\n",
    "               </div>\n",
    "            </div>\n",
    "         </div>\n",
    "      </div>\n",
    "      <div class=\"form-horizontal algo_configuration \" id=\"algocommon_CLUST\" style=\"display:none\">\n",
    "         <p class=\"algoDescription\"></p>\n",
    "         <h3>Configuration</h3>\n",
    "         <div class=\"form-group\">\n",
    "            <label class=\"control-label col-sm-2\">Feature Columns</label>\n",
    "            <div class=\"col-sm-10\">\n",
    "               <select multiple class=\"form-control multiple-column-selector df-col\">\n",
    "               </select>\n",
    "            </div>\n",
    "         </div>\n",
    "         <div class=\"form-horizontal algo_configuration\" id=\"algo_KMEANS\" style=\"display:none\"></div>\n",
    "         <div class=\"form-horizontal algo_configuration\" id=\"algo_GAUSSMIX\" style=\"display:none\"></div>\n",
    "      </div>\n",
    "      <div class=\"form-horizontal algo_configuration \" id=\"algocommon_CL_REG\" style=\"display:none\">\n",
    "         <p> Note that for the classification algorithms, the label column should contain integers from 0 to N-1, where N the number of classes. </p>\n",
    "         <p class=\"algoDescription\"></p>\n",
    "         <h3>Configuration</h3>\n",
    "         <div class=\"form-group\">\n",
    "            <label class=\"control-label col-sm-2\">Label Column</label>\n",
    "            <div class=\"col-sm-10\">\n",
    "               <select class=\"form-control single-column-selector df-col\">\n",
    "               </select>\n",
    "            </div>\n",
    "         </div>\n",
    "         <div class=\"form-group\">\n",
    "            <label class=\"control-label col-sm-2\">Feature Columns</label>\n",
    "            <div class=\"col-sm-10\">\n",
    "               <select multiple class=\"form-control multiple-column-selector df-col\">\n",
    "               </select>\n",
    "            </div>\n",
    "         </div>\n",
    "         <div class=\"form-horizontal algo_configuration\" id=\"algo_OLS\" style=\"display:none\"></div>\n",
    "         <div class=\"form-horizontal algo_configuration\" id=\"algo_DTR\" style=\"display:none\"></div>\n",
    "         <div class=\"form-horizontal algo_configuration\" id=\"algo_DTC\" style=\"display:none\"></div>\n",
    "         <div class=\"form-horizontal algo_configuration\" id=\"algo_MLP\" style=\"display:none\"> </div>\n",
    "         <div class=\"form-horizontal algo_configuration\" id=\"algo_NB\" style=\"display:none\"></div>\n",
    "         <div class=\"form-horizontal algo_configuration\" id=\"algo_GLM\" style=\"display:none\"></div>\n",
    "         <div class=\"form-horizontal algo_configuration\" id=\"algo_RFR\" style=\"display:none\"></div>\n",
    "         <div class=\"form-horizontal algo_configuration\" id=\"algo_RFC\" style=\"display:none\"></div>\n",
    "         <div class=\"form-horizontal algo_configuration\" id=\"algo_GBTR\" style=\"display:none\"></div>\n",
    "         <div class=\"form-horizontal algo_configuration\" id=\"algo_GBTC\" style=\"display:none\"></div>\n",
    "         <div class=\"form-horizontal algo_configuration\" id=\"algo_LOGRE\" style=\"display:none\"></div>\n",
    "         <div class=\"form-horizontal \" id=\"CL_REG_EVAL\">\n",
    "            <div class=\"form-group\">\n",
    "               <div class=\"btn-group btn-group-toggle\" data-toggle=\"buttons\">\n",
    "                  <label class=\"btn btn-secondary eval-label\" onclick=\"selectEvalMethod('train-test-form',this);\" style=\"font-size:16px\">\n",
    "                  <input type=\"radio\" name=\"validation-method\" id=\"train-test\"> Train-Test Split\n",
    "                  </label>\n",
    "                  <label class=\"btn btn-secondary eval-label\" onclick=\"selectEvalMethod('k-fold-form',this);\" style=\"font-size:16px\">\n",
    "                  <input type=\"radio\" name=\"validation-method\"> K-Fold Cross Validation\n",
    "                  </label>\n",
    "               </div>\n",
    "               <div class=\"form-group eval-form\" id=\"k-fold-form\" style=\"display:none\">\n",
    "                  <label class=\"control-label col-sm-2\">Number of Folds for Cross-Validation</label>\n",
    "                  <div class=\"col-sm-10\">\n",
    "                     <input type=\"number\" min=\"2\" max=\"10\" step=\"1\" value=\"2\"></input>\n",
    "                  </div>\n",
    "               </div>\n",
    "               <div class=\"form-group eval-form\" id=\"train-test-form\" style=\"display:none\">\n",
    "                  <label class=\"control-label col-sm-2\">Percentage % of input data to be used for training</label>\n",
    "                  <div class=\"col-sm-10\">\n",
    "                     <input type=\"number\" min=\"10\" max=\"100\" step=\"5\" value=\"80\"></input>\n",
    "                  </div>\n",
    "               </div>\n",
    "            </div>\n",
    "         </div>\n",
    "      </div>\n",
    "      <div class=\"form-group\">\n",
    "         <button type=\"button\" class=\"btn btn-primary btn-md\" onclick=\"openTab('aec-save-conf',this);\"> Next </button>\n",
    "      </div>\n",
    "   </div>\n",
    "   <div style=\"display:none;\" id=\"aec-save-conf\">\n",
    "      <div class=\"form-group\">\n",
    "         <label style=\"width: 20%;\" class=\"qblabel\">Select Dataset</label>\n",
    "         <div style=\"display: inline-block;width: 60%;\">\n",
    "            <select style=\"margin-left:4px !important;\" id=\"datasetSaveSelect\" class=\"form-control\">\n",
    "            </select>\n",
    "         </div>\n",
    "         <button type=\"button\" class=\"btn btn-primary btn-md\" onclick=\"updateBrowser();\"> Update Browser </button>\n",
    "      </div>\n",
    "      <div class=\"form-group\">\n",
    "         <label style=\"width: 20%;\" class=\"qblabel\">Result Folder Name</label>\n",
    "         <div style=\"display: inline-block;width: 60%;\">\n",
    "            <input style=\"margin-left:4px !important;\" id=\"folderNameSelect\" class=\"form-control\">\n",
    "            </input>\n",
    "         </div>\n",
    "      </div>\n",
    "      <!--\n",
    "         <div class=\"form-group\">\n",
    "           <label style=\"width: 20%;\" class=\"qblabel\">Include Header</label>\n",
    "           <div style=\"display: inline-block;width: 60%;\">\n",
    "         <input type=\"checkbox\" id=\"saveHeaderCheckbox\" value=\"with\" checked>                   \n",
    "             </input> \n",
    "           </div>\n",
    "         </div>\n",
    "         -->\n",
    "      <div class=\"form-group\">\n",
    "         <p>If the folder you specified exists, <b> it will be overwritten </b> with the results of the last algorithm execution.</p>\n",
    "         <button type=\"button\" class=\"btn btn-primary btn-md\" onclick=\"updateOverview();openTab('aec-overview',this);\"> Done </button>\n",
    "      </div>\n",
    "   </div>\n",
    "   <div style=\"display:none;\" id=\"aec-apply-model\">\n",
    "      <div class=\"form-group\">\n",
    "         <p>This is only possible for classification and regression models. <b> Make sure that the input file you provide has the same \n",
    "            structure as the one used for training the model,i.e. that its columns correspond to the features used in the training -\n",
    "            no extra columns and no label column should be present in the file. </b>  </p>\n",
    "            <div class=\"form-group\">\n",
    "         <label style=\"width: 20%;\" class=\"qblabel\">Available Datasets</label>\n",
    "         <div style=\"display: inline-block;width: 60%;\">\n",
    "            <select style=\"margin-left:4px !important;\" id=\"datasetModelApplySelect\" class=\"form-control\">\n",
    "            </select>\n",
    "         </div>\n",
    "         <button type=\"button\" class=\"btn btn-primary btn-md\" onclick=\"selectModelApplyDataset()\"> Apply </button>\n",
    "         <button type=\"button\" class=\"btn btn-primary btn-md\" onclick=\"updateBrowser();\"> Refresh </button>\n",
    "      </div>\n",
    "      <div class=\"form-group\">\n",
    "         <label style=\"width: 20%;\" class=\"qblabel\">Available Files</label>\n",
    "         <div style=\"display: inline-block;width: 60%;\">\n",
    "            <select style=\"margin-left:4px !important;\" id=\"fileModelApplySelect\" class=\"form-control\">\n",
    "            </select>\n",
    "         </div>\n",
    "      </div>\n",
    "      <div class=\"form-group\">\n",
    "         <label style=\"width: 20%;\" class=\"qblabel\">CSV Separator</label>\n",
    "         <div style=\"display: inline-block;width: 60%;\">\n",
    "            <select style=\"margin-left:4px !important;\" id=\"sepModelApplySelect\" class=\"form-control\">\n",
    "               <option value=\",\">,</option>\n",
    "               <option value=\";\">;</option>\n",
    "            </select>\n",
    "         </div>\n",
    "         \n",
    "      </div>\n",
    "    <div class=\"form-group\">\n",
    "         <label style=\"width: 20%;\" class=\"qblabel\">Result Folder Name</label>\n",
    "         <div style=\"display: inline-block;width: 60%;\">\n",
    "            <input style=\"margin-left:4px !important;\" id=\"folderModelSelect\" class=\"form-control\">\n",
    "            </input>\n",
    "         </div>\n",
    "        <button type=\"button\" class=\"btn btn-primary btn-md\" onclick=\"selectDataModelApplyfile()\"> Apply Model</button>\n",
    "      </div>\n",
    "    <div id=\"model-results\"></div>\n",
    "      </div>\n",
    "   </div>\n",
    "</div>\n",
    "\n",
    "\n",
    "\n",
    "<script>\n",
    "\n",
    "var tempDF_columns = []\n",
    "var datafile_options = []\n",
    "var datafile_options_ma = []\n",
    "\n",
    "var currentDFpath = \"\"\n",
    "var lines\n",
    "\n",
    "var algorithm_list = {\n",
    "    \"PCA\":\"Principal Component Analysis (PCA)\",\n",
    "    \"ChiSquared\":\"Chi-squared test\",\n",
    "    \"TOKENIZER\":\"Tokenizer\",\n",
    "    \"N-GRAMS\":\"N-Grams\",\n",
    "    \"TF-IDF\":\"TF-IDF\",\n",
    "    \"ALS\":\"Collaborative Filtering (ALS)\",\n",
    "    \"KMEANS\":\"K-MEANS\",\n",
    "    \"GAUSSMIX\":\"Gaussian Mixtures\",\n",
    "    \"OLS\":\"Linear Regression\",\n",
    "    \"DTR\":\"Decision Trees Regression\",\n",
    "    \"DTC\":\"Decision Trees Classifier\",\n",
    "    \"MLP\":\"Multi-Layer Perceptron\",\n",
    "    \"NB\":\"Naive Bayes\",\n",
    "    \"GLM\":\"Generalized Linear Models (GLM)\",\n",
    "    \"RFR\":\"Random Forest Regressor (RFR)\",\n",
    "    \"GBTR\":\"Gradient-boosted tree Regression (GBTR)\",\n",
    "    \"RFC\":\"Random Forest Classifier (RFC)\",\n",
    "    \"GBTC\":\"Gradient-boosted tree Classifier (GBTC)\",\n",
    "    \"LOGRE\":\"Logistic Regression\",\n",
    "    \"FPGrowth\":\"FP Growth\"\n",
    "}\n",
    "\n",
    "var algorithm_descriptions = {\n",
    "    \"PCA\":\"Principal Component Analysis (PCA) is a statistical procedure that uses an orthogonal transformation to \\\n",
    "        convert a set of observations of possibly correlated variables into a set of values of linearly uncorrelated variables\\\n",
    "        called principal components. A PCA class trains a model to project vectors to a low-dimensional space using PCA. \\\n",
    "        <a href=\\\"https://spark.apache.org/docs/2.3.2/ml-features.html#pca\\\" target=\\\"_blank\\\"> More info...</a>\",\n",
    "    \"ChiSquared\":\"Chi-squared stands for Chi-Squared feature selection. It operates on labeled data with categorical \\\n",
    "        features. ChiSqSelector uses the Chi-Squared test of independence to decide which features to choose.test\\\n",
    "        <a href=\\\"https://spark.apache.org/docs/2.3.2/ml-features.html#chisqselector\\\" target=\\\"_blank\\\">More info...</a>\",\n",
    "    \"TOKENIZER\":\"Tokenization is the process of taking text (such as a sentence) and breaking it into individual terms \\\n",
    "        (usually words). Here we use RegexTokenizer that converts the input string to lowercase, \\\n",
    "        removes stopwords and then splits it by white spaces. \\\n",
    "        <a href=\\\"https://spark.apache.org/docs/2.3.2/ml-features.html#tokenizer\\\" target=\\\"_blank\\\"> More info...</a>\",\n",
    "    \"N-GRAMS\":\"An n-gram is a sequence of n tokens (typically words) for some integer n. The NGram class can be used to \\\n",
    "        transform input features into n-grams. The parameter n is used to determine the number of terms in each n-gram. \\\n",
    "        The output will consist of a sequence of n-grams where each n-gram is represented by a space-delimited string \\\n",
    "        of n consecutive words. If the input sequence contains fewer than n strings, no output is produced. \\\n",
    "        <a href=\\\"https://spark.apache.org/docs/2.3.2/ml-features.html#n-gram\\\" target=\\\"_blank\\\"> More info...</a>\",\n",
    "    \"TF-IDF\":\"Term frequency-inverse document frequency (TF-IDF) is a feature vectorization method widely used in \\\n",
    "        text mining to reflect the importance of a term to a document in the corpus. \\\n",
    "        <a href=\\\"https://spark.apache.org/docs/2.3.2/ml-features.html#tf-idf\\\" target=\\\"_blank\\\"> More info...</a>\",\n",
    "    \"ALS\":\"Collaborative filtering is commonly used for recommender systems. These techniques aim to fill in the \\\n",
    "        missing entries of a user-item association matrix. spark.ml currently supports model-based collaborative filtering,\\\n",
    "        in which users and products are described by a small set of latent factors that can be used to predict missing \\\n",
    "        entries. spark.ml uses the alternating least squares (ALS) algorithm to learn these latent factors. \\\n",
    "        <a href=\\\"https://spark.apache.org/docs/2.3.2/ml-collaborative-filtering.html\\\" target=\\\"_blank\\\"> More info...</a>\",\n",
    "    \"KMEANS\":\"K-means is one of the most commonly used clustering algorithms that clusters the data points into a \\\n",
    "        predefined number of clusters (K). \\\n",
    "        <a href=\\\"https://spark.apache.org/docs/2.3.2/ml-clustering.html#k-means\\\" target=\\\"_blank\\\"> More info...</a>\",\n",
    "    \"GAUSSMIX\":\"A Gaussian Mixture Model represents a composite distribution whereby points are drawn from one of \\\n",
    "        k Gaussian sub-distributions, each with its own probability. The spark.ml implementation uses the \\\n",
    "        expectation-maximization algorithm to induce the maximum-likelihood model given a set of samples. \\\n",
    "        <a href=\\\"https://spark.apache.org/docs/2.3.2/ml-clustering.html#gaussian-mixture-model-gmm\\\" target=\\\"_blank\\\"> More info...</a>\",\n",
    "    \"OLS\":\"Ordinary Least squares (OLS) is the simplest and most common linear regressor. The learning objective of OLS \\\n",
    "        is to minimize the sum of squared residuals, in order to estimate the coefficients of the linear regression \\\n",
    "        expression. \\\n",
    "        <a href=\\\"https://spark.apache.org/docs/2.3.2/ml-classification-regression.html#linear-regression\\\" target=\\\"_blank\\\">More info...</a>\",\n",
    "    \"DTR\":\"Decision trees are a popular family of classification and regression methods. \\\n",
    "        More information about the spark.ml implementation can be found \\\n",
    "        <a href=\\\"https://spark.apache.org/docs/2.3.2/ml-classification-regression.html#decision-trees\\\" target=\\\"_blank\\\"> More info...</a>\",\n",
    "    \"DTC\":\"Decision trees are a popular family of classification and regression methods. \\\n",
    "        <a href=\\\"https://spark.apache.org/docs/2.3.2/ml-classification-regression.html#decision-trees\\\" target=\\\"_blank\\\"> More info...</a>\",\n",
    "    \"MLP\":\"Multilayer perceptron classifier (MLPC) is a classifier based on the feedforward artificial neural network. \\\n",
    "        MLPC consists of multiple layers of nodes. Each layer is fully connected to the next layer in the network. \\\n",
    "        Nodes in the input layer represent the input data. The number of nodes N in the output layer corresponds to the \\\n",
    "        number of classes. Here we use two inner layers for which the number of nodes can be selected by the user.\\\n",
    "        <a href=\\\"https://spark.apache.org/docs/2.3.2/ml-classification-regression.html#multilayer-perceptron-classifier\\\" target=\\\"_blank\\\"> More info...</a>\",\n",
    "    \"NB\":\"Naive Bayes classifiers are a family of simple probabilistic classifiers based on applying Bayes’ theorem \\\n",
    "        with strong (naive) independence assumptions between the features.  \\\n",
    "        <a href=\\\"https://spark.apache.org/docs/2.3.2/ml-classification-regression.html#naive-bayes\\\" target=\\\"_blank\\\"> More info...</a>\",\n",
    "    \"GLM\":\"Contrasted with linear regression where the output is assumed to follow a Gaussian distribution, generalized linear models (GLM) are specifications of \\\n",
    "        linear models where the response variable follows some distribution from the exponential family of distributions. \\\n",
    "        <a href=\\\"https://spark.apache.org/docs/2.3.2/ml-classification-regression.html#generalized-linear-regression\\\" target=\\\"_blank\\\"> More info...</a>\",\n",
    "    \"RFR\":\"Random forests are a popular family of classification and regression methods. \\\n",
    "        <a href=\\\"https://spark.apache.org/docs/2.3.2/ml-classification-regression.html#random-forests\\\" target=\\\"_blank\\\"> More info...</a>\",\n",
    "    \"GBTR\":\"Gradient-boosted trees (GBTs) are a popular regression method using ensembles of decision trees. \\\n",
    "        Parameter configuration should be performed with caution, as parameters significantly impact the algorithm's execution time.\\\n",
    "        <a href=\\\"https://spark.apache.org/docs/2.3.2/ml-classification-regression.html#gradient-boosted-tree-regression\\\" target=\\\"_blank\\\"> More info...</a>\",\n",
    "    \"RFC\":\"Random forests are a popular family of classification and regression methods. \\\n",
    "        <a href=\\\"https://spark.apache.org/docs/2.3.2/ml-classification-regression.html#random-forests\\\" target=\\\"_blank\\\"> More info...</a>\",\n",
    "    \"GBTC\":\"Gradient-boosted trees (GBTs) are a popular classification and regression method using ensembles \\\n",
    "        of decision trees. <b> The current implementation supports only binary classification. </b>\\\n",
    "        <a href=\\\"https://spark.apache.org/docs/2.3.2/ml-classification-regression.html#gradient-boosted-tree-classifier\\\" target=\\\"_blank\\\"> More info...</a>\",\n",
    "    \"LOGRE\":\"<p><b>Logistic regression</b> is a popular method to predict a categorical response. \\\n",
    "        It is a special case of Generalized Linear models that predicts the probability of the outcomes. \\\n",
    "        In spark.ml logistic regression can be used to predict a binary outcome by using binomial logistic regression, \\\n",
    "        or it can be used to predict a multiclass outcome by using multinomial logistic regression. \\\n",
    "        <a href=\\\"https://spark.apache.org/docs/2.3.2/ml-classification-regression.html#logistic-regression\\\" target=\\\"_blank\\\"> More info...</a></p>\",\n",
    "    \"FPGrowth\":\"Mining frequent items, itemsets, subsequences, or other substructures is usually among the first steps to analyze a large-scale dataset, \\\n",
    "        which has been an active research topic in data mining for years. The <b>FP Growth</b> algorithm  is described in \\\n",
    "        the paper Han et al., Mining frequent patterns without candidate generation, where “FP” stands for frequent pattern.\\\n",
    "        Given a dataset of transactions, the first step of FP-growth is to calculate item frequencies and identify frequent \\\n",
    "        items. Different from Apriori-like algorithms designed for the same purpose, the second step of FP-growth uses \\\n",
    "        a suffix tree (FP-tree) structure to encode transactions without generating candidate sets explicitly, which are \\\n",
    "        usually expensive to generate. After the second step, the frequent itemsets can be extracted from the FP-tree. \\\n",
    "        <a href=\\\"https://spark.apache.org/docs/2.3.2/ml-frequent-pattern-mining.html#fp-growth\\\" target=\\\"_blank\\\"> More info...</a>\"\n",
    "}\n",
    "\n",
    "var parameters = []\n",
    "var selected_algorithm;\n",
    "var selected_eval = \"train-test\"\n",
    "\n",
    "<!-- Helpers -->\n",
    "\n",
    "<!-- https://www.jstips.co/en/javascript/picking-and-rejecting-object-properties/  -->\n",
    "function reject(obj, keys) {\n",
    "    return Object.keys(obj)\n",
    "        .filter(k => !keys.includes(k))\n",
    "        .map(k => Object.assign({}, {[k]: obj[k]}))\n",
    "        .reduce((res, o) => Object.assign(res, o), {});\n",
    "}\n",
    "\n",
    "function openTab(tab,btn){\n",
    "    $(\".aec-btn\").not(btn).removeClass('active');\n",
    "    $(btn).toggleClass('active');\n",
    "    $(\"#aec\").children().not(\"#\"+tab).hide()\n",
    "    $(\"#\"+tab).slideToggle(\"fast\");\n",
    "}\n",
    "\n",
    "<!-- helper: adds options to help populate algo option dropdowns -->\n",
    "function algoOptionGenerator(instance,paramObj){  \n",
    "    instance\n",
    "    .append($(\"<option></option>\")\n",
    "            .attr(\"value\",paramObj)\n",
    "            .text(algorithm_list[paramObj]));\n",
    "};\n",
    "                                           \n",
    "<!-- helper: adds options to help populate column dropdowns -->\n",
    "function columnOptionGenerator(instance,paramObj){  \n",
    "    instance\n",
    "    .append($(\"<option></option>\")\n",
    "            .attr(\"value\",paramObj)\n",
    "            .text(paramObj));\n",
    "};\n",
    "\n",
    "\n",
    "<!-- Part 1 - Input -->\n",
    "\n",
    "<!-- call python to read file from hdfs, reset filter dropdown and populate column dropdowns -->\n",
    "<!-- aec required -->  \n",
    "function selectDatafile(){\n",
    "    var kernel = IPython.notebook.kernel;\n",
    "    var callbacks = {\n",
    "        iopub : {\n",
    "             output : handle_output,\n",
    "        }\n",
    "    }\n",
    "    dataset = $('#fileSelect').val();\n",
    "    currentDFpath = $('#fileSelect').val();\n",
    "    separator = $('#sepSelect').val();\n",
    "    var msg_id = kernel.execute('read_dataset(\"'+dataset+'\",\"'+separator+'\")', callbacks);\n",
    "}\n",
    "\n",
    "<!-- aec required -->  \n",
    "function selectDataset(){\n",
    "    var kernel = IPython.notebook.kernel;\n",
    "    var callbacks = {\n",
    "        iopub : {\n",
    "             output : getCSVfilesFromFS,\n",
    "        }\n",
    "    }\n",
    "    dataset = $('#datasetSelect').val();\n",
    "    var msg_id = kernel.execute('walk_dataset(\"'+dataset+'\")', callbacks);\n",
    "}\n",
    "\n",
    "<!-- aec required -->  \n",
    "function getCSVfilesFromFS(data){\n",
    "    datafile_options = data.content.text;\n",
    "    datafilesObtained(JSON.parse(datafile_options.replace(/u\\'/g,'\"').replace(/\\'/g,'\"')))\n",
    "}\n",
    "                                                          \n",
    "<!-- aec required -->                                                              \n",
    "function populateAvailableDatasetsList(data){\n",
    "    $.get(\"/hopsworks-api/api/project/\"+data.content.text+\"/dataset/getContent\",  function(data,status){\n",
    "        var user_datasets = []\n",
    "        skipthem = [\"Jupyter\",\"Logs\",\"Models\",\"notebook\",\"Resources\"]\n",
    "        data.forEach(function(element){\n",
    "            if(skipthem.includes(element.name) || element.name.endsWith(\".db\")){\n",
    "                return;\n",
    "            } \n",
    "            user_datasets.push(element.path)\n",
    "        });\n",
    "        var dropl = document.getElementById(\"datasetSelect\");\n",
    "        dropl.options.length = 0;\n",
    "        for (var i = 0; i < user_datasets.length; i++) { \n",
    "            dropl.options[dropl.options.length] = new Option(user_datasets[i].substring(user_datasets[i].lastIndexOf(\"/\")+1),user_datasets[i]);\n",
    "        }\n",
    "        var droplsave = document.getElementById(\"datasetSaveSelect\");\n",
    "        droplsave.options.length = 0;\n",
    "        for (var i = 0; i < user_datasets.length; i++) { \n",
    "            droplsave.options[droplsave.options.length] = new Option(user_datasets[i].substring(user_datasets[i].lastIndexOf(\"/\")+1),user_datasets[i]);\n",
    "        }\n",
    "        var droplmodelapply = document.getElementById(\"datasetModelApplySelect\");\n",
    "        droplmodelapply.options.length = 0;\n",
    "        for (var i = 0; i < user_datasets.length; i++) { \n",
    "            droplmodelapply.options[droplmodelapply.options.length] = new Option(user_datasets[i].substring(user_datasets[i].lastIndexOf(\"/\")+1),user_datasets[i]);\n",
    "        }\n",
    "    })\n",
    "}\n",
    "                                                             \n",
    "<!-- aec required -->                                                               \n",
    "function updateBrowser() {\n",
    "        $.get(\"/hopsworks-api/api/project\", function(data,status){\n",
    "            var pdict = {}\n",
    "            data.forEach(function(e){\n",
    "                pdict[e.project.name]=e.project.id;\n",
    "            });\n",
    "            var kernel = IPython.notebook.kernel;\n",
    "            var callbacks = {\n",
    "                iopub : {\n",
    "                    output : populateAvailableDatasetsList,\n",
    "                }\n",
    "            }\n",
    "            var pdict_parameter = (JSON.stringify(pdict))\n",
    "            var msg_id = kernel.execute('find_current_projectID(\\''+pdict_parameter+'\\')', callbacks);\n",
    "        });    \n",
    "} \n",
    "                                                             \n",
    "function datafilesObtained(options){\n",
    "    var dropl = document.getElementById(\"fileSelect\");\n",
    "    dropl.options.length = 0;\n",
    "    for (opt in options) {\n",
    "        dropl.options[dropl.options.length] = new Option(options[opt]);\n",
    "    }\n",
    "}                                                             \n",
    "\n",
    "                                                          \n",
    "<!-- Part 1.1 Preview -->\n",
    "function getTempDFpreview(){\n",
    "    var kernel = IPython.notebook.kernel;\n",
    "    var callbacks = {\n",
    "        iopub : {\n",
    "             output : show_preview_new,\n",
    "        }\n",
    "    }\n",
    "    var msg_id = kernel.execute('get_tempdf_10first_lines_new()', callbacks); \n",
    "}\n",
    "\n",
    "function show_preview_new(data){\n",
    "    lines = data.content.text.slice(1,-1).replace(/ u\\'/g,'\"').replace(/\\'/g,'\"').replace(/\\[/g,'').split('],');\n",
    "    $('#preview-table').empty();\n",
    "    newline=\"<tr>\"\n",
    "    $.each( JSON.parse(tempDF_columns.replace(/\\'/g,'\"')), function( key, val ) {\n",
    "        newline = newline.concat(\"<th>\"+ val +\"</th>\")\n",
    "    });\n",
    "    newline = newline.concat(\"</tr>\")\n",
    "    $(\"#preview-table\").append(newline)\n",
    "    $.each( lines, function( key, val ) {\n",
    "        newline = \"<tr>\"\n",
    "        valarray = val.split(',');\n",
    "        $.each( valarray, function( k, v ) {\n",
    "            newline = newline.concat(\"<td>\"+ v +\"</td>\")\n",
    "        });\n",
    "        newline = newline.concat(\"</tr>\")\n",
    "        $(\"#preview-table\").append(newline)\n",
    "    });\n",
    "    \n",
    "    \n",
    "    if ($('#data-preview').is(\":hidden\")){$('#data-preview').show()}\n",
    "}\n",
    " \n",
    "\n",
    "<!-- Part 2 - Algo Population & Selection -->\n",
    "\n",
    "<!-- Part 3 - Algo Configuration -->\n",
    "\n",
    "                       \n",
    "function add_conf_forms(data){\n",
    "    dataDict = JSON.parse(data.content.text.replace(/'/g,'\"'));\n",
    "    var temp_algo = Object.keys(dataDict)[0];\n",
    "    params = dataDict[temp_algo];\n",
    "    for(var p in params){\n",
    "        var keepLabel = params[p][\"label\"];\n",
    "        var attributes = reject(params[p],[\"label\"]);\n",
    "        if(attributes[\"type\"]!==\"select\"){\n",
    "            if(attributes[\"grid\"]===\"Y\"){\n",
    "                var $label = $(\"<label>\",{\"class\":\"control-label col-sm-2\",text:keepLabel.concat(\" MinValue\")});\n",
    "                \n",
    "                var attrDict = {};\n",
    "                for(attr in attributes){attrDict[attr]=attributes[attr]}\n",
    "                attrDict[\"grid_\"]=\"min\"\n",
    "                $(\"#algo_\".concat(temp_algo)).append(\n",
    "                    $(\"<div/>\",{class:\"form-group\"}).append(\n",
    "                        $label,\n",
    "                        $(\"<div/>\",{\"class\":\"col-sm-10\"}).append(\n",
    "                            $(\"<input/>\",attrDict\n",
    "                             )\n",
    "                        )\n",
    "                    )\n",
    "                );\n",
    "                var $label = $(\"<label>\",{\"class\":\"control-label col-sm-2\",text:keepLabel.concat(\" MaxValue\")});\n",
    "                attrDict[\"grid_\"]=\"max\"\n",
    "                $(\"#algo_\".concat(temp_algo)).append(\n",
    "                    $(\"<div/>\",{class:\"form-group\"}).append(\n",
    "                        $label,\n",
    "                        $(\"<div/>\",{\"class\":\"col-sm-10\"}).append(\n",
    "                            $(\"<input/>\",attrDict\n",
    "                             )\n",
    "                        )\n",
    "                    )\n",
    "                );\n",
    "            }\n",
    "            else{\n",
    "                var $label = $(\"<label>\",{\"class\":\"control-label col-sm-2\",text:keepLabel});\n",
    "                var attrDict = {};\n",
    "               for(attr in attributes){attrDict[attr]=attributes[attr]}\n",
    "                $(\"#algo_\".concat(temp_algo)).append(\n",
    "                    $(\"<div/>\",{class:\"form-group\"}).append(\n",
    "                        $label,\n",
    "                        $(\"<div/>\",{\"class\":\"col-sm-10\"}).append(\n",
    "                            $(\"<input/>\",attrDict\n",
    "                             )\n",
    "                        )\n",
    "                    )\n",
    "                ); \n",
    "            }\n",
    "        }\n",
    "        else{\n",
    "            var $label = $(\"<label>\",{\"class\":\"control-label col-sm-2\",text:keepLabel});\n",
    "            var attrDict = {};\n",
    "            var newselect = $('<select>')\n",
    "            $(attributes.options).each(function() { newselect.append($(\"<option>\").attr('value',this).text(this));});\n",
    "            var select_div = $($(\"#algo_\".concat(temp_algo)).append(\n",
    "            $(\"<div/>\",{class:\"form-group\"}).append(\n",
    "                $label,\n",
    "                $(\"<div/>\",{\"class\":\"col-sm-10\"}).append(\n",
    "                    $(newselect)\n",
    "                )\n",
    "            )\n",
    "        ));\n",
    "            \n",
    "        }\n",
    "    };\n",
    "    \n",
    "}\n",
    "\n",
    "                                                    \n",
    "<!-- add tempDF columns to the dropdown selectors -->\n",
    "<!-- aec required -->  \n",
    "function handle_output(data){\n",
    "    tempDF_columns = data.content.text;\n",
    "    addColumnFields(tempDF_columns);\n",
    "}\n",
    "\n",
    "\n",
    "function initAlgoConfigurations(){\n",
    "    for(var k in algorithm_list){\n",
    "        var kernel = IPython.notebook.kernel;\n",
    "        var callbacks = {\n",
    "        iopub : {\n",
    "             output : add_conf_forms,\n",
    "        }\n",
    "    }\n",
    "    var msg_id = kernel.execute('get_algorithm_parameters(\"'+k+'\")', callbacks);\n",
    "    }\n",
    "}\n",
    "                                                   \n",
    "function loadAlgorithms(){\n",
    "    var kernel = IPython.notebook.kernel;\n",
    "    var callbacks = {\n",
    "        iopub : {\n",
    "             output : showAlgorithmList,\n",
    "        }\n",
    "    }\n",
    "    family = $('#algoFamilySelect').val();\n",
    "    var msg_id = kernel.execute('get_algos_in_family(\"'+family+'\")', callbacks);\n",
    "}\n",
    "                                                          \n",
    "                                                          \n",
    "function showAlgorithmList(data){\n",
    "    cols = JSON.parse(data.content.text.replace(/\\'/g,'\"'))\n",
    "    $(\"#algoSelect\").children('option:not(:first)').remove();\n",
    "    for(var i = 0; i < cols.length; i++){\n",
    "        algoOptionGenerator($(\"#algoSelect\"),cols[i].trim());\n",
    "    }  \n",
    "    $(\"#algoSelect\")[0].selectedIndex = 0;\n",
    "}\n",
    "                                                \n",
    "<!-- populate column dropdowns -->\n",
    "function addColumnFields(columns){\n",
    "    arraycols = JSON.parse(columns.replace(/\\'/g,'\"'))\n",
    "    $('.df-col').each(function(index){\n",
    "        if($(this).hasClass(\"empty-option-allowed\")){\n",
    "            $(this).children('option:not(:first)').remove();\n",
    "        }\n",
    "        else{\n",
    "            $(this).children('option').remove();\n",
    "        }\n",
    "        \n",
    "        for(var i = 0; i < arraycols.length; i++){\n",
    "            columnOptionGenerator($(this),arraycols[i].trim());\n",
    "        }\n",
    "        \n",
    "    });\n",
    "    \n",
    "}\n",
    "                                           \n",
    "function populate_algo_conf_form(){\n",
    "    selected_algorithm = $(\"#algoSelect\").val();\n",
    "    selected_algorithm_family = $(\"#algoFamilySelect\").val();\n",
    "    algoDiv = \"#algo_\".concat(selected_algorithm);\n",
    "    familyDiv = \"#algocommon_\".concat(selected_algorithm_family);\n",
    "    $(\".algo_configuration\").hide();\n",
    "    $(familyDiv.concat(\" .algoDescription\")).html(algorithm_descriptions[selected_algorithm])\n",
    "    $(familyDiv).show();\n",
    "    $(algoDiv).show();\n",
    "}\n",
    "                                           \n",
    "function selectEvalMethod(form,label){\n",
    "    $(\".eval-form\").not(\"#\"+form).hide()\n",
    "    $(\"#\"+form).show(); \n",
    "    selected_eval = form.replace(\"-form\",\"\");\n",
    "}\n",
    "\n",
    "\n",
    "<!-- Part 4 - Algo Execution -->\n",
    "\n",
    "\n",
    "function executeAlgo(){\n",
    "    \n",
    "    var algoParamDict = {}\n",
    "    \n",
    "    $(\"#algo-results\").html(\"-- pending --\")\n",
    "    \n",
    "    selected_algorithm = $(\"#algoSelect\").val();\n",
    "    selected_algorithm_family = $(\"#algoFamilySelect\").val();\n",
    "    algoDiv = \"#algo_\".concat(selected_algorithm);\n",
    "    familyDiv = \"#algocommon_\".concat(selected_algorithm_family);\n",
    "    \n",
    "    if(selected_algorithm === \"ALS\"){\n",
    "        user_col = $(\"#als-user-id-col\").val()\n",
    "        algoParamDict.user = user_col\n",
    "        item_col = $(\"#als-item-id-col\").val()\n",
    "        algoParamDict.item = item_col\n",
    "        rating_col = $(\"#als-rating-col\").val()\n",
    "        algoParamDict.rating = rating_col\n",
    "        \n",
    "    }\n",
    "    \n",
    "    else if(selected_algorithm_family===\"NLP\"){\n",
    "        text_col = $(\"#nlp-text-col\").val()\n",
    "        algoParamDict.textcol = text_col\n",
    "    }\n",
    "    \n",
    "    else if(selected_algorithm===\"FPGrowth\"){\n",
    "        items_col = $(\"#fpginput-col\").val()\n",
    "        algoParamDict.itemscol = items_col\n",
    "    }\n",
    "    \n",
    "    else{\n",
    "    \n",
    "    if(selected_algorithm_family===\"CL_REG\"){\n",
    "        label_col = $(familyDiv+\" .single-column-selector.df-col\").val()\n",
    "        algoParamDict.label = label_col\n",
    "        if(selected_eval===\"k-fold\"){\n",
    "            algoParamDict.eval = \"k-fold\"\n",
    "            algoParamDict.evalVal = ($(\"#k-fold-form div input\")[0].value) \n",
    "        }\n",
    "        else{\n",
    "            algoParamDict.eval = \"train-test\"\n",
    "            algoParamDict.evalVal = ($(\"#train-test-form div input\")[0].value) \n",
    "        }\n",
    "    }\n",
    "    \n",
    "    else if(selected_algorithm===\"ChiSquared\"){\n",
    "        label_col = $(algoDiv+\" .single-column-selector.df-col\").val()\n",
    "        algoParamDict.label = label_col\n",
    "    }\n",
    "    \n",
    "    feature_cols = $(familyDiv+\" .multiple-column-selector.df-col\").val()\n",
    "    algoParamDict.features=feature_cols\n",
    "    }\n",
    "    \n",
    "    $(algoDiv+\" .form-group\").each(function( index ) {\n",
    "        var label = $(this).children(\"label\")[0];\n",
    "        var value = $($(this).children(\"div\")[0]).children()[0];\n",
    "        algoParamDict[label.innerText]=value.value\n",
    "    });\n",
    "    \n",
    "    \n",
    "    out_dataset = $(\"#datasetSaveSelect\").val()\n",
    "    out_folder = ($(\"#folderNameSelect\").val() !== \"\" ? $(\"#folderNameSelect\").val() : \"unknown_folder_AEC\")\n",
    "    out_path = out_dataset.concat(\"/\",out_folder)\n",
    "    algoParamDict[\"output_path\"]=out_path\n",
    "    lines = algoParamDict\n",
    "    agh = (JSON.stringify(algoParamDict))\n",
    "    \n",
    "    var kernel = IPython.notebook.kernel;\n",
    "    var callbacks = {\n",
    "        iopub : {\n",
    "             output : algo_run_show,\n",
    "        }\n",
    "    }\n",
    "    var msg_id = kernel.execute('run_algo(\"'+selected_algorithm+'\",\\''+agh+'\\')', callbacks); \n",
    "}\n",
    "\n",
    "function algo_run_show(data){\n",
    "    lines = data;\n",
    "    if(data.content.name === \"stderr\"){\n",
    "        $(\"#algo-results\").text(data.content.text)\n",
    "    }\n",
    "    else{\n",
    "        try{\n",
    "            results = JSON.parse(data.content.text.replace(/'/g,'\"'))[\"results\"];\n",
    "            var newcontent = '<br/><label style=\"width: 20%;\" class=\"qblabel\">Results Summary</label>'\n",
    "            $.each( results, function( key, val ) {\n",
    "                newcontent = newcontent.concat(\"<p><b>\",key,\":</b>\",val,\"</p>\")\n",
    "            });\n",
    "            $(\"#algo-results\").html(newcontent);\n",
    "        }\n",
    "        catch(err) {\n",
    "            $(\"#algo-results\").text(\"Oups! Something went wrong: \".concat(data.content.text))\n",
    "        }  \n",
    "    }\n",
    "    \n",
    "}\n",
    "\n",
    "<!-- Part 5 - Output -->\n",
    "\n",
    "<!-- Part 6 - Overview -->\n",
    "function updateOverview(){\n",
    "    out_dataset = ($(\"#datasetSaveSelect\").val() !== \"\" ? $(\"#datasetSaveSelect\").val() : \"--- missing ---\")\n",
    "    out_folder = ($(\"#folderNameSelect\").val() !== \"\" ? $(\"#folderNameSelect\").val() : \"--- missing ---\")\n",
    "    in_file = (currentDFpath !== \"\" ? currentDFpath : \"---missing---\")\n",
    "    $(\"#overview-algorithm\").text($(\"#algoSelect :selected\").text());\n",
    "    $(\"#overview-input\").text(in_file);\n",
    "    $(\"#overview-output\").html(\"Dataset: \".concat(out_dataset,\" </br>Folder: \",out_folder));\n",
    "};\n",
    "\n",
    "                                                   \n",
    "                                                   \n",
    "<!-- Part 7 - Errors -->\n",
    "\n",
    "\n",
    "<!-- Part 8 - Apply Model -->\n",
    "<!-- aec required -->  \n",
    "function selectModelApplyDataset(){\n",
    "    var kernel = IPython.notebook.kernel;\n",
    "    var callbacks = {\n",
    "        iopub : {\n",
    "             output : getCSVfilesFromFS_MA,\n",
    "        }\n",
    "    }\n",
    "    dataset = $('#datasetModelApplySelect').val();\n",
    "    var msg_id = kernel.execute('walk_dataset(\"'+dataset+'\")', callbacks);\n",
    "}\n",
    "                                                           \n",
    "function getCSVfilesFromFS_MA(data){\n",
    "    datafile_options_ma = data.content.text;\n",
    "    datafilesObtainedModelApply(JSON.parse(datafile_options_ma.replace(/u\\'/g,'\"').replace(/\\'/g,'\"')))\n",
    "}\n",
    "                                                           \n",
    "function datafilesObtainedModelApply(options){\n",
    "    var dropl = document.getElementById(\"fileModelApplySelect\");\n",
    "    dropl.options.length = 0;\n",
    "    for (opt in options) {\n",
    "        dropl.options[dropl.options.length] = new Option(options[opt]);\n",
    "    }\n",
    "}  \n",
    "                                                                       \n",
    "function selectDataModelApplyfile(){\n",
    "    var kernel = IPython.notebook.kernel;\n",
    "    var callbacks = {\n",
    "        iopub : {\n",
    "             output : handle_output_model,\n",
    "        }\n",
    "    }\n",
    "    dataset = $('#fileModelApplySelect').val();\n",
    "    separator = $('#sepModelApplySelect').val();\n",
    "    resultFolder = $('#datasetModelApplySelect').val().concat(\"/\").concat($('#folderModelSelect').val());\n",
    "    var msg_id = kernel.execute('apply_model(\"'+dataset+'\",\"'+separator+'\",\"'+resultFolder+'\")', callbacks);\n",
    "}\n",
    "                                                                       \n",
    "function handle_output_model(data){\n",
    "    lines = data;\n",
    "    if(data.content.name === \"stderr\"){\n",
    "        $(\"#model-results\").text(data.content.text)\n",
    "    }\n",
    "    else{\n",
    "        try{\n",
    "            results = JSON.parse(data.content.text.replace(/'/g,'\"'))[\"results\"];\n",
    "            preview_result = results[\"Result preview\"];\n",
    "            var newcontent = '<br/><label style=\"width: 20%;\" class=\"qblabel\">Preview</label>'\n",
    "            $.each( preview_result, function( key, val ) {\n",
    "                newcontent = newcontent.concat(\"<p>\")\n",
    "                $.each( val, function( k, v ) {\n",
    "                newcontent = newcontent.concat(\"<b>\",k,\":</b>\",v,\"  \")\n",
    "            })\n",
    "            newcontent = newcontent.concat(\"</p>\")});\n",
    "            newcontent = newcontent.concat(\"<p>\",results[\"Output\"],\"</p>\")                                               \n",
    "            $(\"#model-results\").html(newcontent);\n",
    "        }\n",
    "        catch(err) {\n",
    "            $(\"#model-results\").text(data.content.text)\n",
    "        }  \n",
    "    }\n",
    "}\n",
    "                                                      \n",
    "</script>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "metadata": {
    "hideCode": true
   },
   "outputs": [
    {
     "data": {
      "application/javascript": [
       "updateBrowser();\n",
       "initAlgoConfigurations();"
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "%%javascript\n",
    "updateBrowser();\n",
    "initAlgoConfigurations();"
   ]
  }
 ],
 "metadata": {
  "celltoolbar": "Hide code",
  "hide_input": true,
  "kernelspec": {
   "display_name": "PySpark",
   "language": "",
   "name": "pysparkkernel"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "python",
    "version": 2
   },
   "mimetype": "text/x-python",
   "name": "pyspark",
   "pygments_lexer": "python2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}