2-training-inference/2.3-automl-training/remote-compute/binaryclassification-employee-attrition-autoaml-remote-amlcompute.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# AutoML Classification experiment using Remote-AML-Compute and AML-Datasets\n",
    "## Data: IBM Employee Attrition dataset loaded from Azure ML Dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##  Get Azure ML Workspace to use"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Check core SDK version number\n",
    "import azureml.core\n",
    "\n",
    "print(\"SDK version:\", azureml.core.VERSION)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# azureml-core of version 1.0.72 or higher is required\n",
    "from azureml.core import Workspace, Dataset\n",
    "\n",
    "# Get Workspace defined in by default config.json file\n",
    "ws = Workspace.from_config()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from azureml.core.authentication import InteractiveLoginAuthentication\n",
    "interactive_auth = InteractiveLoginAuthentication(tenant_id=\"72f988bf-86f1-41af-91ab-2d7cd011db47\")\n",
    "\n",
    "ws = Workspace(subscription_id=\"3c8972d9-f541-46b2-b70b-d81baba3595d\",\n",
    "                  resource_group=\"amlworkshop\",\n",
    "                  workspace_name=\"amlworkshop\",\n",
    "                  auth=interactive_auth)\n",
    "print(ws.name, ws.resource_group, ws.location, ws.subscription_id, sep='\\n')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load data from Azure ML Datasets \n",
    "Pandas DataFrame only used to check out the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load Data\n",
    "aml_dataset = ws.datasets['IBM-Employee-Attrition']\n",
    "\n",
    "# Use Pandas DataFrame just to sneak peak some data and schema\n",
    "full_df = aml_dataset.to_pandas_dataframe()\n",
    "# .to_pandas_dataframe().dropna()\n",
    "full_df.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Use Pandas DataFrame just to investigate the dataset's schema and info\n",
    "full_df.describe()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Clean up the initial dataset \n",
    "#### (Using AML Tabular Dataset .drop_columns() method )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Dropping Employee count as all values are 1 and hence attrition is independent of this feature\n",
    "# Dropping Employee Number since it is merely an identifier\n",
    "# Dropping Over18 since all employees are over 18\n",
    "# Dropping StandardHours Since all values are 80\n",
    "  \n",
    "aml_dataset = aml_dataset.drop_columns(['EmployeeCount','EmployeeNumber','Over18','StandardHours'])\n",
    "\n",
    "full_df = aml_dataset.to_pandas_dataframe()\n",
    "full_df.describe()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Split original AML Tabular Dataset in two test/train AML Tabular Datasets (using AML DS function)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Split using Azure Tabular Datasets (Better for Remote Compute)\n",
    "# https://docs.microsoft.com/en-us/python/api/azureml-core/azureml.data.tabulardataset?view=azure-ml-py#random-split-percentage--seed-none-\n",
    "\n",
    "train_dataset, test_dataset = aml_dataset.random_split(0.9, seed=1)\n",
    "\n",
    "# Use Pandas DF only to check the data\n",
    "train_dataset_df = train_dataset.to_pandas_dataframe()\n",
    "test_dataset_df = test_dataset.to_pandas_dataframe()\n",
    "\n",
    "print(train_dataset_df.describe())\n",
    "# print(test_dataset_df.describe())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## List remote AML compute targets available"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from azureml.core.compute import AmlCompute\n",
    "from azureml.core.compute import ComputeTarget\n",
    "\n",
    "ComputeTarget.list(ws)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Connect to Remote AML Compute (Existing AML cluster)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define remote compute target to use\n",
    "# Further docs on Remote Compute Target: https://docs.microsoft.com/en-us/azure/machine-learning/how-to-auto-train-remote\n",
    "\n",
    "# Choose a name for your cluster.\n",
    "amlcompute_cluster_name = \"cpu-cluster\"\n",
    "\n",
    "found = False\n",
    "# Check if this compute target already exists in the workspace.\n",
    "cts = ws.compute_targets\n",
    "\n",
    "if amlcompute_cluster_name in cts and cts[amlcompute_cluster_name].type == 'AmlCompute':\n",
    "     found = True\n",
    "     print('Found existing training cluster.')\n",
    "     # Get existing cluster\n",
    "     # Method 1:\n",
    "     aml_remote_compute = cts[amlcompute_cluster_name]\n",
    "     # Method 2:\n",
    "     # aml_remote_compute = ComputeTarget(ws, amlcompute_cluster_name)\n",
    "    \n",
    "if not found:\n",
    "     print('Creating a new training cluster...')\n",
    "     provisioning_config = AmlCompute.provisioning_configuration(vm_size = \"STANDARD_D13_V2\", # for GPU, use \"STANDARD_NC12\"\n",
    "                                                                 #vm_priority = 'lowpriority', # optional\n",
    "                                                                 max_nodes = 20)\n",
    "     # Create the cluster.\n",
    "     aml_remote_compute = ComputeTarget.create(ws, amlcompute_cluster_name, provisioning_config)\n",
    "    \n",
    "print('Checking cluster status...')\n",
    "# Can poll for a minimum number of nodes and for a specific timeout.\n",
    "# If no min_node_count is provided, it will use the scale settings for the cluster.\n",
    "aml_remote_compute.wait_for_completion(show_output = True, min_node_count = 0, timeout_in_minutes = 20)\n",
    "    \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# For additional details of current AmlCompute status:\n",
    "aml_remote_compute.get_status().serialize()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## List and select primary metric to drive the AutoML classification problem"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from azureml.train import automl\n",
    "\n",
    "# List of possible primary metrics is here:\n",
    "# https://docs.microsoft.com/en-us/azure/machine-learning/how-to-configure-auto-train#primary-metric\n",
    "    \n",
    "# Get a list of valid metrics for your given task\n",
    "automl.utilities.get_primary_metrics('classification')\n",
    "\n",
    "# We'll use 'accuracy' as primary metric (Closer to 1.00 is better)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Define AutoML Experiment settings (With AML Remote Compute)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import logging\n",
    "import os\n",
    "# You can provide additional settings as a **kwargs parameter for the AutoMLConfig object\n",
    "# automl_settings = {\n",
    "#     \"whitelist_models\": 'XGBoostClassifier'\n",
    "# }\n",
    "\n",
    "from azureml.train.automl import AutoMLConfig\n",
    "\n",
    "project_folder = './automl'\n",
    "os.makedirs(project_folder, exist_ok=True)\n",
    "\n",
    "automl_config = AutoMLConfig(compute_target=aml_remote_compute,\n",
    "                             task='classification',\n",
    "                             primary_metric='accuracy',\n",
    "                             experiment_timeout_minutes=15,                            \n",
    "                             training_data=train_dataset,\n",
    "                             label_column_name=\"Attrition\",\n",
    "                             n_cross_validations=5,\n",
    "                             # blacklist_models='XGBoostClassifier', \n",
    "                             # iteration_timeout_minutes=5,                                                    \n",
    "                             enable_early_stopping=True,\n",
    "                             featurization='auto',\n",
    "                             debug_log='automated_ml_errors.log',\n",
    "                             verbosity=logging.INFO,\n",
    "                             path=project_folder\n",
    "                             # **automl_settings\n",
    "                             )\n",
    "\n",
    "# PaperCut?: Why is drop_column_names only supported by Time Series Forecast? - If used for classification, you get:\n",
    "# drop_column_names= ['EmployeeCount','EmployeeNumber','Over18','StandardHours'], # Clean up dataset by dropping not needed columns\n",
    "# WARNING - Received unrecognized parameter: drop_column_names ['EmployeeCount', 'EmployeeNumber', 'Over18', 'StandardHours']\n",
    "# In documentation it doesn't state that it is only supported for Forecast...:\n",
    "# https://docs.microsoft.com/en-us/python/api/azureml-train-automl-client/azureml.train.automl.automlconfig.automlconfig?view=azure-ml-py\n",
    "\n",
    "# Explanation of Settings: https://docs.microsoft.com/en-us/azure/machine-learning/how-to-configure-auto-train#configure-your-experiment-settings\n",
    "\n",
    "# AutoMLConfig info on: \n",
    "# https://docs.microsoft.com/en-us/python/api/azureml-train-automl-client/azureml.train.automl.automlconfig.automlconfig\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Run Experiment (on AML Remote Compute) with multiple child runs under the covers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from azureml.core import Experiment\n",
    "from datetime import datetime\n",
    "\n",
    "now = datetime.now()\n",
    "time_string = now.strftime(\"%m-%d-%Y-%H\")\n",
    "experiment_name = \"classif-automl-remote-{0}\".format(time_string)\n",
    "print(experiment_name)\n",
    "\n",
    "experiment = Experiment(workspace=ws, name=experiment_name)\n",
    "\n",
    "import time\n",
    "start_time = time.time()\n",
    "            \n",
    "run = experiment.submit(automl_config, show_output=True)\n",
    "\n",
    "print('Manual run timing: --- %s seconds needed for running the whole Remote AutoML Experiment ---' % (time.time() - start_time))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Explore results with Widget"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from azureml.widgets import RunDetails\n",
    "RunDetails(run).show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Measure Parent Run Time needed for the whole AutoML process "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import time\n",
    "import datetime as dt\n",
    "\n",
    "run_details = run.get_details()\n",
    "\n",
    "# Like: 2020-01-12T23:11:56.292703Z\n",
    "end_time_utc_str = run_details['endTimeUtc'].split(\".\")[0]\n",
    "start_time_utc_str = run_details['startTimeUtc'].split(\".\")[0]\n",
    "timestamp_end = time.mktime(datetime.strptime(end_time_utc_str, \"%Y-%m-%dT%H:%M:%S\").timetuple())\n",
    "timestamp_start = time.mktime(datetime.strptime(start_time_utc_str, \"%Y-%m-%dT%H:%M:%S\").timetuple())\n",
    "\n",
    "parent_run_time = timestamp_end - timestamp_start\n",
    "print('Run Timing: --- %s seconds needed for running the whole Remote AutoML Experiment ---' % (parent_run_time))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Retrieve the 'Best Model' (Scikit-Learn model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "best_run, fitted_model = run.get_output()\n",
    "print(best_run)\n",
    "print(fitted_model)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Make Predictions"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Extract X values (feature columns) from test dataset and convert to NumPi array for predicting "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "\n",
    "#Remove Label/y column\n",
    "if 'Attrition' in test_dataset_df.columns:\n",
    "    y_test_df = test_dataset_df.pop('Attrition')\n",
    "\n",
    "x_test_df = test_dataset_df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Make the actual Predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Try the best model\n",
    "y_predictions = fitted_model.predict(x_test_df)\n",
    "\n",
    "print('10 predictions: ')\n",
    "print(y_predictions[:10])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_predictions.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Calculate the Accuracy with Test Dataset (Not used for training)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import accuracy_score\n",
    "\n",
    "print('Accuracy:')\n",
    "accuracy_score(y_test_df, y_predictions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.6 - AzureML",
   "language": "python",
   "name": "python3-azureml"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}