Network Threat Detection.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 5,
  "metadata": {
    "kernelspec": {
      "display_name": "gpu2",
      "language": "python",
      "name": "gpu2"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.6.13"
    },
    "colab": {
      "name": "Capstone.ipynb",
      "provenance": []
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "d9014e0a"
      },
      "source": [
        "import pandas as pd\n",
        "import numpy as np\n",
        "import matplotlib.pyplot as plt\n",
        "import sklearn\n",
        "from sklearn.neighbors import KNeighborsClassifier\n",
        "from sklearn.model_selection import cross_val_score\n",
        "from sklearn.model_selection import train_test_split"
      ],
      "id": "d9014e0a",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "84e5eade"
      },
      "source": [
        "## Assigning data to Pandas DataFrame"
      ],
      "id": "84e5eade"
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fde23323"
      },
      "source": [
        "Creates a string variable with the folder path to current data storage."
      ],
      "id": "fde23323"
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "a0f36dad"
      },
      "source": [
        "folder_path = 'D:\\\\AMLI Capstone\\\\'"
      ],
      "id": "a0f36dad",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "a31e3725"
      },
      "source": [
        "### Creating the Pandas DataFrame and naming the columns"
      ],
      "id": "a31e3725"
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0df781b9"
      },
      "source": [
        "Inputs the auth.txt file into a DataFrame."
      ],
      "id": "0df781b9"
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5d9cb690"
      },
      "source": [
        "auth_df = pd.read_csv(\n",
        "folder_path + 'auth.txt',\n",
        "names = ['time',\n",
        "      'source_user@domain',\n",
        "      'destination_user@domain',\n",
        "      'source_computer',\n",
        "      'destination_computer',\n",
        "      'authentication_type',\n",
        "      'logon_type',\n",
        "      'authentication_orientation',\n",
        "      'success/failure'])"
      ],
      "id": "5d9cb690",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "38e79384"
      },
      "source": [
        "auth_df"
      ],
      "id": "38e79384",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8b4d1c36"
      },
      "source": [
        "Inputs the redteam.txt file into a DataFrame."
      ],
      "id": "8b4d1c36"
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "af4a96a6"
      },
      "source": [
        "redteam_df = pd.read_csv(folder_path + 'redteam.txt', \n",
        "                      names= ['time',\n",
        "                              'user@domain',\n",
        "                              'source_computer',\n",
        "                              'destination_computer'])"
      ],
      "id": "af4a96a6",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "16263600"
      },
      "source": [
        "redteam_df"
      ],
      "id": "16263600",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ba80b3fe"
      },
      "source": [
        "## Labeling and splitting the data"
      ],
      "id": "ba80b3fe"
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "c7bd114f"
      },
      "source": [
        "Creates the threat column classifier, 0 equals non-threat, 1 equals threat"
      ],
      "id": "c7bd114f"
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "83dcd200"
      },
      "source": [
        "auth_df['threat'] = 0"
      ],
      "id": "83dcd200",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "cfa6a84f"
      },
      "source": [
        "Creates the threats DataFrame for future use for appending"
      ],
      "id": "cfa6a84f"
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "e9547598"
      },
      "source": [
        "threats = pd.DataFrame()\n",
        "\n",
        "threats.rename(\n",
        "columns = ['time',\n",
        "      'source_user@domain',\n",
        "      'destination_user@domain',\n",
        "      'source_computer',\n",
        "      'destination_computer',\n",
        "      'authentication_type',\n",
        "      'logon_type',\n",
        "      'authentication_orientation',\n",
        "      'success/failure'])\n",
        "print(threats)"
      ],
      "id": "e9547598",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0b55e5ac"
      },
      "source": [
        "This cell iterates through each element in the redteam_df DataFrame.\n",
        "\n",
        "Each element gets their time value pulled and matched with a list gathered from the auth_df DataFrame with the same times.\n",
        "\n",
        "The same process happens with the source_user@domain column with the time matched list.\n",
        "\n",
        "The same process happens with the source_computer column with the source_user@domain matched list.\n",
        "\n",
        "The same process happens with the destination_computer column with source_computer matched list.\n",
        "\n",
        "Once the redteam_df element is matched an auth_df element, it sets the element to threat = 1 and the element is appended to the threats DataFrame. The index of the threat is popped off the auth_df DataFrame for strafied splitting later."
      ],
      "id": "0b55e5ac"
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8a57c388"
      },
      "source": [
        "for i in range(redteam_df.shape[0] - 1):\n",
        "    print('Redteam event num: ' + str(i))\n",
        "    \n",
        "    print('Finding matches')\n",
        "    time_df = auth_df.loc[auth_df['time'] == redteam_df.loc[i,'time']]\n",
        "    \n",
        "    user_df = time_df.loc[time_df['source_user@domain'] == redteam_df.loc[i,'user@domain']]\n",
        "    \n",
        "    comp_df = user_df.loc[user_df['source_computer'] == redteam_df.loc[i,'source_computer']]\n",
        "    \n",
        "    final_df = comp_df.loc[comp_df['destination_computer'] == redteam_df.loc[i,'destination_computer']]\n",
        "    \n",
        "    \n",
        "    \n",
        "    index = final_df.index.tolist()\n",
        "    \n",
        "    if (len(index) != 0):\n",
        "        print('Index: ', index, end='\\n\\n')\n",
        "        print('Marking event as a threat')\n",
        "        threats = pd.concat([threats, final_df])\n",
        "    else:\n",
        "        print('No match found in auth_df', end='\\n\\n')\n",
        "        \n",
        "threats['threat'] = 1"
      ],
      "id": "8a57c388",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "99586b4b"
      },
      "source": [
        "auth_df.loc[threats.index, 'threat'] = 1"
      ],
      "id": "99586b4b",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cc4b1cba"
      },
      "source": [
        "print(auth_df.iloc[29627158])"
      ],
      "id": "cc4b1cba",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "f80b3ceb"
      },
      "source": [
        "auth_df.shape"
      ],
      "id": "f80b3ceb",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "f9b8d6ab"
      },
      "source": [
        "## Data Preprocessing: String data to ID"
      ],
      "id": "f9b8d6ab"
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "32120ee5"
      },
      "source": [
        "Each cell gathers a list of unique catagories from their corresponding column, enumerates each catagory, and divides them by the total number of unique catagories to scale their values."
      ],
      "id": "32120ee5"
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2fa5ef90"
      },
      "source": [
        "### source_user@domain"
      ],
      "id": "2fa5ef90"
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "b25c06be"
      },
      "source": [
        "source_user_domain = sorted(list(pd.unique(auth_df['source_user@domain'].values.ravel())))\n",
        "\n",
        "source_user_domain_to_id = [i[0]/len(source_user_domain) for i in enumerate(source_user_domain)]\n",
        "\n",
        "source_user_domain_dictionary = {\n",
        "    source_user_domain[i]:\n",
        "    source_user_domain_to_id[i]\n",
        "    for i in range(len(source_user_domain))\n",
        "}"
      ],
      "id": "b25c06be",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9798c4c2"
      },
      "source": [
        "### destination_user@domain"
      ],
      "id": "9798c4c2"
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8a067fe1"
      },
      "source": [
        "destination_user_domain = sorted(list(pd.unique(auth_df['destination_user@domain'].values.ravel())))\n",
        "\n",
        "destination_user_domain_to_id = [i[0]/len(destination_user_domain) for i in enumerate(destination_user_domain)]\n",
        "\n",
        "destination_user_domain_dictionary = {\n",
        "    destination_user_domain[i]:\n",
        "    destination_user_domain_to_id[i] \n",
        "    for i in range(len(destination_user_domain))\n",
        "}"
      ],
      "id": "8a067fe1",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "51f454f5"
      },
      "source": [
        "### source_computer"
      ],
      "id": "51f454f5"
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4b8b53db"
      },
      "source": [
        "source_computer = sorted(list(pd.unique(auth_df['source_computer'].values.ravel())))\n",
        "\n",
        "source_computer_to_id = [i[0]/len(source_computer) for i in enumerate(source_computer)]\n",
        "\n",
        "source_computer_dictionary = {\n",
        "    source_computer[i]:\n",
        "    source_computer_to_id[i]\n",
        "    for i in range(len(source_computer))\n",
        "}"
      ],
      "id": "4b8b53db",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fffa7529"
      },
      "source": [
        "### destination_computer"
      ],
      "id": "fffa7529"
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "d5f06eb8"
      },
      "source": [
        "destination_computer = sorted(list(pd.unique(auth_df['destination_computer'].values.ravel())))\n",
        "\n",
        "destination_computer_to_id = [i[0]/len(destination_computer) for i in enumerate(destination_computer)]\n",
        "\n",
        "destination_computer_dictionary = {\n",
        "    destination_computer[i]:\n",
        "    destination_computer_to_id[i]\n",
        "    for i in range(len(destination_computer))\n",
        "}"
      ],
      "id": "d5f06eb8",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "b5cc94f5"
      },
      "source": [
        "### authentication_type"
      ],
      "id": "b5cc94f5"
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cef0b170"
      },
      "source": [
        "authentication_type = sorted(list(pd.unique(auth_df['authentication_type'].values.ravel())))\n",
        "\n",
        "authentication_type_to_id = [i[0]/len(authentication_type) for i in enumerate(authentication_type)]\n",
        "\n",
        "authentication_type_dictionary = {\n",
        "    authentication_type[i]:\n",
        "    authentication_type_to_id[i]\n",
        "    for i in range(len(authentication_type))\n",
        "}"
      ],
      "id": "cef0b170",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0f2d3e4c"
      },
      "source": [
        "### logon_type"
      ],
      "id": "0f2d3e4c"
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "b6bee0c9"
      },
      "source": [
        "logon_type = sorted(list(pd.unique(auth_df['logon_type'].values.ravel())))\n",
        "\n",
        "logon_type_to_id = [i[0]/len(logon_type) for i in enumerate(logon_type)]\n",
        "\n",
        "logon_type_dictionary = {\n",
        "    logon_type[i]:\n",
        "    logon_type_to_id[i]\n",
        "    for i in range(len(logon_type))\n",
        "}"
      ],
      "id": "b6bee0c9",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "37646880"
      },
      "source": [
        "### authentication_orientation"
      ],
      "id": "37646880"
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8486d978"
      },
      "source": [
        "authentication_orientation = sorted(list(pd.unique(auth_df['authentication_orientation'].values.ravel())))\n",
        "\n",
        "authentication_orientation_to_id = [i[0]/len(authentication_orientation) for i in enumerate(authentication_orientation)]\n",
        "\n",
        "authentication_orientation_dictionary = {\n",
        "    authentication_orientation[i]:\n",
        "    authentication_orientation_to_id[i]\n",
        "    for i in range(len(authentication_orientation))\n",
        "}"
      ],
      "id": "8486d978",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5b69b24c"
      },
      "source": [
        "### success/failure"
      ],
      "id": "5b69b24c"
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "d6ca7efc"
      },
      "source": [
        "success_failure = list(pd.unique(auth_df['success/failure'].values.ravel()))\n",
        "\n",
        "success_failure"
      ],
      "id": "d6ca7efc",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "nAiJfVaFstpR"
      },
      "source": [
        "### Data Preprocessing and Scaling"
      ],
      "id": "nAiJfVaFstpR"
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bc6438cc"
      },
      "source": [
        "processed_train_set = pd.DataFrame(columns= train_set.columns)\n",
        "\n",
        "processed = pd.DataFrame(columns= train_set.columns)\n",
        "\n",
        "for i in range(len(train_set)):\n",
        "    processed.loc[0] = 0\n",
        "\n",
        "    processed['time'] = train_set['time'].iloc[i]\n",
        "\n",
        "    processed['source_user@domain'] = source_user_domain_dictionary[train_set['source_user@domain'].iloc[i]]\n",
        "\n",
        "    processed['destination_user@domain'] = destination_user_domain_dictionary[train_set['destination_user@domain'].iloc[i]]\n",
        "\n",
        "    processed['source_computer'] = source_computer_dictionary[train_set['source_computer'].iloc[i]]\n",
        "\n",
        "    processed['destination_computer'] = destination_computer_dictionary[train_set['destination_computer'].iloc[i]]\n",
        "\n",
        "    processed['authentication_type'] = authentication_type_dictionary[train_set['authentication_type'].iloc[i]]\n",
        "\n",
        "    processed['logon_type'] = logon_type_dictionary[train_set['logon_type'].iloc[i]]\n",
        "\n",
        "    processed['authentication_orientation'] = authentication_orientation_dictionary[train_set['authentication_orientation'].iloc[i]]\n",
        "    \n",
        "    if train_set['success/failure'].iloc[i] == 'Success':\n",
        "        processed['success/failure'] = 1\n",
        "    else:\n",
        "        processed['success/failure'] = 0\n",
        "\n",
        "    processed['threat'] = train_set['threat'].iloc[i]\n",
        "    \n",
        "    processed_train_set = processed_train_set.append(processed)\n",
        "\n",
        "processed_train_set"
      ],
      "id": "bc6438cc",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "d825630d"
      },
      "source": [
        "Splits the data into a 18% train set and 2% test set of data"
      ],
      "id": "d825630d"
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "92a1ed80"
      },
      "source": [
        "x_train, x_test, y_train, y_test = train_test_split(\n",
        "    processed_train_set[FEATURES], processed_train_set[TARGETS], test_size = 0.2, stratify = processed_train_set['threat']\n",
        ")\n",
        "\n",
        "y_train= y_train.astype('int')"
      ],
      "id": "92a1ed80",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1baf1378"
      },
      "source": [
        "## Building the model"
      ],
      "id": "1baf1378"
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5589c980"
      },
      "source": [
        "Features and targets classification"
      ],
      "id": "5589c980"
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "71cb12ff"
      },
      "source": [
        "FEATURES = [x for x in auth_df.columns if x != 'threat' and x != 'time']\n",
        "\n",
        "TARGETS = 'threat'\n",
        "\n",
        "FEATURES, TARGETS"
      ],
      "id": "71cb12ff",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "d3b521eb"
      },
      "source": [
        "### K-nearest neighbor"
      ],
      "id": "d3b521eb"
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ac09b27d"
      },
      "source": [
        "estimator = KNeighborsClassifier(n_neighbors= int(x_train.shape[0]**0.5), p=2, metric='euclidean')\n",
        "\n",
        "scores = cross_val_score(\n",
        "    estimator,\n",
        "    x_train,\n",
        "    y_train,\n",
        "    verbose = 1,\n",
        "    cv = 5\n",
        ")"
      ],
      "id": "ac09b27d",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "a0478bb7"
      },
      "source": [
        "scores.mean()"
      ],
      "id": "a0478bb7",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "daaddcb3"
      },
      "source": [
        "KNN = KNeighborsClassifier(n_neighbors= int(x_train.shape[0]**0.5), p=2, metric='euclidean')\n",
        "KNN.fit(x_train, y_train)\n",
        "\n",
        "y_pred = KNN.predict(x_test)"
      ],
      "id": "daaddcb3",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "b86605cf"
      },
      "source": [
        "np.unique(y_pred)"
      ],
      "id": "b86605cf",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3b703ea2"
      },
      "source": [
        "from sklearn.metrics import confusion_matrix\n",
        "from sklearn.metrics import f1_score\n",
        "from sklearn.metrics import accuracy_score\n",
        "\n",
        "cm = confusion_matrix(y_test,y_pred)\n",
        "f1 = f1_score(y_test,y_pred)\n",
        "accuracy = accuracy_score(y_test,y_pred)\n",
        "\n",
        "\n",
        "print('The confusion matrix is', cm)\n",
        "print('The F1 score is', f1)\n",
        "print('The accuracy score is', accuracy)"
      ],
      "id": "3b703ea2",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "dc62f818"
      },
      "source": [
        "## Redteam Exploratory Data Analysis"
      ],
      "id": "dc62f818"
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "706c2f71"
      },
      "source": [
        "source_unique = redteam_df['destination_computer'].groupby(redteam_df['source_computer']).count()\n",
        "\n",
        "plt.bar(source_unique.index, source_unique)\n",
        "plt.xticks(rotation=90)\n",
        "plt.show()"
      ],
      "id": "706c2f71",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "e6e46b41"
      },
      "source": [
        "sns.heatmap(auth_df.corr(),cmap='inferno')"
      ],
      "id": "e6e46b41",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "166ba6d4"
      },
      "source": [
        "## DataFrames for other data"
      ],
      "id": "166ba6d4"
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "15a2e7fa"
      },
      "source": [
        "proc_df = pd.read_csv(folder_path + 'proc.txt',\n",
        "                     names = ['time',\n",
        "                              'user@domain',\n",
        "                              'computer',\n",
        "                              'process_name',\n",
        "                              'start/end'])"
      ],
      "id": "15a2e7fa",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "88a93f9d"
      },
      "source": [
        "flows_df = pd.read_csv(folder_path + 'flows.txt', \n",
        "                      names = ['time', \n",
        "                               'duration', \n",
        "                               'source_computer', \n",
        "                               'source_port', \n",
        "                               'destination_computer', \n",
        "                               'destination_port', \n",
        "                               'protocol', \n",
        "                               'packet_count', \n",
        "                               'byte_count'])"
      ],
      "id": "88a93f9d",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bf3eb3f7"
      },
      "source": [
        "dns_df = pd.read_csv(folder_path + 'dns.txt', \n",
        "                     names = ['time','source_computer','computer_resolved'])"
      ],
      "id": "bf3eb3f7",
      "execution_count": null,
      "outputs": []
    }
  ]
}