utility_calc.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'EUs': 1    4.0\n",
      "0   -4.0\n",
      "dtype: float64, 'optimal': 1}\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "def decide(probs=None, utils=None):\n",
    "    if probs is None and utils is None:\n",
    "        raise ValueError(\"Either 'probs' or 'utils' must be given\")\n",
    "\n",
    "    if utils is None:\n",
    "        if len(probs.shape) > 1:\n",
    "            raise ValueError(\"'probs' is not a vector, please specify array of utilities\")\n",
    "        utils = np.diag(probs)\n",
    "\n",
    "    elif probs is None:\n",
    "        probs = np.full(utils.shape[1], 1/utils.shape[1])\n",
    "\n",
    "    if len(probs) != utils.shape[1]:\n",
    "        raise ValueError('Mismatch between inference and utility variates')\n",
    "\n",
    "    if not isinstance(utils, pd.DataFrame):\n",
    "        utils = pd.DataFrame(utils)\n",
    "\n",
    "    # Calculate expected utilities (matrix product)\n",
    "    exputils = np.dot(utils, probs)\n",
    "    # If expected utilites are the same for every class, choose random class\n",
    "    if(len(set(exputils)) == 1):\n",
    "        exputils = pd.Series(exputils, index=utils.index).sort_values(ascending=False)\n",
    "        return {'EUs': exputils, 'optimal': random.randint(0,len(exputils)-1)}\n",
    "\n",
    "        \n",
    "    exputils = pd.Series(exputils, index=utils.index).sort_values(ascending=False)\n",
    "    #Select one decision with max expected utility\n",
    "    optimal = exputils.idxmax()\n",
    "    return {'EUs': exputils, 'optimal': optimal}\n",
    "\n",
    "# Example usage\n",
    "probs = np.array([0.3, 0.7])\n",
    "utils = np.array([[10, -10], [-10, 10]])\n",
    "result = decide(probs=probs, utils=utils)\n",
    "print(result)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.97603963, 0.02396037],\n",
       "       [0.00717049, 0.99282951]])"
      ]
     },
     "execution_count": 93,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Probabilities from RNopm infer\n",
    "prob_predict_cat = [0.976039626021315, 0.0239603739786851]\n",
    "prob_predict_dog = [0.00717048738691545, 0.992829512613085]\n",
    "\n",
    "prob = np.array([prob_predict_cat, prob_predict_dog])\n",
    "prob"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [],
   "source": [
    "#CM_df = pd.read_csv('C:\\HVL\\OneDrive - Høgskulen på Vestlandet\\ADA511\\Python\\project\\cd_dataframe.csv')\n",
    "CM_df = pd.read_csv('/Users/eilertskram/Library/CloudStorage/OneDrive-HøgskulenpåVestlandet/ADA511/Python/project/cd_dataframe.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>True Cat</th>\n",
       "      <th>True Dog</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Predicted Cat</td>\n",
       "      <td>1744</td>\n",
       "      <td>38</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Predicted Dog</td>\n",
       "      <td>6</td>\n",
       "      <td>1712</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      Unnamed: 0  True Cat  True Dog\n",
       "0  Predicted Cat      1744        38\n",
       "1  Predicted Dog         6      1712"
      ]
     },
     "execution_count": 95,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "CM_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [],
   "source": [
    "CM = np.array([[1744, 38], [6, 1712]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [],
   "source": [
    "util = np.array([[1, -2], [-10, 1]]) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [],
   "source": [
    "expected_RN = np.multiply(CM, util)\n",
    "expected_RN = expected_RN.sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [],
   "source": [
    "#test_df = pd.read_csv('C:\\\\HVL\\\\OneDrive - Høgskulen på Vestlandet\\\\ADA511\\\\Python\\\\project\\\\test_dataframe.csv')\n",
    "test_df = pd.read_csv('/Users/eilertskram/Library/CloudStorage/OneDrive-HøgskulenpåVestlandet/ADA511/Python/project/test_dataframe.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [],
   "source": [
    "cm_rn = np.array([[0, 0], [0, 0]]) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [],
   "source": [
    "def func_expected_opm(df,util, cm):\n",
    "    expected_opm = 0\n",
    "    number_of_overrides = 0\n",
    "\n",
    "    for index, row in df.iterrows():\n",
    "        predicted = row['Predicted']\n",
    "        actual = row['Actual']\n",
    "\n",
    "        if predicted == 'dog':\n",
    "            optimal = decide(prob_predict_dog, util).get(\"optimal\")\n",
    "            \n",
    "            if optimal == 1:\n",
    "                if 'dog' == actual: #RPopm did not override and resnet predicted dog and it was dog\n",
    "                    expected_opm += util[1][1]\n",
    "                    cm[1][1] += 1\n",
    "                else:   #RPopm did not override and resnet predicted dog and but it was cat\n",
    "                    expected_opm += util[1][0]\n",
    "                    cm[1][0] += 1\n",
    "            else:\n",
    "                number_of_overrides += 1\n",
    "                if 'cat' == actual: #RPopm override and guessed cat and it was cat\n",
    "                    expected_opm += util[0][0] \n",
    "                    cm[0][0] += 1  \n",
    "                else: #RPopm override and guessed cat and but it was dog\n",
    "                    expected_opm += util[0][1]\n",
    "                    cm[0][1] += 1\n",
    "        else:\n",
    "            optimal = decide(prob_predict_cat, util).get(\"optimal\")\n",
    "\n",
    "            if optimal == 0:\n",
    "                if 'cat' == actual: #RPopm did not override and resnet predicted cat and it was cat\n",
    "                    expected_opm += util[0][0]\n",
    "                    cm[0][0] += 1\n",
    "                else:   #RPopm did not override and resnet predicted cat and but it was dog\n",
    "                    expected_opm += util[0][1]\n",
    "                    cm[0][1] += 1\n",
    "            else:\n",
    "                number_of_overrides += 1\n",
    "                if 'dog' == actual: #RPopm override and guessed dog and it was dog\n",
    "                    expected_opm += util[1][1]  \n",
    "                    cm[1][1] += 1\n",
    "                else: #RPopm override and guessed dog and but it was cat\n",
    "                    expected_opm += util[1][0]\n",
    "                    cm[1][0] += 1\n",
    "\n",
    "    return expected_opm, number_of_overrides\n",
    "\n",
    "    \n",
    "\n",
    "            \n",
    "        \n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Trivial utility-matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Expected utilities with resnet alone: 3320\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Expected utilities with OPM: 3320, total number over overrides 0\n"
     ]
    }
   ],
   "source": [
    "print(f'Expected utilities with resnet alone: {np.multiply(CM, util).sum()}')\n",
    "expected_opm = func_expected_opm(test_df, util, cm_rn)\n",
    "print(f\"Expected utilities with OPM: {expected_opm[0]}, total number over overrides {expected_opm[1]}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1744,   38],\n",
       "       [   6, 1712]])"
      ]
     },
     "execution_count": 103,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "CM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1744,   38],\n",
       "       [   6, 1712]])"
      ]
     },
     "execution_count": 104,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm_rn"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Slightly above threshold(p)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [],
   "source": [
    "cm_rn = np.array([[0, 0], [0, 0]]) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [],
   "source": [
    "p = 0. + (1.-0.)*(0.98/(1-0.98))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [],
   "source": [
    "u_above = np.array([[1.,0.], [0, p]]) #utility matrix using slightly above threshold(p)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Expected utilities with resnet alone: 85631.99999999993\n",
      "Expected utilities with OPM: 85749.99999999999, total number over overrides 1782\n"
     ]
    }
   ],
   "source": [
    "print(f'Expected utilities with resnet alone: {np.multiply(CM, u_above).sum()}')\n",
    "expected_opm = func_expected_opm(test_df, u_above, cm_rn)\n",
    "print(f\"Expected utilities with OPM: {expected_opm[0]}, total number over overrides {expected_opm[1]}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1744,   38],\n",
       "       [   6, 1712]])"
      ]
     },
     "execution_count": 109,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "CM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[   0,    0],\n",
       "       [1750, 1750]])"
      ]
     },
     "execution_count": 110,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm_rn #confusion matrix "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Slightly below threshold(p)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {},
   "outputs": [],
   "source": [
    "cm_rn = np.array([[0, 0], [0, 0]]) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [],
   "source": [
    "p = 0. + (1.-0.)*(0.96/(1-0.96))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [],
   "source": [
    "u_below = np.array([[1.,0.], [0, p]]) #utility matrix using sligthly below threshold(p)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Expected utilities with resnet alone: 42831.99999999996\n",
      "Expected utilities with OPM: 42832.0, total number over overrides 0\n"
     ]
    }
   ],
   "source": [
    "print(f'Expected utilities with resnet alone: {np.multiply(CM, u_below).sum()}')\n",
    "expected_opm = func_expected_opm(test_df, u_below, cm_rn)\n",
    "print(f\"Expected utilities with OPM: {expected_opm[0]}, total number over overrides {expected_opm[1]}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1744,   38],\n",
       "       [   6, 1712]])"
      ]
     },
     "execution_count": 115,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "CM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1744,   38],\n",
       "       [   6, 1712]])"
      ]
     },
     "execution_count": 116,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm_rn"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "EQUAL\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {},
   "outputs": [],
   "source": [
    "cm_rn = np.array([[0, 0], [0, 0]]) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [],
   "source": [
    "p = 0. + (1.-0.)*(0.98/(1-0.98))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [],
   "source": [
    "u_eq = np.array([[0.,0.], [-p, p]]) #utility matrix using slightly above threshold(p)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Expected utilities with resnet alone: 83593.99999999993\n",
      "Expected utilities with OPM: 83593.99999999999, total number over overrides 0\n"
     ]
    }
   ],
   "source": [
    "print(f'Expected utilities with resnet alone: {np.multiply(CM, u_eq).sum()}')\n",
    "expected_opm = func_expected_opm(test_df, u_eq, cm_rn)\n",
    "print(f\"Expected utilities with OPM: {expected_opm[0]}, total number over overrides {expected_opm[1]}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1744,   38],\n",
       "       [   6, 1712]])"
      ]
     },
     "execution_count": 121,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm_rn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}