analysis.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Unlearning performance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "# Plot options\n",
    "plt.rcParams['lines.linewidth'] = 2.5\n",
    "plt.rcParams['font.size'] = 15\n",
    "plt.rcParams['font.weight'] = 'bold'\n",
    "plt.rcParams[\"axes.labelweight\"] = \"bold\"\n",
    "plt.rcParams['figure.figsize'] = (6,4)\n",
    "plt.rcParams['figure.constrained_layout.use'] = True\n",
    "plt.rcParams['legend.loc'] = 'lower right'\n",
    "plt.rcParams['legend.fontsize'] = 11\n",
    "\n",
    "rc = {\"font.family\" : \"serif\", \n",
    "      \"mathtext.fontset\" : \"stix\",\n",
    "      }\n",
    "plt.rcParams.update(rc)\n",
    "plt.rcParams[\"font.serif\"] = [\"Times New Roman\"] + plt.rcParams[\"font.serif\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_type = \"nn_mixer\" # change this\n",
    "\n",
    "\n",
    "method_list = [\"ori\", \"retrain\", \"unlearn\"]\n",
    "creteria_list = [\"mse\", \"mape\", \"cost\"]\n",
    "unlearn_prop_list = [0.05, 0.1, 0.15, 0.2, 0.25, 0.3]\n",
    "\n",
    "mse_remain_ori = []\n",
    "mse_remain_unlearn = []\n",
    "mse_unlearn_ori = []\n",
    "mse_unlearn_unlearn = []\n",
    "mse_test_ori = []\n",
    "mse_test_unlearn = []\n",
    "\n",
    "mape_remain_ori = []\n",
    "mape_remain_unlearn = []\n",
    "mape_unlearn_ori = []\n",
    "mape_unlearn_unlearn = []\n",
    "mape_test_ori = []\n",
    "mape_test_unlearn = []\n",
    "\n",
    "cost_remain_ori = []\n",
    "cost_remain_unlearn = []\n",
    "cost_unlearn_ori = []\n",
    "cost_unlearn_unlearn = []\n",
    "cost_test_ori = []\n",
    "cost_test_unlearn = []\n",
    "\n",
    "for unlearn_prop in unlearn_prop_list:\n",
    "    print('unlearn_prop: ', unlearn_prop)\n",
    "    model_dir = f\"simulation_result/{model_type}/unlearning_{unlearn_prop}.npy\"\n",
    "    metric = np.load(model_dir, allow_pickle=True).item()\n",
    "    for creteria in creteria_list:\n",
    "        for method in method_list:\n",
    "            print(f\"{creteria}_{method}:\")\n",
    "            data = metric[creteria + \"_\" + method]\n",
    "            print(data)\n",
    "            \n",
    "            if creteria == \"mse\" and method == \"ori\":\n",
    "                mse_remain_ori.append(data[\"remain\"])\n",
    "                mse_unlearn_ori.append(data[\"unlearn\"])\n",
    "                mse_test_ori.append(data[\"test\"])\n",
    "            if creteria == \"mse\" and method == \"unlearn\":\n",
    "                mse_remain_unlearn.append(data[\"remain\"])\n",
    "                mse_unlearn_unlearn.append(data[\"unlearn\"])\n",
    "                mse_test_unlearn.append(data[\"test\"])\n",
    "            \n",
    "            if creteria == \"cost\" and method == \"ori\":\n",
    "                cost_remain_ori.append(data[\"remain\"])\n",
    "                cost_unlearn_ori.append(data[\"unlearn\"])\n",
    "                cost_test_ori.append(data[\"test\"])\n",
    "            if creteria == \"cost\" and method == \"unlearn\":\n",
    "                cost_remain_unlearn.append(data[\"remain\"])\n",
    "                cost_unlearn_unlearn.append(data[\"unlearn\"])\n",
    "                cost_test_unlearn.append(data[\"test\"])\n",
    "                \n",
    "            if creteria == \"mape\" and method == \"ori\":\n",
    "                mape_remain_ori.append(data[\"remain\"])\n",
    "                mape_unlearn_ori.append(data[\"unlearn\"])\n",
    "                mape_test_ori.append(data[\"test\"])\n",
    "            if creteria == \"mape\" and method == \"unlearn\":\n",
    "                mape_remain_unlearn.append(data[\"remain\"])\n",
    "                mape_unlearn_unlearn.append(data[\"unlearn\"])\n",
    "                mape_test_unlearn.append(data[\"test\"])\n",
    "            \n",
    "    print(\"\\n\")\n",
    "\n",
    "mse_remain_ori = np.array(mse_remain_ori)\n",
    "mse_remain_unlearn = np.array(mse_remain_unlearn)\n",
    "mse_unlearn_ori = np.array(mse_unlearn_ori)\n",
    "mse_unlearn_unlearn = np.array(mse_unlearn_unlearn)\n",
    "mse_test_ori = np.array(mse_test_ori)\n",
    "mse_test_unlearn = np.array(mse_test_unlearn)\n",
    "\n",
    "cost_remain_ori = np.array(cost_remain_ori)\n",
    "cost_remain_unlearn = np.array(cost_remain_unlearn)\n",
    "cost_unlearn_ori = np.array(cost_unlearn_ori)\n",
    "cost_unlearn_unlearn = np.array(cost_unlearn_unlearn)\n",
    "cost_test_ori = np.array(cost_test_ori)\n",
    "cost_test_unlearn = np.array(cost_test_unlearn)\n",
    "\n",
    "mape_remain_ori = np.array(mape_remain_ori)\n",
    "mape_remain_unlearn = np.array(mape_remain_unlearn)\n",
    "mape_unlearn_ori = np.array(mape_unlearn_ori)\n",
    "mape_unlearn_unlearn = np.array(mape_unlearn_unlearn)\n",
    "mape_test_ori = np.array(mape_test_ori)\n",
    "mape_test_unlearn = np.array(mape_test_unlearn)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure()\n",
    "p11 = plt.plot(mse_remain_ori * 1e3, color = \"blue\", linestyle=':', marker = \"o\")\n",
    "p12 = plt.plot(mse_remain_unlearn* 1e3, color = \"blue\", linestyle = \"-\", marker = \"o\")\n",
    "p13 = plt.plot([0], marker='None', linestyle='None')\n",
    "p21 = plt.plot(mse_unlearn_ori* 1e3, color = \"red\", linestyle = \":\", marker = \"o\")\n",
    "p22 = plt.plot(mse_unlearn_unlearn* 1e3, color = \"red\", linestyle = \"-\", marker = \"o\")\n",
    "p23 = plt.plot([0], marker='None', linestyle='None')\n",
    "p31 = plt.plot(mse_test_ori* 1e3, color = \"green\", linestyle = \":\", marker = \"o\")\n",
    "p32 = plt.plot(mse_test_unlearn* 1e3, color = \"green\", linestyle = \"-\", marker = \"o\")\n",
    "p33 = plt.plot([0], marker='None', linestyle='None')\n",
    "plt.xticks(np.arange(len(unlearn_prop_list)), np.array(unlearn_prop_list) * 100)\n",
    "plt.grid()\n",
    "plt.xlabel(\"Unlearning proportion (%)\")\n",
    "plt.ylabel(r\"MSE ($\\times 1e^{-3}$)\")\n",
    "if model_type == \"linear\":\n",
    "    plt.ylim([0, 11])\n",
    "elif model_type == \"nn_conv\":\n",
    "    plt.ylim([0, 9])\n",
    "elif model_type == \"nn_mixer\":\n",
    "    plt.ylim([0, 8])\n",
    "leg1 = plt.legend([p13[0], p11[0], p12[0], p13[0], p21[0], p22[0], p13[0], p31[0], p32[0]], [\"Remain set\", \"Ori.Model\", \"Unl.Model\", \"Unlearn set\", \"Ori.Model\", \"Unl.Model\", \"Test set\", \"Ori.Model\", \"Unl.Model\"],\n",
    "                ncol = 3, loc = \"best\")\n",
    "plt.savefig(f\"figs/{model_type}_unlearn_mse.pdf\", bbox_inches='tight')\n",
    "plt.show()\n",
    "\n",
    "plt.figure()\n",
    "p11 = plt.plot(mape_remain_ori, color = \"blue\", linestyle = \":\", marker = \"o\")\n",
    "p12 = plt.plot(mape_remain_unlearn, color = \"blue\", linestyle = \"-\", marker = \"o\")\n",
    "p13 = plt.plot([0], marker='None', linestyle='None')\n",
    "p21 = plt.plot(mape_unlearn_ori, color = \"red\", linestyle = \":\", marker = \"o\")\n",
    "p22 = plt.plot(mape_unlearn_unlearn, color = \"red\", linestyle = \"-\", marker = \"o\")\n",
    "p23 = plt.plot([0], marker='None', linestyle='None')\n",
    "p31 = plt.plot(mape_test_ori, color = \"green\", linestyle = \":\", marker = \"o\")\n",
    "p32 = plt.plot(mape_test_unlearn, color = \"green\", linestyle = \"-\", marker = \"o\")\n",
    "p33 = plt.plot([0], marker='None', linestyle='None')\n",
    "plt.xticks(np.arange(len(unlearn_prop_list)), np.array(unlearn_prop_list) * 100)\n",
    "plt.grid()\n",
    "plt.xlabel(\"Unlearning proportion (%)\")\n",
    "plt.ylabel(r\"MAPE (%)\")\n",
    "if model_type == \"linear\":\n",
    "    plt.ylim([5, 20])\n",
    "elif model_type == \"nn_conv\":\n",
    "    plt.ylim([2, 12.1])\n",
    "elif model_type == \"nn_mixer\":\n",
    "    plt.ylim([2, 12])\n",
    "\n",
    "# leg1 = plt.legend([p13[0], p11[0], p12[0], p13[0], p21[0], p22[0], p13[0], p31[0], p32[0]], [\"Remain\", \"Original\", \"Unlearned\", \"Unlearn\", \"Original\", \"Unlearned\", \"Test\", \"Original\", \"Unlearned\"],\n",
    "#                 ncol = 3, loc = \"best\")\n",
    "plt.savefig(f\"figs/{model_type}_unlearn_mape.pdf\", bbox_inches='tight')\n",
    "\n",
    "plt.figure()\n",
    "p11 = plt.plot(cost_remain_ori / 1e3, color = \"blue\", linestyle = \":\", marker = \"o\")\n",
    "p12 = plt.plot(cost_remain_unlearn/ 1e3, color = \"blue\", linestyle = \"-\", marker = \"o\")\n",
    "p13 = plt.plot([0], marker='None', linestyle='None')\n",
    "p21 = plt.plot(cost_unlearn_ori/ 1e3, color = \"red\", linestyle = \":\", marker = \"o\")\n",
    "p22 = plt.plot(cost_unlearn_unlearn/ 1e3, color = \"red\", linestyle = \"-\", marker = \"o\")\n",
    "p23 = plt.plot([0], marker='None', linestyle='None')\n",
    "p31 = plt.plot(cost_test_ori/ 1e3, color = \"green\", linestyle = \":\", marker = \"o\")\n",
    "p32 = plt.plot(cost_test_unlearn/ 1e3, color = \"green\", linestyle = \"-\", marker = \"o\")\n",
    "p33 = plt.plot([0], marker='None', linestyle='None')\n",
    "plt.xticks(np.arange(len(unlearn_prop_list)), np.array(unlearn_prop_list) * 100)\n",
    "plt.grid()\n",
    "plt.xlabel(\"Unlearning proportion (%)\")\n",
    "plt.ylabel(r\"Cost ($\\times 1e^{3}$)\")\n",
    "if model_type == \"linear\":\n",
    "    plt.ylim([0.6, 1.4])\n",
    "elif model_type == \"nn_conv\":\n",
    "    plt.ylim([0.4, 0.61])\n",
    "elif model_type == \"nn_mixer\":\n",
    "    plt.ylim([0.3, 0.7])\n",
    "# leg1 = plt.legend([p13[0], p11[0], p12[0], p13[0], p21[0], p22[0], p13[0], p31[0], p32[0]], [\"Remain\", \"Original\", \"Unlearned\", \"Unlearn\", \"Original\", \"Unlearned\", \"Test\", \"Original\", \"Unlearned\"],\n",
    "#                 ncol = 3, loc = \"best\")\n",
    "plt.savefig(f\"figs/{model_type}_unlearn_cost.pdf\", bbox_inches='tight')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Sensitivity analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mse_scores = np.load(f\"simulation_result/{model_type}/unchange_0.25_mse_scores.npy\", allow_pickle=True).item()\n",
    "mape_scores = np.load(f\"simulation_result/{model_type}/unchange_0.25_mape_scores.npy\", allow_pickle=True).item()\n",
    "cost_scores = np.load(f\"simulation_result/{model_type}/unchange_0.25_cost_scores.npy\", allow_pickle=True).item()\n",
    "\n",
    "mismatch = mse_scores['mismatch_remain']\n",
    "over_index = np.where(mismatch > 0)[0]\n",
    "under_index = np.where(mismatch <= 0)[0]\n",
    "print('over number: ', len(over_index), 'under number: ', len(under_index))\n",
    "\n",
    "np.random.seed(10)\n",
    "\n",
    "number_point = 500\n",
    "over_index = np.random.choice(over_index, number_point, replace = False)\n",
    "under_index = np.random.choice(under_index, number_point, replace = False)\n",
    "\n",
    "plt.figure()\n",
    "plt.scatter(mse_scores['remain'][under_index] * 1e3, cost_scores['remain'][under_index], s = 10, color = 'green', label = 'Under-Generation')\n",
    "plt.scatter(mse_scores['remain'][over_index] * 1e3, cost_scores['remain'][over_index], s = 10, color = 'red', label = 'Over-Genenation')\n",
    "plt.xlabel(r\"MSE Sensitivity($\\times 1e^{-3}$)\")\n",
    "plt.ylabel(r\"Cost Sensitivity\")\n",
    "# plt.legend()\n",
    "plt.grid()\n",
    "plt.savefig(f\"figs/{model_type}_sensitivity_mse-cost.pdf\", bbox_inches='tight')\n",
    "plt.show()\n",
    "\n",
    "plt.figure()\n",
    "plt.scatter(mape_scores['remain'][under_index] * 1e3, cost_scores['remain'][under_index], s = 10, color = 'green', label = 'Under-Generation')\n",
    "plt.scatter(mape_scores['remain'][over_index] * 1e3, cost_scores['remain'][over_index], s = 10, color = 'red', label = 'Over-Genenation')\n",
    "plt.xlabel(r\"MAPE Sensitivity($\\times 1e^{-3}$)\")\n",
    "plt.ylabel(r\"Cost Sensitivity\")\n",
    "# plt.legend()\n",
    "plt.grid()\n",
    "plt.savefig(f\"figs/{model_type}_sensitivity_mape-cost.pdf\", bbox_inches='tight')\n",
    "plt.show()\n",
    "\n",
    "plt.figure()\n",
    "plt.scatter(mse_scores['remain'][under_index] * 1e3, mape_scores['remain'][under_index] * 1e3, s = 10, color = 'green', label = 'Under-Generation')\n",
    "plt.scatter(mse_scores['remain'][over_index] * 1e3, mape_scores['remain'][over_index] * 1e3, s = 10, color = 'red', label = 'Over-Genenation')\n",
    "plt.xlabel(r\"MSE Sensitivity($\\times 1e^{-3}$)\")\n",
    "plt.ylabel(r\"MAPE Sensitivity($\\times 1e^{-3}$)\")\n",
    "plt.legend()\n",
    "plt.grid()\n",
    "plt.savefig(f\"figs/{model_type}_sensitivity_mse-mape.pdf\", bbox_inches='tight')\n",
    "plt.show()\n",
    "\n",
    "# the correlations\n",
    "from scipy.stats import pearsonr\n",
    "r_mse_mape = pearsonr(mse_scores['remain'], mape_scores['remain'])\n",
    "r_mse_cost = pearsonr(mse_scores['remain'], cost_scores['remain'])\n",
    "r_mape_cost = pearsonr(mape_scores['remain'], cost_scores['remain'])\n",
    "\n",
    "print(\"r_mse_mape: \", r_mse_mape)\n",
    "print(\"r_mse_cost: \", r_mse_cost)\n",
    "print(\"r_mape_cost: \", r_mape_cost)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Performance unchanged"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "unchange_mse = np.load(f\"simulation_result/{model_type}/unchange_0.25_mse.npy\", allow_pickle=True).item()\n",
    "unchange_mape = np.load(f\"simulation_result/{model_type}/unchange_0.25_mape.npy\", allow_pickle=True).item()\n",
    "unchange_cost = np.load(f\"simulation_result/{model_type}/unchange_0.25_cost.npy\", allow_pickle=True).item()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "xmin = np.min(unchange_mse[\"l1_constraints\"])\n",
    "xmax = np.max(unchange_mse[\"l1_constraints\"])\n",
    "\n",
    "plt.figure()\n",
    "plt.plot(unchange_mse[\"l1_constraints\"], np.array(unchange_mse[\"mse_test\"]) * 1e3, label = \"MSE-PAMU\", linestyle = \"--\", marker = \"o\", color = \"red\")\n",
    "plt.plot(unchange_mape[\"l1_constraints\"], np.array(unchange_mape[\"mse_test\"]) * 1e3, label = \"MAPE-PAMU\", linestyle = \"--\", marker = \"o\", color = \"blue\")\n",
    "plt.plot(unchange_cost[\"l1_constraints\"], np.array(unchange_cost[\"mse_test\"]) * 1e3, label = \"TAMU\", linestyle = \"--\", marker = \"o\", color = \"green\")\n",
    "plt.hlines(y = unchange_mse[\"mse_test_ori\"] * 1e3, xmin = xmin, xmax = xmax, linestyle = \":\", color = \"black\", label = \"Original model\")\n",
    "plt.hlines(y = unchange_mse[\"mse_test_unlearn\"] * 1e3, xmin = xmin, xmax = xmax, linestyle = \":\", color = \"orange\", label = \"Complete unlearnt model\")\n",
    "plt.legend(loc = \"best\")\n",
    "plt.xlabel(r\"$L_1$ constraint\")\n",
    "plt.ylabel(r\"MSE ($\\times 1e^{-3}$)\")\n",
    "plt.grid()\n",
    "plt.savefig(f\"figs/{model_type}_unchange_mse.pdf\", bbox_inches='tight')\n",
    "plt.show()\n",
    "\n",
    "plt.figure()\n",
    "plt.plot(unchange_mse[\"l1_constraints\"], unchange_mse[\"mape_test\"], label = \"MSE\", linestyle = \"--\", marker = \"o\", color = \"red\")\n",
    "plt.plot(unchange_mape[\"l1_constraints\"], unchange_mape[\"mape_test\"], label = \"MAPE\", linestyle = \"--\", marker = \"o\", color = \"blue\")\n",
    "plt.plot(unchange_cost[\"l1_constraints\"], unchange_cost[\"mape_test\"], label = \"Cost\", linestyle = \"--\", marker = \"o\", color = \"green\")\n",
    "plt.hlines(y = unchange_mape[\"mape_test_ori\"], xmin = xmin, xmax = xmax, linestyle = \":\", color = \"black\", label = \"Original\")\n",
    "plt.hlines(y = unchange_mape[\"mape_test_unlearn\"], xmin = xmin, xmax = xmax, linestyle = \":\", color = \"orange\", label = \"Unlearn\")\n",
    "# plt.legend(loc = \"best\")\n",
    "plt.xlabel(r\"$L_1$ constraint\")\n",
    "plt.ylabel(\"MAPE (%)\")\n",
    "plt.grid()\n",
    "plt.savefig(f\"figs/{model_type}_unchange_mape.pdf\", bbox_inches='tight')\n",
    "plt.show()\n",
    "\n",
    "plt.figure()\n",
    "plt.plot(unchange_mse[\"l1_constraints\"], np.array(unchange_mse[\"cost_test\"]) / 1e3, label = \"MSE\", linestyle = \"--\", marker = \"o\", color = \"red\")\n",
    "plt.plot(unchange_mape[\"l1_constraints\"], np.array(unchange_mape[\"cost_test\"]) / 1e3, label = \"MAPE\", linestyle = \"--\", marker = \"o\", color = \"blue\")\n",
    "plt.plot(unchange_cost[\"l1_constraints\"], np.array(unchange_cost[\"cost_test\"]) / 1e3, label = \"Cost\", linestyle = \"--\", marker = \"o\", color = \"green\")\n",
    "plt.hlines(y = unchange_cost[\"cost_test_ori\"] / 1e3, xmin = xmin, xmax = xmax, linestyle = \":\", color = \"black\", label = \"Original\")\n",
    "plt.hlines(y = unchange_cost[\"cost_test_unlearn\"] / 1e3, xmin = xmin, xmax = xmax, linestyle = \":\", color = \"orange\", label = \"Unlearn\")\n",
    "# plt.legend(loc = \"best\")\n",
    "plt.xlabel(r\"$L_1$ constraint\")\n",
    "plt.ylabel(r\"Cost ($\\times 1e^{3}$)\")\n",
    "plt.grid()\n",
    "plt.savefig(f\"figs/{model_type}_unchange_cost.pdf\", bbox_inches='tight')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "if model_type == 'linear':\n",
    "    plt.figure()\n",
    "    plt.plot(unchange_cost['cost_test'], unchange_cost['parameter_diff'], linestyle = \"--\", marker = \"o\", color = \"red\")\n",
    "    plt.xlabel(\"Cost\")\n",
    "    plt.ylabel(\"Parameter difference\")\n",
    "    plt.vlines(x = unchange_cost[\"cost_test_ori\"], ymin = 0, ymax = 0.6, linestyle = \":\", colors='black', label = \"Original model\")\n",
    "    plt.vlines(x = unchange_cost[\"cost_test_unlearn\"], ymin = 0, ymax = 0.6, linestyle = \":\", colors='orange', label = \"Complete unlearnt model\")\n",
    "    plt.legend(loc = 'best')\n",
    "    plt.text(x = 970, y = 0.32, s = \"trade-off between \\ncompleteness and \\nperformance\", color = \"red\", fontsize = 16)\n",
    "    plt.text(x = 810, y = 0.10, s = \"continuely \\nimprove on \\nperformance\", color = \"red\", fontsize = 16)\n",
    "    plt.grid()\n",
    "    plt.savefig(f\"figs/tradeoff.pdf\", bbox_inches='tight')\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "spo",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.16"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}