pate_varchala.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "pate_varchala.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "toc_visible": true,
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/Varchala/Secure_Private_AI/blob/main/pate_varchala.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "S_ex5WHmPGRy",
        "outputId": "e4b7f52a-97a5-4558-c80d-c63d24432006"
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive') #, force_remount = True)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mounted at /content/drive\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5cmMVScVP8an"
      },
      "source": [
        "import keras\n",
        "from keras.datasets import cifar100\n",
        "from keras.models import Sequential\n",
        "from keras.layers import Dense, Dropout, Activation, Flatten\n",
        "from keras.layers import Conv2D, MaxPooling2D, ZeroPadding2D\n",
        "from keras.regularizers import l2\n",
        "from keras.callbacks import Callback, LearningRateScheduler, TensorBoard, ModelCheckpoint\n",
        "from keras.preprocessing.image import ImageDataGenerator\n",
        "from keras import backend as K\n",
        "from keras import utils as np_utils\n",
        "import sys\n",
        "import os\n",
        "import numpy as np\n",
        "import matplotlib.pyplot as plt\n",
        "from keras.utils import np_utils"
      ],
      "execution_count": 1,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "g2aZABM740EP"
      },
      "source": [
        "import tensorflow as tf"
      ],
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zeopwgHpHxys"
      },
      "source": [
        "def extract_n_classes( data, labels,n, cat_y = True):\n",
        "    data_cl=[]\n",
        "    y=[]\n",
        "    # print(data.shape)\n",
        "    for i in range(n):\n",
        "            # print(data[ np.argwhere( labels == i ).reshape(-1) ].shape)\n",
        "            # print(labels.shape)\n",
        "            # print(data[ np.argwhere( labels.reshape(-1) == i ).reshape(-1)].shape,data.shape)\n",
        "            data_cl.append(data[ np.argwhere( labels.reshape(-1) == i ).reshape(-1) ][ : ])\n",
        "            y.extend(np.full((data_cl[i].shape[0]), i, dtype=int))\n",
        "\n",
        "    # print(np.array(data_cl).shape)\n",
        "    x = np.vstack( (data_cl) )\n",
        "    y = np.array(y)\n",
        "    # print(\"In extract classes function\")\n",
        "    # print(x.shape,y.shape)\n",
        "    return x, y"
      ],
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "F9ckROp9HJSK"
      },
      "source": [
        "# Every teacher has instance of only one class"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Y4LW871PH0ii"
      },
      "source": [
        "# 10 Teachers"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4D4tRvM_H0ii"
      },
      "source": [
        "n_instances = 500\n",
        "n_teachers = 10\n",
        "\n",
        "# load data and transform it\n",
        "(x_train, y_train), (x_test, y_test) = keras.datasets.cifar100.load_data()\n",
        "\n",
        "x_train = x_train.astype( float ) / 255.\n",
        "x_test = x_test.astype( float ) / 255.\n",
        "\n",
        "x_train = x_train.reshape( -1, 32, 32, 3)\n",
        "x_test = x_test.reshape( -1, 32, 32, 3)\n",
        "\n",
        "x_train, y_train = extract_n_classes( x_train, y_train ,n=100,cat_y=False)\n",
        "\n",
        "y_train = keras.utils.np_utils.to_categorical( y_train )\n",
        "y_test = keras.utils.np_utils.to_categorical( y_test )\n",
        "\n",
        "# shuffle data\n",
        "idx = np.arange( len( x_train ) )\n",
        "np.random.shuffle( idx )\n",
        "x_train = x_train[ idx ]\n",
        "y_train = y_train[ idx ]\n",
        "\n",
        "# gather the teacher data\n",
        "teacher_data_x = np.array([ x_train[ i * n_instances : ( i + 1 ) * n_instances ] for i in range( n_teachers ) ])\n",
        "teacher_data_y = np.array([ y_train[ i * n_instances : ( i + 1 ) * n_instances ] for i in range( n_teachers ) ])\n",
        "\n",
        "# gather the student data\n",
        "student_data_x = np.array(x_train[ n_teachers * n_instances : ( n_teachers + 1 ) * n_instances ] )\n",
        "student_data_y = np.array(y_train[ n_teachers * n_instances : ( n_teachers + 1 ) * n_instances ]  )"
      ],
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "2bPEVTeYH0ii",
        "outputId": "a391aa69-b025-4d4c-f195-48be1cb614cc"
      },
      "source": [
        "teacher_data_x.shape, teacher_data_y.shape"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "((10, 500, 32, 32, 3), (10, 500, 100))"
            ]
          },
          "metadata": {},
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "u4-lrk16H0ij",
        "outputId": "74f91e46-267e-4328-86bc-a921fe2074a4"
      },
      "source": [
        "student_data_x.shape,student_data_y.shape"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "((500, 32, 32, 3), (500, 100))"
            ]
          },
          "metadata": {},
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "uAgAaLQhH0ij",
        "outputId": "c8a9ad8a-2f1c-4099-d641-91db2d243d66"
      },
      "source": [
        "## train the teacher models\n",
        "# train the teacher models according to cifar100 dataset in tensorflow\n",
        "def get_model():\n",
        "  # Create the model\n",
        "  model = Sequential()\n",
        "  model.add(Conv2D(16, kernel_size=1, activation='relu'))\n",
        "  model.add(Conv2D(16, kernel_size=3, activation='relu'))\n",
        "  model.add(MaxPooling2D(pool_size=(2, 2)))\n",
        "  model.add(Conv2D(32, kernel_size=3, activation='relu'))\n",
        "  model.add(Conv2D(32, (3, 3), activation='relu'))\n",
        "  model.add(MaxPooling2D(pool_size=(2, 2)))\n",
        "  model.add(Flatten())\n",
        "  model.add(Dense(100, activation='softmax'))\n",
        "\n",
        "\n",
        "  model.compile( loss = 'categorical_crossentropy', optimizer = 'adam', metrics = [ 'accuracy' ],run_eagerly=True)\n",
        "\n",
        "  return model\n",
        "    \n",
        "\n",
        "# list of teacher models\n",
        "teacher_models = [ get_model() for i in range( n_teachers ) ]\n",
        "\n",
        "for i, (model, x, y) in enumerate( zip( teacher_models, teacher_data_x, teacher_data_y ) ):\n",
        "    print( 'teacher', i )\n",
        "    model.fit( x, y, batch_size = 32, epochs = 10, verbose = 0 )\n",
        "    print( 'test accuracy:', model.evaluate( x_test, y_test, verbose = 0 )[ 1 ] )\n",
        "    model.save( '/content/drive/MyDrive/SPAI_projectphase2/teacher_model_{}.h5'.format( i ) )"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "teacher 0\n",
            "test accuracy: 0.06809999793767929\n",
            "teacher 1\n",
            "test accuracy: 0.0568000003695488\n",
            "teacher 2\n",
            "test accuracy: 0.05570000037550926\n",
            "teacher 3\n",
            "test accuracy: 0.0608999989926815\n",
            "teacher 4\n",
            "test accuracy: 0.05550000071525574\n",
            "teacher 5\n",
            "test accuracy: 0.0560000017285347\n",
            "teacher 6\n",
            "test accuracy: 0.057500001043081284\n",
            "teacher 7\n",
            "test accuracy: 0.0575999990105629\n",
            "teacher 8\n",
            "test accuracy: 0.0706000030040741\n",
            "teacher 9\n",
            "test accuracy: 0.060100000351667404\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "B8WkTstwH0ij"
      },
      "source": [
        "## training the student model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BXsCjYeWH0ij"
      },
      "source": [
        "tf.config.run_functions_eagerly(True)\n",
        "tf.data.experimental.enable_debug_mode()"
      ],
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JL1lrJKPH0ij"
      },
      "source": [
        "# student_data_x"
      ],
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uwyrZNjPH0ij"
      },
      "source": [
        "# label the data\n",
        "labels = np.array([ teacher.predict( student_data_x ) for teacher in teacher_models ])"
      ],
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "nsbSeQx_H0ij",
        "outputId": "1034e51f-2a68-4f03-fa6b-a88a144b0ad0"
      },
      "source": [
        "labels.shape"
      ],
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(10, 500, 100)"
            ]
          },
          "metadata": {},
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "9pRJdM5HH0ij",
        "outputId": "5f2da15f-d09c-4155-9c81-580224a6c81e"
      },
      "source": [
        "student_data_x.shape"
      ],
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(500, 32, 32, 3)"
            ]
          },
          "metadata": {},
          "execution_count": 13
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "VQ9BTnzWH0ij",
        "outputId": "532a544a-01b6-4979-aeee-d71ff14c5139"
      },
      "source": [
        "len(student_data_x)"
      ],
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "500"
            ]
          },
          "metadata": {},
          "execution_count": 14
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5vedhvYiH0ik"
      },
      "source": [
        "# preform the voting\n",
        "votes = np.zeros( ( student_data_x.shape[ 0 ], 100 ), dtype=np.float )\n",
        "for i in range( len( student_data_x ) ):\n",
        "  for j in range( n_teachers ):\n",
        "    label = np.argmax( labels[ j ][ i ] )\n",
        "    votes[ i, label ] += 1\n",
        "  # add the noise per class\n",
        "  for j in range( 10 ):\n",
        "    votes[ i, j ] += np.random.laplace(loc=0.0, scale=5 )"
      ],
      "execution_count": 15,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "VuUj1DOcH0ik",
        "outputId": "0b2361f8-f24a-4171-e81b-0c4346cdc66d"
      },
      "source": [
        "student_data_x.shape,student_data_y.shape,votes.shape"
      ],
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "((500, 32, 32, 3), (500, 100), (500, 100))"
            ]
          },
          "metadata": {},
          "execution_count": 16
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PdckbZOeH0ik"
      },
      "source": [
        "ll = np.argmax( votes, axis=1 )"
      ],
      "execution_count": 17,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "X2edpLEpH0ik",
        "outputId": "16560529-8188-48a6-ed75-b1ebbd9c82ee"
      },
      "source": [
        "ll.shape"
      ],
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(500,)"
            ]
          },
          "metadata": {},
          "execution_count": 18
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "zWqHgODvH0ik",
        "outputId": "5ee71da1-0f81-4e0e-e2e5-8cecf69281a4"
      },
      "source": [
        "student_data_y.shape,keras.utils.np_utils.to_categorical( ll,num_classes=100 ).shape"
      ],
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "((500, 100), (500, 100))"
            ]
          },
          "metadata": {},
          "execution_count": 19
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Y273T1i-H0ik",
        "outputId": "1382bc44-1a18-4fe0-cf6e-6a29edb6d1a1"
      },
      "source": [
        "student_data_y = keras.utils.np_utils.to_categorical( np.argmax( votes, axis=1 ),num_classes=100 )\n",
        "\n",
        "# train model\n",
        "student_model = get_model()\n",
        "print( 'training student model' )\n",
        "student_model.fit( x, y, epochs=16, verbose=0 )\n",
        "print( 'test accuracy:', student_model.evaluate( student_data_x, student_data_y, verbose=0 )[ 1 ] )"
      ],
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "training student model\n",
            "test accuracy: 0.024000000208616257\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fYW7TTvTH0ik"
      },
      "source": [
        "# 20 Teachers"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wyUrrqtkH0ik"
      },
      "source": [
        "n_instances = 500 \n",
        "n_teachers = 20\n",
        "\n",
        "# load data and transform it\n",
        "(x_train, y_train), (x_test, y_test) = keras.datasets.cifar100.load_data()\n",
        "\n",
        "x_train = x_train.astype( float ) / 255.\n",
        "x_test = x_test.astype( float ) / 255.\n",
        "\n",
        "x_train = x_train.reshape( -1, 32, 32, 3)\n",
        "x_test = x_test.reshape( -1, 32, 32, 3)\n",
        "\n",
        "x_train, y_train = extract_n_classes( x_train, y_train ,n=100,cat_y=False)\n",
        "\n",
        "y_train = keras.utils.np_utils.to_categorical( y_train )\n",
        "y_test = keras.utils.np_utils.to_categorical( y_test )\n",
        "\n",
        "# shuffle data\n",
        "idx = np.arange( len( x_train ) )\n",
        "np.random.shuffle( idx )\n",
        "x_train = x_train[ idx ]\n",
        "y_train = y_train[ idx ]\n",
        "\n",
        "# gather the teacher data\n",
        "teacher_data_x = np.array([ x_train[ i * n_instances : ( i + 1 ) * n_instances ] for i in range( n_teachers ) ])\n",
        "teacher_data_y = np.array([ y_train[ i * n_instances : ( i + 1 ) * n_instances ] for i in range( n_teachers ) ])\n",
        "\n",
        "# gather the student data\n",
        "student_data_x = np.array(x_train[ n_teachers * n_instances : ( n_teachers + 1 ) * n_instances ] )\n",
        "student_data_y = np.array(y_train[ n_teachers * n_instances : ( n_teachers + 1 ) * n_instances ]  )"
      ],
      "execution_count": 21,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "YPte0s5XH0ik",
        "outputId": "71f2fafc-e2f3-4b6e-8cae-1323822b65ee"
      },
      "source": [
        "teacher_data_x.shape, teacher_data_y.shape"
      ],
      "execution_count": 22,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "((20, 500, 32, 32, 3), (20, 500, 100))"
            ]
          },
          "metadata": {},
          "execution_count": 22
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "SQ4NAfk3H0ik",
        "outputId": "31e7624f-ab2e-4bd1-91c8-a29f335fcc85"
      },
      "source": [
        "## train the teacher models\n",
        "# train the teacher models according to cifar100 dataset in tensorflow\n",
        "def get_model():\n",
        "  # Create the model\n",
        "  model = Sequential()\n",
        "  model.add(Conv2D(16, kernel_size=1, activation='relu'))\n",
        "  model.add(Conv2D(16, kernel_size=3, activation='relu'))\n",
        "  model.add(MaxPooling2D(pool_size=(2, 2)))\n",
        "  model.add(Conv2D(32, kernel_size=3, activation='relu'))\n",
        "  model.add(Conv2D(32, (3, 3), activation='relu'))\n",
        "  model.add(MaxPooling2D(pool_size=(2, 2)))\n",
        "  model.add(Flatten())\n",
        "  model.add(Dense(100, activation='softmax'))\n",
        "\n",
        "\n",
        "  model.compile( loss = 'categorical_crossentropy', optimizer = 'adam', metrics = [ 'accuracy' ] )\n",
        "\n",
        "  return model\n",
        "    \n",
        "\n",
        "# list of teacher models\n",
        "teacher_models = [ get_model() for i in range( n_teachers ) ]\n",
        "\n",
        "for i, (model, x, y) in enumerate( zip( teacher_models, teacher_data_x, teacher_data_y ) ):\n",
        "    print( 'teacher', i )\n",
        "    model.fit( x, y, batch_size = 32, epochs = 10, verbose = 0 )\n",
        "    print( 'test accuracy:', model.evaluate( x_test, y_test, verbose = 0 )[ 1 ] )\n",
        "    model.save( '/content/drive/MyDrive/SPAI_projectphase2/teacher_model_{}.h5'.format( i ) )"
      ],
      "execution_count": 23,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "teacher 0\n",
            "test accuracy: 0.06509999930858612\n",
            "teacher 1\n",
            "test accuracy: 0.07100000232458115\n",
            "teacher 2\n",
            "test accuracy: 0.05790000036358833\n",
            "teacher 3\n",
            "test accuracy: 0.06650000065565109\n",
            "teacher 4\n",
            "test accuracy: 0.05350000038743019\n",
            "teacher 5\n",
            "test accuracy: 0.0658000037074089\n",
            "teacher 6\n",
            "test accuracy: 0.059700001031160355\n",
            "teacher 7\n",
            "test accuracy: 0.06369999796152115\n",
            "teacher 8\n",
            "test accuracy: 0.06129999831318855\n",
            "teacher 9\n",
            "test accuracy: 0.05400000140070915\n",
            "teacher 10\n",
            "test accuracy: 0.065700002014637\n",
            "teacher 11\n",
            "test accuracy: 0.05550000071525574\n",
            "teacher 12\n",
            "test accuracy: 0.05660000070929527\n",
            "teacher 13\n",
            "test accuracy: 0.06719999760389328\n",
            "teacher 14\n",
            "test accuracy: 0.0632999986410141\n",
            "teacher 15\n",
            "test accuracy: 0.06260000169277191\n",
            "teacher 16\n",
            "test accuracy: 0.047600001096725464\n",
            "teacher 17\n",
            "test accuracy: 0.05739999935030937\n",
            "teacher 18\n",
            "test accuracy: 0.06520000100135803\n",
            "teacher 19\n",
            "test accuracy: 0.06780000030994415\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "G0ICyY8hH0il"
      },
      "source": [
        "## training the student model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TkX5Fk7dH0il"
      },
      "source": [
        "# label the data\n",
        "labels = np.array([ teacher.predict( student_data_x ) for teacher in teacher_models ])"
      ],
      "execution_count": 24,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "8HlLwtKAH0il",
        "outputId": "bcfb9887-32c2-427c-9cf9-0f1d6b1d6e0c"
      },
      "source": [
        "labels.shape"
      ],
      "execution_count": 25,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(20, 500, 100)"
            ]
          },
          "metadata": {},
          "execution_count": 25
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "SctWD1srH0il",
        "outputId": "32588554-5b69-48ab-856e-d5fe6bd593ee"
      },
      "source": [
        "student_data_x.shape"
      ],
      "execution_count": 26,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(500, 32, 32, 3)"
            ]
          },
          "metadata": {},
          "execution_count": 26
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "8nADoTV9H0il",
        "outputId": "53ca6d16-eee7-49cf-b5f8-051cccc81eef"
      },
      "source": [
        "len(student_data_x)"
      ],
      "execution_count": 27,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "500"
            ]
          },
          "metadata": {},
          "execution_count": 27
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iNgASYmWH0il"
      },
      "source": [
        "# preform the voting\n",
        "votes = np.zeros( ( student_data_x.shape[ 0 ], 100 ), dtype=np.float )\n",
        "for i in range( len( student_data_x ) ):\n",
        "  for j in range( n_teachers ):\n",
        "    label = np.argmax( labels[ j ][ i ] )\n",
        "    votes[ i, label ] += 1\n",
        "  # add the noise per class\n",
        "  for j in range( 10 ):\n",
        "    votes[ i, j ] += np.random.laplace(loc=0.0, scale=5 )"
      ],
      "execution_count": 28,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ArUlWtBUH0il",
        "outputId": "cf9950df-7d7e-4926-f187-ed22c1ad5490"
      },
      "source": [
        "student_data_x.shape,student_data_y.shape,votes.shape"
      ],
      "execution_count": 29,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "((500, 32, 32, 3), (500, 100), (500, 100))"
            ]
          },
          "metadata": {},
          "execution_count": 29
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kst07lt6H0il"
      },
      "source": [
        "ll = np.argmax( votes, axis=1 )"
      ],
      "execution_count": 30,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "3GRDbxnYH0il",
        "outputId": "760099bc-4c65-4637-a322-a300e1edd02f"
      },
      "source": [
        "ll.shape"
      ],
      "execution_count": 31,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(500,)"
            ]
          },
          "metadata": {},
          "execution_count": 31
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "SQ2Wcee-H0il",
        "outputId": "963e4a39-467a-41f1-fc5f-4f9d7bdcf69a"
      },
      "source": [
        "student_data_y.shape,keras.utils.np_utils.to_categorical( ll,num_classes=100 ).shape"
      ],
      "execution_count": 32,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "((500, 100), (500, 100))"
            ]
          },
          "metadata": {},
          "execution_count": 32
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "74CBS2ciH0il",
        "outputId": "b585849d-3f3f-4b04-f787-7a5a7df144db"
      },
      "source": [
        "student_data_y = keras.utils.np_utils.to_categorical( np.argmax( votes, axis=1 ),num_classes=100 )\n",
        "\n",
        "# train model\n",
        "student_model = get_model()\n",
        "print( 'training student model' )\n",
        "student_model.fit( x, y, epochs=16, verbose=0 )\n",
        "print( 'test accuracy:', student_model.evaluate( student_data_x, student_data_y, verbose=0 )[ 1 ] )"
      ],
      "execution_count": 33,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "training student model\n",
            "test accuracy: 0.02199999988079071\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "lW6wLVUGH0il"
      },
      "source": [
        "# 50 Teachers"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8QiDi8ndH0im"
      },
      "source": [
        "n_instances = 500 \n",
        "n_teachers = 50\n",
        "\n",
        "# load data and transform it\n",
        "(x_train, y_train), (x_test, y_test) = keras.datasets.cifar100.load_data()\n",
        "\n",
        "x_train = x_train.astype( float ) / 255.\n",
        "x_test = x_test.astype( float ) / 255.\n",
        "\n",
        "x_train = x_train.reshape( -1, 32, 32, 3)\n",
        "x_test = x_test.reshape( -1, 32, 32, 3)\n",
        "\n",
        "x_train, y_train = extract_n_classes( x_train, y_train ,n=100,cat_y=False)\n",
        "\n",
        "y_train = keras.utils.np_utils.to_categorical( y_train )\n",
        "y_test = keras.utils.np_utils.to_categorical( y_test )\n",
        "\n",
        "# shuffle data\n",
        "idx = np.arange( len( x_train ) )\n",
        "np.random.shuffle( idx )\n",
        "x_train = x_train[ idx ]\n",
        "y_train = y_train[ idx ]\n",
        "\n",
        "# gather the teacher data\n",
        "teacher_data_x = np.array([ x_train[ i * n_instances : ( i + 1 ) * n_instances ] for i in range( n_teachers ) ])\n",
        "teacher_data_y = np.array([ y_train[ i * n_instances : ( i + 1 ) * n_instances ] for i in range( n_teachers ) ])\n",
        "\n",
        "# gather the student data\n",
        "student_data_x = np.array(x_train[ n_teachers * n_instances : ( n_teachers + 1 ) * n_instances ] )\n",
        "student_data_y = np.array(y_train[ n_teachers * n_instances : ( n_teachers + 1 ) * n_instances ]  )"
      ],
      "execution_count": 34,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "uhhVIp16H0im",
        "outputId": "dbb3fd26-90e1-4dc0-84ec-f897adad0fc7"
      },
      "source": [
        "teacher_data_x.shape, teacher_data_y.shape"
      ],
      "execution_count": 35,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "((50, 500, 32, 32, 3), (50, 500, 100))"
            ]
          },
          "metadata": {},
          "execution_count": 35
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "qik1r9ssH0im",
        "outputId": "d098e9ba-724e-43e4-e3f0-0f99191e42b9"
      },
      "source": [
        "## train the teacher models\n",
        "# train the teacher models according to cifar100 dataset in tensorflow\n",
        "def get_model():\n",
        "  # Create the model\n",
        "  model = Sequential()\n",
        "  model.add(Conv2D(16, kernel_size=1, activation='relu'))\n",
        "  model.add(Conv2D(16, kernel_size=3, activation='relu'))\n",
        "  model.add(MaxPooling2D(pool_size=(2, 2)))\n",
        "  model.add(Conv2D(32, kernel_size=3, activation='relu'))\n",
        "  model.add(Conv2D(32, (3, 3), activation='relu'))\n",
        "  model.add(MaxPooling2D(pool_size=(2, 2)))\n",
        "  model.add(Flatten())\n",
        "  model.add(Dense(100, activation='softmax'))\n",
        "\n",
        "\n",
        "  model.compile( loss = 'categorical_crossentropy', optimizer = 'adam', metrics = [ 'accuracy' ] )\n",
        "\n",
        "  return model\n",
        "    \n",
        "\n",
        "# list of teacher models\n",
        "teacher_models = [ get_model() for i in range( n_teachers ) ]\n",
        "\n",
        "for i, (model, x, y) in enumerate( zip( teacher_models, teacher_data_x, teacher_data_y ) ):\n",
        "    print( 'teacher', i )\n",
        "    model.fit( x, y, batch_size = 32, epochs = 10, verbose = 0 )\n",
        "    print( 'test accuracy:', model.evaluate( x_test, y_test, verbose = 0 )[ 1 ] )\n",
        "    model.save( '/content/drive/MyDrive/SPAI_projectphase2/teacher_model_{}.h5'.format( i ) )"
      ],
      "execution_count": 36,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "teacher 0\n",
            "test accuracy: 0.06279999762773514\n",
            "teacher 1\n",
            "test accuracy: 0.05900000035762787\n",
            "teacher 2\n",
            "test accuracy: 0.04919999837875366\n",
            "teacher 3\n",
            "test accuracy: 0.058800000697374344\n",
            "teacher 4\n",
            "test accuracy: 0.05810000002384186\n",
            "teacher 5\n",
            "test accuracy: 0.06210000067949295\n",
            "teacher 6\n",
            "test accuracy: 0.06769999861717224\n",
            "teacher 7\n",
            "test accuracy: 0.05649999901652336\n",
            "teacher 8\n",
            "test accuracy: 0.05849999934434891\n",
            "teacher 9\n",
            "test accuracy: 0.06620000302791595\n",
            "teacher 10\n",
            "test accuracy: 0.06030000001192093\n",
            "teacher 11\n",
            "test accuracy: 0.06310000270605087\n",
            "teacher 12\n",
            "test accuracy: 0.055799998342990875\n",
            "teacher 13\n",
            "test accuracy: 0.05480000004172325\n",
            "teacher 14\n",
            "test accuracy: 0.05689999833703041\n",
            "teacher 15\n",
            "test accuracy: 0.06069999933242798\n",
            "teacher 16\n",
            "test accuracy: 0.05999999865889549\n",
            "teacher 17\n",
            "test accuracy: 0.06759999692440033\n",
            "teacher 18\n",
            "test accuracy: 0.053700000047683716\n",
            "teacher 19\n",
            "test accuracy: 0.05920000001788139\n",
            "teacher 20\n",
            "test accuracy: 0.06310000270605087\n",
            "teacher 21\n",
            "test accuracy: 0.0625\n",
            "teacher 22\n",
            "test accuracy: 0.06589999794960022\n",
            "teacher 23\n",
            "test accuracy: 0.06989999860525131\n",
            "teacher 24\n",
            "test accuracy: 0.05460000038146973\n",
            "teacher 25\n",
            "test accuracy: 0.06419999897480011\n",
            "teacher 26\n",
            "test accuracy: 0.06669999659061432\n",
            "teacher 27\n",
            "test accuracy: 0.06069999933242798\n",
            "teacher 28\n",
            "test accuracy: 0.05009999871253967\n",
            "teacher 29\n",
            "test accuracy: 0.05810000002384186\n",
            "teacher 30\n",
            "test accuracy: 0.060600001364946365\n",
            "teacher 31\n",
            "test accuracy: 0.05719999969005585\n",
            "teacher 32\n",
            "test accuracy: 0.05640000104904175\n",
            "teacher 33\n",
            "test accuracy: 0.06719999760389328\n",
            "teacher 34\n",
            "test accuracy: 0.06080000102519989\n",
            "teacher 35\n",
            "test accuracy: 0.060499999672174454\n",
            "teacher 36\n",
            "test accuracy: 0.057999998331069946\n",
            "teacher 37\n",
            "test accuracy: 0.06419999897480011\n",
            "teacher 38\n",
            "test accuracy: 0.04610000178217888\n",
            "teacher 39\n",
            "test accuracy: 0.05550000071525574\n",
            "teacher 40\n",
            "test accuracy: 0.0674000009894371\n",
            "teacher 41\n",
            "test accuracy: 0.066600002348423\n",
            "teacher 42\n",
            "test accuracy: 0.055799998342990875\n",
            "teacher 43\n",
            "test accuracy: 0.06530000269412994\n",
            "teacher 44\n",
            "test accuracy: 0.062199998646974564\n",
            "teacher 45\n",
            "test accuracy: 0.06859999895095825\n",
            "teacher 46\n",
            "test accuracy: 0.0568000003695488\n",
            "teacher 47\n",
            "test accuracy: 0.061900001019239426\n",
            "teacher 48\n",
            "test accuracy: 0.07000000029802322\n",
            "teacher 49\n",
            "test accuracy: 0.0560000017285347\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "L2jwk0udH0im"
      },
      "source": [
        "## training the student model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EdAR_cTJH0im"
      },
      "source": [
        "# label the data\n",
        "labels = np.array([ teacher.predict( student_data_x ) for teacher in teacher_models ])"
      ],
      "execution_count": 37,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "UpFbVzU2H0im",
        "outputId": "a83064c8-d49c-4103-ab5a-e5b8a0807e6b"
      },
      "source": [
        "labels.shape"
      ],
      "execution_count": 38,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(50, 500, 100)"
            ]
          },
          "metadata": {},
          "execution_count": 38
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "GhDqnSg6H0im",
        "outputId": "8ba516a4-6a58-4bca-cde0-5a62706b259c"
      },
      "source": [
        "student_data_x.shape"
      ],
      "execution_count": 39,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(500, 32, 32, 3)"
            ]
          },
          "metadata": {},
          "execution_count": 39
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "4SoiaRghH0im",
        "outputId": "0b9f1d06-b543-4531-ffc9-b524784ed157"
      },
      "source": [
        "len(student_data_x)"
      ],
      "execution_count": 40,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "500"
            ]
          },
          "metadata": {},
          "execution_count": 40
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8RLa0qJTH0im"
      },
      "source": [
        "# preform the voting\n",
        "votes = np.zeros( ( student_data_x.shape[ 0 ], 100 ), dtype=np.float )\n",
        "for i in range( len( student_data_x ) ):\n",
        "  for j in range( n_teachers ):\n",
        "    label = np.argmax( labels[ j ][ i ] )\n",
        "    votes[ i, label ] += 1\n",
        "  # add the noise per class\n",
        "  for j in range( 10 ):\n",
        "    votes[ i, j ] += np.random.laplace(loc=0.0, scale=5 )"
      ],
      "execution_count": 41,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "V9m8GRmAH0im",
        "outputId": "bac07ee4-5415-470b-c86d-771e3a53d0c5"
      },
      "source": [
        "student_data_x.shape,student_data_y.shape,votes.shape"
      ],
      "execution_count": 42,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "((500, 32, 32, 3), (500, 100), (500, 100))"
            ]
          },
          "metadata": {},
          "execution_count": 42
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cVXIJeZvH0in"
      },
      "source": [
        "ll = np.argmax( votes, axis=1 )"
      ],
      "execution_count": 43,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "sacmmmd-H0in",
        "outputId": "4c5e864e-8c7b-4874-b052-19eb00efe697"
      },
      "source": [
        "ll.shape"
      ],
      "execution_count": 44,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(500,)"
            ]
          },
          "metadata": {},
          "execution_count": 44
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "K69mFOPJH0in",
        "outputId": "27f0baaa-879b-4b0e-e4a7-0d906a10c8a0"
      },
      "source": [
        "student_data_y.shape,keras.utils.np_utils.to_categorical( ll,num_classes=100 ).shape"
      ],
      "execution_count": 45,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "((500, 100), (500, 100))"
            ]
          },
          "metadata": {},
          "execution_count": 45
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "hC8r3heBH0in",
        "outputId": "f34ea906-dc47-4861-d4d6-61662685e1ca"
      },
      "source": [
        "student_data_y = keras.utils.np_utils.to_categorical( np.argmax( votes, axis=1 ),num_classes=100 )\n",
        "\n",
        "# train model\n",
        "student_model = get_model()\n",
        "print( 'training student model' )\n",
        "student_model.fit( x, y, epochs=16, verbose=0 )\n",
        "print( 'test accuracy:', student_model.evaluate( student_data_x, student_data_y, verbose=0 )[ 1 ] )"
      ],
      "execution_count": 46,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "training student model\n",
            "test accuracy: 0.05000000074505806\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "wN_ZOuUQH0in"
      },
      "source": [
        "# 100 Teachers"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wdhpOflMH0in"
      },
      "source": [
        "n_instances = 300\n",
        "n_teachers = 100\n",
        "\n",
        "# load data and transform it\n",
        "(x_train, y_train), (x_test, y_test) = keras.datasets.cifar100.load_data()\n",
        "\n",
        "x_train = x_train.astype( float ) / 255.\n",
        "x_test = x_test.astype( float ) / 255.\n",
        "\n",
        "x_train = x_train.reshape( -1, 32, 32, 3)\n",
        "x_test = x_test.reshape( -1, 32, 32, 3)\n",
        "\n",
        "x_train, y_train = extract_n_classes( x_train, y_train ,n=100,cat_y=False)\n",
        "\n",
        "y_train = keras.utils.np_utils.to_categorical( y_train )\n",
        "y_test = keras.utils.np_utils.to_categorical( y_test )\n",
        "\n",
        "# shuffle data\n",
        "idx = np.arange( len( x_train ) )\n",
        "np.random.shuffle( idx )\n",
        "x_train = x_train[ idx ]\n",
        "y_train = y_train[ idx ]\n",
        "\n",
        "# gather the teacher data\n",
        "teacher_data_x = np.array([ x_train[ i * n_instances : ( i + 1 ) * n_instances ] for i in range( n_teachers ) ])\n",
        "teacher_data_y = np.array([ y_train[ i * n_instances : ( i + 1 ) * n_instances ] for i in range( n_teachers ) ])\n",
        "\n",
        "# gather the student data\n",
        "student_data_x = np.array(x_train[ n_teachers * n_instances : ( n_teachers + 1 ) * n_instances ] )\n",
        "student_data_y = np.array(y_train[ n_teachers * n_instances : ( n_teachers + 1 ) * n_instances ]  )"
      ],
      "execution_count": 33,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "7sidZd8dH0in",
        "outputId": "8031a662-2db3-4594-a010-9eafe6a6f1b1"
      },
      "source": [
        "teacher_data_x.shape, teacher_data_y.shape"
      ],
      "execution_count": 34,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "((100, 300, 32, 32, 3), (100, 300, 100))"
            ]
          },
          "metadata": {},
          "execution_count": 34
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "M1mpTjagH0in",
        "outputId": "36707a28-4808-4af3-9ab8-94fec1ae00c7"
      },
      "source": [
        "## train the teacher models\n",
        "# train the teacher models according to cifar100 dataset in tensorflow\n",
        "def get_model():\n",
        "  # Create the model\n",
        "  model = Sequential()\n",
        "  model.add(Conv2D(16, kernel_size=1, activation='relu'))\n",
        "  model.add(Conv2D(16, kernel_size=3, activation='relu'))\n",
        "  model.add(MaxPooling2D(pool_size=(2, 2)))\n",
        "  model.add(Conv2D(32, kernel_size=3, activation='relu'))\n",
        "  model.add(Conv2D(32, (3, 3), activation='relu'))\n",
        "  model.add(MaxPooling2D(pool_size=(2, 2)))\n",
        "  model.add(Flatten())\n",
        "  model.add(Dense(100, activation='softmax'))\n",
        "\n",
        "\n",
        "  model.compile( loss = 'categorical_crossentropy', optimizer = 'adam', metrics = [ 'accuracy' ] )\n",
        "\n",
        "  return model\n",
        "    \n",
        "\n",
        "# list of teacher models\n",
        "teacher_models = [ get_model() for i in range( n_teachers ) ]\n",
        "\n",
        "for i, (model, x, y) in enumerate( zip( teacher_models, teacher_data_x, teacher_data_y ) ):\n",
        "    print( 'teacher', i )\n",
        "    model.fit( x, y, batch_size = 32, epochs = 10, verbose = 0 )\n",
        "    print( 'test accuracy:', model.evaluate( x_test, y_test, verbose = 0 )[ 1 ] )\n",
        "    model.save( '/content/drive/MyDrive/SPAI_projectphase2/teacher_model_{}.h5'.format( i ) )"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "teacher 0\n",
            "test accuracy: 0.059300001710653305\n",
            "teacher 1\n",
            "test accuracy: 0.06610000133514404\n",
            "teacher 2\n",
            "test accuracy: 0.06440000236034393\n",
            "teacher 3\n",
            "test accuracy: 0.050599999725818634\n",
            "teacher 4\n",
            "test accuracy: 0.06939999759197235\n",
            "teacher 5\n",
            "test accuracy: 0.05310000106692314\n",
            "teacher 6\n",
            "test accuracy: 0.07020000368356705\n",
            "teacher 7\n",
            "test accuracy: 0.06509999930858612\n",
            "teacher 8\n",
            "test accuracy: 0.05979999899864197\n",
            "teacher 9\n",
            "test accuracy: 0.05959999933838844\n",
            "teacher 10\n",
            "test accuracy: 0.06870000064373016\n",
            "teacher 11\n",
            "test accuracy: 0.06459999829530716\n",
            "teacher 12\n",
            "test accuracy: 0.06809999793767929\n",
            "teacher 13\n",
            "test accuracy: 0.049300000071525574\n",
            "teacher 14\n",
            "test accuracy: 0.05950000137090683\n",
            "teacher 15\n",
            "test accuracy: 0.05739999935030937\n",
            "teacher 16\n",
            "test accuracy: 0.06300000101327896\n",
            "teacher 17\n",
            "test accuracy: 0.06379999965429306\n",
            "teacher 18\n",
            "test accuracy: 0.053199999034404755\n",
            "teacher 19\n",
            "test accuracy: 0.06499999761581421\n",
            "teacher 20\n",
            "test accuracy: 0.05609999969601631\n",
            "teacher 21\n",
            "test accuracy: 0.05260000005364418\n",
            "teacher 22\n",
            "test accuracy: 0.05310000106692314\n",
            "teacher 23\n",
            "test accuracy: 0.06340000033378601\n",
            "teacher 24\n",
            "test accuracy: 0.0560000017285347\n",
            "teacher 25\n",
            "test accuracy: 0.056699998676776886\n",
            "teacher 26\n",
            "test accuracy: 0.061400000005960464\n",
            "teacher 27\n",
            "test accuracy: 0.0608999989926815\n",
            "teacher 28\n",
            "test accuracy: 0.06109999865293503\n",
            "teacher 29\n",
            "test accuracy: 0.06939999759197235\n",
            "teacher 30\n",
            "test accuracy: 0.05389999970793724\n",
            "teacher 31\n",
            "test accuracy: 0.06419999897480011\n",
            "teacher 32\n",
            "test accuracy: 0.04960000142455101\n",
            "teacher 33\n",
            "test accuracy: 0.06620000302791595\n",
            "teacher 34\n",
            "test accuracy: 0.06599999964237213\n",
            "teacher 35\n",
            "test accuracy: 0.051899999380111694\n",
            "teacher 36\n",
            "test accuracy: 0.04540000110864639\n",
            "teacher 37\n",
            "test accuracy: 0.05350000038743019\n",
            "teacher 38\n",
            "test accuracy: 0.07199999690055847\n",
            "teacher 39\n",
            "test accuracy: 0.048700001090765\n",
            "teacher 40\n",
            "test accuracy: 0.055399999022483826\n",
            "teacher 41\n",
            "test accuracy: 0.061900001019239426\n",
            "teacher 42\n",
            "test accuracy: 0.0681999996304512\n",
            "teacher 43\n",
            "test accuracy: 0.07360000163316727\n",
            "teacher 44\n",
            "test accuracy: 0.06239999830722809\n",
            "teacher 45\n",
            "test accuracy: 0.065700002014637\n",
            "teacher 46\n",
            "test accuracy: 0.06400000303983688\n",
            "teacher 47\n",
            "test accuracy: 0.06909999996423721\n",
            "teacher 48\n",
            "test accuracy: 0.07050000131130219\n",
            "teacher 49\n",
            "test accuracy: 0.06279999762773514\n",
            "teacher 50\n",
            "test accuracy: 0.046799998730421066\n",
            "teacher 51\n",
            "test accuracy: 0.06650000065565109\n",
            "teacher 52\n",
            "test accuracy: 0.0608999989926815\n",
            "teacher 53\n",
            "test accuracy: 0.06030000001192093\n",
            "teacher 54\n",
            "test accuracy: 0.06610000133514404\n",
            "teacher 55\n",
            "test accuracy: 0.0674000009894371\n",
            "teacher 56\n",
            "test accuracy: 0.06300000101327896\n",
            "teacher 57\n",
            "test accuracy: 0.0632999986410141\n",
            "teacher 58\n",
            "test accuracy: 0.06610000133514404\n",
            "teacher 59\n",
            "test accuracy: 0.06030000001192093\n",
            "teacher 60\n",
            "test accuracy: 0.06129999831318855\n",
            "teacher 61\n",
            "test accuracy: 0.06120000034570694\n",
            "teacher 62\n",
            "test accuracy: 0.05460000038146973\n",
            "teacher 63\n",
            "test accuracy: 0.05490000173449516\n",
            "teacher 64\n",
            "test accuracy: 0.07240000367164612\n",
            "teacher 65\n",
            "test accuracy: 0.06719999760389328\n",
            "teacher 66\n",
            "test accuracy: 0.05950000137090683\n",
            "teacher 67\n",
            "test accuracy: 0.07180000096559525\n",
            "teacher 68\n",
            "test accuracy: 0.05719999969005585\n",
            "teacher 69\n",
            "test accuracy: 0.0658000037074089\n",
            "teacher 70\n",
            "test accuracy: 0.07119999825954437\n",
            "teacher 71\n",
            "test accuracy: 0.06629999727010727\n",
            "teacher 72\n",
            "test accuracy: 0.0575999990105629\n",
            "teacher 73\n",
            "test accuracy: 0.052299998700618744\n",
            "teacher 74\n",
            "test accuracy: 0.058800000697374344\n",
            "teacher 75\n",
            "test accuracy: 0.052000001072883606\n",
            "teacher 76\n",
            "test accuracy: 0.05380000174045563\n",
            "teacher 77\n",
            "test accuracy: 0.06030000001192093\n",
            "teacher 78\n",
            "test accuracy: 0.06530000269412994\n",
            "teacher 79\n",
            "test accuracy: 0.058400001376867294\n",
            "teacher 80\n",
            "test accuracy: 0.06520000100135803\n",
            "teacher 81\n",
            "test accuracy: 0.06509999930858612\n",
            "teacher 82\n",
            "test accuracy: 0.05999999865889549\n",
            "teacher 83\n",
            "test accuracy: 0.05999999865889549\n",
            "teacher 84\n",
            "test accuracy: 0.05900000035762787\n",
            "teacher 85\n",
            "test accuracy: 0.062199998646974564\n",
            "teacher 86\n",
            "test accuracy: 0.04580000042915344\n",
            "teacher 87\n",
            "test accuracy: 0.057999998331069946\n",
            "teacher 88\n",
            "test accuracy: 0.05299999937415123\n",
            "teacher 89\n",
            "test accuracy: 0.056299999356269836\n",
            "teacher 90\n",
            "test accuracy: 0.05790000036358833\n",
            "teacher 91\n",
            "test accuracy: 0.05999999865889549\n",
            "teacher 92\n",
            "test accuracy: 0.05730000138282776\n",
            "teacher 93\n",
            "test accuracy: 0.06719999760389328\n",
            "teacher 94\n",
            "test accuracy: 0.06620000302791595\n",
            "teacher 95\n",
            "test accuracy: 0.07090000063180923\n",
            "teacher 96\n",
            "test accuracy: 0.06639999896287918\n",
            "teacher 97\n",
            "test accuracy: 0.05959999933838844\n",
            "teacher 98\n",
            "test accuracy: 0.053700000047683716\n",
            "teacher 99\n",
            "test accuracy: 0.06080000102519989\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "mGE8PGdWH0in"
      },
      "source": [
        "## training the student model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NUqUcPlzH0io"
      },
      "source": [
        "# label the data\n",
        "labels = np.array([ teacher.predict( student_data_x ) for teacher in teacher_models ])"
      ],
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "y-k9_DJ2H0io",
        "outputId": "8bf3322a-2ee0-4696-9777-bd61d3180578"
      },
      "source": [
        "labels.shape"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(100, 300, 100)"
            ]
          },
          "metadata": {},
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ttc0GfBWH0io",
        "outputId": "57be2cda-74fd-41d9-d737-307cae191cda"
      },
      "source": [
        "student_data_x.shape"
      ],
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(300, 32, 32, 3)"
            ]
          },
          "metadata": {},
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "peAY6FyzH0io",
        "outputId": "86565c1d-fde4-401c-a330-83ba48a23f3c"
      },
      "source": [
        "len(student_data_x)"
      ],
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "300"
            ]
          },
          "metadata": {},
          "execution_count": 13
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vxRdG4XtH0io"
      },
      "source": [
        "# preform the voting\n",
        "votes = np.zeros( ( student_data_x.shape[ 0 ], 100 ), dtype=np.float )\n",
        "for i in range( len( student_data_x ) ):\n",
        "  for j in range( n_teachers ):\n",
        "    label = np.argmax( labels[ j ][ i ] )\n",
        "    votes[ i, label ] += 1\n",
        "  # add the noise per class\n",
        "  for j in range( 10 ):\n",
        "    votes[ i, j ] += np.random.laplace(loc=0.0, scale=5 )"
      ],
      "execution_count": 14,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "4v9jqUYlH0io",
        "outputId": "24a2ced4-0277-4739-b188-9031edc5dd7c"
      },
      "source": [
        "student_data_x.shape,student_data_y.shape,votes.shape"
      ],
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "((300, 32, 32, 3), (300, 100), (300, 100))"
            ]
          },
          "metadata": {},
          "execution_count": 15
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "k6yJBWelH0io"
      },
      "source": [
        "ll = np.argmax( votes, axis=1 )"
      ],
      "execution_count": 16,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yUDW-vZxH0io"
      },
      "source": [
        "ll.shape"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "D3TcyainH0io",
        "outputId": "8fe53709-6873-4f8b-b9dd-dfb0458051e9"
      },
      "source": [
        "student_data_y.shape,keras.utils.np_utils.to_categorical( ll,num_classes=100 ).shape"
      ],
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "((300, 100), (300, 100))"
            ]
          },
          "metadata": {},
          "execution_count": 18
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_ML0W1koH0io",
        "outputId": "e9b2de12-57ba-40cd-dd64-9f4a648674be"
      },
      "source": [
        "student_data_y = keras.utils.np_utils.to_categorical( np.argmax( votes, axis=1 ),num_classes=100 )\n",
        "\n",
        "# train model\n",
        "student_model = get_model()\n",
        "print( 'training student model' )\n",
        "student_model.fit( x, y, epochs=16, verbose=0 )\n",
        "print( 'test accuracy:', student_model.evaluate( student_data_x, student_data_y, verbose=0 )[ 1 ] )"
      ],
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "training student model\n",
            "test accuracy: 0.06333333253860474\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "SJgKXeBNG8pG"
      },
      "source": [
        "# Every teacher has instances of every class:"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "CzGT1G5t5DPo"
      },
      "source": [
        "# 10 Teachers"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WG9_BdHz5DPo"
      },
      "source": [
        "n_instances = 500\n",
        "n_teachers = 10\n",
        "\n",
        "# load data and transform it\n",
        "(x_train, y_train), (x_test, y_test) = keras.datasets.cifar100.load_data()\n",
        "\n",
        "x_train = x_train.astype( float ) / 255.\n",
        "x_test = x_test.astype( float ) / 255.\n",
        "\n",
        "x_train = x_train.reshape( -1, 32, 32, 3)\n",
        "x_test = x_test.reshape( -1, 32, 32, 3)\n",
        "\n",
        "y_train = keras.utils.np_utils.to_categorical( y_train )\n",
        "y_test = keras.utils.np_utils.to_categorical( y_test )\n",
        "\n",
        "# shuffle data\n",
        "idx = np.arange( len( x_train ) )\n",
        "np.random.shuffle( idx )\n",
        "x_train = x_train[ idx ]\n",
        "y_train = y_train[ idx ]\n",
        "\n",
        "# gather the teacher data\n",
        "teacher_data_x = np.array([ x_train[ i * n_instances : ( i + 1 ) * n_instances ] for i in range( n_teachers ) ])\n",
        "teacher_data_y = np.array([ y_train[ i * n_instances : ( i + 1 ) * n_instances ] for i in range( n_teachers ) ])\n",
        "\n",
        "# gather the student data\n",
        "student_data_x = np.array(x_train[ n_teachers * n_instances : ( n_teachers + 1 ) * n_instances ] )\n",
        "student_data_y = np.array(y_train[ n_teachers * n_instances : ( n_teachers + 1 ) * n_instances ]  )"
      ],
      "execution_count": 69,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "6eoDfGwB5DPo",
        "outputId": "b3d7dded-83ea-4abf-9dc2-e74e1ba21e78"
      },
      "source": [
        "teacher_data_x.shape, teacher_data_y.shape"
      ],
      "execution_count": 70,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "((10, 500, 32, 32, 3), (10, 500, 100))"
            ]
          },
          "metadata": {},
          "execution_count": 70
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "cCksvAyDGQ2w",
        "outputId": "f781c4f2-4096-4f43-a05e-3b3b55d0357b"
      },
      "source": [
        "student_data_x.shape,student_data_y.shape"
      ],
      "execution_count": 71,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "((500, 32, 32, 3), (500, 100))"
            ]
          },
          "metadata": {},
          "execution_count": 71
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Xd2oGemh5DPo",
        "outputId": "37c9b924-04fd-4102-9562-92e43ae3911c"
      },
      "source": [
        "## train the teacher models\n",
        "# train the teacher models according to cifar100 dataset in tensorflow\n",
        "def get_model():\n",
        "  # Create the model\n",
        "  model = Sequential()\n",
        "  model.add(Conv2D(16, kernel_size=1, activation='relu'))\n",
        "  model.add(Conv2D(16, kernel_size=3, activation='relu'))\n",
        "  model.add(MaxPooling2D(pool_size=(2, 2)))\n",
        "  model.add(Conv2D(32, kernel_size=3, activation='relu'))\n",
        "  model.add(Conv2D(32, (3, 3), activation='relu'))\n",
        "  model.add(MaxPooling2D(pool_size=(2, 2)))\n",
        "  model.add(Flatten())\n",
        "  model.add(Dense(100, activation='softmax'))\n",
        "\n",
        "\n",
        "  model.compile( loss = 'categorical_crossentropy', optimizer = 'adam', metrics = [ 'accuracy' ],run_eagerly=True)\n",
        "\n",
        "  return model\n",
        "    \n",
        "\n",
        "# list of teacher models\n",
        "teacher_models = [ get_model() for i in range( n_teachers ) ]\n",
        "\n",
        "for i, (model, x, y) in enumerate( zip( teacher_models, teacher_data_x, teacher_data_y ) ):\n",
        "    print( 'teacher', i )\n",
        "    model.fit( x, y, batch_size = 32, epochs = 10, verbose = 0 )\n",
        "    print( 'test accuracy:', model.evaluate( x_test, y_test, verbose = 0 )[ 1 ] )\n",
        "    model.save( '/content/drive/MyDrive/SPAI_projectphase2/teacher_model_{}.h5'.format( i ) )"
      ],
      "execution_count": 72,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "teacher 0\n",
            "test accuracy: 0.06350000202655792\n",
            "teacher 1\n",
            "test accuracy: 0.07129999995231628\n",
            "teacher 2\n",
            "test accuracy: 0.06679999828338623\n",
            "teacher 3\n",
            "test accuracy: 0.058800000697374344\n",
            "teacher 4\n",
            "test accuracy: 0.07419999688863754\n",
            "teacher 5\n",
            "test accuracy: 0.06520000100135803\n",
            "teacher 6\n",
            "test accuracy: 0.06129999831318855\n",
            "teacher 7\n",
            "test accuracy: 0.054999999701976776\n",
            "teacher 8\n",
            "test accuracy: 0.04610000178217888\n",
            "teacher 9\n",
            "test accuracy: 0.05829999968409538\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_ZOnnzt95DPp"
      },
      "source": [
        "## training the student model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "O57oI-4mFo7o"
      },
      "source": [
        "tf.config.run_functions_eagerly(True)\n",
        "tf.data.experimental.enable_debug_mode()"
      ],
      "execution_count": 73,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uZw8Wpu7GJXw"
      },
      "source": [
        "# student_data_x"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Eabg_jWW5DPp"
      },
      "source": [
        "# label the data\n",
        "labels = np.array([ teacher.predict( student_data_x ) for teacher in teacher_models ])"
      ],
      "execution_count": 76,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Unzd-zf45DPp",
        "outputId": "1728af44-2b45-4aa1-879d-883de0fbf9fd"
      },
      "source": [
        "labels.shape"
      ],
      "execution_count": 77,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(10, 500, 100)"
            ]
          },
          "metadata": {},
          "execution_count": 77
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "LXJi190L5DPp",
        "outputId": "b9a8d97d-c5c4-4aa0-b4bf-97f2a720fea1"
      },
      "source": [
        "student_data_x.shape"
      ],
      "execution_count": 79,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(500, 32, 32, 3)"
            ]
          },
          "metadata": {},
          "execution_count": 79
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "dtV8cAup5DPp",
        "outputId": "0a9c73d0-1379-4d7c-918f-6c71d3ca969e"
      },
      "source": [
        "len(student_data_x)"
      ],
      "execution_count": 80,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "500"
            ]
          },
          "metadata": {},
          "execution_count": 80
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XIRHgbEr5DPp"
      },
      "source": [
        "# preform the voting\n",
        "votes = np.zeros( ( student_data_x.shape[ 0 ], 100 ), dtype=np.float )\n",
        "for i in range( len( student_data_x ) ):\n",
        "  for j in range( n_teachers ):\n",
        "    label = np.argmax( labels[ j ][ i ] )\n",
        "    votes[ i, label ] += 1\n",
        "  # add the noise per class\n",
        "  for j in range( 10 ):\n",
        "    votes[ i, j ] += np.random.laplace(loc=0.0, scale=5 )"
      ],
      "execution_count": 81,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "mBzxY-dE5DPp",
        "outputId": "4fc5c765-3c97-47d6-e39d-e24f4a4f0a89"
      },
      "source": [
        "student_data_x.shape,student_data_y.shape,votes.shape"
      ],
      "execution_count": 82,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "((500, 32, 32, 3), (500, 100), (500, 100))"
            ]
          },
          "metadata": {},
          "execution_count": 82
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Db33Wum_5DPp"
      },
      "source": [
        "ll = np.argmax( votes, axis=1 )"
      ],
      "execution_count": 83,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "SFhyWy7O5DPp",
        "outputId": "3e61522f-b970-4d0d-9872-94f811c7b23f"
      },
      "source": [
        "ll.shape"
      ],
      "execution_count": 84,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(500,)"
            ]
          },
          "metadata": {},
          "execution_count": 84
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "atnlUcDB5DPq",
        "outputId": "4331b8da-52f8-498b-c9ec-8f9126447357"
      },
      "source": [
        "student_data_y.shape,keras.utils.np_utils.to_categorical( ll,num_classes=100 ).shape"
      ],
      "execution_count": 85,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "((500, 100), (500, 100))"
            ]
          },
          "metadata": {},
          "execution_count": 85
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "0ylLCVSz5DPq",
        "outputId": "fe70f4fa-9ddd-4fd4-bc4a-845232850e29"
      },
      "source": [
        "student_data_y = keras.utils.np_utils.to_categorical( np.argmax( votes, axis=1 ),num_classes=100 )\n",
        "\n",
        "# train model\n",
        "student_model = get_model()\n",
        "print( 'training student model' )\n",
        "student_model.fit( x, y, epochs=16, verbose=0 )\n",
        "print( 'test accuracy:', student_model.evaluate( student_data_x, student_data_y, verbose=0 )[ 1 ] )"
      ],
      "execution_count": 86,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "training student model\n",
            "test accuracy: 0.009999999776482582\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4Wc1xMyb43Yu"
      },
      "source": [
        "# 20 Teachers"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FipNWUGL43Yv"
      },
      "source": [
        "n_instances = 500 \n",
        "n_teachers = 20\n",
        "\n",
        "# load data and transform it\n",
        "(x_train, y_train), (x_test, y_test) = keras.datasets.cifar100.load_data()\n",
        "\n",
        "x_train = x_train.astype( float ) / 255.\n",
        "x_test = x_test.astype( float ) / 255.\n",
        "\n",
        "x_train = x_train.reshape( -1, 32, 32, 3)\n",
        "x_test = x_test.reshape( -1, 32, 32, 3)\n",
        "\n",
        "y_train = keras.utils.np_utils.to_categorical( y_train )\n",
        "y_test = keras.utils.np_utils.to_categorical( y_test )\n",
        "\n",
        "# shuffle data\n",
        "idx = np.arange( len( x_train ) )\n",
        "np.random.shuffle( idx )\n",
        "x_train = x_train[ idx ]\n",
        "y_train = y_train[ idx ]\n",
        "\n",
        "# gather the teacher data\n",
        "teacher_data_x = np.array([ x_train[ i * n_instances : ( i + 1 ) * n_instances ] for i in range( n_teachers ) ])\n",
        "teacher_data_y = np.array([ y_train[ i * n_instances : ( i + 1 ) * n_instances ] for i in range( n_teachers ) ])\n",
        "\n",
        "# gather the student data\n",
        "student_data_x = np.array(x_train[ n_teachers * n_instances : ( n_teachers + 1 ) * n_instances ] )\n",
        "student_data_y = np.array(y_train[ n_teachers * n_instances : ( n_teachers + 1 ) * n_instances ]  )"
      ],
      "execution_count": 87,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "nbTbEfnO43Yv",
        "outputId": "64715472-88c2-4b0a-f281-94ee4a55d43e"
      },
      "source": [
        "teacher_data_x.shape, teacher_data_y.shape"
      ],
      "execution_count": 88,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "((20, 500, 32, 32, 3), (20, 500, 100))"
            ]
          },
          "metadata": {},
          "execution_count": 88
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "PQqCD55m43Yw",
        "outputId": "c1419381-aadf-4179-ab1a-a41ae6238e4a"
      },
      "source": [
        "## train the teacher models\n",
        "# train the teacher models according to cifar100 dataset in tensorflow\n",
        "def get_model():\n",
        "  # Create the model\n",
        "  model = Sequential()\n",
        "  model.add(Conv2D(16, kernel_size=1, activation='relu'))\n",
        "  model.add(Conv2D(16, kernel_size=3, activation='relu'))\n",
        "  model.add(MaxPooling2D(pool_size=(2, 2)))\n",
        "  model.add(Conv2D(32, kernel_size=3, activation='relu'))\n",
        "  model.add(Conv2D(32, (3, 3), activation='relu'))\n",
        "  model.add(MaxPooling2D(pool_size=(2, 2)))\n",
        "  model.add(Flatten())\n",
        "  model.add(Dense(100, activation='softmax'))\n",
        "\n",
        "\n",
        "  model.compile( loss = 'categorical_crossentropy', optimizer = 'adam', metrics = [ 'accuracy' ] )\n",
        "\n",
        "  return model\n",
        "    \n",
        "\n",
        "# list of teacher models\n",
        "teacher_models = [ get_model() for i in range( n_teachers ) ]\n",
        "\n",
        "for i, (model, x, y) in enumerate( zip( teacher_models, teacher_data_x, teacher_data_y ) ):\n",
        "    print( 'teacher', i )\n",
        "    model.fit( x, y, batch_size = 32, epochs = 10, verbose = 0 )\n",
        "    print( 'test accuracy:', model.evaluate( x_test, y_test, verbose = 0 )[ 1 ] )\n",
        "    model.save( '/content/drive/MyDrive/SPAI_projectphase2/teacher_model_{}.h5'.format( i ) )"
      ],
      "execution_count": 89,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "teacher 0\n",
            "test accuracy: 0.05220000073313713\n",
            "teacher 1\n",
            "test accuracy: 0.05829999968409538\n",
            "teacher 2\n",
            "test accuracy: 0.04540000110864639\n",
            "teacher 3\n",
            "test accuracy: 0.06939999759197235\n",
            "teacher 4\n",
            "test accuracy: 0.05609999969601631\n",
            "teacher 5\n",
            "test accuracy: 0.05550000071525574\n",
            "teacher 6\n",
            "test accuracy: 0.0714000016450882\n",
            "teacher 7\n",
            "test accuracy: 0.04969999939203262\n",
            "teacher 8\n",
            "test accuracy: 0.0568000003695488\n",
            "teacher 9\n",
            "test accuracy: 0.05460000038146973\n",
            "teacher 10\n",
            "test accuracy: 0.0723000019788742\n",
            "teacher 11\n",
            "test accuracy: 0.056299999356269836\n",
            "teacher 12\n",
            "test accuracy: 0.06080000102519989\n",
            "teacher 13\n",
            "test accuracy: 0.06800000369548798\n",
            "teacher 14\n",
            "test accuracy: 0.06159999966621399\n",
            "teacher 15\n",
            "test accuracy: 0.05249999836087227\n",
            "teacher 16\n",
            "test accuracy: 0.05810000002384186\n",
            "teacher 17\n",
            "test accuracy: 0.06629999727010727\n",
            "teacher 18\n",
            "test accuracy: 0.0568000003695488\n",
            "teacher 19\n",
            "test accuracy: 0.05429999902844429\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "oqKNuk5V43Yw"
      },
      "source": [
        "## training the student model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "N3w5g1xL43Yw"
      },
      "source": [
        "# label the data\n",
        "labels = np.array([ teacher.predict( student_data_x ) for teacher in teacher_models ])"
      ],
      "execution_count": 90,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "5doEIn4G43Yx",
        "outputId": "65dc2492-69b2-4797-b1b2-936fec6a5ff2"
      },
      "source": [
        "labels.shape"
      ],
      "execution_count": 91,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(20, 500, 100)"
            ]
          },
          "metadata": {},
          "execution_count": 91
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "TabnzdMR43Yx",
        "outputId": "b9bd5595-b5c0-4e00-a253-9e4e22ff1b76"
      },
      "source": [
        "student_data_x.shape"
      ],
      "execution_count": 92,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(500, 32, 32, 3)"
            ]
          },
          "metadata": {},
          "execution_count": 92
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "RzMZLJ5843Yx",
        "outputId": "e699a860-f407-41f8-bb13-5b0cb6eb0574"
      },
      "source": [
        "len(student_data_x)"
      ],
      "execution_count": 93,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "500"
            ]
          },
          "metadata": {},
          "execution_count": 93
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HtVLWD8443Yx"
      },
      "source": [
        "# preform the voting\n",
        "votes = np.zeros( ( student_data_x.shape[ 0 ], 100 ), dtype=np.float )\n",
        "for i in range( len( student_data_x ) ):\n",
        "  for j in range( n_teachers ):\n",
        "    label = np.argmax( labels[ j ][ i ] )\n",
        "    votes[ i, label ] += 1\n",
        "  # add the noise per class\n",
        "  for j in range( 10 ):\n",
        "    votes[ i, j ] += np.random.laplace(loc=0.0, scale=5 )"
      ],
      "execution_count": 94,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "kfgkN9MN43Yx",
        "outputId": "8d88fd53-e656-4687-a01f-aaa943300340"
      },
      "source": [
        "student_data_x.shape,student_data_y.shape,votes.shape"
      ],
      "execution_count": 95,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "((500, 32, 32, 3), (500, 100), (500, 100))"
            ]
          },
          "metadata": {},
          "execution_count": 95
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hUE10KFe43Yy"
      },
      "source": [
        "ll = np.argmax( votes, axis=1 )"
      ],
      "execution_count": 96,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "CPemZOx743Yy",
        "outputId": "1ebf6379-2ee0-4b2f-a4a3-a671b25e7628"
      },
      "source": [
        "ll.shape"
      ],
      "execution_count": 97,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(500,)"
            ]
          },
          "metadata": {},
          "execution_count": 97
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "9PtLYG6P43Yy",
        "outputId": "b5a72333-a98d-4b83-e2cd-cfbaf68fcb02"
      },
      "source": [
        "student_data_y.shape,keras.utils.np_utils.to_categorical( ll,num_classes=100 ).shape"
      ],
      "execution_count": 98,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "((500, 100), (500, 100))"
            ]
          },
          "metadata": {},
          "execution_count": 98
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-4PhC71p43Yy",
        "outputId": "661630cd-9eea-4f8a-c4c2-b80e0364cd6a"
      },
      "source": [
        "student_data_y = keras.utils.np_utils.to_categorical( np.argmax( votes, axis=1 ),num_classes=100 )\n",
        "\n",
        "# train model\n",
        "student_model = get_model()\n",
        "print( 'training student model' )\n",
        "student_model.fit( x, y, epochs=16, verbose=0 )\n",
        "print( 'test accuracy:', student_model.evaluate( student_data_x, student_data_y, verbose=0 )[ 1 ] )"
      ],
      "execution_count": 99,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "training student model\n",
            "test accuracy: 0.014000000432133675\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "D7K_ArB640ab"
      },
      "source": [
        "# 50 Teachers"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bkMDKX7H40ab"
      },
      "source": [
        "n_instances = 500 \n",
        "n_teachers = 50\n",
        "\n",
        "# load data and transform it\n",
        "(x_train, y_train), (x_test, y_test) = keras.datasets.cifar100.load_data()\n",
        "\n",
        "x_train = x_train.astype( float ) / 255.\n",
        "x_test = x_test.astype( float ) / 255.\n",
        "\n",
        "x_train = x_train.reshape( -1, 32, 32, 3)\n",
        "x_test = x_test.reshape( -1, 32, 32, 3)\n",
        "\n",
        "y_train = keras.utils.np_utils.to_categorical( y_train )\n",
        "y_test = keras.utils.np_utils.to_categorical( y_test )\n",
        "\n",
        "# shuffle data\n",
        "idx = np.arange( len( x_train ) )\n",
        "np.random.shuffle( idx )\n",
        "x_train = x_train[ idx ]\n",
        "y_train = y_train[ idx ]\n",
        "\n",
        "# gather the teacher data\n",
        "teacher_data_x = np.array([ x_train[ i * n_instances : ( i + 1 ) * n_instances ] for i in range( n_teachers ) ])\n",
        "teacher_data_y = np.array([ y_train[ i * n_instances : ( i + 1 ) * n_instances ] for i in range( n_teachers ) ])\n",
        "\n",
        "# gather the student data\n",
        "student_data_x = np.array(x_train[ n_teachers * n_instances : ( n_teachers + 1 ) * n_instances ] )\n",
        "student_data_y = np.array(y_train[ n_teachers * n_instances : ( n_teachers + 1 ) * n_instances ]  )"
      ],
      "execution_count": 20,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "CyJ9mpVD40ab",
        "outputId": "189a6878-76c3-45cf-9eaf-9c6790c2a524"
      },
      "source": [
        "teacher_data_x.shape, teacher_data_y.shape"
      ],
      "execution_count": 21,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "((50, 500, 32, 32, 3), (50, 500, 100))"
            ]
          },
          "metadata": {},
          "execution_count": 21
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "gprLnvrr40ab",
        "outputId": "c7bc3d29-3cd0-4205-e57f-6e1fd6ce8544"
      },
      "source": [
        "## train the teacher models\n",
        "# train the teacher models according to cifar100 dataset in tensorflow\n",
        "def get_model():\n",
        "  # Create the model\n",
        "  model = Sequential()\n",
        "  model.add(Conv2D(16, kernel_size=1, activation='relu'))\n",
        "  model.add(Conv2D(16, kernel_size=3, activation='relu'))\n",
        "  model.add(MaxPooling2D(pool_size=(2, 2)))\n",
        "  model.add(Conv2D(32, kernel_size=3, activation='relu'))\n",
        "  model.add(Conv2D(32, (3, 3), activation='relu'))\n",
        "  model.add(MaxPooling2D(pool_size=(2, 2)))\n",
        "  model.add(Flatten())\n",
        "  model.add(Dense(100, activation='softmax'))\n",
        "\n",
        "\n",
        "  model.compile( loss = 'categorical_crossentropy', optimizer = 'adam', metrics = [ 'accuracy' ] )\n",
        "\n",
        "  return model\n",
        "    \n",
        "\n",
        "# list of teacher models\n",
        "teacher_models = [ get_model() for i in range( n_teachers ) ]\n",
        "\n",
        "for i, (model, x, y) in enumerate( zip( teacher_models, teacher_data_x, teacher_data_y ) ):\n",
        "    print( 'teacher', i )\n",
        "    model.fit( x, y, batch_size = 32, epochs = 10, verbose = 0 )\n",
        "    print( 'test accuracy:', model.evaluate( x_test, y_test, verbose = 0 )[ 1 ] )\n",
        "    model.save( '/content/drive/MyDrive/SPAI_projectphase2/teacher_model_{}.h5'.format( i ) )"
      ],
      "execution_count": 22,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "teacher 0\n",
            "test accuracy: 0.06769999861717224\n",
            "teacher 1\n",
            "test accuracy: 0.06300000101327896\n",
            "teacher 2\n",
            "test accuracy: 0.05620000138878822\n",
            "teacher 3\n",
            "test accuracy: 0.06360000371932983\n",
            "teacher 4\n",
            "test accuracy: 0.05920000001788139\n",
            "teacher 5\n",
            "test accuracy: 0.06689999997615814\n",
            "teacher 6\n",
            "test accuracy: 0.052799999713897705\n",
            "teacher 7\n",
            "test accuracy: 0.0649000033736229\n",
            "teacher 8\n",
            "test accuracy: 0.056299999356269836\n",
            "teacher 9\n",
            "test accuracy: 0.05640000104904175\n",
            "teacher 10\n",
            "test accuracy: 0.06790000200271606\n",
            "teacher 11\n",
            "test accuracy: 0.0568000003695488\n",
            "teacher 12\n",
            "test accuracy: 0.0502999983727932\n",
            "teacher 13\n",
            "test accuracy: 0.06499999761581421\n",
            "teacher 14\n",
            "test accuracy: 0.06750000268220901\n",
            "teacher 15\n",
            "test accuracy: 0.06719999760389328\n",
            "teacher 16\n",
            "test accuracy: 0.048500001430511475\n",
            "teacher 17\n",
            "test accuracy: 0.05640000104904175\n",
            "teacher 18\n",
            "test accuracy: 0.053700000047683716\n",
            "teacher 19\n",
            "test accuracy: 0.06120000034570694\n",
            "teacher 20\n",
            "test accuracy: 0.06400000303983688\n",
            "teacher 21\n",
            "test accuracy: 0.06040000170469284\n",
            "teacher 22\n",
            "test accuracy: 0.06499999761581421\n",
            "teacher 23\n",
            "test accuracy: 0.05249999836087227\n",
            "teacher 24\n",
            "test accuracy: 0.04170000180602074\n",
            "teacher 25\n",
            "test accuracy: 0.05339999869465828\n",
            "teacher 26\n",
            "test accuracy: 0.06199999898672104\n",
            "teacher 27\n",
            "test accuracy: 0.0575999990105629\n",
            "teacher 28\n",
            "test accuracy: 0.06260000169277191\n",
            "teacher 29\n",
            "test accuracy: 0.056299999356269836\n",
            "teacher 30\n",
            "test accuracy: 0.05900000035762787\n",
            "teacher 31\n",
            "test accuracy: 0.05429999902844429\n",
            "teacher 32\n",
            "test accuracy: 0.06040000170469284\n",
            "teacher 33\n",
            "test accuracy: 0.05920000001788139\n",
            "teacher 34\n",
            "test accuracy: 0.05770000070333481\n",
            "teacher 35\n",
            "test accuracy: 0.0560000017285347\n",
            "teacher 36\n",
            "test accuracy: 0.062300000339746475\n",
            "teacher 37\n",
            "test accuracy: 0.060600001364946365\n",
            "teacher 38\n",
            "test accuracy: 0.05299999937415123\n",
            "teacher 39\n",
            "test accuracy: 0.06469999998807907\n",
            "teacher 40\n",
            "test accuracy: 0.07039999961853027\n",
            "teacher 41\n",
            "test accuracy: 0.066600002348423\n",
            "teacher 42\n",
            "test accuracy: 0.06019999831914902\n",
            "teacher 43\n",
            "test accuracy: 0.05660000070929527\n",
            "teacher 44\n",
            "test accuracy: 0.06310000270605087\n",
            "teacher 45\n",
            "test accuracy: 0.06400000303983688\n",
            "teacher 46\n",
            "test accuracy: 0.06970000267028809\n",
            "teacher 47\n",
            "test accuracy: 0.05689999833703041\n",
            "teacher 48\n",
            "test accuracy: 0.05829999968409538\n",
            "teacher 49\n",
            "test accuracy: 0.06210000067949295\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "if7-uptK40ac"
      },
      "source": [
        "## training the student model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iiPLJocM40ac"
      },
      "source": [
        "# label the data\n",
        "labels = np.array([ teacher.predict( student_data_x ) for teacher in teacher_models ])"
      ],
      "execution_count": 23,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ooeWoXYt40ac",
        "outputId": "c974b137-60e7-492f-c05b-493e93c20da9"
      },
      "source": [
        "labels.shape"
      ],
      "execution_count": 24,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(50, 500, 100)"
            ]
          },
          "metadata": {},
          "execution_count": 24
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "P8qqcGGf40ac",
        "outputId": "cadf0c45-38d9-4c64-8840-221ccf23c6f9"
      },
      "source": [
        "student_data_x.shape"
      ],
      "execution_count": 25,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(500, 32, 32, 3)"
            ]
          },
          "metadata": {},
          "execution_count": 25
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "sSEKJCF140ac",
        "outputId": "ef46c835-efb1-44cf-ac97-784700fcb712"
      },
      "source": [
        "len(student_data_x)"
      ],
      "execution_count": 26,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "500"
            ]
          },
          "metadata": {},
          "execution_count": 26
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gTzmAwJc40ac"
      },
      "source": [
        "# preform the voting\n",
        "votes = np.zeros( ( student_data_x.shape[ 0 ], 100 ), dtype=np.float )\n",
        "for i in range( len( student_data_x ) ):\n",
        "  for j in range( n_teachers ):\n",
        "    label = np.argmax( labels[ j ][ i ] )\n",
        "    votes[ i, label ] += 1\n",
        "  # add the noise per class\n",
        "  for j in range( 10 ):\n",
        "    votes[ i, j ] += np.random.laplace(loc=0.0, scale=5 )"
      ],
      "execution_count": 27,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "X1UrCXbh40ad",
        "outputId": "5d968a23-cbaf-41a0-bf93-3ae7ff361009"
      },
      "source": [
        "student_data_x.shape,student_data_y.shape,votes.shape"
      ],
      "execution_count": 28,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "((500, 32, 32, 3), (500, 100), (500, 100))"
            ]
          },
          "metadata": {},
          "execution_count": 28
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vtzhQNrF40ad"
      },
      "source": [
        "ll = np.argmax( votes, axis=1 )"
      ],
      "execution_count": 29,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "FGvmzjGc40ad",
        "outputId": "a5ce9e5a-e882-44ad-964d-801f024f754c"
      },
      "source": [
        "ll.shape"
      ],
      "execution_count": 30,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(500,)"
            ]
          },
          "metadata": {},
          "execution_count": 30
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "BC1PqYAA40ad",
        "outputId": "34e5ad08-b7aa-4381-b2df-834f70402100"
      },
      "source": [
        "student_data_y.shape,keras.utils.np_utils.to_categorical( ll,num_classes=100 ).shape"
      ],
      "execution_count": 31,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "((500, 100), (500, 100))"
            ]
          },
          "metadata": {},
          "execution_count": 31
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "btEyY46840ad",
        "outputId": "b41d08ec-289d-42b3-a64d-6cf20024fa41"
      },
      "source": [
        "student_data_y = keras.utils.np_utils.to_categorical( np.argmax( votes, axis=1 ),num_classes=100 )\n",
        "\n",
        "# train model\n",
        "student_model = get_model()\n",
        "print( 'training student model' )\n",
        "student_model.fit( x, y, epochs=16, verbose=0 )\n",
        "print( 'test accuracy:', student_model.evaluate( student_data_x, student_data_y, verbose=0 )[ 1 ] )"
      ],
      "execution_count": 32,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "training student model\n",
            "test accuracy: 0.057999998331069946\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "TfEtllDd3XWn"
      },
      "source": [
        "# 100 Teachers"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vygfiD4JGqPD"
      },
      "source": [
        "n_instances = 300 \n",
        "n_teachers = 100\n",
        "\n",
        "# load data and transform it\n",
        "(x_train, y_train), (x_test, y_test) = keras.datasets.cifar100.load_data()\n",
        "\n",
        "x_train = x_train.astype( float ) / 255.\n",
        "x_test = x_test.astype( float ) / 255.\n",
        "\n",
        "x_train = x_train.reshape( -1, 32, 32, 3)\n",
        "x_test = x_test.reshape( -1, 32, 32, 3)\n",
        "\n",
        "y_train = keras.utils.np_utils.to_categorical( y_train )\n",
        "y_test = keras.utils.np_utils.to_categorical( y_test )\n",
        "\n",
        "# shuffle data\n",
        "idx = np.arange( len( x_train ) )\n",
        "np.random.shuffle( idx )\n",
        "x_train = x_train[ idx ]\n",
        "y_train = y_train[ idx ]\n",
        "\n",
        "# gather the teacher data\n",
        "teacher_data_x = np.array([ x_train[ i * n_instances : ( i + 1 ) * n_instances ] for i in range( n_teachers ) ])\n",
        "teacher_data_y = np.array([ y_train[ i * n_instances : ( i + 1 ) * n_instances ] for i in range( n_teachers ) ])\n",
        "\n",
        "# gather the student data\n",
        "student_data_x = np.array(x_train[ n_teachers * n_instances : ( n_teachers + 1 ) * n_instances ] )\n",
        "student_data_y = np.array(y_train[ n_teachers * n_instances : ( n_teachers + 1 ) * n_instances ]  )"
      ],
      "execution_count": 39,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "zEzAuVdpvJII",
        "outputId": "8e4accc1-f54e-4806-fd9e-9f4e2cd7f636"
      },
      "source": [
        "teacher_data_x.shape, teacher_data_y.shape"
      ],
      "execution_count": 40,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "((100, 300, 32, 32, 3), (100, 300, 100))"
            ]
          },
          "metadata": {},
          "execution_count": 40
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Eu3-eciWIvTh",
        "outputId": "67ab8899-97e9-47c2-97a1-803fabcff9b3"
      },
      "source": [
        "## train the teacher models\n",
        "# train the teacher models according to cifar100 dataset in tensorflow\n",
        "def get_model():\n",
        "  # Create the model\n",
        "  model = Sequential()\n",
        "  model.add(Conv2D(16, kernel_size=1, activation='relu'))\n",
        "  model.add(Conv2D(16, kernel_size=3, activation='relu'))\n",
        "  model.add(MaxPooling2D(pool_size=(2, 2)))\n",
        "  model.add(Conv2D(32, kernel_size=3, activation='relu'))\n",
        "  model.add(Conv2D(32, (3, 3), activation='relu'))\n",
        "  model.add(MaxPooling2D(pool_size=(2, 2)))\n",
        "  model.add(Flatten())\n",
        "  model.add(Dense(100, activation='softmax'))\n",
        "\n",
        "\n",
        "  model.compile( loss = 'categorical_crossentropy', optimizer = 'adam', metrics = [ 'accuracy' ] )\n",
        "\n",
        "  return model\n",
        "    \n",
        "\n",
        "# list of teacher models\n",
        "teacher_models = [ get_model() for i in range( n_teachers ) ]\n",
        "\n",
        "for i, (model, x, y) in enumerate( zip( teacher_models, teacher_data_x, teacher_data_y ) ):\n",
        "    print( 'teacher', i )\n",
        "    model.fit( x, y, batch_size = 32, epochs = 10, verbose = 0 )\n",
        "    print( 'test accuracy:', model.evaluate( x_test, y_test, verbose = 0 )[ 1 ] )\n",
        "    model.save( '/content/drive/MyDrive/SPAI_projectphase2/teacher_model_{}.h5'.format( i ) )"
      ],
      "execution_count": 41,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "teacher 0\n",
            "test accuracy: 0.039400000125169754\n",
            "teacher 1\n",
            "test accuracy: 0.03999999910593033\n",
            "teacher 2\n",
            "test accuracy: 0.0333000011742115\n",
            "teacher 3\n",
            "test accuracy: 0.05220000073313713\n",
            "teacher 4\n",
            "test accuracy: 0.0502999983727932\n",
            "teacher 5\n",
            "test accuracy: 0.04800000041723251\n",
            "teacher 6\n",
            "test accuracy: 0.04390000179409981\n",
            "teacher 7\n",
            "test accuracy: 0.048900000751018524\n",
            "teacher 8\n",
            "test accuracy: 0.05000000074505806\n",
            "teacher 9\n",
            "test accuracy: 0.04190000146627426\n",
            "teacher 10\n",
            "test accuracy: 0.03849999979138374\n",
            "teacher 11\n",
            "test accuracy: 0.04659999907016754\n",
            "teacher 12\n",
            "test accuracy: 0.04470000043511391\n",
            "teacher 13\n",
            "test accuracy: 0.048700001090765\n",
            "teacher 14\n",
            "test accuracy: 0.045899998396635056\n",
            "teacher 15\n",
            "test accuracy: 0.050999999046325684\n",
            "teacher 16\n",
            "test accuracy: 0.05820000171661377\n",
            "teacher 17\n",
            "test accuracy: 0.03759999945759773\n",
            "teacher 18\n",
            "test accuracy: 0.04780000075697899\n",
            "teacher 19\n",
            "test accuracy: 0.03689999878406525\n",
            "teacher 20\n",
            "test accuracy: 0.05000000074505806\n",
            "teacher 21\n",
            "test accuracy: 0.041600000113248825\n",
            "teacher 22\n",
            "test accuracy: 0.05389999970793724\n",
            "teacher 23\n",
            "test accuracy: 0.051500000059604645\n",
            "teacher 24\n",
            "test accuracy: 0.04699999839067459\n",
            "teacher 25\n",
            "test accuracy: 0.04490000009536743\n",
            "teacher 26\n",
            "test accuracy: 0.046799998730421066\n",
            "teacher 27\n",
            "test accuracy: 0.030799999833106995\n",
            "teacher 28\n",
            "test accuracy: 0.034299999475479126\n",
            "teacher 29\n",
            "test accuracy: 0.047600001096725464\n",
            "teacher 30\n",
            "test accuracy: 0.050700001418590546\n",
            "teacher 31\n",
            "test accuracy: 0.042399998754262924\n",
            "teacher 32\n",
            "test accuracy: 0.043699998408555984\n",
            "teacher 33\n",
            "test accuracy: 0.04529999941587448\n",
            "teacher 34\n",
            "test accuracy: 0.03739999979734421\n",
            "teacher 35\n",
            "test accuracy: 0.05249999836087227\n",
            "teacher 36\n",
            "test accuracy: 0.039000000804662704\n",
            "teacher 37\n",
            "test accuracy: 0.043699998408555984\n",
            "teacher 38\n",
            "test accuracy: 0.04520000144839287\n",
            "teacher 39\n",
            "test accuracy: 0.05429999902844429\n",
            "teacher 40\n",
            "test accuracy: 0.05810000002384186\n",
            "teacher 41\n",
            "test accuracy: 0.04179999977350235\n",
            "teacher 42\n",
            "test accuracy: 0.047200001776218414\n",
            "teacher 43\n",
            "test accuracy: 0.04619999974966049\n",
            "teacher 44\n",
            "test accuracy: 0.04179999977350235\n",
            "teacher 45\n",
            "test accuracy: 0.04479999840259552\n",
            "teacher 46\n",
            "test accuracy: 0.04580000042915344\n",
            "teacher 47\n",
            "test accuracy: 0.05380000174045563\n",
            "teacher 48\n",
            "test accuracy: 0.032600000500679016\n",
            "teacher 49\n",
            "test accuracy: 0.0421999990940094\n",
            "teacher 50\n",
            "test accuracy: 0.038600001484155655\n",
            "teacher 51\n",
            "test accuracy: 0.05040000006556511\n",
            "teacher 52\n",
            "test accuracy: 0.04430000111460686\n",
            "teacher 53\n",
            "test accuracy: 0.06030000001192093\n",
            "teacher 54\n",
            "test accuracy: 0.05849999934434891\n",
            "teacher 55\n",
            "test accuracy: 0.04529999941587448\n",
            "teacher 56\n",
            "test accuracy: 0.042399998754262924\n",
            "teacher 57\n",
            "test accuracy: 0.042100001126527786\n",
            "teacher 58\n",
            "test accuracy: 0.03889999911189079\n",
            "teacher 59\n",
            "test accuracy: 0.04780000075697899\n",
            "teacher 60\n",
            "test accuracy: 0.04360000044107437\n",
            "teacher 61\n",
            "test accuracy: 0.04839999973773956\n",
            "teacher 62\n",
            "test accuracy: 0.04600000008940697\n",
            "teacher 63\n",
            "test accuracy: 0.04149999842047691\n",
            "teacher 64\n",
            "test accuracy: 0.05260000005364418\n",
            "teacher 65\n",
            "test accuracy: 0.04149999842047691\n",
            "teacher 66\n",
            "test accuracy: 0.04610000178217888\n",
            "teacher 67\n",
            "test accuracy: 0.039799999445676804\n",
            "teacher 68\n",
            "test accuracy: 0.044599998742341995\n",
            "teacher 69\n",
            "test accuracy: 0.05139999836683273\n",
            "teacher 70\n",
            "test accuracy: 0.04349999874830246\n",
            "teacher 71\n",
            "test accuracy: 0.03840000182390213\n",
            "teacher 72\n",
            "test accuracy: 0.04450000077486038\n",
            "teacher 73\n",
            "test accuracy: 0.04349999874830246\n",
            "teacher 74\n",
            "test accuracy: 0.046300001442432404\n",
            "teacher 75\n",
            "test accuracy: 0.05339999869465828\n",
            "teacher 76\n",
            "test accuracy: 0.042899999767541885\n",
            "teacher 77\n",
            "test accuracy: 0.04780000075697899\n",
            "teacher 78\n",
            "test accuracy: 0.04019999876618385\n",
            "teacher 79\n",
            "test accuracy: 0.053599998354911804\n",
            "teacher 80\n",
            "test accuracy: 0.039400000125169754\n",
            "teacher 81\n",
            "test accuracy: 0.051600001752376556\n",
            "teacher 82\n",
            "test accuracy: 0.038600001484155655\n",
            "teacher 83\n",
            "test accuracy: 0.04650000110268593\n",
            "teacher 84\n",
            "test accuracy: 0.049400001764297485\n",
            "teacher 85\n",
            "test accuracy: 0.04659999907016754\n",
            "teacher 86\n",
            "test accuracy: 0.04839999973773956\n",
            "teacher 87\n",
            "test accuracy: 0.051899999380111694\n",
            "teacher 88\n",
            "test accuracy: 0.04670000076293945\n",
            "teacher 89\n",
            "test accuracy: 0.04050000011920929\n",
            "teacher 90\n",
            "test accuracy: 0.029600000008940697\n",
            "teacher 91\n",
            "test accuracy: 0.039000000804662704\n",
            "teacher 92\n",
            "test accuracy: 0.03999999910593033\n",
            "teacher 93\n",
            "test accuracy: 0.039900001138448715\n",
            "teacher 94\n",
            "test accuracy: 0.04659999907016754\n",
            "teacher 95\n",
            "test accuracy: 0.04430000111460686\n",
            "teacher 96\n",
            "test accuracy: 0.041600000113248825\n",
            "teacher 97\n",
            "test accuracy: 0.041600000113248825\n",
            "teacher 98\n",
            "test accuracy: 0.039400000125169754\n",
            "teacher 99\n",
            "test accuracy: 0.05079999938607216\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "pMytb6U1Qq9X"
      },
      "source": [
        "## training the student model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wqwj9kCiQqbA"
      },
      "source": [
        "# label the data\n",
        "labels = np.array([ teacher.predict( student_data_x ) for teacher in teacher_models ])"
      ],
      "execution_count": 42,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "JmKghEptubn1",
        "outputId": "a2c16442-1299-4fc4-c9fb-809146a91a2d"
      },
      "source": [
        "labels.shape"
      ],
      "execution_count": 43,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(100, 300, 100)"
            ]
          },
          "metadata": {},
          "execution_count": 43
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "vS5eA4GbwkUB",
        "outputId": "2cac2825-6128-4560-83f9-643766019e82"
      },
      "source": [
        "student_data_x.shape"
      ],
      "execution_count": 44,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(300, 32, 32, 3)"
            ]
          },
          "metadata": {},
          "execution_count": 44
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "BVXqgTEAwoGo",
        "outputId": "60dee2b5-499c-494d-dd60-eb9393b03470"
      },
      "source": [
        "len(student_data_x)"
      ],
      "execution_count": 45,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "300"
            ]
          },
          "metadata": {},
          "execution_count": 45
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "B46Gf18fQy3e"
      },
      "source": [
        "# preform the voting\n",
        "votes = np.zeros( ( student_data_x.shape[ 0 ], 100 ), dtype=np.float )\n",
        "for i in range( len( student_data_x ) ):\n",
        "  for j in range( n_teachers ):\n",
        "    label = np.argmax( labels[ j ][ i ] )\n",
        "    votes[ i, label ] += 1\n",
        "  # add the noise per class\n",
        "  for j in range( 10 ):\n",
        "    votes[ i, j ] += np.random.laplace(loc=0.0, scale=5 )"
      ],
      "execution_count": 46,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "9T3bhdpIn44l",
        "outputId": "3eddd201-3dd4-46d6-88bc-a7c76892d1df"
      },
      "source": [
        "student_data_x.shape,student_data_y.shape,votes.shape"
      ],
      "execution_count": 47,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "((300, 32, 32, 3), (300, 100), (300, 100))"
            ]
          },
          "metadata": {},
          "execution_count": 47
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "x3uW4tdDxxm_"
      },
      "source": [
        "ll = np.argmax( votes, axis=1 )"
      ],
      "execution_count": 48,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "V-VWkZy_x_2u",
        "outputId": "ffc3e1c6-d273-4084-a4f8-4fb9fa31b074"
      },
      "source": [
        "ll.shape"
      ],
      "execution_count": 49,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(300,)"
            ]
          },
          "metadata": {},
          "execution_count": 49
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "3D3_GU6OrcAi",
        "outputId": "b6a605b5-a1ee-4302-b90b-dd345396cce2"
      },
      "source": [
        "student_data_y.shape,keras.utils.np_utils.to_categorical( ll,num_classes=100 ).shape"
      ],
      "execution_count": 50,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "((300, 100), (300, 100))"
            ]
          },
          "metadata": {},
          "execution_count": 50
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Qf3LsX80UWi9",
        "outputId": "03a95b8c-020b-4749-a9d8-c6e6039d01dd"
      },
      "source": [
        "student_data_y = keras.utils.np_utils.to_categorical( np.argmax( votes, axis=1 ),num_classes=100 )\n",
        "\n",
        "# train model\n",
        "student_model = get_model()\n",
        "print( 'training student model' )\n",
        "student_model.fit( x, y, epochs=16, verbose=0 )\n",
        "print( 'test accuracy:', student_model.evaluate( student_data_x, student_data_y, verbose=0 )[ 1 ] )"
      ],
      "execution_count": 51,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "training student model\n",
            "test accuracy: 0.05000000074505806\n"
          ]
        }
      ]
    }
  ]
}