reward.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Allow reload of objects\n",
    "%load_ext autoreload\n",
    "%autoreload\n",
    "from pelenet.utils import Utils\n",
    "from pelenet.experiments.reward import RewardExperiment\n",
    "\n",
    "# Official modules\n",
    "import numpy as np\n",
    "import nxsdk.api.n2a as nx\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "from time import time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Initilizes the experiment and utils\n",
    "utils = Utils.instance()\n",
    "exp = RewardExperiment()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Build network\n",
    "exp.build()\n",
    "\n",
    "# Get spectral radius\n",
    "#utils.getSpectralRadius(exp.net.initialWeights)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Prepare some variables\n",
    "#nxNet = exp.net.nxNet\n",
    "\n",
    "nExCores = int(np.ceil(exp.net.p.reservoirExSize / exp.net.p.neuronsPerCore))\n",
    "nInCores = int(np.ceil(exp.net.p.reservoirInSize / exp.net.p.neuronsPerCore))\n",
    "nAllCores = nExCores + nInCores\n",
    "\n",
    "# Create compartment prototypes\n",
    "compProto = nx.CompartmentPrototype(compartmentVoltageDecay=exp.net.p.compartmentVoltageDecay,\n",
    "                                    refractoryDelay=exp.net.p.refractoryDelay, logicalCoreId=1)#nAllCores+1)\n",
    "\n",
    "compProtoLrn = nx.CompartmentPrototype(compartmentVoltageDecay=exp.net.p.compartmentVoltageDecay,\n",
    "                                       refractoryDelay=exp.net.p.refractoryDelay, logicalCoreId=1,#nAllCores+1,\n",
    "                                       enableSpikeBackprop=1, enableSpikeBackpropFromSelf=1)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "Difference neuron\n",
    "\"\"\"\n",
    "\n",
    "diffCompPlus = exp.net.nxNet.createCompartmentGroup(size=1, prototype=compProto)\n",
    "#diffCompPlus = exp.net.nxNet.createCompartment(prototype=compProto)\n",
    "diffCompMinus = exp.net.nxNet.createCompartmentGroup(size=1, prototype=compProto)\n",
    "#diffCompMinus = exp.net.nxNet.createCompartment(prototype=compProto)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "Output neuron\n",
    "\"\"\"\n",
    "\n",
    "#outComp = exp.net.nxNet.createCompartmentGroup(size=1, prototype=compProtoLrn)\n",
    "outComp = exp.net.nxNet.createCompartment(prototype=compProtoLrn)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "Input generator\n",
    "\"\"\"\n",
    "\n",
    "inputSg = exp.net.nxNet.createSpikeGenProcess(numPorts=1)\n",
    "\n",
    "# Generate spikes\n",
    "rnd = np.random.rand(exp.net.p.totalSteps)\n",
    "thresh = (exp.net.p.totalSteps-np.arange(exp.net.p.totalSteps))/exp.net.p.totalSteps\n",
    "spike = (rnd < thresh)\n",
    "\n",
    "# Index of every second spike\n",
    "idxSecSpike = np.where(spike)[0][::2]\n",
    "\n",
    "# Add spikes to generator\n",
    "inputSg.addSpikes(spikeInputPortNodeIds=0, spikeTimes=list(idxSecSpike))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<nxsdk.net.nodes.connections.ConnectionCC at 0x7f54adc5bef0>"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "exConnProto = nx.ConnectionPrototype(weight=255, numTagBits=8, signMode=nx.SYNAPSE_SIGN_MODE.EXCITATORY)\n",
    "inConnProto = nx.ConnectionPrototype(weight=255, numTagBits=8, signMode=nx.SYNAPSE_SIGN_MODE.INHIBITORY)\n",
    "\n",
    "\"\"\"\n",
    "Connect input, output, difference\n",
    "\"\"\"\n",
    "# Connect input to difference neurons\n",
    "inputSg.connect(diffCompPlus, connectionMask=np.array([1]), weight=np.array([255]), prototype=exConnProto)\n",
    "#inputSg.connect(diffCompPlus, prototype=exConnProto)\n",
    "inputSg.connect(diffCompMinus, connectionMask=np.array([1]), weight=-np.array([255]), prototype=inConnProto)\n",
    "#inputSg.connect(diffCompMinus, prototype=inConnProto)\n",
    "\n",
    "# Connect output to difference neurons\n",
    "#outComp.connect(diffCompPlus, connectionMask=np.array([1]), weight=np.array([255]), prototype=exConnProto)\n",
    "outComp.connect(diffCompPlus, prototype=exConnProto)\n",
    "#outComp.connect(diffCompMinus, connectionMask=np.array([1]), weight=-np.array([255]), prototype=inConnProto)\n",
    "outComp.connect(diffCompMinus, prototype=inConnProto)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "ename": "AttributeError",
     "evalue": "'Compartment' object has no attribute 'addSrcConnectionGroup'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-9-8abf36c020d6>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     19\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mexp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnet\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexReservoirChunks\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     20\u001b[0m     \u001b[0mfr\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mto\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mi\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0mexp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnet\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mneuronsPerCore\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0mexp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnet\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mneuronsPerCore\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 21\u001b[0;31m     \u001b[0mexp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnet\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexReservoirChunks\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconnect\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moutComp\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mconnectionMask\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mma\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mfr\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mto\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mweight\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mwe\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mfr\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mto\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mprototype\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mconnLrnProto\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     22\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     23\u001b[0m \u001b[0;31m# Connect difference neurons to reward channel\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/share/virtualenvs/seq-r0_WvjRo/lib/python3.5/site-packages/nxsdk/net/groups.py\u001b[0m in \u001b[0;36mconnect\u001b[0;34m(self, dstGrp, prototype, prototypeMap, connectionMask, name, sharedConnGrp, synapseSharingOnly, **kwargs)\u001b[0m\n\u001b[1;32m    421\u001b[0m                                                   \u001b[0mprototype\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mprototypeMap\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    422\u001b[0m                                                   \u001b[0mconnectionMask\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 423\u001b[0;31m                                                   **kwargs)\n\u001b[0m\u001b[1;32m    424\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    425\u001b[0m             sharedConnGrp.shareConnections(self, dstGrp,\n",
      "\u001b[0;32m~/.local/share/virtualenvs/seq-r0_WvjRo/lib/python3.5/site-packages/nxsdk/net/net.py\u001b[0m in \u001b[0;36mcreateConnectionGroup\u001b[0;34m(self, name, src, dst, prototype, prototypeMap, connectionMask, **kwargs)\u001b[0m\n\u001b[1;32m    898\u001b[0m                 \u001b[0mprototypeMap\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mprototypeMap\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    899\u001b[0m                 \u001b[0mconnectionMask\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mconnectionMask\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 900\u001b[0;31m                 **kwargs)\n\u001b[0m\u001b[1;32m    901\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    902\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mg\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/share/virtualenvs/seq-r0_WvjRo/lib/python3.5/site-packages/nxsdk/net/net.py\u001b[0m in \u001b[0;36m_buildConnectionGroup\u001b[0;34m(self, src, dst, nodeType, name, prototype, prototypeMap, connectionMask, **kwargs)\u001b[0m\n\u001b[1;32m   1092\u001b[0m             \u001b[0mg\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0maddSrcDstGroups\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msrc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdst\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1093\u001b[0m             \u001b[0msrc\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0maddDstConnectionGroup\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mg\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1094\u001b[0;31m             \u001b[0mdst\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0maddSrcConnectionGroup\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mg\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1095\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1096\u001b[0m             \u001b[0;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Unknown group type.\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mAttributeError\u001b[0m: 'Compartment' object has no attribute 'addSrcConnectionGroup'"
     ]
    }
   ],
   "source": [
    "\"\"\"\n",
    "Connect reservoir to output and add reward\n",
    "\"\"\"\n",
    "\n",
    "# Create learning rule and connection prototype for learning\n",
    "lr = exp.net.nxNet.createLearningRule(dt='x1*y0-x0*y1-2^-2*u0*t', dw='2*u0*r1*t',\n",
    "                                   x1Impulse=40, x1TimeConstant=4,\n",
    "                                   y1Impulse=40, y1TimeConstant=4,\n",
    "                                   r1Impulse=20, r1TimeConstant=4,\n",
    "                                   tEpoch=2)\n",
    "connLrnProto = nx.ConnectionPrototype(enableLearning=1, learningRule=lr, numTagBits=8,\n",
    "                                      signMode=nx.SYNAPSE_SIGN_MODE.MIXED)\n",
    "\n",
    "# Create mask and weights\n",
    "ma = np.ones((1, exp.net.p.reservoirExSize)).astype(int)\n",
    "we = np.random.normal(1, 5, exp.net.p.reservoirExSize).astype(int)\n",
    "\n",
    "# Connect reservoir to output neuron\n",
    "for i in range(len(exp.net.exReservoirChunks)):\n",
    "    fr, to = i*exp.net.p.neuronsPerCore, (i+1)*exp.net.p.neuronsPerCore\n",
    "    exp.net.exReservoirChunks[i].connect(outComp, connectionMask=ma[:,fr:to], weight=we[fr:to], prototype=connLrnProto)\n",
    "\n",
    "# Connect difference neurons to reward channel\n",
    "#inputSg.connect(lr.reinforcementChannel)\n",
    "#diffCompPlus.connect(lr.reinforcementChannel)\n",
    "diffCompPlus.connect(lr.reinforcementChannel, prototype=connLrnProto)\n",
    "#diffCompMinus.connect(lr.reinforcementChannel)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "diffCompPlus"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "lr.reinforcementChannel"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#dir(lr.reinforcementChannel)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "Probes\n",
    "\"\"\"\n",
    "\n",
    "exp.net.outSpikeProbe = outComp.probe([nx.ProbeParameter.SPIKE])[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure(figsize=(16,5))\n",
    "plt.eventplot(idxSecSpike)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "exp.run()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure(figsize=(16,5))\n",
    "exp.net.outSpikeProbe.plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.sum(exp.net.outSpikeProbe.data)/1000"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "exp.net.plot.reservoirSpikeTrain()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "exp.net.plot.reservoirRates()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}