JDE-v2.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "e53f33d3-8a7d-402d-bfc0-566a905faa6f",
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import numpy as np\n",
    "import pickle\n",
    "#from torchsummary import summary\n",
    "from collections import OrderedDict\n",
    "from torch.utils.tensorboard import SummaryWriter\n",
    "import datetime\n",
    "import time\n",
    "import copy\n",
    "import torchvision.datasets as datasets\n",
    "mnist_trainset = datasets.MNIST(root='./data', train=True, download=True, transform=None)\n",
    "mnist_testset = datasets.MNIST(root='./data', train=False, download=True, transform=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "b7cb97ff-7acb-44ef-b9d6-979b457b79da",
   "metadata": {},
   "outputs": [],
   "source": [
    "class DNN(torch.nn.Module):\n",
    "    def __init__(self, layers):\n",
    "        super(DNN, self).__init__()\n",
    "        # depth of the neural network and activation function\n",
    "        self.depth = len(layers)-1\n",
    "        self.activation = torch.nn.ReLU\n",
    "        # layers: (depth-1)x 'linear' + 'activation' + last one 'linear'\n",
    "        layer_list = list()\n",
    "        for i in range(self.depth-1): \n",
    "            layer_list.append(\n",
    "                ('layer_%d' % i, torch.nn.Linear(layers[i], layers[i+1]))\n",
    "            )\n",
    "            layer_list.append(('activation_%d' % i, self.activation())) \n",
    "        layer_list.append(\n",
    "            ('layer_%d' % (self.depth - 1), torch.nn.Linear(layers[-2], layers[-1]))\n",
    "        )\n",
    "        layerDict = OrderedDict(layer_list)\n",
    "        self.layers = torch.nn.Sequential(layerDict)\n",
    "        \n",
    "    def forward(self, x):\n",
    "        out = self.layers(x)\n",
    "        return out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "99960ef9-e149-4b11-8df1-c52f218fab7d",
   "metadata": {},
   "outputs": [],
   "source": [
    "## Asssume the dim of Traing and Testing are in shape [N,C,H,W]\n",
    "\n",
    "class DE_MLP():\n",
    "    def __init__(self, outdim=1,maxdepth=70,mindepth=5,minneuron=4,maxneuron=10,bsize=10,epoch=100,initSize=20,maxiter=10,stopcount=3,\\\n",
    "                 trainingset=None,validationset=None,trainingTarget=None,validateTarget=None,crossover=1):\n",
    "        self.best=[]\n",
    "        self.mean=[]\n",
    "        self.outdim=outdim\n",
    "        self.maxdepth=maxdepth\n",
    "        self.mindepth=mindepth\n",
    "        self.minneuron = minneuron\n",
    "        self.maxneuron = maxneuron\n",
    "        self.bsize = bsize\n",
    "        self.epoch = epoch\n",
    "        self.stopcount = stopcount\n",
    "        self.pplSize = initSize\n",
    "        self.maxiter = maxiter\n",
    "        self.training = trainingset.reshape((trainingset.shape[0],-1))\n",
    "        self.validationSet = validationset.reshape((validationset.shape[0],-1))\n",
    "        self.target=trainingTarget\n",
    "        self.validationTarget = validateTarget\n",
    "        self.MLPlayerlist = []\n",
    "        self.depthlist=np.random.choice(range(self.mindepth,self.maxdepth),self.pplSize,replace=True)\n",
    "        self.crossover=crossover\n",
    "        self.adap_conf = (0.1,0.1,0.1,0.9)\n",
    "        self.tb = SummaryWriter('./Statistic/'+datetime.datetime.now().strftime(\"%Y%m%d-%H%M%S\"))\n",
    "        \n",
    "        if torch.cuda.is_available():\n",
    "            self.device = torch.device('cuda')\n",
    "        else:\n",
    "            self.device = torch.device('cpu')\n",
    "\n",
    "        # Generate initial population\n",
    "        for i in range(self.pplSize):\n",
    "            depth = self.depthlist[i]\n",
    "            tmp = []\n",
    "            # the number of neurons for the first layer is the dimension of the element in training data (in our case the size of the image)\n",
    "            tmp.append(self.training.shape[1])\n",
    "            for j in range(depth):\n",
    "                # generate the number of neurons for each layer\n",
    "                tmp.append(np.random.choice(range(self.minneuron,self.maxneuron),1,replace=False)[0])\n",
    "            tmp.append(self.outdim) # last layer consist of 1 neuron by default\n",
    "            tmp=np.array(tmp)\n",
    "            self.MLPlayerlist.append(tmp)\n",
    "    \n",
    "    # define fit function - it calculates the fitness of one individual (one NN) \n",
    "    def fit(self,config,id_,p=None):\n",
    "        dnn = DNN(config) # define DNN based on configurations (layers and neurons)\n",
    "        dnn.layers.to(self.device)\n",
    "        best = float('inf')\n",
    "        stop=0\n",
    "        opt = torch.optim.Adam(dnn.layers.parameters(), lr=0.001)\n",
    "        loss = torch.nn.BCEWithLogitsLoss()\n",
    "        batch = self.training.shape[0]//self.bsize\n",
    "        vbatch = self.validationSet.shape[0]//self.bsize\n",
    "        idxs = [x for x in range(self.training.shape[0])]\n",
    "        vidxs = [x for x in range(self.validationSet.shape[0])]\n",
    "        for e in range(self.epoch):\n",
    "            start=time.time()\n",
    "            # training\n",
    "            np.random.shuffle(idxs)\n",
    "            dnn.layers.train()\n",
    "            batchloss=0\n",
    "            for i in range(batch):\n",
    "                idx=idxs[i*self.bsize:i*self.bsize+self.bsize]\n",
    "                opt.zero_grad()\n",
    "                data = torch.tensor(self.training[idx]).float().to(self.device)\n",
    "                y = torch.tensor(self.target[idx]).float().to(self.device)\n",
    "                yhat = dnn(data)\n",
    "                l = loss(yhat,y)\n",
    "                batchloss+=l.item()\n",
    "                l.backward()\n",
    "                opt.step()\n",
    "            # validating\n",
    "            dnn.layers.eval()\n",
    "            np.random.shuffle(vidxs)\n",
    "            vloss=0\n",
    "            for i in range(vbatch):\n",
    "                vidx=vidxs[i*self.bsize:i*self.bsize+self.bsize]\n",
    "                vdata = torch.tensor(self.validationSet[vidx]).float().to(self.device)\n",
    "                vy = torch.tensor(self.validationTarget[vidx]).float().to(self.device)\n",
    "                vyhat = dnn(vdata)\n",
    "                vl = loss(vyhat,vy)\n",
    "                vloss += vl.item()\n",
    "            vloss = vloss/vbatch\n",
    "            # updating best loss\n",
    "            if(vloss<best): best=vloss\n",
    "            # updating stopping condition\n",
    "            else: stop+=1\n",
    "            end=time.time()\n",
    "            if(p is not None):\n",
    "                print(f'Process: {p:3d}, ConfigID: {id_:3d}, Epoch: {e:3d}, Training Loss: {(batchloss/batch):10.8f}, Validation Loss: {(vloss):10.8f},\\\n",
    "                    Best: {best:10.8f}, StopCount/Limit: {stop:3d}/{self.stopcount:3d}, Time:{(end-start):10.8f}')\n",
    "            else:\n",
    "                print(f'ConfigID: {id_:3d}, Epoch: {e:3d}, Training Loss: {(batchloss/batch):10.8f}, Validation Loss: {(vloss):10.8f},\\\n",
    "                    Best: {best:10.8f}, StopCount/Limit: {stop:3d}/{self.stopcount:3d}, Time:{(end-start):10.8f}')\n",
    "            # stopping condition and stopping\n",
    "            if(stop>=self.stopcount):\n",
    "                return best,config,id_ \n",
    "\n",
    "    def jde_params(self,beta,cr):\n",
    "        tau1,tau2,beta1,betau = self.adap_conf\n",
    "        r1,r2,r3,r4 = np.random.uniform(0,1,4)\n",
    "        if(r2 < tau1): beta = round(beta1 + r1 * betau,3) # else, keep the beta same\n",
    "        if(r4 < tau2): cr = r3\n",
    "        return beta,cr\n",
    "    \n",
    "    def mutation_rand_1_z(self,x1,xs,beta,debug=False):\n",
    "        indim = x1[0]\n",
    "        x1 = x1[1:-1] # remove in/out dim\n",
    "        xs[0] = xs[0][1:-1]\n",
    "        xs[1] = xs[1][1:-1]\n",
    "        if(debug):\n",
    "            print(f'M1 : x1 len {x1.shape[0]} xs0 len {xs[0].shape[0]} xs1 len {xs[1].shape[0]}')\n",
    "            print(f'M1 : x1 {x1} \\nM1 : xs0 {xs[0]} \\nM1 : xs1 len {xs[1]}')\n",
    "        #\n",
    "        # A. Mutating the # of layers\n",
    "        minlen = np.min([x1.shape[0],xs[0].shape[0],xs[1].shape[0]])\n",
    "        if(debug): print(f'M1 : minlen {minlen}')          \n",
    "        newminlen = minlen\n",
    "        targetlen=int(np.floor((x1.shape[0]) + beta * (xs[0].shape[0] - xs[1].shape[0])))\n",
    "        # check the sign of targetlen: if the new length == 0 , set it back to target len , if <0 , take abs\n",
    "        if(targetlen==0): targetlen=x1.shape[0]\n",
    "        elif(targetlen<0): targetlen=abs(targetlen)\n",
    "        # check if new length is between mindepth and maxdepth\n",
    "        if(targetlen < self.mindepth): targetlen = self.mindepth\n",
    "        elif(targetlen > self.maxdepth): targetlen = self.maxdepth\n",
    "        # new minimum length is min of minlen and targetlen\n",
    "        if(targetlen < minlen): newminlen=targetlen\n",
    "        if(debug): print(f'M1 : New Min Len :{newminlen}, Length Mutation :{targetlen}')\n",
    "        #    \n",
    "        # B. Mutating the # of neurons\n",
    "        # As lengths of x1, xs[0], xs[1] and new length can possibly be different, \n",
    "        # 1) do the mutation for # of neurons for new minlen, \n",
    "        # 2) apply the same rule to remaining if needed\n",
    "        # xa = np.zeros((targetlen),dtype=int)\n",
    "        # Mutating the number of neurons up to min len layers\n",
    "        xa = x1[:newminlen] + beta * (xs[0][:newminlen] - xs[1][:newminlen]) # mutate on node with minlen\n",
    "        # Mutating the number of neurons for the rest layers\n",
    "        if(targetlen>minlen):\n",
    "            xaa = np.zeros((targetlen-minlen))\n",
    "            a,b,c=None,None,None\n",
    "            for i in range(targetlen-newminlen): # if number of neurons missing in vector, generate random from range (min)\n",
    "                if(x1.shape[0]<=newminlen+i): a=np.random.choice(range(self.minneuron,self.maxneuron),1,replace=False)[0]\n",
    "                elif(x1.shape[0]>newminlen+i): a=x1[newminlen+i] \n",
    "                if(xs[0].shape[0]<=newminlen+i): b=np.random.choice(range(self.minneuron,self.maxneuron),1,replace=False)[0]\n",
    "                elif(xs[0].shape[0]>newminlen+i): b=xs[0][newminlen+i]  \n",
    "                if(xs[1].shape[0]<=newminlen+i): c=np.random.choice(range(self.minneuron,self.maxneuron),1,replace=False)[0]\n",
    "                elif(xs[1].shape[0]>newminlen+i): c=xs[1][newminlen+i]      \n",
    "                xaa[i]=a + beta * (b - c)\n",
    "            xa = np.concatenate((xa, xaa), axis=None)\n",
    "        # check if numbers of neurons are in allowed range  \n",
    "        for i in range(xa.shape[0]):\n",
    "            if(xa[i]>self.maxneuron): xa[i]=self.maxneuron\n",
    "            elif(xa[i]<self.minneuron): xa[i]=self.minneuron\n",
    "            xa[i] = np.floor(xa[i])\n",
    "        xa = np.concatenate((np.array(indim,dtype=int),np.array(xa,dtype=int),np.array(self.outdim,dtype=int)), axis=None,dtype=int)\n",
    "        return xa\n",
    "    \n",
    "    def crossoverMean(self,parent,u):\n",
    "        order = [parent[1:-1],u[1:-1]]\n",
    "        if(parent.shape[0] > u.shape[0]): order = [u[1:-1],parent[1:-1]]\n",
    "        order[0] = np.resize(order[0],order[1].shape[0])\n",
    "        middle = np.mean(order,axis=0,dtype=int)\n",
    "        child=np.insert(middle,0,parent[0])\n",
    "        child=np.append(child,parent[-1])\n",
    "        return child.copy()\n",
    "    \n",
    "    def crossoverRandomSwap(self,parent,u):\n",
    "        # the first one is with min len\n",
    "        order = [parent[1:-1],u[1:-1]]\n",
    "        child = [parent[0]]\n",
    "        if(parent.shape[0] > u.shape[0]): order = [u[1:-1],parent[1:-1]]\n",
    "        order[0] = np.resize(order[0],order[1].shape[0])\n",
    "        swap = np.random.randint(0,2,order[0].shape[0])\n",
    "        for i in range(len(swap)):\n",
    "            if(swap[i]==0): child.append(order[0][i])\n",
    "            else: child.append(order[1][i])\n",
    "        child.append(parent[-1])\n",
    "        return np.array(child).copy()\n",
    "    \n",
    "    def crossoverUnif(self,parent,unit,cr):\n",
    "        r = np.random.uniform(0,1,1)[0]\n",
    "        p = parent[1:-1]\n",
    "        u = unit[1:-1]\n",
    "        child = [parent[0]]\n",
    "        if(p.shape[0] > u.shape[0]): u = np.resize(u,p.shape[0])\n",
    "        elif(u.shape[0] > p.shape[0]): p = np.resize(p,u.shape[0])\n",
    "        jr = np.random.choice(range(u.shape[0]),size=1)[0]\n",
    "        for j in range(u.shape[0]):\n",
    "            if r <= cr or j == jr: child.append(u[j])\n",
    "            else: child.append(p[j])\n",
    "        child.append(parent[-1])\n",
    "        return np.array(child)\n",
    "    \n",
    "    def run(self,beta=0.5,cr=0.9):\n",
    "        current_gen=self.MLPlayerlist\n",
    "        scores = np.zeros((self.pplSize))\n",
    "        #initial Run\n",
    "        print('Initial Run Start')\n",
    "        for i in range(len(self.MLPlayerlist)):\n",
    "            b,_,_ = self.fit(self.MLPlayerlist[i],i)\n",
    "            scores[i]=b\n",
    "        print('Initial Run End')\n",
    "        currentbest = np.min(scores)\n",
    "        currentmean = np.mean(scores)\n",
    "        currentbestidx = np.argmin(scores)\n",
    "        print(f'Init Run Best: {currentbest}, Mean: {currentmean}, ID:{currentbestidx}, config: {current_gen[currentbestidx]}')\n",
    "        #Generation Run\n",
    "        for i in range(self.maxiter):\n",
    "            structureStatistic=np.zeros((self.pplSize,5))\n",
    "            updatecount=0\n",
    "            start=time.time()\n",
    "            print(f'Gen {i} Run Start')\n",
    "            betas = np.ones(self.pplSize)*beta\n",
    "            crs = np.ones(self.pplSize)*cr\n",
    "            for j in range(self.pplSize):\n",
    "                parent = current_gen[j]\n",
    "                # factors\n",
    "                betas[j],crs[j] = self.jde_params(betas[j],crs[j])\n",
    "                # mutation\n",
    "                idx0,idx1,idxt = np.random.choice(range(0,self.pplSize),3,replace=False)\n",
    "                target = current_gen[idxt]\n",
    "                diff = [current_gen[idx0],current_gen[idx1]]\n",
    "                unitvector = self.mutation_rand_1_z(target,diff,betas[j])\n",
    "                # crossover\n",
    "                nextGen = self.crossoverUnif(parent,unitvector,crs[j])\n",
    "                print(f'Next Gen: {nextGen}')\n",
    "                structureStatistic[j,0]= nextGen.shape[0]-2\n",
    "                structureStatistic[j,1]= np.mean(nextGen[1:-1])\n",
    "                structureStatistic[j,2]= np.median(nextGen[1:-1])\n",
    "                structureStatistic[j,3]= np.quantile(nextGen[1:-1],0.25)\n",
    "                structureStatistic[j,4]= np.quantile(nextGen[1:-1],0.75)\n",
    "                # selection\n",
    "                s,_,_ = self.fit(nextGen,j)\n",
    "                if(s<scores[j]):\n",
    "                    updatecount+=1\n",
    "                    scores[j]=s\n",
    "                    current_gen[j]=nextGen\n",
    "            print(f'Gen {i} Run End')\n",
    "            end=time.time()\n",
    "            currentbest = np.min(scores)\n",
    "            currentmean = np.mean(scores)\n",
    "            currentmedian = np.median(scores)\n",
    "            currentq25 = np.quantile(scores,0.25)\n",
    "            currentq75 = np.quantile(scores,0.75)\n",
    "            currentbestidx = np.argmin(scores)\n",
    "            genMeanLen = np.mean(structureStatistic[:,0])\n",
    "            genMedianLen = np.median(structureStatistic[:,0])\n",
    "            genq25Len = np.quantile(structureStatistic[:,0],0.25)\n",
    "            genq75Len = np.quantile(structureStatistic[:,0],0.75)\n",
    "            genMeanNode=np.median(structureStatistic[:,1])\n",
    "            genMedianNode=np.median(structureStatistic[:,2])\n",
    "            genq25Node = np.median(structureStatistic[:,3])\n",
    "            genq75Node = np.median(structureStatistic[:,4])\n",
    "            print(f'Run {i:3d} Best: {currentbest}, Mean: {currentmean}, ID:{currentbestidx}, config: {current_gen[currentbestidx]}, updatecount: {updatecount:3d}, Generation RunTime: {(end-start):10.8f}')\n",
    "            self.tb.add_scalars(\"Scores Statistic (Generation)\", {'best':currentbest,'mean':currentmean,'median':currentmedian,'q25':currentq25,'q75':currentq75}, i)\n",
    "            self.tb.add_scalars(\"Structure Statistic (Generation) #HiddenLayer\", {'mean':genMeanLen,'median':genMedianLen,'q25':genq25Len,'q75':genq75Len}, i)\n",
    "            self.tb.add_scalars(\"Structure Statistic (Generation) #Node\", {'mean':genMeanNode,'median':genMedianNode,'q25':genq25Node,'q75':genq75Node}, i)\n",
    "            self.tb.add_scalar('Update Count',updatecount,i)\n",
    "            self.tb.add_scalar('RunTime',(end-start),i)\n",
    "        print(f'Run Completed : Best Score(loss): {np.min(scores)} , Config: {current_gen[np.argmin(scores)]}')    \n",
    "        return \n",
    "                "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "a332b483-a48e-46f8-bc94-b4f36144eb2f",
   "metadata": {},
   "outputs": [],
   "source": [
    "trainingt = torch.nn.functional.one_hot(mnist_trainset.targets,num_classes=10)\n",
    "validationt = torch.nn.functional.one_hot(mnist_testset.targets,num_classes=10)\n",
    "# (self, outdim=1,maxdepth=70,mindepth=5,minneuron=4,maxneuron=10,bsize=10,epoch=100,initSize=20,maxiter=10,stopcount=3,\n",
    "#   trainingset=None,validationset=None,trainingTarget=None,validateTarget=None)\n",
    "d = DE_MLP(outdim=10,maxdepth=10,initSize=3,trainingset=mnist_trainset.data, validationset=mnist_testset.data, trainingTarget=trainingt,validateTarget=validationt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "7b02b92e-097f-4204-9639-c8a195e64ce3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Initial Run Start\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/var/folders/hk/kbn1q_sj4jz4lh0m7x05vff80000gn/T/ipykernel_4211/3137338923.py:67: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  data = torch.tensor(self.training[idx]).float().to(self.device)\n",
      "/var/folders/hk/kbn1q_sj4jz4lh0m7x05vff80000gn/T/ipykernel_4211/3137338923.py:68: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  y = torch.tensor(self.target[idx]).float().to(self.device)\n",
      "/var/folders/hk/kbn1q_sj4jz4lh0m7x05vff80000gn/T/ipykernel_4211/3137338923.py:80: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  vdata = torch.tensor(self.validationSet[vidx]).float().to(self.device)\n",
      "/var/folders/hk/kbn1q_sj4jz4lh0m7x05vff80000gn/T/ipykernel_4211/3137338923.py:81: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  vy = torch.tensor(self.validationTarget[vidx]).float().to(self.device)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ConfigID:   0, Epoch:   0, Training Loss: 0.28295244, Validation Loss: 0.23718860,                    Best: 0.23718860, StopCount/Limit:   0/  3, Time:18.97487426\n",
      "ConfigID:   0, Epoch:   1, Training Loss: 0.22950007, Validation Loss: 0.22033542,                    Best: 0.22033542, StopCount/Limit:   0/  3, Time:12.37673497\n",
      "ConfigID:   0, Epoch:   2, Training Loss: 0.21896783, Validation Loss: 0.21416655,                    Best: 0.21416655, StopCount/Limit:   0/  3, Time:13.69720292\n",
      "ConfigID:   0, Epoch:   3, Training Loss: 0.21391426, Validation Loss: 0.21637192,                    Best: 0.21416655, StopCount/Limit:   1/  3, Time:18.22707582\n",
      "ConfigID:   0, Epoch:   4, Training Loss: 0.21080502, Validation Loss: 0.21014287,                    Best: 0.21014287, StopCount/Limit:   1/  3, Time:16.91261721\n",
      "ConfigID:   0, Epoch:   5, Training Loss: 0.20872166, Validation Loss: 0.20735238,                    Best: 0.20735238, StopCount/Limit:   1/  3, Time:12.52605081\n",
      "ConfigID:   0, Epoch:   6, Training Loss: 0.20692449, Validation Loss: 0.20795651,                    Best: 0.20735238, StopCount/Limit:   2/  3, Time:16.66685176\n",
      "ConfigID:   0, Epoch:   7, Training Loss: 0.20642406, Validation Loss: 0.21899443,                    Best: 0.20735238, StopCount/Limit:   3/  3, Time:19.61515999\n",
      "ConfigID:   1, Epoch:   0, Training Loss: 0.32366439, Validation Loss: 0.27932380,                    Best: 0.27932380, StopCount/Limit:   0/  3, Time:20.60889101\n",
      "ConfigID:   1, Epoch:   1, Training Loss: 0.24519236, Validation Loss: 0.22304911,                    Best: 0.22304911, StopCount/Limit:   0/  3, Time:18.18474603\n",
      "ConfigID:   1, Epoch:   2, Training Loss: 0.21501640, Validation Loss: 0.20421561,                    Best: 0.20421561, StopCount/Limit:   0/  3, Time:11.09411693\n",
      "ConfigID:   1, Epoch:   3, Training Loss: 0.19598608, Validation Loss: 0.18850258,                    Best: 0.18850258, StopCount/Limit:   0/  3, Time:8.08731699\n",
      "ConfigID:   1, Epoch:   4, Training Loss: 0.18675335, Validation Loss: 0.18192868,                    Best: 0.18192868, StopCount/Limit:   0/  3, Time:7.88620019\n",
      "ConfigID:   1, Epoch:   5, Training Loss: 0.17135266, Validation Loss: 0.16232014,                    Best: 0.16232014, StopCount/Limit:   0/  3, Time:8.80261993\n",
      "ConfigID:   1, Epoch:   6, Training Loss: 0.16166699, Validation Loss: 0.15932384,                    Best: 0.15932384, StopCount/Limit:   0/  3, Time:12.65174079\n",
      "ConfigID:   1, Epoch:   7, Training Loss: 0.15534798, Validation Loss: 0.15075164,                    Best: 0.15075164, StopCount/Limit:   0/  3, Time:10.55759311\n",
      "ConfigID:   1, Epoch:   8, Training Loss: 0.15115144, Validation Loss: 0.14823086,                    Best: 0.14823086, StopCount/Limit:   0/  3, Time:17.22707701\n",
      "ConfigID:   1, Epoch:   9, Training Loss: 0.14630845, Validation Loss: 0.14512674,                    Best: 0.14512674, StopCount/Limit:   0/  3, Time:14.76342392\n",
      "ConfigID:   1, Epoch:  10, Training Loss: 0.13743843, Validation Loss: 0.13176271,                    Best: 0.13176271, StopCount/Limit:   0/  3, Time:10.85243225\n",
      "ConfigID:   1, Epoch:  11, Training Loss: 0.12819525, Validation Loss: 0.12426737,                    Best: 0.12426737, StopCount/Limit:   0/  3, Time:11.19941115\n",
      "ConfigID:   1, Epoch:  12, Training Loss: 0.12291345, Validation Loss: 0.12776133,                    Best: 0.12426737, StopCount/Limit:   1/  3, Time:12.54391694\n",
      "ConfigID:   1, Epoch:  13, Training Loss: 0.11854106, Validation Loss: 0.12238646,                    Best: 0.12238646, StopCount/Limit:   1/  3, Time:9.58505392\n",
      "ConfigID:   1, Epoch:  14, Training Loss: 0.11654989, Validation Loss: 0.11835375,                    Best: 0.11835375, StopCount/Limit:   1/  3, Time:12.37783504\n",
      "ConfigID:   1, Epoch:  15, Training Loss: 0.11493322, Validation Loss: 0.12003854,                    Best: 0.11835375, StopCount/Limit:   2/  3, Time:11.25835896\n",
      "ConfigID:   1, Epoch:  16, Training Loss: 0.11400428, Validation Loss: 0.11356059,                    Best: 0.11356059, StopCount/Limit:   2/  3, Time:11.50455999\n",
      "ConfigID:   1, Epoch:  17, Training Loss: 0.11323358, Validation Loss: 0.11545541,                    Best: 0.11356059, StopCount/Limit:   3/  3, Time:10.19786882\n",
      "ConfigID:   2, Epoch:   0, Training Loss: 0.24777698, Validation Loss: 0.18225380,                    Best: 0.18225380, StopCount/Limit:   0/  3, Time:13.29918003\n",
      "ConfigID:   2, Epoch:   1, Training Loss: 0.17237773, Validation Loss: 0.17142959,                    Best: 0.17142959, StopCount/Limit:   0/  3, Time:15.36614513\n",
      "ConfigID:   2, Epoch:   2, Training Loss: 0.16302523, Validation Loss: 0.16329437,                    Best: 0.16329437, StopCount/Limit:   0/  3, Time:19.18211603\n",
      "ConfigID:   2, Epoch:   3, Training Loss: 0.15967423, Validation Loss: 0.16038277,                    Best: 0.16038277, StopCount/Limit:   0/  3, Time:12.63294220\n",
      "ConfigID:   2, Epoch:   4, Training Loss: 0.15717534, Validation Loss: 0.16025279,                    Best: 0.16025279, StopCount/Limit:   0/  3, Time:13.12544107\n",
      "ConfigID:   2, Epoch:   5, Training Loss: 0.15508416, Validation Loss: 0.15926201,                    Best: 0.15926201, StopCount/Limit:   0/  3, Time:14.12941384\n",
      "ConfigID:   2, Epoch:   6, Training Loss: 0.15412933, Validation Loss: 0.15873805,                    Best: 0.15873805, StopCount/Limit:   0/  3, Time:15.10536194\n",
      "ConfigID:   2, Epoch:   7, Training Loss: 0.15327603, Validation Loss: 0.15645795,                    Best: 0.15645795, StopCount/Limit:   0/  3, Time:14.69961023\n",
      "ConfigID:   2, Epoch:   8, Training Loss: 0.15306343, Validation Loss: 0.15755685,                    Best: 0.15645795, StopCount/Limit:   1/  3, Time:16.10904121\n",
      "ConfigID:   2, Epoch:   9, Training Loss: 0.15368918, Validation Loss: 0.15502740,                    Best: 0.15502740, StopCount/Limit:   1/  3, Time:16.02749324\n",
      "ConfigID:   2, Epoch:  10, Training Loss: 0.15278510, Validation Loss: 0.15513815,                    Best: 0.15502740, StopCount/Limit:   2/  3, Time:15.08167720\n",
      "ConfigID:   2, Epoch:  11, Training Loss: 0.15208109, Validation Loss: 0.15399027,                    Best: 0.15399027, StopCount/Limit:   2/  3, Time:14.55724788\n",
      "ConfigID:   2, Epoch:  12, Training Loss: 0.15167111, Validation Loss: 0.15654477,                    Best: 0.15399027, StopCount/Limit:   3/  3, Time:15.58632708\n",
      "Initial Run End\n",
      "Init Run Best: 0.11356059002690017, Mean: 0.15830107795633375, ID:1, config: [784   9   5   5   5   6  10]\n",
      "Gen 0 Run Start\n",
      "Next Gen: [784   6   6   9   8   6   9   6   5  10]\n",
      "ConfigID:   0, Epoch:   0, Training Loss: 0.34820222, Validation Loss: 0.32082012,                    Best: 0.32082012, StopCount/Limit:   0/  3, Time:14.61625290\n",
      "ConfigID:   0, Epoch:   1, Training Loss: 0.30825523, Validation Loss: 0.28880082,                    Best: 0.28880082, StopCount/Limit:   0/  3, Time:15.83083534\n",
      "ConfigID:   0, Epoch:   2, Training Loss: 0.27972937, Validation Loss: 0.27377716,                    Best: 0.27377716, StopCount/Limit:   0/  3, Time:14.42480898\n",
      "ConfigID:   0, Epoch:   3, Training Loss: 0.27279987, Validation Loss: 0.27133479,                    Best: 0.27133479, StopCount/Limit:   0/  3, Time:10.84653306\n",
      "ConfigID:   0, Epoch:   4, Training Loss: 0.26898740, Validation Loss: 0.26645255,                    Best: 0.26645255, StopCount/Limit:   0/  3, Time:10.35347605\n",
      "ConfigID:   0, Epoch:   5, Training Loss: 0.26460967, Validation Loss: 0.26227129,                    Best: 0.26227129, StopCount/Limit:   0/  3, Time:10.06718707\n",
      "ConfigID:   0, Epoch:   6, Training Loss: 0.26168966, Validation Loss: 0.26046713,                    Best: 0.26046713, StopCount/Limit:   0/  3, Time:9.55482173\n",
      "ConfigID:   0, Epoch:   7, Training Loss: 0.25927463, Validation Loss: 0.25862203,                    Best: 0.25862203, StopCount/Limit:   0/  3, Time:9.42360592\n",
      "ConfigID:   0, Epoch:   8, Training Loss: 0.25713355, Validation Loss: 0.25966568,                    Best: 0.25862203, StopCount/Limit:   1/  3, Time:9.80507207\n",
      "ConfigID:   0, Epoch:   9, Training Loss: 0.25449138, Validation Loss: 0.25467364,                    Best: 0.25467364, StopCount/Limit:   1/  3, Time:9.44143629\n",
      "ConfigID:   0, Epoch:  10, Training Loss: 0.25356960, Validation Loss: 0.25374207,                    Best: 0.25374207, StopCount/Limit:   1/  3, Time:9.52576184\n",
      "ConfigID:   0, Epoch:  11, Training Loss: 0.25286563, Validation Loss: 0.25609843,                    Best: 0.25374207, StopCount/Limit:   2/  3, Time:9.69277024\n",
      "ConfigID:   0, Epoch:  12, Training Loss: 0.25233559, Validation Loss: 0.25288956,                    Best: 0.25288956, StopCount/Limit:   2/  3, Time:9.89386702\n",
      "ConfigID:   0, Epoch:  13, Training Loss: 0.25209617, Validation Loss: 0.25222045,                    Best: 0.25222045, StopCount/Limit:   2/  3, Time:9.28070498\n",
      "ConfigID:   0, Epoch:  14, Training Loss: 0.25189916, Validation Loss: 0.25292913,                    Best: 0.25222045, StopCount/Limit:   3/  3, Time:9.42241883\n",
      "Next Gen: [784   4   7  10   9   6   8   7   5   5   7  10]\n",
      "ConfigID:   1, Epoch:   0, Training Loss: 0.30323987, Validation Loss: 0.24442893,                    Best: 0.24442893, StopCount/Limit:   0/  3, Time:10.71265721\n",
      "ConfigID:   1, Epoch:   1, Training Loss: 0.23908278, Validation Loss: 0.22334419,                    Best: 0.22334419, StopCount/Limit:   0/  3, Time:10.74063897\n",
      "ConfigID:   1, Epoch:   2, Training Loss: 0.22445907, Validation Loss: 0.21865691,                    Best: 0.21865691, StopCount/Limit:   0/  3, Time:10.76170707\n",
      "ConfigID:   1, Epoch:   3, Training Loss: 0.21708629, Validation Loss: 0.21175972,                    Best: 0.21175972, StopCount/Limit:   0/  3, Time:10.82225299\n",
      "ConfigID:   1, Epoch:   4, Training Loss: 0.20934010, Validation Loss: 0.20634682,                    Best: 0.20634682, StopCount/Limit:   0/  3, Time:11.85627294\n",
      "ConfigID:   1, Epoch:   5, Training Loss: 0.20742873, Validation Loss: 0.20999822,                    Best: 0.20634682, StopCount/Limit:   1/  3, Time:10.69718623\n",
      "ConfigID:   1, Epoch:   6, Training Loss: 0.20761473, Validation Loss: 0.20491126,                    Best: 0.20491126, StopCount/Limit:   1/  3, Time:10.74519181\n",
      "ConfigID:   1, Epoch:   7, Training Loss: 0.20436191, Validation Loss: 0.20370433,                    Best: 0.20370433, StopCount/Limit:   1/  3, Time:10.87397766\n",
      "ConfigID:   1, Epoch:   8, Training Loss: 0.20202269, Validation Loss: 0.20489582,                    Best: 0.20370433, StopCount/Limit:   2/  3, Time:10.62151408\n",
      "ConfigID:   1, Epoch:   9, Training Loss: 0.20071219, Validation Loss: 0.19808289,                    Best: 0.19808289, StopCount/Limit:   2/  3, Time:10.65878201\n",
      "ConfigID:   1, Epoch:  10, Training Loss: 0.20170216, Validation Loss: 0.19940744,                    Best: 0.19808289, StopCount/Limit:   3/  3, Time:10.74516797\n",
      "Next Gen: [784   4   7  10   9   6   6   7   4   5   9  10]\n",
      "ConfigID:   2, Epoch:   0, Training Loss: 0.26215240, Validation Loss: 0.19255244,                    Best: 0.19255244, StopCount/Limit:   0/  3, Time:10.73235297\n",
      "ConfigID:   2, Epoch:   1, Training Loss: 0.17965859, Validation Loss: 0.17586186,                    Best: 0.17586186, StopCount/Limit:   0/  3, Time:10.62450504\n",
      "ConfigID:   2, Epoch:   2, Training Loss: 0.16678725, Validation Loss: 0.16352006,                    Best: 0.16352006, StopCount/Limit:   0/  3, Time:10.52551603\n",
      "ConfigID:   2, Epoch:   3, Training Loss: 0.16157476, Validation Loss: 0.16102852,                    Best: 0.16102852, StopCount/Limit:   0/  3, Time:10.61722803\n",
      "ConfigID:   2, Epoch:   4, Training Loss: 0.16029549, Validation Loss: 0.15908567,                    Best: 0.15908567, StopCount/Limit:   0/  3, Time:10.35277200\n",
      "ConfigID:   2, Epoch:   5, Training Loss: 0.15984631, Validation Loss: 0.16374265,                    Best: 0.15908567, StopCount/Limit:   1/  3, Time:10.70644712\n",
      "ConfigID:   2, Epoch:   6, Training Loss: 0.15802661, Validation Loss: 0.15943023,                    Best: 0.15908567, StopCount/Limit:   2/  3, Time:11.05058312\n",
      "ConfigID:   2, Epoch:   7, Training Loss: 0.15673246, Validation Loss: 0.15762219,                    Best: 0.15762219, StopCount/Limit:   2/  3, Time:10.78191304\n",
      "ConfigID:   2, Epoch:   8, Training Loss: 0.16202091, Validation Loss: 0.15838359,                    Best: 0.15762219, StopCount/Limit:   3/  3, Time:10.40758014\n",
      "Gen 0 Run End\n",
      "Run   0 Best: 0.11356059002690017, Mean: 0.15830107795633375, ID:1, config: [784   9   5   5   5   6  10], updatecount:   0, Generation RunTime: 377.25320697\n",
      "Gen 1 Run Start\n",
      "Next Gen: [784   9   4   5   5   6   9   4   5  10]\n",
      "ConfigID:   0, Epoch:   0, Training Loss: 0.30338526, Validation Loss: 0.26538627,                    Best: 0.26538627, StopCount/Limit:   0/  3, Time:21.58334184\n",
      "ConfigID:   0, Epoch:   1, Training Loss: 0.26273649, Validation Loss: 0.25648886,                    Best: 0.25648886, StopCount/Limit:   0/  3, Time:9.39202213\n",
      "ConfigID:   0, Epoch:   2, Training Loss: 0.20950467, Validation Loss: 0.17187764,                    Best: 0.17187764, StopCount/Limit:   0/  3, Time:9.58661771\n",
      "ConfigID:   0, Epoch:   3, Training Loss: 0.16049506, Validation Loss: 0.15489960,                    Best: 0.15489960, StopCount/Limit:   0/  3, Time:9.38060188\n",
      "ConfigID:   0, Epoch:   4, Training Loss: 0.14481065, Validation Loss: 0.14313188,                    Best: 0.14313188, StopCount/Limit:   0/  3, Time:9.98775697\n",
      "ConfigID:   0, Epoch:   5, Training Loss: 0.13749652, Validation Loss: 0.13745100,                    Best: 0.13745100, StopCount/Limit:   0/  3, Time:9.62865901\n",
      "ConfigID:   0, Epoch:   6, Training Loss: 0.13332506, Validation Loss: 0.13662830,                    Best: 0.13662830, StopCount/Limit:   0/  3, Time:9.29788709\n",
      "ConfigID:   0, Epoch:   7, Training Loss: 0.13052574, Validation Loss: 0.13538147,                    Best: 0.13538147, StopCount/Limit:   0/  3, Time:9.99395108\n",
      "ConfigID:   0, Epoch:   8, Training Loss: 0.12701481, Validation Loss: 0.12800341,                    Best: 0.12800341, StopCount/Limit:   0/  3, Time:9.70184016\n",
      "ConfigID:   0, Epoch:   9, Training Loss: 0.12326049, Validation Loss: 0.12654610,                    Best: 0.12654610, StopCount/Limit:   0/  3, Time:9.33890724\n",
      "ConfigID:   0, Epoch:  10, Training Loss: 0.12210789, Validation Loss: 0.12582107,                    Best: 0.12582107, StopCount/Limit:   0/  3, Time:9.24305296\n",
      "ConfigID:   0, Epoch:  11, Training Loss: 0.11966564, Validation Loss: 0.12153085,                    Best: 0.12153085, StopCount/Limit:   0/  3, Time:9.29011393\n",
      "ConfigID:   0, Epoch:  12, Training Loss: 0.11596339, Validation Loss: 0.11838216,                    Best: 0.11838216, StopCount/Limit:   0/  3, Time:9.34275699\n",
      "ConfigID:   0, Epoch:  13, Training Loss: 0.11120939, Validation Loss: 0.11498748,                    Best: 0.11498748, StopCount/Limit:   0/  3, Time:9.21070313\n",
      "ConfigID:   0, Epoch:  14, Training Loss: 0.10740415, Validation Loss: 0.11516343,                    Best: 0.11498748, StopCount/Limit:   1/  3, Time:9.34900713\n",
      "ConfigID:   0, Epoch:  15, Training Loss: 0.10552599, Validation Loss: 0.11024789,                    Best: 0.11024789, StopCount/Limit:   1/  3, Time:9.31721306\n",
      "ConfigID:   0, Epoch:  16, Training Loss: 0.10331614, Validation Loss: 0.10666950,                    Best: 0.10666950, StopCount/Limit:   1/  3, Time:11.05596972\n",
      "ConfigID:   0, Epoch:  17, Training Loss: 0.10132878, Validation Loss: 0.10464478,                    Best: 0.10464478, StopCount/Limit:   1/  3, Time:9.32443500\n",
      "ConfigID:   0, Epoch:  18, Training Loss: 0.09943956, Validation Loss: 0.10065020,                    Best: 0.10065020, StopCount/Limit:   1/  3, Time:9.36646509\n",
      "ConfigID:   0, Epoch:  19, Training Loss: 0.08808410, Validation Loss: 0.09168887,                    Best: 0.09168887, StopCount/Limit:   1/  3, Time:10.36097622\n",
      "ConfigID:   0, Epoch:  20, Training Loss: 0.08286817, Validation Loss: 0.09515504,                    Best: 0.09168887, StopCount/Limit:   2/  3, Time:9.42693615\n",
      "ConfigID:   0, Epoch:  21, Training Loss: 0.08113102, Validation Loss: 0.09061624,                    Best: 0.09061624, StopCount/Limit:   2/  3, Time:9.46830893\n",
      "ConfigID:   0, Epoch:  22, Training Loss: 0.07995427, Validation Loss: 0.08184287,                    Best: 0.08184287, StopCount/Limit:   2/  3, Time:9.30999804\n",
      "ConfigID:   0, Epoch:  23, Training Loss: 0.07959113, Validation Loss: 0.07929998,                    Best: 0.07929998, StopCount/Limit:   2/  3, Time:9.38600922\n",
      "ConfigID:   0, Epoch:  24, Training Loss: 0.07863760, Validation Loss: 0.08391658,                    Best: 0.07929998, StopCount/Limit:   3/  3, Time:9.42307472\n",
      "Next Gen: [784   7   5   6   6   6  10   6   4   4   5  10]\n",
      "ConfigID:   1, Epoch:   0, Training Loss: 0.29224197, Validation Loss: 0.26655615,                    Best: 0.26655615, StopCount/Limit:   0/  3, Time:10.50197101\n",
      "ConfigID:   1, Epoch:   1, Training Loss: 0.26220857, Validation Loss: 0.25226742,                    Best: 0.25226742, StopCount/Limit:   0/  3, Time:10.56519389\n",
      "ConfigID:   1, Epoch:   2, Training Loss: 0.24959008, Validation Loss: 0.24799752,                    Best: 0.24799752, StopCount/Limit:   0/  3, Time:10.56491590\n",
      "ConfigID:   1, Epoch:   3, Training Loss: 0.24681705, Validation Loss: 0.24589258,                    Best: 0.24589258, StopCount/Limit:   0/  3, Time:10.84337711\n",
      "ConfigID:   1, Epoch:   4, Training Loss: 0.24531786, Validation Loss: 0.24412424,                    Best: 0.24412424, StopCount/Limit:   0/  3, Time:10.54765201\n",
      "ConfigID:   1, Epoch:   5, Training Loss: 0.24420185, Validation Loss: 0.24560655,                    Best: 0.24412424, StopCount/Limit:   1/  3, Time:10.60536408\n",
      "ConfigID:   1, Epoch:   6, Training Loss: 0.24390217, Validation Loss: 0.24565862,                    Best: 0.24412424, StopCount/Limit:   2/  3, Time:10.96853113\n",
      "ConfigID:   1, Epoch:   7, Training Loss: 0.24271749, Validation Loss: 0.24263683,                    Best: 0.24263683, StopCount/Limit:   2/  3, Time:11.09973979\n",
      "ConfigID:   1, Epoch:   8, Training Loss: 0.24249290, Validation Loss: 0.24363173,                    Best: 0.24263683, StopCount/Limit:   3/  3, Time:10.65724015\n",
      "Next Gen: [784   5   7   4   8   6   9   9   5   5  10]\n",
      "ConfigID:   2, Epoch:   0, Training Loss: 0.33868459, Validation Loss: 0.32239001,                    Best: 0.32239001, StopCount/Limit:   0/  3, Time:9.75142193\n",
      "ConfigID:   2, Epoch:   1, Training Loss: 0.31250072, Validation Loss: 0.29914774,                    Best: 0.29914774, StopCount/Limit:   0/  3, Time:9.77903104\n",
      "ConfigID:   2, Epoch:   2, Training Loss: 0.29689777, Validation Loss: 0.29325957,                    Best: 0.29325957, StopCount/Limit:   0/  3, Time:9.69580793\n",
      "ConfigID:   2, Epoch:   3, Training Loss: 0.29333590, Validation Loss: 0.29032959,                    Best: 0.29032959, StopCount/Limit:   0/  3, Time:10.73603010\n",
      "ConfigID:   2, Epoch:   4, Training Loss: 0.29055215, Validation Loss: 0.28954926,                    Best: 0.28954926, StopCount/Limit:   0/  3, Time:9.90358400\n",
      "ConfigID:   2, Epoch:   5, Training Loss: 0.28925257, Validation Loss: 0.28776010,                    Best: 0.28776010, StopCount/Limit:   0/  3, Time:9.78423095\n",
      "ConfigID:   2, Epoch:   6, Training Loss: 0.28755830, Validation Loss: 0.28456586,                    Best: 0.28456586, StopCount/Limit:   0/  3, Time:9.70729899\n",
      "ConfigID:   2, Epoch:   7, Training Loss: 0.28440726, Validation Loss: 0.28167857,                    Best: 0.28167857, StopCount/Limit:   0/  3, Time:9.74514318\n",
      "ConfigID:   2, Epoch:   8, Training Loss: 0.28264836, Validation Loss: 0.28039097,                    Best: 0.28039097, StopCount/Limit:   0/  3, Time:9.73336673\n",
      "ConfigID:   2, Epoch:   9, Training Loss: 0.28187894, Validation Loss: 0.28001801,                    Best: 0.28001801, StopCount/Limit:   0/  3, Time:9.74477124\n",
      "ConfigID:   2, Epoch:  10, Training Loss: 0.28132008, Validation Loss: 0.27973641,                    Best: 0.27973641, StopCount/Limit:   0/  3, Time:9.90056515\n",
      "ConfigID:   2, Epoch:  11, Training Loss: 0.28090171, Validation Loss: 0.27910511,                    Best: 0.27910511, StopCount/Limit:   0/  3, Time:10.26067805\n",
      "ConfigID:   2, Epoch:  12, Training Loss: 0.28052846, Validation Loss: 0.27860241,                    Best: 0.27860241, StopCount/Limit:   0/  3, Time:9.90144420\n",
      "ConfigID:   2, Epoch:  13, Training Loss: 0.27967074, Validation Loss: 0.27771275,                    Best: 0.27771275, StopCount/Limit:   0/  3, Time:9.90154314\n",
      "ConfigID:   2, Epoch:  14, Training Loss: 0.27003412, Validation Loss: 0.26231714,                    Best: 0.26231714, StopCount/Limit:   0/  3, Time:9.89981079\n",
      "ConfigID:   2, Epoch:  15, Training Loss: 0.26380351, Validation Loss: 0.25955462,                    Best: 0.25955462, StopCount/Limit:   0/  3, Time:9.85449481\n",
      "ConfigID:   2, Epoch:  16, Training Loss: 0.25935957, Validation Loss: 0.25713970,                    Best: 0.25713970, StopCount/Limit:   0/  3, Time:10.25568104\n",
      "ConfigID:   2, Epoch:  17, Training Loss: 0.25881493, Validation Loss: 0.25816538,                    Best: 0.25713970, StopCount/Limit:   1/  3, Time:14.16035509\n",
      "ConfigID:   2, Epoch:  18, Training Loss: 0.25855940, Validation Loss: 0.25712035,                    Best: 0.25712035, StopCount/Limit:   1/  3, Time:13.77399302\n",
      "ConfigID:   2, Epoch:  19, Training Loss: 0.25533396, Validation Loss: 0.25478451,                    Best: 0.25478451, StopCount/Limit:   1/  3, Time:10.04834485\n",
      "ConfigID:   2, Epoch:  20, Training Loss: 0.25413132, Validation Loss: 0.25328858,                    Best: 0.25328858, StopCount/Limit:   1/  3, Time:10.79527807\n",
      "ConfigID:   2, Epoch:  21, Training Loss: 0.25295029, Validation Loss: 0.25168946,                    Best: 0.25168946, StopCount/Limit:   1/  3, Time:9.87955523\n",
      "ConfigID:   2, Epoch:  22, Training Loss: 0.25271502, Validation Loss: 0.25917047,                    Best: 0.25168946, StopCount/Limit:   2/  3, Time:9.92669106\n",
      "ConfigID:   2, Epoch:  23, Training Loss: 0.24949057, Validation Loss: 0.24773686,                    Best: 0.24773686, StopCount/Limit:   2/  3, Time:9.74638200\n",
      "ConfigID:   2, Epoch:  24, Training Loss: 0.24680294, Validation Loss: 0.24661403,                    Best: 0.24661403, StopCount/Limit:   2/  3, Time:9.71870685\n",
      "ConfigID:   2, Epoch:  25, Training Loss: 0.24451436, Validation Loss: 0.24422408,                    Best: 0.24422408, StopCount/Limit:   2/  3, Time:9.79693890\n",
      "ConfigID:   2, Epoch:  26, Training Loss: 0.24241292, Validation Loss: 0.24210287,                    Best: 0.24210287, StopCount/Limit:   2/  3, Time:9.82478499\n",
      "ConfigID:   2, Epoch:  27, Training Loss: 0.24829896, Validation Loss: 0.26264327,                    Best: 0.24210287, StopCount/Limit:   3/  3, Time:9.68119907\n",
      "Gen 1 Run End\n",
      "Run   1 Best: 0.07929997807461768, Mean: 0.11561694564515106, ID:0, config: [784   9   4   5   5   6   9   4   5  10], updatecount:   1, Generation RunTime: 633.07021499\n",
      "Gen 2 Run Start\n",
      "Next Gen: [784  10   4   4   4   6   8  10   4  10]\n",
      "ConfigID:   0, Epoch:   0, Training Loss: 0.23783296, Validation Loss: 0.15717612,                    Best: 0.15717612, StopCount/Limit:   0/  3, Time:9.03730631\n",
      "ConfigID:   0, Epoch:   1, Training Loss: 0.13884498, Validation Loss: 0.12664866,                    Best: 0.12664866, StopCount/Limit:   0/  3, Time:9.40566206\n",
      "ConfigID:   0, Epoch:   2, Training Loss: 0.11477722, Validation Loss: 0.10615750,                    Best: 0.10615750, StopCount/Limit:   0/  3, Time:9.09103823\n",
      "ConfigID:   0, Epoch:   3, Training Loss: 0.10480661, Validation Loss: 0.10058266,                    Best: 0.10058266, StopCount/Limit:   0/  3, Time:8.99837589\n",
      "ConfigID:   0, Epoch:   4, Training Loss: 0.09678954, Validation Loss: 0.09904997,                    Best: 0.09904997, StopCount/Limit:   0/  3, Time:9.12255120\n",
      "ConfigID:   0, Epoch:   5, Training Loss: 0.09204009, Validation Loss: 0.09351854,                    Best: 0.09351854, StopCount/Limit:   0/  3, Time:9.14367008\n",
      "ConfigID:   0, Epoch:   6, Training Loss: 0.08810952, Validation Loss: 0.09026912,                    Best: 0.09026912, StopCount/Limit:   0/  3, Time:10.99772692\n",
      "ConfigID:   0, Epoch:   7, Training Loss: 0.08541852, Validation Loss: 0.09153864,                    Best: 0.09026912, StopCount/Limit:   1/  3, Time:9.89552426\n",
      "ConfigID:   0, Epoch:   8, Training Loss: 0.08431176, Validation Loss: 0.08705330,                    Best: 0.08705330, StopCount/Limit:   1/  3, Time:9.29174399\n",
      "ConfigID:   0, Epoch:   9, Training Loss: 0.08303483, Validation Loss: 0.09342257,                    Best: 0.08705330, StopCount/Limit:   2/  3, Time:9.47202206\n",
      "ConfigID:   0, Epoch:  10, Training Loss: 0.08142562, Validation Loss: 0.08600472,                    Best: 0.08600472, StopCount/Limit:   2/  3, Time:9.10999489\n",
      "ConfigID:   0, Epoch:  11, Training Loss: 0.08076412, Validation Loss: 0.08861229,                    Best: 0.08600472, StopCount/Limit:   3/  3, Time:10.06802320\n",
      "Next Gen: [784   7   5   6   6   6  10   5   4   4   7  10]\n",
      "ConfigID:   1, Epoch:   0, Training Loss: 0.33994420, Validation Loss: 0.32501816,                    Best: 0.32501816, StopCount/Limit:   0/  3, Time:10.42537308\n",
      "ConfigID:   1, Epoch:   1, Training Loss: 0.32503406, Validation Loss: 0.32494891,                    Best: 0.32494891, StopCount/Limit:   0/  3, Time:11.17164111\n",
      "ConfigID:   1, Epoch:   2, Training Loss: 0.32503238, Validation Loss: 0.32501144,                    Best: 0.32494891, StopCount/Limit:   1/  3, Time:10.44798183\n",
      "ConfigID:   1, Epoch:   3, Training Loss: 0.32504208, Validation Loss: 0.32499709,                    Best: 0.32494891, StopCount/Limit:   2/  3, Time:10.39117479\n",
      "ConfigID:   1, Epoch:   4, Training Loss: 0.32504734, Validation Loss: 0.32504108,                    Best: 0.32494891, StopCount/Limit:   3/  3, Time:10.42469501\n",
      "Next Gen: [784  10   4   4   4   6  10   4   4   4  10]\n",
      "ConfigID:   2, Epoch:   0, Training Loss: 0.28946729, Validation Loss: 0.25966264,                    Best: 0.25966264, StopCount/Limit:   0/  3, Time:9.66156316\n",
      "ConfigID:   2, Epoch:   1, Training Loss: 0.25446863, Validation Loss: 0.25254988,                    Best: 0.25254988, StopCount/Limit:   0/  3, Time:9.81218982\n",
      "ConfigID:   2, Epoch:   2, Training Loss: 0.24993439, Validation Loss: 0.24899834,                    Best: 0.24899834, StopCount/Limit:   0/  3, Time:9.76006794\n",
      "ConfigID:   2, Epoch:   3, Training Loss: 0.24665107, Validation Loss: 0.24781470,                    Best: 0.24781470, StopCount/Limit:   0/  3, Time:9.79731917\n",
      "ConfigID:   2, Epoch:   4, Training Loss: 0.24439794, Validation Loss: 0.24556099,                    Best: 0.24556099, StopCount/Limit:   0/  3, Time:9.77640390\n",
      "ConfigID:   2, Epoch:   5, Training Loss: 0.24271833, Validation Loss: 0.24667522,                    Best: 0.24556099, StopCount/Limit:   1/  3, Time:9.76687288\n",
      "ConfigID:   2, Epoch:   6, Training Loss: 0.23600453, Validation Loss: 0.21181068,                    Best: 0.21181068, StopCount/Limit:   1/  3, Time:10.06274509\n",
      "ConfigID:   2, Epoch:   7, Training Loss: 0.19502809, Validation Loss: 0.18994213,                    Best: 0.18994213, StopCount/Limit:   1/  3, Time:9.88971591\n",
      "ConfigID:   2, Epoch:   8, Training Loss: 0.18312628, Validation Loss: 0.17938618,                    Best: 0.17938618, StopCount/Limit:   1/  3, Time:10.23424196\n",
      "ConfigID:   2, Epoch:   9, Training Loss: 0.17569221, Validation Loss: 0.17280288,                    Best: 0.17280288, StopCount/Limit:   1/  3, Time:9.75711894\n",
      "ConfigID:   2, Epoch:  10, Training Loss: 0.16545115, Validation Loss: 0.16510759,                    Best: 0.16510759, StopCount/Limit:   1/  3, Time:9.81374788\n",
      "ConfigID:   2, Epoch:  11, Training Loss: 0.15495204, Validation Loss: 0.16517656,                    Best: 0.16510759, StopCount/Limit:   2/  3, Time:9.81931806\n",
      "ConfigID:   2, Epoch:  12, Training Loss: 0.14722248, Validation Loss: 0.14721258,                    Best: 0.14721258, StopCount/Limit:   2/  3, Time:11.72736692\n",
      "ConfigID:   2, Epoch:  13, Training Loss: 0.14299999, Validation Loss: 0.14268860,                    Best: 0.14268860, StopCount/Limit:   2/  3, Time:10.26645708\n",
      "ConfigID:   2, Epoch:  14, Training Loss: 0.14062334, Validation Loss: 0.14245956,                    Best: 0.14245956, StopCount/Limit:   2/  3, Time:11.23396897\n",
      "ConfigID:   2, Epoch:  15, Training Loss: 0.13702637, Validation Loss: 0.14055244,                    Best: 0.14055244, StopCount/Limit:   2/  3, Time:10.96774578\n",
      "ConfigID:   2, Epoch:  16, Training Loss: 0.13539454, Validation Loss: 0.13752330,                    Best: 0.13752330, StopCount/Limit:   2/  3, Time:10.02662992\n",
      "ConfigID:   2, Epoch:  17, Training Loss: 0.13153574, Validation Loss: 0.13585687,                    Best: 0.13585687, StopCount/Limit:   2/  3, Time:9.75886798\n",
      "ConfigID:   2, Epoch:  18, Training Loss: 0.13035856, Validation Loss: 0.13956736,                    Best: 0.13585687, StopCount/Limit:   3/  3, Time:9.75558972\n",
      "Gen 2 Run End\n",
      "Run   2 Best: 0.07929997807461768, Mean: 0.1095724790111805, ID:0, config: [784   9   4   5   5   6   9   4   5  10], updatecount:   1, Generation RunTime: 358.41405416\n",
      "Gen 3 Run Start\n",
      "Next Gen: [784   8   4   5   5   6   8   8   4  10]\n",
      "ConfigID:   0, Epoch:   0, Training Loss: 0.33640056, Validation Loss: 0.32510251,                    Best: 0.32510251, StopCount/Limit:   0/  3, Time:9.09712291\n",
      "ConfigID:   0, Epoch:   1, Training Loss: 0.32372645, Validation Loss: 0.28730048,                    Best: 0.28730048, StopCount/Limit:   0/  3, Time:9.12355375\n",
      "ConfigID:   0, Epoch:   2, Training Loss: 0.24977892, Validation Loss: 0.21968041,                    Best: 0.21968041, StopCount/Limit:   0/  3, Time:9.50213575\n",
      "ConfigID:   0, Epoch:   3, Training Loss: 0.21167849, Validation Loss: 0.20460124,                    Best: 0.20460124, StopCount/Limit:   0/  3, Time:9.10080075\n",
      "ConfigID:   0, Epoch:   4, Training Loss: 0.20268837, Validation Loss: 0.20037767,                    Best: 0.20037767, StopCount/Limit:   0/  3, Time:9.01505184\n",
      "ConfigID:   0, Epoch:   5, Training Loss: 0.19916523, Validation Loss: 0.20289749,                    Best: 0.20037767, StopCount/Limit:   1/  3, Time:9.00869417\n",
      "ConfigID:   0, Epoch:   6, Training Loss: 0.19588371, Validation Loss: 0.19418827,                    Best: 0.19418827, StopCount/Limit:   1/  3, Time:9.03411508\n",
      "ConfigID:   0, Epoch:   7, Training Loss: 0.19506879, Validation Loss: 0.19379016,                    Best: 0.19379016, StopCount/Limit:   1/  3, Time:9.48797512\n",
      "ConfigID:   0, Epoch:   8, Training Loss: 0.19208042, Validation Loss: 0.19382394,                    Best: 0.19379016, StopCount/Limit:   2/  3, Time:8.94138503\n",
      "ConfigID:   0, Epoch:   9, Training Loss: 0.19161094, Validation Loss: 0.18944916,                    Best: 0.18944916, StopCount/Limit:   2/  3, Time:9.07977414\n",
      "ConfigID:   0, Epoch:  10, Training Loss: 0.18822591, Validation Loss: 0.18835756,                    Best: 0.18835756, StopCount/Limit:   2/  3, Time:8.95713305\n",
      "ConfigID:   0, Epoch:  11, Training Loss: 0.18677483, Validation Loss: 0.18683153,                    Best: 0.18683153, StopCount/Limit:   2/  3, Time:11.53249812\n",
      "ConfigID:   0, Epoch:  12, Training Loss: 0.18532295, Validation Loss: 0.18476980,                    Best: 0.18476980, StopCount/Limit:   2/  3, Time:9.45879602\n",
      "ConfigID:   0, Epoch:  13, Training Loss: 0.18548660, Validation Loss: 0.19619663,                    Best: 0.18476980, StopCount/Limit:   3/  3, Time:9.08699512\n",
      "Next Gen: [784   9   5   5   5   6  10]\n",
      "ConfigID:   1, Epoch:   0, Training Loss: 0.32332645, Validation Loss: 0.27973570,                    Best: 0.27973570, StopCount/Limit:   0/  3, Time:6.99616408\n",
      "ConfigID:   1, Epoch:   1, Training Loss: 0.20789442, Validation Loss: 0.17379959,                    Best: 0.17379959, StopCount/Limit:   0/  3, Time:6.88008404\n",
      "ConfigID:   1, Epoch:   2, Training Loss: 0.16517264, Validation Loss: 0.15954727,                    Best: 0.15954727, StopCount/Limit:   0/  3, Time:6.80330515\n",
      "ConfigID:   1, Epoch:   3, Training Loss: 0.15404929, Validation Loss: 0.14163964,                    Best: 0.14163964, StopCount/Limit:   0/  3, Time:6.83759212\n",
      "ConfigID:   1, Epoch:   4, Training Loss: 0.11563300, Validation Loss: 0.10664788,                    Best: 0.10664788, StopCount/Limit:   0/  3, Time:6.93719316\n",
      "ConfigID:   1, Epoch:   5, Training Loss: 0.10328569, Validation Loss: 0.10066203,                    Best: 0.10066203, StopCount/Limit:   0/  3, Time:6.89946270\n",
      "ConfigID:   1, Epoch:   6, Training Loss: 0.09907974, Validation Loss: 0.09798343,                    Best: 0.09798343, StopCount/Limit:   0/  3, Time:7.41637611\n",
      "ConfigID:   1, Epoch:   7, Training Loss: 0.09618482, Validation Loss: 0.10161082,                    Best: 0.09798343, StopCount/Limit:   1/  3, Time:6.96609688\n",
      "ConfigID:   1, Epoch:   8, Training Loss: 0.09465523, Validation Loss: 0.09591824,                    Best: 0.09591824, StopCount/Limit:   1/  3, Time:6.98001909\n",
      "ConfigID:   1, Epoch:   9, Training Loss: 0.09411020, Validation Loss: 0.09771967,                    Best: 0.09591824, StopCount/Limit:   2/  3, Time:6.73474574\n",
      "ConfigID:   1, Epoch:  10, Training Loss: 0.09187706, Validation Loss: 0.09497010,                    Best: 0.09497010, StopCount/Limit:   2/  3, Time:8.22955918\n",
      "ConfigID:   1, Epoch:  11, Training Loss: 0.09100319, Validation Loss: 0.09436136,                    Best: 0.09436136, StopCount/Limit:   2/  3, Time:7.47772384\n",
      "ConfigID:   1, Epoch:  12, Training Loss: 0.08972985, Validation Loss: 0.09620788,                    Best: 0.09436136, StopCount/Limit:   3/  3, Time:7.14456081\n",
      "Next Gen: [784   8   5   5   5   6   8   5   5   5  10]\n",
      "ConfigID:   2, Epoch:   0, Training Loss: 0.34738015, Validation Loss: 0.32504152,                    Best: 0.32504152, StopCount/Limit:   0/  3, Time:9.87837410\n",
      "ConfigID:   2, Epoch:   1, Training Loss: 0.32501548, Validation Loss: 0.32498929,                    Best: 0.32498929, StopCount/Limit:   0/  3, Time:9.92083287\n",
      "ConfigID:   2, Epoch:   2, Training Loss: 0.32502111, Validation Loss: 0.32495788,                    Best: 0.32495788, StopCount/Limit:   0/  3, Time:9.84210801\n",
      "ConfigID:   2, Epoch:   3, Training Loss: 0.32501518, Validation Loss: 0.32506869,                    Best: 0.32495788, StopCount/Limit:   1/  3, Time:9.90433192\n",
      "ConfigID:   2, Epoch:   4, Training Loss: 0.32501750, Validation Loss: 0.32497734,                    Best: 0.32495788, StopCount/Limit:   2/  3, Time:10.28649187\n",
      "ConfigID:   2, Epoch:   5, Training Loss: 0.32501804, Validation Loss: 0.32495365,                    Best: 0.32495365, StopCount/Limit:   2/  3, Time:9.84428596\n",
      "ConfigID:   2, Epoch:   6, Training Loss: 0.32501410, Validation Loss: 0.32496540,                    Best: 0.32495365, StopCount/Limit:   3/  3, Time:11.31470799\n",
      "Gen 3 Run End\n",
      "Run   3 Best: 0.07929997807461768, Mean: 0.10317273633508012, ID:0, config: [784   9   4   5   5   6   9   4   5  10], updatecount:   1, Generation RunTime: 293.75026298\n",
      "Gen 4 Run Start\n",
      "Next Gen: [784   9   5   4   4   6   9   5   4  10]\n",
      "ConfigID:   0, Epoch:   0, Training Loss: 0.28338865, Validation Loss: 0.23593480,                    Best: 0.23593480, StopCount/Limit:   0/  3, Time:9.33167911\n",
      "ConfigID:   0, Epoch:   1, Training Loss: 0.22447862, Validation Loss: 0.21237777,                    Best: 0.21237777, StopCount/Limit:   0/  3, Time:9.21662712\n",
      "ConfigID:   0, Epoch:   2, Training Loss: 0.21021319, Validation Loss: 0.20139214,                    Best: 0.20139214, StopCount/Limit:   0/  3, Time:9.17569900\n",
      "ConfigID:   0, Epoch:   3, Training Loss: 0.20006369, Validation Loss: 0.19517684,                    Best: 0.19517684, StopCount/Limit:   0/  3, Time:9.50652719\n",
      "ConfigID:   0, Epoch:   4, Training Loss: 0.19562340, Validation Loss: 0.19236105,                    Best: 0.19236105, StopCount/Limit:   0/  3, Time:9.32160091\n",
      "ConfigID:   0, Epoch:   5, Training Loss: 0.19272817, Validation Loss: 0.19475515,                    Best: 0.19236105, StopCount/Limit:   1/  3, Time:9.20512891\n",
      "ConfigID:   0, Epoch:   6, Training Loss: 0.19302765, Validation Loss: 0.19032665,                    Best: 0.19032665, StopCount/Limit:   1/  3, Time:9.19989109\n",
      "ConfigID:   0, Epoch:   7, Training Loss: 0.19113352, Validation Loss: 0.18959251,                    Best: 0.18959251, StopCount/Limit:   1/  3, Time:9.29487705\n",
      "ConfigID:   0, Epoch:   8, Training Loss: 0.18896238, Validation Loss: 0.18329820,                    Best: 0.18329820, StopCount/Limit:   1/  3, Time:11.44183493\n",
      "ConfigID:   0, Epoch:   9, Training Loss: 0.18896244, Validation Loss: 0.19115700,                    Best: 0.18329820, StopCount/Limit:   2/  3, Time:9.25672984\n",
      "ConfigID:   0, Epoch:  10, Training Loss: 0.19034908, Validation Loss: 0.18696080,                    Best: 0.18329820, StopCount/Limit:   3/  3, Time:9.35358906\n",
      "Next Gen: [784   9   4   4   4   6  10   4   5   7   6  10]\n",
      "ConfigID:   1, Epoch:   0, Training Loss: 0.33624164, Validation Loss: 0.32502288,                    Best: 0.32502288, StopCount/Limit:   0/  3, Time:10.58174276\n",
      "ConfigID:   1, Epoch:   1, Training Loss: 0.32507972, Validation Loss: 0.32513808,                    Best: 0.32502288, StopCount/Limit:   1/  3, Time:10.48433113\n",
      "ConfigID:   1, Epoch:   2, Training Loss: 0.32507412, Validation Loss: 0.32506421,                    Best: 0.32502288, StopCount/Limit:   2/  3, Time:10.74126482\n",
      "ConfigID:   1, Epoch:   3, Training Loss: 0.32509558, Validation Loss: 0.32493994,                    Best: 0.32493994, StopCount/Limit:   2/  3, Time:10.51742196\n",
      "ConfigID:   1, Epoch:   4, Training Loss: 0.32507087, Validation Loss: 0.32499070,                    Best: 0.32493994, StopCount/Limit:   3/  3, Time:10.47469282\n",
      "Next Gen: [784   9   4   4   4   6   9   4   4   6   6  10]\n",
      "ConfigID:   2, Epoch:   0, Training Loss: 0.27001146, Validation Loss: 0.23451207,                    Best: 0.23451207, StopCount/Limit:   0/  3, Time:10.37436700\n",
      "ConfigID:   2, Epoch:   1, Training Loss: 0.21041526, Validation Loss: 0.19320372,                    Best: 0.19320372, StopCount/Limit:   0/  3, Time:11.84999084\n",
      "ConfigID:   2, Epoch:   2, Training Loss: 0.19203285, Validation Loss: 0.18613925,                    Best: 0.18613925, StopCount/Limit:   0/  3, Time:10.76177382\n",
      "ConfigID:   2, Epoch:   3, Training Loss: 0.18689118, Validation Loss: 0.18023201,                    Best: 0.18023201, StopCount/Limit:   0/  3, Time:11.15789127\n",
      "ConfigID:   2, Epoch:   4, Training Loss: 0.18086351, Validation Loss: 0.17456821,                    Best: 0.17456821, StopCount/Limit:   0/  3, Time:10.46323991\n",
      "ConfigID:   2, Epoch:   5, Training Loss: 0.17505345, Validation Loss: 0.16847804,                    Best: 0.16847804, StopCount/Limit:   0/  3, Time:10.54757190\n",
      "ConfigID:   2, Epoch:   6, Training Loss: 0.17153172, Validation Loss: 0.16712928,                    Best: 0.16712928, StopCount/Limit:   0/  3, Time:10.36628890\n",
      "ConfigID:   2, Epoch:   7, Training Loss: 0.16925534, Validation Loss: 0.16759815,                    Best: 0.16712928, StopCount/Limit:   1/  3, Time:10.40922594\n",
      "ConfigID:   2, Epoch:   8, Training Loss: 0.16841875, Validation Loss: 0.16508328,                    Best: 0.16508328, StopCount/Limit:   1/  3, Time:10.88555288\n",
      "ConfigID:   2, Epoch:   9, Training Loss: 0.16688606, Validation Loss: 0.16573778,                    Best: 0.16508328, StopCount/Limit:   2/  3, Time:12.16005516\n",
      "ConfigID:   2, Epoch:  10, Training Loss: 0.16567103, Validation Loss: 0.16194680,                    Best: 0.16194680, StopCount/Limit:   2/  3, Time:16.73552990\n",
      "ConfigID:   2, Epoch:  11, Training Loss: 0.16548156, Validation Loss: 0.16167876,                    Best: 0.16167876, StopCount/Limit:   2/  3, Time:16.47265530\n",
      "ConfigID:   2, Epoch:  12, Training Loss: 0.16446243, Validation Loss: 0.15911041,                    Best: 0.15911041, StopCount/Limit:   2/  3, Time:13.04816794\n",
      "ConfigID:   2, Epoch:  13, Training Loss: 0.16400715, Validation Loss: 0.15799730,                    Best: 0.15799730, StopCount/Limit:   2/  3, Time:14.72330070\n",
      "ConfigID:   2, Epoch:  14, Training Loss: 0.16463930, Validation Loss: 0.16416776,                    Best: 0.15799730, StopCount/Limit:   3/  3, Time:11.57345295\n",
      "Gen 4 Run End\n",
      "Run   4 Best: 0.07929997807461768, Mean: 0.10317273633508012, ID:0, config: [784   9   4   5   5   6   9   4   5  10], updatecount:   0, Generation RunTime: 338.66250825\n",
      "Gen 5 Run Start\n",
      "Next Gen: [784  10   4   4   4   6   9   4  10  10]\n",
      "ConfigID:   0, Epoch:   0, Training Loss: 0.26132470, Validation Loss: 0.21799074,                    Best: 0.21799074, StopCount/Limit:   0/  3, Time:9.49838281\n",
      "ConfigID:   0, Epoch:   1, Training Loss: 0.20779225, Validation Loss: 0.19571872,                    Best: 0.19571872, StopCount/Limit:   0/  3, Time:9.54431081\n",
      "ConfigID:   0, Epoch:   2, Training Loss: 0.19410051, Validation Loss: 0.18798954,                    Best: 0.18798954, StopCount/Limit:   0/  3, Time:9.44335604\n",
      "ConfigID:   0, Epoch:   3, Training Loss: 0.17553105, Validation Loss: 0.16571614,                    Best: 0.16571614, StopCount/Limit:   0/  3, Time:9.52237511\n",
      "ConfigID:   0, Epoch:   4, Training Loss: 0.16661783, Validation Loss: 0.16400900,                    Best: 0.16400900, StopCount/Limit:   0/  3, Time:10.66208506\n",
      "ConfigID:   0, Epoch:   5, Training Loss: 0.16342100, Validation Loss: 0.16608587,                    Best: 0.16400900, StopCount/Limit:   1/  3, Time:10.67668319\n",
      "ConfigID:   0, Epoch:   6, Training Loss: 0.16356787, Validation Loss: 0.15859116,                    Best: 0.15859116, StopCount/Limit:   1/  3, Time:9.43718910\n",
      "ConfigID:   0, Epoch:   7, Training Loss: 0.16090530, Validation Loss: 0.16411212,                    Best: 0.15859116, StopCount/Limit:   2/  3, Time:9.56839108\n",
      "ConfigID:   0, Epoch:   8, Training Loss: 0.15806563, Validation Loss: 0.16062860,                    Best: 0.15859116, StopCount/Limit:   3/  3, Time:9.29672980\n",
      "Next Gen: [784   9   5   4   4   6  10]\n",
      "ConfigID:   1, Epoch:   0, Training Loss: 0.32548546, Validation Loss: 0.28630893,                    Best: 0.28630893, StopCount/Limit:   0/  3, Time:7.37000608\n",
      "ConfigID:   1, Epoch:   1, Training Loss: 0.24792777, Validation Loss: 0.21090576,                    Best: 0.21090576, StopCount/Limit:   0/  3, Time:7.27393293\n",
      "ConfigID:   1, Epoch:   2, Training Loss: 0.20515853, Validation Loss: 0.21278340,                    Best: 0.21090576, StopCount/Limit:   1/  3, Time:7.07597899\n",
      "ConfigID:   1, Epoch:   3, Training Loss: 0.20027797, Validation Loss: 0.20083004,                    Best: 0.20083004, StopCount/Limit:   1/  3, Time:7.21685696\n",
      "ConfigID:   1, Epoch:   4, Training Loss: 0.19730022, Validation Loss: 0.19880512,                    Best: 0.19880512, StopCount/Limit:   1/  3, Time:7.18723893\n",
      "ConfigID:   1, Epoch:   5, Training Loss: 0.19519354, Validation Loss: 0.19926663,                    Best: 0.19880512, StopCount/Limit:   2/  3, Time:7.23003578\n",
      "ConfigID:   1, Epoch:   6, Training Loss: 0.19310501, Validation Loss: 0.19913676,                    Best: 0.19880512, StopCount/Limit:   3/  3, Time:7.09718394\n",
      "Next Gen: [784   8   4   5   5   6   7   8   4   5  10]\n",
      "ConfigID:   2, Epoch:   0, Training Loss: 0.33881395, Validation Loss: 0.31732458,                    Best: 0.31732458, StopCount/Limit:   0/  3, Time:10.44975424\n",
      "ConfigID:   2, Epoch:   1, Training Loss: 0.29584712, Validation Loss: 0.27818401,                    Best: 0.27818401, StopCount/Limit:   0/  3, Time:10.26838613\n",
      "ConfigID:   2, Epoch:   2, Training Loss: 0.26995744, Validation Loss: 0.26770221,                    Best: 0.26770221, StopCount/Limit:   0/  3, Time:10.01288795\n",
      "ConfigID:   2, Epoch:   3, Training Loss: 0.26416007, Validation Loss: 0.26634229,                    Best: 0.26634229, StopCount/Limit:   0/  3, Time:10.19998503\n",
      "ConfigID:   2, Epoch:   4, Training Loss: 0.26137323, Validation Loss: 0.26416441,                    Best: 0.26416441, StopCount/Limit:   0/  3, Time:10.15100884\n",
      "ConfigID:   2, Epoch:   5, Training Loss: 0.25959408, Validation Loss: 0.26225214,                    Best: 0.26225214, StopCount/Limit:   0/  3, Time:9.95557380\n",
      "ConfigID:   2, Epoch:   6, Training Loss: 0.25829524, Validation Loss: 0.25987152,                    Best: 0.25987152, StopCount/Limit:   0/  3, Time:9.92914820\n",
      "ConfigID:   2, Epoch:   7, Training Loss: 0.25685243, Validation Loss: 0.25968380,                    Best: 0.25968380, StopCount/Limit:   0/  3, Time:14.68882513\n",
      "ConfigID:   2, Epoch:   8, Training Loss: 0.25523765, Validation Loss: 0.26111560,                    Best: 0.25968380, StopCount/Limit:   1/  3, Time:17.37135100\n",
      "ConfigID:   2, Epoch:   9, Training Loss: 0.25373624, Validation Loss: 0.25521961,                    Best: 0.25521961, StopCount/Limit:   1/  3, Time:10.35020494\n",
      "ConfigID:   2, Epoch:  10, Training Loss: 0.25305255, Validation Loss: 0.26210358,                    Best: 0.25521961, StopCount/Limit:   2/  3, Time:10.16343498\n",
      "ConfigID:   2, Epoch:  11, Training Loss: 0.25216298, Validation Loss: 0.25565776,                    Best: 0.25521961, StopCount/Limit:   3/  3, Time:10.11847782\n",
      "Gen 5 Run End\n",
      "Run   5 Best: 0.07929997807461768, Mean: 0.10317273633508012, ID:0, config: [784   9   4   5   5   6   9   4   5  10], updatecount:   0, Generation RunTime: 271.79614496\n",
      "Gen 6 Run Start\n",
      "Next Gen: [784  10   4   4   4   6  10   4   4   5   9  10]\n",
      "ConfigID:   0, Epoch:   0, Training Loss: 0.27899917, Validation Loss: 0.22558780,                    Best: 0.22558780, StopCount/Limit:   0/  3, Time:11.43687391\n",
      "ConfigID:   0, Epoch:   1, Training Loss: 0.19124316, Validation Loss: 0.17229143,                    Best: 0.17229143, StopCount/Limit:   0/  3, Time:10.79900980\n",
      "ConfigID:   0, Epoch:   2, Training Loss: 0.16566415, Validation Loss: 0.15656445,                    Best: 0.15656445, StopCount/Limit:   0/  3, Time:10.75519180\n",
      "ConfigID:   0, Epoch:   3, Training Loss: 0.15547307, Validation Loss: 0.15480842,                    Best: 0.15480842, StopCount/Limit:   0/  3, Time:10.66414905\n",
      "ConfigID:   0, Epoch:   4, Training Loss: 0.14780318, Validation Loss: 0.14105173,                    Best: 0.14105173, StopCount/Limit:   0/  3, Time:10.61430812\n",
      "ConfigID:   0, Epoch:   5, Training Loss: 0.14195215, Validation Loss: 0.14661663,                    Best: 0.14105173, StopCount/Limit:   1/  3, Time:10.61215520\n",
      "ConfigID:   0, Epoch:   6, Training Loss: 0.13853281, Validation Loss: 0.14054186,                    Best: 0.14054186, StopCount/Limit:   1/  3, Time:10.74076390\n",
      "ConfigID:   0, Epoch:   7, Training Loss: 0.13547302, Validation Loss: 0.12877798,                    Best: 0.12877798, StopCount/Limit:   1/  3, Time:10.55425286\n",
      "ConfigID:   0, Epoch:   8, Training Loss: 0.13309436, Validation Loss: 0.13167049,                    Best: 0.12877798, StopCount/Limit:   2/  3, Time:11.09588695\n",
      "ConfigID:   0, Epoch:   9, Training Loss: 0.12953494, Validation Loss: 0.12676677,                    Best: 0.12676677, StopCount/Limit:   2/  3, Time:9.64681292\n",
      "ConfigID:   0, Epoch:  10, Training Loss: 0.12685756, Validation Loss: 0.12347321,                    Best: 0.12347321, StopCount/Limit:   2/  3, Time:14.60648012\n",
      "ConfigID:   0, Epoch:  11, Training Loss: 0.12572211, Validation Loss: 0.12031860,                    Best: 0.12031860, StopCount/Limit:   2/  3, Time:11.09570622\n",
      "ConfigID:   0, Epoch:  12, Training Loss: 0.12119193, Validation Loss: 0.12209638,                    Best: 0.12031860, StopCount/Limit:   3/  3, Time:12.45533586\n",
      "Next Gen: [784  10   4   4   4   6  10   4   4   5   9  10]\n",
      "ConfigID:   1, Epoch:   0, Training Loss: 0.33445909, Validation Loss: 0.32511493,                    Best: 0.32511493, StopCount/Limit:   0/  3, Time:10.57331109\n",
      "ConfigID:   1, Epoch:   1, Training Loss: 0.32506271, Validation Loss: 0.32516556,                    Best: 0.32511493, StopCount/Limit:   1/  3, Time:10.39564586\n",
      "ConfigID:   1, Epoch:   2, Training Loss: 0.32506298, Validation Loss: 0.32502911,                    Best: 0.32502911, StopCount/Limit:   1/  3, Time:10.37300301\n",
      "ConfigID:   1, Epoch:   3, Training Loss: 0.32508166, Validation Loss: 0.32492930,                    Best: 0.32492930, StopCount/Limit:   1/  3, Time:10.46296620\n",
      "ConfigID:   1, Epoch:   4, Training Loss: 0.32504546, Validation Loss: 0.32520615,                    Best: 0.32492930, StopCount/Limit:   2/  3, Time:10.52707219\n",
      "ConfigID:   1, Epoch:   5, Training Loss: 0.32503148, Validation Loss: 0.32507730,                    Best: 0.32492930, StopCount/Limit:   3/  3, Time:10.34683609\n",
      "Next Gen: [784   9   4   4   4   6  10   4   4   5   8  10]\n",
      "ConfigID:   2, Epoch:   0, Training Loss: 0.29753843, Validation Loss: 0.24736848,                    Best: 0.24736848, StopCount/Limit:   0/  3, Time:10.53317690\n",
      "ConfigID:   2, Epoch:   1, Training Loss: 0.23250993, Validation Loss: 0.21414250,                    Best: 0.21414250, StopCount/Limit:   0/  3, Time:10.50790095\n",
      "ConfigID:   2, Epoch:   2, Training Loss: 0.20773358, Validation Loss: 0.20252443,                    Best: 0.20252443, StopCount/Limit:   0/  3, Time:10.67451429\n",
      "ConfigID:   2, Epoch:   3, Training Loss: 0.20120858, Validation Loss: 0.19940784,                    Best: 0.19940784, StopCount/Limit:   0/  3, Time:12.36291814\n",
      "ConfigID:   2, Epoch:   4, Training Loss: 0.19538913, Validation Loss: 0.19149823,                    Best: 0.19149823, StopCount/Limit:   0/  3, Time:10.76725578\n",
      "ConfigID:   2, Epoch:   5, Training Loss: 0.18809458, Validation Loss: 0.18237734,                    Best: 0.18237734, StopCount/Limit:   0/  3, Time:10.63811898\n",
      "ConfigID:   2, Epoch:   6, Training Loss: 0.17784457, Validation Loss: 0.17428864,                    Best: 0.17428864, StopCount/Limit:   0/  3, Time:10.57508087\n",
      "ConfigID:   2, Epoch:   7, Training Loss: 0.17059074, Validation Loss: 0.16736393,                    Best: 0.16736393, StopCount/Limit:   0/  3, Time:10.53842688\n",
      "ConfigID:   2, Epoch:   8, Training Loss: 0.16652307, Validation Loss: 0.17412796,                    Best: 0.16736393, StopCount/Limit:   1/  3, Time:10.60064697\n",
      "ConfigID:   2, Epoch:   9, Training Loss: 0.16334563, Validation Loss: 0.17193644,                    Best: 0.16736393, StopCount/Limit:   2/  3, Time:11.08182216\n",
      "ConfigID:   2, Epoch:  10, Training Loss: 0.16003207, Validation Loss: 0.16125428,                    Best: 0.16125428, StopCount/Limit:   2/  3, Time:12.38416100\n",
      "ConfigID:   2, Epoch:  11, Training Loss: 0.15629731, Validation Loss: 0.16425724,                    Best: 0.16125428, StopCount/Limit:   3/  3, Time:10.69427681\n",
      "Gen 6 Run End\n",
      "Run   6 Best: 0.07929997807461768, Mean: 0.10317273633508012, ID:0, config: [784   9   4   5   5   6   9   4   5  10], updatecount:   0, Generation RunTime: 339.14623594\n",
      "Gen 7 Run Start\n",
      "Next Gen: [784  10   4   4   4   6   7   5  10  10]\n",
      "ConfigID:   0, Epoch:   0, Training Loss: 0.24636830, Validation Loss: 0.16704659,                    Best: 0.16704659, StopCount/Limit:   0/  3, Time:9.15324092\n",
      "ConfigID:   0, Epoch:   1, Training Loss: 0.15049559, Validation Loss: 0.13354363,                    Best: 0.13354363, StopCount/Limit:   0/  3, Time:9.18385506\n",
      "ConfigID:   0, Epoch:   2, Training Loss: 0.13095366, Validation Loss: 0.13148263,                    Best: 0.13148263, StopCount/Limit:   0/  3, Time:10.35628700\n",
      "ConfigID:   0, Epoch:   3, Training Loss: 0.12258641, Validation Loss: 0.11757117,                    Best: 0.11757117, StopCount/Limit:   0/  3, Time:9.73722100\n",
      "ConfigID:   0, Epoch:   4, Training Loss: 0.11396909, Validation Loss: 0.11029378,                    Best: 0.11029378, StopCount/Limit:   0/  3, Time:9.36263824\n",
      "ConfigID:   0, Epoch:   5, Training Loss: 0.10800624, Validation Loss: 0.11092692,                    Best: 0.11029378, StopCount/Limit:   1/  3, Time:9.27340984\n",
      "ConfigID:   0, Epoch:   6, Training Loss: 0.10484430, Validation Loss: 0.10356497,                    Best: 0.10356497, StopCount/Limit:   1/  3, Time:9.12520194\n",
      "ConfigID:   0, Epoch:   7, Training Loss: 0.10184100, Validation Loss: 0.10047118,                    Best: 0.10047118, StopCount/Limit:   1/  3, Time:9.33005500\n",
      "ConfigID:   0, Epoch:   8, Training Loss: 0.09796345, Validation Loss: 0.09967922,                    Best: 0.09967922, StopCount/Limit:   1/  3, Time:9.29848719\n",
      "ConfigID:   0, Epoch:   9, Training Loss: 0.09706649, Validation Loss: 0.09479570,                    Best: 0.09479570, StopCount/Limit:   1/  3, Time:9.31848311\n",
      "ConfigID:   0, Epoch:  10, Training Loss: 0.09518758, Validation Loss: 0.09469336,                    Best: 0.09469336, StopCount/Limit:   1/  3, Time:9.24174213\n",
      "ConfigID:   0, Epoch:  11, Training Loss: 0.09381809, Validation Loss: 0.09743224,                    Best: 0.09469336, StopCount/Limit:   2/  3, Time:9.29420090\n",
      "ConfigID:   0, Epoch:  12, Training Loss: 0.09230255, Validation Loss: 0.09207558,                    Best: 0.09207558, StopCount/Limit:   2/  3, Time:9.33712101\n",
      "ConfigID:   0, Epoch:  13, Training Loss: 0.08593299, Validation Loss: 0.08670492,                    Best: 0.08670492, StopCount/Limit:   2/  3, Time:9.88095212\n",
      "ConfigID:   0, Epoch:  14, Training Loss: 0.08276783, Validation Loss: 0.08093053,                    Best: 0.08093053, StopCount/Limit:   2/  3, Time:9.42132306\n",
      "ConfigID:   0, Epoch:  15, Training Loss: 0.08119141, Validation Loss: 0.08009711,                    Best: 0.08009711, StopCount/Limit:   2/  3, Time:9.37830400\n",
      "ConfigID:   0, Epoch:  16, Training Loss: 0.08048443, Validation Loss: 0.08132513,                    Best: 0.08009711, StopCount/Limit:   3/  3, Time:9.34871292\n",
      "Next Gen: [784   8   4   5   5   6   7  10]\n",
      "ConfigID:   1, Epoch:   0, Training Loss: 0.33439524, Validation Loss: 0.32227591,                    Best: 0.32227591, StopCount/Limit:   0/  3, Time:11.03256392\n",
      "ConfigID:   1, Epoch:   1, Training Loss: 0.30097394, Validation Loss: 0.28097604,                    Best: 0.28097604, StopCount/Limit:   0/  3, Time:9.11379814\n",
      "ConfigID:   1, Epoch:   2, Training Loss: 0.27281540, Validation Loss: 0.27161542,                    Best: 0.27161542, StopCount/Limit:   0/  3, Time:8.12016201\n",
      "ConfigID:   1, Epoch:   3, Training Loss: 0.26825732, Validation Loss: 0.27127600,                    Best: 0.27127600, StopCount/Limit:   0/  3, Time:7.79212284\n",
      "ConfigID:   1, Epoch:   4, Training Loss: 0.26391149, Validation Loss: 0.24562510,                    Best: 0.24562510, StopCount/Limit:   0/  3, Time:7.90183091\n",
      "ConfigID:   1, Epoch:   5, Training Loss: 0.23562156, Validation Loss: 0.23511057,                    Best: 0.23511057, StopCount/Limit:   0/  3, Time:8.09246683\n",
      "ConfigID:   1, Epoch:   6, Training Loss: 0.22833479, Validation Loss: 0.23216997,                    Best: 0.23216997, StopCount/Limit:   0/  3, Time:8.08538985\n",
      "ConfigID:   1, Epoch:   7, Training Loss: 0.22049602, Validation Loss: 0.22193490,                    Best: 0.22193490, StopCount/Limit:   0/  3, Time:7.68333006\n",
      "ConfigID:   1, Epoch:   8, Training Loss: 0.21495959, Validation Loss: 0.22359739,                    Best: 0.22193490, StopCount/Limit:   1/  3, Time:7.68295503\n",
      "ConfigID:   1, Epoch:   9, Training Loss: 0.21155642, Validation Loss: 0.22196746,                    Best: 0.22193490, StopCount/Limit:   2/  3, Time:7.70891762\n",
      "ConfigID:   1, Epoch:  10, Training Loss: 0.20927303, Validation Loss: 0.22122475,                    Best: 0.22122475, StopCount/Limit:   2/  3, Time:7.78574014\n",
      "ConfigID:   1, Epoch:  11, Training Loss: 0.20656638, Validation Loss: 0.21968314,                    Best: 0.21968314, StopCount/Limit:   2/  3, Time:7.96867704\n",
      "ConfigID:   1, Epoch:  12, Training Loss: 0.20532990, Validation Loss: 0.21517895,                    Best: 0.21517895, StopCount/Limit:   2/  3, Time:7.81048012\n",
      "ConfigID:   1, Epoch:  13, Training Loss: 0.20425814, Validation Loss: 0.22646228,                    Best: 0.21517895, StopCount/Limit:   3/  3, Time:7.82248998\n",
      "Next Gen: [784   8   4   4   4   6  10   4   4   4  10]\n",
      "ConfigID:   2, Epoch:   0, Training Loss: 0.33739507, Validation Loss: 0.32496178,                    Best: 0.32496178, StopCount/Limit:   0/  3, Time:9.79596710\n",
      "ConfigID:   2, Epoch:   1, Training Loss: 0.32507514, Validation Loss: 0.32505399,                    Best: 0.32496178, StopCount/Limit:   1/  3, Time:9.78539610\n",
      "ConfigID:   2, Epoch:   2, Training Loss: 0.32506027, Validation Loss: 0.32497581,                    Best: 0.32496178, StopCount/Limit:   2/  3, Time:9.84438276\n",
      "ConfigID:   2, Epoch:   3, Training Loss: 0.32503500, Validation Loss: 0.32509876,                    Best: 0.32496178, StopCount/Limit:   3/  3, Time:10.33734226\n",
      "Gen 7 Run End\n",
      "Run   7 Best: 0.07929997807461768, Mean: 0.10317273633508012, ID:0, config: [784   9   4   5   5   6   9   4   5  10], updatecount:   0, Generation RunTime: 314.43521690\n",
      "Gen 8 Run Start\n",
      "Next Gen: [784   8   5   5   5   6   8   5   5  10]\n",
      "ConfigID:   0, Epoch:   0, Training Loss: 0.27220513, Validation Loss: 0.22007918,                    Best: 0.22007918, StopCount/Limit:   0/  3, Time:9.19776511\n",
      "ConfigID:   0, Epoch:   1, Training Loss: 0.21619198, Validation Loss: 0.21100464,                    Best: 0.21100464, StopCount/Limit:   0/  3, Time:11.02519202\n",
      "ConfigID:   0, Epoch:   2, Training Loss: 0.20754413, Validation Loss: 0.20369762,                    Best: 0.20369762, StopCount/Limit:   0/  3, Time:10.34710884\n",
      "ConfigID:   0, Epoch:   3, Training Loss: 0.20001851, Validation Loss: 0.18939317,                    Best: 0.18939317, StopCount/Limit:   0/  3, Time:9.34450579\n",
      "ConfigID:   0, Epoch:   4, Training Loss: 0.18606908, Validation Loss: 0.18142290,                    Best: 0.18142290, StopCount/Limit:   0/  3, Time:10.21607399\n",
      "ConfigID:   0, Epoch:   5, Training Loss: 0.17697727, Validation Loss: 0.17251612,                    Best: 0.17251612, StopCount/Limit:   0/  3, Time:10.24036026\n",
      "ConfigID:   0, Epoch:   6, Training Loss: 0.16992298, Validation Loss: 0.16168946,                    Best: 0.16168946, StopCount/Limit:   0/  3, Time:14.17906213\n",
      "ConfigID:   0, Epoch:   7, Training Loss: 0.16117041, Validation Loss: 0.15666595,                    Best: 0.15666595, StopCount/Limit:   0/  3, Time:10.23282671\n",
      "ConfigID:   0, Epoch:   8, Training Loss: 0.15767106, Validation Loss: 0.15681290,                    Best: 0.15666595, StopCount/Limit:   1/  3, Time:9.44261694\n",
      "ConfigID:   0, Epoch:   9, Training Loss: 0.15358462, Validation Loss: 0.15022626,                    Best: 0.15022626, StopCount/Limit:   1/  3, Time:9.64123797\n",
      "ConfigID:   0, Epoch:  10, Training Loss: 0.14742567, Validation Loss: 0.14624725,                    Best: 0.14624725, StopCount/Limit:   1/  3, Time:9.55876994\n",
      "ConfigID:   0, Epoch:  11, Training Loss: 0.14292493, Validation Loss: 0.14370016,                    Best: 0.14370016, StopCount/Limit:   1/  3, Time:9.91456008\n",
      "ConfigID:   0, Epoch:  12, Training Loss: 0.14221970, Validation Loss: 0.14031448,                    Best: 0.14031448, StopCount/Limit:   1/  3, Time:9.42741299\n",
      "ConfigID:   0, Epoch:  13, Training Loss: 0.14079823, Validation Loss: 0.13670629,                    Best: 0.13670629, StopCount/Limit:   1/  3, Time:9.50470781\n",
      "ConfigID:   0, Epoch:  14, Training Loss: 0.13936543, Validation Loss: 0.13684707,                    Best: 0.13670629, StopCount/Limit:   2/  3, Time:9.38733220\n",
      "ConfigID:   0, Epoch:  15, Training Loss: 0.13932522, Validation Loss: 0.14206895,                    Best: 0.13670629, StopCount/Limit:   3/  3, Time:9.30614400\n",
      "Next Gen: [784   9   4   4   4   6  10   4   4   4   4  10]\n",
      "ConfigID:   1, Epoch:   0, Training Loss: 0.34223478, Validation Loss: 0.32496673,                    Best: 0.32496673, StopCount/Limit:   0/  3, Time:10.45386910\n",
      "ConfigID:   1, Epoch:   1, Training Loss: 0.32500648, Validation Loss: 0.32501019,                    Best: 0.32496673, StopCount/Limit:   1/  3, Time:10.87627411\n",
      "ConfigID:   1, Epoch:   2, Training Loss: 0.32500275, Validation Loss: 0.32508169,                    Best: 0.32496673, StopCount/Limit:   2/  3, Time:10.55273104\n",
      "ConfigID:   1, Epoch:   3, Training Loss: 0.32500748, Validation Loss: 0.32506077,                    Best: 0.32496673, StopCount/Limit:   3/  3, Time:10.47878814\n",
      "Next Gen: [784   9   4   4   4   6  10   4   5   6   4  10]\n",
      "ConfigID:   2, Epoch:   0, Training Loss: 0.31398034, Validation Loss: 0.26756187,                    Best: 0.26756187, StopCount/Limit:   0/  3, Time:14.66042876\n",
      "ConfigID:   2, Epoch:   1, Training Loss: 0.25939737, Validation Loss: 0.25996590,                    Best: 0.25996590, StopCount/Limit:   0/  3, Time:10.96003366\n",
      "ConfigID:   2, Epoch:   2, Training Loss: 0.23294051, Validation Loss: 0.21000092,                    Best: 0.21000092, StopCount/Limit:   0/  3, Time:10.84686184\n",
      "ConfigID:   2, Epoch:   3, Training Loss: 0.19690566, Validation Loss: 0.19581363,                    Best: 0.19581363, StopCount/Limit:   0/  3, Time:11.01515603\n",
      "ConfigID:   2, Epoch:   4, Training Loss: 0.18920050, Validation Loss: 0.18996909,                    Best: 0.18996909, StopCount/Limit:   0/  3, Time:10.83345604\n",
      "ConfigID:   2, Epoch:   5, Training Loss: 0.18520092, Validation Loss: 0.18840712,                    Best: 0.18840712, StopCount/Limit:   0/  3, Time:10.84853101\n",
      "ConfigID:   2, Epoch:   6, Training Loss: 0.18282335, Validation Loss: 0.19013452,                    Best: 0.18840712, StopCount/Limit:   1/  3, Time:12.69540906\n",
      "ConfigID:   2, Epoch:   7, Training Loss: 0.18103647, Validation Loss: 0.18440898,                    Best: 0.18440898, StopCount/Limit:   1/  3, Time:13.85177398\n",
      "ConfigID:   2, Epoch:   8, Training Loss: 0.17910856, Validation Loss: 0.18825468,                    Best: 0.18440898, StopCount/Limit:   2/  3, Time:13.74781609\n",
      "ConfigID:   2, Epoch:   9, Training Loss: 0.17756561, Validation Loss: 0.18485852,                    Best: 0.18440898, StopCount/Limit:   3/  3, Time:16.09307170\n",
      "Gen 8 Run End\n",
      "Run   8 Best: 0.07929997807461768, Mean: 0.10317273633508012, ID:0, config: [784   9   4   5   5   6   9   4   5  10], updatecount:   0, Generation RunTime: 328.90917087\n",
      "Gen 9 Run Start\n",
      "Next Gen: [784   9   5   5   5   6   9   4   5  10]\n",
      "ConfigID:   0, Epoch:   0, Training Loss: 0.33935048, Validation Loss: 0.32495067,                    Best: 0.32495067, StopCount/Limit:   0/  3, Time:12.10118484\n",
      "ConfigID:   0, Epoch:   1, Training Loss: 0.32502390, Validation Loss: 0.32497925,                    Best: 0.32495067, StopCount/Limit:   1/  3, Time:9.74355102\n",
      "ConfigID:   0, Epoch:   2, Training Loss: 0.32503989, Validation Loss: 0.32506524,                    Best: 0.32495067, StopCount/Limit:   2/  3, Time:9.61818194\n",
      "ConfigID:   0, Epoch:   3, Training Loss: 0.32505527, Validation Loss: 0.32497708,                    Best: 0.32495067, StopCount/Limit:   3/  3, Time:9.67109704\n",
      "Next Gen: [784  10   4   4   4   6  10   4   4   4   5  10]\n",
      "ConfigID:   1, Epoch:   0, Training Loss: 0.26528915, Validation Loss: 0.22144433,                    Best: 0.22144433, StopCount/Limit:   0/  3, Time:10.97419596\n",
      "ConfigID:   1, Epoch:   1, Training Loss: 0.20785122, Validation Loss: 0.19710351,                    Best: 0.19710351, StopCount/Limit:   0/  3, Time:12.33420706\n",
      "ConfigID:   1, Epoch:   2, Training Loss: 0.18741728, Validation Loss: 0.18117108,                    Best: 0.18117108, StopCount/Limit:   0/  3, Time:10.93106318\n",
      "ConfigID:   1, Epoch:   3, Training Loss: 0.17691917, Validation Loss: 0.17927649,                    Best: 0.17927649, StopCount/Limit:   0/  3, Time:11.37641597\n",
      "ConfigID:   1, Epoch:   4, Training Loss: 0.17111752, Validation Loss: 0.16890419,                    Best: 0.16890419, StopCount/Limit:   0/  3, Time:11.64018726\n",
      "ConfigID:   1, Epoch:   5, Training Loss: 0.16729769, Validation Loss: 0.16613039,                    Best: 0.16613039, StopCount/Limit:   0/  3, Time:11.02435374\n",
      "ConfigID:   1, Epoch:   6, Training Loss: 0.16146473, Validation Loss: 0.15903660,                    Best: 0.15903660, StopCount/Limit:   0/  3, Time:10.80018401\n",
      "ConfigID:   1, Epoch:   7, Training Loss: 0.15097748, Validation Loss: 0.14419688,                    Best: 0.14419688, StopCount/Limit:   0/  3, Time:10.87213206\n",
      "ConfigID:   1, Epoch:   8, Training Loss: 0.13983149, Validation Loss: 0.14289352,                    Best: 0.14289352, StopCount/Limit:   0/  3, Time:10.70062065\n",
      "ConfigID:   1, Epoch:   9, Training Loss: 0.13466742, Validation Loss: 0.13514901,                    Best: 0.13514901, StopCount/Limit:   0/  3, Time:10.73497820\n",
      "ConfigID:   1, Epoch:  10, Training Loss: 0.13002378, Validation Loss: 0.13410114,                    Best: 0.13410114, StopCount/Limit:   0/  3, Time:10.88752198\n",
      "ConfigID:   1, Epoch:  11, Training Loss: 0.11872038, Validation Loss: 0.12192000,                    Best: 0.12192000, StopCount/Limit:   0/  3, Time:10.81528902\n",
      "ConfigID:   1, Epoch:  12, Training Loss: 0.11426362, Validation Loss: 0.11370727,                    Best: 0.11370727, StopCount/Limit:   0/  3, Time:10.91560078\n",
      "ConfigID:   1, Epoch:  13, Training Loss: 0.11177029, Validation Loss: 0.11624483,                    Best: 0.11370727, StopCount/Limit:   1/  3, Time:10.81733203\n",
      "ConfigID:   1, Epoch:  14, Training Loss: 0.11311247, Validation Loss: 0.11389116,                    Best: 0.11370727, StopCount/Limit:   2/  3, Time:10.83505630\n",
      "ConfigID:   1, Epoch:  15, Training Loss: 0.10844963, Validation Loss: 0.11150489,                    Best: 0.11150489, StopCount/Limit:   2/  3, Time:10.81566525\n",
      "ConfigID:   1, Epoch:  16, Training Loss: 0.10753225, Validation Loss: 0.10560526,                    Best: 0.10560526, StopCount/Limit:   2/  3, Time:11.26612210\n",
      "ConfigID:   1, Epoch:  17, Training Loss: 0.10581539, Validation Loss: 0.10764058,                    Best: 0.10560526, StopCount/Limit:   3/  3, Time:11.59201407\n",
      "Next Gen: [784  10   4   4   4   6  10   4   4   4   5  10]\n",
      "ConfigID:   2, Epoch:   0, Training Loss: 0.23845021, Validation Loss: 0.16733924,                    Best: 0.16733924, StopCount/Limit:   0/  3, Time:15.05499816\n",
      "ConfigID:   2, Epoch:   1, Training Loss: 0.15645515, Validation Loss: 0.13826808,                    Best: 0.13826808, StopCount/Limit:   0/  3, Time:10.78567982\n",
      "ConfigID:   2, Epoch:   2, Training Loss: 0.12927403, Validation Loss: 0.11591259,                    Best: 0.11591259, StopCount/Limit:   0/  3, Time:10.86297011\n",
      "ConfigID:   2, Epoch:   3, Training Loss: 0.11441727, Validation Loss: 0.10547776,                    Best: 0.10547776, StopCount/Limit:   0/  3, Time:11.47930813\n",
      "ConfigID:   2, Epoch:   4, Training Loss: 0.09901820, Validation Loss: 0.09628053,                    Best: 0.09628053, StopCount/Limit:   0/  3, Time:10.93947077\n",
      "ConfigID:   2, Epoch:   5, Training Loss: 0.09592544, Validation Loss: 0.09054091,                    Best: 0.09054091, StopCount/Limit:   0/  3, Time:10.81800199\n",
      "ConfigID:   2, Epoch:   6, Training Loss: 0.09226479, Validation Loss: 0.09222959,                    Best: 0.09054091, StopCount/Limit:   1/  3, Time:10.76520491\n",
      "ConfigID:   2, Epoch:   7, Training Loss: 0.09179446, Validation Loss: 0.09042752,                    Best: 0.09042752, StopCount/Limit:   1/  3, Time:10.81980896\n",
      "ConfigID:   2, Epoch:   8, Training Loss: 0.08257299, Validation Loss: 0.07953897,                    Best: 0.07953897, StopCount/Limit:   1/  3, Time:10.93914318\n",
      "ConfigID:   2, Epoch:   9, Training Loss: 0.07840238, Validation Loss: 0.08007113,                    Best: 0.07953897, StopCount/Limit:   2/  3, Time:10.87706518\n",
      "ConfigID:   2, Epoch:  10, Training Loss: 0.07521411, Validation Loss: 0.07713584,                    Best: 0.07713584, StopCount/Limit:   2/  3, Time:10.89199066\n",
      "ConfigID:   2, Epoch:  11, Training Loss: 0.07427025, Validation Loss: 0.07224944,                    Best: 0.07224944, StopCount/Limit:   2/  3, Time:10.95111108\n",
      "ConfigID:   2, Epoch:  12, Training Loss: 0.07380312, Validation Loss: 0.07949645,                    Best: 0.07224944, StopCount/Limit:   3/  3, Time:11.83742118\n",
      "Gen 9 Run End\n",
      "Run   9 Best: 0.07224943741853349, Mean: 0.08197025916391673, ID:2, config: [784  10   4   4   4   6  10   4   4   4   5  10], updatecount:   1, Generation RunTime: 387.52803087\n",
      "Run Completed : Best Score(loss): 0.07224943741853349 , Config: [784  10   4   4   4   6  10   4   4   4   5  10]\n"
     ]
    }
   ],
   "source": [
    "d.run()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6df60840",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}