diff --git a/__generated/trainerfunc.jl b/__generated/trainerfunc.jl index 4cf6781..6738652 100644 --- a/__generated/trainerfunc.jl +++ b/__generated/trainerfunc.jl @@ -26,18 +26,21 @@ function trainer(m, func, ln::Real=5) schedule = Interpolator(Step(initial_lr, 0.5, [25, 10]), es) optim = func(initial_lr) - logger = TensorBoardBackend("tblogs") - - logcb = (LogMetrics(logger),)# LogHyperParams(logger)) - valcb = Metrics(Metric(accfn; phase = TrainingPhase, name = "train_acc"), - Metric(accfn; phase = ValidationPhase, name = "val_acc")) - learner = Learner(m, lossfn; - data = (trainloader, valloader), - optimizer = optim, - callbacks = [ToGPU(), logcb..., valcb]) - - FluxTraining.fit!(learner, ln) - close(logger.logger) + logger = TensorBoardBackend("tblog") + try + + logcb = (LogMetrics(logger),)# LogHyperParams(logger)) + valcb = Metrics(Metric(accfn; phase = TrainingPhase, name = "train_acc"), + Metric(accfn; phase = ValidationPhase, name = "val_acc")) + learner = Learner(m, lossfn; + data = (trainloader, valloader), + optimizer = optim, + callbacks = [ToGPU(), logcb..., valcb]) + + FluxTraining.fit!(learner, ln) + finally + close(logger.logger) + end # save model m = learner.model |> cpu diff --git a/_tutorials/trainerfunc.jl b/_tutorials/trainerfunc.jl index 08dc051..74fdfdf 100644 --- a/_tutorials/trainerfunc.jl +++ b/_tutorials/trainerfunc.jl @@ -26,18 +26,23 @@ function trainer(m, func, ln::Real=5) schedule = Interpolator(Step(initial_lr, 0.5, [25, 10]), es) optim = func(initial_lr) # callbacks - logger = TensorBoardBackend("tblogs") - # schcb = Scheduler(LearningRate => schedule) - logcb = (LogMetrics(logger),)# LogHyperParams(logger)) - valcb = Metrics(Metric(accfn; phase = TrainingPhase, name = "train_acc"), - Metric(accfn; phase = ValidationPhase, name = "val_acc")) - learner = Learner(m, lossfn; - data = (trainloader, valloader), - optimizer = optim, - callbacks = [ToGPU(), logcb..., valcb]) - FluxTraining.fit!(learner, ln) - close(logger.logger) + logger = TensorBoardBackend("tblog") + try + + # schcb = Scheduler(LearningRate => schedule) + logcb = (LogMetrics(logger),)# LogHyperParams(logger)) + valcb = Metrics(Metric(accfn; phase = TrainingPhase, name = "train_acc"), + Metric(accfn; phase = ValidationPhase, name = "val_acc")) + learner = Learner(m, lossfn; + data = (trainloader, valloader), + optimizer = optim, + callbacks = [ToGPU(), logcb..., valcb]) + + FluxTraining.fit!(learner, ln) + finally + close(logger.logger) + end ## save model m = learner.model |> cpu @@ -56,4 +61,4 @@ end #m = resize_nobn(m) #m = desaturate(m) #m = rebn(m) -#m = trainer(m, 1) \ No newline at end of file +#m = trainer(m, 1) diff --git a/assets/tutorials.tar.gz b/assets/tutorials.tar.gz index 1ff948b..fc2073c 100644 Binary files a/assets/tutorials.tar.gz and b/assets/tutorials.tar.gz differ diff --git a/assets/tutorials/bitstream/code/output/ex10.res b/assets/tutorials/bitstream/code/output/ex10.res index 5e18ba7..0220082 100644 --- a/assets/tutorials/bitstream/code/output/ex10.res +++ b/assets/tutorials/bitstream/code/output/ex10.res @@ -1 +1 @@ -0.0040000000000000036 \ No newline at end of file +0.01100000000000001 \ No newline at end of file diff --git a/assets/tutorials/bitstream/code/output/ex4.res b/assets/tutorials/bitstream/code/output/ex4.res index b2204e8..ae0c5c5 100644 --- a/assets/tutorials/bitstream/code/output/ex4.res +++ b/assets/tutorials/bitstream/code/output/ex4.res @@ -1 +1 @@ -SBit(pos = true, neg = false) \ No newline at end of file +SBit(pos = false, neg = false) \ No newline at end of file diff --git a/assets/tutorials/pruning/code/output/ex7.out b/assets/tutorials/pruning/code/output/ex7.out index 3e33968..7691d5b 100644 --- a/assets/tutorials/pruning/code/output/ex7.out +++ b/assets/tutorials/pruning/code/output/ex7.out @@ -1 +1 @@ -Propagated MobileNet Mults 4992057 Adds 4887057 +Propagated MobileNet Mults 4968641 Adds 4866825 diff --git a/assets/tutorials/pruning/code/output/ex8.out b/assets/tutorials/pruning/code/output/ex8.out index 66f77ca..12a0f28 100644 --- a/assets/tutorials/pruning/code/output/ex8.out +++ b/assets/tutorials/pruning/code/output/ex8.out @@ -1 +1 @@ -Resized MobileNet Mults 3671590 Adds 3518013 +Resized MobileNet Mults 3638364 Adds 3487521 diff --git a/tutorials/bitstream/index.html b/tutorials/bitstream/index.html index 948223b..0feefca 100644 --- a/tutorials/bitstream/index.html +++ b/tutorials/bitstream/index.html @@ -3,7 +3,7 @@ x = SBitstream(0.3)
SBitstream{Float64}(value = 0.3)
     with 0 bits.

Here, we created a SBitstream (the type in BitSAD for stochastic bitstreams) encoding the real value 0.3. SBitstream will keep track of the mean of the Bernoulli distribution, which we can recover with float.

float(x)
0.3

You'll also notice that there were "0 bits enqueue" in x. This refers to the fact that the bitstream, x, is a sequence of samples. Currently, we have not drawn any samples from x. We can try that now:

-
xt = pop!(x)
SBit(pos = true, neg = false)
+
xt = pop!(x)
SBit(pos = false, neg = false)

Now, we have a single sample, xt, which is of type SBit. An SBit is a "stochastic bit" which is just a convenient alias for a NamedTuple with two parts –- the positive part (pos) and the negative part (neg).

Wait, I thought stochastic bitstreams were a single bit sequence?
–- You (probably)

@@ -51,7 +51,7 @@

end -abs(estimate(z) - float(z))
0.0040000000000000036
+abs(estimate(z) - float(z))
0.01100000000000001

We used a helper function, multiply_sbit to multiply the positive and negative channel of each SBit separately. This resulted in a new SBit, zbit, which we pushed onto z. When we take the empirical average of all these zbits, we see that it is close to the true mean of z.

Hopefully, you can now see why stochastic computing can be so resource efficient. Each channel of multiply_sbit only needed to multiply two 1-bit numbers. This can be done with a single AND gate.

diff --git a/tutorials/pruning/index.html b/tutorials/pruning/index.html index 90a4942..f4ca407 100644 --- a/tutorials/pruning/index.html +++ b/tutorials/pruning/index.html @@ -24,13 +24,13 @@

m_pruned = keepprune(m_ch_pruned) m_prop = prune_propagate(m_pruned) mults, adds, output_size = compute_dot_prods(m_prop, (96, 96, 3, 1)) -println("Propagated MobileNet Mults ", mults, " Adds ", adds)
Propagated MobileNet Mults 4992057 Adds 4887057
+println("Propagated MobileNet Mults ", mults, " Adds ", adds)
Propagated MobileNet Mults 4968641 Adds 4866825

Resizing the propagated model

If enough nodes get pruned out, there would be slices in the model which accomplish nothing, computationally. Instead of wasting resources on passing these kernels full of zeros around, they can be eliminated from the structure of our model.

m_resized = resize(m_prop)
 mults, adds, output_size = compute_dot_prods(m_resized, (96, 96, 3, 1))
-println("Resized MobileNet Mults ", mults, " Adds ", adds)
Resized MobileNet Mults 3671590 Adds 3518013
+println("Resized MobileNet Mults ", mults, " Adds ", adds)
Resized MobileNet Mults 3638364 Adds 3487521

Pruning and Finetuning pipeline

Now that we seen how to prune our model, let's try to finetune it to recover some of the accuracy we lost. A basic template setup for training the model is provided by trainer function, and can be used as a starting point for your own training methodology.