diff --git a/CHANGELOG.md b/CHANGELOG.md index 2b606f6..24fa326 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,10 @@ +# 0.4.3 + +- Bumped version of `sprint` from `1.0.2+3` to `1.0.3`. +- Updated repository, homepage and issue tracker links. +- Refactored and made formatting and style changes to bring the project up to + par. + # 0.4.2+1 - Updated package description. diff --git a/README.md b/README.md index b951fbd..e2746ae 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,8 @@ # synadart -The `synadart` library can be used to create neural networks of any complexity, as well as learn from the source code by studying its extremely clean implementation. +The `synadart` library can be used to create neural networks of any complexity, +as well as learn from the source code by studying its extremely clean +implementation. ## Launching our first network @@ -10,25 +12,31 @@ To begin using the `synadart`, we must first import it into our project: import 'package:synadart/synadart.dart'; ``` -Next, we must create a network of our chosen type. Let's create a sequential network, in which every layer has one input and one output tensor. This should be pretty easy: +Next, we must create a network of our chosen type. Let's create a sequential +network, in which every layer has one input and one output tensor. This should +be pretty easy: ```dart final network = Sequential(learningRate: 0.3); ``` -Our network is currently empty; it contains no layers and therefore no neurons. Let's add three layers; the input layer, one hidden layer and the output layer: +Our network is currently empty; it contains no layers and therefore no neurons. +Let's add three layers; the input layer, one hidden layer and the output layer: ```dart network.addLayers([ - Dense(15, activationAlgorithm: ActivationAlgorithm.sigmoid), - Dense(5, activationAlgorithm: ActivationAlgorithm.sigmoid), - Dense(1, activationAlgorithm: ActivationAlgorithm.sigmoid), + Dense(15, activationAlgorithm: ActivationAlgorithm.sigmoid), + Dense(5, activationAlgorithm: ActivationAlgorithm.sigmoid), + Dense(1, activationAlgorithm: ActivationAlgorithm.sigmoid), ]); ``` -Now that our network has some structure to it, we can begin using it.. No, not quite yet. Our network is still not trained, and has no clue what it is doing. Time to train it. +Now that our network has some structure to it, we can begin using it.. No, not +quite yet. Our network is still not trained, and has no clue what it is doing. +Time to train it. -Firstly, we will create a list of *expected* values, i.e. *values we are expecting the network to output*. Here, we are expecting to get the number '5'. +Firstly, we will create a list of _expected_ values, i.e. _values we are +expecting the network to output_. Here, we are expecting to get the number '5'. ```dart final expected = [ @@ -45,9 +53,12 @@ final expected = [ ]; ``` -Fantastic, we are now expecting our infantile network to magically output a number 5, not having taught it a thing. Oh, right - that's where the training data part comes in! +Fantastic, we are now expecting our infantile network to magically output a +number 5, not having taught it a thing. Oh, right - that's where the training +data part comes in! -We must now tell the network what each of our expected output values is associated with. Let's teach it some numbers: +We must now tell the network what each of our expected output values is +associated with. Let's teach it some numbers: ```dart final trainingData = [ @@ -64,15 +75,19 @@ final trainingData = [ ]; ``` -Now that we granted our network a grand total of 10 numbers to learn, we can begin training the network using the values we've set up: +Now that we granted our network a grand total of 10 numbers to learn, we can +begin training the network using the values we've set up: ```dart network.train(inputs: trainingData, expected: expected, iterations: 5000); ``` -Wonderful! We've trained our network using the pixel representation of number images, and our network is now able to recognise the number '5' with relative confidence. The last step is to test our network's capabilities ourselves. +Wonderful! We've trained our network using the pixel representation of number +images, and our network is now able to recognise the number '5' with relative +confidence. The last step is to test our network's capabilities ourselves. -Let's give our network a couple pixel representations of distorted images of the number '5': +Let's give our network a couple pixel representations of distorted images of the +number '5': ```dart final testData = [ @@ -91,4 +106,4 @@ To check the confidence of the network in recognising distorted '5's: for (final test in testData) { print('Confidence in recognising a distorted 5: ${network.process(test)}'); } -``` \ No newline at end of file +``` diff --git a/example/example.dart b/example/example.dart index fb35302..dcf8a26 100644 --- a/example/example.dart +++ b/example/example.dart @@ -2,24 +2,22 @@ import 'package:synadart/src/layers/core/dense.dart'; import 'package:synadart/synadart.dart'; void main() { - final network = Sequential(learningRate: 0.2); - - network.addLayer(Dense( - size: 15, - activation: ActivationAlgorithm.sigmoid, - )); + final network = Sequential(learningRate: 0.2, layers: [ + Dense( + size: 15, + activation: ActivationAlgorithm.sigmoid, + ), + Dense( + size: 5, + activation: ActivationAlgorithm.sigmoid, + ), + Dense( + size: 1, + activation: ActivationAlgorithm.sigmoid, + ) + ]); - network.addLayer(Dense( - size: 5, - activation: ActivationAlgorithm.sigmoid, - )); - - network.addLayer(Dense( - size: 1, - activation: ActivationAlgorithm.sigmoid, - )); - - // We are expecting to get the number '5' + // We are expecting to get the number '5'. final expected = [ [0.01], [0.01], @@ -33,21 +31,22 @@ void main() { [0.01], ]; - // Training data contains different number patterns + // Training data contains different number patterns. final trainingData = [ '111101101101111'.split('').map(double.parse).toList(), '001001001001001'.split('').map(double.parse).toList(), '111001111100111'.split('').map(double.parse).toList(), '111001111001111'.split('').map(double.parse).toList(), '101101111001001'.split('').map(double.parse).toList(), - '111100111001111'.split('').map(double.parse).toList(), // This is the number 5 + // This is the number 5 + '111100111001111'.split('').map(double.parse).toList(), '111100111101111'.split('').map(double.parse).toList(), '111001001001001'.split('').map(double.parse).toList(), '111101111101111'.split('').map(double.parse).toList(), '111101111001111'.split('').map(double.parse).toList(), ]; - // Test data which contains distorted patterns of the number 5 + // Test data which contains distorted patterns of the number 5. final testData = [ '111100111000111'.split('').map(double.parse).toList(), '111100010001111'.split('').map(double.parse).toList(), @@ -57,10 +56,10 @@ void main() { '111100101001111'.split('').map(double.parse).toList(), ]; - // The number 5 itself + // The number 5 itself. final numberFive = trainingData[5]; - // Train the network using the training and expected data + // Train the network using the training and expected data. network.train(inputs: trainingData, expected: expected, iterations: 5000); print('Confidence in recognising a 5: ${network.process(numberFive)}'); diff --git a/lib/src/activation.dart b/lib/src/activation.dart index 49d4bca..1b46af5 100644 --- a/lib/src/activation.dart +++ b/lib/src/activation.dart @@ -2,129 +2,145 @@ import 'dart:math'; import 'package:synadart/src/utils/mathematical_operations.dart'; +/// Type defining an activation function taking as a parameter the function to +/// obtain the weighted sum of the inputs and the weights. typedef ActivationFunction = double Function(double Function()); -/// Resolves an `ActivationAlgorithm` to a mathematical function in the form of an `ActivationFunction` -ActivationFunction resolveActivationAlgorithm(ActivationAlgorithm activationAlgorithm) { - switch (activationAlgorithm) { - case ActivationAlgorithm.sigmoid: - return (double Function() weightedSum) => sigmoid(weightedSum()); - - case ActivationAlgorithm.relu: - return (double Function() weightedSum) => relu(weightedSum()); - case ActivationAlgorithm.lrelu: - return (double Function() weightedSum) => lrelu(weightedSum()); - case ActivationAlgorithm.elu: - return (double Function() weightedSum) => elu(weightedSum()); - case ActivationAlgorithm.selu: - return (double Function() weightedSum) => selu(weightedSum()); - - case ActivationAlgorithm.tanh: - return (double Function() weightedSum) => tanh(weightedSum()); - - case ActivationAlgorithm.softplus: - return (double Function() weightedSum) => softplus(weightedSum()); - case ActivationAlgorithm.softsign: - return (double Function() weightedSum) => softsign(weightedSum()); - case ActivationAlgorithm.swish: - return (double Function() weightedSum) => swish(weightedSum()); - case ActivationAlgorithm.gaussian: - return (double Function() weightedSum) => gaussian(weightedSum()); - } -} +/// Type defining a bare activation function. +typedef ActivationFunctionSignature = double Function(double); -/// Resolves an `ActivationAlgorithm` to the derivative of the mathematical function in -/// the form of an `ActivationFunction` -ActivationFunction resolveDerivative(ActivationAlgorithm activationAlgorithm) { - switch (activationAlgorithm) { - case ActivationAlgorithm.sigmoid: - return (double Function() weightedSum) => sigmoidPrime(weightedSum()); - - case ActivationAlgorithm.relu: - return (double Function() weightedSum) => reluPrime(weightedSum()); - case ActivationAlgorithm.lrelu: - return (double Function() weightedSum) => lreluPrime(weightedSum()); - case ActivationAlgorithm.elu: - return (double Function() weightedSum) => eluPrime(weightedSum()); - case ActivationAlgorithm.selu: - return (double Function() weightedSum) => seluPrime(weightedSum()); - - case ActivationAlgorithm.tanh: - return (double Function() weightedSum) => tanhPrime(weightedSum()); - - case ActivationAlgorithm.softplus: - return (double Function() weightedSum) => softplusPrime(weightedSum()); - case ActivationAlgorithm.softsign: - return (double Function() weightedSum) => softsignPrime(weightedSum()); - case ActivationAlgorithm.swish: - return (double Function() weightedSum) => swishPrime(weightedSum()); - case ActivationAlgorithm.gaussian: - return (double Function() weightedSum) => gaussianPrime(weightedSum()); - } -} +/// Map containing all available activation algorithms and their derivatives. +const algorithms = >{ + ActivationAlgorithm.sigmoid: [sigmoid, sigmoidPrime], + ActivationAlgorithm.relu: [relu, reluPrime], + ActivationAlgorithm.lrelu: [lrelu, lreluPrime], + ActivationAlgorithm.elu: [elu, eluPrime], + ActivationAlgorithm.selu: [selu, seluPrime], + ActivationAlgorithm.tanh: [tanh, tanhPrime], + ActivationAlgorithm.softplus: [softplus, softplusPrime], + ActivationAlgorithm.softsign: [softsign, softsignPrime], + ActivationAlgorithm.swish: [swish, swishPrime], + ActivationAlgorithm.gaussian: [gaussian, gaussianPrime], +}; + +/// Resolves an `ActivationAlgorithm` to a mathematical function in the form of +/// an `ActivationFunction`. +ActivationFunction resolveActivationAlgorithm( + ActivationAlgorithm activationAlgorithm, +) => + (weightedSum) => algorithms[activationAlgorithm]![0](weightedSum()); + +/// Resolves an `ActivationAlgorithm` to the derivative of the mathematical +/// function in the form of an `ActivationFunction` +ActivationFunction resolveActivationDerivative( + ActivationAlgorithm activationAlgorithm, +) => + (weightedSum) => algorithms[activationAlgorithm]![1](weightedSum()); -/// Shrinks the range of values to inbetween 0 and 1 using exponentials. Results can be driven into saturation, -/// which makes the sigmoid function unsuited for deep networks with random initialisation. +/// Shrinks the range of values to inbetween 0 and 1 using exponentials. Results +/// can be driven into saturation, which makes the sigmoid function unsuited for +/// deep networks with random initialisation. double sigmoid(double x) => 1 / (1 + exp(-x)); -/// Rectified Linear Unit - Negative integers adjusted to 0, leaving positive ones untouched + +/// Rectified Linear Unit - Negative integers adjusted to 0, leaving positive +/// ones untouched. double relu(double x) => max(0, x); -/// Leaky Linear Unit - Shallow line is seen in the negative x- and y-axes, instead of reducing result to 0 like ReLU. + +/// Leaky Linear Unit - Shallow line is seen in the negative x- and y-axes, +/// instead of reducing result to 0 like ReLU. double lrelu(double x) => max(0.01 * x, x); -/// Exponential Linear Unit function which provides a smooth descent below 0, towards the negative of -/// [hyperparameter], or returns [x] if above or equal to 0 -double elu(double x, [double hyperparameter = 1]) => x >= 0 ? x : hyperparameter * (exp(x) - 1); -/// Scaled Exponential Linear Unit - Ensures a slope larger than one for positive inputs + +/// Exponential Linear Unit - Provides a smooth descent below 0, towards the +/// negative of [hyperparameter], or returns [x] if above or equal to 0. +double elu(double x, [double hyperparameter = 1]) => + x >= 0 ? x : hyperparameter * (exp(x) - 1); + +/// Scaled Exponential Linear Unit - Ensures a slope larger than one for +/// positive inputs. double selu(double x) => 1.0507 * (x < 0 ? 1.67326 * (exp(x) - 1) : x); -/// Hyperbolic Tangent, which uses exponentials to shrink a range to inbetween -1 and 1, -1 and 1 being -/// the function's asymptotes, towards which the lines tend. + +/// Hyperbolic Tangent - Utilises exponentials in order to shrink a range of +/// numbers to strictly in-between -1 and 1, -1 and 1 being the mfunction's +/// asymptotes, towards which the curve tends. double tanh(double x) => (exp(x) - exp(-x)) / (exp(x) + exp(-x)); -/// Similar to ReLU, but there is a smooth (soft) curve as result approaches zero on the negative x-axis. -/// Softplus is strictly positive and monotonic. + +/// Similar to ReLU, but there is a smooth (soft) curve as the result approaches +/// zero on the negative x-axis. Softplus is strictly positive and monotonic. double softplus(double x) => log(exp(x) + 1); -/// Works similarly to the hyperbolic tangent but its tails are quadratic polynomials rather -/// than exponentials, therefore approaching its asymptotes much slower. + +/// Similar to the hyperbolic tangent, but its tails are quadratic polynomials, +/// rather than exponentials, therefore causing the curve to approach its +/// asymptotes much more slowly. double softsign(double x) => x / (1 + abs(x)); -/// Similar to ReLU and softplus, negative results do occur but approach 0 up until x ≈ -10. -/// Delivers equal or better results to ReLU. + +/// Similar to ReLU and Softplus; negative results do occur, but they approach 0 +/// up until x ≈ -10. Delivers comparable or superior results to ReLU. double swish(double x) => x * sigmoid(x); -/// Symmetrical and bell-shaped graph with a peak at 1 and a smooth approach to 0 for both sides -/// of the x-axis + +/// Symmetrical and bell-shaped graph with a peak at 1 and a smooth approach to +/// 0 for both sides of the x-axis. double gaussian(double x) => exp(-pow(x, 2)); -/// The derivative of the Sigmoid +/// The derivative of the Sigmoid. double sigmoidPrime(double x) => sigmoid(x) * (1 - sigmoid(x)); -/// The derivative of the Rectified Linear Unit + +/// The derivative of ReLU. double reluPrime(double x) => x < 0 ? 0 : 1; -/// The derivative of the Leaky Linear Unit + +/// The derivative of LReLU. double lreluPrime(double x) => x < 0 ? 0.01 : 1; -/// The derivative of the Exponential Linear Unit -double eluPrime(double x, [double hyperparameter = 1]) => x > 0 ? 1 : elu(x, hyperparameter) + hyperparameter; + +/// The derivative of ELU. +double eluPrime(double x, [double hyperparameter = 1]) => + x > 0 ? 1 : elu(x, hyperparameter) + hyperparameter; + /// The derivative of the Scaled Exponential Linear Unit double seluPrime(double x) => 1.0507 * (x < 0 ? 1.67326 * exp(x) : 1); + /// The derivative of the Hyperbolic Tangent double tanhPrime(double x) => 1.0 - pow(tanh(x), 2); + /// The derivative of the Softplus double softplusPrime(double x) => sigmoid(x); + /// The derivative of the Softsign -double softsignPrime(double x) => 1 / pow((1 + abs(x)), 2); +double softsignPrime(double x) => 1 / pow(1 + abs(x), 2); + /// The derivative of the Swish double swishPrime(double x) => swish(x) + sigmoid(x) * (1 - swish(x)); + /// The derivative of the Gaussian double gaussianPrime(double x) => -2 * x * gaussian(x); /// Algorithms which can be used for activating `Neuron`s enum ActivationAlgorithm { + /// Sigmoid sigmoid, - relu, // Rectified Linear Unit - lrelu, // Leaky Rectified Linear Unit - elu, // Exponential Linear Unit - selu, // Scaled Exponential Linear Unit + /// Rectified Linear Unit + relu, - tanh, // Hyperbolic tangent + /// Leaky Rectified Linear Unit + lrelu, + /// Exponential Linear Unit + elu, + + /// Scaled Exponential Linear Unit + selu, + + /// Hyperbolic tangent + tanh, + + /// Softplus softplus, + + /// Softsign softsign, + + /// Swish swish, + + /// Gaussian gaussian, -} \ No newline at end of file +} diff --git a/lib/src/layers/core/dense.dart b/lib/src/layers/core/dense.dart index d00e3c4..db3bccf 100644 --- a/lib/src/layers/core/dense.dart +++ b/lib/src/layers/core/dense.dart @@ -1,14 +1,12 @@ import 'package:synadart/src/activation.dart'; import 'package:synadart/src/layers/layer.dart'; -/// `Layer` in which all `Neurons` are connected to all `Neurons` in the previous `Layer` +/// A `Layer` in which every `Neuron` is connected to every other `Neurons` in +/// the preceding `Layer`. class Dense extends Layer { - /// Construct a dense layer using the [activation] and [size] + /// Construct a dense layer using the [activation] algorithm and [size]. Dense({ required int size, required ActivationAlgorithm activation, - }) : super( - size: size, - activation: activation, - ); -} \ No newline at end of file + }) : super(size: size, activation: activation); +} diff --git a/lib/src/layers/layer.dart b/lib/src/layers/layer.dart index 3277383..28e823f 100644 --- a/lib/src/layers/layer.dart +++ b/lib/src/layers/layer.dart @@ -6,32 +6,33 @@ import 'package:synadart/src/activation.dart'; import 'package:synadart/src/neurons/neuron.dart'; import 'package:synadart/src/utils/mathematical_operations.dart'; -/// Representation of a single `Layer` inside a `Network` - a 'column' of `Neurons` which can be -/// manipulated through accepting new data, propagated through training or simply processed the output of. +/// Representation of a single `Layer` inside a `Network`, more accurately a +/// 'column' of `Neurons` that can be manipulated through accepting new data and +/// trained. class Layer { - /// I don't know how to document this one + /// `Sprint` instance for logging messages. final Sprint log = Sprint('Layer'); - /// The `ActivationAlgorithm` used for activating `Neurons` inside this `Layer`. + /// The algorithm used for activating `Neurons`. final ActivationAlgorithm activation; - /// List containing the `Neuron`s inside this `Layer`. + /// The `Neurons` part of this `Layer`. final List neurons = []; - /// The amount of `Neurons` this `Layer` comprises. + /// The number of `Neurons` this `Layer` comprises. final int size; - /// Specifies whether this `Layer` is an input `Layer`. This is used to determine how inputs should be - /// accepted by each neuron in this `Layer`. + /// Specifies whether or not this `Layer` is an input `Layer`. This is used + /// to determine how inputs should be accepted by each neuron in this `Layer`. bool isInput = false; - /// Creates a `Layer` with the specified `ActivationAlgorithm` which is then passed to and resolved by `Neuron`s. + /// Creates a `Layer` with the specified activation algorithm that is then + /// passed to and resolved by `Neurons`. /// - /// [size] - The amount of `Neuron`s this `Layer` has. + /// [size] - The number of `Neurons` this `Layer` is to house. /// - /// [activation] - The algorithm used for 'activating' this `Layer`'s `Neuron`s, or indicating - /// how 'active' this `Layer`'s `Neuron`s are by shrinking the weighted sum of a `Neuron`'s [weights] and [inputs] - /// to a more controlled range, such as 0 to 1. + /// [activation] - The algorithm used for determining how active `Neurons` are + /// contained within this layer. Layer({ required this.size, required this.activation, @@ -42,15 +43,17 @@ class Layer { } } - /// Initialises this `Layer` with parameters passed in by the `Network` + /// Initialises this `Layer` using the parameters passed into it by the + /// `Network` in which the `Layer` is housed. /// - /// [parentNeuronCount] - The amount of 'connections' this `Layer` has, or how many `Neuron`s the previous - /// `Layer` contains. + /// [parentLayerSize] - The number of 'connections' this `Layer` is in + /// disposition of. In other words, the number of `Neurons` the previous + /// `Layer` houses. This number be equal to zero if this `Layer` is an input + /// `Layer`. /// - /// This number will equal zero if this `Layer` is an input `Layer`. - /// - /// [learningRate] - A value between 0 (exclusive) and 1 (inclusive) that indicates how sensitive - /// this `Layer`'s `Neuron`s are to adjustments of their [weights]. + /// [learningRate] - A value between 0 (exclusive) and 1 (inclusive), + /// indicating how sensitive the `Neurons` within this `Layer` are to + /// adjustments of their weights. void initialise({ required int parentLayerSize, required double learningRate, @@ -61,19 +64,21 @@ class Layer { size, (_) => Neuron( activationAlgorithm: activation, - parentNeuronCount: parentLayerSize, + parentLayerSize: parentLayerSize, learningRate: learningRate, ), )); } - /// Accept a single [input] or multiple [inputs] by assigning them to every of this `Layer`'s `Neuron`'s [inputs]. + /// Accept a single input or multiple [inputs] by assigning them sequentially + /// to the inputs of the `Neurons` housed within this `Layer`. /// - /// If [isInput] is true, each `Neuron` in this `Layer` will only accept a single input corresponding - /// to its index in the [neurons] list. + /// If [isInput] is equal to true, each `Neuron` within this `Layer` will only + /// accept a single input corresponding to its index within the [neurons] + /// list. void accept(List inputs) { if (isInput) { - for (int index = 0; index < neurons.length; index++) { + for (var index = 0; index < neurons.length; index++) { neurons[index].accept(input: inputs[index]); } return; @@ -84,17 +89,18 @@ class Layer { } } - /// Adjust weights of each `Neuron` based on its respective [weightMargin] and return - /// new [weightMargins] for the previous `Layer` (We are moving backwards during propagation). + /// Adjusts weights of each `Neuron` based on its respective weight margin, + /// and returns the new [weightMargins] for the previous `Layer` (We are + /// moving backwards during propagation). List propagate(List weightMargins) { - final List> newWeightMargins = []; + final newWeightMargins = >[]; for (final neuron in neurons) { newWeightMargins .add(neuron.adjust(weightMargin: weightMargins.removeAt(0))); } - return newWeightMargins.reduce((a, b) => add(a, b)); + return newWeightMargins.reduce(add); } /// Returns a list of this `Layer`'s `Neuron`s' outputs diff --git a/lib/src/layers/recurrent/lstm.dart b/lib/src/layers/recurrent/lstm.dart index d600d22..0033dd9 100644 --- a/lib/src/layers/recurrent/lstm.dart +++ b/lib/src/layers/recurrent/lstm.dart @@ -1,53 +1,59 @@ import 'package:synadart/src/activation.dart'; import 'package:synadart/src/layers/layer.dart'; -/// `Layer` in which all `Neurons` are connected to all `Neurons` in the previous `Layer` +/// A `Layer` in which every `Neuron` is connected to every other `Neuron` in +/// the preceding `Layer`, with additional memory/retention capabilities. class LSTM extends Layer { - /// Function used for LSTM activation - late final ActivationFunction recurrentActivation; + /// Activation function of the LSTM layer. + late final ActivationFunction recurrenceActivation; - /// Memory carried across all `Neurons` inside the layer + /// Memory carried across all `Neurons` inside the layer. double longTermMemory = 1; - /// Memory carried over from the current `Neuron` to the next `Neuron` + + /// Memory carried over from the current `Neuron` to the following `Neuron`. double shortTermMemory = 1; - /// Construct a LSTM layer - /// - /// [size] - How many `Neurons` this `LSTM` has - /// - /// [activation] - `ActivationAlgorithm` used for `Neuron` activation - /// - /// [recurrentActivation] - `ActivationAlgorithm` used for `LSTM` activation + /// Construct an LSTM layer. + /// + /// [size] - How many `Neurons` this `LSTM` has. + /// + /// [activation] - Algorithm used to activate `Neurons` in this `Layer`. + /// + /// [recurrenceActivation] - Algorithm used to activate recurrence + /// connections. LSTM({ required int size, required ActivationAlgorithm activation, - required ActivationAlgorithm recurrentActivation, + required ActivationAlgorithm recurrenceActivation, }) : super( - size: size, - activation: activation, - ) { - this.recurrentActivation = resolveActivationAlgorithm(recurrentActivation); + size: size, + activation: activation, + ) { + this.recurrenceActivation = + resolveActivationAlgorithm(recurrenceActivation); } - /// Obtain the output by applying the recurrent memory algorithm + /// Obtain the output by applying the recurrent memory algorithm. @override List get output { - final List output = []; + final output = []; for (final neuron in neurons) { - double neuronOutput = neuron.output; - double hiddenActivationComponent = neuron.activation(() => neuronOutput); - double hiddenRecurrentComponent = recurrentActivation(() => neuronOutput); + final neuronOutput = neuron.output; + final hiddenActivationComponent = neuron.activation(() => neuronOutput); + final hiddenRecurrentComponent = recurrenceActivation(() => neuronOutput); // Forget gate longTermMemory = hiddenActivationComponent * longTermMemory; // Update gate - longTermMemory = (hiddenActivationComponent * hiddenRecurrentComponent) + longTermMemory; + longTermMemory = (hiddenActivationComponent * hiddenRecurrentComponent) + + longTermMemory; // Output gate - double stateRecurrentComponent = recurrentActivation(() => longTermMemory); + final stateRecurrentComponent = + recurrenceActivation(() => longTermMemory); shortTermMemory = hiddenActivationComponent * stateRecurrentComponent; output.add(shortTermMemory); } return output; } -} \ No newline at end of file +} diff --git a/lib/src/networks/network.dart b/lib/src/networks/network.dart index 93f3390..6a9af20 100644 --- a/lib/src/networks/network.dart +++ b/lib/src/networks/network.dart @@ -2,33 +2,43 @@ import 'package:sprint/sprint.dart'; import 'package:synadart/src/layers/layer.dart'; -/// Representation of a Neural Network, which contains `Layers`, each containing a number of `Neurons`. A `Network` takes -/// an input in the form of several entries and returns an output by processing the data, passing the data through each -/// layer. +/// Representation of a neural network containing `Layers`, which each further +/// house a number of `Neurons`. A `Network` takes an input in the form of +/// several entries and returns an output by processing the data by running the +/// data through the layers. /// -/// `Network`s must have a training mixin in order to be able to learn. +/// In order to train a `Network`, the selected `Network` must have a training +/// algorithm mixed into it - most commonly `Backpropagation`. class Network { - /// Used for performance analysis as well as general information logging + /// Used for performance analysis as well as general information logging. final Stopwatch stopwatch = Stopwatch(); + /// `Sprint` instance for logging messages. final Sprint log = Sprint('Network'); - /// List containing the `Layers` inside this `Network`. + /// The `Layers` part of this `Network`. final List layers = []; + /// The degree of radicality at which the `Network` will adjust its `Neurons` + /// weights. double learningRate; - /// Creates a `Network` with optional `Layers` + /// Creates a `Network` with optional `Layers`. + /// + /// [learningRate] - The level of aggressiveness at which this `Network` will + /// adjust its `Neurons`' weights during training. + /// + /// [layers] - (Optional) The `Layers` of this `Network`. Network({required this.learningRate, List? layers}) { if (layers != null) { addLayers(layers); } } - /// Processes [inputs], propagating them across every `Layer`, processing each `Layer` one-by-one + /// Processes the [inputs] by propagating them across every `Layer`. /// and returns the output. List process(List inputs) { - List output = inputs; + var output = inputs; for (final layer in layers) { layer.accept(output); @@ -38,7 +48,7 @@ class Network { return output; } - /// Adds a `Layer` to this `Network` + /// Adds a `Layer` to this `Network`. void addLayer(Layer layer) { layer.initialise( parentLayerSize: layers.isEmpty ? 0 : layers[layers.length - 1].size, @@ -49,17 +59,18 @@ class Network { log.info('Added layer of size ${layer.neurons.length}.'); } - /// Adds a list of `Layers` to this `Network` + /// Adds a list of `Layers` to this `Network`. void addLayers(List layers) { for (final layer in layers) { addLayer(layer); } } - /// Resets the `Network` by removing all `Layers` - void reset() { + /// Clears the `Network` by removing all `Layers`, thereby returning it to its + /// initial, empty state. + void clear() { if (layers.isEmpty) { - log.warning('Attempted to reset an already empty network'); + log.warning('Attempted to reset an already empty network.'); return; } diff --git a/lib/src/networks/sequential.dart b/lib/src/networks/sequential.dart index d765c1e..9745dab 100644 --- a/lib/src/networks/sequential.dart +++ b/lib/src/networks/sequential.dart @@ -2,7 +2,12 @@ import 'package:synadart/src/layers/layer.dart'; import 'package:synadart/src/networks/network.dart'; import 'package:synadart/src/networks/training/backpropagation.dart'; -/// Network model in which every layer has one input and one output tensor +/// A `Network` model in which every `Layer` has one input and one output +/// tensor. class Sequential extends Network with Backpropagation { - Sequential({required double learningRate, List? layers}) : super(learningRate: learningRate, layers: layers); -} \ No newline at end of file + /// Creates a `Sequential` model network. + Sequential({ + required double learningRate, + List? layers, + }) : super(learningRate: learningRate, layers: layers); +} diff --git a/lib/src/networks/training/backpropagation.dart b/lib/src/networks/training/backpropagation.dart index f2d3340..b300ff6 100644 --- a/lib/src/networks/training/backpropagation.dart +++ b/lib/src/networks/training/backpropagation.dart @@ -4,29 +4,31 @@ import 'package:synadart/src/networks/network.dart'; import 'package:synadart/src/utils/mathematical_operations.dart'; import 'package:synadart/src/utils/utils.dart'; -/// Extension to `Network` that allows it to train by performing backpropagation +/// Extension to `Network` that allows it to train by performing +/// backpropagation. mixin Backpropagation on Network { - /// Perform the backpropagation algorithm by comparing the observed with the expected - /// values, and propagate each layer using the knowledge of the network's `cost`, which - /// indicates how bad the network is performing. + /// Performs the backpropagation algorithm by comparing the observed values + /// with the expected values, and propagates each layer using the knowledge of + /// the network's 'cost', which indicates how bad the network is performing. void propagateBackwards(List input, List expected) { final observed = process(input); - List errors = subtract(expected, observed); + var errors = subtract(expected, observed); - for (int index = layers.length - 1; index > 0; index--) { + for (var index = layers.length - 1; index > 0; index--) { errors = layers[index].propagate(errors); } } - /// Train the network by passing in [inputs], their respective [expected] results - /// as well as the number of iterations to make during training + /// Trains the network using the passed [inputs], their respective [expected] + /// results, as well as the number of iterations to make during training. /// - /// [inputs] - The values we pass into the `Network` + /// [inputs] - The values we pass into the `Network`. /// - /// [expected] - What the `Network` is expected to output + /// [expected] - What we expect the `Network` to output. /// - /// [iterations] - How many times the `Network` should train, given the inputs and expected values + /// [iterations] - How many times the `Network` should perform backpropagation + /// using the provided inputs and expected values. void train({ required List> inputs, required List> expected, @@ -39,29 +41,32 @@ mixin Backpropagation on Network { } if (inputs.length != expected.length) { - log.severe('Inputs and expected result lists must be of the same length.'); + log.severe( + 'Inputs and expected result lists must be of the same length.'); return; } if (iterations < 1) { log.severe( - 'You cannot train a network without granting it at least one iteration.'); + 'You cannot train a network without granting it at least one ' + 'iteration.', + ); return; } // Perform backpropagation without any additional metrics overhead if (quiet) { - for (int iteration = 0; iteration < iterations; iteration++) { - for (int index = 0; index < inputs.length; index++) { + for (var iteration = 0; iteration < iterations; iteration++) { + for (var index = 0; index < inputs.length; index++) { propagateBackwards(inputs[index], expected[index]); } } return; } - for (int iteration = 0; iteration < iterations; iteration++) { + for (var iteration = 0; iteration < iterations; iteration++) { stopwatch.start(); - for (int index = 0; index < inputs.length; index++) { + for (var index = 0; index < inputs.length; index++) { propagateBackwards(inputs[index], expected[index]); } stopwatch.stop(); diff --git a/lib/src/neurons/neuron.dart b/lib/src/neurons/neuron.dart index 593fcc4..283bbde 100644 --- a/lib/src/neurons/neuron.dart +++ b/lib/src/neurons/neuron.dart @@ -7,84 +7,103 @@ import 'package:synadart/src/activation.dart'; import 'package:synadart/src/utils/mathematical_operations.dart'; import 'package:synadart/src/utils/value_generator.dart'; -/// Representation of a single `Neuron` inside a `Network`. A `Neuron` can take on multiple forms and functions, -/// the most basic of which being the taking of the weighted sum of [inputs] and [weights], and passing it further -/// to the next `Neuron`. +/// A representation of a single `Neuron` within a `Network` of `Layers` (of +/// `Neurons`). A `Neuron` can take on multiple forms and functions, the most +/// basic of which being the taking of the weighted sum of [inputs] and +/// [weights], and passing it on to the next `Neuron`. class Neuron { + /// `Sprint` instance for logging messages. final Sprint log = Sprint('Neuron'); - /// `ActivationAlgorithm` in the form of a mathematical function which can be resolved when needed. + /// The activation algorithm used for determining this `Neuron`'s level of + /// activation. late final ActivationFunction activation; - /// The derivative of the `ActivationAlgorithm`. + /// The derivative of the activation algorithm. late final ActivationFunction activationPrime; - /// The sensitivity of this `Neuron` to the function adjusting [weights] during training. + /// The sensitivity of this `Neuron` to the function adjusting [weights] + /// during training. + /// /// - /// The [learningRate] must be a value between 0 (exclusive) and 1 (inclusive). final double learningRate; - /// Weights of connections to previous `Neuron`s, which can be imagined as how influential each - /// connection is on this `Neuron`'s [output]. + /// The weights of connections to the precedent `Neurons`, which can be + /// imagined as how influential each `Neuron` in the preceding layer is on + /// this `Neuron`'s [output]. List weights = []; - /// Inputs received from the previous `Layer`, which are combined with [weights] of connections to - /// `Neurons`s from the previous `Layer` in order to create an [output]. + /// The inputs received from the precedent `Layer`, combined with [weights] or + /// connections to `Neurons` from the previous `Layer` in order to create an + /// [output]. List inputs = []; - /// Specifies whether this `Neuron` belongs to an input `Layer`. This is used to determine the source of - /// output because a parentless `Neuron` will not have any connections. + /// Specifies whether this `Neuron` belongs to an input `Layer`. This is used + /// to determine the source of output, because a parentless `Neuron` will not + /// have any connections. bool isInput = false; - /// Creates a `Neuron` with the specified `ActivationAlgorithm` which is then resolved to an `ActivationFunction`. + /// Creates a `Neuron` with the specified `ActivationAlgorithm`, which is then + /// resolved to an `ActivationFunction`. /// - /// [activationAlgorithm] - The algorithm used for 'activating' this `Neuron`, or indicating - /// how 'active' this `Neuron` is by shrinking the weighted sum of this `Neuron`'s [weights] and [inputs] - /// to a more controlled range, such as 0 to 1. + /// [activationAlgorithm] - The algorithm used for 'activating' this `Neuron`, + /// i.e. determining how 'active' this `Neuron` is by shrinking the weighted + /// sum of this `Neuron`'s [weights] and [inputs] to a more appropriate range, + /// such as 0–1. /// - /// [parentNeuronCount] - The amount of 'connections' this `Neuron` has, or how many `Neuron`s - /// from the previous layer this `Neuron` is connected to. + /// [parentLayerSize] - The number of connections this `Neuron` has, i.e. how + /// many `Neurons` from the previous layer this `Neuron` is connected to. /// - /// [learningRate] - A value between 0 (exclusive) and 1 (inclusive) that indicates how sensitive - /// this `Neuron` is to adjustments of [weights]. + /// [learningRate] - A value between 0 (exclusive) and 1 (inclusive) that + /// indicates how sensitive this `Neuron` is to [weights] adjustments. /// - /// [weights] - (Optional) Weights of connections to `Neuron`s in the previous `Layer`. - /// If [weights] have not been provided, they will be generated randomly. - Neuron( - {required ActivationAlgorithm activationAlgorithm, - required int parentNeuronCount, - required this.learningRate, - List weights = const []}) { + /// [weights] - (Optional) Weights of connections to `Neuron`s in the previous + /// `Layer`. If the [weights] aren't provided, they will be generated + /// randomly. + Neuron({ + required ActivationAlgorithm activationAlgorithm, + required int parentLayerSize, + required this.learningRate, + List weights = const [], + }) { activation = resolveActivationAlgorithm(activationAlgorithm); - activationPrime = resolveDerivative(activationAlgorithm); + activationPrime = resolveActivationDerivative(activationAlgorithm); - if (parentNeuronCount == 0) { - this.isInput = true; + if (parentLayerSize == 0) { + isInput = true; this.weights.add(1); return; } if (weights.isEmpty) { - // Calculate the range extrema based on the number of parent neurons - final double limit = 1 / sqrt(parentNeuronCount); - // Generate random values for the weights of connections - this.weights.addAll(ValueGenerator.generateListWithRandomDoubles( - size: parentNeuronCount, from: -limit, to: limit)); + // Calculate the range extreme based on the number of parent neurons. + final limit = 1 / sqrt(parentLayerSize); + // Generate random values for the connection weights. + this.weights.addAll( + generateListWithRandomDoubles( + size: parentLayerSize, + from: -limit, + to: limit, + ), + ); return; } - if (weights.length != parentNeuronCount) { + if (weights.length != parentLayerSize) { log.severe( - 'The amount of weights supplied to this neuron does not match the amount' - 'of connections to neurons in the parent layer.'); + 'The number of weights supplied to this neuron does not match the ' + 'number of connections to neurons in the parent layer.', + ); exit(0); } + // ignore: prefer_initializing_formals this.weights = weights; return; } - /// Accept a single [input] or multiple [inputs] by assigning them to this `Neuron`'s [inputs]. + /// Accepts a single [input] or multiple [inputs] by assigning them to the + /// [inputs] of this `Neuron`. void accept({List? inputs, double? input}) { if (inputs == null && input == null) { log.severe('Attempted to accept without any inputs.'); @@ -103,14 +122,14 @@ class Neuron { } } - /// Adjust this `Neuron`'s [weights] and return their adjusted values + /// Adjusts this `Neuron`'s [weights] and returns their adjusted values. List adjust({required double weightMargin}) { - final List adjustedWeights = []; + final adjustedWeights = []; - for (int index = 0; index < weights.length; index++) { + for (var index = 0; index < weights.length; index++) { adjustedWeights.add(weightMargin * weights[index]); weights[index] -= learningRate * - -weightMargin * // TODO: Shouldn't this logically be δC/δa? + -weightMargin * activationPrime(() => dot(inputs, weights)) * // δa/δz inputs[index]; // δz/δw } @@ -118,9 +137,10 @@ class Neuron { return adjustedWeights; } - /// If this `Neuron` is an 'input' `Neuron`, it should output its sole input because it has no parent neurons - /// and therefore no weights of connections. - /// Otherwise, it should output the weighted sum of the [inputs] and [weights], passed through the [activationFunction]. + /// If this `Neuron` is an 'input' `Neuron`, it will output its sole input + /// because it has no parent neurons and therefore has no weights. Otherwise, + /// it will output the weighted sum of the [inputs] and [weights], passed + /// through the activation function. double get output => weights.isEmpty ? inputs[0] : activation(() => dot(inputs, weights)); } diff --git a/lib/src/utils/mathematical_operations.dart b/lib/src/utils/mathematical_operations.dart index c7c2465..d15393a 100644 --- a/lib/src/utils/mathematical_operations.dart +++ b/lib/src/utils/mathematical_operations.dart @@ -1,7 +1,7 @@ /// Calculates the dot product of two lists double dot(List a, List b) { - double result = 0; - for (int index = 0; index < a.length; index++) { + var result = 0.0; + for (var index = 0; index < a.length; index++) { result += a[index] * b[index]; } return result; @@ -9,8 +9,8 @@ double dot(List a, List b) { /// Adds values in list [b] to list [a] List add(List a, List b) { - final List result = []; - for (int index = 0; index < a.length; index++) { + final result = []; + for (var index = 0; index < a.length; index++) { result.add(a[index] + b[index]); } return result; @@ -18,12 +18,12 @@ List add(List a, List b) { /// Subtracts values in list [b] from list [a] List subtract(List a, List b) { - final List result = []; - for (int index = 0; index < a.length; index++) { + final result = []; + for (var index = 0; index < a.length; index++) { result.add(a[index] - b[index]); } return result; } /// Gets the absolute value of [x] -double abs(double x) => x < 0 ? -x : x; \ No newline at end of file +double abs(double x) => x < 0 ? -x : x; diff --git a/lib/src/utils/utils.dart b/lib/src/utils/utils.dart index faf732c..ad3e660 100644 --- a/lib/src/utils/utils.dart +++ b/lib/src/utils/utils.dart @@ -1,14 +1,15 @@ -/// Converts seconds to an Estimated Time of Arrival string +/// Converts seconds to an estimated time of arrival (ETA) string. String secondsToETA(int seconds) { - String eta = ''; + var eta = ''; - // Break down the total number of seconds into larger units of time - int hours = seconds ~/ (60 * 60); - seconds -= hours * (60 * 60); - int minutes = seconds ~/ 60; - seconds -= minutes * 60; + // Compose the total number of seconds into larger units of time. + var secondsSink = seconds; + final hours = seconds ~/ (60 * 60); + secondsSink -= hours * (60 * 60); + final minutes = seconds ~/ 60; + secondsSink -= minutes * 60; - // Append a zero to single-digit time representations + // Append a zero to single-digit time representations. if (hours < 10) { eta += '0'; } @@ -19,10 +20,9 @@ String secondsToETA(int seconds) { } eta += '$minutes:'; - if (seconds < 10) { + if (secondsSink < 10) { eta += '0'; } - eta += '$seconds'; - return eta; + return eta += '$secondsSink'; } diff --git a/lib/src/utils/value_generator.dart b/lib/src/utils/value_generator.dart index 3c73e3f..b71a625 100644 --- a/lib/src/utils/value_generator.dart +++ b/lib/src/utils/value_generator.dart @@ -1,21 +1,21 @@ import 'dart:math'; -/// Class with static functions that allow for generating random values between maxima -class ValueGenerator { - static final Random random = Random.secure(); +final Random _random = Random.secure(); - /// Generates a single `double` inside the range [from] (inclusive) - [to] (exclusive) - static double nextDouble({double from = 0, double to = 0}) => random.nextDouble() * (to - from) + from; +/// Generates a single `double` inside the range [from] (inclusive) - [to] +/// (exclusive). +double nextDouble({double from = 0, double to = 0}) => + _random.nextDouble() * (to - from) + from; - /// Generator function for `double`s inside the range [from] (inclusive) - [to] (exclusive) - static Iterable doubleIterableSync({double from = 0, double to = 0}) sync* { - while (true) { - yield nextDouble(from: from, to: to); - } - } - - /// Generates a list of size [size], filled with random `double`s - static List generateListWithRandomDoubles({required int size, double from = 0, double to = 0}) { - return doubleIterableSync(from: from, to: to).take(size).toList(); +/// Generator function for `double`s inside the range [from] (inclusive) - [to] +/// (exclusive). +Iterable doubleIterableSync({double from = 0, double to = 0}) sync* { + while (true) { + yield nextDouble(from: from, to: to); } } + +/// Generates a list of size [size], filled with random `double` values. +List generateListWithRandomDoubles( + {required int size, double from = 0, double to = 0}) => + doubleIterableSync(from: from, to: to).take(size).toList(); diff --git a/lib/synadart.dart b/lib/synadart.dart index 82cafcc..d87b0c8 100644 --- a/lib/synadart.dart +++ b/lib/synadart.dart @@ -2,4 +2,4 @@ library synadart; export 'src/activation.dart'; -export 'src/networks/sequential.dart'; \ No newline at end of file +export 'src/networks/sequential.dart'; diff --git a/pubspec.yaml b/pubspec.yaml index bb22a2b..2867e78 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -1,22 +1,22 @@ publish_to: https://pub.dev name: synadart -version: 0.4.2+1 +version: 0.4.3 description: >- A simple-to-grasp, complete and fully documented Neural Network library, written from scratch in Dart. -homepage: https://github.com/vxern/synadart -repository: https://github.com/vxern/synadart -issue_tracker: https://github.com/vxern/synadart/issues +homepage: https://github.com/wordcollector/synadart +repository: https://github.com/wordcollector/synadart +issue_tracker: https://github.com/wordcollector/synadart/issues environment: sdk: '>=2.12.0 <3.0.0' dependencies: # Logging - sprint: ^1.0.2+3 + sprint: ^1.0.3 dev_dependencies: words: ^0.0.2+1