I recently just finished programming a neural network in c#, and it seems like it's working. My question is if I'm doing it right. It's a very confusing process so I will explain.
Basically every neuron in the network has a bias (except the first layer) and fully connected weights to the next. every neuron also has a double[] desiredBias which holds all the desired changes to the neuron for every iteration in a given mini batch. The same thing for the weights. So I start from the last layer, and from there I calculate the "sensitivity" you might say between the cost of a given neuron on that last layer and the bias, and i take note of it (put it into changes). Then for every weight connected to the layer behind, I also calculate the sensitivity for that one. And I keep doing that until I get to the second to first layer. And to take note of the desiredChanges for neurons not in the last layer, I use this array called "changes". For the first layer it's basically just a list of all the derivatives of the activations (z) with respect to the actual activations (sigmoid(z)) respectively. After I shift to the layer behind, and behind, every time I do that, I create a new changes array, which is the size of the layer behind, and for each item in the previous changes array, I take the sum of all of them multiplied by the partial derivative of that same z-activation with respect to the previous layer's z activation. I do this for every neuron in the previous layer.
after all of that I end up with for every neuron a list of desired changes to it's bias, and a list of lists which contain desired changes to the weights of each neuron. Then for each neuron (except for the first of course) I average together all of the biases and SUBTRACT it from the neuron's bias. I do the same thing for the weights.
Is this the correct process to go through for exactly one batch of training data?
here's my code:
using System;
namespace NeuralNetwork {
public class Functions {
// Activation Functions
public static double sigmoid(double x) {
return 1 / (1 + Math.Pow(Math.E, -x));
}
public static double ReLU(double x) {
return Math.Max(0, x);
}
public static double Tanh(double x) {
return Math.Tanh(x);
}
public static double Linear(double x) {
return x;
}
// Derivatives
public static double sigmoidDerivative(double x) {
return sigmoid(x) * (1 - sigmoid(x));
}
public static double ReLUDerivative(double x) {
return (x > 0) ? 1 : 0;
}
public static double TanhDerivative(double x) {
return 1 - (Math.Pow(Tanh(x) * Tanh(x), 2));
}
public static double LinearDerivative(double x) {
return 1;
}
public enum ActFunction {
Sigmoid,
ReLU,
Tanh,
Linear
}
// Loss Functions
public static double MSE(double[] outputs, double[] desired) {
double sum = 0;
for (int i = 0; i < outputs.Length; i++) {
sum += Math.Pow((outputs[i] - desired[i]), 2);
}
return sum / outputs.Length;
}
// Derivatives
public static double[] MSEDerivative(double[] outputs, double[] desired) {
double[] derivatives = new double[outputs.Length];
for (int i = 0; i < outputs.Length; i++) {
derivatives[i] = 2 * (outputs[i] - desired[i]);
}
return derivatives;
}
public enum LossFunction {
MSE
}
}
public class Neuron {
public double activation = 0;
public Functions.ActFunction activationFunction = Functions.ActFunction.Linear;
public double bias;
public double[] weights = new double[] {};
public int x;
public int y;
public Neuron[][] network;
public double[] desiredBias;
public double[][] desiredWeights;
public void fire() {
for (int i = 0; i < network[x + 1].Length; i++) {
Neuron next = network[x + 1][i];
next.activation += activation * weights[i];
}
}
public void activate() {
double n_activation = activation + bias;
switch(activationFunction) {
case Functions.ActFunction.Sigmoid:
activation = Functions.sigmoid(n_activation);
break;
case Functions.ActFunction.ReLU:
activation = Functions.ReLU(n_activation);
break;
case Functions.ActFunction.Tanh:
activation = Functions.Tanh(n_activation);
break;
case Functions.ActFunction.Linear:
activation = Functions.Linear(n_activation);
break;
}
}
}
public class Network {
public Neuron[][] network;
public double learningRate;
public int iterations;
public Functions.LossFunction lossFunction;
public int trainingLength;
public Network(int[] size, double _learningRate, int _iterations, int _trainingLength, Functions.LossFunction _lossFunction) {
learningRate = _learningRate;
iterations = _iterations;
trainingLength = _trainingLength;
lossFunction = _lossFunction;
network = new Neuron[size.Length][];
for (int i = 0; i < size.Length; i++) {
network[i] = new Neuron[size[i]];
}
for (int i = 0; i < size.Length; i++) {
for (int j = 0; j < size[i]; j++) {
network[i][j] = new Neuron();
}
}
for (int i = 0; i < network.Length - 1; i++) {
for (int j = 0; j < network[i].Length; j++) {
Neuron neuron = network[i][j];
neuron.weights = new double[network[i + 1].Length];
neuron.desiredWeights = new double[neuron.weights.Length][]; // i would have put iterations right here but the compiler thinks otherwise
for (int k = 0; k < neuron.desiredWeights.Length; k++) {
neuron.desiredWeights[k] = new double[trainingLength/iterations];
}
}
}
for (int i = 1; i < network.Length; i++) {
for (int j = 0; j < network[i].Length; j++) {
Neuron neuron = network[i][j];
neuron.desiredBias = new double[trainingLength/iterations];
}
}
}
public void init() {
Random rand = new Random();
for (int i = 0; i < network.Length; i++) {
for (int j = 0; j < network[i].Length; j++) {
Neuron neuron = network[i][j];
neuron.x = i;
neuron.y = j;
neuron.network = network;
if (i != 0) {
neuron.bias = (rand.NextDouble() * 2) - 1;
}
if (i != network.Length - 1) {
for (int k = 0; k < network[neuron.x + 1].Length; k++) {
neuron.weights[k] = (rand.NextDouble() * 20) - 1;
}
}
}
}
}
public void info() {
Console.WriteLine("Here are the statistics for the network:\n");
for (int i = 0; i < network.Length; i++) {
for (int j = 0; j < network[i].Length; j++) {
Neuron neuron = network[i][j];
Console.WriteLine("Neuron " + j + " of layer " + i);
//Console.WriteLine("Activation: " + neuron.activation);
Console.WriteLine("Bias: " + neuron.bias);
Console.WriteLine("X and Y: " + neuron.x + ", " + neuron.y);
string weights = "Weights: ";
foreach (double weight in neuron.weights) {
weights += weight + " ";
}
Console.WriteLine(weights);
Console.WriteLine();
}
}
Console.WriteLine();
}
public double[] run(double[] inputs) {
// initialize inputs
for (int i = 0; i < network[0].Length; i++) {
network[0][i].activation = inputs[i];
}
for (int i = 0; i < network.Length - 1; i++) {
for (int j = 0; j < network[i].Length; j++) {
Neuron neuron = network[i][j];
neuron.fire();
}
for (int j = 0; j < network[i + 1].Length; j++) {
Neuron neuron = network[i + 1][j];
neuron.activate();
}
}
// get outputs
Neuron[] outputs = network[network.Length - 1];
double[] activations = new double[outputs.Length];
for (int i = 0; i < outputs.Length; i++) {
activations[i] = outputs[i].activation;
}
return activations;
}
public double iteration() {
Random rand = new Random();
int iterationLength = trainingLength/iterations;
double[] costs = new double[iterationLength];
// training
for (int iter = 0; iter < iterationLength; iter++) {
double[] inputs = { (rand.NextDouble() ), ( rand.NextDouble() )};
double[] answers = { inputs[0] + inputs[1] };
double[] outputs = run(inputs);
for (int i = 0; i < network.Length; i++) {
for (int j = 0; j < network[i].Length; j++) {
Neuron neuron = network[i][j];
neuron.desiredBias = new double[iterationLength];
if (i == network.Length - 1) continue;
neuron.desiredWeights = new double[network[i + 1].Length][];
for (int k = 0; k < neuron.desiredWeights.Length; k++) {
neuron.desiredWeights[k] = new double[iterationLength];
}
}
}
double cost = 0;
switch(lossFunction) {
case Functions.LossFunction.MSE:
cost = Functions.MSE(outputs, answers);
break;
}
costs[iter] = cost;
double[] changes = new double[network[network.Length - 1].Length];
switch(lossFunction) {
case Functions.LossFunction.MSE:
changes = Functions.MSEDerivative(outputs, answers);
break;
}
for (int i = 0; i < changes.Length; i++) {
Neuron neuron = network[network.Length - 1][i];
double act = neuron.activation;
switch(neuron.activationFunction) {
case Functions.ActFunction.Sigmoid:
changes[i] *= Functions.sigmoidDerivative(Math.Log(act / (1 - act)));
break;
case Functions.ActFunction.Linear:
changes[i] *= Functions.LinearDerivative(act);
break;
}
}
for (int i = network.Length - 1; i > 0; i--) {
for (int j = 0; j < network[i].Length; j++) {
Neuron bNeuron = network[i][j];
bNeuron.desiredBias[iter] += changes[j]; // bias partial derivative is always one
for (int k = 0; k < network[i - 1].Length; k++) {
Neuron aNeuron = network[i - 1][k];
aNeuron.desiredWeights[bNeuron.y][iter] += aNeuron.weights[bNeuron.y] * changes[j]; // chain rule
}
}
double[] holderChanges = changes;
changes = new double[network[i - 1].Length];
for (int j = 0; j < changes.Length; j++) {
for (int k = 0; k < holderChanges.Length; k++) {
Neuron aNeuron = network[i - 1][j];
Neuron bNeuron = network[i][k];
switch(aNeuron.activationFunction) {
case Functions.ActFunction.Sigmoid:
double activation = aNeuron.activation;
changes[j] += Functions.sigmoidDerivative(Math.Log(activation / 1 - activation)) * aNeuron.weights[bNeuron.y] * holderChanges[k];
break;
case Functions.ActFunction.Linear:
activation = aNeuron.activation;
changes[j] += Functions.LinearDerivative(activation) * aNeuron.weights[bNeuron.y] * holderChanges[k];
break;
}
}
}
}
// reset activations
for (int i = 0; i < network.Length; i++) {
for (int j = 0; j < network[i].Length; j++) {
Neuron neuron = network[i][j];
neuron.activation = 0;
}
}
}
// after everything we now actually add the desired changes
// the weights
for (int i = 0; i < network.Length - 1; i++) {
for (int j = 0; j < network[i].Length; j++) {
Neuron neuron = network[i][j];
for (int k = 0; k < neuron.desiredWeights.Length; k++) {
double weightSum = 0;
foreach (double weight in neuron.desiredWeights[k]) {
weightSum += weight;
}
weightSum /= iterationLength;
neuron.weights[k] -= learningRate * weightSum;
}
}
}
// the biases
for (int i = 1; i < network.Length; i++) {
for (int j = 0; j < network[i].Length; j++) {
Neuron neuron = network[i][j];
double biasSum = 0;
foreach (double bias in neuron.desiredBias) {
biasSum += bias;
}
biasSum /= iterationLength;
neuron.bias -= learningRate * biasSum;
}
}
double netcost = 0;
foreach (double cost in costs) {
netcost += cost;
}
return netcost/costs.Length;
}
public void epoch(bool showInfo) {
Console.WriteLine("Starting training...\n");
for (int iter = 0; iter < iterations; iter++) {
double cost = iteration();
if (showInfo) info();
Console.WriteLine("Cost for iteration " + iter + ": " + cost + "\n");
// reset desired
for (int k = 0; k < network.Length - 1; k++) {
for (int l = 0; l < network[k].Length; l++) {
Neuron neuron = network[k][l];
for (int i = 0; i < neuron.desiredWeights.Length; i++) {
for (int j = 0; j < neuron.desiredWeights[i].Length; j++) {
neuron.desiredWeights[i][j] = 0;
}
}
}
}
for (int k = 1; k < network.Length; k++) {
for (int l = 0; l < network[k].Length; l++) {
Neuron neuron = network[k][l];
for (int i = 0; i < neuron.desiredBias.Length; i++) {
neuron.desiredBias[i] = 0;
}
}
}
}
Console.WriteLine("Training finished!\n");
}
public void prompt() {
while(true) {
Console.WriteLine("What would you like to do?");
Console.WriteLine("Train, info, run or end?");
string answer = Console.ReadLine();
if (answer.ToLower() == "end") {
Console.WriteLine("...");
break;
}
switch(answer.ToLower()) {
case "info":
Console.WriteLine();
info();
Console.WriteLine();
break;
case "run":
double[] runanswer = new double[network[0].Length];
while (true) {
try {
for (int i = 0; i < network[0].Length; i++) {
Console.WriteLine("Enter input " + i);
runanswer[i] = Convert.ToDouble(Console.ReadLine());
}
Console.WriteLine("Running...\n");
double[] outputs = run(runanswer);
string _string = "";
foreach (double output in outputs) {
_string += output;
}
Console.WriteLine("Here were the outputs:");
Console.WriteLine(_string + "\n");
break;
} catch (FormatException exception) {
Console.WriteLine("Only numbers!");
continue;
}
}
break;
case "train":
int trainanswer = 0;
int trainanswer2 = 0;
double trainanswer3 = 0;
while (true) {
try {
Console.WriteLine("Enter epoch length:");
trainanswer = Convert.ToInt32(Console.ReadLine());
trainingLength = trainanswer;
Console.WriteLine("\nEnter batch size:");
trainanswer2 = Convert.ToInt32(Console.ReadLine());
iterations = trainanswer2;
Console.WriteLine("\nEnter learning rate:");
trainanswer3 = Convert.ToDouble(Console.ReadLine());
learningRate = trainanswer3;
for (int i = 0; i < trainanswer2; i++) {
double cost = iteration();
Console.WriteLine("The cost was: " + cost + "\n");
}
break;
} catch (FormatException exception) {
Console.WriteLine("Only numbers!");
continue;
}
}
break;
default:
Console.WriteLine("Invalid command!\n");
break;
}
}
}
}
public class Run {
public static void Main(string[] args) {
Network brain = new Network(new int[] { 2, 1 }, .01, 50, 50, Functions.LossFunction.MSE);
brain.init();
int i = 7;
while (i > 0) {
brain.epoch(false);
i--;
}
brain.prompt();
}
}
}
*** to clarify if you don't understand my question, I'm just asking if i'm implementing backpropagation correctly
