I am really new to neural networks, so i was following along with a video series, created by '3blue1brown' on youtube. I created an implementation of the network he explained in c++. I am attempting to train the network to recognize hand written characters, using the MNIST data set. What seems to be happening is, rather than actually learn how to recognize the characters, it is just learning how many of each input there is in the data set and doing it with probability. When testing on a smaller dataset this is more noticeable, for example i was testing on a set with 100, and the numbers that were more frequent would always have a slightly higher activation at the end, and others where very close to 0. Here is my code if it helps:
#include <random>
#include <vector>
#include <iostream>
#include <fstream>
#include <cmath>
double weightMutationRate = 1;
double biasMutationRate = 1;
//Keeps track of the weights, biases, last activation and the derivatives
//of the weights and biases for a single node in a neural network.
struct Node
{
std::vector<double> weights;
std::vector<double> derivWeights;
double activation;
double derivActivation;
double bias;
double derivBias;
};
//Struct to hold the nodes in each layer of a network.
struct Layer
{
std::vector<struct Node> nodes;
};
//Struct to hold the layers in a network.
struct Network
{
std::vector<struct Layer> layers;
double cost;
};
//Stores the inputs and outputs for a single training example.
struct Data
{
std::vector<double> inputs;
std::vector<double> answers;
};
//Stores all of the data to be used to train the neural network.
struct DataSet
{
std::vector<struct Data> data;
};
//Generates a double by creating a uniform distribution between the two arguments.
double RandomDouble(double min, double max)
{
std::random_device seed;
std::mt19937 random(seed());
std::uniform_real_distribution<> dist(min, max);
return dist(random);
}
//Constructs a network with the node count in each layer defined with 'layers'.
//the first layer will not have any weights and biases and will simply have
//the activation of the input data.
struct Network CreateNetwork(std::vector<int> layers, double minWeight = -1, double maxWeight = 1, double minBias = -1, double maxBias = 1)
{
//Network to construct.
struct Network network;
//Used to store the nodes in the previous layer.
int prevLayerNodes;
//Iterates through the layers vector and constructs a neural network with the values in
//the vector determining how many nodes that are in each of the layers.
bool isFirstLayer = true;
for (int layerNodes : layers)
{
//Layer to construct.
struct Layer layer;
//Creating the nodes for the current layer.
for (int i = 0; i < layerNodes; i++)
{
//Node to construct
struct Node node;
//Checks to see if the current layer is not the input layer, which does not have
//any weights or biases.
if (!isFirstLayer)
{
//Creating weights for the connections between this node
//and the nodes in the previous layer.
for (int i = 0; i < prevLayerNodes; i++)
{
//Getting a random double for the weight, between the bounds set in the arguments.
double inputWeight = RandomDouble(minWeight, maxWeight);
//Adding the inputWeight to the current node.
node.weights.push_back(inputWeight);
//Adding a 0 to the deriv weights for the weight just added.
node.derivWeights.push_back(0.0);
}
//Getting a random double for the bias, between the bounds set in the arguments.
double bias = RandomDouble(minBias, maxBias);
//Adding the bias to the current node.
node.bias = bias;
//Adding the node to the layer.
}
layer.nodes.push_back(node);
}
//Updating the isFirstLayer variable if the current layer is the input layer.
if (isFirstLayer)
{
isFirstLayer = false;
}
//Updating the prevLayerNodes variable for use in the next layer.
prevLayerNodes = layerNodes;
//Adding the layer to the network.
network.layers.push_back(layer);
}
//Returning the constructed network.
return network;
}
//Outputs the network passed to the networkPrint.txt file.
void PrintNetwork(struct Network network)
{
std::cout << "Printing network ..." << std::endl;
std::ofstream networkPrintFile;
networkPrintFile.open("networkPrint.txt");
//Iterates through each of the layers in teh network.
for (int i = 0; i < network.layers.size(); i++)
{
std::cout << "Layer : " << i << std::endl;
networkPrintFile << "Layer " << i << ":" << std::endl;
//Iterates through each of the nodes in the current layer.
for (int j = 0; j < network.layers[i].nodes.size(); j++)
{
networkPrintFile << "\t" << "Node " << j << ":" << std::endl;
//Outputs the node's activation into networkPrintFile.
double activation = network.layers[i].nodes[j].activation;
networkPrintFile << "\t\t" << "Activation" << ": " << activation << std::endl;
//Outputs the node's derivActivation into networkPrintFile.
double derivActivation = network.layers[i].nodes[j].derivActivation;
networkPrintFile << "\t\t" << "Deriv Activation" << ": " << derivActivation << std::endl;
double bias = network.layers[i].nodes[j].bias;
double derivBias = network.layers[i].nodes[j].derivBias;
//Outputs the bias and derivative of the bias.
networkPrintFile << "\t\t" << "Bias" << ": " << bias << std::endl;
networkPrintFile << "\t\t" << "Deriv Bias" << ": " << derivBias << std::endl;
//Iterates through all of the inputWeights in the current node.
networkPrintFile << "\t\t" << "Weights" << ":" << std::endl;
for (int k = 0; k < network.layers[i].nodes[j].weights.size(); k++)
{
double inputWeight = network.layers[i].nodes[j].weights[k];
double derivWeight = network.layers[i].nodes[j].derivWeights[k];
networkPrintFile << "\t\t\t" << "Weight " << k << ":" << std::endl;
networkPrintFile << "\t\t\t\t" << "Value" << ":" << inputWeight << std::endl;
networkPrintFile << "\t\t\t\t" << "Derivative" << ":" << derivWeight << std::endl;
}
}
}
std::cout << "Done" << std::endl;
}
//Takes and input and peforms a mathematical sigmoid
//function on it and returns the value.
// 1
// σ(x) = ---------
// 1 + e^x
double Sigmoid(double input)
{
double expInput = std::exp(-input);
double denom = expInput + 1;
double value = 1 / denom;
return value;
}
//Returns the activation of the node passed in give the previous layer.
double CalculateNode(struct Node &node, struct Layer &prevLayer)
{
//Keeps a runing total of the weights and activations added up so far.
double total = 0.0;
int weightCount = node.weights.size();
//Iterated through each of the weights, and thus each of the
//nodes in the previous layer to find the weight * activation.
for (int i = 0; i < weightCount; i++)
{
//Calculated the current weight and activation and
//adds it to the 'total' variable.
double weight = node.weights[i];
double input = prevLayer.nodes[i].activation;
double value = weight * input;
total += value;
}
//Add the node's bias to the total.
total += node.bias;
//Normalises the node's activation value by passing it through
//a sigmoid function, which bounds it between 0 and 1.
double normTotal = Sigmoid(total);
//Returns the caclulated value for this node.
return normTotal;
}
//Adds the activation values to a layer passed in, given the previous layer.
void CaclulateLayer(struct Layer &layer, struct Layer &prevLayer)
{
//Iterates through all of the nodes and calculated their activations.
for (struct Node &node : layer.nodes)
{
double activation = CalculateNode(node, prevLayer);
//Setting the activation to the node.
node.activation = activation;
}
}
//Takes in the first layer of the neural network and interates through the
//nodes and sets each input to each node in a loop.
void SetInputs(struct Layer &layer, std::vector<double> inputs)
{
for (int i = 0; i < layer.nodes.size(); i++)
{
//Setting the node's activation to the corrosponding input.
layer.nodes[i].activation = inputs[i];
}
}
//Takes in a network and inputs and calculates the value of
//activation for every node for a single input vector.
void CalculateNetwork(struct Network &network, std::vector<double> inputs)
{
//Setting the activations of the first layer to the inputs vector.
SetInputs(network.layers[0], inputs);
//Iterates through all of the layers, apart from the first layer, and
//calculated the activations of the nodes in that layer.
for (int i = 1; i < network.layers.size(); i++)
{
//Getting the layer to calculate to activations on and the
//previous layer, which already has it's activations calculated.
struct Layer currentLayer = network.layers[i];
struct Layer prevLayer = network.layers[i - 1];
//Calculating the nodes on the current layer.
CaclulateLayer(currentLayer, prevLayer);
//Setting the currentLayer back into the network struct with
//all of the activations in it now calculated.
network.layers[i] = currentLayer;
}
}
//Caclulates the sum of the differences between the outputs and the correct
//values squared.
//
// Cost = Σ((a-y)^2)
//
double CalculateCost(struct Network &network, std::vector<double> correctOutputs)
{
//Keeps track of the current sum of the costs.
double totalCost = 0.0;
//The layer of the network that holds the calculated values, the
//last layer in the network.
struct Layer outputLayer = network.layers[network.layers.size() - 1];
//Loops through all the node sin the output layer and compared them
//to their corresponding correctOutput value, calculates the cost
//and adds it to the running total, totalCoat.
for (int i = 0; i < outputLayer.nodes.size(); i++)
{
struct Node node = outputLayer.nodes[i];
double calculatedActivation = node.activation;
double correctActivation = correctOutputs[i];
double diff = calculatedActivation - correctActivation;
double modDiff = diff * diff;
//Adding the cost to the sum of the other costs.
totalCost += modDiff;
}
//Returning the value of the calculated cost.
return totalCost;
}
//Takes in the output layer of the network and calculates the derivatives of the
//cost function with respect to the activations in each node. this value is then
//stored on the Node struct.
void LastLayerDerivActivations(struct Layer &layer, std::vector<double> correctOutputs)
{
//Iterating through all the nodes in the layer.
for (int i = 0; i < layer.nodes.size(); i++)
{
//Getting the values of the node output and correct output.
double activation = layer.nodes[i].activation;
double correctOutput = correctOutputs[i];
//Caclulating the partial derivative of the cost function with respect
//to the current node's activation value.
double activationDiff = activation - correctOutput;
double derivActivation = 2 * activationDiff;
//Setting the activation partial derivative to the layer passed in.
layer.nodes[i].derivActivation = derivActivation;
}
}
//Returns the derivative of the sigmoid function.
// d
// ---- σ(x) = σ(x)(1 - σ(x))
// dx
double DerivSigma(double input)
{
double sigma = Sigmoid(input);
double value = sigma * (1 - sigma);
return sigma;
}
//Takes in a node and the layer that the node takes inputs from and adds.
//to the derivWeight and derivBias of the node and adds to each of the
//deriv activations in the previous layer for them to be used in this
//function to calculate their derivatives.
void NodeDeriv(struct Node &node, struct Layer &prevLayer)
{
//Starting the total at the bias.
double total = node.bias;
//Looping through all the weights and biases to find z(x).
// z(x) = a w + a w + ... + a w + b
// 1 1 2 2 n n
for (int i = 0; i < node.weights.size(); i++)
{
double weight = node.weights[i];
double activation = prevLayer.nodes[i].activation;
double value = weight * activation;
//Adding to the running total for z(x).
total += value;
}
//Finding the derivative of the cost function with respect to the
//z(x) by multiplying the DerivSigma() by the node's derivActivation
//using the chain rule.
double derivAZ = DerivSigma(total);
double derivCZ = derivAZ * node.derivActivation;
//The derivative of the cost with respect to the bias is the same as
//the derivative of the cost function with respect to z(x) since
//d/db z(x) = 1
node.derivBias += derivCZ;
//Iterating through all of the nodes and weights to find the derivatives
//of all of the weights for the node and the activations on the
//previous layer.
for (int i = 0; i < node.weights.size(); i++)
{
// dc/dw = dc/dz * activation
double derivCW = derivCZ * prevLayer.nodes[i].activation;
// dc/da = dc/dz * weight
double derivCA = derivCZ * node.weights[i];
//Adding the weights and activations to the node objects.
node.derivWeights[i] += derivCW;
prevLayer.nodes[i].derivActivation += derivCA;
}
//Resetting the activation derivative.
node.derivActivation = 0;
}
//Takes in a layer and iterates through all the nodes in order to find
//the derivatives of thw weight and biases in the current layer and the
//activations in the previous layer.
void LayerDeriv(struct Layer &layer, struct Layer &prevLayer)
{
for (int i = 0; i < layer.nodes.size(); i++)
{
NodeDeriv(layer.nodes[i], prevLayer);
}
}
//Takes in a network and uses backpropogation to find the derivatives of
//all the nodes for a single training example.
void NetworkDeriv(struct Network &network, std::vector<double> expectedOutputs)
{
//Calculating the derivatives of the activations in the last layer.
LastLayerDerivActivations(network.layers[network.layers.size() - 1], expectedOutputs);
//Looping through all the layers to find the derivatives of all of
//the weights and activations in the network for this training example.
for (int i = network.layers.size() - 1; i > 0; i--)
{
LayerDeriv(network.layers[i], network.layers[i - 1]);
}
}
//Takes in an input string and char and will return a vector of the string split
//by the char. The char is lost in this conversion.
std::vector<std::string> SplitString(std::string stringToSplit, char delimiter)
{
//Creating the output vector.
std::vector<std::string> outputVector;
//Initialising the lastDelimiter to -1, since the first string should be split as if
//the char before it was the splitter.
int lastDelimiterIndex = -1;
for (int i = 0; i < stringToSplit.size(); i++)
{
//Getting the current char.
char chr = stringToSplit[i];
//If the current char is the delimiter, create a new substring in the vector.
if (chr == delimiter)
{
//Creating the new substring at the delimiter and adding it to the end
//of the output vector.
std::string subString = stringToSplit.substr(lastDelimiterIndex + 1, i - lastDelimiterIndex - 1);
outputVector.push_back(subString);
//Setting the last delimiter variable to the current character.
lastDelimiterIndex = i;
}
}
//Adding the last section of the string to the output vector, since there is no
//delimiter and will not be added in the for loop.
std::string subString = stringToSplit.substr(lastDelimiterIndex + 1, stringToSplit.size() - lastDelimiterIndex - 1);
outputVector.push_back(subString);
//Returning the split string as a vector of strings.
return outputVector;
}
//Takes in a vector of strings and converts it to a vector of doubles. normalise argument
//sets what value will be taken to be 1, and other numbers will be a fraction of that.
//Set normalise to 0 to disable normalisation.
std::vector<double> ConvertStringVectorToDoubleVector(std::vector<std::string> input, int normalise = 0)
{
std::vector<double> convertedVector;
//Iterating through all the strings int the input vector.
for (std::string str : input)
{
//Converting the string into a double.
double value = stod(str);
//Checks to see if normalisation is enabled.
if (normalise != 0)
{
//Normalising the double.
value /= normalise;
}
//Adding the double to the output vector.
convertedVector.push_back(value);
}
//Returning the converted vector.
return convertedVector;
}
//Takes in a string of data and uses it to create a DataSet object to
//be used in the training of the neural network.
struct DataSet FormatData(std::string dataString)
{
struct DataSet dataSet;
//Splitting the input string into the seperate images.
std::vector<std::string> imageSplit = SplitString(dataString, '|');
//Looping through all of the images.
for (int i = 0; i < imageSplit.size(); i++)
{
//Getting the current image string.
std::string imageData = imageSplit[i];
//Splitting the image between the inputs and expected outputs.
std::vector<std::string> ioSplit = SplitString(imageData, '/');
std::string inputs = ioSplit[0];
std::string outputs = ioSplit[1];
//converting the input and output strings into string arrays of the values.
std::vector<std::string> inputVectorString = SplitString(inputs, ',');
std::vector<std::string> outputVectorString = SplitString(outputs, ',');
//Converting the string arrays into double arrays and normalising the input doubles.
std::vector<double> inputVector = ConvertStringVectorToDoubleVector(inputVectorString, 255);
std::vector<double> outputVector = ConvertStringVectorToDoubleVector(outputVectorString);
//Creating a new Data object.
struct Data data;
data.inputs = inputVector;
data.answers = outputVector;
//Adding the object to the dataset.
dataSet.data.push_back(data);
}
//Returning the completed dataset.
return dataSet;
}
//Takes in a filename and extracts all of the ascii data from the
//file and calls the FormatData function to create a DataSet object.
struct DataSet CreateDataSetFromFile(std::string fileName)
{
//Opening file.
std::ifstream dataFile;
dataFile.open(fileName);
//Storing fild data in a string.
std::string data;
dataFile >> data;
//Creating DataSet object.
struct DataSet dataSet = FormatData(data);
//Returning completed DataSet object.
return dataSet;
}
//Takes in a network and a Data object and runs the network and adds
//to the derivatives of the network for that one training example.
void NetworkIteration(struct Network &network, struct Data data)
{
//Extracting the input and output data from the data object.
std::vector<double> inputs = data.inputs;
std::vector<double> outputs = data.answers;
//Caclulating the activations of the network for this data.
CalculateNetwork(network, inputs);
//Caclulating the cost for this iteration and adding it to the total.
double cost = CalculateCost(network, outputs);
network.cost += cost;
//Caclulating the derivatives for the network weights and biases
//for this training example.
NetworkDeriv(network, outputs);
}
//Takes in a node and caclulates the average of the derivatives over
//the dataset and then multiplies them by a fixes mutation rate and
//applies the derivatives to the node's values.
void GradientDecentNode(struct Node &node, int dataCount)
{
//Iterating through all of the weights of the node.
for (int i = 0; i < node.weights.size(); i++)
{
double weight = node.weights[i];
double derivWeight = node.derivWeights[i];
//Getting the average over all of the training data.
derivWeight /= dataCount;
//Applying a constant multiplier to alter the rate at which is mutates.
derivWeight *= weightMutationRate;
//Subtracting the derivative from the weight.
node.weights[i] -= derivWeight;
//Reseting the weight derivative
node.derivWeights[i] = 0;
}
double bias = node.bias;
double derivBias = node.derivBias;
//Applying a constant multiplier to alter the rate at which is mutates.
derivBias *= biasMutationRate;
//Subtracting the derivative from the bias.
node.bias -= derivBias;
//Resetting the bias derivative.
node.derivBias = 0;
}
//Takes in a layer and iterated through all of the nodes in the layer
//and applies all of their derivatives to them.
void GradientDecentLayer(struct Layer &layer, int dataCount)
{
for (struct Node &node : layer.nodes)
{
GradientDecentNode(node, dataCount);
}
}
//Takes in a network and iterated through all of the layers and applies
//all of the derivatives to them.
void GradientDecentNetwork(struct Network &network, int dataCount)
{
for (int i = 1; i < network.layers.size(); i++)
{
GradientDecentLayer(network.layers[i], dataCount);
}
}
//Iterates through all of the training data in dataSet and calculates the derivatives
//of the weights and biases and then peforms the gradient decent using the derivatives.
void TrainNetworkSingle(struct Network &network, struct DataSet dataSet)
{
//Iterating through all of the training data.
for (struct Data data : dataSet.data)
{
//Caclulating the network for a single training example.
NetworkIteration(network, data);
}
//Peforming the derivatives.
GradientDecentNetwork(network, dataSet.data.size());
}
void TrainNetwork(struct Network &network, struct DataSet dataSet, int iterations)
{
for (int i = 0; true/*i < iterations*/; i++)
{
TrainNetworkSingle(network, dataSet);
std::cout << network.cost << std::endl;
network.cost = 0;
}
}
int main()
{
struct Network network = CreateNetwork({784, 784, 16, 16, 10});
struct DataSet dataSet = CreateDataSetFromFile("data.txt");
TrainNetwork(network, dataSet, 100);
PrintNetwork(network);
return 0;
}
```
