# Why doesnt my Neural Network work?

I Build this NN in c++. I reviewed it since 3 days. I checked every line 100 times, but I cant find my error. If someone can please help me find the Bugs: 1. The output is garbage 2. The weights go from 2e^79 down to -1.8e^80 after approximatly 400 iterations.

mat flip(mat m) {
mat out(m.n_cols,m.n_rows);

for (int i = 0; i < m.n_rows; ++i)
for (int j = 0; j < m.n_cols; ++j)
out(j, i) = m(i, j);

return out;
}

Layer::Layer(int nodes) :
rand_engine(time(0))
{

y = mat (nodes, 1);
net = mat(nodes, 1);
e = mat(nodes, 1);
}

Layer::Layer(int nodes, int next_nodes) :
Layer(nodes)
{

this->next_l = next_l;

auto random = bind(uniform_real_distribution<double>{-1, 1}, rand_engine);

w = mat(next_nodes,nodes);

for (int i = 0; i < w.n_rows; ++i) {

for (int j = 0; j < w.n_cols; ++j) {
w(i,j) = random();
}
}
}

Layer::Layer(int nodes, Layer* next_l) :
Layer(nodes,next_l->y.n_rows)
{
this->next_l = next_l;
}

void Layer::feed_forward()
{
next_l->net = w*y;

for (int i = 0; i < next_l->y.n_rows;++i)
next_l->y[i] = sig(next_l->net[i]);

}

void Layer::backprop()
{

for (double d : w)
cout << d << "\t";

e = flip(w)*next_l->e;

for (int i = 0; i < e.n_rows; ++i) {
e[i] *= net[i] * (1 - net[i]);
cout << e[i] << '\t';
}

w += l_rate*(next_l->e*flip(y));

}

void Layer::backprop_last(mat t)
{
for (int i = 0; i < e.n_rows; ++i) {
e[i] = net[i] * (1 - net[i])*(t[i] - y[i]);
cout << e[i] << '\t';
}

}

void Layer::feed_forward(Layer* next_l)
{
this->next_l = next_l;

feed_forward();

}

double Layer::sig(double x)
{
return 1 / (1 + exp(-x));
}

Network::Network(vector<int> top):
top(top)
{

network = new Layer*[top.size()];
network[top.size() - 1] = new Layer(top.back());

for (int i = top.size()-2; i > -1; --i)
network[i] = new Layer(top[i], network[i + 1]);

}

Network::~Network()
{
delete[] network;
}

void Network::forward()
{
for(int i = 0; i < top.front();++i)
network[0]->y[i] = input[i];

for (int i = 0; i < top.size() - 1; ++i)
network[i]->feed_forward();
}

void Network::forward(vector<double> input)
{
set_input(input);
forward();
}

void Network::backprop()
{

network[top.size() - 1]->backprop_last(t_vals);

for (int i = top.size() - 2; i > -1; --i) {
network[i]->backprop();
}

}

void Network::backprop(vector<double> t_vals)
{
set_t_vals(t_vals);
backprop();
}


I know its a bunch of code but im really desprate since I cant find whats wrong. I tested it with a simple XOR.

Edit: Heres my Main code:

    #include "Network.h"
#include <iomanip>

using namespace std;

vector<vector<double>> input = { {0,0},{0,1},{1,1},{1,0} };

vector<vector<double>> true_vals = { {0},{1},{0},{1} };

int main() {

ifstream f("out.txt", fstream::out);
f.clear();

cout << fixed;
cout << setprecision(5);

Network net({2,5,1});

vector<double> in,t,out;

auto buf = cout.rdbuf();

for (int i = 0; i < 1000; ++i) {
cout.rdbuf(f.rdbuf());

in = input[i % 4];

net.forward(in);

out = net.get_output();

t = true_vals[i % 4];

net.backprop(t);
cout << '\n';
cout.rdbuf(buf);
if ((i %101))continue;

cout << "it: " << i << '\n';

cout << "in:\t";
for (double d : in)
cout << d << ' ';

cout << '\n';

cout << "out:\t";

for (double d : out)
cout << d << ' ';

cout << '\n';

cout << "true:\t";

for (double d : t)
cout << d << ' ';

cout << '\n';

double err = net.get_error();

cout <<"err:\t"<< err << '\n' << '\n';

}

cout.rdbuf(NULL);
f.close();
return system("pause");
}

• What is this code for? Problem statement? – kiner_shah Dec 31 '16 at 13:49
• @kiner_shah that's the code I made. And I want to know if anyone knows why it doesn't work. – Miemels Dec 31 '16 at 13:51
• What is that code for? XOR function or AND function or something else? And this is not your entire code! Post your whole code with main() function – kiner_shah Dec 31 '16 at 13:51
• @kiner_shah I added the main code – Miemels Dec 31 '16 at 14:06
• This question is off-topic here, sorry. – Mithical Dec 31 '16 at 18:01

XOR input space is not linearly separable. It means that you cannot separate the input points in a 2D space into 1 area and 0 area by simply drawing a line between them. It requires at least 2 lines to separate the XOR input space and consequently 2 output nodes (used as classifiers rather than regression). You can easily find its details in google. Search "XOR problem in Neural Net".

You can manually implement the desired Neural Network with two output Nodes acting as classifiers as follows :

Where A & B are two output Nodes which act as classifiers (by forming AA' and BB' decision lines respectively during training by Backpropagation). The interpretation of the Outputs of the nodes is given in the table where Net column represents the Overall output to be interpreted.

I showed the above manual implementation just to give you the idea of how classification is done behind the scenes.

Here is the actual Automatic implementation :

Here , all the task is performed by the Neural Net behind the scenes and you get the desired output from the output node in the topmost layer

• Since you said the 4 required outputs are used as classifiers, why do I need 4 classifiers for a binary output and how to I interpret it. – Miemels Dec 31 '16 at 14:27
• Now I'm confused. On the link the network only has 1 output node. You stated it needs 2. Please explain what I'm getting wrong. – Miemels Dec 31 '16 at 14:42
• Ok... Ignore it. I am editing my answer to make you understand. – Programmer Dec 31 '16 at 14:54
• Ok thank you very much. I think I understand it now. – Miemels Dec 31 '16 at 15:20