I tried training a 2 hidden layer network using the mnist dataset, but I am not getting any results. I have tried tuning the learning rate(tried 0.1 and 0.0001) and the number of epochs(tried 10 and 50). I even changed the size of hidden layer from 10 to 250. First i had initialized the weights between 0 and 1 and was getting the same classification for all test samples but added (-) sign to 50% of them(chose the figure of 50% by myself) and that problem was solved. Now I cant figure out why it is not working.
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import math
from sklearn.preprocessing import StandardScaler
scaler = StandardScaler()
def to_array(img):
img = np.asarray(img)
return img
'''def standardize(gray):
st_gray = (gray-np.amin(gray))/(np.amax(gray)-np.amin(gray))
return st_gray
'''
def activ_func(x):
for i in range(x.shape[0]):
'''x[i][0]=(1-np.e**(-2*x[i][0]))/(1+np.e**(-2*x[i][0]))'''
x[i][0] = 1/(1+np.e**(-x[i][0]))
return x
def deriv_activ_func(x):
for i in range(x.shape[0]):
'''x[i][0] = 1-math.pow(x[i][0],2)'''
x[i][0] = (x[i][0])*(1-x[i][0])
return x
def cost(out_layer, label, ind):
cost = (out_layer[ind]-label)**2
return cost
def update(x, grad, r):
for i in range(x.shape[0]):
x[i][0] = x[i][0]+r*grad[i][0]
return x
path = "mnist/mnist_train.csv"
gray = pd.read_csv(path)
labels = gray['label']
gray = gray.drop(['label'], axis=1)
gray = to_array(gray)
labels = to_array(labels)
st_gray = np.empty(shape=(gray.shape[1],1))
def rand_sign(w):
n = np.random.randint(2,size=w.shape[0]*w.shape[1]).reshape(w.shape[0],w.shape[1])
for i in range(w.shape[0]):
for j in range(w.shape[1]):
if(n[i][j]==1):
w[i][j]=(-1)*w[i][j]
return w
def initialize():
in_layer = np.empty(shape=(st_gray.shape[0],1))
out_layer = np.unique(labels).reshape(-1,1)
w1 = rand_sign(np.random.rand(250,in_layer.shape[0]))
b1 = rand_sign(np.random.rand(250,1))
l1 = np.empty(shape=(250,1))
w2 = rand_sign(np.random.rand(250,l1.shape[0]))
b2 = rand_sign(np.random.rand(250,1))
l2 = np.empty(shape=(250,1))
w3 = rand_sign(np.random.rand(out_layer.shape[0],l2.shape[0]))
b3 = rand_sign(np.random.rand(out_layer.shape[0],1))
l3 = np.empty_like(out_layer)
return l1,l2,l3,w1,w2,w3,b1,b2,b3,in_layer,out_layer
def feed_forward(l1,l2,l3,w1,w2,w3,b1,b2,b3,in_layer,i):
st_gray = scaler.fit_transform(gray[i][:].reshape(-1,1))
in_layer = st_gray
l1 = np.dot(w1,in_layer)+b1
l1 = activ_func(l1)
l2 = np.dot(w2,l1)+b2
l2 = activ_func(l2)
l3 = np.dot(w3,l2)+b3
l3 = activ_func(l3)
return l1,l2,l3,w1,w2,w3,b1,b2,b3,in_layer
def one_hot(out_layer,label):
for j in range(out_layer.shape[0]):
if(out_layer[j][0]==label):
out_layer[j][0] = 1
else:
out_layer[j][0] = 0
return out_layer
def back_prop(l1,l2,l3,w1,w2,w3,b1,b2,b3,in_layer,out_layer,lr):
error = out_layer-l3
grad = np.dot(error*deriv_activ_func(l3),l2.T)
w3 = update(w3, grad, lr)
grad = error*deriv_activ_func(l3)
b3 = update(b3, grad, lr)
grad = np.dot(w3.T,error*deriv_activ_func(l3))
error = grad
grad = error*deriv_activ_func(l2)
w2 = update(w2, grad, lr)
grad = error*deriv_activ_func(l2)
b2 = update(b2, grad, lr)
grad = np.dot(w2.T,error*deriv_activ_func(l2))
error = grad
grad = error*deriv_activ_func(l1)
w1 = update(w1, grad, lr)
grad = error*deriv_activ_func(l1)
b1 = update(b1, grad, lr)
return l1,l2,l3,w1,w2,w3,b1,b2,b3,in_layer
def predict(l3):
out = np.amax(l3)
count = 0
for j in range(l3.shape[0]):
count=count+1
if(l3[j]==out):
break
return count
def trainer():
l1,l2,l3,w1,w2,w3,b1,b2,b3,in_layer,out_layer = initialize()
for epochs in range(50):
for i in range(gray.shape[0]):
out_layer = np.unique(labels).reshape(-1,1)
l1,l2,l3,w1,w2,w3,b1,b2,b3,in_layer = feed_forward(l1,l2,l3,w1,w2,w3,b1,b2,b3,in_layer,i)
out_layer = one_hot(out_layer,labels[i])
l1,l2,l3,w1,w2,w3,b1,b2,b3,in_layer = back_prop(l1,l2,l3,w1,w2,w3,b1,b2,b3,in_layer,out_layer,0.0001)
print("End of epoch :",epochs+1)
return l1,l2,l3,w1,w2,w3,b1,b2,b3,in_layer,out_layer
l1,l2,l3,w1,w2,w3,b1,b2,b3,in_layer,out_layer = trainer()
path = "mnist/mnist_train.csv"
gray = pd.read_csv(path)
labels = gray['label']
gray = gray.drop(['label'], axis=1)
gray = to_array(gray)
labels = to_array(labels)
st_gray = np.empty(shape=(gray.shape[1],1))
for i in range(10):
st_gray = scaler.fit_transform(gray[i][:].reshape(-1,1))
in_layer = st_gray
l1 = np.dot(w1,in_layer)+b1
l1 = activ_func(l1)
l2 = np.dot(w2,l1)+b2
l2 = activ_func(l2)
l3 = np.dot(w3,l2)+b3
l3 = activ_func(l3)
count = predict(l3)
print("Expected: ",labels[i]," Predicted: ",count)