1
$\begingroup$

I'm a beginner in machine learning and I was trying to make a test neural network for digits recognition from scratch using Numpy. I used MNIST dataset for training and testing. Input layer have 28*28 neurons which correspond to each pixel of image that must be recognized. Output layer have 10 neurons which correspond to each digit (0-9), and return values from 0 to 1 which mean chance that the corresponding digit is displayed on the image. Class Layer represents a separate layer and contains links to previous and next layer (if prevLayer is None, the current layer is input layer; if nextLayer is None, the current layer is output). The forward() method is responsible for passing data through neural network. The backprop() method is responsible for training of neural network via Backpropagation algorithm. A Layer object contains weights (W) between previous and current layer (input layer object doesn't contain weights). 'data_in' property contains a vector of calculated values before passing them into activation function. Property 'data' contains the values after activation function. But, unfortunately, it doesn't work: returned value of loss function doesn't decrease during training and neural network returns the same result during testing. I assume that bugs might be associated with backprop() and softmax_derivatime() methods. I tried in vain to find all bugs. Here's my code:

import numpy as np

def ReLU(x):
    return np.maximum(0, x)

def ReLU_derivative(x):
    return np.greater(x, 0).astype(int)

def softmax(x):
    shift = x - np.max(x)
    return np.exp(shift) / np.sum(np.exp(shift))

def softmax_derivative(x):
    sm_array = softmax(x)
    J = np.zeros((x.size, x.size))
    for i in range(x.size):
        for j in range(x.size):
            delta = np.equal(i, j).astype(int)
            J[j, i] = sm_array[0][i] * (delta - sm_array[0][j])
    return J

class Layer:
    def __init__(self, size, prev_layer=None):
        self.size = size
        self.prevLayer = prev_layer
        self.nextLayer = None
        self.data = None
        self.data_in = None
        if prev_layer is not None:
            self.prevLayer.nextLayer = self
            self.W = np.random.random((self.prevLayer.size, size))
            self.W_bias = np.array([np.random.random(size)])
        else:
            self.W = None
            self.W_bias = None

    def forward(self):
        if self.prevLayer is not None:
            self.data_in = np.dot(self.prevLayer.data, self.W)
            self.data_in += np.dot([[1]], self.W_bias)
            if self.nextLayer is not None:
                self.data = ReLU(self.data_in)
                self.nextLayer.forward()
            else:
                self.data = softmax(self.data_in)
        else:
            self.nextLayer.forward()

    def backprop(self, expected_output=None, prev_delta=None):
        if prev_delta is None:
            #print(self.data_in)
            delta = np.dot(-(expected_output - self.data), softmax_derivative(self.data_in))
            delta_bias = delta
        else:
            delta = np.dot(prev_delta, self.nextLayer.W.T) * ReLU_derivative(self.data_in)
            delta_bias = np.dot(prev_delta, self.nextLayer.W_bias.T) * ReLU_derivative(self.data_in)
        training_velocity = 0.1
        W_dif = np.dot(self.prevLayer.data.T, delta) * training_velocity
        W_bias_dif = np.dot([[1]], delta_bias) * training_velocity
        if self.prevLayer.prevLayer is not None:
            self.prevLayer.backprop(prev_delta=delta)
        self.W -= W_dif
        self.W_bias -= W_bias_dif

f_images = open("train-images.idx3-ubyte", "br")
f_images.seek(4)
f_labels = open("train-labels.idx1-ubyte", "br")
f_labels.seek(8)
images_number = int.from_bytes(f_images.read(4), byteorder='big')
rows_number = int.from_bytes(f_images.read(4), byteorder='big')
cols_number = int.from_bytes(f_images.read(4), byteorder='big')

input_layer = Layer(rows_number*cols_number)
hidden_layer1 = Layer(rows_number*cols_number*7//10, input_layer)
hidden_layer2 = Layer(rows_number*cols_number*7//10, hidden_layer1)
output_layer = Layer(10, hidden_layer2)
digits = np.array([np.zeros(10)])

input_image = np.array([np.zeros(rows_number * cols_number)])
for k in range(images_number):
    for i in range(rows_number):
        for j in range(cols_number):
            input_image[0][i*cols_number+j] = int.from_bytes(f_images.read(1), byteorder='big') / 255.0 * 2 - 1
    input_layer.data = input_image
    input_layer.forward()
    current_digit = int.from_bytes(f_labels.read(1), byteorder='big')
    digits[0][current_digit] = 1
    output_layer.backprop(expected_output=digits)
    print(np.sum((digits - output_layer.data)**2)/2)
    digits[0][current_digit] = 0
    if((k+1) % 1000 == 0):
        print(str(k+1) + " / " + str(images_number))
f_images.close()
f_labels.close()

f_images = open("t10k-images.idx3-ubyte", "br")
f_images.seek(4)
f_labels = open("t10k-labels.idx1-ubyte", "br")
f_labels.seek(8)
images_number = int.from_bytes(f_images.read(4), byteorder='big')
rows_number = int.from_bytes(f_images.read(4), byteorder='big')
cols_number = int.from_bytes(f_images.read(4), byteorder='big')

for k in range(images_number):
    for i in range(rows_number):
        for j in range(cols_number):
            input_image[0][i*cols_number+j] = int.from_bytes(f_images.read(1), byteorder='big')
    input_layer.data = input_image
    input_layer.forward()
    current_digit = int.from_bytes(f_labels.read(1), byteorder='big')
    print(output_layer.data)

f_images.close()
f_labels.close()

I would appreciate for any help. Thanks in advance!

$\endgroup$

closed as off-topic by Ben N Feb 16 at 19:38

This question appears to be off-topic. The users who voted to close gave this specific reason:

  • "This question does not appear to be about artificial intelligence, within the scope defined in the help center." – Ben N
If this question can be reworded to fit the rules in the help center, please edit the question.

  • $\begingroup$ welcome to Ai,however can you try to go through our community guidelines ,for effective feedback. $\endgroup$ – quintumnia Feb 9 at 11:51
1
$\begingroup$

It seems I've solved the issue. There was several mistakes:
1. I've generated random weights from 0 to 1. As a result, too big numbers passed through softmax function (>10000), and the function wasn't calculated correctly. I divided each initial weight on the number of neurons in previous layer and solved the issue.
2. I've calculated separate delta for biases while delta must be the same for main weights and biases.
If anyone is interested, here is the correct code (83% and 89% precision after first and second launch):

import numpy as np

def ReLU(x):
    return np.maximum(0, x)

def ReLU_derivative(x):
    return np.greater(x, 0).astype(int)

def softmax(x):
    shift = x - np.max(x)
    return np.exp(shift) / np.sum(np.exp(shift))

def softmax_derivative(x):
    sm_array = softmax(x)
    J = np.zeros((x.size, x.size))
    for i in range(x.size):
        for j in range(x.size):
            delta = np.equal(i, j).astype(int)
            J[j, i] = sm_array[0][i] * (delta - sm_array[0][j])
    return J

class Layer:
    def __init__(self, size, prev_layer=None):
        self.size = size
        self.prevLayer = prev_layer
        self.nextLayer = None
        self.data = None
        self.data_in = None
        if prev_layer is not None:
            self.prevLayer.nextLayer = self
            self.W = np.random.random((self.prevLayer.size, size)) / (self.prevLayer.size + 1)
            self.W_bias = np.random.random((1, size)) / (self.prevLayer.size + 1)
        else:
            self.W = None
            self.W_bias = None

    def forward(self):
        if self.prevLayer is not None:
            self.data_in = np.dot(self.prevLayer.data, self.W)
            self.data_in += np.dot([[1]], self.W_bias)
            if self.nextLayer is not None:
                self.data = ReLU(self.data_in)
                self.nextLayer.forward()
            else:
                self.data = softmax(self.data_in)
        else:
            self.nextLayer.forward()

    def backprop(self, expected_output=None, prev_delta=None):
        if prev_delta is None:
            delta = np.dot(-(expected_output - self.data), softmax_derivative(self.data_in))
        else:
            delta = np.dot(prev_delta, self.nextLayer.W.T) * ReLU_derivative(self.data_in)
        training_velocity = 0.1
        W_dif = np.dot(self.prevLayer.data.T, delta) * training_velocity
        W_bias_dif = np.dot([[1]], delta) * training_velocity
        if self.prevLayer.prevLayer is not None:
            self.prevLayer.backprop(prev_delta=delta)
        self.W -= W_dif
        self.W_bias -= W_bias_dif

f_images = open("train-images.idx3-ubyte", "br")
f_images.seek(4)
f_labels = open("train-labels.idx1-ubyte", "br")
f_labels.seek(8)
images_number = int.from_bytes(f_images.read(4), byteorder='big')
rows_number = int.from_bytes(f_images.read(4), byteorder='big')
cols_number = int.from_bytes(f_images.read(4), byteorder='big')

input_layer = Layer(rows_number*cols_number)
hidden_layer1 = Layer(rows_number*cols_number*7//10, input_layer)
hidden_layer2 = Layer(rows_number*cols_number*7//10, hidden_layer1)
output_layer = Layer(10, hidden_layer2)
digits = np.array([np.zeros(10)])

print("Training:")
input_image = np.array([np.zeros(rows_number * cols_number)])
for k in range(images_number):
    for i in range(rows_number):
        for j in range(cols_number):
            input_image[0][i*cols_number+j] = int.from_bytes(f_images.read(1), byteorder='big') / 255.0
    input_layer.data = input_image
    input_layer.forward()
    current_digit = int.from_bytes(f_labels.read(1), byteorder='big')
    digits[0][current_digit] = 1
    output_layer.backprop(expected_output=digits)
    digits[0][current_digit] = 0
    if((k+1) % 1000 == 0):
        print(str(k+1) + " / " + str(images_number))
f_images.close()
f_labels.close()

f_images = open("t10k-images.idx3-ubyte", "br")
f_images.seek(4)
f_labels = open("t10k-labels.idx1-ubyte", "br")
f_labels.seek(8)
images_number = int.from_bytes(f_images.read(4), byteorder='big')
rows_number = int.from_bytes(f_images.read(4), byteorder='big')
cols_number = int.from_bytes(f_images.read(4), byteorder='big')

print("\r\nTesting:")
correct = 0
for k in range(images_number):
    for i in range(rows_number):
        for j in range(cols_number):
            input_image[0][i*cols_number+j] = int.from_bytes(f_images.read(1), byteorder='big')
    input_layer.data = input_image
    input_layer.forward()
    current_digit = int.from_bytes(f_labels.read(1), byteorder='big')
    if np.argmax(output_layer.data[0]) == current_digit:
        correct += 1
    if((k+1) % 1000 == 0):
        print(str(k+1) + " / " + str(images_number))

print("\r\nCorrect: " + str(correct) + " / " + str(images_number))
f_images.close()
f_labels.close()
$\endgroup$

Not the answer you're looking for? Browse other questions tagged or ask your own question.