Skip to main content
deleted 2241 characters in body; edited tags
Source Link
nbro
  • 41.4k
  • 12
  • 114
  • 205

I'm working on an implementation of the backpropagation algorithm for a simple neural network, which predicts a probability of survival (1 or 0) and.

However, I can't get it above 80%, no matter how much I try to set the right hyperparameters. I suspect that's because my backpropagation is implemented incorrectly, since I tried 2 different types of code and both give me the same results. 

Is my backpropagation implemented correctly? Also how can I improve my model to givethere a better predictionway to determine whether my implementation of backpropagation is correct?

class NeuralNetwork(object):

def __init__(self, input_nodes, hidden_nodes, output_nodes, learning_rate):
    # Set number of nodes in input, hidden and output layers.
    self.input_nodes = input_nodes
    self.hidden_nodes = hidden_nodes
    self.output_nodes = output_nodes
    self.lr = learning_rate
    
    # Initialize weights
    self.input_hidden_weights = np.random.randn(hidden_nodes, input_nodes) # 10x7
    self.hidden_output_weights = np.random.randn(output_nodes, hidden_nodes) # 1x10
    
    
    # Sigmoid activation funciton
    self.sigmoid = lambda x: 1/(1+np.exp(-x))
    self.diff_sigm = lambda x: x*(1-x)
    
def train(self, input_list, label_list):
    
    # Create an array of inputs and labels
    inputs = np.array(input_list, ndmin=2).T # 7x1
    labels = np.array(label_list, ndmin=2) # 1x1
            
    # Forward propagation
    hidden_layer = self.sigmoid(np.dot(self.input_hidden_weights, inputs))        
    output_layer = self.sigmoid(np.dot(self.hidden_output_weights, hidden_layer))
    
    final_output = output_layer
    
    
    # Error function
    output_errors = labels-final_output
    
    # Backpropagation  
    output_delta = output_errors * self.diff_sigm(output_layer)
    hidden_delta = np.dot(self.hidden_output_weights.T, output_delta) * self.diff_sigm(hidden_layer)
    
    # Update the weights
    self.hidden_output_weights += np.dot(output_delta, hidden_layer.T) * self.lr
    self.input_hidden_weights += np.dot(hidden_delta, inputs.T) * self.lr
    
    
    """
    # Backpropagation
    hidden_errors = np.dot(self.hidden_output_weights.T, output_errors)        
    hidden_grad = hidden_layer * (1.0 - hidden_layer)
    
    # Update the weights
    self.hidden_output_weights += self.lr * np.dot(output_errors.T, output_layer.T) # update hidden-to-output weights with gradient descent step    
    self.input_hidden_weights += self.lr * np.dot(hidden_errors * hidden_grad, inputs.T)  # update input-to-hidden weights with gradient descent step
    """

I'm working on implementation of the backpropagation algorithm for a simple neural network which predicts a probability of survival (1 or 0) and I can't get it above 80% no matter how much I try to set the right hyperparameters. I suspect that's because my backpropagation is implemented incorrectly since I tried 2 different types of code and both give me same results. Is my backpropagation implemented correctly? Also how can I improve my model to give a better prediction?

class NeuralNetwork(object):

def __init__(self, input_nodes, hidden_nodes, output_nodes, learning_rate):
    # Set number of nodes in input, hidden and output layers.
    self.input_nodes = input_nodes
    self.hidden_nodes = hidden_nodes
    self.output_nodes = output_nodes
    self.lr = learning_rate
    
    # Initialize weights
    self.input_hidden_weights = np.random.randn(hidden_nodes, input_nodes) # 10x7
    self.hidden_output_weights = np.random.randn(output_nodes, hidden_nodes) # 1x10
    
    
    # Sigmoid activation funciton
    self.sigmoid = lambda x: 1/(1+np.exp(-x))
    self.diff_sigm = lambda x: x*(1-x)
    
def train(self, input_list, label_list):
    
    # Create an array of inputs and labels
    inputs = np.array(input_list, ndmin=2).T # 7x1
    labels = np.array(label_list, ndmin=2) # 1x1
            
    # Forward propagation
    hidden_layer = self.sigmoid(np.dot(self.input_hidden_weights, inputs))        
    output_layer = self.sigmoid(np.dot(self.hidden_output_weights, hidden_layer))
    
    final_output = output_layer
    
    
    # Error function
    output_errors = labels-final_output
    
    # Backpropagation  
    output_delta = output_errors * self.diff_sigm(output_layer)
    hidden_delta = np.dot(self.hidden_output_weights.T, output_delta) * self.diff_sigm(hidden_layer)
    
    # Update the weights
    self.hidden_output_weights += np.dot(output_delta, hidden_layer.T) * self.lr
    self.input_hidden_weights += np.dot(hidden_delta, inputs.T) * self.lr
    
    
    """
    # Backpropagation
    hidden_errors = np.dot(self.hidden_output_weights.T, output_errors)        
    hidden_grad = hidden_layer * (1.0 - hidden_layer)
    
    # Update the weights
    self.hidden_output_weights += self.lr * np.dot(output_errors.T, output_layer.T) # update hidden-to-output weights with gradient descent step    
    self.input_hidden_weights += self.lr * np.dot(hidden_errors * hidden_grad, inputs.T)  # update input-to-hidden weights with gradient descent step
    """

I'm working on an implementation of the backpropagation algorithm for a simple neural network, which predicts a probability of survival (1 or 0).

However, I can't get it above 80%, no matter how much I try to set the right hyperparameters. I suspect that's because my backpropagation is implemented incorrectly, since I tried 2 different types of code and both give me the same results. 

Is there a way to determine whether my implementation of backpropagation is correct?

Source Link

How do I know if my backpropagation is implemented correctly?

I'm working on implementation of the backpropagation algorithm for a simple neural network which predicts a probability of survival (1 or 0) and I can't get it above 80% no matter how much I try to set the right hyperparameters. I suspect that's because my backpropagation is implemented incorrectly since I tried 2 different types of code and both give me same results. Is my backpropagation implemented correctly? Also how can I improve my model to give a better prediction?

class NeuralNetwork(object):

def __init__(self, input_nodes, hidden_nodes, output_nodes, learning_rate):
    # Set number of nodes in input, hidden and output layers.
    self.input_nodes = input_nodes
    self.hidden_nodes = hidden_nodes
    self.output_nodes = output_nodes
    self.lr = learning_rate
    
    # Initialize weights
    self.input_hidden_weights = np.random.randn(hidden_nodes, input_nodes) # 10x7
    self.hidden_output_weights = np.random.randn(output_nodes, hidden_nodes) # 1x10
    
    
    # Sigmoid activation funciton
    self.sigmoid = lambda x: 1/(1+np.exp(-x))
    self.diff_sigm = lambda x: x*(1-x)
    
def train(self, input_list, label_list):
    
    # Create an array of inputs and labels
    inputs = np.array(input_list, ndmin=2).T # 7x1
    labels = np.array(label_list, ndmin=2) # 1x1
            
    # Forward propagation
    hidden_layer = self.sigmoid(np.dot(self.input_hidden_weights, inputs))        
    output_layer = self.sigmoid(np.dot(self.hidden_output_weights, hidden_layer))
    
    final_output = output_layer
    
    
    # Error function
    output_errors = labels-final_output
    
    # Backpropagation  
    output_delta = output_errors * self.diff_sigm(output_layer)
    hidden_delta = np.dot(self.hidden_output_weights.T, output_delta) * self.diff_sigm(hidden_layer)
    
    # Update the weights
    self.hidden_output_weights += np.dot(output_delta, hidden_layer.T) * self.lr
    self.input_hidden_weights += np.dot(hidden_delta, inputs.T) * self.lr
    
    
    """
    # Backpropagation
    hidden_errors = np.dot(self.hidden_output_weights.T, output_errors)        
    hidden_grad = hidden_layer * (1.0 - hidden_layer)
    
    # Update the weights
    self.hidden_output_weights += self.lr * np.dot(output_errors.T, output_layer.T) # update hidden-to-output weights with gradient descent step    
    self.input_hidden_weights += self.lr * np.dot(hidden_errors * hidden_grad, inputs.T)  # update input-to-hidden weights with gradient descent step
    """