I am confused by where the recurrence happens in RNNs, especially in the context of deep neural networks. I am trying to transform an ordinary neural network into a recurrent one from scratch.
nn = NeuralNetwork()
nn.add(Layer(2, 2)) # fully connected layer
nn.add(ReLU()) # activation layer/function
nn.add(Layer(2, 1))
nn.add(Tanh())
Does the recurrence happen at the last hidden layer of the network or after each fully connected layer?
I have heard of stacking RNNs, but I am not sure why we can't simply add a layer to an existing RNN. Is it because the recurrence is after the activation function/layer, hence the following code won't work?
import numpy as np
class Layer():
'''
Fullly connected layer
'''
def __init__(self, input_size, output_size, randomize=True):
# initialize weights and biases
if randomize:
self.W = np.random.randn(input_size, output_size)
else:
self.W = np.ones((input_size, output_size))
self.b = np.zeros((1, output_size))
def forward(self, X):
return np.dot(X, self.W) + self.b
class NeuralNetwork():
'''
Neural network divided into layers
'''
def __init__(self):
self.layers = []
def add(self, layer):
'''
Add layer
'''
self.layers.append(layer)
def forward(self, X):
'''
Propagate input forward through each layer
'''
for layer in self.layers:
# output of current layer becomes input of next layer
X = layer.forward(X)
return X
class ReLU():
'''
ReLU activation layer
'''
def forward(self, X):
self.X = X
return np.maximum(0, X)
class RecurrentLayer():
def __init__(self, input_size, output_size, randomize=True):
self.layer = Layer(input_size, output_size, randomize)
if randomize:
self.V = np.random.randn(output_size, output_size)
else:
self.V = np.ones((output_size, output_size))
def forward(self, X_seq):
hidden_state = []
for x_t in X_seq:
h_t = self.layer.forward(x_t)
try:
h_t += np.dot(hidden_state[-1], self.V)
except:
pass
hidden_state.append(h_t)
return hidden_state
X_sequence = [[1, 1], [2, 2], [3, 3]]
nn = NeuralNetwork()
nn.add(RecurrentLayer(2, 2, randomize=False))
nn.add(ReLU())
nn.add(RecurrentLayer(2, 1, randomize=False))
nn.forward(X_sequence)
[array([[4.]]), array([[20.]]), array([[64.]])]
