# Neural network returns about the same output(mean) for every input

I tried to build a neural network from scratch to build a cat or dog binary classifier using a sigmoid output unit. I seem to get the output value around 0.5(+/- 0.002) for every input. This seems really weird to me. Here's my code, Please let me know if there is a mistake in the implementation.

def initialize_parameters_deep(layer_dims):
l=len(layer_dims)
parameters={}
for l in range(1,len(layer_dims)):
parameters['W'+str(l)]=np.random.randn(layer_dims[l],layer_dims[l-1])*0.01
parameters['b'+str(l)]=np.zeros((layer_dims[l],1))
return parameters

def linear_forward(A,W,b):
Z=np.dot(W,A)+b
cache=(A,W,b)
return Z,cache

def sigmoid(Z):
A = 1/(1+np.exp(-Z))
cache=Z
return A, cache

def relu(Z):
A = np.maximum(0,Z)

assert(A.shape == Z.shape)

cache = Z
return A, cache

def relu_backward(dA, cache):
Z = cache
dZ = np.array(dA, copy=True) # just converting dz to a correct object.

# When z <= 0, you should set dz to 0 as well.
dZ[Z <= 0] = 0

assert (dZ.shape == Z.shape)

return dZ

def sigmoid_backward(dA, cache):
Z = cache

s = 1/(1+np.exp(-Z))
dZ = dA * s * (1-s)

assert (dZ.shape == Z.shape)

return dZ

def linear_activation_forward(A_prev,W,b,activation):
if(activation=='sigmoid'):
Z,linear_cache=linear_forward(A_prev,W,b)
A,activation_cache=sigmoid(Z)
elif activation=='relu':
Z,linear_cache=linear_forward(A_prev,W,b)
A,activation_cache=relu(Z)
cache=(linear_cache,activation_cache)
return A,cache

def L_model_forward(X,parameters):
A=X
L=len(parameters)//2
caches=[]
for l in range(1,L):
A,cache=linear_activation_forward(A,parameters['W'+str(l)],parameters['b'+str(l)],'relu')
caches.append(cache)
AL,cache=linear_activation_forward(A,parameters['W'+str(L)],parameters['b'+str(L)],'sigmoid')
caches.append(cache)
return AL,caches

def compute_cost(AL,Y):
m=Y.shape[1]
cost=-1/m*np.sum(np.multiply(np.log(AL),Y)+np.multiply(np.log(1-AL),1-Y))
return cost

def linear_backward(dZ,cache):
A_prev,W,b=cache
m=A_prev.shape[1]
dW = np.dot(dZ,A_prev.T)/m
db = np.sum(dZ,axis=1,keepdims=True)/m
dA_prev = np.dot(W.T,dZ)
return dA_prev,dW,db

def linear_activation_backward(activation,dA_prev,cache):
linear_cache,activation_cache=cache
if activation=='sigmoid':

dZ=sigmoid_backward(dA_prev,activation_cache)
dA_prev,dW,db=linear_backward(dZ,linear_cache)
if activation=='relu':
dZ=relu_backward(dA_prev,activation_cache)
dA_prev,dW,db=linear_backward(dZ,linear_cache)
return dA_prev,dW,db

def L_model_backward(AL,Y,caches):
L=len(caches)
m = AL.shape[1]
Y = Y.reshape(AL.shape)
dAL = - (np.divide(Y, AL) - np.divide(1 - Y, 1 - AL))

current_cache=caches[-1]

for l in reversed(range(L-1)):
current_cache=caches[l]
grads["dW" + str(l + 1)] = dW_temp
grads["db" + str(l + 1)] = db_temp
L=len(parameters)//2
for l in range(L):
return parameters

def L_layer_model(X,Y,learning_rate,num_iter,layer_dims):
parameters=initialize_parameters_deep(layer_dims)
costs=[]
for i in range(num_iter):
AL,caches=L_model_forward(X,parameters)
cost=compute_cost(AL,Y)
if i%100==0:
print(cost)
costs.append(cost)
plt.plot(np.squeeze(costs))
def predict(X,parameters):
AL,caches=L_model_forward(X,parameters)
prediction=(AL>0.5)
return AL,prediction

L_layer_model(x_train,y_train,0.0075,12000,[12288,20,7,5,1])
prediction=predict(x_train,initialize_parameters_deep([12288,20,7,5,1]))

• Typical. Fell for the same thing when I made my first one. Commented Apr 19, 2018 at 12:13
• Uh... Which language are you using? Reminds me, we need to make tags for those. Commented Apr 19, 2018 at 12:17
• Python...Can you make suggestions please?Is there anything I need to change? Commented Apr 19, 2018 at 12:31
• Yes. Why are you starting from scratch? You should rethink about making your own AI library... Very hard. Commented Apr 19, 2018 at 12:33
• Just want to understand how exactly it works.. Commented Apr 19, 2018 at 12:34

There is a technique called Gradient checking.

With it, you can assert if you are calculating the correct gradient in the components of your ANN. The code implementation is:

def gradient_check_n(parameters, gradients, X, Y, epsilon = 1e-7):

parameters_values, _ = dictionary_to_vector(parameters)
num_parameters = parameters_values.shape[0]
J_plus = np.zeros((num_parameters, 1))
J_minus = np.zeros((num_parameters, 1))

for i in range(num_parameters):

thetaplus = np.copy(parameters_values)
thetaplus[i][0] = thetaplus[i][0]+  epsilon
J_plus[i], _ = forward_propagation_n(X, Y, vector_to_dictionary( thetaplus  ))

thetaminus =  np.copy(parameters_values)
thetaminus[i][0] = thetaplus[i][0]-  epsilon
J_minus[i], _ = forward_propagation_n(X, Y, vector_to_dictionary( thetaminus  ))