1
$\begingroup$

I have made sure that layers,parameters, hyperparameters,kernel_initialization, bias_initialization, seed and dataset are all equal. But still the output for both the models are different.

from __future__ import division, print_function, absolute_import
import tensorflow.keras.datasets.mnist as mnist
import tensorflow as tf
tf.reset_default_graph()
tf.random.set_random_seed(43)
from input_data import Dataset
# Import MNIST data
import input_data
# mnist = input_data_old.read_data_sets("MNSITData/raw/", one_hot=True)

# Seed value
# Apparently you may use different seed values at each stage
seed_value= 43

# 1. Set the `PYTHONHASHSEED` environment variable at a fixed value
import os
os.environ['PYTHONHASHSEED']=str(seed_value)

# 2. Set the `python` built-in pseudo-random generator at a fixed value
import random
random.seed(seed_value)

# 3. Set the `numpy` pseudo-random generator at a fixed value
import numpy as np
np.random.seed(seed_value)

# 4. Set the `tensorflow` pseudo-random generator at a fixed value
tf.set_random_seed(seed_value)
# (x_train, y_train), (x_test, y_test) = mnist.load_data()
# x_train, x_test = x_train / 255, x_test / 255

# # Add a channels dimension
# x_train = x_train[..., None]
# x_test = x_test[..., None]
# train_ds = tf.data.Dataset.from_tensor_slices(
#     (x_train, y_train)).batch(64)
# test_ds = tf.data.Dataset.from_tensor_slices((x_test, y_test)).batch(64)
# Training Parameters
learning_rate = 0.001
batch = 5
display_step = 10

dataset={
    "file_path":"/home/pat-011/Desktop/deep_learning/playground/flower_photos",
    "ratio":(0.8,0.1,0.1),
    "target_size":(150,150),
    "color_mode":"rgb",
    "class_mode":'sparse'
}
train,test,val_ds= Dataset(dataset)()
np.save("train",train[0])
np.save("test",test[0])

# Network Parameters
num_classes = 5# MNIST total classes (0-9 digits)
dropout = 0.25 # Dropout, probability to keep units

# tf Graph input
X = tf.placeholder(tf.float32, [None, 150,150,3])
Y = tf.placeholder(tf.float32, [None, num_classes])
keep_prob = tf.placeholder(tf.float32) # dropout (keep probability)


# Create some wrappers for simplicity
def conv2d(x, W, b, strides=1):
    print("shape of x",x.shape.as_list(),type(x.shape.as_list()),x.shape.as_list()[0],x.shape.as_list()[1])
    # Conv2D wrapper, with bias and relu activation
    tf.compat.v1.summary.scalar('conv_weights', W )
    tf.compat.v1.summary.scalar('conv_bias', b)
    x = tf.nn.conv2d(x, W, strides=[1, strides, strides, 1], padding='VALID')
    x = tf.nn.bias_add(x, b)
    return tf.nn.relu(x)

def maxpool2d(x, k=2):
    return tf.nn.max_pool(x, ksize=[1, k, k, 1], strides=[1, k, k, 1],
                          padding='VALID')


# Create model
def conv_net(x, weights, biases):
    # MNIST data input is a 1-D vector of 784 features (128*128 pixels)
    # Reshape to match picture format [Height x Width x Channel]
    # Tensor input become 4-D: [Batch Size, Height, Width, Channel]
    print(type(x.shape[-1]),x.shape[-1],type(int(x.shape[-1])))
    x = tf.reshape(x, shape=[-1, 150, 150, 3])
    # Convolution Layerint()
    conv1 = conv2d(x, weights['wc1'], biases['bc1'])
    tf.compat.v1.summary.scalar(conv1.name, conv1)
    # Max Pooling (down-sampling)
    print(conv1.shape)
    conv1 = maxpool2d(conv1, k=2)
    print(conv1.shape)
    # Convolution Layer
    conv2 = conv2d(conv1, weights['wc2'], biases['bc2'])
    # Max Pooling (down-sampling)
    print(conv2.shape)
    conv2 = maxpool2d(conv2, k=2)
    print(conv2.shape)

    # Fully connected layer
    # Reshape conv2 output to fit fully connected layer input
    print("weights_____________________________",[-1, weights['wd1'].get_shape().as_list()[0]])

    fc1 = tf.reshape(conv2, [-1, weights['wd1'].get_shape().as_list()[0]])
    print(fc1.shape,)
    fc1 = tf.add(tf.matmul(fc1, weights['wd1']), biases['bd1'])
    fc1 = tf.nn.relu(fc1)
    # Apply Dropout
    print("dropout++++++++++++++++++++++++++++++++++",dropout)
    fc1 = tf.nn.dropout(fc1, rate = dropout,seed=43)
    print(fc1.shape)

    fc1 = tf.reshape(fc1, [-1, weights['out'].get_shape().as_list()[0]])
    out = tf.add(tf.matmul(fc1, weights['out']), biases['out'])
    tf.compat.v1.summary.scalar('dense_weights', weights['out'] )
    tf.compat.v1.summary.scalar('dense_bias', biases['out'])
    print(out.shape)
    tf.compat.v1.summary.scalar(out.name, out)
    return out

# Store layers weight & bias
weights = {
    # 5x5 conv, 1 input, 32 outputs
    'wc1': tf.Variable(tf.random_normal([5, 5, 3, 32],mean=0.0,
    stddev=1.0, seed=43)),
    # 'wc1': tf.Variable(np.load("conv_w.npy")),
    'wc2': tf.Variable(tf.random_normal([5, 5, 32, 64],mean=0.0,
    stddev=1.0, seed=43)),
    # fully connected, 7*7*64 inputs, 1024 outputs
    'wd1': tf.Variable(tf.random_normal([34*34*64, 1024],mean=0.0,
    stddev=1.0, seed=43)),
    # 1024 inputs, 10 outputs (class prediction)
    # 'out': tf.Variable(np.load("dense_w.npy"))
    'out': tf.Variable(tf.random_normal([1024, num_classes],mean=0.0,
    stddev=1.0, seed=43))
}

biases = {
    'bc1': tf.Variable(tf.random_normal([32],mean=0.0,
    stddev=1.0, seed=43)),
    # 'bc1': tf.Variable(np.load("conv_b.npy")),
    'bc2': tf.Variable(tf.random_normal([64],mean=0.0,
    stddev=1.0, seed=43)),
    'bd1': tf.Variable(tf.random_normal([1024],mean=0.0,
    stddev=1.0, seed=43)),
    'out': tf.Variable(tf.random_normal([num_classes], mean=0.0,
    stddev=1.0, seed=43))
    # 'out': tf.Variable(np.load("dense_b.npy"))
}
print(weights,biases)
# Construct model
logits = conv_net(X, weights, biases)
prediction = tf.nn.softmax(logits)

# Define loss and optimizer
loss_op = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(
    logits=logits, labels=Y))
optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate)
train_op = optimizer.minimize(loss_op)
tf.compat.v1.summary.scalar('loss_op', loss_op)

# Evaluate model
correct_pred = tf.equal(tf.argmax(prediction, 1), tf.argmax(Y, 1))
accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))
tf.compat.v1.summary.scalar('accuracy', accuracy)

# Merge all the summaries and write them out to
# /tmp/tensorflow/mnist/logs/mnist_with_summaries (by default)
merged = tf.compat.v1.summary.merge_all()
# Initialize thcorrect_prede variables (i.e. assign their default value)
init = tf.global_variables_initializer()

# Start training
with tf.Session() as sess:
    writer = tf.summary.FileWriter("/tmp/tf_withoutjson/",sess.graph)
    # Run the initializer
    sess.run(init)
    for i in range(5):
        for step in range(1, int(len(train[0])/batch)):
            # Run optimization op (backprop)
            offset = (step * batch) % (train[1].shape[0] - batch)
            # Generate a minibatch.
            batch_x = train[0][offset:(offset + batch), :]
            batch_y = train[1][offset:(offset + batch), :]
            # np.save("arrays/xtrain"+str(step),batch_x)
            # np.save("arrays/ytrain"+str(step),batch_y)
            sess.run(train_op, feed_dict={X: batch_x, Y: batch_y})
            if step % display_step == 0 or step == 1:
                # Calculate batch loss and accuracy

                loss, acc ,cw,cb,dw,db= sess.run([loss_op, accuracy,weights["wc1"],biases["bc1"],weights["out"],biases['out']], feed_dict={X: batch_x,
                                                                    Y: batch_y
                                                                    })
                np.save("weightBias/conv_w"+str(step),cw)
                np.save("weightBias/conv_b"+str(step),cb)
                np.save("weightBias/dense_w"+str(step),dw)
                np.save("weightBias/dense_b"+str(step),db)
                print("Step " + str(step) + ", Minibatch Loss= " + \
                    "{:.4f}".format(loss) + ", Training Accuracy= " + \
                    "{:.3f}".format(acc))

    print("Optimization Finished!")

    # Calculate accuracy for 256 MNIST test images
    print("Testing Accuracy:", \
        sess.run(accuracy, feed_dict={X: test[0],
                                      Y: test[1],
                                      keep_prob: 0.75}))

This is the tensorflow flow and I have also saved the weights generated in each layer.

Now for the keras flow,

import keras
from keras.layers import *
from keras.models import Model
from keras import optimizers
from keras.datasets import mnist
from keras import backend as K
from keras.utils import multi_gpu_model
import tensorflow as tf
from input_data import Dataset
from numpy import sqrt
from keras import initializers

# Seed value
# Apparently you may use different seed values at each stage
seed_value= 43

# 1. Set the `PYTHONHASHSEED` environment variable at a fixed value
import os
os.environ['PYTHONHASHSEED']=str(seed_value)

# 2. Set the `python` built-in pseudo-random generator at a fixed value
import random
random.seed(seed_value)

# 3. Set the `numpy` pseudo-random generator at a fixed value
import numpy as np
np.random.seed(seed_value)

# 4. Set the `tensorflow` pseudo-random generator at a fixed value
import tensorflow as tf
tf.set_random_seed(seed_value)

# 5. Configure a new global `tensorflow` session
from keras import backend as K
session_conf = tf.ConfigProto(intra_op_parallelism_threads=1, inter_op_parallelism_threads=1)
sess = tf.Session(graph=tf.get_default_graph(), config=session_conf)
K.set_session(sess)


img_rows, img_cols = 150, 150

dataset={
    "file_path":"/home/pat-011/Desktop/deep_learning/playground/flower_photos",
    "ratio":(0.8,0.1,0.1),
    "target_size":(150,150),
    "color_mode":"rgb",
    "class_mode":'sparse'
}

(x_train, y_train), (x_test, y_test), (x_val, y_val) = Dataset(dataset)()

in_shape = x_train.shape[1]
out_shape = y_train.shape[1]
n_train = x_train.shape[0]
n_test = x_test.shape[0]


# def get_network_utils():
#     dropout_ph = tf.placeholder_with_default(0.25, shape=())
#     optimizer = tf.train.AdamOptimizer(learning_rate=lr, name='Trainer')
#     activation = tf.nn.relu

#     def loss(y_true, y_pred):
#         return tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=y_true, logits=y_pred))

#     def accuracy(y_true, y_pred):
#         correct = tf.equal(tf.argmax(y_true, 1), tf.argmax(y_pred, 1))
#         return tf.reduce_mean(tf.cast(correct, tf.float32))

#     def weight_initializer(shape, dtype=None, partition_info=None):
#         init_range = sqrt(6.0 / (shape[0] + shape[1]))
#         return tf.get_variable('weights', shape=shape, dtype=dtype,
#                                initializer=tf.random_uniform_initializer(-init_range, init_range))

#     def bias_initializer(shape, dtype=None, partition_info=None):
#         return tf.Variable(name='bias', initial_value=tf.random_normal(shape))

#     return (dropout_ph, optimizer, activation, loss, accuracy,
#             weight_initializer, bias_initializer)

x_train = x_train.astype('float32')
x_test = x_test.astype('float32')
x_train /= 255
x_test /= 255
print('x_train shape:', x_train.shape)
print(x_train.shape[0], 'train samples')
print(x_test.shape[0], 'test samples')

lr = 0.001

# _, optimizer, activation, loss, accuracy, weight_initializer, bias_initializer = get_network_utils()

# # convert class vectors to binary class matrices
# y_train = keras.utils.to_categorical(y_train, num_classes)
# y_test = keras.utils.to_categorical(y_test, num_classes)

inputs = Input(shape=(150, 150, 3))
x = Conv2D(32, (5, 5), activation='relu', strides = (1, 1), padding = 'VALID', kernel_initializer=initializers.RandomNormal(mean=0.0,
    stddev=1.0,seed = 43), bias_initializer=initializers.RandomNormal(mean=0.0,
    stddev=1.0, seed = 43))(inputs)
x = MaxPool2D(pool_size=2, strides = (2, 2), padding = 'VALID')(x)
x = Conv2D(64, (5, 5),activation='relu', strides = (1, 1), padding = 'VALID',kernel_initializer=initializers.RandomNormal(mean=0.0,
    stddev=1.0, seed = 43), bias_initializer=initializers.RandomNormal(mean=0.0,
    stddev=1.0, seed = 43))(x)
x = MaxPool2D(pool_size=2, strides = (2, 2), padding = 'VALID')(x)
x = Flatten()(x)
x = Dense(1024, activation='relu', kernel_initializer=initializers.RandomNormal(mean=0.0,
    stddev=1.0, seed = 43), bias_initializer=initializers.RandomNormal(mean=0.0,
    stddev=1.0, seed = 43))(x)
x = Dropout(rate = 0.25, seed = 43)(x)
predictions = Dense(5, activation='softmax', kernel_initializer=initializers.RandomNormal(mean=0.0,
    stddev=1.0, seed = 43), bias_initializer=initializers.RandomNormal(mean=0.0,
    stddev=1.0, seed = 43))(x)
model = Model(inputs=inputs, outputs=predictions, name='mnist_model')

model.compile(loss='categorical_crossentropy',
            optimizer=optimizers.Adam(),
            metrics=['accuracy'])

history = model.fit(x_train, y_train,
                    batch_size=5,
                    epochs=10)
model.summary()

test_scores = model.evaluate(x_test, y_test, verbose=0)
print('Test loss:', test_scores[0])
print('Test accuracy:', test_scores[1])

weights = []
for layer in model.layers:
    weights = layer.get_weights()
    np.save("../weightBias/"+layer.name,weights)

Each layer weights are different though initial weights are initialized with the same random distribution.

Please someone point out why I am getting different output for the same model

Tensorflow accuracy-

Step 1, Minibatch Loss= 68458.3750, Training Accuracy= 0.800
Step 10, Minibatch Loss= 451470.5625, Training Accuracy= 0.200
Step 20, Minibatch Loss= 582654.3750, Training Accuracy= 0.200
Step 30, Minibatch Loss= 185989.4219, Training Accuracy= 0.400
Step 1, Minibatch Loss= 161536.2188, Training Accuracy= 0.600
Step 10, Minibatch Loss= 286743.1875, Training Accuracy= 0.400
Step 20, Minibatch Loss= 206555.5000, Training Accuracy= 0.600
Step 30, Minibatch Loss= 201791.6250, Training Accuracy= 0.800
Step 1, Minibatch Loss= 212732.3438, Training Accuracy= 0.600
Step 10, Minibatch Loss= 108325.5469, Training Accuracy= 0.800
Step 20, Minibatch Loss= 80389.1641, Training Accuracy= 0.800
Step 30, Minibatch Loss= 34269.3750, Training Accuracy= 0.800
Step 1, Minibatch Loss= 134766.6406, Training Accuracy= 0.400
Step 10, Minibatch Loss= 166675.9531, Training Accuracy= 0.400
Step 20, Minibatch Loss= 66996.0938, Training Accuracy= 0.600
Step 30, Minibatch Loss= 29773.1660, Training Accuracy= 0.800
Step 1, Minibatch Loss= 33207.5234, Training Accuracy= 0.600
Step 10, Minibatch Loss= 154839.6250, Training Accuracy= 0.800
Step 20, Minibatch Loss= 23777.3125, Training Accuracy= 0.800
Step 30, Minibatch Loss= 57576.0742, Training Accuracy= 0.400
Optimization Finished!
Testing Accuracy: 0.2

Keras Accuracy -

156/156 [==============================] - 6s 41ms/step - loss: 13.0185 - acc: 0.1923 
Epoch 2/10
156/156 [==============================] - 3s 18ms/step - loss: 12.9151 - acc: 0.1987
Epoch 3/10
156/156 [==============================] - 3s 18ms/step - loss: 13.1218 - acc: 0.1859
Epoch 4/10
156/156 [==============================] - 3s 18ms/step - loss: 12.9151 - acc: 0.1987
Epoch 5/10
156/156 [==============================] - 3s 19ms/step - loss: 13.1218 - acc: 0.1859
Epoch 6/10
156/156 [==============================] - 3s 19ms/step - loss: 12.9151 - acc: 0.1987
Epoch 7/10
156/156 [==============================] - 3s 19ms/step - loss: 12.8118 - acc: 0.2051
Epoch 8/10
156/156 [==============================] - 3s 19ms/step - loss: 12.8118 - acc: 0.2051
Epoch 9/10
156/156 [==============================] - 3s 19ms/step - loss: 12.8118 - acc: 0.2051
Epoch 10/10
156/156 [==============================] - 3s 19ms/step - loss: 12.9151 - acc: 0.1987
$\endgroup$

closed as off-topic by nbro, DukeZhou Sep 23 at 19:11

This question appears to be off-topic. The users who voted to close gave this specific reason:

  • "This question does not appear to be about artificial intelligence, within the scope defined in the help center." – nbro, DukeZhou
If this question can be reworded to fit the rules in the help center, please edit the question.

  • $\begingroup$ Apologies. This forum focuses on theory, as opposed to specific troubleshooting. This question is likely a better fit for Data Science or Overflow. $\endgroup$ – DukeZhou Sep 23 at 19:11
0
$\begingroup$

I'm writing here because I can't comment yet. Just for clarification. Does your code run on the same machine and/or with cpu or a gpu? For example this article suggest that even more randomness is added to models, if you use a gpu. This might yield different results of models.

Also you might want to make sure, that keras is really using the tensorflow backend on the same version, as suggested here.

Hope to hear from you :)

Edit:

I just saw the "multi-gpu" import...Can you try to disable the gpu and try again on cpu only.

$\endgroup$
  • $\begingroup$ yes i have got identical results after running my model in cpu and using Adagrad optimizer and glorot uniform initializer $\endgroup$ – Subham Tiwari Sep 25 at 10:17

Not the answer you're looking for? Browse other questions tagged or ask your own question.