2
$\begingroup$

I am currently learning neural networks. Using data from http://www.mariofrank.net/touchalytics/index.html, I am trying to predict "User ID" by training the neural network model shown below. However, the model loss is increasing after every iteration. What am I doing wrong? Loss graph

import time
import os
BATCH_SIZE=32
embedding_size=256
sequence_length=200
BUFFER_SIZE=10000
input_size=41
learning_rate=0.001

inputs_as_tensors=tf.data.Dataset.from_tensor_slices(train_data_features_array)
targets_as_tensors=tf.data.Dataset.from_tensor_slices(train_data_labels_categorical_array)
training_data=tf.data.Dataset.zip((inputs_as_tensors,targets_as_tensors))
#training_data=training_data.batch(sequence_length,drop_remainder=True)
training_dataset=training_data.shuffle(BUFFER_SIZE).batch(BATCH_SIZE, drop_remainder=True)
print(training_dataset)

def build_model(vocab_size,batch_size):
    modelf=tf.keras.Sequential([
        tf.keras.layers.Dense(10,activation="sigmoid",input_shape=(None,10)),

                                tf.keras.layers.Dense(30,activation="relu",use_bias=True),  

                                     tf.keras.layers.Dropout(0.2),                    
                                   tf.keras.layers.Dense(vocab_size)


                                ])
    return modelf

def training_step(inputs,targets,optimizer):
    with tf.GradientTape() as tape:
        predictions=model(inputs)
        loss=tf.reduce_mean(tf.keras.losses.categorical_crossentropy(targets,predictions,from_logits=True))

        grads=tape.gradient(loss,model.trainable_variables)
        optimizer.apply_gradients(zip(grads,model.trainable_variables))
        return loss,predictions

model=build_model(input_size,BATCH_SIZE)
i=0
inner_loop=0
checkpoint_dir ='Moses_Model_x'
checkpoint_prefix = os.path.join(checkpoint_dir, "ckpt_{i}")

while(1):
    start = time.time()
    for x,y in training_dataset:
        loss,predictions=training_step(x,y,tf.keras.optimizers.RMSprop(learning_rate=0.002))
    print ('Epoch {} Loss {:.4f}'.format(i, loss))
    print ('Time taken for iteration {} is {} sec\n'.format(i,time.time() - start))
    model.save_weights(checkpoint_prefix.format(i=i))
    i=i+1
$\endgroup$
  • 1
    $\begingroup$ Can you provide a plot? It’s normal for loss to fluctuate from batch to batch. How does your loss behave from epoch to epoch? $\endgroup$ – SpiderRico Mar 22 at 20:11
  • $\begingroup$ @SpiderRico please check the updates $\endgroup$ – onexpeters Mar 23 at 9:17
  • 1
    $\begingroup$ Can you please describe exactly what your neural network is supposed to do? Which task are you trying to solve? What does the dataset contain? Which loss are you using? Some of these questions may be answered by looking at the code, but, ideally, on this site, we should be able to answer your question without looking at the code, so provide these details, please! $\endgroup$ – nbro Mar 23 at 22:51

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service, privacy policy and cookie policy

Browse other questions tagged or ask your own question.