LSTM network doesn't converge, what should be changed? [closed]

Question

I'm testing out TensorFlow LSTM layer text generation task, not classification task; but something is wrong with my code, it doesn't converge. What changes should be done?

Source code:

import tensorflow as tf;

# t=0  t=1    t=2  t=3     
#[the, brown, fox, is,     quick]
#   0  1      2    3       4
#[the, red,   fox, jumps,  high]
#   0  5      2    6       7

#t0 x=[[the],  [the]]
#   y=[[brown],[red]]
#t1 ...
#t2
#t3
bsize = 2;
times = 4;

#data
x = [];
y = [];
#t0        the:     the:
x.append([[0/6],   [0/6]]); #normalise: x divided by 6 (max x)
#          brown:   red:
y.append([[1/7],   [5/7]]); #normalise: y divided by 7 (max y)
#t1
x.append([[1/6],   [5/6]]);
y.append([[2/7],   [2/7]]);
#t2
x.append([[2/6],   [2/6]]);
y.append([[3/7],   [6/7]]);
#t3
x.append([[3/6],   [6/6]]);
y.append([[4/7],   [7/7]]);

#model
inputs  = tf.placeholder(tf.float32,[times,bsize,1]) #4,2,1
exps    = tf.placeholder(tf.float32,[times,bsize,1]);

layer1  = tf.keras.layers.LSTMCell(20) 
hids1,_ = tf.nn.static_rnn(layer1,tf.split(inputs,times),dtype=tf.float32);

w2      = tf.Variable(tf.random_uniform([20,1],-1,1));
b2      = tf.Variable(tf.random_uniform([   1],-1,1));
outs    = tf.sigmoid(tf.matmul(hids1,w2) + b2);

loss  = tf.reduce_sum(tf.square(exps-outs))
optim = tf.train.GradientDescentOptimizer(1e-1)
train = optim.minimize(loss)

#train
s    = tf.Session();
init = tf.global_variables_initializer();
s.run(init)

feed = {inputs:x, exps:y}
for i in range(10000):
  if i%1000==0:
    lossval = s.run(loss,feed)
    print("loss:",lossval)
  #end if
  s.run(train,feed)
#end for

lastloss = s.run(loss,feed)
print("loss:",lastloss,"(last)");
#eof

Output showing loss values (a little different every run):

loss: 3.020703
loss: 1.8259083
loss: 1.812584
loss: 1.8101325
loss: 1.8081319
loss: 1.8070083
loss: 1.8065354
loss: 1.8063282
loss: 1.8062303
loss: 1.8061805
loss: 1.8061543 (last)

Colab link: https://colab.research.google.com/drive/1TsHjmucuynCPOgKuo4a0hiM8B8UaOWQo

Answer 1

writing here my suggestion, because i haven't earned the right to comment yet.

Your main "problem" could be your loss function. It converges, this is why your loss value is decreasing. So I suggest to let it maybe train longer.

Alternatively you could change the loss function to fit your need. For example you could use:

loss  = tf.reduce_mean(tf.square(exps-outs))

You will get a smaller loss value which decreases clearly after every output.

I hope this helps :)

Answer 2

I'm still working on how to make the code work for text generation, but the following converges and work for text classification:

import tensorflow as tf;
tf.reset_default_graph();

#data
'''
t0      t1      t2
british gray    is => cat (y=0)
0       1       2
white   samoyed is => dog (y=1)
3       4       2 
'''
Bsize = 2;
Times = 3;
Max_X = 4;
Max_Y = 1;

X = [[[0],[1],[2]], [[3],[4],[2]]];
Y = [[0],           [1]          ];

#normalise
for I in range(len(X)):
  for J in range(len(X[I])):
    X[I][J][0] /= Max_X;

for I in range(len(Y)):
  Y[I][0] /= Max_Y;

#model
Inputs   = tf.placeholder(tf.float32, [Bsize,Times,1]);
Expected = tf.placeholder(tf.float32, [Bsize,      1]);

#single LSTM layer
#'''
Layer1   = tf.keras.layers.LSTM(20);
Hidden1  = Layer1(Inputs);
#'''

#multi LSTM layers
'''
Layers = tf.keras.layers.RNN([
  tf.keras.layers.LSTMCell(30), #hidden 1
  tf.keras.layers.LSTMCell(20)  #hidden 2
]);
Hidden2 = Layers(Inputs);
'''

Weight3  = tf.Variable(tf.random_uniform([20,1], -1,1));
Bias3    = tf.Variable(tf.random_uniform([   1], -1,1));
Output   = tf.sigmoid(tf.matmul(Hidden1,Weight3) + Bias3);

Loss     = tf.reduce_sum(tf.square(Expected-Output));
Optim    = tf.train.GradientDescentOptimizer(1e-1);
Training = Optim.minimize(Loss);

#train
Sess = tf.Session();
Init = tf.global_variables_initializer();
Sess.run(Init);

Feed = {Inputs:X, Expected:Y};
for I in range(1000): #number of feeds, 1 feed = 1 batch
  if I%100==0: 
    Lossvalue = Sess.run(Loss,Feed);
    print("Loss:",Lossvalue);
  #end if

  Sess.run(Training,Feed);
#end for

Lastloss = Sess.run(Loss,Feed);
print("Loss:",Lastloss,"(Last)");

#eval
Results = Sess.run(Output,Feed);
print("\nEval:");
print(Results);

print("\nDone.");
#eof