I'm testing out TensorFlow LSTM layer text generation task, not classification task; but something is wrong with my code, it doesn't converge. What changes should be done?
Source code:
import tensorflow as tf;
# t=0 t=1 t=2 t=3
#[the, brown, fox, is, quick]
# 0 1 2 3 4
#[the, red, fox, jumps, high]
# 0 5 2 6 7
#t0 x=[[the], [the]]
# y=[[brown],[red]]
#t1 ...
#t2
#t3
bsize = 2;
times = 4;
#data
x = [];
y = [];
#t0 the: the:
x.append([[0/6], [0/6]]); #normalise: x divided by 6 (max x)
# brown: red:
y.append([[1/7], [5/7]]); #normalise: y divided by 7 (max y)
#t1
x.append([[1/6], [5/6]]);
y.append([[2/7], [2/7]]);
#t2
x.append([[2/6], [2/6]]);
y.append([[3/7], [6/7]]);
#t3
x.append([[3/6], [6/6]]);
y.append([[4/7], [7/7]]);
#model
inputs = tf.placeholder(tf.float32,[times,bsize,1]) #4,2,1
exps = tf.placeholder(tf.float32,[times,bsize,1]);
layer1 = tf.keras.layers.LSTMCell(20)
hids1,_ = tf.nn.static_rnn(layer1,tf.split(inputs,times),dtype=tf.float32);
w2 = tf.Variable(tf.random_uniform([20,1],-1,1));
b2 = tf.Variable(tf.random_uniform([ 1],-1,1));
outs = tf.sigmoid(tf.matmul(hids1,w2) + b2);
loss = tf.reduce_sum(tf.square(exps-outs))
optim = tf.train.GradientDescentOptimizer(1e-1)
train = optim.minimize(loss)
#train
s = tf.Session();
init = tf.global_variables_initializer();
s.run(init)
feed = {inputs:x, exps:y}
for i in range(10000):
if i%1000==0:
lossval = s.run(loss,feed)
print("loss:",lossval)
#end if
s.run(train,feed)
#end for
lastloss = s.run(loss,feed)
print("loss:",lastloss,"(last)");
#eof
Output showing loss values (a little different every run):
loss: 3.020703
loss: 1.8259083
loss: 1.812584
loss: 1.8101325
loss: 1.8081319
loss: 1.8070083
loss: 1.8065354
loss: 1.8063282
loss: 1.8062303
loss: 1.8061805
loss: 1.8061543 (last)
Colab link: https://colab.research.google.com/drive/1TsHjmucuynCPOgKuo4a0hiM8B8UaOWQo