# Q Learning Algorithm not converging

I am trying to run Deep Q-learning algorithm on a game which I made in Python using pygame library. The algorithm accepts the game screen (4 frames) as input to neural network which used as the function approximator.

The game looks like this: Player can move both the paddles and randomly a white ball is generated from the center of screen. If the paddle touches the white ball, reward of +1 is awarded if it misses the reward of -1 is awarded and the same reward is passed in the Q-learning algorithm to learn. Only 3 actions are possible move to left, move to right and stay.

Here is the code for my Deep Q learning algorithm:

from __future__ import division, print_function
from keras.models import Sequential
from keras.layers.core import *
from keras.layers.convolutional import Conv2D
from scipy.misc import imresize
import collections
import numpy as np

import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
import wrapped

# stack the four frames together
def preprocess_images(image):

if image.shape < 4:
xt_list = []
for i in range(image.shape):
x_t = imresize(image[i],(100,100))
x_t = x_t.astype('float')
x_t /= 255.0
xt_list.append(x_t)
num = 4 - len(xt_list)
length = len(xt_list)
for x in range(num):
x_t = xt_list[length-1]
xt_list.append(x_t)
s_t = np.stack((xt_list,xt_list,xt_list,xt_list),axis=2)

else:
xt_list = []
for i in range(image.shape):
x_t = imresize(image[i],(100,100))
x_t = x_t.astype('float')
x_t /= 255.0
xt_list.append(x_t)
s_t = np.stack((xt_list,xt_list,xt_list,xt_list),axis=2)
s_t = np.expand_dims(s_t,axis=0)
return s_t

# generate data to train neural network
def gen_next_batch(experience,model,num_actions,gamma,batch_size):

batch_indices = np.random.randint(low=0,high=len(experience),size=batch_size)
batch = [experience[i] for i in batch_indices]
X = np.zeros((batch_size,100,100,4))
Y = np.zeros((batch_size,num_actions))
for i in range(len(batch)):
s_t, a_t, r_t, s_t1, game_over = batch[i]
X[i] = s_t
Y[i] = model.predict(s_t)
Q_sa = np.max(model.predict(s_t1))
if game_over:
Y[i,a_t] = r_t
else:
Y[i,a_t] = r_t + gamma*Q_sa
return X,Y

# Neural Network model implemented using keras
model = Sequential()
model.compile(loss='mse',optimizer=opt)

NUM_ACTIONS = 3
GAMMA = 0.99
INITIAL_EPSILON = 0.1
FINAL_EPSILON = 0.0001
NUM_EPOCHS = 10000
MEMORY_SIZE = 50000
BATCH_SIZE = 64
EPSILON = INITIAL_EPSILON

experience = collections.deque(maxlen=MEMORY_SIZE)

for x in range(1000):
game.reset()
game_over = False

a_0 = 2
x_t, r_0, game_over = game.step(a_0)
s_t = preprocess_images(x_t)

while  not game_over:
s_t1 = s_t
a_t = np.random.randint(low=0,high=NUM_ACTIONS,size=1)

x_t, r_t, game_over = game.step(a_t)
s_t = preprocess_images(x_t)
experience.append((s_t1,a_t,r_t,s_t1,game_over))
print('Random Actions')
print(len(experience))

Reward_list = []
for i in range(NUM_EPOCHS):
game.reset()
loss = 0.0
R = 0

a_0 = 2
x_t, r_0, game_over = game.step(a_0)
s_t = preprocess_images(x_t)

while not game_over:

s_t1 = s_t

if np.random.rand() <= EPSILON:
a_t = np.random.randint(low=0,high=NUM_ACTIONS,size=1)
else:
q = model.predict(s_t)
a_t = np.argmax(q)

x_t, r_t, game_over = game.step(a_t)
s_t = preprocess_images(x_t)

experience.append((s_t1,a_t,r_t,s_t,game_over)) # stores experiences

X,Y = gen_next_batch(experience,model,NUM_ACTIONS,GAMMA,BATCH_SIZE)
loss += model.train_on_batch(X,Y)

R += r_t

if EPSILON > FINAL_EPSILON:
EPSILON -= (INITIAL_EPSILON - FINAL_EPSILON)/NUM_EPOCHS

print('Episode : %d | Epsilon: %f | Reward : %f | Loss : %f'%(i+1,EPSILON,R,loss))
Reward_list.append(R)

if (i+1) % 1000 == 0:
plt.plot(Reward_list)
plt.xlabel('Episodes')
plt.ylabel('Reward')
plt.savefig('/output/reward.png')
model.save("/output/RL_MODEL.h5",overwrite=True)


I trained the Neural Network for 10000 Epochs and the Total Reward per episode looks like this, which clearly indicates the algorithm is not learning (Max of +1 and Min of -1 reward is possible in a episode). Can anyone suggest me what I am doing wrong? I am having a very hard time in implementing the reinforcement learning algorithms. I have tried to implement same algorithm on an another game but had the same issue. Is it related to my epsilon-greedy policy, or I am not training enough or something else? Please help.

In the gen_next_batch method, you return X, Y while inside the for-loop. This results in the entire array of X and Y filled with zeros, except for the first one (index 0); thus, considerably slowing down your training.