# Q Learning Algorithm not converging

I am trying to run Deep Q-learning algorithm on a game which i made in python using pygame library. The algorithm accepts the game screen (4 frames) as input to neural network which used as the function approximator.

The game looks like this ... Player can move both the paddles and randomly a white ball is generated from the center of screen. If the paddle touches the white ball reward of +1 is awarded if it misses the reward of -1 is awarded and the same reward is passed in the Q-learning algorithm to learn. Only 3 actions are possible move to left, move to right and stay

Here is the code for my Deep Q learning algorithm...

from __future__ import division, print_function
from keras.models import Sequential
from keras.layers.core import *
from keras.layers.convolutional import Conv2D
from scipy.misc import imresize
import collections
import numpy as np

import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
import wrapped

# stack the four frames together
def preprocess_images(image):

if image.shape < 4:
xt_list = []
for i in range(image.shape):
x_t = imresize(image[i],(100,100))
x_t = x_t.astype('float')
x_t /= 255.0
xt_list.append(x_t)
num = 4 - len(xt_list)
length = len(xt_list)
for x in range(num):
x_t = xt_list[length-1]
xt_list.append(x_t)
s_t = np.stack((xt_list,xt_list,xt_list,xt_list),axis=2)

else:
xt_list = []
for i in range(image.shape):
x_t = imresize(image[i],(100,100))
x_t = x_t.astype('float')
x_t /= 255.0
xt_list.append(x_t)
s_t = np.stack((xt_list,xt_list,xt_list,xt_list),axis=2)
s_t = np.expand_dims(s_t,axis=0)
return s_t

# generate data to train neural network
def gen_next_batch(experience,model,num_actions,gamma,batch_size):

batch_indices = np.random.randint(low=0,high=len(experience),size=batch_size)
batch = [experience[i] for i in batch_indices]
X = np.zeros((batch_size,100,100,4))
Y = np.zeros((batch_size,num_actions))
for i in range(len(batch)):
s_t, a_t, r_t, s_t1, game_over = batch[i]
X[i] = s_t
Y[i] = model.predict(s_t)
Q_sa = np.max(model.predict(s_t1))
if game_over:
Y[i,a_t] = r_t
else:
Y[i,a_t] = r_t + gamma*Q_sa
return X,Y

# Neural Network model implemented using keras
model = Sequential()
model.compile(loss='mse',optimizer=opt)

NUM_ACTIONS = 3
GAMMA = 0.99
INITIAL_EPSILON = 0.1
FINAL_EPSILON = 0.0001
NUM_EPOCHS = 10000
MEMORY_SIZE = 50000
BATCH_SIZE = 64
EPSILON = INITIAL_EPSILON

experience = collections.deque(maxlen=MEMORY_SIZE)

for x in range(1000):
game.reset()
game_over = False

a_0 = 2
x_t, r_0, game_over = game.step(a_0)
s_t = preprocess_images(x_t)

while  not game_over:
s_t1 = s_t
a_t = np.random.randint(low=0,high=NUM_ACTIONS,size=1)

x_t, r_t, game_over = game.step(a_t)
s_t = preprocess_images(x_t)
experience.append((s_t1,a_t,r_t,s_t1,game_over))
print('Random Actions')
print(len(experience))

Reward_list = []
for i in range(NUM_EPOCHS):
game.reset()
loss = 0.0
R = 0

a_0 = 2
x_t, r_0, game_over = game.step(a_0)
s_t = preprocess_images(x_t)

while not game_over:

s_t1 = s_t

if np.random.rand() <= EPSILON:
a_t = np.random.randint(low=0,high=NUM_ACTIONS,size=1)
else:
q = model.predict(s_t)
a_t = np.argmax(q)

x_t, r_t, game_over = game.step(a_t)
s_t = preprocess_images(x_t)

experience.append((s_t1,a_t,r_t,s_t,game_over)) # stores experiences

X,Y = gen_next_batch(experience,model,NUM_ACTIONS,GAMMA,BATCH_SIZE)
loss += model.train_on_batch(X,Y)

R += r_t

if EPSILON > FINAL_EPSILON:
EPSILON -= (INITIAL_EPSILON - FINAL_EPSILON)/NUM_EPOCHS

print('Episode : %d | Epsilon: %f | Reward : %f | Loss : %f'%(i+1,EPSILON,R,loss))
Reward_list.append(R)

if (i+1) % 1000 == 0:
plt.plot(Reward_list)
plt.xlabel('Episodes')
plt.ylabel('Reward')
plt.savefig('/output/reward.png')
model.save("/output/RL_MODEL.h5",overwrite=True)


I Trained the Neural Network for 10000 Epochs and the Total Reward per episode looks like this, which clearly indicates the algorithm is not learning...(Max of +1 and Min of -1 reward is possible in a episode) Can any one suggest me what i am doing wrong. I am having very hard time in implementing the reinforcement learning algorithms. I have tried to implement same algorithm on an another game but had the same issue. Is it related to my epsilon-greedy policy, or i am not training enough or something else. Please Help me...

In the gen_next_batch method, you return X, Y while inside the for-loop. This results in the entire array of X and Y filled with zeros, except for the first one (index 0); thus, considerably slowing down your training.