I am trying to run Deep Q-learning algorithm on a game which I made in Python using pygame library. The algorithm accepts the game screen (4 frames) as input to neural network which used as the function approximator.
The game looks like this:
Player can move both the paddles and randomly a white ball is generated from the center of screen. If the paddle touches the white ball, reward of +1 is awarded if it misses the reward of -1 is awarded and the same reward is passed in the Q-learning algorithm to learn. Only 3 actions are possible move to left, move to right and stay.
Here is the code for my Deep Q learning algorithm:
from __future__ import division, print_function
from keras.models import Sequential
from keras.layers.core import *
from keras.layers.convolutional import Conv2D
from keras.optimizers import Adam
from scipy.misc import imresize
import collections
import numpy as np
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
import wrapped
# stack the four frames together
def preprocess_images(image):
if image.shape[0] < 4:
xt_list = []
for i in range(image.shape[0]):
x_t = imresize(image[i],(100,100))
x_t = x_t.astype('float')
x_t /= 255.0
xt_list.append(x_t)
num = 4 - len(xt_list)
length = len(xt_list)
for x in range(num):
x_t = xt_list[length-1]
xt_list.append(x_t)
s_t = np.stack((xt_list[0],xt_list[1],xt_list[2],xt_list[3]),axis=2)
else:
xt_list = []
for i in range(image.shape[0]):
x_t = imresize(image[i],(100,100))
x_t = x_t.astype('float')
x_t /= 255.0
xt_list.append(x_t)
s_t = np.stack((xt_list[0],xt_list[1],xt_list[2],xt_list[3]),axis=2)
s_t = np.expand_dims(s_t,axis=0)
return s_t
# generate data to train neural network
def gen_next_batch(experience,model,num_actions,gamma,batch_size):
batch_indices = np.random.randint(low=0,high=len(experience),size=batch_size)
batch = [experience[i] for i in batch_indices]
X = np.zeros((batch_size,100,100,4))
Y = np.zeros((batch_size,num_actions))
for i in range(len(batch)):
s_t, a_t, r_t, s_t1, game_over = batch[i]
X[i] = s_t
Y[i] = model.predict(s_t)[0]
Q_sa = np.max(model.predict(s_t1)[0])
if game_over:
Y[i,a_t] = r_t
else:
Y[i,a_t] = r_t + gamma*Q_sa
return X,Y
# Neural Network model implemented using keras
model = Sequential()
model.add(Conv2D(32,kernel_size=8,strides=4,kernel_initializer='normal',padding="same",input_shape=(100,100,4)))
model.add(Activation('relu'))
model.add(Conv2D(64,kernel_size=4,strides=2,kernel_initializer='normal',padding='same'))
model.add(Activation('relu'))
model.add(Conv2D(32,kernel_size=3,strides=1,kernel_initializer='normal',padding='same'))
model.add(Activation('relu'))
model.add(Flatten())
model.add(Dense(512,kernel_initializer='normal'))
model.add(Activation('relu'))
model.add(Dense(3,kernel_initializer='normal'))
model.add(Activation('linear'))
opt = Adam(lr=1e-06)
model.compile(loss='mse',optimizer=opt)
NUM_ACTIONS = 3
GAMMA = 0.99
INITIAL_EPSILON = 0.1
FINAL_EPSILON = 0.0001
NUM_EPOCHS = 10000
MEMORY_SIZE = 50000
BATCH_SIZE = 64
EPSILON = INITIAL_EPSILON
experience = collections.deque(maxlen=MEMORY_SIZE)
game = wrapped.Paddle()
for x in range(1000):
game.reset()
game_over = False
a_0 = 2
x_t, r_0, game_over = game.step(a_0)
s_t = preprocess_images(x_t)
while not game_over:
s_t1 = s_t
a_t = np.random.randint(low=0,high=NUM_ACTIONS,size=1)[0]
x_t, r_t, game_over = game.step(a_t)
s_t = preprocess_images(x_t)
experience.append((s_t1,a_t,r_t,s_t1,game_over))
print('Random Actions')
print(len(experience))
Reward_list = []
for i in range(NUM_EPOCHS):
game.reset()
loss = 0.0
R = 0
a_0 = 2
x_t, r_0, game_over = game.step(a_0)
s_t = preprocess_images(x_t)
while not game_over:
s_t1 = s_t
if np.random.rand() <= EPSILON:
a_t = np.random.randint(low=0,high=NUM_ACTIONS,size=1)[0]
else:
q = model.predict(s_t)[0]
a_t = np.argmax(q)
x_t, r_t, game_over = game.step(a_t)
s_t = preprocess_images(x_t)
experience.append((s_t1,a_t,r_t,s_t,game_over)) # stores experiences
X,Y = gen_next_batch(experience,model,NUM_ACTIONS,GAMMA,BATCH_SIZE)
loss += model.train_on_batch(X,Y)
R += r_t
if EPSILON > FINAL_EPSILON:
EPSILON -= (INITIAL_EPSILON - FINAL_EPSILON)/NUM_EPOCHS
print('Episode : %d | Epsilon: %f | Reward : %f | Loss : %f'%(i+1,EPSILON,R,loss))
Reward_list.append(R)
if (i+1) % 1000 == 0:
plt.plot(Reward_list)
plt.xlabel('Episodes')
plt.ylabel('Reward')
plt.savefig('/output/reward.png')
model.save("/output/RL_MODEL.h5",overwrite=True)
I trained the Neural Network for 10000 Epochs and the Total Reward per episode looks like this, which clearly indicates the algorithm is not learning (Max of +1 and Min of -1 reward is possible in a episode).
Can anyone suggest me what I am doing wrong? I am having a very hard time in implementing the reinforcement learning algorithms. I have tried to implement same algorithm on an another game but had the same issue. Is it related to my epsilon-greedy policy, or I am not training enough or something else? Please help.