# CartPoleV0 model is not getting trained in even after 1500+ episodes using deep Q-learning

I am new to deep Q learning and trying to train the open AI cartpole_V0 game using deep Q learning. Here is my code:

import gym
import os
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'
import tensorflow as tf
from collections import deque
import numpy as np
import random
import matplotlib
matplotlib.use('tkagg')
import matplotlib.pyplot as plt

EPISODES = 5000
output_dir = "/home/ug2018/mst/18114017/ML/"
EPSILON = 1
REPLAY_MEMORY = deque(maxlen=800)
MIN_EPSILON = 0.01
DECAY_RATE = 0.995
MINIBATCH = 750
GAMMA = 0.99

env = gym.make('CartPole-v0')
state_size = env.observation_space.shape[0]
action_size = env.action_space.n

class DQNagent:

def __init__(self):

self.fit_model = self.create_model()

self.predict_model = self.create_model()
self.predict_model.set_weights(self.fit_model.get_weights())

self.targets = []
self.states = []

def create_model(self):

model = tf.keras.models.Sequential()
return model

def model_summary(self,model):
return model.summary()

def get_q(self, state):
return self.predict_model.predict(state)

def train(self,batch_size):
minibatch = random.sample(REPLAY_MEMORY, batch_size)
for state, reward, action, new_state, done in minibatch:
if done :
target = reward
else:
target = reward + (GAMMA * np.amax(self.get_q(new_state)[0]))
target_f = self.get_q(state)
target_f[0][action] = target

self.states.append(state[0])
self.targets.append(target_f[0])

self.fit_weights(self.states,self.targets)

def fit_weights(self, states, targets):
self.fit_model.fit(np.array(states), np.array(targets), batch_size = MINIBATCH, epochs = 1 ,verbose=0)

def predict_save(self, name):
self.predict_model.save_weights(name)
def fit_save(self, name):
self.fit_model.save_weights(name)

agent = DQNagent()
print(agent.fit_model.summary())

x=[]
y=[]
z=[]

def update_graph(z,y):
plt.xlabel("Episodes")
plt.ylabel("Score")
plt.plot(z,y)
plt.pause(0.5)
plt.show()

for eps in range(EPISODES):
env.reset()
done = False
state = env.reset()
state = np.reshape(state, [1,state_size])
time = 0
exp=0
elp=0
while not done:

if EPSILON >= np.random.rand():
exp +=1
action = random.randrange(action_size)
else:
elp +=1
action = np.argmax(agent.get_q(state)[0])
new_state, reward, done, _ = env.step(action)
new_state = np.reshape(new_state,[1, state_size])
if not done:
reward = -10
else:
reward = 10
REPLAY_MEMORY.append((state,reward,action,new_state,done))
state = new_state
time += 1
x.append([eps,exp,elp,time,EPSILON])
y.append(time)
z.append(eps)
update_graph(z,y)
if (len(REPLAY_MEMORY)) >= MINIBATCH:
agent.train(MINIBATCH)
if EPSILON > MIN_EPSILON:
EPSILON *= DECAY_RATE
if eps % 50 == 0:
agent.predict_save(output_dir + "predict_weights_" + '{:04d}'.format(eps) + ".hdf5")
agent.fit_save(output_dir + "fit_weights_" + '{:04d}'.format(eps) + ".hdf5")
with open("score_vs_eps.txt", "w") as output:
output.write("Episodes"+"   "+"Exploration"+"   " + "Exploitation" + "  "+ "Score" + "  " + "Epsilon"+"\n")
for eps,exp,elp,time,epsilon in x:
output.write("      "+str(eps)+"        "+str(exp)+"        "+str(elp)+"        "+str(time)+"       "+"{:.4f}".format(epsilon) +"\n")

agent.predict_model.save('CartPole_predict_model')
agent.predict_model.save('CartPole_fit_model')


Code is running perfectly but the model is taking too many episodes to get trained and even after it scores 200, there is no continuity of it. Could please help me with how can I train the model in fewer episodes and maintain the continuity of the 200 scores?

Here are some of the steps listed:

Episodes Exploration Exploitation Score    Epsilon
0        18        0        18       1.0000
1        32        0        32       1.0000
2        43        0        43       1.0000
3        17        0        17       1.0000
4        17        0        17       1.0000
5        16        0        16       1.0000
6        13        0        13       1.0000
7        21        0        21       1.0000
8        16        0        16       1.0000
9        20        0        20       1.0000
10        35       0        35       1.0000
11        14       0        14       1.0000
12        13       0        13       1.0000
13        12       0        12       1.0000
14        16       0        16       1.0000
15        17       0        17       1.0000
16        27       0        27       1.0000
17        24       0        24       1.0000
18        14       0        14       1.0000
19        28       0        28       1.0000
20        20       0        20       1.0000
21        13       0        13       1.0000
22        12       0        12       1.0000
23        23       0        23       1.0000
24        17       0        17       1.0000
25        43       0        43       1.0000
26        61       0        61       1.0000
27        29       0        29       1.0000
28        21       0        21       1.0000
29        17       0        17       1.0000
30        41       0        41       1.0000
31        9        0        9        1.0000
32        18       0        18       1.0000
33        23       0        23       1.0000
34        28       0        28       0.9950
35        24       0        24       0.9900
36        25       0        25       0.9851
37        28       1        29       0.9801
38        26       1        27       0.9752
39        35       2        37       0.9704
40        19       0        19       0.9655
41        48       0        48       0.9607
42        25       2        27       0.9559
43        13       0        13       0.9511
44        20       2        22       0.9464
45        21       0        21       0.9416
46        13       0        13       0.9369
47        28       5        33       0.9322
48        23       3        26       0.9276
49        24       1        25       0.9229
50        20       2        22       0.9183
51        13       0        13       0.9137
52        19       1        20       0.9092
53        13       1        14       0.9046
54        18       1        19       0.9001
55        12       1        13       0.8956
56        29       7        36       0.8911
57        28       2        30       0.8867
58        16       1        17       0.8822
59        28       6        34       0.8778
60        13       3        16       0.8734
61        15       4        19       0.8691
62        19       2        21       0.8647
63        27       4        31       0.8604
64        19       4        23       0.8561
65        16       1        17       0.8518
66        60       9        69       0.8475
67        24       1        25       0.8433
68        21       7        28       0.8391
69        14       0        14       0.8349
70        31       4        35       0.8307
71        64       13       77       0.8266
72        58       13       71       0.8224
73        32       9        41       0.8183
74        15       1        16       0.8142
75        23       6        29       0.8102
76        27       5        32       0.8061
77        66       6        82       0.8021
78        30       6        36       0.7981
79        74       22       96       0.7941
80        14       1        15       0.7901
81        18       1        19       0.7862
82        28       7        35       0.7822
83        28       4        32       0.7783
84        12       2        14       0.7744
85        10       2        12       0.7705
86        21       4        25       0.7667
87        13       6        19       0.7629
88        19       6        25       0.7590
89        16       4        20       0.7553
90        46       16       62       0.7515
91        12       1        13       0.7477
92        30       15       45       0.7440
93        38       9        47       0.7403
94        14       7        21       0.7366
95        10       1        11       0.7329
96        16       8        24       0.7292
97        10       2        12       0.7256
98        20       5        25       0.7219
99        19       7        26       0.7183
100       31       9        40       0.7147
.
.
.
1522        0        104       104      0.0100
1523        0        35        35       0.0100
1524        0        27        27       0.0100
1525        0        52        52       0.0100
1526        0        25        25       0.0100
1527        1        199       200      0.0100
1528        0        30        30       0.0100
1529        0        57        57       0.0100
1530        0        35        35       0.0100
1531        0        25        25       0.0100
1532        0        22        22       0.0100
1533        0        24        24       0.0100
1534        1        199       200      0.0100
1535        0        68        68       0.0100
1536        0        200       200      0.0100
1537        0        22        22       0.0100
1538        2        42        44       0.0100
1539        1        111       112      0.0100
1540        0        91        91       0.0100
1541        0        45        45       0.0100
1542        2        108       110      0.0100
1543        1        181       182      0.0100
1544        0        30        30       0.0100
1545        0        21        21       0.0100
1546        1        25        26       0.0100
1547        4        196       200      0.0100
1548        0        95        95       0.0100
1549        0        53        53       0.0100
1550        0        55        55       0.0100
1551        0        29        29       0.0100
1552        0        40        40       0.0100
1553        0        25        25       0.0100
1554        0        33        33       0.0100
1555        0        63        63       0.0100
1556        0        23        23       0.0100
1557        0        45        45       0.0100
1558        0        25        25       0.0100
1559        0        36        36       0.0100
1560        0        24        24       0.0100
1561        1        31        32       0.0100
1562        0        30        30       0.0100
1563        1        56        57       0.0100
1564        0        22        22       0.0100
1565        0        20        20       0.0100
1566        1        22        23       0.0100
1567        0        45        45       0.0100
1568        1        50        51       0.0100
1569        0        25        25       0.0100
1570        0        30        30       0.0100
1571        2        198       200      0.0100
1572        2        198       200      0.0100
1573        1        185       186      0.0100
1574        0        26        26       0.0100
1575        4        196       200      0.0100
1576        3        197       200      0.0100
1577        1        29        30       0.0100
1578        0        25        25       0.0100
1579        0        32        32       0.0100
1580        3        197       200      0.0100
1581        1        23        24       0.0100
1582        0        25        25       0.0100
1583        0        66        66       0.0100
1584        1        27        28       0.0100
1585        0        32        32       0.0100
1586        0        21        21       0.0100
1587        0        23        23       0.0100
1588        1        47        48       0.0100
1589        0        42        42       0.0100
1590        0        26        26       0.0100
1591        0        47        47       0.0100
1592        0        200       200      0.0100
1593        2        52        54       0.0100
1594        1        19        20       0.0100
1595        0        33        33       0.0100
1596        0        27        27       0.0100
1597        1        79        80       0.0100
1598        0        54        54       0.0100
1599        0        50        50       0.0100
1600        0        25        25       0.0100


I have initiated the epsilon with 1 and it has already reached its minimum possible value. Still, the result is very fluctuating from the score's point of view. Why is this happening? How can I maintain the continuity?

• Please help me with this, how can I train this model to score 200 scores continuously. Am I doing anything wrong in applying the concept? May 24 at 9:35
• You make the reward 10 for it being done and -10 for it staying alive. Reversing those should help. Right now, it’s being trained to fall as quickly as possible. Another problem is that your neural network is way larger than it needs to be for this simple environment. May 24 at 10:32
• @S2673 , Thank you for pointing out the error. Could you please tell me how to decide how many layers will be good for optimal performance? May 24 at 10:44
• You can make a guess based on how complex the environment is and your experience with similar environments, but there is no one way I know of to easily decide the number of layers. From my experience with CartPole, you need very few layers or none at all. Around 2 should probably work for you. May 24 at 11:21
• @S2673 , As I understand the value stored in the done variable is false until the episode gets terminated. From that point of view, I have written the code rightly. It would have not been reached the score of 200 if I had implemented it wrongly. May 24 at 16:29