# My Deep Q-Learning Network does not learn for OpenAI gym's cartpole problem

I am implementing OpenAI gym's cartpole problem using Deep Q-Learning (DQN). I followed tutorials (video and otherwise) and learned all about it. I implemented a code for myself and I thought it should work, but the agent is not learning. I will really really really appreciate if someone can pinpoint where I am doing wrong.

Note that I have a target neuaral network and a policy network already there. The code is as below.

import numpy as np
import gym
import random
from keras.models import Sequential
from keras.layers import Dense
from collections import deque

env = gym.make('CartPole-v0')

EPISODES = 2000
BATCH_SIZE = 32
DISCOUNT = 0.95
UPDATE_TARGET_EVERY = 5
STATE_SIZE = env.observation_space.shape[0]
ACTION_SIZE = env.action_space.n
SHOW_EVERY = 50

class DQNAgents:

def __init__(self, state_size, action_size):
self.state_size = state_size
self.action_size = action_size
self.replay_memory = deque(maxlen = 2000)
self.gamma = 0.95
self.epsilon = 1
self.epsilon_decay = 0.995
self.epsilon_min = 0.01
self.model = self._build_model()
self.target_model = self.model

self.target_update_counter = 0
print('Initialize the agent')

def _build_model(self):
model = Sequential()
model.add(Dense(20, input_dim = self.state_size, activation = 'relu'))
model.compile(loss = 'mse', optimizer = Adam(lr = 0.001))

return model

def update_replay_memory(self, current_state, action, reward, next_state, done):
self.replay_memory.append((current_state, action, reward, next_state, done))

def train(self, terminal_state):

# Sample from replay memory
minibatch = random.sample(self.replay_memory, BATCH_SIZE)

#Picks the current states from the randomly selected minibatch
current_states = np.array([t[0] for t in minibatch])
current_qs_list= self.model.predict(current_states) #gives the Q value for the policy network
new_state = np.array([t[3] for t in minibatch])
future_qs_list = self.target_model.predict(new_state)

X = []
Y = []

# This loop will run 32 times (actually minibatch times)
for index, (current_state, action, reward, next_state, done) in enumerate(minibatch):

if not done:
new_q = reward + DISCOUNT * np.max(future_qs_list)
else:
new_q = reward

# Update Q value for given state
current_qs = current_qs_list[index]
current_qs[action] = new_q

X.append(current_state)
Y.append(current_qs)

# Fitting the weights, i.e. reducing the loss using gradient descent
self.model.fit(np.array(X), np.array(Y), batch_size = BATCH_SIZE, verbose = 0, shuffle = False)

# Update target network counter every episode
if terminal_state:
self.target_update_counter += 1

# If counter reaches set value, update target network with weights of main network
if self.target_update_counter > UPDATE_TARGET_EVERY:
self.target_model.set_weights(self.model.get_weights())
self.target_update_counter = 0

def get_qs(self, state):
return self.model.predict(np.array(state).reshape(-1, *state.shape))[0]

''' We start here'''

agent = DQNAgents(STATE_SIZE, ACTION_SIZE)

for e in range(EPISODES):

done = False
current_state = env.reset()
time = 0
total_reward = 0
while not done:
if np.random.random() > agent.epsilon:
action = np.argmax(agent.get_qs(current_state))
else:
action = env.action_space.sample()

next_state, reward, done, _ = env.step(action)

agent.update_replay_memory(current_state, action, reward, next_state, done)

if len(agent.replay_memory) < BATCH_SIZE:
pass
else:
agent.train(done)

time+=1
current_state = next_state
total_reward += reward

print(f'episode : {e}, steps {time}, epsilon : {agent.epsilon}')

if agent.epsilon > agent.epsilon_min:
agent.epsilon *= agent.epsilon_decay


Results for first 40ish iterations are below (look for the number of steps, they should be increasing and should reach a maximum of 199)

episode : 0, steps 14, epsilon : 1
episode : 1, steps 13, epsilon : 0.995
episode : 2, steps 17, epsilon : 0.990025
episode : 3, steps 12, epsilon : 0.985074875
episode : 4, steps 29, epsilon : 0.9801495006250001
episode : 5, steps 14, epsilon : 0.9752487531218751
episode : 6, steps 11, epsilon : 0.9703725093562657
episode : 7, steps 13, epsilon : 0.9655206468094844
episode : 8, steps 11, epsilon : 0.960693043575437
episode : 9, steps 14, epsilon : 0.9558895783575597
episode : 10, steps 39, epsilon : 0.9511101304657719
episode : 11, steps 14, epsilon : 0.946354579813443
episode : 12, steps 19, epsilon : 0.9416228069143757
episode : 13, steps 16, epsilon : 0.9369146928798039
episode : 14, steps 14, epsilon : 0.9322301194154049
episode : 15, steps 18, epsilon : 0.9275689688183278
episode : 16, steps 31, epsilon : 0.9229311239742362
episode : 17, steps 14, epsilon : 0.918316468354365
episode : 18, steps 21, epsilon : 0.9137248860125932
episode : 19, steps 9, epsilon : 0.9091562615825302
episode : 20, steps 26, epsilon : 0.9046104802746175
episode : 21, steps 20, epsilon : 0.9000874278732445
episode : 22, steps 53, epsilon : 0.8955869907338783
episode : 23, steps 24, epsilon : 0.8911090557802088
episode : 24, steps 14, epsilon : 0.8866535105013078
episode : 25, steps 40, epsilon : 0.8822202429488013
episode : 26, steps 10, epsilon : 0.8778091417340573
episode : 27, steps 60, epsilon : 0.8734200960253871
episode : 28, steps 17, epsilon : 0.8690529955452602
episode : 29, steps 11, epsilon : 0.8647077305675338
episode : 30, steps 42, epsilon : 0.8603841919146962
episode : 31, steps 16, epsilon : 0.8560822709551227
episode : 32, steps 12, epsilon : 0.851801859600347
episode : 33, steps 12, epsilon : 0.8475428503023453
episode : 34, steps 10, epsilon : 0.8433051360508336
episode : 35, steps 30, epsilon : 0.8390886103705794
episode : 36, steps 21, epsilon : 0.8348931673187264
episode : 37, steps 24, epsilon : 0.8307187014821328
episode : 38, steps 33, epsilon : 0.8265651079747222
episode : 39, steps 32, epsilon : 0.8224322824348486
episode : 40, steps 15, epsilon : 0.8183201210226743
episode : 41, steps 20, epsilon : 0.8142285204175609
episode : 42, steps 37, epsilon : 0.810157377815473
episode : 43, steps 11, epsilon : 0.8061065909263957
episode : 44, steps 30, epsilon : 0.8020760579717637
episode : 45, steps 11, epsilon : 0.798065677681905
episode : 46, steps 34, epsilon : 0.7940753492934954
episode : 47, steps 12, epsilon : 0.7901049725470279
episode : 48, steps 26, epsilon : 0.7861544476842928
episode : 49, steps 19, epsilon : 0.7822236754458713
episode : 50, steps 20, epsilon : 0.778312557068642

• I have also tried by setting the epsilon (parameter for epsilon-greedy) to 0.1 from the beginning. The learning still remain worse than what it is when epsilon starts from 1. Apparently there is some issue in updating the best Q value for the selected action. But I cannot find what exactly I am missing, doing wrong, or what I should add. – Kashan Aug 11 '20 at 15:49
• You know that your epsilon needs to be high in the beginning since it's the way you search though your search space... (eploitation vs exploration) – hal9000 Aug 15 '20 at 22:01

There is a really small mistake in here that causes the problem:


for index, (current_state, action, reward, next_state, done) in enumerate(minibatch):
if not done:
new_q = reward + DISCOUNT * np.max(future_qs_list) #HERE
else:
new_q = reward

# Update Q value for given state
current_qs = current_qs_list[index]
current_qs[action] = new_q

X.append(current_state)
Y.append(current_qs)


Since np.max(future_qs_list) should be np.max(future_qs_list[index]) since you're now getting the highest Q of the entire batch. Instead of the getting the highest Q from the current next state.

It's like this after changing that (remember an epsilon of 1 means that you get 100% of your actions taken by the a dice roll so I let it go for a few more epochs, also tried it with the old code but indeed didn't get more then 50 steps (even after 400 epochs/episodes))

episode : 52, steps 16, epsilon : 0.7705488893118823
episode : 53, steps 25, epsilon : 0.7666961448653229
episode : 54, steps 25, epsilon : 0.7628626641409962
episode : 55, steps 36, epsilon : 0.7590483508202912
episode : 56, steps 32, epsilon : 0.7552531090661897
episode : 57, steps 22, epsilon : 0.7514768435208588
episode : 58, steps 55, epsilon : 0.7477194593032545
episode : 59, steps 24, epsilon : 0.7439808620067382
episode : 60, steps 46, epsilon : 0.7402609576967045
episode : 61, steps 11, epsilon : 0.736559652908221
episode : 62, steps 14, epsilon : 0.7328768546436799
episode : 63, steps 13, epsilon : 0.7292124703704616
episode : 64, steps 113, epsilon : 0.7255664080186093
episode : 65, steps 33, epsilon : 0.7219385759785162
episode : 66, steps 33, epsilon : 0.7183288830986236
episode : 67, steps 39, epsilon : 0.7147372386831305
episode : 68, steps 27, epsilon : 0.7111635524897149
episode : 69, steps 22, epsilon : 0.7076077347272662
episode : 70, steps 60, epsilon : 0.7040696960536299
episode : 71, steps 40, epsilon : 0.7005493475733617
episode : 72, steps 67, epsilon : 0.697046600835495
episode : 73, steps 115, epsilon : 0.6935613678313175
episode : 74, steps 61, epsilon : 0.6900935609921609
episode : 75, steps 43, epsilon : 0.6866430931872001
episode : 76, steps 21, epsilon : 0.6832098777212641
episode : 77, steps 65, epsilon : 0.6797938283326578
episode : 78, steps 45, epsilon : 0.6763948591909945
episode : 79, steps 93, epsilon : 0.6730128848950395
episode : 80, steps 200, epsilon : 0.6696478204705644
episode : 81, steps 200, epsilon : 0.6662995813682115


I think the problem is with openAI gym CartPole-v0 environment reward structure. The reward is always +1 for each time step. So if pole falls reward is +1 itself. So we need to check and redefine the reward for this case. So in the train function try this:

if not done:
new_q = reward + DISCOUNT * np.max(future_qs_list)
else:
# if done assign some negative reward
new_q = -20


(Or change the reward during replay buffer update)

Check the lines 81 and 82 in Qlearning.py code in this repo for further clarification.

• You really don't need to edit the reward structure. The reward structure is fine. Such a high negative reward can even have negative effects. – hal9000 Aug 15 '20 at 22:53
• @hal I’m not sure that that’s true (that you don’t need to change it). If the environment gives +1 on termination then that can be problematic for the agent, it is better to change it to 0 or -1. – David Ireland Aug 15 '20 at 23:12
• yeah, I have to agree on that, 0 or -1 seems reasonable to me too but the reward structure is fine enough to be able to solve without doing that. – hal9000 Aug 15 '20 at 23:15
• @hal I experimented with different reward stuctures for cartpole using classical Q-learning(Not DQN). I found that reward between -10 to -20 for terminal state working better. My experimentation code and graph can be found here. But I found this code having reward of -200 for terminal state in official gym website. – Girish Dattatray Hegde Aug 16 '20 at 4:36