My Deep Q-Learning Network does not learn for OpenAI gym's cartpole problem

Question

I am implementing OpenAI gym's cartpole problem using Deep Q-Learning (DQN). I followed tutorials (video and otherwise) and learned all about it. I implemented a code for myself and I thought it should work, but the agent is not learning. I will really really really appreciate if someone can pinpoint where I am doing wrong.

Note that I have a target neuaral network and a policy network already there. The code is as below.

import numpy as np
import gym
import random
from keras.optimizers import Adam
from keras.models import Sequential
from keras.layers import Dense
from collections import deque

env = gym.make('CartPole-v0')

EPISODES = 2000
BATCH_SIZE = 32
DISCOUNT = 0.95
UPDATE_TARGET_EVERY = 5
STATE_SIZE = env.observation_space.shape[0]
ACTION_SIZE = env.action_space.n
SHOW_EVERY = 50

class DQNAgents:
    
    def __init__(self, state_size, action_size):
        self.state_size = state_size
        self.action_size = action_size
        self.replay_memory = deque(maxlen = 2000)
        self.gamma = 0.95
        self.epsilon = 1
        self.epsilon_decay = 0.995
        self.epsilon_min = 0.01
        self.model = self._build_model()
        self.target_model = self.model
        
        self.target_update_counter = 0
        print('Initialize the agent')
        
    def _build_model(self):
        model = Sequential()
        model.add(Dense(20, input_dim = self.state_size, activation = 'relu'))
        model.add(Dense(10, activation = 'relu'))
        model.add(Dense(self.action_size, activation = 'linear'))
        model.compile(loss = 'mse', optimizer = Adam(lr = 0.001))
        
        return model

    def update_replay_memory(self, current_state, action, reward, next_state, done):
        self.replay_memory.append((current_state, action, reward, next_state, done))
        
    def train(self, terminal_state):
        
        # Sample from replay memory
        minibatch = random.sample(self.replay_memory, BATCH_SIZE)
        
        #Picks the current states from the randomly selected minibatch
        current_states = np.array([t[0] for t in minibatch])
        current_qs_list= self.model.predict(current_states) #gives the Q value for the policy network
        new_state = np.array([t[3] for t in minibatch])
        future_qs_list = self.target_model.predict(new_state)
        
        X = []
        Y = []
        
        # This loop will run 32 times (actually minibatch times)
        for index, (current_state, action, reward, next_state, done) in enumerate(minibatch):
            
            if not done:
                new_q = reward + DISCOUNT * np.max(future_qs_list)
            else:
                new_q = reward
                
            # Update Q value for given state
            current_qs = current_qs_list[index]
            current_qs[action] = new_q
            
            X.append(current_state)
            Y.append(current_qs)
        
        # Fitting the weights, i.e. reducing the loss using gradient descent
        self.model.fit(np.array(X), np.array(Y), batch_size = BATCH_SIZE, verbose = 0, shuffle = False)
        
       # Update target network counter every episode
        if terminal_state:
            self.target_update_counter += 1
            
        # If counter reaches set value, update target network with weights of main network
        if self.target_update_counter > UPDATE_TARGET_EVERY:
            self.target_model.set_weights(self.model.get_weights())
            self.target_update_counter = 0
    
    def get_qs(self, state):
        return self.model.predict(np.array(state).reshape(-1, *state.shape))[0]
            

''' We start here'''

agent = DQNAgents(STATE_SIZE, ACTION_SIZE)

for e in range(EPISODES):
    
    done = False
    current_state = env.reset()
    time = 0 
    total_reward = 0
    while not done:
        if np.random.random() > agent.epsilon:
            action = np.argmax(agent.get_qs(current_state))
        else:
            action = env.action_space.sample()
        
        next_state, reward, done, _ = env.step(action)

        agent.update_replay_memory(current_state, action, reward, next_state, done)
        
        if len(agent.replay_memory) < BATCH_SIZE:
            pass
        else:
            agent.train(done)
            
        time+=1    
        current_state = next_state
        total_reward += reward
        
    print(f'episode : {e}, steps {time}, epsilon : {agent.epsilon}')
    
    if agent.epsilon > agent.epsilon_min:
        agent.epsilon *= agent.epsilon_decay

Results for first 40ish iterations are below (look for the number of steps, they should be increasing and should reach a maximum of 199)

episode : 0, steps 14, epsilon : 1
episode : 1, steps 13, epsilon : 0.995
episode : 2, steps 17, epsilon : 0.990025
episode : 3, steps 12, epsilon : 0.985074875
episode : 4, steps 29, epsilon : 0.9801495006250001
episode : 5, steps 14, epsilon : 0.9752487531218751
episode : 6, steps 11, epsilon : 0.9703725093562657
episode : 7, steps 13, epsilon : 0.9655206468094844
episode : 8, steps 11, epsilon : 0.960693043575437
episode : 9, steps 14, epsilon : 0.9558895783575597
episode : 10, steps 39, epsilon : 0.9511101304657719
episode : 11, steps 14, epsilon : 0.946354579813443
episode : 12, steps 19, epsilon : 0.9416228069143757
episode : 13, steps 16, epsilon : 0.9369146928798039
episode : 14, steps 14, epsilon : 0.9322301194154049
episode : 15, steps 18, epsilon : 0.9275689688183278
episode : 16, steps 31, epsilon : 0.9229311239742362
episode : 17, steps 14, epsilon : 0.918316468354365
episode : 18, steps 21, epsilon : 0.9137248860125932
episode : 19, steps 9, epsilon : 0.9091562615825302
episode : 20, steps 26, epsilon : 0.9046104802746175
episode : 21, steps 20, epsilon : 0.9000874278732445
episode : 22, steps 53, epsilon : 0.8955869907338783
episode : 23, steps 24, epsilon : 0.8911090557802088
episode : 24, steps 14, epsilon : 0.8866535105013078
episode : 25, steps 40, epsilon : 0.8822202429488013
episode : 26, steps 10, epsilon : 0.8778091417340573
episode : 27, steps 60, epsilon : 0.8734200960253871
episode : 28, steps 17, epsilon : 0.8690529955452602
episode : 29, steps 11, epsilon : 0.8647077305675338
episode : 30, steps 42, epsilon : 0.8603841919146962
episode : 31, steps 16, epsilon : 0.8560822709551227
episode : 32, steps 12, epsilon : 0.851801859600347
episode : 33, steps 12, epsilon : 0.8475428503023453
episode : 34, steps 10, epsilon : 0.8433051360508336
episode : 35, steps 30, epsilon : 0.8390886103705794
episode : 36, steps 21, epsilon : 0.8348931673187264
episode : 37, steps 24, epsilon : 0.8307187014821328
episode : 38, steps 33, epsilon : 0.8265651079747222
episode : 39, steps 32, epsilon : 0.8224322824348486
episode : 40, steps 15, epsilon : 0.8183201210226743
episode : 41, steps 20, epsilon : 0.8142285204175609
episode : 42, steps 37, epsilon : 0.810157377815473
episode : 43, steps 11, epsilon : 0.8061065909263957
episode : 44, steps 30, epsilon : 0.8020760579717637
episode : 45, steps 11, epsilon : 0.798065677681905
episode : 46, steps 34, epsilon : 0.7940753492934954
episode : 47, steps 12, epsilon : 0.7901049725470279
episode : 48, steps 26, epsilon : 0.7861544476842928
episode : 49, steps 19, epsilon : 0.7822236754458713
episode : 50, steps 20, epsilon : 0.778312557068642

Answer 1

There is a really small mistake in here that causes the problem:

for index, (current_state, action, reward, next_state, done) in enumerate(minibatch):
            if not done:
                new_q = reward + DISCOUNT * np.max(future_qs_list) #HERE 
            else:
                new_q = reward
                
            # Update Q value for given state
            current_qs = current_qs_list[index]
            current_qs[action] = new_q
            
            X.append(current_state)
            Y.append(current_qs)

Since np.max(future_qs_list) should be np.max(future_qs_list[index]) since you're now getting the highest Q of the entire batch. Instead of the getting the highest Q from the current next state.

It's like this after changing that (remember an epsilon of 1 means that you get 100% of your actions taken by the a dice roll so I let it go for a few more epochs, also tried it with the old code but indeed didn't get more then 50 steps (even after 400 epochs/episodes))

episode : 52, steps 16, epsilon : 0.7705488893118823
episode : 53, steps 25, epsilon : 0.7666961448653229
episode : 54, steps 25, epsilon : 0.7628626641409962
episode : 55, steps 36, epsilon : 0.7590483508202912
episode : 56, steps 32, epsilon : 0.7552531090661897
episode : 57, steps 22, epsilon : 0.7514768435208588
episode : 58, steps 55, epsilon : 0.7477194593032545
episode : 59, steps 24, epsilon : 0.7439808620067382
episode : 60, steps 46, epsilon : 0.7402609576967045
episode : 61, steps 11, epsilon : 0.736559652908221
episode : 62, steps 14, epsilon : 0.7328768546436799
episode : 63, steps 13, epsilon : 0.7292124703704616
episode : 64, steps 113, epsilon : 0.7255664080186093
episode : 65, steps 33, epsilon : 0.7219385759785162
episode : 66, steps 33, epsilon : 0.7183288830986236
episode : 67, steps 39, epsilon : 0.7147372386831305
episode : 68, steps 27, epsilon : 0.7111635524897149
episode : 69, steps 22, epsilon : 0.7076077347272662
episode : 70, steps 60, epsilon : 0.7040696960536299
episode : 71, steps 40, epsilon : 0.7005493475733617
episode : 72, steps 67, epsilon : 0.697046600835495
episode : 73, steps 115, epsilon : 0.6935613678313175
episode : 74, steps 61, epsilon : 0.6900935609921609
episode : 75, steps 43, epsilon : 0.6866430931872001
episode : 76, steps 21, epsilon : 0.6832098777212641
episode : 77, steps 65, epsilon : 0.6797938283326578
episode : 78, steps 45, epsilon : 0.6763948591909945
episode : 79, steps 93, epsilon : 0.6730128848950395
episode : 80, steps 200, epsilon : 0.6696478204705644
episode : 81, steps 200, epsilon : 0.6662995813682115

Answer 2

I think the problem is with openAI gym CartPole-v0 environment reward structure. The reward is always +1 for each time step. So if pole falls reward is +1 itself. So we need to check and redefine the reward for this case. So in the train function try this:

if not done:
    new_q = reward + DISCOUNT * np.max(future_qs_list)
else:
    # if done assign some negative reward
    new_q = -20

(Or change the reward during replay buffer update)

Check the lines 81 and 82 in Qlearning.py code in this repo for further clarification.