0
$\begingroup$

Im trying to solve pong by a DQN approach. These are my hyper parameters:

    replay_buffer = deque(maxlen=100000)
    batch_size    = 50
    update_factor = 1000

    gamma = 0.99  # Discount future rate
    alpha = 0.0001  # Learning rate

    epsilon      = 1.0
    epsilon_min  = 0.01
    decay_rate   = 0.00005

and my network looks like:

def create_net(self):
    # Neural-net
    model = Sequential()
    model.add(Dense(24, input_dim=self.amount_obs, activation='relu'))
    model.add(Dense(24, activation='relu'))
    model.add(Dense(24, activation='relu'))
    model.add(Dense(24, activation='relu'))
    model.add(Dense(self.amount_actions, activation='linear'))
    model.compile(loss='mse', optimizer=Adam(lr=self.alpha))
    return model

My logic is as follows:

  1. Get 5 frames/steps from the environment

  2. Append these to my replay buffer

  3. Sample a minibatch from my replay buffer

  4. Train on this minibatch

  5. Repeat

I update my target network after 1k frames/steps

With this approach I get a perfect score on cartpole after 50k/60k frames/steps.

But I cant even get a slight improvement on pong. And after 150k frames, I have over trained and my agent wont even move from the bottom.

What can I be doing wrong? Is my logic correct? What are some common mistakes?

Very glad for any help!

code:

import gym
from gym import spaces
import numpy as np
import random
from collections import deque
from keras.optimizers import Adam
from keras.models import Sequential, save_model, load_model
from keras.layers import Activation, Dense
import copy
import sys

# Deep Q-learning Agent
class Model:
    def __init__(self, environment, model_name = ''):
    self.env = gym.make(environment)
    self.global_state = self.env.reset()

    self.amount_obs = len(self.env.observation_space.sample())
    self.amount_actions = self.env.action_space.n


    self.f = open('learning_history', 'a')

    self.replay_buffer = deque(maxlen=100000)
    self.batch_size    = 50
    self.amount_frames = 5
    self.update_factor = 1000
    self.max_frames    = 100000

    self.gamma = 0.99  # Discount future rate
    self.alpha = 0.0001  # Learning rate

    # Epsilon = explore/exploite factor
    self.epsilon      = 1.0 # 100% explore initally
    self.epsilon_min  = 0.01
    self.decay_rate   = 0.00005

    if model_name:
        print("using existing model")
        self.network = load_model(model_name)
    else:
        self.network  = self.create_net()

    self.target_network = copy.deepcopy(self.network)

    def create_net(self):
    # Neural-net
    model = Sequential()
    model.add(Dense(24, input_dim=self.amount_obs, activation='relu'))
    model.add(Dense(24, activation='relu'))
    model.add(Dense(24, activation='relu'))
    model.add(Dense(24, activation='relu'))
    model.add(Dense(self.amount_actions, activation='linear'))
    model.compile(loss='mse', optimizer=Adam(lr=self.alpha))
    return model

    def act(self, state, always_exploit=False):
    # Reshape into a column vector
    state = np.reshape(state, [1, self.amount_obs])

    # Explore or exploite
    if random.random() <= self.epsilon and not always_exploit:
        # Random action (explore)
        return random.randrange(self.amount_actions)
    # Predict an action (exploite)
    possible_actions = self.network.predict(state)
    return np.argmax(possible_actions[0])

    def generate_data(self):
    state = self.global_state
    for episode in range(self.amount_frames):
        # Decide action
        action = self.act(state)
        next_state, reward, done, _ = self.env.step(action)

        # Reshape into a column vectors
        state = np.reshape(state, [1, self.amount_obs])
        next_state = np.reshape(next_state, [1, self.amount_obs])
        # save the state
        self.replay_buffer.append((state, action, reward, next_state, done))
        if not done:
            state = next_state
        else:
            state = self.env.reset()

    self.global_state = state

    def train(self):
    frames = 0
    while True:
        # Get training data
        self.generate_data()
        # Only if we dont have enough data
        if len(self.replay_buffer) > self.batch_size:
            states        = []
            target_states = []
            sample_batch = random.sample(self.replay_buffer, self.batch_size)
            for state, action, reward, next_state, done in sample_batch:

               if done:
                    target = reward
               else:
                    target = reward + self.gamma * \
                           np.amax(self.target_network.predict(next_state))

               target_state = self.network.predict(state)
               target_state[0][action] = target

               # Saves states into a list
               state = np.squeeze(state)
               states.append(state)

               # Saves target state into a list
               target_state = np.squeeze(target_state)
               target_states.append(target_state)

            self.network.fit(np.asarray(states), np.asarray(target_states), \
                    batch_size=self.batch_size, \
                    epochs=2, verbose=0)

            # Decrease epsilon
            if self.epsilon > self.epsilon_min:
                self.epsilon -= self.decay_rate

            # Update target network after each n:th step
            if frames%self.update_factor==0 and not frames==0:
                print("{} frames".format(frames))
                print("Update target network and evaluate")

                # Update target network
                self.target_network = copy.deepcopy(self.network)

                # play games to determain how good network is
                play_average = self.play(5)
                self.f.write(str(frames) + ', ' + str(play_average) + '\n')

                print("new epsilon {}".format(self.epsilon))

            # Save network
            if frames%10000==0 and not frames==0:
                # Save the episode number and its average score
                name = "model_" + str(frames)
                self.network.save(name)
                print("{} saved...".format(name))

            # Increment loop
            frames += self.amount_frames

    def play(self, episodes):
    total_reward = 0
    for episode in range(1, episodes+1):
        state = self.env.reset()
        episode_reward = 0
        done  = False
        while not done:
            self.env.render()
            # Always get best action
            action = self.act(state, always_exploit=True)
            next_state, reward, done, _ = self.env.step(action)
            episode_reward += reward
            state = next_state
            if done:
                break

        total_reward += episode_reward
        print("game: {} with score {}".format(episode, episode_reward))

    average = total_reward/episodes
    print("Average score of {} games: {}".format(episodes, average))
    return average

if len(sys.argv) > 1:
    model = Model('Pong-ram-v0', sys.argv[1])
else:
    model = Model('Pong-ram-v0')

#model.train()
model.play(10)
$\endgroup$
  • $\begingroup$ I have small question: Does the model plays pong directly or does the model simulates the pong game with the aim of model predictive control? $\endgroup$ – Manuel Rodriguez Mar 11 at 20:44
  • $\begingroup$ @ManuelRodriguez It does play it! But in a environmentet so that it gets +1 reward if it scores and -1 reward if it gets scored on $\endgroup$ – Felix Rosén Mar 11 at 20:47
  • $\begingroup$ This explanation makes sense. The neural network controls the pong game directly and improving the network is equal to training the controller to play better. The overall system contains of a single instance which is the DQN-network. Around the DQN-network is only the environment given, which is the game. I'm not sure if this special case was described in the Arxiv-literature yet. $\endgroup$ – Manuel Rodriguez Mar 11 at 21:14
  • $\begingroup$ @ManuelRodriguez Yes, I think you get it. Would you please elaborate why this is a special case? $\endgroup$ – Felix Rosén Mar 11 at 21:16

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service, privacy policy and cookie policy

Browse other questions tagged or ask your own question.