2
$\begingroup$

I've created a deep Q network. My model does not get better, and can't see what I'm doing wrong. I'm new to RL.

Replay Memory

class ReplayMemory(object):

def __init__(self, input_shape, mem_size=100000):
    self.states = np.zeros((mem_size, input_shape))
    self.actions = np.zeros(mem_size, dtype=np.int32)
    self.next_states = np.zeros((mem_size, input_shape))
    self.rewards = np.zeros(mem_size)
    self.terminals = np.zeros(mem_size)

    self.mem_size = mem_size
    self.mem_count = 0

def push(self, state, action, next_state, reward, terminal):

    idx = self.mem_count % self.mem_size

    self.states[idx] = state
    self.actions[idx] = action
    self.next_states[idx] = next_state
    self.rewards[idx] = reward
    self.terminals[idx] = terminal

    self.mem_count += 1

def sample(self, batch_size):
    batch_index = np.random.randint(0, min(self.mem_count, self.mem_size), batch_size)

    states = self.states[batch_index]
    actions = self.actions[batch_index]
    next_states = self.next_states[batch_index]
    rewards = self.rewards[batch_index]
    terminals = self.terminals[batch_index]

    return (states, actions, next_states, rewards, terminals)

def __len__(self):
    return min(self.mem_count, self.mem_size)

DQN Agent

class DQN_Agent(object):

  def __init__(self, n_actions,n_states, ALPHA=0.001, GAMMA=0.99, eps_start=1 , eps_end=0.01, eps_decay=0.005):
      self.n_actions = n_actions
      self.n_states = n_states

      self.memory = ReplayMemory(n_states)

      self.ALPHA = ALPHA
      self.GAMMA = GAMMA

      self.eps_start = eps_start
      self.eps_end = eps_end
      self.eps_decay = eps_decay


      self.model = self.create_net()
      self.target = self.create_net()

      self.target.set_weights(self.model.get_weights())

      self.steps_counter = 0

  def create_net(self):

    model = Sequential([
        Dense(64, activation="relu", input_shape=(self.n_states,)),
        Dense(32, activation="relu"),
        Dense(self.n_actions)
    ])

    model.compile(loss="huber_loss", optimizer=Adam(lr=0.0005))

    return model

def select_action(self, state):
    ratio = self.eps_end + (self.eps_start-self.eps_end)*np.exp(-1*self.eps_decay*self.steps_counter)
    rand = random.random()
    self.steps_counter += 1

    if ratio > rand:
        #print("random")
        return np.random.randint(0, self.n_actions)
    else:
        #print("not random")
        return np.argmax(self.model.predict(state))


def train_model(self, batch_size):
    if len(self.memory) < batch_size:
        return None

    states, actions, next_states, rewards, terminals = self.memory.sample(batch_size)

    q_curr = self.model.predict(states)
    q_next = self.target.predict(next_states)
    q_target = q_curr.copy()


    batch_index = np.arange(batch_size, dtype=np.int32)

    q_target[batch_index, actions] = rewards + self.GAMMA*np.max(q_next, axis=1)*terminals

    _ = self.model.fit(states, q_target, verbose = 0)

    if self.steps_counter % 10 == 0:
        self.target.set_weights(self.model.get_weights())

My training loop

n_games = 50000
agent = DQN_Agent(2, 4)

scores = []
avg_scores = []

for epoch in range(n_games):
    done = False
    score = 0
    state = env.reset()

    while not done:
       #env.render()
       action = agent.select_action(state.reshape(1,-1)) 
       next_state, reward, done, _ = env.step(action)


       score += reward

       agent.memory.push(state, action, next_state, reward, done)

       state = next_state
       agent.train_model(64)


   avg_score = np.mean(scores[max(0, epoch-100):epoch + 1])
   avg_scores.append(avg_score)
   scores.append(score)
   print(score, avg_score)
$\endgroup$

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service, privacy policy and cookie policy

Browse other questions tagged or ask your own question.