Simple DQN too slow to train [closed]

I have been trying to solve the OpenAI lunar lander game with a DQN taken from this paper

https://arxiv.org/pdf/2006.04938v2.pdf

The issue is that it takes 12 hours to train 50 episodes so something must be wrong.

import os
import random
import gym
import numpy as np
from collections import deque
import tensorflow as tf
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense
from tensorflow.keras.optimizers import Adam
from tensorflow.keras import Model

ENV_NAME = "LunarLander-v2"

DISCOUNT_FACTOR = 0.9
LEARNING_RATE = 0.001

MEMORY_SIZE = 2000
TRAIN_START = 1000
BATCH_SIZE = 24

EXPLORATION_MAX = 1.0
EXPLORATION_MIN = 0.01
EXPLORATION_DECAY = 0.99

class MyModel(Model):
def __init__(self, input_size, output_size):
super(MyModel, self).__init__()
self.d1 = Dense(128, input_shape=(input_size,), activation="relu")
self.d2 = Dense(128, activation="relu")
self.d3 = Dense(output_size, activation="linear")

def call(self, x):
x = self.d1(x)
x = self.d2(x)
return self.d3(x)

class DQNSolver():

def __init__(self, observation_space, action_space):
self.exploration_rate = EXPLORATION_MAX

self.action_space = action_space
self.memory = deque(maxlen=MEMORY_SIZE)

self.model = MyModel(observation_space,action_space)

def remember(self, state, action, reward, next_state, done):
self.memory.append((state, action, reward, next_state, done))

def act(self, state):
if np.random.rand() < self.exploration_rate:
return random.randrange(self.action_space)
q_values = self.model.predict(state)
return np.argmax(q_values[0])

def experience_replay(self):
if len(self.memory) < BATCH_SIZE:
return
batch = random.sample(self.memory, BATCH_SIZE)
state_batch, q_values_batch = [], []
for state, action, reward, state_next, terminal in batch:
# q-value prediction for a given state
q_values_cs = self.model.predict(state)
# target q-value
max_q_value_ns = np.amax(self.model.predict(state_next)[0])
# correction on the Q value for the action used
if terminal:
q_values_cs[0][action] = reward
else:
q_values_cs[0][action] = reward + DISCOUNT_FACTOR * max_q_value_ns
state_batch.append(state[0])
q_values_batch.append(q_values_cs[0])
# train the Q network
self.model.fit(np.array(state_batch),
np.array(q_values_batch),
batch_size = BATCH_SIZE,
epochs = 1, verbose = 0)
self.exploration_rate *= EXPLORATION_DECAY
self.exploration_rate = max(EXPLORATION_MIN, self.exploration_rate)

def lunar_lander():
env = gym.make(ENV_NAME)
observation_space = env.observation_space.shape[0]
action_space = env.action_space.n
dqn_solver = DQNSolver(observation_space, action_space)
episode = 0
print("Running")
while True:
episode += 1
state = env.reset()
state = np.reshape(state, [1, observation_space])
scores = []
score = 0
while True:
action = dqn_solver.act(state)
state_next, reward, terminal, _ = env.step(action)
state_next = np.reshape(state_next, [1, observation_space])
dqn_solver.remember(state, action, reward, state_next, terminal)
dqn_solver.experience_replay()
state = state_next
score += reward
if terminal:
print("Episode: " + str(episode) + ", exploration: " + str(dqn_solver.exploration_rate) + ", score: " + str(score))
scores.append(score)
break
if np.mean(scores[-min(100, len(scores)):]) >= 195:
print("Problem is solved in {} episodes.".format(episode))
break
env.close
if __name__ == "__main__":
lunar_lander()


Here are the logs

root@b11438e3d3e8:~# /usr/bin/python3 /root/test.py
2021-01-03 13:42:38.055593: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcudart.so.10.1
2021-01-03 13:42:39.338231: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcuda.so.1
2021-01-03 13:42:39.368192: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:982] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2021-01-03 13:42:39.368693: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1716] Found device 0 with properties:
pciBusID: 0000:01:00.0 name: GeForce GTX 1080 computeCapability: 6.1
coreClock: 1.8095GHz coreCount: 20 deviceMemorySize: 7.92GiB deviceMemoryBandwidth: 298.32GiB/s
2021-01-03 13:42:39.368729: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcudart.so.10.1
2021-01-03 13:42:39.370269: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcublas.so.10
2021-01-03 13:42:39.371430: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcufft.so.10
2021-01-03 13:42:39.371704: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcurand.so.10
2021-01-03 13:42:39.373318: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcusolver.so.10
2021-01-03 13:42:39.374243: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcusparse.so.10
2021-01-03 13:42:39.377939: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcudnn.so.7
2021-01-03 13:42:39.378118: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:982] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2021-01-03 13:42:39.378702: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:982] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2021-01-03 13:42:39.379127: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1858] Adding visible gpu devices: 0
2021-01-03 13:42:39.386525: I tensorflow/core/platform/profile_utils/cpu_utils.cc:104] CPU Frequency: 3411185000 Hz
2021-01-03 13:42:39.386867: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x4fb44c0 initialized for platform Host (this does not guarantee that XLA will be used). Devices:
2021-01-03 13:42:39.386891: I tensorflow/compiler/xla/service/service.cc:176]   StreamExecutor device (0): Host, Default Version
2021-01-03 13:42:39.498097: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:982] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2021-01-03 13:42:39.498786: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x4fdf030 initialized for platform CUDA (this does not guarantee that XLA will be used). Devices:
2021-01-03 13:42:39.498814: I tensorflow/compiler/xla/service/service.cc:176]   StreamExecutor device (0): GeForce GTX 1080, Compute Capability 6.1
2021-01-03 13:42:39.498987: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:982] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2021-01-03 13:42:39.499416: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1716] Found device 0 with properties:
pciBusID: 0000:01:00.0 name: GeForce GTX 1080 computeCapability: 6.1
coreClock: 1.8095GHz coreCount: 20 deviceMemorySize: 7.92GiB deviceMemoryBandwidth: 298.32GiB/s
2021-01-03 13:42:39.499448: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcudart.so.10.1
2021-01-03 13:42:39.499483: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcublas.so.10
2021-01-03 13:42:39.499504: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcufft.so.10
2021-01-03 13:42:39.499523: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcurand.so.10
2021-01-03 13:42:39.499543: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcusolver.so.10
2021-01-03 13:42:39.499562: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcusparse.so.10
2021-01-03 13:42:39.499581: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcudnn.so.7
2021-01-03 13:42:39.499643: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:982] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2021-01-03 13:42:39.500113: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:982] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2021-01-03 13:42:39.500730: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1858] Adding visible gpu devices: 0
2021-01-03 13:42:39.500772: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcudart.so.10.1
2021-01-03 13:42:39.915228: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1257] Device interconnect StreamExecutor with strength 1 edge matrix:
2021-01-03 13:42:39.915298: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1263]      0
2021-01-03 13:42:39.915322: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1276] 0:   N
2021-01-03 13:42:39.915568: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:982] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2021-01-03 13:42:39.916104: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:982] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2021-01-03 13:42:39.916555: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1402] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 6668 MB memory) -> physical GPU (device: 0, name: GeForce GTX 1080, pci bus id: 0000:01:00.0, compute capability: 6.1)
Running
2021-01-03 13:42:40.267699: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcublas.so.10


This is the GPU stats

+-----------------------------------------------------------------------------+
| NVIDIA-SMI 450.66       Driver Version: 450.66       CUDA Version: 11.0     |
|-------------------------------+----------------------+----------------------+
| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |
| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |
|                               |                      |               MIG M. |
|===============================+======================+======================|
|   0  GeForce GTX 1080    Off  | 00000000:01:00.0  On |                  N/A |
|  0%   53C    P2    46W / 198W |   7718MiB /  8111MiB |      0%      Default |
|                               |                      |                  N/A |
+-------------------------------+----------------------+----------------------+


As you can see, TensorFlow does not compute on the GPU but reserves the memory so I'm assuming it's because the inputs of the neural networks are too small and it uses the CPU instead.

To make sure the GPU was installed properly, I ran a sample from their documentation and it uses the GPU.

Is it an issue with the algorithm or the code? Is there a way to utilize the GPU in this case?

Thanks!

• Programming issues are off-topic here. We focus on the theoretical aspects of AI. Please, take a look at ai.stackexchange.com/help/on-topic for more info. So, this post seems to be off-topic. If I'm wrong, please, let me know. – nbro Jan 3 at 16:33
• Would it be better on stack overflow? – Marc Jan 3 at 19:53
• I think so. On SO, you will probably find more people interested in these issues. – nbro Jan 3 at 20:03
• Would it be possible to move datascience.stackexchange.com/questions/87458/… to SO? – Marc Jan 3 at 21:09
• I'm not a moderator at Data Science SE (so I can't directly do it myself), but I think if you ping a moderator there, he/she may do it. In any case, this question may also be on-topic on DS SE. – nbro Jan 3 at 21:11

When it comes to GPU usage,

nvidia-smi


shows the usage at the time it was executed. You should try running

watch -n0.01 nvidia-smi


to see the usage of GPU every 0.01 second. It should output some small usage for current model, like 5%. You could try to increase you model, to e.g.

self.d1 = Dense(1024, input_shape=(input_size,), activation="relu")
self.d2 = Dense(1024, activation="relu")
self.d3 = Dense(output_size, activation="linear")


to see if the usage of GPU increased.

• Thanks for the quick reply. I forgot to mention the command. I was already monitoring the GPU with nvidea-smi -l 1 and the usage was 0%. With nvidea-smi -l 0.01, I get usage between 0 and 2%. I tried to increase the size of the hidden layers without any luck. – Marc Jan 3 at 16:13

Is it training at all? Or is agent performance not improving over time? Q learning can be pretty unstable. I would recommend logging the sum of rewards received by the agent at the end of each episode and the model loss to help in the debugging process. The sum of rewards will show you if the agent is improving over time and the model loss will give you a rough idea about how stable the convergence is. I would recommend using tensorboard to log these metrics (https://www.tensorflow.org/tensorboard/get_started#using_tensorboard_with_other_methods). You will be able to monitor these metrics throughout the training process. You could also just print these metrics at the end of every epoch and monitor them in your console. You really just need someway to see what's going on during training.

In the paper you linked, it also mentioned double q learning, which in your code does not seem to be implemented. Vanilla q learning can have a reputation of being overoptimistic in the values that it assigns to states. This results in compounding approximation errors, which tend to destabilize learning. Using double q learning may help speed up convergence. If you need help with double q learning check out this paper: https://arxiv.org/pdf/1509.06461.pdf, and this github page: https://github.com/jihoonerd/Deep-Reinforcement-Learning-with-Double-Q-learning/blob/master/ddqn/agent/ddqn_agent.py

If you use double q learning, you may have to write your own custom training loop. This can be achieved by using the gradient tape object. Make sure to wrap this new function in a tf.function decorator. This will tell the TensorFlow back-end to compile that bit of code, making it run faster (https://www.tensorflow.org/guide/function). There are also some handy speed up tips in this post (https://www.tensorflow.org/tutorials/reinforcement_learning/actor_critic). They even wrap the environment step functions in tf.functions.The article uses actor-critic, which is a combination of policy gradient and q learning techniques, but you can swap out their neural network update code with the q learning functionality that you need.

If you need help with double q learning check out this paper: https://arxiv.org/pdf/1509.06461.pdf, and this github page: https://github.com/jihoonerd/Deep-Reinforcement-Learning-with-Double-Q-learning/blob/master/ddqn/agent/ddqn_agent.py

• Thanks for the detailed answer. I was trying to understand Q learning before switching to double Q learning. The agent overall is learning, it doesn't land yet but it learned how to fly between the flags and the score is improving. I will try to implement the optimization mentioned in your answer but I would like to know based on your experience if the slowness observed is normal. It played only 50 episodes in 12 hours (overnight) – Marc Jan 3 at 16:20
• I haven't messed with the lunar lander environment specifically, but the OpenAI page for the environment mentions that an environment ends only when the lander crashes or comes to a rest. It could be the case that the lander is just flying for a really long time, which would extend the timesteps for a given episode. Maybe try swapping lunar lander out for cart-pole and see if you still get long training times. Cart-pole has a fixed number of timesteps per episode. If you still get long training times, something else in your code is likely wrong. – Rohin Dasari Jan 3 at 18:24