I'm making a simple deep Q learning algorithm, with cartpole-v1 env.
Like you can see in chart, after many episodes the reward decrease, some possible reasons?
The exploration vs axplotation algorithm is epsilon-decay, I used a target network (used every mini-batch gradient descent update, in calculating actual Q values, and next Q values, is it right?)
The neural network is made from scratch the complete code is here: https://github.com/LorenzoTinfena/deep-q-learning-itt-final-project
# %%
from core.dqn_agent import DQNAgent
from cartpole.cartpole_neural_network import CartPoleNeuralNetwork
from cartpole.cartpole_wrapper import CartPoleWrapper
import gym
import numpy as np
import torch
from tqdm import tqdm
import glob
import os
from IPython.display import Video
import matplotlib
import matplotlib.pyplot as plt
import numpy as np
from itertools import cycle
import sys
import shutil
from pathlib import Path
import shutil
import pyvirtualdisplay
_display = pyvirtualdisplay.Display(visible=False, size=(1400, 900))
_ = _display.start()
# %% [markdown]
# Initialize deep Q-learning agent, neural network, and parameters
# %%
np.random.seed(20)
agent = DQNAgent(env=CartPoleWrapper(gym.make("CartPole-v1")),
nn=CartPoleNeuralNetwork(), replay_memory_max_size=5000, batch_size=30)
DISCOUNT_FACTOR = 0.995
LEARNING_RATE = 0.0001
n_episodes = []
total_rewards = []
number_steps = []
total_episodes = 0
# %% [markdown]
# Training
# %%
while total_episodes <= 10000:
total_reward, steps = agent.start_episode_and_evaluate(DISCOUNT_FACTOR, LEARNING_RATE, epsilon=0, min_epsilon=0, render=False, optimize=False)
print(f'\ntotal_episodes_training: {total_episodes}\tsteps: {steps}\ttotal_reward: {total_reward}', flush = True)
n_episodes.append(total_episodes)
total_rewards.append(total_reward)
number_steps.append(steps)
for i in tqdm(range(50), 'learning...'):
agent.start_episode_and_evaluate(DISCOUNT_FACTOR, LEARNING_RATE, epsilon=1, epsilon_decay=0.99, min_epsilon=0.01, render=False, optimize=True)
total_episodes += i+1
