I'm trying to build a DQN Agent to take a set of 10 best actions simultaneously (integer values from 1 to 100) as outputs per episode. The input is a float. The goal is to find the optimal combination of (10) actions per episode.
Currently, the set up is having a single NN output 10 actions w/ the highest q-valules for each episode. But in the Memory Replay process, each individual Set (of 10 fixed actions obtained from the exploration phase) is being treated as a single action. Because the target network also takes the output of the list of 10-action from the main NN. Hence I can see the agent repeatedly trying certain Set (with a fixed 10 actions) in the replay/retrain part, whereas our goal is to find the optimal combination of 10 actions per episode, Not the optimal Set of fixed combinations. So in essence, I would like the agent to pick out and mix up the actions from the Sets with higher Q-values (known from the exploration phase) to form new optimal "Sets" in the Replay process.
I was thinking maybe instead of using a single NN with 10 outputs I could do 10 NNs with single outputs for each episode so that each action is treated separately. And I suppose I will have 10 q-networks and target networks as well, then I could combine the results by the end of each episodes. But, I am not sure if that is necessarily the best way to fix the problem of having repetitive sets of fixed action in the replay process.
Alternatively, I think the problem could be treated as a multi-armed bandit problem, except each arm here has "sub-arms" too so to speak, but that could require some changes to the custom environment I am working with and I don't want to touch that unless necessary.
Maybe there is a clever manipulation within the retrain process given my current setup that I am not seeing. Here is a snippet of the code for some more clarity.
class DQNAgent():
def __init__(self,optimizer):
# Initialize atributes
self._state_size = 1
self._action_size = 76
self._optimizer = optimizer
self.experience_replay = deque(maxlen=2000)
# # Initialize discount and exploration rate
# self.gamma = 0.6
# self.epsilon = 0.5
self.gamma = 0.95
self.epsilon = 1.0
self.epsilon_min = 0.01
self.epsilon_decay = 0.95
self.learning_rate = 0.01
# Build networks
self.q_network = self._build_compile_model()
self.target_network = self._build_compile_model()
def store(self, state, action, reward, next_state, terminated):
self.experience_replay.append((state, action, reward, next_state, terminated))
def _build_compile_model(self):
model = Sequential()
model.add(InputLayer(input_shape=(self._state_size,)))
model.add(Dense(100, activation='relu'))
model.add(Dense(100, activation='relu'))
model.add(Dense(self._action_size, activation='linear'))
model.compile(loss='mse', optimizer=self._optimizer)
return model
def alighn_target_model(self):
self.target_network.set_weights(self.q_network.get_weights())
def retrain(self, batch_size):
if len(self.expirience_replay) < batch_size:
return
minibatch = random.sample(self.expirience_replay, batch_size)
for state, action, reward, next_state, terminated in minibatch:
target = self.q_network.predict(np.reshape(np.array(state), (-1,1)))
print('target size :', np.shape(target))
if terminated:
target[0][action] = reward
else:
t = self.target_network.predict(np.reshape(np.array(next_state), (-1,1)))
target[0][action] = reward + self.gamma * np.amax(t)
self.q_network.fit(np.reshape(np.array(state), (-1,1)), target, epochs=1, verbose=0)
def act(self,state):
self.epsilon *= self.epsilon_decay
self.epsilon = max(self.epsilon_min, self.epsilon)
action_space = [ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17,
18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34,
35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51,
52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68,
69, 70, 71, 72, 73, 74, 75, 76] #all 76 available nodes
if np.random.rand() <= self.epsilon:
return np.array(random.sample(action_space,10))-1 #-1 to match control's index
q_values = self.q_network.predict(np.reshape(np.array(state), (-1,1)))
print("q_vals shape",np.shape(q_values))
print('q_vals type',type(q_values))
top_actions_idx = q_values[0].argsort()[-10:][::-1]
