I'm having trouble implementing AC for continuous action space. As far as I can tell, my code doesn't seem to have any bugs! The agent is learning "something" as its behaviour seems to vary dramatically after several episodes, but it never seems to ever approach a type of behaviour which I'd think is reasonable.
I've used very similar code and things have gone smoothly in discrete space and little has changed other than changes to the output (mean and variance).
Below is the relevant code:
class Actor(object): def __init__(self, sess, s_size, h_size, a_size, env, lr=1e-3): mu = tf.layers.dense(self.hidden_1, self.a_size, activation=tf.nn.tanh, bias_initializer=None) sigma = tf.layers.dense(self.hidden_1, self.a_size, activation=tf.nn.softplus, bias_initializer=None) sigma = sigma + 1e-10 self.normal_dist = tf.contrib.distributions.Normal(mu, sigma) self.action = tf.clip_by_value(self.normal_dist.sample(1), env.action_space.low, env.action_space.high) self.adv = tf.placeholder(dtype=tf.float32) # get log prob of the actions taken in _samples self.acts = tf.placeholder(shape=[None, a_size], dtype=tf.float32) self.log_prob = self.normal_dist.log_prob(self.acts) self.loss = -self.log_prob * self.adv
The environment I'm using is the LunarLanderContinuous-v2. I've tested DDPG in this same environment and the agent learns incredibly quickly in comparison with the same learning rate and model size which is making me very confused. If anyone has any input it would be very much appreciated. Thanks
class Critic(object): def __init__(self, sess, s_size, h_size, env, gamma=0.99, lr=1e-3): self.gamma = gamma self.replay_buffer =  self.input = tf.placeholder(shape=[None, s_size], dtype=tf.float32) self.hidden_1 = tf.layers.dense(self.input, h_size, activation=tf.nn.relu, bias_initializer=None) self.hidden_2 = tf.layers.dense(self.hidden_1, h_size, activation=tf.nn.relu, bias_initializer=None) self.value = tf.layers.dense(self.hidden_2, 1, activation=None, bias_initializer=None) self.q_value = tf.placeholder(shape=[None,], dtype=tf.float32) self.advantage = self.q_value - self.value self.loss = tf.reduce_mean(tf.square(self.advantage)) self.lr = lr optimizer = tf.train.AdamOptimizer(self.lr) self.update = optimizer.minimize(self.loss)