I've pieced together this A3C w/ PPO Gym Pendulum example, but I'm finding after a while, when attempting to get the action from the model, I get a NaN return:

a = self.sess.run(self.sample_op, {self.tfs: s})[0]

It runs okay for a while, but then errors. To me that implies perhaps a invalid update happens at some point that corrupts the model. But I've put debugging output in the code and everything appears to be fine - it's not submitting any NaNs or outliers to the model as far as I can see.

After a bit of playing around I find that if I comment out the code to update the actor model then the code executes fine. Obviously it doesn't learn much, but it appears it's something to do with this update:

[self.sess.run(self.atrain_op, {self.tfs: s, self.tfa: a, self.tfadv: adv}) for _ in range(UPDATE_STEP)]

Can anyone see what the problem might be? I've been playing around with this for hours and it's got me stumped.

In code example below I set N_WORKERS = 1 so the debug is easier to read, but if you want to hit the error faster then increase that figure to the number of CPU cores you have.

tensorflow 1.8.0
gym 0.9.2

import tensorflow as tf
import numpy as np
import matplotlib.pyplot as plt
import gym, threading, queue, math

EP_MAX = 600
EP_LEN = 200
N_WORKER = 1                # parallel workers
GAMMA = 0.9                 # reward discount factor
A_LR = 0.0001               # learning rate for actor
C_LR = 0.0001               # learning rate for critic
MIN_BATCH_SIZE = 64         # minimum batch size for updating PPO
UPDATE_STEP = 15            # loop update operation n-steps
EPSILON = 0.2               # for clipping surrogate objective
GAME = 'Pendulum-v0'

env = gym.make(GAME)
S_DIM = env.observation_space.shape[0]
A_DIM = 1 # not available in pendulum env.action_space.n

class PPONet(object):
    def __init__(self):
        self.sess = tf.Session()
        self.tfs = tf.placeholder(tf.float32, [None, S_DIM], 'state')
        self.tfa = tf.placeholder(tf.float32, [None, ], 'action')
        self.tfadv = tf.placeholder(tf.float32, [None, 1], 'advantage')        

        # critic        
        with tf.variable_scope('critic'):
            w_init = tf.random_normal_initializer(0., .1)
            lc = tf.layers.dense(self.tfs, 200, tf.nn.relu, kernel_initializer=w_init, name='layer1-critic')
            self.v = tf.layers.dense(lc, 1)

        with tf.variable_scope('ctrain'):
            self.tfdc_r = tf.placeholder(tf.float32, [None, 1], 'discounted_r')
            self.advantage = self.tfdc_r - self.v
            self.closs = tf.reduce_mean(tf.square(self.advantage))
            self.ctrain_op = tf.train.AdamOptimizer(C_LR).minimize(self.closs)

        # actor
        pi, pi_params = self._build_anet('pi', trainable=True)
        oldpi, oldpi_params = self._build_anet('oldpi', trainable=False)

        with tf.variable_scope('sample_action'):
            self.sample_op = tf.squeeze(pi.sample(1), axis=0)       # choosing action

        with tf.variable_scope('update_oldpi'):
            self.update_oldpi_op = [oldp.assign(p) for p, oldp in zip(pi_params, oldpi_params)]

        with tf.variable_scope('loss'):
            with tf.variable_scope('surrogate_pp'):
                ratio = pi.prob(self.tfa) / oldpi.prob(self.tfa)
                surr = ratio * self.tfadv

            self.aloss = -tf.reduce_mean(tf.minimum(        # clipped surrogate objective
                tf.clip_by_value(ratio, 1. - EPSILON, 1. + EPSILON) * self.tfadv))

        with tf.variable_scope('atrain'):
            self.atrain_op = tf.train.AdamOptimizer(A_LR).minimize(self.aloss)


    def update(self):
        while not COORD.should_stop():
            if GLOBAL_EP < EP_MAX:
                UPDATE_EVENT.wait()                     # wait until get batch of data
                self.sess.run(self.update_oldpi_op)     # copy pi to old pi
                data = [QUEUE.get() for _ in range(QUEUE.qsize())]      # collect data from all workers
                data = np.vstack(data)
                s, a, r = data[:, :S_DIM], data[:, S_DIM: S_DIM + 1].ravel(), data[:, -1:]
                adv = self.sess.run(self.advantage, {self.tfs: s, self.tfdc_r: r})

                print("Updating: s: {}, a: {}, r: {}, adv: {}".format(s, a, r, adv))

                # update actor and critic in a update loop
                [self.sess.run(self.atrain_op, {self.tfs: s, self.tfa: a, self.tfadv: adv}) for _ in range(UPDATE_STEP)] #ERR 
                [self.sess.run(self.ctrain_op, {self.tfs: s, self.tfdc_r: r}) for _ in range(UPDATE_STEP)]
                UPDATE_EVENT.clear()        # updating finished
                GLOBAL_UPDATE_COUNTER = 0   # reset counter
                ROLLING_EVENT.set()         # set roll-out available

    def _build_anet(self, name, trainable): # Build the current & hold structure for the policies 
        with tf.variable_scope(name):
            l1 = tf.layers.dense(self.tfs, 200, tf.nn.relu, trainable=trainable)
            mu = 2 * tf.layers.dense(l1, A_DIM, tf.nn.tanh, trainable=trainable, name = 'mu_'+name)
            sigma = tf.layers.dense(l1, A_DIM, tf.nn.softplus, trainable=trainable,name ='sigma_'+name )
            norm_dist = tf.distributions.Normal(loc=mu, scale=sigma) # Loc is the mean
        params = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope=name) # Collects the weights of the layers l1, mu / 2, sigma
        return norm_dist, params    

    def choose_action(self, s):
        s = s[np.newaxis, :]
        print("Action s: {}".format(s))
        a = self.sess.run(self.sample_op, {self.tfs: s})[0]
        if math.isnan(a):
            print("Action is NaN - stopping")
        return np.clip(a, -2, 2)

    def get_v(self, s):
        if s.ndim < 2: s = s[np.newaxis, :]
        return self.sess.run(self.v, {self.tfs: s})[0, 0]

class Worker(object):
    def __init__(self, wid):
        self.wid = wid
        self.env = gym.make(GAME).unwrapped
        self.ppo = GLOBAL_PPO

    def work(self):
        while not COORD.should_stop():
            s = self.env.reset()            
            ep_r = 0
            buffer_s, buffer_a, buffer_r = [], [], []
            for t in range(EP_LEN):
                if not ROLLING_EVENT.is_set():                  # while global PPO is updating
                    ROLLING_EVENT.wait()                        # wait until PPO is updated
                    buffer_s, buffer_a, buffer_r = [], [], []   # clear history buffer, use new policy to collect data

                a = self.ppo.choose_action(s)
                s_, r, done, _ = self.env.step(a)
                print("Step returns: s_: {}, r: {}, done: {}".format(s_, r, done))
                buffer_r.append((r+8)/8)    # normalize reward, find to be useful
                s = s_
                ep_r += r

                GLOBAL_UPDATE_COUNTER += 1                      # count to minimum batch size, no need to wait other workers
                if t == EP_LEN - 1 or GLOBAL_UPDATE_COUNTER >= MIN_BATCH_SIZE or done:
                    if done:
                        v_s_ = 0                                # end of episode
                        v_s_ = self.ppo.get_v(s_)

                    discounted_r = []                           # compute discounted reward
                    for r in buffer_r[::-1]:
                        v_s_ = r + GAMMA * v_s_

                    bs, ba, br = np.vstack(buffer_s), np.vstack(buffer_a), np.array(discounted_r)[:, None]
                    buffer_s, buffer_a, buffer_r = [], [], []
                    QUEUE.put(np.hstack((bs, ba, br)))          # put data in the queue
                    if GLOBAL_UPDATE_COUNTER >= MIN_BATCH_SIZE:
                        ROLLING_EVENT.clear()       # stop collecting data
                        UPDATE_EVENT.set()          # globalPPO update

                    if GLOBAL_EP >= EP_MAX:         # stop training

                    if done: break

            # record reward changes, plot later
            if len(GLOBAL_RUNNING_R) == 0: GLOBAL_RUNNING_R.append(ep_r)
            else: GLOBAL_RUNNING_R.append(GLOBAL_RUNNING_R[-1]*0.9+ep_r*0.1)
            GLOBAL_EP += 1
            print('{0:.1f}%'.format(GLOBAL_EP/EP_MAX*100), '|W%i' % self.wid,  '|Ep_r: %.2f' % ep_r,)

if __name__ == '__main__':
    UPDATE_EVENT, ROLLING_EVENT = threading.Event(), threading.Event()
    UPDATE_EVENT.clear()            # not update now
    ROLLING_EVENT.set()             # start to roll out
    workers = [Worker(wid=i) for i in range(N_WORKER)]

    COORD = tf.train.Coordinator()
    QUEUE = queue.Queue()           # workers putting data in this queue
    threads = []
    for worker in workers:          # worker threads
        t = threading.Thread(target=worker.work, args=())
        t.start()                   # training
    # add a PPO updating thread

    # plot reward change and test
    plt.plot(np.arange(len(GLOBAL_RUNNING_R)), GLOBAL_RUNNING_R)
    plt.xlabel('Episode'); plt.ylabel('Moving reward'); plt.ion(); plt.show()
    env = gym.make('CartPole-v0')
    while True:
        s = env.reset()
        for t in range(1000):
            s, r, done, info = env.step(GLOBAL_PPO.choose_action(s))
            if done:

closed as off-topic by nbro, Dennis Soemers, DuttaA, malioboro, Philip Raeisghasem Mar 18 at 16:22

This question appears to be off-topic. The users who voted to close gave this specific reason:

  • "This question does not appear to be about artificial intelligence, within the scope defined in the help center." – Dennis Soemers, DuttaA, malioboro, Philip Raeisghasem
If this question can be reworded to fit the rules in the help center, please edit the question.


I found the error: division by zero when calculating the ratio, here.

ratio = pi.prob(self.tfa) / oldpi.prob(self.tfa)

I changed to:

ratio = tf.divide(pi.prob(self.tfa), tf.maximum(oldpi.prob(self.tfa), 1e-5))

Not the answer you're looking for? Browse other questions tagged or ask your own question.