Why does the implementation of REINFORCE algorithm minimize the gradient term but not the loss?

Question

I read the book "Foundation of Deep Reinforcement Learning, Laura Graesser and Wah Loon Keng", and when I go through the REINFORCE algorithm, they show the objective function:

$$ J\left(\pi_{\theta}\right)=\mathbb{E}_{\tau \sim \pi_{\theta}}[R(\tau)]=\mathbb{E}_{\tau \sim \pi_{\theta}}\left[\sum_{t=0}^{T} \gamma^{t} r_{t}\right] $$

and the gradient of the objective:

$$ \nabla_{\theta} J\left(\pi_{\theta}\right)=\mathbb{E}_{\tau \sim \pi_{\theta}}\left[\sum_{t=0}^{T} R_{t}(\tau) \nabla_{\theta} \log \pi_{\theta}\left(a_{t} \mid s_{t}\right)\right] $$

But when they implement it,

class Pi(nn.Module):
    def __init__(self, in_dim, out_dim):
        super(Pi, self).__init__()
        layers = [
                nn.Linear(in_dim, 64),
                nn.ReLU(),
                nn.Linear(64, out_dim)
        ]
        self.model = nn.Sequential(*layers)
        self.onpolicy_reset()
        self.train()

    def onpolicy_reset(self):
        self.log_probs = []
        self.rewards = []

    def forward(self, x):
        pdparam = self.model(x)
        return pdparam

    def act(self, state):
        x = torch.from_numpy(state.astype(np.float32))
        pdparam = self.forward(x) # (1, num_action), each number represent the raw logits for that specific action
        # model contain the paremeters theta of the policy, pd is the probability 
        # distribution parameterized by model's theta 
        pd = Categorical(logits = pdparam)
        action = pd.sample()
        log_prob = pd.log_prob(action)
        self.log_probs.append(log_prob)
        return action.item()

def train(pi, optimizer):
    T = len(pi.rewards)
    rets = np.empty(T, dtype = np.float32)
    future_ret = 0.0
    for t in reversed(range(T)):
        future_ret = pi.rewards[t] + gamma*future_ret
        rets[t] = future_ret

    rets = torch.tensor(rets)
    log_probs = torch.stack(pi.log_probs)
    loss = -log_probs*rets
    loss = torch.sum(loss)
    optimizer.zero_grad()
    loss.backward()
    optimizer.step()
    return loss

def main():
    env = gym.make('CartPole-v0')
    # in_dim is the state dimension
    in_dim = env.observation_space.shape[0]
    # out_dim is the action dimension
    out_dim = env.action_space.n
    pi = Pi(in_dim, out_dim)
    optimizer = optim.Adam(pi.parameters(), lr = 0.005)
    for epi in range(300):
        state = env.reset()
        for t in range(200): # max timstep of cartpole is 200
            action = pi.act(state)
            state, reward, done, _ = env.step(action)
            pi.rewards.append(reward)
        # env.render(mode='rgb_array')
            if done:
                break
        loss = train(pi, optimizer)
        total_reward = sum(pi.rewards)
        solved = total_reward > 195.0
        pi.onpolicy_reset()
        print(f'Episode {epi}, loss: {loss}, total reward: {total_reward}, solve: {solved}')
    return pi

In train(), they minimize the gradient term, and I can not understand why is that.

Can someone shed light on that?

I am new to this so please forget me if this question is stupid.

Answer 1

They are not maximizing the gradient, the gradient is of the form \begin{equation} \nabla_{\theta} J \approx \sum_{t=0}^T G_t \nabla_{\theta} \log(\pi_{\theta}(a_t|s_t)) \end{equation} that means that when implementing it in software you can form your objective as \begin{equation} J = \sum_{t=0}^T G_t \log(\pi_{\theta}(a_t|s_t)) \end{equation} and then taking the gradient of that objective is equal to the policy gradient.