1
$\begingroup$

I tried the first neural network architecture and the second one, but keeping all other variables constants, I am getting better results with the second architecture. Why are these same neural network architecture giving different results? Or am I making some mistakes?

First one:

def __init__(self, state_size, action_size, seed, hidden_advantage=[512, 512],
             hidden_state_value=[512,512]):
    super(DuelingQNetwork, self).__init__()
    self.seed = torch.manual_seed(seed)
    hidden_layers = [state_size] + hidden_advantage
    self.adv_network = nn.Sequential(nn.Linear(hidden_layers[0], hidden_layers[1]),
                                     nn.ReLU(),
                                     nn.Linear(hidden_layers[1], hidden_layers[2]),
                                     nn.ReLU(),
                                     nn.Linear(hidden_layers[2], action_size))

    hidden_layers = [state_size] + hidden_state_value
    self.val_network = nn.Sequential(nn.Linear(hidden_layers[0], hidden_layers[1]),
                                     nn.ReLU(),
                                     nn.Linear(hidden_layers[1], hidden_layers[2]),
                                     nn.ReLU(),
                                     nn.Linear(hidden_layers[2], 1))                                                           
def forward(self, state):
    """Build a network that maps state -> action values."""
    # Perform a feed-forward pass through the networks
    advantage = self.adv_network(state)
    value = self.val_network(state)
    return advantage.sub_(advantage.mean()).add_(value)

Second one:

def __init__(self, state_size, action_size, seed, hidden_advantage=[512, 512],
             hidden_state_value=[512,512]):
    super(DuelingQNetwork, self).__init__()
    self.seed = torch.manual_seed(seed)

    hidden_layers = [state_size] + hidden_advantage
    advantage_layers = OrderedDict()
    for idx, (hl_in, hl_out) in enumerate(zip(hidden_layers[:-1],hidden_layers[1:])):
        advantage_layers['adv_fc_'+str(idx)] = nn.Linear(hl_in, hl_out)
        advantage_layers['adv_activation_'+str(idx)] = nn.ReLU()

    advantage_layers['adv_output'] = nn.Linear(hidden_layers[-1], action_size)

    self.network_advantage = nn.Sequential(advantage_layers)

    value_layers = OrderedDict()
    hidden_layers = [state_size] + hidden_state_value

    # Iterate over the parameters to create the value network
    for idx, (hl_in, hl_out) in enumerate(zip(hidden_layers[:-1],hidden_layers[1:])):
        # Add a linear layer
        value_layers['val_fc_'+str(idx)] = nn.Linear(hl_in, hl_out)
        # Add an activation function
        value_layers['val_activation_'+str(idx)] = nn.ReLU()

    # Create the output layer for the value network
    value_layers['val_output'] = nn.Linear(hidden_layers[-1], 1)

    # Create the value network
    self.network_value = nn.Sequential(value_layers)

def forward(self, state):
    """Build a network that maps state -> action values."""

    # Perform a feed-forward pass through the networks
    advantage = self.network_advantage(state)
    value = self.network_value(state)
    return advantage.sub_(advantage.mean()).add_(value)
$\endgroup$
  • $\begingroup$ I suspect the problem comes from the weight initialization. I suggest to manually initialize your weights like this: def init_weights(m): if type(m) == nn.Linear: torch.nn.init.xavier_uniform(m.weight) m.bias.data.fill_(0.01) net.apply(init_weights) and then go through the calculations layer by layer. $\endgroup$ – razvanc92 Jul 14 at 10:38

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service, privacy policy and cookie policy

Browse other questions tagged or ask your own question.