I wanted to have a common critic for all my agents so I defined it as following but because the critic will be changing while training in every iteration, I am not sure if all agents still share the same critic network or they will eventually have the different critic in the process? Also, If they will eventually have different critics, how can I implement the critic such that all the agents share the same critic weights even after training every iteration?
import numpy as np import random import copy from collections import namedtuple, deque from model import Actor, Critic import torch import torch.nn.functional as F import torch.optim as optim BUFFER_SIZE = int(1e6) # replay buffer size BATCH_SIZE = 128 # minibatch size LR_ACTOR = 1e-3 # learning rate of the actor LR_CRITIC = 1e-3 # learning rate of the critic WEIGHT_DECAY = 0 # L2 weight decay LEARN_EVERY = 1 # learning timestep interval LEARN_NUM = 5 # number of learning passes GAMMA = 0.99 # discount factor TAU = 8e-3 # for soft update of target parameters OU_SIGMA = 0.2 # Ornstein-Uhlenbeck noise parameter, volatility OU_THETA = 0.15 # Ornstein-Uhlenbeck noise parameter, speed of mean reversion EPS_START = 5.0 # initial value for epsilon in noise decay process in Agent.act() EPS_EP_END = 300 # episode to end the noise decay process EPS_FINAL = 0 # final value for epsilon after decay device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") #Critic Network (w/ Target Network)(every agent share common critic) **critic_local = Critic(24, 2, 0).to(device) critic_target = Critic(24, 2, 0).to(device) critic_optimizer = optim.Adam(critic_local.parameters(), lr=LR_CRITIC, weight_decay=WEIGHT_DECAY)** class Agent(): """Interacts with and learns from the environment.""" def __init__(self, state_size, action_size, num_agents, random_seed): """Initialize an Agent object. Params ====== state_size (int): dimension of each state action_size (int): dimension of each action num_agents (int): number of agents random_seed (int): random seed """ self.state_size = state_size self.action_size = action_size self.num_agents = num_agents self.seed = random.seed(random_seed) self.eps = EPS_START self.eps_decay = 1/(EPS_EP_END*LEARN_NUM) # set decay rate based on epsilon end target self.timestep = 0 # Actor Network (w/ Target Network) self.actor_local = Actor(state_size, action_size, random_seed).to(device) self.actor_target = Actor(state_size, action_size, random_seed).to(device) self.actor_optimizer = optim.Adam(self.actor_local.parameters(), lr=LR_ACTOR) # Noise process self.noise = OUNoise((num_agents, action_size), random_seed) # Replay memory self.memory = ReplayBuffer(action_size, BUFFER_SIZE, BATCH_SIZE, random_seed)