0
$\begingroup$

I am trying to implement the DQN algorithm for the task of HVAC control. I have the algorithm implemented using Pytorch. I know that the HVAC simulator is working as it works for other control methods.

When I implement DQN, the algorithm does not converge as it should. I have included a graph showing the convergence with a single hidden layer NN with 5 units (red curve) and another with 2 hidden layers, each with 32 hidden units (blue curve) for one run.

Are there any simple explanations that would explain why the algorithm isn't converging? I am currently using a linearly decreasing epsilon: e = e*0.9997, starting from 0.9 and ending at 0.05. I've also included some of the RL code I am using.

Any help would be appreciated.

Thanks.

Convergence Graphs

```class ReplayMemory(object):

def __init__(self, capacity):
    self.capacity = capacity
    self.memory = []
    self.position = 0

def push(self, *args):
    if len(self.memory) < self.capacity:
        self.memory.append(None)
    self.memory[self.position] = Transition(*args)
    self.position = (self.position + 1) % self.capacity

def sample(self, batch_size):
    return random.sample(self.memory, batch_size)

def __len__(self):
    return len(self.memory)

    def __init__(self):
        super(DQN, self).__init__()
        self.hidden = nn.Linear(4, 32)
        self.hidden1 = nn.Linear(32, 32)
        self.head = nn.Linear(32, 2)

    def forward(self, x):

        x = F.relu(self.hidden(x))
        x = F.relu(self.hidden1(x))
        return self.head(x.view(x.size(0), -1))








```BATCH_SIZE = 128
GAMMA = 0.999
EPS_START = 0.9
EPS_END = 0.05
EPS_DECAY = 200 
TARGET_UPDATE = 10

policy_net = DQN().to(device)
target_net = DQN().to(device)
target_net.load_state_dict(policy_net.state_dict())
target_net.eval()

optimizer = optim.RMSprop(policy_net.parameters())
memory = ReplayMemory(10000)


steps_done = 0
global curEpisode
curEpisode=0


```def select_action(state):
    global steps_done
    sample = random.random()


    global curEpisode
    eps_threshold = EPS_START*(0.9997**curEpisode) 
    if(eps_threshold<EPS_END):
        eps_threshold = EPS_END

    if sample > eps_threshold:
        with torch.no_grad():
            return policy_net(state).max(1)[1].view(1, 1)
    else:
        return torch.tensor([[random.randrange(2)]], device=device, dtype=torch.long)







def optimize_model():

    if len(memory) < BATCH_SIZE:
        return

    transitions = memory.sample(BATCH_SIZE)

    batch = Transition(*zip(*transitions))


    non_final_mask = torch.tensor(tuple(map(lambda s: s is not None,
                                          batch.next_state)), device=device, dtype=torch.uint8)
    non_final_next_states = torch.cat([s for s in batch.next_state
                                                if s is not None])
    state_batch = torch.cat(batch.state)
    action_batch = torch.cat(batch.action)
    reward_batch = torch.cat(batch.reward)


    state_action_values = policy_net(state_batch).gather(1, action_batch)


    next_state_values = torch.zeros(BATCH_SIZE, device=device)
    next_state_values[non_final_mask] = target_net(non_final_next_states).max(1)[0].detach()

    expected_state_action_values = (next_state_values * GAMMA) + reward_batch 


    loss = F.smooth_l1_loss(state_action_values, expected_state_action_values.unsqueeze(1))


    optimizer.zero_grad()
    loss.backward()
    for param in policy_net.parameters():
        param.grad.data.clamp_(-1, 1)
    optimizer.step()



runs = 1



for i_runs in range (runs):
    hvac = HVAC()
    num_episodes = 20000 
    bestReward = -1.0e20
    convergence = list()

    for i_episode in range(num_episodes):
        curEpisode = i_episode
        print('Episode ',i_episode)
        state = hvac.reset()
        total_reward=0
        done=False
        t=0

        while (not done):

            state = torch.FloatTensor([state], device=device)
            action = select_action(state)
            actionList=list()
            if(int(action)==0):
                actionList.append(100) 
                actionList.append(0)
            else:
                actionList.append(0)
                actionList.append(100)

            next_state, reward, done, info = hvac.step(actionList)

            total_reward=total_reward+reward
            reward = torch.FloatTensor([reward], device=device)

            next_state_tensor = torch.FloatTensor([next_state], device=device)

            memory.push(state, action, next_state_tensor, reward)

            state = next_state
            if done:
                next_state = None

            t=t+1
            optimize_model()

        print('total_reward ',total_reward)
        convergence.append(total_reward)



        if(i_episode%5==0):
            print('Episode %s: Best reward %s ' %(str(i_episode),str(bestReward)))


        if i_episode % TARGET_UPDATE == 0:
            target_net.load_state_dict(policy_net.state_dict())


```
$\endgroup$

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service, privacy policy and cookie policy

Browse other questions tagged or ask your own question.