I am trying to teach an agent to make any random 1-qubit state reach uniform superposition. So basically, the full circuit will be State -> measurement -> new_state (|0> if 0, |1> if 1) -> Hadamard gate
. It just needs to perform 2 actions
. That's all. So it's more of an RL problem rather than QC.
I am using reinforcement learning to train the model but it doesn't seem to learn anything. The reward keeps on decreasing and even after 3 million episodes, the agent doesn't seem to converge anywhere. This is how I am training:
def get_exploration_rate(self, time_step):
return self.epsilon_min + (self.epsilon - self.epsilon_min)*\
math.exp(1.*time_step*self.epsilon_decay)
def act(self, data,t): #state
rate = self.get_exploration_rate(t)
if random.random() < rate:
options = self.model.predict(data) #state
options = np.squeeze(options)
action = random.randrange(self.action_size)
else:
options = self.model.predict(data) #state
options = np.squeeze(options)
action = options.argmax()
return action, options, rate
def train(self):
batch_size = 200
t = 0 #increment
states, prob_actions, dlogps, drs, proj_data, reward_data =[], [], [], [], [], []
tr_x, tr_y = [],[]
avg_reward = []
reward_sum = 0
ep_number = 0
prev_state = None
first_step = True
new_state = self.value
data_inp = self.data
while ep_number<3000000:
prev_data = data_inp
prev_state = new_state
states.append(new_state)
action, probs, rate = self.act(data_inp,t)
prob_actions.append(probs)
y = np.zeros([self.action_size])
y[action] = 1
new_state = eval(command[action])
proj = projection(new_state, self.final_state)
data_inp = [proj,action]
data_inp = np.reshape(data_inp,(1,1,len(data_inp)))
tr_x.append(data_inp)
if(t==0):
rw = reward(proj,0)
drs.append(rw)
reward_sum+=rw
elif(t<4):
rw = reward(new_state, self.final_state)
drs.append(rw)
print("present reward: ", rw)
reward_sum+=rw
elif(t==4):
if not np.allclose(new_state, self.final_state):
rw = -1
drs.append(rw)
reward_sum+=rw
else:
rw = 1
drs.append(rw)
reward_sum+=rw
print("reward till now: ",reward_sum)
dlogps.append(np.array(y).astype('float32') * probs)
print("dlogps before time step: ", len(dlogps))
print("time step: ",t)
del(probs, action)
t+=1
if(t==5 or np.allclose(new_state,self.final_state)): #### Done State
ep_number+=1
ep_x = np.vstack(tr_x) #states
ep_dlogp = np.vstack(dlogps)
ep_reward = np.vstack(drs)
disc_rw = discounted_reward(ep_reward,self.gamma)
disc_rw = disc_rw.astype('float32')
disc_rw -= np.mean(disc_rw)
disc_rw /= np.std(disc_rw)
tr_y_len = len(ep_dlogp)
ep_dlogp*=disc_rw
if ep_number % batch_size == 0:
input_tr_y = prob_actions - self.learning_rate * ep_dlogp
input_tr_y = np.reshape(input_tr_y, (tr_y_len,1,6))
self.model.train_on_batch(ep_x, input_tr_y)
tr_x, dlogps, drs, states, prob_actions, reward_data = [],[],[],[],[],[]
env = Environment()
new_state = env.reset()
proj = projection(state, self.final_state)
data_inp = [proj,5]
data_inp = np.reshape(data_inp,(1,1,len(data_inp)))
print("State after resetting: ", new_state)
t=0
I have tried various things like changing the inputs, reward function, even added exploration
rate. I have assigned max time step as 5
even though it should complete in just 2
.
What am I doing wrong? Any suggestions?