#Policy Iteration solver for FUN
class PolicyEvaluation:
def __init__(self, policies):
self.N = 3
self.pi = policies
self.actions = [0, 1] # a0 and a1
self.discount = 0.7
self.states = [i for i in range(self.N + 1)]
def terminalState(self, state):
return state == 3
# assume p = q = 0.5
def succProbReward(self, state):
# (newState, probability, reward)
spr_list = []
if (state == 0 and self.pi[state] == 0):
spr_list.append([1, 1.0, 1])
elif (state == 0 and self.pi[state] == 1):
spr_list.append([2, 1.0, 0])
elif (state == 1 and self.pi[state] == 0):
spr_list.append([2, 0.5, 0])
spr_list.append([0, 0.5, 0])
elif (state == 2 and self.pi[state] == 0):
spr_list.append([1, 1.0, 0])
elif (state == 2 and self.pi[state] == 1):
spr_list.append([3, 0.5, 10])
spr_list.append([2, 0.5, 0])
return spr_list
def policyEvaluation(mdp):
# initialize
V = {} # state -> Vopt[state]
for state in mdp.states:
V[state] = 0
def V_pi(state):
return sum(prob * (reward + mdp.discount*V[newState]) for prob, reward, newState in
mdp.succProbReward(state))
while True:
# compute new values (newV) given old values (V)
newV = {}
for state in mdp.states:
if mdp.terminalState(state):
newV[state] = 0
else:
newV[state] = V_pi(state)
if max(abs(V[state] - newV[state]) for state in mdp.states) < 1e-10:
break
V = newV
print(V)
print(V)
pE = PolicyEvaluation({0:1, 1:0, 2:0})
print(pE.states)
print(pE.succProbReward(0))
policyIteration(pE)
