0
$\begingroup$

I have this code:

import gym
import numpy as np
import copy
import math
import random

class Connection:
    def __init__(self, connectedNeuron):
        self.connectedNeuron = connectedNeuron
        self.weight = np.random.normal()
        self.dWeight = 0.0


class Neuron:
    eta = 1
    alpha = 0.01

    def __init__(self, layer):
        self.dendrons = []
        self.error = 0.0
        self.gradient = 0.0
        self.output = 0.0
        if layer is None:
            pass
        else:
            for neuron in layer:
                con = Connection(neuron)
                self.dendrons.append(con)

    def addError(self, err):
        self.error = self.error + err

    def sigmoid(self, x):
        return 1 / (1 + math.exp(-x * 1.0))

    def dSigmoid(self, x):
        return x * (1.0 - x)

    def setError(self, err):
        self.error = err

    def setOutput(self, output):
        self.output = output

    def getOutput(self):
        return self.output

    def feedForword(self):
        sumOutput = 0
        if len(self.dendrons) == 0:
            return
        for dendron in self.dendrons:
            sumOutput = sumOutput + dendron.connectedNeuron.getOutput() * dendron.weight
        self.output = self.sigmoid(sumOutput)

    def backPropagate(self):
        self.gradient = self.error * self.dSigmoid(self.output);
        
        for dendron in self.dendrons:
            dendron.dWeight = Neuron.eta * (
            self.gradient * dendron.connectedNeuron.output) + self.alpha * dendron.dWeight;
            dendron.weight = dendron.weight + dendron.dWeight;
            dendron.connectedNeuron.addError(dendron.weight * self.gradient);
        self.error = 0;


class Network:
    def __init__(self, topology):
        self.layers = []
        for numNeuron in topology:
            layer = []
            for i in range(numNeuron):
                if (len(self.layers) == 0):
                    layer.append(Neuron(None))
                else:
                    layer.append(Neuron(self.layers[-1]))
            layer.append(Neuron(None))
            layer[-1].setOutput(1)
            self.layers.append(layer)

    def setInput(self, inputs):
        for i in range(len(inputs)):
            self.layers[0][i].setOutput(inputs[i])

    def feedForword(self):
        for layer in self.layers[1:]:
            for neuron in layer:
                neuron.feedForword();

    def backPropagate(self, loss):
        for i in range(len(loss)):
            #self.layers[-1][i].setError(self.layers[-1][i].getOutput()- target[i]) was originally this
            self.layers[-1][i].setError(loss[i])
        for layer in self.layers[::-1]:
            for neuron in layer:
                neuron.backPropagate()

    def getError(self, target):
        err = 0
        for i in range(len(target)):
            e = (target[i] - self.layers[-1][i].getOutput())
            err = err + e ** 2
        err = err / len(target)
        err = math.sqrt(err)
        return err

    def getResults(self):
        output = []
        for neuron in self.layers[-1]:
            output.append(neuron.getOutput())
        output.pop()
        return output

    def getThResults(self):
        output = []
        for neuron in self.layers[-1]:
            o = neuron.getOutput()
            print(o)
            if (o > 0.5):
                o = 1
            else:
                o = 0
            output.append(o)
        output.pop()
        return output


policynetwork = Network([4,10,10,2])
targetnetwork = copy.deepcopy(policynetwork)
environment = gym.make("CartPole-v0")
environment.reset()
explorationorexploitation = 0
exploitationtreshold = 0
exploitationrise = 0.00005
oldobservation = [0,0,0,0]
observation = [0,0,0,0]
experiences = []
memorysize = 20000
gamma = 0.99
whentoupdatethenetwork = 10
howmanytimesrange = 20000
samplespertrainingtime = 20

for howmanytimes in range(0, howmanytimesrange):
    exploitationtreshold += exploitationrise
    print(howmanytimes)
    
    exploitationtreshold = exploitationtreshold
    while True:
        policynetwork.setInput(observation)
        policynetwork.feedForword()
        explorationorexploitation = np.random.uniform()
        if explorationorexploitation > exploitationtreshold:
            action = random.randint(0,1)
        else:
            
            action = np.argmax(policynetwork.getResults())
        
        
        

        observation, reward, done, info = environment.step(action)
        
        experiences.append([oldobservation, action, reward, observation])
        #print(experiences[0])
        if len(experiences) > memorysize-1:
            totalloss = 0

            for i in range(samplespertrainingtime):
                sampletobotrained = random.choice(experiences)
                if len(experiences) == 0:
                    print("huutista")
                




                policynetwork.setInput(sampletobotrained[0])
                policynetwork.feedForword()
                qsa = policynetwork.getResults()



                targetnetwork.setInput(sampletobotrained[3])
                targetnetwork.feedForword()
                qprimesa = targetnetwork.getResults()
                loss = []
                for paska in range(0, len(qprimesa)):
                    loss.append(((float(sampletobotrained[2])+(float(gamma)*float(max(qprimesa))))-(qsa[sampletobotrained[1]])))
                    print(((float(sampletobotrained[2])+(float(gamma)*float(max(qprimesa))))-(qsa[sampletobotrained[1]])))
                    totalloss += ((float(sampletobotrained[2])+(float(gamma)*float(max(qprimesa))))-(qsa[sampletobotrained[1]])) #originally was this whole thing squared on both of these
                #print(loss)

            

                policynetwork.backPropagate(loss)
                #print(loss)


            print("Average total loss:" +str(totalloss/samplespertrainingtime))

            experiences.pop(0)

            
            

        
        oldobservation = observation
        #environment.render()
        if done:
            environment.reset()
            break

    if howmanytimes % whentoupdatethenetwork == 0:
        targetnetwork = copy.deepcopy(policynetwork)
    

    howmanytimes = howmanytimes + 1

print("Training complete.")
while True:
    targetnetwork.setInput(observation)
    targetnetwork.feedForword()
    action = np.argmax([policynetwork.layers[-1][0].output, policynetwork.layers[-1][1].output])

    observation,_,done,_ =environment.step(action)
    environment.render()
    if done:
        environment.reset()

the code is supposed to do the classical q learning example of the balancing pole thingy, but this code fails to decrease the loss. What i am doing wrong? I tried nudging all of the values, but nothing seems to help. I am trying to follow this tutorial series: here, the episodes which are conserned with actually training the deep q networks. My guess is that i have understood something wrong in calculating the loss. The neural network class is third party, but i had to modify some bits, for example you can see my comment on the backpropagate() function of the neural network.

And also this somehow kinda works:

import gym
import numpy as np
import copy
import math
import random

class Connection:
    def __init__(self, connectedNeuron):
        self.connectedNeuron = connectedNeuron
        self.weight = np.random.normal()
        self.dWeight = 0.0


class Neuron:
    eta = 0.01
    alpha = 0.01

    def __init__(self, layer):
        self.dendrons = []
        self.error = 0.0
        self.gradient = 0.0
        self.output = 0.0
        if layer is None:
            pass
        else:
            for neuron in layer:
                con = Connection(neuron)
                self.dendrons.append(con)

    def addError(self, err):
        self.error = self.error + err

    def sigmoid(self, x):
        return 1 / (1 + math.exp(-x * 1.0))

    def dSigmoid(self, x):
        return x * (1.0 - x)

    def setError(self, err):
        self.error = err

    def setOutput(self, output):
        self.output = output

    def getOutput(self):
        return self.output

    def feedForword(self):
        sumOutput = 0
        if len(self.dendrons) == 0:
            return
        for dendron in self.dendrons:
            sumOutput = sumOutput + dendron.connectedNeuron.getOutput() * dendron.weight
        self.output = self.sigmoid(sumOutput)

    def backPropagate(self):
        self.gradient = self.error * self.dSigmoid(self.output);
        
        for dendron in self.dendrons:
            dendron.dWeight = Neuron.eta * (
            self.gradient * dendron.connectedNeuron.output) + self.alpha * dendron.dWeight;
            dendron.weight = dendron.weight + dendron.dWeight;
            dendron.connectedNeuron.addError(dendron.weight * self.gradient);
        self.error = 0;


class Network:
    def __init__(self, topology):
        self.layers = []
        for numNeuron in topology:
            layer = []
            for i in range(numNeuron):
                if (len(self.layers) == 0):
                    layer.append(Neuron(None))
                else:
                    layer.append(Neuron(self.layers[-1]))
            layer.append(Neuron(None))
            layer[-1].setOutput(1)
            self.layers.append(layer)

    def setInput(self, inputs):
        for i in range(len(inputs)):
            self.layers[0][i].setOutput(inputs[i])

    def feedForword(self):
        for layer in self.layers[1:]:
            for neuron in layer:
                neuron.feedForword();

    def backPropagate(self, loss):
        for i in range(len(loss)):
            #self.layers[-1][i].setError(self.layers[-1][i].getOutput()- target[i]) was originally this
            self.layers[-1][i].setError(loss[i])
        for layer in self.layers[::-1]:
            for neuron in layer:
                neuron.backPropagate()

    def getError(self, target):
        err = 0
        for i in range(len(target)):
            e = (target[i] - self.layers[-1][i].getOutput())
            err = err + e ** 2
        err = err / len(target)
        err = math.sqrt(err)
        return err

    def getResults(self):
        output = []
        for neuron in self.layers[-1]:
            output.append(neuron.getOutput())
        output.pop()
        return output

    def getThResults(self):
        output = []
        for neuron in self.layers[-1]:
            o = neuron.getOutput()
            print(o)
            if (o > 0.5):
                o = 1
            else:
                o = 0
            output.append(o)
        output.pop()
        return output


policynetwork = Network([4,10,2])
targetnetwork = copy.deepcopy(policynetwork)
environment = gym.make("CartPole-v0")
environment.reset()
explorationorexploitation = 0
exploitationtreshold = 0
exploitationrise = 0.0001
oldobservation = [0,0,0,0]
observation = [0,0,0,0]
experiences = []
memorysize = 2000
gamma = 0.99
whentoupdatethenetwork = 10
howmanytimesrange = 1000
samplespertrainingtime = 20

for howmanytimes in range(0, howmanytimesrange):
    exploitationtreshold += exploitationrise
    print(howmanytimes)
    
    exploitationtreshold = exploitationtreshold
    while True:
        targetnetwork.setInput(observation)
        targetnetwork.feedForword()
        explorationorexploitation = np.random.uniform()
        if explorationorexploitation > exploitationtreshold:
            action = random.randint(0,1)
        else:
            
            action = np.argmax([targetnetwork.layers[-1][0].output, targetnetwork.layers[-1][1].output])
        
        
        

        observation, reward, done, info = environment.step(action)
        
        experiences.append([oldobservation, action, reward, observation])
        #print(experiences[0])
        if len(experiences) > memorysize-1:

            for i in range(samplespertrainingtime):
                sampletobotrained = random.choice(experiences)




                policynetwork.setInput(sampletobotrained[0])
                policynetwork.feedForword()
                qsa = policynetwork.getResults()



                targetnetwork.setInput(sampletobotrained[3])
                targetnetwork.feedForword()
                qprimesa = targetnetwork.getResults()
                loss = []
                for paska in range(0, len(qprimesa)):
                    loss.append(((float(sampletobotrained[2])+(float(gamma)*float(max(qprimesa))))-(qsa[paska]))**2)

            

                policynetwork.backPropagate(loss)
                #print(loss)




            experiences.pop(random.randrange(len(experiences)))

            
            
            

        
        oldobservation = observation
        #environment.render()
        if done:
            environment.reset()
            break

    if howmanytimes % whentoupdatethenetwork == 0:
        targetnetwork = copy.deepcopy(policynetwork)
    

    howmanytimes = howmanytimes + 1

print("Training complete.")
while True:
    targetnetwork.setInput(observation)
    targetnetwork.feedForword()
    action = np.argmax([targetnetwork.layers[-1][0].output, targetnetwork.layers[-1][1].output])

    observation,_,done,_ =environment.step(action)
    environment.render()
    if done:
        environment.reset()

I am lost, so i will cling onto any kind of help that i can get my hands on.

$\endgroup$
1
  • $\begingroup$ Welcome to SE:AI! This has been voted for closure b/c it's a programming question. If you can render it a theory question about the underlying method, that would be suitable. $\endgroup$ – DukeZhou Dec 3 '20 at 1:34

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service, privacy policy and cookie policy

Browse other questions tagged or ask your own question.