基于relu的python神经网络

Question

我想用你的神经网络来加上整数/浮点数，输出值大于1或等于0。我不使用sigmoid函数，而是使用ReLU函数

我用python实现，并使用ReLU方法对神经网络进行numpy，我有这个：

from numpy import exp, array, random, dot

class NeuralNetwork():
    def __init__(self):
        random.seed(1)

        # setting the number of nodes in layer 2 and layer 3
        # more nodes --> more confidence in predictions (?)
        l2 = 5
        l3 = 4

        # assign random weights to matrices in network
        # format is (no. of nodes in previous layer) x (no. of nodes in following layer)
        self.synaptic_weights1 = 2 * random.random((3, l2)) -1
        self.synaptic_weights2 = 2 * random.random((l2, l3)) -1
        self.synaptic_weights3 = 2 * random.random((l3, 1)) -1

    def ReLU(self, x):
        return abs(x) * (x > 0)
        #return self.relu(x, False)
        #return 1/(1+exp(-x))

    # derivative of sigmoid function, indicates confidence about existing weight
    def ReLU_derivative(self, x):
        return 1. * (x > 0)
        #return self.relu(x, True)
        #return x*(1-x)

    # train neural network, adusting synaptic weights each time
    def train(self, training_set_inputs, training_set_outputs, number_of_training_iterations):
        for iteration in range(number_of_training_iterations):

            # pass training set through our neural network
            # a2 means the activations fed to second layer
            a2 = self.ReLU(dot(training_set_inputs, self.synaptic_weights1))
            a3 = self.ReLU(dot(a2, self.synaptic_weights2))
            output = self.ReLU(dot(a3, self.synaptic_weights3))

            # calculate 'error'
            del4 = (training_set_outputs - output)*self.ReLU_derivative(output)

            # find 'errors' in each layer
            del3 = dot(self.synaptic_weights3, del4.T)*(self.ReLU_derivative(a3).T)
            del2 = dot(self.synaptic_weights2, del3)*(self.ReLU_derivative(a2).T)

            # get adjustments (gradients) for each layer
            adjustment3 = dot(a3.T, del4)
            adjustment2 = dot(a2.T, del3.T)
            adjustment1 = dot(training_set_inputs.T, del2.T)

            # adjust weights accordingly
            self.synaptic_weights1 += adjustment1
            self.synaptic_weights2 += adjustment2
            self.synaptic_weights3 += adjustment3

    def forward_pass(self, inputs):
        # pass our inputs through our neural network
        a2 = self.ReLU(dot(inputs, self.synaptic_weights1))
        a3 = self.ReLU(dot(a2, self.synaptic_weights2))
        output = self.ReLU(dot(a3, self.synaptic_weights3)) 
        return output

if __name__ == "__main__":
    # initialise single neuron neural network
    neural_network = NeuralNetwork()

    print ("Random starting synaptic weights (layer 1): ")
    print (neural_network.synaptic_weights1)
    print ("\nRandom starting synaptic weights (layer 2): ")
    print (neural_network.synaptic_weights2)
    print ("\nRandom starting synaptic weights (layer 3): ")
    print (neural_network.synaptic_weights3)

    # the training set.
    training_set_inputs = array([[2,2,1],[1,1,1],[1,2,1],[2,2,2]])
    training_set_outputs = array([[5,3,4,6]]).T

    neural_network.train(training_set_inputs, training_set_outputs, 10000)

    print ("\nNew synaptic weights (layer 1) after training: ")
    print (neural_network.synaptic_weights1)
    print ("\nNew synaptic weights (layer 2) after training: ")
    print (neural_network.synaptic_weights2)
    print ("\nNew synaptic weights (layer 3) after training: ")
    print (neural_network.synaptic_weights3)

    # test with new input
    print ("\nConsidering new situation [1,2,2] -> ?")
    print (neural_network.forward_pass(array([1,2,2])))

问题是我的代码返回0而不是5 为什么？ 谢谢你的帮助