Sudarshan Sunder Sudarshan Sunder - 13 days ago 8
Python Question

XOR classification using multilayer perceptrons is outputting 1 for all inputs

I'm using a neural network with 1 hidden layer (2 neurons) and 1 output neuron for solving the XOR problem.

Here's the code I'm using. It contains the main run file xor.py which creates a model defined in model.py. Each neuron is defined by the class Neuron in neuron.py

xor.py

from model import Model
import numpy as np

inputs = [[0,0], [0,1], [1,0], [1,1]]
outputs = [0, 1, 1, 0]

m = Model()

m.train(inputs, outputs)

for i in inputs:
p = m.predict(i)
print str(i) + ' => ' + str(p)


model.py

from neuron import HiddenNeuron, OutputNeuron
import numpy as np

class Model(object):

def __init__(self):
self.hidden = [HiddenNeuron(2) for i in range(2)]
self.output = OutputNeuron(2)

def predict(self, input):
temp = []
for x in range(2):
self.hidden[x].forward(input)
temp.append(self.hidden[x].out)
self.output.forward(temp)
return self.output.out

def train(self, inputs, targets):
it = 0
i = 0
size = len(inputs)
while it < 4:
if i == size:
i = 0
feature = inputs[i]
print '\n\nFeature : ' + str(feature) + '\n'
print 'Output weights : ' + str(self.output.weights)
print 'Hidden 1 weights : ' + str(self.hidden[0].weights)
print 'Hidden 2 weights : ' + str(self.hidden[1].weights)
temp = []
for x in range(2):
self.hidden[x].forward(feature)
temp.append(self.hidden[x].out)
self.output.forward(temp)
self.output.backward(targets[i])
deltas = []
deltas.append(self.output.error)
weights = []
weights.append([self.output.weights[0]])
weights.append([self.output.weights[1]])
for x in range(2):
self.hidden[x].backward(deltas, weights[x])
for x in range(2):
self.hidden[x].update(feature)
self.output.update(temp)
it += 1
i += 1


neuron.py

import numpy as np
from random import uniform

class Neuron(object):

def activation(self, fx):
return 1/(1 + np.exp(-fx))

def __init__(self, dim, lrate):
self.dim = dim
self.weights = np.empty([dim])
self.weights = [uniform(0,1) for x in range(dim)]
self.bias = uniform(0, 1)
self.lrate = lrate
self.out = None
self.error = None

def update(self, input):
j = 0
for i in input:
delta = self.lrate * self.error
self.weights[j] -= (delta*i)
self.bias += delta
j+=1

def forward(self, input):
j = 0
sum = self.bias
for f in input:
sum += f * self.weights[j]
j+=1
self.out = self.activation(sum)

def backward(self):
pass

class OutputNeuron(Neuron):

def __init__(self, dim, lrate=0.2):
super(OutputNeuron, self).__init__(dim, lrate)

def backward(self, target):
self.error = self.out * (1 - self.out) * (self.out - target)


class HiddenNeuron(Neuron):

def __init__(self, dim, lrate=0.2):
super(HiddenNeuron, self).__init__(dim, lrate)

def backward(self, deltas, weights):
sum = 0
size = len(deltas)
for x in range(size):
sum += deltas[x] * weights[x]
self.error = self.out * (1 - self.out) * sum


The final output is

[0, 0] => 0.999999991272
[0, 1] => 0.999999970788
[1, 0] => 0.999999952345
[1, 1] => 0.999715564446

Answer

I think the error is in neuron.py in the function update(). If you change self.bias += delta to self.bias -= delta it should work, at least it does for me. Otherwise you would modify your biases to ascend towards a maximum on the error surface.

Below you can see the output after 100000 training epochs.

[0, 0] => 0.0174550173543 [0, 1] => 0.983899954593 [1, 0] => 0.983895388655 [1, 1] => 0.0164172288168