#Import plotter
 
import pylab
import numpy
 
 
 
from pybrain.datasets import ClassificationDataSet
 
from pybrain.supervised.trainers import BackpropTrainer
 
from pybrain.structure.modules import LinearLayer, SigmoidLayer, SoftmaxLayer
 
from pybrain.structure import FullConnection, FeedForwardNetwork, BiasUnit
 
 
 
#Dataset Filename
 
filename = 'irisN.dat'
 
 
 
#Output Data
 
TrainingPoints = []
 
TestPoints = []
 
xAxis = []
 
 
 
#Parameters
 
INPUT = 4     #No of input dimensions
 
OUTPUT = 3    #No of Output Class
 
HIDDEN0 = 3   #No of Hidden Neurons 1st layer
 
HIDDEN1 = 2   #Second Layer
 
LEARNING_RATE = 0.05
 
MOMENTUM = 0.0
 
IMPROVEMENT_EPOCHS = 3  #No of training cycles used
 
WEIGHT_DECAY = 0.0
 
BIAS = True
 
#Data structures
AccuracyData = []
 
 
 
myDataset = ClassificationDataSet(INPUT, OUTPUT)
 
 
 
#Configure input file (text) into input list and output list
 
def readMyData(filename):
 
	global myDataset
 
	file = open(filename, 'r')
 
	for line in file.readlines():
 
		L = line.split(" ")
 
		inSample = []
 
		outSample = []
 
		for i in range(INPUT):
 
			inSample.append(float(L[i]))
 
		for j in range(OUTPUT):
 
			outSample.append(float(L[j+INPUT]))
 
		myDataset.addSample(inSample,outSample)
 
 
 
#current Error Measure - Youcould add your own
 
def SumSquareError(Actual, Desired):
 
	error = 0.
 
	for i in range(len(Desired)):
 
		for j in range(len(Desired[i])):
 
			error = error + ((Actual[i])[j] - (Desired[i])[j])*((Actual[i])[j] - (Desired[i])[j])
 
	return error 
 
def MeanSquareError(Actual, Desired):
	error = 0.
	for i in range(len(Desired)):
		for j in range(len(Desired[i])):
			error = error + ((Actual[i])[j] - (Desired[i])[j])*((Actual[i])[j] - (Desired[i])[j])
			error = error / len(Desired)
	return error
 
 
 
for LEARNING_RATE in range(0, 101, 10):
 
	#Load float format iris dataset
 
	readMyData(filename)
 
 
 
	#create baseline network
 
	network = FeedForwardNetwork()
 
 
 
	# Add a bias if desired
 
	if BIAS:
 
		bias = BiasUnit()
 
		network.addModule(bias)
 
 
 
	#Build Architecture
 
	inLayer = LinearLayer(INPUT)
 
	hiddenLayer0 = SigmoidLayer(HIDDEN0)
 
	hiddenLayer1 = SigmoidLayer(HIDDEN1)
 
	outLayer = SoftmaxLayer(OUTPUT)
 
	network.addInputModule(inLayer)
 
	network.addModule(hiddenLayer0)
 
	network.addModule(hiddenLayer1)
 
	network.addOutputModule(outLayer)
 
 
 
	#Make connections
 
	in_to_hidden = FullConnection(inLayer, hiddenLayer0)
 
	hidden_to_hidden = FullConnection(hiddenLayer0, hiddenLayer1)
 
	hidden_to_out = FullConnection(hiddenLayer1, outLayer)
 
	network.addConnection(in_to_hidden)
 
	network.addConnection(hidden_to_hidden)
 
	network.addConnection(hidden_to_out)
 
 
 
	#initialize
 
	network.sortModules()
 
 
 
	#split the data randomly into 90% training, 10% test
 
	testData, trainData = myDataset.splitWithProportion(0.1)
 
 
 
	#create the trainer environment for backprop and  train network
 
	trainer = BackpropTrainer(network, dataset = trainData, learningrate = LEARNING_RATE/100.0, momentum=MOMENTUM, weightdecay=WEIGHT_DECAY)
 
 
	pylab.close()
	pylab.figure()
 
	epochCounter = 0
	trnresultsum = 0
	tstresultsum = 0
	print "Learning rate: " + str(LEARNING_RATE)
	while epochCounter < IMPROVEMENT_EPOCHS:
 
	#for i in range(TrainingEpochs):
 
		trainer.trainEpochs(1)
 
 
 
		#Print Current Errors (comment out when not needed)
 
		trnresult = SumSquareError(network.activateOnDataset(dataset=trainData), trainData['target'])
 
		tstresult = SumSquareError(network.activateOnDataset(dataset=testData), testData['target'])
 
		#print "epoch: %4d" % trainer.totalepochs, \
 
		#	" train error: %5.2f" % trnresult, \
 
		#	" test error: %5.2f" % tstresult
 
 
		if trainer.totalepochs == 1:
			lowTrnResult = trnresult
			lowTstResult = tstresult
		elif (trnresult < lowTrnResult) or (tstresult < lowTstResult):
			if trnresult < lowTrnResult:
				lowTrnResult = trnresult
			if tstresult < lowTstResult:
				lowTstResult = tstresult
			epochCounter = 0
		else:
			epochCounter += 1
			trnresultsum += trnresult
			tstresultsum += tstresult
			#print epochCounter
 
		#Build Lists for plotting 
 
		TrainingPoints.append(trnresult)
 
		TestPoints.append(tstresult)
 
		xAxis.append(trainer.totalepochs - 1)
 
	#Print results
	AccuracyData.append([lowTstResult, lowTrnResult, trnresultsum / 20, tstresultsum / 20])
	print "It took %4d" % trainer.totalepochs, "epochs to reach train error of %5.2f" % lowTrnResult, "and test error of %5.2f" %lowTstResult
	print "Average train error was " + str(trnresultsum / 20) + " and average test error was " + str(tstresultsum / 20)
 
 
 
	#Compare actual test results (comment out when not needed)
 
	actualTestOutput = network.activateOnDataset(dataset=testData)
 
	desiredTestOutput = testData['target']
 
	#print "Actual vs Desired Test Values"
 
	#for m in range(len(actualTestOutput)):
 
	#	print
 
	#	for n in range(len(actualTestOutput[m])):
 
	#		print "%2.1f" % float(actualTestOutput[m][n]), '\t',
 
	#		print desiredTestOutput[m][n]
 
 
# ATTEMPT ONE
# Create new figure.
#	fig = pylab.figure()
# Create new axes
#	rect = [0.5, 0.5, 0.5, 0.5]
#	ax = fig.add_subplot(111)
#	ax.hold(True)
#	ax.set_anchor('SW')
#	ax.set_position([0, 0, 1, 1])
# Plot data.
#	ax.plot(xAxis, TrainingPoints, 'b-')
#	ax.plot(xAxis, TestPoints, 'r-')
#	ax.set_title('Plot of Iris Training Errors')
#	ax.set_xlabel('Epochs')
#	ax.set_ylabel('Sum Squared Error')
 
# EXAMPLE CODE I WAS PROVIDED WITH
#	pylab.figure()
 
#	pylab.plot(xAxis, TrainingPoints, 'b-')
 
#	pylab.plot(xAxis, TestPoints, 'r-')
 
#	pylab.xlabel('Epochs')
 
#	pylab.ylabel('Sum Squared Error')
 
#	pylab.title('Plot of Iris Training Errors')
 
# LATEST ATTEMPT
# Create new figure.
	fig = pylab.figure()
# Clear figure.
	fig.clf()
# Add new axis.
	ax = fig.add_subplot(1, 1, 1)
	for lines in ax.lines:
		ax.lines.remove(line)
	print ax.lines
# Plot data.
	TrainLine, = ax.plot(xAxis, TrainingPoints, 'b-')
	TestLine, = ax.plot(xAxis, TestPoints, 'r-')
	ax.set_title('Plot of Iris Training Errors')
	ax.set_xlabel('Epochs')
	ax.set_ylabel('Sum Squared Error')
 
# Save image.
	outfilename = str(int(LEARNING_RATE)) + ".png"
	print outfilename
#	pylab.ylim(ymin=0)
	pylab.savefig(outfilename)
	ax.lines.remove(TrainLine)
	ax.lines.remove(TestLine)
#	pylab.cla()
#	pylab.clf()
print AccuracyData