Renato Medeiros Renato Medeiros - 17 days ago 6
Python Question

Predicting time-series values with MLP and Tensorflow

I'm having a hard time trying to set up a multilayer perceptron neural network to predict the next value of a time-series using Tensorflow.

I read the time-series from a file, split it into three arrays and use those arrays to train, test and validate the network. Unfortunately, my network answers 0.9999 to every input I give to it.

The image below shows the values I expect my network to outcome, note that they range from 2.8 to 4.2

This is the real values my network is supposed to output. Note that it goes from 2.8 to 4.2

Now, these are the values my network predicts. Though they seem all the same, they're actually 0.9999... (and some difference in the 9th decimal place).

These are the values my network predicts

import csv
import numpy as np

from statsmodels.tsa.tsatools import lagmat
import tensorflow as tf

# Data split (values represent percentage)
perc_train = 0.5
perc_test = 0.4
perc_eval = 0.1

# Parameters
learning_rate = 10 ** -3
min_step_size_train = 10 ** -5
training_epochs = 250
display_step = 1

# Network Parameters
n_input = 15
n_classes = 1
n_hidden = (n_input + n_classes) / 2


def get_nn_sets(pmX, pmY):
'''
Splits data into three subsets
'''
trainningIndex = int(len(pmX) * perc_train)
validationIndex = int(len(pmX) * perc_test) + trainningIndex

pmXFit = pmX[:trainningIndex, :]
pmYFit = pmY[:trainningIndex]

pmXTest = pmX[trainningIndex:validationIndex, :]
pmYTest = pmY[trainningIndex:validationIndex]

pmxEvaluate = pmX[validationIndex:, :]
pmYEvaluate = pmY[validationIndex:]

return pmXFit, pmYFit, pmXTest, pmYTest, pmxEvaluate, pmYEvaluate


def read_dollar_file(clip_first = 4000):
'''
Reads the CSV file containing the dollar value for Brazilian real during the years
-----
RETURNS:
A matrix with the file contents
'''

str_vals = []
with open('dolar.csv', 'rb') as csvfile:
spamreader = csv.reader(csvfile, delimiter=',')
for row in spamreader:
# retrieving the first column of the file (the dollar value)
str_vals.append(row[1])

# removing title
str_vals = str_vals[1:]
# removing the empty strings (sunday and holidays have no values)
y = filter(None, str_vals)
# converting from string to float values
y = np.array(y).astype(np.float)

# checking if initial elements should be discarded
if (clip_first > 0):
y = y[clip_first:]
return y

# Create model
def get_multilayer_perceptron(x):
# Store layers weight & bias

weights = {
'h1': tf.Variable(tf.random_normal([n_input, n_hidden], dtype=tf.float64)),
'out': tf.Variable(tf.random_normal([n_hidden, n_classes], dtype=tf.float64))
}
biases = {
'b1': tf.Variable(tf.random_normal([n_hidden], dtype=tf.float64)),
'out': tf.Variable(tf.random_normal([n_classes], dtype=tf.float64))
}
# Hidden layer with relu activation
layer_1 = tf.add(tf.matmul(x, weights['h1']), biases['b1'])
layer_1 = tf.nn.relu(layer_1)
# Output layer with tanh activation
out_layer = tf.matmul(layer_1, weights['out']) + biases['out']
out_layer = tf.nn.tanh(out_layer)
return out_layer

def run_mlp(inp, outp):

pmXFit, pmYFit, pmXTest, pmYTest, pmXEvaluate, pmYEvaluate = get_nn_sets(inp, outp)

# tf Graph input
x = tf.placeholder("float64", [None, n_input])
y = tf.placeholder("float64", [None, n_classes])

# Construct model
pred = get_multilayer_perceptron(x)

# Define loss and optimizer
cost = tf.nn.l2_loss(tf.sub(pred, y))
optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(cost)

# Initializing the variables
init = tf.initialize_all_variables()

# Launch the graph
with tf.Session() as sess:
sess.run(init)

# Training cycle
last_cost = min_step_size_train + 1
for epoch in range(training_epochs):

# Trainning data
for i in range(len(pmXFit)):
batch_x = np.reshape(pmXFit[i,:], (1, n_input))
batch_y = np.reshape(pmYFit[i], (1, n_classes))

# Run optimization
sess.run(optimizer, feed_dict={x: batch_x, y: batch_y})


# Calculating data error
c = 0.0
for i in range(len(pmXTest)):
batch_x = np.reshape(pmXTest[i,:], (1, n_input))
batch_y = np.reshape(pmYTest[i], (1, n_classes))

# Run Cost function
c += sess.run(cost, feed_dict={x: batch_x, y: batch_y})

c /= len(pmXTest)
# Display logs per epoch step
if epoch % display_step == 0:
print("Epoch:", '%04d' % (epoch+1), "cost=", \
"{:.30f}".format(c))

if abs(c - last_cost) < min_step_size_train:
break
last_cost = c

nn_predictions = np.array([])
for i in range(len(pmXEvaluate)):
batch_x = np.reshape(pmXEvaluate[i,:], (1, n_input))
nn_predictions = np.append(nn_predictions, sess.run(pred, feed_dict={x: batch_x})[0])

print("Optimization Finished!")

nn_predictions.flatten()
return [pmYEvaluate, nn_predictions]

inp = lagmat(read_dollar_file(), n_input, trim='both')
outp = inp[1:, 0]
inp = inp[:-1]

real_value, predicted_value = run_mlp(inp, outp)


I also tried different cost functions and it didn't work. I know I may be missing something really stupid, so I really appreciate your help.

Thanks.

Answer

From your code:

out_layer = tf.nn.tanh(out_layer)

tanh can only output values between (-1.0, 1.0), remove this line will make it do better.

Comments