9. Recurrent Neural Networks¶
Overview video. See also lecture on Thursday October 22 and examples from week 42.
9.1. Recurrent neural networks: Overarching view¶
Till now our focus has been, including convolutional neural networks as well, on feedforward neural networks. The output or the activations flow only in one direction, from the input layer to the output layer.
A recurrent neural network (RNN) looks very much like a feedforward neural network, except that it also has connections pointing backward.
RNNs are used to analyze time series data such as stock prices, and tell you when to buy or sell. In autonomous driving systems, they can anticipate car trajectories and help avoid accidents. More generally, they can work on sequences of arbitrary lengths, rather than on fixed-sized inputs like all the nets we have discussed so far. For example, they can take sentences, documents, or audio samples as input, making them extremely useful for natural language processing systems such as automatic translation and speech-to-text.
9.2. Set up of an RNN¶
Text to come.
9.3. A simple example¶
%matplotlib inline
# Start importing packages
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import tensorflow as tf
from tensorflow.keras import datasets, layers, models
from tensorflow.keras.layers import Input
from tensorflow.keras.models import Model, Sequential
from tensorflow.keras.layers import Dense, SimpleRNN, LSTM, GRU
from tensorflow.keras import optimizers
from tensorflow.keras import regularizers
from tensorflow.keras.utils import to_categorical
# convert into dataset matrix
def convertToMatrix(data, step):
X, Y =[], []
for i in range(len(data)-step):
d=i+step
X.append(data[i:d,])
Y.append(data[d,])
return np.array(X), np.array(Y)
step = 4
N = 1000
Tp = 800
t=np.arange(0,N)
x=np.sin(0.02*t)+2*np.random.rand(N)
df = pd.DataFrame(x)
df.head()
plt.plot(df)
plt.show()
values=df.values
train,test = values[0:Tp,:], values[Tp:N,:]
# add step elements into train and test
test = np.append(test,np.repeat(test[-1,],step))
train = np.append(train,np.repeat(train[-1,],step))
trainX,trainY =convertToMatrix(train,step)
testX,testY =convertToMatrix(test,step)
trainX = np.reshape(trainX, (trainX.shape[0], 1, trainX.shape[1]))
testX = np.reshape(testX, (testX.shape[0], 1, testX.shape[1]))
model = Sequential()
model.add(SimpleRNN(units=32, input_shape=(1,step), activation="relu"))
model.add(Dense(8, activation="relu"))
model.add(Dense(1))
model.compile(loss='mean_squared_error', optimizer='rmsprop')
model.summary()
model.fit(trainX,trainY, epochs=100, batch_size=16, verbose=2)
trainPredict = model.predict(trainX)
testPredict= model.predict(testX)
predicted=np.concatenate((trainPredict,testPredict),axis=0)
trainScore = model.evaluate(trainX, trainY, verbose=0)
print(trainScore)
index = df.index.values
plt.plot(index,df)
plt.plot(index,predicted)
plt.axvline(df.index[Tp], c="r")
plt.show()
Model: "sequential"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
simple_rnn (SimpleRNN) (None, 32) 1184
_________________________________________________________________
dense (Dense) (None, 8) 264
_________________________________________________________________
dense_1 (Dense) (None, 1) 9
=================================================================
Total params: 1,457
Trainable params: 1,457
Non-trainable params: 0
_________________________________________________________________
Epoch 1/100
50/50 - 0s - loss: 2.1500
Epoch 2/100
50/50 - 0s - loss: 0.4827
Epoch 3/100
50/50 - 0s - loss: 0.4041
Epoch 4/100
50/50 - 0s - loss: 0.3986
Epoch 5/100
50/50 - 0s - loss: 0.3980
Epoch 6/100
50/50 - 0s - loss: 0.3996
Epoch 7/100
50/50 - 0s - loss: 0.3982
Epoch 8/100
50/50 - 0s - loss: 0.3962
Epoch 9/100
50/50 - 0s - loss: 0.3965
Epoch 10/100
50/50 - 0s - loss: 0.3934
Epoch 11/100
50/50 - 0s - loss: 0.3933
Epoch 12/100
50/50 - 0s - loss: 0.3951
Epoch 13/100
50/50 - 0s - loss: 0.3947
Epoch 14/100
50/50 - 0s - loss: 0.3924
Epoch 15/100
50/50 - 0s - loss: 0.3932
Epoch 16/100
50/50 - 0s - loss: 0.3932
Epoch 17/100
50/50 - 0s - loss: 0.3924
Epoch 18/100
50/50 - 0s - loss: 0.3914
Epoch 19/100
50/50 - 0s - loss: 0.3918
Epoch 20/100
50/50 - 0s - loss: 0.3914
Epoch 21/100
50/50 - 0s - loss: 0.3918
Epoch 22/100
50/50 - 0s - loss: 0.3889
Epoch 23/100
50/50 - 0s - loss: 0.3912
Epoch 24/100
50/50 - 0s - loss: 0.3892
Epoch 25/100
50/50 - 0s - loss: 0.3884
Epoch 26/100
50/50 - 0s - loss: 0.3881
Epoch 27/100
50/50 - 0s - loss: 0.3886
Epoch 28/100
50/50 - 0s - loss: 0.3884
Epoch 29/100
50/50 - 0s - loss: 0.3867
Epoch 30/100
50/50 - 0s - loss: 0.3878
Epoch 31/100
50/50 - 0s - loss: 0.3871
Epoch 32/100
50/50 - 0s - loss: 0.3866
Epoch 33/100
50/50 - 0s - loss: 0.3869
Epoch 34/100
50/50 - 0s - loss: 0.3869
Epoch 35/100
50/50 - 0s - loss: 0.3854
Epoch 36/100
50/50 - 0s - loss: 0.3848
Epoch 37/100
50/50 - 0s - loss: 0.3854
Epoch 38/100
50/50 - 0s - loss: 0.3855
Epoch 39/100
50/50 - 0s - loss: 0.3839
Epoch 40/100
50/50 - 0s - loss: 0.3845
Epoch 41/100
50/50 - 0s - loss: 0.3838
Epoch 42/100
50/50 - 0s - loss: 0.3824
Epoch 43/100
50/50 - 0s - loss: 0.3820
Epoch 44/100
50/50 - 0s - loss: 0.3828
Epoch 45/100
50/50 - 0s - loss: 0.3818
Epoch 46/100
50/50 - 0s - loss: 0.3808
Epoch 47/100
50/50 - 0s - loss: 0.3827
Epoch 48/100
50/50 - 0s - loss: 0.3818
Epoch 49/100
50/50 - 0s - loss: 0.3807
Epoch 50/100
50/50 - 0s - loss: 0.3827
Epoch 51/100
50/50 - 0s - loss: 0.3796
Epoch 52/100
50/50 - 0s - loss: 0.3808
Epoch 53/100
50/50 - 0s - loss: 0.3798
Epoch 54/100
50/50 - 0s - loss: 0.3807
Epoch 55/100
50/50 - 0s - loss: 0.3776
Epoch 56/100
50/50 - 0s - loss: 0.3777
Epoch 57/100
50/50 - 0s - loss: 0.3793
Epoch 58/100
50/50 - 0s - loss: 0.3785
Epoch 59/100
50/50 - 0s - loss: 0.3778
Epoch 60/100
50/50 - 0s - loss: 0.3782
Epoch 61/100
50/50 - 0s - loss: 0.3776
Epoch 62/100
50/50 - 0s - loss: 0.3775
Epoch 63/100
50/50 - 0s - loss: 0.3767
Epoch 64/100
50/50 - 0s - loss: 0.3761
Epoch 65/100
50/50 - 0s - loss: 0.3762
Epoch 66/100
50/50 - 0s - loss: 0.3760
Epoch 67/100
50/50 - 0s - loss: 0.3762
Epoch 68/100
50/50 - 0s - loss: 0.3745
Epoch 69/100
50/50 - 0s - loss: 0.3747
Epoch 70/100
50/50 - 0s - loss: 0.3756
Epoch 71/100
50/50 - 0s - loss: 0.3754
Epoch 72/100
50/50 - 0s - loss: 0.3761
Epoch 73/100
50/50 - 0s - loss: 0.3758
Epoch 74/100
50/50 - 0s - loss: 0.3744
Epoch 75/100
50/50 - 0s - loss: 0.3732
Epoch 76/100
50/50 - 0s - loss: 0.3733
Epoch 77/100
50/50 - 0s - loss: 0.3724
Epoch 78/100
50/50 - 0s - loss: 0.3726
Epoch 79/100
50/50 - 0s - loss: 0.3726
Epoch 80/100
50/50 - 0s - loss: 0.3710
Epoch 81/100
50/50 - 0s - loss: 0.3700
Epoch 82/100
50/50 - 0s - loss: 0.3711
Epoch 83/100
50/50 - 0s - loss: 0.3718
Epoch 84/100
50/50 - 0s - loss: 0.3714
Epoch 85/100
50/50 - 0s - loss: 0.3712
Epoch 86/100
50/50 - 0s - loss: 0.3713
Epoch 87/100
50/50 - 0s - loss: 0.3710
Epoch 88/100
50/50 - 0s - loss: 0.3714
Epoch 89/100
50/50 - 0s - loss: 0.3688
Epoch 90/100
50/50 - 0s - loss: 0.3694
Epoch 91/100
50/50 - 0s - loss: 0.3696
Epoch 92/100
50/50 - 0s - loss: 0.3695
Epoch 93/100
50/50 - 0s - loss: 0.3694
Epoch 94/100
50/50 - 0s - loss: 0.3704
Epoch 95/100
50/50 - 0s - loss: 0.3676
Epoch 96/100
50/50 - 0s - loss: 0.3693
Epoch 97/100
50/50 - 0s - loss: 0.3664
Epoch 98/100
50/50 - 0s - loss: 0.3673
Epoch 99/100
50/50 - 0s - loss: 0.3687
Epoch 100/100
50/50 - 0s - loss: 0.3675
0.36208778619766235
9.4. An extrapolation example¶
The following code provides an example of how recurrent neural networks can be used to extrapolate to unknown values of physics data sets. Specifically, the data sets used in this program come from a quantum mechanical many-body calculation of energies as functions of the number of particles.
# For matrices and calculations
import numpy as np
# For machine learning (backend for keras)
import tensorflow as tf
# User-friendly machine learning library
# Front end for TensorFlow
import tensorflow.keras
# Different methods from Keras needed to create an RNN
# This is not necessary but it shortened function calls
# that need to be used in the code.
from tensorflow.keras import datasets, layers, models
from tensorflow.keras.layers import Input
from tensorflow.keras import regularizers
from tensorflow.keras.models import Model, Sequential
from tensorflow.keras.layers import Dense, SimpleRNN, LSTM, GRU
# For timing the code
from timeit import default_timer as timer
# For plotting
import matplotlib.pyplot as plt
# The data set
datatype='VaryDimension'
X_tot = np.arange(2, 42, 2)
y_tot = np.array([-0.03077640549, -0.08336233266, -0.1446729567, -0.2116753732, -0.2830637392, -0.3581341341, -0.436462435, -0.5177783846,
-0.6019067271, -0.6887363571, -0.7782028952, -0.8702784034, -0.9649652536, -1.062292565, -1.16231451,
-1.265109911, -1.370782966, -1.479465113, -1.591317992, -1.70653767])
9.5. Formatting the Data¶
The way the recurrent neural networks are trained in this program differs from how machine learning algorithms are usually trained. Typically a machine learning algorithm is trained by learning the relationship between the x data and the y data. In this program, the recurrent neural network will be trained to recognize the relationship in a sequence of y values. This is type of data formatting is typically used time series forcasting, but it can also be used in any extrapolation (time series forecasting is just a specific type of extrapolation along the time axis). This method of data formatting does not use the x data and assumes that the y data are evenly spaced.
For a standard machine learning algorithm, the training data has the form of (x,y) so the machine learning algorithm learns to assiciate a y value with a given x value. This is useful when the test data has x values within the same range as the training data. However, for this application, the x values of the test data are outside of the x values of the training data and the traditional method of training a machine learning algorithm does not work as well. For this reason, the recurrent neural network is trained on sequences of y values of the form ((y1, y2), y3), so that the network is concerned with learning the pattern of the y data and not the relation between the x and y data. As long as the pattern of y data outside of the training region stays relatively stable compared to what was inside the training region, this method of training can produce accurate extrapolations to y values far removed from the training data set.
# FORMAT_DATA
def format_data(data, length_of_sequence = 2):
"""
Inputs:
data(a numpy array): the data that will be the inputs to the recurrent neural
network
length_of_sequence (an int): the number of elements in one iteration of the
sequence patter. For a function approximator use length_of_sequence = 2.
Returns:
rnn_input (a 3D numpy array): the input data for the recurrent neural network. Its
dimensions are length of data - length of sequence, length of sequence,
dimnsion of data
rnn_output (a numpy array): the training data for the neural network
Formats data to be used in a recurrent neural network.
"""
X, Y = [], []
for i in range(len(data)-length_of_sequence):
# Get the next length_of_sequence elements
a = data[i:i+length_of_sequence]
# Get the element that immediately follows that
b = data[i+length_of_sequence]
# Reshape so that each data point is contained in its own array
a = np.reshape (a, (len(a), 1))
X.append(a)
Y.append(b)
rnn_input = np.array(X)
rnn_output = np.array(Y)
return rnn_input, rnn_output
# ## Defining the Recurrent Neural Network Using Keras
#
# The following method defines a simple recurrent neural network in keras consisting of one input layer, one hidden layer, and one output layer.
def rnn(length_of_sequences, batch_size = None, stateful = False):
"""
Inputs:
length_of_sequences (an int): the number of y values in "x data". This is determined
when the data is formatted
batch_size (an int): Default value is None. See Keras documentation of SimpleRNN.
stateful (a boolean): Default value is False. See Keras documentation of SimpleRNN.
Returns:
model (a Keras model): The recurrent neural network that is built and compiled by this
method
Builds and compiles a recurrent neural network with one hidden layer and returns the model.
"""
# Number of neurons in the input and output layers
in_out_neurons = 1
# Number of neurons in the hidden layer
hidden_neurons = 200
# Define the input layer
inp = Input(batch_shape=(batch_size,
length_of_sequences,
in_out_neurons))
# Define the hidden layer as a simple RNN layer with a set number of neurons and add it to
# the network immediately after the input layer
rnn = SimpleRNN(hidden_neurons,
return_sequences=False,
stateful = stateful,
name="RNN")(inp)
# Define the output layer as a dense neural network layer (standard neural network layer)
#and add it to the network immediately after the hidden layer.
dens = Dense(in_out_neurons,name="dense")(rnn)
# Create the machine learning model starting with the input layer and ending with the
# output layer
model = Model(inputs=[inp],outputs=[dens])
# Compile the machine learning model using the mean squared error function as the loss
# function and an Adams optimizer.
model.compile(loss="mean_squared_error", optimizer="adam")
return model
9.6. Predicting New Points With A Trained Recurrent Neural Network¶
def test_rnn (x1, y_test, plot_min, plot_max):
"""
Inputs:
x1 (a list or numpy array): The complete x component of the data set
y_test (a list or numpy array): The complete y component of the data set
plot_min (an int or float): the smallest x value used in the training data
plot_max (an int or float): the largest x valye used in the training data
Returns:
None.
Uses a trained recurrent neural network model to predict future points in the
series. Computes the MSE of the predicted data set from the true data set, saves
the predicted data set to a csv file, and plots the predicted and true data sets w
while also displaying the data range used for training.
"""
# Add the training data as the first dim points in the predicted data array as these
# are known values.
y_pred = y_test[:dim].tolist()
# Generate the first input to the trained recurrent neural network using the last two
# points of the training data. Based on how the network was trained this means that it
# will predict the first point in the data set after the training data. All of the
# brackets are necessary for Tensorflow.
next_input = np.array([[[y_test[dim-2]], [y_test[dim-1]]]])
# Save the very last point in the training data set. This will be used later.
last = [y_test[dim-1]]
# Iterate until the complete data set is created.
for i in range (dim, len(y_test)):
# Predict the next point in the data set using the previous two points.
next = model.predict(next_input)
# Append just the number of the predicted data set
y_pred.append(next[0][0])
# Create the input that will be used to predict the next data point in the data set.
next_input = np.array([[last, next[0]]], dtype=np.float64)
last = next
# Print the mean squared error between the known data set and the predicted data set.
print('MSE: ', np.square(np.subtract(y_test, y_pred)).mean())
# Save the predicted data set as a csv file for later use
name = datatype + 'Predicted'+str(dim)+'.csv'
np.savetxt(name, y_pred, delimiter=',')
# Plot the known data set and the predicted data set. The red box represents the region that was used
# for the training data.
fig, ax = plt.subplots()
ax.plot(x1, y_test, label="true", linewidth=3)
ax.plot(x1, y_pred, 'g-.',label="predicted", linewidth=4)
ax.legend()
# Created a red region to represent the points used in the training data.
ax.axvspan(plot_min, plot_max, alpha=0.25, color='red')
plt.show()
# Check to make sure the data set is complete
assert len(X_tot) == len(y_tot)
# This is the number of points that will be used in as the training data
dim=12
# Separate the training data from the whole data set
X_train = X_tot[:dim]
y_train = y_tot[:dim]
# Generate the training data for the RNN, using a sequence of 2
rnn_input, rnn_training = format_data(y_train, 2)
# Create a recurrent neural network in Keras and produce a summary of the
# machine learning model
model = rnn(length_of_sequences = rnn_input.shape[1])
model.summary()
# Start the timer. Want to time training+testing
start = timer()
# Fit the model using the training data genenerated above using 150 training iterations and a 5%
# validation split. Setting verbose to True prints information about each training iteration.
hist = model.fit(rnn_input, rnn_training, batch_size=None, epochs=150,
verbose=True,validation_split=0.05)
for label in ["loss","val_loss"]:
plt.plot(hist.history[label],label=label)
plt.ylabel("loss")
plt.xlabel("epoch")
plt.title("The final validation loss: {}".format(hist.history["val_loss"][-1]))
plt.legend()
plt.show()
# Use the trained neural network to predict more points of the data set
test_rnn(X_tot, y_tot, X_tot[0], X_tot[dim-1])
# Stop the timer and calculate the total time needed.
end = timer()
print('Time: ', end-start)
Model: "functional_1"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
input_1 (InputLayer) [(None, 2, 1)] 0
_________________________________________________________________
RNN (SimpleRNN) (None, 200) 40400
_________________________________________________________________
dense (Dense) (None, 1) 201
=================================================================
Total params: 40,601
Trainable params: 40,601
Non-trainable params: 0
_________________________________________________________________
Epoch 1/150
1/1 [==============================] - ETA: 0s - loss: 0.1614
1/1 [==============================] - 0s 168ms/step - loss: 0.1614 - val_loss: 0.1927
Epoch 2/150
1/1 [==============================] - ETA: 0s - loss: 0.0681
1/1 [==============================] - 0s 17ms/step - loss: 0.0681 - val_loss: 0.0356
Epoch 3/150
1/1 [==============================] - ETA: 0s - loss: 0.0154
1/1 [==============================] - 0s 16ms/step - loss: 0.0154 - val_loss: 0.0018
Epoch 4/150
1/1 [==============================] - ETA: 0s - loss: 4.3675e-04
1/1 [==============================] - 0s 17ms/step - loss: 4.3675e-04 - val_loss: 0.0539
Epoch 5/150
1/1 [==============================] - ETA: 0s - loss: 0.0127
1/1 [==============================] - 0s 18ms/step - loss: 0.0127 - val_loss: 0.1238
Epoch 6/150
1/1 [==============================] - ETA: 0s - loss: 0.0318
1/1 [==============================] - 0s 17ms/step - loss: 0.0318 - val_loss: 0.1571
Epoch 7/150
1/1 [==============================] - ETA: 0s - loss: 0.0409
1/1 [==============================] - 0s 17ms/step - loss: 0.0409 - val_loss: 0.1457
Epoch 8/150
1/1 [==============================] - ETA: 0s - loss: 0.0373
1/1 [==============================] - 0s 16ms/step - loss: 0.0373 - val_loss: 0.1070
Epoch 9/150
1/1 [==============================] - ETA: 0s - loss: 0.0263
1/1 [==============================] - 0s 16ms/step - loss: 0.0263 - val_loss: 0.0618
Epoch 10/150
1/1 [==============================] - ETA: 0s - loss: 0.0140
1/1 [==============================] - 0s 16ms/step - loss: 0.0140 - val_loss: 0.0252
Epoch 11/150
1/1 [==============================] - ETA: 0s - loss: 0.0049
1/1 [==============================] - 0s 16ms/step - loss: 0.0049 - val_loss: 0.0047
Epoch 12/150
1/1 [==============================] - ETA: 0s - loss: 8.9601e-04
1/1 [==============================] - 0s 17ms/step - loss: 8.9601e-04 - val_loss: 1.7706e-04
Epoch 13/150
1/1 [==============================] - ETA: 0s - loss: 0.0015
1/1 [==============================] - 0s 16ms/step - loss: 0.0015 - val_loss: 0.0066
Epoch 14/150
1/1 [==============================] - ETA: 0s - loss: 0.0050
1/1 [==============================] - 0s 17ms/step - loss: 0.0050 - val_loss: 0.0169
Epoch 15/150
1/1 [==============================] - ETA: 0s - loss: 0.0092
1/1 [==============================] - 0s 17ms/step - loss: 0.0092 - val_loss: 0.0252
Epoch 16/150
1/1 [==============================] - ETA: 0s - loss: 0.0122
1/1 [==============================] - 0s 17ms/step - loss: 0.0122 - val_loss: 0.0280
Epoch 17/150
1/1 [==============================] - ETA: 0s - loss: 0.0131
1/1 [==============================] - 0s 16ms/step - loss: 0.0131 - val_loss: 0.0249
Epoch 18/150
1/1 [==============================] - ETA: 0s - loss: 0.0119
1/1 [==============================] - 0s 16ms/step - loss: 0.0119 - val_loss: 0.0178
Epoch 19/150
1/1 [==============================] - ETA: 0s - loss: 0.0091
1/1 [==============================] - 0s 17ms/step - loss: 0.0091 - val_loss: 0.0095
Epoch 20/150
1/1 [==============================] - ETA: 0s - loss: 0.0057
1/1 [==============================] - 0s 16ms/step - loss: 0.0057 - val_loss: 0.0029
Epoch 21/150
1/1 [==============================] - ETA: 0s - loss: 0.0028
1/1 [==============================] - 0s 16ms/step - loss: 0.0028 - val_loss: 5.1024e-05
Epoch 22/150
1/1 [==============================] - ETA: 0s - loss: 9.3047e-04
1/1 [==============================] - 0s 16ms/step - loss: 9.3047e-04 - val_loss: 0.0015
Epoch 23/150
1/1 [==============================] - ETA: 0s - loss: 4.2170e-04
1/1 [==============================] - 0s 17ms/step - loss: 4.2170e-04 - val_loss: 0.0064
Epoch 24/150
1/1 [==============================] - ETA: 0s - loss: 0.0011
1/1 [==============================] - 0s 16ms/step - loss: 0.0011 - val_loss: 0.0127
Epoch 25/150
1/1 [==============================] - ETA: 0s - loss: 0.0024
1/1 [==============================] - 0s 16ms/step - loss: 0.0024 - val_loss: 0.0182
Epoch 26/150
1/1 [==============================] - ETA: 0s - loss: 0.0037
1/1 [==============================] - 0s 16ms/step - loss: 0.0037 - val_loss: 0.0210
Epoch 27/150
1/1 [==============================] - ETA: 0s - loss: 0.0044
1/1 [==============================] - 0s 16ms/step - loss: 0.0044 - val_loss: 0.0205
Epoch 28/150
1/1 [==============================] - ETA: 0s - loss: 0.0043
1/1 [==============================] - 0s 16ms/step - loss: 0.0043 - val_loss: 0.0170
Epoch 29/150
1/1 [==============================] - ETA: 0s - loss: 0.0035
1/1 [==============================] - 0s 16ms/step - loss: 0.0035 - val_loss: 0.0118
Epoch 30/150
1/1 [==============================] - ETA: 0s - loss: 0.0024
1/1 [==============================] - 0s 16ms/step - loss: 0.0024 - val_loss: 0.0066
Epoch 31/150
1/1 [==============================] - ETA: 0s - loss: 0.0012
1/1 [==============================] - 0s 16ms/step - loss: 0.0012 - val_loss: 0.0026
Epoch 32/150
1/1 [==============================] - ETA: 0s - loss: 4.1938e-04
1/1 [==============================] - 0s 16ms/step - loss: 4.1938e-04 - val_loss: 4.2108e-04
Epoch 33/150
1/1 [==============================] - ETA: 0s - loss: 1.4696e-04
1/1 [==============================] - 0s 16ms/step - loss: 1.4696e-04 - val_loss: 4.9351e-05
Epoch 34/150
1/1 [==============================] - ETA: 0s - loss: 3.3515e-04
1/1 [==============================] - 0s 17ms/step - loss: 3.3515e-04 - val_loss: 8.7654e-04
Epoch 35/150
1/1 [==============================] - ETA: 0s - loss: 7.8864e-04
1/1 [==============================] - 0s 16ms/step - loss: 7.8864e-04 - val_loss: 0.0021
Epoch 36/150
1/1 [==============================] - ETA: 0s - loss: 0.0013
1/1 [==============================] - 0s 16ms/step - loss: 0.0013 - val_loss: 0.0030
Epoch 37/150
1/1 [==============================] - ETA: 0s - loss: 0.0015
1/1 [==============================] - 0s 16ms/step - loss: 0.0015 - val_loss: 0.0031
Epoch 38/150
1/1 [==============================] - ETA: 0s - loss: 0.0015
1/1 [==============================] - 0s 17ms/step - loss: 0.0015 - val_loss: 0.0026
Epoch 39/150
1/1 [==============================] - ETA: 0s - loss: 0.0013
1/1 [==============================] - 0s 16ms/step - loss: 0.0013 - val_loss: 0.0016
Epoch 40/150
1/1 [==============================] - ETA: 0s - loss: 8.4817e-04
1/1 [==============================] - 0s 16ms/step - loss: 8.4817e-04 - val_loss: 5.9712e-04
Epoch 41/150
1/1 [==============================] - ETA: 0s - loss: 4.1960e-04
1/1 [==============================] - 0s 16ms/step - loss: 4.1960e-04 - val_loss: 5.1834e-05
Epoch 42/150
1/1 [==============================] - ETA: 0s - loss: 1.2261e-04
1/1 [==============================] - 0s 16ms/step - loss: 1.2261e-04 - val_loss: 1.0502e-04
Epoch 43/150
1/1 [==============================] - ETA: 0s - loss: 2.5307e-05
1/1 [==============================] - 0s 16ms/step - loss: 2.5307e-05 - val_loss: 6.7755e-04
Epoch 44/150
1/1 [==============================] - ETA: 0s - loss: 1.1302e-04
1/1 [==============================] - 0s 16ms/step - loss: 1.1302e-04 - val_loss: 0.0015
Epoch 45/150
1/1 [==============================] - ETA: 0s - loss: 3.0417e-04
1/1 [==============================] - 0s 18ms/step - loss: 3.0417e-04 - val_loss: 0.0022
Epoch 46/150
1/1 [==============================] - ETA: 0s - loss: 4.9066e-04
1/1 [==============================] - 0s 16ms/step - loss: 4.9066e-04 - val_loss: 0.0025
Epoch 47/150
1/1 [==============================] - ETA: 0s - loss: 5.8441e-04
1/1 [==============================] - 0s 16ms/step - loss: 5.8441e-04 - val_loss: 0.0023
Epoch 48/150
1/1 [==============================] - ETA: 0s - loss: 5.5018e-04
1/1 [==============================] - 0s 17ms/step - loss: 5.5018e-04 - val_loss: 0.0017
Epoch 49/150
1/1 [==============================] - ETA: 0s - loss: 4.1198e-04
1/1 [==============================] - 0s 16ms/step - loss: 4.1198e-04 - val_loss: 9.9305e-04
Epoch 50/150
1/1 [==============================] - ETA: 0s - loss: 2.3364e-04
1/1 [==============================] - 0s 16ms/step - loss: 2.3364e-04 - val_loss: 3.8335e-04
Epoch 51/150
1/1 [==============================] - ETA: 0s - loss: 8.5796e-05
1/1 [==============================] - 0s 16ms/step - loss: 8.5796e-05 - val_loss: 4.9130e-05
Epoch 52/150
1/1 [==============================] - ETA: 0s - loss: 1.5314e-05
1/1 [==============================] - 0s 16ms/step - loss: 1.5314e-05 - val_loss: 2.2907e-05
Epoch 53/150
1/1 [==============================] - ETA: 0s - loss: 2.9857e-05
1/1 [==============================] - 0s 16ms/step - loss: 2.9857e-05 - val_loss: 2.1308e-04
Epoch 54/150
1/1 [==============================] - ETA: 0s - loss: 1.0133e-04
1/1 [==============================] - 0s 16ms/step - loss: 1.0133e-04 - val_loss: 4.6367e-04
Epoch 55/150
1/1 [==============================] - ETA: 0s - loss: 1.8355e-04
1/1 [==============================] - 0s 16ms/step - loss: 1.8355e-04 - val_loss: 6.2971e-04
Epoch 56/150
1/1 [==============================] - ETA: 0s - loss: 2.3446e-04
1/1 [==============================] - 0s 17ms/step - loss: 2.3446e-04 - val_loss: 6.3475e-04
Epoch 57/150
1/1 [==============================] - ETA: 0s - loss: 2.3285e-04
1/1 [==============================] - 0s 16ms/step - loss: 2.3285e-04 - val_loss: 4.9020e-04
Epoch 58/150
1/1 [==============================] - ETA: 0s - loss: 1.8382e-04
1/1 [==============================] - 0s 17ms/step - loss: 1.8382e-04 - val_loss: 2.7457e-04
Epoch 59/150
1/1 [==============================] - ETA: 0s - loss: 1.1231e-04
1/1 [==============================] - 0s 16ms/step - loss: 1.1231e-04 - val_loss: 8.7585e-05
Epoch 60/150
1/1 [==============================] - ETA: 0s - loss: 4.9274e-05
1/1 [==============================] - 0s 16ms/step - loss: 4.9274e-05 - val_loss: 2.2688e-06
Epoch 61/150
1/1 [==============================] - ETA: 0s - loss: 1.7310e-05
1/1 [==============================] - 0s 16ms/step - loss: 1.7310e-05 - val_loss: 3.5831e-05
Epoch 62/150
1/1 [==============================] - ETA: 0s - loss: 2.1970e-05
1/1 [==============================] - 0s 16ms/step - loss: 2.1970e-05 - val_loss: 1.4969e-04
Epoch 63/150
1/1 [==============================] - ETA: 0s - loss: 5.1827e-05
1/1 [==============================] - 0s 17ms/step - loss: 5.1827e-05 - val_loss: 2.7475e-04
Epoch 64/150
1/1 [==============================] - ETA: 0s - loss: 8.6183e-05
1/1 [==============================] - 0s 16ms/step - loss: 8.6183e-05 - val_loss: 3.4722e-04
Epoch 65/150
1/1 [==============================] - ETA: 0s - loss: 1.0592e-04
1/1 [==============================] - 0s 16ms/step - loss: 1.0592e-04 - val_loss: 3.3682e-04
Epoch 66/150
1/1 [==============================] - ETA: 0s - loss: 1.0207e-04
1/1 [==============================] - 0s 16ms/step - loss: 1.0207e-04 - val_loss: 2.5532e-04
Epoch 67/150
1/1 [==============================] - ETA: 0s - loss: 7.8321e-05
1/1 [==============================] - 0s 17ms/step - loss: 7.8321e-05 - val_loss: 1.4419e-04
Epoch 68/150
1/1 [==============================] - ETA: 0s - loss: 4.7236e-05
1/1 [==============================] - 0s 16ms/step - loss: 4.7236e-05 - val_loss: 5.0422e-05
Epoch 69/150
1/1 [==============================] - ETA: 0s - loss: 2.2810e-05
1/1 [==============================] - 0s 16ms/step - loss: 2.2810e-05 - val_loss: 3.9370e-06
Epoch 70/150
1/1 [==============================] - ETA: 0s - loss: 1.3631e-05
1/1 [==============================] - 0s 19ms/step - loss: 1.3631e-05 - val_loss: 7.0204e-06
Epoch 71/150
1/1 [==============================] - ETA: 0s - loss: 1.9720e-05
1/1 [==============================] - 0s 17ms/step - loss: 1.9720e-05 - val_loss: 3.8779e-05
Epoch 72/150
1/1 [==============================] - ETA: 0s - loss: 3.3982e-05
1/1 [==============================] - 0s 16ms/step - loss: 3.3982e-05 - val_loss: 6.9903e-05
Epoch 73/150
1/1 [==============================] - ETA: 0s - loss: 4.6795e-05
1/1 [==============================] - 0s 19ms/step - loss: 4.6795e-05 - val_loss: 7.8865e-05
Epoch 74/150
1/1 [==============================] - ETA: 0s - loss: 5.1028e-05
1/1 [==============================] - 0s 16ms/step - loss: 5.1028e-05 - val_loss: 6.1482e-05
Epoch 75/150
1/1 [==============================] - ETA: 0s - loss: 4.5036e-05
1/1 [==============================] - 0s 16ms/step - loss: 4.5036e-05 - val_loss: 3.0208e-05
Epoch 76/150
1/1 [==============================] - ETA: 0s - loss: 3.2523e-05
1/1 [==============================] - 0s 16ms/step - loss: 3.2523e-05 - val_loss: 5.0685e-06
Epoch 77/150
1/1 [==============================] - ETA: 0s - loss: 1.9844e-05
1/1 [==============================] - 0s 17ms/step - loss: 1.9844e-05 - val_loss: 1.9610e-06
Epoch 78/150
1/1 [==============================] - ETA: 0s - loss: 1.2452e-05
1/1 [==============================] - 0s 16ms/step - loss: 1.2452e-05 - val_loss: 2.4494e-05
Epoch 79/150
1/1 [==============================] - ETA: 0s - loss: 1.2335e-05
1/1 [==============================] - 0s 16ms/step - loss: 1.2335e-05 - val_loss: 6.3044e-05
Epoch 80/150
1/1 [==============================] - ETA: 0s - loss: 1.7583e-05
1/1 [==============================] - 0s 16ms/step - loss: 1.7583e-05 - val_loss: 1.0059e-04
Epoch 81/150
1/1 [==============================] - ETA: 0s - loss: 2.3980e-05
1/1 [==============================] - 0s 16ms/step - loss: 2.3980e-05 - val_loss: 1.2157e-04
Epoch 82/150
1/1 [==============================] - ETA: 0s - loss: 2.7513e-05
1/1 [==============================] - 0s 16ms/step - loss: 2.7513e-05 - val_loss: 1.1890e-04
Epoch 83/150
1/1 [==============================] - ETA: 0s - loss: 2.6387e-05
1/1 [==============================] - 0s 16ms/step - loss: 2.6387e-05 - val_loss: 9.5984e-05
Epoch 84/150
1/1 [==============================] - ETA: 0s - loss: 2.1564e-05
1/1 [==============================] - 0s 16ms/step - loss: 2.1564e-05 - val_loss: 6.3446e-05
Epoch 85/150
1/1 [==============================] - ETA: 0s - loss: 1.5803e-05
1/1 [==============================] - 0s 17ms/step - loss: 1.5803e-05 - val_loss: 3.3027e-05
Epoch 86/150
1/1 [==============================] - ETA: 0s - loss: 1.1910e-05
1/1 [==============================] - 0s 16ms/step - loss: 1.1910e-05 - val_loss: 1.2295e-05
Epoch 87/150
1/1 [==============================] - ETA: 0s - loss: 1.1264e-05
1/1 [==============================] - 0s 16ms/step - loss: 1.1264e-05 - val_loss: 2.4889e-06
Epoch 88/150
1/1 [==============================] - ETA: 0s - loss: 1.3337e-05
1/1 [==============================] - 0s 16ms/step - loss: 1.3337e-05 - val_loss: 4.8689e-08
Epoch 89/150
1/1 [==============================] - ETA: 0s - loss: 1.6291e-05
1/1 [==============================] - 0s 16ms/step - loss: 1.6291e-05 - val_loss: 1.6905e-07
Epoch 90/150
1/1 [==============================] - ETA: 0s - loss: 1.8174e-05
1/1 [==============================] - 0s 17ms/step - loss: 1.8174e-05 - val_loss: 8.0057e-08
Epoch 91/150
1/1 [==============================] - ETA: 0s - loss: 1.7968e-05
1/1 [==============================] - 0s 19ms/step - loss: 1.7968e-05 - val_loss: 2.7600e-07
Epoch 92/150
1/1 [==============================] - ETA: 0s - loss: 1.5971e-05
1/1 [==============================] - 0s 16ms/step - loss: 1.5971e-05 - val_loss: 3.3847e-06
Epoch 93/150
1/1 [==============================] - ETA: 0s - loss: 1.3408e-05
1/1 [==============================] - 0s 16ms/step - loss: 1.3408e-05 - val_loss: 1.1733e-05
Epoch 94/150
1/1 [==============================] - ETA: 0s - loss: 1.1615e-05
1/1 [==============================] - 0s 16ms/step - loss: 1.1615e-05 - val_loss: 2.5258e-05
Epoch 95/150
1/1 [==============================] - ETA: 0s - loss: 1.1290e-05
1/1 [==============================] - 0s 18ms/step - loss: 1.1290e-05 - val_loss: 4.0986e-05
Epoch 96/150
1/1 [==============================] - ETA: 0s - loss: 1.2221e-05
1/1 [==============================] - 0s 17ms/step - loss: 1.2221e-05 - val_loss: 5.4292e-05
Epoch 97/150
1/1 [==============================] - ETA: 0s - loss: 1.3550e-05
1/1 [==============================] - 0s 16ms/step - loss: 1.3550e-05 - val_loss: 6.1085e-05
Epoch 98/150
1/1 [==============================] - ETA: 0s - loss: 1.4361e-05
1/1 [==============================] - 0s 19ms/step - loss: 1.4361e-05 - val_loss: 5.9625e-05
Epoch 99/150
1/1 [==============================] - ETA: 0s - loss: 1.4195e-05
1/1 [==============================] - 0s 16ms/step - loss: 1.4195e-05 - val_loss: 5.1104e-05
Epoch 100/150
1/1 [==============================] - ETA: 0s - loss: 1.3226e-05
1/1 [==============================] - 0s 16ms/step - loss: 1.3226e-05 - val_loss: 3.8767e-05
Epoch 101/150
1/1 [==============================] - ETA: 0s - loss: 1.2059e-05
1/1 [==============================] - 0s 16ms/step - loss: 1.2059e-05 - val_loss: 2.6280e-05
Epoch 102/150
1/1 [==============================] - ETA: 0s - loss: 1.1312e-05
1/1 [==============================] - 0s 16ms/step - loss: 1.1312e-05 - val_loss: 1.6264e-05
Epoch 103/150
1/1 [==============================] - ETA: 0s - loss: 1.1259e-05
1/1 [==============================] - 0s 16ms/step - loss: 1.1259e-05 - val_loss: 9.6858e-06
Epoch 104/150
1/1 [==============================] - ETA: 0s - loss: 1.1740e-05
1/1 [==============================] - 0s 16ms/step - loss: 1.1740e-05 - val_loss: 6.1976e-06
Epoch 105/150
1/1 [==============================] - ETA: 0s - loss: 1.2320e-05
1/1 [==============================] - 0s 16ms/step - loss: 1.2320e-05 - val_loss: 4.9763e-06
Epoch 106/150
1/1 [==============================] - ETA: 0s - loss: 1.2591e-05
1/1 [==============================] - 0s 17ms/step - loss: 1.2591e-05 - val_loss: 5.4740e-06
Epoch 107/150
1/1 [==============================] - ETA: 0s - loss: 1.2401e-05
1/1 [==============================] - 0s 16ms/step - loss: 1.2401e-05 - val_loss: 7.6209e-06
Epoch 108/150
1/1 [==============================] - ETA: 0s - loss: 1.1895e-05
1/1 [==============================] - 0s 16ms/step - loss: 1.1895e-05 - val_loss: 1.1510e-05
Epoch 109/150
1/1 [==============================] - ETA: 0s - loss: 1.1385e-05
1/1 [==============================] - 0s 16ms/step - loss: 1.1385e-05 - val_loss: 1.6893e-05
Epoch 110/150
1/1 [==============================] - ETA: 0s - loss: 1.1131e-05
1/1 [==============================] - 0s 17ms/step - loss: 1.1131e-05 - val_loss: 2.2888e-05
Epoch 111/150
1/1 [==============================] - ETA: 0s - loss: 1.1203e-05
1/1 [==============================] - 0s 17ms/step - loss: 1.1203e-05 - val_loss: 2.8135e-05
Epoch 112/150
1/1 [==============================] - ETA: 0s - loss: 1.1467e-05
1/1 [==============================] - 0s 16ms/step - loss: 1.1467e-05 - val_loss: 3.1299e-05
Epoch 113/150
1/1 [==============================] - ETA: 0s - loss: 1.1704e-05
1/1 [==============================] - 0s 16ms/step - loss: 1.1704e-05 - val_loss: 3.1629e-05
Epoch 114/150
1/1 [==============================] - ETA: 0s - loss: 1.1751e-05
1/1 [==============================] - 0s 17ms/step - loss: 1.1751e-05 - val_loss: 2.9233e-05
Epoch 115/150
1/1 [==============================] - ETA: 0s - loss: 1.1592e-05
1/1 [==============================] - 0s 16ms/step - loss: 1.1592e-05 - val_loss: 2.4977e-05
Epoch 116/150
1/1 [==============================] - ETA: 0s - loss: 1.1341e-05
1/1 [==============================] - 0s 16ms/step - loss: 1.1341e-05 - val_loss: 2.0057e-05
Epoch 117/150
1/1 [==============================] - ETA: 0s - loss: 1.1148e-05
1/1 [==============================] - 0s 17ms/step - loss: 1.1148e-05 - val_loss: 1.5546e-05
Epoch 118/150
1/1 [==============================] - ETA: 0s - loss: 1.1104e-05
1/1 [==============================] - 0s 16ms/step - loss: 1.1104e-05 - val_loss: 1.2102e-05
Epoch 119/150
1/1 [==============================] - ETA: 0s - loss: 1.1196e-05
1/1 [==============================] - 0s 16ms/step - loss: 1.1196e-05 - val_loss: 9.9551e-06
Epoch 120/150
1/1 [==============================] - ETA: 0s - loss: 1.1329e-05
1/1 [==============================] - 0s 16ms/step - loss: 1.1329e-05 - val_loss: 9.0678e-06
Epoch 121/150
1/1 [==============================] - ETA: 0s - loss: 1.1403e-05
1/1 [==============================] - 0s 16ms/step - loss: 1.1403e-05 - val_loss: 9.3212e-06
Epoch 122/150
1/1 [==============================] - ETA: 0s - loss: 1.1372e-05
1/1 [==============================] - 0s 16ms/step - loss: 1.1372e-05 - val_loss: 1.0588e-05
Epoch 123/150
1/1 [==============================] - ETA: 0s - loss: 1.1265e-05
1/1 [==============================] - 0s 16ms/step - loss: 1.1265e-05 - val_loss: 1.2699e-05
Epoch 124/150
1/1 [==============================] - ETA: 0s - loss: 1.1155e-05
1/1 [==============================] - 0s 17ms/step - loss: 1.1155e-05 - val_loss: 1.5344e-05
Epoch 125/150
1/1 [==============================] - ETA: 0s - loss: 1.1104e-05
1/1 [==============================] - 0s 16ms/step - loss: 1.1104e-05 - val_loss: 1.8049e-05
Epoch 126/150
1/1 [==============================] - ETA: 0s - loss: 1.1126e-05
1/1 [==============================] - 0s 16ms/step - loss: 1.1126e-05 - val_loss: 2.0259e-05
Epoch 127/150
1/1 [==============================] - ETA: 0s - loss: 1.1188e-05
1/1 [==============================] - 0s 16ms/step - loss: 1.1188e-05 - val_loss: 2.1508e-05
Epoch 128/150
1/1 [==============================] - ETA: 0s - loss: 1.1237e-05
1/1 [==============================] - 0s 16ms/step - loss: 1.1237e-05 - val_loss: 2.1579e-05
Epoch 129/150
1/1 [==============================] - ETA: 0s - loss: 1.1239e-05
1/1 [==============================] - 0s 16ms/step - loss: 1.1239e-05 - val_loss: 2.0558e-05
Epoch 130/150
1/1 [==============================] - ETA: 0s - loss: 1.1197e-05
1/1 [==============================] - 0s 16ms/step - loss: 1.1197e-05 - val_loss: 1.8797e-05
Epoch 131/150
1/1 [==============================] - ETA: 0s - loss: 1.1140e-05
1/1 [==============================] - 0s 16ms/step - loss: 1.1140e-05 - val_loss: 1.6756e-05
Epoch 132/150
1/1 [==============================] - ETA: 0s - loss: 1.1104e-05
1/1 [==============================] - 0s 17ms/step - loss: 1.1104e-05 - val_loss: 1.4866e-05
Epoch 133/150
1/1 [==============================] - ETA: 0s - loss: 1.1104e-05
1/1 [==============================] - 0s 16ms/step - loss: 1.1104e-05 - val_loss: 1.3434e-05
Epoch 134/150
1/1 [==============================] - ETA: 0s - loss: 1.1131e-05
1/1 [==============================] - 0s 16ms/step - loss: 1.1131e-05 - val_loss: 1.2624e-05
Epoch 135/150
1/1 [==============================] - ETA: 0s - loss: 1.1157e-05
1/1 [==============================] - 0s 16ms/step - loss: 1.1157e-05 - val_loss: 1.2484e-05
Epoch 136/150
1/1 [==============================] - ETA: 0s - loss: 1.1164e-05
1/1 [==============================] - 0s 17ms/step - loss: 1.1164e-05 - val_loss: 1.2973e-05
Epoch 137/150
1/1 [==============================] - ETA: 0s - loss: 1.1146e-05
1/1 [==============================] - 0s 17ms/step - loss: 1.1146e-05 - val_loss: 1.3983e-05
Epoch 138/150
1/1 [==============================] - ETA: 0s - loss: 1.1119e-05
1/1 [==============================] - 0s 17ms/step - loss: 1.1119e-05 - val_loss: 1.5332e-05
Epoch 139/150
1/1 [==============================] - ETA: 0s - loss: 1.1098e-05
1/1 [==============================] - 0s 16ms/step - loss: 1.1098e-05 - val_loss: 1.6776e-05
Epoch 140/150
1/1 [==============================] - ETA: 0s - loss: 1.1095e-05
1/1 [==============================] - 0s 17ms/step - loss: 1.1095e-05 - val_loss: 1.8043e-05
Epoch 141/150
1/1 [==============================] - ETA: 0s - loss: 1.1106e-05
1/1 [==============================] - 0s 19ms/step - loss: 1.1106e-05 - val_loss: 1.8897e-05
Epoch 142/150
1/1 [==============================] - ETA: 0s - loss: 1.1120e-05
1/1 [==============================] - 0s 19ms/step - loss: 1.1120e-05 - val_loss: 1.9197e-05
Epoch 143/150
1/1 [==============================] - ETA: 0s - loss: 1.1125e-05
1/1 [==============================] - 0s 17ms/step - loss: 1.1125e-05 - val_loss: 1.8938e-05
Epoch 144/150
1/1 [==============================] - ETA: 0s - loss: 1.1118e-05
1/1 [==============================] - 0s 16ms/step - loss: 1.1118e-05 - val_loss: 1.8237e-05
Epoch 145/150
1/1 [==============================] - ETA: 0s - loss: 1.1104e-05
1/1 [==============================] - 0s 18ms/step - loss: 1.1104e-05 - val_loss: 1.7295e-05
Epoch 146/150
1/1 [==============================] - ETA: 0s - loss: 1.1093e-05
1/1 [==============================] - 0s 16ms/step - loss: 1.1093e-05 - val_loss: 1.6333e-05
Epoch 147/150
1/1 [==============================] - ETA: 0s - loss: 1.1091e-05
1/1 [==============================] - 0s 16ms/step - loss: 1.1091e-05 - val_loss: 1.5537e-05
Epoch 148/150
1/1 [==============================] - ETA: 0s - loss: 1.1096e-05
1/1 [==============================] - 0s 17ms/step - loss: 1.1096e-05 - val_loss: 1.5039e-05
Epoch 149/150
1/1 [==============================] - ETA: 0s - loss: 1.1103e-05
1/1 [==============================] - 0s 18ms/step - loss: 1.1103e-05 - val_loss: 1.4898e-05
Epoch 150/150
1/1 [==============================] - ETA: 0s - loss: 1.1105e-05
1/1 [==============================] - 0s 18ms/step - loss: 1.1105e-05 - val_loss: 1.5105e-05
MSE: 9.884935143687282e-05
Time: 4.521348709999998
9.7. Other Things to Try¶
Changing the size of the recurrent neural network and its parameters can drastically change the results you get from the model. The below code takes the simple recurrent neural network from above and adds a second hidden layer, changes the number of neurons in the hidden layer, and explicitly declares the activation function of the hidden layers to be a sigmoid function. The loss function and optimizer can also be changed but are kept the same as the above network. These parameters can be tuned to provide the optimal result from the network. For some ideas on how to improve the performance of a recurrent neural network.
def rnn_2layers(length_of_sequences, batch_size = None, stateful = False):
"""
Inputs:
length_of_sequences (an int): the number of y values in "x data". This is determined
when the data is formatted
batch_size (an int): Default value is None. See Keras documentation of SimpleRNN.
stateful (a boolean): Default value is False. See Keras documentation of SimpleRNN.
Returns:
model (a Keras model): The recurrent neural network that is built and compiled by this
method
Builds and compiles a recurrent neural network with two hidden layers and returns the model.
"""
# Number of neurons in the input and output layers
in_out_neurons = 1
# Number of neurons in the hidden layer, increased from the first network
hidden_neurons = 500
# Define the input layer
inp = Input(batch_shape=(batch_size,
length_of_sequences,
in_out_neurons))
# Create two hidden layers instead of one hidden layer. Explicitly set the activation
# function to be the sigmoid function (the default value is hyperbolic tangent)
rnn1 = SimpleRNN(hidden_neurons,
return_sequences=True, # This needs to be True if another hidden layer is to follow
stateful = stateful, activation = 'sigmoid',
name="RNN1")(inp)
rnn2 = SimpleRNN(hidden_neurons,
return_sequences=False, activation = 'sigmoid',
stateful = stateful,
name="RNN2")(rnn1)
# Define the output layer as a dense neural network layer (standard neural network layer)
#and add it to the network immediately after the hidden layer.
dens = Dense(in_out_neurons,name="dense")(rnn2)
# Create the machine learning model starting with the input layer and ending with the
# output layer
model = Model(inputs=[inp],outputs=[dens])
# Compile the machine learning model using the mean squared error function as the loss
# function and an Adams optimizer.
model.compile(loss="mean_squared_error", optimizer="adam")
return model
# Check to make sure the data set is complete
assert len(X_tot) == len(y_tot)
# This is the number of points that will be used in as the training data
dim=12
# Separate the training data from the whole data set
X_train = X_tot[:dim]
y_train = y_tot[:dim]
# Generate the training data for the RNN, using a sequence of 2
rnn_input, rnn_training = format_data(y_train, 2)
# Create a recurrent neural network in Keras and produce a summary of the
# machine learning model
model = rnn_2layers(length_of_sequences = 2)
model.summary()
# Start the timer. Want to time training+testing
start = timer()
# Fit the model using the training data genenerated above using 150 training iterations and a 5%
# validation split. Setting verbose to True prints information about each training iteration.
hist = model.fit(rnn_input, rnn_training, batch_size=None, epochs=150,
verbose=True,validation_split=0.05)
# This section plots the training loss and the validation loss as a function of training iteration.
# This is not required for analyzing the couple cluster data but can help determine if the network is
# being overtrained.
for label in ["loss","val_loss"]:
plt.plot(hist.history[label],label=label)
plt.ylabel("loss")
plt.xlabel("epoch")
plt.title("The final validation loss: {}".format(hist.history["val_loss"][-1]))
plt.legend()
plt.show()
# Use the trained neural network to predict more points of the data set
test_rnn(X_tot, y_tot, X_tot[0], X_tot[dim-1])
# Stop the timer and calculate the total time needed.
end = timer()
print('Time: ', end-start)
Model: "functional_3"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
input_2 (InputLayer) [(None, 2, 1)] 0
_________________________________________________________________
RNN1 (SimpleRNN) (None, 2, 500) 251000
_________________________________________________________________
RNN2 (SimpleRNN) (None, 500) 500500
_________________________________________________________________
dense (Dense) (None, 1) 501
=================================================================
Total params: 752,001
Trainable params: 752,001
Non-trainable params: 0
_________________________________________________________________
Epoch 1/150
1/1 [==============================] - ETA: 0s - loss: 1.4176
1/1 [==============================] - 0s 227ms/step - loss: 1.4176 - val_loss: 3.6488
Epoch 2/150
1/1 [==============================] - ETA: 0s - loss: 5.4802
1/1 [==============================] - 0s 17ms/step - loss: 5.4802 - val_loss: 0.5738
Epoch 3/150
1/1 [==============================] - ETA: 0s - loss: 1.4330
1/1 [==============================] - 0s 18ms/step - loss: 1.4330 - val_loss: 0.8091
Epoch 4/150
1/1 [==============================] - ETA: 0s - loss: 0.2697
1/1 [==============================] - 0s 17ms/step - loss: 0.2697 - val_loss: 3.5191
Epoch 5/150
1/1 [==============================] - ETA: 0s - loss: 2.1554
1/1 [==============================] - 0s 17ms/step - loss: 2.1554 - val_loss: 3.2451
Epoch 6/150
1/1 [==============================] - ETA: 0s - loss: 1.9444
1/1 [==============================] - 0s 17ms/step - loss: 1.9444 - val_loss: 1.2933
Epoch 7/150
1/1 [==============================] - ETA: 0s - loss: 0.5533
1/1 [==============================] - 0s 18ms/step - loss: 0.5533 - val_loss: 0.1028
Epoch 8/150
1/1 [==============================] - ETA: 0s - loss: 0.0521
1/1 [==============================] - 0s 17ms/step - loss: 0.0521 - val_loss: 0.1031
Epoch 9/150
1/1 [==============================] - ETA: 0s - loss: 0.5942
1/1 [==============================] - 0s 17ms/step - loss: 0.5942 - val_loss: 0.3703
Epoch 10/150
1/1 [==============================] - ETA: 0s - loss: 1.1041
1/1 [==============================] - 0s 17ms/step - loss: 1.1041 - val_loss: 0.2869
Epoch 11/150
1/1 [==============================] - ETA: 0s - loss: 0.9592
1/1 [==============================] - 0s 18ms/step - loss: 0.9592 - val_loss: 0.0406
Epoch 12/150
1/1 [==============================] - ETA: 0s - loss: 0.4307
1/1 [==============================] - 0s 17ms/step - loss: 0.4307 - val_loss: 0.0672
Epoch 13/150
1/1 [==============================] - ETA: 0s - loss: 0.0683
1/1 [==============================] - 0s 17ms/step - loss: 0.0683 - val_loss: 0.5012
Epoch 14/150
1/1 [==============================] - ETA: 0s - loss: 0.1235
1/1 [==============================] - 0s 18ms/step - loss: 0.1235 - val_loss: 1.0556
Epoch 15/150
1/1 [==============================] - ETA: 0s - loss: 0.4083
1/1 [==============================] - 0s 19ms/step - loss: 0.4083 - val_loss: 1.3256
Epoch 16/150
1/1 [==============================] - ETA: 0s - loss: 0.5737
1/1 [==============================] - 0s 17ms/step - loss: 0.5737 - val_loss: 1.1606
Epoch 17/150
1/1 [==============================] - ETA: 0s - loss: 0.4712
1/1 [==============================] - 0s 17ms/step - loss: 0.4712 - val_loss: 0.7278
Epoch 18/150
1/1 [==============================] - ETA: 0s - loss: 0.2276
1/1 [==============================] - 0s 17ms/step - loss: 0.2276 - val_loss: 0.3054
Epoch 19/150
1/1 [==============================] - ETA: 0s - loss: 0.0589
1/1 [==============================] - 0s 18ms/step - loss: 0.0589 - val_loss: 0.0652
Epoch 20/150
1/1 [==============================] - ETA: 0s - loss: 0.0695
1/1 [==============================] - 0s 18ms/step - loss: 0.0695 - val_loss: 8.7287e-04
Epoch 21/150
1/1 [==============================] - ETA: 0s - loss: 0.1958
1/1 [==============================] - 0s 17ms/step - loss: 0.1958 - val_loss: 0.0067
Epoch 22/150
1/1 [==============================] - ETA: 0s - loss: 0.2957
1/1 [==============================] - 0s 17ms/step - loss: 0.2957 - val_loss: 0.0046
Epoch 23/150
1/1 [==============================] - ETA: 0s - loss: 0.2815
1/1 [==============================] - 0s 20ms/step - loss: 0.2815 - val_loss: 0.0030
Epoch 24/150
1/1 [==============================] - ETA: 0s - loss: 0.1768
1/1 [==============================] - 0s 20ms/step - loss: 0.1768 - val_loss: 0.0605
Epoch 25/150
1/1 [==============================] - ETA: 0s - loss: 0.0727
1/1 [==============================] - 0s 17ms/step - loss: 0.0727 - val_loss: 0.2095
Epoch 26/150
1/1 [==============================] - ETA: 0s - loss: 0.0430
1/1 [==============================] - 0s 17ms/step - loss: 0.0430 - val_loss: 0.4103
Epoch 27/150
1/1 [==============================] - ETA: 0s - loss: 0.0896
1/1 [==============================] - 0s 19ms/step - loss: 0.0896 - val_loss: 0.5719
Epoch 28/150
1/1 [==============================] - ETA: 0s - loss: 0.1536
1/1 [==============================] - 0s 18ms/step - loss: 0.1536 - val_loss: 0.6165
Epoch 29/150
1/1 [==============================] - ETA: 0s - loss: 0.1739
1/1 [==============================] - 0s 17ms/step - loss: 0.1739 - val_loss: 0.5322
Epoch 30/150
1/1 [==============================] - ETA: 0s - loss: 0.1365
1/1 [==============================] - 0s 19ms/step - loss: 0.1365 - val_loss: 0.3724
Epoch 31/150
1/1 [==============================] - ETA: 0s - loss: 0.0773
1/1 [==============================] - 0s 18ms/step - loss: 0.0773 - val_loss: 0.2118
Epoch 32/150
1/1 [==============================] - ETA: 0s - loss: 0.0432
1/1 [==============================] - 0s 17ms/step - loss: 0.0432 - val_loss: 0.0999
Epoch 33/150
1/1 [==============================] - ETA: 0s - loss: 0.0528
1/1 [==============================] - 0s 18ms/step - loss: 0.0528 - val_loss: 0.0440
Epoch 34/150
1/1 [==============================] - ETA: 0s - loss: 0.0865
1/1 [==============================] - 0s 19ms/step - loss: 0.0865 - val_loss: 0.0261
Epoch 35/150
1/1 [==============================] - ETA: 0s - loss: 0.1092
1/1 [==============================] - 0s 17ms/step - loss: 0.1092 - val_loss: 0.0315
Epoch 36/150
1/1 [==============================] - ETA: 0s - loss: 0.1012
1/1 [==============================] - 0s 18ms/step - loss: 0.1012 - val_loss: 0.0612
Epoch 37/150
1/1 [==============================] - ETA: 0s - loss: 0.0719
1/1 [==============================] - 0s 19ms/step - loss: 0.0719 - val_loss: 0.1231
Epoch 38/150
1/1 [==============================] - ETA: 0s - loss: 0.0466
1/1 [==============================] - 0s 18ms/step - loss: 0.0466 - val_loss: 0.2126
Epoch 39/150
1/1 [==============================] - ETA: 0s - loss: 0.0432
1/1 [==============================] - 0s 18ms/step - loss: 0.0432 - val_loss: 0.3043
Epoch 40/150
1/1 [==============================] - ETA: 0s - loss: 0.0587
1/1 [==============================] - 0s 18ms/step - loss: 0.0587 - val_loss: 0.3636
Epoch 41/150
1/1 [==============================] - ETA: 0s - loss: 0.0747
1/1 [==============================] - 0s 17ms/step - loss: 0.0747 - val_loss: 0.3676
Epoch 42/150
1/1 [==============================] - ETA: 0s - loss: 0.0759
1/1 [==============================] - 0s 18ms/step - loss: 0.0759 - val_loss: 0.3193
Epoch 43/150
1/1 [==============================] - ETA: 0s - loss: 0.0623
1/1 [==============================] - 0s 18ms/step - loss: 0.0623 - val_loss: 0.2431
Epoch 44/150
1/1 [==============================] - ETA: 0s - loss: 0.0468
1/1 [==============================] - 0s 18ms/step - loss: 0.0468 - val_loss: 0.1684
Epoch 45/150
1/1 [==============================] - ETA: 0s - loss: 0.0417
1/1 [==============================] - 0s 18ms/step - loss: 0.0417 - val_loss: 0.1143
Epoch 46/150
1/1 [==============================] - ETA: 0s - loss: 0.0485
1/1 [==============================] - 0s 18ms/step - loss: 0.0485 - val_loss: 0.0857
Epoch 47/150
1/1 [==============================] - ETA: 0s - loss: 0.0580
1/1 [==============================] - 0s 17ms/step - loss: 0.0580 - val_loss: 0.0803
Epoch 48/150
1/1 [==============================] - ETA: 0s - loss: 0.0605
1/1 [==============================] - 0s 19ms/step - loss: 0.0605 - val_loss: 0.0955
Epoch 49/150
1/1 [==============================] - ETA: 0s - loss: 0.0540
1/1 [==============================] - 0s 19ms/step - loss: 0.0540 - val_loss: 0.1299
Epoch 50/150
1/1 [==============================] - ETA: 0s - loss: 0.0452
1/1 [==============================] - 0s 19ms/step - loss: 0.0452 - val_loss: 0.1782
Epoch 51/150
1/1 [==============================] - ETA: 0s - loss: 0.0415
1/1 [==============================] - 0s 17ms/step - loss: 0.0415 - val_loss: 0.2284
Epoch 52/150
1/1 [==============================] - ETA: 0s - loss: 0.0448
1/1 [==============================] - 0s 17ms/step - loss: 0.0448 - val_loss: 0.2639
Epoch 53/150
1/1 [==============================] - ETA: 0s - loss: 0.0502
1/1 [==============================] - 0s 17ms/step - loss: 0.0502 - val_loss: 0.2725
Epoch 54/150
1/1 [==============================] - ETA: 0s - loss: 0.0518
1/1 [==============================] - 0s 17ms/step - loss: 0.0518 - val_loss: 0.2531
Epoch 55/150
1/1 [==============================] - ETA: 0s - loss: 0.0483
1/1 [==============================] - 0s 18ms/step - loss: 0.0483 - val_loss: 0.2155
Epoch 56/150
1/1 [==============================] - ETA: 0s - loss: 0.0434
1/1 [==============================] - 0s 19ms/step - loss: 0.0434 - val_loss: 0.1742
Epoch 57/150
1/1 [==============================] - ETA: 0s - loss: 0.0415
1/1 [==============================] - 0s 18ms/step - loss: 0.0415 - val_loss: 0.1411
Epoch 58/150
1/1 [==============================] - ETA: 0s - loss: 0.0435
1/1 [==============================] - 0s 18ms/step - loss: 0.0435 - val_loss: 0.1224
Epoch 59/150
1/1 [==============================] - ETA: 0s - loss: 0.0464
1/1 [==============================] - 0s 18ms/step - loss: 0.0464 - val_loss: 0.1194
Epoch 60/150
1/1 [==============================] - ETA: 0s - loss: 0.0470
1/1 [==============================] - 0s 18ms/step - loss: 0.0470 - val_loss: 0.1310
Epoch 61/150
1/1 [==============================] - ETA: 0s - loss: 0.0448
1/1 [==============================] - 0s 17ms/step - loss: 0.0448 - val_loss: 0.1544
Epoch 62/150
1/1 [==============================] - ETA: 0s - loss: 0.0422
1/1 [==============================] - 0s 19ms/step - loss: 0.0422 - val_loss: 0.1836
Epoch 63/150
1/1 [==============================] - ETA: 0s - loss: 0.0415
1/1 [==============================] - 0s 19ms/step - loss: 0.0415 - val_loss: 0.2101
Epoch 64/150
1/1 [==============================] - ETA: 0s - loss: 0.0429
1/1 [==============================] - 0s 17ms/step - loss: 0.0429 - val_loss: 0.2253
Epoch 65/150
1/1 [==============================] - ETA: 0s - loss: 0.0444
1/1 [==============================] - 0s 18ms/step - loss: 0.0444 - val_loss: 0.2246
Epoch 66/150
1/1 [==============================] - ETA: 0s - loss: 0.0443
1/1 [==============================] - 0s 18ms/step - loss: 0.0443 - val_loss: 0.2096
Epoch 67/150
1/1 [==============================] - ETA: 0s - loss: 0.0428
1/1 [==============================] - 0s 17ms/step - loss: 0.0428 - val_loss: 0.1869
Epoch 68/150
1/1 [==============================] - ETA: 0s - loss: 0.0415
1/1 [==============================] - 0s 18ms/step - loss: 0.0415 - val_loss: 0.1645
Epoch 69/150
1/1 [==============================] - ETA: 0s - loss: 0.0415
1/1 [==============================] - 0s 17ms/step - loss: 0.0415 - val_loss: 0.1485
Epoch 70/150
1/1 [==============================] - ETA: 0s - loss: 0.0425
1/1 [==============================] - 0s 20ms/step - loss: 0.0425 - val_loss: 0.1423
Epoch 71/150
1/1 [==============================] - ETA: 0s - loss: 0.0431
1/1 [==============================] - 0s 19ms/step - loss: 0.0431 - val_loss: 0.1461
Epoch 72/150
1/1 [==============================] - ETA: 0s - loss: 0.0427
1/1 [==============================] - 0s 18ms/step - loss: 0.0427 - val_loss: 0.1584
Epoch 73/150
1/1 [==============================] - ETA: 0s - loss: 0.0418
1/1 [==============================] - 0s 19ms/step - loss: 0.0418 - val_loss: 0.1752
Epoch 74/150
1/1 [==============================] - ETA: 0s - loss: 0.0413
1/1 [==============================] - 0s 20ms/step - loss: 0.0413 - val_loss: 0.1915
Epoch 75/150
1/1 [==============================] - ETA: 0s - loss: 0.0416
1/1 [==============================] - 0s 19ms/step - loss: 0.0416 - val_loss: 0.2018
Epoch 76/150
1/1 [==============================] - ETA: 0s - loss: 0.0421
1/1 [==============================] - 0s 18ms/step - loss: 0.0421 - val_loss: 0.2030
Epoch 77/150
1/1 [==============================] - ETA: 0s - loss: 0.0422
1/1 [==============================] - 0s 19ms/step - loss: 0.0422 - val_loss: 0.1955
Epoch 78/150
1/1 [==============================] - ETA: 0s - loss: 0.0418
1/1 [==============================] - 0s 18ms/step - loss: 0.0418 - val_loss: 0.1827
Epoch 79/150
1/1 [==============================] - ETA: 0s - loss: 0.0413
1/1 [==============================] - 0s 21ms/step - loss: 0.0413 - val_loss: 0.1693
Epoch 80/150
1/1 [==============================] - ETA: 0s - loss: 0.0413
1/1 [==============================] - 0s 18ms/step - loss: 0.0413 - val_loss: 0.1595
Epoch 81/150
1/1 [==============================] - ETA: 0s - loss: 0.0416
1/1 [==============================] - 0s 19ms/step - loss: 0.0416 - val_loss: 0.1557
Epoch 82/150
1/1 [==============================] - ETA: 0s - loss: 0.0418
1/1 [==============================] - 0s 17ms/step - loss: 0.0418 - val_loss: 0.1584
Epoch 83/150
1/1 [==============================] - ETA: 0s - loss: 0.0416
1/1 [==============================] - 0s 17ms/step - loss: 0.0416 - val_loss: 0.1662
Epoch 84/150
1/1 [==============================] - ETA: 0s - loss: 0.0413
1/1 [==============================] - 0s 17ms/step - loss: 0.0413 - val_loss: 0.1764
Epoch 85/150
1/1 [==============================] - ETA: 0s - loss: 0.0411
1/1 [==============================] - 0s 17ms/step - loss: 0.0411 - val_loss: 0.1856
Epoch 86/150
1/1 [==============================] - ETA: 0s - loss: 0.0413
1/1 [==============================] - 0s 18ms/step - loss: 0.0413 - val_loss: 0.1906
Epoch 87/150
1/1 [==============================] - ETA: 0s - loss: 0.0414
1/1 [==============================] - 0s 17ms/step - loss: 0.0414 - val_loss: 0.1899
Epoch 88/150
1/1 [==============================] - ETA: 0s - loss: 0.0414
1/1 [==============================] - 0s 17ms/step - loss: 0.0414 - val_loss: 0.1844
Epoch 89/150
1/1 [==============================] - ETA: 0s - loss: 0.0412
1/1 [==============================] - 0s 20ms/step - loss: 0.0412 - val_loss: 0.1764
Epoch 90/150
1/1 [==============================] - ETA: 0s - loss: 0.0411
1/1 [==============================] - 0s 18ms/step - loss: 0.0411 - val_loss: 0.1688
Epoch 91/150
1/1 [==============================] - ETA: 0s - loss: 0.0411
1/1 [==============================] - 0s 18ms/step - loss: 0.0411 - val_loss: 0.1641
Epoch 92/150
1/1 [==============================] - ETA: 0s - loss: 0.0412
1/1 [==============================] - 0s 18ms/step - loss: 0.0412 - val_loss: 0.1634
Epoch 93/150
1/1 [==============================] - ETA: 0s - loss: 0.0412
1/1 [==============================] - 0s 18ms/step - loss: 0.0412 - val_loss: 0.1666
Epoch 94/150
1/1 [==============================] - ETA: 0s - loss: 0.0411
1/1 [==============================] - 0s 18ms/step - loss: 0.0411 - val_loss: 0.1723
Epoch 95/150
1/1 [==============================] - ETA: 0s - loss: 0.0410
1/1 [==============================] - 0s 17ms/step - loss: 0.0410 - val_loss: 0.1783
Epoch 96/150
1/1 [==============================] - ETA: 0s - loss: 0.0410
1/1 [==============================] - 0s 17ms/step - loss: 0.0410 - val_loss: 0.1826
Epoch 97/150
1/1 [==============================] - ETA: 0s - loss: 0.0411
1/1 [==============================] - 0s 18ms/step - loss: 0.0411 - val_loss: 0.1836
Epoch 98/150
1/1 [==============================] - ETA: 0s - loss: 0.0411
1/1 [==============================] - 0s 17ms/step - loss: 0.0411 - val_loss: 0.1814
Epoch 99/150
1/1 [==============================] - ETA: 0s - loss: 0.0410
1/1 [==============================] - 0s 17ms/step - loss: 0.0410 - val_loss: 0.1769
Epoch 100/150
1/1 [==============================] - ETA: 0s - loss: 0.0409
1/1 [==============================] - 0s 19ms/step - loss: 0.0409 - val_loss: 0.1720
Epoch 101/150
1/1 [==============================] - ETA: 0s - loss: 0.0409
1/1 [==============================] - 0s 20ms/step - loss: 0.0409 - val_loss: 0.1684
Epoch 102/150
1/1 [==============================] - ETA: 0s - loss: 0.0409
1/1 [==============================] - 0s 18ms/step - loss: 0.0409 - val_loss: 0.1673
Epoch 103/150
1/1 [==============================] - ETA: 0s - loss: 0.0409
1/1 [==============================] - 0s 18ms/step - loss: 0.0409 - val_loss: 0.1687
Epoch 104/150
1/1 [==============================] - ETA: 0s - loss: 0.0409
1/1 [==============================] - 0s 18ms/step - loss: 0.0409 - val_loss: 0.1719
Epoch 105/150
1/1 [==============================] - ETA: 0s - loss: 0.0409
1/1 [==============================] - 0s 19ms/step - loss: 0.0409 - val_loss: 0.1756
Epoch 106/150
1/1 [==============================] - ETA: 0s - loss: 0.0408
1/1 [==============================] - 0s 18ms/step - loss: 0.0408 - val_loss: 0.1784
Epoch 107/150
1/1 [==============================] - ETA: 0s - loss: 0.0408
1/1 [==============================] - 0s 20ms/step - loss: 0.0408 - val_loss: 0.1793
Epoch 108/150
1/1 [==============================] - ETA: 0s - loss: 0.0408
1/1 [==============================] - 0s 20ms/step - loss: 0.0408 - val_loss: 0.1781
Epoch 109/150
1/1 [==============================] - ETA: 0s - loss: 0.0408
1/1 [==============================] - 0s 19ms/step - loss: 0.0408 - val_loss: 0.1754
Epoch 110/150
1/1 [==============================] - ETA: 0s - loss: 0.0408
1/1 [==============================] - 0s 18ms/step - loss: 0.0408 - val_loss: 0.1723
Epoch 111/150
1/1 [==============================] - ETA: 0s - loss: 0.0408
1/1 [==============================] - 0s 21ms/step - loss: 0.0408 - val_loss: 0.1701
Epoch 112/150
1/1 [==============================] - ETA: 0s - loss: 0.0408
1/1 [==============================] - 0s 18ms/step - loss: 0.0408 - val_loss: 0.1693
Epoch 113/150
1/1 [==============================] - ETA: 0s - loss: 0.0407
1/1 [==============================] - 0s 18ms/step - loss: 0.0407 - val_loss: 0.1702
Epoch 114/150
1/1 [==============================] - ETA: 0s - loss: 0.0407
1/1 [==============================] - 0s 18ms/step - loss: 0.0407 - val_loss: 0.1722
Epoch 115/150
1/1 [==============================] - ETA: 0s - loss: 0.0407
1/1 [==============================] - 0s 18ms/step - loss: 0.0407 - val_loss: 0.1745
Epoch 116/150
1/1 [==============================] - ETA: 0s - loss: 0.0407
1/1 [==============================] - 0s 17ms/step - loss: 0.0407 - val_loss: 0.1760
Epoch 117/150
1/1 [==============================] - ETA: 0s - loss: 0.0407
1/1 [==============================] - 0s 18ms/step - loss: 0.0407 - val_loss: 0.1764
Epoch 118/150
1/1 [==============================] - ETA: 0s - loss: 0.0407
1/1 [==============================] - 0s 17ms/step - loss: 0.0407 - val_loss: 0.1754
Epoch 119/150
1/1 [==============================] - ETA: 0s - loss: 0.0406
1/1 [==============================] - 0s 18ms/step - loss: 0.0406 - val_loss: 0.1736
Epoch 120/150
1/1 [==============================] - ETA: 0s - loss: 0.0406
1/1 [==============================] - 0s 19ms/step - loss: 0.0406 - val_loss: 0.1718
Epoch 121/150
1/1 [==============================] - ETA: 0s - loss: 0.0406
1/1 [==============================] - 0s 17ms/step - loss: 0.0406 - val_loss: 0.1705
Epoch 122/150
1/1 [==============================] - ETA: 0s - loss: 0.0406
1/1 [==============================] - 0s 17ms/step - loss: 0.0406 - val_loss: 0.1703
Epoch 123/150
1/1 [==============================] - ETA: 0s - loss: 0.0406
1/1 [==============================] - 0s 18ms/step - loss: 0.0406 - val_loss: 0.1711
Epoch 124/150
1/1 [==============================] - ETA: 0s - loss: 0.0405
1/1 [==============================] - 0s 18ms/step - loss: 0.0405 - val_loss: 0.1724
Epoch 125/150
1/1 [==============================] - ETA: 0s - loss: 0.0405
1/1 [==============================] - 0s 18ms/step - loss: 0.0405 - val_loss: 0.1737
Epoch 126/150
1/1 [==============================] - ETA: 0s - loss: 0.0405
1/1 [==============================] - 0s 18ms/step - loss: 0.0405 - val_loss: 0.1744
Epoch 127/150
1/1 [==============================] - ETA: 0s - loss: 0.0405
1/1 [==============================] - 0s 17ms/step - loss: 0.0405 - val_loss: 0.1742
Epoch 128/150
1/1 [==============================] - ETA: 0s - loss: 0.0405
1/1 [==============================] - 0s 20ms/step - loss: 0.0405 - val_loss: 0.1734
Epoch 129/150
1/1 [==============================] - ETA: 0s - loss: 0.0405
1/1 [==============================] - 0s 18ms/step - loss: 0.0405 - val_loss: 0.1721
Epoch 130/150
1/1 [==============================] - ETA: 0s - loss: 0.0404
1/1 [==============================] - 0s 17ms/step - loss: 0.0404 - val_loss: 0.1711
Epoch 131/150
1/1 [==============================] - ETA: 0s - loss: 0.0404
1/1 [==============================] - 0s 18ms/step - loss: 0.0404 - val_loss: 0.1706
Epoch 132/150
1/1 [==============================] - ETA: 0s - loss: 0.0404
1/1 [==============================] - 0s 18ms/step - loss: 0.0404 - val_loss: 0.1707
Epoch 133/150
1/1 [==============================] - ETA: 0s - loss: 0.0404
1/1 [==============================] - 0s 19ms/step - loss: 0.0404 - val_loss: 0.1714
Epoch 134/150
1/1 [==============================] - ETA: 0s - loss: 0.0404
1/1 [==============================] - 0s 18ms/step - loss: 0.0404 - val_loss: 0.1722
Epoch 135/150
1/1 [==============================] - ETA: 0s - loss: 0.0403
1/1 [==============================] - 0s 20ms/step - loss: 0.0403 - val_loss: 0.1728
Epoch 136/150
1/1 [==============================] - ETA: 0s - loss: 0.0403
1/1 [==============================] - 0s 17ms/step - loss: 0.0403 - val_loss: 0.1729
Epoch 137/150
1/1 [==============================] - ETA: 0s - loss: 0.0403
1/1 [==============================] - 0s 17ms/step - loss: 0.0403 - val_loss: 0.1725
Epoch 138/150
1/1 [==============================] - ETA: 0s - loss: 0.0403
1/1 [==============================] - 0s 17ms/step - loss: 0.0403 - val_loss: 0.1717
Epoch 139/150
1/1 [==============================] - ETA: 0s - loss: 0.0403
1/1 [==============================] - 0s 18ms/step - loss: 0.0403 - val_loss: 0.1710
Epoch 140/150
1/1 [==============================] - ETA: 0s - loss: 0.0402
1/1 [==============================] - 0s 18ms/step - loss: 0.0402 - val_loss: 0.1705
Epoch 141/150
1/1 [==============================] - ETA: 0s - loss: 0.0402
1/1 [==============================] - 0s 18ms/step - loss: 0.0402 - val_loss: 0.1704
Epoch 142/150
1/1 [==============================] - ETA: 0s - loss: 0.0402
1/1 [==============================] - 0s 18ms/step - loss: 0.0402 - val_loss: 0.1707
Epoch 143/150
1/1 [==============================] - ETA: 0s - loss: 0.0402
1/1 [==============================] - 0s 18ms/step - loss: 0.0402 - val_loss: 0.1712
Epoch 144/150
1/1 [==============================] - ETA: 0s - loss: 0.0402
1/1 [==============================] - 0s 18ms/step - loss: 0.0402 - val_loss: 0.1716
Epoch 145/150
1/1 [==============================] - ETA: 0s - loss: 0.0402
1/1 [==============================] - 0s 18ms/step - loss: 0.0402 - val_loss: 0.1717
Epoch 146/150
1/1 [==============================] - ETA: 0s - loss: 0.0401
1/1 [==============================] - 0s 19ms/step - loss: 0.0401 - val_loss: 0.1715
Epoch 147/150
1/1 [==============================] - ETA: 0s - loss: 0.0401
1/1 [==============================] - 0s 18ms/step - loss: 0.0401 - val_loss: 0.1710
Epoch 148/150
1/1 [==============================] - ETA: 0s - loss: 0.0401
1/1 [==============================] - 0s 18ms/step - loss: 0.0401 - val_loss: 0.1705
Epoch 149/150
1/1 [==============================] - ETA: 0s - loss: 0.0401
1/1 [==============================] - 0s 18ms/step - loss: 0.0401 - val_loss: 0.1701
Epoch 150/150
1/1 [==============================] - ETA: 0s - loss: 0.0401
1/1 [==============================] - 0s 20ms/step - loss: 0.0401 - val_loss: 0.1699
MSE: 0.32920351498520983
Time: 6.260979576
9.8. Other Types of Recurrent Neural Networks¶
Besides a simple recurrent neural network layer, there are two other commonly used types of recurrent neural network layers: Long Short Term Memory (LSTM) and Gated Recurrent Unit (GRU). For a short introduction to these layers see https://medium.com/mindboard/lstm-vs-gru-experimental-comparison-955820c21e8b and https://medium.com/mindboard/lstm-vs-gru-experimental-comparison-955820c21e8b.
The first network created below is similar to the previous network, but it replaces the SimpleRNN layers with LSTM layers. The second network below has two hidden layers made up of GRUs, which are preceeded by two dense (feeddorward) neural network layers. These dense layers “preprocess” the data before it reaches the recurrent layers. This architecture has been shown to improve the performance of recurrent neural networks (see the link above and also https://arxiv.org/pdf/1807.02857.pdf.
def lstm_2layers(length_of_sequences, batch_size = None, stateful = False):
"""
Inputs:
length_of_sequences (an int): the number of y values in "x data". This is determined
when the data is formatted
batch_size (an int): Default value is None. See Keras documentation of SimpleRNN.
stateful (a boolean): Default value is False. See Keras documentation of SimpleRNN.
Returns:
model (a Keras model): The recurrent neural network that is built and compiled by this
method
Builds and compiles a recurrent neural network with two LSTM hidden layers and returns the model.
"""
# Number of neurons on the input/output layer and the number of neurons in the hidden layer
in_out_neurons = 1
hidden_neurons = 250
# Input Layer
inp = Input(batch_shape=(batch_size,
length_of_sequences,
in_out_neurons))
# Hidden layers (in this case they are LSTM layers instead if SimpleRNN layers)
rnn= LSTM(hidden_neurons,
return_sequences=True,
stateful = stateful,
name="RNN", use_bias=True, activation='tanh')(inp)
rnn1 = LSTM(hidden_neurons,
return_sequences=False,
stateful = stateful,
name="RNN1", use_bias=True, activation='tanh')(rnn)
# Output layer
dens = Dense(in_out_neurons,name="dense")(rnn1)
# Define the midel
model = Model(inputs=[inp],outputs=[dens])
# Compile the model
model.compile(loss='mean_squared_error', optimizer='adam')
# Return the model
return model
def dnn2_gru2(length_of_sequences, batch_size = None, stateful = False):
"""
Inputs:
length_of_sequences (an int): the number of y values in "x data". This is determined
when the data is formatted
batch_size (an int): Default value is None. See Keras documentation of SimpleRNN.
stateful (a boolean): Default value is False. See Keras documentation of SimpleRNN.
Returns:
model (a Keras model): The recurrent neural network that is built and compiled by this
method
Builds and compiles a recurrent neural network with four hidden layers (two dense followed by
two GRU layers) and returns the model.
"""
# Number of neurons on the input/output layers and hidden layers
in_out_neurons = 1
hidden_neurons = 250
# Input layer
inp = Input(batch_shape=(batch_size,
length_of_sequences,
in_out_neurons))
# Hidden Dense (feedforward) layers
dnn = Dense(hidden_neurons/2, activation='relu', name='dnn')(inp)
dnn1 = Dense(hidden_neurons/2, activation='relu', name='dnn1')(dnn)
# Hidden GRU layers
rnn1 = GRU(hidden_neurons,
return_sequences=True,
stateful = stateful,
name="RNN1", use_bias=True)(dnn1)
rnn = GRU(hidden_neurons,
return_sequences=False,
stateful = stateful,
name="RNN", use_bias=True)(rnn1)
# Output layer
dens = Dense(in_out_neurons,name="dense")(rnn)
# Define the model
model = Model(inputs=[inp],outputs=[dens])
# Compile the mdoel
model.compile(loss='mean_squared_error', optimizer='adam')
# Return the model
return model
# Check to make sure the data set is complete
assert len(X_tot) == len(y_tot)
# This is the number of points that will be used in as the training data
dim=12
# Separate the training data from the whole data set
X_train = X_tot[:dim]
y_train = y_tot[:dim]
# Generate the training data for the RNN, using a sequence of 2
rnn_input, rnn_training = format_data(y_train, 2)
# Create a recurrent neural network in Keras and produce a summary of the
# machine learning model
# Change the method name to reflect which network you want to use
model = dnn2_gru2(length_of_sequences = 2)
model.summary()
# Start the timer. Want to time training+testing
start = timer()
# Fit the model using the training data genenerated above using 150 training iterations and a 5%
# validation split. Setting verbose to True prints information about each training iteration.
hist = model.fit(rnn_input, rnn_training, batch_size=None, epochs=150,
verbose=True,validation_split=0.05)
# This section plots the training loss and the validation loss as a function of training iteration.
# This is not required for analyzing the couple cluster data but can help determine if the network is
# being overtrained.
for label in ["loss","val_loss"]:
plt.plot(hist.history[label],label=label)
plt.ylabel("loss")
plt.xlabel("epoch")
plt.title("The final validation loss: {}".format(hist.history["val_loss"][-1]))
plt.legend()
plt.show()
# Use the trained neural network to predict more points of the data set
test_rnn(X_tot, y_tot, X_tot[0], X_tot[dim-1])
# Stop the timer and calculate the total time needed.
end = timer()
print('Time: ', end-start)
# ### Training Recurrent Neural Networks in the Standard Way (i.e. learning the relationship between the X and Y data)
#
# Finally, comparing the performace of a recurrent neural network using the standard data formatting to the performance of the network with time sequence data formatting shows the benefit of this type of data formatting with extrapolation.
# Check to make sure the data set is complete
assert len(X_tot) == len(y_tot)
# This is the number of points that will be used in as the training data
dim=12
# Separate the training data from the whole data set
X_train = X_tot[:dim]
y_train = y_tot[:dim]
# Reshape the data for Keras specifications
X_train = X_train.reshape((dim, 1))
y_train = y_train.reshape((dim, 1))
# Create a recurrent neural network in Keras and produce a summary of the
# machine learning model
# Set the sequence length to 1 for regular data formatting
model = rnn(length_of_sequences = 1)
model.summary()
# Start the timer. Want to time training+testing
start = timer()
# Fit the model using the training data genenerated above using 150 training iterations and a 5%
# validation split. Setting verbose to True prints information about each training iteration.
hist = model.fit(X_train, y_train, batch_size=None, epochs=150,
verbose=True,validation_split=0.05)
# This section plots the training loss and the validation loss as a function of training iteration.
# This is not required for analyzing the couple cluster data but can help determine if the network is
# being overtrained.
for label in ["loss","val_loss"]:
plt.plot(hist.history[label],label=label)
plt.ylabel("loss")
plt.xlabel("epoch")
plt.title("The final validation loss: {}".format(hist.history["val_loss"][-1]))
plt.legend()
plt.show()
# Use the trained neural network to predict the remaining data points
X_pred = X_tot[dim:]
X_pred = X_pred.reshape((len(X_pred), 1))
y_model = model.predict(X_pred)
y_pred = np.concatenate((y_tot[:dim], y_model.flatten()))
# Plot the known data set and the predicted data set. The red box represents the region that was used
# for the training data.
fig, ax = plt.subplots()
ax.plot(X_tot, y_tot, label="true", linewidth=3)
ax.plot(X_tot, y_pred, 'g-.',label="predicted", linewidth=4)
ax.legend()
# Created a red region to represent the points used in the training data.
ax.axvspan(X_tot[0], X_tot[dim], alpha=0.25, color='red')
plt.show()
# Stop the timer and calculate the total time needed.
end = timer()
print('Time: ', end-start)
Model: "functional_5"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
input_3 (InputLayer) [(None, 2, 1)] 0
_________________________________________________________________
dnn (Dense) (None, 2, 125) 250
_________________________________________________________________
dnn1 (Dense) (None, 2, 125) 15750
_________________________________________________________________
RNN1 (GRU) (None, 2, 250) 282750
_________________________________________________________________
RNN (GRU) (None, 250) 376500
_________________________________________________________________
dense (Dense) (None, 1) 251
=================================================================
Total params: 675,501
Trainable params: 675,501
Non-trainable params: 0
_________________________________________________________________
Epoch 1/150
1/1 [==============================] - ETA: 0s - loss: 0.2339
1/1 [==============================] - 1s 653ms/step - loss: 0.2339 - val_loss: 0.5383
Epoch 2/150
1/1 [==============================] - ETA: 0s - loss: 0.1652
1/1 [==============================] - 0s 19ms/step - loss: 0.1652 - val_loss: 0.3519
Epoch 3/150
1/1 [==============================] - ETA: 0s - loss: 0.1008
1/1 [==============================] - 0s 19ms/step - loss: 0.1008 - val_loss: 0.1726
Epoch 4/150
1/1 [==============================] - ETA: 0s - loss: 0.0430
1/1 [==============================] - 0s 18ms/step - loss: 0.0430 - val_loss: 0.0348
Epoch 5/150
1/1 [==============================] - ETA: 0s - loss: 0.0070
1/1 [==============================] - 0s 20ms/step - loss: 0.0070 - val_loss: 0.0070
Epoch 6/150
1/1 [==============================] - ETA: 0s - loss: 0.0194
1/1 [==============================] - 0s 18ms/step - loss: 0.0194 - val_loss: 0.0534
Epoch 7/150
1/1 [==============================] - ETA: 0s - loss: 0.0503
1/1 [==============================] - 0s 18ms/step - loss: 0.0503 - val_loss: 0.0415
Epoch 8/150
1/1 [==============================] - ETA: 0s - loss: 0.0430
1/1 [==============================] - 0s 18ms/step - loss: 0.0430 - val_loss: 0.0088
Epoch 9/150
1/1 [==============================] - ETA: 0s - loss: 0.0204
1/1 [==============================] - 0s 18ms/step - loss: 0.0204 - val_loss: 0.0014
Epoch 10/150
1/1 [==============================] - ETA: 0s - loss: 0.0061
1/1 [==============================] - 0s 18ms/step - loss: 0.0061 - val_loss: 0.0239
Epoch 11/150
1/1 [==============================] - ETA: 0s - loss: 0.0052
1/1 [==============================] - 0s 19ms/step - loss: 0.0052 - val_loss: 0.0584
Epoch 12/150
1/1 [==============================] - ETA: 0s - loss: 0.0120
1/1 [==============================] - 0s 19ms/step - loss: 0.0120 - val_loss: 0.0860
Epoch 13/150
1/1 [==============================] - ETA: 0s - loss: 0.0191
1/1 [==============================] - 0s 18ms/step - loss: 0.0191 - val_loss: 0.0977
Epoch 14/150
1/1 [==============================] - ETA: 0s - loss: 0.0225
1/1 [==============================] - 0s 18ms/step - loss: 0.0225 - val_loss: 0.0928
Epoch 15/150
1/1 [==============================] - ETA: 0s - loss: 0.0213
1/1 [==============================] - 0s 18ms/step - loss: 0.0213 - val_loss: 0.0754
Epoch 16/150
1/1 [==============================] - ETA: 0s - loss: 0.0166
1/1 [==============================] - 0s 18ms/step - loss: 0.0166 - val_loss: 0.0516
Epoch 17/150
1/1 [==============================] - ETA: 0s - loss: 0.0106
1/1 [==============================] - 0s 19ms/step - loss: 0.0106 - val_loss: 0.0278
Epoch 18/150
1/1 [==============================] - ETA: 0s - loss: 0.0056
1/1 [==============================] - 0s 18ms/step - loss: 0.0056 - val_loss: 0.0099
Epoch 19/150
1/1 [==============================] - ETA: 0s - loss: 0.0032
1/1 [==============================] - 0s 18ms/step - loss: 0.0032 - val_loss: 0.0011
Epoch 20/150
1/1 [==============================] - ETA: 0s - loss: 0.0042
1/1 [==============================] - 0s 17ms/step - loss: 0.0042 - val_loss: 4.2492e-04
Epoch 21/150
1/1 [==============================] - ETA: 0s - loss: 0.0072
1/1 [==============================] - 0s 18ms/step - loss: 0.0072 - val_loss: 0.0028
Epoch 22/150
1/1 [==============================] - ETA: 0s - loss: 0.0096
1/1 [==============================] - 0s 18ms/step - loss: 0.0096 - val_loss: 0.0035
Epoch 23/150
1/1 [==============================] - ETA: 0s - loss: 0.0094
1/1 [==============================] - 0s 18ms/step - loss: 0.0094 - val_loss: 0.0016
Epoch 24/150
1/1 [==============================] - ETA: 0s - loss: 0.0068
1/1 [==============================] - 0s 18ms/step - loss: 0.0068 - val_loss: 2.1770e-05
Epoch 25/150
1/1 [==============================] - ETA: 0s - loss: 0.0037
1/1 [==============================] - 0s 17ms/step - loss: 0.0037 - val_loss: 0.0014
Epoch 26/150
1/1 [==============================] - ETA: 0s - loss: 0.0017
1/1 [==============================] - 0s 20ms/step - loss: 0.0017 - val_loss: 0.0061
Epoch 27/150
1/1 [==============================] - ETA: 0s - loss: 0.0015
1/1 [==============================] - 0s 18ms/step - loss: 0.0015 - val_loss: 0.0121
Epoch 28/150
1/1 [==============================] - ETA: 0s - loss: 0.0025
1/1 [==============================] - 0s 18ms/step - loss: 0.0025 - val_loss: 0.0166
Epoch 29/150
1/1 [==============================] - ETA: 0s - loss: 0.0037
1/1 [==============================] - 0s 18ms/step - loss: 0.0037 - val_loss: 0.0177
Epoch 30/150
1/1 [==============================] - ETA: 0s - loss: 0.0042
1/1 [==============================] - 0s 18ms/step - loss: 0.0042 - val_loss: 0.0151
Epoch 31/150
1/1 [==============================] - ETA: 0s - loss: 0.0037
1/1 [==============================] - 0s 18ms/step - loss: 0.0037 - val_loss: 0.0101
Epoch 32/150
1/1 [==============================] - ETA: 0s - loss: 0.0024
1/1 [==============================] - 0s 18ms/step - loss: 0.0024 - val_loss: 0.0046
Epoch 33/150
1/1 [==============================] - ETA: 0s - loss: 0.0011
1/1 [==============================] - 0s 19ms/step - loss: 0.0011 - val_loss: 8.8886e-04
Epoch 34/150
1/1 [==============================] - ETA: 0s - loss: 2.7391e-04
1/1 [==============================] - 0s 18ms/step - loss: 2.7391e-04 - val_loss: 6.9638e-05
Epoch 35/150
1/1 [==============================] - ETA: 0s - loss: 3.0805e-04
1/1 [==============================] - 0s 18ms/step - loss: 3.0805e-04 - val_loss: 0.0016
Epoch 36/150
1/1 [==============================] - ETA: 0s - loss: 9.6065e-04
1/1 [==============================] - 0s 18ms/step - loss: 9.6065e-04 - val_loss: 0.0037
Epoch 37/150
1/1 [==============================] - ETA: 0s - loss: 0.0016
1/1 [==============================] - 0s 18ms/step - loss: 0.0016 - val_loss: 0.0045
Epoch 38/150
1/1 [==============================] - ETA: 0s - loss: 0.0016
1/1 [==============================] - 0s 18ms/step - loss: 0.0016 - val_loss: 0.0035
Epoch 39/150
1/1 [==============================] - ETA: 0s - loss: 0.0011
1/1 [==============================] - 0s 18ms/step - loss: 0.0011 - val_loss: 0.0016
Epoch 40/150
1/1 [==============================] - ETA: 0s - loss: 3.6501e-04
1/1 [==============================] - 0s 18ms/step - loss: 3.6501e-04 - val_loss: 2.5101e-04
Epoch 41/150
1/1 [==============================] - ETA: 0s - loss: 5.2387e-05
1/1 [==============================] - 0s 19ms/step - loss: 5.2387e-05 - val_loss: 4.9809e-05
Epoch 42/150
1/1 [==============================] - ETA: 0s - loss: 2.7319e-04
1/1 [==============================] - 0s 19ms/step - loss: 2.7319e-04 - val_loss: 5.0112e-04
Epoch 43/150
1/1 [==============================] - ETA: 0s - loss: 7.0581e-04
1/1 [==============================] - 0s 18ms/step - loss: 7.0581e-04 - val_loss: 7.5034e-04
Epoch 44/150
1/1 [==============================] - ETA: 0s - loss: 9.3238e-04
1/1 [==============================] - 0s 19ms/step - loss: 9.3238e-04 - val_loss: 5.0648e-04
Epoch 45/150
1/1 [==============================] - ETA: 0s - loss: 8.0866e-04
1/1 [==============================] - 0s 18ms/step - loss: 8.0866e-04 - val_loss: 9.4408e-05
Epoch 46/150
1/1 [==============================] - ETA: 0s - loss: 4.7288e-04
1/1 [==============================] - 0s 18ms/step - loss: 4.7288e-04 - val_loss: 5.9254e-05
Epoch 47/150
1/1 [==============================] - ETA: 0s - loss: 1.8597e-04
1/1 [==============================] - 0s 20ms/step - loss: 1.8597e-04 - val_loss: 6.7079e-04
Epoch 48/150
1/1 [==============================] - ETA: 0s - loss: 1.2584e-04
1/1 [==============================] - 0s 20ms/step - loss: 1.2584e-04 - val_loss: 0.0017
Epoch 49/150
1/1 [==============================] - ETA: 0s - loss: 2.6867e-04
1/1 [==============================] - 0s 18ms/step - loss: 2.6867e-04 - val_loss: 0.0024
Epoch 50/150
1/1 [==============================] - ETA: 0s - loss: 4.3826e-04
1/1 [==============================] - 0s 19ms/step - loss: 4.3826e-04 - val_loss: 0.0025
Epoch 51/150
1/1 [==============================] - ETA: 0s - loss: 4.6733e-04
1/1 [==============================] - 0s 18ms/step - loss: 4.6733e-04 - val_loss: 0.0018
Epoch 52/150
1/1 [==============================] - ETA: 0s - loss: 3.3438e-04
1/1 [==============================] - 0s 18ms/step - loss: 3.3438e-04 - val_loss: 9.4701e-04
Epoch 53/150
1/1 [==============================] - ETA: 0s - loss: 1.5339e-04
1/1 [==============================] - 0s 18ms/step - loss: 1.5339e-04 - val_loss: 2.6460e-04
Epoch 54/150
1/1 [==============================] - ETA: 0s - loss: 5.4331e-05
1/1 [==============================] - 0s 18ms/step - loss: 5.4331e-05 - val_loss: 6.5156e-06
Epoch 55/150
1/1 [==============================] - ETA: 0s - loss: 7.7712e-05
1/1 [==============================] - 0s 18ms/step - loss: 7.7712e-05 - val_loss: 6.1830e-05
Epoch 56/150
1/1 [==============================] - ETA: 0s - loss: 1.6479e-04
1/1 [==============================] - 0s 18ms/step - loss: 1.6479e-04 - val_loss: 1.8163e-04
Epoch 57/150
1/1 [==============================] - ETA: 0s - loss: 2.2423e-04
1/1 [==============================] - 0s 18ms/step - loss: 2.2423e-04 - val_loss: 1.9299e-04
Epoch 58/150
1/1 [==============================] - ETA: 0s - loss: 2.0515e-04
1/1 [==============================] - 0s 19ms/step - loss: 2.0515e-04 - val_loss: 9.4929e-05
Epoch 59/150
1/1 [==============================] - ETA: 0s - loss: 1.2535e-04
1/1 [==============================] - 0s 18ms/step - loss: 1.2535e-04 - val_loss: 6.2223e-06
Epoch 60/150
1/1 [==============================] - ETA: 0s - loss: 4.6826e-05
1/1 [==============================] - 0s 18ms/step - loss: 4.6826e-05 - val_loss: 3.4402e-05
Epoch 61/150
1/1 [==============================] - ETA: 0s - loss: 2.2165e-05
1/1 [==============================] - 0s 18ms/step - loss: 2.2165e-05 - val_loss: 1.7489e-04
Epoch 62/150
1/1 [==============================] - ETA: 0s - loss: 5.5691e-05
1/1 [==============================] - 0s 18ms/step - loss: 5.5691e-05 - val_loss: 3.1733e-04
Epoch 63/150
1/1 [==============================] - ETA: 0s - loss: 1.0595e-04
1/1 [==============================] - 0s 18ms/step - loss: 1.0595e-04 - val_loss: 3.4700e-04
Epoch 64/150
1/1 [==============================] - ETA: 0s - loss: 1.2525e-04
1/1 [==============================] - 0s 18ms/step - loss: 1.2525e-04 - val_loss: 2.4500e-04
Epoch 65/150
1/1 [==============================] - ETA: 0s - loss: 9.9924e-05
1/1 [==============================] - 0s 18ms/step - loss: 9.9924e-05 - val_loss: 9.6135e-05
Epoch 66/150
1/1 [==============================] - ETA: 0s - loss: 5.5901e-05
1/1 [==============================] - 0s 18ms/step - loss: 5.5901e-05 - val_loss: 6.8225e-06
Epoch 67/150
1/1 [==============================] - ETA: 0s - loss: 2.9911e-05
1/1 [==============================] - 0s 18ms/step - loss: 2.9911e-05 - val_loss: 1.7879e-05
Epoch 68/150
1/1 [==============================] - ETA: 0s - loss: 3.7141e-05
1/1 [==============================] - 0s 18ms/step - loss: 3.7141e-05 - val_loss: 8.5801e-05
Epoch 69/150
1/1 [==============================] - ETA: 0s - loss: 6.3071e-05
1/1 [==============================] - 0s 18ms/step - loss: 6.3071e-05 - val_loss: 1.3386e-04
Epoch 70/150
1/1 [==============================] - ETA: 0s - loss: 8.0889e-05
1/1 [==============================] - 0s 18ms/step - loss: 8.0889e-05 - val_loss: 1.1862e-04
Epoch 71/150
1/1 [==============================] - ETA: 0s - loss: 7.5159e-05
1/1 [==============================] - 0s 20ms/step - loss: 7.5159e-05 - val_loss: 5.8102e-05
Epoch 72/150
1/1 [==============================] - ETA: 0s - loss: 5.2129e-05
1/1 [==============================] - 0s 18ms/step - loss: 5.2129e-05 - val_loss: 7.2266e-06
Epoch 73/150
1/1 [==============================] - ETA: 0s - loss: 3.1149e-05
1/1 [==============================] - 0s 20ms/step - loss: 3.1149e-05 - val_loss: 7.1946e-06
Epoch 74/150
1/1 [==============================] - ETA: 0s - loss: 2.6803e-05
1/1 [==============================] - 0s 20ms/step - loss: 2.6803e-05 - val_loss: 5.2427e-05
Epoch 75/150
1/1 [==============================] - ETA: 0s - loss: 3.7385e-05
1/1 [==============================] - 0s 24ms/step - loss: 3.7385e-05 - val_loss: 1.0039e-04
Epoch 76/150
1/1 [==============================] - ETA: 0s - loss: 4.8700e-05
1/1 [==============================] - 0s 18ms/step - loss: 4.8700e-05 - val_loss: 1.1159e-04
Epoch 77/150
1/1 [==============================] - ETA: 0s - loss: 4.8312e-05
1/1 [==============================] - 0s 20ms/step - loss: 4.8312e-05 - val_loss: 8.1357e-05
Epoch 78/150
1/1 [==============================] - ETA: 0s - loss: 3.6325e-05
1/1 [==============================] - 0s 21ms/step - loss: 3.6325e-05 - val_loss: 3.6670e-05
Epoch 79/150
1/1 [==============================] - ETA: 0s - loss: 2.3281e-05
1/1 [==============================] - 0s 19ms/step - loss: 2.3281e-05 - val_loss: 6.9527e-06
Epoch 80/150
1/1 [==============================] - ETA: 0s - loss: 1.9011e-05
1/1 [==============================] - 0s 18ms/step - loss: 1.9011e-05 - val_loss: 6.2134e-08
Epoch 81/150
1/1 [==============================] - ETA: 0s - loss: 2.4047e-05
1/1 [==============================] - 0s 20ms/step - loss: 2.4047e-05 - val_loss: 3.3985e-06
Epoch 82/150
1/1 [==============================] - ETA: 0s - loss: 3.0829e-05
1/1 [==============================] - 0s 25ms/step - loss: 3.0829e-05 - val_loss: 3.1615e-06
Epoch 83/150
1/1 [==============================] - ETA: 0s - loss: 3.1812e-05
1/1 [==============================] - 0s 20ms/step - loss: 3.1812e-05 - val_loss: 1.7273e-08
Epoch 84/150
1/1 [==============================] - ETA: 0s - loss: 2.6124e-05
1/1 [==============================] - 0s 19ms/step - loss: 2.6124e-05 - val_loss: 6.8793e-06
Epoch 85/150
1/1 [==============================] - ETA: 0s - loss: 1.9199e-05
1/1 [==============================] - 0s 19ms/step - loss: 1.9199e-05 - val_loss: 3.3150e-05
Epoch 86/150
1/1 [==============================] - ETA: 0s - loss: 1.6832e-05
1/1 [==============================] - 0s 18ms/step - loss: 1.6832e-05 - val_loss: 7.2050e-05
Epoch 87/150
1/1 [==============================] - ETA: 0s - loss: 1.9787e-05
1/1 [==============================] - 0s 19ms/step - loss: 1.9787e-05 - val_loss: 1.0364e-04
Epoch 88/150
1/1 [==============================] - ETA: 0s - loss: 2.3840e-05
1/1 [==============================] - 0s 19ms/step - loss: 2.3840e-05 - val_loss: 1.1049e-04
Epoch 89/150
1/1 [==============================] - ETA: 0s - loss: 2.4491e-05
1/1 [==============================] - 0s 24ms/step - loss: 2.4491e-05 - val_loss: 9.1017e-05
Epoch 90/150
1/1 [==============================] - ETA: 0s - loss: 2.1253e-05
1/1 [==============================] - 0s 20ms/step - loss: 2.1253e-05 - val_loss: 5.8702e-05
Epoch 91/150
1/1 [==============================] - ETA: 0s - loss: 1.7418e-05
1/1 [==============================] - 0s 20ms/step - loss: 1.7418e-05 - val_loss: 2.9923e-05
Epoch 92/150
1/1 [==============================] - ETA: 0s - loss: 1.6259e-05
1/1 [==============================] - 0s 19ms/step - loss: 1.6259e-05 - val_loss: 1.2810e-05
Epoch 93/150
1/1 [==============================] - ETA: 0s - loss: 1.7905e-05
1/1 [==============================] - 0s 18ms/step - loss: 1.7905e-05 - val_loss: 6.0232e-06
Epoch 94/150
1/1 [==============================] - ETA: 0s - loss: 1.9781e-05
1/1 [==============================] - 0s 18ms/step - loss: 1.9781e-05 - val_loss: 5.3134e-06
Epoch 95/150
1/1 [==============================] - ETA: 0s - loss: 1.9548e-05
1/1 [==============================] - 0s 19ms/step - loss: 1.9548e-05 - val_loss: 9.3216e-06
Epoch 96/150
1/1 [==============================] - ETA: 0s - loss: 1.7296e-05
1/1 [==============================] - 0s 19ms/step - loss: 1.7296e-05 - val_loss: 1.9159e-05
Epoch 97/150
1/1 [==============================] - ETA: 0s - loss: 1.5052e-05
1/1 [==============================] - 0s 18ms/step - loss: 1.5052e-05 - val_loss: 3.3857e-05
Epoch 98/150
1/1 [==============================] - ETA: 0s - loss: 1.4498e-05
1/1 [==============================] - 0s 19ms/step - loss: 1.4498e-05 - val_loss: 4.7879e-05
Epoch 99/150
1/1 [==============================] - ETA: 0s - loss: 1.5386e-05
1/1 [==============================] - 0s 19ms/step - loss: 1.5386e-05 - val_loss: 5.4022e-05
Epoch 100/150
1/1 [==============================] - ETA: 0s - loss: 1.6115e-05
1/1 [==============================] - 0s 18ms/step - loss: 1.6115e-05 - val_loss: 4.8949e-05
Epoch 101/150
1/1 [==============================] - ETA: 0s - loss: 1.5549e-05
1/1 [==============================] - 0s 19ms/step - loss: 1.5549e-05 - val_loss: 3.5722e-05
Epoch 102/150
1/1 [==============================] - ETA: 0s - loss: 1.4079e-05
1/1 [==============================] - 0s 18ms/step - loss: 1.4079e-05 - val_loss: 2.1059e-05
Epoch 103/150
1/1 [==============================] - ETA: 0s - loss: 1.2963e-05
1/1 [==============================] - 0s 20ms/step - loss: 1.2963e-05 - val_loss: 1.0307e-05
Epoch 104/150
1/1 [==============================] - ETA: 0s - loss: 1.2911e-05
1/1 [==============================] - 0s 19ms/step - loss: 1.2911e-05 - val_loss: 4.7690e-06
Epoch 105/150
1/1 [==============================] - ETA: 0s - loss: 1.3470e-05
1/1 [==============================] - 0s 18ms/step - loss: 1.3470e-05 - val_loss: 2.9656e-06
Epoch 106/150
1/1 [==============================] - ETA: 0s - loss: 1.3678e-05
1/1 [==============================] - 0s 20ms/step - loss: 1.3678e-05 - val_loss: 3.4672e-06
Epoch 107/150
1/1 [==============================] - ETA: 0s - loss: 1.3126e-05
1/1 [==============================] - 0s 18ms/step - loss: 1.3126e-05 - val_loss: 6.0939e-06
Epoch 108/150
1/1 [==============================] - ETA: 0s - loss: 1.2266e-05
1/1 [==============================] - 0s 20ms/step - loss: 1.2266e-05 - val_loss: 1.0808e-05
Epoch 109/150
1/1 [==============================] - ETA: 0s - loss: 1.1796e-05
1/1 [==============================] - 0s 19ms/step - loss: 1.1796e-05 - val_loss: 1.6232e-05
Epoch 110/150
1/1 [==============================] - ETA: 0s - loss: 1.1902e-05
1/1 [==============================] - 0s 22ms/step - loss: 1.1902e-05 - val_loss: 1.9843e-05
Epoch 111/150
1/1 [==============================] - ETA: 0s - loss: 1.2143e-05
1/1 [==============================] - 0s 18ms/step - loss: 1.2143e-05 - val_loss: 1.9755e-05
Epoch 112/150
1/1 [==============================] - ETA: 0s - loss: 1.2022e-05
1/1 [==============================] - 0s 18ms/step - loss: 1.2022e-05 - val_loss: 1.6197e-05
Epoch 113/150
1/1 [==============================] - ETA: 0s - loss: 1.1506e-05
1/1 [==============================] - 0s 20ms/step - loss: 1.1506e-05 - val_loss: 1.1183e-05
Epoch 114/150
1/1 [==============================] - ETA: 0s - loss: 1.0982e-05
1/1 [==============================] - 0s 18ms/step - loss: 1.0982e-05 - val_loss: 6.8771e-06
Epoch 115/150
1/1 [==============================] - ETA: 0s - loss: 1.0778e-05
1/1 [==============================] - 0s 18ms/step - loss: 1.0778e-05 - val_loss: 4.3037e-06
Epoch 116/150
1/1 [==============================] - ETA: 0s - loss: 1.0826e-05
1/1 [==============================] - 0s 19ms/step - loss: 1.0826e-05 - val_loss: 3.4128e-06
Epoch 117/150
1/1 [==============================] - ETA: 0s - loss: 1.0803e-05
1/1 [==============================] - 0s 18ms/step - loss: 1.0803e-05 - val_loss: 3.9190e-06
Epoch 118/150
1/1 [==============================] - ETA: 0s - loss: 1.0525e-05
1/1 [==============================] - 0s 19ms/step - loss: 1.0525e-05 - val_loss: 5.7593e-06
Epoch 119/150
1/1 [==============================] - ETA: 0s - loss: 1.0114e-05
1/1 [==============================] - 0s 18ms/step - loss: 1.0114e-05 - val_loss: 8.7378e-06
Epoch 120/150
1/1 [==============================] - ETA: 0s - loss: 9.8203e-06
1/1 [==============================] - 0s 18ms/step - loss: 9.8203e-06 - val_loss: 1.2031e-05
Epoch 121/150
1/1 [==============================] - ETA: 0s - loss: 9.7284e-06
1/1 [==============================] - 0s 19ms/step - loss: 9.7284e-06 - val_loss: 1.4340e-05
Epoch 122/150
1/1 [==============================] - ETA: 0s - loss: 9.6966e-06
1/1 [==============================] - 0s 19ms/step - loss: 9.6966e-06 - val_loss: 1.4696e-05
Epoch 123/150
1/1 [==============================] - ETA: 0s - loss: 9.5506e-06
1/1 [==============================] - 0s 20ms/step - loss: 9.5506e-06 - val_loss: 1.3124e-05
Epoch 124/150
1/1 [==============================] - ETA: 0s - loss: 9.2787e-06
1/1 [==============================] - 0s 18ms/step - loss: 9.2787e-06 - val_loss: 1.0547e-05
Epoch 125/150
1/1 [==============================] - ETA: 0s - loss: 9.0185e-06
1/1 [==============================] - 0s 18ms/step - loss: 9.0185e-06 - val_loss: 8.0814e-06
Epoch 126/150
1/1 [==============================] - ETA: 0s - loss: 8.8755e-06
1/1 [==============================] - 0s 18ms/step - loss: 8.8755e-06 - val_loss: 6.4479e-06
Epoch 127/150
1/1 [==============================] - ETA: 0s - loss: 8.8137e-06
1/1 [==============================] - 0s 18ms/step - loss: 8.8137e-06 - val_loss: 5.8674e-06
Epoch 128/150
1/1 [==============================] - ETA: 0s - loss: 8.7187e-06
1/1 [==============================] - 0s 18ms/step - loss: 8.7187e-06 - val_loss: 6.2909e-06
Epoch 129/150
1/1 [==============================] - ETA: 0s - loss: 8.5374e-06
1/1 [==============================] - 0s 18ms/step - loss: 8.5374e-06 - val_loss: 7.5345e-06
Epoch 130/150
1/1 [==============================] - ETA: 0s - loss: 8.3250e-06
1/1 [==============================] - 0s 18ms/step - loss: 8.3250e-06 - val_loss: 9.2095e-06
Epoch 131/150
1/1 [==============================] - ETA: 0s - loss: 8.1651e-06
1/1 [==============================] - 0s 18ms/step - loss: 8.1651e-06 - val_loss: 1.0688e-05
Epoch 132/150
1/1 [==============================] - ETA: 0s - loss: 8.0688e-06
1/1 [==============================] - 0s 18ms/step - loss: 8.0688e-06 - val_loss: 1.1327e-05
Epoch 133/150
1/1 [==============================] - ETA: 0s - loss: 7.9771e-06
1/1 [==============================] - 0s 19ms/step - loss: 7.9771e-06 - val_loss: 1.0835e-05
Epoch 134/150
1/1 [==============================] - ETA: 0s - loss: 7.8467e-06
1/1 [==============================] - 0s 18ms/step - loss: 7.8467e-06 - val_loss: 9.4450e-06
Epoch 135/150
1/1 [==============================] - ETA: 0s - loss: 7.6825e-06
1/1 [==============================] - 0s 19ms/step - loss: 7.6825e-06 - val_loss: 7.7252e-06
Epoch 136/150
1/1 [==============================] - ETA: 0s - loss: 7.5362e-06
1/1 [==============================] - 0s 18ms/step - loss: 7.5362e-06 - val_loss: 6.2324e-06
Epoch 137/150
1/1 [==============================] - ETA: 0s - loss: 7.4299e-06
1/1 [==============================] - 0s 18ms/step - loss: 7.4299e-06 - val_loss: 5.2841e-06
Epoch 138/150
1/1 [==============================] - ETA: 0s - loss: 7.3370e-06
1/1 [==============================] - 0s 18ms/step - loss: 7.3370e-06 - val_loss: 4.9452e-06
Epoch 139/150
1/1 [==============================] - ETA: 0s - loss: 7.2211e-06
1/1 [==============================] - 0s 19ms/step - loss: 7.2211e-06 - val_loss: 5.1307e-06
Epoch 140/150
1/1 [==============================] - ETA: 0s - loss: 7.0810e-06
1/1 [==============================] - 0s 18ms/step - loss: 7.0810e-06 - val_loss: 5.6509e-06
Epoch 141/150
1/1 [==============================] - ETA: 0s - loss: 6.9456e-06
1/1 [==============================] - 0s 18ms/step - loss: 6.9456e-06 - val_loss: 6.2259e-06
Epoch 142/150
1/1 [==============================] - ETA: 0s - loss: 6.8358e-06
1/1 [==============================] - 0s 18ms/step - loss: 6.8358e-06 - val_loss: 6.5445e-06
Epoch 143/150
1/1 [==============================] - ETA: 0s - loss: 6.7413e-06
1/1 [==============================] - 0s 18ms/step - loss: 6.7413e-06 - val_loss: 6.4017e-06
Epoch 144/150
1/1 [==============================] - ETA: 0s - loss: 6.6380e-06
1/1 [==============================] - 0s 18ms/step - loss: 6.6380e-06 - val_loss: 5.8084e-06
Epoch 145/150
1/1 [==============================] - ETA: 0s - loss: 6.5196e-06
1/1 [==============================] - 0s 18ms/step - loss: 6.5196e-06 - val_loss: 4.9715e-06
Epoch 146/150
1/1 [==============================] - ETA: 0s - loss: 6.4007e-06
1/1 [==============================] - 0s 19ms/step - loss: 6.4007e-06 - val_loss: 4.1612e-06
Epoch 147/150
1/1 [==============================] - ETA: 0s - loss: 6.2981e-06
1/1 [==============================] - 0s 18ms/step - loss: 6.2981e-06 - val_loss: 3.5793e-06
Epoch 148/150
1/1 [==============================] - ETA: 0s - loss: 6.2091e-06
1/1 [==============================] - 0s 18ms/step - loss: 6.2091e-06 - val_loss: 3.3095e-06
Epoch 149/150
1/1 [==============================] - ETA: 0s - loss: 6.1165e-06
1/1 [==============================] - 0s 18ms/step - loss: 6.1165e-06 - val_loss: 3.3423e-06
Epoch 150/150
1/1 [==============================] - ETA: 0s - loss: 6.0133e-06
1/1 [==============================] - 0s 18ms/step - loss: 6.0133e-06 - val_loss: 3.6012e-06
MSE: 1.9124180119966666e-05
Time: 9.464447478
Model: "functional_7"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
input_4 (InputLayer) [(None, 1, 1)] 0
_________________________________________________________________
RNN (SimpleRNN) (None, 200) 40400
_________________________________________________________________
dense (Dense) (None, 1) 201
=================================================================
Total params: 40,601
Trainable params: 40,601
Non-trainable params: 0
_________________________________________________________________
Epoch 1/150
1/1 [==============================] - ETA: 0s - loss: 1.4102
1/1 [==============================] - 0s 154ms/step - loss: 1.4102 - val_loss: 2.1967
Epoch 2/150
1/1 [==============================] - ETA: 0s - loss: 0.9243
1/1 [==============================] - 0s 18ms/step - loss: 0.9243 - val_loss: 1.4521
Epoch 3/150
1/1 [==============================] - ETA: 0s - loss: 0.5465
1/1 [==============================] - 0s 17ms/step - loss: 0.5465 - val_loss: 0.8738
Epoch 4/150
1/1 [==============================] - ETA: 0s - loss: 0.2756
1/1 [==============================] - 0s 17ms/step - loss: 0.2756 - val_loss: 0.4570
Epoch 5/150
1/1 [==============================] - ETA: 0s - loss: 0.1066
1/1 [==============================] - 0s 20ms/step - loss: 0.1066 - val_loss: 0.1890
Epoch 6/150
1/1 [==============================] - ETA: 0s - loss: 0.0284
1/1 [==============================] - 0s 17ms/step - loss: 0.0284 - val_loss: 0.0477
Epoch 7/150
1/1 [==============================] - ETA: 0s - loss: 0.0229
1/1 [==============================] - 0s 16ms/step - loss: 0.0229 - val_loss: 0.0012
Epoch 8/150
1/1 [==============================] - ETA: 0s - loss: 0.0654
1/1 [==============================] - 0s 17ms/step - loss: 0.0654 - val_loss: 0.0122
Epoch 9/150
1/1 [==============================] - ETA: 0s - loss: 0.1283
1/1 [==============================] - 0s 18ms/step - loss: 0.1283 - val_loss: 0.0453
Epoch 10/150
1/1 [==============================] - ETA: 0s - loss: 0.1870
1/1 [==============================] - 0s 17ms/step - loss: 0.1870 - val_loss: 0.0747
Epoch 11/150
1/1 [==============================] - ETA: 0s - loss: 0.2252
1/1 [==============================] - 0s 17ms/step - loss: 0.2252 - val_loss: 0.0870
Epoch 12/150
1/1 [==============================] - ETA: 0s - loss: 0.2363
1/1 [==============================] - 0s 17ms/step - loss: 0.2363 - val_loss: 0.0801
Epoch 13/150
1/1 [==============================] - ETA: 0s - loss: 0.2218
1/1 [==============================] - 0s 16ms/step - loss: 0.2218 - val_loss: 0.0593
Epoch 14/150
1/1 [==============================] - ETA: 0s - loss: 0.1884
1/1 [==============================] - 0s 16ms/step - loss: 0.1884 - val_loss: 0.0332
Epoch 15/150
1/1 [==============================] - ETA: 0s - loss: 0.1445
1/1 [==============================] - 0s 17ms/step - loss: 0.1445 - val_loss: 0.0111
Epoch 16/150
1/1 [==============================] - ETA: 0s - loss: 0.0988
1/1 [==============================] - 0s 18ms/step - loss: 0.0988 - val_loss: 3.2715e-04
Epoch 17/150
1/1 [==============================] - ETA: 0s - loss: 0.0585
1/1 [==============================] - 0s 17ms/step - loss: 0.0585 - val_loss: 0.0055
Epoch 18/150
1/1 [==============================] - ETA: 0s - loss: 0.0286
1/1 [==============================] - 0s 16ms/step - loss: 0.0286 - val_loss: 0.0275
Epoch 19/150
1/1 [==============================] - ETA: 0s - loss: 0.0116
1/1 [==============================] - 0s 17ms/step - loss: 0.0116 - val_loss: 0.0639
Epoch 20/150
1/1 [==============================] - ETA: 0s - loss: 0.0072
1/1 [==============================] - 0s 17ms/step - loss: 0.0072 - val_loss: 0.1091
Epoch 21/150
1/1 [==============================] - ETA: 0s - loss: 0.0129
1/1 [==============================] - 0s 16ms/step - loss: 0.0129 - val_loss: 0.1561
Epoch 22/150
1/1 [==============================] - ETA: 0s - loss: 0.0249
1/1 [==============================] - 0s 16ms/step - loss: 0.0249 - val_loss: 0.1975
Epoch 23/150
1/1 [==============================] - ETA: 0s - loss: 0.0388
1/1 [==============================] - 0s 17ms/step - loss: 0.0388 - val_loss: 0.2272
Epoch 24/150
1/1 [==============================] - ETA: 0s - loss: 0.0505
1/1 [==============================] - 0s 17ms/step - loss: 0.0505 - val_loss: 0.2413
Epoch 25/150
1/1 [==============================] - ETA: 0s - loss: 0.0572
1/1 [==============================] - 0s 18ms/step - loss: 0.0572 - val_loss: 0.2389
Epoch 26/150
1/1 [==============================] - ETA: 0s - loss: 0.0578
1/1 [==============================] - 0s 18ms/step - loss: 0.0578 - val_loss: 0.2215
Epoch 27/150
1/1 [==============================] - ETA: 0s - loss: 0.0526
1/1 [==============================] - 0s 17ms/step - loss: 0.0526 - val_loss: 0.1927
Epoch 28/150
1/1 [==============================] - ETA: 0s - loss: 0.0430
1/1 [==============================] - 0s 17ms/step - loss: 0.0430 - val_loss: 0.1571
Epoch 29/150
1/1 [==============================] - ETA: 0s - loss: 0.0313
1/1 [==============================] - 0s 17ms/step - loss: 0.0313 - val_loss: 0.1197
Epoch 30/150
1/1 [==============================] - ETA: 0s - loss: 0.0198
1/1 [==============================] - 0s 17ms/step - loss: 0.0198 - val_loss: 0.0845
Epoch 31/150
1/1 [==============================] - ETA: 0s - loss: 0.0105
1/1 [==============================] - 0s 17ms/step - loss: 0.0105 - val_loss: 0.0548
Epoch 32/150
1/1 [==============================] - ETA: 0s - loss: 0.0045
1/1 [==============================] - 0s 17ms/step - loss: 0.0045 - val_loss: 0.0320
Epoch 33/150
1/1 [==============================] - ETA: 0s - loss: 0.0022
1/1 [==============================] - 0s 18ms/step - loss: 0.0022 - val_loss: 0.0165
Epoch 34/150
1/1 [==============================] - ETA: 0s - loss: 0.0032
1/1 [==============================] - 0s 22ms/step - loss: 0.0032 - val_loss: 0.0071
Epoch 35/150
1/1 [==============================] - ETA: 0s - loss: 0.0065
1/1 [==============================] - 0s 17ms/step - loss: 0.0065 - val_loss: 0.0024
Epoch 36/150
1/1 [==============================] - ETA: 0s - loss: 0.0105
1/1 [==============================] - 0s 19ms/step - loss: 0.0105 - val_loss: 5.8779e-04
Epoch 37/150
1/1 [==============================] - ETA: 0s - loss: 0.0140
1/1 [==============================] - 0s 18ms/step - loss: 0.0140 - val_loss: 9.0393e-05
Epoch 38/150
1/1 [==============================] - ETA: 0s - loss: 0.0161
1/1 [==============================] - 0s 17ms/step - loss: 0.0161 - val_loss: 2.5875e-05
Epoch 39/150
1/1 [==============================] - ETA: 0s - loss: 0.0163
1/1 [==============================] - 0s 17ms/step - loss: 0.0163 - val_loss: 1.0453e-04
Epoch 40/150
1/1 [==============================] - ETA: 0s - loss: 0.0146
1/1 [==============================] - 0s 18ms/step - loss: 0.0146 - val_loss: 5.5861e-04
Epoch 41/150
1/1 [==============================] - ETA: 0s - loss: 0.0117
1/1 [==============================] - 0s 19ms/step - loss: 0.0117 - val_loss: 0.0019
Epoch 42/150
1/1 [==============================] - ETA: 0s - loss: 0.0083
1/1 [==============================] - 0s 16ms/step - loss: 0.0083 - val_loss: 0.0047
Epoch 43/150
1/1 [==============================] - ETA: 0s - loss: 0.0050
1/1 [==============================] - 0s 17ms/step - loss: 0.0050 - val_loss: 0.0091
Epoch 44/150
1/1 [==============================] - ETA: 0s - loss: 0.0027
1/1 [==============================] - 0s 18ms/step - loss: 0.0027 - val_loss: 0.0151
Epoch 45/150
1/1 [==============================] - ETA: 0s - loss: 0.0015
1/1 [==============================] - 0s 18ms/step - loss: 0.0015 - val_loss: 0.0221
Epoch 46/150
1/1 [==============================] - ETA: 0s - loss: 0.0015
1/1 [==============================] - 0s 17ms/step - loss: 0.0015 - val_loss: 0.0294
Epoch 47/150
1/1 [==============================] - ETA: 0s - loss: 0.0023
1/1 [==============================] - 0s 17ms/step - loss: 0.0023 - val_loss: 0.0358
Epoch 48/150
1/1 [==============================] - ETA: 0s - loss: 0.0036
1/1 [==============================] - 0s 18ms/step - loss: 0.0036 - val_loss: 0.0407
Epoch 49/150
1/1 [==============================] - ETA: 0s - loss: 0.0048
1/1 [==============================] - 0s 18ms/step - loss: 0.0048 - val_loss: 0.0432
Epoch 50/150
1/1 [==============================] - ETA: 0s - loss: 0.0056
1/1 [==============================] - 0s 18ms/step - loss: 0.0056 - val_loss: 0.0432
Epoch 51/150
1/1 [==============================] - ETA: 0s - loss: 0.0057
1/1 [==============================] - 0s 17ms/step - loss: 0.0057 - val_loss: 0.0409
Epoch 52/150
1/1 [==============================] - ETA: 0s - loss: 0.0053
1/1 [==============================] - 0s 17ms/step - loss: 0.0053 - val_loss: 0.0368
Epoch 53/150
1/1 [==============================] - ETA: 0s - loss: 0.0043
1/1 [==============================] - 0s 18ms/step - loss: 0.0043 - val_loss: 0.0315
Epoch 54/150
1/1 [==============================] - ETA: 0s - loss: 0.0032
1/1 [==============================] - 0s 17ms/step - loss: 0.0032 - val_loss: 0.0258
Epoch 55/150
1/1 [==============================] - ETA: 0s - loss: 0.0023
1/1 [==============================] - 0s 17ms/step - loss: 0.0023 - val_loss: 0.0203
Epoch 56/150
1/1 [==============================] - ETA: 0s - loss: 0.0016
1/1 [==============================] - 0s 19ms/step - loss: 0.0016 - val_loss: 0.0156
Epoch 57/150
1/1 [==============================] - ETA: 0s - loss: 0.0013
1/1 [==============================] - 0s 17ms/step - loss: 0.0013 - val_loss: 0.0118
Epoch 58/150
1/1 [==============================] - ETA: 0s - loss: 0.0014
1/1 [==============================] - 0s 18ms/step - loss: 0.0014 - val_loss: 0.0089
Epoch 59/150
1/1 [==============================] - ETA: 0s - loss: 0.0017
1/1 [==============================] - 0s 18ms/step - loss: 0.0017 - val_loss: 0.0071
Epoch 60/150
1/1 [==============================] - ETA: 0s - loss: 0.0022
1/1 [==============================] - 0s 17ms/step - loss: 0.0022 - val_loss: 0.0060
Epoch 61/150
1/1 [==============================] - ETA: 0s - loss: 0.0025
1/1 [==============================] - 0s 17ms/step - loss: 0.0025 - val_loss: 0.0055
Epoch 62/150
1/1 [==============================] - ETA: 0s - loss: 0.0026
1/1 [==============================] - 0s 17ms/step - loss: 0.0026 - val_loss: 0.0057
Epoch 63/150
1/1 [==============================] - ETA: 0s - loss: 0.0026
1/1 [==============================] - 0s 17ms/step - loss: 0.0026 - val_loss: 0.0064
Epoch 64/150
1/1 [==============================] - ETA: 0s - loss: 0.0023
1/1 [==============================] - 0s 16ms/step - loss: 0.0023 - val_loss: 0.0076
Epoch 65/150
1/1 [==============================] - ETA: 0s - loss: 0.0019
1/1 [==============================] - 0s 17ms/step - loss: 0.0019 - val_loss: 0.0093
Epoch 66/150
1/1 [==============================] - ETA: 0s - loss: 0.0016
1/1 [==============================] - 0s 16ms/step - loss: 0.0016 - val_loss: 0.0113
Epoch 67/150
1/1 [==============================] - ETA: 0s - loss: 0.0013
1/1 [==============================] - 0s 17ms/step - loss: 0.0013 - val_loss: 0.0136
Epoch 68/150
1/1 [==============================] - ETA: 0s - loss: 0.0012
1/1 [==============================] - 0s 17ms/step - loss: 0.0012 - val_loss: 0.0160
Epoch 69/150
1/1 [==============================] - ETA: 0s - loss: 0.0012
1/1 [==============================] - 0s 17ms/step - loss: 0.0012 - val_loss: 0.0181
Epoch 70/150
1/1 [==============================] - ETA: 0s - loss: 0.0013
1/1 [==============================] - 0s 18ms/step - loss: 0.0013 - val_loss: 0.0198
Epoch 71/150
1/1 [==============================] - ETA: 0s - loss: 0.0014
1/1 [==============================] - 0s 17ms/step - loss: 0.0014 - val_loss: 0.0209
Epoch 72/150
1/1 [==============================] - ETA: 0s - loss: 0.0015
1/1 [==============================] - 0s 17ms/step - loss: 0.0015 - val_loss: 0.0213
Epoch 73/150
1/1 [==============================] - ETA: 0s - loss: 0.0016
1/1 [==============================] - 0s 16ms/step - loss: 0.0016 - val_loss: 0.0210
Epoch 74/150
1/1 [==============================] - ETA: 0s - loss: 0.0015
1/1 [==============================] - 0s 17ms/step - loss: 0.0015 - val_loss: 0.0202
Epoch 75/150
1/1 [==============================] - ETA: 0s - loss: 0.0014
1/1 [==============================] - 0s 16ms/step - loss: 0.0014 - val_loss: 0.0189
Epoch 76/150
1/1 [==============================] - ETA: 0s - loss: 0.0013
1/1 [==============================] - 0s 17ms/step - loss: 0.0013 - val_loss: 0.0174
Epoch 77/150
1/1 [==============================] - ETA: 0s - loss: 0.0011
1/1 [==============================] - 0s 17ms/step - loss: 0.0011 - val_loss: 0.0158
Epoch 78/150
1/1 [==============================] - ETA: 0s - loss: 0.0011
1/1 [==============================] - 0s 17ms/step - loss: 0.0011 - val_loss: 0.0143
Epoch 79/150
1/1 [==============================] - ETA: 0s - loss: 0.0010
1/1 [==============================] - 0s 17ms/step - loss: 0.0010 - val_loss: 0.0130
Epoch 80/150
1/1 [==============================] - ETA: 0s - loss: 0.0010
1/1 [==============================] - 0s 17ms/step - loss: 0.0010 - val_loss: 0.0120
Epoch 81/150
1/1 [==============================] - ETA: 0s - loss: 0.0011
1/1 [==============================] - 0s 17ms/step - loss: 0.0011 - val_loss: 0.0114
Epoch 82/150
1/1 [==============================] - ETA: 0s - loss: 0.0011
1/1 [==============================] - 0s 17ms/step - loss: 0.0011 - val_loss: 0.0110
Epoch 83/150
1/1 [==============================] - ETA: 0s - loss: 0.0011
1/1 [==============================] - 0s 17ms/step - loss: 0.0011 - val_loss: 0.0110
Epoch 84/150
1/1 [==============================] - ETA: 0s - loss: 0.0011
1/1 [==============================] - 0s 17ms/step - loss: 0.0011 - val_loss: 0.0113
Epoch 85/150
1/1 [==============================] - ETA: 0s - loss: 0.0011
1/1 [==============================] - 0s 17ms/step - loss: 0.0011 - val_loss: 0.0118
Epoch 86/150
1/1 [==============================] - ETA: 0s - loss: 0.0011
1/1 [==============================] - 0s 17ms/step - loss: 0.0011 - val_loss: 0.0125
Epoch 87/150
1/1 [==============================] - ETA: 0s - loss: 9.9906e-04
1/1 [==============================] - 0s 17ms/step - loss: 9.9906e-04 - val_loss: 0.0133
Epoch 88/150
1/1 [==============================] - ETA: 0s - loss: 9.5899e-04
1/1 [==============================] - 0s 16ms/step - loss: 9.5899e-04 - val_loss: 0.0142
Epoch 89/150
1/1 [==============================] - ETA: 0s - loss: 9.3780e-04
1/1 [==============================] - 0s 17ms/step - loss: 9.3780e-04 - val_loss: 0.0150
Epoch 90/150
1/1 [==============================] - ETA: 0s - loss: 9.3488e-04
1/1 [==============================] - 0s 16ms/step - loss: 9.3488e-04 - val_loss: 0.0156
Epoch 91/150
1/1 [==============================] - ETA: 0s - loss: 9.4357e-04
1/1 [==============================] - 0s 17ms/step - loss: 9.4357e-04 - val_loss: 0.0161
Epoch 92/150
1/1 [==============================] - ETA: 0s - loss: 9.5451e-04
1/1 [==============================] - 0s 17ms/step - loss: 9.5451e-04 - val_loss: 0.0163
Epoch 93/150
1/1 [==============================] - ETA: 0s - loss: 9.5936e-04
1/1 [==============================] - 0s 17ms/step - loss: 9.5936e-04 - val_loss: 0.0163
Epoch 94/150
1/1 [==============================] - ETA: 0s - loss: 9.5352e-04
1/1 [==============================] - 0s 17ms/step - loss: 9.5352e-04 - val_loss: 0.0161
Epoch 95/150
1/1 [==============================] - ETA: 0s - loss: 9.3717e-04
1/1 [==============================] - 0s 16ms/step - loss: 9.3717e-04 - val_loss: 0.0156
Epoch 96/150
1/1 [==============================] - ETA: 0s - loss: 9.1432e-04
1/1 [==============================] - 0s 17ms/step - loss: 9.1432e-04 - val_loss: 0.0150
Epoch 97/150
1/1 [==============================] - ETA: 0s - loss: 8.9074e-04
1/1 [==============================] - 0s 17ms/step - loss: 8.9074e-04 - val_loss: 0.0144
Epoch 98/150
1/1 [==============================] - ETA: 0s - loss: 8.7159e-04
1/1 [==============================] - 0s 17ms/step - loss: 8.7159e-04 - val_loss: 0.0138
Epoch 99/150
1/1 [==============================] - ETA: 0s - loss: 8.5960e-04
1/1 [==============================] - 0s 17ms/step - loss: 8.5960e-04 - val_loss: 0.0132
Epoch 100/150
1/1 [==============================] - ETA: 0s - loss: 8.5449e-04
1/1 [==============================] - 0s 17ms/step - loss: 8.5449e-04 - val_loss: 0.0127
Epoch 101/150
1/1 [==============================] - ETA: 0s - loss: 8.5362e-04
1/1 [==============================] - 0s 17ms/step - loss: 8.5362e-04 - val_loss: 0.0124
Epoch 102/150
1/1 [==============================] - ETA: 0s - loss: 8.5331e-04
1/1 [==============================] - 0s 18ms/step - loss: 8.5331e-04 - val_loss: 0.0122
Epoch 103/150
1/1 [==============================] - ETA: 0s - loss: 8.5041e-04
1/1 [==============================] - 0s 17ms/step - loss: 8.5041e-04 - val_loss: 0.0121
Epoch 104/150
1/1 [==============================] - ETA: 0s - loss: 8.4335e-04
1/1 [==============================] - 0s 16ms/step - loss: 8.4335e-04 - val_loss: 0.0121
Epoch 105/150
1/1 [==============================] - ETA: 0s - loss: 8.3246e-04
1/1 [==============================] - 0s 17ms/step - loss: 8.3246e-04 - val_loss: 0.0122
Epoch 106/150
1/1 [==============================] - ETA: 0s - loss: 8.1954e-04
1/1 [==============================] - 0s 16ms/step - loss: 8.1954e-04 - val_loss: 0.0124
Epoch 107/150
1/1 [==============================] - ETA: 0s - loss: 8.0688e-04
1/1 [==============================] - 0s 17ms/step - loss: 8.0688e-04 - val_loss: 0.0127
Epoch 108/150
1/1 [==============================] - ETA: 0s - loss: 7.9636e-04
1/1 [==============================] - 0s 17ms/step - loss: 7.9636e-04 - val_loss: 0.0129
Epoch 109/150
1/1 [==============================] - ETA: 0s - loss: 7.8874e-04
1/1 [==============================] - 0s 17ms/step - loss: 7.8874e-04 - val_loss: 0.0131
Epoch 110/150
1/1 [==============================] - ETA: 0s - loss: 7.8361e-04
1/1 [==============================] - 0s 17ms/step - loss: 7.8361e-04 - val_loss: 0.0132
Epoch 111/150
1/1 [==============================] - ETA: 0s - loss: 7.7972e-04
1/1 [==============================] - 0s 17ms/step - loss: 7.7972e-04 - val_loss: 0.0133
Epoch 112/150
1/1 [==============================] - ETA: 0s - loss: 7.7562e-04
1/1 [==============================] - 0s 17ms/step - loss: 7.7562e-04 - val_loss: 0.0132
Epoch 113/150
1/1 [==============================] - ETA: 0s - loss: 7.7023e-04
1/1 [==============================] - 0s 17ms/step - loss: 7.7023e-04 - val_loss: 0.0131
Epoch 114/150
1/1 [==============================] - ETA: 0s - loss: 7.6324e-04
1/1 [==============================] - 0s 18ms/step - loss: 7.6324e-04 - val_loss: 0.0129
Epoch 115/150
1/1 [==============================] - ETA: 0s - loss: 7.5506e-04
1/1 [==============================] - 0s 16ms/step - loss: 7.5506e-04 - val_loss: 0.0126
Epoch 116/150
1/1 [==============================] - ETA: 0s - loss: 7.4655e-04
1/1 [==============================] - 0s 17ms/step - loss: 7.4655e-04 - val_loss: 0.0123
Epoch 117/150
1/1 [==============================] - ETA: 0s - loss: 7.3860e-04
1/1 [==============================] - 0s 17ms/step - loss: 7.3860e-04 - val_loss: 0.0121
Epoch 118/150
1/1 [==============================] - ETA: 0s - loss: 7.3174e-04
1/1 [==============================] - 0s 17ms/step - loss: 7.3174e-04 - val_loss: 0.0118
Epoch 119/150
1/1 [==============================] - ETA: 0s - loss: 7.2605e-04
1/1 [==============================] - 0s 16ms/step - loss: 7.2605e-04 - val_loss: 0.0115
Epoch 120/150
1/1 [==============================] - ETA: 0s - loss: 7.2114e-04
1/1 [==============================] - 0s 19ms/step - loss: 7.2114e-04 - val_loss: 0.0114
Epoch 121/150
1/1 [==============================] - ETA: 0s - loss: 7.1643e-04
1/1 [==============================] - 0s 18ms/step - loss: 7.1643e-04 - val_loss: 0.0112
Epoch 122/150
1/1 [==============================] - ETA: 0s - loss: 7.1139e-04
1/1 [==============================] - 0s 18ms/step - loss: 7.1139e-04 - val_loss: 0.0111
Epoch 123/150
1/1 [==============================] - ETA: 0s - loss: 7.0577e-04
1/1 [==============================] - 0s 17ms/step - loss: 7.0577e-04 - val_loss: 0.0111
Epoch 124/150
1/1 [==============================] - ETA: 0s - loss: 6.9964e-04
1/1 [==============================] - 0s 19ms/step - loss: 6.9964e-04 - val_loss: 0.0111
Epoch 125/150
1/1 [==============================] - ETA: 0s - loss: 6.9330e-04
1/1 [==============================] - 0s 18ms/step - loss: 6.9330e-04 - val_loss: 0.0112
Epoch 126/150
1/1 [==============================] - ETA: 0s - loss: 6.8710e-04
1/1 [==============================] - 0s 17ms/step - loss: 6.8710e-04 - val_loss: 0.0112
Epoch 127/150
1/1 [==============================] - ETA: 0s - loss: 6.8134e-04
1/1 [==============================] - 0s 17ms/step - loss: 6.8134e-04 - val_loss: 0.0112
Epoch 128/150
1/1 [==============================] - ETA: 0s - loss: 6.7611e-04
1/1 [==============================] - 0s 18ms/step - loss: 6.7611e-04 - val_loss: 0.0113
Epoch 129/150
1/1 [==============================] - ETA: 0s - loss: 6.7125e-04
1/1 [==============================] - 0s 18ms/step - loss: 6.7125e-04 - val_loss: 0.0113
Epoch 130/150
1/1 [==============================] - ETA: 0s - loss: 6.6659e-04
1/1 [==============================] - 0s 17ms/step - loss: 6.6659e-04 - val_loss: 0.0112
Epoch 131/150
1/1 [==============================] - ETA: 0s - loss: 6.6186e-04
1/1 [==============================] - 0s 17ms/step - loss: 6.6186e-04 - val_loss: 0.0112
Epoch 132/150
1/1 [==============================] - ETA: 0s - loss: 6.5694e-04
1/1 [==============================] - 0s 17ms/step - loss: 6.5694e-04 - val_loss: 0.0111
Epoch 133/150
1/1 [==============================] - ETA: 0s - loss: 6.5184e-04
1/1 [==============================] - 0s 18ms/step - loss: 6.5184e-04 - val_loss: 0.0110
Epoch 134/150
1/1 [==============================] - ETA: 0s - loss: 6.4666e-04
1/1 [==============================] - 0s 19ms/step - loss: 6.4666e-04 - val_loss: 0.0108
Epoch 135/150
1/1 [==============================] - ETA: 0s - loss: 6.4156e-04
1/1 [==============================] - 0s 18ms/step - loss: 6.4156e-04 - val_loss: 0.0107
Epoch 136/150
1/1 [==============================] - ETA: 0s - loss: 6.3666e-04
1/1 [==============================] - 0s 17ms/step - loss: 6.3666e-04 - val_loss: 0.0105
Epoch 137/150
1/1 [==============================] - ETA: 0s - loss: 6.3201e-04
1/1 [==============================] - 0s 18ms/step - loss: 6.3201e-04 - val_loss: 0.0104
Epoch 138/150
1/1 [==============================] - ETA: 0s - loss: 6.2756e-04
1/1 [==============================] - 0s 18ms/step - loss: 6.2756e-04 - val_loss: 0.0103
Epoch 139/150
1/1 [==============================] - ETA: 0s - loss: 6.2321e-04
1/1 [==============================] - 0s 17ms/step - loss: 6.2321e-04 - val_loss: 0.0102
Epoch 140/150
1/1 [==============================] - ETA: 0s - loss: 6.1888e-04
1/1 [==============================] - 0s 16ms/step - loss: 6.1888e-04 - val_loss: 0.0102
Epoch 141/150
1/1 [==============================] - ETA: 0s - loss: 6.1449e-04
1/1 [==============================] - 0s 16ms/step - loss: 6.1449e-04 - val_loss: 0.0101
Epoch 142/150
1/1 [==============================] - ETA: 0s - loss: 6.1007e-04
1/1 [==============================] - 0s 17ms/step - loss: 6.1007e-04 - val_loss: 0.0101
Epoch 143/150
1/1 [==============================] - ETA: 0s - loss: 6.0565e-04
1/1 [==============================] - 0s 17ms/step - loss: 6.0565e-04 - val_loss: 0.0101
Epoch 144/150
1/1 [==============================] - ETA: 0s - loss: 6.0129e-04
1/1 [==============================] - 0s 18ms/step - loss: 6.0129e-04 - val_loss: 0.0101
Epoch 145/150
1/1 [==============================] - ETA: 0s - loss: 5.9706e-04
1/1 [==============================] - 0s 17ms/step - loss: 5.9706e-04 - val_loss: 0.0100
Epoch 146/150
1/1 [==============================] - ETA: 0s - loss: 5.9295e-04
1/1 [==============================] - 0s 17ms/step - loss: 5.9295e-04 - val_loss: 0.0100
Epoch 147/150
1/1 [==============================] - ETA: 0s - loss: 5.8894e-04
1/1 [==============================] - 0s 16ms/step - loss: 5.8894e-04 - val_loss: 0.0100
Epoch 148/150
1/1 [==============================] - ETA: 0s - loss: 5.8501e-04
1/1 [==============================] - 0s 17ms/step - loss: 5.8501e-04 - val_loss: 0.0100
Epoch 149/150
1/1 [==============================] - ETA: 0s - loss: 5.8111e-04
1/1 [==============================] - 0s 17ms/step - loss: 5.8111e-04 - val_loss: 0.0099
Epoch 150/150
1/1 [==============================] - ETA: 0s - loss: 5.7722e-04
1/1 [==============================] - 0s 19ms/step - loss: 5.7722e-04 - val_loss: 0.0098
Time: 4.337630193999999
10. Solving ODEs with Deep Learning¶
The Universal Approximation Theorem states that a neural network can approximate any function at a single hidden layer along with one input and output layer to any given precision.
10.1. Ordinary Differential Equations¶
An ordinary differential equation (ODE) is an equation involving functions having one variable.
In general, an ordinary differential equation looks like
\[
\begin{equation} \label{ode} \tag{1}
f\left(x, \, g(x), \, g'(x), \, g''(x), \, \dots \, , \, g^{(n)}(x)\right) = 0
\end{equation}
\]
where \(g(x)\) is the function to find, and \(g^{(n)}(x)\) is the \(n\)-th derivative of \(g(x)\).
The \(f\left(x, g(x), g'(x), g''(x), \, \dots \, , g^{(n)}(x)\right)\) is just a way to write that there is an expression involving \(x\) and \(g(x), \ g'(x), \ g''(x), \, \dots \, , \text{ and } g^{(n)}(x)\) on the left side of the equality sign in (1). The highest order of derivative, that is the value of \(n\), determines to the order of the equation. The equation is referred to as a \(n\)-th order ODE. Along with (1), some additional conditions of the function \(g(x)\) are typically given for the solution to be unique.
10.2. The trial solution¶
Let the trial solution \(g_t(x)\) be
\[
\begin{equation}
g_t(x) = h_1(x) + h_2(x,N(x,P))
\label{_auto1} \tag{2}
\end{equation}
\]
where \(h_1(x)\) is a function that makes \(g_t(x)\) satisfy a given set of conditions, \(N(x,P)\) a neural network with weights and biases described by \(P\) and \(h_2(x, N(x,P))\) some expression involving the neural network. The role of the function \(h_2(x, N(x,P))\), is to ensure that the output from \(N(x,P)\) is zero when \(g_t(x)\) is evaluated at the values of \(x\) where the given conditions must be satisfied. The function \(h_1(x)\) should alone make \(g_t(x)\) satisfy the conditions.
But what about the network \(N(x,P)\)?
As described previously, an optimization method could be used to minimize the parameters of a neural network, that being its weights and biases, through backward propagation.
10.3. Minimization process¶
For the minimization to be defined, we need to have a cost function at hand to minimize.
It is given that \(f\left(x, \, g(x), \, g'(x), \, g''(x), \, \dots \, , \, g^{(n)}(x)\right)\) should be equal to zero in (1). We can choose to consider the mean squared error as the cost function for an input \(x\). Since we are looking at one input, the cost function is just \(f\) squared. The cost function \(c\left(x, P \right)\) can therefore be expressed as
\[
C\left(x, P\right) = \big(f\left(x, \, g(x), \, g'(x), \, g''(x), \, \dots \, , \, g^{(n)}(x)\right)\big)^2
\]
If \(N\) inputs are given as a vector \(\boldsymbol{x}\) with elements \(x_i\) for \(i = 1,\dots,N\), the cost function becomes
\[
\begin{equation} \label{cost} \tag{3}
C\left(\boldsymbol{x}, P\right) = \frac{1}{N} \sum_{i=1}^N \big(f\left(x_i, \, g(x_i), \, g'(x_i), \, g''(x_i), \, \dots \, , \, g^{(n)}(x_i)\right)\big)^2
\end{equation}
\]
The neural net should then find the parameters \(P\) that minimizes the cost function in (3) for a set of \(N\) training samples \(x_i\).
10.4. Minimizing the cost function using gradient descent and automatic differentiation¶
To perform the minimization using gradient descent, the gradient of \(C\left(\boldsymbol{x}, P\right)\) is needed. It might happen so that finding an analytical expression of the gradient of \(C(\boldsymbol{x}, P)\) from (3) gets too messy, depending on which cost function one desires to use.
Luckily, there exists libraries that makes the job for us through automatic differentiation. Automatic differentiation is a method of finding the derivatives numerically with very high precision.
10.5. Example: Exponential decay¶
An exponential decay of a quantity \(g(x)\) is described by the equation
\[
\begin{equation} \label{solve_expdec} \tag{4}
g'(x) = -\gamma g(x)
\end{equation}
\]
with \(g(0) = g_0\) for some chosen initial value \(g_0\).
The analytical solution of (4) is
\[
\begin{equation}
g(x) = g_0 \exp\left(-\gamma x\right)
\label{_auto2} \tag{5}
\end{equation}
\]
Having an analytical solution at hand, it is possible to use it to compare how well a neural network finds a solution of (4).
10.6. The function to solve for¶
The program will use a neural network to solve
\[
\begin{equation} \label{solveode} \tag{6}
g'(x) = -\gamma g(x)
\end{equation}
\]
where \(g(0) = g_0\) with \(\gamma\) and \(g_0\) being some chosen values.
In this example, \(\gamma = 2\) and \(g_0 = 10\).
10.7. The trial solution¶
To begin with, a trial solution \(g_t(t)\) must be chosen. A general trial solution for ordinary differential equations could be
\[
g_t(x, P) = h_1(x) + h_2(x, N(x, P))
\]
with \(h_1(x)\) ensuring that \(g_t(x)\) satisfies some conditions and \(h_2(x,N(x, P))\) an expression involving \(x\) and the output from the neural network \(N(x,P)\) with \(P \) being the collection of the weights and biases for each layer. For now, it is assumed that the network consists of one input layer, one hidden layer, and one output layer.
10.8. Setup of Network¶
In this network, there are no weights and bias at the input layer, so \(P = \{ P_{\text{hidden}}, P_{\text{output}} \}\). If there are \(N_{\text{hidden} }\) neurons in the hidden layer, then \(P_{\text{hidden}}\) is a \(N_{\text{hidden} } \times (1 + N_{\text{input}})\) matrix, given that there are \(N_{\text{input}}\) neurons in the input layer.
The first column in \(P_{\text{hidden} }\) represents the bias for each neuron in the hidden layer and the second column represents the weights for each neuron in the hidden layer from the input layer. If there are \(N_{\text{output} }\) neurons in the output layer, then \(P_{\text{output}} \) is a \(N_{\text{output} } \times (1 + N_{\text{hidden} })\) matrix.
Its first column represents the bias of each neuron and the remaining columns represents the weights to each neuron.
It is given that \(g(0) = g_0\). The trial solution must fulfill this condition to be a proper solution of (6). A possible way to ensure that \(g_t(0, P) = g_0\), is to let \(F(N(x,P)) = x \cdot N(x,P)\) and \(A(x) = g_0\). This gives the following trial solution:
\[
\begin{equation} \label{trial} \tag{7}
g_t(x, P) = g_0 + x \cdot N(x, P)
\end{equation}
\]
10.9. Reformulating the problem¶
We wish that our neural network manages to minimize a given cost function.
A reformulation of out equation, (6), must therefore be done, such that it describes the problem a neural network can solve for.
The neural network must find the set of weights and biases \(P\) such that the trial solution in (7) satisfies (6).
The trial solution
\[
g_t(x, P) = g_0 + x \cdot N(x, P)
\]
has been chosen such that it already solves the condition \(g(0) = g_0\). What remains, is to find \(P\) such that
\[
\begin{equation} \label{nnmin} \tag{8}
g_t'(x, P) = - \gamma g_t(x, P)
\end{equation}
\]
is fulfilled as best as possible.
10.10. More technicalities¶
The left hand side and right hand side of (8) must be computed separately, and then the neural network must choose weights and biases, contained in \(P\), such that the sides are equal as best as possible. This means that the absolute or squared difference between the sides must be as close to zero, ideally equal to zero. In this case, the difference squared shows to be an appropriate measurement of how erroneous the trial solution is with respect to \(P\) of the neural network.
This gives the following cost function our neural network must solve for:
\[
\min_{P}\Big\{ \big(g_t'(x, P) - ( -\gamma g_t(x, P) \big)^2 \Big\}
\]
(the notation \(\min_{P}\{ f(x, P) \}\) means that we desire to find \(P\) that yields the minimum of \(f(x, P)\))
or, in terms of weights and biases for the hidden and output layer in our network:
\[
\min_{P_{\text{hidden} }, \ P_{\text{output} }}\Big\{ \big(g_t'(x, \{ P_{\text{hidden} }, P_{\text{output} }\}) - ( -\gamma g_t(x, \{ P_{\text{hidden} }, P_{\text{output} }\}) \big)^2 \Big\}
\]
for an input value \(x\).
10.11. More details¶
If the neural network evaluates \(g_t(x, P)\) at more values for \(x\), say \(N\) values \(x_i\) for \(i = 1, \dots, N\), then the total error to minimize becomes
\[
\begin{equation} \label{min} \tag{9}
\min_{P}\Big\{\frac{1}{N} \sum_{i=1}^N \big(g_t'(x_i, P) - ( -\gamma g_t(x_i, P) \big)^2 \Big\}
\end{equation}
\]
Letting \(\boldsymbol{x}\) be a vector with elements \(x_i\) and \(C(\boldsymbol{x}, P) = \frac{1}{N} \sum_i \big(g_t'(x_i, P) - ( -\gamma g_t(x_i, P) \big)^2\) denote the cost function, the minimization problem that our network must solve, becomes
\[
\min_{P} C(\boldsymbol{x}, P)
\]
In terms of \(P_{\text{hidden} }\) and \(P_{\text{output} }\), this could also be expressed as
\[
\min_{P_{\text{hidden} }, \ P_{\text{output} }} C(\boldsymbol{x}, \{P_{\text{hidden} }, P_{\text{output} }\})
\]
10.12. A possible implementation of a neural network¶
For simplicity, it is assumed that the input is an array \(\boldsymbol{x} = (x_1, \dots, x_N)\) with \(N\) elements. It is at these points the neural network should find \(P\) such that it fulfills (9).
First, the neural network must feed forward the inputs. This means that \(\boldsymbol{x}s\) must be passed through an input layer, a hidden layer and a output layer. The input layer in this case, does not need to process the data any further. The input layer will consist of \(N_{\text{input} }\) neurons, passing its element to each neuron in the hidden layer. The number of neurons in the hidden layer will be \(N_{\text{hidden} }\).
10.13. Technicalities¶
For the \(i\)-th in the hidden layer with weight \(w_i^{\text{hidden} }\) and bias \(b_i^{\text{hidden} }\), the weighting from the \(j\)-th neuron at the input layer is:
\[\begin{split}
\begin{aligned}
z_{i,j}^{\text{hidden}} &= b_i^{\text{hidden}} + w_i^{\text{hidden}}x_j \\
&=
\begin{pmatrix}
b_i^{\text{hidden}} & w_i^{\text{hidden}}
\end{pmatrix}
\begin{pmatrix}
1 \\
x_j
\end{pmatrix}
\end{aligned}
\end{split}\]
10.14. Final technicalities I¶
The result after weighting the inputs at the \(i\)-th hidden neuron can be written as a vector:
\[\begin{split}
\begin{aligned}
\boldsymbol{z}_{i}^{\text{hidden}} &= \Big( b_i^{\text{hidden}} + w_i^{\text{hidden}}x_1 , \ b_i^{\text{hidden}} + w_i^{\text{hidden}} x_2, \ \dots \, , \ b_i^{\text{hidden}} + w_i^{\text{hidden}} x_N\Big) \\
&=
\begin{pmatrix}
b_i^{\text{hidden}} & w_i^{\text{hidden}}
\end{pmatrix}
\begin{pmatrix}
1 & 1 & \dots & 1 \\
x_1 & x_2 & \dots & x_N
\end{pmatrix} \\
&= \boldsymbol{p}_{i, \text{hidden}}^T X
\end{aligned}
\end{split}\]
10.15. Final technicalities II¶
The vector \(\boldsymbol{p}_{i, \text{hidden}}^T\) constitutes each row in \(P_{\text{hidden} }\), which contains the weights for the neural network to minimize according to (9).
After having found \(\boldsymbol{z}_{i}^{\text{hidden}} \) for every \(i\)-th neuron within the hidden layer, the vector will be sent to an activation function \(a_i(\boldsymbol{z})\).
In this example, the sigmoid function has been chosen to be the activation function for each hidden neuron:
\[
f(z) = \frac{1}{1 + \exp{(-z)}}
\]
It is possible to use other activations functions for the hidden layer also.
The output \(\boldsymbol{x}_i^{\text{hidden}}\) from each \(i\)-th hidden neuron is:
\[
\boldsymbol{x}_i^{\text{hidden} } = f\big( \boldsymbol{z}_{i}^{\text{hidden}} \big)
\]
The outputs \(\boldsymbol{x}_i^{\text{hidden} } \) are then sent to the output layer.
The output layer consists of one neuron in this case, and combines the output from each of the neurons in the hidden layers. The output layer combines the results from the hidden layer using some weights \(w_i^{\text{output}}\) and biases \(b_i^{\text{output}}\). In this case, it is assumes that the number of neurons in the output layer is one.
10.16. Final technicalities III¶
The procedure of weighting the output neuron \(j\) in the hidden layer to the \(i\)-th neuron in the output layer is similar as for the hidden layer described previously.
\[\begin{split}
\begin{aligned}
z_{1,j}^{\text{output}} & =
\begin{pmatrix}
b_1^{\text{output}} & \boldsymbol{w}_1^{\text{output}}
\end{pmatrix}
\begin{pmatrix}
1 \\
\boldsymbol{x}_j^{\text{hidden}}
\end{pmatrix}
\end{aligned}
\end{split}\]
10.17. Final technicalities IV¶
Expressing \(z_{1,j}^{\text{output}}\) as a vector gives the following way of weighting the inputs from the hidden layer:
\[\begin{split}
\boldsymbol{z}_{1}^{\text{output}} =
\begin{pmatrix}
b_1^{\text{output}} & \boldsymbol{w}_1^{\text{output}}
\end{pmatrix}
\begin{pmatrix}
1 & 1 & \dots & 1 \\
\boldsymbol{x}_1^{\text{hidden}} & \boldsymbol{x}_2^{\text{hidden}} & \dots & \boldsymbol{x}_N^{\text{hidden}}
\end{pmatrix}
\end{split}\]
In this case we seek a continuous range of values since we are approximating a function. This means that after computing \(\boldsymbol{z}_{1}^{\text{output}}\) the neural network has finished its feed forward step, and \(\boldsymbol{z}_{1}^{\text{output}}\) is the final output of the network.
10.18. Back propagation¶
The next step is to decide how the parameters should be changed such that they minimize the cost function.
The chosen cost function for this problem is
\[
C(\boldsymbol{x}, P) = \frac{1}{N} \sum_i \big(g_t'(x_i, P) - ( -\gamma g_t(x_i, P) \big)^2
\]
In order to minimize the cost function, an optimization method must be chosen.
Here, gradient descent with a constant step size has been chosen.
10.19. Gradient descent¶
The idea of the gradient descent algorithm is to update parameters in a direction where the cost function decreases goes to a minimum.
In general, the update of some parameters \(\boldsymbol{\omega}\) given a cost function defined by some weights \(\boldsymbol{\omega}\), \(C(\boldsymbol{x}, \boldsymbol{\omega})\), goes as follows:
\[
\boldsymbol{\omega}_{\text{new} } = \boldsymbol{\omega} - \lambda \nabla_{\boldsymbol{\omega}} C(\boldsymbol{x}, \boldsymbol{\omega})
\]
for a number of iterations or until \( \big|\big| \boldsymbol{\omega}_{\text{new} } - \boldsymbol{\omega} \big|\big|\) becomes smaller than some given tolerance.
The value of \(\lambda\) decides how large steps the algorithm must take in the direction of \( \nabla_{\boldsymbol{\omega}} C(\boldsymbol{x}, \boldsymbol{\omega})\). The notation \(\nabla_{\boldsymbol{\omega}}\) express the gradient with respect to the elements in \(\boldsymbol{\omega}\).
In our case, we have to minimize the cost function \(C(\boldsymbol{x}, P)\) with respect to the two sets of weights and biases, that is for the hidden layer \(P_{\text{hidden} }\) and for the output layer \(P_{\text{output} }\) .
This means that \(P_{\text{hidden} }\) and \(P_{\text{output} }\) is updated by
\[\begin{split}
\begin{aligned}
P_{\text{hidden},\text{new}} &= P_{\text{hidden}} - \lambda \nabla_{P_{\text{hidden}}} C(\boldsymbol{x}, P) \\
P_{\text{output},\text{new}} &= P_{\text{output}} - \lambda \nabla_{P_{\text{output}}} C(\boldsymbol{x}, P)
\end{aligned}
\end{split}\]
10.20. The code for solving the ODE¶
import autograd.numpy as np
from autograd import grad, elementwise_grad
import autograd.numpy.random as npr
from matplotlib import pyplot as plt
def sigmoid(z):
return 1/(1 + np.exp(-z))
# Assuming one input, hidden, and output layer
def neural_network(params, x):
# Find the weights (including and biases) for the hidden and output layer.
# Assume that params is a list of parameters for each layer.
# The biases are the first element for each array in params,
# and the weights are the remaning elements in each array in params.
w_hidden = params[0]
w_output = params[1]
# Assumes input x being an one-dimensional array
num_values = np.size(x)
x = x.reshape(-1, num_values)
# Assume that the input layer does nothing to the input x
x_input = x
## Hidden layer:
# Add a row of ones to include bias
x_input = np.concatenate((np.ones((1,num_values)), x_input ), axis = 0)
z_hidden = np.matmul(w_hidden, x_input)
x_hidden = sigmoid(z_hidden)
## Output layer:
# Include bias:
x_hidden = np.concatenate((np.ones((1,num_values)), x_hidden ), axis = 0)
z_output = np.matmul(w_output, x_hidden)
x_output = z_output
return x_output
# The trial solution using the deep neural network:
def g_trial(x,params, g0 = 10):
return g0 + x*neural_network(params,x)
# The right side of the ODE:
def g(x, g_trial, gamma = 2):
return -gamma*g_trial
# The cost function:
def cost_function(P, x):
# Evaluate the trial function with the current parameters P
g_t = g_trial(x,P)
# Find the derivative w.r.t x of the neural network
d_net_out = elementwise_grad(neural_network,1)(P,x)
# Find the derivative w.r.t x of the trial function
d_g_t = elementwise_grad(g_trial,0)(x,P)
# The right side of the ODE
func = g(x, g_t)
err_sqr = (d_g_t - func)**2
cost_sum = np.sum(err_sqr)
return cost_sum / np.size(err_sqr)
# Solve the exponential decay ODE using neural network with one input, hidden, and output layer
def solve_ode_neural_network(x, num_neurons_hidden, num_iter, lmb):
## Set up initial weights and biases
# For the hidden layer
p0 = npr.randn(num_neurons_hidden, 2 )
# For the output layer
p1 = npr.randn(1, num_neurons_hidden + 1 ) # +1 since bias is included
P = [p0, p1]
print('Initial cost: %g'%cost_function(P, x))
## Start finding the optimal weights using gradient descent
# Find the Python function that represents the gradient of the cost function
# w.r.t the 0-th input argument -- that is the weights and biases in the hidden and output layer
cost_function_grad = grad(cost_function,0)
# Let the update be done num_iter times
for i in range(num_iter):
# Evaluate the gradient at the current weights and biases in P.
# The cost_grad consist now of two arrays;
# one for the gradient w.r.t P_hidden and
# one for the gradient w.r.t P_output
cost_grad = cost_function_grad(P, x)
P[0] = P[0] - lmb * cost_grad[0]
P[1] = P[1] - lmb * cost_grad[1]
print('Final cost: %g'%cost_function(P, x))
return P
def g_analytic(x, gamma = 2, g0 = 10):
return g0*np.exp(-gamma*x)
# Solve the given problem
if __name__ == '__main__':
# Set seed such that the weight are initialized
# with same weights and biases for every run.
npr.seed(15)
## Decide the vales of arguments to the function to solve
N = 10
x = np.linspace(0, 1, N)
## Set up the initial parameters
num_hidden_neurons = 10
num_iter = 10000
lmb = 0.001
# Use the network
P = solve_ode_neural_network(x, num_hidden_neurons, num_iter, lmb)
# Print the deviation from the trial solution and true solution
res = g_trial(x,P)
res_analytical = g_analytic(x)
print('Max absolute difference: %g'%np.max(np.abs(res - res_analytical)))
# Plot the results
plt.figure(figsize=(10,10))
plt.title('Performance of neural network solving an ODE compared to the analytical solution')
plt.plot(x, res_analytical)
plt.plot(x, res[0,:])
plt.legend(['analytical','nn'])
plt.xlabel('x')
plt.ylabel('g(x)')
plt.show()
Initial cost: 367.01
Final cost: 0.0666807
Max absolute difference: 0.0437499
10.22. Example: Population growth¶
A logistic model of population growth assumes that a population converges toward an equilibrium. The population growth can be modeled by
\[
\begin{equation} \label{log} \tag{10}
g'(t) = \alpha g(t)(A - g(t))
\end{equation}
\]
where \(g(t)\) is the population density at time \(t\), \(\alpha > 0\) the growth rate and \(A > 0\) is the maximum population number in the environment. Also, at \(t = 0\) the population has the size \(g(0) = g_0\), where \(g_0\) is some chosen constant.
In this example, similar network as for the exponential decay using Autograd has been used to solve the equation. However, as the implementation might suffer from e.g numerical instability and high execution time (this might be more apparent in the examples solving PDEs), using a library like TensorFlow is recommended. Here, we stay with a more simple approach and implement for comparison, the simple forward Euler method.
10.23. Setting up the problem¶
Here, we will model a population \(g(t)\) in an environment having carrying capacity \(A\). The population follows the model
\[
\begin{equation} \label{solveode_population} \tag{11}
g'(t) = \alpha g(t)(A - g(t))
\end{equation}
\]
where \(g(0) = g_0\).
In this example, we let \(\alpha = 2\), \(A = 1\), and \(g_0 = 1.2\).
10.24. The trial solution¶
We will get a slightly different trial solution, as the boundary conditions are different compared to the case for exponential decay.
A possible trial solution satisfying the condition \(g(0) = g_0\) could be
\[
h_1(t) = g_0 + t \cdot N(t,P)
\]
with \(N(t,P)\) being the output from the neural network with weights and biases for each layer collected in the set \(P\).
The analytical solution is
\[
g(t) = \frac{Ag_0}{g_0 + (A - g_0)\exp(-\alpha A t)}
\]
10.25. The program using Autograd¶
The network will be the similar as for the exponential decay example, but with some small modifications for our problem.
import autograd.numpy as np
from autograd import grad, elementwise_grad
import autograd.numpy.random as npr
from matplotlib import pyplot as plt
def sigmoid(z):
return 1/(1 + np.exp(-z))
# Function to get the parameters.
# Done such that one can easily change the paramaters after one's liking.
def get_parameters():
alpha = 2
A = 1
g0 = 1.2
return alpha, A, g0
def deep_neural_network(P, x):
# N_hidden is the number of hidden layers
N_hidden = np.size(P) - 1 # -1 since params consist of parameters to all the hidden layers AND the output layer
# Assumes input x being an one-dimensional array
num_values = np.size(x)
x = x.reshape(-1, num_values)
# Assume that the input layer does nothing to the input x
x_input = x
# Due to multiple hidden layers, define a variable referencing to the
# output of the previous layer:
x_prev = x_input
## Hidden layers:
for l in range(N_hidden):
# From the list of parameters P; find the correct weigths and bias for this layer
w_hidden = P[l]
# Add a row of ones to include bias
x_prev = np.concatenate((np.ones((1,num_values)), x_prev ), axis = 0)
z_hidden = np.matmul(w_hidden, x_prev)
x_hidden = sigmoid(z_hidden)
# Update x_prev such that next layer can use the output from this layer
x_prev = x_hidden
## Output layer:
# Get the weights and bias for this layer
w_output = P[-1]
# Include bias:
x_prev = np.concatenate((np.ones((1,num_values)), x_prev), axis = 0)
z_output = np.matmul(w_output, x_prev)
x_output = z_output
return x_output
def cost_function_deep(P, x):
# Evaluate the trial function with the current parameters P
g_t = g_trial_deep(x,P)
# Find the derivative w.r.t x of the trial function
d_g_t = elementwise_grad(g_trial_deep,0)(x,P)
# The right side of the ODE
func = f(x, g_t)
err_sqr = (d_g_t - func)**2
cost_sum = np.sum(err_sqr)
return cost_sum / np.size(err_sqr)
# The right side of the ODE:
def f(x, g_trial):
alpha,A, g0 = get_parameters()
return alpha*g_trial*(A - g_trial)
# The trial solution using the deep neural network:
def g_trial_deep(x, params):
alpha,A, g0 = get_parameters()
return g0 + x*deep_neural_network(params,x)
# The analytical solution:
def g_analytic(t):
alpha,A, g0 = get_parameters()
return A*g0/(g0 + (A - g0)*np.exp(-alpha*A*t))
def solve_ode_deep_neural_network(x, num_neurons, num_iter, lmb):
# num_hidden_neurons is now a list of number of neurons within each hidden layer
# Find the number of hidden layers:
N_hidden = np.size(num_neurons)
## Set up initial weigths and biases
# Initialize the list of parameters:
P = [None]*(N_hidden + 1) # + 1 to include the output layer
P[0] = npr.randn(num_neurons[0], 2 )
for l in range(1,N_hidden):
P[l] = npr.randn(num_neurons[l], num_neurons[l-1] + 1) # +1 to include bias
# For the output layer
P[-1] = npr.randn(1, num_neurons[-1] + 1 ) # +1 since bias is included
print('Initial cost: %g'%cost_function_deep(P, x))
## Start finding the optimal weigths using gradient descent
# Find the Python function that represents the gradient of the cost function
# w.r.t the 0-th input argument -- that is the weights and biases in the hidden and output layer
cost_function_deep_grad = grad(cost_function_deep,0)
# Let the update be done num_iter times
for i in range(num_iter):
# Evaluate the gradient at the current weights and biases in P.
# The cost_grad consist now of N_hidden + 1 arrays; the gradient w.r.t the weights and biases
# in the hidden layers and output layers evaluated at x.
cost_deep_grad = cost_function_deep_grad(P, x)
for l in range(N_hidden+1):
P[l] = P[l] - lmb * cost_deep_grad[l]
print('Final cost: %g'%cost_function_deep(P, x))
return P
if __name__ == '__main__':
npr.seed(4155)
## Decide the vales of arguments to the function to solve
Nt = 10
T = 1
t = np.linspace(0,T, Nt)
## Set up the initial parameters
num_hidden_neurons = [100, 50, 25]
num_iter = 1000
lmb = 1e-3
P = solve_ode_deep_neural_network(t, num_hidden_neurons, num_iter, lmb)
g_dnn_ag = g_trial_deep(t,P)
g_analytical = g_analytic(t)
# Find the maximum absolute difference between the solutons:
diff_ag = np.max(np.abs(g_dnn_ag - g_analytical))
print("The max absolute difference between the solutions is: %g"%diff_ag)
plt.figure(figsize=(10,10))
plt.title('Performance of neural network solving an ODE compared to the analytical solution')
plt.plot(t, g_analytical)
plt.plot(t, g_dnn_ag[0,:])
plt.legend(['analytical','nn'])
plt.xlabel('t')
plt.ylabel('g(t)')
plt.show()
10.26. Using forward Euler to solve the ODE¶
A straightforward way of solving an ODE numerically, is to use Euler’s method.
Euler’s method uses Taylor series to approximate the value at a function \(f\) at a step \(\Delta x\) from \(x\):
\[
f(x + \Delta x) \approx f(x) + \Delta x f'(x)
\]
In our case, using Euler’s method to approximate the value of \(g\) at a step \(\Delta t\) from \(t\) yields
\[\begin{split}
\begin{aligned}
g(t + \Delta t) &\approx g(t) + \Delta t g'(t) \\
&= g(t) + \Delta t \big(\alpha g(t)(A - g(t))\big)
\end{aligned}
\end{split}\]
along with the condition that \(g(0) = g_0\).
Let \(t_i = i \cdot \Delta t\) where \(\Delta t = \frac{T}{N_t-1}\) where \(T\) is the final time our solver must solve for and \(N_t\) the number of values for \(t \in [0, T]\) for \(i = 0, \dots, N_t-1\).
For \(i \geq 1\), we have that
\[\begin{split}
\begin{aligned}
t_i &= i\Delta t \\
&= (i - 1)\Delta t + \Delta t \\
&= t_{i-1} + \Delta t
\end{aligned}
\end{split}\]
Now, if \(g_i = g(t_i)\) then
\[\begin{split}
\begin{equation}
\begin{aligned}
g_i &= g(t_i) \\
&= g(t_{i-1} + \Delta t) \\
&\approx g(t_{i-1}) + \Delta t \big(\alpha g(t_{i-1})(A - g(t_{i-1}))\big) \\
&= g_{i-1} + \Delta t \big(\alpha g_{i-1}(A - g_{i-1})\big)
\end{aligned}
\end{equation} \label{odenum} \tag{12}
\end{split}\]
for \(i \geq 1\) and \(g_0 = g(t_0) = g(0) = g_0\).
Equation (12) could be implemented in the following way, extending the program that uses the network using Autograd:
# Assume that all function definitions from the example program using Autograd
# are located here.
if __name__ == '__main__':
npr.seed(4155)
## Decide the vales of arguments to the function to solve
Nt = 10
T = 1
t = np.linspace(0,T, Nt)
## Set up the initial parameters
num_hidden_neurons = [100,50,25]
num_iter = 1000
lmb = 1e-3
P = solve_ode_deep_neural_network(t, num_hidden_neurons, num_iter, lmb)
g_dnn_ag = g_trial_deep(t,P)
g_analytical = g_analytic(t)
# Find the maximum absolute difference between the solutons:
diff_ag = np.max(np.abs(g_dnn_ag - g_analytical))
print("The max absolute difference between the solutions is: %g"%diff_ag)
plt.figure(figsize=(10,10))
plt.title('Performance of neural network solving an ODE compared to the analytical solution')
plt.plot(t, g_analytical)
plt.plot(t, g_dnn_ag[0,:])
plt.legend(['analytical','nn'])
plt.xlabel('t')
plt.ylabel('g(t)')
## Find an approximation to the funtion using forward Euler
alpha, A, g0 = get_parameters()
dt = T/(Nt - 1)
# Perform forward Euler to solve the ODE
g_euler = np.zeros(Nt)
g_euler[0] = g0
for i in range(1,Nt):
g_euler[i] = g_euler[i-1] + dt*(alpha*g_euler[i-1]*(A - g_euler[i-1]))
# Print the errors done by each method
diff1 = np.max(np.abs(g_euler - g_analytical))
diff2 = np.max(np.abs(g_dnn_ag[0,:] - g_analytical))
print('Max absolute difference between Euler method and analytical: %g'%diff1)
print('Max absolute difference between deep neural network and analytical: %g'%diff2)
# Plot results
plt.figure(figsize=(10,10))
plt.plot(t,g_euler)
plt.plot(t,g_analytical)
plt.plot(t,g_dnn_ag[0,:])
plt.legend(['euler','analytical','dnn'])
plt.xlabel('Time t')
plt.ylabel('g(t)')
plt.show()
10.27. Example: Solving the one dimensional Poisson equation¶
The Poisson equation for \(g(x)\) in one dimension is
\[
\begin{equation} \label{poisson} \tag{13}
-g''(x) = f(x)
\end{equation}
\]
where \(f(x)\) is a given function for \(x \in (0,1)\).
The conditions that \(g(x)\) is chosen to fulfill, are
\[\begin{split}
\begin{align*}
g(0) &= 0 \\
g(1) &= 0
\end{align*}
\end{split}\]
This equation can be solved numerically using programs where e.g Autograd and TensorFlow are used. The results from the networks can then be compared to the analytical solution. In addition, it could be interesting to see how a typical method for numerically solving second order ODEs compares to the neural networks.
10.28. The specific equation to solve for¶
Here, the function \(g(x)\) to solve for follows the equation
\[
-g''(x) = f(x),\qquad x \in (0,1)
\]
where \(f(x)\) is a given function, along with the chosen conditions
\[
\begin{aligned}
g(0) = g(1) = 0
\end{aligned}\label{cond} \tag{14}
\]
In this example, we consider the case when \(f(x) = (3x + x^2)\exp(x)\).
For this case, a possible trial solution satisfying the conditions could be
\[
g_t(x) = x \cdot (1-x) \cdot N(P,x)
\]
The analytical solution for this problem is
\[
g(x) = x(1 - x)\exp(x)
\]
10.29. Solving the equation using Autograd¶
import autograd.numpy as np
from autograd import grad, elementwise_grad
import autograd.numpy.random as npr
from matplotlib import pyplot as plt
def sigmoid(z):
return 1/(1 + np.exp(-z))
def deep_neural_network(deep_params, x):
# N_hidden is the number of hidden layers
N_hidden = np.size(deep_params) - 1 # -1 since params consist of parameters to all the hidden layers AND the output layer
# Assumes input x being an one-dimensional array
num_values = np.size(x)
x = x.reshape(-1, num_values)
# Assume that the input layer does nothing to the input x
x_input = x
# Due to multiple hidden layers, define a variable referencing to the
# output of the previous layer:
x_prev = x_input
## Hidden layers:
for l in range(N_hidden):
# From the list of parameters P; find the correct weigths and bias for this layer
w_hidden = deep_params[l]
# Add a row of ones to include bias
x_prev = np.concatenate((np.ones((1,num_values)), x_prev ), axis = 0)
z_hidden = np.matmul(w_hidden, x_prev)
x_hidden = sigmoid(z_hidden)
# Update x_prev such that next layer can use the output from this layer
x_prev = x_hidden
## Output layer:
# Get the weights and bias for this layer
w_output = deep_params[-1]
# Include bias:
x_prev = np.concatenate((np.ones((1,num_values)), x_prev), axis = 0)
z_output = np.matmul(w_output, x_prev)
x_output = z_output
return x_output
def solve_ode_deep_neural_network(x, num_neurons, num_iter, lmb):
# num_hidden_neurons is now a list of number of neurons within each hidden layer
# Find the number of hidden layers:
N_hidden = np.size(num_neurons)
## Set up initial weigths and biases
# Initialize the list of parameters:
P = [None]*(N_hidden + 1) # + 1 to include the output layer
P[0] = npr.randn(num_neurons[0], 2 )
for l in range(1,N_hidden):
P[l] = npr.randn(num_neurons[l], num_neurons[l-1] + 1) # +1 to include bias
# For the output layer
P[-1] = npr.randn(1, num_neurons[-1] + 1 ) # +1 since bias is included
print('Initial cost: %g'%cost_function_deep(P, x))
## Start finding the optimal weigths using gradient descent
# Find the Python function that represents the gradient of the cost function
# w.r.t the 0-th input argument -- that is the weights and biases in the hidden and output layer
cost_function_deep_grad = grad(cost_function_deep,0)
# Let the update be done num_iter times
for i in range(num_iter):
# Evaluate the gradient at the current weights and biases in P.
# The cost_grad consist now of N_hidden + 1 arrays; the gradient w.r.t the weights and biases
# in the hidden layers and output layers evaluated at x.
cost_deep_grad = cost_function_deep_grad(P, x)
for l in range(N_hidden+1):
P[l] = P[l] - lmb * cost_deep_grad[l]
print('Final cost: %g'%cost_function_deep(P, x))
return P
## Set up the cost function specified for this Poisson equation:
# The right side of the ODE
def f(x):
return (3*x + x**2)*np.exp(x)
def cost_function_deep(P, x):
# Evaluate the trial function with the current parameters P
g_t = g_trial_deep(x,P)
# Find the derivative w.r.t x of the trial function
d2_g_t = elementwise_grad(elementwise_grad(g_trial_deep,0))(x,P)
right_side = f(x)
err_sqr = (-d2_g_t - right_side)**2
cost_sum = np.sum(err_sqr)
return cost_sum/np.size(err_sqr)
# The trial solution:
def g_trial_deep(x,P):
return x*(1-x)*deep_neural_network(P,x)
# The analytic solution;
def g_analytic(x):
return x*(1-x)*np.exp(x)
if __name__ == '__main__':
npr.seed(4155)
## Decide the vales of arguments to the function to solve
Nx = 10
x = np.linspace(0,1, Nx)
## Set up the initial parameters
num_hidden_neurons = [200,100]
num_iter = 1000
lmb = 1e-3
P = solve_ode_deep_neural_network(x, num_hidden_neurons, num_iter, lmb)
g_dnn_ag = g_trial_deep(x,P)
g_analytical = g_analytic(x)
# Find the maximum absolute difference between the solutons:
max_diff = np.max(np.abs(g_dnn_ag - g_analytical))
print("The max absolute difference between the solutions is: %g"%max_diff)
plt.figure(figsize=(10,10))
plt.title('Performance of neural network solving an ODE compared to the analytical solution')
plt.plot(x, g_analytical)
plt.plot(x, g_dnn_ag[0,:])
plt.legend(['analytical','nn'])
plt.xlabel('x')
plt.ylabel('g(x)')
plt.show()
10.30. Comparing with a numerical scheme¶
The Poisson equation is possible to solve using Taylor series to approximate the second derivative.
Using Taylor series, the second derivative can be expressed as
\[
g''(x) = \frac{g(x + \Delta x) - 2g(x) + g(x-\Delta x)}{\Delta x^2} + E_{\Delta x}(x)
\]
where \(\Delta x\) is a small step size and \(E_{\Delta x}(x)\) being the error term.
Looking away from the error terms gives an approximation to the second derivative:
\[
\begin{equation} \label{approx} \tag{15}
g''(x) \approx \frac{g(x + \Delta x) - 2g(x) + g(x-\Delta x)}{\Delta x^2}
\end{equation}
\]
If \(x_i = i \Delta x = x_{i-1} + \Delta x\) and \(g_i = g(x_i)\) for \(i = 1,\dots N_x - 2\) with \(N_x\) being the number of values for \(x\), (15) becomes
\[\begin{split}
\begin{aligned}
g''(x_i) &\approx \frac{g(x_i + \Delta x) - 2g(x_i) + g(x_i -\Delta x)}{\Delta x^2} \\
&= \frac{g_{i+1} - 2g_i + g_{i-1}}{\Delta x^2}
\end{aligned}
\end{split}\]
Since we know from our problem that
\[\begin{split}
\begin{aligned}
-g''(x) &= f(x) \\
&= (3x + x^2)\exp(x)
\end{aligned}
\end{split}\]
along with the conditions \(g(0) = g(1) = 0\), the following scheme can be used to find an approximate solution for \(g(x)\) numerically:
\[\begin{split}
\begin{equation}
\begin{aligned}
-\Big( \frac{g_{i+1} - 2g_i + g_{i-1}}{\Delta x^2} \Big) &= f(x_i) \\
-g_{i+1} + 2g_i - g_{i-1} &= \Delta x^2 f(x_i)
\end{aligned}
\end{equation} \label{odesys} \tag{16}
\end{split}\]
for \(i = 1, \dots, N_x - 2\) where \(g_0 = g_{N_x - 1} = 0\) and \(f(x_i) = (3x_i + x_i^2)\exp(x_i)\), which is given for our specific problem.
The equation can be rewritten into a matrix equation:
\[\begin{split}
\begin{aligned}
\begin{pmatrix}
2 & -1 & 0 & \dots & 0 \\
-1 & 2 & -1 & \dots & 0 \\
\vdots & & \ddots & & \vdots \\
0 & \dots & -1 & 2 & -1 \\
0 & \dots & 0 & -1 & 2\\
\end{pmatrix}
\begin{pmatrix}
g_1 \\
g_2 \\
\vdots \\
g_{N_x - 3} \\
g_{N_x - 2}
\end{pmatrix}
&=
\Delta x^2
\begin{pmatrix}
f(x_1) \\
f(x_2) \\
\vdots \\
f(x_{N_x - 3}) \\
f(x_{N_x - 2})
\end{pmatrix} \\
\boldsymbol{A}\boldsymbol{g} &= \boldsymbol{f},
\end{aligned}
\end{split}\]
which makes it possible to solve for the vector \(\boldsymbol{g}\).
10.31. Setting up the code¶
We can then compare the result from this numerical scheme with the output from our network using Autograd:
import autograd.numpy as np
from autograd import grad, elementwise_grad
import autograd.numpy.random as npr
from matplotlib import pyplot as plt
def sigmoid(z):
return 1/(1 + np.exp(-z))
def deep_neural_network(deep_params, x):
# N_hidden is the number of hidden layers
N_hidden = np.size(deep_params) - 1 # -1 since params consist of parameters to all the hidden layers AND the output layer
# Assumes input x being an one-dimensional array
num_values = np.size(x)
x = x.reshape(-1, num_values)
# Assume that the input layer does nothing to the input x
x_input = x
# Due to multiple hidden layers, define a variable referencing to the
# output of the previous layer:
x_prev = x_input
## Hidden layers:
for l in range(N_hidden):
# From the list of parameters P; find the correct weigths and bias for this layer
w_hidden = deep_params[l]
# Add a row of ones to include bias
x_prev = np.concatenate((np.ones((1,num_values)), x_prev ), axis = 0)
z_hidden = np.matmul(w_hidden, x_prev)
x_hidden = sigmoid(z_hidden)
# Update x_prev such that next layer can use the output from this layer
x_prev = x_hidden
## Output layer:
# Get the weights and bias for this layer
w_output = deep_params[-1]
# Include bias:
x_prev = np.concatenate((np.ones((1,num_values)), x_prev), axis = 0)
z_output = np.matmul(w_output, x_prev)
x_output = z_output
return x_output
def solve_ode_deep_neural_network(x, num_neurons, num_iter, lmb):
# num_hidden_neurons is now a list of number of neurons within each hidden layer
# Find the number of hidden layers:
N_hidden = np.size(num_neurons)
## Set up initial weigths and biases
# Initialize the list of parameters:
P = [None]*(N_hidden + 1) # + 1 to include the output layer
P[0] = npr.randn(num_neurons[0], 2 )
for l in range(1,N_hidden):
P[l] = npr.randn(num_neurons[l], num_neurons[l-1] + 1) # +1 to include bias
# For the output layer
P[-1] = npr.randn(1, num_neurons[-1] + 1 ) # +1 since bias is included
print('Initial cost: %g'%cost_function_deep(P, x))
## Start finding the optimal weigths using gradient descent
# Find the Python function that represents the gradient of the cost function
# w.r.t the 0-th input argument -- that is the weights and biases in the hidden and output layer
cost_function_deep_grad = grad(cost_function_deep,0)
# Let the update be done num_iter times
for i in range(num_iter):
# Evaluate the gradient at the current weights and biases in P.
# The cost_grad consist now of N_hidden + 1 arrays; the gradient w.r.t the weights and biases
# in the hidden layers and output layers evaluated at x.
cost_deep_grad = cost_function_deep_grad(P, x)
for l in range(N_hidden+1):
P[l] = P[l] - lmb * cost_deep_grad[l]
print('Final cost: %g'%cost_function_deep(P, x))
return P
## Set up the cost function specified for this Poisson equation:
# The right side of the ODE
def f(x):
return (3*x + x**2)*np.exp(x)
def cost_function_deep(P, x):
# Evaluate the trial function with the current parameters P
g_t = g_trial_deep(x,P)
# Find the derivative w.r.t x of the trial function
d2_g_t = elementwise_grad(elementwise_grad(g_trial_deep,0))(x,P)
right_side = f(x)
err_sqr = (-d2_g_t - right_side)**2
cost_sum = np.sum(err_sqr)
return cost_sum/np.size(err_sqr)
# The trial solution:
def g_trial_deep(x,P):
return x*(1-x)*deep_neural_network(P,x)
# The analytic solution;
def g_analytic(x):
return x*(1-x)*np.exp(x)
if __name__ == '__main__':
npr.seed(4155)
## Decide the vales of arguments to the function to solve
Nx = 10
x = np.linspace(0,1, Nx)
## Set up the initial parameters
num_hidden_neurons = [200,100]
num_iter = 1000
lmb = 1e-3
P = solve_ode_deep_neural_network(x, num_hidden_neurons, num_iter, lmb)
g_dnn_ag = g_trial_deep(x,P)
g_analytical = g_analytic(x)
# Find the maximum absolute difference between the solutons:
plt.figure(figsize=(10,10))
plt.title('Performance of neural network solving an ODE compared to the analytical solution')
plt.plot(x, g_analytical)
plt.plot(x, g_dnn_ag[0,:])
plt.legend(['analytical','nn'])
plt.xlabel('x')
plt.ylabel('g(x)')
## Perform the computation using the numerical scheme
dx = 1/(Nx - 1)
# Set up the matrix A
A = np.zeros((Nx-2,Nx-2))
A[0,0] = 2
A[0,1] = -1
for i in range(1,Nx-3):
A[i,i-1] = -1
A[i,i] = 2
A[i,i+1] = -1
A[Nx - 3, Nx - 4] = -1
A[Nx - 3, Nx - 3] = 2
# Set up the vector f
f_vec = dx**2 * f(x[1:-1])
# Solve the equation
g_res = np.linalg.solve(A,f_vec)
g_vec = np.zeros(Nx)
g_vec[1:-1] = g_res
# Print the differences between each method
max_diff1 = np.max(np.abs(g_dnn_ag - g_analytical))
max_diff2 = np.max(np.abs(g_vec - g_analytical))
print("The max absolute difference between the analytical solution and DNN Autograd: %g"%max_diff1)
print("The max absolute difference between the analytical solution and numerical scheme: %g"%max_diff2)
# Plot the results
plt.figure(figsize=(10,10))
plt.plot(x,g_vec)
plt.plot(x,g_analytical)
plt.plot(x,g_dnn_ag[0,:])
plt.legend(['numerical scheme','analytical','dnn'])
plt.show()
10.32. Partial Differential Equations¶
A partial differential equation (PDE) has a solution here the function is defined by multiple variables. The equation may involve all kinds of combinations of which variables the function is differentiated with respect to.
In general, a partial differential equation for a function \(g(x_1,\dots,x_N)\) with \(N\) variables may be expressed as
\[
\begin{equation} \label{PDE} \tag{17}
f\left(x_1, \, \dots \, , x_N, \frac{\partial g(x_1,\dots,x_N) }{\partial x_1}, \dots , \frac{\partial g(x_1,\dots,x_N) }{\partial x_N}, \frac{\partial g(x_1,\dots,x_N) }{\partial x_1\partial x_2}, \, \dots \, , \frac{\partial^n g(x_1,\dots,x_N) }{\partial x_N^n} \right) = 0
\end{equation}
\]
where \(f\) is an expression involving all kinds of possible mixed derivatives of \(g(x_1,\dots,x_N)\) up to an order \(n\). In order for the solution to be unique, some additional conditions must also be given.
10.33. Type of problem¶
The problem our network must solve for, is similar to the ODE case. We must have a trial solution \(g_t\) at hand.
For instance, the trial solution could be expressed as
\[
\begin{align*}
g_t(x_1,\dots,x_N) = h_1(x_1,\dots,x_N) + h_2(x_1,\dots,x_N,N(x_1,\dots,x_N,P))
\end{align*}
\]
where \(h_1(x_1,\dots,x_N)\) is a function that ensures \(g_t(x_1,\dots,x_N)\) satisfies some given conditions. The neural network \(N(x_1,\dots,x_N,P)\) has weights and biases described by \(P\) and \(h_2(x_1,\dots,x_N,N(x_1,\dots,x_N,P))\) is an expression using the output from the neural network in some way.
The role of the function \(h_2(x_1,\dots,x_N,N(x_1,\dots,x_N,P))\), is to ensure that the output of \(N(x_1,\dots,x_N,P)\) is zero when \(g_t(x_1,\dots,x_N)\) is evaluated at the values of \(x_1,\dots,x_N\) where the given conditions must be satisfied. The function \(h_1(x_1,\dots,x_N)\) should alone make \(g_t(x_1,\dots,x_N)\) satisfy the conditions.
10.34. Network requirements¶
The network tries then the minimize the cost function following the same ideas as described for the ODE case, but now with more than one variables to consider. The concept still remains the same; find a set of parameters \(P\) such that the expression \(f\) in (17) is as close to zero as possible.
As for the ODE case, the cost function is the mean squared error that the network must try to minimize. The cost function for the network to minimize is
\[
C\left(x_1, \dots, x_N, P\right) = \left( f\left(x_1, \, \dots \, , x_N, \frac{\partial g(x_1,\dots,x_N) }{\partial x_1}, \dots , \frac{\partial g(x_1,\dots,x_N) }{\partial x_N}, \frac{\partial g(x_1,\dots,x_N) }{\partial x_1\partial x_2}, \, \dots \, , \frac{\partial^n g(x_1,\dots,x_N) }{\partial x_N^n} \right) \right)^2
\]
10.35. More details¶
If we let \(\boldsymbol{x} = \big( x_1, \dots, x_N \big)\) be an array containing the values for \(x_1, \dots, x_N\) respectively, the cost function can be reformulated into the following:
\[
C\left(\boldsymbol{x}, P\right) = f\left( \left( \boldsymbol{x}, \frac{\partial g(\boldsymbol{x}) }{\partial x_1}, \dots , \frac{\partial g(\boldsymbol{x}) }{\partial x_N}, \frac{\partial g(\boldsymbol{x}) }{\partial x_1\partial x_2}, \, \dots \, , \frac{\partial^n g(\boldsymbol{x}) }{\partial x_N^n} \right) \right)^2
\]
If we also have \(M\) different sets of values for \(x_1, \dots, x_N\), that is \(\boldsymbol{x}_i = \big(x_1^{(i)}, \dots, x_N^{(i)}\big)\) for \(i = 1,\dots,M\) being the rows in matrix \(X\), the cost function can be generalized into
\[
C\left(X, P \right) = \sum_{i=1}^M f\left( \left( \boldsymbol{x}_i, \frac{\partial g(\boldsymbol{x}_i) }{\partial x_1}, \dots , \frac{\partial g(\boldsymbol{x}_i) }{\partial x_N}, \frac{\partial g(\boldsymbol{x}_i) }{\partial x_1\partial x_2}, \, \dots \, , \frac{\partial^n g(\boldsymbol{x}_i) }{\partial x_N^n} \right) \right)^2.
\]
10.36. Example: The diffusion equation¶
In one spatial dimension, the equation reads
\[
\frac{\partial g(x,t)}{\partial t} = \frac{\partial^2 g(x,t)}{\partial x^2}
\]
where a possible choice of conditions are
\[\begin{split}
\begin{align*}
g(0,t) &= 0 ,\qquad t \geq 0 \\
g(1,t) &= 0, \qquad t \geq 0 \\
g(x,0) &= u(x),\qquad x\in [0,1]
\end{align*}
\end{split}\]
with \(u(x)\) being some given function.
10.37. Defining the problem¶
For this case, we want to find \(g(x,t)\) such that
\[
\begin{equation}
\frac{\partial g(x,t)}{\partial t} = \frac{\partial^2 g(x,t)}{\partial x^2}
\end{equation} \label{diffonedim} \tag{18}
\]
and
\[\begin{split}
\begin{align*}
g(0,t) &= 0 ,\qquad t \geq 0 \\
g(1,t) &= 0, \qquad t \geq 0 \\
g(x,0) &= u(x),\qquad x\in [0,1]
\end{align*}
\end{split}\]
with \(u(x) = \sin(\pi x)\).
First, let us set up the deep neural network. The deep neural network will follow the same structure as discussed in the examples solving the ODEs. First, we will look into how Autograd could be used in a network tailored to solve for bivariate functions.
10.38. Setting up the network using Autograd¶
The only change to do here, is to extend our network such that functions of multiple parameters are correctly handled. In this case we have two variables in our function to solve for, that is time \(t\) and position \(x\). The variables will be represented by a one-dimensional array in the program. The program will evaluate the network at each possible pair \((x,t)\), given an array for the desired \(x\)-values and \(t\)-values to approximate the solution at.
def sigmoid(z):
return 1/(1 + np.exp(-z))
def deep_neural_network(deep_params, x):
# x is now a point and a 1D numpy array; make it a column vector
num_coordinates = np.size(x,0)
x = x.reshape(num_coordinates,-1)
num_points = np.size(x,1)
# N_hidden is the number of hidden layers
N_hidden = np.size(deep_params) - 1 # -1 since params consist of parameters to all the hidden layers AND the output layer
# Assume that the input layer does nothing to the input x
x_input = x
x_prev = x_input
## Hidden layers:
for l in range(N_hidden):
# From the list of parameters P; find the correct weigths and bias for this layer
w_hidden = deep_params[l]
# Add a row of ones to include bias
x_prev = np.concatenate((np.ones((1,num_points)), x_prev ), axis = 0)
z_hidden = np.matmul(w_hidden, x_prev)
x_hidden = sigmoid(z_hidden)
# Update x_prev such that next layer can use the output from this layer
x_prev = x_hidden
## Output layer:
# Get the weights and bias for this layer
w_output = deep_params[-1]
# Include bias:
x_prev = np.concatenate((np.ones((1,num_points)), x_prev), axis = 0)
z_output = np.matmul(w_output, x_prev)
x_output = z_output
return x_output[0][0]
10.39. Setting up the network using Autograd; The trial solution¶
The cost function must then iterate through the given arrays containing values for \(x\) and \(t\), defines a point \((x,t)\) the deep neural network and the trial solution is evaluated at, and then finds the Jacobian of the trial solution.
A possible trial solution for this PDE is
\[
g_t(x,t) = h_1(x,t) + x(1-x)tN(x,t,P)
\]
with \(A(x,t)\) being a function ensuring that \(g_t(x,t)\) satisfies our given conditions, and \(N(x,t,P)\) being the output from the deep neural network using weights and biases for each layer from \(P\).
To fulfill the conditions, \(A(x,t)\) could be:
\[
h_1(x,t) = (1-t)\Big(u(x) - \big((1-x)u(0) + x u(1)\big)\Big) = (1-t)u(x) = (1-t)\sin(\pi x)
\]
since \((0) = u(1) = 0\) and \(u(x) = \sin(\pi x)\).
10.40. Why the jacobian?¶
The Jacobian is used because the program must find the derivative of the trial solution with respect to \(x\) and \(t\).
This gives the necessity of computing the Jacobian matrix, as we want to evaluate the gradient with respect to \(x\) and \(t\) (note that the Jacobian of a scalar-valued multivariate function is simply its gradient).
In Autograd, the differentiation is by default done with respect to the first input argument of your Python function. Since the points is an array representing \(x\) and \(t\), the Jacobian is calculated using the values of \(x\) and \(t\).
To find the second derivative with respect to \(x\) and \(t\), the Jacobian can be found for the second time. The result is a Hessian matrix, which is the matrix containing all the possible second order mixed derivatives of \(g(x,t)\).
# Set up the trial function:
def u(x):
return np.sin(np.pi*x)
def g_trial(point,P):
x,t = point
return (1-t)*u(x) + x*(1-x)*t*deep_neural_network(P,point)
# The right side of the ODE:
def f(point):
return 0.
# The cost function:
def cost_function(P, x, t):
cost_sum = 0
g_t_jacobian_func = jacobian(g_trial)
g_t_hessian_func = hessian(g_trial)
for x_ in x:
for t_ in t:
point = np.array([x_,t_])
g_t = g_trial(point,P)
g_t_jacobian = g_t_jacobian_func(point,P)
g_t_hessian = g_t_hessian_func(point,P)
g_t_dt = g_t_jacobian[1]
g_t_d2x = g_t_hessian[0][0]
func = f(point)
err_sqr = ( (g_t_dt - g_t_d2x) - func)**2
cost_sum += err_sqr
return cost_sum
10.41. Setting up the network using Autograd; The full program¶
Having set up the network, along with the trial solution and cost function, we can now see how the deep neural network performs by comparing the results to the analytical solution.
The analytical solution of our problem is
\[
g(x,t) = \exp(-\pi^2 t)\sin(\pi x)
\]
A possible way to implement a neural network solving the PDE, is given below. Be aware, though, that it is fairly slow for the parameters used. A better result is possible, but requires more iterations, and thus longer time to complete.
Indeed, the program below is not optimal in its implementation, but rather serves as an example on how to implement and use a neural network to solve a PDE. Using TensorFlow results in a much better execution time. Try it!
import autograd.numpy as np
from autograd import jacobian,hessian,grad
import autograd.numpy.random as npr
from matplotlib import cm
from matplotlib import pyplot as plt
from mpl_toolkits.mplot3d import axes3d
## Set up the network
def sigmoid(z):
return 1/(1 + np.exp(-z))
def deep_neural_network(deep_params, x):
# x is now a point and a 1D numpy array; make it a column vector
num_coordinates = np.size(x,0)
x = x.reshape(num_coordinates,-1)
num_points = np.size(x,1)
# N_hidden is the number of hidden layers
N_hidden = np.size(deep_params) - 1 # -1 since params consist of parameters to all the hidden layers AND the output layer
# Assume that the input layer does nothing to the input x
x_input = x
x_prev = x_input
## Hidden layers:
for l in range(N_hidden):
# From the list of parameters P; find the correct weigths and bias for this layer
w_hidden = deep_params[l]
# Add a row of ones to include bias
x_prev = np.concatenate((np.ones((1,num_points)), x_prev ), axis = 0)
z_hidden = np.matmul(w_hidden, x_prev)
x_hidden = sigmoid(z_hidden)
# Update x_prev such that next layer can use the output from this layer
x_prev = x_hidden
## Output layer:
# Get the weights and bias for this layer
w_output = deep_params[-1]
# Include bias:
x_prev = np.concatenate((np.ones((1,num_points)), x_prev), axis = 0)
z_output = np.matmul(w_output, x_prev)
x_output = z_output
return x_output[0][0]
## Define the trial solution and cost function
def u(x):
return np.sin(np.pi*x)
def g_trial(point,P):
x,t = point
return (1-t)*u(x) + x*(1-x)*t*deep_neural_network(P,point)
# The right side of the ODE:
def f(point):
return 0.
# The cost function:
def cost_function(P, x, t):
cost_sum = 0
g_t_jacobian_func = jacobian(g_trial)
g_t_hessian_func = hessian(g_trial)
for x_ in x:
for t_ in t:
point = np.array([x_,t_])
g_t = g_trial(point,P)
g_t_jacobian = g_t_jacobian_func(point,P)
g_t_hessian = g_t_hessian_func(point,P)
g_t_dt = g_t_jacobian[1]
g_t_d2x = g_t_hessian[0][0]
func = f(point)
err_sqr = ( (g_t_dt - g_t_d2x) - func)**2
cost_sum += err_sqr
return cost_sum /( np.size(x)*np.size(t) )
## For comparison, define the analytical solution
def g_analytic(point):
x,t = point
return np.exp(-np.pi**2*t)*np.sin(np.pi*x)
## Set up a function for training the network to solve for the equation
def solve_pde_deep_neural_network(x,t, num_neurons, num_iter, lmb):
## Set up initial weigths and biases
N_hidden = np.size(num_neurons)
## Set up initial weigths and biases
# Initialize the list of parameters:
P = [None]*(N_hidden + 1) # + 1 to include the output layer
P[0] = npr.randn(num_neurons[0], 2 + 1 ) # 2 since we have two points, +1 to include bias
for l in range(1,N_hidden):
P[l] = npr.randn(num_neurons[l], num_neurons[l-1] + 1) # +1 to include bias
# For the output layer
P[-1] = npr.randn(1, num_neurons[-1] + 1 ) # +1 since bias is included
print('Initial cost: ',cost_function(P, x, t))
cost_function_grad = grad(cost_function,0)
# Let the update be done num_iter times
for i in range(num_iter):
cost_grad = cost_function_grad(P, x , t)
for l in range(N_hidden+1):
P[l] = P[l] - lmb * cost_grad[l]
print('Final cost: ',cost_function(P, x, t))
return P
if __name__ == '__main__':
### Use the neural network:
npr.seed(15)
## Decide the vales of arguments to the function to solve
Nx = 10; Nt = 10
x = np.linspace(0, 1, Nx)
t = np.linspace(0,1,Nt)
## Set up the parameters for the network
num_hidden_neurons = [100, 25]
num_iter = 250
lmb = 0.01
P = solve_pde_deep_neural_network(x,t, num_hidden_neurons, num_iter, lmb)
## Store the results
g_dnn_ag = np.zeros((Nx, Nt))
G_analytical = np.zeros((Nx, Nt))
for i,x_ in enumerate(x):
for j, t_ in enumerate(t):
point = np.array([x_, t_])
g_dnn_ag[i,j] = g_trial(point,P)
G_analytical[i,j] = g_analytic(point)
# Find the map difference between the analytical and the computed solution
diff_ag = np.abs(g_dnn_ag - G_analytical)
print('Max absolute difference between the analytical solution and the network: %g'%np.max(diff_ag))
## Plot the solutions in two dimensions, that being in position and time
T,X = np.meshgrid(t,x)
fig = plt.figure(figsize=(10,10))
ax = fig.gca(projection='3d')
ax.set_title('Solution from the deep neural network w/ %d layer'%len(num_hidden_neurons))
s = ax.plot_surface(T,X,g_dnn_ag,linewidth=0,antialiased=False,cmap=cm.viridis)
ax.set_xlabel('Time $t$')
ax.set_ylabel('Position $x$');
fig = plt.figure(figsize=(10,10))
ax = fig.gca(projection='3d')
ax.set_title('Analytical solution')
s = ax.plot_surface(T,X,G_analytical,linewidth=0,antialiased=False,cmap=cm.viridis)
ax.set_xlabel('Time $t$')
ax.set_ylabel('Position $x$');
fig = plt.figure(figsize=(10,10))
ax = fig.gca(projection='3d')
ax.set_title('Difference')
s = ax.plot_surface(T,X,diff_ag,linewidth=0,antialiased=False,cmap=cm.viridis)
ax.set_xlabel('Time $t$')
ax.set_ylabel('Position $x$');
## Take some slices of the 3D plots just to see the solutions at particular times
indx1 = 0
indx2 = int(Nt/2)
indx3 = Nt-1
t1 = t[indx1]
t2 = t[indx2]
t3 = t[indx3]
# Slice the results from the DNN
res1 = g_dnn_ag[:,indx1]
res2 = g_dnn_ag[:,indx2]
res3 = g_dnn_ag[:,indx3]
# Slice the analytical results
res_analytical1 = G_analytical[:,indx1]
res_analytical2 = G_analytical[:,indx2]
res_analytical3 = G_analytical[:,indx3]
# Plot the slices
plt.figure(figsize=(10,10))
plt.title("Computed solutions at time = %g"%t1)
plt.plot(x, res1)
plt.plot(x,res_analytical1)
plt.legend(['dnn','analytical'])
plt.figure(figsize=(10,10))
plt.title("Computed solutions at time = %g"%t2)
plt.plot(x, res2)
plt.plot(x,res_analytical2)
plt.legend(['dnn','analytical'])
plt.figure(figsize=(10,10))
plt.title("Computed solutions at time = %g"%t3)
plt.plot(x, res3)
plt.plot(x,res_analytical3)
plt.legend(['dnn','analytical'])
plt.show()
10.42. Example: Solving the wave equation with Neural Networks¶
The wave equation is
\[
\frac{\partial^2 g(x,t)}{\partial t^2} = c^2\frac{\partial^2 g(x,t)}{\partial x^2}
\]
with \(c\) being the specified wave speed.
Here, the chosen conditions are
\[\begin{split}
\begin{align*}
g(0,t) &= 0 \\
g(1,t) &= 0 \\
g(x,0) &= u(x) \\
\frac{\partial g(x,t)}{\partial t} \Big |_{t = 0} &= v(x)
\end{align*}
\end{split}\]
where \(\frac{\partial g(x,t)}{\partial t} \Big |_{t = 0}\) means the derivative of \(g(x,t)\) with respect to \(t\) is evaluated at \(t = 0\), and \(u(x)\) and \(v(x)\) being given functions.
10.43. The problem to solve for¶
The wave equation to solve for, is
\[
\begin{equation} \label{wave} \tag{19}
\frac{\partial^2 g(x,t)}{\partial t^2} = c^2 \frac{\partial^2 g(x,t)}{\partial x^2}
\end{equation}
\]
where \(c\) is the given wave speed. The chosen conditions for this equation are
\[\begin{split}
\begin{aligned}
g(0,t) &= 0, &t \geq 0 \\
g(1,t) &= 0, &t \geq 0 \\
g(x,0) &= u(x), &x\in[0,1] \\
\frac{\partial g(x,t)}{\partial t}\Big |_{t = 0} &= v(x), &x \in [0,1]
\end{aligned} \label{condwave} \tag{20}
\end{split}\]
In this example, let \(c = 1\) and \(u(x) = \sin(\pi x)\) and \(v(x) = -\pi\sin(\pi x)\).
10.44. The trial solution¶
Setting up the network is done in similar matter as for the example of solving the diffusion equation. The only things we have to change, is the trial solution such that it satisfies the conditions from (20) and the cost function.
The trial solution becomes slightly different since we have other conditions than in the example of solving the diffusion equation. Here, a possible trial solution \(g_t(x,t)\) is
\[
g_t(x,t) = h_1(x,t) + x(1-x)t^2N(x,t,P)
\]
where
\[
h_1(x,t) = (1-t^2)u(x) + tv(x)
\]
Note that this trial solution satisfies the conditions only if \(u(0) = v(0) = u(1) = v(1) = 0\), which is the case in this example.
10.45. The analytical solution¶
The analytical solution for our specific problem, is
\[
g(x,t) = \sin(\pi x)\cos(\pi t) - \sin(\pi x)\sin(\pi t)
\]
10.46. Solving the wave equation - the full program using Autograd¶
import autograd.numpy as np
from autograd import hessian,grad
import autograd.numpy.random as npr
from matplotlib import cm
from matplotlib import pyplot as plt
from mpl_toolkits.mplot3d import axes3d
## Set up the trial function:
def u(x):
return np.sin(np.pi*x)
def v(x):
return -np.pi*np.sin(np.pi*x)
def h1(point):
x,t = point
return (1 - t**2)*u(x) + t*v(x)
def g_trial(point,P):
x,t = point
return h1(point) + x*(1-x)*t**2*deep_neural_network(P,point)
## Define the cost function
def cost_function(P, x, t):
cost_sum = 0
g_t_hessian_func = hessian(g_trial)
for x_ in x:
for t_ in t:
point = np.array([x_,t_])
g_t_hessian = g_t_hessian_func(point,P)
g_t_d2x = g_t_hessian[0][0]
g_t_d2t = g_t_hessian[1][1]
err_sqr = ( (g_t_d2t - g_t_d2x) )**2
cost_sum += err_sqr
return cost_sum / (np.size(t) * np.size(x))
## The neural network
def sigmoid(z):
return 1/(1 + np.exp(-z))
def deep_neural_network(deep_params, x):
# x is now a point and a 1D numpy array; make it a column vector
num_coordinates = np.size(x,0)
x = x.reshape(num_coordinates,-1)
num_points = np.size(x,1)
# N_hidden is the number of hidden layers
N_hidden = np.size(deep_params) - 1 # -1 since params consist of parameters to all the hidden layers AND the output layer
# Assume that the input layer does nothing to the input x
x_input = x
x_prev = x_input
## Hidden layers:
for l in range(N_hidden):
# From the list of parameters P; find the correct weigths and bias for this layer
w_hidden = deep_params[l]
# Add a row of ones to include bias
x_prev = np.concatenate((np.ones((1,num_points)), x_prev ), axis = 0)
z_hidden = np.matmul(w_hidden, x_prev)
x_hidden = sigmoid(z_hidden)
# Update x_prev such that next layer can use the output from this layer
x_prev = x_hidden
## Output layer:
# Get the weights and bias for this layer
w_output = deep_params[-1]
# Include bias:
x_prev = np.concatenate((np.ones((1,num_points)), x_prev), axis = 0)
z_output = np.matmul(w_output, x_prev)
x_output = z_output
return x_output[0][0]
## The analytical solution
def g_analytic(point):
x,t = point
return np.sin(np.pi*x)*np.cos(np.pi*t) - np.sin(np.pi*x)*np.sin(np.pi*t)
def solve_pde_deep_neural_network(x,t, num_neurons, num_iter, lmb):
## Set up initial weigths and biases
N_hidden = np.size(num_neurons)
## Set up initial weigths and biases
# Initialize the list of parameters:
P = [None]*(N_hidden + 1) # + 1 to include the output layer
P[0] = npr.randn(num_neurons[0], 2 + 1 ) # 2 since we have two points, +1 to include bias
for l in range(1,N_hidden):
P[l] = npr.randn(num_neurons[l], num_neurons[l-1] + 1) # +1 to include bias
# For the output layer
P[-1] = npr.randn(1, num_neurons[-1] + 1 ) # +1 since bias is included
print('Initial cost: ',cost_function(P, x, t))
cost_function_grad = grad(cost_function,0)
# Let the update be done num_iter times
for i in range(num_iter):
cost_grad = cost_function_grad(P, x , t)
for l in range(N_hidden+1):
P[l] = P[l] - lmb * cost_grad[l]
print('Final cost: ',cost_function(P, x, t))
return P
if __name__ == '__main__':
### Use the neural network:
npr.seed(15)
## Decide the vales of arguments to the function to solve
Nx = 10; Nt = 10
x = np.linspace(0, 1, Nx)
t = np.linspace(0,1,Nt)
## Set up the parameters for the network
num_hidden_neurons = [50,20]
num_iter = 1000
lmb = 0.01
P = solve_pde_deep_neural_network(x,t, num_hidden_neurons, num_iter, lmb)
## Store the results
res = np.zeros((Nx, Nt))
res_analytical = np.zeros((Nx, Nt))
for i,x_ in enumerate(x):
for j, t_ in enumerate(t):
point = np.array([x_, t_])
res[i,j] = g_trial(point,P)
res_analytical[i,j] = g_analytic(point)
diff = np.abs(res - res_analytical)
print("Max difference between analytical and solution from nn: %g"%np.max(diff))
## Plot the solutions in two dimensions, that being in position and time
T,X = np.meshgrid(t,x)
fig = plt.figure(figsize=(10,10))
ax = fig.gca(projection='3d')
ax.set_title('Solution from the deep neural network w/ %d layer'%len(num_hidden_neurons))
s = ax.plot_surface(T,X,res,linewidth=0,antialiased=False,cmap=cm.viridis)
ax.set_xlabel('Time $t$')
ax.set_ylabel('Position $x$');
fig = plt.figure(figsize=(10,10))
ax = fig.gca(projection='3d')
ax.set_title('Analytical solution')
s = ax.plot_surface(T,X,res_analytical,linewidth=0,antialiased=False,cmap=cm.viridis)
ax.set_xlabel('Time $t$')
ax.set_ylabel('Position $x$');
fig = plt.figure(figsize=(10,10))
ax = fig.gca(projection='3d')
ax.set_title('Difference')
s = ax.plot_surface(T,X,diff,linewidth=0,antialiased=False,cmap=cm.viridis)
ax.set_xlabel('Time $t$')
ax.set_ylabel('Position $x$');
## Take some slices of the 3D plots just to see the solutions at particular times
indx1 = 0
indx2 = int(Nt/2)
indx3 = Nt-1
t1 = t[indx1]
t2 = t[indx2]
t3 = t[indx3]
# Slice the results from the DNN
res1 = res[:,indx1]
res2 = res[:,indx2]
res3 = res[:,indx3]
# Slice the analytical results
res_analytical1 = res_analytical[:,indx1]
res_analytical2 = res_analytical[:,indx2]
res_analytical3 = res_analytical[:,indx3]
# Plot the slices
plt.figure(figsize=(10,10))
plt.title("Computed solutions at time = %g"%t1)
plt.plot(x, res1)
plt.plot(x,res_analytical1)
plt.legend(['dnn','analytical'])
plt.figure(figsize=(10,10))
plt.title("Computed solutions at time = %g"%t2)
plt.plot(x, res2)
plt.plot(x,res_analytical2)
plt.legend(['dnn','analytical'])
plt.figure(figsize=(10,10))
plt.title("Computed solutions at time = %g"%t3)
plt.plot(x, res3)
plt.plot(x,res_analytical3)
plt.legend(['dnn','analytical'])
plt.show()