@@ -1056,13 +1066,13 @@ example of the functionality of Scikit-Learn.
The intercept alpha:
- [1.94485679]
+ [2.07549007]
Coefficient beta :
- [[5.13059483]]
-Mean squared error: 0.32
-Variance score: 0.87
+ [[5.14029264]]
+Mean squared error: 0.23
+Variance score: 0.89
Mean squared log error: 0.01
-Mean absolute error: 0.45
+Mean absolute error: 0.38
@@ -1162,7 +1172,7 @@ a linear \(x\)-dependence we s
-0.004999999999999996
+0.004999999999999987
@@ -1373,7 +1383,7 @@ the Hadamard product, meaning element-wise multiplication.
Old accuracy on training data: 0.1440501043841336
/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_22411/953065564.py:4: RuntimeWarning: overflow encountered in exp
+/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_57122/953065564.py:4: RuntimeWarning: overflow encountered in exp
return 1/(1 + np.exp(-x))
@@ -1707,7 +1717,7 @@ Lambda = 10.0
Accuracy score on test set: 0.19166666666666668
/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_22411/953065564.py:4: RuntimeWarning: overflow encountered in exp
+/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_57122/953065564.py:4: RuntimeWarning: overflow encountered in exp
return 1/(1 + np.exp(-x))
@@ -1716,7 +1726,7 @@ Lambda = 1e-05
Accuracy score on test set: 0.10555555555555556
/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_22411/953065564.py:4: RuntimeWarning: overflow encountered in exp
+/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_57122/953065564.py:4: RuntimeWarning: overflow encountered in exp
return 1/(1 + np.exp(-x))
@@ -1725,7 +1735,7 @@ Lambda = 0.0001
Accuracy score on test set: 0.08611111111111111
/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_22411/953065564.py:4: RuntimeWarning: overflow encountered in exp
+/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_57122/953065564.py:4: RuntimeWarning: overflow encountered in exp
return 1/(1 + np.exp(-x))
@@ -1734,7 +1744,7 @@ Lambda = 0.001
Accuracy score on test set: 0.10555555555555556
/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_22411/953065564.py:4: RuntimeWarning: overflow encountered in exp
+/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_57122/953065564.py:4: RuntimeWarning: overflow encountered in exp
return 1/(1 + np.exp(-x))
@@ -1743,7 +1753,7 @@ Lambda = 0.01
Accuracy score on test set: 0.08888888888888889
/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_22411/953065564.py:4: RuntimeWarning: overflow encountered in exp
+/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_57122/953065564.py:4: RuntimeWarning: overflow encountered in exp
return 1/(1 + np.exp(-x))
@@ -1752,7 +1762,7 @@ Lambda = 0.1
Accuracy score on test set: 0.08611111111111111
/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_22411/953065564.py:4: RuntimeWarning: overflow encountered in exp
+/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_57122/953065564.py:4: RuntimeWarning: overflow encountered in exp
return 1/(1 + np.exp(-x))
@@ -1761,191 +1771,34 @@ Lambda = 1.0
Accuracy score on test set: 0.08888888888888889
/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_22411/953065564.py:4: RuntimeWarning: overflow encountered in exp
+/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_57122/953065564.py:4: RuntimeWarning: overflow encountered in exp
return 1/(1 + np.exp(-x))
-Learning rate = 0.1
-Lambda = 10.0
-Accuracy score on test set: 0.09166666666666666
-
-
-/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_22411/953065564.py:4: RuntimeWarning: overflow encountered in exp
- return 1/(1 + np.exp(-x))
-/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_22411/1630775253.py:43: RuntimeWarning: overflow encountered in exp
- exp_term = np.exp(self.z_o)
-/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_22411/1630775253.py:44: RuntimeWarning: invalid value encountered in true_divide
- self.probabilities = exp_term / np.sum(exp_term, axis=1, keepdims=True)
-
-
-Learning rate = 1.0
-Lambda = 1e-05
-Accuracy score on test set: 0.07777777777777778
-
-
-/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_22411/953065564.py:4: RuntimeWarning: overflow encountered in exp
- return 1/(1 + np.exp(-x))
-/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_22411/1630775253.py:43: RuntimeWarning: overflow encountered in exp
- exp_term = np.exp(self.z_o)
-/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_22411/1630775253.py:44: RuntimeWarning: invalid value encountered in true_divide
- self.probabilities = exp_term / np.sum(exp_term, axis=1, keepdims=True)
-
-
-Learning rate = 1.0
-Lambda = 0.0001
-Accuracy score on test set: 0.07777777777777778
-
-
-/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_22411/953065564.py:4: RuntimeWarning: overflow encountered in exp
- return 1/(1 + np.exp(-x))
-/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_22411/1630775253.py:43: RuntimeWarning: overflow encountered in exp
- exp_term = np.exp(self.z_o)
-/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_22411/1630775253.py:44: RuntimeWarning: invalid value encountered in true_divide
- self.probabilities = exp_term / np.sum(exp_term, axis=1, keepdims=True)
-
-
-Learning rate = 1.0
-Lambda = 0.001
-Accuracy score on test set: 0.07777777777777778
-
-
-/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_22411/953065564.py:4: RuntimeWarning: overflow encountered in exp
- return 1/(1 + np.exp(-x))
-/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_22411/1630775253.py:43: RuntimeWarning: overflow encountered in exp
- exp_term = np.exp(self.z_o)
-/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_22411/1630775253.py:44: RuntimeWarning: invalid value encountered in true_divide
- self.probabilities = exp_term / np.sum(exp_term, axis=1, keepdims=True)
-
-
-Learning rate = 1.0
-Lambda = 0.01
-Accuracy score on test set: 0.07777777777777778
-
-
-/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_22411/953065564.py:4: RuntimeWarning: overflow encountered in exp
- return 1/(1 + np.exp(-x))
-/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_22411/1630775253.py:43: RuntimeWarning: overflow encountered in exp
- exp_term = np.exp(self.z_o)
-/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_22411/1630775253.py:44: RuntimeWarning: invalid value encountered in true_divide
- self.probabilities = exp_term / np.sum(exp_term, axis=1, keepdims=True)
-
-
-Learning rate = 1.0
-Lambda = 0.1
-Accuracy score on test set: 0.07777777777777778
-
-
-/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_22411/953065564.py:4: RuntimeWarning: overflow encountered in exp
- return 1/(1 + np.exp(-x))
-
-
-Learning rate = 1.0
-Lambda = 1.0
-Accuracy score on test set: 0.10555555555555556
-
-
-/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_22411/953065564.py:4: RuntimeWarning: overflow encountered in exp
- return 1/(1 + np.exp(-x))
-/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_22411/1630775253.py:43: RuntimeWarning: overflow encountered in exp
- exp_term = np.exp(self.z_o)
-/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_22411/1630775253.py:44: RuntimeWarning: invalid value encountered in true_divide
- self.probabilities = exp_term / np.sum(exp_term, axis=1, keepdims=True)
-
-
-Learning rate = 1.0
-Lambda = 10.0
-Accuracy score on test set: 0.07777777777777778
-
-
-/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_22411/953065564.py:4: RuntimeWarning: overflow encountered in exp
- return 1/(1 + np.exp(-x))
-/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_22411/1630775253.py:43: RuntimeWarning: overflow encountered in exp
- exp_term = np.exp(self.z_o)
-/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_22411/1630775253.py:44: RuntimeWarning: invalid value encountered in true_divide
- self.probabilities = exp_term / np.sum(exp_term, axis=1, keepdims=True)
-
-
-Learning rate = 10.0
-Lambda = 1e-05
-Accuracy score on test set: 0.07777777777777778
-
-
-/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_22411/953065564.py:4: RuntimeWarning: overflow encountered in exp
- return 1/(1 + np.exp(-x))
-/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_22411/1630775253.py:43: RuntimeWarning: overflow encountered in exp
- exp_term = np.exp(self.z_o)
-/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_22411/1630775253.py:44: RuntimeWarning: invalid value encountered in true_divide
- self.probabilities = exp_term / np.sum(exp_term, axis=1, keepdims=True)
-
-
-Learning rate = 10.0
-Lambda = 0.0001
-Accuracy score on test set: 0.07777777777777778
-
-
-/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_22411/953065564.py:4: RuntimeWarning: overflow encountered in exp
- return 1/(1 + np.exp(-x))
-/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_22411/1630775253.py:43: RuntimeWarning: overflow encountered in exp
- exp_term = np.exp(self.z_o)
-/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_22411/1630775253.py:44: RuntimeWarning: invalid value encountered in true_divide
- self.probabilities = exp_term / np.sum(exp_term, axis=1, keepdims=True)
-
-
-Learning rate = 10.0
-Lambda = 0.001
-Accuracy score on test set: 0.07777777777777778
-
-
-/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_22411/953065564.py:4: RuntimeWarning: overflow encountered in exp
- return 1/(1 + np.exp(-x))
-/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_22411/1630775253.py:43: RuntimeWarning: overflow encountered in exp
- exp_term = np.exp(self.z_o)
-/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_22411/1630775253.py:44: RuntimeWarning: invalid value encountered in true_divide
- self.probabilities = exp_term / np.sum(exp_term, axis=1, keepdims=True)
-
-
-Learning rate = 10.0
-Lambda = 0.01
-Accuracy score on test set: 0.07777777777777778
-
-
-/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_22411/953065564.py:4: RuntimeWarning: overflow encountered in exp
- return 1/(1 + np.exp(-x))
-/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_22411/1630775253.py:43: RuntimeWarning: overflow encountered in exp
- exp_term = np.exp(self.z_o)
-/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_22411/1630775253.py:44: RuntimeWarning: invalid value encountered in true_divide
- self.probabilities = exp_term / np.sum(exp_term, axis=1, keepdims=True)
-
-
-Learning rate = 10.0
-Lambda = 0.1
-Accuracy score on test set: 0.07777777777777778
-
-
-/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_22411/953065564.py:4: RuntimeWarning: overflow encountered in exp
- return 1/(1 + np.exp(-x))
-/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_22411/1630775253.py:43: RuntimeWarning: overflow encountered in exp
- exp_term = np.exp(self.z_o)
-/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_22411/1630775253.py:44: RuntimeWarning: invalid value encountered in true_divide
- self.probabilities = exp_term / np.sum(exp_term, axis=1, keepdims=True)
-
-
-Learning rate = 10.0
-Lambda = 1.0
-Accuracy score on test set: 0.07777777777777778
-
-
-/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_22411/953065564.py:4: RuntimeWarning: overflow encountered in exp
- return 1/(1 + np.exp(-x))
-/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_22411/1630775253.py:43: RuntimeWarning: overflow encountered in exp
- exp_term = np.exp(self.z_o)
-/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_22411/1630775253.py:44: RuntimeWarning: invalid value encountered in true_divide
- self.probabilities = exp_term / np.sum(exp_term, axis=1, keepdims=True)
-
-
-Learning rate = 10.0
-Lambda = 10.0
-Accuracy score on test set: 0.07777777777777778
+---------------------------------------------------------------------------
+KeyboardInterrupt Traceback (most recent call last)
+Cell In[8], line 11
+ 8 for j, lmbd in enumerate(lmbd_vals):
+ 9 dnn = NeuralNetwork(X_train, Y_train_onehot, eta=eta, lmbd=lmbd, epochs=epochs, batch_size=batch_size,
+ 10 n_hidden_neurons=n_hidden_neurons, n_categories=n_categories)
+---> 11 dnn.train()
+ 13 DNN_numpy[i][j] = dnn
+ 15 test_predict = dnn.predict(X_test)
+
+Cell In[6], line 98, in NeuralNetwork.train(self)
+ 95 self.X_data = self.X_data_full[chosen_datapoints]
+ 96 self.Y_data = self.Y_data_full[chosen_datapoints]
+---> 98 self.feed_forward()
+ 99 self.backpropagation()
+
+Cell In[6], line 38, in NeuralNetwork.feed_forward(self)
+ 36 def feed_forward(self):
+ 37 # feed-forward for training
+---> 38 self.z_h = np.matmul(self.X_data, self.hidden_weights) + self.hidden_bias
+ 39 self.a_h = sigmoid(self.z_h)
+ 41 self.z_o = np.matmul(self.a_h, self.output_weights) + self.output_bias
+
+KeyboardInterrupt:
@@ -1991,22 +1844,6 @@ Accuracy score on test set: 0.07777777777777778
/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_22411/953065564.py:4: RuntimeWarning: overflow encountered in exp
- return 1/(1 + np.exp(-x))
-/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_22411/953065564.py:4: RuntimeWarning: overflow encountered in exp
- return 1/(1 + np.exp(-x))
-/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_22411/953065564.py:4: RuntimeWarning: overflow encountered in exp
- return 1/(1 + np.exp(-x))
-/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_22411/953065564.py:4: RuntimeWarning: overflow encountered in exp
- return 1/(1 + np.exp(-x))
-/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_22411/953065564.py:4: RuntimeWarning: overflow encountered in exp
- return 1/(1 + np.exp(-x))
-
-
-/Users/mhjensen/miniforge3/envs/myenv/lib/python3.9/site-packages/sklearn/neural_network/_multilayer_perceptron.py:691: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (100) reached and the optimization hasn't converged yet.
- warnings.warn(
-Learning rate = 1e-05
-Lambda = 1e-05
-Accuracy score on test set: 0.18333333333333332
-/Users/mhjensen/miniforge3/envs/myenv/lib/python3.9/site-packages/sklearn/neural_network/_multilayer_perceptron.py:691: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (100) reached and the optimization hasn't converged yet.
- warnings.warn(
-Learning rate = 1e-05
-Lambda = 0.0001
-Accuracy score on test set: 0.18611111111111112
-/Users/mhjensen/miniforge3/envs/myenv/lib/python3.9/site-packages/sklearn/neural_network/_multilayer_perceptron.py:691: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (100) reached and the optimization hasn't converged yet.
- warnings.warn(
-Learning rate = 1e-05
-Lambda = 0.001
-Accuracy score on test set: 0.13055555555555556
-/Users/mhjensen/miniforge3/envs/myenv/lib/python3.9/site-packages/sklearn/neural_network/_multilayer_perceptron.py:691: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (100) reached and the optimization hasn't converged yet.
- warnings.warn(
-Learning rate = 1e-05
-Lambda = 0.01
-Accuracy score on test set: 0.24444444444444444
-/Users/mhjensen/miniforge3/envs/myenv/lib/python3.9/site-packages/sklearn/neural_network/_multilayer_perceptron.py:691: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (100) reached and the optimization hasn't converged yet.
- warnings.warn(
-Learning rate = 1e-05
-Lambda = 0.1
-Accuracy score on test set: 0.23333333333333334
-/Users/mhjensen/miniforge3/envs/myenv/lib/python3.9/site-packages/sklearn/neural_network/_multilayer_perceptron.py:691: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (100) reached and the optimization hasn't converged yet.
- warnings.warn(
-Learning rate = 1e-05
-Lambda = 1.0
-Accuracy score on test set: 0.12777777777777777
-/Users/mhjensen/miniforge3/envs/myenv/lib/python3.9/site-packages/sklearn/neural_network/_multilayer_perceptron.py:691: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (100) reached and the optimization hasn't converged yet.
- warnings.warn(
-Learning rate = 1e-05
-Lambda = 10.0
-Accuracy score on test set: 0.1527777777777778
-/Users/mhjensen/miniforge3/envs/myenv/lib/python3.9/site-packages/sklearn/neural_network/_multilayer_perceptron.py:691: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (100) reached and the optimization hasn't converged yet.
- warnings.warn(
-Learning rate = 0.0001
-Lambda = 1e-05
-Accuracy score on test set: 0.9111111111111111
-/Users/mhjensen/miniforge3/envs/myenv/lib/python3.9/site-packages/sklearn/neural_network/_multilayer_perceptron.py:691: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (100) reached and the optimization hasn't converged yet.
- warnings.warn(
-Learning rate = 0.0001
-Lambda = 0.0001
-Accuracy score on test set: 0.8888888888888888
-/Users/mhjensen/miniforge3/envs/myenv/lib/python3.9/site-packages/sklearn/neural_network/_multilayer_perceptron.py:691: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (100) reached and the optimization hasn't converged yet.
- warnings.warn(
-Learning rate = 0.0001
-Lambda = 0.001
-Accuracy score on test set: 0.8722222222222222
-/Users/mhjensen/miniforge3/envs/myenv/lib/python3.9/site-packages/sklearn/neural_network/_multilayer_perceptron.py:691: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (100) reached and the optimization hasn't converged yet.
- warnings.warn(
-Learning rate = 0.0001
-Lambda = 0.01
-Accuracy score on test set: 0.8305555555555556
-/Users/mhjensen/miniforge3/envs/myenv/lib/python3.9/site-packages/sklearn/neural_network/_multilayer_perceptron.py:691: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (100) reached and the optimization hasn't converged yet.
- warnings.warn(
-Learning rate = 0.0001
-Lambda = 0.1
-Accuracy score on test set: 0.8888888888888888
-/Users/mhjensen/miniforge3/envs/myenv/lib/python3.9/site-packages/sklearn/neural_network/_multilayer_perceptron.py:691: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (100) reached and the optimization hasn't converged yet.
- warnings.warn(
-Learning rate = 0.0001
-Lambda = 1.0
-Accuracy score on test set: 0.8805555555555555
-/Users/mhjensen/miniforge3/envs/myenv/lib/python3.9/site-packages/sklearn/neural_network/_multilayer_perceptron.py:691: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (100) reached and the optimization hasn't converged yet.
- warnings.warn(
-Learning rate = 0.0001
-Lambda = 10.0
-Accuracy score on test set: 0.8944444444444445
-/Users/mhjensen/miniforge3/envs/myenv/lib/python3.9/site-packages/sklearn/neural_network/_multilayer_perceptron.py:691: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (100) reached and the optimization hasn't converged yet.
- warnings.warn(
-Learning rate = 0.001
-Lambda = 1e-05
-Accuracy score on test set: 0.975
-/Users/mhjensen/miniforge3/envs/myenv/lib/python3.9/site-packages/sklearn/neural_network/_multilayer_perceptron.py:691: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (100) reached and the optimization hasn't converged yet.
- warnings.warn(
-Learning rate = 0.001
-Lambda = 0.0001
-Accuracy score on test set: 0.9777777777777777
-/Users/mhjensen/miniforge3/envs/myenv/lib/python3.9/site-packages/sklearn/neural_network/_multilayer_perceptron.py:691: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (100) reached and the optimization hasn't converged yet.
- warnings.warn(
-Learning rate = 0.001
-Lambda = 0.001
-Accuracy score on test set: 0.9805555555555555
-/Users/mhjensen/miniforge3/envs/myenv/lib/python3.9/site-packages/sklearn/neural_network/_multilayer_perceptron.py:691: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (100) reached and the optimization hasn't converged yet.
- warnings.warn(
-Learning rate = 0.001
-Lambda = 0.01
-Accuracy score on test set: 0.9861111111111112
-/Users/mhjensen/miniforge3/envs/myenv/lib/python3.9/site-packages/sklearn/neural_network/_multilayer_perceptron.py:691: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (100) reached and the optimization hasn't converged yet.
- warnings.warn(
-Learning rate = 0.001
-Lambda = 0.1
-Accuracy score on test set: 0.9805555555555555
-/Users/mhjensen/miniforge3/envs/myenv/lib/python3.9/site-packages/sklearn/neural_network/_multilayer_perceptron.py:691: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (100) reached and the optimization hasn't converged yet.
- warnings.warn(
-Learning rate = 0.001
-Lambda = 1.0
-Accuracy score on test set: 0.9777777777777777
-/Users/mhjensen/miniforge3/envs/myenv/lib/python3.9/site-packages/sklearn/neural_network/_multilayer_perceptron.py:691: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (100) reached and the optimization hasn't converged yet.
- warnings.warn(
-Learning rate = 0.001
-Lambda = 10.0
-Accuracy score on test set: 0.9444444444444444
-/Users/mhjensen/miniforge3/envs/myenv/lib/python3.9/site-packages/sklearn/neural_network/_multilayer_perceptron.py:691: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (100) reached and the optimization hasn't converged yet.
- warnings.warn(
-Learning rate = 0.01
-Lambda = 1e-05
-Accuracy score on test set: 0.9861111111111112
-Learning rate = 0.01
-Lambda = 0.0001
-Accuracy score on test set: 0.9888888888888889
-Learning rate = 0.01
-Lambda = 0.001
-Accuracy score on test set: 0.9888888888888889
-Learning rate = 0.01
-Lambda = 0.01
-Accuracy score on test set: 0.9861111111111112
-Learning rate = 0.01
-Lambda = 0.1
-Accuracy score on test set: 0.9888888888888889
-Learning rate = 0.01
-Lambda = 1.0
-Accuracy score on test set: 0.9722222222222222
-
-Learning rate = 0.01
-Lambda = 10.0
-Accuracy score on test set: 0.9527777777777777
-Learning rate = 0.1
-Lambda = 1e-05
-Accuracy score on test set: 0.9027777777777778
-Learning rate = 0.1
-Lambda = 0.0001
-Accuracy score on test set: 0.8583333333333333
-Learning rate = 0.1
-Lambda = 0.001
-Accuracy score on test set: 0.8722222222222222
-Learning rate = 0.1
-Lambda = 0.01
-Accuracy score on test set: 0.9055555555555556
-Learning rate = 0.1
-Lambda = 0.1
-Accuracy score on test set: 0.8805555555555555
-Learning rate = 0.1
-Lambda = 1.0
-Accuracy score on test set: 0.8722222222222222
-Learning rate = 0.1
-Lambda = 10.0
-Accuracy score on test set: 0.8666666666666667
-Learning rate = 1.0
-Lambda = 1e-05
-Accuracy score on test set: 0.08611111111111111
-
-Learning rate = 1.0
-Lambda = 0.0001
-Accuracy score on test set: 0.10555555555555556
-Learning rate = 1.0
-Lambda = 0.001
-Accuracy score on test set: 0.10555555555555556
-
-Learning rate = 1.0
-Lambda = 0.01
-Accuracy score on test set: 0.17777777777777778
-Learning rate = 1.0
-Lambda = 0.1
-Accuracy score on test set: 0.08333333333333333
-
-Learning rate = 1.0
-Lambda = 1.0
-Accuracy score on test set: 0.08888888888888889
-Learning rate = 1.0
-Lambda = 10.0
-Accuracy score on test set: 0.09444444444444444
-
-Learning rate = 10.0
-Lambda = 1e-05
-Accuracy score on test set: 0.17222222222222222
-Learning rate = 10.0
-Lambda = 0.0001
-Accuracy score on test set: 0.11666666666666667
-
-Learning rate = 10.0
-Lambda = 0.001
-Accuracy score on test set: 0.10555555555555556
-Learning rate = 10.0
-Lambda = 0.01
-Accuracy score on test set: 0.1388888888888889
-
-Learning rate = 10.0
-Lambda = 0.1
-Accuracy score on test set: 0.11388888888888889
-Learning rate = 10.0
-Lambda = 1.0
-Accuracy score on test set: 0.10555555555555556
-
-Learning rate = 10.0
-Lambda = 10.0
-Accuracy score on test set: 0.09444444444444444
-
-
- Cell In[12], line 1
- conda create -n tf tensorflow
- ^
-SyntaxError: invalid syntax
-To install the current release of GPU TensorFlow
@@ -2662,6 +2672,43 @@ Using TensorFlow results in a much better execution time. Try it!
19 x = tuple(args[i] for i in argnum) ---> 20 return unary_operator(unary_f, x, *nary_op_args, **nary_op_kwargs) +File ~/miniforge3/envs/myenv/lib/python3.9/site-packages/autograd/differential_operators.py:60, in jacobian(fun, x) + 50 @unary_to_nary + 51 def jacobian(fun, x): + 52 """ + 53 Returns a function which computes the Jacobian of `fun` with respect to + 54 positional argument number `argnum`, which must be a scalar or array. Unlike + (...) + 58 (out1, out2, ...) then the Jacobian has shape (out1, out2, ..., in1, in2, ...). + 59 """ +---> 60 vjp, ans = _make_vjp(fun, x) + 61 ans_vspace = vspace(ans) + 62 jacobian_shape = ans_vspace.shape + vspace(x).shape + +File ~/miniforge3/envs/myenv/lib/python3.9/site-packages/autograd/core.py:10, in make_vjp(fun, x) + 8 def make_vjp(fun, x): + 9 start_node = VJPNode.new_root() +---> 10 end_value, end_node = trace(start_node, fun, x) + 11 if end_node is None: + 12 def vjp(g): return vspace(x).zeros() + +File ~/miniforge3/envs/myenv/lib/python3.9/site-packages/autograd/tracer.py:10, in trace(start_node, fun, x) + 8 with trace_stack.new_trace() as t: + 9 start_box = new_box(x, t, start_node) +---> 10 end_box = fun(start_box) + 11 if isbox(end_box) and end_box._trace == start_box._trace: + 12 return end_box._value, end_box._node + +File ~/miniforge3/envs/myenv/lib/python3.9/site-packages/autograd/wrap_util.py:15, in unary_to_nary.<locals>.nary_operator.<locals>.nary_f.<locals>.unary_f(x) + 13 else: + 14 subargs = subvals(args, zip(argnum, x)) +---> 15 return fun(*subargs, **kwargs) + +File ~/miniforge3/envs/myenv/lib/python3.9/site-packages/autograd/wrap_util.py:20, in unary_to_nary.<locals>.nary_operator.<locals>.nary_f(*args, **kwargs) + 18 else: + 19 x = tuple(args[i] for i in argnum) +---> 20 return unary_operator(unary_f, x, *nary_op_args, **nary_op_kwargs) + File ~/miniforge3/envs/myenv/lib/python3.9/site-packages/autograd/differential_operators.py:64, in jacobian(fun, x) 62 jacobian_shape = ans_vspace.shape + vspace(x).shape 63 grads = map(vjp, ans_vspace.standard_basis()) @@ -2695,43 +2742,58 @@ Using TensorFlow results in a much better execution time. Try it! 22 for parent, ingrad in zip(node.parents, ingrads): 23 outgrads[parent] = add_outgrads(outgrads.get(parent), ingrad) -File ~/miniforge3/envs/myenv/lib/python3.9/site-packages/autograd/core.py:78, in defvjp.<locals>.vjp_argnums.<locals>.<lambda>(g) - 76 vjp_0 = vjp_0_fun(ans, *args, **kwargs) - 77 vjp_1 = vjp_1_fun(ans, *args, **kwargs) ----> 78 return lambda g: (vjp_0(g), vjp_1(g)) - 79 else: - 80 vjps = [vjps_dict[argnum](ans, *args, **kwargs) for argnum in argnums] +File ~/miniforge3/envs/myenv/lib/python3.9/site-packages/autograd/core.py:67, in defvjp.<locals>.vjp_argnums.<locals>.<lambda>(g) + 64 raise NotImplementedError( + 65 "VJP of {} wrt argnum 0 not defined".format(fun.__name__)) + 66 vjp = vjpfun(ans, *args, **kwargs) +---> 67 return lambda g: (vjp(g),) + 68 elif L == 2: + 69 argnum_0, argnum_1 = argnums -File ~/miniforge3/envs/myenv/lib/python3.9/site-packages/autograd/numpy/numpy_vjps.py:660, in unbroadcast_f.<locals>.<lambda>(g) - 658 def unbroadcast_f(target, f): - 659 target_meta = anp.metadata(target) ---> 660 return lambda g: unbroadcast(f(g), target_meta) +File ~/miniforge3/envs/myenv/lib/python3.9/site-packages/autograd/numpy/numpy_vjps.py:423, in matmul_vjp_1.<locals>.<lambda>(g) + 421 A_ndim = anp.ndim(A) + 422 B_meta = anp.metadata(B) +--> 423 return lambda g: matmul_adjoint_1(A, g, A_ndim, B_meta) -File ~/miniforge3/envs/myenv/lib/python3.9/site-packages/autograd/numpy/numpy_vjps.py:34, in <lambda>(g) - 30 # ----- Binary ufuncs ----- - 32 defvjp(anp.add, lambda ans, x, y : unbroadcast_f(x, lambda g: g), - 33 lambda ans, x, y : unbroadcast_f(y, lambda g: g)) ----> 34 defvjp(anp.multiply, lambda ans, x, y : unbroadcast_f(x, lambda g: y * g), - 35 lambda ans, x, y : unbroadcast_f(y, lambda g: x * g)) - 36 defvjp(anp.subtract, lambda ans, x, y : unbroadcast_f(x, lambda g: g), - 37 lambda ans, x, y : unbroadcast_f(y, lambda g: -g)) - 38 defvjp(anp.divide, lambda ans, x, y : unbroadcast_f(x, lambda g: g / y), - 39 lambda ans, x, y : unbroadcast_f(y, lambda g: - g * x / y**2)) +File ~/miniforge3/envs/myenv/lib/python3.9/site-packages/autograd/numpy/numpy_vjps.py:413, in matmul_adjoint_1(A, G, A_ndim, B_meta) + 411 if B_is_vec: + 412 result = anp.squeeze(result, anp.ndim(G) - 1) +--> 413 return unbroadcast(result, B_meta) -File ~/miniforge3/envs/myenv/lib/python3.9/site-packages/autograd/numpy/numpy_boxes.py:27, in ArrayBox.__mul__(self, other) ----> 27 def __mul__(self, other): return anp.multiply(self, other) +File ~/miniforge3/envs/myenv/lib/python3.9/site-packages/autograd/numpy/numpy_vjps.py:653, in unbroadcast(x, target_meta, broadcast_idx) + 651 for axis, size in enumerate(target_shape): + 652 if size == 1: +--> 653 x = anp.sum(x, axis=axis, keepdims=True) + 654 if anp.iscomplexobj(x) and not target_iscomplex: + 655 x = anp.real(x) -File ~/miniforge3/envs/myenv/lib/python3.9/site-packages/autograd/tracer.py:44, in primitive.<locals>.f_wrapped(*args, **kwargs) - 42 parents = tuple(box._node for _ , box in boxed_args) - 43 argnums = tuple(argnum for argnum, _ in boxed_args) ----> 44 ans = f_wrapped(*argvals, **kwargs) - 45 node = node_constructor(ans, f_wrapped, argvals, kwargs, argnums, parents) - 46 return new_box(ans, trace, node) - -File ~/miniforge3/envs/myenv/lib/python3.9/site-packages/autograd/tracer.py:48, in primitive.<locals>.f_wrapped(*args, **kwargs) +File ~/miniforge3/envs/myenv/lib/python3.9/site-packages/autograd/tracer.py:45, in primitive.<locals>.f_wrapped(*args, **kwargs) + 43 argnums = tuple(argnum for argnum, _ in boxed_args) + 44 ans = f_wrapped(*argvals, **kwargs) +---> 45 node = node_constructor(ans, f_wrapped, argvals, kwargs, argnums, parents) 46 return new_box(ans, trace, node) 47 else: ----> 48 return f_raw(*args, **kwargs) + +File ~/miniforge3/envs/myenv/lib/python3.9/site-packages/autograd/core.py:36, in VJPNode.__init__(self, value, fun, args, kwargs, parent_argnums, parents) + 33 fun_name = getattr(fun, '__name__', fun) + 34 raise NotImplementedError("VJP of {} wrt argnums {} not defined" + 35 .format(fun_name, parent_argnums)) +---> 36 self.vjp = vjpmaker(parent_argnums, value, args, kwargs) + +File ~/miniforge3/envs/myenv/lib/python3.9/site-packages/autograd/core.py:66, in defvjp.<locals>.vjp_argnums(argnums, ans, args, kwargs) + 63 except KeyError: + 64 raise NotImplementedError( + 65 "VJP of {} wrt argnum 0 not defined".format(fun.__name__)) +---> 66 vjp = vjpfun(ans, *args, **kwargs) + 67 return lambda g: (vjp(g),) + 68 elif L == 2: + +File ~/miniforge3/envs/myenv/lib/python3.9/site-packages/autograd/numpy/numpy_vjps.py:297, in grad_np_sum(ans, x, axis, keepdims, dtype) + 294 return lambda g: anp.sum(g, axis=broadcast_axes, keepdims=True) + 295 defvjp(anp.broadcast_to, grad_broadcast_to) +--> 297 def grad_np_sum(ans, x, axis=None, keepdims=False, dtype=None): + 298 shape, dtype = anp.shape(x), anp.result_type(x) + 299 return lambda g: repeat_to_match_shape(g, shape, dtype, axis, keepdims)[0] KeyboardInterrupt:
@@ -1235,12 +1245,12 @@ labels = (n_inputs) = (1797,)
2 from tensorflow.keras.layers import Input
3 from tensorflow.keras.models import Sequential #This allows appending layers to existing models
-File ~/miniforge3/envs/myenv/lib/python3.9/site-packages/tensorflow/__init__.py:443
- 441 _plugin_dir = _os.path.join(_s, 'tensorflow-plugins')
- 442 if _os.path.exists(_plugin_dir):
---> 443 _ll.load_library(_plugin_dir)
- 444 # Load Pluggable Device Library
- 445 _ll.load_pluggable_device_library(_plugin_dir)
+File ~/miniforge3/envs/myenv/lib/python3.9/site-packages/tensorflow/__init__.py:440
+ 438 _plugin_dir = _os.path.join(_s, 'tensorflow-plugins')
+ 439 if _os.path.exists(_plugin_dir):
+--> 440 _ll.load_library(_plugin_dir)
+ 441 # Load Pluggable Device Library
+ 442 _ll.load_pluggable_device_library(_plugin_dir)
File ~/miniforge3/envs/myenv/lib/python3.9/site-packages/tensorflow/python/framework/load_library.py:151, in load_library(library_location)
148 kernel_libraries = [library_location]
diff --git a/doc/LectureNotes/_build/html/chapter13.html b/doc/LectureNotes/_build/html/chapter13.html
index 3828cc5ae..c80a1f73e 100644
--- a/doc/LectureNotes/_build/html/chapter13.html
+++ b/doc/LectureNotes/_build/html/chapter13.html
@@ -323,6 +323,16 @@ const thebe_selector_output = ".output, .cell_output"
Week 41 Neural networks and constructing a neural network code
+
@@ -647,12 +657,12 @@ systems such as automatic translation and speech-to-text.
@@ -1305,10 +1315,10 @@ covariance matrix through the np.linalg.eig() function. We see here that we reach a plateau for the Ridge results. Writing out the coefficients \(\boldsymbol{\beta}\), we observe that they are getting smaller and smaller and our error stabilizes since the predicted values of \(\tilde{\boldsymbol{y}}\) approach zero.
@@ -859,10 +869,10 @@ number \(i\) is left out. Usin
The bias-variance tradeoff summarizes the fundamental tension in
@@ -1647,12 +1661,12 @@ Mean squared error on test data: 877.21517262
Degree of polynomial: 23
Mean squared error on training data: 0.00085892
Mean squared error on test data: 5567.04664255
-Degree of polynomial: 24
-Mean squared error on training data: 0.00084707
-Mean squared error on test data: 1325.26124692
-
diff --git a/doc/LectureNotes/_build/html/chapter5.html b/doc/LectureNotes/_build/html/chapter5.html
index 394c0c69a..1fac69acc 100644
--- a/doc/LectureNotes/_build/html/chapter5.html
+++ b/doc/LectureNotes/_build/html/chapter5.html
@@ -323,6 +323,16 @@ const thebe_selector_output = ".output, .cell_output"
Week 41 Neural networks and constructing a neural network code
+
diff --git a/doc/LectureNotes/_build/html/chapter6.html b/doc/LectureNotes/_build/html/chapter6.html
index 9c904f401..5e85b0cbf 100644
--- a/doc/LectureNotes/_build/html/chapter6.html
+++ b/doc/LectureNotes/_build/html/chapter6.html
@@ -323,6 +323,16 @@ const thebe_selector_output = ".output, .cell_output"
Week 41 Neural networks and constructing a neural network code
+
@@ -787,9 +797,9 @@ predicting the target features of query instances is as follows:
diff --git a/doc/LectureNotes/_build/html/chapter8.html b/doc/LectureNotes/_build/html/chapter8.html
index d85525649..a38643236 100644
--- a/doc/LectureNotes/_build/html/chapter8.html
+++ b/doc/LectureNotes/_build/html/chapter8.html
@@ -323,6 +323,16 @@ const thebe_selector_output = ".output, .cell_output"
Week 41 Neural networks and constructing a neural network code
+
@@ -741,10 +751,10 @@ covariance matrix through the np.linalg.eig() function.
diff --git a/doc/LectureNotes/_build/html/chapteroptimization.html b/doc/LectureNotes/_build/html/chapteroptimization.html
index 637c870f3..838dfe57c 100644
--- a/doc/LectureNotes/_build/html/chapteroptimization.html
+++ b/doc/LectureNotes/_build/html/chapteroptimization.html
@@ -323,6 +323,16 @@ const thebe_selector_output = ".output, .cell_output"
Week 41 Neural networks and constructing a neural network code
+
diff --git a/doc/LectureNotes/_build/html/clustering.html b/doc/LectureNotes/_build/html/clustering.html
index ee79e09c1..44e1749ec 100644
--- a/doc/LectureNotes/_build/html/clustering.html
+++ b/doc/LectureNotes/_build/html/clustering.html
@@ -323,6 +323,16 @@ const thebe_selector_output = ".output, .cell_output"
Week 41 Neural networks and constructing a neural network code
+
@@ -600,12 +610,12 @@ Gaussian distribution.
diff --git a/doc/LectureNotes/_build/html/exercisesweek35.html b/doc/LectureNotes/_build/html/exercisesweek35.html
index 2f0422528..552519e48 100644
--- a/doc/LectureNotes/_build/html/exercisesweek35.html
+++ b/doc/LectureNotes/_build/html/exercisesweek35.html
@@ -323,6 +323,16 @@ const thebe_selector_output = ".output, .cell_output"
Week 41 Neural networks and constructing a neural network code
+
diff --git a/doc/LectureNotes/_build/html/exercisesweek36.html b/doc/LectureNotes/_build/html/exercisesweek36.html
index b2187bd49..8f3f40509 100644
--- a/doc/LectureNotes/_build/html/exercisesweek36.html
+++ b/doc/LectureNotes/_build/html/exercisesweek36.html
@@ -323,6 +323,16 @@ const thebe_selector_output = ".output, .cell_output"
Week 41 Neural networks and constructing a neural network code
+
diff --git a/doc/LectureNotes/_build/html/exercisesweek37.html b/doc/LectureNotes/_build/html/exercisesweek37.html
index f6a46f580..b9223bc23 100644
--- a/doc/LectureNotes/_build/html/exercisesweek37.html
+++ b/doc/LectureNotes/_build/html/exercisesweek37.html
@@ -323,6 +323,16 @@ const thebe_selector_output = ".output, .cell_output"
Week 41 Neural networks and constructing a neural network code
+
diff --git a/doc/LectureNotes/_build/html/exercisesweek38.html b/doc/LectureNotes/_build/html/exercisesweek38.html
index 8bea48d90..80ddafe25 100644
--- a/doc/LectureNotes/_build/html/exercisesweek38.html
+++ b/doc/LectureNotes/_build/html/exercisesweek38.html
@@ -323,6 +323,16 @@ const thebe_selector_output = ".output, .cell_output"
Week 41 Neural networks and constructing a neural network code
+
diff --git a/doc/LectureNotes/_build/html/exercisesweek39.html b/doc/LectureNotes/_build/html/exercisesweek39.html
index 74bb4cb9b..fa3740090 100644
--- a/doc/LectureNotes/_build/html/exercisesweek39.html
+++ b/doc/LectureNotes/_build/html/exercisesweek39.html
@@ -321,6 +321,16 @@ const thebe_selector_output = ".output, .cell_output"
Week 41 Neural networks and constructing a neural network code
+
diff --git a/doc/LectureNotes/_build/html/exercisesweek41.html b/doc/LectureNotes/_build/html/exercisesweek41.html
index 6ae56b051..39e5531ee 100644
--- a/doc/LectureNotes/_build/html/exercisesweek41.html
+++ b/doc/LectureNotes/_build/html/exercisesweek41.html
@@ -804,15 +804,15 @@ regression. October 14-18, 2024 October 11-18, 2024 Date: Deadline is Friday October 18 at midnight The aim of the exercises this week is to get started with implementing a neural network. There are a lot of technical and finicky parts of implementing a neutal network, so take your time. This week, you will implement only the feed-forward pass. Next week, you will implement backpropagation. We recommend that you do the exercises this week by editing and running this notebook file, as it includes several checks along the way that you have implemented the pieces of the feed-forward pass correctly. If you have trouble running a notebook, or importing pytorch, you can run this notebook in google colab instead: (LINK TO COLAB), though we recommend that you set up VSCode and your python environment to run code like this locally. This week, you will implement only the feed-forward pass and updating the network parameters with simple gradient descent, the gradient will be computed using autograd using code we provide. Next week, you will implement backpropagation. We recommend that you do the exercises this week by editing and running this notebook file, as it includes some checks along the way that you have implemented the pieces of the feed-forward pass correctly, and running small parts of the code at a time will be important for understanding the methods. If you have trouble running a notebook, you can run this notebook in google colab instead (https://colab.research.google.com/drive/1OCQm1tlTWB6hZSf9I7gGUgW9M8SbVeQu#offline=true&sandboxMode=true), an updated link will be provided on the course discord (you can also send an email to k.h.fredly@fys.uio.no if you encounter any trouble), though we recommend that you set up VSCode and your python environment to run code like this locally. Complete the following parts to compute the activation of the first layer. In this exercise you will compute the activation of the first layer. You only need to change the code in the cells right below an exercise, the rest works out of the box. Feel free to make changes and see how stuff works though! a) Define the bias of the first layer, a) Given the shape of the first layer weight matrix, what is the input shape of the neural network? What is the output shape of the first layer? b) Define the bias of the first layer, b) Compute the intermediary c) Compute the intermediary c) Compute the activation d) Compute the activation Confirm that you got the correct activation with the test below. Confirm that you got the correct activation with the test below. Make sure that you define Compute the activation of the second layer with an output of length 8 and ReLU activation. a) Define the weight and bias of the second layer with the right shapes. Now we will add a layer to the network with an output of length 8 and ReLU activation. a) What is the input of the second layer? What is its shape? b) Define the weight and bias of the second layer with the right shapes. b) Compute intermediary c) Compute the intermediary Confirm that you got the correct activation shape with the test below. a) Complete the function below so that it returns a list b) Comple the function below so that it evaluates the intermediate b) Comple the function below so that it evaluates the intermediary c) Create a network with input size 8 and layers with output sizes 10, 16, 6, 2. Evaluate it and make sure that you get the correct size vectors along the way. So far, every layer has used the same activation, ReLU. We often want to use other types of activation however, so we need to update our code to support multiple types of activation. Make sure that you have completed every previous exercise before trying this one. a) Make the d) Why is a neural network with no activation functions always mathematically equivelent to a neural network with only one layer? So far, every layer has used the same activation, ReLU. We often want to use other types of activation however, so we need to update our code to support multiple types of activation functions. Make sure that you have completed every previous exercise before trying this one. a) Complete the b) Make a list with three activation functions(don’t call them yet! you can make a list with function names as elements, and then call these elements of the list later), two ReLU and one sigmoid. (If you add other functions than the ones defined at the start of the notebook, make sure everything is defined using autograd’s numpy wrapper, like above, since we want to use automatic differentiation on all of these functions later.) Then evaluate a network with three layers and these activation functions. So far, the feed forward function has taken one input vector as an input. This vector then undergoes a linear transformation and then an element-wise non-linear operation for each layer. This approach of sending one vector in at a time is great for interpreting how the network transforms data with its linear and non-linear operations, but not the best for numerical efficiency. Now, we want to be able to send many inputs through the network at once. This will make the code a bit harder to understand, but it will make it faster, and more compact. It will be worth the trouble. To process multiple inputs at once, while still performing the same operations, you will only need to flip a couple things around. a) Complete the function b) Update the b) Make a matrix of inputs with the shape (number of features, number of inputs), you choose the number of inputs and features per input. Then complete the function c) Create and evaluate a neural network with 4 inputs and layers with output sizes 12, 10, 3 and activations ReLU, ReLU, softmax. The final exercise will hopefully be very simple if everything has worked so far. You will evaluate your neural network on the iris data set (https://scikit-learn.org/1.5/auto_examples/datasets/plot_iris_dataset.html). This dataset contains data on 150 flowers of 3 different types which can be separated pretty well using the four features given for each flower, which includes the width and length of their leaves. You are not expected to do any training of the network or actual classification, unless you feel like it, in that case you can do exercise 5. You should use this batched approach moving forward, as it will lead to much more compact code. However, remember that each input is still treated separately, and that you will need to keep in mind the transposed weight matrix and other details when implementing backpropagation. You will now evaluate your neural network on the iris data set (https://scikit-learn.org/1.5/auto_examples/datasets/plot_iris_dataset.html). This dataset contains data on 150 flowers of 3 different types which can be separated pretty well using the four features given for each flower, which includes the width and length of their leaves. You are will later train your network to actually make good predictions. c) Loop over the iris dataset( a) What should the input size for the network be with this dataset? What should the output shape of the last layer be? b) Create a network with two hidden layers, the first with sigmoid activation and the last with softmax, the first layer should have 8 “nodes”, the second has the number of nodes you found in exercise a). Softmax returns a “probability distribution”, in the sense that the numbers in the output are positive and add up to 1 and, their magnitude are in some sense relative to their magnitude before going through the softmax function. Remember to use the batched version of the create_layers and feed forward functions. c) Evaluate your model on the entire iris dataset! For later purposes, we will split the data into train and test sets, and compute gradients on smaller batches of the training data. But for now, evaluate the network on the whole thing at once. d) Compute the accuracy of your model using the accuracy function defined above. Recreate your model a couple times and see how the accuracy changes. a) Make the iris target values into one-hot vectors. b) Define the cross-entropy loss function to evaluate the performance of your network on the data set. c) Use the autograd package to take the gradient of the cross entropy wrt. the weights and biases of the network. d) Use gradient descent of some sort to optimize the parameters. e) Evaluate the accuracy of the network. e) Show off how you did in a group session! To be able to actually do anything useful with your neural network, you need to train it. For this, we need a cost function and a way to take the gradient of the cost function wrt. the network parameters. The following exercises guide you through taking the gradient using autograd, and updating the network parameters using the gradient. Feel free to implement gradient methods like ADAM if you finish everything. The cross-entropy loss function can evaluate performance on classification tasks. It sees if your prediction is “most certain” on the correct target. a) What shape should the gradient of the cost function wrt. weights and biases be? b) Use the c) Finish the e) What do we call the gradient method used above? d) Train your network and see how the accuracy changes! Make a plot if you want. e) How high of an accuracy is it possible to acheive with a neural network on this dataset, if we use the whole thing as training data?
@@ -3610,10 +3620,10 @@ features).
-
@@ -1345,10 +1355,10 @@ a more brute force way. Here we scale the mean values for each column of the des
-0.009134699065945493
-4.0244965108017645
-[[0.85613835 2.50655379]
- [2.50655379 8.3404509 ]]
+
0.04718566894028431
+4.11080997912276
+[[ 1.10517643 3.48455788]
+ [ 3.48455788 12.00216162]]
-
@@ -1378,30 +1388,30 @@ this matrix we easily see that it is a positive definite matrix.
0.07971187802560528
-1.800782161095708
-[[1. 0.59411814]
- [0.59411814 1. ]]
+
0.07836997022107646
+1.1378267322316808
+[[1. 0.63980097]
+ [0.63980097 1. ]]
-
@@ -1458,37 +1468,37 @@ this matrix we easily see that it is a positive definite matrix.
[[ 0.81395716 1.89155934]
- [-1.34726166 -4.13453411]
- [-0.46229544 -2.34061974]
- [ 0.24429334 1.4051634 ]
- [ 0.41971814 1.6405671 ]
- [ 2.02456235 5.03973227]
- [-1.97311824 -4.72521196]
- [ 0.10738656 0.24578123]
- [-0.52702419 -2.34023682]
- [ 0.69978197 3.31779928]]
+
[[ 1.34931214 3.06139439]
+ [-0.44476964 -2.60794187]
+ [ 0.02225493 0.16388664]
+ [-1.91193672 -3.82324216]
+ [-0.2044881 -1.56027537]
+ [-1.15572395 -3.25982474]
+ [ 0.94217756 1.49888671]
+ [ 0.28472162 2.92474572]
+ [ 2.38943 7.14118216]
+ [-1.27097785 -3.5388115 ]]
0 1
-0 0.813957 1.891559
-1 -1.347262 -4.134534
-2 -0.462295 -2.340620
-3 0.244293 1.405163
-4 0.419718 1.640567
-5 2.024562 5.039732
-6 -1.973118 -4.725212
-7 0.107387 0.245781
-8 -0.527024 -2.340237
-9 0.699782 3.317799
+0 1.349312 3.061394
+1 -0.444770 -2.607942
+2 0.022255 0.163887
+3 -1.911937 -3.823242
+4 -0.204488 -1.560275
+5 -1.155724 -3.259825
+6 0.942178 1.498887
+7 0.284722 2.924746
+8 2.389430 7.141182
+9 -1.270978 -3.538811
0 1
-0 1.000000 0.969413
-1 0.969413 1.000000
+0 1.000000 0.950873
+1 0.950873 1.000000
@@ -1969,11 +1979,13 @@ We select values of the hyperparameter
0 1 2 3 4 5 6 7 \
0 0.0 0.000000 0.000000 0.000000 0.000000 0.000000 0.000000 0.000000
-1 0.0 0.092746 0.090302 0.090561 0.088058 0.085463 0.081530 0.078898
-2 0.0 0.090302 0.088694 0.089052 0.086797 0.084423 0.080286 0.077782
-3 0.0 0.090561 0.089052 0.095106 0.092533 0.089858 0.089404 0.086489
-4 0.0 0.088058 0.086797 0.092533 0.090114 0.087585 0.086913 0.084127
-5 0.0 0.085463 0.084423 0.089858 0.087585 0.085197 0.084340 0.081681
-6 0.0 0.081530 0.080286 0.089404 0.086913 0.084340 0.086471 0.083596
-7 0.0 0.078898 0.077782 0.086489 0.084127 0.081681 0.083596 0.080849
-8 0.0 0.076334 0.075334 0.083645 0.081405 0.079080 0.080793 0.078170
-9 0.0 0.073841 0.072946 0.080875 0.078753 0.076543 0.078067 0.075562
-10 0.0 0.072973 0.071789 0.082361 0.079982 0.077541 0.081296 0.078544
-11 0.0 0.070485 0.069391 0.079518 0.077254 0.074928 0.078454 0.075823
-12 0.0 0.068088 0.067077 0.076777 0.074622 0.072404 0.075712 0.073198
-13 0.0 0.065778 0.064845 0.074134 0.072083 0.069970 0.073071 0.070667
-14 0.0 0.063554 0.062693 0.071588 0.069637 0.067623 0.070527 0.068229
+1 0.0 0.074334 0.080585 0.077061 0.078751 0.080220 0.070657 0.071406
+2 0.0 0.080585 0.088425 0.082009 0.084289 0.086338 0.074009 0.075052
+3 0.0 0.077061 0.082009 0.085147 0.086339 0.087297 0.081324 0.081796
+4 0.0 0.078751 0.084289 0.086339 0.087789 0.089007 0.081926 0.082537
+5 0.0 0.080220 0.086338 0.087297 0.089007 0.090492 0.082307 0.083061
+6 0.0 0.070657 0.074009 0.081324 0.081926 0.082307 0.079874 0.080032
+7 0.0 0.071406 0.075052 0.081796 0.082537 0.083061 0.080032 0.080271
+8 0.0 0.072148 0.076101 0.082240 0.083128 0.083801 0.080150 0.080474
+9 0.0 0.072902 0.077180 0.082670 0.083714 0.084548 0.080237 0.080651
+10 0.0 0.063646 0.065859 0.075320 0.075498 0.075472 0.075478 0.075409
+11 0.0 0.064071 0.066452 0.075576 0.075838 0.075897 0.075542 0.075525
+12 0.0 0.064514 0.067074 0.075838 0.076189 0.076340 0.075602 0.075639
+13 0.0 0.064980 0.067731 0.076108 0.076555 0.076803 0.075658 0.075753
+14 0.0 0.065472 0.068429 0.076389 0.076938 0.077292 0.075711 0.075868
8 9 10 11 12 13 14
0 0.000000 0.000000 0.000000 0.000000 0.000000 0.000000 0.000000
-1 0.076334 0.073841 0.072973 0.070485 0.068088 0.065778 0.063554
-2 0.075334 0.072946 0.071789 0.069391 0.067077 0.064845 0.062693
-3 0.083645 0.080875 0.082361 0.079518 0.076777 0.074134 0.071588
-4 0.081405 0.078753 0.079982 0.077254 0.074622 0.072083 0.069637
-5 0.079080 0.076543 0.077541 0.074928 0.072404 0.069970 0.067623
-6 0.080793 0.078067 0.081296 0.078454 0.075712 0.073071 0.070527
-7 0.078170 0.075562 0.078544 0.075823 0.073198 0.070667 0.068229
-8 0.075609 0.073115 0.075864 0.073260 0.070747 0.068324 0.065989
-9 0.073115 0.070730 0.073260 0.070769 0.068364 0.066044 0.063809
-10 0.075864 0.073260 0.077608 0.074866 0.072223 0.069676 0.067224
-11 0.073260 0.070769 0.074866 0.072241 0.069711 0.067273 0.064924
-12 0.070747 0.068364 0.072223 0.069711 0.067288 0.064954 0.062705
-13 0.068324 0.066044 0.069676 0.067273 0.064954 0.062719 0.060565
-14 0.065989 0.063809 0.067224 0.064924 0.062705 0.060565 0.058503
+1 0.072148 0.072902 0.063646 0.064071 0.064514 0.064980 0.065472
+2 0.076101 0.077180 0.065859 0.066452 0.067074 0.067731 0.068429
+3 0.082240 0.082670 0.075320 0.075576 0.075838 0.076108 0.076389
+4 0.083128 0.083714 0.075498 0.075838 0.076189 0.076555 0.076938
+5 0.083801 0.084548 0.075472 0.075897 0.076340 0.076803 0.077292
+6 0.080150 0.080237 0.075478 0.075542 0.075602 0.075658 0.075711
+7 0.080474 0.080651 0.075409 0.075525 0.075639 0.075753 0.075868
+8 0.080766 0.081038 0.075293 0.075463 0.075634 0.075809 0.075988
+9 0.081038 0.081411 0.075136 0.075363 0.075595 0.075834 0.076082
+10 0.075293 0.075136 0.072406 0.072329 0.072240 0.072140 0.072028
+11 0.075463 0.075363 0.072329 0.072286 0.072234 0.072173 0.072101
+12 0.075634 0.075595 0.072240 0.072234 0.072220 0.072199 0.072171
+13 0.075809 0.075834 0.072140 0.072173 0.072199 0.072221 0.072238
+14 0.075988 0.076082 0.072028 0.072101 0.072171 0.072238 0.072303
+
+[2. 2.]
-Training MSE for OLS
+
-Training MSE for OLS
3.0
+
-
+
+
@@ -1081,7 +1091,7 @@ theorem.
Runtime: 0.146141 sec
+
Runtime: 0.154751 sec
Jackknife Statistics :
original bias std. error
- 100.139 100.129 0.148776
+ 99.9896 99.9796 0.149524
@@ -1288,7 +1298,9 @@ Error: 0.08426840630693411
Bias^2: 0.0796891867672603
Var: 0.004579219539673834
0.08426840630693411 >= 0.0796891867672603 + 0.004579219539673834 = 0.08426840630693413
-Polynomial degree: 2
+Bootstrap Statistics :
original bias std. error
- 99.989 15.1792 99.9878 0.152149
+ 100.307 14.9693 100.309 0.149416
+
+Polynomial degree: 2
Error: 0.10398646080125035
Bias^2: 0.1007711427354898
Var: 0.0032153180657605116
@@ -1315,7 +1327,9 @@ Error: 0.037813671417389005
Bias^2: 0.033657685071527665
Var: 0.00415598634586135
0.037813671417389005 >= 0.033657685071527665 + 0.00415598634586135 = 0.03781367141738902
-Polynomial degree: 7
+
+
+Polynomial degree: 7
Error: 0.02760977349102253
Bias^2: 0.022999498260366312
Var: 0.004610275230656212
@@ -1342,21 +1356,21 @@ Error: 0.07160048164233104
Bias^2: 0.014436800088904942
Var: 0.05716368155342608
0.07160048164233104 >= 0.014436800088904942 + 0.05716368155342608 = 0.07160048164233102
-Polynomial degree: 12
+
-
-Polynomial degree: 12
Error: 0.11547777218872497
Bias^2: 0.01628578269596628
Var: 0.09919198949275869
0.11547777218872497 >= 0.01628578269596628 + 0.09919198949275869 = 0.11547777218872497
-
-Polynomial degree: 13
+Polynomial degree: 13
Error: 0.22842468702219465
Bias^2: 0.01975416527185249
Var: 0.20867052175034223
0.22842468702219465 >= 0.01975416527185249 + 0.20867052175034223 = 0.2284246870221947
+
Degree of polynomial: 25
+
-Degree of polynomial: 24
+Mean squared error on training data: 0.00084707
+Mean squared error on test data: 1325.26124692
+Degree of polynomial: 25
Mean squared error on training data: 0.00079125
Mean squared error on test data: 129012.83870189
Degree of polynomial: 26
@@ -1661,19 +1675,19 @@ Mean squared error on test data: 18388.59354079
Degree of polynomial: 27
Mean squared error on training data: 0.00069123
Mean squared error on test data: 2351.97979891
-Degree of polynomial: 28
-Mean squared error on training data: 0.00062592
-Mean squared error on test data: 3983.63037846
Degree of polynomial: 29
+
-Degree of polynomial: 28
+Mean squared error on training data: 0.00062592
+Mean squared error on test data: 3983.63037846
+Degree of polynomial: 29
Mean squared error on training data: 0.00060704
Mean squared error on test data: 3262.26814548
/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_22458/626635268.py:73: RuntimeWarning: divide by zero encountered in log10
+
@@ -1907,7 +1921,7 @@ cross-validation (LOOCV).
/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_57183/626635268.py:73: RuntimeWarning: divide by zero encountered in log10
plt.plot(polynomial, np.log10(trainingerror), label='Training Error')
-/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_22458/626635268.py:74: RuntimeWarning: divide by zero encountered in log10
+/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_57183/626635268.py:74: RuntimeWarning: divide by zero encountered in log10
plt.plot(polynomial, np.log10(testerror), label='Test Error')
-/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_22458/3817475779.py:63: RuntimeWarning: divide by zero encountered in log10
+
@@ -2796,7 +2810,7 @@ linear system as an equation would reduce this down to
/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_57183/3817475779.py:63: RuntimeWarning: divide by zero encountered in log10
plt.plot(polynomial, np.log10(estimated_mse_sklearn), label='Test Error')
-/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_22458/4162706317.py:7: UserWarning: set_ticklabels() should only be used with a fixed number of ticks, i.e. after set_ticks() or using a FixedLocator.
+
@@ -2940,7 +2954,7 @@ with the form utilized in linear regression, viz.
/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_57183/4162706317.py:7: UserWarning: set_ticklabels() should only be used with a fixed number of ticks, i.e. after set_ticks() or using a FixedLocator.
cb.ax.set_yticklabels(cb.ax.get_yticklabels(), fontsize=18)
-/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_22458/3777801602.py:7: UserWarning: set_ticklabels() should only be used with a fixed number of ticks, i.e. after set_ticks() or using a FixedLocator.
+
@@ -2980,7 +2994,7 @@ cost function is given by
/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_57183/3777801602.py:7: UserWarning: set_ticklabels() should only be used with a fixed number of ticks, i.e. after set_ticks() or using a FixedLocator.
cb.ax.set_yticklabels(cb.ax.get_yticklabels(), fontsize=18)
-/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_22458/438060758.py:10: UserWarning: set_ticklabels() should only be used with a fixed number of ticks, i.e. after set_ticks() or using a FixedLocator.
+
@@ -3015,7 +3029,7 @@ cost function is given by
/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_57183/438060758.py:10: UserWarning: set_ticklabels() should only be used with a fixed number of ticks, i.e. after set_ticks() or using a FixedLocator.
cb.ax.set_yticklabels(cb.ax.get_yticklabels(), fontsize=18)
-/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_22458/3544313922.py:9: UserWarning: set_ticklabels() should only be used with a fixed number of ticks, i.e. after set_ticks() or using a FixedLocator.
+
@@ -3068,43 +3082,43 @@ constant as opposed to ridge and OLS. We get a sparse solution with
/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_57183/3544313922.py:9: UserWarning: set_ticklabels() should only be used with a fixed number of ticks, i.e. after set_ticks() or using a FixedLocator.
cb.ax.set_yticklabels(cb.ax.get_yticklabels(), fontsize=18)
- 0%| | 0/10 [00:00<?, ?it/s]
+
0%| | 0/10 [00:00<?, ?it/s]
-/Users/mhjensen/miniforge3/envs/myenv/lib/python3.9/site-packages/sklearn/linear_model/_coordinate_descent.py:628: ConvergenceWarning: Objective did not converge. You might want to increase the number of iterations, check the scale of the features or consider increasing regularisation. Duality gap: 3.924e+00, tolerance: 1.797e+00
model = cd_fast.enet_coordinate_descent(
- 10%|███████████████████▏ | 1/10 [00:00<00:06, 1.29it/s]
+ 10%|█████████████▍ | 1/10 [00:00<00:07, 1.14it/s]
20%|██████████████████████████████████████▍ | 2/10 [00:01<00:06, 1.18it/s]
+
- 20%|██████████████████████████▊ | 2/10 [00:01<00:06, 1.16it/s]
30%|█████████████████████████████████████████████████████████▌ | 3/10 [00:02<00:05, 1.29it/s]
+
- 30%|████████████████████████████████████████▏ | 3/10 [00:02<00:05, 1.31it/s]
- 40%|████████████████████████████████████████████████████████████████████████████▊ | 4/10 [00:03<00:04, 1.32it/s]
+
- 40%|█████████████████████████████████████████████████████▌ | 4/10 [00:03<00:04, 1.39it/s]
@@ -1653,7 +1655,7 @@ It provides composable transformations of Python+NumPy programs: differentiate,
return asarray(x, dtype=self.dtype)
50%|████████████████████████████████████████████████████████████████████████████████████████████████ | 5/10 [00:03<00:03, 1.54it/s]
+
- 50%|███████████████████████████████████████████████████████████████████ | 5/10 [00:03<00:03, 1.38it/s]
60%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 6/10 [00:04<00:02, 1.65it/s]
+
- 60%|████████████████████████████████████████████████████████████████████████████████▍ | 6/10 [00:04<00:02, 1.44it/s]
@@ -1578,9 +1580,9 @@ theta from own sdg with momentum
70%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 7/10 [00:04<00:02, 1.44it/s]
+
- 70%|█████████████████████████████████████████████████████████████████████████████████████████████▊ | 7/10 [00:05<00:02, 1.45it/s]
@@ -1490,9 +1492,9 @@ theta from own sdg with momentum
80%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 8/10 [00:05<00:01, 1.50it/s]
+
- 80%|███████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 8/10 [00:05<00:01, 1.55it/s]
90%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 9/10 [00:06<00:00, 1.47it/s]
+
- 90%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 9/10 [00:06<00:00, 1.54it/s]
100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 10/10 [00:06<00:00, 1.52it/s]
+
-100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 10/10 [00:07<00:00, 1.48it/s]
100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 10/10 [00:06<00:00, 1.45it/s]
+
100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 10/10 [00:07<00:00, 1.43it/s]
diff --git a/doc/LectureNotes/_build/html/chapter4.html b/doc/LectureNotes/_build/html/chapter4.html
index 3e81bb26c..1abcd6de0 100644
--- a/doc/LectureNotes/_build/html/chapter4.html
+++ b/doc/LectureNotes/_build/html/chapter4.html
@@ -323,6 +323,16 @@ const thebe_selector_output = ".output, .cell_output"
Week 41 Neural networks and constructing a neural network code
+
2nd degree coefficients:
-zero power: -5.030164788184997
-first power: 0.001268544905013411
-second power: -5.234332597247826e-05
+zero power: -0.22158725223474995
+first power: 0.24121476598003452
+second power: -0.0009532583857747255
diff --git a/doc/LectureNotes/_build/html/chapter7.html b/doc/LectureNotes/_build/html/chapter7.html
index 1e6480c23..2b7035e16 100644
--- a/doc/LectureNotes/_build/html/chapter7.html
+++ b/doc/LectureNotes/_build/html/chapter7.html
@@ -323,6 +323,16 @@ const thebe_selector_output = ".output, .cell_output"
Week 41 Neural networks and constructing a neural network code
+
-
@@ -784,10 +794,10 @@ a more brute force way. Here we scale the mean values for each column of the des
-0.012423940191689783
-4.101008878523571
-[[0.89527291 2.65532045]
- [2.65532045 8.81987609]]
+
0.1001408041761458
+4.2807716628772665
+[[ 1.15654145 3.54867722]
+ [ 3.54867722 11.70485195]]
-
@@ -816,30 +826,30 @@ this matrix we easily see that it is a positive definite matrix.
0.08705631913312815
-1.7026908764394864
-[[1. 0.65870313]
- [0.65870313 1. ]]
+
0.09543871010617433
+1.6888043337746685
+[[1. 0.7167077]
+ [0.7167077 1. ]]
-
@@ -896,37 +906,37 @@ this matrix we easily see that it is a positive definite matrix.
[[-1.7755649 -4.56778296]
- [-0.81015037 -2.80072356]
- [ 0.73628249 1.95206335]
- [ 0.97366347 1.61130099]
- [ 0.7271324 1.97965627]
- [ 0.36881837 0.56037913]
- [-1.33163086 -2.59391196]
- [-0.68953877 -1.58298728]
- [ 0.19982428 -1.08010965]
- [ 1.60116388 6.52211567]]
+
[[-0.20575734 0.01384583]
+ [-0.89876098 -3.04065686]
+ [-0.76289128 -3.17080691]
+ [-0.0334136 0.16124569]
+ [ 2.73970542 9.28885103]
+ [ 0.75413023 2.98474769]
+ [-1.87894459 -5.48121459]
+ [-1.26814205 -2.4848097 ]
+ [ 0.18114057 -0.9889962 ]
+ [ 1.37293361 2.71779401]]
0 1
-0 -1.775565 -4.567783
-1 -0.810150 -2.800724
-2 0.736282 1.952063
-3 0.973663 1.611301
-4 0.727132 1.979656
-5 0.368818 0.560379
-6 -1.331631 -2.593912
-7 -0.689539 -1.582987
-8 0.199824 -1.080110
-9 1.601164 6.522116
- 0 1
-0 1.00000 0.94335
-1 0.94335 1.00000
+0 -0.205757 0.013846
+1 -0.898761 -3.040657
+2 -0.762891 -3.170807
+3 -0.033414 0.161246
+4 2.739705 9.288851
+5 0.754130 2.984748
+6 -1.878945 -5.481215
+7 -1.268142 -2.484810
+8 0.181141 -0.988996
+9 1.372934 2.717794
+ 0 1
+0 1.000000 0.970965
+1 0.970965 1.000000
@@ -1115,10 +1125,12 @@ We can write our own code or simply use either the functionaly of numpy<
0 1 2 3 4 5 6 7 \
0 0.0 0.000000 0.000000 0.000000 0.000000 0.000000 0.000000 0.000000
-1 0.0 0.088650 0.084209 0.092689 0.091653 0.090476 0.085764 0.085378
-2 0.0 0.084209 0.080559 0.088599 0.087876 0.087020 0.082478 0.082249
-3 0.0 0.092689 0.088599 0.102209 0.101292 0.100209 0.097667 0.097380
-4 0.0 0.091653 0.087876 0.101292 0.100523 0.099588 0.097017 0.096811
-5 0.0 0.090476 0.087020 0.100209 0.099588 0.098803 0.096203 0.096078
-6 0.0 0.085764 0.082478 0.097667 0.097017 0.096203 0.095425 0.095278
-7 0.0 0.085378 0.082249 0.097380 0.096811 0.096078 0.095278 0.095178
-8 0.0 0.084976 0.082002 0.097060 0.096570 0.095915 0.095090 0.095037
-9 0.0 0.084550 0.081730 0.096700 0.096287 0.095710 0.094857 0.094849
-10 0.0 0.077672 0.075070 0.090429 0.090002 0.089421 0.089826 0.089786
-11 0.0 0.077490 0.074976 0.090319 0.089940 0.089408 0.089800 0.089790
-12 0.0 0.077310 0.074882 0.090204 0.089872 0.089386 0.089763 0.089783
-13 0.0 0.077131 0.074787 0.090081 0.089795 0.089354 0.089714 0.089763
-14 0.0 0.076951 0.074688 0.089949 0.089708 0.089311 0.089653 0.089730
+1 0.0 0.090565 0.089065 0.093226 0.091674 0.090154 0.086287 0.084847
+2 0.0 0.089065 0.087946 0.092421 0.091068 0.089735 0.085988 0.084674
+3 0.0 0.093226 0.092421 0.102147 0.100816 0.099494 0.098144 0.096731
+4 0.0 0.091674 0.091068 0.100816 0.099622 0.098431 0.097115 0.095803
+5 0.0 0.090154 0.089735 0.099494 0.098431 0.097365 0.096077 0.094862
+6 0.0 0.086287 0.085988 0.098144 0.097115 0.096077 0.096630 0.095395
+7 0.0 0.084847 0.084674 0.096731 0.095803 0.094862 0.095395 0.094243
+8 0.0 0.083459 0.083405 0.095358 0.094527 0.093680 0.094189 0.093115
+9 0.0 0.082121 0.082180 0.094027 0.093288 0.092530 0.093011 0.092013
+10 0.0 0.078708 0.078711 0.091732 0.090935 0.090118 0.091871 0.090804
+11 0.0 0.077431 0.077523 0.090387 0.089668 0.088926 0.090626 0.089626
+12 0.0 0.076203 0.076378 0.089086 0.088441 0.087772 0.089417 0.088481
+13 0.0 0.075021 0.075274 0.087828 0.087255 0.086655 0.088242 0.087369
+14 0.0 0.073883 0.074212 0.086611 0.086107 0.085573 0.087102 0.086289
8 9 10 11 12 13 14
0 0.000000 0.000000 0.000000 0.000000 0.000000 0.000000 0.000000
-1 0.084976 0.084550 0.077672 0.077490 0.077310 0.077131 0.076951
-2 0.082002 0.081730 0.075070 0.074976 0.074882 0.074787 0.074688
-3 0.097060 0.096700 0.090429 0.090319 0.090204 0.090081 0.089949
-4 0.096570 0.096287 0.090002 0.089940 0.089872 0.089795 0.089708
-5 0.095915 0.095710 0.089421 0.089408 0.089386 0.089354 0.089311
-6 0.095090 0.094857 0.089826 0.089800 0.089763 0.089714 0.089653
-7 0.095037 0.094849 0.089786 0.089790 0.089783 0.089763 0.089730
-8 0.094941 0.094797 0.089704 0.089738 0.089760 0.089769 0.089763
-9 0.094797 0.094697 0.089576 0.089639 0.089690 0.089726 0.089748
-10 0.089704 0.089576 0.085655 0.085690 0.085712 0.085722 0.085716
-11 0.089738 0.089639 0.085690 0.085747 0.085790 0.085819 0.085833
-12 0.089760 0.089690 0.085712 0.085790 0.085853 0.085902 0.085935
-13 0.089769 0.089726 0.085722 0.085819 0.085902 0.085970 0.086021
-14 0.089763 0.089748 0.085716 0.085833 0.085935 0.086021 0.086092
+1 0.083459 0.082121 0.078708 0.077431 0.076203 0.075021 0.073883
+2 0.083405 0.082180 0.078711 0.077523 0.076378 0.075274 0.074212
+3 0.095358 0.094027 0.091732 0.090387 0.089086 0.087828 0.086611
+4 0.094527 0.093288 0.090935 0.089668 0.088441 0.087255 0.086107
+5 0.093680 0.092530 0.090118 0.088926 0.087772 0.086655 0.085573
+6 0.094189 0.093011 0.091871 0.090626 0.089417 0.088242 0.087102
+7 0.093115 0.092013 0.090804 0.089626 0.088481 0.087369 0.086289
+8 0.092064 0.091034 0.089755 0.088642 0.087560 0.086508 0.085486
+9 0.091034 0.090075 0.088726 0.087675 0.086653 0.085659 0.084694
+10 0.089755 0.088726 0.088455 0.087327 0.086227 0.085155 0.084112
+11 0.088642 0.087675 0.087327 0.086256 0.085212 0.084195 0.083203
+12 0.087560 0.086653 0.086227 0.085212 0.084222 0.083257 0.082316
+13 0.086508 0.085659 0.085155 0.084195 0.083257 0.082342 0.081450
+14 0.085486 0.084694 0.084112 0.083203 0.082316 0.081450 0.080605
@@ -1145,8 +1157,8 @@ Our own code here is not very elegant and asks for obvious improvements. It is t
+
+ 0 1
-0 3.914672 1.954823
-1 1.954823 1.963858
-[[3.91467223 1.95482298]
- [1.95482298 1.96385798]]
+0 4.114499 2.071143
+1 2.071143 2.061388
+
[[4.11449851 2.07114326]
+ [2.07114326 2.0613875 ]]
Centered covariance using own code
-[[3.91467223 1.95482298]
- [1.95482298 1.96385798]]
+[[4.11449851 2.07114326]
+ [2.07114326 2.0613875 ]]
@@ -1206,16 +1218,16 @@ questions.
diff --git a/doc/LectureNotes/_build/html/chapter9.html b/doc/LectureNotes/_build/html/chapter9.html
index 82abdae5b..b577b1a11 100644
--- a/doc/LectureNotes/_build/html/chapter9.html
+++ b/doc/LectureNotes/_build/html/chapter9.html
@@ -323,6 +323,16 @@ const thebe_selector_output = ".output, .cell_output"
Week 41 Neural networks and constructing a neural network code
+ Eigenvalues of Covariance matrix
-5.123928000581825
-0.7546022135629743
+5.399533503407795
+0.776352510835556
First eigenvector
-[0.85043503 0.5260801 ]
+[0.84973247 0.52721412]
Second eigenvector
-[-0.5260801 0.85043503]
+[-0.52721412 0.84973247]
Eigenvector of largest eigenvalue
-[0.85043503 0.5260801 ]
+[-0.84973247 -0.52721412]
Own inversion
-[[4.04881585]
- [2.96745096]]
-Eigenvalues of Hessian Matrix:[0.37175588 4.15690073]
+[[4.1729993]
+ [3.0170097]]
+Eigenvalues of Hessian Matrix:[0.28192769 4.68753434]
theta from own gd
-[[4.04881585]
- [2.96745096]]
+[[4.1729993]
+ [3.0170097]]
theta from own sdg
-[[4.04015455]
- [2.95745651]]
+[[4.14043884]
+ [2.99071523]]
@@ -934,14 +934,14 @@ first example shows results with ordinary leats squares.
@@ -1131,17 +1131,17 @@ theta from own gd wth momentum
Own inversion
-[[3.37007195]
- [3.48441798]]
-Eigenvalues of Hessian Matrix:[0.32411274 4.30450049]
+[[4.03696458]
+ [3.0324793 ]]
+Eigenvalues of Hessian Matrix:[0.30959659 4.4150026 ]
theta from own gd
-[[3.37007195]
- [3.48441798]]
+[[4.03696458]
+ [3.0324793 ]]
@@ -1012,73 +1012,73 @@ Eigenvalues of Hessian Matrix:[0.32411274 4.30450049]
Own inversion
[[4.]
[3.]]
-Eigenvalues of Hessian Matrix:[0.27227895 4.125757 ]
-0 [-11.75978215] [-12.89770892]
-1 [-0.06807123] [0.06136823]
-2 [-0.06357887] [0.05731824]
-3 [-0.05938299] [0.05353553]
-4 [-0.05546402] [0.05000245]
-5 [-0.05180367] [0.04670255]
-6 [-0.04838489] [0.04362042]
-7 [-0.04519174] [0.04074169]
-8 [-0.04220931] [0.03805295]
-9 [-0.03942371] [0.03554165]
-10 [-0.03682195] [0.03319608]
-11 [-0.03439189] [0.03100531]
-12 [-0.0321222] [0.02895911]
-13 [-0.0300023] [0.02704796]
-14 [-0.0280223] [0.02526293]
-15 [-0.02617297] [0.02359571]
-16 [-0.02444569] [0.02203851]
-17 [-0.0228324] [0.02058408]
-18 [-0.02132557] [0.01922564]
-19 [-0.01991819] [0.01795684]
-20 [-0.0186037] [0.01677178]
-21 [-0.01737595] [0.01566493]
-22 [-0.01622922] [0.01463112]
-23 [-0.01515818] [0.01366555]
-24 [-0.01415781] [0.01276369]
-25 [-0.01322347] [0.01192135]
-26 [-0.01235079] [0.0111346]
-27 [-0.0115357] [0.01039977]
-28 [-0.0107744] [0.00971344]
-29 [-0.01006335] [0.0090724]
+Eigenvalues of Hessian Matrix:[0.23469347 4.90407685]
+0 [-18.02987043] [-22.42501752]
+1 [-0.32329897] [0.25206371]
+2 [-0.30782691] [0.24000075]
+3 [-0.2930953] [0.22851508]
+4 [-0.27906869] [0.21757908]
+5 [-0.26571335] [0.20716644]
+6 [-0.25299716] [0.19725211]
+7 [-0.24088952] [0.18781225]
+8 [-0.22936132] [0.17882416]
+9 [-0.21838482] [0.17026621]
+10 [-0.20793362] [0.16211781]
+11 [-0.19798258] [0.15435937]
+12 [-0.18850776] [0.14697222]
+13 [-0.17948638] [0.1399386]
+14 [-0.17089674] [0.13324158]
+15 [-0.16271817] [0.12686507]
+16 [-0.15493099] [0.12079371]
+17 [-0.14751649] [0.11501291]
+18 [-0.14045682] [0.10950876]
+19 [-0.13373501] [0.10426802]
+20 [-0.12733488] [0.09927808]
+21 [-0.12124104] [0.09452695]
+22 [-0.11543883] [0.09000319]
+23 [-0.10991429] [0.08569593]
+24 [-0.10465415] [0.08159479]
+25 [-0.09964573] [0.07768993]
+26 [-0.094877] [0.07397193]
+27 [-0.09033649] [0.07043187]
+28 [-0.08601328] [0.06706123]
+29 [-0.08189696] [0.06385189]
theta from own gd
-[[3.96547946]
- [3.03112129]]
-0 [-0.00939922] [0.00847367]
-1 [-0.00877892] [0.00791445]
-2 [-0.00801346] [0.00722437]
-3 [-0.00725498] [0.00654058]
-4 [-0.00654864] [0.00590379]
-5 [-0.00590456] [0.00532314]
-6 [-0.00532167] [0.00479764]
-7 [-0.0047956] [0.00432337]
-8 [-0.00432129] [0.00389577]
-9 [-0.00389382] [0.00351039]
-10 [-0.0035086] [0.00316311]
-11 [-0.00316149] [0.00285017]
-12 [-0.00284871] [0.0025682]
-13 [-0.00256688] [0.00231412]
-14 [-0.00231293] [0.00208517]
-15 [-0.0020841] [0.00187888]
-16 [-0.00187791] [0.00169299]
-17 [-0.00169212] [0.0015255]
-18 [-0.00152471] [0.00137458]
-19 [-0.00137387] [0.00123858]
-20 [-0.00123795] [0.00111605]
-21 [-0.00111547] [0.00100563]
-22 [-0.00100511] [0.00090614]
-23 [-0.00090567] [0.00081649]
-24 [-0.00081607] [0.00073571]
-25 [-0.00073534] [0.00066293]
-26 [-0.00066259] [0.00059734]
-27 [-0.00059703] [0.00053824]
-28 [-0.00053797] [0.00048499]
-29 [-0.00048474] [0.00043701]
+[[3.66774691]
+ [3.25904489]]
+0 [-0.07797763] [0.06079614]
+1 [-0.07424587] [0.05788664]
+2 [-0.06957317] [0.05424351]
+3 [-0.06484181] [0.05055465]
+4 [-0.06031928] [0.04702861]
+5 [-0.05607583] [0.04372016]
+6 [-0.05211919] [0.04063532]
+7 [-0.04843794] [0.03776519]
+8 [-0.04501548] [0.03509683]
+9 [-0.04183444] [0.0326167]
+10 [-0.03887807] [0.03031173]
+11 [-0.03613058] [0.02816961]
+12 [-0.03357723] [0.02617887]
+13 [-0.03120433] [0.02432881]
+14 [-0.02899912] [0.02260949]
+15 [-0.02694975] [0.02101168]
+16 [-0.02504521] [0.01952679]
+17 [-0.02327527] [0.01814683]
+18 [-0.0216304] [0.01686439]
+19 [-0.02010178] [0.01567258]
+20 [-0.01868119] [0.014565]
+21 [-0.01736099] [0.01353569]
+22 [-0.01613409] [0.01257912]
+23 [-0.01499389] [0.01169016]
+24 [-0.01393427] [0.01086401]
+25 [-0.01294954] [0.01009625]
+26 [-0.01203439] [0.00938275]
+27 [-0.01118392] [0.00871967]
+28 [-0.01039355] [0.00810345]
+29 [-0.00965904] [0.00753078]
theta from own gd wth momentum
-[[3.99839581]
- [3.00144622]]
+[[3.96175251]
+ [3.02982009]]
@@ -1230,18 +1230,20 @@ beta from own Newton code
Own inversion
-[[3.81818338]
- [3.12840176]]
-Eigenvalues of Hessian Matrix:[0.26024513 4.64291046]
-0 [-16.3824865] [-20.20867716]
-1 [9.71640303e-15] [1.05750493e-14]
-2 [-2.05998413e-17] [-2.11570114e-16]
-3 [6.96057795e-17] [1.90885733e-16]
-4 [6.96057795e-17] [1.90885733e-16]
+[[4.15451852]
+ [2.83230774]]
+Eigenvalues of Hessian Matrix:[0.30616802 4.24299211]
+0 [-10.57502449] [-11.57610367]
+1 [-4.47905601e-15] [-4.07372439e-16]
+2 [-6.9388939e-16] [-7.21432413e-16]
+3 [-6.9388939e-16] [-7.21432413e-16]
+4 [-6.9388939e-16] [-7.21432413e-16]
beta from own Newton code
-[[3.81818338]
- [3.12840176]]
+[[4.15451852]
+ [2.83230774]]
@@ -1323,15 +1325,15 @@ theta from own gd
-Own inversion
-[[3.6955259]
- [3.2809076]]
-Eigenvalues of Hessian Matrix:[0.31381731 4.50516278]
-theta from own gd
-[[3.6955259]
- [3.2809076]]
+[[4.04601419]
+ [3.12204312]]
+Eigenvalues of Hessian Matrix:[0.33604208 4.45709724]
+
+
+theta from own gd
+[[4.04601419]
+ [3.12204312]]
+
theta from own sdg
-[[3.6805151 ]
- [3.33045013]]
+[[4.02781444]
+ [3.13976073]]
@@ -1406,9 +1408,9 @@ theta from own sdg with momentum
Own inversion
-[[3.98751068]
- [2.92901115]]
-Eigenvalues of Hessian Matrix:[0.28244905 4.61501312]
+[[3.96417888]
+ [3.06634473]]
+Eigenvalues of Hessian Matrix:[0.32962444 4.18715465]
theta from own gd
-[[3.98556236]
- [2.93059014]]
+[[3.9639885]
+ [3.0665111]]
theta from own sdg with momentum
-[[4.01133846]
- [2.92452609]]
+[[4.00842216]
+ [3.14285244]]
theta from own AdaGrad
-[[1.99974365]
- [3.0013839 ]
- [3.99861193]]
+[[1.99969895]
+ [3.00167058]
+ [3.99835872]]
theta from own RMSprop
-[[1.99757634]
- [2.9983289 ]
- [3.99759503]]
+[[1.99852187]
+ [3.03868311]
+ [3.95744254]]
theta from own ADAM
-[[1.99993596]
- [3.00035483]
- [3.99963172]]
+[[1.99996471]
+ [3.00026784]
+ [3.99973141]]
[<matplotlib.lines.Line2D at 0x1360ef5e0>]
+
[<matplotlib.lines.Line2D at 0x11892e7f0>]
@@ -1688,7 +1690,7 @@ It provides composable transformations of Python+NumPy programs: differentiate,
-<matplotlib.collections.PathCollection at 0x135f8b1f0>
+
<matplotlib.collections.PathCollection at 0x118995f70>
diff --git a/doc/LectureNotes/_build/html/exercisesweek42.html b/doc/LectureNotes/_build/html/exercisesweek42.html
index f044e2489..f4d0fc025 100644
--- a/doc/LectureNotes/_build/html/exercisesweek42.html
+++ b/doc/LectureNotes/_build/html/exercisesweek42.html
@@ -445,13 +445,23 @@ const thebe_selector_output = ".output, .cell_output"
Exercises week 42¶
-
Overarching aims of the exercises this week¶
-
@@ -541,47 +587,43 @@ doconce format html exercisesweek41.do.txt -->
import autograd.numpy as np
-from autograd import grad
+
import autograd.numpy as np # We need to use this numpy wrapper to make automatic differentiation work later
+from sklearn import datasets
+import matplotlib.pyplot as plt
+from sklearn.metrics import accuracy_score
+
+
+# Defining some activation functions
+def ReLU(z):
+ return np.where(z > 0, z, 0)
+
+
+def sigmoid(z):
+ return 1 / (1 + np.exp(-z))
+
+
+def softmax(z):
+ """Compute softmax values for each set of scores in the rows of the matrix z.
+ Used with batched input data."""
+ e_z = np.exp(z - np.max(z, axis=0))
+ return e_z / np.sum(e_z, axis=1)[:, np.newaxis]
+
+
+def softmax_vec(z):
+ """Compute softmax values for each set of scores in the vector z.
+ Use this function when you use the activation function on one vector at a time"""
+ e_z = np.exp(z - np.max(z))
+ return e_z / np.sum(e_z)
Exercise 1¶
-
-
np.random.seed(2024)
-
-def ReLU(z):
- return np.where(z > 0, z, 0)
-
-
-x = np.random.randn(2) # network input
+x = np.random.randn(2) # network input. This is a single input with two features
W1 = np.random.randn(4, 2) # first layer weights
b1with the correct shapeb1with the correct shape. (Run the next cell right after the previous to get the random generated values to line up with the test solution below)
-
-b1 = np.random.randn(4)
+
b1 = ...
z1 for the first layerz1 for the first layer
-
-z1 = W1 @ x + b1
+
z1 = ...
a1 for the first layer using the ReLU activation function defined earlier.a1 for the first layer using the ReLU activation function defined earlier.
-
-a1 = ReLU(z1)
+
a1 = ...
b1 with the randn function right after you define W1.
sol1 = np.array([0.60610368, 4.0076268, 0.0, 0.56469864])
@@ -591,7 +633,41 @@ doconce format html exercisesweek41.do.txt -->
-
@@ -599,21 +675,22 @@ doconce format html exercisesweek41.do.txt -->
True
+
---------------------------------------------------------------------------
+TypeError Traceback (most recent call last)
+Cell In[6], line 3
+ 1 sol1 = np.array([0.60610368, 4.0076268, 0.0, 0.56469864])
+----> 3 print(np.allclose(a1, sol1))
+
+File ~/miniforge3/envs/myenv/lib/python3.9/site-packages/autograd/tracer.py:48, in primitive.<locals>.f_wrapped(*args, **kwargs)
+ 46 return new_box(ans, trace, node)
+ 47 else:
+---> 48 return f_raw(*args, **kwargs)
+
+File <__array_function__ internals>:180, in allclose(*args, **kwargs)
+
+File ~/miniforge3/envs/myenv/lib/python3.9/site-packages/numpy/core/numeric.py:2251, in allclose(a, b, rtol, atol, equal_nan)
+ 2180 @array_function_dispatch(_allclose_dispatcher)
+ 2181 def allclose(a, b, rtol=1.e-5, atol=1.e-8, equal_nan=False):
+ 2182 """
+ 2183 Returns True if two arrays are element-wise equal within a tolerance.
+ 2184
+ (...)
+ 2249
+ 2250 """
+-> 2251 res = all(isclose(a, b, rtol=rtol, atol=atol, equal_nan=equal_nan))
+ 2252 return bool(res)
+
+File <__array_function__ internals>:180, in isclose(*args, **kwargs)
+
+File ~/miniforge3/envs/myenv/lib/python3.9/site-packages/numpy/core/numeric.py:2358, in isclose(a, b, rtol, atol, equal_nan)
+ 2355 dt = multiarray.result_type(y, 1.)
+ 2356 y = asanyarray(y, dtype=dt)
+-> 2358 xfin = isfinite(x)
+ 2359 yfin = isfinite(y)
+ 2360 if all(xfin) and all(yfin):
+
+TypeError: ufunc 'isfinite' not supported for the input types, and the inputs could not be safely coerced to any supported types according to the casting rule ''safe''
Exercise 2¶
-
-
-W2 = np.random.randn(8, 4)
-b2 = np.random.randn(8)
+
W2 = ...
+b2 = ...
z2 and activation a2 for the second layer.z2 and activation a2 for the second layer.
-
@@ -621,12 +698,9 @@ doconce format html exercisesweek41.do.txt -->
z2 = W2 @ a1
-a2 = ReLU(z2)
+
z2 = ...
+a2 = ...
-
-
-
-print(a2.shape == (8,))
-
-
@@ -638,65 +712,115 @@ doconce format html exercisesweek41.do.txt -->
True
+
print(
+ np.allclose(np.exp(len(a2)), 2980.9579870417283)
+) # This should evaluate to True if a2 has the correct shape :)
layers of weight and bias tuples (W, b) for each layer, in order, with the correct shapes that we can use later as our network parameters.
-
-def create_layers(network_input_size, output_sizes):
+
def create_layers(network_input_size, layer_output_sizes):
layers = []
i_size = network_input_size
- for output_size in output_sizes:
- W = np.random.rand(output_size, i_size)
- b = np.random.rand(output_size)
+ for layer_output_size in layer_output_sizes:
+ W = ...
+ b = ...
layers.append((W, b))
- i_size = output_size
+ i_size = layer_output_size
return layers
z and activation a for each layer, and returns the final activation a. This is the complete feed-forward pass, a full neural network!z and activation a for each layer, with ReLU actication, and returns the final activation a. This is the complete feed-forward pass, a full neural network!
-
-def feed_forward(layers, input):
+
def feed_forward_all_relu(layers, input):
a = input
for W, b in layers:
- z = W @ a + b
- a = ReLU(z)
+ z = ...
+ a = ...
return a
-Exercise 4¶
-create_layers function also accept a list of activation functions, which is used to add activation functions to each of the tuples in layers. Make new functions to not mess with the old ones.
-
+
+
+def create_layers_4(network_input_size, output_sizes, activation_funcs):
+
+
+input_size = ...
+layer_output_sizes = [...]
+
+x = np.random.rand(input_size)
+layers = ...
+predict = ...
+print(predict)
+
+
+Exercise 4 - Custom activation for each layer¶
+feed_forward function which accepts a list of activation functions as an argument, and which evaluates these activation functions at each layer.
+
+
+
+
+
+def feed_forward(input, layers, activations):
+ a = input
+ for (W, b), activation in zip(layers, activations):
+ z = ...
+ a = ...
+ return a
+
+
+
+
+
+
+network_input_size = ...
+layer_output_sizes = [...]
+activations = [...]
+layers = ...
+
+x = np.random.randn(network_input_size)
+feed_forward(x, layers, activations)
+
+Exercise 5 - Processing multiple inputs at once¶
+create_layers_batch so that the weight matrix is the transpose of what it was when you only sent in one input at a time.
+
-
+
def create_layers_batch(network_input_size, layer_output_sizes):
layers = []
i_size = network_input_size
- for output_size, activation in zip(output_sizes, activation_funcs):
- W = np.random.rand(output_size, i_size)
- b = np.random.rand(output_size)
- layers.append((W, b, activation))
+ for layer_output_size in layer_output_sizes:
+ W = ...
+ b = ...
+ layers.append((W, b))
- i_size = output_size
+ i_size = layer_output_size
return layers
feed_forward function to support this change.feed_forward_batch so that you can process this matrix of inputs with only one matrix multiplication and one broadcasted vector addition per layer. (Hint: You will only need to swap two variable around from your previous implementation, but remember to test that you get the same results for equivelent inputs!)
-def feed_forward_4(layers, input):
- a = input
- for W, b, activation in layers:
- z = W @ a + b
- a = activation(z)
+
@@ -705,28 +829,26 @@ doconce format html exercisesweek41.do.txt -->
inputs = np.random.rand(1000, 4)
+
+
+def feed_forward_batch(inputs, layers, activations):
+ a = inputs
+ for (W, b), activation in zip(layers, activations):
+ z = ...
+ a = ...
return a
-
+
-from scipy.special import softmax
-
-network_input_size = 4
-output_sizes = [12, 10, 3]
-activation_funcs = [ReLU, ReLU, softmax]
-layers = create_layers_4(network_input_size, output_sizes, activation_funcs)
+
network_input_size = ...
+layer_output_sizes = [...]
+activations = [...]
+layers = create_layers_batch(network_input_size, layer_output_sizes)
x = np.random.randn(network_input_size)
-predict = feed_forward_4(layers, x)
+feed_forward_batch(inputs, layers, activations)
+Exercise 6 - Predicting on real data¶
+
-
-
-# Loading and plotting iris dataset
-from sklearn import datasets
-import matplotlib.pyplot as plt
-
-iris = datasets.load_iris()
+
iris = datasets.load_iris()
_, ax = plt.subplots()
scatter = ax.scatter(iris.data[:, 0], iris.data[:, 1], c=iris.target)
@@ -737,29 +859,114 @@ doconce format html exercisesweek41.do.txt -->
-
-
iris.data) and evaluate the network for each data point.
-
-
+
+# No need to change this cell! Just make sure it works!
-for x in iris.data:
- prediction = feed_forward_4(layers, x)
+
+
+inputs = iris.data
+
+# Since each prediction is a vector with a score for each of the three types of flowers,
+# we need to make each target a vector with a 1 for the correct flower and a 0 for the others.
+targets = np.zeros((len(iris.data), 3))
+for i, t in enumerate(iris.target):
+ targets[i, t] = 1
+
+
+def accuracy(predictions, targets):
+ one_hot_predictions = np.zeros(predictions.shape)
+
+ for i, prediction in enumerate(predictions):
+ one_hot_predictions[i, np.argmax(prediction)] = 1
+ return accuracy_score(one_hot_predictions, targets)
+
+
+
+
+
+
+...
+layers = ...
+
+
+
+
+
+
+predictions = feed_forward_batch(inputs, layers, activations)
+
+
+
print(accuracy(predictions, targets))
-
-
+
diff --git a/doc/LectureNotes/_build/html/week42.html b/doc/LectureNotes/_build/html/week42.html
index f8178314f..3a6eaabc7 100644
--- a/doc/LectureNotes/_build/html/week42.html
+++ b/doc/LectureNotes/_build/html/week42.html
@@ -3378,7 +3378,7 @@ the Hadamard product, meaning element-wise multiplication.
Exercise 5 (Very optional and very hard :)¶
-
+
-
+
@@ -323,6 +323,16 @@ const thebe_selector_output = ".output, .cell_output"
Week 41 Neural networks and constructing a neural network code
+ Exercise 6 - Training on real data¶
+
+
+
+
+
+
+from autograd import grad
+
+
+def cost(input, layers, activations, target):
+ predict = feed_forward_batch(input, layers, activations)
+ return cross_entropy(predict, target)
+
+
+def cross_entropy(predict, target):
+ return np.sum(-target * np.log(predict))
+
+
+gradient_func = grad(
+ cross_entropy, 1
+) # Taking the gradient wrt. the second input to the cost function
+
gradient_func function to take the gradient of the cross entropy wrt. the weights and biases of the network. Check the shapes of what’s inside. What does the grad func from autograd actually do?
+
+
+
+
+
+layers_grad = gradient_func(inputs, layers, activations, targets) # Don't change this
+
train_network function.
+
+
+
+
+
+def train_network(
+ inputs, layers, activations, targets, learning_rate=0.001, epochs=100
+):
+ for i in range(epochs):
+ layers_grad = gradient_func(inputs, layers, activations, targets)
+ for (W, b), (W_g, b_g) in zip(layers, layers_grad):
+ W -= ...
+ b -= ...
+
+
+
+
+
+
+...
+
-Old accuracy on training data: 0.1440501043841336
-/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_55841/953065564.py:4: RuntimeWarning: overflow encountered in exp
+
@@ -3714,7 +3714,7 @@ Lambda = 10.0
Accuracy score on test set: 0.19166666666666668
/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_57345/953065564.py:4: RuntimeWarning: overflow encountered in exp
return 1/(1 + np.exp(-x))
-/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_55841/953065564.py:4: RuntimeWarning: overflow encountered in exp
+
@@ -3723,7 +3723,7 @@ Lambda = 1e-05
Accuracy score on test set: 0.10555555555555556
/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_57345/953065564.py:4: RuntimeWarning: overflow encountered in exp
return 1/(1 + np.exp(-x))
-/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_55841/953065564.py:4: RuntimeWarning: overflow encountered in exp
+
@@ -3732,7 +3732,7 @@ Lambda = 0.0001
Accuracy score on test set: 0.08611111111111111
/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_57345/953065564.py:4: RuntimeWarning: overflow encountered in exp
return 1/(1 + np.exp(-x))
-/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_55841/953065564.py:4: RuntimeWarning: overflow encountered in exp
+
@@ -3741,7 +3741,7 @@ Lambda = 0.001
Accuracy score on test set: 0.10555555555555556
/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_57345/953065564.py:4: RuntimeWarning: overflow encountered in exp
return 1/(1 + np.exp(-x))
-/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_55841/953065564.py:4: RuntimeWarning: overflow encountered in exp
+
@@ -3750,7 +3750,7 @@ Lambda = 0.01
Accuracy score on test set: 0.08888888888888889
/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_57345/953065564.py:4: RuntimeWarning: overflow encountered in exp
return 1/(1 + np.exp(-x))
-/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_55841/953065564.py:4: RuntimeWarning: overflow encountered in exp
+
@@ -3759,7 +3759,7 @@ Lambda = 0.1
Accuracy score on test set: 0.08611111111111111
/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_57345/953065564.py:4: RuntimeWarning: overflow encountered in exp
return 1/(1 + np.exp(-x))
-/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_55841/953065564.py:4: RuntimeWarning: overflow encountered in exp
+
@@ -3768,7 +3768,7 @@ Lambda = 1.0
Accuracy score on test set: 0.08888888888888889
/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_57345/953065564.py:4: RuntimeWarning: overflow encountered in exp
return 1/(1 + np.exp(-x))
-/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_55841/953065564.py:4: RuntimeWarning: overflow encountered in exp
+
@@ -3777,11 +3777,11 @@ Lambda = 10.0
Accuracy score on test set: 0.09166666666666666
/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_57345/953065564.py:4: RuntimeWarning: overflow encountered in exp
return 1/(1 + np.exp(-x))
-/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_55841/953065564.py:4: RuntimeWarning: overflow encountered in exp
+
@@ -3790,11 +3790,11 @@ Lambda = 1e-05
Accuracy score on test set: 0.07777777777777778
/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_57345/953065564.py:4: RuntimeWarning: overflow encountered in exp
return 1/(1 + np.exp(-x))
-/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_55841/1630775253.py:43: RuntimeWarning: overflow encountered in exp
+/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_57345/1630775253.py:43: RuntimeWarning: overflow encountered in exp
exp_term = np.exp(self.z_o)
-/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_55841/1630775253.py:44: RuntimeWarning: invalid value encountered in true_divide
+/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_57345/1630775253.py:44: RuntimeWarning: invalid value encountered in true_divide
self.probabilities = exp_term / np.sum(exp_term, axis=1, keepdims=True)
-/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_55841/953065564.py:4: RuntimeWarning: overflow encountered in exp
+
@@ -3803,11 +3803,11 @@ Lambda = 0.0001
Accuracy score on test set: 0.07777777777777778
/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_57345/953065564.py:4: RuntimeWarning: overflow encountered in exp
return 1/(1 + np.exp(-x))
-/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_55841/1630775253.py:43: RuntimeWarning: overflow encountered in exp
+/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_57345/1630775253.py:43: RuntimeWarning: overflow encountered in exp
exp_term = np.exp(self.z_o)
-/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_55841/1630775253.py:44: RuntimeWarning: invalid value encountered in true_divide
+/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_57345/1630775253.py:44: RuntimeWarning: invalid value encountered in true_divide
self.probabilities = exp_term / np.sum(exp_term, axis=1, keepdims=True)
-/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_55841/953065564.py:4: RuntimeWarning: overflow encountered in exp
+
@@ -3816,11 +3816,11 @@ Lambda = 0.001
Accuracy score on test set: 0.07777777777777778
/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_57345/953065564.py:4: RuntimeWarning: overflow encountered in exp
return 1/(1 + np.exp(-x))
-/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_55841/1630775253.py:43: RuntimeWarning: overflow encountered in exp
+/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_57345/1630775253.py:43: RuntimeWarning: overflow encountered in exp
exp_term = np.exp(self.z_o)
-/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_55841/1630775253.py:44: RuntimeWarning: invalid value encountered in true_divide
+/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_57345/1630775253.py:44: RuntimeWarning: invalid value encountered in true_divide
self.probabilities = exp_term / np.sum(exp_term, axis=1, keepdims=True)
-/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_55841/953065564.py:4: RuntimeWarning: overflow encountered in exp
+
@@ -3829,11 +3829,11 @@ Lambda = 0.01
Accuracy score on test set: 0.07777777777777778
/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_57345/953065564.py:4: RuntimeWarning: overflow encountered in exp
return 1/(1 + np.exp(-x))
-/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_55841/1630775253.py:43: RuntimeWarning: overflow encountered in exp
+/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_57345/1630775253.py:43: RuntimeWarning: overflow encountered in exp
exp_term = np.exp(self.z_o)
-/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_55841/1630775253.py:44: RuntimeWarning: invalid value encountered in true_divide
+/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_57345/1630775253.py:44: RuntimeWarning: invalid value encountered in true_divide
self.probabilities = exp_term / np.sum(exp_term, axis=1, keepdims=True)
-/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_55841/953065564.py:4: RuntimeWarning: overflow encountered in exp
+
@@ -3842,7 +3842,7 @@ Lambda = 0.1
Accuracy score on test set: 0.07777777777777778
/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_57345/953065564.py:4: RuntimeWarning: overflow encountered in exp
return 1/(1 + np.exp(-x))
-/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_55841/1630775253.py:43: RuntimeWarning: overflow encountered in exp
+/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_57345/1630775253.py:43: RuntimeWarning: overflow encountered in exp
exp_term = np.exp(self.z_o)
-/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_55841/1630775253.py:44: RuntimeWarning: invalid value encountered in true_divide
+/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_57345/1630775253.py:44: RuntimeWarning: invalid value encountered in true_divide
self.probabilities = exp_term / np.sum(exp_term, axis=1, keepdims=True)
-/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_55841/953065564.py:4: RuntimeWarning: overflow encountered in exp
+
@@ -3851,11 +3851,11 @@ Lambda = 1.0
Accuracy score on test set: 0.10555555555555556
/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_57345/953065564.py:4: RuntimeWarning: overflow encountered in exp
return 1/(1 + np.exp(-x))
-/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_55841/953065564.py:4: RuntimeWarning: overflow encountered in exp
+
@@ -3864,11 +3864,11 @@ Lambda = 10.0
Accuracy score on test set: 0.07777777777777778
/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_57345/953065564.py:4: RuntimeWarning: overflow encountered in exp
return 1/(1 + np.exp(-x))
-/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_55841/1630775253.py:43: RuntimeWarning: overflow encountered in exp
+/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_57345/1630775253.py:43: RuntimeWarning: overflow encountered in exp
exp_term = np.exp(self.z_o)
-/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_55841/1630775253.py:44: RuntimeWarning: invalid value encountered in true_divide
+/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_57345/1630775253.py:44: RuntimeWarning: invalid value encountered in true_divide
self.probabilities = exp_term / np.sum(exp_term, axis=1, keepdims=True)
-/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_55841/953065564.py:4: RuntimeWarning: overflow encountered in exp
+
@@ -3877,11 +3877,11 @@ Lambda = 1e-05
Accuracy score on test set: 0.07777777777777778
/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_57345/953065564.py:4: RuntimeWarning: overflow encountered in exp
return 1/(1 + np.exp(-x))
-/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_55841/1630775253.py:43: RuntimeWarning: overflow encountered in exp
+/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_57345/1630775253.py:43: RuntimeWarning: overflow encountered in exp
exp_term = np.exp(self.z_o)
-/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_55841/1630775253.py:44: RuntimeWarning: invalid value encountered in true_divide
+/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_57345/1630775253.py:44: RuntimeWarning: invalid value encountered in true_divide
self.probabilities = exp_term / np.sum(exp_term, axis=1, keepdims=True)
-/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_55841/953065564.py:4: RuntimeWarning: overflow encountered in exp
+
@@ -3890,11 +3890,11 @@ Lambda = 0.0001
Accuracy score on test set: 0.07777777777777778
/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_57345/953065564.py:4: RuntimeWarning: overflow encountered in exp
return 1/(1 + np.exp(-x))
-/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_55841/1630775253.py:43: RuntimeWarning: overflow encountered in exp
+/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_57345/1630775253.py:43: RuntimeWarning: overflow encountered in exp
exp_term = np.exp(self.z_o)
-/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_55841/1630775253.py:44: RuntimeWarning: invalid value encountered in true_divide
+/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_57345/1630775253.py:44: RuntimeWarning: invalid value encountered in true_divide
self.probabilities = exp_term / np.sum(exp_term, axis=1, keepdims=True)
-/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_55841/953065564.py:4: RuntimeWarning: overflow encountered in exp
+
@@ -3903,11 +3903,11 @@ Lambda = 0.001
Accuracy score on test set: 0.07777777777777778
/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_57345/953065564.py:4: RuntimeWarning: overflow encountered in exp
return 1/(1 + np.exp(-x))
-/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_55841/1630775253.py:43: RuntimeWarning: overflow encountered in exp
+/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_57345/1630775253.py:43: RuntimeWarning: overflow encountered in exp
exp_term = np.exp(self.z_o)
-/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_55841/1630775253.py:44: RuntimeWarning: invalid value encountered in true_divide
+/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_57345/1630775253.py:44: RuntimeWarning: invalid value encountered in true_divide
self.probabilities = exp_term / np.sum(exp_term, axis=1, keepdims=True)
-/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_55841/953065564.py:4: RuntimeWarning: overflow encountered in exp
+
@@ -3916,11 +3916,11 @@ Lambda = 0.01
Accuracy score on test set: 0.07777777777777778
/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_57345/953065564.py:4: RuntimeWarning: overflow encountered in exp
return 1/(1 + np.exp(-x))
-/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_55841/1630775253.py:43: RuntimeWarning: overflow encountered in exp
+/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_57345/1630775253.py:43: RuntimeWarning: overflow encountered in exp
exp_term = np.exp(self.z_o)
-/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_55841/1630775253.py:44: RuntimeWarning: invalid value encountered in true_divide
+/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_57345/1630775253.py:44: RuntimeWarning: invalid value encountered in true_divide
self.probabilities = exp_term / np.sum(exp_term, axis=1, keepdims=True)
-/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_55841/953065564.py:4: RuntimeWarning: overflow encountered in exp
+
@@ -3929,11 +3929,11 @@ Lambda = 0.1
Accuracy score on test set: 0.07777777777777778
/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_57345/953065564.py:4: RuntimeWarning: overflow encountered in exp
return 1/(1 + np.exp(-x))
-/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_55841/1630775253.py:43: RuntimeWarning: overflow encountered in exp
+/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_57345/1630775253.py:43: RuntimeWarning: overflow encountered in exp
exp_term = np.exp(self.z_o)
-/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_55841/1630775253.py:44: RuntimeWarning: invalid value encountered in true_divide
+/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_57345/1630775253.py:44: RuntimeWarning: invalid value encountered in true_divide
self.probabilities = exp_term / np.sum(exp_term, axis=1, keepdims=True)
-/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_55841/953065564.py:4: RuntimeWarning: overflow encountered in exp
+
@@ -3942,11 +3942,11 @@ Lambda = 1.0
Accuracy score on test set: 0.07777777777777778
/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_57345/953065564.py:4: RuntimeWarning: overflow encountered in exp
return 1/(1 + np.exp(-x))
-/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_55841/1630775253.py:43: RuntimeWarning: overflow encountered in exp
+/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_57345/1630775253.py:43: RuntimeWarning: overflow encountered in exp
exp_term = np.exp(self.z_o)
-/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_55841/1630775253.py:44: RuntimeWarning: invalid value encountered in true_divide
+/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_57345/1630775253.py:44: RuntimeWarning: invalid value encountered in true_divide
self.probabilities = exp_term / np.sum(exp_term, axis=1, keepdims=True)
/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_55841/953065564.py:4: RuntimeWarning: overflow encountered in exp
+
@@ -3999,15 +3999,15 @@ Accuracy score on test set: 0.07777777777777778
/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_57345/953065564.py:4: RuntimeWarning: overflow encountered in exp
return 1/(1 + np.exp(-x))
-/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_55841/1630775253.py:43: RuntimeWarning: overflow encountered in exp
+/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_57345/1630775253.py:43: RuntimeWarning: overflow encountered in exp
exp_term = np.exp(self.z_o)
-/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_55841/1630775253.py:44: RuntimeWarning: invalid value encountered in true_divide
+/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_57345/1630775253.py:44: RuntimeWarning: invalid value encountered in true_divide
self.probabilities = exp_term / np.sum(exp_term, axis=1, keepdims=True)
-
-
-/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_55841/953065564.py:4: RuntimeWarning: overflow encountered in exp
+
@@ -4324,18 +4324,17 @@ Accuracy score on test set: 0.10555555555555556
Learning rate = 1.0
Lambda = 0.01
Accuracy score on test set: 0.17777777777777778
-/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_57345/953065564.py:4: RuntimeWarning: overflow encountered in exp
return 1/(1 + np.exp(-x))
-/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_55841/953065564.py:4: RuntimeWarning: overflow encountered in exp
+/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_57345/953065564.py:4: RuntimeWarning: overflow encountered in exp
return 1/(1 + np.exp(-x))
-/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_55841/953065564.py:4: RuntimeWarning: overflow encountered in exp
+/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_57345/953065564.py:4: RuntimeWarning: overflow encountered in exp
return 1/(1 + np.exp(-x))
-/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_55841/953065564.py:4: RuntimeWarning: overflow encountered in exp
+/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_57345/953065564.py:4: RuntimeWarning: overflow encountered in exp
return 1/(1 + np.exp(-x))
-/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_55841/953065564.py:4: RuntimeWarning: overflow encountered in exp
+/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_57345/953065564.py:4: RuntimeWarning: overflow encountered in exp
return 1/(1 + np.exp(-x))
Learning rate = 1.0
-Lambda = 0.1
-Accuracy score on test set: 0.08333333333333333
Learning rate = 1.0
-Lambda = 1.0
-Accuracy score on test set: 0.08888888888888889
+Lambda = 0.1
+Accuracy score on test set: 0.08333333333333333
-
-Learning rate = 1.0
+Lambda = 1.0
+Accuracy score on test set: 0.08888888888888889
+
+Learning rate = 1.0
Lambda = 10.0
Accuracy score on test set: 0.09444444444444444
@@ -4355,9 +4354,8 @@ Accuracy score on test set: 0.10555555555555556
Learning rate = 10.0
Lambda = 0.01
Accuracy score on test set: 0.1388888888888889
-
diff --git a/doc/LectureNotes/_build/jupyter_execute/chapter1.ipynb b/doc/LectureNotes/_build/jupyter_execute/chapter1.ipynb
index 7759019af..d4be4ab07 100644
--- a/doc/LectureNotes/_build/jupyter_execute/chapter1.ipynb
+++ b/doc/LectureNotes/_build/jupyter_execute/chapter1.ipynb
@@ -343,7 +343,7 @@
"outputs": [
{
"data": {
- "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjYAAAHFCAYAAADhWLMfAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy81sbWrAAAACXBIWXMAAA9hAAAPYQGoP6dpAABH1UlEQVR4nO3dd3xUdb7/8fcQQqgJvSYQsNFBQSAgTYr3CoqwXL1iQVd/NlzBchV0V8RVo7gqrhfEXV3ZXaU81FiuroVuLCDVRbEbJCCdJUGUQJLz++PsyWSSmWQmmZlT5vV8PPKAnJkk3+TMzPnM9/v5fj4+wzAMAQAAeEAduwcAAAAQLQQ2AADAMwhsAACAZxDYAAAAzyCwAQAAnkFgAwAAPIPABgAAeAaBDQAA8AwCGwAA4BkENoCk9evXa+LEierYsaNSUlLUpk0bZWVl6fbbbw+434gRIzRixAhbxrhjxw75fD4tWrQoat8zMzNT48ePr/I+V111lTIzM6P2M+NtxIgR8vl8ZR/169dX9+7d9cADD+jEiRN2Dy9uMjMzddVVV9k9DCDm6to9AMBub731li688EKNGDFCc+fOVbt27bRnzx5t3LhRS5cu1WOPPVZ23wULFtg4Unv87ne/0/Tp0+0eRq106dJFL774oiTpwIEDevbZZ/W73/1OO3fu1J/+9CebRxcfr776qlJTU+0eBhBzPnpFIdENHz5cu3fv1pdffqm6dQNj/dLSUtWp44yJzR07dqhz5856/vnno/bOOzMzUz179tSbb74Zle9nB8MwdPz4cTVo0CDo7SNGjNDBgwf12WeflR0rLi5W9+7d9cMPP6igoED169eP13B18uRJ+Xy+So81ANHhjFdswEaHDh1Sy5Ytg15oKgY1FZeirOWhRx99VI888ogyMzPVoEEDjRgxQl9//bVOnjypmTNnqn379kpLS9PEiRO1f//+gO9pLQe9+uqr6t27t+rXr68uXbroj3/8Y1jj/+abbzRlyhS1bt1aKSkp6tatm+bPnx/5HyKEYEtRPp9PN998s/7+97+rW7duatiwofr06RM0QApnfMePH9ftt9+uvn37Ki0tTc2bN1dWVpZef/31St/P+tkLFy5Ut27dlJKSor/+9a8R/U5169ZV3759deLECR05cqTsuGEYWrBggfr27asGDRqoWbNmmjx5sr7//vuArzcMQw899JA6deqk+vXrq3///lq+fHmlx8eaNWvk8/n097//Xbfffrs6dOiglJQUffvtt5KkFStWaNSoUUpNTVXDhg01ZMgQrVy5MuBnHThwQNddd50yMjKUkpKiVq1aaciQIVqxYkXZfbZs2aLx48eX/Y3bt2+vcePGadeuXWX3CbYUtXPnTl1++eUB5+axxx5TaWlp2X2sx/gf/vAHPf744+rcubMaN26srKwsrVu3LqK/OxAPvGVAwsvKytKzzz6rW265RZdddpnOOussJScnR/Q95s+fr969e2v+/Pk6cuSIbr/9dl1wwQUaOHCgkpOT9Ze//EU//PCD7rjjDl177bV64403Ar5+69atmjFjhu677z61bdtWL774oqZPn64TJ07ojjvuCPlzt2/frsGDB6tjx4567LHH1LZtW7377ru65ZZbdPDgQc2ePbtGf5NwvPXWW9qwYYPuv/9+NW7cWHPnztXEiRP11VdfqUuXLhGNr6ioSIcPH9Ydd9yhDh066MSJE1qxYoUmTZqk559/XldeeWXAz37ttdeUm5ure++9V23btlXr1q0jHn9eXp6aNm2qVq1alR27/vrrtWjRIt1yyy165JFHdPjwYd1///0aPHiwPv30U7Vp00aSdM899yg7O1vXXXedJk2apPz8fF177bU6efKkTj/99Eo/a9asWcrKytLChQtVp04dtW7dWi+88IKuvPJKTZgwQX/961+VnJysZ555Ruedd57effddjRo1SpJ0xRVXaPPmzXrwwQd1+umn68iRI9q8ebMOHTokSTp27JjGjBmjzp07a/78+WrTpo327t2r1atX6+jRoyF//wMHDmjw4ME6ceKEfv/73yszM1Nvvvmm7rjjDn333XeVll3nz5+vrl27at68eZLMJcrzzz9feXl5SktLi/jvD8SMASS4gwcPGuecc44hyZBkJCcnG4MHDzays7ONo0ePBtx3+PDhxvDhw8s+z8vLMyQZffr0MUpKSsqOz5s3z5BkXHjhhQFfP2PGDEOSUVBQUHasU6dOhs/nM7Zu3Rpw3zFjxhipqanGsWPHAn7W888/X3af8847z0hPTw/4foZhGDfffLNRv3594/Dhw1X+7p06dTLGjRtX5X2mTp1qdOrUKeCYJKNNmzZGYWFh2bG9e/caderUMbKzs2s9vuLiYuPkyZPGNddcY5x55pmVfnZaWlq1v5tl+PDhRo8ePYyTJ08aJ0+eNPbs2WPce++9hiRj4cKFZff7+OOPDUnGY489FvD1+fn5RoMGDYw777zTMAzDOHz4sJGSkmJccsklAfezvr7842P16tWGJGPYsGEB9z127JjRvHlz44ILLgg4XlJSYvTp08cYMGBA2bHGjRsbM2bMCPn7bdy40ZBkvPbaa1X+HTp16mRMnTq17POZM2cakoz169cH3O/GG280fD6f8dVXXxmG4X/c9erVyyguLi673yeffGJIMpYsWVLlzwXijaUoJLwWLVooNzdXGzZs0MMPP6wJEybo66+/1qxZs9SrVy8dPHiw2u9x/vnnByxbdevWTZI0bty4gPtZx3fu3BlwvEePHurTp0/AsSlTpqiwsFCbN28O+jOPHz+ulStXauLEiWrYsKGKi4vLPs4//3wdP348pksFI0eOVJMmTco+b9OmjVq3bq0ffvihRuN76aWXNGTIEDVu3Fh169ZVcnKynnvuOX3xxReVfva5556rZs2ahT3Wzz//XMnJyUpOTla7du10//33a9asWbr++uvL7vPmm2/K5/Pp8ssvDxhr27Zt1adPH61Zs0aStG7dOhUVFeniiy8O+BmDBg0KuXvsV7/6VcDnH330kQ4fPqypU6cG/KzS0lL9x3/8hzZs2KBjx45JkgYMGKBFixbpgQce0Lp163Ty5MmA73XqqaeqWbNmuuuuu7Rw4UJt3749rL/JqlWr1L17dw0YMCDg+FVXXSXDMLRq1aqA4+PGjVNSUlLZ571795aksvMNOAWBDfBv/fv311133aWXXnpJP/74o2699Vbt2LFDc+fOrfZrmzdvHvB5vXr1qjx+/PjxgONt27at9D2tY9aSQ0WHDh1ScXGxnnrqqbKLtvVx/vnnS1JYQVlNtWjRotKxlJQU/fLLLxGPLycnRxdffLE6dOigF154QR9//LE2bNigX//615X+VpLUrl27iMZ6yimnaMOGDfrkk0/00ksvqU+fPsrOztbSpUvL7rNv3z4ZhqE2bdpUGu+6devKxmqdD2tZqrxgx4KNd9++fZKkyZMnV/pZjzzyiAzD0OHDhyVJy5Yt09SpU/Xss88qKytLzZs315VXXqm9e/dKktLS0rR27Vr17dtXd999t3r06KH27dtr9uzZlYKg8g4dOhT079i+ffuA39NS8XynpKRIUtn5BpyCHBsgiOTkZM2ePVtPPPFEwG6aWLEuUsGOBQsgJKlZs2ZKSkrSFVdcoWnTpgW9T+fOnaM3yAhFMr4XXnhBnTt31rJly+Tz+cpuLyoqCvp15e8TDivBV5LOPvtsjRw5Uj169NCMGTM0fvx4NW7cWC1btpTP51Nubm7ZRbs865h1PqzgpLy9e/cGnbWpON6WLVtKkp566ikNGjQo6JitIKlly5aaN2+e5s2bp507d+qNN97QzJkztX//fr3zzjuSpF69emnp0qUyDEP//Oc/tWjRIt1///1q0KCBZs6cGfT7t2jRQnv27Kl0/McffwwYI+A2BDZIeHv27An6ztVaArHewcbS559/rk8//TRgOWrx4sVq0qSJzjrrrKBf07BhQ40cOVJbtmxR7969y2aDnCKS8fl8PtWrVy8gANi7d2/QXVHR0KJFCz388MO6+uqr9dRTT2nWrFkaP368Hn74Ye3evbvSMlN5AwcOVEpKipYtW6ZJkyaVHV+3bp1++OGHsIoZDhkyRE2bNtX27dt18803hz3ujh076uabb9bKlSv14YcfVrrd5/OpT58+euKJJ7Ro0aKQy5iSNGrUKGVnZ2vz5s0Bj7G//e1v8vl8GjlyZNjjApyEwAYJ77zzzlN6erouuOACde3aVaWlpdq6dasee+wxNW7cOC7F6dq3b68LL7xQ9913n9q1a6cXXnhBy5cv1yOPPKKGDRuG/Lonn3xS55xzjoYOHaobb7xRmZmZOnr0qL799lv93//9X6U8iWD27t2rl19+udLxzMzMslmOmgp3fOPHj1dOTo5uuukmTZ48Wfn5+fr973+vdu3a6ZtvvqnVGEK58sor9fjjj+sPf/iDpk2bpiFDhui6667T1VdfrY0bN2rYsGFq1KiR9uzZow8++EC9evXSjTfeqObNm+u2225Tdna2mjVrpokTJ2rXrl2aM2eO2rVrF1bdo8aNG+upp57S1KlTdfjwYU2ePFmtW7fWgQMH9Omnn+rAgQN6+umnVVBQoJEjR2rKlCnq2rWrmjRpog0bNuidd94pC6refPNNLViwQBdddJG6dOkiwzCUk5OjI0eOaMyYMSHHcOutt+pvf/ubxo0bp/vvv1+dOnXSW2+9pQULFujGG28MursLcAMCGyS83/72t3r99df1xBNPaM+ePSoqKlK7du00evRozZo1qyzhN5b69u2rq6++WrNnz9Y333yj9u3b6/HHH9ett95a5dd1795dmzdv1u9//3v99re/1f79+9W0aVOddtppZXks1dm0aZP+67/+q9LxqVOn1rp9Q7jju/rqq7V//34tXLhQf/nLX9SlSxfNnDmzLGCIhTp16ujhhx/WuHHjNG/ePN1777165plnNGjQID3zzDNasGCBSktL1b59ew0ZMiQgyfbBBx9Uo0aNtHDhQj3//PPq2rWrnn76ad1zzz1q2rRpWD//8ssvV8eOHTV37lxdf/31Onr0qFq3bq2+ffuW1ZupX7++Bg4cqL///e/asWOHTp48qY4dO+quu+7SnXfeKUk67bTT1LRpU82dO1c//vij6tWrpzPOOEOLFi3S1KlTQ/78Vq1a6aOPPtKsWbM0a9YsFRYWqkuXLpo7d65uu+22Gv9dAbtReRiwmReq/8Ksi9O1a1fNnj1bd999t93DARIWMzYAEKFPP/1US5Ys0eDBg5WamqqvvvpKc+fOVWpqqq655hq7hwckNAIbAIhQo0aNtHHjRj333HM6cuSI0tLSNGLECD344IMht3wDiA+WogAAgGfYXqDvvvvuk8/nC/gIVqwMAACgOo5YiurRo0dAp9ryZbsBAADC5YjApm7duszSAACAWnNEYGPV7UhJSdHAgQP10EMPqUuXLkHvW1RUFFBmvbS0VIcPH1aLFi0iLrMOAADsYRiGjh49qvbt24dV2DJcticPv/322/r55591+umna9++fXrggQf05Zdf6vPPPw/aI+e+++6LWcEuAAAQX/n5+UpPT4/a97M9sKno2LFjOuWUU3TnnXcGrX5ZccamoKBAHTt2VH5+vlJTU+M5VAAAUEOFhYXKyMgoK5kQLY5YiiqvUaNG6tWrV8j+MCkpKUE776amphLYAADgMtFOI7F9u3dFRUVF+uKLL4J2WwYAAKiK7YHNHXfcobVr1yovL0/r16/X5MmTVVhYWGXzNgAAgGBsX4ratWuXLr30Uh08eFCtWrXSoEGDtG7dOnXq1MnuoQEAAJexPbBZunSp3UMAAAAeYftSFAAAQLQQ2AAAAM8gsAEAAJ5BYAMAADyDwAYAAHgGgQ0AAPAMAhsAAOAZBDYAAMAzCGwAAIBnENgAAADPILABAACeQWADAAA8g8AGAAB4BoENAADwDAIbAADgGQQ2AADAMwhsAACAZxDYAAAAzyCwAQAAnkFgAwAAPIPABgAAeAaBDQAA8AwCGwAA4BkENgAAwDMIbAAAgGcQ2AAAAM8gsAEAAJ5BYAMAADyDwAYAAHgGgQ0AAPAMAhsAAOAZBDYAAMAzCGwAAIBnENgAAADPILABAACeQWADAAA8g8AGAAB4BoENAADwDAIbAADgGQQ2AADAMwhsAACAZxDYAAAAzyCwAQAAnkFgAwAAPIPABgAAeAaBDQAA8AwCGwAA4BkENgAAwDMIbAAAgGcQ2AAAAM8gsAEAAJ5BYAMAADyDwAYAAHgGgQ0AAPAMAhsAAOAZBDYAAMAzCGwAAIBnENgAAADPILABAACeQWADAAA8g8AGAAB4BoENAADwDAIbAADgGY4KbLKzs+Xz+TRjxgy7hwIAAFzIMYHNhg0b9Kc//Um9e/e2eygAAMClHBHY/PTTT7rsssv05z//Wc2aNbN7OAAAwKUcEdhMmzZN48aN0+jRo6u9b1FRkQoLCwM+AAAAJKmu3QNYunSpNm/erA0bNoR1/+zsbM2ZMyfGowIAAG5k64xNfn6+pk+frhdeeEH169cP62tmzZqlgoKCso/8/PwYjxIAALiFzzAMw64f/tprr2nixIlKSkoqO1ZSUiKfz6c6deqoqKgo4LZgCgsLlZaWpoKCAqWmpsZ6yAAAIApidf22dSlq1KhR2rZtW8Cxq6++Wl27dtVdd91VbVADAABQnq2BTZMmTdSzZ8+AY40aNVKLFi0qHQcAAKiOI3ZFAQAARIPtu6IqWrNmjd1DAAAALsWMDQAA8AwCGwAA4BkENgAAwDMIbAAAgGcQ2AAAAM8gsAEAAJ5BYAMAADyDwAYAAHgGgQ0AAPAMAhsAAOAZBDYAAMAzCGwAAIBnENgAAADPILABAACeQWADAAA8g8AGAAB4BoENAADwDAIbAADgGQQ2AADAMwhsAACAZxDYAAAAzyCwAQAAnkFgAwAAPIPABgAAeAaBDQAA8AwCGwAA4BkENgAAwDMIbAAAgGcQ2AAAAM+oa/cAAACAjUpKpNxcac8eqV07aehQKSnJ7lHVGIENAACJKidHmj5d2rXLfyw9XXrySWnSJPvGVQssRQEAkIhycqTJkwODGknavds8npNjz7hqicAGAIBEU1JiztQYRuXbrGMzZpj3cxkCGwAAEk1ubuWZmvIMQ8rPN+/nMgQ2AAAkmj17ons/ByGwAQAg0bRrF937OQiBDQAAiWboUHP3k88X/HafT8rIMO/nMgQ2AAAkmqQkc0u3VDm4sT6fN8+V9WwIbAAASESTJkkvvyx16BB4PD3dPO7SOjYU6AMAIFFNmiRNmEDlYQAA4BFJSdKIEXaPImpYigIAAJ5BYAMAADyDwAYAAHgGgQ0AAPAMAhsAAOAZBDYAAMAzCGwAAIBnENgAAADPILABAACeQWADAAA8g5YKALytpMRTfXAAVI3ABoB35eRI06dLu3b5j6WnS08+6drOxQCqxlIUAG/KyZEmTw4MaiRp927zeE6OPeMCEFMENgC8p6TEnKkxjMq3WcdmzDDvB8BTCGwAeE9ubuWZmvIMQ8rPN+8HwFMIbAB4z5490b0fANcgsAHgPe3aRfd+AFyDwAaA9wwdau5+8vmC3+7zSRkZ5v0AeAqBDQDvSUoyt3RLlYMb6/N586hnA3gQgQ0Ab5o0SXr5ZalDh8Dj6enmcerYAJ5EgT4A3jVpkjRhApWHgQRCYAPA25KSpBEj7B4FgDixfSnq6aefVu/evZWamqrU1FRlZWXp7bfftntYAADAhWyfsUlPT9fDDz+sU089VZL017/+VRMmTNCWLVvUo0cPm0cHAEAU0ZQ15nyGEazmuL2aN2+uRx99VNdcc0219y0sLFRaWpoKCgqUmpoah9EBAFADNGUNEKvrt+1LUeWVlJRo6dKlOnbsmLKysoLep6ioSIWFhQEfAAA4Gk1Z48YRgc22bdvUuHFjpaSk6IYbbtCrr76q7t27B71vdna20tLSyj4yMjLiPFoAiEBJibRmjbRkifkvjTcTD01Z48oRS1EnTpzQzp07deTIEb3yyit69tlntXbt2qDBTVFRkYqKiso+LywsVEZGhj1LUayVAs7klOcmSw+QzIB25Mjq77d6dULt4IvVUpTtycOSVK9evbLk4f79+2vDhg168skn9cwzz1S6b0pKilJSUuI9xMp4wQKcySnPTWvpoeJ7R2vpgSKBiYOmrHHliKWoigzDCJiVcRzWSgFncspzk6UHlEdT1riyPbC5++67lZubqx07dmjbtm265557tGbNGl122WV2Dy04XrAAZ3LSczM3t3JwVXE8+fnm/eB9TmrKmgA5X7YHNvv27dMVV1yhM844Q6NGjdL69ev1zjvvaMyYMXYPLThesABnctJzk6UHlOeUpqw5OVJmppnvM2WK+W9mpudWGWzPsXnuuefsHkJkeMECnMlJz02WHlCR1ZQ1WP7XvHnh51vVNDE+gXK+bA9sXIcXLMCZnPTctJYedu8OvjTm85m3x2PpAc5R26asNU2Mr26Z1uczl2knTPDEzl5HbPeujbhXHi4pMafuqnvBysvzxAMEcA2nPTetd8hS4HispQcPvUNGHISacQnn8eTQ7eYJUXnYFZyyVgogkNOem9bSQ4cOgcfT0wlqEJnaJsY7aZk2DghsaoIXLMCZnPbcnDRJ2rHDfCe8eLH5b14erxGITG0T4520TBsH5NjUVG3XSgHEhtOem0lJCVVNFjFQ2xmXBMv5IrCpDV6wAGfiuQkvqe2Mi7VMO3myGcQEy/nyUAoFS1EAAHfzetG5aBT4c9oybQwxYwMAcC+n9AaLpWjNuDhtmTZG2O4NAHCn2myBdqNgQVxGRmQF/hwkVtdvAhsAgPtYdYtC7Rbyak2xmlYedqBYXb9ZigIAuE8kW6C9lEjulsR4GwMwAhsAgPskWNE5V7E574ldUQAA90mwonOuYeU9VZxNs5ptxqGTOIENAMB9orEFGtFV29YPUUJgAwCx5PUaK3ZxWm8w1L71Q5QQ2ABwFzcFCjk55s6dkSOlKVPMfzMz4zIdnxASqOicKzgk74nkYQDu4aZibKFqrFi5Blx4oyNBis65gkPynqhjA8Ad3FSMLVFrrCCxWY/76ppt/vtxH6vrN0tRAJzPIUmJYXNIrgEQVw7JeyKwAeB8bgsUHJJrAMSdA/KeyLEB4HxuCxQckmsA2MLmvCcCGwDO57ZAwaqxUl2ugZNqrHioBxEcwMbWDyxFAXA+txVjc0iuQdjYll41N5UYAIENABdwW6AgOSLXICwOKIHvaAR9oTk04GO7NwD3CFbHJiPDDGqcEihU5OQlHralV81NJQbiLQo1pWJ1/SawAeAuTg4U3GbNGnMGojqrV9uWL2Ebgr7QohTwOaKOTX5+ftR+MADUiJWUeOml5r+JdlGJJrftNosnt5UYiBcX1JSKKLDp2rWrfve73+nYsWOxGg8QXw5dIwbiwm27zaKtquc/QV9wLgj4Igpsli9frvfee0+nnXaann/++ViNCYgPkgKR6Ny22yyaqnv+J3rQF4oLAr6IApvBgwdr/fr1evjhh3XvvffqzDPP1Jo1a2I0NCCG2AkCuHO3WTSE8/xP5KCvKi4I+Gq03fvKK6/U119/rQsuuEDjxo3TxIkT9e2330Z7bEBsuGCNGIgbt2xLj5Zwn/9SYgZ91XFBwFfjOjaGYWjs2LG67rrr9MYbb6hnz566/fbbdfTo0WiOD4g+F6wRA3E1aZK0Y4e5+2nxYvPfvDzvBTVSZM//RAv6wuGCWb6IWiosXLhQGzZs0IYNG/TFF18oKSlJvXv31rRp09S3b1+9+OKL6t69u1599VX1798/VmMGascFa8RA3NlYAj+uIn3+29z3yJGsgC9YHRsH1JSKKLB58MEHNWjQIE2dOlWDBg1S//79lZKSUnb7r3/9az300EO66qqr9Nlnn0V9sEBUuGCNGECM1OT5nyhBXyQcHPBFvUDfvn371L59e5XEKT+BAn2ImFV4q7oGhYlYeAvwOp7/juGIAn3haN26tVatWhXtbwtEjwvWiAHECM9/z4t6YOPz+TR8+PBof1sgukgKBBIXz39Po1cUEht9h4DExfPfVrG6fkeUPAx4DkmBQOLi+e9JUV+KAgAAsAuBDQAA8AyWogAAQPTYnLtEYAMAAKIjJyd4ReInn4zbbjOWogAAQO2F0zU9DghsAABA7YTbNT0OXQkIbAAAQO1E0jU9xghsAABA7UTaNT2GCGwAAEDt1KRreowQ2AAAgNoZOtTc/VSxsajF55MyMsz7xRiBDQB3KimR1qyRliwx/41DUiKAEBzUNZ3ABoD75ORImZnSyJHSlCnmv5mZcdtOCiAIh3RNp7s3AHexamVUfOmy3hXG8QUUQBBhVh6O1fWbwAaAe5SUmDMzobaV+nzmu8O8vLiWcAcQuVhdv1mKAuAeDqqVAeDfHJbvRq8oAO7hoFoZAOSI3lAVMWMDwD0cVCsDSHgO6Q1VEYENAPdwUK0MIKE5qDdURQQ2ANzDQbUygITm4Hw3AhsA7uKQWhlAQnNwvhvJwwDcZ9IkacKEsGplAIgBB+e7EdgAcKekJGnECLtHASQmK99t9+7geTZWTSkb8t1YigIAAJFxcL4bgQ0AAIicQ/PdbA9ssrOzdfbZZ6tJkyZq3bq1LrroIn311Vd2DwsAAFRn0iRpxw5p9Wpp8WLz37w8W5P4bc+xWbt2raZNm6azzz5bxcXFuueeezR27Fht375djRo1snt4AACgKg7Ld3NcE8wDBw6odevWWrt2rYYNG1bt/WmCCQCA+8Tq+m37jE1FBQUFkqTmzZsHvb2oqEhFRUVlnxcWFsZlXADgKSUlbJeHJ9meY1OeYRi67bbbdM4556hnz55B75Odna20tLSyj4yMjDiPEgBcLidHysyURo6Upkwx/83MtK23DxBNjlqKmjZtmt566y198MEHSk9PD3qfYDM2GRkZLEUBQDisxoUVX/qtLbpUb0acxGopyjGBzW9+8xu99tprev/999W5c+ewv44cGwAIU0mJOTMTqsePVVQtL49lKcRcrK7fti9FGYahm2++WTk5OVq1alVEQQ2QUEpKpDVrpCVLzH9t6JoLl3Nw40IgWmxPHp42bZoWL16s119/XU2aNNHevXslSWlpaWrQoIHNowMcIidHmj498KKUnm5W/mTZAOFycONCIFpsn7F5+umnVVBQoBEjRqhdu3ZlH8uWLbN7aIAzWDkRFd9p795tHifhE+FycONCRFkCz/A6JsempsixgaeRE4Fosh5P1TUu5PHkbNVt1XfJDK9nc2wAVIGcCESTgxsXIkzVbdV3ywxvYaH09tsx+dYENoCTkROBaHNo40KEobqg5aWXzJmaYLNx1rEZM+xZlioqktaule69Vxo8WGreXPrv/47Jj7I9eRhAFcLNddi+3VxHp3psbHmlWu+kSdKECd74XRJFSUnVQYvPJ02bJh04EPp7lJ/hjXVvp9JS6Z//lFaskFaulN5/X/r558D7dOkiff991H80OTaAk1WXE1GRA9fRPcMleQvwqDVrzGWnaFi8WLr00uh8r/Ly8sxAZsUKadUq6eDBwNtbt5ZGj5ZGjZJGjVJhs2aJ0SsKQDlWTsTkyeY7suqCG2tKmiWF6ApVrZe/N+IlmsvN0dr1duCAGcCsXGkGM3l5gbc3biwNH+4PZnr2DMztilGvR2ZsADcINlsQCjtbooudaXCCcGdsWrUyZ0pisevt2DFzGctaXtq6NfD2unWlrCwziBk9WhowQEpODvntEqa7N4AgyudErFwpPfBA6PvGcx09EUSyM42/N2Jl6FAzKKluq/7jj0sXX1x5hrcmu96Ki6UNG/zLSx9/LJ08GXif3r39MzLDhpmzNDYjsAHcIinJvHAm4k4pO5N2E/HvDeepalm6fNBi7XoLlg9m3R6KYZgbEaylpTVrpKNHA+/TqZMZyIweLZ17rpk34zAENoDbJFr1WLuTdhPt7w3nCjdoiWTXW36+GchYwcy/2xqVad68LNlXo0ebO5kq1kByGHJsALdJpOqxoZJ2rRfWeCTtJtLfG+5QmxnMf/3LnImxlpe+/jrw9gYNzO9nLS/17SvViU3Ju1hdvwlsADeyLvhS8ClpL+zScVLSbiL8veFNx49LH37on5HZtMmsMWOpU8dM8rVmZLKypJSUuAyN5GEAfrVZR3cLJyXtJsLfG95QUiJt2eLfufTBB2ZwU163bv4ZmeHDpaZNbRlqrBDYAG5jTUMXFUmLFpnH9u/3XvVYpyXtUq0XTmQY0jff+GdkVq82l5vKa98+MOG3YjuNWLEp6Z/ABnCTqhJpvbbV2IlJu9bONMBOe/cGJvzm5wfenpZm1ryxlpfOOCP+Cb82Jv2TYwO4hRMSaeOJpF3AdPSo2UDSSvj9/PPA2+vVk4YM8S8v9etnFsuzS5ivVSQPh0Bgg4TgpETaeCJpF4noxAlp3Tr/jMz69YEduX0+6cwz/ctLQ4ZIDRvaN97yInitKjx2jORhIGE5KZE2nkjaRSIoLZW2bfMn/K5dW7kT9qmn+peWRo6UWrSwZ6zVieS16qyzYjIEAhvADZyWSBtPJO3Ci/Ly/DMyK1cG74RtBTKjRpkVf93AAa9VBDaAG4SbILt9u1l8y2sXfpJ24XYHDwZ2wv7++8DbGzXyd8IePbpyJ2y3cEDSPzk2gBtUl0hbUTxbDljs7OcEOM2xY2YNGWtGZsuWwNvr1pUGDQrshF2vnj1jjaYIkv7JsQESWVUN8ILZvdu8b7ySa+3u5wTYzeqEbc3IfPRR5U7YvXr5Z2SGDpWaNLFnrLEUbrPOGL7pYcYGcJNgAUQo8doplWjb0AHJfLx/8YV/RmbNGqmwMPA+GRnmktIpp5gJvxMmJM4sZrDXqoyMgKR/tnuHQGCDhGMt+axcKT3wQPX3X706dvkpiboNHYlp167AhN+KCbDNm5uVfUeNMp8bDz+c2LOY1SxP0ysKgMlKpHXA7oOE3YaOxHDkSGAn7K++Cry9fn1/J+zRo/2dsEPNYsZ7idhuNiX9E9gAbuWA3QeOCK6AaDl+3MyNsWZkNm6s3An77LMDO2HXrx/4PUpKzCWYYIshhmHOYs6YkVjLUnFGYAO41dCh5tR2dbsPhg6N3RicEFwBNVVSIm3d6p+RCdYJu2tX/4xMOJ2wmcW0HYEN4FYO2H3giOAKCJdhSN9+65+RWbUqeCfs8oXxIu2EzSym7QhsADezu+WAE4IroCr79gUm/O7cGXh7ampgJ+yuXWtXGI9ZTNuxKwrwAruL44WxtROIi6NHpfff9y8vffZZ4O316kmDB/uXl6LdCZuu9GFju3cIBDaAQ9gdXCExnThhdr+2ZmTWrzeL5VmsTtjWjMw558S+E7aXutLH8HnNdm+v4MUfXkU/J8RDaak5C2PNyLz/vtm+oLxTTvHnyIwcKbVsGd8x2r1EHC0urSjOjE08ufRBAgC22rEjME/mwIHA21u1Ckz4zcy0Y5SVlX8j27q1eWz/fne8qY1DRXGWokJwTWBD2XkAXhOrGeiDB82K2VYg8913gbdbnbCtYKZnT7PGjFO57U1tuBXFv/3WrPtTw/NPYBOCKwIbys4D8JpoXqx//tnfCXvFCrO2TPlLU1KS2QnbmpEZONA9nbDd+KZ2zRpzCa86LVuaQaglwvNPYBOCKwKbcB8ksezpAwDRUtuLdXGxWdXXmpH56CMzCbi8nj39O5eGDXNmJ+zqZqzc+qZ2yRJpypTIvy7CYI3kYTejYBMAr6hJywDDkL780j8jE6oT9pgx5ozMuedKbdvG+jepnXBmrNxahbimNXaqaxlRMRDs06fWQw2GwCYeKNgEwCvCvVjn5Ei//OIPZiq+cWvWzAxgrOWlU0+tXWG8eAq3yaVb39RWV1G8KqGCtWCBYPv2URluRQQ28UDZeQBeEe5F+OKLAz+3OmFbCb99+0a+/OKEchmRzFi59U1tVRXFw1X+cRIqEPzxx9qNMwQCm3ig7DwArwj3IuzzmZ2wrRmZwYMrd8KOhFN2FkWyvOTmN7WhavG0alV5u30w1uOkqkAwRhy8P85jrAdJxYZq6enOzIoHgPJKSqRNm6R166SUlKrv27KlefFbv1568EFzyam2Qc3kyZUDCmvpJyen5t87UpEsL1lvaqXKy2xueFM7aZJZQ2j1amnxYvPfXbvM61aoZUOfz8yXsoK16gLBGGDGJp4mTTKnJ+2eSgWQGGqzdGMYZv2Y8p2wDx+u+musi90zz0gtWtRu7JaaJCvHUqTLS26vQhysongkKxA25A+x3RsAvKgmSzf79pkBjBXM/PBD4O1NmpilK0aPNlsb/OEPsW986rRyGTVtcnnihLRggRksnnKKdNNN7qnFE0y4jW+rOH+FktIk6thURGADABWEW2fG6oRttSvYti3w/snJ0pAh/oTf/v0DO2HHI5k33JoqixdLl14a3Z8dSqRNLp2SHxRt4Zz/KgJBApsQCGwAoJwTJ8xcvvIVYStKTZV69arcCVsyO2FbCb/nnGO2L7CT02ZsLOHOWLix8nC0hQgECWxCILCJEydsswRQtZwc6YYbwtu1YunSxV/h145O2NWp6dJPvMbmxcrDsRAkECzs0EFpu3cT2FREYBMHXp1GBbwk1MxAKNdeK919t9S5c2zHFQ2RLv04hVNnm+xSIRAs7NNHac2b01IBcRZuhU23YiYKbnfokJkfc801kdUKuewydwQ1Uvx3FkXrdcEJlYed9BpXcYdVxbYaUUJgg9Ccts0y2piJghtZnbCthN8tWyILaJxcFK4q8SqXEc3XBbsrDyfoaxxLUQjNy9OoJPTBLYqLzcJ41hbsDz+s3Ak7PT38Img+H4/vUKL9umBnfpALXuNidf0msEFoTtxmGQ0k9MHJrE7Y1ozMmjVSQUHgfTIy/DuXzj1X+uqr8N6EtGolLVxo+wUtqqK11BKr1wU78oOc/BpX7nwVpqYqbfx4cmwQR3ZPo8ZKJL1e3DYTBXfavdsMZKxgpmJzwGbN/IXxRo+u3Am7devquzG3amU+7t1cFK6iaC61xOp1wY7Kw059jQt2vmKAwAahubmBW1WckNCHxFZQYM7EWMtLX3wReHtKSmAn7DPPrPqddTiNdhcu9F5QE82NDbF8XYh3Ox0nvsZFumuvFghsEJpXu5J7dSYqGCftiEhkRUXSxx+bgcyKFdKGDWZLAkudOlK/foGdsBs0iOxnuL0nUSRisbEh1q8LwXouxYrTXuPi3OGbHBtUL9wKm27h5IJf0ZSgOyIcobRU2rrVv7SUmyv98kvgfc44wz8jM2KEudwUDYkQzMZiY4OXXhec9ruEOF+xqjzMjA2q57Wu5F6diSrP6fWHvHbxNQzp++/9MzKrV5v1Zcpr29Y/IzNqlPnmIBbiOTNgl1gstXjpdcFpv0u8l/UNlysoKDAkGQUFBXYPBW7zyiuGkZ5uGObT3vzIyDCPu1lxceXfq/yHz2f+nsXF9owv2N89Pd19f/d9+wxjyRLDuOYaw+jUqfLfuUkTw7jgAsN48knD+PxzwygttXvE3rF6dejHd/mP1asj/95eel1wyu8S4nwVSDG5frMUhcTmtZkDydn1h1xQWyOkn34K7IT9z38G3p6cbObGWMtLZ58d2Akb0RPrpRYvvS444XcJcb5YigJiwa5p+1i+2DhxR4TkvkrWJ09Kn3ziX15at65yJ+y+fc06Mm3amNupO3d27kXQCRe4aIn1UouXlvOc8LtUdb5igMAGiLdYJ/U6bUeExam1Ncr//M8+88/IrF1rztKU16WLf0Zm5EhzrPFM0K5pcOLFRPJE2gXmBaHOVwywFAXEUzyWYpy2I8LixErWO3f6Z2RWrZL27Qu8vWVLf7LvqFFmYGOJ97JaTYMTNy//hcNLM1GJIA6VhwlsgHiJZ5lzO8q4V8cJuT+HD5vf3wpmvv028PaGDaVhw/y7l3r3NmvMVBTvkvU1DU6cXFofCY9eUSEQ2MA14n1hd1r9ITtmkn75JbAT9ubNgT87KUkaONC/vDRoUHjVeeN5LmsTnDghmARCiNX1mxwbIF7indTrtPpD8aitUVLi74S9YoX00Udm1d/yevTwz8gMHy7V5AU1nueyNrlJTk0kB2LI9sDm/fff16OPPqpNmzZpz549evXVV3XRRRfZPSznY13ZfexI6nXCjojyop3waRhmZ2trRmb16sqdsNPT/c0jzz03On/feJ7L2gQnTk0kB2LI9sDm2LFj6tOnj66++mr96le/sns47uDFHQ6JwKtNRSNV25mkH3/0BzIrV5p/z/KaNg3shH3aaYGdsKMhnueyNsEJjzkkIEfl2Ph8vohnbBIux8brOxy8zolJvU5XUGBuvbaWl4J1wj7nHP/y0llnxWf2Ml7nsra5STzm4FDk2PxbUVGRisqtmRcWFto4mjhzW4EzVEbtjepZnbCtWZkNG8zHvsXn83fCHj26Zp2wo6G6czlhgpm8W9vl4trmJrnhMcfSOqLIdTM29913n+bMmVPpeELM2LDDwTt4IfcrLZU+/dQ/IxOsE/bpp/tnZEaMkJo3t2WoQQU7l6+/Hv3l4trucnPqY46l9YSVENu9wwlsgs3YZGRkJEZg48QCZ0BNlO+EvWpV5U7Ybdr4Z2Ri2Qk7FmK5XOzU4KSmWFpPaCxF/VtKSopSUlLsHoY92OGAmnDCxXD/fjOAsZaXduwIvL1JE3PrtRXMdO8e/YTfeIh0uTjSc+O0XW61wdI6YsR1gU1CY4cDImXXNP9PP5kXbGvn0qefBt6enCxlZflnZM4+2zzmZiUl0lNPhV9z5vDhxF6CcXrvMLiW7YHNTz/9pG/LlTXPy8vT1q1b1bx5c3Xs2NHGkTlQPAqcwTtCTfPv3m0ej+Y0v9UJ25qRWbfOPFZenz7+GZmhQ6VGjQJvd8LMUk0FCyCr8vrr5nM5HufGqSgeiFgxbLZ69WpDUqWPqVOnhvX1BQUFhiSjoKAgtgN1kldeMYz0dMMwXxbNj4wM8zgSU3GxYaxebRiLF5v/FhVVfoyU//D5zMdMcXHNfl5pqWFs22YYTzxhGOPHG0bjxpV/RmamYfy//2cYS5caxv79VX+/YI/p9HR3PKZfecX8e4b6Wwf7aNkydufGLVavDu9vtXq13SNFjMTq+u2o5OGaSLg6NhY3v7tFdAWbLWjZUjp4sPqvjWQH3c6dgYXxKnbCbtHC33OpYifs6sbv1gTS6vo4VeTzmefmwIHq7+v13Y1O7UKPuCF5GIG8lESImgsVFIQT1EhVT/NbnbCtYOabbwJvb9jQDKit5aVQnbCr4vYE0uryRMqzArXLLjOXjKvj9SUYNy6t84bSFQhsALeqKigIV/kddL/8In34oX9GZtOmyp2wBwzwz8gMGmRW/a0NtyeQRhJ8WAXxmjcPL7BJhN2NbigeaKHejmsQ2ABuFclsQUU+n9ShgxmYZGebwcyHH1buhN29u39GpqadsKvi9gTScIOPJ56QfvMb/xZvdjf6Oa0LfTDxTMRHrRHYAG5Vm4u9YZhLTYMHBx7v0CGwE3b79rUbY3XcXpsp3BIMVlAjuXMJJtacvLTu9uXSBBThgjgAxwj3Yt+kSfDjP/8spaVJEydK//u/0pdfmss+ixZJl18e+6BG8gcGoYrx+Xxm1WGnzl5YQYpU+XeoKkixlmA6dAg8np7Ou3+niWS5FI7AjA3gVtXNFliOHvX/PznZ7IQ9ZoyZJ9Ovn73vMr0we1HTPBE3LMHA/culCYjABnCrpCTpD3+Q/vu/q75f//7+bdhDhtjTCbsqbkogDaWmQYqTl2BgcvtyaQKijg3gJlYnbGsLdm6uuaRUUePG0g03SLNmOasTdlXYSgsnot5OzFDHBkhUVifslSvNRpIVa9S0aSONHCl17Gi+wPbq5c6ggNkLOJEXlksTDIEN4DQHDpgBjBXM5OUF3t64sRkAWMtLPXpE1gmbmREgMl5YLk0gBDaA3Y4d83fCXrEieCfsQYP8hfEGDKh5J2yKjAE1Q7K3a5BjA8TbyZPShg3+PJmPPw7eCduakRk61JylqS0392QCooUZS8eI1fWbwAaINcOQtm/3Ly2tWRO4BVuSOnXyb8E+91ypdevojqG6Zo2JkADJBQ3MWDoKycOAm+TnB3bC3rs38PYWLcwApnwn7EjyZCLl9p5MtcUFDbRFSBgENkA0/OtfgZ2wv/468PYGDaRhw/zLS336RN4JuzYSucgYFzTQFiGhENgANXH8uL8T9ooV0ubNZo0ZS506gZ2ws7Jq3wm7NhK1yBgXNEjMWCYYAhsgHCUlZvBizch88EHwTtjWjMzw4WYfJqcIt1mjU3sy1RQXNEiJPWOZgAhsgGAMQ/rmm8DCeEeOBN7H6oQ9apT5EY+mkTWVqEXGuKBBStwZywRFYANY9u71z8isWFH5nX5amlnh1wpmzjgjtgm/0ZaIRca4oEFK3BnLBMV2bySuwkJp7Vp/MPP554G316tnNo0cPdr8OOssqa4H3gsk0rZn+vzEn1MfX1YSuRR8xpIk8rhjuzdQWydOSOvW+WdkPvnEfBG2+Hxm8FK+E3bDhvaNN1YSqSdToi7B2cXJ2+oTccYyQTFjA+8qLZX++U//jMz771fuhH3qqf4ZmREjzPoybuHUd8ZOFOyCm5HBBS2a3FLZOpGfNw773ak8HAKBDQLk5fkTfleurNwJu3Vr/4zMqFFmxV83cvI7Y6dy2Iu6p1DZ2vkc+JpBYBMCgU2CO3jQ3wl7xYrKnbAbNTJnYqxApmdPdyX8BuOWd8ZIHGvWmIn11Vm9OnGWQZ3Eoa8Z5NgAkr8TtrW8tHVr4O116/o7YY8eXbtO2E5EwTk4EdvqnSsBXzMIbOBsxcVmJ2xreemjjyp3wu7d27+8NGxYdDphOxUF5+BEbKt3rgR8zSCwgbMYhvTFF/6lpbVrzW3Z5XXq5J+RiUUnbCfjnTGciDoxzpWArxkENrDfrl2BCb8Vn2DNm/s7YY8eHftO2E7GO2N3SLREZbbVO1cCvmaQPIz4+9e/zGRDK5j56qvA2xs0MC8E1vJS377x7YTtZBSccz4H7j6JG7bVO4+DXzPYFRUCgY0LHD9u5sZYy0ubNlXuhH322f4ZGbs7YTsdFVSdy6G7T+Iq0War3MChrxkENiEQ2DhQSYm0ZYt/RuaDD8zgprxu3QI7YTdtastQXYt3xs5DLRc4mQNfMwhsQiCwcQDDkL791j8js3q1udxUXvv2gQm/HTrYM1Yv4Z2xs1DLBU7nsNcM6thUJzfX3D3jgJOVEKxO2FY9mfz8wNvT0vyF8UaPdl8nbDdIpJ5PbpCAu0/gMgnymuGdwGb8eP//EyVRL56OHjW3XlvLS599Fni71QnbWl7q188bnbCBcCXg7hPAibyzFCWpbCIrkRL1YuXECWn9+sBO2MXF/tt9PunMM/2tCs45x5udsIFwOXj3CTzCYUtJtcVSVCScUCbabQ/A0lJp2zb/jMz775vtC8o79VT/jMzIke7qhA3EGrVcEEuJXEYgQt6csSnPjkQ9tzwAd+zwz8isWiUdOBB4e6tW/hmZUaPMd6MAqubA3SdwOY+WEWBXVAjVBjaLF0uXXhq/ATn5AXjwoBnoWcHM998H3t6okbn1unwnbArjAZFz24wtnMvDZQRYiqqpeCbqOa2L6s8/my+u1vLSli2Bt1udsK3lpQEDzCRgALWTILtPEAcJ2MSytrwb2NjRdM3uB2BxsbRxo39G5uOPzSTg8nr18s/IDBsmNWkS/XEAAKKDMgIR82ZgY1eiXrwfgOU7Ya9caRYIq9gJu2PHwMJ4bdpE52eHwhQ8AEQPZQQi5s3AJj3dnkS9eDwAd+0KLIwXqhO2tbx0yinxK4znlqRpAHCLoUPN19HqygjEc3XC4byTPPzmm0q1u/JwLOpYHDkS2An7yy8Db69f3/x9reWlvn3t+d2dnDQNAG7m0CaWtcWuqBAc1yuqtg9AqxO2NSOzcWPwTtjWjExWlhnc2MnDWfsA4AgeLCNAYBOC4wIbKbIHYEmJtHWrf0YmN7dyJ+yuXf0zMiNGOK8TNs3/ACD2PJbDyHZvN5k0ydzSHewBaHXCtmZkVq2q3Am7aVNzVmbKFGnMGOd3wiZrHwBijzICYUmMwMaOKLf8A3DfPmnZMn8ws3Nn4H1TU83u1998Y+bUHDkiLV9u7nhKTXX+NCNZ+wAAh/D+UpQdO3WOHjV7LVnLS9u2Bd5er540eLB/eSk/X7rkEvcm3tL8DwAQIXJsQqjyDxOvnTpWJ2xrRmb9+uCdsK2E3/KdsL2SeOvRrH0AQGwQ2IQQ8g8Ty4ChtFT67DP/jMzatZU7YZ9ySmAn7JYtg38vuxNvo7lM58GsfSAkjyVyAvFG8nCkot3eYMeOwITf/fsDb2/Vyt8Fe9QoqXPn8MZpZ+JttJfpqkqaBryEYpSAY3k3sKltwHDokBnAWMHMd98F3t6okdlryWpXUNNO2HYl3oZaptu92zxe06UjsvbhdbF67gCICu8uRUW6xPPzz9IHHwR2wi7/p0lKCuyEPXBgdDph25F465W8HiDeeO4AUcNSVKTC6a/RqpW5bDJnjlntt2In7J49Azthx6IAYFKSOX09ebI5pmCJt9Fu5ml3F3LArXjuAI7n3cCmqoBBMj/fv1+6917/sYyMwE7YbdvGZ6yTJpnT18HW7GOReEtBPaBmeO4AjufdwEYyl4tuvll67jlzqamiZs0CO2Gfemr8OmFXFM/EWwrqATXDcwdwPG/l2BiGvxP2ihWVO2EnJ0u9epmzOGPH2tcJ224U1ANqhucOEDXk2FTn3HPNhN+KnbD79/fPyAwebH8nbCewI68H8AKeO4DjeWfGRlKqZPZcKt8Ju1kzewdot6qKiFFQD6gZnjtArVF5OISyP8zTTyt1/HhzGhimcIqIUT0VqBmeO0CtENiEEKs/jOvFq08WAAA1EKvrdw1K5cLxSkrMmZpgMat1bMYM836omZISM1F9yRLzX/6WAOAIjghsFixYoM6dO6t+/frq16+fcnNz7R6Su0VSRAyRy8kxd8aMHClNmWL+m5lpHgcA2Mr2wGbZsmWaMWOG7rnnHm3ZskVDhw7Vf/7nf2rnzp12D829KCIWWm1nWqwlvoqBo9UniOAGAGxle2Dz+OOP65prrtG1116rbt26ad68ecrIyNDTTz9t99DciyJiwdV2poUlPgBwPFsDmxMnTmjTpk0aO3ZswPGxY8fqo48+smlUDlGbmQWrT1aoKso+n7k1dejQaIzUHaIx08ISHwA4nq0F+g4ePKiSkhK1adMm4HibNm20d+/eoF9TVFSkoqKiss8LCgokmdnVnvHGG9Jdd0k//ug/1r699Mgj0oUXhvc9srOlK64IfpthSA89JB07VvuxukFJifSb31Q903LLLeYMTlXbdb/7Lryf99130llnRT5OAEgg1nU72puzHVF52FdhZsEwjErHLNnZ2ZozZ06l4xkZGTEZm2P8+GPoQKUmovm9vGD3bql58+h8r2uvNT8AANU6dOiQ0tLSovb9bA1sWrZsqaSkpEqzM/v37680i2OZNWuWbrvttrLPjxw5ok6dOmnnzp1R/cMgcoWFhcrIyFB+fj41hRyA8+EcnAvn4Fw4R0FBgTp27Kjm0XpT+W+2Bjb16tVTv379tHz5ck2cOLHs+PLlyzVhwoSgX5OSkqKUlJRKx9PS0niQOkRqairnwkE4H87BuXAOzoVz1KkT3XRf25eibrvtNl1xxRXq37+/srKy9Kc//Uk7d+7UDTfcYPfQAACAy9ge2FxyySU6dOiQ7r//fu3Zs0c9e/bUP/7xD3Xq1MnuoQEAAJexPbCRpJtuukk33XRTjb42JSVFs2fPDro8hfjiXDgL58M5OBfOwblwjlidC9c3wQQAALDYXnkYAAAgWghsAACAZxDYAAAAzyCwAQAAnuGKwGbBggXq3Lmz6tevr379+im3miaDa9euVb9+/VS/fn116dJFCxcujNNIvS+Sc5GTk6MxY8aoVatWSk1NVVZWlt599904jtbbIn1eWD788EPVrVtXffv2je0AE0yk56OoqEj33HOPOnXqpJSUFJ1yyin6y1/+EqfReluk5+LFF19Unz591LBhQ7Vr105XX321Dh06FKfRetf777+vCy64QO3bt5fP59Nrr71W7ddE5fptONzSpUuN5ORk489//rOxfft2Y/r06UajRo2MH374Iej9v//+e6Nhw4bG9OnTje3btxt//vOfjeTkZOPll1+O88i9J9JzMX36dOORRx4xPvnkE+Prr782Zs2aZSQnJxubN2+O88i9J9JzYTly5IjRpUsXY+zYsUafPn3iM9gEUJPzceGFFxoDBw40li9fbuTl5Rnr1683PvzwwziO2psiPRe5ublGnTp1jCeffNL4/vvvjdzcXKNHjx7GRRddFOeRe88//vEP45577jFeeeUVQ5Lx6quvVnn/aF2/HR/YDBgwwLjhhhsCjnXt2tWYOXNm0PvfeeedRteuXQOOXX/99cagQYNiNsZEEem5CKZ79+7GnDlzoj20hFPTc3HJJZcYv/3tb43Zs2cT2ERRpOfj7bffNtLS0oxDhw7FY3gJJdJz8eijjxpdunQJOPbHP/7RSE9Pj9kYE1E4gU20rt+OXoo6ceKENm3apLFjxwYcHzt2rD766KOgX/Pxxx9Xuv95552njRs36uTJkzEbq9fV5FxUVFpaqqNHj0a94Vmiqem5eP755/Xdd99p9uzZsR5iQqnJ+XjjjTfUv39/zZ07Vx06dNDpp5+uO+64Q7/88ks8huxZNTkXgwcP1q5du/SPf/xDhmFo3759evnllzVu3Lh4DBnlROv67YjKw6EcPHhQJSUllTp9t2nTplJHcMvevXuD3r+4uFgHDx5Uu3btYjZeL6vJuajoscce07Fjx3TxxRfHYogJoybn4ptvvtHMmTOVm5urunUd/bR3nZqcj++//14ffPCB6tevr1dffVUHDx7UTTfdpMOHD5NnUws1OReDBw/Wiy++qEsuuUTHjx9XcXGxLrzwQj311FPxGDLKidb129EzNhafzxfwuWEYlY5Vd/9gxxG5SM+FZcmSJbrvvvu0bNkytW7dOlbDSyjhnouSkhJNmTJFc+bM0emnnx6v4SWcSJ4bpaWl8vl8evHFFzVgwACdf/75evzxx7Vo0SJmbaIgknOxfft23XLLLbr33nu1adMmvfPOO8rLy6MRs02icf129Fu3li1bKikpqVKkvX///kpRnaVt27ZB71+3bl21aNEiZmP1upqcC8uyZct0zTXX6KWXXtLo0aNjOcyEEOm5OHr0qDZu3KgtW7bo5ptvlmReWA3DUN26dfXee+/p3HPPjcvYvagmz4127dqpQ4cOSktLKzvWrVs3GYahXbt26bTTTovpmL2qJuciOztbQ4YM0f/8z/9Iknr37q1GjRpp6NCheuCBB5jlj6NoXb8dPWNTr1499evXT8uXLw84vnz5cg0ePDjo12RlZVW6/3vvvaf+/fsrOTk5ZmP1upqcC8mcqbnqqqu0ePFi1qyjJNJzkZqaqm3btmnr1q1lHzfccIPOOOMMbd26VQMHDozX0D2pJs+NIUOG6Mcff9RPP/1Uduzrr79WnTp1lJ6eHtPxellNzsXPP/+sOnUCL4VJSUmS/LMFiI+oXb8jSjW2gbV177nnnjO2b99uzJgxw2jUqJGxY8cOwzAMY+bMmcYVV1xRdn9ru9itt95qbN++3XjuuefY7h0lkZ6LxYsXG3Xr1jXmz59v7Nmzp+zjyJEjdv0KnhHpuaiIXVHRFen5OHr0qJGenm5MnjzZ+Pzzz421a9cap512mnHttdfa9St4RqTn4vnnnzfq1q1rLFiwwPjuu++MDz74wOjfv78xYMAAu34Fzzh69KixZcsWY8uWLYYk4/HHHze2bNlStvU+Vtdvxwc2hmEY8+fPNzp16mTUq1fPOOuss4y1a9eW3TZ16lRj+PDhAfdfs2aNceaZZxr16tUzMjMzjaeffjrOI/auSM7F8OHDDUmVPqZOnRr/gXtQpM+L8ghsoi/S8/HFF18Yo0ePNho0aGCkp6cbt912m/Hzzz/HedTeFOm5+OMf/2h0797daNCggdGuXTvjsssuM3bt2hXnUXvP6tWrq7wGxOr67TMM5toAAIA3ODrHBgAAIBIENgAAwDMIbAAAgGcQ2AAAAM8gsAEAAJ5BYAMAADyDwAYAAHgGgQ0AAPAMAhsAAOAZBDYAAMAzCGwAOM6SJUtUv3597d69u+zYtddeq969e6ugoMDGkQFwOnpFAXAcwzDUt29fDR06VP/7v/+rOXPm6Nlnn9W6devUoUMHu4cHwMHq2j0AAKjI5/PpwQcf1OTJk9W+fXs9+eSTys3NJagBUC1mbAA41llnnaXPP/9c7733noYPH273cAC4ADk2ABzp3Xff1ZdffqmSkhK1adPG7uEAcAlmbAA4zubNmzVixAjNnz9fS5cuVcOGDfXSSy/ZPSwALkCODQBH2bFjh8aNG6eZM2fqiiuuUPfu3XX22Wdr06ZN6tevn93DA+BwzNgAcIzDhw9ryJAhGjZsmJ555pmy4xMmTFBRUZHeeecdG0cHwA0IbAAAgGeQPAwAADyDwAYAAHgGgQ0AAPAMAhsAAOAZBDYAAMAzCGwAAIBnENgAAADPILABAACeQWADAAA8g8AGAAB4BoENAADwDAIbAADgGf8fhxvmCVJdzuQAAAAASUVORK5CYII=",
+ "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjYAAAHFCAYAAADhWLMfAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy81sbWrAAAACXBIWXMAAA9hAAAPYQGoP6dpAABNj0lEQVR4nO3dd3hT5eIH8G/oZLVs6KIFBdkgIHvKUCmz1ua5eBG9+rivIPpTcAFetYBXBRXECfeKQBhFEBRBpIgDZDpAVLTIsGVV2grSkb6/P96bpmmTNklzctb38zx9oOecpG9z0pzveadFCCFAREREZAC11C4AERERUaAw2BAREZFhMNgQERGRYTDYEBERkWEw2BAREZFhMNgQERGRYTDYEBERkWEw2BAREZFhMNgQERGRYTDYEAHYvXs3JkyYgJYtWyIiIgLNmzdH37598dBDD7kcN2TIEAwZMkSVMh47dgwWiwVLly4N2HMmJSVh9OjRVR5z6623IikpKWA/M9iGDBkCi8VS9hUZGYkOHTrgmWeeQVFRkdrFC5qkpCTceuutaheDSHGhaheASG2bNm3C2LFjMWTIEMybNw8xMTHIzs7G3r17sXLlSrzwwgtlxy5atEjFkqrjySefxJQpU9QuRo20bt0a7733HgDg7NmzeOutt/Dkk0/i+PHjeOONN1QuXXCsW7cOUVFRaheDSHEWrhVFZjd48GCcOnUKR44cQWioa9YvLS1FrVraqNg8duwYWrVqhSVLlgTszjspKQmdOnXCxo0bA/J8ahBC4PLly6hdu7bb/UOGDMG5c+fw/fffl20rKSlBhw4d8NtvvyEvLw+RkZHBKi6Ki4thsVgqvdeIKDC08YlNpKLz58+jSZMmbi80FUNNxaYoR/PQ888/j7lz5yIpKQm1a9fGkCFD8NNPP6G4uBjTp09HbGwsoqOjMWHCBJw5c8blOR3NQevWrUOXLl0QGRmJ1q1b4+WXX/aq/D///DMmTpyIZs2aISIiAu3bt8fChQt9fyE8cNcUZbFYcP/99+Pdd99F+/btUadOHXTt2tVtQPKmfJcvX8ZDDz2Ebt26ITo6Go0aNULfvn2xfv36Ss/n+NmLFy9G+/btERERgf/85z8+/U6hoaHo1q0bioqKcOHChbLtQggsWrQI3bp1Q+3atdGwYUOkpqbi119/dXm8EALPPfccEhMTERkZiZ49e2Lr1q2V3h+ZmZmwWCx499138dBDDyEuLg4RERE4evQoAOCTTz7BsGHDEBUVhTp16qB///7Ytm2by886e/Ys7rzzTiQkJCAiIgJNmzZF//798cknn5Qdc+DAAYwePbrsNY6NjUVycjJOnjxZdoy7pqjjx4/j73//u8u5eeGFF1BaWlp2jOM9/u9//xsvvvgiWrVqhXr16qFv377YtWuXT687UTDwloFMr2/fvnjrrbfwwAMP4Oabb0b37t0RFhbm03MsXLgQXbp0wcKFC3HhwgU89NBDGDNmDHr37o2wsDC88847+O233/Dwww/jjjvuwIYNG1wef/DgQUydOhWzZs1CixYt8N5772HKlCkoKirCww8/7PHnHj58GP369UPLli3xwgsvoEWLFvj444/xwAMP4Ny5c5g5c6Zfr4k3Nm3ahD179uDpp59GvXr1MG/ePEyYMAE//vgjWrdu7VP5CgsLkZubi4cffhhxcXEoKirCJ598gpSUFCxZsgS33HKLy89+//33sXPnTjz11FNo0aIFmjVr5nP5s7Ky0KBBAzRt2rRs21133YWlS5figQcewNy5c5Gbm4unn34a/fr1wzfffIPmzZsDAB5//HGkp6fjzjvvREpKCk6cOIE77rgDxcXFaNu2baWfNWPGDPTt2xeLFy9GrVq10KxZMyxbtgy33HILxo0bh//85z8ICwvD66+/juuuuw4ff/wxhg0bBgCYNGkS9u/fj2effRZt27bFhQsXsH//fpw/fx4AcPHiRYwYMQKtWrXCwoUL0bx5c+Tk5GD79u0oKCjw+PufPXsW/fr1Q1FREf71r38hKSkJGzduxMMPP4xffvmlUrPrwoUL0a5dO8yfPx+AbKIcNWoUsrKyEB0d7fPrT6QYQWRy586dEwMGDBAABAARFhYm+vXrJ9LT00VBQYHLsYMHDxaDBw8u+z4rK0sAEF27dhV2u71s+/z58wUAMXbsWJfHT506VQAQeXl5ZdsSExOFxWIRBw8edDl2xIgRIioqSly8eNHlZy1ZsqTsmOuuu07Ex8e7PJ8QQtx///0iMjJS5ObmVvm7JyYmiuTk5CqPmTx5skhMTHTZBkA0b95c5Ofnl23LyckRtWrVEunp6TUuX0lJiSguLha33367uPrqqyv97Ojo6Gp/N4fBgweLjh07iuLiYlFcXCyys7PFU089JQCIxYsXlx331VdfCQDihRdecHn8iRMnRO3atcUjjzwihBAiNzdXRERECKvV6nKc4/Hl3x/bt28XAMSgQYNcjr148aJo1KiRGDNmjMt2u90uunbtKnr16lW2rV69emLq1Kkef7+9e/cKAOL999+v8nVITEwUkydPLvt++vTpAoDYvXu3y3H33HOPsFgs4scffxRCON93nTt3FiUlJWXHff311wKAWLFiRZU/lyjY2BRFpte4cWPs3LkTe/bswZw5czBu3Dj89NNPmDFjBjp37oxz585V+xyjRo1yabZq3749ACA5OdnlOMf248ePu2zv2LEjunbt6rJt4sSJyM/Px/79+93+zMuXL2Pbtm2YMGEC6tSpg5KSkrKvUaNG4fLly4o2FQwdOhT169cv+7558+Zo1qwZfvvtN7/Kt3r1avTv3x/16tVDaGgowsLC8Pbbb+OHH36o9LOvvfZaNGzY0OuyHjp0CGFhYQgLC0NMTAyefvppzJgxA3fddVfZMRs3boTFYsHf//53l7K2aNECXbt2RWZmJgBg165dKCwsRFpamsvP6NOnj8fRYzfeeKPL919++SVyc3MxefJkl59VWlqK66+/Hnv27MHFixcBAL169cLSpUvxzDPPYNeuXSguLnZ5riuvvBINGzbEo48+isWLF+Pw4cNevSaffvopOnTogF69erlsv/XWWyGEwKeffuqyPTk5GSEhIWXfd+nSBQDKzjeRVjDYEP1Pz5498eijj2L16tX4/fff8eCDD+LYsWOYN29etY9t1KiRy/fh4eFVbr98+bLL9hYtWlR6Tsc2R5NDRefPn0dJSQleeeWVsou242vUqFEA4FUo81fjxo0rbYuIiMBff/3lc/kyMjKQlpaGuLg4LFu2DF999RX27NmDf/zjH5VeKwCIiYnxqaxXXHEF9uzZg6+//hqrV69G165dkZ6ejpUrV5Ydc/r0aQgh0Lx580rl3bVrV1lZHefD0SxVnrtt7sp7+vRpAEBqamqlnzV37lwIIZCbmwsAsNlsmDx5Mt566y307dsXjRo1wi233IKcnBwAQHR0NHbs2IFu3brhscceQ8eOHREbG4uZM2dWCkHlnT9/3u3rGBsb6/J7OlQ83xEREQBQdr6JtIJ9bIjcCAsLw8yZM/HSSy+5jKZRiuMi5W6buwABAA0bNkRISAgmTZqE++67z+0xrVq1ClwhfeRL+ZYtW4ZWrVrBZrPBYrGU7S8sLHT7uPLHeMPRwRcArrnmGgwdOhQdO3bE1KlTMXr0aNSrVw9NmjSBxWLBzp07yy7a5Tm2Oc6HI5yUl5OT47bWpmJ5mzRpAgB45ZVX0KdPH7dldoSkJk2aYP78+Zg/fz6OHz+ODRs2YPr06Thz5gw2b94MAOjcuTNWrlwJIQS+/fZbLF26FE8//TRq166N6dOnu33+xo0bIzs7u9L233//3aWMRHrDYEOml52d7fbO1dEE4riDVdKhQ4fwzTffuDRHLV++HPXr10f37t3dPqZOnToYOnQoDhw4gC5dupTVBmmFL+WzWCwIDw93CQA5OTluR0UFQuPGjTFnzhzcdttteOWVVzBjxgyMHj0ac+bMwalTpyo1M5XXu3dvREREwGazISUlpWz7rl278Ntvv3k1mWH//v3RoEEDHD58GPfff7/X5W7ZsiXuv/9+bNu2DV988UWl/RaLBV27dsVLL72EpUuXemzGBIBhw4YhPT0d+/fvd3mP/fe//4XFYsHQoUO9LheRljDYkOldd911iI+Px5gxY9CuXTuUlpbi4MGDeOGFF1CvXr2gTE4XGxuLsWPHYtasWYiJicGyZcuwdetWzJ07F3Xq1PH4uAULFmDAgAEYOHAg7rnnHiQlJaGgoABHjx7FBx98UKmfhDs5OTlYs2ZNpe1JSUlltRz+8rZ8o0ePRkZGBu69916kpqbixIkT+Ne//oWYmBj8/PPPNSqDJ7fccgtefPFF/Pvf/8Z9992H/v37484778Rtt92GvXv3YtCgQahbty6ys7Px+eefo3PnzrjnnnvQqFEjTJs2Denp6WjYsCEmTJiAkydPYvbs2YiJifFq3qN69erhlVdeweTJk5Gbm4vU1FQ0a9YMZ8+exTfffIOzZ8/itddeQ15eHoYOHYqJEyeiXbt2qF+/Pvbs2YPNmzeXhaqNGzdi0aJFGD9+PFq3bg0hBDIyMnDhwgWMGDHCYxkefPBB/Pe//0VycjKefvppJCYmYtOmTVi0aBHuuecet6O7iPSAwYZM74knnsD69evx0ksvITs7G4WFhYiJicHw4cMxY8aMsg6/SurWrRtuu+02zJw5Ez///DNiY2Px4osv4sEHH6zycR06dMD+/fvxr3/9C0888QTOnDmDBg0aoE2bNmX9WKqzb98+3HTTTZW2T548ucbLN3hbvttuuw1nzpzB4sWL8c4776B169aYPn16WWBQQq1atTBnzhwkJydj/vz5eOqpp/D666+jT58+eP3117Fo0SKUlpYiNjYW/fv3d+lk++yzz6Ju3bpYvHgxlixZgnbt2uG1117D448/jgYNGnj18//+97+jZcuWmDdvHu666y4UFBSgWbNm6NatW9l8M5GRkejduzfeffddHDt2DMXFxWjZsiUeffRRPPLIIwCANm3aoEGDBpg3bx5+//13hIeH46qrrsLSpUsxefJkjz+/adOm+PLLLzFjxgzMmDED+fn5aN26NebNm4dp06b5/boSqY0zDxOpzAiz/5KcF6ddu3aYOXMmHnvsMbWLQ2RarLEhIvLRN998gxUrVqBfv36IiorCjz/+iHnz5iEqKgq333672sUjMjUGGyIiH9WtWxd79+7F22+/jQsXLiA6OhpDhgzBs88+63HINxEFB5uiiIiIyDBUn6Bv1qxZsFgsLl/uJisjIiIiqo4mmqI6duzoslJt+Wm7iYiIiLyliWATGhrKWhoiIiKqMU0EG8e8HREREejduzeee+45tG7d2u2xhYWFLtOsl5aWIjc3F40bN/Z5mnUiIiJShxACBQUFiI2N9WpiS2+p3nn4o48+wqVLl9C2bVucPn0azzzzDI4cOYJDhw65XSNn1qxZik3YRURERMF14sQJxMfHB+z5VA82FV28eBFXXHEFHnnkEbezX1asscnLy0PLli1x4sQJREVFBbOoRERE5Kf8/HwkJCSUTZkQKJpoiiqvbt266Ny5s8f1YSIiItyuvBsVFcVgQ0REpDOB7kai+nDvigoLC/HDDz+4XW2ZiIiIqCqqB5uHH34YO3bsQFZWFnbv3o3U1FTk5+dXuXgbERERkTuqN0WdPHkSf/vb33Du3Dk0bdoUffr0wa5du5CYmKh20YiIiEhnVA82K1euVLsIREREZBCqN0URERERBQqDDRERERkGgw0REREZBoMNERERGQaDDRERERkGgw0REREZBoMNERERGQaDDRERERkGgw0REREZBoMNERERGQaDDRERERkGgw0REREZBoMNERERGQaDDRERERkGgw0REREZBoMNERERGQaDDRERERkGgw0REREZBoMNERERGQaDDRERERkGgw0REREZBoMNERERGQaDDRERERkGgw0REREZBoMNERERGQaDDRERERkGgw0REREZBoMNERERGQaDDRERERkGgw0REREZBoMNERERGQaDDRERERkGgw0REREZBoMNERERGQaDDRERERkGgw0REREZRqjaBSAiIjIMux3YuRPIzgZiYoCBA4GQELVLZSoMNkRERIGQkQFMmQKcPOncFh8PLFgApKSoVy6TYVMUERFRTWVkAKmprqEGAE6dktszMtQplwkx2BAREdWE3S5raoSovM+xbepUeRwpjsGGiIioJnburFxTU54QwIkT8jhSHIMNERFRTWRnB/Y4qhEGGyIiopqIiQnscVQjDDZEREQ1MXCgHP1ksbjfb7EACQnyOFIcgw0REVFNhITIId1A5XDj+H7+fM5nEyQMNkRERDWVkgKsWQPExbluj4+X2zmPTdBwgj4iIqJASEkBxo3jzMMqY7AhIiIKlJAQYMgQtUthamyKIiIiIsNgsCEiIiLDYLAhIiIiw2CwISIiIsNgsCEiIiLDYLAhIiIiw2CwISIiIsNgsCEiIiLDYLAhIiIiw2CwISIiIsNgsCEiIiLDYLAhIiIiw2CwISIiIsNgsCEiIiLDYLAhIiIiw9BUsElPT4fFYsHUqVPVLgoREVVktwOZmcCKFfJfu13tEhFVEqp2ARz27NmDN954A126dFG7KEREVFFGBjBlCnDypHNbfDywYAGQkqJeuYgq0ESNzZ9//ombb74Zb775Jho2bKh2cYiIqLyMDCA11TXUAMCpU3J7RoY65SJyQxPB5r777kNycjKGDx9e7bGFhYXIz893+SIiIoXY7bKmRojK+xzbpk5lsxRphurBZuXKldi/fz/S09O9Oj49PR3R0dFlXwkJCQqXkIjIxHburFxTU54QwIkT8jgiDVA12Jw4cQJTpkzBsmXLEBkZ6dVjZsyYgby8vLKvEydOKFxKIiITy84O7HFEClO18/C+fftw5swZ9OjRo2yb3W7HZ599hldffRWFhYUICQlxeUxERAQiIiKCXVQiInOKiQnscUQKUzXYDBs2DN99953Ltttuuw3t2rXDo48+WinUEBFRkA0cKEc/nTrlvp+NxSL3DxwY/LIRuaFqsKlfvz46derksq1u3bpo3Lhxpe1ERKSCkBA5pDs1VYaY8uHGYpH/zp8vjyPSANU7DxMRkcalpABr1gBxca7b4+Plds5jQxpiEcJd3aJ+5OfnIzo6Gnl5eYiKilK7OERExmW3y9FP2dmyT83AgaypIb8pdf3WzMzDRESkcSEhwJAhapeCqEpsiiIiIiLDYLAhIiIiw2CwISIiIsNgsCEiIiLDYLAhIiIiw+CoKCIj4XBcIjI5Bhsio8jIAKZMcV2JOT5ezhrLCdSIyCTYFEVkBBkZcsr78qEGkOv7pKbK/UREJsBgQ6R3drusqXE3ibhj29Sp8jgiIoNjsCHSu507K9fUlCcEcOKEPI6IyOAYbIj0Ljs7sMcREekYgw2R3sXEBPY4IiIdY7Ah0ruBA+XoJ4vF/X6LBUhIkMcRERkcgw2R3oWEyCHdQOVw4/h+/nzOZ0NEpsBgQ2QEKSnAmjVAXJzr9vh4uZ3z2BCRSXCCPiKjSEkBxo3jzMNEZGoMNkRGEhICDBmidimIiFTDpigiIiIyDAYbIiIiMgwGGyIiIjIMBhsiIiIyDAYbIiIiMgwGGyIiIjIMBhsiIiIyDAYbIiIiMgwGGyIiIjIMBhsiIiIyDAYbIiIiMgwGGyIiIjIMBhsiIiIyDAYbIiIiMgwGGyIiIjIMBhsiIiIyDAYbIiIiMgwGGyIiIjIMBhsiIiIyDAYbIiIiMgwGGyIiIjIMBhsiIiIyDAYbIiIiMgwGGyIiIjIMBhsiIiIyDAYbIiIiMgwGGyIiIjIMBhsiIiIyDAYbIiIiMgwGGyIiIjIMBhsiIiIyDAYbIiIiMgwGGyIiIjKMULULQERERBpntwM7dwLZ2UBMDDBwIBASonap3GKwISIiIs8yMoApU4CTJ53b4uOBBQuAlBT1yuUBm6KIiIjIvYwMIDXVNdQAwKlTcntGhjrlqgKDDREREVVmt8uaGiEq73NsmzpVHqchDDZERERU2c6dlWtqyhMCOHFCHqchDDZERERUWXZ2YI8LEnYeJiIifdLRSB1diokJ7HFBwhobIiLSn4wMICkJGDoUmDhR/puUpMnOrLo1cKAc/WSxuN9vsQAJCfI4DWGwISIifdHhSB1dCgmRQ7qByuHG8f38+ZqrJWOwISIi/dDpSB3dSkkB1qwB4uJct8fHy+0anMeGfWyIiEg/fBmpM2RI0IplaCkpwLhxuunPxGBDRET6odOROroXEqKboKh6U9Rrr72GLl26ICoqClFRUejbty8++ugjtYtFpA12O5CZCaxYIf9l9TqZnU5H6lDwqB5s4uPjMWfOHOzduxd79+7Ftddei3HjxuHQoUNqF41IXRz1QVSZTkfqUPBYhHDXA0tdjRo1wvPPP4/bb7+92mPz8/MRHR2NvLw8REVFBaF0REHgGPVR8c/T8WGu0U57REHh+PsAXP9G+PehK0pdv1WvsSnPbrdj5cqVuHjxIvr27ev2mMLCQuTn57t8ERkKR30QVU2HI3UoeDTRefi7775D3759cfnyZdSrVw/r1q1Dhw4d3B6bnp6O2bNnB7mEREHEUR9E1dPZSB0KHk00RRUVFeH48eO4cOEC1q5di7feegs7duxwG24KCwtRWFhY9n1+fj4SEhLYFEXGsWKF7FNTneXLgb/9TfnyEBEpQKmmKE3U2ISHh+PKK68EAPTs2RN79uzBggUL8Prrr1c6NiIiAhEREcEuIlHwcNQHEZHfNNXHxkEI4VIrQ2QqHPVBROQ31WtsHnvsMdxwww1ISEhAQUEBVq5ciczMTGzevFntohGpw7E+S2qqDDHuRn1ocH0WIk3git+mp3qNzenTpzFp0iRcddVVGDZsGHbv3o3NmzdjxIgRaheNSD0c9UHkO879RNBI5+Ga4Dw2ZGi8+yTyDud+0h2lrt8MNkREpG92u6yZ8TRNgsUiazuzsnhjoCGmmKCPiIjIZ77M/USGx2BDRET6xhW/qRwGGyIi0jfO/UTlMNgQEZG+ce4nKofBhoiI9M0x9xNQOdxw7ifTYbAhIiL949xP9D+qzzxMREQUEFzxm8BgQ0RERhISAgwZonYpSEVsiiIiIiLDYLAhIiIiw2CwISIiIsNgsCEiIiLDYLAhIiIiw+CoKCIiInLPbtfd8HkGGyIiIqosIwOYMsV15fT4eDnLs4YnPGRTFBEREbnKyABSU11DDQCcOiW3Z2SoUy4vMNgQERGRk90ua2qEqLzPsW3qVHmcBjHYEBERkdPOnZVrasoTAjhxQh6nQexjQ0REpCVqd9jNzg7scUHGYENERKQVWuiwGxMT2OOCjE1RRFpntwOZmcCKFfJfjbZrE1ENaaXD7sCBMkxZLO73WyxAQoI8ToMYbIjUVlVwycgAkpKAoUOBiRPlv0lJmh6RQER+0FKH3ZAQWUMEVA43ju/nz9fsfDYWIdy9ivqRn5+P6Oho5OXlISoqSu3i+E7ttlTynhLnqqpqZ0DepVX8E3V8sKxZo+m5JIjIB5mZ8salOtu3A0OGKF0ayd3nU0KCDDUB+OxR6vrNPjZq0kJbKnlHiXPlqHauGFxOnQJuvBFo3Njz3ZvFIu/exo1jECYyAi122E1JkZ8xOrv5ZlOUWrTSlkrVU+JceVPtfP6858drfLglEflIqx12Q0JkDdHf/ib/1XioAXwMNidOnFCqHOaipbZUqppS56q6eSK8pdHhlqRR7IiuXTrvsKslPgWbdu3a4cknn8TFixeVKo856HzyI1NR6lwFKpBodLglaRA7omubzjvsaolPwWbr1q3YsmUL2rRpgyVLlihVJuPTYlsquafUuappIOHdG/mCTd/6kJIiBwXExbluj4/nYAEf+BRs+vXrh927d2POnDl46qmncPXVVyMzM1OhohmYVttSqTKlzpU31c6NG8t/efdGNcGmb31JSQGOHZOjn5Yvl/9mZTHU+MCvzsO33HILfvrpJ4wZMwbJycmYMGECjh49GuiyGRfbUvVDqXPlTbXzG2/w7o1qjk3f+qPDDrta4veoKCEERo4ciTvvvBMbNmxAp06d8NBDD6GgoCCQ5TMmtqXqh5LnyptqZ969UU2x6ZtMxqcJ+hYvXow9e/Zgz549+OGHHxASEoIuXbqgT58+6NatG9577z389NNPWLduHXr27KlkucvoeoI+hSc/ogBS8lxxkkZSkhYnfiOCctdvn4JNQkIC+vTpU/bVs2dPREREuBzz3HPPYfny5fj+++8DVsiq6DrYALyo6QnPFemR3S5HP5065b6fjcUiawmzsvh+1judfUZpIth44/Tp04iNjYU9SB3RdB9siIiU5hgVBbiGGy7PYRw6nMleqet3wGcebtasGT799NNAPy0REfmLw4iNjcP5XXARTCIis9BZUwV5wdHU6Gnkm4abGrkIJhER1YxjGDEZhy/D+U1y7rkIJhERkV5xOH8lDDZERER6xZnsK2GwISIi0ivOZF8Jgw0REZFecSb7ShhsiIiI9MwxnD821nV7XJwph/Mz2BARERmBp+Yok2GwISIi0jNO0OeCwYaIiEiv7Ha5lIK7uXYd26ZOlceZBIMNERGRXvkyQZ9JcOZhIjMqKgIWLQJ++QW44grg3nuB8HC1S0VEvtLrBH12O7BjhyJPzWBDZDaPPAK8+KJr1fTDDwPTpgHz5qlXLiLynZ4m6CstBT7/HLDZ5GitM2cU+TEMNkRm8sgjwPPPV95utzu3M9wQ6Ydjgr5Tp9z3s3EsgqnWBH2lpcCuXTLMrF7tWnPUsCHwxx8B/5Fc3ZvILIqKgDp1qu5EGBICXLrEZikiPXGMigJcw41j+Hew57IRAti7V4aZVatkHx+H6GhgwgTAakX+NdcgukkTru5NRH5atKj6kRF2uzxu6tSgFMkQ7HbZMTM7W1b3DxxoqlleSQMcE/RNmeLakTg+Xs46HIxQIwRw8KAzzGRlOffVrw+MGwdYrcCIEUBEhNyen69IURhsiMzil18CexzJO2V3F5MFC0w32yupLCVFhodghmwhgO+/l0HGZgN+/tm5r04dYMwYGWauvx6oXVu5clTAYENkFldcEdjjzM5R/V+xNd8xKZoJp7InlYWEAEOGKP9zjhyRQcZmA374wbk9MhJITpZhZtQooG5d5cviBvvYEJkF+9gEjt0OJCV5nj/E0WEzK4vNUmQMR486m5m+/da5PTxc1shYrbKGpn59r59Sqes3a2yIzCI8XA7pdjcqymHaNIYab/gyKVow7qCJlHDsmLOZaf9+5/bQUGDkSBlmxo2THYI1hMGGyEwcQ7krzmMTEsJ5bHyh10nRiKpz4oQclm2zAV9/7dweEgIMGwakpclRTY0aqVfGajDYEJnNvHnAM89w5uGa0NOkaETVyc6WfcJsNuCLL5zba9UCBg+WNTMpKUDTpuqV0QfsY0NE5CtHH5vqJkVjHxvSqjNngLVrZZj57DPn+9hiAQYMkGHmxhuBFi0UKwL72BARaUVIiBzSnZoqLwTuJkWbP5+hhrTl/Hk5mm/VKuDTT+WswA59+sgwc9NNQFycemUMAAYbIiJ/aGFSNKLqXLgAvP++rJn55BOgpMS5r2dPZ5hJTFSrhAHHYENE5C81JkUjqk5+PrBhgwwzH38MFBc793XrJjsAp6UZds4qBhsico9LBXgnWJOiEVXl4kVg40YZZj78ECgsdO7r2FHWzKSlAVddpV4Zg0T1YJOeno6MjAwcOXIEtWvXRr9+/TB37lxcZYIXn0izuFQAkfb99ZcMMTabDDV//eXcd9VVzjDTsaPr4wx+06J6sNmxYwfuu+8+XHPNNSgpKcHjjz+OkSNH4vDhw6ir0nTMRKbGpQKItKuwENi8WXYA3rAB+PNP577WrWWYsVqBLl2cHdnLM8FNi+aGe589exbNmjXDjh07MGjQoGqP53BvogDiUgFE2lNUJDv+2myyI3D5VbETE2WtjNUKdO/uPsw4eLppcTwmyDctphnunZeXBwBo5GFWw8LCQhSWazvMV2jZcyJT4lIBRNpQUgJs3y7DTEYG8Mcfzn1xcXIkk9UK9O5ddZhxsNtlTY27ugwh5HNMnSo7wwf6piXITV+aCjZCCEybNg0DBgxAp06d3B6Tnp6O2bNnB7lkRCbBpQKI1GO3y8nybDY5ed65c859zZs7w0y/fnJWYF+oddNSVdPX8OGB+znlaCrY3H///fj222/x+eefezxmxowZmDZtWtn3+fn5SEhICEbxiIyPSwUQBYa3tRSlpXIZg1WrZFNQTo5zX5MmsukoLQ0YNKhmtRxq3LRU11/vv/8N3M8qRzPB5p///Cc2bNiAzz77DPHx8R6Pi4iIQERERBBLRmQiAwfKu6nqlgoYODD4ZSPSi+o66AoB7N4ta2ZWr5Z/bw4NG8pjrFZg6FC5knYgBPumxZumr+nTA/OzKlC987AQAv/85z+xbt06ZGZmok2bNj49np2HiQLMcZcFuF8qgKOiiDyrqoOuEMDYscDBg8Dx4859UVFyxWyrVa6grcSCtMFe3ywzUwazKuQDiAYCfv32sZEu8O677z4sW7YMy5cvR/369ZGTk4OcnBz8VX48PhEFj2OpgIrrxcTHM9QQVaW6WgpADtE+fhyoVw+YOBFYv14uSLl0KXDDDcqEGsC5vhlQubOxEuubqdgPT/UaG4uH3txLlizBrbfeWu3jWWNDpBCDT+JFFHBe1FIAAGbPBv7v/4DatRUvUiXumskSErxb38yXzwQVa2xU72OjsWl0iMiBSwUQee/HH4GFC707tk0bdUIN4P/6Zr5O7OdNf73YWNf+RQGierAhIiITMVJN4K+/yg7ANhvwzTfeP07tUYW+3rT4Mxu5o+krNdXZv8jB0VIzZw4waZJfv0JVVG+Kqik2RRER6YQRpvP/7Tc5NHvVKmDvXuf20FA5L8vu3cCFC8HpoBsMNZ2NvIqmr/zhwxW5fjPYEBGR8jQ2nb9PTp2Sw7JtNmDXLuf2WrWAa6+Vo5kmTAAaNzbeqEJv+w1t3+65FshDLZ1pllQgIqIAU7v5R83p/P2VkyNDiM0GlJ801mIBBg+WYSYlBWjWzPVxjlGF7mqmvOmgqzWBmNgvyP31GGyIiPTA33CiheYfvaxBdvasfL1sNmDHDjkrsEP//jLMpKZW30fG3w66WqTD2cgZbNS+kyEiqo6/4cSfTp9K0PIaZLm5wLp1Msx8+qm8Jjj07i3DzE03ydfbF0YZVajD2cjNHWy0cCdDRFQVf8OJlpp/tHbXn5cnJ8az2YAtW+RK2g49esi1mdLSZKdZs/NmdFMgJ/YLAPN2HtZzRzYiMoeajEh5+mlg5szqf0ZVnT4DJdjT+btTUAB88IEMM5s3A0VFzn1dusiambQ04Morlfn5eleTif08UKrzsDmDTU2HrxERBYO/I1IyMoAbb/TuZyxfDvztb/6UzjdqjBa6eBHYtEkOzd60Cbh82bmvQwdnmGnXLrA/16gC3HWDo6ICSS8d2YjI3Pzpm+JogvJWsJp/gjVa6PJl4KOPZM3MBx8Aly4597VpI8OM1Qp06hSYn2cmOuk3ZM5go+WObEREDv70Tanuxq28hITgdvpUarRQYaHsK2OzyUUmCwqc+1q1ctbMdOtWeQFIMhxzBhutdWQjInLHnxEpvtyQqdHpM1B3/cXFwLZtMsysWyc7BDskJMggY7UCPXsyzJiMOYONDoevEZEJ+TMixdsbstmz9TdAoqRE9juy2WSfndxc577YWDksOy0N6NNHzgpMpmTOzsOA8aa9JiLj8mVESnUjkAB543bsmD4GRzg6rK5aJT+Xz5517mvWTH6OW63AgAEMMzrDUVEe1OiFUWD4GhGRInwZkaL3G7fSUuCrr2TNzJo1rs1rjRvLEV9WKzBokFx8knSJwcaDGr8wnHmYiIxIbzduQgBffy3DzOrVruVu0EAuMmm1ykUnw8JUK6Zm6fBaxmDjAVf3JiLyQOsXOyGAAwdkmFm1SjaPOdSvD4wfL8PMiBFAeLhapaw5pc+DTmfRZ7DxgMGGiCgIAnVxFgL47jtnmDl61Lmvbl1g7FjZAfj664HIyMCVXy1Khw4dz6LPYOMBgw0RkcICcXE+fFgGGZsNOHLEub12bSA5WdbMjBoF1KkT2LKrSenQofNZ9BlsPNBUsNF6tS+RWfFv0381uTj//LMMMjYb8P33zu0REcANN8gwM3o0UK+eMmVXUzBCh79LbmgEl1TQOp22cRIZntp/m3oOVf6sEJ6V5WxmOnDAeXxYGDBypAwz48YBat+IKi0YS/dwFn23GGwCwdMdzalTcruG2ziJDE3tv021Q1VNeXtxXrtW/muzAXv2OPeHhADDh8swM3480LCh4kX2iZKhMxihg7Pou8WmqJrSeRsnKUTPd+lGofbfpo47dZZZsQKYONG3x9SqJZtH0tLk79ekiTJlqymlQ2cwmomqm4xR49cfpa7fnKaxpnypbiRzyMiQHzZDh8qLwtCh8vuMDLVLZi5q/m1W14QDyCYcuz3wPzuQfLnTHzQIWLgQ+P134JNPgDvv1HaoSU2t/P5w1OQF4m/VsXSPp3WqLJaaL0LqWHLD8XwVnx9QZz0wlTHY1BTbOKm8YHxgknfU/Ns0yg3PwIFyDaaqNGgAHD8O7NgB3Hsv0Lx5UIrmt2CFzmCFjpQUWfsXF+e6PT5eH7WCCmCwqSm2cZKDUe7S9cpul9X/K1bIf5s18+5xSvxt6v2G548/gHfekcOwc3LcH2OxyK+335Y1D3pR09BZ8X1W1d9zsEJHSoqc3HD7dmD5cvlvVpYpQw3AzsM1x5XCySEYoyDIPXf9JeLi5LpCubnB/9vU4w1Pfj6wfr3sALxlC1Bc7NzXqpV8HfPynNvi47W7PENVahI6/emXk5IiR4Ep3ecuJISfK//DYFNTjurG1FT5QeluwTkTtnGakt7v0vXKUyfd3393bgv236Zebnj+/BP44AMZZjZvBgoLnfs6d5YdgNPSgLZtjdMh3t/QWZMRdgwdQcVRUYGitwXnKPB0PlmWLnkz8qlRIzm7bbD/NrW6wvalS8CHH8ows2kT8Ndfzn3t2smh2WlpQIcOwS9bMPgzkkjtEXYGxZmHPdBMsAGMc0dD/tH50EtV1PRvxtsw+ckn8nmD/bfpzw2PEp8jly/LGplVq4ANG4CLF537rrzSGWY6d/Y8ikdravI6+Ro6edOiCMWu30Ln8vLyBACRl5endlGIhFi7VgiLRX7Jj0z55di2dq3aJdSOtWuFiI93fZ3i4317jZYvd328p6/ly5X7PapTUiLE9u2yDNu3y+89CcRr4lBYKMTGjUJMmiREVJTrcyYlCfHII0Ls2ydEaam/v5l6AvE6uXuOhAT3z6HG+8yX941OKXX9ZrAhfdDTH7kvH5hm5QiAFS8MvgbA7du9u+Bs367kbxMYgXhNioqE2LxZiNtuE6JBg8oX/mnThNi9W59hxiFQ7x0hvP9cCfb7LJABtyLH77xsmRAvvST/VekzlcHGAwYbE1Dyj1wpegpiwVZSUvl8VrxAJSR495o5nsvdhc7X51JTTV6TkhIhtm0T4s47hWjc2PVxLVoI8c9/CvH550LY7cH/vQItkO8df35uMN5ngQxu7p7b0+unwmcqg40HDDYGp+QfOakj0He/Rmj+8/U1sduF2LFDiHvvFaJZM9djmjYV4p57hMjM1H6g85WaNXTBeJ8pGdw8fZZWfP4g/r0odf3mBH2kXZzwzpgCPSzeCDOvevu77tgh3/MJCcDgwcCiRcCZM3Lk1x13AFu3ymHuixbJ/UbrpK7mlAqe3mdxccCsWXJ02fz5wHvvVT9xnydKzVhd1WdpRQb4TOU8NqRdnPDOmJSYvC5Yk6ApxdvfddYs5/+jo4EJE+SIpmHDgLAwRYqmOF9GN6k98WHF99nPPwNvvAHMnFn5WH8W1FQquFX3WepgkM9UBhvSLk54Z0xKTV6n50nQqntNHOrVA8aPl0OzR44EIiKCVkRF+DqTrxYmPnS8zzIyZND0dL5Onqx+4r6KlApuvn5G6vwzlU1RpF1q352RMrgicWUhIcBDD1Udah55RDY7vfsuMGaMMUKNrwvGauW9o1TTjlIrgvv6Ganzz1QGG9Iupf7ISX1G6BcTCEeOALNnAx07Ag8+6P6Y+Hhg7Vpg7lw5g7IR1KT/nBbeO/407XhDqeDm+CytjkE+U9kURdrFdbiMTe/9Yvx19KicAdhmA7791rk9PBy4/nrgpptkZ+C8POO+JjXtP6f2e0fJph1HcHPXRDd/vvy9MzN9+73Lf5ZWVcskhCE+UxlsSNuq+yM3y529Uem5X4wvjh1zhpn9+53bQ0NlXxmrVV6woqNVK2LQ2O3Atm3eHVtVIFDzvaN0046n4LZ+feU1q7ztpOz4LL3zTuD8ed/KozNcK4r0getwkd6cPAmsXi3DzO7dzu0hIXIUU1qaHNXUqJF6ZQw2d52Fq6LVtZeqWxTTIZDrw3laXdyXRVU1tpgnF8H0gMGGiDQjO1teYGw24IsvnNstFnmBtlrlxadpU9WKqBpPF2Z39LBgrDe/j8USmH4/gQokGlvMU6nrN5uiiIhq4swZ2bl31So5gV75C92AATLMpKYCLVqoV0a1+TKKSC/95zw1kztUt4q7LwI1p5dJptBgsCEi8tX588C6dbJm5tNPgdJS574+fWSYuemmyiN3zMrbUUSAvvrPle8Lc+oUcPasrI2Liwtsc3mgAolJptBgsCEi8saFC8D778sw88knQEmJc1/PnrLPTFoakJioVgm1y9sL8xNPyEnvtFxTU1EwOjEHKpBoYYLDIGCwISLyJD8f+OADGWY+/hgoKnLu69pV1sykpQFXXKFeGfXA2wvzsGH6CjXBEqhAYpIpNBhsiIjKu3gR2LhRhpkPPwQKC537OnZ0hpmrrlKvjHpjkpqCGvM0+jOQgcQEU2hwVBQR0V9/yRCzapUMNZcuOfe1bSvDjNUqgw35xzGKCHB/YTbTjNPueLNulrtj/O2krIEpNDjc2wMGGyLyS2GhbF6y2YANG4A//3Tua93aGWa6dPG8rAf5JpAXZiPxZY4aDQSSQGGw8YDBhoi8VlQkO/7abLIjcH6+c1/LlrKJyWoFevRgmFGKgS7MAaGxSfOCifPYEBH5o6RETjhms8k74z/+cO6Li5PDsq1WoHdvhplgMMsyGt4K1Bw1VIbBhsiIzH5XbLcDn30mw8zatcC5c859zZvLMJOWBvTvD9Sq5ftzm/m1DSS+lqaZNC+YGGyIjMabTohGVFoqlzFYtUr2ScjJce5r0gS48UZZMzNokP8XT7O+tkrgaymZZNK8YGIfGyIjCcRCeXoihFxg0maTC06eOuXc17Ch/F2tVrk+TmgN7+PM9toqia+lk6OPTXVD4dnHxmsMNkRGYZZOiEIA+/bJMLNqFXD8uHNfVBQwfrwMM8OHA+HhgfmZZnltg4GvZWUmHQqv1PXbx8ZlItIsXzoh6o0QwDffAI89BrRpA1xzDfDvf8tQU68eMHEisH69XJDyP/8BRo0KXKgBjP3aBhtfy8ock+ZVXFssPt6woUZJ7GNDFExKdpY0YifEQ4ecNTM//ujcXrs2MGaM7AA8apT8XklGfG3VwtfSvfILapq5M3UAMNgQBYvSnSW13gnR21D3448yyNhsMtg4RETIEGO1AqNHA3XrBq/sWn9t9YSvpWccCh8Q7GNDFAzB6Cyp5U6I1YW6X3+VQcZmk01ODmFhwPXXyzAzZozsQ6MGLb+2esPXkv6HnYc9YLAhzQtmZ0ktdkKsKtQJIVfG/uUX5/bQUNnx12qVHYEbNAhmaT3T4murV3wtCew8TIFktwOZmcCKFfJfu13tEhlbMDtLaq0Tot0ua2rc3T85tv3yi7ygDR8OvPmmnH/mo4+AW2/VTqgBtPfa6hlfS1IQ+9iYDSfFCr5gd5bUUifE6kKdw9q1wIQJypenprT02uodX0tSiOrB5rPPPsPzzz+Pffv2ITs7G+vWrcP48ePVLpYxeWoSOHVKbuedkjLU6CypdifEs2fl++3ll707/vJlZcsTSGq/tkbC15IUoHpT1MWLF9G1a1e8+uqrahdF36prXvKmSWDqVDZLKWHgQFkr5mmBRYsFSEiQx+lZbi7w9tvAyJEypN19N3D4sHePNeMIGCJShOo1NjfccANuuOEGtYuhb940L3EFWfWEhMhzkZrq7DDr4Ag78+frswo+L09OjGezAVu2yJW0Hbp3l4tNLlgAnD5d9QgYvYc6ItIM1YONrwoLC1FYWFj2fX5+voql0QBvm5c4KZa6HJ0l3QXQ+fP11QRYUAB88IEMM5s3A0VFzn1dusjRTGlpwJVXym1t2xoz1BGRJuku2KSnp2P27NlqF0Mbqmteslhk89K4cZwUSwv03Fny0iVg0yYZZjZtcu0T0769DDNWK9CuXeXHGinUEZHmaWoeG4vFUm3nYXc1NgkJCeacxyYzU65aXJ3t2+UFlJNikS8uX5bDrm02WUNz6ZJzX5s2zjDTsaPn/kPlKbmcBBHpjlLz2OiuxiYiIgIRERFqF0MbfGleMnI/DwqcwkLZV2bVKtl3pqDAua9VK9nEZLUC3bp5F2bK4wgYIgoC3QUbKsfX5iU2CWiPFmoxiouBbdtkzcy6dbJDsENCgjPM9Ozpe5ghIgoy1YPNn3/+iaNHj5Z9n5WVhYMHD6JRo0Zo2bKliiXTAccw4uqal8qPONFzPw+jUXOyxJIS2ZRps8ly5OY698XGytFMaWlAnz5ALdVnhSAi8prqfWwyMzMx1E0/kcmTJ2Pp0qXVPt70a0X5uuaKFmoIKDiLYlZktwOffy7DzNq1wJkzzn3NmsnyWK3AgAEMM0SkOC6C6YHpgw3g/s4/IaFy8xKXU9CGYC6KWVoKfPWVDDNr1rj2y2rcGLjxRhlmBg2Si08SEQUJg40HDDb/U11NjBo1BOSeL6PZ/OlsKwSwZ48MM6tWuQaoBg3kmkxWK3DttUBYmO/PXxXWCBKRlzgqiqpW1YgTX+a74UVIeUpMligEcOCAM8wcO+bcV78+MH68DDMjRgDh4b6U1nusESQiDWCwMQMup6AtgZosUQjgu+9kkLHZgHKd8FG3LjBmjAwz118PREb6X15vcIFVItIIBhsz4HIK2uLPaLbyfvhBBhmbDThyxLm9dm0gOVmGmVGjgDp1lCl/RawRJCINYbAxAy6noC3+TJb488/OZqbvvnNuDw8HbrhBhpkxY4B69YLyK7hgjSARaQiDjRnUtIaAAs+byRKzspzNTAcOOI8JCwNGjpRhZuxYIDo66MV3wRpBItIQBhszCAkBXnpJTrpWEZdTUI+7yRKTkuQcM716yZFNDiEhwPDhctK8CROAhg1VK3YlrBEkIg1hsDGDjAzgwQfd7+NyCuoKCQHatgW+/RZ49VXgyy+d+2rVkk03Vqs8P02aqFbMKrFGkIg0xFzBxoxzbHgareLwwgsMNWo4fVrWzNhs8j3pOD8Wi3xfWq1y8rzmzdUtpze4wCoRaYh5Jugz4xwbwZzhlqp37px8H65aJSffKy117uvbV4aZ1FQgLk69MtaEtzNgExGBMw975NULY9ZZd5We4Zaq98cfwPvvy5qZTz6RYdPhmmtkmLnpJsAoC76asVaUiPzCmYf9ZeY5NjhaRR35+cD69TLMbNkCFBc79119tewAnJYGtG6tXhmVUtUM2EREQWD8YGPmOTY4WiV4/vwT2LhRhpmPPgIKC537OnWSNTNpabKjMBERKcb4wcbMtRYcraKsS5eADz+UYWbTJuCvv5z72rVzhpkOHdQrIxGRyRg/2Ji51oKjVQLv8mVg82bZAXjDBuDiRee+K66QYcZqBTp3dr7GREQUNMYPNmavtfBmhluqWlERsHWrrJlZv172oXFITHSGmauvZpgxCnaCJtIt4wcb1lq4n+GWH9RVKy4GPv1Uhpl164ALF5z74uJkE5PVKmcIZpgxFm+mhmDwIdIscwz3BjjHBlXNbpfD47dvB775Rs4AnJvr3N+ihRyWbbXKOWdq1VKtqKQgb6aGAMw3JxaRAjiPjQc+vTC8y6KKSkuB554D5sxx7S8DAFFRwM03y9oZvleMz5sJLRs1As6fd78PMO6cWEQK4Dw2gcA5NgiQYWbXLtkB+L//lZPouVNQIBee5HvGHLyZGsJdqHHsM/KcWEQ6Yq5go0WsRQoOIYC9e2WfmVWr5NxF3uCFyjxqOuWDkefEItIRBhs1mXH9qmASAjh40BlmsrKc++rVA/r0kcscVPV4XqjMI1BTPhhxTiwiHWEPSLU4OilWrPo+dUpuz8hQp1xG8P33wJNPAlddBXTvDsydK0NNnTqy829GBnDmDPCPf3j3fLxQmYNjaoiajnIz4pxYRDrCGhs1mHn9KqUcOeKsmTl82Lk9MhJITpYdgJOTgbp1nfvMPHkjVebN1BCNGsnRcmacE4tIJxhs1KDH9au02Bfo6FEZZGw24NtvndvDw4Hrr5e1M2PGAPXru3+82SdvpMqqm9ASMPecWEQ6wGCjBr2tX6WlvkDHjjnDzP79zu2hocDIkbJmZtw4oEGD6p+LkzeSO9VNaMmZvIk0zVzz2GhFZiYwdGj1x23frn6NjTcTlin9YX7yJLB6tQwzu3c7t4eEANdeK2tmJkyQzQT+4OSN5Cst1mAS6Qwn6PNAl8HGMRFYdU0gWVnqflh6M2GZUuXMzpahyWYDvvjC9WcOGSLDTEoK0LRpYH4eL1REREHFCfqMRC9NIMHuC3TmDLB2rWxq2rHD9XUZMECGmdRUubxBoHHyxuBjmCQiBTDYqEUPq24Hoy9Qbq5sCrLZ5KKTpaXOfX36yDBz001y4UkyDi312yIiQ2GwUZPWV91Wajj0hQvA++/LmpmtW4GSEue+nj1lB+C0NCAx0bfnJX3w1G/LMYcT11siohpgHxvyLJB9gQoKgA0bZM3Mxx8DRUXOfV27ypqZtDTgiisC+iuQxqjZb4uINIV9bCj4atoX6OJFYONGWTPz4YfA5cvOfR07OsPMVVcp9isoin1EfKfHOZyISFcYbKhqvvYF+usv4KOPZM3Mxo3ApUvOfW3byjBjtcpgo2fsI+Ifvc3hRES6w2BD1auuL1BhoWxestlkc9Offzof27q1M8x06VLzdXi0gH1E/MdlLIhIYexjQ/4pKgK2bZNh5v33gbw8576WLWUTk9UK9OhhjDDjwD4iNaOXOZyISHHsY0PqKymRsyHbbLLW4o8/nPtiY52jmfr0MVaYKY99RGpGL3M4EZFuMdhQ1ex24LPPZAfgtWuBs2ed+5o3lxcoqxXo3x+oVUu9cgYL+4jUnB7mcCIi3TJ2sOGoFf+UlgJffilrZtasAXJynPuaNAFuvFGGmUGDzPd6so9IYGh9Dici0i3jBhuOWvGNEHKBSZtNLjh56pRzX8OGcpFJq1UuOhlq3LdNtQYOlO+j6vqIDBwY/LLpDZexICIFGPMKZaZRKzWplRIC2L9fhplVq4DffnPui4oCxo+XYWb4cCA8XJHi6w77iBARaZrxRkWZadSKP7VSQgDffusMM7/84txXrx4wdqzsAHzddUBkpLLl1zN3r31CAvuIEBF5SalRUcYLNpmZwNCh1T9w+3Z9V4N7qpVy1BpUrJU6dEgGGZsN+PFH5/batYHRo2XNzKhR8nvyDvtwERH5jcO9vWWGUSt2u6wtcJdJhZDhZupUoH17GXBsNhlsHCIiZIixWoHkZFlTQ75jHxEiIs0xXrAxw6gVb+dS6dDBuS0sTDYvWa2yuYmTGRIRkQEZL9iYYdSKt7VNtWoBI0fKMDN+PNCggZKlIiIiUp3xgo0ZRq14W9uUkSHnCtED9lchIqIAMOZUsY6ZTePiXLfHx+t7qHdODrBwIfDkk1UfZ7HIETqjRwenXDWVkSFHsg0dCkycKP9NSpLbiYiIfGC8UVHlGaEW4Nw5uZSBzQbs2CFnBa6Kp1FRWuXr6C4iIjIEDvf2wJCre+fmAuvWyeHZ27bJgObQq5fsM3PTTcCePfqeS8VMcw4REZELDvc2urw8YP16WTOzdStQXOzc1727DDNpaTIIOCQk6Hu9Ha6UTUREAcZgo6aCAuCDD2SY2bwZKCpy7uvSRQaZtDSgTRvPz6HnuVTMMOcQEREFFYNNsF26BGzaJMPMpk3A5cvOfe3bO2tm2rdXr4zBYoY5h4j0zAj9FMl0GGyC4fJl4KOPZJj54AMZbhzatHGGmU6dnJ1mzcAMcw6R73gx1QZ/1qIj0gAGG6UUFgJbtsgOwOvXy2Ynh6QkGWasVqBbN3OFmfLMMOcQ+YYXU23wNFrx1Cm5naMVScM4KiqQiovlKCabTY5qystz7ktIkLUyVivQs6d5w0xFdjvw7LPywpWb69yup9FdFBgc+q8NHK1IQcJRUTWhZNV2SYlcUdxmkx/M5S/OMTFyWLbVCvTpI5c4ICd3d+eNGsltjz/OD00z8XZh13Hj+L5QGkcrks4ZP9goUbVttwOffy7DzNq1wJkzzn3Nmsm7TqsVGDCAYcYTT3fnf/wBzJol+xvx7tw8eDHVDo5WJJ0zdrAJZDtxaSnw1VcyzKxZ4/pH3bixfB6rFRg8GAg19staY7w7p4p4MdUOjlYknTPuFTgQF08h5Oy+NpvsBFz+jrJBA2DCBBlmrr0WCAtT4rcwJt6dU0W8mGoHRyuSzhk32Ph78RQCOHDAGWaOHXPuq18fGD9edgIeORIID1eo8AbHu3OqiBdT7eBoRdI54wYbXy6eQgDffSeDjM0GHD3q3F+3LjBmjKyZuf56IDJSmfKaCe/OqSJeTLUlJUU2ubvrn8jRiqRxmujZumjRIrRq1QqRkZHo0aMHdu7cWfMn9fai+PHHQIcOQNeuctjx0aMyvAwaJP+o16wBli2TNTUMNYHhuDv3NOTdYpHDvXl3bi6Oi2lcnOv2+HgO9VZDSoqssd6+HVi+XP6blcXzQJqn+jw2NpsNkyZNwqJFi9C/f3+8/vrreOutt3D48GG0bNmy2sd7HAfvmIvBU9V2ReHhwA03AK1aAatXy8c5cIKwwHN07Abc353zQmZenHmYyBSUmsdG9WDTu3dvdO/eHa+99lrZtvbt22P8+PFIT0+v9vFVvjCeLp4OISGyeclqBcaOlZPrcYKw4HE3FJ8T8xERmYJSwUbVpqiioiLs27cPI0eOdNk+cuRIfPnllzV78hMnZLVp69aVg0pkJHDvvcDZs8DGjcCkSUC9elWPogLkKCq7vWblIidWdRMRUYCp2nn43LlzsNvtaN68ucv25s2bIycnx+1jCgsLUVhYWPZ93v+WLcjPz5dV1+vXy0nzvv7a9YFdush+NDfcIGtpHFXb+fnyX29HUW3ezL4fgda9u/P/Fy+qVw4iIgqa/P9dfwPdcKSJUVGWCp1IhRCVtjmkp6dj9uzZlbYnJCRU/UO+/VZ+vfuu3+UEAIweXbPHExERUZnz588jOjo6YM+narBp0qQJQkJCKtXOnDlzplItjsOMGTMwbdq0su8vXLiAxMREHD9+PKAvDPkuPz8fCQkJOHHihPoLkhLPh4bwXGgHz4V25OXloWXLlmjUqFFAn1fVYBMeHo4ePXpg69atmDBhQtn2rVu3Yty4cW4fExERgYiIiErbo6Oj+SbViKioKJ4LDeH50A6eC+3gudCOWgFeU1H1pqhp06Zh0qRJ6NmzJ/r27Ys33ngDx48fx91336120YiIiEhnVA82VqsV58+fx9NPP43s7Gx06tQJH374IRITE9UuGhEREemM6sEGAO69917ce++9fj02IiICM2fOdNs8RcHFc6EtPB/awXOhHTwX2qHUuVB9gj4iIiKiQNHEWlFEREREgcBgQ0RERIbBYENERESGwWBDREREhqGLYLNo0SK0atUKkZGR6NGjB3bu3Fnl8Tt27ECPHj0QGRmJ1q1bY/HixUEqqfH5ci4yMjIwYsQING3aFFFRUejbty8+/vjjIJbW2Hz9u3D44osvEBoaim7duilbQJPx9XwUFhbi8ccfR2JiIiIiInDFFVfgnXfeCVJpjc3Xc/Hee++ha9euqFOnDmJiYnDbbbfh/PnzQSqtcX322WcYM2YMYmNjYbFY8P7771f7mIBcv4XGrVy5UoSFhYk333xTHD58WEyZMkXUrVtX/Pbbb26P//XXX0WdOnXElClTxOHDh8Wbb74pwsLCxJo1a4JccuPx9VxMmTJFzJ07V3z99dfip59+EjNmzBBhYWFi//79QS658fh6LhwuXLggWrduLUaOHCm6du0anMKagD/nY+zYsaJ3795i69atIisrS+zevVt88cUXQSy1Mfl6Lnbu3Clq1aolFixYIH799Vexc+dO0bFjRzF+/Pggl9x4PvzwQ/H444+LtWvXCgBi3bp1VR4fqOu35oNNr169xN133+2yrV27dmL69Oluj3/kkUdEu3btXLbdddddok+fPoqV0Sx8PRfudOjQQcyePTvQRTMdf8+F1WoVTzzxhJg5cyaDTQD5ej4++ugjER0dLc6fPx+M4pmKr+fi+eefF61bt3bZ9vLLL4v4+HjFymhG3gSbQF2/Nd0UVVRUhH379mHkyJEu20eOHIkvv/zS7WO++uqrSsdfd9112Lt3L4qLixUrq9H5cy4qKi0tRUFBQcAXPDMbf8/FkiVL8Msvv2DmzJlKF9FU/DkfGzZsQM+ePTFv3jzExcWhbdu2ePjhh/HXX38Fo8iG5c+56NevH06ePIkPP/wQQgicPn0aa9asQXJycjCKTOUE6vqtiZmHPTl37hzsdnullb6bN29eaUVwh5ycHLfHl5SU4Ny5c4iJiVGsvEbmz7mo6IUXXsDFixeRlpamRBFNw59z8fPPP2P69OnYuXMnQkM1/WevO/6cj19//RWff/45IiMjsW7dOpw7dw733nsvcnNz2c+mBvw5F/369cN7770Hq9WKy5cvo6SkBGPHjsUrr7wSjCJTOYG6fmu6xsbBYrG4fC+EqLStuuPdbSff+XouHFasWIFZs2bBZrOhWbNmShXPVLw9F3a7HRMnTsTs2bPRtm3bYBXPdHz52ygtLYXFYsF7772HXr16YdSoUXjxxRexdOlS1toEgC/n4vDhw3jggQfw1FNPYd++fdi8eTOysrK4ELNKAnH91vStW5MmTRASElIpaZ85c6ZSqnNo0aKF2+NDQ0PRuHFjxcpqdP6cCwebzYbbb78dq1evxvDhw5Uspin4ei4KCgqwd+9eHDhwAPfffz8AeWEVQiA0NBRbtmzBtddeG5SyG5E/fxsxMTGIi4tDdHR02bb27dtDCIGTJ0+iTZs2ipbZqPw5F+np6ejfvz/+7//+DwDQpUsX1K1bFwMHDsQzzzzDWv4gCtT1W9M1NuHh4ejRowe2bt3qsn3r1q3o16+f28f07du30vFbtmxBz549ERYWplhZjc6fcwHImppbb70Vy5cvZ5t1gPh6LqKiovDdd9/h4MGDZV933303rrrqKhw8eBC9e/cOVtENyZ+/jf79++P333/Hn3/+Wbbtp59+Qq1atRAfH69oeY3Mn3Nx6dIl1KrleikMCQkB4KwtoOAI2PXbp67GKnAM3Xv77bfF4cOHxdSpU0XdunXFsWPHhBBCTJ8+XUyaNKnseMdwsQcffFAcPnxYvP322xzuHSC+novly5eL0NBQsXDhQpGdnV32deHCBbV+BcPw9VxUxFFRgeXr+SgoKBDx8fEiNTVVHDp0SOzYsUO0adNG3HHHHWr9Cobh67lYsmSJCA0NFYsWLRK//PKL+Pzzz0XPnj1Fr1691PoVDKOgoEAcOHBAHDhwQAAQL774ojhw4EDZ0Hulrt+aDzZCCLFw4UKRmJgowsPDRffu3cWOHTvK9k2ePFkMHjzY5fjMzExx9dVXi/DwcJGUlCRee+21IJfYuHw5F4MHDxYAKn1Nnjw5+AU3IF//LspjsAk8X8/HDz/8IIYPHy5q164t4uPjxbRp08SlS5eCXGpj8vVcvPzyy6JDhw6idu3aIiYmRtx8883i5MmTQS618Wzfvr3Ka4BS12+LEKxrIyIiImPQdB8bIiIiIl8w2BAREZFhMNgQERGRYTDYEBERkWEw2BAREZFhMNgQERGRYTDYEBERkWEw2BAREZFhMNgQERGRYTDYEBERkWEw2BCR5qxYsQKRkZE4depU2bY77rgDXbp0QV5enoolIyKt41pRRKQ5Qgh069YNAwcOxKuvvorZs2fjrbfewq5duxAXF6d28YhIw0LVLgARUUUWiwXPPvssUlNTERsbiwULFmDnzp0MNURULdbYEJFmde/eHYcOHcKWLVswePBgtYtDRDrAPjZEpEkff/wxjhw5ArvdjubNm6tdHCLSCdbYEJHm7N+/H0OGDMHChQuxcuVK1KlTB6tXr1a7WESkA+xjQ0SacuzYMSQnJ2P69OmYNGkSOnTogGuuuQb79u1Djx491C4eEWkca2yISDNyc3PRv39/DBo0CK+//nrZ9nHjxqGwsBCbN29WsXREpAcMNkRERGQY7DxMREREhsFgQ0RERIbBYENERESGwWBDREREhsFgQ0RERIbBYENERESGwWBDREREhsFgQ0RERIbBYENERESGwWBDREREhsFgQ0RERIbBYENERESG8f+ADUMJH70M0wAAAABJRU5ErkJggg==",
"text/plain": [
"Learning rate = 10.0
+
+Learning rate = 10.0
Lambda = 0.1
Accuracy score on test set: 0.11388888888888889