"""
Simple code that tests XOR, OR and AND gates with linear regression
"""
-
+# import necessary packages
import numpy as np
+import matplotlib.pyplot as plt
+from sklearn import datasets
+
# Design matrix
X = np.array([ [1, 0, 0], [1, 0, 1], [1, 1, 0],[1, 1, 1]],dtype=np.float64)
@@ -1570,9 +1573,77 @@ yXOR = np.array( [ 0, 0, 1 ,1, 1])
# The AND gate
yAND = np.array( [ 0, 0 ,0, 1])
++ + +
def sigmoid(x):
+ return 1/(1 + np.exp(-x))
+
+def feed_forward(X):
+ # weighted sum of inputs to the hidden layer
+ z_h = np.matmul(X, hidden_weights) + hidden_bias
+ # activation in the hidden layer
+ a_h = sigmoid(z_h)
+
+ # weighted sum of inputs to the output layer
+ z_o = np.matmul(a_h, output_weights) + output_bias
+ # softmax output
+ # axis 0 holds each input and axis 1 the probabilities of each category
+ probabilities = sigmoid(z_o)
+ return probabilities
+
+# we obtain a prediction by taking the class with the highest likelihood
+def predict(X):
+ probabilities = feed_forward(X)
+ return np.argmax(probabilities, axis=1)
+
+
+
+# ensure the same random numbers appear every time
+np.random.seed(0)
+
+
+# Defining the neural network
+n_inputs, n_features = X.shape
+n_hidden_neurons = 2
+n_categories = 1
+n_features = 2
+
+# we make the weights normally distributed using numpy.random.randn
+
+# weights and bias in the hidden layer
+hidden_weights = np.random.randn(n_features, n_hidden_neurons)
+hidden_bias = np.zeros(n_hidden_neurons) + 0.01
+
+# weights and bias in the output layer
+output_weights = np.random.randn(n_hidden_neurons, n_categories)
+output_bias = np.zeros(n_categories) + 0.01
+
+probabilities = feed_forward(X)
+print(probabilities)
+
+
+predictions = predict(X)
+print(predictions)
++Not an impressive result. Let us now add the full network with the back-propagation algorithm discussed above. +
+ + +
"""
Simple code that tests XOR, OR and AND gates with linear regression
"""
-
+# import necessary packages
import numpy as np
+import matplotlib.pyplot as plt
+from sklearn import datasets
+
# Design matrix
X = np.array([ [1, 0, 0], [1, 0, 1], [1, 1, 0],[1, 1, 1]],dtype=np.float64)
@@ -1575,9 +1586,76 @@ yXOR = np.array( [ 0, 0, 1 ,1, 1])
# The AND gate
yAND = np.array( [ 0, 0 ,0, 1])
+
+
-#print(f"The values of theta for the AND gate:{ThetaAND}")
-#print(f"The linear regression prediction for the AND gate:{X @ ThetaAND}")
+
+ + +
def sigmoid(x):
+ return 1/(1 + np.exp(-x))
+
+def feed_forward(X):
+ # weighted sum of inputs to the hidden layer
+ z_h = np.matmul(X, hidden_weights) + hidden_bias
+ # activation in the hidden layer
+ a_h = sigmoid(z_h)
+
+ # weighted sum of inputs to the output layer
+ z_o = np.matmul(a_h, output_weights) + output_bias
+ # softmax output
+ # axis 0 holds each input and axis 1 the probabilities of each category
+ probabilities = sigmoid(z_o)
+ return probabilities
+
+# we obtain a prediction by taking the class with the highest likelihood
+def predict(X):
+ probabilities = feed_forward(X)
+ return np.argmax(probabilities, axis=1)
+
+
+
+# ensure the same random numbers appear every time
+np.random.seed(0)
+
+
+# Defining the neural network
+n_inputs, n_features = X.shape
+n_hidden_neurons = 2
+n_categories = 1
+n_features = 2
+
+# we make the weights normally distributed using numpy.random.randn
+
+# weights and bias in the hidden layer
+hidden_weights = np.random.randn(n_features, n_hidden_neurons)
+hidden_bias = np.zeros(n_hidden_neurons) + 0.01
+
+# weights and bias in the output layer
+output_weights = np.random.randn(n_hidden_neurons, n_categories)
+output_bias = np.zeros(n_categories) + 0.01
+
+probabilities = feed_forward(X)
+print(probabilities)
+
+
+predictions = predict(X)
+print(predictions)
++Not an impressive result. Let us now add the full network with the back-propagation algorithm discussed above. + +
+
+
+
+ + +
diff --git a/doc/pub/week41/html/week41.html b/doc/pub/week41/html/week41.html
index c152409d4..9b52c171f 100644
--- a/doc/pub/week41/html/week41.html
+++ b/doc/pub/week41/html/week41.html
@@ -145,6 +145,14 @@ div { text-align: justify; text-justify: inter-word; }
2,
None,
'setting-up-the-neural-network'),
+ ('Then the first Feed Forward pass',
+ 2,
+ None,
+ 'then-the-first-feed-forward-pass'),
+ ('The full Network for the Various Gates',
+ 2,
+ None,
+ 'the-full-network-for-the-various-gates'),
('Building neural networks in Tensorflow and Keras',
2,
None,
@@ -1569,8 +1577,11 @@ We define first our design matrix and the various input vectors.
"""
Simple code that tests XOR, OR and AND gates with linear regression
"""
-
+# import necessary packages
import numpy as np
+import matplotlib.pyplot as plt
+from sklearn import datasets
+
# Design matrix
X = np.array([ [1, 0, 0], [1, 0, 1], [1, 1, 0],[1, 1, 1]],dtype=np.float64)
@@ -1580,9 +1591,76 @@ yXOR = np.= np.array( [ 0, 1 ,1, 1])
# The AND gate
yAND = np.array( [ 0, 0 ,0, 1])
+
+
-#print(f"The values of theta for the AND gate:{ThetaAND}")
-#print(f"The linear regression prediction for the AND gate:{X @ ThetaAND}")
+
+ + +
def sigmoid(x):
+ return 1/(1 + np.exp(-x))
+
+def feed_forward(X):
+ # weighted sum of inputs to the hidden layer
+ z_h = np.matmul(X, hidden_weights) + hidden_bias
+ # activation in the hidden layer
+ a_h = sigmoid(z_h)
+
+ # weighted sum of inputs to the output layer
+ z_o = np.matmul(a_h, output_weights) + output_bias
+ # softmax output
+ # axis 0 holds each input and axis 1 the probabilities of each category
+ probabilities = sigmoid(z_o)
+ return probabilities
+
+# we obtain a prediction by taking the class with the highest likelihood
+def predict(X):
+ probabilities = feed_forward(X)
+ return np.argmax(probabilities, axis=1)
+
+
+
+# ensure the same random numbers appear every time
+np.random.seed(0)
+
+
+# Defining the neural network
+n_inputs, n_features = X.shape
+n_hidden_neurons = 2
+n_categories = 1
+n_features = 2
+
+# we make the weights normally distributed using numpy.random.randn
+
+# weights and bias in the hidden layer
+hidden_weights = np.random.randn(n_features, n_hidden_neurons)
+hidden_bias = np.zeros(n_hidden_neurons) + 0.01
+
+# weights and bias in the output layer
+output_weights = np.random.randn(n_hidden_neurons, n_categories)
+output_bias = np.zeros(n_categories) + 0.01
+
+probabilities = feed_forward(X)
+print(probabilities)
+
+
+predictions = predict(X)
+print(predictions)
++Not an impressive result. Let us now add the full network with the back-propagation algorithm discussed above. + +
+
+
+
+ + +
diff --git a/doc/pub/week41/ipynb/ipynb-week41-src.tar.gz b/doc/pub/week41/ipynb/ipynb-week41-src.tar.gz
index f66080706..18192ecbc 100644
Binary files a/doc/pub/week41/ipynb/ipynb-week41-src.tar.gz and b/doc/pub/week41/ipynb/ipynb-week41-src.tar.gz differ
diff --git a/doc/pub/week41/ipynb/week41.ipynb b/doc/pub/week41/ipynb/week41.ipynb
index 04023feac..af4c3a2bd 100644
--- a/doc/pub/week41/ipynb/week41.ipynb
+++ b/doc/pub/week41/ipynb/week41.ipynb
@@ -1473,8 +1473,11 @@
"\"\"\"\n",
"Simple code that tests XOR, OR and AND gates with linear regression\n",
"\"\"\"\n",
- "\n",
+ "# import necessary packages\n",
"import numpy as np\n",
+ "import matplotlib.pyplot as plt\n",
+ "from sklearn import datasets\n",
+ "\n",
"# Design matrix\n",
"X = np.array([ [1, 0, 0], [1, 0, 1], [1, 1, 0],[1, 1, 1]],dtype=np.float64)\n",
"\n",
@@ -1483,10 +1486,85 @@
"# The OR gate \n",
"yOR = np.array( [ 0, 1 ,1, 1])\n",
"# The AND gate \n",
- "yAND = np.array( [ 0, 0 ,0, 1])\n",
+ "yAND = np.array( [ 0, 0 ,0, 1])"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Then the first Feed Forward pass"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "collapsed": false,
+ "editable": true
+ },
+ "outputs": [],
+ "source": [
"\n",
- "#print(f\"The values of theta for the AND gate:{ThetaAND}\")\n",
- "#print(f\"The linear regression prediction for the AND gate:{X @ ThetaAND}\")"
+ "\n",
+ "def sigmoid(x):\n",
+ " return 1/(1 + np.exp(-x))\n",
+ "\n",
+ "def feed_forward(X):\n",
+ " # weighted sum of inputs to the hidden layer\n",
+ " z_h = np.matmul(X, hidden_weights) + hidden_bias\n",
+ " # activation in the hidden layer\n",
+ " a_h = sigmoid(z_h)\n",
+ " \n",
+ " # weighted sum of inputs to the output layer\n",
+ " z_o = np.matmul(a_h, output_weights) + output_bias\n",
+ " # softmax output\n",
+ " # axis 0 holds each input and axis 1 the probabilities of each category\n",
+ " probabilities = sigmoid(z_o)\n",
+ " return probabilities\n",
+ "\n",
+ "# we obtain a prediction by taking the class with the highest likelihood\n",
+ "def predict(X):\n",
+ " probabilities = feed_forward(X)\n",
+ " return np.argmax(probabilities, axis=1)\n",
+ "\n",
+ "\n",
+ "\n",
+ "# ensure the same random numbers appear every time\n",
+ "np.random.seed(0)\n",
+ "\n",
+ "\n",
+ "# Defining the neural network\n",
+ "n_inputs, n_features = X.shape\n",
+ "n_hidden_neurons = 2\n",
+ "n_categories = 1\n",
+ "n_features = 2\n",
+ "\n",
+ "# we make the weights normally distributed using numpy.random.randn\n",
+ "\n",
+ "# weights and bias in the hidden layer\n",
+ "hidden_weights = np.random.randn(n_features, n_hidden_neurons)\n",
+ "hidden_bias = np.zeros(n_hidden_neurons) + 0.01\n",
+ "\n",
+ "# weights and bias in the output layer\n",
+ "output_weights = np.random.randn(n_hidden_neurons, n_categories)\n",
+ "output_bias = np.zeros(n_categories) + 0.01\n",
+ "\n",
+ "probabilities = feed_forward(X)\n",
+ "print(probabilities)\n",
+ "\n",
+ "\n",
+ "predictions = predict(X)\n",
+ "print(predictions)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Not an impressive result. Let us now add the full network with the back-propagation algorithm discussed above.\n",
+ "\n",
+ "## The full Network for the Various Gates"
]
},
{
diff --git a/doc/src/week41/nn.py b/doc/src/week41/nn.py
new file mode 100644
index 000000000..cee3150f3
--- /dev/null
+++ b/doc/src/week41/nn.py
@@ -0,0 +1,67 @@
+"""
+Simple code that tests XOR, OR and AND gates with linear regression
+"""
+
+# import necessary packages
+import numpy as np
+import matplotlib.pyplot as plt
+from sklearn import datasets
+
+def sigmoid(x):
+ return 1/(1 + np.exp(-x))
+
+def feed_forward(X):
+ # weighted sum of inputs to the hidden layer
+ z_h = np.matmul(X, hidden_weights) + hidden_bias
+ # activation in the hidden layer
+ a_h = sigmoid(z_h)
+
+ # weighted sum of inputs to the output layer
+ z_o = np.matmul(a_h, output_weights) + output_bias
+ # softmax output
+ # axis 0 holds each input and axis 1 the probabilities of each category
+ probabilities = sigmoid(z_o)
+ return probabilities
+
+# we obtain a prediction by taking the class with the highest likelihood
+def predict(X):
+ probabilities = feed_forward(X)
+ return np.argmax(probabilities, axis=1)
+
+
+
+# ensure the same random numbers appear every time
+np.random.seed(0)
+
+# Design matrix
+X = np.array([ [0, 0], [0, 1], [1, 0],[1, 1]],dtype=np.float64)
+
+# The XOR gate
+yXOR = np.array( [ 0, 1 ,1, 0])
+# The OR gate
+yOR = np.array( [ 0, 1 ,1, 1])
+# The AND gate
+yAND = np.array( [ 0, 0 ,0, 1])
+
+# Defining the neural network
+n_inputs, n_features = X.shape
+n_hidden_neurons = 2
+n_categories = 1
+n_features = 2
+
+# we make the weights normally distributed using numpy.random.randn
+
+# weights and bias in the hidden layer
+hidden_weights = np.random.randn(n_features, n_hidden_neurons)
+hidden_bias = np.zeros(n_hidden_neurons) + 0.01
+
+# weights and bias in the output layer
+output_weights = np.random.randn(n_hidden_neurons, n_categories)
+output_bias = np.zeros(n_categories) + 0.01
+
+probabilities = feed_forward(X)
+print(probabilities)
+
+
+predictions = predict(X)
+print(predictions)
diff --git a/doc/src/week41/week41.do.txt b/doc/src/week41/week41.do.txt
index 0830ab643..442c8902a 100644
--- a/doc/src/week41/week41.do.txt
+++ b/doc/src/week41/week41.do.txt
@@ -1131,8 +1131,11 @@ We define first our design matrix and the various input vectors.
"""
Simple code that tests XOR, OR and AND gates with linear regression
"""
-
+# import necessary packages
import numpy as np
+import matplotlib.pyplot as plt
+from sklearn import datasets
+
# Design matrix
X = np.array([ [1, 0, 0], [1, 0, 1], [1, 1, 0],[1, 1, 1]],dtype=np.float64)
@@ -1142,12 +1145,73 @@ yXOR = np.array( [ 0, 1 ,1, 0])
yOR = np.array( [ 0, 1 ,1, 1])
# The AND gate
yAND = np.array( [ 0, 0 ,0, 1])
-
-#print(f"The values of theta for the AND gate:{ThetaAND}")
-#print(f"The linear regression prediction for the AND gate:{X @ ThetaAND}")
!ec
+!split
+===== Then the first Feed Forward pass =====
+
+!bc pycod
+
+
+def sigmoid(x):
+ return 1/(1 + np.exp(-x))
+
+def feed_forward(X):
+ # weighted sum of inputs to the hidden layer
+ z_h = np.matmul(X, hidden_weights) + hidden_bias
+ # activation in the hidden layer
+ a_h = sigmoid(z_h)
+
+ # weighted sum of inputs to the output layer
+ z_o = np.matmul(a_h, output_weights) + output_bias
+ # softmax output
+ # axis 0 holds each input and axis 1 the probabilities of each category
+ probabilities = sigmoid(z_o)
+ return probabilities
+
+# we obtain a prediction by taking the class with the highest likelihood
+def predict(X):
+ probabilities = feed_forward(X)
+ return np.argmax(probabilities, axis=1)
+
+
+
+# ensure the same random numbers appear every time
+np.random.seed(0)
+
+
+# Defining the neural network
+n_inputs, n_features = X.shape
+n_hidden_neurons = 2
+n_categories = 1
+n_features = 2
+
+# we make the weights normally distributed using numpy.random.randn
+
+# weights and bias in the hidden layer
+hidden_weights = np.random.randn(n_features, n_hidden_neurons)
+hidden_bias = np.zeros(n_hidden_neurons) + 0.01
+
+# weights and bias in the output layer
+output_weights = np.random.randn(n_hidden_neurons, n_categories)
+output_bias = np.zeros(n_categories) + 0.01
+
+probabilities = feed_forward(X)
+print(probabilities)
+
+
+predictions = predict(X)
+print(predictions)
+!ec
+Not an impressive result. Let us now add the full network with the back-propagation algorithm discussed above.
+
+!split
+===== The full Network for the Various Gates =====
+!bc pycod
+
+!ec
+
!split
===== Building neural networks in Tensorflow and Keras =====