update week 42
This commit is contained in:
Morten Hjorth-Jensen
+12
-186
@@ -6,12 +6,15 @@ DATE: October 13-17, 2025
|
||||
===== Lecture October 13, 2025 =====
|
||||
!bblock
|
||||
o Building our own Feed-forward Neural Network and discussion of project 2
|
||||
o Project 2 is available at URL:"https://github.com/CompPhysics/MachineLearning/blob/master/doc/Projects/2025/Project2/ipynb/Project2.ipynb"
|
||||
!eblock
|
||||
!bblock Readings and videos
|
||||
!split
|
||||
===== Readings and videos =====
|
||||
!bblock
|
||||
o These lecture notes
|
||||
#o "Video of lecture":"https://youtu.be/7B2F35gNj2Y"
|
||||
#o "Whiteboard notes":"https://github.com/CompPhysics/MachineLearning/blob/master/doc/HandWrittenNotes/2024/NotesOct14.pdf"
|
||||
o For a more in depth discussion on neural networks we recommend Goodfellow et al chapters 6 and 7.
|
||||
o For a more in depth discussion on neural networks we recommend Goodfellow et al chapters 6 and 7. For the optimization part, see chapter 8.
|
||||
o Neural Networks demystified at URL:"https://www.youtube.com/watch?v=bxe2T-V8XRs&list=PLiaHhY2iBX9hdHaRr6b7XevZtgZRa1PoU&ab_channel=WelchLabs"
|
||||
o Building Neural Networks from scratch at URL:"https://www.youtube.com/watch?v=Wo5dMEP_BbI&list=PLQVvvaa0QuDcjD5BAw2DxE6OF2tius3V3&ab_channel=sentdex"
|
||||
o Video on Neural Networks at URL:"https://www.youtube.com/watch?v=CqOfi41LfDw"
|
||||
@@ -20,17 +23,15 @@ I also recommend Michael Nielsen's intuitive approach to the neural networks an
|
||||
!eblock
|
||||
|
||||
!split
|
||||
===== Material for the active learning sessions on Tuesday and Wednesday =====
|
||||
===== Material for the lab sessions on Tuesday and Wednesday =====
|
||||
!bblock
|
||||
* Exercise on starting to write a code for neural networks, feed forward part. We will also continue ur discussions of gradient descent methods from last week. If you have time, start considering the back-propagation part as well (exercises for next week)
|
||||
* Discussion of project 2
|
||||
o Exercises on writing a code for neural networks, back propagation part, see exercises for week 42 at URL:"https://compphysics.github.io/MachineLearning/doc/LectureNotes/_build/html/exercisesweek42.html"
|
||||
o Discussion of project 2
|
||||
!eblock
|
||||
|
||||
_Note_: some of the codes will also be discussed next week in connection with the solution of differential equations.
|
||||
|
||||
|
||||
!split
|
||||
===== Writing a code which implements a feed-forward neural network =====
|
||||
===== Lecture material: Writing a code which implements a feed-forward neural network =====
|
||||
|
||||
Last week we discussed the basics of neural networks and deep learning
|
||||
and the basics of automatic differentiation. We looked also at
|
||||
@@ -40,7 +41,7 @@ inputs and ouputs and no or just one hidden layers.
|
||||
|
||||
We ended our discussions with the derivation of the equations for a
|
||||
neural network with one hidden layers and two input variables and two
|
||||
hidden nodes but only one output node.
|
||||
hidden nodes but only one output node. We did almost finish the derivation of the back propagation algorithm.
|
||||
|
||||
|
||||
!split
|
||||
@@ -69,7 +70,7 @@ o Goodfellow et al, chapter 6 and 7 contain most of the neural network backgroun
|
||||
|
||||
|
||||
!split
|
||||
===== First network example, simple percepetron with one input =====
|
||||
===== Reminder from last week: First network example, simple percepetron with one input =====
|
||||
|
||||
As yet another example we define now a simple perceptron model with
|
||||
all quantities given by scalars. We consider only one input variable
|
||||
@@ -565,7 +566,7 @@ all inputs $\bm{x}$ are given by $\bm{\tilde{y}}_i$.
|
||||
|
||||
|
||||
!split
|
||||
===== Layout of a neural network with three hidden layers (last later = $l=L=4$, first layer $l=0$) =====
|
||||
===== Layout of a neural network with three hidden layers (last layer = $l=L=4$, first layer $l=0$) =====
|
||||
|
||||
FIGURE: [figures/nn2.pdf, width=900 frac=1.0]
|
||||
|
||||
@@ -2574,181 +2575,6 @@ plt.show()
|
||||
|
||||
|
||||
|
||||
!split
|
||||
===== The Breast Cancer Data, now with Keras =====
|
||||
|
||||
!bc pycod
|
||||
|
||||
import tensorflow as tf
|
||||
from tensorflow.keras.layers import Input
|
||||
from tensorflow.keras.models import Sequential #This allows appending layers to existing models
|
||||
from tensorflow.keras.layers import Dense #This allows defining the characteristics of a particular layer
|
||||
from tensorflow.keras import optimizers #This allows using whichever optimiser we want (sgd,adam,RMSprop)
|
||||
from tensorflow.keras import regularizers #This allows using whichever regularizer we want (l1,l2,l1_l2)
|
||||
from tensorflow.keras.utils import to_categorical #This allows using categorical cross entropy as the cost function
|
||||
import numpy as np
|
||||
import matplotlib.pyplot as plt
|
||||
import seaborn as sns
|
||||
from sklearn.model_selection import train_test_split as splitter
|
||||
from sklearn.datasets import load_breast_cancer
|
||||
import pickle
|
||||
import os
|
||||
|
||||
|
||||
"""Load breast cancer dataset"""
|
||||
|
||||
np.random.seed(0) #create same seed for random number every time
|
||||
|
||||
cancer=load_breast_cancer() #Download breast cancer dataset
|
||||
|
||||
inputs=cancer.data #Feature matrix of 569 rows (samples) and 30 columns (parameters)
|
||||
outputs=cancer.target #Label array of 569 rows (0 for benign and 1 for malignant)
|
||||
labels=cancer.feature_names[0:30]
|
||||
|
||||
print('The content of the breast cancer dataset is:') #Print information about the datasets
|
||||
print(labels)
|
||||
print('-------------------------')
|
||||
print("inputs = " + str(inputs.shape))
|
||||
print("outputs = " + str(outputs.shape))
|
||||
print("labels = "+ str(labels.shape))
|
||||
|
||||
x=inputs #Reassign the Feature and Label matrices to other variables
|
||||
y=outputs
|
||||
|
||||
#%%
|
||||
|
||||
# Visualisation of dataset (for correlation analysis)
|
||||
|
||||
plt.figure()
|
||||
plt.scatter(x[:,0],x[:,2],s=40,c=y,cmap=plt.cm.Spectral)
|
||||
plt.xlabel('Mean radius',fontweight='bold')
|
||||
plt.ylabel('Mean perimeter',fontweight='bold')
|
||||
plt.show()
|
||||
|
||||
plt.figure()
|
||||
plt.scatter(x[:,5],x[:,6],s=40,c=y, cmap=plt.cm.Spectral)
|
||||
plt.xlabel('Mean compactness',fontweight='bold')
|
||||
plt.ylabel('Mean concavity',fontweight='bold')
|
||||
plt.show()
|
||||
|
||||
|
||||
plt.figure()
|
||||
plt.scatter(x[:,0],x[:,1],s=40,c=y,cmap=plt.cm.Spectral)
|
||||
plt.xlabel('Mean radius',fontweight='bold')
|
||||
plt.ylabel('Mean texture',fontweight='bold')
|
||||
plt.show()
|
||||
|
||||
plt.figure()
|
||||
plt.scatter(x[:,2],x[:,1],s=40,c=y,cmap=plt.cm.Spectral)
|
||||
plt.xlabel('Mean perimeter',fontweight='bold')
|
||||
plt.ylabel('Mean compactness',fontweight='bold')
|
||||
plt.show()
|
||||
|
||||
|
||||
# Generate training and testing datasets
|
||||
|
||||
#Select features relevant to classification (texture,perimeter,compactness and symmetery)
|
||||
#and add to input matrix
|
||||
|
||||
temp1=np.reshape(x[:,1],(len(x[:,1]),1))
|
||||
temp2=np.reshape(x[:,2],(len(x[:,2]),1))
|
||||
X=np.hstack((temp1,temp2))
|
||||
temp=np.reshape(x[:,5],(len(x[:,5]),1))
|
||||
X=np.hstack((X,temp))
|
||||
temp=np.reshape(x[:,8],(len(x[:,8]),1))
|
||||
X=np.hstack((X,temp))
|
||||
|
||||
X_train,X_test,y_train,y_test=splitter(X,y,test_size=0.1) #Split datasets into training and testing
|
||||
|
||||
y_train=to_categorical(y_train) #Convert labels to categorical when using categorical cross entropy
|
||||
y_test=to_categorical(y_test)
|
||||
|
||||
del temp1,temp2,temp
|
||||
|
||||
# %%
|
||||
|
||||
# Define tunable parameters"
|
||||
|
||||
eta=np.logspace(-3,-1,3) #Define vector of learning rates (parameter to SGD optimiser)
|
||||
lamda=0.01 #Define hyperparameter
|
||||
n_layers=2 #Define number of hidden layers in the model
|
||||
n_neuron=np.logspace(0,3,4,dtype=int) #Define number of neurons per layer
|
||||
epochs=100 #Number of reiterations over the input data
|
||||
batch_size=100 #Number of samples per gradient update
|
||||
|
||||
# %%
|
||||
|
||||
"""Define function to return Deep Neural Network model"""
|
||||
|
||||
def NN_model(inputsize,n_layers,n_neuron,eta,lamda):
|
||||
model=Sequential()
|
||||
for i in range(n_layers): #Run loop to add hidden layers to the model
|
||||
if (i==0): #First layer requires input dimensions
|
||||
model.add(Dense(n_neuron,activation='relu',kernel_regularizer=regularizers.l2(lamda),input_dim=inputsize))
|
||||
else: #Subsequent layers are capable of automatic shape inferencing
|
||||
model.add(Dense(n_neuron,activation='relu',kernel_regularizer=regularizers.l2(lamda)))
|
||||
model.add(Dense(2,activation='softmax')) #2 outputs - ordered and disordered (softmax for prob)
|
||||
sgd=optimizers.SGD(lr=eta)
|
||||
model.compile(loss='categorical_crossentropy',optimizer=sgd,metrics=['accuracy'])
|
||||
return model
|
||||
|
||||
|
||||
Train_accuracy=np.zeros((len(n_neuron),len(eta))) #Define matrices to store accuracy scores as a function
|
||||
Test_accuracy=np.zeros((len(n_neuron),len(eta))) #of learning rate and number of hidden neurons for
|
||||
|
||||
for i in range(len(n_neuron)): #run loops over hidden neurons and learning rates to calculate
|
||||
for j in range(len(eta)): #accuracy scores
|
||||
DNN_model=NN_model(X_train.shape[1],n_layers,n_neuron[i],eta[j],lamda)
|
||||
DNN_model.fit(X_train,y_train,epochs=epochs,batch_size=batch_size,verbose=1)
|
||||
Train_accuracy[i,j]=DNN_model.evaluate(X_train,y_train)[1]
|
||||
Test_accuracy[i,j]=DNN_model.evaluate(X_test,y_test)[1]
|
||||
|
||||
|
||||
def plot_data(x,y,data,title=None):
|
||||
|
||||
# plot results
|
||||
fontsize=16
|
||||
|
||||
|
||||
fig = plt.figure()
|
||||
ax = fig.add_subplot(111)
|
||||
cax = ax.matshow(data, interpolation='nearest', vmin=0, vmax=1)
|
||||
|
||||
cbar=fig.colorbar(cax)
|
||||
cbar.ax.set_ylabel('accuracy (%)',rotation=90,fontsize=fontsize)
|
||||
cbar.set_ticks([0,.2,.4,0.6,0.8,1.0])
|
||||
cbar.set_ticklabels(['0%','20%','40%','60%','80%','100%'])
|
||||
|
||||
# put text on matrix elements
|
||||
for i, x_val in enumerate(np.arange(len(x))):
|
||||
for j, y_val in enumerate(np.arange(len(y))):
|
||||
c = "${0:.1f}\\%$".format( 100*data[j,i])
|
||||
ax.text(x_val, y_val, c, va='center', ha='center')
|
||||
|
||||
# convert axis vaues to to string labels
|
||||
x=[str(i) for i in x]
|
||||
y=[str(i) for i in y]
|
||||
|
||||
|
||||
ax.set_xticklabels(['']+x)
|
||||
ax.set_yticklabels(['']+y)
|
||||
|
||||
ax.set_xlabel('$\\mathrm{learning\\ rate}$',fontsize=fontsize)
|
||||
ax.set_ylabel('$\\mathrm{hidden\\ neurons}$',fontsize=fontsize)
|
||||
if title is not None:
|
||||
ax.set_title(title)
|
||||
|
||||
plt.tight_layout()
|
||||
|
||||
plt.show()
|
||||
|
||||
plot_data(eta,n_neuron,Train_accuracy, 'training')
|
||||
plot_data(eta,n_neuron,Test_accuracy, 'testing')
|
||||
|
||||
!ec
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
!split
|
||||
|
||||
Reference in New Issue
Block a user