updating week 35
This commit is contained in:
Morten Hjorth-Jensen
@@ -147,11 +147,18 @@ Automatically generated HTML file from DocOnce source
|
||||
None,
|
||||
'example-of-own-standard-scaling'),
|
||||
('Min-Max Scaling', 2, None, 'min-max-scaling'),
|
||||
('Simple preprocessing examples, Franke function and regression',
|
||||
('Testing the Means Squared Error as function of Complexity',
|
||||
2,
|
||||
None,
|
||||
'simple-preprocessing-examples-franke-function-and-regression'),
|
||||
('Friday September 3', 2, None, 'friday-september-3'),
|
||||
'testing-the-means-squared-error-as-function-of-complexity'),
|
||||
('More preprocessing examples, Franke function and regression',
|
||||
2,
|
||||
None,
|
||||
'more-preprocessing-examples-franke-function-and-regression'),
|
||||
('Mathematical Interpretation of Ordinary Least Squares',
|
||||
2,
|
||||
None,
|
||||
'mathematical-interpretation-of-ordinary-least-squares'),
|
||||
('The singular value decomposition',
|
||||
2,
|
||||
None,
|
||||
@@ -169,7 +176,7 @@ Automatically generated HTML file from DocOnce source
|
||||
('Economy-size SVD', 2, None, 'economy-size-svd'),
|
||||
('Codes for the SVD', 2, None, 'codes-for-the-svd'),
|
||||
('Mathematical Properties', 2, None, 'mathematical-properties'),
|
||||
('Friday September 12', 2, None, 'friday-september-12'),
|
||||
('Friday September 3', 2, None, 'friday-september-3'),
|
||||
('Ridge and LASSO Regression',
|
||||
2,
|
||||
None,
|
||||
@@ -213,7 +220,19 @@ Automatically generated HTML file from DocOnce source
|
||||
2,
|
||||
None,
|
||||
'rewriting-the-covariance-and-or-correlation-matrix'),
|
||||
('Linking with SVD', 2, None, 'linking-with-svd')]}
|
||||
('Linking with SVD', 2, None, 'linking-with-svd'),
|
||||
('Exercises for week 37, September 6-10',
|
||||
2,
|
||||
None,
|
||||
'exercises-for-week-37-september-6-10'),
|
||||
('Exercise 1: Adding Ridge and Lasso Regression',
|
||||
2,
|
||||
None,
|
||||
'exercise-1-adding-ridge-and-lasso-regression'),
|
||||
('Exercise: Linear Regression for a two-dimensional function',
|
||||
3,
|
||||
None,
|
||||
'exercise-linear-regression-for-a-two-dimensional-function')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -251,63 +270,67 @@ MathJax.Hub.Config({
|
||||
<li class="dropdown">
|
||||
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
|
||||
<ul class="dropdown-menu">
|
||||
<!-- navigation toc: --> <li><a href="._week35-bs001.html#plans-for-week-35-august-24-28" style="font-size: 80%;">Plans for week 35, August 24-28</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week35-bs002.html#thursday-august-27" style="font-size: 80%;">Thursday August 27</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week35-bs003.html#why-linear-regression-aka-ordinary-least-squares-and-family-repeat-from-last-week" style="font-size: 80%;">Why Linear Regression (aka Ordinary Least Squares and family), repeat from last week</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week35-bs004.html#regression-analysis-overarching-aims" style="font-size: 80%;">Regression analysis, overarching aims</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week35-bs005.html#regression-analysis-overarching-aims-ii" style="font-size: 80%;">Regression analysis, overarching aims II</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week35-bs022.html#examples" style="font-size: 80%;">Examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week35-bs007.html#general-linear-models" style="font-size: 80%;">General linear models</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week35-bs008.html#rewriting-the-fitting-procedure-as-a-linear-algebra-problem" style="font-size: 80%;">Rewriting the fitting procedure as a linear algebra problem</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week35-bs009.html#rewriting-the-fitting-procedure-as-a-linear-algebra-problem-more-details" style="font-size: 80%;">Rewriting the fitting procedure as a linear algebra problem, more details</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week35-bs011.html#generalizing-the-fitting-procedure-as-a-linear-algebra-problem" style="font-size: 80%;">Generalizing the fitting procedure as a linear algebra problem</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week35-bs011.html#generalizing-the-fitting-procedure-as-a-linear-algebra-problem" style="font-size: 80%;">Generalizing the fitting procedure as a linear algebra problem</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week35-bs012.html#optimizing-our-parameters" style="font-size: 80%;">Optimizing our parameters</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week35-bs013.html#our-model-for-the-nuclear-binding-energies" style="font-size: 80%;">Our model for the nuclear binding energies</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week35-bs014.html#optimizing-our-parameters-more-details" style="font-size: 80%;">Optimizing our parameters, more details</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week35-bs018.html#interpretations-and-optimizing-our-parameters" style="font-size: 80%;">Interpretations and optimizing our parameters</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week35-bs018.html#interpretations-and-optimizing-our-parameters" style="font-size: 80%;">Interpretations and optimizing our parameters</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week35-bs017.html#some-useful-matrix-and-vector-expressions" style="font-size: 80%;">Some useful matrix and vector expressions</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week35-bs018.html#interpretations-and-optimizing-our-parameters" style="font-size: 80%;">Interpretations and optimizing our parameters</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week35-bs019.html#own-code-for-ordinary-least-squares" style="font-size: 80%;">Own code for Ordinary Least Squares</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week35-bs020.html#adding-error-analysis-and-training-set-up" style="font-size: 80%;">Adding error analysis and training set up</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week35-bs021.html#splitting-our-data-in-training-and-test-data" style="font-size: 80%;">Splitting our Data in Training and Test data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week35-bs022.html#examples" style="font-size: 80%;">Examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week35-bs023.html#making-your-own-test-train-splitting" style="font-size: 80%;">Making your own test-train splitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week35-bs024.html#the-boston-housing-data-example" style="font-size: 80%;">The Boston housing data example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week35-bs025.html#housing-data-the-code" style="font-size: 80%;">Housing data, the code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week35-bs026.html#reducing-the-number-of-degrees-of-freedom-overarching-view" style="font-size: 80%;">Reducing the number of degrees of freedom, overarching view</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week35-bs027.html#preprocessing-our-data" style="font-size: 80%;">Preprocessing our data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week35-bs028.html#functionality-in-scikit-learn" style="font-size: 80%;">Functionality in Scikit-Learn</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week35-bs029.html#more-preprocessing" style="font-size: 80%;">More preprocessing</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week35-bs030.html#frequently-used-scaling-functions" style="font-size: 80%;">Frequently used scaling functions</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week35-bs031.html#example-of-own-standard-scaling" style="font-size: 80%;">Example of own Standard scaling</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week35-bs032.html#min-max-scaling" style="font-size: 80%;">Min-Max Scaling</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week35-bs033.html#simple-preprocessing-examples-franke-function-and-regression" style="font-size: 80%;">Simple preprocessing examples, Franke function and regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week35-bs034.html#friday-september-3" style="font-size: 80%;">Friday September 3</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week35-bs035.html#the-singular-value-decomposition" style="font-size: 80%;">The singular value decomposition</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week35-bs036.html#linear-regression-problems" style="font-size: 80%;">Linear Regression Problems</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week35-bs037.html#fixing-the-singularity" style="font-size: 80%;">Fixing the singularity</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week35-bs038.html#basic-math-of-the-svd" style="font-size: 80%;">Basic math of the SVD</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week35-bs039.html#the-svd-a-fantastic-algorithm" style="font-size: 80%;">The SVD, a Fantastic Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week35-bs040.html#economy-size-svd" style="font-size: 80%;">Economy-size SVD</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week35-bs041.html#codes-for-the-svd" style="font-size: 80%;">Codes for the SVD</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week35-bs042.html#mathematical-properties" style="font-size: 80%;">Mathematical Properties</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week35-bs043.html#friday-september-12" style="font-size: 80%;">Friday September 12</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week35-bs044.html#ridge-and-lasso-regression" style="font-size: 80%;">Ridge and LASSO Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week35-bs045.html#more-on-ridge-regression" style="font-size: 80%;">More on Ridge Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week35-bs046.html#interpreting-the-ridge-results" style="font-size: 80%;">Interpreting the Ridge results</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week35-bs047.html#more-interpretations" style="font-size: 80%;">More interpretations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week35-bs048.html#a-better-understanding-of-regularization" style="font-size: 80%;">A better understanding of regularization</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week35-bs049.html#decomposing-the-ols-and-ridge-expressions" style="font-size: 80%;">Decomposing the OLS and Ridge expressions</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week35-bs050.html#introducing-the-covariance-and-correlation-functions" style="font-size: 80%;">Introducing the Covariance and Correlation functions</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week35-bs051.html#correlation-function-and-design-feature-matrix" style="font-size: 80%;">Correlation Function and Design/Feature Matrix</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week35-bs052.html#covariance-matrix-examples" style="font-size: 80%;">Covariance Matrix Examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week35-bs053.html#correlation-matrix" style="font-size: 80%;">Correlation Matrix</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week35-bs054.html#correlation-matrix-with-pandas" style="font-size: 80%;">Correlation Matrix with Pandas</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week35-bs055.html#correlation-matrix-with-pandas-and-the-franke-function" style="font-size: 80%;">Correlation Matrix with Pandas and the Franke function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week35-bs056.html#rewriting-the-covariance-and-or-correlation-matrix" style="font-size: 80%;">Rewriting the Covariance and/or Correlation Matrix</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week35-bs057.html#linking-with-svd" style="font-size: 80%;">Linking with SVD</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week35-bs001.html#plans-for-week-35-august-24-28" style="font-size: 80%;"><b>Plans for week 35, August 24-28</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week35-bs002.html#thursday-august-27" style="font-size: 80%;"><b>Thursday August 27</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week35-bs003.html#why-linear-regression-aka-ordinary-least-squares-and-family-repeat-from-last-week" style="font-size: 80%;"><b>Why Linear Regression (aka Ordinary Least Squares and family), repeat from last week</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week35-bs004.html#regression-analysis-overarching-aims" style="font-size: 80%;"><b>Regression analysis, overarching aims</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week35-bs005.html#regression-analysis-overarching-aims-ii" style="font-size: 80%;"><b>Regression analysis, overarching aims II</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week35-bs022.html#examples" style="font-size: 80%;"><b>Examples</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week35-bs007.html#general-linear-models" style="font-size: 80%;"><b>General linear models</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week35-bs008.html#rewriting-the-fitting-procedure-as-a-linear-algebra-problem" style="font-size: 80%;"><b>Rewriting the fitting procedure as a linear algebra problem</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week35-bs009.html#rewriting-the-fitting-procedure-as-a-linear-algebra-problem-more-details" style="font-size: 80%;"><b>Rewriting the fitting procedure as a linear algebra problem, more details</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week35-bs011.html#generalizing-the-fitting-procedure-as-a-linear-algebra-problem" style="font-size: 80%;"><b>Generalizing the fitting procedure as a linear algebra problem</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week35-bs011.html#generalizing-the-fitting-procedure-as-a-linear-algebra-problem" style="font-size: 80%;"><b>Generalizing the fitting procedure as a linear algebra problem</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week35-bs012.html#optimizing-our-parameters" style="font-size: 80%;"><b>Optimizing our parameters</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week35-bs013.html#our-model-for-the-nuclear-binding-energies" style="font-size: 80%;"><b>Our model for the nuclear binding energies</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week35-bs014.html#optimizing-our-parameters-more-details" style="font-size: 80%;"><b>Optimizing our parameters, more details</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week35-bs018.html#interpretations-and-optimizing-our-parameters" style="font-size: 80%;"><b>Interpretations and optimizing our parameters</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week35-bs018.html#interpretations-and-optimizing-our-parameters" style="font-size: 80%;"><b>Interpretations and optimizing our parameters</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week35-bs017.html#some-useful-matrix-and-vector-expressions" style="font-size: 80%;"><b>Some useful matrix and vector expressions</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week35-bs018.html#interpretations-and-optimizing-our-parameters" style="font-size: 80%;"><b>Interpretations and optimizing our parameters</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week35-bs019.html#own-code-for-ordinary-least-squares" style="font-size: 80%;"><b>Own code for Ordinary Least Squares</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week35-bs020.html#adding-error-analysis-and-training-set-up" style="font-size: 80%;"><b>Adding error analysis and training set up</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week35-bs021.html#splitting-our-data-in-training-and-test-data" style="font-size: 80%;"><b>Splitting our Data in Training and Test data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week35-bs022.html#examples" style="font-size: 80%;"><b>Examples</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week35-bs023.html#making-your-own-test-train-splitting" style="font-size: 80%;"><b>Making your own test-train splitting</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week35-bs024.html#the-boston-housing-data-example" style="font-size: 80%;"><b>The Boston housing data example</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week35-bs025.html#housing-data-the-code" style="font-size: 80%;"><b>Housing data, the code</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week35-bs026.html#reducing-the-number-of-degrees-of-freedom-overarching-view" style="font-size: 80%;"><b>Reducing the number of degrees of freedom, overarching view</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week35-bs027.html#preprocessing-our-data" style="font-size: 80%;"><b>Preprocessing our data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week35-bs028.html#functionality-in-scikit-learn" style="font-size: 80%;"><b>Functionality in Scikit-Learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week35-bs029.html#more-preprocessing" style="font-size: 80%;"><b>More preprocessing</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week35-bs030.html#frequently-used-scaling-functions" style="font-size: 80%;"><b>Frequently used scaling functions</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week35-bs031.html#example-of-own-standard-scaling" style="font-size: 80%;"><b>Example of own Standard scaling</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week35-bs032.html#min-max-scaling" style="font-size: 80%;"><b>Min-Max Scaling</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week35-bs033.html#testing-the-means-squared-error-as-function-of-complexity" style="font-size: 80%;"><b>Testing the Means Squared Error as function of Complexity</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week35-bs034.html#more-preprocessing-examples-franke-function-and-regression" style="font-size: 80%;"><b>More preprocessing examples, Franke function and regression</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week35-bs035.html#mathematical-interpretation-of-ordinary-least-squares" style="font-size: 80%;"><b>Mathematical Interpretation of Ordinary Least Squares</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week35-bs036.html#the-singular-value-decomposition" style="font-size: 80%;"><b>The singular value decomposition</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week35-bs037.html#linear-regression-problems" style="font-size: 80%;"><b>Linear Regression Problems</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week35-bs038.html#fixing-the-singularity" style="font-size: 80%;"><b>Fixing the singularity</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week35-bs039.html#basic-math-of-the-svd" style="font-size: 80%;"><b>Basic math of the SVD</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week35-bs040.html#the-svd-a-fantastic-algorithm" style="font-size: 80%;"><b>The SVD, a Fantastic Algorithm</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week35-bs041.html#economy-size-svd" style="font-size: 80%;"><b>Economy-size SVD</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week35-bs042.html#codes-for-the-svd" style="font-size: 80%;"><b>Codes for the SVD</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week35-bs043.html#mathematical-properties" style="font-size: 80%;"><b>Mathematical Properties</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week35-bs044.html#friday-september-3" style="font-size: 80%;"><b>Friday September 3</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week35-bs045.html#ridge-and-lasso-regression" style="font-size: 80%;"><b>Ridge and LASSO Regression</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week35-bs046.html#more-on-ridge-regression" style="font-size: 80%;"><b>More on Ridge Regression</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week35-bs047.html#interpreting-the-ridge-results" style="font-size: 80%;"><b>Interpreting the Ridge results</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week35-bs048.html#more-interpretations" style="font-size: 80%;"><b>More interpretations</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week35-bs049.html#a-better-understanding-of-regularization" style="font-size: 80%;"><b>A better understanding of regularization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week35-bs050.html#decomposing-the-ols-and-ridge-expressions" style="font-size: 80%;"><b>Decomposing the OLS and Ridge expressions</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week35-bs051.html#introducing-the-covariance-and-correlation-functions" style="font-size: 80%;"><b>Introducing the Covariance and Correlation functions</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week35-bs052.html#correlation-function-and-design-feature-matrix" style="font-size: 80%;"><b>Correlation Function and Design/Feature Matrix</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week35-bs053.html#covariance-matrix-examples" style="font-size: 80%;"><b>Covariance Matrix Examples</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week35-bs054.html#correlation-matrix" style="font-size: 80%;"><b>Correlation Matrix</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week35-bs055.html#correlation-matrix-with-pandas" style="font-size: 80%;"><b>Correlation Matrix with Pandas</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week35-bs056.html#correlation-matrix-with-pandas-and-the-franke-function" style="font-size: 80%;"><b>Correlation Matrix with Pandas and the Franke function</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week35-bs057.html#rewriting-the-covariance-and-or-correlation-matrix" style="font-size: 80%;"><b>Rewriting the Covariance and/or Correlation Matrix</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week35-bs058.html#linking-with-svd" style="font-size: 80%;"><b>Linking with SVD</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week35-bs059.html#exercises-for-week-37-september-6-10" style="font-size: 80%;"><b>Exercises for week 37, September 6-10</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week35-bs059.html#exercise-1-adding-ridge-and-lasso-regression" style="font-size: 80%;"><b>Exercise 1: Adding Ridge and Lasso Regression</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week35-bs059.html#exercise-linear-regression-for-a-two-dimensional-function" style="font-size: 80%;"> Exercise: Linear Regression for a two-dimensional function</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -366,7 +389,7 @@ MathJax.Hub.Config({
|
||||
<li><a href="._week35-bs008.html">9</a></li>
|
||||
<li><a href="._week35-bs009.html">10</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._week35-bs057.html">58</a></li>
|
||||
<li><a href="._week35-bs059.html">60</a></li>
|
||||
<li><a href="._week35-bs001.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -1318,12 +1318,15 @@ XPandas = pd.DataFrame(X)
|
||||
display(XPandas)
|
||||
<span style="color: #658b00">print</span>(XPandas.mean())
|
||||
<span style="color: #658b00">print</span>(XPandas.std())
|
||||
XPandas = XPandas -XPandas,mean()
|
||||
XPandas = (XPandas -XPandas.mean())
|
||||
display(XPandas)
|
||||
scaler = StandardScaler()
|
||||
Xscaled = scaler.transform(a)
|
||||
<span style="color: #658b00">print</span>(Xscaled)
|
||||
scaler = StandardScaler(with_std=<span style="color: #8B008B; font-weight: bold">False</span>)
|
||||
scaler.fit(X)
|
||||
Xscaled = scaler.transform(X)
|
||||
display(XPandas-Xscaled)
|
||||
</pre></div>
|
||||
<p>
|
||||
Small exercise: perform the standars scaling by including the standard deviation.
|
||||
</section>
|
||||
|
||||
|
||||
@@ -1347,7 +1350,73 @@ where \( \min(x_j) \) and \( \max(x_j) \) return the minimum and maximum value o
|
||||
|
||||
|
||||
<section>
|
||||
<h2 id="simple-preprocessing-examples-franke-function-and-regression">Simple preprocessing examples, Franke function and regression </h2>
|
||||
<h2 id="testing-the-means-squared-error-as-function-of-complexity">Testing the Means Squared Error as function of Complexity </h2>
|
||||
One of
|
||||
the aims is to reproduce Figure 2.11 of <a href="https://github.com/CompPhysics/MLErasmus/blob/master/doc/Textbooks/elementsstat.pdf" target="_blank">Hastie et al</a>.
|
||||
We will also use Ridge and Lasso regression.
|
||||
|
||||
<p>
|
||||
Our data is defined by \( x\in [-3,3] \) with a total of for example \( 100 \) data points.
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="highlight" style="background: #eeeedd"><pre style="font-size: 80%; line-height: 125%;"><span></span>np.random.seed()
|
||||
n = <span style="color: #B452CD">100</span>
|
||||
maxdegree = <span style="color: #B452CD">14</span>
|
||||
<span style="color: #228B22"># Make data set.</span>
|
||||
x = np.linspace(-<span style="color: #B452CD">3</span>, <span style="color: #B452CD">3</span>, n).reshape(-<span style="color: #B452CD">1</span>, <span style="color: #B452CD">1</span>)
|
||||
y = np.exp(-x**<span style="color: #B452CD">2</span>) + <span style="color: #B452CD">1.5</span> * np.exp(-(x-<span style="color: #B452CD">2</span>)**<span style="color: #B452CD">2</span>)+ np.random.normal(<span style="color: #B452CD">0</span>, <span style="color: #B452CD">0.1</span>, x.shape)
|
||||
</pre></div>
|
||||
<p>
|
||||
where \( y \) is the function we want to fit with a given polynomial.
|
||||
|
||||
<p>
|
||||
Write a first code which sets up a design matrix \( X \) defined by a fifth-order polynomial. Scale your data and split it in training and test data.
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="highlight" style="background: #eeeedd"><pre style="font-size: 80%; line-height: 125%;"><span></span><span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">matplotlib.pyplot</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">plt</span>
|
||||
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">numpy</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">np</span>
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.linear_model</span> <span style="color: #8B008B; font-weight: bold">import</span> LinearRegression, Ridge, Lasso
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.preprocessing</span> <span style="color: #8B008B; font-weight: bold">import</span> PolynomialFeatures
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.model_selection</span> <span style="color: #8B008B; font-weight: bold">import</span> train_test_split
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.pipeline</span> <span style="color: #8B008B; font-weight: bold">import</span> make_pipeline
|
||||
|
||||
|
||||
np.random.seed(<span style="color: #B452CD">2018</span>)
|
||||
n = <span style="color: #B452CD">50</span>
|
||||
maxdegree = <span style="color: #B452CD">5</span>
|
||||
<span style="color: #228B22"># Make data set.</span>
|
||||
x = np.linspace(-<span style="color: #B452CD">3</span>, <span style="color: #B452CD">3</span>, n).reshape(-<span style="color: #B452CD">1</span>, <span style="color: #B452CD">1</span>)
|
||||
y = np.exp(-x**<span style="color: #B452CD">2</span>) + <span style="color: #B452CD">1.5</span> * np.exp(-(x-<span style="color: #B452CD">2</span>)**<span style="color: #B452CD">2</span>)+ np.random.normal(<span style="color: #B452CD">0</span>, <span style="color: #B452CD">0.1</span>, x.shape)
|
||||
TestError = np.zeros(maxdegree)
|
||||
TrainError = np.zeros(maxdegree)
|
||||
polydegree = np.zeros(maxdegree)
|
||||
x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=<span style="color: #B452CD">0.2</span>)
|
||||
scaler = StandardScaler()
|
||||
scaler.fit(X_train)
|
||||
x_train_scaled = scaler.transform(x_train)
|
||||
x_test_scaled = scaler.transform(x_test)
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">for</span> degree <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(maxdegree):
|
||||
model = make_pipeline(PolynomialFeatures(degree=degree), LinearRegression(fit_intercept=<span style="color: #8B008B; font-weight: bold">False</span>))
|
||||
clf = model.fit(x_train_scale,y_train)
|
||||
y_fit = clf.predict(x_train_scaled)
|
||||
y_pred = clf.predict(x_test_scaled)
|
||||
polydegree[degree] = degree
|
||||
TestError[degree] = np.mean( np.mean((y_test - y_pred)**<span style="color: #B452CD">2</span>) )
|
||||
TrainError[degree] = np.mean( np.mean((y_train - y_fit)**<span style="color: #B452CD">2</span>) )
|
||||
|
||||
plt.plot(polydegree, TestError, label=<span style="color: #CD5555">'Test Error'</span>)
|
||||
plt.plot(polydegree, TrainError, label=<span style="color: #CD5555">'Train Error'</span>)
|
||||
plt.legend()
|
||||
plt.show()
|
||||
</pre></div>
|
||||
</section>
|
||||
|
||||
|
||||
<section>
|
||||
<h2 id="more-preprocessing-examples-franke-function-and-regression">More preprocessing examples, Franke function and regression </h2>
|
||||
|
||||
<p>
|
||||
|
||||
@@ -1449,10 +1518,10 @@ clf = skl.LinearRegression().fit(X_train_scaled, y_train)
|
||||
|
||||
|
||||
<section>
|
||||
<h2 id="friday-september-3">Friday September 3 </h2>
|
||||
<h2 id="mathematical-interpretation-of-ordinary-least-squares">Mathematical Interpretation of Ordinary Least Squares </h2>
|
||||
|
||||
<p>
|
||||
Lasso and Ridge regression
|
||||
What is presented here is a mathematical analysis of various regression algorithms (ordinary least squares, Ridge and Lasso Regression). The analysis is based on an important algorithm in linear algebra, the so-called Singular Value Decomposition (SVD).
|
||||
</section>
|
||||
|
||||
|
||||
@@ -1465,12 +1534,15 @@ Lasso and Ridge regression
|
||||
<p>
|
||||
The examples we have looked at so far are cases where we normally can
|
||||
invert the matrix \( \boldsymbol{X}^T\boldsymbol{X} \). Using a polynomial expansion as we
|
||||
did both for the masses and the fitting of the equation of state,
|
||||
leads to row vectors of the design matrix which are essentially
|
||||
orthogonal due to the polynomial character of our model. Obtaining the inverse of the design matrix is then often done via a so-called LU, QR or Cholesky decomposition.
|
||||
did both for the masses and the fitting of various functions leads to
|
||||
row vectors of the design matrix which are essentially orthogonal due
|
||||
to the polynomial character of our model. Obtaining the inverse of the
|
||||
design matrix is then often done via a so-called LU, QR or Cholesky
|
||||
decomposition.
|
||||
|
||||
<p>
|
||||
This may
|
||||
As we will also see in the first project,
|
||||
this may
|
||||
however not the be case in general and a standard matrix inversion
|
||||
algorithm based on say LU, QR or Cholesky decomposition may lead to singularities. We will see examples of this below.
|
||||
|
||||
@@ -1849,10 +1921,10 @@ $$ n > p$$
|
||||
|
||||
|
||||
<section>
|
||||
<h2 id="friday-september-12">Friday September 12 </h2>
|
||||
<h2 id="friday-september-3">Friday September 3 </h2>
|
||||
|
||||
<p>
|
||||
<a href="https://www.uio.no/studier/emner/matnat/fys/FYS-STK4155/h20/forelesningsvideoer/LectureSeptember11.mp4?vrtx=view-as-webpage" target="_blank">Video of Lecture</a> and <a href="https://github.com/CompPhysics/MachineLearning/blob/master/doc/HandWrittenNotes/NotesSeptember11.pdf" target="_blank">handwritten notes</a>
|
||||
<a href="https://www.uio.no/studier/emner/matnat/fys/FYS-STK4155/h20/forelesningsvideoer/LectureSeptember11.mp4?vrtx=view-as-webpage" target="_blank">Video of Lecture from 2020</a> and <a href="https://github.com/CompPhysics/MachineLearning/blob/master/doc/HandWrittenNotes/NotesSeptember11.pdf" target="_blank">handwritten notes</a>
|
||||
|
||||
<p>
|
||||
More material will be added here, see handwritten notes also.
|
||||
@@ -2492,6 +2564,334 @@ It is easy to generalize this to a matrix \( \boldsymbol{X}\in {\mathbb{R}}^{n\t
|
||||
|
||||
<section>
|
||||
<h2 id="linking-with-svd">Linking with SVD </h2>
|
||||
|
||||
<p>
|
||||
More material will be added here.
|
||||
</section>
|
||||
|
||||
|
||||
<section>
|
||||
<h2 id="exercises-for-week-37-september-6-10">Exercises for week 37, September 6-10 </h2>
|
||||
|
||||
<p>
|
||||
The exercises here are meant to prepare you for work with project 1. The first exercise is a follow-up of exercise 2 from week 36 August 30-September 3).
|
||||
|
||||
<p>
|
||||
<!-- --- begin exercise --- -->
|
||||
|
||||
<h2 id="exercise-1-adding-ridge-and-lasso-regression">Exercise 1: Adding Ridge and Lasso Regression </h2>
|
||||
|
||||
<p>
|
||||
This exercise is a continuation of exercise 2 from exercise set 1 (week 36, August 30-September 3)). We will
|
||||
use the same function to generate our data set, still staying with a
|
||||
simple function \( y(x) \) which we want to fit using linear regression,
|
||||
but now extending the analysis to include the Ridge and the Lasso
|
||||
regression methods.
|
||||
|
||||
<p>
|
||||
We will thus again generate our own dataset for a function \( y(x) \) where
|
||||
\( x \in [0,1] \) and defined by random numbers computed with the uniform
|
||||
distribution. The function \( y \) is a quadratic polynomial in \( x \) with
|
||||
added stochastic noise according to the normal distribution \( \cal{N}(0,1) \).
|
||||
|
||||
<p>
|
||||
The following simple Python instructions define our \( x \) and \( y \) values (with 100 data points).
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="highlight" style="background: #eeeedd"><pre style="font-size: 80%; line-height: 125%;"><span></span>x = np.random.rand(<span style="color: #B452CD">100</span>)
|
||||
y = <span style="color: #B452CD">2.0</span>+<span style="color: #B452CD">5</span>*x*x+<span style="color: #B452CD">0.1</span>*np.random.randn(<span style="color: #B452CD">100</span>)
|
||||
</pre></div>
|
||||
<p>
|
||||
Write your own code for the Ridge method (see chapter 3.4 of Hastie <em>et al.</em>, equations (3.43) and (3.44)) and compute the parametrization for different values of \( \lambda \). Compare and analyze your results with those from exercise 3. Study the dependence on \( \lambda \) while also varying the strength of the noise in your expression for \( y(x) \).
|
||||
|
||||
<p>
|
||||
The code here allows you to perform your own Ridge calculation and perform calculations for various values of the regularization parameter \( \lambda \). This program can easily be extended upon.
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="highlight" style="background: #eeeedd"><pre style="font-size: 80%; line-height: 125%;"><span></span><span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">os</span>
|
||||
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">numpy</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">np</span>
|
||||
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">pandas</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">pd</span>
|
||||
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">matplotlib.pyplot</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">plt</span>
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.model_selection</span> <span style="color: #8B008B; font-weight: bold">import</span> train_test_split
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.preprocessing</span> <span style="color: #8B008B; font-weight: bold">import</span> StandardScaler
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">R2</span>(y_data, y_model):
|
||||
<span style="color: #8B008B; font-weight: bold">return</span> <span style="color: #B452CD">1</span> - np.sum((y_data - y_model) ** <span style="color: #B452CD">2</span>) / np.sum((y_data - np.mean(y_data)) ** <span style="color: #B452CD">2</span>)
|
||||
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">MSE</span>(y_data,y_model):
|
||||
n = np.size(y_model)
|
||||
<span style="color: #8B008B; font-weight: bold">return</span> np.sum((y_data-y_model)**<span style="color: #B452CD">2</span>)/n
|
||||
|
||||
|
||||
<span style="color: #228B22"># A seed just to ensure that the random numbers are the same for every run.</span>
|
||||
<span style="color: #228B22"># Useful for eventual debugging.</span>
|
||||
np.random.seed(<span style="color: #B452CD">3155</span>)
|
||||
|
||||
x = np.random.rand(<span style="color: #B452CD">100</span>)
|
||||
y = <span style="color: #B452CD">2.0</span>+<span style="color: #B452CD">5</span>*x*x+<span style="color: #B452CD">0.1</span>*np.random.randn(<span style="color: #B452CD">100</span>)
|
||||
|
||||
<span style="color: #228B22"># number of features p (here degree of polynomial</span>
|
||||
p = <span style="color: #B452CD">3</span>
|
||||
<span style="color: #228B22"># The design matrix now as function of a given polynomial</span>
|
||||
X = np.zeros((<span style="color: #658b00">len</span>(x),p))
|
||||
X[:,<span style="color: #B452CD">0</span>] = <span style="color: #B452CD">1.0</span>
|
||||
X[:,<span style="color: #B452CD">1</span>] = x
|
||||
X[:,<span style="color: #B452CD">2</span>] = x*x
|
||||
<span style="color: #228B22"># We split the data in test and training data</span>
|
||||
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=<span style="color: #B452CD">0.2</span>)
|
||||
scaler = StandardScaler()
|
||||
scaler.fit(X_train)
|
||||
X_train_scaled = scaler.transform(X_train)
|
||||
X_test_scaled = scaler.transform(X_test)
|
||||
|
||||
<span style="color: #228B22"># matrix inversion to find beta</span>
|
||||
OLSbeta = np.linalg.inv(X_train.T @ X_train) @ X_train.T @ y_train
|
||||
<span style="color: #658b00">print</span>(OLSbeta)
|
||||
<span style="color: #228B22"># and then make the prediction</span>
|
||||
ytildeOLS = X_train @ OLSbeta
|
||||
<span style="color: #658b00">print</span>(<span style="color: #CD5555">"Training R2 for OLS"</span>)
|
||||
<span style="color: #658b00">print</span>(R2(y_train,ytildeOLS))
|
||||
<span style="color: #658b00">print</span>(<span style="color: #CD5555">"Training MSE for OLS"</span>)
|
||||
<span style="color: #658b00">print</span>(MSE(y_train,ytildeOLS))
|
||||
ypredictOLS = X_test @ OLSbeta
|
||||
<span style="color: #658b00">print</span>(<span style="color: #CD5555">"Test R2 for OLS"</span>)
|
||||
<span style="color: #658b00">print</span>(R2(y_test,ypredictOLS))
|
||||
<span style="color: #658b00">print</span>(<span style="color: #CD5555">"Test MSE OLS"</span>)
|
||||
<span style="color: #658b00">print</span>(MSE(y_test,ypredictOLS))
|
||||
|
||||
<span style="color: #228B22"># Repeat now for Ridge regression and various values of the regularization parameter</span>
|
||||
I = np.eye(p,p)
|
||||
<span style="color: #228B22"># Decide which values of lambda to use</span>
|
||||
nlambdas = <span style="color: #B452CD">20</span>
|
||||
MSEPredict = np.zeros(nlambdas)
|
||||
MSETrain = np.zeros(nlambdas)
|
||||
lambdas = np.logspace(-<span style="color: #B452CD">4</span>, <span style="color: #B452CD">1</span>, nlambdas)
|
||||
<span style="color: #8B008B; font-weight: bold">for</span> i <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(nlambdas):
|
||||
lmb = lambdas[i]
|
||||
Ridgebeta = np.linalg.inv(X_train.T @ X_train+lmb*I) @ X_train.T @ y_train
|
||||
<span style="color: #228B22"># and then make the prediction</span>
|
||||
ytildeRidge = X_train @ Ridgebeta
|
||||
ypredictRidge = X_test @ Ridgebeta
|
||||
MSEPredict[i] = MSE(y_test,ypredictRidge)
|
||||
MSETrain[i] = MSE(y_train,ytildeRidge)
|
||||
<span style="color: #228B22"># Now plot the results</span>
|
||||
plt.figure()
|
||||
plt.plot(np.log10(lambdas), MSETrain, label = <span style="color: #CD5555">'MSE Ridge train'</span>)
|
||||
plt.plot(np.log10(lambdas), MSEPredict, <span style="color: #CD5555">'r--'</span>, label = <span style="color: #CD5555">'MSE Ridge Test'</span>)
|
||||
plt.xlabel(<span style="color: #CD5555">'log10(lambda)'</span>)
|
||||
plt.ylabel(<span style="color: #CD5555">'MSE'</span>)
|
||||
plt.legend()
|
||||
plt.show()
|
||||
</pre></div>
|
||||
<p>
|
||||
Repeat the above but using the functionality of <b>Scikit-Learn</b>. Compare your code with the results from <b>Scikit-Learn</b>. Remember to run with the same random numbers for generating \( x \) and \( y \).
|
||||
|
||||
<p>
|
||||
Finally, using <b>Scikit-Learn</b> or your own code, compute also the mean square error, a risk metric corresponding to the expected value of the squared (quadratic) error defined as
|
||||
<p> <br>
|
||||
$$ MSE(\hat{y},\hat{\tilde{y}}) = \frac{1}{n}
|
||||
\sum_{i=0}^{n-1}(y_i-\tilde{y}_i)^2,
|
||||
$$
|
||||
<p> <br>
|
||||
|
||||
and the \( R^2 \) score function.
|
||||
If \( \tilde{\hat{y}}_i \) is the predicted value of the \( i-th \) sample and \( y_i \) is the corresponding true value, then the score \( R^2 \) is defined as
|
||||
<p> <br>
|
||||
$$
|
||||
R^2(\hat{y}, \tilde{\hat{y}}) = 1 - \frac{\sum_{i=0}^{n - 1} (y_i - \tilde{y}_i)^2}{\sum_{i=0}^{n - 1} (y_i - \bar{y})^2},
|
||||
$$
|
||||
<p> <br>
|
||||
|
||||
where we have defined the mean value of \( \hat{y} \) as
|
||||
<p> <br>
|
||||
$$
|
||||
\bar{y} = \frac{1}{n} \sum_{i=0}^{n - 1} y_i.
|
||||
$$
|
||||
<p> <br>
|
||||
|
||||
Discuss these quantities as functions of the variable \( \lambda \) in the Ridge and Lasso regression methods.
|
||||
|
||||
<h3 id="exercise-linear-regression-for-a-two-dimensional-function">Exercise: Linear Regression for a two-dimensional function </h3>
|
||||
|
||||
<p>
|
||||
This is a longer exercise and the aim is to study in more detail various
|
||||
regression methods, including the Ordinary Least Squares (OLS) method,
|
||||
Ridge regression and finally Lasso regression.
|
||||
This exercise forms a part of project 1.
|
||||
|
||||
<p>
|
||||
We will study how to fit polynomials to a specific
|
||||
two-dimensional function called <a href="http://www.dtic.mil/dtic/tr/fulltext/u2/a081688.pdf" target="_blank">Franke's
|
||||
function</a>. This
|
||||
is a function which has been widely used when testing various
|
||||
interpolation and fitting algorithms.
|
||||
|
||||
<p>
|
||||
The Franke function, which is a weighted sum of four exponentials reads as follows
|
||||
<p> <br>
|
||||
$$
|
||||
\begin{align*}
|
||||
f(x,y) &= \frac{3}{4}\exp{\left(-\frac{(9x-2)^2}{4} - \frac{(9y-2)^2}{4}\right)}+\frac{3}{4}\exp{\left(-\frac{(9x+1)^2}{49}- \frac{(9y+1)}{10}\right)} \\
|
||||
&+\frac{1}{2}\exp{\left(-\frac{(9x-7)^2}{4} - \frac{(9y-3)^2}{4}\right)} -\frac{1}{5}\exp{\left(-(9x-4)^2 - (9y-7)^2\right) }.
|
||||
\end{align*}
|
||||
$$
|
||||
<p> <br>
|
||||
|
||||
<p>
|
||||
The function will be defined for \( x,y\in [0,1] \). Our first step will
|
||||
be to perform an OLS regression analysis of this function, trying out
|
||||
a polynomial fit with an \( x \) and \( y \) dependence of the form \( [x, y,
|
||||
x^2, y^2, xy, \dots] \). We will fit a
|
||||
function (for example a polynomial) of \( x \) and \( y \). Thereafter we
|
||||
will repeat much of the same procedure using the Ridge and Lasso
|
||||
regression methods, introducing thus a dependence on the bias
|
||||
(penalty) \( \lambda \).
|
||||
|
||||
<p>
|
||||
The Python fucntion for the Franke function is included here (it performs also a three-dimensional plot of it)
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="highlight" style="background: #eeeedd"><pre style="font-size: 80%; line-height: 125%;"><span></span><span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">mpl_toolkits.mplot3d</span> <span style="color: #8B008B; font-weight: bold">import</span> Axes3D
|
||||
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">matplotlib.pyplot</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">plt</span>
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">matplotlib</span> <span style="color: #8B008B; font-weight: bold">import</span> cm
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">matplotlib.ticker</span> <span style="color: #8B008B; font-weight: bold">import</span> LinearLocator, FormatStrFormatter
|
||||
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">numpy</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">np</span>
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">random</span> <span style="color: #8B008B; font-weight: bold">import</span> random, seed
|
||||
|
||||
fig = plt.figure()
|
||||
ax = fig.gca(projection=<span style="color: #CD5555">'3d'</span>)
|
||||
|
||||
<span style="color: #228B22"># Make data.</span>
|
||||
x = np.arange(<span style="color: #B452CD">0</span>, <span style="color: #B452CD">1</span>, <span style="color: #B452CD">0.05</span>)
|
||||
y = np.arange(<span style="color: #B452CD">0</span>, <span style="color: #B452CD">1</span>, <span style="color: #B452CD">0.05</span>)
|
||||
x, y = np.meshgrid(x,y)
|
||||
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">FrankeFunction</span>(x,y):
|
||||
term1 = <span style="color: #B452CD">0.75</span>*np.exp(-(<span style="color: #B452CD">0.25</span>*(<span style="color: #B452CD">9</span>*x-<span style="color: #B452CD">2</span>)**<span style="color: #B452CD">2</span>) - <span style="color: #B452CD">0.25</span>*((<span style="color: #B452CD">9</span>*y-<span style="color: #B452CD">2</span>)**<span style="color: #B452CD">2</span>))
|
||||
term2 = <span style="color: #B452CD">0.75</span>*np.exp(-((<span style="color: #B452CD">9</span>*x+<span style="color: #B452CD">1</span>)**<span style="color: #B452CD">2</span>)/<span style="color: #B452CD">49.0</span> - <span style="color: #B452CD">0.1</span>*(<span style="color: #B452CD">9</span>*y+<span style="color: #B452CD">1</span>))
|
||||
term3 = <span style="color: #B452CD">0.5</span>*np.exp(-(<span style="color: #B452CD">9</span>*x-<span style="color: #B452CD">7</span>)**<span style="color: #B452CD">2</span>/<span style="color: #B452CD">4.0</span> - <span style="color: #B452CD">0.25</span>*((<span style="color: #B452CD">9</span>*y-<span style="color: #B452CD">3</span>)**<span style="color: #B452CD">2</span>))
|
||||
term4 = -<span style="color: #B452CD">0.2</span>*np.exp(-(<span style="color: #B452CD">9</span>*x-<span style="color: #B452CD">4</span>)**<span style="color: #B452CD">2</span> - (<span style="color: #B452CD">9</span>*y-<span style="color: #B452CD">7</span>)**<span style="color: #B452CD">2</span>)
|
||||
<span style="color: #8B008B; font-weight: bold">return</span> term1 + term2 + term3 + term4
|
||||
|
||||
|
||||
z = FrankeFunction(x, y)
|
||||
|
||||
<span style="color: #228B22"># Plot the surface.</span>
|
||||
surf = ax.plot_surface(x, y, z, cmap=cm.coolwarm,
|
||||
linewidth=<span style="color: #B452CD">0</span>, antialiased=<span style="color: #8B008B; font-weight: bold">False</span>)
|
||||
|
||||
<span style="color: #228B22"># Customize the z axis.</span>
|
||||
ax.set_zlim(-<span style="color: #B452CD">0.10</span>, <span style="color: #B452CD">1.40</span>)
|
||||
ax.zaxis.set_major_locator(LinearLocator(<span style="color: #B452CD">10</span>))
|
||||
ax.zaxis.set_major_formatter(FormatStrFormatter(<span style="color: #CD5555">'%.02f'</span>))
|
||||
|
||||
<span style="color: #228B22"># Add a color bar which maps values to colors.</span>
|
||||
fig.colorbar(surf, shrink=<span style="color: #B452CD">0.5</span>, aspect=<span style="color: #B452CD">5</span>)
|
||||
|
||||
plt.show()
|
||||
</pre></div>
|
||||
<p>
|
||||
We will generate our own dataset for a function
|
||||
\( \mathrm{FrankeFunction}(x,y) \) with \( x,y \in [0,1] \). The function
|
||||
\( f(x,y) \) is the Franke function. You should explore also the addition
|
||||
an added stochastic noise to this function using the normal
|
||||
distribution \( \cal{N}(0,1) \).
|
||||
|
||||
<p>
|
||||
Write your own code (using either a matrix inversion or a singular
|
||||
value decomposition from e.g., <b>numpy</b> ) or use your code and perform a standard least square regression
|
||||
analysis using polynomials in \( x \) and \( y \) up to fifth order. You can use <b>scikit-learn</b> as well.
|
||||
|
||||
<p>
|
||||
Evaluate the Mean Squared error (MSE)
|
||||
|
||||
<p> <br>
|
||||
$$ MSE(\hat{y},\hat{\tilde{y}}) = \frac{1}{n}
|
||||
\sum_{i=0}^{n-1}(y_i-\tilde{y}_i)^2,
|
||||
$$
|
||||
<p> <br>
|
||||
|
||||
<p>
|
||||
and the \( R^2 \) score function. If \( \tilde{\hat{y}}_i \) is the predicted
|
||||
value of the \( i-th \) sample and \( y_i \) is the corresponding true value,
|
||||
then the score \( R^2 \) is defined as
|
||||
|
||||
<p> <br>
|
||||
$$
|
||||
R^2(\hat{y}, \tilde{\hat{y}}) = 1 - \frac{\sum_{i=0}^{n - 1} (y_i - \tilde{y}_i)^2}{\sum_{i=0}^{n - 1} (y_i - \bar{y})^2},
|
||||
$$
|
||||
<p> <br>
|
||||
|
||||
<p>
|
||||
where we have defined the mean value of \( \hat{y} \) as
|
||||
|
||||
<p> <br>
|
||||
$$
|
||||
\bar{y} = \frac{1}{n} \sum_{i=0}^{n - 1} y_i.
|
||||
$$
|
||||
<p> <br>
|
||||
|
||||
<p>
|
||||
You should split your data in train and test and also consider scaling the data.
|
||||
|
||||
<p>
|
||||
To set up the design matrix, the following code can be used
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="highlight" style="background: #eeeedd"><pre style="font-size: 80%; line-height: 125%;"><span></span><span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">FrankeFunction</span>(x,y):
|
||||
term1 = <span style="color: #B452CD">0.75</span>*np.exp(-(<span style="color: #B452CD">0.25</span>*(<span style="color: #B452CD">9</span>*x-<span style="color: #B452CD">2</span>)**<span style="color: #B452CD">2</span>) - <span style="color: #B452CD">0.25</span>*((<span style="color: #B452CD">9</span>*y-<span style="color: #B452CD">2</span>)**<span style="color: #B452CD">2</span>))
|
||||
term2 = <span style="color: #B452CD">0.75</span>*np.exp(-((<span style="color: #B452CD">9</span>*x+<span style="color: #B452CD">1</span>)**<span style="color: #B452CD">2</span>)/<span style="color: #B452CD">49.0</span> - <span style="color: #B452CD">0.1</span>*(<span style="color: #B452CD">9</span>*y+<span style="color: #B452CD">1</span>))
|
||||
term3 = <span style="color: #B452CD">0.5</span>*np.exp(-(<span style="color: #B452CD">9</span>*x-<span style="color: #B452CD">7</span>)**<span style="color: #B452CD">2</span>/<span style="color: #B452CD">4.0</span> - <span style="color: #B452CD">0.25</span>*((<span style="color: #B452CD">9</span>*y-<span style="color: #B452CD">3</span>)**<span style="color: #B452CD">2</span>))
|
||||
term4 = -<span style="color: #B452CD">0.2</span>*np.exp(-(<span style="color: #B452CD">9</span>*x-<span style="color: #B452CD">4</span>)**<span style="color: #B452CD">2</span> - (<span style="color: #B452CD">9</span>*y-<span style="color: #B452CD">7</span>)**<span style="color: #B452CD">2</span>)
|
||||
<span style="color: #8B008B; font-weight: bold">return</span> term1 + term2 + term3 + term4
|
||||
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">create_X</span>(x, y, n ):
|
||||
<span style="color: #8B008B; font-weight: bold">if</span> <span style="color: #658b00">len</span>(x.shape) > <span style="color: #B452CD">1</span>:
|
||||
x = np.ravel(x)
|
||||
y = np.ravel(y)
|
||||
|
||||
N = <span style="color: #658b00">len</span>(x)
|
||||
l = <span style="color: #658b00">int</span>((n+<span style="color: #B452CD">1</span>)*(n+<span style="color: #B452CD">2</span>)/<span style="color: #B452CD">2</span>) <span style="color: #228B22"># Number of elements in beta</span>
|
||||
X = np.ones((N,l))
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">for</span> i <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(<span style="color: #B452CD">1</span>,n+<span style="color: #B452CD">1</span>):
|
||||
q = <span style="color: #658b00">int</span>((i)*(i+<span style="color: #B452CD">1</span>)/<span style="color: #B452CD">2</span>)
|
||||
<span style="color: #8B008B; font-weight: bold">for</span> k <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(i+<span style="color: #B452CD">1</span>):
|
||||
X[:,q+k] = (x**(i-k))*(y**k)
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">return</span> X
|
||||
|
||||
|
||||
<span style="color: #228B22"># Making meshgrid of datapoints and compute Franke's function</span>
|
||||
n = <span style="color: #B452CD">5</span>
|
||||
N = <span style="color: #B452CD">1000</span>
|
||||
x = np.sort(np.random.uniform(<span style="color: #B452CD">0</span>, <span style="color: #B452CD">1</span>, N))
|
||||
y = np.sort(np.random.uniform(<span style="color: #B452CD">0</span>, <span style="color: #B452CD">1</span>, N))
|
||||
z = FrankeFunction(x, y)
|
||||
X = create_X(x, y, n=n)
|
||||
</pre></div>
|
||||
<p>
|
||||
Write then your own code for the Ridge method or use <b>Scikit-Learn</b>.
|
||||
Perform the same analysis as you did for ordinary Least Squares (for the same polynomials) but now for different values of \( \lambda \). Compare and
|
||||
analyze your results with those obtained with ordinary Least Squares. Study the
|
||||
dependence on \( \lambda \).
|
||||
|
||||
<p>
|
||||
This part is essentially a repeat of the previous ones, but now
|
||||
with Lasso regression. Write either your own code or
|
||||
use the functionalities of <b>Scikit-Learn</b> (recommended).
|
||||
Give a
|
||||
critical discussion of the three methods and a judgement of which
|
||||
model fits the data best.
|
||||
|
||||
<p>
|
||||
<!-- --- end exercise --- -->
|
||||
</section>
|
||||
|
||||
|
||||
|
||||
@@ -167,11 +167,18 @@ div { text-align: justify; text-justify: inter-word; }
|
||||
None,
|
||||
'example-of-own-standard-scaling'),
|
||||
('Min-Max Scaling', 2, None, 'min-max-scaling'),
|
||||
('Simple preprocessing examples, Franke function and regression',
|
||||
('Testing the Means Squared Error as function of Complexity',
|
||||
2,
|
||||
None,
|
||||
'simple-preprocessing-examples-franke-function-and-regression'),
|
||||
('Friday September 3', 2, None, 'friday-september-3'),
|
||||
'testing-the-means-squared-error-as-function-of-complexity'),
|
||||
('More preprocessing examples, Franke function and regression',
|
||||
2,
|
||||
None,
|
||||
'more-preprocessing-examples-franke-function-and-regression'),
|
||||
('Mathematical Interpretation of Ordinary Least Squares',
|
||||
2,
|
||||
None,
|
||||
'mathematical-interpretation-of-ordinary-least-squares'),
|
||||
('The singular value decomposition',
|
||||
2,
|
||||
None,
|
||||
@@ -189,7 +196,7 @@ div { text-align: justify; text-justify: inter-word; }
|
||||
('Economy-size SVD', 2, None, 'economy-size-svd'),
|
||||
('Codes for the SVD', 2, None, 'codes-for-the-svd'),
|
||||
('Mathematical Properties', 2, None, 'mathematical-properties'),
|
||||
('Friday September 12', 2, None, 'friday-september-12'),
|
||||
('Friday September 3', 2, None, 'friday-september-3'),
|
||||
('Ridge and LASSO Regression',
|
||||
2,
|
||||
None,
|
||||
@@ -233,7 +240,19 @@ div { text-align: justify; text-justify: inter-word; }
|
||||
2,
|
||||
None,
|
||||
'rewriting-the-covariance-and-or-correlation-matrix'),
|
||||
('Linking with SVD', 2, None, 'linking-with-svd')]}
|
||||
('Linking with SVD', 2, None, 'linking-with-svd'),
|
||||
('Exercises for week 37, September 6-10',
|
||||
2,
|
||||
None,
|
||||
'exercises-for-week-37-september-6-10'),
|
||||
('Exercise 1: Adding Ridge and Lasso Regression',
|
||||
2,
|
||||
None,
|
||||
'exercise-1-adding-ridge-and-lasso-regression'),
|
||||
('Exercise: Linear Regression for a two-dimensional function',
|
||||
3,
|
||||
None,
|
||||
'exercise-linear-regression-for-a-two-dimensional-function')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -1394,12 +1413,16 @@ XPandas = pd.DataFrame(X)
|
||||
display(XPandas)
|
||||
<span style="color: #658b00">print</span>(XPandas.mean())
|
||||
<span style="color: #658b00">print</span>(XPandas.std())
|
||||
XPandas = XPandas -XPandas,mean()
|
||||
XPandas = (XPandas -XPandas.mean())
|
||||
display(XPandas)
|
||||
scaler = StandardScaler()
|
||||
Xscaled = scaler.transform(a)
|
||||
<span style="color: #658b00">print</span>(Xscaled)
|
||||
scaler = StandardScaler(with_std=<span style="color: #8B008B; font-weight: bold">False</span>)
|
||||
scaler.fit(X)
|
||||
Xscaled = scaler.transform(X)
|
||||
display(XPandas-Xscaled)
|
||||
</pre></div>
|
||||
<p>
|
||||
Small exercise: perform the standars scaling by including the standard deviation.
|
||||
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
@@ -1420,7 +1443,72 @@ where \( \min(x_j) \) and \( \max(x_j) \) return the minimum and maximum value o
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="simple-preprocessing-examples-franke-function-and-regression">Simple preprocessing examples, Franke function and regression </h2>
|
||||
<h2 id="testing-the-means-squared-error-as-function-of-complexity">Testing the Means Squared Error as function of Complexity </h2>
|
||||
One of
|
||||
the aims is to reproduce Figure 2.11 of <a href="https://github.com/CompPhysics/MLErasmus/blob/master/doc/Textbooks/elementsstat.pdf" target="_blank">Hastie et al</a>.
|
||||
We will also use Ridge and Lasso regression.
|
||||
|
||||
<p>
|
||||
Our data is defined by \( x\in [-3,3] \) with a total of for example \( 100 \) data points.
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="highlight" style="background: #eeeedd"><pre style="line-height: 125%;"><span></span>np.random.seed()
|
||||
n = <span style="color: #B452CD">100</span>
|
||||
maxdegree = <span style="color: #B452CD">14</span>
|
||||
<span style="color: #228B22"># Make data set.</span>
|
||||
x = np.linspace(-<span style="color: #B452CD">3</span>, <span style="color: #B452CD">3</span>, n).reshape(-<span style="color: #B452CD">1</span>, <span style="color: #B452CD">1</span>)
|
||||
y = np.exp(-x**<span style="color: #B452CD">2</span>) + <span style="color: #B452CD">1.5</span> * np.exp(-(x-<span style="color: #B452CD">2</span>)**<span style="color: #B452CD">2</span>)+ np.random.normal(<span style="color: #B452CD">0</span>, <span style="color: #B452CD">0.1</span>, x.shape)
|
||||
</pre></div>
|
||||
<p>
|
||||
where \( y \) is the function we want to fit with a given polynomial.
|
||||
|
||||
<p>
|
||||
Write a first code which sets up a design matrix \( X \) defined by a fifth-order polynomial. Scale your data and split it in training and test data.
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="highlight" style="background: #eeeedd"><pre style="line-height: 125%;"><span></span><span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">matplotlib.pyplot</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">plt</span>
|
||||
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">numpy</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">np</span>
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.linear_model</span> <span style="color: #8B008B; font-weight: bold">import</span> LinearRegression, Ridge, Lasso
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.preprocessing</span> <span style="color: #8B008B; font-weight: bold">import</span> PolynomialFeatures
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.model_selection</span> <span style="color: #8B008B; font-weight: bold">import</span> train_test_split
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.pipeline</span> <span style="color: #8B008B; font-weight: bold">import</span> make_pipeline
|
||||
|
||||
|
||||
np.random.seed(<span style="color: #B452CD">2018</span>)
|
||||
n = <span style="color: #B452CD">50</span>
|
||||
maxdegree = <span style="color: #B452CD">5</span>
|
||||
<span style="color: #228B22"># Make data set.</span>
|
||||
x = np.linspace(-<span style="color: #B452CD">3</span>, <span style="color: #B452CD">3</span>, n).reshape(-<span style="color: #B452CD">1</span>, <span style="color: #B452CD">1</span>)
|
||||
y = np.exp(-x**<span style="color: #B452CD">2</span>) + <span style="color: #B452CD">1.5</span> * np.exp(-(x-<span style="color: #B452CD">2</span>)**<span style="color: #B452CD">2</span>)+ np.random.normal(<span style="color: #B452CD">0</span>, <span style="color: #B452CD">0.1</span>, x.shape)
|
||||
TestError = np.zeros(maxdegree)
|
||||
TrainError = np.zeros(maxdegree)
|
||||
polydegree = np.zeros(maxdegree)
|
||||
x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=<span style="color: #B452CD">0.2</span>)
|
||||
scaler = StandardScaler()
|
||||
scaler.fit(X_train)
|
||||
x_train_scaled = scaler.transform(x_train)
|
||||
x_test_scaled = scaler.transform(x_test)
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">for</span> degree <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(maxdegree):
|
||||
model = make_pipeline(PolynomialFeatures(degree=degree), LinearRegression(fit_intercept=<span style="color: #8B008B; font-weight: bold">False</span>))
|
||||
clf = model.fit(x_train_scale,y_train)
|
||||
y_fit = clf.predict(x_train_scaled)
|
||||
y_pred = clf.predict(x_test_scaled)
|
||||
polydegree[degree] = degree
|
||||
TestError[degree] = np.mean( np.mean((y_test - y_pred)**<span style="color: #B452CD">2</span>) )
|
||||
TrainError[degree] = np.mean( np.mean((y_train - y_fit)**<span style="color: #B452CD">2</span>) )
|
||||
|
||||
plt.plot(polydegree, TestError, label=<span style="color: #CD5555">'Test Error'</span>)
|
||||
plt.plot(polydegree, TrainError, label=<span style="color: #CD5555">'Train Error'</span>)
|
||||
plt.legend()
|
||||
plt.show()
|
||||
</pre></div>
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="more-preprocessing-examples-franke-function-and-regression">More preprocessing examples, Franke function and regression </h2>
|
||||
|
||||
<p>
|
||||
|
||||
@@ -1521,10 +1609,10 @@ clf = skl.LinearRegression().fit(X_train_scaled, y_train)
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="friday-september-3">Friday September 3 </h2>
|
||||
<h2 id="mathematical-interpretation-of-ordinary-least-squares">Mathematical Interpretation of Ordinary Least Squares </h2>
|
||||
|
||||
<p>
|
||||
Lasso and Ridge regression
|
||||
What is presented here is a mathematical analysis of various regression algorithms (ordinary least squares, Ridge and Lasso Regression). The analysis is based on an important algorithm in linear algebra, the so-called Singular Value Decomposition (SVD).
|
||||
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
@@ -1539,12 +1627,15 @@ Lasso and Ridge regression
|
||||
<p>
|
||||
The examples we have looked at so far are cases where we normally can
|
||||
invert the matrix \( \boldsymbol{X}^T\boldsymbol{X} \). Using a polynomial expansion as we
|
||||
did both for the masses and the fitting of the equation of state,
|
||||
leads to row vectors of the design matrix which are essentially
|
||||
orthogonal due to the polynomial character of our model. Obtaining the inverse of the design matrix is then often done via a so-called LU, QR or Cholesky decomposition.
|
||||
did both for the masses and the fitting of various functions leads to
|
||||
row vectors of the design matrix which are essentially orthogonal due
|
||||
to the polynomial character of our model. Obtaining the inverse of the
|
||||
design matrix is then often done via a so-called LU, QR or Cholesky
|
||||
decomposition.
|
||||
|
||||
<p>
|
||||
This may
|
||||
As we will also see in the first project,
|
||||
this may
|
||||
however not the be case in general and a standard matrix inversion
|
||||
algorithm based on say LU, QR or Cholesky decomposition may lead to singularities. We will see examples of this below.
|
||||
|
||||
@@ -1872,10 +1963,10 @@ Here we have that $${\bf X} = {\bf U}{\bf \Sigma}{\bf V}^T$$, with $$\Sigma$$ be
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="friday-september-12">Friday September 12 </h2>
|
||||
<h2 id="friday-september-3">Friday September 3 </h2>
|
||||
|
||||
<p>
|
||||
<a href="https://www.uio.no/studier/emner/matnat/fys/FYS-STK4155/h20/forelesningsvideoer/LectureSeptember11.mp4?vrtx=view-as-webpage" target="_blank">Video of Lecture</a> and <a href="https://github.com/CompPhysics/MachineLearning/blob/master/doc/HandWrittenNotes/NotesSeptember11.pdf" target="_blank">handwritten notes</a>
|
||||
<a href="https://www.uio.no/studier/emner/matnat/fys/FYS-STK4155/h20/forelesningsvideoer/LectureSeptember11.mp4?vrtx=view-as-webpage" target="_blank">Video of Lecture from 2020</a> and <a href="https://github.com/CompPhysics/MachineLearning/blob/master/doc/HandWrittenNotes/NotesSeptember11.pdf" target="_blank">handwritten notes</a>
|
||||
|
||||
<p>
|
||||
More material will be added here, see handwritten notes also.
|
||||
@@ -2446,6 +2537,318 @@ It is easy to generalize this to a matrix \( \boldsymbol{X}\in {\mathbb{R}}^{n\t
|
||||
<h2 id="linking-with-svd">Linking with SVD </h2>
|
||||
|
||||
<p>
|
||||
More material will be added here.
|
||||
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="exercises-for-week-37-september-6-10">Exercises for week 37, September 6-10 </h2>
|
||||
|
||||
<p>
|
||||
The exercises here are meant to prepare you for work with project 1. The first exercise is a follow-up of exercise 2 from week 36 August 30-September 3).
|
||||
|
||||
<p>
|
||||
<!-- --- begin exercise --- -->
|
||||
|
||||
<h2 id="exercise-1-adding-ridge-and-lasso-regression">Exercise 1: Adding Ridge and Lasso Regression </h2>
|
||||
|
||||
<p>
|
||||
This exercise is a continuation of exercise 2 from exercise set 1 (week 36, August 30-September 3)). We will
|
||||
use the same function to generate our data set, still staying with a
|
||||
simple function \( y(x) \) which we want to fit using linear regression,
|
||||
but now extending the analysis to include the Ridge and the Lasso
|
||||
regression methods.
|
||||
|
||||
<p>
|
||||
We will thus again generate our own dataset for a function \( y(x) \) where
|
||||
\( x \in [0,1] \) and defined by random numbers computed with the uniform
|
||||
distribution. The function \( y \) is a quadratic polynomial in \( x \) with
|
||||
added stochastic noise according to the normal distribution \( \cal{N}(0,1) \).
|
||||
|
||||
<p>
|
||||
The following simple Python instructions define our \( x \) and \( y \) values (with 100 data points).
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="highlight" style="background: #eeeedd"><pre style="line-height: 125%;"><span></span>x = np.random.rand(<span style="color: #B452CD">100</span>)
|
||||
y = <span style="color: #B452CD">2.0</span>+<span style="color: #B452CD">5</span>*x*x+<span style="color: #B452CD">0.1</span>*np.random.randn(<span style="color: #B452CD">100</span>)
|
||||
</pre></div>
|
||||
<p>
|
||||
Write your own code for the Ridge method (see chapter 3.4 of Hastie <em>et al.</em>, equations (3.43) and (3.44)) and compute the parametrization for different values of \( \lambda \). Compare and analyze your results with those from exercise 3. Study the dependence on \( \lambda \) while also varying the strength of the noise in your expression for \( y(x) \).
|
||||
|
||||
<p>
|
||||
The code here allows you to perform your own Ridge calculation and perform calculations for various values of the regularization parameter \( \lambda \). This program can easily be extended upon.
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="highlight" style="background: #eeeedd"><pre style="line-height: 125%;"><span></span><span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">os</span>
|
||||
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">numpy</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">np</span>
|
||||
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">pandas</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">pd</span>
|
||||
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">matplotlib.pyplot</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">plt</span>
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.model_selection</span> <span style="color: #8B008B; font-weight: bold">import</span> train_test_split
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.preprocessing</span> <span style="color: #8B008B; font-weight: bold">import</span> StandardScaler
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">R2</span>(y_data, y_model):
|
||||
<span style="color: #8B008B; font-weight: bold">return</span> <span style="color: #B452CD">1</span> - np.sum((y_data - y_model) ** <span style="color: #B452CD">2</span>) / np.sum((y_data - np.mean(y_data)) ** <span style="color: #B452CD">2</span>)
|
||||
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">MSE</span>(y_data,y_model):
|
||||
n = np.size(y_model)
|
||||
<span style="color: #8B008B; font-weight: bold">return</span> np.sum((y_data-y_model)**<span style="color: #B452CD">2</span>)/n
|
||||
|
||||
|
||||
<span style="color: #228B22"># A seed just to ensure that the random numbers are the same for every run.</span>
|
||||
<span style="color: #228B22"># Useful for eventual debugging.</span>
|
||||
np.random.seed(<span style="color: #B452CD">3155</span>)
|
||||
|
||||
x = np.random.rand(<span style="color: #B452CD">100</span>)
|
||||
y = <span style="color: #B452CD">2.0</span>+<span style="color: #B452CD">5</span>*x*x+<span style="color: #B452CD">0.1</span>*np.random.randn(<span style="color: #B452CD">100</span>)
|
||||
|
||||
<span style="color: #228B22"># number of features p (here degree of polynomial</span>
|
||||
p = <span style="color: #B452CD">3</span>
|
||||
<span style="color: #228B22"># The design matrix now as function of a given polynomial</span>
|
||||
X = np.zeros((<span style="color: #658b00">len</span>(x),p))
|
||||
X[:,<span style="color: #B452CD">0</span>] = <span style="color: #B452CD">1.0</span>
|
||||
X[:,<span style="color: #B452CD">1</span>] = x
|
||||
X[:,<span style="color: #B452CD">2</span>] = x*x
|
||||
<span style="color: #228B22"># We split the data in test and training data</span>
|
||||
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=<span style="color: #B452CD">0.2</span>)
|
||||
scaler = StandardScaler()
|
||||
scaler.fit(X_train)
|
||||
X_train_scaled = scaler.transform(X_train)
|
||||
X_test_scaled = scaler.transform(X_test)
|
||||
|
||||
<span style="color: #228B22"># matrix inversion to find beta</span>
|
||||
OLSbeta = np.linalg.inv(X_train.T @ X_train) @ X_train.T @ y_train
|
||||
<span style="color: #658b00">print</span>(OLSbeta)
|
||||
<span style="color: #228B22"># and then make the prediction</span>
|
||||
ytildeOLS = X_train @ OLSbeta
|
||||
<span style="color: #658b00">print</span>(<span style="color: #CD5555">"Training R2 for OLS"</span>)
|
||||
<span style="color: #658b00">print</span>(R2(y_train,ytildeOLS))
|
||||
<span style="color: #658b00">print</span>(<span style="color: #CD5555">"Training MSE for OLS"</span>)
|
||||
<span style="color: #658b00">print</span>(MSE(y_train,ytildeOLS))
|
||||
ypredictOLS = X_test @ OLSbeta
|
||||
<span style="color: #658b00">print</span>(<span style="color: #CD5555">"Test R2 for OLS"</span>)
|
||||
<span style="color: #658b00">print</span>(R2(y_test,ypredictOLS))
|
||||
<span style="color: #658b00">print</span>(<span style="color: #CD5555">"Test MSE OLS"</span>)
|
||||
<span style="color: #658b00">print</span>(MSE(y_test,ypredictOLS))
|
||||
|
||||
<span style="color: #228B22"># Repeat now for Ridge regression and various values of the regularization parameter</span>
|
||||
I = np.eye(p,p)
|
||||
<span style="color: #228B22"># Decide which values of lambda to use</span>
|
||||
nlambdas = <span style="color: #B452CD">20</span>
|
||||
MSEPredict = np.zeros(nlambdas)
|
||||
MSETrain = np.zeros(nlambdas)
|
||||
lambdas = np.logspace(-<span style="color: #B452CD">4</span>, <span style="color: #B452CD">1</span>, nlambdas)
|
||||
<span style="color: #8B008B; font-weight: bold">for</span> i <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(nlambdas):
|
||||
lmb = lambdas[i]
|
||||
Ridgebeta = np.linalg.inv(X_train.T @ X_train+lmb*I) @ X_train.T @ y_train
|
||||
<span style="color: #228B22"># and then make the prediction</span>
|
||||
ytildeRidge = X_train @ Ridgebeta
|
||||
ypredictRidge = X_test @ Ridgebeta
|
||||
MSEPredict[i] = MSE(y_test,ypredictRidge)
|
||||
MSETrain[i] = MSE(y_train,ytildeRidge)
|
||||
<span style="color: #228B22"># Now plot the results</span>
|
||||
plt.figure()
|
||||
plt.plot(np.log10(lambdas), MSETrain, label = <span style="color: #CD5555">'MSE Ridge train'</span>)
|
||||
plt.plot(np.log10(lambdas), MSEPredict, <span style="color: #CD5555">'r--'</span>, label = <span style="color: #CD5555">'MSE Ridge Test'</span>)
|
||||
plt.xlabel(<span style="color: #CD5555">'log10(lambda)'</span>)
|
||||
plt.ylabel(<span style="color: #CD5555">'MSE'</span>)
|
||||
plt.legend()
|
||||
plt.show()
|
||||
</pre></div>
|
||||
<p>
|
||||
Repeat the above but using the functionality of <b>Scikit-Learn</b>. Compare your code with the results from <b>Scikit-Learn</b>. Remember to run with the same random numbers for generating \( x \) and \( y \).
|
||||
|
||||
<p>
|
||||
Finally, using <b>Scikit-Learn</b> or your own code, compute also the mean square error, a risk metric corresponding to the expected value of the squared (quadratic) error defined as
|
||||
$$ MSE(\hat{y},\hat{\tilde{y}}) = \frac{1}{n}
|
||||
\sum_{i=0}^{n-1}(y_i-\tilde{y}_i)^2,
|
||||
$$
|
||||
|
||||
and the \( R^2 \) score function.
|
||||
If \( \tilde{\hat{y}}_i \) is the predicted value of the \( i-th \) sample and \( y_i \) is the corresponding true value, then the score \( R^2 \) is defined as
|
||||
$$
|
||||
R^2(\hat{y}, \tilde{\hat{y}}) = 1 - \frac{\sum_{i=0}^{n - 1} (y_i - \tilde{y}_i)^2}{\sum_{i=0}^{n - 1} (y_i - \bar{y})^2},
|
||||
$$
|
||||
|
||||
where we have defined the mean value of \( \hat{y} \) as
|
||||
$$
|
||||
\bar{y} = \frac{1}{n} \sum_{i=0}^{n - 1} y_i.
|
||||
$$
|
||||
|
||||
Discuss these quantities as functions of the variable \( \lambda \) in the Ridge and Lasso regression methods.
|
||||
|
||||
<h3 id="exercise-linear-regression-for-a-two-dimensional-function">Exercise: Linear Regression for a two-dimensional function </h3>
|
||||
|
||||
<p>
|
||||
This is a longer exercise and the aim is to study in more detail various
|
||||
regression methods, including the Ordinary Least Squares (OLS) method,
|
||||
Ridge regression and finally Lasso regression.
|
||||
This exercise forms a part of project 1.
|
||||
|
||||
<p>
|
||||
We will study how to fit polynomials to a specific
|
||||
two-dimensional function called <a href="http://www.dtic.mil/dtic/tr/fulltext/u2/a081688.pdf" target="_blank">Franke's
|
||||
function</a>. This
|
||||
is a function which has been widely used when testing various
|
||||
interpolation and fitting algorithms.
|
||||
|
||||
<p>
|
||||
The Franke function, which is a weighted sum of four exponentials reads as follows
|
||||
$$
|
||||
\begin{align*}
|
||||
f(x,y) &= \frac{3}{4}\exp{\left(-\frac{(9x-2)^2}{4} - \frac{(9y-2)^2}{4}\right)}+\frac{3}{4}\exp{\left(-\frac{(9x+1)^2}{49}- \frac{(9y+1)}{10}\right)} \\
|
||||
&+\frac{1}{2}\exp{\left(-\frac{(9x-7)^2}{4} - \frac{(9y-3)^2}{4}\right)} -\frac{1}{5}\exp{\left(-(9x-4)^2 - (9y-7)^2\right) }.
|
||||
\end{align*}
|
||||
$$
|
||||
|
||||
<p>
|
||||
The function will be defined for \( x,y\in [0,1] \). Our first step will
|
||||
be to perform an OLS regression analysis of this function, trying out
|
||||
a polynomial fit with an \( x \) and \( y \) dependence of the form \( [x, y,
|
||||
x^2, y^2, xy, \dots] \). We will fit a
|
||||
function (for example a polynomial) of \( x \) and \( y \). Thereafter we
|
||||
will repeat much of the same procedure using the Ridge and Lasso
|
||||
regression methods, introducing thus a dependence on the bias
|
||||
(penalty) \( \lambda \).
|
||||
|
||||
<p>
|
||||
The Python fucntion for the Franke function is included here (it performs also a three-dimensional plot of it)
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="highlight" style="background: #eeeedd"><pre style="line-height: 125%;"><span></span><span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">mpl_toolkits.mplot3d</span> <span style="color: #8B008B; font-weight: bold">import</span> Axes3D
|
||||
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">matplotlib.pyplot</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">plt</span>
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">matplotlib</span> <span style="color: #8B008B; font-weight: bold">import</span> cm
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">matplotlib.ticker</span> <span style="color: #8B008B; font-weight: bold">import</span> LinearLocator, FormatStrFormatter
|
||||
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">numpy</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">np</span>
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">random</span> <span style="color: #8B008B; font-weight: bold">import</span> random, seed
|
||||
|
||||
fig = plt.figure()
|
||||
ax = fig.gca(projection=<span style="color: #CD5555">'3d'</span>)
|
||||
|
||||
<span style="color: #228B22"># Make data.</span>
|
||||
x = np.arange(<span style="color: #B452CD">0</span>, <span style="color: #B452CD">1</span>, <span style="color: #B452CD">0.05</span>)
|
||||
y = np.arange(<span style="color: #B452CD">0</span>, <span style="color: #B452CD">1</span>, <span style="color: #B452CD">0.05</span>)
|
||||
x, y = np.meshgrid(x,y)
|
||||
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">FrankeFunction</span>(x,y):
|
||||
term1 = <span style="color: #B452CD">0.75</span>*np.exp(-(<span style="color: #B452CD">0.25</span>*(<span style="color: #B452CD">9</span>*x-<span style="color: #B452CD">2</span>)**<span style="color: #B452CD">2</span>) - <span style="color: #B452CD">0.25</span>*((<span style="color: #B452CD">9</span>*y-<span style="color: #B452CD">2</span>)**<span style="color: #B452CD">2</span>))
|
||||
term2 = <span style="color: #B452CD">0.75</span>*np.exp(-((<span style="color: #B452CD">9</span>*x+<span style="color: #B452CD">1</span>)**<span style="color: #B452CD">2</span>)/<span style="color: #B452CD">49.0</span> - <span style="color: #B452CD">0.1</span>*(<span style="color: #B452CD">9</span>*y+<span style="color: #B452CD">1</span>))
|
||||
term3 = <span style="color: #B452CD">0.5</span>*np.exp(-(<span style="color: #B452CD">9</span>*x-<span style="color: #B452CD">7</span>)**<span style="color: #B452CD">2</span>/<span style="color: #B452CD">4.0</span> - <span style="color: #B452CD">0.25</span>*((<span style="color: #B452CD">9</span>*y-<span style="color: #B452CD">3</span>)**<span style="color: #B452CD">2</span>))
|
||||
term4 = -<span style="color: #B452CD">0.2</span>*np.exp(-(<span style="color: #B452CD">9</span>*x-<span style="color: #B452CD">4</span>)**<span style="color: #B452CD">2</span> - (<span style="color: #B452CD">9</span>*y-<span style="color: #B452CD">7</span>)**<span style="color: #B452CD">2</span>)
|
||||
<span style="color: #8B008B; font-weight: bold">return</span> term1 + term2 + term3 + term4
|
||||
|
||||
|
||||
z = FrankeFunction(x, y)
|
||||
|
||||
<span style="color: #228B22"># Plot the surface.</span>
|
||||
surf = ax.plot_surface(x, y, z, cmap=cm.coolwarm,
|
||||
linewidth=<span style="color: #B452CD">0</span>, antialiased=<span style="color: #8B008B; font-weight: bold">False</span>)
|
||||
|
||||
<span style="color: #228B22"># Customize the z axis.</span>
|
||||
ax.set_zlim(-<span style="color: #B452CD">0.10</span>, <span style="color: #B452CD">1.40</span>)
|
||||
ax.zaxis.set_major_locator(LinearLocator(<span style="color: #B452CD">10</span>))
|
||||
ax.zaxis.set_major_formatter(FormatStrFormatter(<span style="color: #CD5555">'%.02f'</span>))
|
||||
|
||||
<span style="color: #228B22"># Add a color bar which maps values to colors.</span>
|
||||
fig.colorbar(surf, shrink=<span style="color: #B452CD">0.5</span>, aspect=<span style="color: #B452CD">5</span>)
|
||||
|
||||
plt.show()
|
||||
</pre></div>
|
||||
<p>
|
||||
We will generate our own dataset for a function
|
||||
\( \mathrm{FrankeFunction}(x,y) \) with \( x,y \in [0,1] \). The function
|
||||
\( f(x,y) \) is the Franke function. You should explore also the addition
|
||||
an added stochastic noise to this function using the normal
|
||||
distribution \( \cal{N}(0,1) \).
|
||||
|
||||
<p>
|
||||
Write your own code (using either a matrix inversion or a singular
|
||||
value decomposition from e.g., <b>numpy</b> ) or use your code and perform a standard least square regression
|
||||
analysis using polynomials in \( x \) and \( y \) up to fifth order. You can use <b>scikit-learn</b> as well.
|
||||
|
||||
<p>
|
||||
Evaluate the Mean Squared error (MSE)
|
||||
|
||||
$$ MSE(\hat{y},\hat{\tilde{y}}) = \frac{1}{n}
|
||||
\sum_{i=0}^{n-1}(y_i-\tilde{y}_i)^2,
|
||||
$$
|
||||
|
||||
<p>
|
||||
and the \( R^2 \) score function. If \( \tilde{\hat{y}}_i \) is the predicted
|
||||
value of the \( i-th \) sample and \( y_i \) is the corresponding true value,
|
||||
then the score \( R^2 \) is defined as
|
||||
|
||||
$$
|
||||
R^2(\hat{y}, \tilde{\hat{y}}) = 1 - \frac{\sum_{i=0}^{n - 1} (y_i - \tilde{y}_i)^2}{\sum_{i=0}^{n - 1} (y_i - \bar{y})^2},
|
||||
$$
|
||||
|
||||
<p>
|
||||
where we have defined the mean value of \( \hat{y} \) as
|
||||
|
||||
$$
|
||||
\bar{y} = \frac{1}{n} \sum_{i=0}^{n - 1} y_i.
|
||||
$$
|
||||
|
||||
<p>
|
||||
You should split your data in train and test and also consider scaling the data.
|
||||
|
||||
<p>
|
||||
To set up the design matrix, the following code can be used
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="highlight" style="background: #eeeedd"><pre style="line-height: 125%;"><span></span><span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">FrankeFunction</span>(x,y):
|
||||
term1 = <span style="color: #B452CD">0.75</span>*np.exp(-(<span style="color: #B452CD">0.25</span>*(<span style="color: #B452CD">9</span>*x-<span style="color: #B452CD">2</span>)**<span style="color: #B452CD">2</span>) - <span style="color: #B452CD">0.25</span>*((<span style="color: #B452CD">9</span>*y-<span style="color: #B452CD">2</span>)**<span style="color: #B452CD">2</span>))
|
||||
term2 = <span style="color: #B452CD">0.75</span>*np.exp(-((<span style="color: #B452CD">9</span>*x+<span style="color: #B452CD">1</span>)**<span style="color: #B452CD">2</span>)/<span style="color: #B452CD">49.0</span> - <span style="color: #B452CD">0.1</span>*(<span style="color: #B452CD">9</span>*y+<span style="color: #B452CD">1</span>))
|
||||
term3 = <span style="color: #B452CD">0.5</span>*np.exp(-(<span style="color: #B452CD">9</span>*x-<span style="color: #B452CD">7</span>)**<span style="color: #B452CD">2</span>/<span style="color: #B452CD">4.0</span> - <span style="color: #B452CD">0.25</span>*((<span style="color: #B452CD">9</span>*y-<span style="color: #B452CD">3</span>)**<span style="color: #B452CD">2</span>))
|
||||
term4 = -<span style="color: #B452CD">0.2</span>*np.exp(-(<span style="color: #B452CD">9</span>*x-<span style="color: #B452CD">4</span>)**<span style="color: #B452CD">2</span> - (<span style="color: #B452CD">9</span>*y-<span style="color: #B452CD">7</span>)**<span style="color: #B452CD">2</span>)
|
||||
<span style="color: #8B008B; font-weight: bold">return</span> term1 + term2 + term3 + term4
|
||||
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">create_X</span>(x, y, n ):
|
||||
<span style="color: #8B008B; font-weight: bold">if</span> <span style="color: #658b00">len</span>(x.shape) > <span style="color: #B452CD">1</span>:
|
||||
x = np.ravel(x)
|
||||
y = np.ravel(y)
|
||||
|
||||
N = <span style="color: #658b00">len</span>(x)
|
||||
l = <span style="color: #658b00">int</span>((n+<span style="color: #B452CD">1</span>)*(n+<span style="color: #B452CD">2</span>)/<span style="color: #B452CD">2</span>) <span style="color: #228B22"># Number of elements in beta</span>
|
||||
X = np.ones((N,l))
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">for</span> i <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(<span style="color: #B452CD">1</span>,n+<span style="color: #B452CD">1</span>):
|
||||
q = <span style="color: #658b00">int</span>((i)*(i+<span style="color: #B452CD">1</span>)/<span style="color: #B452CD">2</span>)
|
||||
<span style="color: #8B008B; font-weight: bold">for</span> k <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(i+<span style="color: #B452CD">1</span>):
|
||||
X[:,q+k] = (x**(i-k))*(y**k)
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">return</span> X
|
||||
|
||||
|
||||
<span style="color: #228B22"># Making meshgrid of datapoints and compute Franke's function</span>
|
||||
n = <span style="color: #B452CD">5</span>
|
||||
N = <span style="color: #B452CD">1000</span>
|
||||
x = np.sort(np.random.uniform(<span style="color: #B452CD">0</span>, <span style="color: #B452CD">1</span>, N))
|
||||
y = np.sort(np.random.uniform(<span style="color: #B452CD">0</span>, <span style="color: #B452CD">1</span>, N))
|
||||
z = FrankeFunction(x, y)
|
||||
X = create_X(x, y, n=n)
|
||||
</pre></div>
|
||||
<p>
|
||||
Write then your own code for the Ridge method or use <b>Scikit-Learn</b>.
|
||||
Perform the same analysis as you did for ordinary Least Squares (for the same polynomials) but now for different values of \( \lambda \). Compare and
|
||||
analyze your results with those obtained with ordinary Least Squares. Study the
|
||||
dependence on \( \lambda \).
|
||||
|
||||
<p>
|
||||
This part is essentially a repeat of the previous ones, but now
|
||||
with Lasso regression. Write either your own code or
|
||||
use the functionalities of <b>Scikit-Learn</b> (recommended).
|
||||
Give a
|
||||
critical discussion of the three methods and a judgement of which
|
||||
model fits the data best.
|
||||
|
||||
<p>
|
||||
<!-- --- end exercise --- -->
|
||||
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
|
||||
+421
-18
@@ -172,11 +172,18 @@ div { text-align: justify; text-justify: inter-word; }
|
||||
None,
|
||||
'example-of-own-standard-scaling'),
|
||||
('Min-Max Scaling', 2, None, 'min-max-scaling'),
|
||||
('Simple preprocessing examples, Franke function and regression',
|
||||
('Testing the Means Squared Error as function of Complexity',
|
||||
2,
|
||||
None,
|
||||
'simple-preprocessing-examples-franke-function-and-regression'),
|
||||
('Friday September 3', 2, None, 'friday-september-3'),
|
||||
'testing-the-means-squared-error-as-function-of-complexity'),
|
||||
('More preprocessing examples, Franke function and regression',
|
||||
2,
|
||||
None,
|
||||
'more-preprocessing-examples-franke-function-and-regression'),
|
||||
('Mathematical Interpretation of Ordinary Least Squares',
|
||||
2,
|
||||
None,
|
||||
'mathematical-interpretation-of-ordinary-least-squares'),
|
||||
('The singular value decomposition',
|
||||
2,
|
||||
None,
|
||||
@@ -194,7 +201,7 @@ div { text-align: justify; text-justify: inter-word; }
|
||||
('Economy-size SVD', 2, None, 'economy-size-svd'),
|
||||
('Codes for the SVD', 2, None, 'codes-for-the-svd'),
|
||||
('Mathematical Properties', 2, None, 'mathematical-properties'),
|
||||
('Friday September 12', 2, None, 'friday-september-12'),
|
||||
('Friday September 3', 2, None, 'friday-september-3'),
|
||||
('Ridge and LASSO Regression',
|
||||
2,
|
||||
None,
|
||||
@@ -238,7 +245,19 @@ div { text-align: justify; text-justify: inter-word; }
|
||||
2,
|
||||
None,
|
||||
'rewriting-the-covariance-and-or-correlation-matrix'),
|
||||
('Linking with SVD', 2, None, 'linking-with-svd')]}
|
||||
('Linking with SVD', 2, None, 'linking-with-svd'),
|
||||
('Exercises for week 37, September 6-10',
|
||||
2,
|
||||
None,
|
||||
'exercises-for-week-37-september-6-10'),
|
||||
('Exercise 1: Adding Ridge and Lasso Regression',
|
||||
2,
|
||||
None,
|
||||
'exercise-1-adding-ridge-and-lasso-regression'),
|
||||
('Exercise: Linear Regression for a two-dimensional function',
|
||||
3,
|
||||
None,
|
||||
'exercise-linear-regression-for-a-two-dimensional-function')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -1399,12 +1418,16 @@ XPandas <span style="color: #666666">=</span> pd<span style="color: #666666">.</
|
||||
display(XPandas)
|
||||
<span style="color: #008000">print</span>(XPandas<span style="color: #666666">.</span>mean())
|
||||
<span style="color: #008000">print</span>(XPandas<span style="color: #666666">.</span>std())
|
||||
XPandas <span style="color: #666666">=</span> XPandas <span style="color: #666666">-</span>XPandas,mean()
|
||||
XPandas <span style="color: #666666">=</span> (XPandas <span style="color: #666666">-</span>XPandas<span style="color: #666666">.</span>mean())
|
||||
display(XPandas)
|
||||
scaler <span style="color: #666666">=</span> StandardScaler()
|
||||
Xscaled <span style="color: #666666">=</span> scaler<span style="color: #666666">.</span>transform(a)
|
||||
<span style="color: #008000">print</span>(Xscaled)
|
||||
scaler <span style="color: #666666">=</span> StandardScaler(with_std<span style="color: #666666">=</span><span style="color: #008000; font-weight: bold">False</span>)
|
||||
scaler<span style="color: #666666">.</span>fit(X)
|
||||
Xscaled <span style="color: #666666">=</span> scaler<span style="color: #666666">.</span>transform(X)
|
||||
display(XPandas<span style="color: #666666">-</span>Xscaled)
|
||||
</pre></div>
|
||||
<p>
|
||||
Small exercise: perform the standars scaling by including the standard deviation.
|
||||
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
@@ -1425,7 +1448,72 @@ where \( \min(x_j) \) and \( \max(x_j) \) return the minimum and maximum value o
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="simple-preprocessing-examples-franke-function-and-regression">Simple preprocessing examples, Franke function and regression </h2>
|
||||
<h2 id="testing-the-means-squared-error-as-function-of-complexity">Testing the Means Squared Error as function of Complexity </h2>
|
||||
One of
|
||||
the aims is to reproduce Figure 2.11 of <a href="https://github.com/CompPhysics/MLErasmus/blob/master/doc/Textbooks/elementsstat.pdf" target="_blank">Hastie et al</a>.
|
||||
We will also use Ridge and Lasso regression.
|
||||
|
||||
<p>
|
||||
Our data is defined by \( x\in [-3,3] \) with a total of for example \( 100 \) data points.
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%;"><span></span>np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>seed()
|
||||
n <span style="color: #666666">=</span> <span style="color: #666666">100</span>
|
||||
maxdegree <span style="color: #666666">=</span> <span style="color: #666666">14</span>
|
||||
<span style="color: #408080; font-style: italic"># Make data set.</span>
|
||||
x <span style="color: #666666">=</span> np<span style="color: #666666">.</span>linspace(<span style="color: #666666">-3</span>, <span style="color: #666666">3</span>, n)<span style="color: #666666">.</span>reshape(<span style="color: #666666">-1</span>, <span style="color: #666666">1</span>)
|
||||
y <span style="color: #666666">=</span> np<span style="color: #666666">.</span>exp(<span style="color: #666666">-</span>x<span style="color: #666666">**2</span>) <span style="color: #666666">+</span> <span style="color: #666666">1.5</span> <span style="color: #666666">*</span> np<span style="color: #666666">.</span>exp(<span style="color: #666666">-</span>(x<span style="color: #666666">-2</span>)<span style="color: #666666">**2</span>)<span style="color: #666666">+</span> np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>normal(<span style="color: #666666">0</span>, <span style="color: #666666">0.1</span>, x<span style="color: #666666">.</span>shape)
|
||||
</pre></div>
|
||||
<p>
|
||||
where \( y \) is the function we want to fit with a given polynomial.
|
||||
|
||||
<p>
|
||||
Write a first code which sets up a design matrix \( X \) defined by a fifth-order polynomial. Scale your data and split it in training and test data.
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%;"><span></span><span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">matplotlib.pyplot</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">plt</span>
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">numpy</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">np</span>
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.linear_model</span> <span style="color: #008000; font-weight: bold">import</span> LinearRegression, Ridge, Lasso
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.preprocessing</span> <span style="color: #008000; font-weight: bold">import</span> PolynomialFeatures
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.model_selection</span> <span style="color: #008000; font-weight: bold">import</span> train_test_split
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.pipeline</span> <span style="color: #008000; font-weight: bold">import</span> make_pipeline
|
||||
|
||||
|
||||
np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>seed(<span style="color: #666666">2018</span>)
|
||||
n <span style="color: #666666">=</span> <span style="color: #666666">50</span>
|
||||
maxdegree <span style="color: #666666">=</span> <span style="color: #666666">5</span>
|
||||
<span style="color: #408080; font-style: italic"># Make data set.</span>
|
||||
x <span style="color: #666666">=</span> np<span style="color: #666666">.</span>linspace(<span style="color: #666666">-3</span>, <span style="color: #666666">3</span>, n)<span style="color: #666666">.</span>reshape(<span style="color: #666666">-1</span>, <span style="color: #666666">1</span>)
|
||||
y <span style="color: #666666">=</span> np<span style="color: #666666">.</span>exp(<span style="color: #666666">-</span>x<span style="color: #666666">**2</span>) <span style="color: #666666">+</span> <span style="color: #666666">1.5</span> <span style="color: #666666">*</span> np<span style="color: #666666">.</span>exp(<span style="color: #666666">-</span>(x<span style="color: #666666">-2</span>)<span style="color: #666666">**2</span>)<span style="color: #666666">+</span> np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>normal(<span style="color: #666666">0</span>, <span style="color: #666666">0.1</span>, x<span style="color: #666666">.</span>shape)
|
||||
TestError <span style="color: #666666">=</span> np<span style="color: #666666">.</span>zeros(maxdegree)
|
||||
TrainError <span style="color: #666666">=</span> np<span style="color: #666666">.</span>zeros(maxdegree)
|
||||
polydegree <span style="color: #666666">=</span> np<span style="color: #666666">.</span>zeros(maxdegree)
|
||||
x_train, x_test, y_train, y_test <span style="color: #666666">=</span> train_test_split(x, y, test_size<span style="color: #666666">=0.2</span>)
|
||||
scaler <span style="color: #666666">=</span> StandardScaler()
|
||||
scaler<span style="color: #666666">.</span>fit(X_train)
|
||||
x_train_scaled <span style="color: #666666">=</span> scaler<span style="color: #666666">.</span>transform(x_train)
|
||||
x_test_scaled <span style="color: #666666">=</span> scaler<span style="color: #666666">.</span>transform(x_test)
|
||||
|
||||
<span style="color: #008000; font-weight: bold">for</span> degree <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(maxdegree):
|
||||
model <span style="color: #666666">=</span> make_pipeline(PolynomialFeatures(degree<span style="color: #666666">=</span>degree), LinearRegression(fit_intercept<span style="color: #666666">=</span><span style="color: #008000; font-weight: bold">False</span>))
|
||||
clf <span style="color: #666666">=</span> model<span style="color: #666666">.</span>fit(x_train_scale,y_train)
|
||||
y_fit <span style="color: #666666">=</span> clf<span style="color: #666666">.</span>predict(x_train_scaled)
|
||||
y_pred <span style="color: #666666">=</span> clf<span style="color: #666666">.</span>predict(x_test_scaled)
|
||||
polydegree[degree] <span style="color: #666666">=</span> degree
|
||||
TestError[degree] <span style="color: #666666">=</span> np<span style="color: #666666">.</span>mean( np<span style="color: #666666">.</span>mean((y_test <span style="color: #666666">-</span> y_pred)<span style="color: #666666">**2</span>) )
|
||||
TrainError[degree] <span style="color: #666666">=</span> np<span style="color: #666666">.</span>mean( np<span style="color: #666666">.</span>mean((y_train <span style="color: #666666">-</span> y_fit)<span style="color: #666666">**2</span>) )
|
||||
|
||||
plt<span style="color: #666666">.</span>plot(polydegree, TestError, label<span style="color: #666666">=</span><span style="color: #BA2121">'Test Error'</span>)
|
||||
plt<span style="color: #666666">.</span>plot(polydegree, TrainError, label<span style="color: #666666">=</span><span style="color: #BA2121">'Train Error'</span>)
|
||||
plt<span style="color: #666666">.</span>legend()
|
||||
plt<span style="color: #666666">.</span>show()
|
||||
</pre></div>
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="more-preprocessing-examples-franke-function-and-regression">More preprocessing examples, Franke function and regression </h2>
|
||||
|
||||
<p>
|
||||
|
||||
@@ -1526,10 +1614,10 @@ clf <span style="color: #666666">=</span> skl<span style="color: #666666">.</spa
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="friday-september-3">Friday September 3 </h2>
|
||||
<h2 id="mathematical-interpretation-of-ordinary-least-squares">Mathematical Interpretation of Ordinary Least Squares </h2>
|
||||
|
||||
<p>
|
||||
Lasso and Ridge regression
|
||||
What is presented here is a mathematical analysis of various regression algorithms (ordinary least squares, Ridge and Lasso Regression). The analysis is based on an important algorithm in linear algebra, the so-called Singular Value Decomposition (SVD).
|
||||
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
@@ -1544,12 +1632,15 @@ Lasso and Ridge regression
|
||||
<p>
|
||||
The examples we have looked at so far are cases where we normally can
|
||||
invert the matrix \( \boldsymbol{X}^T\boldsymbol{X} \). Using a polynomial expansion as we
|
||||
did both for the masses and the fitting of the equation of state,
|
||||
leads to row vectors of the design matrix which are essentially
|
||||
orthogonal due to the polynomial character of our model. Obtaining the inverse of the design matrix is then often done via a so-called LU, QR or Cholesky decomposition.
|
||||
did both for the masses and the fitting of various functions leads to
|
||||
row vectors of the design matrix which are essentially orthogonal due
|
||||
to the polynomial character of our model. Obtaining the inverse of the
|
||||
design matrix is then often done via a so-called LU, QR or Cholesky
|
||||
decomposition.
|
||||
|
||||
<p>
|
||||
This may
|
||||
As we will also see in the first project,
|
||||
this may
|
||||
however not the be case in general and a standard matrix inversion
|
||||
algorithm based on say LU, QR or Cholesky decomposition may lead to singularities. We will see examples of this below.
|
||||
|
||||
@@ -1877,10 +1968,10 @@ Here we have that $${\bf X} = {\bf U}{\bf \Sigma}{\bf V}^T$$, with $$\Sigma$$ be
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="friday-september-12">Friday September 12 </h2>
|
||||
<h2 id="friday-september-3">Friday September 3 </h2>
|
||||
|
||||
<p>
|
||||
<a href="https://www.uio.no/studier/emner/matnat/fys/FYS-STK4155/h20/forelesningsvideoer/LectureSeptember11.mp4?vrtx=view-as-webpage" target="_blank">Video of Lecture</a> and <a href="https://github.com/CompPhysics/MachineLearning/blob/master/doc/HandWrittenNotes/NotesSeptember11.pdf" target="_blank">handwritten notes</a>
|
||||
<a href="https://www.uio.no/studier/emner/matnat/fys/FYS-STK4155/h20/forelesningsvideoer/LectureSeptember11.mp4?vrtx=view-as-webpage" target="_blank">Video of Lecture from 2020</a> and <a href="https://github.com/CompPhysics/MachineLearning/blob/master/doc/HandWrittenNotes/NotesSeptember11.pdf" target="_blank">handwritten notes</a>
|
||||
|
||||
<p>
|
||||
More material will be added here, see handwritten notes also.
|
||||
@@ -2451,6 +2542,318 @@ It is easy to generalize this to a matrix \( \boldsymbol{X}\in {\mathbb{R}}^{n\t
|
||||
<h2 id="linking-with-svd">Linking with SVD </h2>
|
||||
|
||||
<p>
|
||||
More material will be added here.
|
||||
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="exercises-for-week-37-september-6-10">Exercises for week 37, September 6-10 </h2>
|
||||
|
||||
<p>
|
||||
The exercises here are meant to prepare you for work with project 1. The first exercise is a follow-up of exercise 2 from week 36 August 30-September 3).
|
||||
|
||||
<p>
|
||||
<!-- --- begin exercise --- -->
|
||||
|
||||
<h2 id="exercise-1-adding-ridge-and-lasso-regression">Exercise 1: Adding Ridge and Lasso Regression </h2>
|
||||
|
||||
<p>
|
||||
This exercise is a continuation of exercise 2 from exercise set 1 (week 36, August 30-September 3)). We will
|
||||
use the same function to generate our data set, still staying with a
|
||||
simple function \( y(x) \) which we want to fit using linear regression,
|
||||
but now extending the analysis to include the Ridge and the Lasso
|
||||
regression methods.
|
||||
|
||||
<p>
|
||||
We will thus again generate our own dataset for a function \( y(x) \) where
|
||||
\( x \in [0,1] \) and defined by random numbers computed with the uniform
|
||||
distribution. The function \( y \) is a quadratic polynomial in \( x \) with
|
||||
added stochastic noise according to the normal distribution \( \cal{N}(0,1) \).
|
||||
|
||||
<p>
|
||||
The following simple Python instructions define our \( x \) and \( y \) values (with 100 data points).
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%;"><span></span>x <span style="color: #666666">=</span> np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>rand(<span style="color: #666666">100</span>)
|
||||
y <span style="color: #666666">=</span> <span style="color: #666666">2.0+5*</span>x<span style="color: #666666">*</span>x<span style="color: #666666">+0.1*</span>np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>randn(<span style="color: #666666">100</span>)
|
||||
</pre></div>
|
||||
<p>
|
||||
Write your own code for the Ridge method (see chapter 3.4 of Hastie <em>et al.</em>, equations (3.43) and (3.44)) and compute the parametrization for different values of \( \lambda \). Compare and analyze your results with those from exercise 3. Study the dependence on \( \lambda \) while also varying the strength of the noise in your expression for \( y(x) \).
|
||||
|
||||
<p>
|
||||
The code here allows you to perform your own Ridge calculation and perform calculations for various values of the regularization parameter \( \lambda \). This program can easily be extended upon.
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%;"><span></span><span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">os</span>
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">numpy</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">np</span>
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">pandas</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">pd</span>
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">matplotlib.pyplot</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">plt</span>
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.model_selection</span> <span style="color: #008000; font-weight: bold">import</span> train_test_split
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.preprocessing</span> <span style="color: #008000; font-weight: bold">import</span> StandardScaler
|
||||
|
||||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">R2</span>(y_data, y_model):
|
||||
<span style="color: #008000; font-weight: bold">return</span> <span style="color: #666666">1</span> <span style="color: #666666">-</span> np<span style="color: #666666">.</span>sum((y_data <span style="color: #666666">-</span> y_model) <span style="color: #666666">**</span> <span style="color: #666666">2</span>) <span style="color: #666666">/</span> np<span style="color: #666666">.</span>sum((y_data <span style="color: #666666">-</span> np<span style="color: #666666">.</span>mean(y_data)) <span style="color: #666666">**</span> <span style="color: #666666">2</span>)
|
||||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">MSE</span>(y_data,y_model):
|
||||
n <span style="color: #666666">=</span> np<span style="color: #666666">.</span>size(y_model)
|
||||
<span style="color: #008000; font-weight: bold">return</span> np<span style="color: #666666">.</span>sum((y_data<span style="color: #666666">-</span>y_model)<span style="color: #666666">**2</span>)<span style="color: #666666">/</span>n
|
||||
|
||||
|
||||
<span style="color: #408080; font-style: italic"># A seed just to ensure that the random numbers are the same for every run.</span>
|
||||
<span style="color: #408080; font-style: italic"># Useful for eventual debugging.</span>
|
||||
np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>seed(<span style="color: #666666">3155</span>)
|
||||
|
||||
x <span style="color: #666666">=</span> np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>rand(<span style="color: #666666">100</span>)
|
||||
y <span style="color: #666666">=</span> <span style="color: #666666">2.0+5*</span>x<span style="color: #666666">*</span>x<span style="color: #666666">+0.1*</span>np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>randn(<span style="color: #666666">100</span>)
|
||||
|
||||
<span style="color: #408080; font-style: italic"># number of features p (here degree of polynomial</span>
|
||||
p <span style="color: #666666">=</span> <span style="color: #666666">3</span>
|
||||
<span style="color: #408080; font-style: italic"># The design matrix now as function of a given polynomial</span>
|
||||
X <span style="color: #666666">=</span> np<span style="color: #666666">.</span>zeros((<span style="color: #008000">len</span>(x),p))
|
||||
X[:,<span style="color: #666666">0</span>] <span style="color: #666666">=</span> <span style="color: #666666">1.0</span>
|
||||
X[:,<span style="color: #666666">1</span>] <span style="color: #666666">=</span> x
|
||||
X[:,<span style="color: #666666">2</span>] <span style="color: #666666">=</span> x<span style="color: #666666">*</span>x
|
||||
<span style="color: #408080; font-style: italic"># We split the data in test and training data</span>
|
||||
X_train, X_test, y_train, y_test <span style="color: #666666">=</span> train_test_split(X, y, test_size<span style="color: #666666">=0.2</span>)
|
||||
scaler <span style="color: #666666">=</span> StandardScaler()
|
||||
scaler<span style="color: #666666">.</span>fit(X_train)
|
||||
X_train_scaled <span style="color: #666666">=</span> scaler<span style="color: #666666">.</span>transform(X_train)
|
||||
X_test_scaled <span style="color: #666666">=</span> scaler<span style="color: #666666">.</span>transform(X_test)
|
||||
|
||||
<span style="color: #408080; font-style: italic"># matrix inversion to find beta</span>
|
||||
OLSbeta <span style="color: #666666">=</span> np<span style="color: #666666">.</span>linalg<span style="color: #666666">.</span>inv(X_train<span style="color: #666666">.</span>T <span style="color: #666666">@</span> X_train) <span style="color: #666666">@</span> X_train<span style="color: #666666">.</span>T <span style="color: #666666">@</span> y_train
|
||||
<span style="color: #008000">print</span>(OLSbeta)
|
||||
<span style="color: #408080; font-style: italic"># and then make the prediction</span>
|
||||
ytildeOLS <span style="color: #666666">=</span> X_train <span style="color: #666666">@</span> OLSbeta
|
||||
<span style="color: #008000">print</span>(<span style="color: #BA2121">"Training R2 for OLS"</span>)
|
||||
<span style="color: #008000">print</span>(R2(y_train,ytildeOLS))
|
||||
<span style="color: #008000">print</span>(<span style="color: #BA2121">"Training MSE for OLS"</span>)
|
||||
<span style="color: #008000">print</span>(MSE(y_train,ytildeOLS))
|
||||
ypredictOLS <span style="color: #666666">=</span> X_test <span style="color: #666666">@</span> OLSbeta
|
||||
<span style="color: #008000">print</span>(<span style="color: #BA2121">"Test R2 for OLS"</span>)
|
||||
<span style="color: #008000">print</span>(R2(y_test,ypredictOLS))
|
||||
<span style="color: #008000">print</span>(<span style="color: #BA2121">"Test MSE OLS"</span>)
|
||||
<span style="color: #008000">print</span>(MSE(y_test,ypredictOLS))
|
||||
|
||||
<span style="color: #408080; font-style: italic"># Repeat now for Ridge regression and various values of the regularization parameter</span>
|
||||
I <span style="color: #666666">=</span> np<span style="color: #666666">.</span>eye(p,p)
|
||||
<span style="color: #408080; font-style: italic"># Decide which values of lambda to use</span>
|
||||
nlambdas <span style="color: #666666">=</span> <span style="color: #666666">20</span>
|
||||
MSEPredict <span style="color: #666666">=</span> np<span style="color: #666666">.</span>zeros(nlambdas)
|
||||
MSETrain <span style="color: #666666">=</span> np<span style="color: #666666">.</span>zeros(nlambdas)
|
||||
lambdas <span style="color: #666666">=</span> np<span style="color: #666666">.</span>logspace(<span style="color: #666666">-4</span>, <span style="color: #666666">1</span>, nlambdas)
|
||||
<span style="color: #008000; font-weight: bold">for</span> i <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(nlambdas):
|
||||
lmb <span style="color: #666666">=</span> lambdas[i]
|
||||
Ridgebeta <span style="color: #666666">=</span> np<span style="color: #666666">.</span>linalg<span style="color: #666666">.</span>inv(X_train<span style="color: #666666">.</span>T <span style="color: #666666">@</span> X_train<span style="color: #666666">+</span>lmb<span style="color: #666666">*</span>I) <span style="color: #666666">@</span> X_train<span style="color: #666666">.</span>T <span style="color: #666666">@</span> y_train
|
||||
<span style="color: #408080; font-style: italic"># and then make the prediction</span>
|
||||
ytildeRidge <span style="color: #666666">=</span> X_train <span style="color: #666666">@</span> Ridgebeta
|
||||
ypredictRidge <span style="color: #666666">=</span> X_test <span style="color: #666666">@</span> Ridgebeta
|
||||
MSEPredict[i] <span style="color: #666666">=</span> MSE(y_test,ypredictRidge)
|
||||
MSETrain[i] <span style="color: #666666">=</span> MSE(y_train,ytildeRidge)
|
||||
<span style="color: #408080; font-style: italic"># Now plot the results</span>
|
||||
plt<span style="color: #666666">.</span>figure()
|
||||
plt<span style="color: #666666">.</span>plot(np<span style="color: #666666">.</span>log10(lambdas), MSETrain, label <span style="color: #666666">=</span> <span style="color: #BA2121">'MSE Ridge train'</span>)
|
||||
plt<span style="color: #666666">.</span>plot(np<span style="color: #666666">.</span>log10(lambdas), MSEPredict, <span style="color: #BA2121">'r--'</span>, label <span style="color: #666666">=</span> <span style="color: #BA2121">'MSE Ridge Test'</span>)
|
||||
plt<span style="color: #666666">.</span>xlabel(<span style="color: #BA2121">'log10(lambda)'</span>)
|
||||
plt<span style="color: #666666">.</span>ylabel(<span style="color: #BA2121">'MSE'</span>)
|
||||
plt<span style="color: #666666">.</span>legend()
|
||||
plt<span style="color: #666666">.</span>show()
|
||||
</pre></div>
|
||||
<p>
|
||||
Repeat the above but using the functionality of <b>Scikit-Learn</b>. Compare your code with the results from <b>Scikit-Learn</b>. Remember to run with the same random numbers for generating \( x \) and \( y \).
|
||||
|
||||
<p>
|
||||
Finally, using <b>Scikit-Learn</b> or your own code, compute also the mean square error, a risk metric corresponding to the expected value of the squared (quadratic) error defined as
|
||||
$$ MSE(\hat{y},\hat{\tilde{y}}) = \frac{1}{n}
|
||||
\sum_{i=0}^{n-1}(y_i-\tilde{y}_i)^2,
|
||||
$$
|
||||
|
||||
and the \( R^2 \) score function.
|
||||
If \( \tilde{\hat{y}}_i \) is the predicted value of the \( i-th \) sample and \( y_i \) is the corresponding true value, then the score \( R^2 \) is defined as
|
||||
$$
|
||||
R^2(\hat{y}, \tilde{\hat{y}}) = 1 - \frac{\sum_{i=0}^{n - 1} (y_i - \tilde{y}_i)^2}{\sum_{i=0}^{n - 1} (y_i - \bar{y})^2},
|
||||
$$
|
||||
|
||||
where we have defined the mean value of \( \hat{y} \) as
|
||||
$$
|
||||
\bar{y} = \frac{1}{n} \sum_{i=0}^{n - 1} y_i.
|
||||
$$
|
||||
|
||||
Discuss these quantities as functions of the variable \( \lambda \) in the Ridge and Lasso regression methods.
|
||||
|
||||
<h3 id="exercise-linear-regression-for-a-two-dimensional-function">Exercise: Linear Regression for a two-dimensional function </h3>
|
||||
|
||||
<p>
|
||||
This is a longer exercise and the aim is to study in more detail various
|
||||
regression methods, including the Ordinary Least Squares (OLS) method,
|
||||
Ridge regression and finally Lasso regression.
|
||||
This exercise forms a part of project 1.
|
||||
|
||||
<p>
|
||||
We will study how to fit polynomials to a specific
|
||||
two-dimensional function called <a href="http://www.dtic.mil/dtic/tr/fulltext/u2/a081688.pdf" target="_blank">Franke's
|
||||
function</a>. This
|
||||
is a function which has been widely used when testing various
|
||||
interpolation and fitting algorithms.
|
||||
|
||||
<p>
|
||||
The Franke function, which is a weighted sum of four exponentials reads as follows
|
||||
$$
|
||||
\begin{align*}
|
||||
f(x,y) &= \frac{3}{4}\exp{\left(-\frac{(9x-2)^2}{4} - \frac{(9y-2)^2}{4}\right)}+\frac{3}{4}\exp{\left(-\frac{(9x+1)^2}{49}- \frac{(9y+1)}{10}\right)} \\
|
||||
&+\frac{1}{2}\exp{\left(-\frac{(9x-7)^2}{4} - \frac{(9y-3)^2}{4}\right)} -\frac{1}{5}\exp{\left(-(9x-4)^2 - (9y-7)^2\right) }.
|
||||
\end{align*}
|
||||
$$
|
||||
|
||||
<p>
|
||||
The function will be defined for \( x,y\in [0,1] \). Our first step will
|
||||
be to perform an OLS regression analysis of this function, trying out
|
||||
a polynomial fit with an \( x \) and \( y \) dependence of the form \( [x, y,
|
||||
x^2, y^2, xy, \dots] \). We will fit a
|
||||
function (for example a polynomial) of \( x \) and \( y \). Thereafter we
|
||||
will repeat much of the same procedure using the Ridge and Lasso
|
||||
regression methods, introducing thus a dependence on the bias
|
||||
(penalty) \( \lambda \).
|
||||
|
||||
<p>
|
||||
The Python fucntion for the Franke function is included here (it performs also a three-dimensional plot of it)
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%;"><span></span><span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">mpl_toolkits.mplot3d</span> <span style="color: #008000; font-weight: bold">import</span> Axes3D
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">matplotlib.pyplot</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">plt</span>
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">matplotlib</span> <span style="color: #008000; font-weight: bold">import</span> cm
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">matplotlib.ticker</span> <span style="color: #008000; font-weight: bold">import</span> LinearLocator, FormatStrFormatter
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">numpy</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">np</span>
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">random</span> <span style="color: #008000; font-weight: bold">import</span> random, seed
|
||||
|
||||
fig <span style="color: #666666">=</span> plt<span style="color: #666666">.</span>figure()
|
||||
ax <span style="color: #666666">=</span> fig<span style="color: #666666">.</span>gca(projection<span style="color: #666666">=</span><span style="color: #BA2121">'3d'</span>)
|
||||
|
||||
<span style="color: #408080; font-style: italic"># Make data.</span>
|
||||
x <span style="color: #666666">=</span> np<span style="color: #666666">.</span>arange(<span style="color: #666666">0</span>, <span style="color: #666666">1</span>, <span style="color: #666666">0.05</span>)
|
||||
y <span style="color: #666666">=</span> np<span style="color: #666666">.</span>arange(<span style="color: #666666">0</span>, <span style="color: #666666">1</span>, <span style="color: #666666">0.05</span>)
|
||||
x, y <span style="color: #666666">=</span> np<span style="color: #666666">.</span>meshgrid(x,y)
|
||||
|
||||
|
||||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">FrankeFunction</span>(x,y):
|
||||
term1 <span style="color: #666666">=</span> <span style="color: #666666">0.75*</span>np<span style="color: #666666">.</span>exp(<span style="color: #666666">-</span>(<span style="color: #666666">0.25*</span>(<span style="color: #666666">9*</span>x<span style="color: #666666">-2</span>)<span style="color: #666666">**2</span>) <span style="color: #666666">-</span> <span style="color: #666666">0.25*</span>((<span style="color: #666666">9*</span>y<span style="color: #666666">-2</span>)<span style="color: #666666">**2</span>))
|
||||
term2 <span style="color: #666666">=</span> <span style="color: #666666">0.75*</span>np<span style="color: #666666">.</span>exp(<span style="color: #666666">-</span>((<span style="color: #666666">9*</span>x<span style="color: #666666">+1</span>)<span style="color: #666666">**2</span>)<span style="color: #666666">/49.0</span> <span style="color: #666666">-</span> <span style="color: #666666">0.1*</span>(<span style="color: #666666">9*</span>y<span style="color: #666666">+1</span>))
|
||||
term3 <span style="color: #666666">=</span> <span style="color: #666666">0.5*</span>np<span style="color: #666666">.</span>exp(<span style="color: #666666">-</span>(<span style="color: #666666">9*</span>x<span style="color: #666666">-7</span>)<span style="color: #666666">**2/4.0</span> <span style="color: #666666">-</span> <span style="color: #666666">0.25*</span>((<span style="color: #666666">9*</span>y<span style="color: #666666">-3</span>)<span style="color: #666666">**2</span>))
|
||||
term4 <span style="color: #666666">=</span> <span style="color: #666666">-0.2*</span>np<span style="color: #666666">.</span>exp(<span style="color: #666666">-</span>(<span style="color: #666666">9*</span>x<span style="color: #666666">-4</span>)<span style="color: #666666">**2</span> <span style="color: #666666">-</span> (<span style="color: #666666">9*</span>y<span style="color: #666666">-7</span>)<span style="color: #666666">**2</span>)
|
||||
<span style="color: #008000; font-weight: bold">return</span> term1 <span style="color: #666666">+</span> term2 <span style="color: #666666">+</span> term3 <span style="color: #666666">+</span> term4
|
||||
|
||||
|
||||
z <span style="color: #666666">=</span> FrankeFunction(x, y)
|
||||
|
||||
<span style="color: #408080; font-style: italic"># Plot the surface.</span>
|
||||
surf <span style="color: #666666">=</span> ax<span style="color: #666666">.</span>plot_surface(x, y, z, cmap<span style="color: #666666">=</span>cm<span style="color: #666666">.</span>coolwarm,
|
||||
linewidth<span style="color: #666666">=0</span>, antialiased<span style="color: #666666">=</span><span style="color: #008000; font-weight: bold">False</span>)
|
||||
|
||||
<span style="color: #408080; font-style: italic"># Customize the z axis.</span>
|
||||
ax<span style="color: #666666">.</span>set_zlim(<span style="color: #666666">-0.10</span>, <span style="color: #666666">1.40</span>)
|
||||
ax<span style="color: #666666">.</span>zaxis<span style="color: #666666">.</span>set_major_locator(LinearLocator(<span style="color: #666666">10</span>))
|
||||
ax<span style="color: #666666">.</span>zaxis<span style="color: #666666">.</span>set_major_formatter(FormatStrFormatter(<span style="color: #BA2121">'</span><span style="color: #BB6688; font-weight: bold">%.02f</span><span style="color: #BA2121">'</span>))
|
||||
|
||||
<span style="color: #408080; font-style: italic"># Add a color bar which maps values to colors.</span>
|
||||
fig<span style="color: #666666">.</span>colorbar(surf, shrink<span style="color: #666666">=0.5</span>, aspect<span style="color: #666666">=5</span>)
|
||||
|
||||
plt<span style="color: #666666">.</span>show()
|
||||
</pre></div>
|
||||
<p>
|
||||
We will generate our own dataset for a function
|
||||
\( \mathrm{FrankeFunction}(x,y) \) with \( x,y \in [0,1] \). The function
|
||||
\( f(x,y) \) is the Franke function. You should explore also the addition
|
||||
an added stochastic noise to this function using the normal
|
||||
distribution \( \cal{N}(0,1) \).
|
||||
|
||||
<p>
|
||||
Write your own code (using either a matrix inversion or a singular
|
||||
value decomposition from e.g., <b>numpy</b> ) or use your code and perform a standard least square regression
|
||||
analysis using polynomials in \( x \) and \( y \) up to fifth order. You can use <b>scikit-learn</b> as well.
|
||||
|
||||
<p>
|
||||
Evaluate the Mean Squared error (MSE)
|
||||
|
||||
$$ MSE(\hat{y},\hat{\tilde{y}}) = \frac{1}{n}
|
||||
\sum_{i=0}^{n-1}(y_i-\tilde{y}_i)^2,
|
||||
$$
|
||||
|
||||
<p>
|
||||
and the \( R^2 \) score function. If \( \tilde{\hat{y}}_i \) is the predicted
|
||||
value of the \( i-th \) sample and \( y_i \) is the corresponding true value,
|
||||
then the score \( R^2 \) is defined as
|
||||
|
||||
$$
|
||||
R^2(\hat{y}, \tilde{\hat{y}}) = 1 - \frac{\sum_{i=0}^{n - 1} (y_i - \tilde{y}_i)^2}{\sum_{i=0}^{n - 1} (y_i - \bar{y})^2},
|
||||
$$
|
||||
|
||||
<p>
|
||||
where we have defined the mean value of \( \hat{y} \) as
|
||||
|
||||
$$
|
||||
\bar{y} = \frac{1}{n} \sum_{i=0}^{n - 1} y_i.
|
||||
$$
|
||||
|
||||
<p>
|
||||
You should split your data in train and test and also consider scaling the data.
|
||||
|
||||
<p>
|
||||
To set up the design matrix, the following code can be used
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%;"><span></span><span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">FrankeFunction</span>(x,y):
|
||||
term1 <span style="color: #666666">=</span> <span style="color: #666666">0.75*</span>np<span style="color: #666666">.</span>exp(<span style="color: #666666">-</span>(<span style="color: #666666">0.25*</span>(<span style="color: #666666">9*</span>x<span style="color: #666666">-2</span>)<span style="color: #666666">**2</span>) <span style="color: #666666">-</span> <span style="color: #666666">0.25*</span>((<span style="color: #666666">9*</span>y<span style="color: #666666">-2</span>)<span style="color: #666666">**2</span>))
|
||||
term2 <span style="color: #666666">=</span> <span style="color: #666666">0.75*</span>np<span style="color: #666666">.</span>exp(<span style="color: #666666">-</span>((<span style="color: #666666">9*</span>x<span style="color: #666666">+1</span>)<span style="color: #666666">**2</span>)<span style="color: #666666">/49.0</span> <span style="color: #666666">-</span> <span style="color: #666666">0.1*</span>(<span style="color: #666666">9*</span>y<span style="color: #666666">+1</span>))
|
||||
term3 <span style="color: #666666">=</span> <span style="color: #666666">0.5*</span>np<span style="color: #666666">.</span>exp(<span style="color: #666666">-</span>(<span style="color: #666666">9*</span>x<span style="color: #666666">-7</span>)<span style="color: #666666">**2/4.0</span> <span style="color: #666666">-</span> <span style="color: #666666">0.25*</span>((<span style="color: #666666">9*</span>y<span style="color: #666666">-3</span>)<span style="color: #666666">**2</span>))
|
||||
term4 <span style="color: #666666">=</span> <span style="color: #666666">-0.2*</span>np<span style="color: #666666">.</span>exp(<span style="color: #666666">-</span>(<span style="color: #666666">9*</span>x<span style="color: #666666">-4</span>)<span style="color: #666666">**2</span> <span style="color: #666666">-</span> (<span style="color: #666666">9*</span>y<span style="color: #666666">-7</span>)<span style="color: #666666">**2</span>)
|
||||
<span style="color: #008000; font-weight: bold">return</span> term1 <span style="color: #666666">+</span> term2 <span style="color: #666666">+</span> term3 <span style="color: #666666">+</span> term4
|
||||
|
||||
|
||||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">create_X</span>(x, y, n ):
|
||||
<span style="color: #008000; font-weight: bold">if</span> <span style="color: #008000">len</span>(x<span style="color: #666666">.</span>shape) <span style="color: #666666">></span> <span style="color: #666666">1</span>:
|
||||
x <span style="color: #666666">=</span> np<span style="color: #666666">.</span>ravel(x)
|
||||
y <span style="color: #666666">=</span> np<span style="color: #666666">.</span>ravel(y)
|
||||
|
||||
N <span style="color: #666666">=</span> <span style="color: #008000">len</span>(x)
|
||||
l <span style="color: #666666">=</span> <span style="color: #008000">int</span>((n<span style="color: #666666">+1</span>)<span style="color: #666666">*</span>(n<span style="color: #666666">+2</span>)<span style="color: #666666">/2</span>) <span style="color: #408080; font-style: italic"># Number of elements in beta</span>
|
||||
X <span style="color: #666666">=</span> np<span style="color: #666666">.</span>ones((N,l))
|
||||
|
||||
<span style="color: #008000; font-weight: bold">for</span> i <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(<span style="color: #666666">1</span>,n<span style="color: #666666">+1</span>):
|
||||
q <span style="color: #666666">=</span> <span style="color: #008000">int</span>((i)<span style="color: #666666">*</span>(i<span style="color: #666666">+1</span>)<span style="color: #666666">/2</span>)
|
||||
<span style="color: #008000; font-weight: bold">for</span> k <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(i<span style="color: #666666">+1</span>):
|
||||
X[:,q<span style="color: #666666">+</span>k] <span style="color: #666666">=</span> (x<span style="color: #666666">**</span>(i<span style="color: #666666">-</span>k))<span style="color: #666666">*</span>(y<span style="color: #666666">**</span>k)
|
||||
|
||||
<span style="color: #008000; font-weight: bold">return</span> X
|
||||
|
||||
|
||||
<span style="color: #408080; font-style: italic"># Making meshgrid of datapoints and compute Franke's function</span>
|
||||
n <span style="color: #666666">=</span> <span style="color: #666666">5</span>
|
||||
N <span style="color: #666666">=</span> <span style="color: #666666">1000</span>
|
||||
x <span style="color: #666666">=</span> np<span style="color: #666666">.</span>sort(np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>uniform(<span style="color: #666666">0</span>, <span style="color: #666666">1</span>, N))
|
||||
y <span style="color: #666666">=</span> np<span style="color: #666666">.</span>sort(np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>uniform(<span style="color: #666666">0</span>, <span style="color: #666666">1</span>, N))
|
||||
z <span style="color: #666666">=</span> FrankeFunction(x, y)
|
||||
X <span style="color: #666666">=</span> create_X(x, y, n<span style="color: #666666">=</span>n)
|
||||
</pre></div>
|
||||
<p>
|
||||
Write then your own code for the Ridge method or use <b>Scikit-Learn</b>.
|
||||
Perform the same analysis as you did for ordinary Least Squares (for the same polynomials) but now for different values of \( \lambda \). Compare and
|
||||
analyze your results with those obtained with ordinary Least Squares. Study the
|
||||
dependence on \( \lambda \).
|
||||
|
||||
<p>
|
||||
This part is essentially a repeat of the previous ones, but now
|
||||
with Lasso regression. Write either your own code or
|
||||
use the functionalities of <b>Scikit-Learn</b> (recommended).
|
||||
Give a
|
||||
critical discussion of the three methods and a judgement of which
|
||||
model fits the data best.
|
||||
|
||||
<p>
|
||||
<!-- --- end exercise --- -->
|
||||
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
|
||||
Binary file not shown.
@@ -1619,17 +1619,20 @@
|
||||
"display(XPandas)\n",
|
||||
"print(XPandas.mean())\n",
|
||||
"print(XPandas.std())\n",
|
||||
"XPandas = XPandas -XPandas,mean()\n",
|
||||
"XPandas = (XPandas -XPandas.mean())\n",
|
||||
"display(XPandas)\n",
|
||||
"scaler = StandardScaler()\n",
|
||||
"Xscaled = scaler.transform(a)\n",
|
||||
"print(Xscaled)"
|
||||
"scaler = StandardScaler(with_std=False)\n",
|
||||
"scaler.fit(X)\n",
|
||||
"Xscaled = scaler.transform(X)\n",
|
||||
"display(XPandas-Xscaled)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"Small exercise: perform the standars scaling by including the standard deviation.\n",
|
||||
"\n",
|
||||
"## Min-Max Scaling\n",
|
||||
"\n",
|
||||
"Another commonly used scaling method is min-max scaling. This is very\n",
|
||||
@@ -1654,7 +1657,93 @@
|
||||
"where $\\min(x_j)$ and $\\max(x_j)$ return the minimum and maximum value of $x_j$ over the data set, respectively.\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"## Simple preprocessing examples, Franke function and regression"
|
||||
"## Testing the Means Squared Error as function of Complexity\n",
|
||||
"One of \n",
|
||||
"the aims is to reproduce Figure 2.11 of [Hastie et al](https://github.com/CompPhysics/MLErasmus/blob/master/doc/Textbooks/elementsstat.pdf).\n",
|
||||
"We will also use Ridge and Lasso regression. \n",
|
||||
"\n",
|
||||
"\n",
|
||||
"Our data is defined by $x\\in [-3,3]$ with a total of for example $100$ data points."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"collapsed": false,
|
||||
"editable": true
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"np.random.seed()\n",
|
||||
"n = 100\n",
|
||||
"maxdegree = 14\n",
|
||||
"# Make data set.\n",
|
||||
"x = np.linspace(-3, 3, n).reshape(-1, 1)\n",
|
||||
"y = np.exp(-x**2) + 1.5 * np.exp(-(x-2)**2)+ np.random.normal(0, 0.1, x.shape)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"where $y$ is the function we want to fit with a given polynomial.\n",
|
||||
"\n",
|
||||
"Write a first code which sets up a design matrix $X$ defined by a fifth-order polynomial. Scale your data and split it in training and test data."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"collapsed": false,
|
||||
"editable": true
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import matplotlib.pyplot as plt\n",
|
||||
"import numpy as np\n",
|
||||
"from sklearn.linear_model import LinearRegression, Ridge, Lasso\n",
|
||||
"from sklearn.preprocessing import PolynomialFeatures\n",
|
||||
"from sklearn.model_selection import train_test_split\n",
|
||||
"from sklearn.pipeline import make_pipeline\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"np.random.seed(2018)\n",
|
||||
"n = 50\n",
|
||||
"maxdegree = 5\n",
|
||||
"# Make data set.\n",
|
||||
"x = np.linspace(-3, 3, n).reshape(-1, 1)\n",
|
||||
"y = np.exp(-x**2) + 1.5 * np.exp(-(x-2)**2)+ np.random.normal(0, 0.1, x.shape)\n",
|
||||
"TestError = np.zeros(maxdegree)\n",
|
||||
"TrainError = np.zeros(maxdegree)\n",
|
||||
"polydegree = np.zeros(maxdegree)\n",
|
||||
"x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.2)\n",
|
||||
"scaler = StandardScaler()\n",
|
||||
"scaler.fit(X_train)\n",
|
||||
"x_train_scaled = scaler.transform(x_train)\n",
|
||||
"x_test_scaled = scaler.transform(x_test)\n",
|
||||
"\n",
|
||||
"for degree in range(maxdegree):\n",
|
||||
" model = make_pipeline(PolynomialFeatures(degree=degree), LinearRegression(fit_intercept=False))\n",
|
||||
" clf = model.fit(x_train_scale,y_train)\n",
|
||||
" y_fit = clf.predict(x_train_scaled)\n",
|
||||
" y_pred = clf.predict(x_test_scaled) \n",
|
||||
" polydegree[degree] = degree\n",
|
||||
" TestError[degree] = np.mean( np.mean((y_test - y_pred)**2) )\n",
|
||||
" TrainError[degree] = np.mean( np.mean((y_train - y_fit)**2) )\n",
|
||||
"\n",
|
||||
"plt.plot(polydegree, TestError, label='Test Error')\n",
|
||||
"plt.plot(polydegree, TrainError, label='Train Error')\n",
|
||||
"plt.legend()\n",
|
||||
"plt.show()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## More preprocessing examples, Franke function and regression"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -1764,10 +1853,9 @@
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Friday September 3\n",
|
||||
"\n",
|
||||
"Lasso and Ridge regression\n",
|
||||
"## Mathematical Interpretation of Ordinary Least Squares\n",
|
||||
"\n",
|
||||
"What is presented here is a mathematical analysis of various regression algorithms (ordinary least squares, Ridge and Lasso Regression). The analysis is based on an important algorithm in linear algebra, the so-called Singular Value Decomposition (SVD). \n",
|
||||
"\n",
|
||||
"\n",
|
||||
"## The singular value decomposition\n",
|
||||
@@ -1775,13 +1863,15 @@
|
||||
"\n",
|
||||
"The examples we have looked at so far are cases where we normally can\n",
|
||||
"invert the matrix $\\boldsymbol{X}^T\\boldsymbol{X}$. Using a polynomial expansion as we\n",
|
||||
"did both for the masses and the fitting of the equation of state,\n",
|
||||
"leads to row vectors of the design matrix which are essentially\n",
|
||||
"orthogonal due to the polynomial character of our model. Obtaining the inverse of the design matrix is then often done via a so-called LU, QR or Cholesky decomposition. \n",
|
||||
"did both for the masses and the fitting of various functions leads to\n",
|
||||
"row vectors of the design matrix which are essentially orthogonal due\n",
|
||||
"to the polynomial character of our model. Obtaining the inverse of the\n",
|
||||
"design matrix is then often done via a so-called LU, QR or Cholesky\n",
|
||||
"decomposition.\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"This may\n",
|
||||
"As we will also see in the first project, \n",
|
||||
"this may\n",
|
||||
"however not the be case in general and a standard matrix inversion\n",
|
||||
"algorithm based on say LU, QR or Cholesky decomposition may lead to singularities. We will see examples of this below.\n",
|
||||
"\n",
|
||||
@@ -2256,9 +2346,9 @@
|
||||
"\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"## Friday September 12\n",
|
||||
"## Friday September 3\n",
|
||||
"\n",
|
||||
"[Video of Lecture](https://www.uio.no/studier/emner/matnat/fys/FYS-STK4155/h20/forelesningsvideoer/LectureSeptember11.mp4?vrtx=view-as-webpage) and [handwritten notes](https://github.com/CompPhysics/MachineLearning/blob/master/doc/HandWrittenNotes/NotesSeptember11.pdf)\n",
|
||||
"[Video of Lecture from 2020](https://www.uio.no/studier/emner/matnat/fys/FYS-STK4155/h20/forelesningsvideoer/LectureSeptember11.mp4?vrtx=view-as-webpage) and [handwritten notes](https://github.com/CompPhysics/MachineLearning/blob/master/doc/HandWrittenNotes/NotesSeptember11.pdf)\n",
|
||||
"\n",
|
||||
"More material will be added here, see handwritten notes also.\n",
|
||||
"\n",
|
||||
@@ -3152,7 +3242,436 @@
|
||||
"It is easy to generalize this to a matrix $\\boldsymbol{X}\\in {\\mathbb{R}}^{n\\times p}$.\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"## Linking with SVD"
|
||||
"## Linking with SVD\n",
|
||||
"\n",
|
||||
"More material will be added here.\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"## Exercises for week 37, September 6-10\n",
|
||||
"\n",
|
||||
"The exercises here are meant to prepare you for work with project 1. The first exercise is a follow-up of exercise 2 from week 36 August 30-September 3).\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"<!-- --- begin exercise --- -->\n",
|
||||
"\n",
|
||||
"## Exercise 1: Adding Ridge and Lasso Regression\n",
|
||||
"\n",
|
||||
"This exercise is a continuation of exercise 2 from exercise set 1 (week 36, August 30-September 3)). We will\n",
|
||||
"use the same function to generate our data set, still staying with a\n",
|
||||
"simple function $y(x)$ which we want to fit using linear regression,\n",
|
||||
"but now extending the analysis to include the Ridge and the Lasso\n",
|
||||
"regression methods. \n",
|
||||
"\n",
|
||||
"We will thus again generate our own dataset for a function $y(x)$ where \n",
|
||||
"$x \\in [0,1]$ and defined by random numbers computed with the uniform\n",
|
||||
"distribution. The function $y$ is a quadratic polynomial in $x$ with\n",
|
||||
"added stochastic noise according to the normal distribution $\\cal{N}(0,1)$.\n",
|
||||
"\n",
|
||||
"The following simple Python instructions define our $x$ and $y$ values (with 100 data points)."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"collapsed": false,
|
||||
"editable": true
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"x = np.random.rand(100)\n",
|
||||
"y = 2.0+5*x*x+0.1*np.random.randn(100)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"Write your own code for the Ridge method (see chapter 3.4 of Hastie *et al.*, equations (3.43) and (3.44)) and compute the parametrization for different values of $\\lambda$. Compare and analyze your results with those from exercise 3. Study the dependence on $\\lambda$ while also varying the strength of the noise in your expression for $y(x)$. \n",
|
||||
"\n",
|
||||
"\n",
|
||||
"The code here allows you to perform your own Ridge calculation and perform calculations for various values of the regularization parameter $\\lambda$. This program can easily be extended upon."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"collapsed": false,
|
||||
"editable": true
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import os\n",
|
||||
"import numpy as np\n",
|
||||
"import pandas as pd\n",
|
||||
"import matplotlib.pyplot as plt\n",
|
||||
"from sklearn.model_selection import train_test_split\n",
|
||||
"from sklearn.preprocessing import StandardScaler\n",
|
||||
"\n",
|
||||
"def R2(y_data, y_model):\n",
|
||||
" return 1 - np.sum((y_data - y_model) ** 2) / np.sum((y_data - np.mean(y_data)) ** 2)\n",
|
||||
"def MSE(y_data,y_model):\n",
|
||||
" n = np.size(y_model)\n",
|
||||
" return np.sum((y_data-y_model)**2)/n\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"# A seed just to ensure that the random numbers are the same for every run.\n",
|
||||
"# Useful for eventual debugging.\n",
|
||||
"np.random.seed(3155)\n",
|
||||
"\n",
|
||||
"x = np.random.rand(100)\n",
|
||||
"y = 2.0+5*x*x+0.1*np.random.randn(100)\n",
|
||||
"\n",
|
||||
"# number of features p (here degree of polynomial\n",
|
||||
"p = 3\n",
|
||||
"# The design matrix now as function of a given polynomial\n",
|
||||
"X = np.zeros((len(x),p))\n",
|
||||
"X[:,0] = 1.0\n",
|
||||
"X[:,1] = x\n",
|
||||
"X[:,2] = x*x\n",
|
||||
"# We split the data in test and training data\n",
|
||||
"X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)\n",
|
||||
"scaler = StandardScaler()\n",
|
||||
"scaler.fit(X_train)\n",
|
||||
"X_train_scaled = scaler.transform(X_train)\n",
|
||||
"X_test_scaled = scaler.transform(X_test)\n",
|
||||
"\n",
|
||||
"# matrix inversion to find beta\n",
|
||||
"OLSbeta = np.linalg.inv(X_train.T @ X_train) @ X_train.T @ y_train\n",
|
||||
"print(OLSbeta)\n",
|
||||
"# and then make the prediction\n",
|
||||
"ytildeOLS = X_train @ OLSbeta\n",
|
||||
"print(\"Training R2 for OLS\")\n",
|
||||
"print(R2(y_train,ytildeOLS))\n",
|
||||
"print(\"Training MSE for OLS\")\n",
|
||||
"print(MSE(y_train,ytildeOLS))\n",
|
||||
"ypredictOLS = X_test @ OLSbeta\n",
|
||||
"print(\"Test R2 for OLS\")\n",
|
||||
"print(R2(y_test,ypredictOLS))\n",
|
||||
"print(\"Test MSE OLS\")\n",
|
||||
"print(MSE(y_test,ypredictOLS))\n",
|
||||
"\n",
|
||||
"# Repeat now for Ridge regression and various values of the regularization parameter\n",
|
||||
"I = np.eye(p,p)\n",
|
||||
"# Decide which values of lambda to use\n",
|
||||
"nlambdas = 20\n",
|
||||
"MSEPredict = np.zeros(nlambdas)\n",
|
||||
"MSETrain = np.zeros(nlambdas)\n",
|
||||
"lambdas = np.logspace(-4, 1, nlambdas)\n",
|
||||
"for i in range(nlambdas):\n",
|
||||
" lmb = lambdas[i]\n",
|
||||
" Ridgebeta = np.linalg.inv(X_train.T @ X_train+lmb*I) @ X_train.T @ y_train\n",
|
||||
" # and then make the prediction\n",
|
||||
" ytildeRidge = X_train @ Ridgebeta\n",
|
||||
" ypredictRidge = X_test @ Ridgebeta\n",
|
||||
" MSEPredict[i] = MSE(y_test,ypredictRidge)\n",
|
||||
" MSETrain[i] = MSE(y_train,ytildeRidge)\n",
|
||||
"# Now plot the results\n",
|
||||
"plt.figure()\n",
|
||||
"plt.plot(np.log10(lambdas), MSETrain, label = 'MSE Ridge train')\n",
|
||||
"plt.plot(np.log10(lambdas), MSEPredict, 'r--', label = 'MSE Ridge Test')\n",
|
||||
"plt.xlabel('log10(lambda)')\n",
|
||||
"plt.ylabel('MSE')\n",
|
||||
"plt.legend()\n",
|
||||
"plt.show()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"Repeat the above but using the functionality of **Scikit-Learn**. Compare your code with the results from **Scikit-Learn**. Remember to run with the same random numbers for generating $x$ and $y$. \n",
|
||||
"\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"Finally, using **Scikit-Learn** or your own code, compute also the mean square error, a risk metric corresponding to the expected value of the squared (quadratic) error defined as"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"$$\n",
|
||||
"MSE(\\hat{y},\\hat{\\tilde{y}}) = \\frac{1}{n}\n",
|
||||
"\\sum_{i=0}^{n-1}(y_i-\\tilde{y}_i)^2,\n",
|
||||
"$$"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"and the $R^2$ score function.\n",
|
||||
"If $\\tilde{\\hat{y}}_i$ is the predicted value of the $i-th$ sample and $y_i$ is the corresponding true value, then the score $R^2$ is defined as"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"$$\n",
|
||||
"R^2(\\hat{y}, \\tilde{\\hat{y}}) = 1 - \\frac{\\sum_{i=0}^{n - 1} (y_i - \\tilde{y}_i)^2}{\\sum_{i=0}^{n - 1} (y_i - \\bar{y})^2},\n",
|
||||
"$$"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"where we have defined the mean value of $\\hat{y}$ as"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"$$\n",
|
||||
"\\bar{y} = \\frac{1}{n} \\sum_{i=0}^{n - 1} y_i.\n",
|
||||
"$$"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"Discuss these quantities as functions of the variable $\\lambda$ in the Ridge and Lasso regression methods.\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"### Exercise: Linear Regression for a two-dimensional function\n",
|
||||
"\n",
|
||||
"This is a longer exercise and the aim is to study in more detail various\n",
|
||||
"regression methods, including the Ordinary Least Squares (OLS) method,\n",
|
||||
"Ridge regression and finally Lasso regression.\n",
|
||||
"This exercise forms a part of project 1.\n",
|
||||
"\n",
|
||||
"We will study how to fit polynomials to a specific\n",
|
||||
"two-dimensional function called [Franke's\n",
|
||||
"function](http://www.dtic.mil/dtic/tr/fulltext/u2/a081688.pdf). This\n",
|
||||
"is a function which has been widely used when testing various\n",
|
||||
"interpolation and fitting algorithms. \n",
|
||||
"\n",
|
||||
"The Franke function, which is a weighted sum of four exponentials reads as follows"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"$$\n",
|
||||
"\\begin{align*}\n",
|
||||
"f(x,y) &= \\frac{3}{4}\\exp{\\left(-\\frac{(9x-2)^2}{4} - \\frac{(9y-2)^2}{4}\\right)}+\\frac{3}{4}\\exp{\\left(-\\frac{(9x+1)^2}{49}- \\frac{(9y+1)}{10}\\right)} \\\\\n",
|
||||
"&+\\frac{1}{2}\\exp{\\left(-\\frac{(9x-7)^2}{4} - \\frac{(9y-3)^2}{4}\\right)} -\\frac{1}{5}\\exp{\\left(-(9x-4)^2 - (9y-7)^2\\right) }.\n",
|
||||
"\\end{align*}\n",
|
||||
"$$"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"The function will be defined for $x,y\\in [0,1]$. Our first step will\n",
|
||||
"be to perform an OLS regression analysis of this function, trying out\n",
|
||||
"a polynomial fit with an $x$ and $y$ dependence of the form $[x, y,\n",
|
||||
"x^2, y^2, xy, \\dots]$. We will fit a\n",
|
||||
"function (for example a polynomial) of $x$ and $y$. Thereafter we\n",
|
||||
"will repeat much of the same procedure using the Ridge and Lasso\n",
|
||||
"regression methods, introducing thus a dependence on the bias\n",
|
||||
"(penalty) $\\lambda$.\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"The Python fucntion for the Franke function is included here (it performs also a three-dimensional plot of it)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"collapsed": false,
|
||||
"editable": true
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from mpl_toolkits.mplot3d import Axes3D\n",
|
||||
"import matplotlib.pyplot as plt\n",
|
||||
"from matplotlib import cm\n",
|
||||
"from matplotlib.ticker import LinearLocator, FormatStrFormatter\n",
|
||||
"import numpy as np\n",
|
||||
"from random import random, seed\n",
|
||||
"\n",
|
||||
"fig = plt.figure()\n",
|
||||
"ax = fig.gca(projection='3d')\n",
|
||||
"\n",
|
||||
"# Make data.\n",
|
||||
"x = np.arange(0, 1, 0.05)\n",
|
||||
"y = np.arange(0, 1, 0.05)\n",
|
||||
"x, y = np.meshgrid(x,y)\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def FrankeFunction(x,y):\n",
|
||||
" term1 = 0.75*np.exp(-(0.25*(9*x-2)**2) - 0.25*((9*y-2)**2))\n",
|
||||
" term2 = 0.75*np.exp(-((9*x+1)**2)/49.0 - 0.1*(9*y+1))\n",
|
||||
" term3 = 0.5*np.exp(-(9*x-7)**2/4.0 - 0.25*((9*y-3)**2))\n",
|
||||
" term4 = -0.2*np.exp(-(9*x-4)**2 - (9*y-7)**2)\n",
|
||||
" return term1 + term2 + term3 + term4\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"z = FrankeFunction(x, y)\n",
|
||||
"\n",
|
||||
"# Plot the surface.\n",
|
||||
"surf = ax.plot_surface(x, y, z, cmap=cm.coolwarm,\n",
|
||||
" linewidth=0, antialiased=False)\n",
|
||||
"\n",
|
||||
"# Customize the z axis.\n",
|
||||
"ax.set_zlim(-0.10, 1.40)\n",
|
||||
"ax.zaxis.set_major_locator(LinearLocator(10))\n",
|
||||
"ax.zaxis.set_major_formatter(FormatStrFormatter('%.02f'))\n",
|
||||
"\n",
|
||||
"# Add a color bar which maps values to colors.\n",
|
||||
"fig.colorbar(surf, shrink=0.5, aspect=5)\n",
|
||||
"\n",
|
||||
"plt.show()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"We will generate our own dataset for a function\n",
|
||||
"$\\mathrm{FrankeFunction}(x,y)$ with $x,y \\in [0,1]$. The function\n",
|
||||
"$f(x,y)$ is the Franke function. You should explore also the addition\n",
|
||||
"an added stochastic noise to this function using the normal\n",
|
||||
"distribution $\\cal{N}(0,1)$.\n",
|
||||
"\n",
|
||||
"Write your own code (using either a matrix inversion or a singular\n",
|
||||
"value decomposition from e.g., **numpy** ) or use your code and perform a standard least square regression\n",
|
||||
"analysis using polynomials in $x$ and $y$ up to fifth order. You can use **scikit-learn** as well.\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"Evaluate the Mean Squared error (MSE)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"$$\n",
|
||||
"MSE(\\hat{y},\\hat{\\tilde{y}}) = \\frac{1}{n}\n",
|
||||
"\\sum_{i=0}^{n-1}(y_i-\\tilde{y}_i)^2,\n",
|
||||
"$$"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"and the $R^2$ score function. If $\\tilde{\\hat{y}}_i$ is the predicted\n",
|
||||
"value of the $i-th$ sample and $y_i$ is the corresponding true value,\n",
|
||||
"then the score $R^2$ is defined as"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"$$\n",
|
||||
"R^2(\\hat{y}, \\tilde{\\hat{y}}) = 1 - \\frac{\\sum_{i=0}^{n - 1} (y_i - \\tilde{y}_i)^2}{\\sum_{i=0}^{n - 1} (y_i - \\bar{y})^2},\n",
|
||||
"$$"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"where we have defined the mean value of $\\hat{y}$ as"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"$$\n",
|
||||
"\\bar{y} = \\frac{1}{n} \\sum_{i=0}^{n - 1} y_i.\n",
|
||||
"$$"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"You should split your data in train and test and also consider scaling the data.\n",
|
||||
"\n",
|
||||
"To set up the design matrix, the following code can be used"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"collapsed": false,
|
||||
"editable": true
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def FrankeFunction(x,y):\n",
|
||||
"\tterm1 = 0.75*np.exp(-(0.25*(9*x-2)**2) - 0.25*((9*y-2)**2))\n",
|
||||
"\tterm2 = 0.75*np.exp(-((9*x+1)**2)/49.0 - 0.1*(9*y+1))\n",
|
||||
"\tterm3 = 0.5*np.exp(-(9*x-7)**2/4.0 - 0.25*((9*y-3)**2))\n",
|
||||
"\tterm4 = -0.2*np.exp(-(9*x-4)**2 - (9*y-7)**2)\n",
|
||||
"\treturn term1 + term2 + term3 + term4\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def create_X(x, y, n ):\n",
|
||||
"\tif len(x.shape) > 1:\n",
|
||||
"\t\tx = np.ravel(x)\n",
|
||||
"\t\ty = np.ravel(y)\n",
|
||||
"\n",
|
||||
"\tN = len(x)\n",
|
||||
"\tl = int((n+1)*(n+2)/2)\t\t# Number of elements in beta\n",
|
||||
"\tX = np.ones((N,l))\n",
|
||||
"\n",
|
||||
"\tfor i in range(1,n+1):\n",
|
||||
"\t\tq = int((i)*(i+1)/2)\n",
|
||||
"\t\tfor k in range(i+1):\n",
|
||||
"\t\t\tX[:,q+k] = (x**(i-k))*(y**k)\n",
|
||||
"\n",
|
||||
"\treturn X\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"# Making meshgrid of datapoints and compute Franke's function\n",
|
||||
"n = 5\n",
|
||||
"N = 1000\n",
|
||||
"x = np.sort(np.random.uniform(0, 1, N))\n",
|
||||
"y = np.sort(np.random.uniform(0, 1, N))\n",
|
||||
"z = FrankeFunction(x, y)\n",
|
||||
"X = create_X(x, y, n=n)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"Write then your own code for the Ridge method or use **Scikit-Learn**.\n",
|
||||
"Perform the same analysis as you did for ordinary Least Squares (for the same polynomials) but now for different values of $\\lambda$. Compare and\n",
|
||||
"analyze your results with those obtained with ordinary Least Squares. Study the\n",
|
||||
"dependence on $\\lambda$.\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"This part is essentially a repeat of the previous ones, but now\n",
|
||||
"with Lasso regression. Write either your own code or\n",
|
||||
"use the functionalities of **Scikit-Learn** (recommended). \n",
|
||||
"Give a\n",
|
||||
"critical discussion of the three methods and a judgement of which\n",
|
||||
"model fits the data best.\n",
|
||||
"\n",
|
||||
"<!-- --- end exercise --- -->"
|
||||
]
|
||||
}
|
||||
],
|
||||
|
||||
+401
-16
@@ -965,14 +965,15 @@ XPandas = pd.DataFrame(X)
|
||||
display(XPandas)
|
||||
print(XPandas.mean())
|
||||
print(XPandas.std())
|
||||
XPandas = XPandas -XPandas,mean()
|
||||
XPandas = (XPandas -XPandas.mean())
|
||||
display(XPandas)
|
||||
scaler = StandardScaler()
|
||||
Xscaled = scaler.transform(a)
|
||||
print(Xscaled)
|
||||
scaler = StandardScaler(with_std=False)
|
||||
scaler.fit(X)
|
||||
Xscaled = scaler.transform(X)
|
||||
display(XPandas-Xscaled)
|
||||
!ec
|
||||
|
||||
|
||||
Small exercise: perform the standars scaling by including the standard deviation.
|
||||
|
||||
!split
|
||||
===== Min-Max Scaling =====
|
||||
@@ -991,7 +992,67 @@ where $\min(x_j)$ and $\max(x_j)$ return the minimum and maximum value of $x_j$
|
||||
|
||||
|
||||
!split
|
||||
===== Simple preprocessing examples, Franke function and regression =====
|
||||
===== Testing the Means Squared Error as function of Complexity =====
|
||||
One of
|
||||
the aims is to reproduce Figure 2.11 of "Hastie et al":"https://github.com/CompPhysics/MLErasmus/blob/master/doc/Textbooks/elementsstat.pdf".
|
||||
We will also use Ridge and Lasso regression.
|
||||
|
||||
|
||||
Our data is defined by $x\in [-3,3]$ with a total of for example $100$ data points.
|
||||
!bc pycod
|
||||
np.random.seed()
|
||||
n = 100
|
||||
maxdegree = 14
|
||||
# Make data set.
|
||||
x = np.linspace(-3, 3, n).reshape(-1, 1)
|
||||
y = np.exp(-x**2) + 1.5 * np.exp(-(x-2)**2)+ np.random.normal(0, 0.1, x.shape)
|
||||
!ec
|
||||
where $y$ is the function we want to fit with a given polynomial.
|
||||
|
||||
Write a first code which sets up a design matrix $X$ defined by a fifth-order polynomial. Scale your data and split it in training and test data.
|
||||
!bc pycod
|
||||
import matplotlib.pyplot as plt
|
||||
import numpy as np
|
||||
from sklearn.linear_model import LinearRegression, Ridge, Lasso
|
||||
from sklearn.preprocessing import PolynomialFeatures
|
||||
from sklearn.model_selection import train_test_split
|
||||
from sklearn.pipeline import make_pipeline
|
||||
|
||||
|
||||
np.random.seed(2018)
|
||||
n = 50
|
||||
maxdegree = 5
|
||||
# Make data set.
|
||||
x = np.linspace(-3, 3, n).reshape(-1, 1)
|
||||
y = np.exp(-x**2) + 1.5 * np.exp(-(x-2)**2)+ np.random.normal(0, 0.1, x.shape)
|
||||
TestError = np.zeros(maxdegree)
|
||||
TrainError = np.zeros(maxdegree)
|
||||
polydegree = np.zeros(maxdegree)
|
||||
x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.2)
|
||||
scaler = StandardScaler()
|
||||
scaler.fit(X_train)
|
||||
x_train_scaled = scaler.transform(x_train)
|
||||
x_test_scaled = scaler.transform(x_test)
|
||||
|
||||
for degree in range(maxdegree):
|
||||
model = make_pipeline(PolynomialFeatures(degree=degree), LinearRegression(fit_intercept=False))
|
||||
clf = model.fit(x_train_scale,y_train)
|
||||
y_fit = clf.predict(x_train_scaled)
|
||||
y_pred = clf.predict(x_test_scaled)
|
||||
polydegree[degree] = degree
|
||||
TestError[degree] = np.mean( np.mean((y_test - y_pred)**2) )
|
||||
TrainError[degree] = np.mean( np.mean((y_train - y_fit)**2) )
|
||||
|
||||
plt.plot(polydegree, TestError, label='Test Error')
|
||||
plt.plot(polydegree, TrainError, label='Train Error')
|
||||
plt.legend()
|
||||
plt.show()
|
||||
!ec
|
||||
|
||||
|
||||
|
||||
!split
|
||||
===== More preprocessing examples, Franke function and regression =====
|
||||
|
||||
!bc pycod
|
||||
# Common imports
|
||||
@@ -1091,10 +1152,9 @@ print("R2 score for scaled data: {:.2f}".format(clf.score(X_test_scaled,y_test)
|
||||
|
||||
|
||||
!split
|
||||
===== Friday September 3 =====
|
||||
|
||||
Lasso and Ridge regression
|
||||
===== Mathematical Interpretation of Ordinary Least Squares =====
|
||||
|
||||
What is presented here is a mathematical analysis of various regression algorithms (ordinary least squares, Ridge and Lasso Regression). The analysis is based on an important algorithm in linear algebra, the so-called Singular Value Decomposition (SVD).
|
||||
|
||||
|
||||
!split
|
||||
@@ -1104,13 +1164,15 @@ Lasso and Ridge regression
|
||||
|
||||
The examples we have looked at so far are cases where we normally can
|
||||
invert the matrix $\bm{X}^T\bm{X}$. Using a polynomial expansion as we
|
||||
did both for the masses and the fitting of the equation of state,
|
||||
leads to row vectors of the design matrix which are essentially
|
||||
orthogonal due to the polynomial character of our model. Obtaining the inverse of the design matrix is then often done via a so-called LU, QR or Cholesky decomposition.
|
||||
did both for the masses and the fitting of various functions leads to
|
||||
row vectors of the design matrix which are essentially orthogonal due
|
||||
to the polynomial character of our model. Obtaining the inverse of the
|
||||
design matrix is then often done via a so-called LU, QR or Cholesky
|
||||
decomposition.
|
||||
|
||||
|
||||
|
||||
This may
|
||||
As we will also see in the first project,
|
||||
this may
|
||||
however not the be case in general and a standard matrix inversion
|
||||
algorithm based on say LU, QR or Cholesky decomposition may lead to singularities. We will see examples of this below.
|
||||
|
||||
@@ -1419,9 +1481,9 @@ Here we have that $${\bf X} = {\bf U}{\bf \Sigma}{\bf V}^T$$, with $$\Sigma$$ be
|
||||
|
||||
|
||||
!split
|
||||
===== Friday September 12 =====
|
||||
===== Friday September 3 =====
|
||||
|
||||
"Video of Lecture":"https://www.uio.no/studier/emner/matnat/fys/FYS-STK4155/h20/forelesningsvideoer/LectureSeptember11.mp4?vrtx=view-as-webpage" and "handwritten notes":"https://github.com/CompPhysics/MachineLearning/blob/master/doc/HandWrittenNotes/NotesSeptember11.pdf"
|
||||
"Video of Lecture from 2020":"https://www.uio.no/studier/emner/matnat/fys/FYS-STK4155/h20/forelesningsvideoer/LectureSeptember11.mp4?vrtx=view-as-webpage" and "handwritten notes":"https://github.com/CompPhysics/MachineLearning/blob/master/doc/HandWrittenNotes/NotesSeptember11.pdf"
|
||||
|
||||
More material will be added here, see handwritten notes also.
|
||||
|
||||
@@ -1980,4 +2042,327 @@ It is easy to generalize this to a matrix $\bm{X}\in {\mathbb{R}}^{n\times p}$.
|
||||
!split
|
||||
===== Linking with SVD =====
|
||||
|
||||
More material will be added here.
|
||||
|
||||
|
||||
!split
|
||||
===== Exercises for week 37, September 6-10 =====
|
||||
|
||||
The exercises here are meant to prepare you for work with project 1. The first exercise is a follow-up of exercise 2 from week 36 August 30-September 3).
|
||||
|
||||
|
||||
===== Exercise: Adding Ridge and Lasso Regression =====
|
||||
|
||||
|
||||
This exercise is a continuation of exercise 2 from exercise set 1 (week 36, August 30-September 3)). We will
|
||||
use the same function to generate our data set, still staying with a
|
||||
simple function $y(x)$ which we want to fit using linear regression,
|
||||
but now extending the analysis to include the Ridge and the Lasso
|
||||
regression methods.
|
||||
|
||||
We will thus again generate our own dataset for a function $y(x)$ where
|
||||
$x \in [0,1]$ and defined by random numbers computed with the uniform
|
||||
distribution. The function $y$ is a quadratic polynomial in $x$ with
|
||||
added stochastic noise according to the normal distribution $\cal{N}(0,1)$.
|
||||
|
||||
The following simple Python instructions define our $x$ and $y$ values (with 100 data points).
|
||||
!bc pycod
|
||||
x = np.random.rand(100)
|
||||
y = 2.0+5*x*x+0.1*np.random.randn(100)
|
||||
!ec
|
||||
|
||||
|
||||
Write your own code for the Ridge method (see chapter 3.4 of Hastie *et al.*, equations (3.43) and (3.44)) and compute the parametrization for different values of $\lambda$. Compare and analyze your results with those from exercise 3. Study the dependence on $\lambda$ while also varying the strength of the noise in your expression for $y(x)$.
|
||||
|
||||
|
||||
The code here allows you to perform your own Ridge calculation and perform calculations for various values of the regularization parameter $\lambda$. This program can easily be extended upon.
|
||||
|
||||
!bc pycod
|
||||
import os
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
import matplotlib.pyplot as plt
|
||||
from sklearn.model_selection import train_test_split
|
||||
from sklearn.preprocessing import StandardScaler
|
||||
|
||||
def R2(y_data, y_model):
|
||||
return 1 - np.sum((y_data - y_model) ** 2) / np.sum((y_data - np.mean(y_data)) ** 2)
|
||||
def MSE(y_data,y_model):
|
||||
n = np.size(y_model)
|
||||
return np.sum((y_data-y_model)**2)/n
|
||||
|
||||
|
||||
# A seed just to ensure that the random numbers are the same for every run.
|
||||
# Useful for eventual debugging.
|
||||
np.random.seed(3155)
|
||||
|
||||
x = np.random.rand(100)
|
||||
y = 2.0+5*x*x+0.1*np.random.randn(100)
|
||||
|
||||
# number of features p (here degree of polynomial
|
||||
p = 3
|
||||
# The design matrix now as function of a given polynomial
|
||||
X = np.zeros((len(x),p))
|
||||
X[:,0] = 1.0
|
||||
X[:,1] = x
|
||||
X[:,2] = x*x
|
||||
# We split the data in test and training data
|
||||
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)
|
||||
scaler = StandardScaler()
|
||||
scaler.fit(X_train)
|
||||
X_train_scaled = scaler.transform(X_train)
|
||||
X_test_scaled = scaler.transform(X_test)
|
||||
|
||||
# matrix inversion to find beta
|
||||
OLSbeta = np.linalg.inv(X_train.T @ X_train) @ X_train.T @ y_train
|
||||
print(OLSbeta)
|
||||
# and then make the prediction
|
||||
ytildeOLS = X_train @ OLSbeta
|
||||
print("Training R2 for OLS")
|
||||
print(R2(y_train,ytildeOLS))
|
||||
print("Training MSE for OLS")
|
||||
print(MSE(y_train,ytildeOLS))
|
||||
ypredictOLS = X_test @ OLSbeta
|
||||
print("Test R2 for OLS")
|
||||
print(R2(y_test,ypredictOLS))
|
||||
print("Test MSE OLS")
|
||||
print(MSE(y_test,ypredictOLS))
|
||||
|
||||
# Repeat now for Ridge regression and various values of the regularization parameter
|
||||
I = np.eye(p,p)
|
||||
# Decide which values of lambda to use
|
||||
nlambdas = 20
|
||||
MSEPredict = np.zeros(nlambdas)
|
||||
MSETrain = np.zeros(nlambdas)
|
||||
lambdas = np.logspace(-4, 1, nlambdas)
|
||||
for i in range(nlambdas):
|
||||
lmb = lambdas[i]
|
||||
Ridgebeta = np.linalg.inv(X_train.T @ X_train+lmb*I) @ X_train.T @ y_train
|
||||
# and then make the prediction
|
||||
ytildeRidge = X_train @ Ridgebeta
|
||||
ypredictRidge = X_test @ Ridgebeta
|
||||
MSEPredict[i] = MSE(y_test,ypredictRidge)
|
||||
MSETrain[i] = MSE(y_train,ytildeRidge)
|
||||
# Now plot the results
|
||||
plt.figure()
|
||||
plt.plot(np.log10(lambdas), MSETrain, label = 'MSE Ridge train')
|
||||
plt.plot(np.log10(lambdas), MSEPredict, 'r--', label = 'MSE Ridge Test')
|
||||
plt.xlabel('log10(lambda)')
|
||||
plt.ylabel('MSE')
|
||||
plt.legend()
|
||||
plt.show()
|
||||
!ec
|
||||
|
||||
|
||||
|
||||
Repeat the above but using the functionality of _Scikit-Learn_. Compare your code with the results from _Scikit-Learn_. Remember to run with the same random numbers for generating $x$ and $y$.
|
||||
|
||||
|
||||
|
||||
Finally, using _Scikit-Learn_ or your own code, compute also the mean square error, a risk metric corresponding to the expected value of the squared (quadratic) error defined as
|
||||
!bt
|
||||
\[ MSE(\hat{y},\hat{\tilde{y}}) = \frac{1}{n}
|
||||
\sum_{i=0}^{n-1}(y_i-\tilde{y}_i)^2,
|
||||
\]
|
||||
!et
|
||||
and the $R^2$ score function.
|
||||
If $\tilde{\hat{y}}_i$ is the predicted value of the $i-th$ sample and $y_i$ is the corresponding true value, then the score $R^2$ is defined as
|
||||
!bt
|
||||
\[
|
||||
R^2(\hat{y}, \tilde{\hat{y}}) = 1 - \frac{\sum_{i=0}^{n - 1} (y_i - \tilde{y}_i)^2}{\sum_{i=0}^{n - 1} (y_i - \bar{y})^2},
|
||||
\]
|
||||
!et
|
||||
where we have defined the mean value of $\hat{y}$ as
|
||||
!bt
|
||||
\[
|
||||
\bar{y} = \frac{1}{n} \sum_{i=0}^{n - 1} y_i.
|
||||
\]
|
||||
!et
|
||||
Discuss these quantities as functions of the variable $\lambda$ in the Ridge and Lasso regression methods.
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
=== Exercise: Linear Regression for a two-dimensional function ===
|
||||
|
||||
This is a longer exercise and the aim is to study in more detail various
|
||||
regression methods, including the Ordinary Least Squares (OLS) method,
|
||||
Ridge regression and finally Lasso regression.
|
||||
This exercise forms a part of project 1.
|
||||
|
||||
We will study how to fit polynomials to a specific
|
||||
two-dimensional function called "Franke's
|
||||
function":"http://www.dtic.mil/dtic/tr/fulltext/u2/a081688.pdf". This
|
||||
is a function which has been widely used when testing various
|
||||
interpolation and fitting algorithms.
|
||||
|
||||
The Franke function, which is a weighted sum of four exponentials reads as follows
|
||||
!bt
|
||||
\begin{align*}
|
||||
f(x,y) &= \frac{3}{4}\exp{\left(-\frac{(9x-2)^2}{4} - \frac{(9y-2)^2}{4}\right)}+\frac{3}{4}\exp{\left(-\frac{(9x+1)^2}{49}- \frac{(9y+1)}{10}\right)} \\
|
||||
&+\frac{1}{2}\exp{\left(-\frac{(9x-7)^2}{4} - \frac{(9y-3)^2}{4}\right)} -\frac{1}{5}\exp{\left(-(9x-4)^2 - (9y-7)^2\right) }.
|
||||
\end{align*}
|
||||
!et
|
||||
|
||||
The function will be defined for $x,y\in [0,1]$. Our first step will
|
||||
be to perform an OLS regression analysis of this function, trying out
|
||||
a polynomial fit with an $x$ and $y$ dependence of the form $[x, y,
|
||||
x^2, y^2, xy, \dots]$. We will fit a
|
||||
function (for example a polynomial) of $x$ and $y$. Thereafter we
|
||||
will repeat much of the same procedure using the Ridge and Lasso
|
||||
regression methods, introducing thus a dependence on the bias
|
||||
(penalty) $\lambda$.
|
||||
|
||||
|
||||
The Python fucntion for the Franke function is included here (it performs also a three-dimensional plot of it)
|
||||
!bc pycod
|
||||
from mpl_toolkits.mplot3d import Axes3D
|
||||
import matplotlib.pyplot as plt
|
||||
from matplotlib import cm
|
||||
from matplotlib.ticker import LinearLocator, FormatStrFormatter
|
||||
import numpy as np
|
||||
from random import random, seed
|
||||
|
||||
fig = plt.figure()
|
||||
ax = fig.gca(projection='3d')
|
||||
|
||||
# Make data.
|
||||
x = np.arange(0, 1, 0.05)
|
||||
y = np.arange(0, 1, 0.05)
|
||||
x, y = np.meshgrid(x,y)
|
||||
|
||||
|
||||
def FrankeFunction(x,y):
|
||||
term1 = 0.75*np.exp(-(0.25*(9*x-2)**2) - 0.25*((9*y-2)**2))
|
||||
term2 = 0.75*np.exp(-((9*x+1)**2)/49.0 - 0.1*(9*y+1))
|
||||
term3 = 0.5*np.exp(-(9*x-7)**2/4.0 - 0.25*((9*y-3)**2))
|
||||
term4 = -0.2*np.exp(-(9*x-4)**2 - (9*y-7)**2)
|
||||
return term1 + term2 + term3 + term4
|
||||
|
||||
|
||||
z = FrankeFunction(x, y)
|
||||
|
||||
# Plot the surface.
|
||||
surf = ax.plot_surface(x, y, z, cmap=cm.coolwarm,
|
||||
linewidth=0, antialiased=False)
|
||||
|
||||
# Customize the z axis.
|
||||
ax.set_zlim(-0.10, 1.40)
|
||||
ax.zaxis.set_major_locator(LinearLocator(10))
|
||||
ax.zaxis.set_major_formatter(FormatStrFormatter('%.02f'))
|
||||
|
||||
# Add a color bar which maps values to colors.
|
||||
fig.colorbar(surf, shrink=0.5, aspect=5)
|
||||
|
||||
plt.show()
|
||||
|
||||
!ec
|
||||
|
||||
|
||||
|
||||
We will generate our own dataset for a function
|
||||
$\mathrm{FrankeFunction}(x,y)$ with $x,y \in [0,1]$. The function
|
||||
$f(x,y)$ is the Franke function. You should explore also the addition
|
||||
an added stochastic noise to this function using the normal
|
||||
distribution $\cal{N}(0,1)$.
|
||||
|
||||
Write your own code (using either a matrix inversion or a singular
|
||||
value decomposition from e.g., _numpy_ ) or use your code and perform a standard least square regression
|
||||
analysis using polynomials in $x$ and $y$ up to fifth order. You can use _scikit-learn_ as well.
|
||||
|
||||
|
||||
Evaluate the Mean Squared error (MSE)
|
||||
|
||||
!bt
|
||||
\[ MSE(\hat{y},\hat{\tilde{y}}) = \frac{1}{n}
|
||||
\sum_{i=0}^{n-1}(y_i-\tilde{y}_i)^2,
|
||||
\]
|
||||
!et
|
||||
|
||||
and the $R^2$ score function. If $\tilde{\hat{y}}_i$ is the predicted
|
||||
value of the $i-th$ sample and $y_i$ is the corresponding true value,
|
||||
then the score $R^2$ is defined as
|
||||
|
||||
!bt
|
||||
\[
|
||||
R^2(\hat{y}, \tilde{\hat{y}}) = 1 - \frac{\sum_{i=0}^{n - 1} (y_i - \tilde{y}_i)^2}{\sum_{i=0}^{n - 1} (y_i - \bar{y})^2},
|
||||
\]
|
||||
!et
|
||||
|
||||
where we have defined the mean value of $\hat{y}$ as
|
||||
|
||||
!bt
|
||||
\[
|
||||
\bar{y} = \frac{1}{n} \sum_{i=0}^{n - 1} y_i.
|
||||
\]
|
||||
!et
|
||||
|
||||
|
||||
You should split your data in train and test and also consider scaling the data.
|
||||
|
||||
To set up the design matrix, the following code can be used
|
||||
!bc pycod
|
||||
def FrankeFunction(x,y):
|
||||
term1 = 0.75*np.exp(-(0.25*(9*x-2)**2) - 0.25*((9*y-2)**2))
|
||||
term2 = 0.75*np.exp(-((9*x+1)**2)/49.0 - 0.1*(9*y+1))
|
||||
term3 = 0.5*np.exp(-(9*x-7)**2/4.0 - 0.25*((9*y-3)**2))
|
||||
term4 = -0.2*np.exp(-(9*x-4)**2 - (9*y-7)**2)
|
||||
return term1 + term2 + term3 + term4
|
||||
|
||||
|
||||
def create_X(x, y, n ):
|
||||
if len(x.shape) > 1:
|
||||
x = np.ravel(x)
|
||||
y = np.ravel(y)
|
||||
|
||||
N = len(x)
|
||||
l = int((n+1)*(n+2)/2) # Number of elements in beta
|
||||
X = np.ones((N,l))
|
||||
|
||||
for i in range(1,n+1):
|
||||
q = int((i)*(i+1)/2)
|
||||
for k in range(i+1):
|
||||
X[:,q+k] = (x**(i-k))*(y**k)
|
||||
|
||||
return X
|
||||
|
||||
|
||||
# Making meshgrid of datapoints and compute Franke's function
|
||||
n = 5
|
||||
N = 1000
|
||||
x = np.sort(np.random.uniform(0, 1, N))
|
||||
y = np.sort(np.random.uniform(0, 1, N))
|
||||
z = FrankeFunction(x, y)
|
||||
X = create_X(x, y, n=n)
|
||||
!ec
|
||||
|
||||
|
||||
|
||||
Write then your own code for the Ridge method or use _Scikit-Learn_.
|
||||
Perform the same analysis as you did for ordinary Least Squares (for the same polynomials) but now for different values of $\lambda$. Compare and
|
||||
analyze your results with those obtained with ordinary Least Squares. Study the
|
||||
dependence on $\lambda$.
|
||||
|
||||
|
||||
This part is essentially a repeat of the previous ones, but now
|
||||
with Lasso regression. Write either your own code or
|
||||
use the functionalities of _Scikit-Learn_ (recommended).
|
||||
Give a
|
||||
critical discussion of the three methods and a judgement of which
|
||||
model fits the data best.
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user