diff --git a/doc/pub/week35/html/._week35-bs000.html b/doc/pub/week35/html/._week35-bs000.html index 2f2b84c35..6714a5fc6 100644 --- a/doc/pub/week35/html/._week35-bs000.html +++ b/doc/pub/week35/html/._week35-bs000.html @@ -38,11 +38,7 @@ doconce format html week35.do.txt --html_style=bootstrap --pygments_html_style=d {'highest level': 2, 'sections': [('Plans for week 35', 2, None, 'plans-for-week-35'), ('Reading recommendations:', 3, None, 'reading-recommendations'), - ('For exercise sessions: Why Linear Regression (aka Ordinary ' - 'Least Squares and family), repeat from last week', - 2, - None, - 'for-exercise-sessions-why-linear-regression-aka-ordinary-least-squares-and-family-repeat-from-last-week'), + ('Reminder from last week', 2, None, 'reminder-from-last-week'), ('The equations for ordinary least squares', 2, None, @@ -122,32 +118,6 @@ doconce format html week35.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'testing-the-means-squared-error-as-function-of-complexity'), - ('More preprocessing examples, two-dimensional example, the ' - 'Franke function', - 2, - None, - 'more-preprocessing-examples-two-dimensional-example-the-franke-function'), - ('To think about, first part', - 2, - None, - 'to-think-about-first-part'), - ('More thinking', 2, None, 'more-thinking'), - ('Still thinking', 2, None, 'still-thinking'), - ('What does centering (subtracting the mean values) mean ' - 'mathematically?', - 2, - None, - 'what-does-centering-subtracting-the-mean-values-mean-mathematically'), - ('Further Manipulations', 2, None, 'further-manipulations'), - ('Wrapping it up', 2, None, 'wrapping-it-up'), - ('Linear Regression code, Intercept handling first', - 2, - None, - 'linear-regression-code-intercept-handling-first'), - ('Material for lecture Monday, August 26', - 2, - None, - 'material-for-lecture-monday-august-26'), ('Mathematical Interpretation of Ordinary Least Squares', 2, None, @@ -279,7 +249,7 @@ MathJax.Hub.Config({ @@ -400,7 +361,7 @@ MathJax.Hub.Config({
  • 9
  • 10
  • ...
    • -
  • 72
    • +
  • 63
  • »
    • diff --git a/doc/pub/week35/html/._week35-bs001.html b/doc/pub/week35/html/._week35-bs001.html index 5bb2d1f4c..1572bb887 100644 --- a/doc/pub/week35/html/._week35-bs001.html +++ b/doc/pub/week35/html/._week35-bs001.html @@ -38,11 +38,7 @@ doconce format html week35.do.txt --html_style=bootstrap --pygments_html_style=d {'highest level': 2, 'sections': [('Plans for week 35', 2, None, 'plans-for-week-35'), ('Reading recommendations:', 3, None, 'reading-recommendations'), - ('For exercise sessions: Why Linear Regression (aka Ordinary ' - 'Least Squares and family), repeat from last week', - 2, - None, - 'for-exercise-sessions-why-linear-regression-aka-ordinary-least-squares-and-family-repeat-from-last-week'), + ('Reminder from last week', 2, None, 'reminder-from-last-week'), ('The equations for ordinary least squares', 2, None, @@ -122,32 +118,6 @@ doconce format html week35.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'testing-the-means-squared-error-as-function-of-complexity'), - ('More preprocessing examples, two-dimensional example, the ' - 'Franke function', - 2, - None, - 'more-preprocessing-examples-two-dimensional-example-the-franke-function'), - ('To think about, first part', - 2, - None, - 'to-think-about-first-part'), - ('More thinking', 2, None, 'more-thinking'), - ('Still thinking', 2, None, 'still-thinking'), - ('What does centering (subtracting the mean values) mean ' - 'mathematically?', - 2, - None, - 'what-does-centering-subtracting-the-mean-values-mean-mathematically'), - ('Further Manipulations', 2, None, 'further-manipulations'), - ('Wrapping it up', 2, None, 'wrapping-it-up'), - ('Linear Regression code, Intercept handling first', - 2, - None, - 'linear-regression-code-intercept-handling-first'), - ('Material for lecture Monday, August 26', - 2, - None, - 'material-for-lecture-monday-august-26'), ('Mathematical Interpretation of Ordinary Least Squares', 2, None, @@ -279,7 +249,7 @@ MathJax.Hub.Config({ @@ -374,12 +335,12 @@ MathJax.Hub.Config({

    Reading recommendations:

      -
    1. These lecture notes -
      2. -
    2. Video for exercises week 35
      3. -
    3. Goodfellow, Bengio and Courville, Deep Learning, chapter 2 on linear algebra and sections 3.1-3.10 on elements of statistics (background)
      4. -
    4. Raschka et al on preprocessing of data, relevant for exercise 3 this week, see chapter 4.
      5. -
    5. For exercise 1 of week 35, the book by A. Aldo Faisal, Cheng Soon Ong, and Marc Peter Deisenroth on the Mathematics of Machine Learning, may be very relevant. In particular chapter 5 at URL"https://mml-book.github.io/" (section 5.5 on derivatives) is very useful for exercise 1 this coming week.
      6. +
    6. These lecture notes + +
      7. +
    7. Goodfellow, Bengio and Courville, Deep Learning, chapter 2 on linear algebra and sections 3.1-3.10 on elements of statistics (background)
      8. +
    8. Raschka et al on preprocessing of data, relevant for exercise 3 this week, see chapter 4.
      9. +
    9. For exercise 1 of week 35, the book by A. Aldo Faisal, Cheng Soon Ong, and Marc Peter Deisenroth on the Mathematics of Machine Learning, may be very relevant. In particular chapter 5 at URL"https://mml-book.github.io/" (section 5.5 on derivatives) is very useful for exercise 1 this coming week.

    @@ -397,7 +358,7 @@ MathJax.Hub.Config({

  • 10
  • 11
  • ...
    • -
  • 72
    • +
  • 63
  • »
    • diff --git a/doc/pub/week35/html/._week35-bs002.html b/doc/pub/week35/html/._week35-bs002.html index 9acec0922..50cc554bf 100644 --- a/doc/pub/week35/html/._week35-bs002.html +++ b/doc/pub/week35/html/._week35-bs002.html @@ -38,11 +38,7 @@ doconce format html week35.do.txt --html_style=bootstrap --pygments_html_style=d {'highest level': 2, 'sections': [('Plans for week 35', 2, None, 'plans-for-week-35'), ('Reading recommendations:', 3, None, 'reading-recommendations'), - ('For exercise sessions: Why Linear Regression (aka Ordinary ' - 'Least Squares and family), repeat from last week', - 2, - None, - 'for-exercise-sessions-why-linear-regression-aka-ordinary-least-squares-and-family-repeat-from-last-week'), + ('Reminder from last week', 2, None, 'reminder-from-last-week'), ('The equations for ordinary least squares', 2, None, @@ -122,32 +118,6 @@ doconce format html week35.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'testing-the-means-squared-error-as-function-of-complexity'), - ('More preprocessing examples, two-dimensional example, the ' - 'Franke function', - 2, - None, - 'more-preprocessing-examples-two-dimensional-example-the-franke-function'), - ('To think about, first part', - 2, - None, - 'to-think-about-first-part'), - ('More thinking', 2, None, 'more-thinking'), - ('Still thinking', 2, None, 'still-thinking'), - ('What does centering (subtracting the mean values) mean ' - 'mathematically?', - 2, - None, - 'what-does-centering-subtracting-the-mean-values-mean-mathematically'), - ('Further Manipulations', 2, None, 'further-manipulations'), - ('Wrapping it up', 2, None, 'wrapping-it-up'), - ('Linear Regression code, Intercept handling first', - 2, - None, - 'linear-regression-code-intercept-handling-first'), - ('Material for lecture Monday, August 26', - 2, - None, - 'material-for-lecture-monday-august-26'), ('Mathematical Interpretation of Ordinary Least Squares', 2, None, @@ -279,7 +249,7 @@ MathJax.Hub.Config({ @@ -360,7 +321,7 @@ MathJax.Hub.Config({

     

     

     

    -

    For exercise sessions: Why Linear Regression (aka Ordinary Least Squares and family), repeat from last week

    +

    Reminder from last week

    We need first a reminder from last week about linear regression.

    @@ -397,7 +358,7 @@ Similarly, Mehta et al
  • 11
  • 12
  • ...
    • -
  • 72
    • +
  • 63
  • »
    • diff --git a/doc/pub/week35/html/._week35-bs003.html b/doc/pub/week35/html/._week35-bs003.html index cad3da412..3c3677438 100644 --- a/doc/pub/week35/html/._week35-bs003.html +++ b/doc/pub/week35/html/._week35-bs003.html @@ -38,11 +38,7 @@ doconce format html week35.do.txt --html_style=bootstrap --pygments_html_style=d {'highest level': 2, 'sections': [('Plans for week 35', 2, None, 'plans-for-week-35'), ('Reading recommendations:', 3, None, 'reading-recommendations'), - ('For exercise sessions: Why Linear Regression (aka Ordinary ' - 'Least Squares and family), repeat from last week', - 2, - None, - 'for-exercise-sessions-why-linear-regression-aka-ordinary-least-squares-and-family-repeat-from-last-week'), + ('Reminder from last week', 2, None, 'reminder-from-last-week'), ('The equations for ordinary least squares', 2, None, @@ -122,32 +118,6 @@ doconce format html week35.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'testing-the-means-squared-error-as-function-of-complexity'), - ('More preprocessing examples, two-dimensional example, the ' - 'Franke function', - 2, - None, - 'more-preprocessing-examples-two-dimensional-example-the-franke-function'), - ('To think about, first part', - 2, - None, - 'to-think-about-first-part'), - ('More thinking', 2, None, 'more-thinking'), - ('Still thinking', 2, None, 'still-thinking'), - ('What does centering (subtracting the mean values) mean ' - 'mathematically?', - 2, - None, - 'what-does-centering-subtracting-the-mean-values-mean-mathematically'), - ('Further Manipulations', 2, None, 'further-manipulations'), - ('Wrapping it up', 2, None, 'wrapping-it-up'), - ('Linear Regression code, Intercept handling first', - 2, - None, - 'linear-regression-code-intercept-handling-first'), - ('Material for lecture Monday, August 26', - 2, - None, - 'material-for-lecture-monday-august-26'), ('Mathematical Interpretation of Ordinary Least Squares', 2, None, @@ -279,7 +249,7 @@ MathJax.Hub.Config({ @@ -421,7 +382,7 @@ values \( \tilde{y}_i \), namely the so-called cost/loss function.
  • 12
  • 13
  • ...
    • -
  • 72
    • +
  • 63
  • »
    • diff --git a/doc/pub/week35/html/._week35-bs004.html b/doc/pub/week35/html/._week35-bs004.html index f5283d363..4bb20012a 100644 --- a/doc/pub/week35/html/._week35-bs004.html +++ b/doc/pub/week35/html/._week35-bs004.html @@ -38,11 +38,7 @@ doconce format html week35.do.txt --html_style=bootstrap --pygments_html_style=d {'highest level': 2, 'sections': [('Plans for week 35', 2, None, 'plans-for-week-35'), ('Reading recommendations:', 3, None, 'reading-recommendations'), - ('For exercise sessions: Why Linear Regression (aka Ordinary ' - 'Least Squares and family), repeat from last week', - 2, - None, - 'for-exercise-sessions-why-linear-regression-aka-ordinary-least-squares-and-family-repeat-from-last-week'), + ('Reminder from last week', 2, None, 'reminder-from-last-week'), ('The equations for ordinary least squares', 2, None, @@ -122,32 +118,6 @@ doconce format html week35.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'testing-the-means-squared-error-as-function-of-complexity'), - ('More preprocessing examples, two-dimensional example, the ' - 'Franke function', - 2, - None, - 'more-preprocessing-examples-two-dimensional-example-the-franke-function'), - ('To think about, first part', - 2, - None, - 'to-think-about-first-part'), - ('More thinking', 2, None, 'more-thinking'), - ('Still thinking', 2, None, 'still-thinking'), - ('What does centering (subtracting the mean values) mean ' - 'mathematically?', - 2, - None, - 'what-does-centering-subtracting-the-mean-values-mean-mathematically'), - ('Further Manipulations', 2, None, 'further-manipulations'), - ('Wrapping it up', 2, None, 'wrapping-it-up'), - ('Linear Regression code, Intercept handling first', - 2, - None, - 'linear-regression-code-intercept-handling-first'), - ('Material for lecture Monday, August 26', - 2, - None, - 'material-for-lecture-monday-august-26'), ('Mathematical Interpretation of Ordinary Least Squares', 2, None, @@ -279,7 +249,7 @@ MathJax.Hub.Config({ @@ -403,7 +364,7 @@ $$
  • 13
  • 14
  • ...
    • -
  • 72
    • +
  • 63
  • »
    • diff --git a/doc/pub/week35/html/._week35-bs005.html b/doc/pub/week35/html/._week35-bs005.html index 586c696eb..07ab45faf 100644 --- a/doc/pub/week35/html/._week35-bs005.html +++ b/doc/pub/week35/html/._week35-bs005.html @@ -38,11 +38,7 @@ doconce format html week35.do.txt --html_style=bootstrap --pygments_html_style=d {'highest level': 2, 'sections': [('Plans for week 35', 2, None, 'plans-for-week-35'), ('Reading recommendations:', 3, None, 'reading-recommendations'), - ('For exercise sessions: Why Linear Regression (aka Ordinary ' - 'Least Squares and family), repeat from last week', - 2, - None, - 'for-exercise-sessions-why-linear-regression-aka-ordinary-least-squares-and-family-repeat-from-last-week'), + ('Reminder from last week', 2, None, 'reminder-from-last-week'), ('The equations for ordinary least squares', 2, None, @@ -122,32 +118,6 @@ doconce format html week35.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'testing-the-means-squared-error-as-function-of-complexity'), - ('More preprocessing examples, two-dimensional example, the ' - 'Franke function', - 2, - None, - 'more-preprocessing-examples-two-dimensional-example-the-franke-function'), - ('To think about, first part', - 2, - None, - 'to-think-about-first-part'), - ('More thinking', 2, None, 'more-thinking'), - ('Still thinking', 2, None, 'still-thinking'), - ('What does centering (subtracting the mean values) mean ' - 'mathematically?', - 2, - None, - 'what-does-centering-subtracting-the-mean-values-mean-mathematically'), - ('Further Manipulations', 2, None, 'further-manipulations'), - ('Wrapping it up', 2, None, 'wrapping-it-up'), - ('Linear Regression code, Intercept handling first', - 2, - None, - 'linear-regression-code-intercept-handling-first'), - ('Material for lecture Monday, August 26', - 2, - None, - 'material-for-lecture-monday-august-26'), ('Mathematical Interpretation of Ordinary Least Squares', 2, None, @@ -279,7 +249,7 @@ MathJax.Hub.Config({ @@ -426,7 +387,7 @@ $$
  • 14
  • 15
  • ...
    • -
  • 72
    • +
  • 63
  • »
    • diff --git a/doc/pub/week35/html/._week35-bs006.html b/doc/pub/week35/html/._week35-bs006.html index 5d6d78ad0..b7801c390 100644 --- a/doc/pub/week35/html/._week35-bs006.html +++ b/doc/pub/week35/html/._week35-bs006.html @@ -38,11 +38,7 @@ doconce format html week35.do.txt --html_style=bootstrap --pygments_html_style=d {'highest level': 2, 'sections': [('Plans for week 35', 2, None, 'plans-for-week-35'), ('Reading recommendations:', 3, None, 'reading-recommendations'), - ('For exercise sessions: Why Linear Regression (aka Ordinary ' - 'Least Squares and family), repeat from last week', - 2, - None, - 'for-exercise-sessions-why-linear-regression-aka-ordinary-least-squares-and-family-repeat-from-last-week'), + ('Reminder from last week', 2, None, 'reminder-from-last-week'), ('The equations for ordinary least squares', 2, None, @@ -122,32 +118,6 @@ doconce format html week35.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'testing-the-means-squared-error-as-function-of-complexity'), - ('More preprocessing examples, two-dimensional example, the ' - 'Franke function', - 2, - None, - 'more-preprocessing-examples-two-dimensional-example-the-franke-function'), - ('To think about, first part', - 2, - None, - 'to-think-about-first-part'), - ('More thinking', 2, None, 'more-thinking'), - ('Still thinking', 2, None, 'still-thinking'), - ('What does centering (subtracting the mean values) mean ' - 'mathematically?', - 2, - None, - 'what-does-centering-subtracting-the-mean-values-mean-mathematically'), - ('Further Manipulations', 2, None, 'further-manipulations'), - ('Wrapping it up', 2, None, 'wrapping-it-up'), - ('Linear Regression code, Intercept handling first', - 2, - None, - 'linear-regression-code-intercept-handling-first'), - ('Material for lecture Monday, August 26', - 2, - None, - 'material-for-lecture-monday-august-26'), ('Mathematical Interpretation of Ordinary Least Squares', 2, None, @@ -279,7 +249,7 @@ MathJax.Hub.Config({ @@ -422,7 +383,7 @@ allow for the usage of direct linear algebra methods such as LU decomposi
  • 15
  • 16
  • ...
    • -
  • 72
    • +
  • 63
  • »
    • diff --git a/doc/pub/week35/html/._week35-bs007.html b/doc/pub/week35/html/._week35-bs007.html index b7188b249..aabb5a016 100644 --- a/doc/pub/week35/html/._week35-bs007.html +++ b/doc/pub/week35/html/._week35-bs007.html @@ -38,11 +38,7 @@ doconce format html week35.do.txt --html_style=bootstrap --pygments_html_style=d {'highest level': 2, 'sections': [('Plans for week 35', 2, None, 'plans-for-week-35'), ('Reading recommendations:', 3, None, 'reading-recommendations'), - ('For exercise sessions: Why Linear Regression (aka Ordinary ' - 'Least Squares and family), repeat from last week', - 2, - None, - 'for-exercise-sessions-why-linear-regression-aka-ordinary-least-squares-and-family-repeat-from-last-week'), + ('Reminder from last week', 2, None, 'reminder-from-last-week'), ('The equations for ordinary least squares', 2, None, @@ -122,32 +118,6 @@ doconce format html week35.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'testing-the-means-squared-error-as-function-of-complexity'), - ('More preprocessing examples, two-dimensional example, the ' - 'Franke function', - 2, - None, - 'more-preprocessing-examples-two-dimensional-example-the-franke-function'), - ('To think about, first part', - 2, - None, - 'to-think-about-first-part'), - ('More thinking', 2, None, 'more-thinking'), - ('Still thinking', 2, None, 'still-thinking'), - ('What does centering (subtracting the mean values) mean ' - 'mathematically?', - 2, - None, - 'what-does-centering-subtracting-the-mean-values-mean-mathematically'), - ('Further Manipulations', 2, None, 'further-manipulations'), - ('Wrapping it up', 2, None, 'wrapping-it-up'), - ('Linear Regression code, Intercept handling first', - 2, - None, - 'linear-regression-code-intercept-handling-first'), - ('Material for lecture Monday, August 26', - 2, - None, - 'material-for-lecture-monday-august-26'), ('Mathematical Interpretation of Ordinary Least Squares', 2, None, @@ -279,7 +249,7 @@ MathJax.Hub.Config({ @@ -409,7 +370,7 @@ $$
  • 16
  • 17
  • ...
    • -
  • 72
    • +
  • 63
  • »
    • diff --git a/doc/pub/week35/html/._week35-bs008.html b/doc/pub/week35/html/._week35-bs008.html index 1cbb749fa..a8c58852b 100644 --- a/doc/pub/week35/html/._week35-bs008.html +++ b/doc/pub/week35/html/._week35-bs008.html @@ -38,11 +38,7 @@ doconce format html week35.do.txt --html_style=bootstrap --pygments_html_style=d {'highest level': 2, 'sections': [('Plans for week 35', 2, None, 'plans-for-week-35'), ('Reading recommendations:', 3, None, 'reading-recommendations'), - ('For exercise sessions: Why Linear Regression (aka Ordinary ' - 'Least Squares and family), repeat from last week', - 2, - None, - 'for-exercise-sessions-why-linear-regression-aka-ordinary-least-squares-and-family-repeat-from-last-week'), + ('Reminder from last week', 2, None, 'reminder-from-last-week'), ('The equations for ordinary least squares', 2, None, @@ -122,32 +118,6 @@ doconce format html week35.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'testing-the-means-squared-error-as-function-of-complexity'), - ('More preprocessing examples, two-dimensional example, the ' - 'Franke function', - 2, - None, - 'more-preprocessing-examples-two-dimensional-example-the-franke-function'), - ('To think about, first part', - 2, - None, - 'to-think-about-first-part'), - ('More thinking', 2, None, 'more-thinking'), - ('Still thinking', 2, None, 'still-thinking'), - ('What does centering (subtracting the mean values) mean ' - 'mathematically?', - 2, - None, - 'what-does-centering-subtracting-the-mean-values-mean-mathematically'), - ('Further Manipulations', 2, None, 'further-manipulations'), - ('Wrapping it up', 2, None, 'wrapping-it-up'), - ('Linear Regression code, Intercept handling first', - 2, - None, - 'linear-regression-code-intercept-handling-first'), - ('Material for lecture Monday, August 26', - 2, - None, - 'material-for-lecture-monday-august-26'), ('Mathematical Interpretation of Ordinary Least Squares', 2, None, @@ -279,7 +249,7 @@ MathJax.Hub.Config({ @@ -407,7 +368,7 @@ vector is differentiable.
  • 17
  • 18
  • ...
    • -
  • 72
    • +
  • 63
  • »
    • diff --git a/doc/pub/week35/html/._week35-bs009.html b/doc/pub/week35/html/._week35-bs009.html index 35e64b1fa..ff32e9e8c 100644 --- a/doc/pub/week35/html/._week35-bs009.html +++ b/doc/pub/week35/html/._week35-bs009.html @@ -38,11 +38,7 @@ doconce format html week35.do.txt --html_style=bootstrap --pygments_html_style=d {'highest level': 2, 'sections': [('Plans for week 35', 2, None, 'plans-for-week-35'), ('Reading recommendations:', 3, None, 'reading-recommendations'), - ('For exercise sessions: Why Linear Regression (aka Ordinary ' - 'Least Squares and family), repeat from last week', - 2, - None, - 'for-exercise-sessions-why-linear-regression-aka-ordinary-least-squares-and-family-repeat-from-last-week'), + ('Reminder from last week', 2, None, 'reminder-from-last-week'), ('The equations for ordinary least squares', 2, None, @@ -122,32 +118,6 @@ doconce format html week35.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'testing-the-means-squared-error-as-function-of-complexity'), - ('More preprocessing examples, two-dimensional example, the ' - 'Franke function', - 2, - None, - 'more-preprocessing-examples-two-dimensional-example-the-franke-function'), - ('To think about, first part', - 2, - None, - 'to-think-about-first-part'), - ('More thinking', 2, None, 'more-thinking'), - ('Still thinking', 2, None, 'still-thinking'), - ('What does centering (subtracting the mean values) mean ' - 'mathematically?', - 2, - None, - 'what-does-centering-subtracting-the-mean-values-mean-mathematically'), - ('Further Manipulations', 2, None, 'further-manipulations'), - ('Wrapping it up', 2, None, 'wrapping-it-up'), - ('Linear Regression code, Intercept handling first', - 2, - None, - 'linear-regression-code-intercept-handling-first'), - ('Material for lecture Monday, August 26', - 2, - None, - 'material-for-lecture-monday-august-26'), ('Mathematical Interpretation of Ordinary Least Squares', 2, None, @@ -279,7 +249,7 @@ MathJax.Hub.Config({ @@ -406,7 +367,7 @@ $$
  • 18
  • 19
  • ...
    • -
  • 72
    • +
  • 63
  • »
    • diff --git a/doc/pub/week35/html/._week35-bs010.html b/doc/pub/week35/html/._week35-bs010.html index 1dceb8be0..80e5828b8 100644 --- a/doc/pub/week35/html/._week35-bs010.html +++ b/doc/pub/week35/html/._week35-bs010.html @@ -38,11 +38,7 @@ doconce format html week35.do.txt --html_style=bootstrap --pygments_html_style=d {'highest level': 2, 'sections': [('Plans for week 35', 2, None, 'plans-for-week-35'), ('Reading recommendations:', 3, None, 'reading-recommendations'), - ('For exercise sessions: Why Linear Regression (aka Ordinary ' - 'Least Squares and family), repeat from last week', - 2, - None, - 'for-exercise-sessions-why-linear-regression-aka-ordinary-least-squares-and-family-repeat-from-last-week'), + ('Reminder from last week', 2, None, 'reminder-from-last-week'), ('The equations for ordinary least squares', 2, None, @@ -122,32 +118,6 @@ doconce format html week35.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'testing-the-means-squared-error-as-function-of-complexity'), - ('More preprocessing examples, two-dimensional example, the ' - 'Franke function', - 2, - None, - 'more-preprocessing-examples-two-dimensional-example-the-franke-function'), - ('To think about, first part', - 2, - None, - 'to-think-about-first-part'), - ('More thinking', 2, None, 'more-thinking'), - ('Still thinking', 2, None, 'still-thinking'), - ('What does centering (subtracting the mean values) mean ' - 'mathematically?', - 2, - None, - 'what-does-centering-subtracting-the-mean-values-mean-mathematically'), - ('Further Manipulations', 2, None, 'further-manipulations'), - ('Wrapping it up', 2, None, 'wrapping-it-up'), - ('Linear Regression code, Intercept handling first', - 2, - None, - 'linear-regression-code-intercept-handling-first'), - ('Material for lecture Monday, August 26', - 2, - None, - 'material-for-lecture-monday-august-26'), ('Mathematical Interpretation of Ordinary Least Squares', 2, None, @@ -279,7 +249,7 @@ MathJax.Hub.Config({ @@ -417,7 +378,7 @@ $$
  • 19
  • 20
  • ...
    • -
  • 72
    • +
  • 63
  • »
    • diff --git a/doc/pub/week35/html/._week35-bs011.html b/doc/pub/week35/html/._week35-bs011.html index fea66bdc4..c3510dc7c 100644 --- a/doc/pub/week35/html/._week35-bs011.html +++ b/doc/pub/week35/html/._week35-bs011.html @@ -38,11 +38,7 @@ doconce format html week35.do.txt --html_style=bootstrap --pygments_html_style=d {'highest level': 2, 'sections': [('Plans for week 35', 2, None, 'plans-for-week-35'), ('Reading recommendations:', 3, None, 'reading-recommendations'), - ('For exercise sessions: Why Linear Regression (aka Ordinary ' - 'Least Squares and family), repeat from last week', - 2, - None, - 'for-exercise-sessions-why-linear-regression-aka-ordinary-least-squares-and-family-repeat-from-last-week'), + ('Reminder from last week', 2, None, 'reminder-from-last-week'), ('The equations for ordinary least squares', 2, None, @@ -122,32 +118,6 @@ doconce format html week35.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'testing-the-means-squared-error-as-function-of-complexity'), - ('More preprocessing examples, two-dimensional example, the ' - 'Franke function', - 2, - None, - 'more-preprocessing-examples-two-dimensional-example-the-franke-function'), - ('To think about, first part', - 2, - None, - 'to-think-about-first-part'), - ('More thinking', 2, None, 'more-thinking'), - ('Still thinking', 2, None, 'still-thinking'), - ('What does centering (subtracting the mean values) mean ' - 'mathematically?', - 2, - None, - 'what-does-centering-subtracting-the-mean-values-mean-mathematically'), - ('Further Manipulations', 2, None, 'further-manipulations'), - ('Wrapping it up', 2, None, 'wrapping-it-up'), - ('Linear Regression code, Intercept handling first', - 2, - None, - 'linear-regression-code-intercept-handling-first'), - ('Material for lecture Monday, August 26', - 2, - None, - 'material-for-lecture-monday-august-26'), ('Mathematical Interpretation of Ordinary Least Squares', 2, None, @@ -279,7 +249,7 @@ MathJax.Hub.Config({ @@ -419,7 +380,7 @@ $$
  • 20
  • 21
  • ...
    • -
  • 72
    • +
  • 63
  • »
    • diff --git a/doc/pub/week35/html/._week35-bs012.html b/doc/pub/week35/html/._week35-bs012.html index 9fc728584..727a44c1f 100644 --- a/doc/pub/week35/html/._week35-bs012.html +++ b/doc/pub/week35/html/._week35-bs012.html @@ -38,11 +38,7 @@ doconce format html week35.do.txt --html_style=bootstrap --pygments_html_style=d {'highest level': 2, 'sections': [('Plans for week 35', 2, None, 'plans-for-week-35'), ('Reading recommendations:', 3, None, 'reading-recommendations'), - ('For exercise sessions: Why Linear Regression (aka Ordinary ' - 'Least Squares and family), repeat from last week', - 2, - None, - 'for-exercise-sessions-why-linear-regression-aka-ordinary-least-squares-and-family-repeat-from-last-week'), + ('Reminder from last week', 2, None, 'reminder-from-last-week'), ('The equations for ordinary least squares', 2, None, @@ -122,32 +118,6 @@ doconce format html week35.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'testing-the-means-squared-error-as-function-of-complexity'), - ('More preprocessing examples, two-dimensional example, the ' - 'Franke function', - 2, - None, - 'more-preprocessing-examples-two-dimensional-example-the-franke-function'), - ('To think about, first part', - 2, - None, - 'to-think-about-first-part'), - ('More thinking', 2, None, 'more-thinking'), - ('Still thinking', 2, None, 'still-thinking'), - ('What does centering (subtracting the mean values) mean ' - 'mathematically?', - 2, - None, - 'what-does-centering-subtracting-the-mean-values-mean-mathematically'), - ('Further Manipulations', 2, None, 'further-manipulations'), - ('Wrapping it up', 2, None, 'wrapping-it-up'), - ('Linear Regression code, Intercept handling first', - 2, - None, - 'linear-regression-code-intercept-handling-first'), - ('Material for lecture Monday, August 26', - 2, - None, - 'material-for-lecture-monday-august-26'), ('Mathematical Interpretation of Ordinary Least Squares', 2, None, @@ -279,7 +249,7 @@ MathJax.Hub.Config({ @@ -420,7 +381,7 @@ $$
  • 21
  • 22
  • ...
    • -
  • 72
    • +
  • 63
  • »
    • diff --git a/doc/pub/week35/html/._week35-bs013.html b/doc/pub/week35/html/._week35-bs013.html index 0fef0efa4..7023a8f1b 100644 --- a/doc/pub/week35/html/._week35-bs013.html +++ b/doc/pub/week35/html/._week35-bs013.html @@ -38,11 +38,7 @@ doconce format html week35.do.txt --html_style=bootstrap --pygments_html_style=d {'highest level': 2, 'sections': [('Plans for week 35', 2, None, 'plans-for-week-35'), ('Reading recommendations:', 3, None, 'reading-recommendations'), - ('For exercise sessions: Why Linear Regression (aka Ordinary ' - 'Least Squares and family), repeat from last week', - 2, - None, - 'for-exercise-sessions-why-linear-regression-aka-ordinary-least-squares-and-family-repeat-from-last-week'), + ('Reminder from last week', 2, None, 'reminder-from-last-week'), ('The equations for ordinary least squares', 2, None, @@ -122,32 +118,6 @@ doconce format html week35.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'testing-the-means-squared-error-as-function-of-complexity'), - ('More preprocessing examples, two-dimensional example, the ' - 'Franke function', - 2, - None, - 'more-preprocessing-examples-two-dimensional-example-the-franke-function'), - ('To think about, first part', - 2, - None, - 'to-think-about-first-part'), - ('More thinking', 2, None, 'more-thinking'), - ('Still thinking', 2, None, 'still-thinking'), - ('What does centering (subtracting the mean values) mean ' - 'mathematically?', - 2, - None, - 'what-does-centering-subtracting-the-mean-values-mean-mathematically'), - ('Further Manipulations', 2, None, 'further-manipulations'), - ('Wrapping it up', 2, None, 'wrapping-it-up'), - ('Linear Regression code, Intercept handling first', - 2, - None, - 'linear-regression-code-intercept-handling-first'), - ('Material for lecture Monday, August 26', - 2, - None, - 'material-for-lecture-monday-august-26'), ('Mathematical Interpretation of Ordinary Least Squares', 2, None, @@ -279,7 +249,7 @@ MathJax.Hub.Config({ @@ -432,7 +393,7 @@ $$
  • 22
  • 23
  • ...
    • -
  • 72
    • +
  • 63
  • »
    • diff --git a/doc/pub/week35/html/._week35-bs014.html b/doc/pub/week35/html/._week35-bs014.html index 9ebd409ce..0c200eea0 100644 --- a/doc/pub/week35/html/._week35-bs014.html +++ b/doc/pub/week35/html/._week35-bs014.html @@ -38,11 +38,7 @@ doconce format html week35.do.txt --html_style=bootstrap --pygments_html_style=d {'highest level': 2, 'sections': [('Plans for week 35', 2, None, 'plans-for-week-35'), ('Reading recommendations:', 3, None, 'reading-recommendations'), - ('For exercise sessions: Why Linear Regression (aka Ordinary ' - 'Least Squares and family), repeat from last week', - 2, - None, - 'for-exercise-sessions-why-linear-regression-aka-ordinary-least-squares-and-family-repeat-from-last-week'), + ('Reminder from last week', 2, None, 'reminder-from-last-week'), ('The equations for ordinary least squares', 2, None, @@ -122,32 +118,6 @@ doconce format html week35.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'testing-the-means-squared-error-as-function-of-complexity'), - ('More preprocessing examples, two-dimensional example, the ' - 'Franke function', - 2, - None, - 'more-preprocessing-examples-two-dimensional-example-the-franke-function'), - ('To think about, first part', - 2, - None, - 'to-think-about-first-part'), - ('More thinking', 2, None, 'more-thinking'), - ('Still thinking', 2, None, 'still-thinking'), - ('What does centering (subtracting the mean values) mean ' - 'mathematically?', - 2, - None, - 'what-does-centering-subtracting-the-mean-values-mean-mathematically'), - ('Further Manipulations', 2, None, 'further-manipulations'), - ('Wrapping it up', 2, None, 'wrapping-it-up'), - ('Linear Regression code, Intercept handling first', - 2, - None, - 'linear-regression-code-intercept-handling-first'), - ('Material for lecture Monday, August 26', - 2, - None, - 'material-for-lecture-monday-august-26'), ('Mathematical Interpretation of Ordinary Least Squares', 2, None, @@ -279,7 +249,7 @@ MathJax.Hub.Config({ @@ -406,7 +367,7 @@ $$
  • 23
  • 24
  • ...
    • -
  • 72
    • +
  • 63
  • »
    • diff --git a/doc/pub/week35/html/._week35-bs015.html b/doc/pub/week35/html/._week35-bs015.html index e614da5ad..51c6cc4a0 100644 --- a/doc/pub/week35/html/._week35-bs015.html +++ b/doc/pub/week35/html/._week35-bs015.html @@ -38,11 +38,7 @@ doconce format html week35.do.txt --html_style=bootstrap --pygments_html_style=d {'highest level': 2, 'sections': [('Plans for week 35', 2, None, 'plans-for-week-35'), ('Reading recommendations:', 3, None, 'reading-recommendations'), - ('For exercise sessions: Why Linear Regression (aka Ordinary ' - 'Least Squares and family), repeat from last week', - 2, - None, - 'for-exercise-sessions-why-linear-regression-aka-ordinary-least-squares-and-family-repeat-from-last-week'), + ('Reminder from last week', 2, None, 'reminder-from-last-week'), ('The equations for ordinary least squares', 2, None, @@ -122,32 +118,6 @@ doconce format html week35.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'testing-the-means-squared-error-as-function-of-complexity'), - ('More preprocessing examples, two-dimensional example, the ' - 'Franke function', - 2, - None, - 'more-preprocessing-examples-two-dimensional-example-the-franke-function'), - ('To think about, first part', - 2, - None, - 'to-think-about-first-part'), - ('More thinking', 2, None, 'more-thinking'), - ('Still thinking', 2, None, 'still-thinking'), - ('What does centering (subtracting the mean values) mean ' - 'mathematically?', - 2, - None, - 'what-does-centering-subtracting-the-mean-values-mean-mathematically'), - ('Further Manipulations', 2, None, 'further-manipulations'), - ('Wrapping it up', 2, None, 'wrapping-it-up'), - ('Linear Regression code, Intercept handling first', - 2, - None, - 'linear-regression-code-intercept-handling-first'), - ('Material for lecture Monday, August 26', - 2, - None, - 'material-for-lecture-monday-august-26'), ('Mathematical Interpretation of Ordinary Least Squares', 2, None, @@ -279,7 +249,7 @@ MathJax.Hub.Config({ @@ -386,10 +347,10 @@ next week) with the variance of the optimal parameters important role in optmization algorithms and Principal Component Analysis as a way to reduce the dimensionality of a machine learning/data analysis problem. +v +Linear algebra question: Can we use the Hessian matrix to say something about properties of the cost function (our optmization problem)? (hint: think about convex or concave problems and how to relate these to a matrix!).

    -

    Linear algebra question: Can we use the Hessian matrix to say something about properties of the cost function (our optmization problem)? (hint: think about convex or concave problems and how to relate these to a matrix!).

    -

    diff --git a/doc/pub/week35/html/._week35-bs016.html b/doc/pub/week35/html/._week35-bs016.html index 9da65cdaa..8404e4964 100644 --- a/doc/pub/week35/html/._week35-bs016.html +++ b/doc/pub/week35/html/._week35-bs016.html @@ -38,11 +38,7 @@ doconce format html week35.do.txt --html_style=bootstrap --pygments_html_style=d {'highest level': 2, 'sections': [('Plans for week 35', 2, None, 'plans-for-week-35'), ('Reading recommendations:', 3, None, 'reading-recommendations'), - ('For exercise sessions: Why Linear Regression (aka Ordinary ' - 'Least Squares and family), repeat from last week', - 2, - None, - 'for-exercise-sessions-why-linear-regression-aka-ordinary-least-squares-and-family-repeat-from-last-week'), + ('Reminder from last week', 2, None, 'reminder-from-last-week'), ('The equations for ordinary least squares', 2, None, @@ -122,32 +118,6 @@ doconce format html week35.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'testing-the-means-squared-error-as-function-of-complexity'), - ('More preprocessing examples, two-dimensional example, the ' - 'Franke function', - 2, - None, - 'more-preprocessing-examples-two-dimensional-example-the-franke-function'), - ('To think about, first part', - 2, - None, - 'to-think-about-first-part'), - ('More thinking', 2, None, 'more-thinking'), - ('Still thinking', 2, None, 'still-thinking'), - ('What does centering (subtracting the mean values) mean ' - 'mathematically?', - 2, - None, - 'what-does-centering-subtracting-the-mean-values-mean-mathematically'), - ('Further Manipulations', 2, None, 'further-manipulations'), - ('Wrapping it up', 2, None, 'wrapping-it-up'), - ('Linear Regression code, Intercept handling first', - 2, - None, - 'linear-regression-code-intercept-handling-first'), - ('Material for lecture Monday, August 26', - 2, - None, - 'material-for-lecture-monday-august-26'), ('Mathematical Interpretation of Ordinary Least Squares', 2, None, @@ -279,7 +249,7 @@ MathJax.Hub.Config({ @@ -410,7 +371,7 @@ $$
  • 25
  • 26
  • ...
    • -
  • 72
    • +
  • 63
  • »
    • diff --git a/doc/pub/week35/html/._week35-bs017.html b/doc/pub/week35/html/._week35-bs017.html index e84d3c4d9..dc7390997 100644 --- a/doc/pub/week35/html/._week35-bs017.html +++ b/doc/pub/week35/html/._week35-bs017.html @@ -38,11 +38,7 @@ doconce format html week35.do.txt --html_style=bootstrap --pygments_html_style=d {'highest level': 2, 'sections': [('Plans for week 35', 2, None, 'plans-for-week-35'), ('Reading recommendations:', 3, None, 'reading-recommendations'), - ('For exercise sessions: Why Linear Regression (aka Ordinary ' - 'Least Squares and family), repeat from last week', - 2, - None, - 'for-exercise-sessions-why-linear-regression-aka-ordinary-least-squares-and-family-repeat-from-last-week'), + ('Reminder from last week', 2, None, 'reminder-from-last-week'), ('The equations for ordinary least squares', 2, None, @@ -122,32 +118,6 @@ doconce format html week35.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'testing-the-means-squared-error-as-function-of-complexity'), - ('More preprocessing examples, two-dimensional example, the ' - 'Franke function', - 2, - None, - 'more-preprocessing-examples-two-dimensional-example-the-franke-function'), - ('To think about, first part', - 2, - None, - 'to-think-about-first-part'), - ('More thinking', 2, None, 'more-thinking'), - ('Still thinking', 2, None, 'still-thinking'), - ('What does centering (subtracting the mean values) mean ' - 'mathematically?', - 2, - None, - 'what-does-centering-subtracting-the-mean-values-mean-mathematically'), - ('Further Manipulations', 2, None, 'further-manipulations'), - ('Wrapping it up', 2, None, 'wrapping-it-up'), - ('Linear Regression code, Intercept handling first', - 2, - None, - 'linear-regression-code-intercept-handling-first'), - ('Material for lecture Monday, August 26', - 2, - None, - 'material-for-lecture-monday-august-26'), ('Mathematical Interpretation of Ordinary Least Squares', 2, None, @@ -279,7 +249,7 @@ MathJax.Hub.Config({ @@ -399,7 +360,7 @@ $$
  • 26
  • 27
  • ...
    • -
  • 72
    • +
  • 63
  • »
    • diff --git a/doc/pub/week35/html/._week35-bs018.html b/doc/pub/week35/html/._week35-bs018.html index 2d92f459d..628fe5f2a 100644 --- a/doc/pub/week35/html/._week35-bs018.html +++ b/doc/pub/week35/html/._week35-bs018.html @@ -38,11 +38,7 @@ doconce format html week35.do.txt --html_style=bootstrap --pygments_html_style=d {'highest level': 2, 'sections': [('Plans for week 35', 2, None, 'plans-for-week-35'), ('Reading recommendations:', 3, None, 'reading-recommendations'), - ('For exercise sessions: Why Linear Regression (aka Ordinary ' - 'Least Squares and family), repeat from last week', - 2, - None, - 'for-exercise-sessions-why-linear-regression-aka-ordinary-least-squares-and-family-repeat-from-last-week'), + ('Reminder from last week', 2, None, 'reminder-from-last-week'), ('The equations for ordinary least squares', 2, None, @@ -122,32 +118,6 @@ doconce format html week35.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'testing-the-means-squared-error-as-function-of-complexity'), - ('More preprocessing examples, two-dimensional example, the ' - 'Franke function', - 2, - None, - 'more-preprocessing-examples-two-dimensional-example-the-franke-function'), - ('To think about, first part', - 2, - None, - 'to-think-about-first-part'), - ('More thinking', 2, None, 'more-thinking'), - ('Still thinking', 2, None, 'still-thinking'), - ('What does centering (subtracting the mean values) mean ' - 'mathematically?', - 2, - None, - 'what-does-centering-subtracting-the-mean-values-mean-mathematically'), - ('Further Manipulations', 2, None, 'further-manipulations'), - ('Wrapping it up', 2, None, 'wrapping-it-up'), - ('Linear Regression code, Intercept handling first', - 2, - None, - 'linear-regression-code-intercept-handling-first'), - ('Material for lecture Monday, August 26', - 2, - None, - 'material-for-lecture-monday-august-26'), ('Mathematical Interpretation of Ordinary Least Squares', 2, None, @@ -279,7 +249,7 @@ MathJax.Hub.Config({ @@ -452,7 +413,7 @@ ytildenp = np.<
  • 27
  • 28
  • ...
    • -
  • 72
    • +
  • 63
  • »
    • diff --git a/doc/pub/week35/html/._week35-bs019.html b/doc/pub/week35/html/._week35-bs019.html index 6b138e20e..d86d98ffd 100644 --- a/doc/pub/week35/html/._week35-bs019.html +++ b/doc/pub/week35/html/._week35-bs019.html @@ -38,11 +38,7 @@ doconce format html week35.do.txt --html_style=bootstrap --pygments_html_style=d {'highest level': 2, 'sections': [('Plans for week 35', 2, None, 'plans-for-week-35'), ('Reading recommendations:', 3, None, 'reading-recommendations'), - ('For exercise sessions: Why Linear Regression (aka Ordinary ' - 'Least Squares and family), repeat from last week', - 2, - None, - 'for-exercise-sessions-why-linear-regression-aka-ordinary-least-squares-and-family-repeat-from-last-week'), + ('Reminder from last week', 2, None, 'reminder-from-last-week'), ('The equations for ordinary least squares', 2, None, @@ -122,32 +118,6 @@ doconce format html week35.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'testing-the-means-squared-error-as-function-of-complexity'), - ('More preprocessing examples, two-dimensional example, the ' - 'Franke function', - 2, - None, - 'more-preprocessing-examples-two-dimensional-example-the-franke-function'), - ('To think about, first part', - 2, - None, - 'to-think-about-first-part'), - ('More thinking', 2, None, 'more-thinking'), - ('Still thinking', 2, None, 'still-thinking'), - ('What does centering (subtracting the mean values) mean ' - 'mathematically?', - 2, - None, - 'what-does-centering-subtracting-the-mean-values-mean-mathematically'), - ('Further Manipulations', 2, None, 'further-manipulations'), - ('Wrapping it up', 2, None, 'wrapping-it-up'), - ('Linear Regression code, Intercept handling first', - 2, - None, - 'linear-regression-code-intercept-handling-first'), - ('Material for lecture Monday, August 26', - 2, - None, - 'material-for-lecture-monday-august-26'), ('Mathematical Interpretation of Ordinary Least Squares', 2, None, @@ -279,7 +249,7 @@ MathJax.Hub.Config({ @@ -493,7 +454,7 @@ Since we are not using Scikit-Learn here we can define our own \( R2 \) f
  • 28
  • 29
  • ...
    • -
  • 72
    • +
  • 63
  • »
    • diff --git a/doc/pub/week35/html/._week35-bs020.html b/doc/pub/week35/html/._week35-bs020.html index 4dc4044e9..2c6e27cfb 100644 --- a/doc/pub/week35/html/._week35-bs020.html +++ b/doc/pub/week35/html/._week35-bs020.html @@ -38,11 +38,7 @@ doconce format html week35.do.txt --html_style=bootstrap --pygments_html_style=d {'highest level': 2, 'sections': [('Plans for week 35', 2, None, 'plans-for-week-35'), ('Reading recommendations:', 3, None, 'reading-recommendations'), - ('For exercise sessions: Why Linear Regression (aka Ordinary ' - 'Least Squares and family), repeat from last week', - 2, - None, - 'for-exercise-sessions-why-linear-regression-aka-ordinary-least-squares-and-family-repeat-from-last-week'), + ('Reminder from last week', 2, None, 'reminder-from-last-week'), ('The equations for ordinary least squares', 2, None, @@ -122,32 +118,6 @@ doconce format html week35.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'testing-the-means-squared-error-as-function-of-complexity'), - ('More preprocessing examples, two-dimensional example, the ' - 'Franke function', - 2, - None, - 'more-preprocessing-examples-two-dimensional-example-the-franke-function'), - ('To think about, first part', - 2, - None, - 'to-think-about-first-part'), - ('More thinking', 2, None, 'more-thinking'), - ('Still thinking', 2, None, 'still-thinking'), - ('What does centering (subtracting the mean values) mean ' - 'mathematically?', - 2, - None, - 'what-does-centering-subtracting-the-mean-values-mean-mathematically'), - ('Further Manipulations', 2, None, 'further-manipulations'), - ('Wrapping it up', 2, None, 'wrapping-it-up'), - ('Linear Regression code, Intercept handling first', - 2, - None, - 'linear-regression-code-intercept-handling-first'), - ('Material for lecture Monday, August 26', - 2, - None, - 'material-for-lecture-monday-august-26'), ('Mathematical Interpretation of Ordinary Least Squares', 2, None, @@ -279,7 +249,7 @@ MathJax.Hub.Config({ @@ -406,7 +367,7 @@ but now splitting the data into a training set and a test set.
  • 29
  • 30
  • ...
    • -
  • 72
    • +
  • 63
  • »
    • diff --git a/doc/pub/week35/html/._week35-bs021.html b/doc/pub/week35/html/._week35-bs021.html index 1e94b90da..2ffc22d63 100644 --- a/doc/pub/week35/html/._week35-bs021.html +++ b/doc/pub/week35/html/._week35-bs021.html @@ -38,11 +38,7 @@ doconce format html week35.do.txt --html_style=bootstrap --pygments_html_style=d {'highest level': 2, 'sections': [('Plans for week 35', 2, None, 'plans-for-week-35'), ('Reading recommendations:', 3, None, 'reading-recommendations'), - ('For exercise sessions: Why Linear Regression (aka Ordinary ' - 'Least Squares and family), repeat from last week', - 2, - None, - 'for-exercise-sessions-why-linear-regression-aka-ordinary-least-squares-and-family-repeat-from-last-week'), + ('Reminder from last week', 2, None, 'reminder-from-last-week'), ('The equations for ordinary least squares', 2, None, @@ -122,32 +118,6 @@ doconce format html week35.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'testing-the-means-squared-error-as-function-of-complexity'), - ('More preprocessing examples, two-dimensional example, the ' - 'Franke function', - 2, - None, - 'more-preprocessing-examples-two-dimensional-example-the-franke-function'), - ('To think about, first part', - 2, - None, - 'to-think-about-first-part'), - ('More thinking', 2, None, 'more-thinking'), - ('Still thinking', 2, None, 'still-thinking'), - ('What does centering (subtracting the mean values) mean ' - 'mathematically?', - 2, - None, - 'what-does-centering-subtracting-the-mean-values-mean-mathematically'), - ('Further Manipulations', 2, None, 'further-manipulations'), - ('Wrapping it up', 2, None, 'wrapping-it-up'), - ('Linear Regression code, Intercept handling first', - 2, - None, - 'linear-regression-code-intercept-handling-first'), - ('Material for lecture Monday, August 26', - 2, - None, - 'material-for-lecture-monday-august-26'), ('Mathematical Interpretation of Ordinary Least Squares', 2, None, @@ -279,7 +249,7 @@ MathJax.Hub.Config({ @@ -450,7 +411,7 @@ ypredict = X_test 30
  • 31
  • ...
    • -
  • 72
    • +
  • 63
  • »
    • diff --git a/doc/pub/week35/html/._week35-bs022.html b/doc/pub/week35/html/._week35-bs022.html index d26b59621..c2358459c 100644 --- a/doc/pub/week35/html/._week35-bs022.html +++ b/doc/pub/week35/html/._week35-bs022.html @@ -38,11 +38,7 @@ doconce format html week35.do.txt --html_style=bootstrap --pygments_html_style=d {'highest level': 2, 'sections': [('Plans for week 35', 2, None, 'plans-for-week-35'), ('Reading recommendations:', 3, None, 'reading-recommendations'), - ('For exercise sessions: Why Linear Regression (aka Ordinary ' - 'Least Squares and family), repeat from last week', - 2, - None, - 'for-exercise-sessions-why-linear-regression-aka-ordinary-least-squares-and-family-repeat-from-last-week'), + ('Reminder from last week', 2, None, 'reminder-from-last-week'), ('The equations for ordinary least squares', 2, None, @@ -122,32 +118,6 @@ doconce format html week35.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'testing-the-means-squared-error-as-function-of-complexity'), - ('More preprocessing examples, two-dimensional example, the ' - 'Franke function', - 2, - None, - 'more-preprocessing-examples-two-dimensional-example-the-franke-function'), - ('To think about, first part', - 2, - None, - 'to-think-about-first-part'), - ('More thinking', 2, None, 'more-thinking'), - ('Still thinking', 2, None, 'still-thinking'), - ('What does centering (subtracting the mean values) mean ' - 'mathematically?', - 2, - None, - 'what-does-centering-subtracting-the-mean-values-mean-mathematically'), - ('Further Manipulations', 2, None, 'further-manipulations'), - ('Wrapping it up', 2, None, 'wrapping-it-up'), - ('Linear Regression code, Intercept handling first', - 2, - None, - 'linear-regression-code-intercept-handling-first'), - ('Material for lecture Monday, August 26', - 2, - None, - 'material-for-lecture-monday-august-26'), ('Mathematical Interpretation of Ordinary Least Squares', 2, None, @@ -279,7 +249,7 @@ MathJax.Hub.Config({ @@ -428,7 +389,7 @@ normally recommend using the latter functionality.
  • 31
  • 32
  • ...
    • -
  • 72
    • +
  • 63
  • »
    • diff --git a/doc/pub/week35/html/._week35-bs023.html b/doc/pub/week35/html/._week35-bs023.html index be5e111e5..ba6d6d682 100644 --- a/doc/pub/week35/html/._week35-bs023.html +++ b/doc/pub/week35/html/._week35-bs023.html @@ -38,11 +38,7 @@ doconce format html week35.do.txt --html_style=bootstrap --pygments_html_style=d {'highest level': 2, 'sections': [('Plans for week 35', 2, None, 'plans-for-week-35'), ('Reading recommendations:', 3, None, 'reading-recommendations'), - ('For exercise sessions: Why Linear Regression (aka Ordinary ' - 'Least Squares and family), repeat from last week', - 2, - None, - 'for-exercise-sessions-why-linear-regression-aka-ordinary-least-squares-and-family-repeat-from-last-week'), + ('Reminder from last week', 2, None, 'reminder-from-last-week'), ('The equations for ordinary least squares', 2, None, @@ -122,32 +118,6 @@ doconce format html week35.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'testing-the-means-squared-error-as-function-of-complexity'), - ('More preprocessing examples, two-dimensional example, the ' - 'Franke function', - 2, - None, - 'more-preprocessing-examples-two-dimensional-example-the-franke-function'), - ('To think about, first part', - 2, - None, - 'to-think-about-first-part'), - ('More thinking', 2, None, 'more-thinking'), - ('Still thinking', 2, None, 'still-thinking'), - ('What does centering (subtracting the mean values) mean ' - 'mathematically?', - 2, - None, - 'what-does-centering-subtracting-the-mean-values-mean-mathematically'), - ('Further Manipulations', 2, None, 'further-manipulations'), - ('Wrapping it up', 2, None, 'wrapping-it-up'), - ('Linear Regression code, Intercept handling first', - 2, - None, - 'linear-regression-code-intercept-handling-first'), - ('Material for lecture Monday, August 26', - 2, - None, - 'material-for-lecture-monday-august-26'), ('Mathematical Interpretation of Ordinary Least Squares', 2, None, @@ -279,7 +249,7 @@ MathJax.Hub.Config({ @@ -415,7 +376,7 @@ visualization.
  • 32
  • 33
  • ...
    • -
  • 72
    • +
  • 63
  • »
    • diff --git a/doc/pub/week35/html/._week35-bs024.html b/doc/pub/week35/html/._week35-bs024.html index 5c76e2186..ef7138610 100644 --- a/doc/pub/week35/html/._week35-bs024.html +++ b/doc/pub/week35/html/._week35-bs024.html @@ -38,11 +38,7 @@ doconce format html week35.do.txt --html_style=bootstrap --pygments_html_style=d {'highest level': 2, 'sections': [('Plans for week 35', 2, None, 'plans-for-week-35'), ('Reading recommendations:', 3, None, 'reading-recommendations'), - ('For exercise sessions: Why Linear Regression (aka Ordinary ' - 'Least Squares and family), repeat from last week', - 2, - None, - 'for-exercise-sessions-why-linear-regression-aka-ordinary-least-squares-and-family-repeat-from-last-week'), + ('Reminder from last week', 2, None, 'reminder-from-last-week'), ('The equations for ordinary least squares', 2, None, @@ -122,32 +118,6 @@ doconce format html week35.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'testing-the-means-squared-error-as-function-of-complexity'), - ('More preprocessing examples, two-dimensional example, the ' - 'Franke function', - 2, - None, - 'more-preprocessing-examples-two-dimensional-example-the-franke-function'), - ('To think about, first part', - 2, - None, - 'to-think-about-first-part'), - ('More thinking', 2, None, 'more-thinking'), - ('Still thinking', 2, None, 'still-thinking'), - ('What does centering (subtracting the mean values) mean ' - 'mathematically?', - 2, - None, - 'what-does-centering-subtracting-the-mean-values-mean-mathematically'), - ('Further Manipulations', 2, None, 'further-manipulations'), - ('Wrapping it up', 2, None, 'wrapping-it-up'), - ('Linear Regression code, Intercept handling first', - 2, - None, - 'linear-regression-code-intercept-handling-first'), - ('Material for lecture Monday, August 26', - 2, - None, - 'material-for-lecture-monday-august-26'), ('Mathematical Interpretation of Ordinary Least Squares', 2, None, @@ -279,7 +249,7 @@ MathJax.Hub.Config({ @@ -410,7 +371,7 @@ the features in a way to avoid such outlier values.
  • 33
  • 34
  • ...
    • -
  • 72
    • +
  • 63
  • »
    • diff --git a/doc/pub/week35/html/._week35-bs025.html b/doc/pub/week35/html/._week35-bs025.html index efe604346..31f7a21cd 100644 --- a/doc/pub/week35/html/._week35-bs025.html +++ b/doc/pub/week35/html/._week35-bs025.html @@ -38,11 +38,7 @@ doconce format html week35.do.txt --html_style=bootstrap --pygments_html_style=d {'highest level': 2, 'sections': [('Plans for week 35', 2, None, 'plans-for-week-35'), ('Reading recommendations:', 3, None, 'reading-recommendations'), - ('For exercise sessions: Why Linear Regression (aka Ordinary ' - 'Least Squares and family), repeat from last week', - 2, - None, - 'for-exercise-sessions-why-linear-regression-aka-ordinary-least-squares-and-family-repeat-from-last-week'), + ('Reminder from last week', 2, None, 'reminder-from-last-week'), ('The equations for ordinary least squares', 2, None, @@ -122,32 +118,6 @@ doconce format html week35.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'testing-the-means-squared-error-as-function-of-complexity'), - ('More preprocessing examples, two-dimensional example, the ' - 'Franke function', - 2, - None, - 'more-preprocessing-examples-two-dimensional-example-the-franke-function'), - ('To think about, first part', - 2, - None, - 'to-think-about-first-part'), - ('More thinking', 2, None, 'more-thinking'), - ('Still thinking', 2, None, 'still-thinking'), - ('What does centering (subtracting the mean values) mean ' - 'mathematically?', - 2, - None, - 'what-does-centering-subtracting-the-mean-values-mean-mathematically'), - ('Further Manipulations', 2, None, 'further-manipulations'), - ('Wrapping it up', 2, None, 'wrapping-it-up'), - ('Linear Regression code, Intercept handling first', - 2, - None, - 'linear-regression-code-intercept-handling-first'), - ('Material for lecture Monday, August 26', - 2, - None, - 'material-for-lecture-monday-august-26'), ('Mathematical Interpretation of Ordinary Least Squares', 2, None, @@ -279,7 +249,7 @@ MathJax.Hub.Config({ @@ -398,7 +359,7 @@ ensures that all features are exactly between \( 0 \) and \( 1 \). The
  • 34
  • 35
  • ...
    • -
  • 72
    • +
  • 63
  • »
    • diff --git a/doc/pub/week35/html/._week35-bs026.html b/doc/pub/week35/html/._week35-bs026.html index 8a8d3546e..566ee51e7 100644 --- a/doc/pub/week35/html/._week35-bs026.html +++ b/doc/pub/week35/html/._week35-bs026.html @@ -38,11 +38,7 @@ doconce format html week35.do.txt --html_style=bootstrap --pygments_html_style=d {'highest level': 2, 'sections': [('Plans for week 35', 2, None, 'plans-for-week-35'), ('Reading recommendations:', 3, None, 'reading-recommendations'), - ('For exercise sessions: Why Linear Regression (aka Ordinary ' - 'Least Squares and family), repeat from last week', - 2, - None, - 'for-exercise-sessions-why-linear-regression-aka-ordinary-least-squares-and-family-repeat-from-last-week'), + ('Reminder from last week', 2, None, 'reminder-from-last-week'), ('The equations for ordinary least squares', 2, None, @@ -122,32 +118,6 @@ doconce format html week35.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'testing-the-means-squared-error-as-function-of-complexity'), - ('More preprocessing examples, two-dimensional example, the ' - 'Franke function', - 2, - None, - 'more-preprocessing-examples-two-dimensional-example-the-franke-function'), - ('To think about, first part', - 2, - None, - 'to-think-about-first-part'), - ('More thinking', 2, None, 'more-thinking'), - ('Still thinking', 2, None, 'still-thinking'), - ('What does centering (subtracting the mean values) mean ' - 'mathematically?', - 2, - None, - 'what-does-centering-subtracting-the-mean-values-mean-mathematically'), - ('Further Manipulations', 2, None, 'further-manipulations'), - ('Wrapping it up', 2, None, 'wrapping-it-up'), - ('Linear Regression code, Intercept handling first', - 2, - None, - 'linear-regression-code-intercept-handling-first'), - ('Material for lecture Monday, August 26', - 2, - None, - 'material-for-lecture-monday-august-26'), ('Mathematical Interpretation of Ordinary Least Squares', 2, None, @@ -279,7 +249,7 @@ MathJax.Hub.Config({ @@ -412,7 +373,7 @@ techniques.
  • 35
  • 36
  • ...
    • -
  • 72
    • +
  • 63
  • »
    • diff --git a/doc/pub/week35/html/._week35-bs027.html b/doc/pub/week35/html/._week35-bs027.html index e7da5f800..cd06a623a 100644 --- a/doc/pub/week35/html/._week35-bs027.html +++ b/doc/pub/week35/html/._week35-bs027.html @@ -38,11 +38,7 @@ doconce format html week35.do.txt --html_style=bootstrap --pygments_html_style=d {'highest level': 2, 'sections': [('Plans for week 35', 2, None, 'plans-for-week-35'), ('Reading recommendations:', 3, None, 'reading-recommendations'), - ('For exercise sessions: Why Linear Regression (aka Ordinary ' - 'Least Squares and family), repeat from last week', - 2, - None, - 'for-exercise-sessions-why-linear-regression-aka-ordinary-least-squares-and-family-repeat-from-last-week'), + ('Reminder from last week', 2, None, 'reminder-from-last-week'), ('The equations for ordinary least squares', 2, None, @@ -122,32 +118,6 @@ doconce format html week35.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'testing-the-means-squared-error-as-function-of-complexity'), - ('More preprocessing examples, two-dimensional example, the ' - 'Franke function', - 2, - None, - 'more-preprocessing-examples-two-dimensional-example-the-franke-function'), - ('To think about, first part', - 2, - None, - 'to-think-about-first-part'), - ('More thinking', 2, None, 'more-thinking'), - ('Still thinking', 2, None, 'still-thinking'), - ('What does centering (subtracting the mean values) mean ' - 'mathematically?', - 2, - None, - 'what-does-centering-subtracting-the-mean-values-mean-mathematically'), - ('Further Manipulations', 2, None, 'further-manipulations'), - ('Wrapping it up', 2, None, 'wrapping-it-up'), - ('Linear Regression code, Intercept handling first', - 2, - None, - 'linear-regression-code-intercept-handling-first'), - ('Material for lecture Monday, August 26', - 2, - None, - 'material-for-lecture-monday-august-26'), ('Mathematical Interpretation of Ordinary Least Squares', 2, None, @@ -279,7 +249,7 @@ MathJax.Hub.Config({ @@ -399,7 +360,7 @@ This ensures that each feature has zero mean and unit standard deviation. For d
  • 36
  • 37
  • ...
    • -
  • 72
    • +
  • 63
  • »
    • diff --git a/doc/pub/week35/html/._week35-bs028.html b/doc/pub/week35/html/._week35-bs028.html index ecc29be05..26c7ec1c1 100644 --- a/doc/pub/week35/html/._week35-bs028.html +++ b/doc/pub/week35/html/._week35-bs028.html @@ -38,11 +38,7 @@ doconce format html week35.do.txt --html_style=bootstrap --pygments_html_style=d {'highest level': 2, 'sections': [('Plans for week 35', 2, None, 'plans-for-week-35'), ('Reading recommendations:', 3, None, 'reading-recommendations'), - ('For exercise sessions: Why Linear Regression (aka Ordinary ' - 'Least Squares and family), repeat from last week', - 2, - None, - 'for-exercise-sessions-why-linear-regression-aka-ordinary-least-squares-and-family-repeat-from-last-week'), + ('Reminder from last week', 2, None, 'reminder-from-last-week'), ('The equations for ordinary least squares', 2, None, @@ -122,32 +118,6 @@ doconce format html week35.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'testing-the-means-squared-error-as-function-of-complexity'), - ('More preprocessing examples, two-dimensional example, the ' - 'Franke function', - 2, - None, - 'more-preprocessing-examples-two-dimensional-example-the-franke-function'), - ('To think about, first part', - 2, - None, - 'to-think-about-first-part'), - ('More thinking', 2, None, 'more-thinking'), - ('Still thinking', 2, None, 'still-thinking'), - ('What does centering (subtracting the mean values) mean ' - 'mathematically?', - 2, - None, - 'what-does-centering-subtracting-the-mean-values-mean-mathematically'), - ('Further Manipulations', 2, None, 'further-manipulations'), - ('Wrapping it up', 2, None, 'wrapping-it-up'), - ('Linear Regression code, Intercept handling first', - 2, - None, - 'linear-regression-code-intercept-handling-first'), - ('Material for lecture Monday, August 26', - 2, - None, - 'material-for-lecture-monday-august-26'), ('Mathematical Interpretation of Ordinary Least Squares', 2, None, @@ -279,7 +249,7 @@ MathJax.Hub.Config({ @@ -440,7 +401,7 @@ display(XPandas-Xscaled)
  • 37
  • 38
  • ...
    • -
  • 72
    • +
  • 63
  • »
    • diff --git a/doc/pub/week35/html/._week35-bs029.html b/doc/pub/week35/html/._week35-bs029.html index 2b41810cd..4b6afc329 100644 --- a/doc/pub/week35/html/._week35-bs029.html +++ b/doc/pub/week35/html/._week35-bs029.html @@ -38,11 +38,7 @@ doconce format html week35.do.txt --html_style=bootstrap --pygments_html_style=d {'highest level': 2, 'sections': [('Plans for week 35', 2, None, 'plans-for-week-35'), ('Reading recommendations:', 3, None, 'reading-recommendations'), - ('For exercise sessions: Why Linear Regression (aka Ordinary ' - 'Least Squares and family), repeat from last week', - 2, - None, - 'for-exercise-sessions-why-linear-regression-aka-ordinary-least-squares-and-family-repeat-from-last-week'), + ('Reminder from last week', 2, None, 'reminder-from-last-week'), ('The equations for ordinary least squares', 2, None, @@ -122,32 +118,6 @@ doconce format html week35.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'testing-the-means-squared-error-as-function-of-complexity'), - ('More preprocessing examples, two-dimensional example, the ' - 'Franke function', - 2, - None, - 'more-preprocessing-examples-two-dimensional-example-the-franke-function'), - ('To think about, first part', - 2, - None, - 'to-think-about-first-part'), - ('More thinking', 2, None, 'more-thinking'), - ('Still thinking', 2, None, 'still-thinking'), - ('What does centering (subtracting the mean values) mean ' - 'mathematically?', - 2, - None, - 'what-does-centering-subtracting-the-mean-values-mean-mathematically'), - ('Further Manipulations', 2, None, 'further-manipulations'), - ('Wrapping it up', 2, None, 'wrapping-it-up'), - ('Linear Regression code, Intercept handling first', - 2, - None, - 'linear-regression-code-intercept-handling-first'), - ('Material for lecture Monday, August 26', - 2, - None, - 'material-for-lecture-monday-august-26'), ('Mathematical Interpretation of Ordinary Least Squares', 2, None, @@ -279,7 +249,7 @@ MathJax.Hub.Config({ @@ -399,7 +360,7 @@ $$
  • 38
  • 39
  • ...
    • -
  • 72
    • +
  • 63
  • »
    • diff --git a/doc/pub/week35/html/._week35-bs030.html b/doc/pub/week35/html/._week35-bs030.html index ebbb25cb1..16c258074 100644 --- a/doc/pub/week35/html/._week35-bs030.html +++ b/doc/pub/week35/html/._week35-bs030.html @@ -38,11 +38,7 @@ doconce format html week35.do.txt --html_style=bootstrap --pygments_html_style=d {'highest level': 2, 'sections': [('Plans for week 35', 2, None, 'plans-for-week-35'), ('Reading recommendations:', 3, None, 'reading-recommendations'), - ('For exercise sessions: Why Linear Regression (aka Ordinary ' - 'Least Squares and family), repeat from last week', - 2, - None, - 'for-exercise-sessions-why-linear-regression-aka-ordinary-least-squares-and-family-repeat-from-last-week'), + ('Reminder from last week', 2, None, 'reminder-from-last-week'), ('The equations for ordinary least squares', 2, None, @@ -122,32 +118,6 @@ doconce format html week35.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'testing-the-means-squared-error-as-function-of-complexity'), - ('More preprocessing examples, two-dimensional example, the ' - 'Franke function', - 2, - None, - 'more-preprocessing-examples-two-dimensional-example-the-franke-function'), - ('To think about, first part', - 2, - None, - 'to-think-about-first-part'), - ('More thinking', 2, None, 'more-thinking'), - ('Still thinking', 2, None, 'still-thinking'), - ('What does centering (subtracting the mean values) mean ' - 'mathematically?', - 2, - None, - 'what-does-centering-subtracting-the-mean-values-mean-mathematically'), - ('Further Manipulations', 2, None, 'further-manipulations'), - ('Wrapping it up', 2, None, 'wrapping-it-up'), - ('Linear Regression code, Intercept handling first', - 2, - None, - 'linear-regression-code-intercept-handling-first'), - ('Material for lecture Monday, August 26', - 2, - None, - 'material-for-lecture-monday-august-26'), ('Mathematical Interpretation of Ordinary Least Squares', 2, None, @@ -279,7 +249,7 @@ MathJax.Hub.Config({ @@ -482,7 +443,7 @@ plt.show()
  • 39
  • 40
  • ...
    • -
  • 72
    • +
  • 63
  • »
    • diff --git a/doc/pub/week35/html/._week35-bs031.html b/doc/pub/week35/html/._week35-bs031.html index 3ee214377..e5aadeb88 100644 --- a/doc/pub/week35/html/._week35-bs031.html +++ b/doc/pub/week35/html/._week35-bs031.html @@ -38,11 +38,7 @@ doconce format html week35.do.txt --html_style=bootstrap --pygments_html_style=d {'highest level': 2, 'sections': [('Plans for week 35', 2, None, 'plans-for-week-35'), ('Reading recommendations:', 3, None, 'reading-recommendations'), - ('For exercise sessions: Why Linear Regression (aka Ordinary ' - 'Least Squares and family), repeat from last week', - 2, - None, - 'for-exercise-sessions-why-linear-regression-aka-ordinary-least-squares-and-family-repeat-from-last-week'), + ('Reminder from last week', 2, None, 'reminder-from-last-week'), ('The equations for ordinary least squares', 2, None, @@ -122,32 +118,6 @@ doconce format html week35.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'testing-the-means-squared-error-as-function-of-complexity'), - ('More preprocessing examples, two-dimensional example, the ' - 'Franke function', - 2, - None, - 'more-preprocessing-examples-two-dimensional-example-the-franke-function'), - ('To think about, first part', - 2, - None, - 'to-think-about-first-part'), - ('More thinking', 2, None, 'more-thinking'), - ('Still thinking', 2, None, 'still-thinking'), - ('What does centering (subtracting the mean values) mean ' - 'mathematically?', - 2, - None, - 'what-does-centering-subtracting-the-mean-values-mean-mathematically'), - ('Further Manipulations', 2, None, 'further-manipulations'), - ('Wrapping it up', 2, None, 'wrapping-it-up'), - ('Linear Regression code, Intercept handling first', - 2, - None, - 'linear-regression-code-intercept-handling-first'), - ('Material for lecture Monday, August 26', - 2, - None, - 'material-for-lecture-monday-august-26'), ('Mathematical Interpretation of Ordinary Least Squares', 2, None, @@ -279,7 +249,7 @@ MathJax.Hub.Config({ @@ -360,122 +321,37 @@ MathJax.Hub.Config({

     

     

     

    -

    More preprocessing examples, two-dimensional example, the Franke function

    +

    Mathematical Interpretation of Ordinary Least Squares

    +

    What is presented here is a mathematical analysis of various regression algorithms (ordinary least squares, Ridge and Lasso Regression). The analysis is based on an important algorithm in linear algebra, the so-called Singular Value Decomposition (SVD).

    - -
    -
    -
    -
    -
    - 
    # Common imports
-import os
-import numpy as np
-import pandas as pd
-import matplotlib.pyplot as plt
-import sklearn.linear_model as skl
-from sklearn.metrics import mean_squared_error
-from sklearn.model_selection import  train_test_split
-from sklearn.preprocessing import MinMaxScaler, StandardScaler, Normalizer
+
    We have shown that in ordinary least squares the optimal parameters \( \beta \) are given by
    
 
-# Where to save the figures and data files
-PROJECT_ROOT_DIR = "Results"
-FIGURE_ID = "Results/FigureFiles"
-DATA_ID = "DataFiles/"
+$$
+\hat{\boldsymbol{\beta}} = \left(\boldsymbol{X}^T\boldsymbol{X}\right)^{-1}\boldsymbol{X}^T\boldsymbol{y}.
+$$
 
-if not os.path.exists(PROJECT_ROOT_DIR):
-    os.mkdir(PROJECT_ROOT_DIR)
+
    The hat over \( \boldsymbol{\beta} \) means we have the optimal parameters after minimization of the cost function.
    
 
-if not os.path.exists(FIGURE_ID):
-    os.makedirs(FIGURE_ID)
+
    This means that our best model is defined as
    
 
-if not os.path.exists(DATA_ID):
-    os.makedirs(DATA_ID)
+$$
+\tilde{\boldsymbol{y}}=\boldsymbol{X}\hat{\boldsymbol{\beta}} = \boldsymbol{X}\left(\boldsymbol{X}^T\boldsymbol{X}\right)^{-1}\boldsymbol{X}^T\boldsymbol{y}.
+$$
 
-def image_path(fig_id):
-    return os.path.join(FIGURE_ID, fig_id)
+
    We now define a matrix
    
+$$
+\boldsymbol{A}=\boldsymbol{X}\left(\boldsymbol{X}^T\boldsymbol{X}\right)^{-1}\boldsymbol{X}^T.
+$$
 
-def data_path(dat_id):
-    return os.path.join(DATA_ID, dat_id)
-
-def save_fig(fig_id):
-    plt.savefig(image_path(fig_id) + ".png", format='png')
-
-
-def FrankeFunction(x,y):
-	term1 = 0.75*np.exp(-(0.25*(9*x-2)**2) - 0.25*((9*y-2)**2))
-	term2 = 0.75*np.exp(-((9*x+1)**2)/49.0 - 0.1*(9*y+1))
-	term3 = 0.5*np.exp(-(9*x-7)**2/4.0 - 0.25*((9*y-3)**2))
-	term4 = -0.2*np.exp(-(9*x-4)**2 - (9*y-7)**2)
-	return term1 + term2 + term3 + term4
-
-
-def create_X(x, y, n ):
-	if len(x.shape) > 1:
-		x = np.ravel(x)
-		y = np.ravel(y)
-
-	N = len(x)
-	l = int((n+1)*(n+2)/2)		# Number of elements in beta
-	X = np.ones((N,l))
-
-	for i in range(1,n+1):
-		q = int((i)*(i+1)/2)
-		for k in range(i+1):
-			X[:,q+k] = (x**(i-k))*(y**k)
-
-	return X
-
-
-# Making meshgrid of datapoints and compute Franke's function
-n = 5
-N = 1000
-x = np.sort(np.random.uniform(0, 1, N))
-y = np.sort(np.random.uniform(0, 1, N))
-z = FrankeFunction(x, y)
-X = create_X(x, y, n=n)    
-# split in training and test data
-X_train, X_test, y_train, y_test = train_test_split(X,z,test_size=0.2)
-
-
-clf = skl.LinearRegression().fit(X_train, y_train)
-
-# The mean squared error and R2 score
-print("MSE before scaling: {:.2f}".format(mean_squared_error(clf.predict(X_test), y_test)))
-print("R2 score before scaling {:.2f}".format(clf.score(X_test,y_test)))
-
-scaler = StandardScaler()
-scaler.fit(X_train)
-X_train_scaled = scaler.transform(X_train)
-X_test_scaled = scaler.transform(X_test)
-
-print("Feature min values before scaling:\n {}".format(X_train.min(axis=0)))
-print("Feature max values before scaling:\n {}".format(X_train.max(axis=0)))
-
-print("Feature min values after scaling:\n {}".format(X_train_scaled.min(axis=0)))
-print("Feature max values after scaling:\n {}".format(X_train_scaled.max(axis=0)))
-
-clf = skl.LinearRegression().fit(X_train_scaled, y_train)
-
-
-print("MSE after  scaling: {:.2f}".format(mean_squared_error(clf.predict(X_test_scaled), y_test)))
-print("R2 score for  scaled data: {:.2f}".format(clf.score(X_test_scaled,y_test)))
-
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    +

    We can rewrite

    +$$ +\tilde{\boldsymbol{y}}=\boldsymbol{X}\hat{\boldsymbol{\beta}} = \boldsymbol{A}\boldsymbol{y}. +$$ +

    The matrix \( \boldsymbol{A} \) has the important property that \( \boldsymbol{A}^2=\boldsymbol{A} \). This is the definition of a projection matrix. +We can then interpret our optimal model \( \tilde{\boldsymbol{y}} \) as being represented by an orthogonal projection of \( \boldsymbol{y} \) onto a space defined by the column vectors of \( \boldsymbol{X} \). In our case here the matrix \( \boldsymbol{A} \) is a square matrix. If it is a general rectangular matrix we have an oblique projection matrix. +

    @@ -502,7 +378,7 @@ clf = skl.40

  • 41
  • ...
    • -
  • 72
    • +
  • 63
  • »
    • diff --git a/doc/pub/week35/html/._week35-bs032.html b/doc/pub/week35/html/._week35-bs032.html index a0aa82370..4f6a5fea8 100644 --- a/doc/pub/week35/html/._week35-bs032.html +++ b/doc/pub/week35/html/._week35-bs032.html @@ -38,11 +38,7 @@ doconce format html week35.do.txt --html_style=bootstrap --pygments_html_style=d {'highest level': 2, 'sections': [('Plans for week 35', 2, None, 'plans-for-week-35'), ('Reading recommendations:', 3, None, 'reading-recommendations'), - ('For exercise sessions: Why Linear Regression (aka Ordinary ' - 'Least Squares and family), repeat from last week', - 2, - None, - 'for-exercise-sessions-why-linear-regression-aka-ordinary-least-squares-and-family-repeat-from-last-week'), + ('Reminder from last week', 2, None, 'reminder-from-last-week'), ('The equations for ordinary least squares', 2, None, @@ -122,32 +118,6 @@ doconce format html week35.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'testing-the-means-squared-error-as-function-of-complexity'), - ('More preprocessing examples, two-dimensional example, the ' - 'Franke function', - 2, - None, - 'more-preprocessing-examples-two-dimensional-example-the-franke-function'), - ('To think about, first part', - 2, - None, - 'to-think-about-first-part'), - ('More thinking', 2, None, 'more-thinking'), - ('Still thinking', 2, None, 'still-thinking'), - ('What does centering (subtracting the mean values) mean ' - 'mathematically?', - 2, - None, - 'what-does-centering-subtracting-the-mean-values-mean-mathematically'), - ('Further Manipulations', 2, None, 'further-manipulations'), - ('Wrapping it up', 2, None, 'wrapping-it-up'), - ('Linear Regression code, Intercept handling first', - 2, - None, - 'linear-regression-code-intercept-handling-first'), - ('Material for lecture Monday, August 26', - 2, - None, - 'material-for-lecture-monday-august-26'), ('Mathematical Interpretation of Ordinary Least Squares', 2, None, @@ -279,7 +249,7 @@ MathJax.Hub.Config({ @@ -360,28 +321,14 @@ MathJax.Hub.Config({

     

     

     

    -

    To think about, first part

    +

    Residual Error

    -

    When you are comparing your own code with for example Scikit-Learn's -library, there are some technicalities to keep in mind. The examples -here demonstrate some of these aspects with potential pitfalls. -

    +

    We have defined the residual error as

    +$$ +\boldsymbol{\epsilon}=\boldsymbol{y}-\tilde{\boldsymbol{y}}=\left[\boldsymbol{I}-\boldsymbol{X}\left(\boldsymbol{X}^T\boldsymbol{X}\right)^{-1}\boldsymbol{X}^T\right]\boldsymbol{y}. +$$ -

    The discussion here focuses on the role of the intercept, how we can -set up the design matrix, what scaling we should use and other topics -which tend confuse us. -

    - -

    The intercept can be interpreted as the expected value of our -target/output variables when all other predictors are set to zero. -Thus, if we cannot assume that the expected outputs/targets are zero -when all predictors are zero (the columns in the design matrix), it -may be a bad idea to implement a model which penalizes the intercept. -Furthermore, in for example Ridge and Lasso regression (to be discussed in moe detail next week), the default solutions -from the library Scikit-Learn (when not shrinking \( \beta_0 \)) for the unknown parameters -\( \boldsymbol{\beta} \), are derived under the assumption that both \( \boldsymbol{y} \) and -\( \boldsymbol{X} \) are zero centered, that is we subtract the mean values. -

    +

    The residual errors are then the projections of \( \boldsymbol{y} \) onto the orthogonal component of the space defined by the column vectors of \( \boldsymbol{X} \).

    @@ -408,7 +355,7 @@ from the library Scikit-Learn (when not shrinking \( \beta_0 \)) for the

  • 41
  • 42
  • ...
    • -
  • 72
    • +
  • 63
  • »
    • diff --git a/doc/pub/week35/html/._week35-bs033.html b/doc/pub/week35/html/._week35-bs033.html index fb9d553e1..a1614c3b4 100644 --- a/doc/pub/week35/html/._week35-bs033.html +++ b/doc/pub/week35/html/._week35-bs033.html @@ -38,11 +38,7 @@ doconce format html week35.do.txt --html_style=bootstrap --pygments_html_style=d {'highest level': 2, 'sections': [('Plans for week 35', 2, None, 'plans-for-week-35'), ('Reading recommendations:', 3, None, 'reading-recommendations'), - ('For exercise sessions: Why Linear Regression (aka Ordinary ' - 'Least Squares and family), repeat from last week', - 2, - None, - 'for-exercise-sessions-why-linear-regression-aka-ordinary-least-squares-and-family-repeat-from-last-week'), + ('Reminder from last week', 2, None, 'reminder-from-last-week'), ('The equations for ordinary least squares', 2, None, @@ -122,32 +118,6 @@ doconce format html week35.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'testing-the-means-squared-error-as-function-of-complexity'), - ('More preprocessing examples, two-dimensional example, the ' - 'Franke function', - 2, - None, - 'more-preprocessing-examples-two-dimensional-example-the-franke-function'), - ('To think about, first part', - 2, - None, - 'to-think-about-first-part'), - ('More thinking', 2, None, 'more-thinking'), - ('Still thinking', 2, None, 'still-thinking'), - ('What does centering (subtracting the mean values) mean ' - 'mathematically?', - 2, - None, - 'what-does-centering-subtracting-the-mean-values-mean-mathematically'), - ('Further Manipulations', 2, None, 'further-manipulations'), - ('Wrapping it up', 2, None, 'wrapping-it-up'), - ('Linear Regression code, Intercept handling first', - 2, - None, - 'linear-regression-code-intercept-handling-first'), - ('Material for lecture Monday, August 26', - 2, - None, - 'material-for-lecture-monday-august-26'), ('Mathematical Interpretation of Ordinary Least Squares', 2, None, @@ -279,7 +249,7 @@ MathJax.Hub.Config({ @@ -360,36 +321,25 @@ MathJax.Hub.Config({

     

     

     

    -

    More thinking

    +

    Simple case

    -

    If our predictors represent different scales, then it is important to -standardize the design matrix \( \boldsymbol{X} \) by subtracting the mean of each -column from the corresponding column and dividing the column with its -standard deviation. Most machine learning libraries do this as a default. This means that if you compare your code with the results from a given library, -the results may differ. -

    +

    If the matrix \( \boldsymbol{X} \) is an orthogonal (or unitary in case of complex values) matrix, we have

    -

    The -Standadscaler -function in Scikit-Learn does this for us. For the data sets we -have been studying in our various examples, the data are in many cases -already scaled and there is no need to scale them. You as a user of different machine learning algorithms, should always perform a -survey of your data, with a critical assessment of them in case you need to scale the data. -

    +$$ +\boldsymbol{X}^T\boldsymbol{X}=\boldsymbol{X}\boldsymbol{X}^T = \boldsymbol{I}. +$$ -

    If you need to scale the data, not doing so will give an unfair -penalization of the parameters since their magnitude depends on the -scale of their corresponding predictor. -

    +

    In this case the matrix \( \boldsymbol{A} \) becomes

    +$$ +\boldsymbol{A}=\boldsymbol{X}\left(\boldsymbol{X}^T\boldsymbol{X}\right)^{-1}\boldsymbol{X}^T)=\boldsymbol{I}, +$$ -

    Suppose as an example that you -you have an input variable given by the heights of different persons. -Human height might be measured in inches or meters or -kilometers. If measured in kilometers, a standard linear regression -model with this predictor would probably give a much bigger -coefficient term, than if measured in millimeters. -This can clearly lead to problems in evaluating the cost/loss functions. -

    +

    and we have the obvious case

    +$$ +\boldsymbol{\epsilon}=\boldsymbol{y}-\tilde{\boldsymbol{y}}=0. +$$ + +

    This serves also as a useful test of our codes.

    @@ -416,7 +366,7 @@ This can clearly lead to problems in evaluating the cost/loss functions.

  • 42
  • 43
  • ...
    • -
  • 72
    • +
  • 63
  • »
    • diff --git a/doc/pub/week35/html/._week35-bs034.html b/doc/pub/week35/html/._week35-bs034.html index 22fc7e5b1..51593d84b 100644 --- a/doc/pub/week35/html/._week35-bs034.html +++ b/doc/pub/week35/html/._week35-bs034.html @@ -38,11 +38,7 @@ doconce format html week35.do.txt --html_style=bootstrap --pygments_html_style=d {'highest level': 2, 'sections': [('Plans for week 35', 2, None, 'plans-for-week-35'), ('Reading recommendations:', 3, None, 'reading-recommendations'), - ('For exercise sessions: Why Linear Regression (aka Ordinary ' - 'Least Squares and family), repeat from last week', - 2, - None, - 'for-exercise-sessions-why-linear-regression-aka-ordinary-least-squares-and-family-repeat-from-last-week'), + ('Reminder from last week', 2, None, 'reminder-from-last-week'), ('The equations for ordinary least squares', 2, None, @@ -122,32 +118,6 @@ doconce format html week35.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'testing-the-means-squared-error-as-function-of-complexity'), - ('More preprocessing examples, two-dimensional example, the ' - 'Franke function', - 2, - None, - 'more-preprocessing-examples-two-dimensional-example-the-franke-function'), - ('To think about, first part', - 2, - None, - 'to-think-about-first-part'), - ('More thinking', 2, None, 'more-thinking'), - ('Still thinking', 2, None, 'still-thinking'), - ('What does centering (subtracting the mean values) mean ' - 'mathematically?', - 2, - None, - 'what-does-centering-subtracting-the-mean-values-mean-mathematically'), - ('Further Manipulations', 2, None, 'further-manipulations'), - ('Wrapping it up', 2, None, 'wrapping-it-up'), - ('Linear Regression code, Intercept handling first', - 2, - None, - 'linear-regression-code-intercept-handling-first'), - ('Material for lecture Monday, August 26', - 2, - None, - 'material-for-lecture-monday-august-26'), ('Mathematical Interpretation of Ordinary Least Squares', 2, None, @@ -279,7 +249,7 @@ MathJax.Hub.Config({ @@ -360,46 +321,47 @@ MathJax.Hub.Config({

     

     

     

    -

    Still thinking

    +

    The singular value decomposition

    -

    Keep in mind that when you transform your data set before training a model, the same transformation needs to be done -on your eventual new data set before making a prediction. If we translate this into a Python code, it would could be implemented as follows -(note that the lines are commented since the model function has not been defined) +

    +
    + + +

    The examples we have looked at so far are cases where we normally can +invert the matrix \( \boldsymbol{X}^T\boldsymbol{X} \). Using a polynomial expansion where we fit of various functions leads to +row vectors of the design matrix which are essentially orthogonal due +to the polynomial character of our model. Obtaining the inverse of the +design matrix is then often done via a so-called LU, QR or Cholesky +decomposition.

    - -
    -
    -
    -
    -
    - 
    #Model training, we compute the mean value of y and X
-y_train_mean = np.mean(y_train)
-X_train_mean = np.mean(X_train,axis=0)
-X_train = X_train - X_train_mean
-y_train = y_train - y_train_mean
+
    As we will also see in the first project, 
+this may
+however not the be case in general and a standard matrix inversion
+algorithm based on say LU, QR or Cholesky decomposition may lead to singularities. We will see examples of this below.
+
    
 
-# The we fit our model with the training data
-#trained_model = some_model.fit(X_train,y_train)
+
    There is however a way to circumvent this problem and also
+gain some insights about the ordinary least squares approach, and
+later shrinkage methods like Ridge and Lasso regressions.
+
    
 
+
    This is given by the Singular Value Decomposition (SVD) algorithm,
+perhaps the most powerful linear algebra algorithm.  The SVD provides
+a numerically stable matrix decomposition that is used in a large
+swath oc applications and the decomposition is always stable
+numerically.
+
    
 
-#Model prediction, we need also to transform our data set used for the prediction.
-X_test = X_test - X_train_mean #Use mean from training data
-#y_pred = trained_model(X_test)
-y_pred = y_pred + y_train_mean
-
    +

    In machine learning it plays a central role in dealing with for +example design matrices that may be near singular or singular. +Furthermore, as we will see here, the singular values can be related +to the covariance matrix (and thereby the correlation matrix) and in +turn the variance of a given quantity. It plays also an important role +in the principal component analysis where high-dimensional data can be +reduced to the statistically relevant features. +

    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    @@ -428,7 +390,7 @@ y_pred = y_pred 43
  • 44
  • ...
    • -
  • 72
    • +
  • 63
  • »
    • diff --git a/doc/pub/week35/html/._week35-bs035.html b/doc/pub/week35/html/._week35-bs035.html index f10422079..c3387ebb0 100644 --- a/doc/pub/week35/html/._week35-bs035.html +++ b/doc/pub/week35/html/._week35-bs035.html @@ -38,11 +38,7 @@ doconce format html week35.do.txt --html_style=bootstrap --pygments_html_style=d {'highest level': 2, 'sections': [('Plans for week 35', 2, None, 'plans-for-week-35'), ('Reading recommendations:', 3, None, 'reading-recommendations'), - ('For exercise sessions: Why Linear Regression (aka Ordinary ' - 'Least Squares and family), repeat from last week', - 2, - None, - 'for-exercise-sessions-why-linear-regression-aka-ordinary-least-squares-and-family-repeat-from-last-week'), + ('Reminder from last week', 2, None, 'reminder-from-last-week'), ('The equations for ordinary least squares', 2, None, @@ -122,32 +118,6 @@ doconce format html week35.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'testing-the-means-squared-error-as-function-of-complexity'), - ('More preprocessing examples, two-dimensional example, the ' - 'Franke function', - 2, - None, - 'more-preprocessing-examples-two-dimensional-example-the-franke-function'), - ('To think about, first part', - 2, - None, - 'to-think-about-first-part'), - ('More thinking', 2, None, 'more-thinking'), - ('Still thinking', 2, None, 'still-thinking'), - ('What does centering (subtracting the mean values) mean ' - 'mathematically?', - 2, - None, - 'what-does-centering-subtracting-the-mean-values-mean-mathematically'), - ('Further Manipulations', 2, None, 'further-manipulations'), - ('Wrapping it up', 2, None, 'wrapping-it-up'), - ('Linear Regression code, Intercept handling first', - 2, - None, - 'linear-regression-code-intercept-handling-first'), - ('Material for lecture Monday, August 26', - 2, - None, - 'material-for-lecture-monday-august-26'), ('Mathematical Interpretation of Ordinary Least Squares', 2, None, @@ -279,7 +249,7 @@ MathJax.Hub.Config({ @@ -360,40 +321,54 @@ MathJax.Hub.Config({

     

     

     

    -

    What does centering (subtracting the mean values) mean mathematically?

    +

    Linear Regression Problems

    -

    Let us try to understand what this may imply mathematically when we -subtract the mean values, also known as zero centering. For -simplicity, we will focus on ordinary regression, as done in the above example. -

    - -

    The cost/loss function for regression is

    -$$ -C(\beta_0, \beta_1, ... , \beta_{p-1}) = \frac{1}{n}\sum_{i=0}^{n} \left(y_i - \beta_0 - \sum_{j=1}^{p-1} X_{ij}\beta_j\right)^2,. -$$ - -

    Recall also that we use the squared value since this leads to an increase of the penalty for higher differences between predicted and output/target values.

    - -

    What we have done is to single out the \( \beta_0 \) term in the definition of the mean squared error (MSE). -The design matrix -\( X \) does in this case not contain any intercept column. -When we take the derivative with respect to \( \beta_0 \), we want the derivative to obey +

    One of the typical problems we encounter with linear regression, in particular +when the matrix \( \boldsymbol{X} \) (our so-called design matrix) is high-dimensional, +are problems with near singular or singular matrices. The column vectors of \( \boldsymbol{X} \) +may be linearly dependent, normally referred to as super-collinearity. +This means that the matrix may be rank deficient and it is basically impossible to +to model the data using linear regression. As an example, consider the matrix

    $$ -\frac{\partial C}{\partial \beta_j} = 0, +\begin{align*} +\mathbf{X} & = \left[ +\begin{array}{rrr} +1 & -1 & 2 +\\ +1 & 0 & 1 +\\ +1 & 2 & -1 +\\ +1 & 1 & 0 +\end{array} \right] +\end{align*} $$ -

    for all \( j \). For \( \beta_0 \) we have

    +

    The columns of \( \boldsymbol{X} \) are linearly dependent. We see this easily since the +the first column is the row-wise sum of the other two columns. The rank (more correct, +the column rank) of a matrix is the dimension of the space spanned by the +column vectors. Hence, the rank of \( \mathbf{X} \) is equal to the number +of linearly independent columns. In this particular case the matrix has rank 2. +

    +

    Super-collinearity of an \( (n \times p) \)-dimensional design matrix \( \mathbf{X} \) implies +that the inverse of the matrix \( \boldsymbol{X}^T\boldsymbol{X} \) (the matrix we need to invert to solve the linear regression equations) is non-invertible. If we have a square matrix that does not have an inverse, we say this matrix singular. The example here demonstrates this +

    $$ -\frac{\partial C}{\partial \beta_0} = -\frac{2}{n}\sum_{i=0}^{n-1} \left(y_i - \beta_0 - \sum_{j=1}^{p-1} X_{ij} \beta_j\right). -$$ - -

    Multiplying away the constant \( 2/n \), we obtain

    -$$ -\sum_{i=0}^{n-1} \beta_0 = \sum_{i=0}^{n-1}y_i - \sum_{i=0}^{n-1} \sum_{j=1}^{p-1} X_{ij} \beta_j. +\begin{align*} +\boldsymbol{X} & = \left[ +\begin{array}{rr} +1 & -1 +\\ +1 & -1 +\end{array} \right]. +\end{align*} $$ +

    We see easily that \( \mbox{det}(\boldsymbol{X}) = x_{11} x_{22} - x_{12} x_{21} = 1 \times (-1) - 1 \times (-1) = 0 \). Hence, \( \mathbf{X} \) is singular and its inverse is undefined. +This is equivalent to saying that the matrix \( \boldsymbol{X} \) has at least an eigenvalue which is zero. +

    @@ -420,7 +395,7 @@ $$

  • 44
  • 45
  • ...
    • -
  • 72
    • +
  • 63
  • »
    • diff --git a/doc/pub/week35/html/._week35-bs036.html b/doc/pub/week35/html/._week35-bs036.html index 715733cbf..5f7291dff 100644 --- a/doc/pub/week35/html/._week35-bs036.html +++ b/doc/pub/week35/html/._week35-bs036.html @@ -38,11 +38,7 @@ doconce format html week35.do.txt --html_style=bootstrap --pygments_html_style=d {'highest level': 2, 'sections': [('Plans for week 35', 2, None, 'plans-for-week-35'), ('Reading recommendations:', 3, None, 'reading-recommendations'), - ('For exercise sessions: Why Linear Regression (aka Ordinary ' - 'Least Squares and family), repeat from last week', - 2, - None, - 'for-exercise-sessions-why-linear-regression-aka-ordinary-least-squares-and-family-repeat-from-last-week'), + ('Reminder from last week', 2, None, 'reminder-from-last-week'), ('The equations for ordinary least squares', 2, None, @@ -122,32 +118,6 @@ doconce format html week35.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'testing-the-means-squared-error-as-function-of-complexity'), - ('More preprocessing examples, two-dimensional example, the ' - 'Franke function', - 2, - None, - 'more-preprocessing-examples-two-dimensional-example-the-franke-function'), - ('To think about, first part', - 2, - None, - 'to-think-about-first-part'), - ('More thinking', 2, None, 'more-thinking'), - ('Still thinking', 2, None, 'still-thinking'), - ('What does centering (subtracting the mean values) mean ' - 'mathematically?', - 2, - None, - 'what-does-centering-subtracting-the-mean-values-mean-mathematically'), - ('Further Manipulations', 2, None, 'further-manipulations'), - ('Wrapping it up', 2, None, 'wrapping-it-up'), - ('Linear Regression code, Intercept handling first', - 2, - None, - 'linear-regression-code-intercept-handling-first'), - ('Material for lecture Monday, August 26', - 2, - None, - 'material-for-lecture-monday-august-26'), ('Mathematical Interpretation of Ordinary Least Squares', 2, None, @@ -279,7 +249,7 @@ MathJax.Hub.Config({ @@ -360,45 +321,29 @@ MathJax.Hub.Config({

     

     

     

    -

    Further Manipulations

    +

    Fixing the singularity

    -

    Let us special first to the case where we have only two parameters \( \beta_0 \) and \( \beta_1 \). -Our result for \( \beta_0 \) simplifies then to +

    If our design matrix \( \boldsymbol{X} \) which enters the linear regression problem

    +$$ +\begin{align} +\boldsymbol{\beta} & = (\boldsymbol{X}^{T} \boldsymbol{X})^{-1} \boldsymbol{X}^{T} \boldsymbol{y}, +\tag{1} +\end{align} +$$ + +

    has linearly dependent column vectors, we will not be able to compute the inverse +of \( \boldsymbol{X}^T\boldsymbol{X} \) and we cannot find the parameters (estimators) \( \beta_i \). +The estimators are only well-defined if \( (\boldsymbol{X}^{T}\boldsymbol{X})^{-1} \) exits. +This is more likely to happen when the matrix \( \boldsymbol{X} \) is high-dimensional. In this case it is likely to encounter a situation where +the regression parameters \( \beta_i \) cannot be estimated.

    + +

    A cheap ad hoc approach is simply to add a small diagonal component to the matrix to invert, that is we change

    $$ -n\beta_0 = \sum_{i=0}^{n-1}y_i - \sum_{i=0}^{n-1} X_{i1} \beta_1. -$$ - -

    We obtain then

    -$$ -\beta_0 = \frac{1}{n}\sum_{i=0}^{n-1}y_i - \beta_1\frac{1}{n}\sum_{i=0}^{n-1} X_{i1}. -$$ - -

    If we define

    -$$ -\mu_1=\frac{1}{n}\sum_{i=0}^{n-1} (X_{i1}, -$$ - -

    and if we define the mean value of the outputs as

    -$$ -\mu_y=\frac{1}{n}\sum_{i=0}^{n-1}y_i, -$$ - -

    we have

    -$$ -\beta_0 = \mu_y - \beta_1\mu_{1}. -$$ - -

    In the general case, that is we have more parameters than \( \beta_0 \) and \( \beta_1 \), we have

    -$$ -\beta_0 = \frac{1}{n}\sum_{i=0}^{n-1}y_i - \frac{1}{n}\sum_{i=0}^{n-1}\sum_{j=1}^{p-1} X_{ij}\beta_j. -$$ - -

    Replacing \( y_i \) with \( y_i - y_i - \overline{\boldsymbol{y}} \) and centering also our design matrix results in a cost function (in vector-matrix disguise)

    -$$ -C(\boldsymbol{\beta}) = (\boldsymbol{\tilde{y}} - \tilde{X}\boldsymbol{\beta})^T(\boldsymbol{\tilde{y}} - \tilde{X}\boldsymbol{\beta}). +\boldsymbol{X}^{T} \boldsymbol{X} \rightarrow \boldsymbol{X}^{T} \boldsymbol{X}+\lambda \boldsymbol{I}, $$ +

    where \( \boldsymbol{I} \) is the identity matrix. When we discuss Ridge regression this is actually what we end up evaluating. The parameter \( \lambda \) is called a hyperparameter. More about this later.

    @@ -425,7 +370,7 @@ $$

  • 45
  • 46
  • ...
    • -
  • 72
    • +
  • 63
  • »
    • diff --git a/doc/pub/week35/html/._week35-bs037.html b/doc/pub/week35/html/._week35-bs037.html index f39627a49..cec97e9ff 100644 --- a/doc/pub/week35/html/._week35-bs037.html +++ b/doc/pub/week35/html/._week35-bs037.html @@ -38,11 +38,7 @@ doconce format html week35.do.txt --html_style=bootstrap --pygments_html_style=d {'highest level': 2, 'sections': [('Plans for week 35', 2, None, 'plans-for-week-35'), ('Reading recommendations:', 3, None, 'reading-recommendations'), - ('For exercise sessions: Why Linear Regression (aka Ordinary ' - 'Least Squares and family), repeat from last week', - 2, - None, - 'for-exercise-sessions-why-linear-regression-aka-ordinary-least-squares-and-family-repeat-from-last-week'), + ('Reminder from last week', 2, None, 'reminder-from-last-week'), ('The equations for ordinary least squares', 2, None, @@ -122,32 +118,6 @@ doconce format html week35.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'testing-the-means-squared-error-as-function-of-complexity'), - ('More preprocessing examples, two-dimensional example, the ' - 'Franke function', - 2, - None, - 'more-preprocessing-examples-two-dimensional-example-the-franke-function'), - ('To think about, first part', - 2, - None, - 'to-think-about-first-part'), - ('More thinking', 2, None, 'more-thinking'), - ('Still thinking', 2, None, 'still-thinking'), - ('What does centering (subtracting the mean values) mean ' - 'mathematically?', - 2, - None, - 'what-does-centering-subtracting-the-mean-values-mean-mathematically'), - ('Further Manipulations', 2, None, 'further-manipulations'), - ('Wrapping it up', 2, None, 'wrapping-it-up'), - ('Linear Regression code, Intercept handling first', - 2, - None, - 'linear-regression-code-intercept-handling-first'), - ('Material for lecture Monday, August 26', - 2, - None, - 'material-for-lecture-monday-august-26'), ('Mathematical Interpretation of Ordinary Least Squares', 2, None, @@ -279,7 +249,7 @@ MathJax.Hub.Config({ @@ -360,24 +321,41 @@ MathJax.Hub.Config({

     

     

     

    -

    Wrapping it up

    +

    Basic math of the SVD

    -

    If we minimize with respect to \( \boldsymbol{\beta} \) we have then

    - -$$ -\hat{\boldsymbol{\beta}} = (\tilde{X}^T\tilde{X})^{-1}\tilde{X}^T\boldsymbol{\tilde{y}}, -$$ - -

    where \( \boldsymbol{\tilde{y}} = \boldsymbol{y} - \overline{\boldsymbol{y}} \) -and \( \tilde{X}_{ij} = X_{ij} - \frac{1}{n}\sum_{k=0}^{n-1}X_{kj} \). +

    From standard linear algebra we know that a square matrix \( \boldsymbol{X} \) can be diagonalized if and only it is +a so-called normal matrix, that is if \( \boldsymbol{X}\in {\mathbb{R}}^{n\times n} \) +we have \( \boldsymbol{X}\boldsymbol{X}^T=\boldsymbol{X}^T\boldsymbol{X} \) or if \( \boldsymbol{X}\in {\mathbb{C}}^{n\times n} \) we have \( \boldsymbol{X}\boldsymbol{X}^{\dagger}=\boldsymbol{X}^{\dagger}\boldsymbol{X} \). +The matrix has then a set of eigenpairs

    -

    For Ridge regression we need to add \( \lambda \boldsymbol{\beta}^T\boldsymbol{\beta} \) to the cost function and get then

    $$ -\hat{\boldsymbol{\beta}} = (\tilde{X}^T\tilde{X} + \lambda I)^{-1}\tilde{X}^T\boldsymbol{\tilde{y}}. +(\lambda_1,\boldsymbol{u}_1),\dots, (\lambda_n,\boldsymbol{u}_n), $$ -

    What does this mean? And why do we insist on all this? Let us look at some examples.

    +

    and the eigenvalues are given by the diagonal matrix

    +$$ +\boldsymbol{\Sigma}=\mathrm{Diag}(\lambda_1, \dots,\lambda_n). +$$ + +

    The matrix \( \boldsymbol{X} \) can be written in terms of an orthogonal/unitary transformation \( \boldsymbol{U} \)

    +$$ +\boldsymbol{X} = \boldsymbol{U}\boldsymbol{\Sigma}\boldsymbol{V}^T, +$$ + +

    with \( \boldsymbol{U}\boldsymbol{U}^T=\boldsymbol{I} \) or \( \boldsymbol{U}\boldsymbol{U}^{\dagger}=\boldsymbol{I} \).

    + +

    Not all square matrices are diagonalizable. A matrix like the one discussed above

    +$$ +\boldsymbol{X} = \begin{bmatrix} +1& -1 \\ +1& -1\\ +\end{bmatrix} +$$ + +

    is not diagonalizable, it is a so-called defective matrix. It is easy to see that the condition +\( \boldsymbol{X}\boldsymbol{X}^T=\boldsymbol{X}^T\boldsymbol{X} \) is not fulfilled. +

    @@ -404,7 +382,7 @@ $$

  • 46
  • 47
  • ...
    • -
  • 72
    • +
  • 63
  • »
    • diff --git a/doc/pub/week35/html/._week35-bs038.html b/doc/pub/week35/html/._week35-bs038.html index 9fec3eeeb..b6d269532 100644 --- a/doc/pub/week35/html/._week35-bs038.html +++ b/doc/pub/week35/html/._week35-bs038.html @@ -38,11 +38,7 @@ doconce format html week35.do.txt --html_style=bootstrap --pygments_html_style=d {'highest level': 2, 'sections': [('Plans for week 35', 2, None, 'plans-for-week-35'), ('Reading recommendations:', 3, None, 'reading-recommendations'), - ('For exercise sessions: Why Linear Regression (aka Ordinary ' - 'Least Squares and family), repeat from last week', - 2, - None, - 'for-exercise-sessions-why-linear-regression-aka-ordinary-least-squares-and-family-repeat-from-last-week'), + ('Reminder from last week', 2, None, 'reminder-from-last-week'), ('The equations for ordinary least squares', 2, None, @@ -122,32 +118,6 @@ doconce format html week35.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'testing-the-means-squared-error-as-function-of-complexity'), - ('More preprocessing examples, two-dimensional example, the ' - 'Franke function', - 2, - None, - 'more-preprocessing-examples-two-dimensional-example-the-franke-function'), - ('To think about, first part', - 2, - None, - 'to-think-about-first-part'), - ('More thinking', 2, None, 'more-thinking'), - ('Still thinking', 2, None, 'still-thinking'), - ('What does centering (subtracting the mean values) mean ' - 'mathematically?', - 2, - None, - 'what-does-centering-subtracting-the-mean-values-mean-mathematically'), - ('Further Manipulations', 2, None, 'further-manipulations'), - ('Wrapping it up', 2, None, 'wrapping-it-up'), - ('Linear Regression code, Intercept handling first', - 2, - None, - 'linear-regression-code-intercept-handling-first'), - ('Material for lecture Monday, August 26', - 2, - None, - 'material-for-lecture-monday-august-26'), ('Mathematical Interpretation of Ordinary Least Squares', 2, None, @@ -279,7 +249,7 @@ MathJax.Hub.Config({ @@ -360,157 +321,53 @@ MathJax.Hub.Config({

     

     

     

    -

    Linear Regression code, Intercept handling first

    +

    The SVD, a Fantastic Algorithm

    -

    This code shows a simple first-order fit to a data set using the above transformed data, where we consider the role of the intercept first, by either excluding it or including it (code example thanks to Øyvind Sigmundson Schøyen). Here our scaling of the data is done by subtracting the mean values only. -Note also that we do not split the data into training and test. +

    However, and this is the strength of the SVD algorithm, any general +matrix \( \boldsymbol{X} \) can be decomposed in terms of a diagonal matrix and +two orthogonal/unitary matrices. The Singular Value Decompostion +(SVD) theorem +states that a general \( m\times n \) matrix \( \boldsymbol{X} \) can be written in +terms of a diagonal matrix \( \boldsymbol{\Sigma} \) of dimensionality \( m\times n \) +and two orthognal matrices \( \boldsymbol{U} \) and \( \boldsymbol{V} \), where the first has +dimensionality \( m \times m \) and the last dimensionality \( n\times n \). +We have then

    +$$ +\boldsymbol{X} = \boldsymbol{U}\boldsymbol{\Sigma}\boldsymbol{V}^T +$$ - -
    -
    -
    -
    -
    - 
    import numpy as np
-import matplotlib.pyplot as plt
+
    As an example, the above defective matrix can be decomposed as
    
 
-from sklearn.linear_model import LinearRegression
+$$
+\boldsymbol{X} = \frac{1}{\sqrt{2}}\begin{bmatrix}  1&  1 \\ 1& -1\\ \end{bmatrix} \begin{bmatrix}  2&  0 \\ 0& 0\\ \end{bmatrix}    \frac{1}{\sqrt{2}}\begin{bmatrix}  1&  -1 \\ 1& 1\\ \end{bmatrix}=\boldsymbol{U}\boldsymbol{\Sigma}\boldsymbol{V}^T, 
+$$
 
-
-np.random.seed(2021)
-
-def MSE(y_data,y_model):
-    n = np.size(y_model)
-    return np.sum((y_data-y_model)**2)/n
-
-
-def fit_beta(X, y):
-    return np.linalg.pinv(X.T @ X) @ X.T @ y
-
-
-true_beta = [2, 0.5, 3.7]
-
-x = np.linspace(0, 1, 11)
-y = np.sum(
-    np.asarray([x ** p * b for p, b in enumerate(true_beta)]), axis=0
-) + 0.1 * np.random.normal(size=len(x))
-
-degree = 3
-X = np.zeros((len(x), degree))
-
-# Include the intercept in the design matrix
-for p in range(degree):
-    X[:, p] = x ** p
-
-beta = fit_beta(X, y)
-
-# Intercept is included in the design matrix
-skl = LinearRegression(fit_intercept=False).fit(X, y)
-
-print(f"True beta: {true_beta}")
-print(f"Fitted beta: {beta}")
-print(f"Sklearn fitted beta: {skl.coef_}")
-ypredictOwn = X @ beta
-ypredictSKL = skl.predict(X)
-print(f"MSE with intercept column")
-print(MSE(y,ypredictOwn))
-print(f"MSE with intercept column from SKL")
-print(MSE(y,ypredictSKL))
-
-
-plt.figure()
-plt.scatter(x, y, label="Data")
-plt.plot(x, X @ beta, label="Fit")
-plt.plot(x, skl.predict(X), label="Sklearn (fit_intercept=False)")
-
-
-# Do not include the intercept in the design matrix
-X = np.zeros((len(x), degree - 1))
-
-for p in range(degree - 1):
-    X[:, p] = x ** (p + 1)
-
-# Intercept is not included in the design matrix
-skl = LinearRegression(fit_intercept=True).fit(X, y)
-
-# Use centered values for X and y when computing coefficients
-y_offset = np.average(y, axis=0)
-X_offset = np.average(X, axis=0)
-
-beta = fit_beta(X - X_offset, y - y_offset)
-intercept = np.mean(y_offset - X_offset @ beta)
-
-print(f"Manual intercept: {intercept}")
-print(f"Fitted beta (wiothout intercept): {beta}")
-print(f"Sklearn intercept: {skl.intercept_}")
-print(f"Sklearn fitted beta (without intercept): {skl.coef_}")
-ypredictOwn = X @ beta
-ypredictSKL = skl.predict(X)
-print(f"MSE with Manual intercept")
-print(MSE(y,ypredictOwn+intercept))
-print(f"MSE with Sklearn intercept")
-print(MSE(y,ypredictSKL))
-
-plt.plot(x, X @ beta + intercept, "--", label="Fit (manual intercept)")
-plt.plot(x, skl.predict(X), "--", label="Sklearn (fit_intercept=True)")
-plt.grid()
-plt.legend()
-
-plt.show()
-
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    - -

    The intercept is the value of our output/target variable -when all our features are zero and our function crosses the \( y \)-axis (for a one-dimensional case). +

    with eigenvalues \( \sigma_1=2 \) and \( \sigma_2=0 \). +The SVD exits always!

    -

    Printing the MSE, we see first that both methods give the same MSE, as -they should. However, when we move to for example Ridge regression (discussed next week), -the way we treat the intercept may give a larger or smaller MSE, -meaning that the MSE can be penalized by the value of the -intercept. Not including the intercept in the fit, means that the -regularization term does not include \( \beta_0 \). For different values -of \( \lambda \), this may lead to differing MSE values. +

    The SVD +decomposition (singular values) gives eigenvalues +\( \sigma_i\geq\sigma_{i+1} \) for all \( i \) and for dimensions larger than \( i=p \), the +eigenvalues (singular values) are zero.

    -

    To remind the reader, the regularization term, with the intercept in Ridge regression is given by

    -$$ -\lambda \vert\vert \boldsymbol{\beta} \vert\vert_2^2 = \lambda \sum_{j=0}^{p-1}\beta_j^2, -$$ - -

    but when we take out the intercept, this equation becomes

    -$$ -\lambda \vert\vert \boldsymbol{\beta} \vert\vert_2^2 = \lambda \sum_{j=1}^{p-1}\beta_j^2. -$$ - -

    For Lasso regression we have

    -$$ -\lambda \vert\vert \boldsymbol{\beta} \vert\vert_1 = \lambda \sum_{j=1}^{p-1}\vert\beta_j\vert. -$$ - -

    It means that, when scaling the design matrix and the outputs/targets, -by subtracting the mean values, we have an optimization problem which -is not penalized by the intercept. The MSE value can then be smaller -since it focuses only on the remaining quantities. If we however bring -back the intercept, we will get an MSE which then contains the -intercept. This becomes more important when we discuss Ridge and Lasso -regression next week. +

    In the general case, where our design matrix \( \boldsymbol{X} \) has dimension +\( n\times p \), the matrix is thus decomposed into an \( n\times n \) +orthogonal matrix \( \boldsymbol{U} \), a \( p\times p \) orthogonal matrix \( \boldsymbol{V} \) +and a diagonal matrix \( \boldsymbol{\Sigma} \) with \( r=\mathrm{min}(n,p) \) +singular values \( \sigma_i\geq 0 \) on the main diagonal and zeros filling +the rest of the matrix. There are at most \( p \) singular values +assuming that \( n > p \). In our regression examples for the nuclear +masses and the equation of state this is indeed the case, while for +the Ising model we have \( p > n \). These are often cases that lead to +near singular or singular matrices.

    +

    The columns of \( \boldsymbol{U} \) are called the left singular vectors while the columns of \( \boldsymbol{V} \) are the right singular vectors.

    +

      @@ -536,7 +393,7 @@ regression next week.
    • 47
    • 48
    • ...
      • -
    • 72
      • +
    • 63
    • »
    diff --git a/doc/pub/week35/html/._week35-bs039.html b/doc/pub/week35/html/._week35-bs039.html index 82e4f122a..a285e77ef 100644 --- a/doc/pub/week35/html/._week35-bs039.html +++ b/doc/pub/week35/html/._week35-bs039.html @@ -38,11 +38,7 @@ doconce format html week35.do.txt --html_style=bootstrap --pygments_html_style=d {'highest level': 2, 'sections': [('Plans for week 35', 2, None, 'plans-for-week-35'), ('Reading recommendations:', 3, None, 'reading-recommendations'), - ('For exercise sessions: Why Linear Regression (aka Ordinary ' - 'Least Squares and family), repeat from last week', - 2, - None, - 'for-exercise-sessions-why-linear-regression-aka-ordinary-least-squares-and-family-repeat-from-last-week'), + ('Reminder from last week', 2, None, 'reminder-from-last-week'), ('The equations for ordinary least squares', 2, None, @@ -122,32 +118,6 @@ doconce format html week35.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'testing-the-means-squared-error-as-function-of-complexity'), - ('More preprocessing examples, two-dimensional example, the ' - 'Franke function', - 2, - None, - 'more-preprocessing-examples-two-dimensional-example-the-franke-function'), - ('To think about, first part', - 2, - None, - 'to-think-about-first-part'), - ('More thinking', 2, None, 'more-thinking'), - ('Still thinking', 2, None, 'still-thinking'), - ('What does centering (subtracting the mean values) mean ' - 'mathematically?', - 2, - None, - 'what-does-centering-subtracting-the-mean-values-mean-mathematically'), - ('Further Manipulations', 2, None, 'further-manipulations'), - ('Wrapping it up', 2, None, 'wrapping-it-up'), - ('Linear Regression code, Intercept handling first', - 2, - None, - 'linear-regression-code-intercept-handling-first'), - ('Material for lecture Monday, August 26', - 2, - None, - 'material-for-lecture-monday-august-26'), ('Mathematical Interpretation of Ordinary Least Squares', 2, None, @@ -279,7 +249,7 @@ MathJax.Hub.Config({ @@ -360,7 +321,27 @@ MathJax.Hub.Config({

     

     

     

    -

    Material for lecture Monday, August 26

    +

    Economy-size SVD

    + +

    If we assume that \( n > p \), then our matrix \( \boldsymbol{U} \) has dimension \( n +\times n \). The last \( n-p \) columns of \( \boldsymbol{U} \) become however +irrelevant in our calculations since they are multiplied with the +zeros in \( \boldsymbol{\Sigma} \). +

    + +

    The economy-size decomposition removes extra rows or columns of zeros +from the diagonal matrix of singular values, \( \boldsymbol{\Sigma} \), along with the columns +in either \( \boldsymbol{U} \) or \( \boldsymbol{V} \) that multiply those zeros in the expression. +Removing these zeros and columns can improve execution time +and reduce storage requirements without compromising the accuracy of +the decomposition. +

    + +

    If \( n > p \), we keep only the first \( p \) columns of \( \boldsymbol{U} \) and \( \boldsymbol{\Sigma} \) has dimension \( p\times p \). +If \( p > n \), then only the first \( n \) columns of \( \boldsymbol{V} \) are computed and \( \boldsymbol{\Sigma} \) has dimension \( n\times n \). +The \( n=p \) case is obvious, we retain the full SVD. +In general the economy-size SVD leads to less FLOPS and still conserving the desired accuracy. +

    @@ -387,7 +368,7 @@ MathJax.Hub.Config({

  • 48
  • 49
  • ...
    • -
  • 72
    • +
  • 63
  • »
    • diff --git a/doc/pub/week35/html/._week35-bs040.html b/doc/pub/week35/html/._week35-bs040.html index 8619d07cd..fa59903d9 100644 --- a/doc/pub/week35/html/._week35-bs040.html +++ b/doc/pub/week35/html/._week35-bs040.html @@ -38,11 +38,7 @@ doconce format html week35.do.txt --html_style=bootstrap --pygments_html_style=d {'highest level': 2, 'sections': [('Plans for week 35', 2, None, 'plans-for-week-35'), ('Reading recommendations:', 3, None, 'reading-recommendations'), - ('For exercise sessions: Why Linear Regression (aka Ordinary ' - 'Least Squares and family), repeat from last week', - 2, - None, - 'for-exercise-sessions-why-linear-regression-aka-ordinary-least-squares-and-family-repeat-from-last-week'), + ('Reminder from last week', 2, None, 'reminder-from-last-week'), ('The equations for ordinary least squares', 2, None, @@ -122,32 +118,6 @@ doconce format html week35.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'testing-the-means-squared-error-as-function-of-complexity'), - ('More preprocessing examples, two-dimensional example, the ' - 'Franke function', - 2, - None, - 'more-preprocessing-examples-two-dimensional-example-the-franke-function'), - ('To think about, first part', - 2, - None, - 'to-think-about-first-part'), - ('More thinking', 2, None, 'more-thinking'), - ('Still thinking', 2, None, 'still-thinking'), - ('What does centering (subtracting the mean values) mean ' - 'mathematically?', - 2, - None, - 'what-does-centering-subtracting-the-mean-values-mean-mathematically'), - ('Further Manipulations', 2, None, 'further-manipulations'), - ('Wrapping it up', 2, None, 'wrapping-it-up'), - ('Linear Regression code, Intercept handling first', - 2, - None, - 'linear-regression-code-intercept-handling-first'), - ('Material for lecture Monday, August 26', - 2, - None, - 'material-for-lecture-monday-august-26'), ('Mathematical Interpretation of Ordinary Least Squares', 2, None, @@ -279,7 +249,7 @@ MathJax.Hub.Config({ @@ -360,36 +321,67 @@ MathJax.Hub.Config({

     

     

     

    -

    Mathematical Interpretation of Ordinary Least Squares

    +

    Codes for the SVD

    -

    What is presented here is a mathematical analysis of various regression algorithms (ordinary least squares, Ridge and Lasso Regression). The analysis is based on an important algorithm in linear algebra, the so-called Singular Value Decomposition (SVD).

    -

    We have shown that in ordinary least squares the optimal parameters \( \beta \) are given by

    + +
    +
    +
    +
    +
    + 
    import numpy as np
+# SVD inversion
+def SVD(A):
+    ''' Takes as input a numpy matrix A and returns inv(A) based on singular value decomposition (SVD).
+    SVD is numerically more stable than the inversion algorithms provided by
+    numpy and scipy.linalg at the cost of being slower.
+    '''
+    U, S, VT = np.linalg.svd(A,full_matrices=True)
+    print('test U')
+    print( (np.transpose(U) @ U - U @np.transpose(U)))
+    print('test VT')
+    print( (np.transpose(VT) @ VT - VT @np.transpose(VT)))
+    print(U)
+    print(S)
+    print(VT)
 
-$$
-\hat{\boldsymbol{\beta}} = \left(\boldsymbol{X}^T\boldsymbol{X}\right)^{-1}\boldsymbol{X}^T\boldsymbol{y}.
-$$
+    D = np.zeros((len(U),len(VT)))
+    for i in range(0,len(VT)):
+        D[i,i]=S[i]
+    return U @ D @ VT
 
-
    The hat over \( \boldsymbol{\beta} \) means we have the optimal parameters after minimization of the cost function.
    
 
-
    This means that our best model is defined as
    
+X = np.array([ [1.0,-1.0], [1.0,-1.0]])
+#X = np.array([[1, 2], [3, 4], [5, 6]])
 
-$$
-\tilde{\boldsymbol{y}}=\boldsymbol{X}\hat{\boldsymbol{\beta}} = \boldsymbol{X}\left(\boldsymbol{X}^T\boldsymbol{X}\right)^{-1}\boldsymbol{X}^T\boldsymbol{y}.
-$$
+print(X)
+C = SVD(X)
+# Print the difference between the original matrix and the SVD one
+print(C-X)
+
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    -

    We now define a matrix

    -$$ -\boldsymbol{A}=\boldsymbol{X}\left(\boldsymbol{X}^T\boldsymbol{X}\right)^{-1}\boldsymbol{X}^T. -$$ - -

    We can rewrite

    -$$ -\tilde{\boldsymbol{y}}=\boldsymbol{X}\hat{\boldsymbol{\beta}} = \boldsymbol{A}\boldsymbol{y}. -$$ - -

    The matrix \( \boldsymbol{A} \) has the important property that \( \boldsymbol{A}^2=\boldsymbol{A} \). This is the definition of a projection matrix. -We can then interpret our optimal model \( \tilde{\boldsymbol{y}} \) as being represented by an orthogonal projection of \( \boldsymbol{y} \) onto a space defined by the column vectors of \( \boldsymbol{X} \). In our case here the matrix \( \boldsymbol{A} \) is a square matrix. If it is a general rectangular matrix we have an oblique projection matrix. +

    The matrix \( \boldsymbol{X} \) has columns that are linearly dependent. The first +column is the row-wise sum of the other two columns. The rank of a +matrix (the column rank) is the dimension of space spanned by the +column vectors. The rank of the matrix is the number of linearly +independent columns, in this case just \( 2 \). We see this from the +singular values when running the above code. Running the standard +inversion algorithm for matrix inversion with \( \boldsymbol{X}^T\boldsymbol{X} \) results +in the program terminating due to a singular matrix.

    @@ -417,7 +409,7 @@ We can then interpret our optimal model \( \tilde{\boldsymbol{y}} \) as being re

  • 49
  • 50
  • ...
    • -
  • 72
    • +
  • 63
  • »
    • diff --git a/doc/pub/week35/html/._week35-bs041.html b/doc/pub/week35/html/._week35-bs041.html index bb6dcbe54..61b0493f6 100644 --- a/doc/pub/week35/html/._week35-bs041.html +++ b/doc/pub/week35/html/._week35-bs041.html @@ -38,11 +38,7 @@ doconce format html week35.do.txt --html_style=bootstrap --pygments_html_style=d {'highest level': 2, 'sections': [('Plans for week 35', 2, None, 'plans-for-week-35'), ('Reading recommendations:', 3, None, 'reading-recommendations'), - ('For exercise sessions: Why Linear Regression (aka Ordinary ' - 'Least Squares and family), repeat from last week', - 2, - None, - 'for-exercise-sessions-why-linear-regression-aka-ordinary-least-squares-and-family-repeat-from-last-week'), + ('Reminder from last week', 2, None, 'reminder-from-last-week'), ('The equations for ordinary least squares', 2, None, @@ -122,32 +118,6 @@ doconce format html week35.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'testing-the-means-squared-error-as-function-of-complexity'), - ('More preprocessing examples, two-dimensional example, the ' - 'Franke function', - 2, - None, - 'more-preprocessing-examples-two-dimensional-example-the-franke-function'), - ('To think about, first part', - 2, - None, - 'to-think-about-first-part'), - ('More thinking', 2, None, 'more-thinking'), - ('Still thinking', 2, None, 'still-thinking'), - ('What does centering (subtracting the mean values) mean ' - 'mathematically?', - 2, - None, - 'what-does-centering-subtracting-the-mean-values-mean-mathematically'), - ('Further Manipulations', 2, None, 'further-manipulations'), - ('Wrapping it up', 2, None, 'wrapping-it-up'), - ('Linear Regression code, Intercept handling first', - 2, - None, - 'linear-regression-code-intercept-handling-first'), - ('Material for lecture Monday, August 26', - 2, - None, - 'material-for-lecture-monday-august-26'), ('Mathematical Interpretation of Ordinary Least Squares', 2, None, @@ -279,7 +249,7 @@ MathJax.Hub.Config({ @@ -360,14 +321,23 @@ MathJax.Hub.Config({

     

     

     

    -

    Residual Error

    +

    Note about SVD Calculations

    -

    We have defined the residual error as

    -$$ -\boldsymbol{\epsilon}=\boldsymbol{y}-\tilde{\boldsymbol{y}}=\left[\boldsymbol{I}-\boldsymbol{X}\left(\boldsymbol{X}^T\boldsymbol{X}\right)^{-1}\boldsymbol{X}^T\right]\boldsymbol{y}. -$$ +

    The \( U \), \( S \), and \( V \) matrices returned from the svd() function +cannot be multiplied directly. +

    -

    The residual errors are then the projections of \( \boldsymbol{y} \) onto the orthogonal component of the space defined by the column vectors of \( \boldsymbol{X} \).

    +

    As you can see from the code, the \( S \) vector must be converted into a +diagonal matrix. This may cause a problem as the size of the matrices +do not fit the rules of matrix multiplication, where the number of +columns in a matrix must match the number of rows in the subsequent +matrix. +

    + +

    If you wish to include the zero singular values, you will need to +resize the matrices and set up a diagonal matrix as done in the above +example +

    @@ -394,7 +364,7 @@ $$

  • 50
  • 51
  • ...
    • -
  • 72
    • +
  • 63
  • »
    • diff --git a/doc/pub/week35/html/._week35-bs042.html b/doc/pub/week35/html/._week35-bs042.html index 33ad58458..7b7f72f0b 100644 --- a/doc/pub/week35/html/._week35-bs042.html +++ b/doc/pub/week35/html/._week35-bs042.html @@ -38,11 +38,7 @@ doconce format html week35.do.txt --html_style=bootstrap --pygments_html_style=d {'highest level': 2, 'sections': [('Plans for week 35', 2, None, 'plans-for-week-35'), ('Reading recommendations:', 3, None, 'reading-recommendations'), - ('For exercise sessions: Why Linear Regression (aka Ordinary ' - 'Least Squares and family), repeat from last week', - 2, - None, - 'for-exercise-sessions-why-linear-regression-aka-ordinary-least-squares-and-family-repeat-from-last-week'), + ('Reminder from last week', 2, None, 'reminder-from-last-week'), ('The equations for ordinary least squares', 2, None, @@ -122,32 +118,6 @@ doconce format html week35.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'testing-the-means-squared-error-as-function-of-complexity'), - ('More preprocessing examples, two-dimensional example, the ' - 'Franke function', - 2, - None, - 'more-preprocessing-examples-two-dimensional-example-the-franke-function'), - ('To think about, first part', - 2, - None, - 'to-think-about-first-part'), - ('More thinking', 2, None, 'more-thinking'), - ('Still thinking', 2, None, 'still-thinking'), - ('What does centering (subtracting the mean values) mean ' - 'mathematically?', - 2, - None, - 'what-does-centering-subtracting-the-mean-values-mean-mathematically'), - ('Further Manipulations', 2, None, 'further-manipulations'), - ('Wrapping it up', 2, None, 'wrapping-it-up'), - ('Linear Regression code, Intercept handling first', - 2, - None, - 'linear-regression-code-intercept-handling-first'), - ('Material for lecture Monday, August 26', - 2, - None, - 'material-for-lecture-monday-august-26'), ('Mathematical Interpretation of Ordinary Least Squares', 2, None, @@ -279,7 +249,7 @@ MathJax.Hub.Config({ @@ -360,25 +321,38 @@ MathJax.Hub.Config({

     

     

     

    -

    Simple case

    +

    Mathematics of the SVD and implications

    -

    If the matrix \( \boldsymbol{X} \) is an orthogonal (or unitary in case of complex values) matrix, we have

    +

    Let us take a closer look at the mathematics of the SVD and the various implications for machine learning studies.

    +

    Our starting point is our design matrix \( \boldsymbol{X} \) of dimension \( n\times p \)

    $$ -\boldsymbol{X}^T\boldsymbol{X}=\boldsymbol{X}\boldsymbol{X}^T = \boldsymbol{I}. +\boldsymbol{X}=\begin{bmatrix} +x_{0,0} & x_{0,1} & x_{0,2}& \dots & \dots x_{0,p-1}\\ +x_{1,0} & x_{1,1} & x_{1,2}& \dots & \dots x_{1,p-1}\\ +x_{2,0} & x_{2,1} & x_{2,2}& \dots & \dots x_{2,p-1}\\ +\dots & \dots & \dots & \dots \dots & \dots \\ +x_{n-2,0} & x_{n-2,1} & x_{n-2,2}& \dots & \dots x_{n-2,p-1}\\ +x_{n-1,0} & x_{n-1,1} & x_{n-1,2}& \dots & \dots x_{n-1,p-1}\\ +\end{bmatrix}. $$ -

    In this case the matrix \( \boldsymbol{A} \) becomes

    +

    We can SVD decompose our matrix as

    $$ -\boldsymbol{A}=\boldsymbol{X}\left(\boldsymbol{X}^T\boldsymbol{X}\right)^{-1}\boldsymbol{X}^T)=\boldsymbol{I}, +\boldsymbol{X}=\boldsymbol{U}\boldsymbol{\Sigma}\boldsymbol{V}^T, $$ -

    and we have the obvious case

    +

    where \( \boldsymbol{U} \) is an orthogonal matrix of dimension \( n\times n \), meaning that \( \boldsymbol{U}\boldsymbol{U}^T=\boldsymbol{U}^T\boldsymbol{U}=\boldsymbol{I}_n \). Here \( \boldsymbol{I}_n \) is the unit matrix of dimension \( n \times n \).

    + +

    Similarly, \( \boldsymbol{V} \) is an orthogonal matrix of dimension \( p\times p \), meaning that \( \boldsymbol{V}\boldsymbol{V}^T=\boldsymbol{V}^T\boldsymbol{V}=\boldsymbol{I}_p \). Here \( \boldsymbol{I}_n \) is the unit matrix of dimension \( p \times p \).

    + +

    Finally \( \boldsymbol{\Sigma} \) contains the singular values \( \sigma_i \). This matrix has dimension \( n\times p \) and the singular values \( \sigma_i \) are all positive. The non-zero values are ordered in descending order, that is

    + $$ -\boldsymbol{\epsilon}=\boldsymbol{y}-\tilde{\boldsymbol{y}}=0. +\sigma_0 > \sigma_1 > \sigma_2 > \dots > \sigma_{p-1} > 0. $$ -

    This serves also as a useful test of our codes.

    +

    All values beyond \( p-1 \) are all zero.

    @@ -405,7 +379,7 @@ $$

  • 51
  • 52
  • ...
    • -
  • 72
    • +
  • 63
  • »
    • diff --git a/doc/pub/week35/html/._week35-bs043.html b/doc/pub/week35/html/._week35-bs043.html index 9e163c18e..f38e34b94 100644 --- a/doc/pub/week35/html/._week35-bs043.html +++ b/doc/pub/week35/html/._week35-bs043.html @@ -38,11 +38,7 @@ doconce format html week35.do.txt --html_style=bootstrap --pygments_html_style=d {'highest level': 2, 'sections': [('Plans for week 35', 2, None, 'plans-for-week-35'), ('Reading recommendations:', 3, None, 'reading-recommendations'), - ('For exercise sessions: Why Linear Regression (aka Ordinary ' - 'Least Squares and family), repeat from last week', - 2, - None, - 'for-exercise-sessions-why-linear-regression-aka-ordinary-least-squares-and-family-repeat-from-last-week'), + ('Reminder from last week', 2, None, 'reminder-from-last-week'), ('The equations for ordinary least squares', 2, None, @@ -122,32 +118,6 @@ doconce format html week35.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'testing-the-means-squared-error-as-function-of-complexity'), - ('More preprocessing examples, two-dimensional example, the ' - 'Franke function', - 2, - None, - 'more-preprocessing-examples-two-dimensional-example-the-franke-function'), - ('To think about, first part', - 2, - None, - 'to-think-about-first-part'), - ('More thinking', 2, None, 'more-thinking'), - ('Still thinking', 2, None, 'still-thinking'), - ('What does centering (subtracting the mean values) mean ' - 'mathematically?', - 2, - None, - 'what-does-centering-subtracting-the-mean-values-mean-mathematically'), - ('Further Manipulations', 2, None, 'further-manipulations'), - ('Wrapping it up', 2, None, 'wrapping-it-up'), - ('Linear Regression code, Intercept handling first', - 2, - None, - 'linear-regression-code-intercept-handling-first'), - ('Material for lecture Monday, August 26', - 2, - None, - 'material-for-lecture-monday-august-26'), ('Mathematical Interpretation of Ordinary Least Squares', 2, None, @@ -279,7 +249,7 @@ MathJax.Hub.Config({ @@ -360,50 +321,63 @@ MathJax.Hub.Config({

     

     

     

    -

    The singular value decomposition

    +

    Example Matrix

    -
    -
    - +

    As an example, consider the following \( 3\times 2 \) example for the matrix \( \boldsymbol{\Sigma} \)

    -

    The examples we have looked at so far are cases where we normally can -invert the matrix \( \boldsymbol{X}^T\boldsymbol{X} \). Using a polynomial expansion where we fit of various functions leads to -row vectors of the design matrix which are essentially orthogonal due -to the polynomial character of our model. Obtaining the inverse of the -design matrix is then often done via a so-called LU, QR or Cholesky -decomposition. +$$ +\boldsymbol{\Sigma}= +\begin{bmatrix} +2& 0 \\ +0 & 1 \\ +0 & 0 \\ +\end{bmatrix} +$$ + +

    The singular values are \( \sigma_0=2 \) and \( \sigma_1=1 \). It is common to rewrite the matrix \( \boldsymbol{\Sigma} \) as

    + +$$ +\boldsymbol{\Sigma}= +\begin{bmatrix} +\boldsymbol{\tilde{\Sigma}}\\ +\boldsymbol{0}\\ +\end{bmatrix}, +$$ + +

    where

    +$$ +\boldsymbol{\tilde{\Sigma}}= +\begin{bmatrix} +2& 0 \\ +0 & 1 \\ +\end{bmatrix}, +$$ + +

    contains only the singular values. Note also (and we will use this below) that

    + +$$ +\boldsymbol{\Sigma}^T\boldsymbol{\Sigma}= +\begin{bmatrix} +4& 0 \\ +0 & 1 \\ +\end{bmatrix}, +$$ + +

    which is a \( 2\times 2 \) matrix while

    +$$ +\boldsymbol{\Sigma}\boldsymbol{\Sigma}^T= +\begin{bmatrix} +4& 0 & 0\\ +0 & 1 & 0\\ +0 & 0 & 0\\ +\end{bmatrix}, +$$ + +

    is a \( 3\times 3 \) matrix. The last row and column of this last matrix +contain only zeros. This will have important consequences for our SVD +decomposition of the design matrix.

    -

    As we will also see in the first project, -this may -however not the be case in general and a standard matrix inversion -algorithm based on say LU, QR or Cholesky decomposition may lead to singularities. We will see examples of this below. -

    - -

    There is however a way to circumvent this problem and also -gain some insights about the ordinary least squares approach, and -later shrinkage methods like Ridge and Lasso regressions. -

    - -

    This is given by the Singular Value Decomposition (SVD) algorithm, -perhaps the most powerful linear algebra algorithm. The SVD provides -a numerically stable matrix decomposition that is used in a large -swath oc applications and the decomposition is always stable -numerically. -

    - -

    In machine learning it plays a central role in dealing with for -example design matrices that may be near singular or singular. -Furthermore, as we will see here, the singular values can be related -to the covariance matrix (and thereby the correlation matrix) and in -turn the variance of a given quantity. It plays also an important role -in the principal component analysis where high-dimensional data can be -reduced to the statistically relevant features. -

    -
    -
    - -

      @@ -429,7 +403,7 @@ reduced to the statistically relevant features.
    • 52
    • 53
    • ...
      • -
    • 72
      • +
    • 63
    • »
    diff --git a/doc/pub/week35/html/._week35-bs044.html b/doc/pub/week35/html/._week35-bs044.html index 01e064f50..d658f6d3f 100644 --- a/doc/pub/week35/html/._week35-bs044.html +++ b/doc/pub/week35/html/._week35-bs044.html @@ -38,11 +38,7 @@ doconce format html week35.do.txt --html_style=bootstrap --pygments_html_style=d {'highest level': 2, 'sections': [('Plans for week 35', 2, None, 'plans-for-week-35'), ('Reading recommendations:', 3, None, 'reading-recommendations'), - ('For exercise sessions: Why Linear Regression (aka Ordinary ' - 'Least Squares and family), repeat from last week', - 2, - None, - 'for-exercise-sessions-why-linear-regression-aka-ordinary-least-squares-and-family-repeat-from-last-week'), + ('Reminder from last week', 2, None, 'reminder-from-last-week'), ('The equations for ordinary least squares', 2, None, @@ -122,32 +118,6 @@ doconce format html week35.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'testing-the-means-squared-error-as-function-of-complexity'), - ('More preprocessing examples, two-dimensional example, the ' - 'Franke function', - 2, - None, - 'more-preprocessing-examples-two-dimensional-example-the-franke-function'), - ('To think about, first part', - 2, - None, - 'to-think-about-first-part'), - ('More thinking', 2, None, 'more-thinking'), - ('Still thinking', 2, None, 'still-thinking'), - ('What does centering (subtracting the mean values) mean ' - 'mathematically?', - 2, - None, - 'what-does-centering-subtracting-the-mean-values-mean-mathematically'), - ('Further Manipulations', 2, None, 'further-manipulations'), - ('Wrapping it up', 2, None, 'wrapping-it-up'), - ('Linear Regression code, Intercept handling first', - 2, - None, - 'linear-regression-code-intercept-handling-first'), - ('Material for lecture Monday, August 26', - 2, - None, - 'material-for-lecture-monday-august-26'), ('Mathematical Interpretation of Ordinary Least Squares', 2, None, @@ -279,7 +249,7 @@ MathJax.Hub.Config({ @@ -360,53 +321,47 @@ MathJax.Hub.Config({

     

     

     

    -

    Linear Regression Problems

    +

    Setting up the Matrix to be inverted

    + +

    The matrix that may cause problems for us is \( \boldsymbol{X}^T\boldsymbol{X} \). Using the SVD we can rewrite this matrix as

    -

    One of the typical problems we encounter with linear regression, in particular -when the matrix \( \boldsymbol{X} \) (our so-called design matrix) is high-dimensional, -are problems with near singular or singular matrices. The column vectors of \( \boldsymbol{X} \) -may be linearly dependent, normally referred to as super-collinearity. -This means that the matrix may be rank deficient and it is basically impossible to -to model the data using linear regression. As an example, consider the matrix -

    $$ -\begin{align*} -\mathbf{X} & = \left[ -\begin{array}{rrr} -1 & -1 & 2 -\\ -1 & 0 & 1 -\\ -1 & 2 & -1 -\\ -1 & 1 & 0 -\end{array} \right] -\end{align*} +\boldsymbol{X}^T\boldsymbol{X}=\boldsymbol{V}\boldsymbol{\Sigma}^T\boldsymbol{U}^T\boldsymbol{U}\boldsymbol{\Sigma}\boldsymbol{V}^T, $$ -

    The columns of \( \boldsymbol{X} \) are linearly dependent. We see this easily since the -the first column is the row-wise sum of the other two columns. The rank (more correct, -the column rank) of a matrix is the dimension of the space spanned by the -column vectors. Hence, the rank of \( \mathbf{X} \) is equal to the number -of linearly independent columns. In this particular case the matrix has rank 2. -

    +

    and using the orthogonality of the matrix \( \boldsymbol{U} \) we have

    -

    Super-collinearity of an \( (n \times p) \)-dimensional design matrix \( \mathbf{X} \) implies -that the inverse of the matrix \( \boldsymbol{X}^T\boldsymbol{X} \) (the matrix we need to invert to solve the linear regression equations) is non-invertible. If we have a square matrix that does not have an inverse, we say this matrix singular. The example here demonstrates this -

    $$ -\begin{align*} -\boldsymbol{X} & = \left[ -\begin{array}{rr} -1 & -1 -\\ -1 & -1 -\end{array} \right]. -\end{align*} +\boldsymbol{X}^T\boldsymbol{X}=\boldsymbol{V}\boldsymbol{\Sigma}^T\boldsymbol{\Sigma}\boldsymbol{V}^T. $$ -

    We see easily that \( \mbox{det}(\boldsymbol{X}) = x_{11} x_{22} - x_{12} x_{21} = 1 \times (-1) - 1 \times (-1) = 0 \). Hence, \( \mathbf{X} \) is singular and its inverse is undefined. -This is equivalent to saying that the matrix \( \boldsymbol{X} \) has at least an eigenvalue which is zero. +

    We define \( \boldsymbol{\Sigma}^T\boldsymbol{\Sigma}=\tilde{\boldsymbol{\Sigma}}^2 \) which is a diagonal matrix containing only the singular values squared. It has dimensionality \( p \times p \).

    + +

    We can now insert the result for the matrix \( \boldsymbol{X}^T\boldsymbol{X} \) into our equation for ordinary least squares where

    + +$$ +\tilde{y}_{\mathrm{OLS}}=\boldsymbol{X}\left(\boldsymbol{X}^T\boldsymbol{X}\right)^{-1}\boldsymbol{X}^T\boldsymbol{y}, +$$ + +

    and using our SVD decomposition of \( \boldsymbol{X} \) we have

    + +$$ +\tilde{y}_{\mathrm{OLS}}=\boldsymbol{U}\boldsymbol{\Sigma}\boldsymbol{V}^T\left(\boldsymbol{V}\tilde{\boldsymbol{\Sigma}}^{2}(\boldsymbol{V}^T\right)^{-1}\boldsymbol{V}\boldsymbol{\Sigma}^T\boldsymbol{U}^T\boldsymbol{y}, +$$ + +

    which gives us, using the orthogonality of the matrix \( \boldsymbol{V} \),

    + +$$ +\tilde{y}_{\mathrm{OLS}}=\boldsymbol{U}\boldsymbol{U}^T\boldsymbol{y}=\sum_{i=0}^{p-1}\boldsymbol{u}_i\boldsymbol{u}^T_i\boldsymbol{y}, +$$ + +

    It means that the ordinary least square model (with the optimal +parameters) \( \boldsymbol{\tilde{y}} \), corresponds to an orthogonal +transformation of the output (or target) vector \( \boldsymbol{y} \) by the +vectors of the matrix \( \boldsymbol{U} \). Note that the summation ends at +\( p-1 \), that is \( \boldsymbol{\tilde{y}}\ne \boldsymbol{y} \). We can thus not use the +orthogonality relation for the matrix \( \boldsymbol{U} \). This can already be +when we multiply the matrices \( \boldsymbol{\Sigma}^T\boldsymbol{U}^T \).

    @@ -434,7 +389,7 @@ This is equivalent to saying that the matrix \( \boldsymbol{X} \) has at least a

  • 53
  • 54
  • ...
    • -
  • 72
    • +
  • 63
  • »
    • diff --git a/doc/pub/week35/html/._week35-bs045.html b/doc/pub/week35/html/._week35-bs045.html index c9d29b265..b3d55ab2d 100644 --- a/doc/pub/week35/html/._week35-bs045.html +++ b/doc/pub/week35/html/._week35-bs045.html @@ -38,11 +38,7 @@ doconce format html week35.do.txt --html_style=bootstrap --pygments_html_style=d {'highest level': 2, 'sections': [('Plans for week 35', 2, None, 'plans-for-week-35'), ('Reading recommendations:', 3, None, 'reading-recommendations'), - ('For exercise sessions: Why Linear Regression (aka Ordinary ' - 'Least Squares and family), repeat from last week', - 2, - None, - 'for-exercise-sessions-why-linear-regression-aka-ordinary-least-squares-and-family-repeat-from-last-week'), + ('Reminder from last week', 2, None, 'reminder-from-last-week'), ('The equations for ordinary least squares', 2, None, @@ -122,32 +118,6 @@ doconce format html week35.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'testing-the-means-squared-error-as-function-of-complexity'), - ('More preprocessing examples, two-dimensional example, the ' - 'Franke function', - 2, - None, - 'more-preprocessing-examples-two-dimensional-example-the-franke-function'), - ('To think about, first part', - 2, - None, - 'to-think-about-first-part'), - ('More thinking', 2, None, 'more-thinking'), - ('Still thinking', 2, None, 'still-thinking'), - ('What does centering (subtracting the mean values) mean ' - 'mathematically?', - 2, - None, - 'what-does-centering-subtracting-the-mean-values-mean-mathematically'), - ('Further Manipulations', 2, None, 'further-manipulations'), - ('Wrapping it up', 2, None, 'wrapping-it-up'), - ('Linear Regression code, Intercept handling first', - 2, - None, - 'linear-regression-code-intercept-handling-first'), - ('Material for lecture Monday, August 26', - 2, - None, - 'material-for-lecture-monday-august-26'), ('Mathematical Interpretation of Ordinary Least Squares', 2, None, @@ -279,7 +249,7 @@ MathJax.Hub.Config({ @@ -360,29 +321,53 @@ MathJax.Hub.Config({

     

     

     

    -

    Fixing the singularity

    +

    Further properties (important for our analyses later)

    -

    If our design matrix \( \boldsymbol{X} \) which enters the linear regression problem

    +

    Let us study again \( \boldsymbol{X}^T\boldsymbol{X} \) in terms of our SVD,

    $$ -\begin{align} -\boldsymbol{\beta} & = (\boldsymbol{X}^{T} \boldsymbol{X})^{-1} \boldsymbol{X}^{T} \boldsymbol{y}, -\tag{1} -\end{align} +\boldsymbol{X}^T\boldsymbol{X}=\boldsymbol{V}\boldsymbol{\Sigma}^T\boldsymbol{U}^T\boldsymbol{U}\boldsymbol{\Sigma}\boldsymbol{V}^T=\boldsymbol{V}\boldsymbol{\Sigma}^T\boldsymbol{\Sigma}\boldsymbol{V}^T. $$ -

    has linearly dependent column vectors, we will not be able to compute the inverse -of \( \boldsymbol{X}^T\boldsymbol{X} \) and we cannot find the parameters (estimators) \( \beta_i \). -The estimators are only well-defined if \( (\boldsymbol{X}^{T}\boldsymbol{X})^{-1} \) exits. -This is more likely to happen when the matrix \( \boldsymbol{X} \) is high-dimensional. In this case it is likely to encounter a situation where -the regression parameters \( \beta_i \) cannot be estimated. +

    If we now multiply from the right with \( \boldsymbol{V} \) (using the orthogonality of \( \boldsymbol{V} \)) we get

    +$$ +\left(\boldsymbol{X}^T\boldsymbol{X}\right)\boldsymbol{V}=\boldsymbol{V}\boldsymbol{\Sigma}^T\boldsymbol{\Sigma}. +$$ + +

    This means the vectors \( \boldsymbol{v}_i \) of the orthogonal matrix \( \boldsymbol{V} \) are the eigenvectors of the matrix \( \boldsymbol{X}^T\boldsymbol{X} \) +with eigenvalues given by the singular values squared, that is +

    +$$ +\left(\boldsymbol{X}^T\boldsymbol{X}\right)\boldsymbol{v}_i=\boldsymbol{v}_i\sigma_i^2. +$$ + +

    Similarly, if we use the SVD decomposition for the matrix \( \boldsymbol{X}\boldsymbol{X}^T \), we have

    +$$ +\boldsymbol{X}\boldsymbol{X}^T=\boldsymbol{U}\boldsymbol{\Sigma}\boldsymbol{V}^T\boldsymbol{V}\boldsymbol{\Sigma}^T\boldsymbol{U}^T=\boldsymbol{U}\boldsymbol{\Sigma}\boldsymbol{\Sigma}^T\boldsymbol{U}^T. +$$ + +

    If we now multiply from the right with \( \boldsymbol{U} \) (using the orthogonality of \( \boldsymbol{U} \)) we get

    +$$ +\left(\boldsymbol{X}\boldsymbol{X}^T\right)\boldsymbol{U}=\boldsymbol{U}\boldsymbol{\Sigma}\boldsymbol{\Sigma}^T. +$$ + +

    This means the vectors \( \boldsymbol{u}_i \) of the orthogonal matrix \( \boldsymbol{U} \) are the eigenvectors of the matrix \( \boldsymbol{X}\boldsymbol{X}^T \) +with eigenvalues given by the singular values squared, that is +

    +$$ +\left(\boldsymbol{X}\boldsymbol{X}^T\right)\boldsymbol{u}_i=\boldsymbol{u}_i\sigma_i^2. +$$ + +

    Important note: we have defined our design matrix \( \boldsymbol{X} \) to be an +\( n\times p \) matrix. In most supervised learning cases we have that \( n +\ge p \), and quite often we have \( n >> p \). For linear algebra based methods like ordinary least squares or Ridge regression, this leads to a matrix \( \boldsymbol{X}^T\boldsymbol{X} \) which is small and thereby easier to handle from a computational point of view (in terms of number of floating point operations).

    -

    A cheap ad hoc approach is simply to add a small diagonal component to the matrix to invert, that is we change

    -$$ -\boldsymbol{X}^{T} \boldsymbol{X} \rightarrow \boldsymbol{X}^{T} \boldsymbol{X}+\lambda \boldsymbol{I}, -$$ - -

    where \( \boldsymbol{I} \) is the identity matrix. When we discuss Ridge regression this is actually what we end up evaluating. The parameter \( \lambda \) is called a hyperparameter. More about this later.

    +

    In our lectures, the number of columns will +always refer to the number of features in our data set, while the +number of rows represents the number of data inputs. Note that in +other texts you may find the opposite notation. This has consequences +for the definition of for example the covariance matrix and its relation to the SVD. +

    @@ -409,7 +394,7 @@ $$

  • 54
  • 55
  • ...
    • -
  • 72
    • +
  • 63
  • »
    • diff --git a/doc/pub/week35/html/._week35-bs046.html b/doc/pub/week35/html/._week35-bs046.html index b473f8311..eb4ff0c8d 100644 --- a/doc/pub/week35/html/._week35-bs046.html +++ b/doc/pub/week35/html/._week35-bs046.html @@ -38,11 +38,7 @@ doconce format html week35.do.txt --html_style=bootstrap --pygments_html_style=d {'highest level': 2, 'sections': [('Plans for week 35', 2, None, 'plans-for-week-35'), ('Reading recommendations:', 3, None, 'reading-recommendations'), - ('For exercise sessions: Why Linear Regression (aka Ordinary ' - 'Least Squares and family), repeat from last week', - 2, - None, - 'for-exercise-sessions-why-linear-regression-aka-ordinary-least-squares-and-family-repeat-from-last-week'), + ('Reminder from last week', 2, None, 'reminder-from-last-week'), ('The equations for ordinary least squares', 2, None, @@ -122,32 +118,6 @@ doconce format html week35.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'testing-the-means-squared-error-as-function-of-complexity'), - ('More preprocessing examples, two-dimensional example, the ' - 'Franke function', - 2, - None, - 'more-preprocessing-examples-two-dimensional-example-the-franke-function'), - ('To think about, first part', - 2, - None, - 'to-think-about-first-part'), - ('More thinking', 2, None, 'more-thinking'), - ('Still thinking', 2, None, 'still-thinking'), - ('What does centering (subtracting the mean values) mean ' - 'mathematically?', - 2, - None, - 'what-does-centering-subtracting-the-mean-values-mean-mathematically'), - ('Further Manipulations', 2, None, 'further-manipulations'), - ('Wrapping it up', 2, None, 'wrapping-it-up'), - ('Linear Regression code, Intercept handling first', - 2, - None, - 'linear-regression-code-intercept-handling-first'), - ('Material for lecture Monday, August 26', - 2, - None, - 'material-for-lecture-monday-august-26'), ('Mathematical Interpretation of Ordinary Least Squares', 2, None, @@ -279,7 +249,7 @@ MathJax.Hub.Config({ @@ -360,40 +321,31 @@ MathJax.Hub.Config({

     

     

     

    -

    Basic math of the SVD

    +

    Meet the Covariance Matrix

    -

    From standard linear algebra we know that a square matrix \( \boldsymbol{X} \) can be diagonalized if and only it is -a so-called normal matrix, that is if \( \boldsymbol{X}\in {\mathbb{R}}^{n\times n} \) -we have \( \boldsymbol{X}\boldsymbol{X}^T=\boldsymbol{X}^T\boldsymbol{X} \) or if \( \boldsymbol{X}\in {\mathbb{C}}^{n\times n} \) we have \( \boldsymbol{X}\boldsymbol{X}^{\dagger}=\boldsymbol{X}^{\dagger}\boldsymbol{X} \). -The matrix has then a set of eigenpairs +

    Before we move on to a discussion of Ridge and Lasso regression, we want to show an important example of the above.

    + +

    We have already noted that the matrix \( \boldsymbol{X}^T\boldsymbol{X} \) in ordinary +least squares is proportional to the second derivative of the cost +function, that is we have

    $$ -(\lambda_1,\boldsymbol{u}_1),\dots, (\lambda_n,\boldsymbol{u}_n), +\frac{\partial^2 C(\boldsymbol{\beta})}{\partial \boldsymbol{\beta}\partial \boldsymbol{\beta}^T} =\frac{2}{n}\boldsymbol{X}^T\boldsymbol{X}. $$ -

    and the eigenvalues are given by the diagonal matrix

    +

    This quantity defines was what is called the Hessian matrix (the second derivative of a function we want to optimize).

    + +

    The Hessian matrix plays an important role and is defined in this course as

    + $$ -\boldsymbol{\Sigma}=\mathrm{Diag}(\lambda_1, \dots,\lambda_n). +\boldsymbol{H}=\boldsymbol{X}^T\boldsymbol{X}. $$ -

    The matrix \( \boldsymbol{X} \) can be written in terms of an orthogonal/unitary transformation \( \boldsymbol{U} \)

    -$$ -\boldsymbol{X} = \boldsymbol{U}\boldsymbol{\Sigma}\boldsymbol{V}^T, -$$ - -

    with \( \boldsymbol{U}\boldsymbol{U}^T=\boldsymbol{I} \) or \( \boldsymbol{U}\boldsymbol{U}^{\dagger}=\boldsymbol{I} \).

    - -

    Not all square matrices are diagonalizable. A matrix like the one discussed above

    -$$ -\boldsymbol{X} = \begin{bmatrix} -1& -1 \\ -1& -1\\ -\end{bmatrix} -$$ - -

    is not diagonalizable, it is a so-called defective matrix. It is easy to see that the condition -\( \boldsymbol{X}\boldsymbol{X}^T=\boldsymbol{X}^T\boldsymbol{X} \) is not fulfilled. +

    The Hessian matrix for ordinary least squares is also proportional to +the covariance matrix. This means also that we can use the SVD to find +the eigenvalues of the covariance matrix and the Hessian matrix in +terms of the singular values. Let us develop these arguments, as they will play an important role in our machine learning studies.

    @@ -421,7 +373,7 @@ $$

  • 55
  • 56
  • ...
    • -
  • 72
    • +
  • 63
  • »
    • diff --git a/doc/pub/week35/html/._week35-bs047.html b/doc/pub/week35/html/._week35-bs047.html index 61d693ebe..d4db14941 100644 --- a/doc/pub/week35/html/._week35-bs047.html +++ b/doc/pub/week35/html/._week35-bs047.html @@ -38,11 +38,7 @@ doconce format html week35.do.txt --html_style=bootstrap --pygments_html_style=d {'highest level': 2, 'sections': [('Plans for week 35', 2, None, 'plans-for-week-35'), ('Reading recommendations:', 3, None, 'reading-recommendations'), - ('For exercise sessions: Why Linear Regression (aka Ordinary ' - 'Least Squares and family), repeat from last week', - 2, - None, - 'for-exercise-sessions-why-linear-regression-aka-ordinary-least-squares-and-family-repeat-from-last-week'), + ('Reminder from last week', 2, None, 'reminder-from-last-week'), ('The equations for ordinary least squares', 2, None, @@ -122,32 +118,6 @@ doconce format html week35.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'testing-the-means-squared-error-as-function-of-complexity'), - ('More preprocessing examples, two-dimensional example, the ' - 'Franke function', - 2, - None, - 'more-preprocessing-examples-two-dimensional-example-the-franke-function'), - ('To think about, first part', - 2, - None, - 'to-think-about-first-part'), - ('More thinking', 2, None, 'more-thinking'), - ('Still thinking', 2, None, 'still-thinking'), - ('What does centering (subtracting the mean values) mean ' - 'mathematically?', - 2, - None, - 'what-does-centering-subtracting-the-mean-values-mean-mathematically'), - ('Further Manipulations', 2, None, 'further-manipulations'), - ('Wrapping it up', 2, None, 'wrapping-it-up'), - ('Linear Regression code, Intercept handling first', - 2, - None, - 'linear-regression-code-intercept-handling-first'), - ('Material for lecture Monday, August 26', - 2, - None, - 'material-for-lecture-monday-august-26'), ('Mathematical Interpretation of Ordinary Least Squares', 2, None, @@ -279,7 +249,7 @@ MathJax.Hub.Config({ @@ -360,53 +321,48 @@ MathJax.Hub.Config({

     

     

     

    -

    The SVD, a Fantastic Algorithm

    +

    Introducing the Covariance and Correlation functions

    -

    However, and this is the strength of the SVD algorithm, any general -matrix \( \boldsymbol{X} \) can be decomposed in terms of a diagonal matrix and -two orthogonal/unitary matrices. The Singular Value Decompostion -(SVD) theorem -states that a general \( m\times n \) matrix \( \boldsymbol{X} \) can be written in -terms of a diagonal matrix \( \boldsymbol{\Sigma} \) of dimensionality \( m\times n \) -and two orthognal matrices \( \boldsymbol{U} \) and \( \boldsymbol{V} \), where the first has -dimensionality \( m \times m \) and the last dimensionality \( n\times n \). -We have then +

    Before we discuss the link between for example Ridge regression and the singular value decomposition, we need to remind ourselves about +the definition of the covariance and the correlation function. These are quantities that play a central role in machine learning methods.

    -$$ -\boldsymbol{X} = \boldsymbol{U}\boldsymbol{\Sigma}\boldsymbol{V}^T +

    Suppose we have defined two vectors +\( \hat{x} \) and \( \hat{y} \) with \( n \) elements each. The covariance matrix \( \boldsymbol{C} \) is defined as +

    +$$ +\boldsymbol{C}[\boldsymbol{x},\boldsymbol{y}] = \begin{bmatrix} \mathrm{cov}[\boldsymbol{x},\boldsymbol{x}] & \mathrm{cov}[\boldsymbol{x},\boldsymbol{y}] \\ + \mathrm{cov}[\boldsymbol{y},\boldsymbol{x}] & \mathrm{cov}[\boldsymbol{y},\boldsymbol{y}] \\ + \end{bmatrix}, $$ -

    As an example, the above defective matrix can be decomposed as

    - +

    where for example

    $$ -\boldsymbol{X} = \frac{1}{\sqrt{2}}\begin{bmatrix} 1& 1 \\ 1& -1\\ \end{bmatrix} \begin{bmatrix} 2& 0 \\ 0& 0\\ \end{bmatrix} \frac{1}{\sqrt{2}}\begin{bmatrix} 1& -1 \\ 1& 1\\ \end{bmatrix}=\boldsymbol{U}\boldsymbol{\Sigma}\boldsymbol{V}^T, +\mathrm{cov}[\boldsymbol{x},\boldsymbol{y}] =\frac{1}{n} \sum_{i=0}^{n-1}(x_i- \overline{x})(y_i- \overline{y}). $$ -

    with eigenvalues \( \sigma_1=2 \) and \( \sigma_2=0 \). -The SVD exits always! -

    +

    With this definition and recalling that the variance is defined as

    +$$ +\mathrm{var}[\boldsymbol{x}]=\frac{1}{n} \sum_{i=0}^{n-1}(x_i- \overline{x})^2, +$$ -

    The SVD -decomposition (singular values) gives eigenvalues -\( \sigma_i\geq\sigma_{i+1} \) for all \( i \) and for dimensions larger than \( i=p \), the -eigenvalues (singular values) are zero. -

    +

    we can rewrite the covariance matrix as

    +$$ +\boldsymbol{C}[\boldsymbol{x},\boldsymbol{y}] = \begin{bmatrix} \mathrm{var}[\boldsymbol{x}] & \mathrm{cov}[\boldsymbol{x},\boldsymbol{y}] \\ + \mathrm{cov}[\boldsymbol{x},\boldsymbol{y}] & \mathrm{var}[\boldsymbol{y}] \\ + \end{bmatrix}. +$$ -

    In the general case, where our design matrix \( \boldsymbol{X} \) has dimension -\( n\times p \), the matrix is thus decomposed into an \( n\times n \) -orthogonal matrix \( \boldsymbol{U} \), a \( p\times p \) orthogonal matrix \( \boldsymbol{V} \) -and a diagonal matrix \( \boldsymbol{\Sigma} \) with \( r=\mathrm{min}(n,p) \) -singular values \( \sigma_i\geq 0 \) on the main diagonal and zeros filling -the rest of the matrix. There are at most \( p \) singular values -assuming that \( n > p \). In our regression examples for the nuclear -masses and the equation of state this is indeed the case, while for -the Ising model we have \( p > n \). These are often cases that lead to -near singular or singular matrices. +

    Note: we have used \( 1/n \) in the above definitions of the sample variance and covariance. We assume then that we can calculate the exact mean value. +What you will find in essentially all statistics texts are equations +with a factor \( 1/(n-1) \). This is called Bessel's correction. This +method corrects the bias in the estimation of the population variance +and covariance. It also partially corrects the bias in the estimation +of the population standard deviation. If you use a library like +Scikit-Learn or nunmpy's function to calculate the covariance, this +quantity will be computed with a factor \( 1/(n-1) \).

    -

    The columns of \( \boldsymbol{U} \) are called the left singular vectors while the columns of \( \boldsymbol{V} \) are the right singular vectors.

    -

      @@ -432,7 +388,7 @@ near singular or singular matrices.
    • 56
    • 57
    • ...
      • -
    • 72
      • +
    • 63
    • »
    diff --git a/doc/pub/week35/html/._week35-bs048.html b/doc/pub/week35/html/._week35-bs048.html index 8908d83be..600721a8d 100644 --- a/doc/pub/week35/html/._week35-bs048.html +++ b/doc/pub/week35/html/._week35-bs048.html @@ -38,11 +38,7 @@ doconce format html week35.do.txt --html_style=bootstrap --pygments_html_style=d {'highest level': 2, 'sections': [('Plans for week 35', 2, None, 'plans-for-week-35'), ('Reading recommendations:', 3, None, 'reading-recommendations'), - ('For exercise sessions: Why Linear Regression (aka Ordinary ' - 'Least Squares and family), repeat from last week', - 2, - None, - 'for-exercise-sessions-why-linear-regression-aka-ordinary-least-squares-and-family-repeat-from-last-week'), + ('Reminder from last week', 2, None, 'reminder-from-last-week'), ('The equations for ordinary least squares', 2, None, @@ -122,32 +118,6 @@ doconce format html week35.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'testing-the-means-squared-error-as-function-of-complexity'), - ('More preprocessing examples, two-dimensional example, the ' - 'Franke function', - 2, - None, - 'more-preprocessing-examples-two-dimensional-example-the-franke-function'), - ('To think about, first part', - 2, - None, - 'to-think-about-first-part'), - ('More thinking', 2, None, 'more-thinking'), - ('Still thinking', 2, None, 'still-thinking'), - ('What does centering (subtracting the mean values) mean ' - 'mathematically?', - 2, - None, - 'what-does-centering-subtracting-the-mean-values-mean-mathematically'), - ('Further Manipulations', 2, None, 'further-manipulations'), - ('Wrapping it up', 2, None, 'wrapping-it-up'), - ('Linear Regression code, Intercept handling first', - 2, - None, - 'linear-regression-code-intercept-handling-first'), - ('Material for lecture Monday, August 26', - 2, - None, - 'material-for-lecture-monday-august-26'), ('Mathematical Interpretation of Ordinary Least Squares', 2, None, @@ -279,7 +249,7 @@ MathJax.Hub.Config({ @@ -360,27 +321,32 @@ MathJax.Hub.Config({

     

     

     

    -

    Economy-size SVD

    +

    Covariance and Correlation Matrix

    -

    If we assume that \( n > p \), then our matrix \( \boldsymbol{U} \) has dimension \( n -\times n \). The last \( n-p \) columns of \( \boldsymbol{U} \) become however -irrelevant in our calculations since they are multiplied with the -zeros in \( \boldsymbol{\Sigma} \). +

    The covariance takes values between zero and infinity and may thus +lead to problems with loss of numerical precision for particularly +large values. It is common to scale the covariance matrix by +introducing instead the correlation matrix defined via the so-called +correlation function

    -

    The economy-size decomposition removes extra rows or columns of zeros -from the diagonal matrix of singular values, \( \boldsymbol{\Sigma} \), along with the columns -in either \( \boldsymbol{U} \) or \( \boldsymbol{V} \) that multiply those zeros in the expression. -Removing these zeros and columns can improve execution time -and reduce storage requirements without compromising the accuracy of -the decomposition. +$$ +\mathrm{corr}[\boldsymbol{x},\boldsymbol{y}]=\frac{\mathrm{cov}[\boldsymbol{x},\boldsymbol{y}]}{\sqrt{\mathrm{var}[\boldsymbol{x}] \mathrm{var}[\boldsymbol{y}]}}. +$$ + +

    The correlation function is then given by values \( \mathrm{corr}[\boldsymbol{x},\boldsymbol{y}] +\in [-1,1] \). This avoids eventual problems with too large values. We +can then define the correlation matrix for the two vectors \( \boldsymbol{x} \) +and \( \boldsymbol{y} \) as

    -

    If \( n > p \), we keep only the first \( p \) columns of \( \boldsymbol{U} \) and \( \boldsymbol{\Sigma} \) has dimension \( p\times p \). -If \( p > n \), then only the first \( n \) columns of \( \boldsymbol{V} \) are computed and \( \boldsymbol{\Sigma} \) has dimension \( n\times n \). -The \( n=p \) case is obvious, we retain the full SVD. -In general the economy-size SVD leads to less FLOPS and still conserving the desired accuracy. -

    +$$ +\boldsymbol{K}[\boldsymbol{x},\boldsymbol{y}] = \begin{bmatrix} 1 & \mathrm{corr}[\boldsymbol{x},\boldsymbol{y}] \\ + \mathrm{corr}[\boldsymbol{y},\boldsymbol{x}] & 1 \\ + \end{bmatrix}, +$$ + +

    In the above example this is the function we constructed using pandas.

    @@ -407,7 +373,7 @@ In general the economy-size SVD leads to less FLOPS and still conserving the des

  • 57
  • 58
  • ...
    • -
  • 72
    • +
  • 63
  • »
    • diff --git a/doc/pub/week35/html/._week35-bs049.html b/doc/pub/week35/html/._week35-bs049.html index 2f1c7df67..67bfceff8 100644 --- a/doc/pub/week35/html/._week35-bs049.html +++ b/doc/pub/week35/html/._week35-bs049.html @@ -38,11 +38,7 @@ doconce format html week35.do.txt --html_style=bootstrap --pygments_html_style=d {'highest level': 2, 'sections': [('Plans for week 35', 2, None, 'plans-for-week-35'), ('Reading recommendations:', 3, None, 'reading-recommendations'), - ('For exercise sessions: Why Linear Regression (aka Ordinary ' - 'Least Squares and family), repeat from last week', - 2, - None, - 'for-exercise-sessions-why-linear-regression-aka-ordinary-least-squares-and-family-repeat-from-last-week'), + ('Reminder from last week', 2, None, 'reminder-from-last-week'), ('The equations for ordinary least squares', 2, None, @@ -122,32 +118,6 @@ doconce format html week35.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'testing-the-means-squared-error-as-function-of-complexity'), - ('More preprocessing examples, two-dimensional example, the ' - 'Franke function', - 2, - None, - 'more-preprocessing-examples-two-dimensional-example-the-franke-function'), - ('To think about, first part', - 2, - None, - 'to-think-about-first-part'), - ('More thinking', 2, None, 'more-thinking'), - ('Still thinking', 2, None, 'still-thinking'), - ('What does centering (subtracting the mean values) mean ' - 'mathematically?', - 2, - None, - 'what-does-centering-subtracting-the-mean-values-mean-mathematically'), - ('Further Manipulations', 2, None, 'further-manipulations'), - ('Wrapping it up', 2, None, 'wrapping-it-up'), - ('Linear Regression code, Intercept handling first', - 2, - None, - 'linear-regression-code-intercept-handling-first'), - ('Material for lecture Monday, August 26', - 2, - None, - 'material-for-lecture-monday-august-26'), ('Mathematical Interpretation of Ordinary Least Squares', 2, None, @@ -279,7 +249,7 @@ MathJax.Hub.Config({ @@ -360,69 +321,66 @@ MathJax.Hub.Config({

     

     

     

    -

    Codes for the SVD

    +

    Correlation Function and Design/Feature Matrix

    - - -
    -
    -
    -
    -
    - 
    import numpy as np
-# SVD inversion
-def SVD(A):
-    ''' Takes as input a numpy matrix A and returns inv(A) based on singular value decomposition (SVD).
-    SVD is numerically more stable than the inversion algorithms provided by
-    numpy and scipy.linalg at the cost of being slower.
-    '''
-    U, S, VT = np.linalg.svd(A,full_matrices=True)
-    print('test U')
-    print( (np.transpose(U) @ U - U @np.transpose(U)))
-    print('test VT')
-    print( (np.transpose(VT) @ VT - VT @np.transpose(VT)))
-    print(U)
-    print(S)
-    print(VT)
-
-    D = np.zeros((len(U),len(VT)))
-    for i in range(0,len(VT)):
-        D[i,i]=S[i]
-    return U @ D @ VT
-
-
-X = np.array([ [1.0,-1.0], [1.0,-1.0]])
-#X = np.array([[1, 2], [3, 4], [5, 6]])
-
-print(X)
-C = SVD(X)
-# Print the difference between the original matrix and the SVD one
-print(C-X)
-
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    - -

    The matrix \( \boldsymbol{X} \) has columns that are linearly dependent. The first -column is the row-wise sum of the other two columns. The rank of a -matrix (the column rank) is the dimension of space spanned by the -column vectors. The rank of the matrix is the number of linearly -independent columns, in this case just \( 2 \). We see this from the -singular values when running the above code. Running the standard -inversion algorithm for matrix inversion with \( \boldsymbol{X}^T\boldsymbol{X} \) results -in the program terminating due to a singular matrix. +

    In our derivation of the various regression algorithms like Ordinary Least Squares or Ridge regression +we defined the design/feature matrix \( \boldsymbol{X} \) as

    +$$ +\boldsymbol{X}=\begin{bmatrix} +x_{0,0} & x_{0,1} & x_{0,2}& \dots & \dots x_{0,p-1}\\ +x_{1,0} & x_{1,1} & x_{1,2}& \dots & \dots x_{1,p-1}\\ +x_{2,0} & x_{2,1} & x_{2,2}& \dots & \dots x_{2,p-1}\\ +\dots & \dots & \dots & \dots \dots & \dots \\ +x_{n-2,0} & x_{n-2,1} & x_{n-2,2}& \dots & \dots x_{n-2,p-1}\\ +x_{n-1,0} & x_{n-1,1} & x_{n-1,2}& \dots & \dots x_{n-1,p-1}\\ +\end{bmatrix}, +$$ + +

    with \( \boldsymbol{X}\in {\mathbb{R}}^{n\times p} \), with the predictors/features \( p \) refering to the column numbers and the +entries \( n \) being the row elements. +We can rewrite the design/feature matrix in terms of its column vectors as +

    +$$ +\boldsymbol{X}=\begin{bmatrix} \boldsymbol{x}_0 & \boldsymbol{x}_1 & \boldsymbol{x}_2 & \dots & \dots & \boldsymbol{x}_{p-1}\end{bmatrix}, +$$ + +

    with a given vector

    +$$ +\boldsymbol{x}_i^T = \begin{bmatrix}x_{0,i} & x_{1,i} & x_{2,i}& \dots & \dots x_{n-1,i}\end{bmatrix}. +$$ + +

    With these definitions, we can now rewrite our \( 2\times 2 \) +correlation/covariance matrix in terms of a moe general design/feature +matrix \( \boldsymbol{X}\in {\mathbb{R}}^{n\times p} \). This leads to a \( p\times p \) +covariance matrix for the vectors \( \boldsymbol{x}_i \) with \( i=0,1,\dots,p-1 \) +

    + +$$ +\boldsymbol{C}[\boldsymbol{x}] = \begin{bmatrix} +\mathrm{var}[\boldsymbol{x}_0] & \mathrm{cov}[\boldsymbol{x}_0,\boldsymbol{x}_1] & \mathrm{cov}[\boldsymbol{x}_0,\boldsymbol{x}_2] & \dots & \dots & \mathrm{cov}[\boldsymbol{x}_0,\boldsymbol{x}_{p-1}]\\ +\mathrm{cov}[\boldsymbol{x}_1,\boldsymbol{x}_0] & \mathrm{var}[\boldsymbol{x}_1] & \mathrm{cov}[\boldsymbol{x}_1,\boldsymbol{x}_2] & \dots & \dots & \mathrm{cov}[\boldsymbol{x}_1,\boldsymbol{x}_{p-1}]\\ +\mathrm{cov}[\boldsymbol{x}_2,\boldsymbol{x}_0] & \mathrm{cov}[\boldsymbol{x}_2,\boldsymbol{x}_1] & \mathrm{var}[\boldsymbol{x}_2] & \dots & \dots & \mathrm{cov}[\boldsymbol{x}_2,\boldsymbol{x}_{p-1}]\\ +\dots & \dots & \dots & \dots & \dots & \dots \\ +\dots & \dots & \dots & \dots & \dots & \dots \\ +\mathrm{cov}[\boldsymbol{x}_{p-1},\boldsymbol{x}_0] & \mathrm{cov}[\boldsymbol{x}_{p-1},\boldsymbol{x}_1] & \mathrm{cov}[\boldsymbol{x}_{p-1},\boldsymbol{x}_{2}] & \dots & \dots & \mathrm{var}[\boldsymbol{x}_{p-1}]\\ +\end{bmatrix}, +$$ + +

    and the correlation matrix

    +$$ +\boldsymbol{K}[\boldsymbol{x}] = \begin{bmatrix} +1 & \mathrm{corr}[\boldsymbol{x}_0,\boldsymbol{x}_1] & \mathrm{corr}[\boldsymbol{x}_0,\boldsymbol{x}_2] & \dots & \dots & \mathrm{corr}[\boldsymbol{x}_0,\boldsymbol{x}_{p-1}]\\ +\mathrm{corr}[\boldsymbol{x}_1,\boldsymbol{x}_0] & 1 & \mathrm{corr}[\boldsymbol{x}_1,\boldsymbol{x}_2] & \dots & \dots & \mathrm{corr}[\boldsymbol{x}_1,\boldsymbol{x}_{p-1}]\\ +\mathrm{corr}[\boldsymbol{x}_2,\boldsymbol{x}_0] & \mathrm{corr}[\boldsymbol{x}_2,\boldsymbol{x}_1] & 1 & \dots & \dots & \mathrm{corr}[\boldsymbol{x}_2,\boldsymbol{x}_{p-1}]\\ +\dots & \dots & \dots & \dots & \dots & \dots \\ +\dots & \dots & \dots & \dots & \dots & \dots \\ +\mathrm{corr}[\boldsymbol{x}_{p-1},\boldsymbol{x}_0] & \mathrm{corr}[\boldsymbol{x}_{p-1},\boldsymbol{x}_1] & \mathrm{corr}[\boldsymbol{x}_{p-1},\boldsymbol{x}_{2}] & \dots & \dots & 1\\ +\end{bmatrix}, +$$ + +

      @@ -448,7 +406,7 @@ in the program terminating due to a singular matrix.
    • 58
    • 59
    • ...
      • -
    • 72
      • +
    • 63
    • »
    diff --git a/doc/pub/week35/html/._week35-bs050.html b/doc/pub/week35/html/._week35-bs050.html index 3a252eb47..841e8bf34 100644 --- a/doc/pub/week35/html/._week35-bs050.html +++ b/doc/pub/week35/html/._week35-bs050.html @@ -38,11 +38,7 @@ doconce format html week35.do.txt --html_style=bootstrap --pygments_html_style=d {'highest level': 2, 'sections': [('Plans for week 35', 2, None, 'plans-for-week-35'), ('Reading recommendations:', 3, None, 'reading-recommendations'), - ('For exercise sessions: Why Linear Regression (aka Ordinary ' - 'Least Squares and family), repeat from last week', - 2, - None, - 'for-exercise-sessions-why-linear-regression-aka-ordinary-least-squares-and-family-repeat-from-last-week'), + ('Reminder from last week', 2, None, 'reminder-from-last-week'), ('The equations for ordinary least squares', 2, None, @@ -122,32 +118,6 @@ doconce format html week35.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'testing-the-means-squared-error-as-function-of-complexity'), - ('More preprocessing examples, two-dimensional example, the ' - 'Franke function', - 2, - None, - 'more-preprocessing-examples-two-dimensional-example-the-franke-function'), - ('To think about, first part', - 2, - None, - 'to-think-about-first-part'), - ('More thinking', 2, None, 'more-thinking'), - ('Still thinking', 2, None, 'still-thinking'), - ('What does centering (subtracting the mean values) mean ' - 'mathematically?', - 2, - None, - 'what-does-centering-subtracting-the-mean-values-mean-mathematically'), - ('Further Manipulations', 2, None, 'further-manipulations'), - ('Wrapping it up', 2, None, 'wrapping-it-up'), - ('Linear Regression code, Intercept handling first', - 2, - None, - 'linear-regression-code-intercept-handling-first'), - ('Material for lecture Monday, August 26', - 2, - None, - 'material-for-lecture-monday-august-26'), ('Mathematical Interpretation of Ordinary Least Squares', 2, None, @@ -279,7 +249,7 @@ MathJax.Hub.Config({ @@ -360,23 +321,61 @@ MathJax.Hub.Config({

     

     

     

    -

    Note about SVD Calculations

    +

    Covariance Matrix Examples

    -

    The \( U \), \( S \), and \( V \) matrices returned from the svd() function -cannot be multiplied directly. +

    The Numpy function np.cov calculates the covariance elements using +the factor \( 1/(n-1) \) instead of \( 1/n \) since it assumes we do not have +the exact mean values. The following simple function uses the +np.vstack function which takes each vector of dimension \( 1\times n \) +and produces a \( 2\times n \) matrix \( \boldsymbol{W} \)

    -

    As you can see from the code, the \( S \) vector must be converted into a -diagonal matrix. This may cause a problem as the size of the matrices -do not fit the rules of matrix multiplication, where the number of -columns in a matrix must match the number of rows in the subsequent -matrix. +

    Note that this assumes you have the features as the rows, and the inputs as columns, that is

    +$$ +\boldsymbol{W} = \begin{bmatrix} x_0 & x_1 & x_2 & \dots & x_{n-2} & x_{n-1} \\ + y_0 & y_1 & y_2 & \dots & y_{n-2} & y_{n-1} \\ + \end{bmatrix}, +$$ + +

    which in turn is converted into into the \( 2\times 2 \) covariance matrix +\( \boldsymbol{C} \) via the Numpy function np.cov(). We note that we can also calculate +the mean value of each set of samples \( \boldsymbol{x} \) etc using the Numpy +function np.mean(x). We can also extract the eigenvalues of the +covariance matrix through the np.linalg.eig() function.

    -

    If you wish to include the zero singular values, you will need to -resize the matrices and set up a diagonal matrix as done in the above -example -

    + + +
    +
    +
    +
    +
    + 
    # Importing various packages
+import numpy as np
+n = 100
+x = np.random.normal(size=n)
+print(np.mean(x))
+y = 4+3*x+np.random.normal(size=n)
+print(np.mean(y))
+W = np.vstack((x, y))
+C = np.cov(W)
+print(C)
+
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +

    @@ -403,7 +402,7 @@ example

  • 59
  • 60
  • ...
    • -
  • 72
    • +
  • 63
  • »
    • diff --git a/doc/pub/week35/html/._week35-bs051.html b/doc/pub/week35/html/._week35-bs051.html index 7567d7363..4d58521d8 100644 --- a/doc/pub/week35/html/._week35-bs051.html +++ b/doc/pub/week35/html/._week35-bs051.html @@ -38,11 +38,7 @@ doconce format html week35.do.txt --html_style=bootstrap --pygments_html_style=d {'highest level': 2, 'sections': [('Plans for week 35', 2, None, 'plans-for-week-35'), ('Reading recommendations:', 3, None, 'reading-recommendations'), - ('For exercise sessions: Why Linear Regression (aka Ordinary ' - 'Least Squares and family), repeat from last week', - 2, - None, - 'for-exercise-sessions-why-linear-regression-aka-ordinary-least-squares-and-family-repeat-from-last-week'), + ('Reminder from last week', 2, None, 'reminder-from-last-week'), ('The equations for ordinary least squares', 2, None, @@ -122,32 +118,6 @@ doconce format html week35.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'testing-the-means-squared-error-as-function-of-complexity'), - ('More preprocessing examples, two-dimensional example, the ' - 'Franke function', - 2, - None, - 'more-preprocessing-examples-two-dimensional-example-the-franke-function'), - ('To think about, first part', - 2, - None, - 'to-think-about-first-part'), - ('More thinking', 2, None, 'more-thinking'), - ('Still thinking', 2, None, 'still-thinking'), - ('What does centering (subtracting the mean values) mean ' - 'mathematically?', - 2, - None, - 'what-does-centering-subtracting-the-mean-values-mean-mathematically'), - ('Further Manipulations', 2, None, 'further-manipulations'), - ('Wrapping it up', 2, None, 'wrapping-it-up'), - ('Linear Regression code, Intercept handling first', - 2, - None, - 'linear-regression-code-intercept-handling-first'), - ('Material for lecture Monday, August 26', - 2, - None, - 'material-for-lecture-monday-august-26'), ('Mathematical Interpretation of Ordinary Least Squares', 2, None, @@ -279,7 +249,7 @@ MathJax.Hub.Config({ @@ -360,38 +321,64 @@ MathJax.Hub.Config({

     

     

     

    -

    Mathematics of the SVD and implications

    +

    Correlation Matrix

    -

    Let us take a closer look at the mathematics of the SVD and the various implications for machine learning studies.

    +

    The previous example can be converted into the correlation matrix by +simply scaling the matrix elements with the variances. We should also +subtract the mean values for each column. This leads to the following +code which sets up the correlations matrix for the previous example in +a more brute force way. Here we scale the mean values for each column of the design matrix, calculate the relevant mean values and variances and then finally set up the \( 2\times 2 \) correlation matrix (since we have only two vectors). +

    -

    Our starting point is our design matrix \( \boldsymbol{X} \) of dimension \( n\times p \)

    -$$ -\boldsymbol{X}=\begin{bmatrix} -x_{0,0} & x_{0,1} & x_{0,2}& \dots & \dots x_{0,p-1}\\ -x_{1,0} & x_{1,1} & x_{1,2}& \dots & \dots x_{1,p-1}\\ -x_{2,0} & x_{2,1} & x_{2,2}& \dots & \dots x_{2,p-1}\\ -\dots & \dots & \dots & \dots \dots & \dots \\ -x_{n-2,0} & x_{n-2,1} & x_{n-2,2}& \dots & \dots x_{n-2,p-1}\\ -x_{n-1,0} & x_{n-1,1} & x_{n-1,2}& \dots & \dots x_{n-1,p-1}\\ -\end{bmatrix}. -$$ -

    We can SVD decompose our matrix as

    -$$ -\boldsymbol{X}=\boldsymbol{U}\boldsymbol{\Sigma}\boldsymbol{V}^T, -$$ + +
    +
    +
    +
    +
    + 
    import numpy as np
+n = 100
+# define two vectors                                                                                           
+x = np.random.random(size=n)
+y = 4+3*x+np.random.normal(size=n)
+#scaling the x and y vectors                                                                                   
+x = x - np.mean(x)
+y = y - np.mean(y)
+variance_x = np.sum(x@x)/n
+variance_y = np.sum(y@y)/n
+print(variance_x)
+print(variance_y)
+cov_xy = np.sum(x@y)/n
+cov_xx = np.sum(x@x)/n
+cov_yy = np.sum(y@y)/n
+C = np.zeros((2,2))
+C[0,0]= cov_xx/variance_x
+C[1,1]= cov_yy/variance_y
+C[0,1]= cov_xy/np.sqrt(variance_y*variance_x)
+C[1,0]= C[0,1]
+print(C)
+
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    -

    where \( \boldsymbol{U} \) is an orthogonal matrix of dimension \( n\times n \), meaning that \( \boldsymbol{U}\boldsymbol{U}^T=\boldsymbol{U}^T\boldsymbol{U}=\boldsymbol{I}_n \). Here \( \boldsymbol{I}_n \) is the unit matrix of dimension \( n \times n \).

    +

    We see that the matrix elements along the diagonal are one as they +should be and that the matrix is symmetric. Furthermore, diagonalizing +this matrix we easily see that it is a positive definite matrix. +

    -

    Similarly, \( \boldsymbol{V} \) is an orthogonal matrix of dimension \( p\times p \), meaning that \( \boldsymbol{V}\boldsymbol{V}^T=\boldsymbol{V}^T\boldsymbol{V}=\boldsymbol{I}_p \). Here \( \boldsymbol{I}_n \) is the unit matrix of dimension \( p \times p \).

    - -

    Finally \( \boldsymbol{\Sigma} \) contains the singular values \( \sigma_i \). This matrix has dimension \( n\times p \) and the singular values \( \sigma_i \) are all positive. The non-zero values are ordered in descending order, that is

    - -$$ -\sigma_0 > \sigma_1 > \sigma_2 > \dots > \sigma_{p-1} > 0. -$$ - -

    All values beyond \( p-1 \) are all zero.

    +

    The above procedure with numpy can be made more compact if we use pandas.

    @@ -418,7 +405,7 @@ $$

  • 60
  • 61
  • ...
    • -
  • 72
    • +
  • 63
  • »
    • diff --git a/doc/pub/week35/html/._week35-bs052.html b/doc/pub/week35/html/._week35-bs052.html index 1123583c0..a148cc62e 100644 --- a/doc/pub/week35/html/._week35-bs052.html +++ b/doc/pub/week35/html/._week35-bs052.html @@ -38,11 +38,7 @@ doconce format html week35.do.txt --html_style=bootstrap --pygments_html_style=d {'highest level': 2, 'sections': [('Plans for week 35', 2, None, 'plans-for-week-35'), ('Reading recommendations:', 3, None, 'reading-recommendations'), - ('For exercise sessions: Why Linear Regression (aka Ordinary ' - 'Least Squares and family), repeat from last week', - 2, - None, - 'for-exercise-sessions-why-linear-regression-aka-ordinary-least-squares-and-family-repeat-from-last-week'), + ('Reminder from last week', 2, None, 'reminder-from-last-week'), ('The equations for ordinary least squares', 2, None, @@ -122,32 +118,6 @@ doconce format html week35.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'testing-the-means-squared-error-as-function-of-complexity'), - ('More preprocessing examples, two-dimensional example, the ' - 'Franke function', - 2, - None, - 'more-preprocessing-examples-two-dimensional-example-the-franke-function'), - ('To think about, first part', - 2, - None, - 'to-think-about-first-part'), - ('More thinking', 2, None, 'more-thinking'), - ('Still thinking', 2, None, 'still-thinking'), - ('What does centering (subtracting the mean values) mean ' - 'mathematically?', - 2, - None, - 'what-does-centering-subtracting-the-mean-values-mean-mathematically'), - ('Further Manipulations', 2, None, 'further-manipulations'), - ('Wrapping it up', 2, None, 'wrapping-it-up'), - ('Linear Regression code, Intercept handling first', - 2, - None, - 'linear-regression-code-intercept-handling-first'), - ('Material for lecture Monday, August 26', - 2, - None, - 'material-for-lecture-monday-august-26'), ('Mathematical Interpretation of Ordinary Least Squares', 2, None, @@ -279,7 +249,7 @@ MathJax.Hub.Config({ @@ -360,62 +321,46 @@ MathJax.Hub.Config({

     

     

     

    -

    Example Matrix

    +

    Correlation Matrix with Pandas

    -

    As an example, consider the following \( 3\times 2 \) example for the matrix \( \boldsymbol{\Sigma} \)

    +

    We whow here how we can set up the correlation matrix using pandas, as done in this simple code

    -$$ -\boldsymbol{\Sigma}= -\begin{bmatrix} -2& 0 \\ -0 & 1 \\ -0 & 0 \\ -\end{bmatrix} -$$ + +
    +
    +
    +
    +
    + 
    import numpy as np
+import pandas as pd
+n = 10
+x = np.random.normal(size=n)
+x = x - np.mean(x)
+y = 4+3*x+np.random.normal(size=n)
+y = y - np.mean(y)
+# Note that we transpose the matrix in order to stay with our ordering n x p
+X = (np.vstack((x, y))).T
+print(X)
+Xpd = pd.DataFrame(X)
+print(Xpd)
+correlation_matrix = Xpd.corr()
+print(correlation_matrix)
+
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    -

    The singular values are \( \sigma_0=2 \) and \( \sigma_1=1 \). It is common to rewrite the matrix \( \boldsymbol{\Sigma} \) as

    - -$$ -\boldsymbol{\Sigma}= -\begin{bmatrix} -\boldsymbol{\tilde{\Sigma}}\\ -\boldsymbol{0}\\ -\end{bmatrix}, -$$ - -

    where

    -$$ -\boldsymbol{\tilde{\Sigma}}= -\begin{bmatrix} -2& 0 \\ -0 & 1 \\ -\end{bmatrix}, -$$ - -

    contains only the singular values. Note also (and we will use this below) that

    - -$$ -\boldsymbol{\Sigma}^T\boldsymbol{\Sigma}= -\begin{bmatrix} -4& 0 \\ -0 & 1 \\ -\end{bmatrix}, -$$ - -

    which is a \( 2\times 2 \) matrix while

    -$$ -\boldsymbol{\Sigma}\boldsymbol{\Sigma}^T= -\begin{bmatrix} -4& 0 & 0\\ -0 & 1 & 0\\ -0 & 0 & 0\\ -\end{bmatrix}, -$$ - -

    is a \( 3\times 3 \) matrix. The last row and column of this last matrix -contain only zeros. This will have important consequences for our SVD -decomposition of the design matrix. -

    +

    We expand this model to the Franke function discussed above.

    @@ -442,7 +387,7 @@ decomposition of the design matrix.

  • 61
  • 62
  • ...
    • -
  • 72
    • +
  • 63
  • »
    • diff --git a/doc/pub/week35/html/._week35-bs053.html b/doc/pub/week35/html/._week35-bs053.html index 4734db895..55180f1ee 100644 --- a/doc/pub/week35/html/._week35-bs053.html +++ b/doc/pub/week35/html/._week35-bs053.html @@ -38,11 +38,7 @@ doconce format html week35.do.txt --html_style=bootstrap --pygments_html_style=d {'highest level': 2, 'sections': [('Plans for week 35', 2, None, 'plans-for-week-35'), ('Reading recommendations:', 3, None, 'reading-recommendations'), - ('For exercise sessions: Why Linear Regression (aka Ordinary ' - 'Least Squares and family), repeat from last week', - 2, - None, - 'for-exercise-sessions-why-linear-regression-aka-ordinary-least-squares-and-family-repeat-from-last-week'), + ('Reminder from last week', 2, None, 'reminder-from-last-week'), ('The equations for ordinary least squares', 2, None, @@ -122,32 +118,6 @@ doconce format html week35.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'testing-the-means-squared-error-as-function-of-complexity'), - ('More preprocessing examples, two-dimensional example, the ' - 'Franke function', - 2, - None, - 'more-preprocessing-examples-two-dimensional-example-the-franke-function'), - ('To think about, first part', - 2, - None, - 'to-think-about-first-part'), - ('More thinking', 2, None, 'more-thinking'), - ('Still thinking', 2, None, 'still-thinking'), - ('What does centering (subtracting the mean values) mean ' - 'mathematically?', - 2, - None, - 'what-does-centering-subtracting-the-mean-values-mean-mathematically'), - ('Further Manipulations', 2, None, 'further-manipulations'), - ('Wrapping it up', 2, None, 'wrapping-it-up'), - ('Linear Regression code, Intercept handling first', - 2, - None, - 'linear-regression-code-intercept-handling-first'), - ('Material for lecture Monday, August 26', - 2, - None, - 'material-for-lecture-monday-august-26'), ('Mathematical Interpretation of Ordinary Least Squares', 2, None, @@ -279,7 +249,7 @@ MathJax.Hub.Config({ @@ -360,47 +321,82 @@ MathJax.Hub.Config({

     

     

     

    -

    Setting up the Matrix to be inverted

    +

    Correlation Matrix with Pandas and the Franke function

    -

    The matrix that may cause problems for us is \( \boldsymbol{X}^T\boldsymbol{X} \). Using the SVD we can rewrite this matrix as

    -$$ -\boldsymbol{X}^T\boldsymbol{X}=\boldsymbol{V}\boldsymbol{\Sigma}^T\boldsymbol{U}^T\boldsymbol{U}\boldsymbol{\Sigma}\boldsymbol{V}^T, -$$ + +
    +
    +
    +
    +
    + 
    # Common imports
+import numpy as np
+import pandas as pd
 
-
    and using the orthogonality of the matrix \( \boldsymbol{U} \) we have
    
 
-$$
-\boldsymbol{X}^T\boldsymbol{X}=\boldsymbol{V}\boldsymbol{\Sigma}^T\boldsymbol{\Sigma}\boldsymbol{V}^T.
-$$
+def FrankeFunction(x,y):
+	term1 = 0.75*np.exp(-(0.25*(9*x-2)**2) - 0.25*((9*y-2)**2))
+	term2 = 0.75*np.exp(-((9*x+1)**2)/49.0 - 0.1*(9*y+1))
+	term3 = 0.5*np.exp(-(9*x-7)**2/4.0 - 0.25*((9*y-3)**2))
+	term4 = -0.2*np.exp(-(9*x-4)**2 - (9*y-7)**2)
+	return term1 + term2 + term3 + term4
 
-
    We define \( \boldsymbol{\Sigma}^T\boldsymbol{\Sigma}=\tilde{\boldsymbol{\Sigma}}^2 \) which is  a diagonal matrix containing only the singular values squared. It has dimensionality \( p \times p \).
    
 
-
    We can now insert the result for the matrix \( \boldsymbol{X}^T\boldsymbol{X} \) into our equation for ordinary least squares where
    
+def create_X(x, y, n ):
+	if len(x.shape) > 1:
+		x = np.ravel(x)
+		y = np.ravel(y)
 
-$$
-\tilde{y}_{\mathrm{OLS}}=\boldsymbol{X}\left(\boldsymbol{X}^T\boldsymbol{X}\right)^{-1}\boldsymbol{X}^T\boldsymbol{y},
-$$
+	N = len(x)
+	l = int((n+1)*(n+2)/2)		# Number of elements in beta
+	X = np.ones((N,l))
 
-
    and using our SVD decomposition of \( \boldsymbol{X} \) we have
    
+	for i in range(1,n+1):
+		q = int((i)*(i+1)/2)
+		for k in range(i+1):
+			X[:,q+k] = (x**(i-k))*(y**k)
 
-$$
-\tilde{y}_{\mathrm{OLS}}=\boldsymbol{U}\boldsymbol{\Sigma}\boldsymbol{V}^T\left(\boldsymbol{V}\tilde{\boldsymbol{\Sigma}}^{2}(\boldsymbol{V}^T\right)^{-1}\boldsymbol{V}\boldsymbol{\Sigma}^T\boldsymbol{U}^T\boldsymbol{y},
-$$
+	return X
 
-
    which gives us, using the orthogonality of the matrix \( \boldsymbol{V} \),
    
 
-$$
-\tilde{y}_{\mathrm{OLS}}=\boldsymbol{U}\boldsymbol{U}^T\boldsymbol{y}=\sum_{i=0}^{p-1}\boldsymbol{u}_i\boldsymbol{u}^T_i\boldsymbol{y},
-$$
+# Making meshgrid of datapoints and compute Franke's function
+n = 4
+N = 100
+x = np.sort(np.random.uniform(0, 1, N))
+y = np.sort(np.random.uniform(0, 1, N))
+z = FrankeFunction(x, y)
+X = create_X(x, y, n=n)    
 
-
    It means that the ordinary least square model (with the optimal
-parameters) \( \boldsymbol{\tilde{y}} \), corresponds to an orthogonal
-transformation of the output (or target) vector \( \boldsymbol{y} \) by the
-vectors of the matrix \( \boldsymbol{U} \). Note that the summation ends at
-\( p-1 \), that is \( \boldsymbol{\tilde{y}}\ne \boldsymbol{y} \). We can thus not use the
-orthogonality relation for the matrix \( \boldsymbol{U} \). This can already be
-when we multiply the matrices \( \boldsymbol{\Sigma}^T\boldsymbol{U}^T \).
+Xpd = pd.DataFrame(X)
+# subtract the mean values and set up the covariance matrix
+Xpd = Xpd - Xpd.mean()
+covariance_matrix = Xpd.cov()
+print(covariance_matrix)
+
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    + +

    We note here that the covariance is zero for the first rows and +columns since all matrix elements in the design matrix were set to one +(we are fitting the function in terms of a polynomial of degree \( n \)). +

    + +

    This means that the variance for these elements will be zero and will +cause problems when we set up the correlation matrix. We can simply +drop these elements and construct a correlation +matrix without these elements.

    @@ -427,8 +423,6 @@ when we multiply the matrices \( \boldsymbol{\Sigma}^T\boldsymbol{U}^T \).

  • 61
  • 62
  • 63
    • -
  • ...
    • -
  • 72
  • »
    • diff --git a/doc/pub/week35/html/._week35-bs054.html b/doc/pub/week35/html/._week35-bs054.html index 19c7bcce6..86df2852f 100644 --- a/doc/pub/week35/html/._week35-bs054.html +++ b/doc/pub/week35/html/._week35-bs054.html @@ -38,11 +38,7 @@ doconce format html week35.do.txt --html_style=bootstrap --pygments_html_style=d {'highest level': 2, 'sections': [('Plans for week 35', 2, None, 'plans-for-week-35'), ('Reading recommendations:', 3, None, 'reading-recommendations'), - ('For exercise sessions: Why Linear Regression (aka Ordinary ' - 'Least Squares and family), repeat from last week', - 2, - None, - 'for-exercise-sessions-why-linear-regression-aka-ordinary-least-squares-and-family-repeat-from-last-week'), + ('Reminder from last week', 2, None, 'reminder-from-last-week'), ('The equations for ordinary least squares', 2, None, @@ -122,32 +118,6 @@ doconce format html week35.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'testing-the-means-squared-error-as-function-of-complexity'), - ('More preprocessing examples, two-dimensional example, the ' - 'Franke function', - 2, - None, - 'more-preprocessing-examples-two-dimensional-example-the-franke-function'), - ('To think about, first part', - 2, - None, - 'to-think-about-first-part'), - ('More thinking', 2, None, 'more-thinking'), - ('Still thinking', 2, None, 'still-thinking'), - ('What does centering (subtracting the mean values) mean ' - 'mathematically?', - 2, - None, - 'what-does-centering-subtracting-the-mean-values-mean-mathematically'), - ('Further Manipulations', 2, None, 'further-manipulations'), - ('Wrapping it up', 2, None, 'wrapping-it-up'), - ('Linear Regression code, Intercept handling first', - 2, - None, - 'linear-regression-code-intercept-handling-first'), - ('Material for lecture Monday, August 26', - 2, - None, - 'material-for-lecture-monday-august-26'), ('Mathematical Interpretation of Ordinary Least Squares', 2, None, @@ -279,7 +249,7 @@ MathJax.Hub.Config({ @@ -360,53 +321,41 @@ MathJax.Hub.Config({

     

     

     

    -

    Further properties (important for our analyses later)

    +

    Rewriting the Covariance and/or Correlation Matrix

    -

    Let us study again \( \boldsymbol{X}^T\boldsymbol{X} \) in terms of our SVD,

    +

    We can rewrite the covariance matrix in a more compact form in terms of the design/feature matrix \( \boldsymbol{X} \) as

    $$ -\boldsymbol{X}^T\boldsymbol{X}=\boldsymbol{V}\boldsymbol{\Sigma}^T\boldsymbol{U}^T\boldsymbol{U}\boldsymbol{\Sigma}\boldsymbol{V}^T=\boldsymbol{V}\boldsymbol{\Sigma}^T\boldsymbol{\Sigma}\boldsymbol{V}^T. +\boldsymbol{C}[\boldsymbol{x}] = \frac{1}{n}\boldsymbol{X}^T\boldsymbol{X}= \mathbb{E}[\boldsymbol{X}^T\boldsymbol{X}]. $$ -

    If we now multiply from the right with \( \boldsymbol{V} \) (using the orthogonality of \( \boldsymbol{V} \)) we get

    +

    To see this let us simply look at a design matrix \( \boldsymbol{X}\in {\mathbb{R}}^{2\times 2} \)

    $$ -\left(\boldsymbol{X}^T\boldsymbol{X}\right)\boldsymbol{V}=\boldsymbol{V}\boldsymbol{\Sigma}^T\boldsymbol{\Sigma}. +\boldsymbol{X}=\begin{bmatrix} +x_{00} & x_{01}\\ +x_{10} & x_{11}\\ +\end{bmatrix}=\begin{bmatrix} +\boldsymbol{x}_{0} & \boldsymbol{x}_{1}\\ +\end{bmatrix}. $$ -

    This means the vectors \( \boldsymbol{v}_i \) of the orthogonal matrix \( \boldsymbol{V} \) are the eigenvectors of the matrix \( \boldsymbol{X}^T\boldsymbol{X} \) -with eigenvalues given by the singular values squared, that is -

    +

    If we then compute the expectation value (note the \( 1/n \) factor instead of \( 1/(n-1) \))

    $$ -\left(\boldsymbol{X}^T\boldsymbol{X}\right)\boldsymbol{v}_i=\boldsymbol{v}_i\sigma_i^2. +\mathbb{E}[\boldsymbol{X}^T\boldsymbol{X}] = \frac{1}{n}\boldsymbol{X}^T\boldsymbol{X}=\frac{1}{n}\begin{bmatrix} +x_{00}^2+x_{10}^2 & x_{00}x_{01}+x_{10}x_{11}\\ +x_{01}x_{00}+x_{11}x_{10} & x_{01}^2+x_{11}^2\\ +\end{bmatrix}, $$ -

    Similarly, if we use the SVD decomposition for the matrix \( \boldsymbol{X}\boldsymbol{X}^T \), we have

    +

    which is just

    $$ -\boldsymbol{X}\boldsymbol{X}^T=\boldsymbol{U}\boldsymbol{\Sigma}\boldsymbol{V}^T\boldsymbol{V}\boldsymbol{\Sigma}^T\boldsymbol{U}^T=\boldsymbol{U}\boldsymbol{\Sigma}\boldsymbol{\Sigma}^T\boldsymbol{U}^T. +\boldsymbol{C}[\boldsymbol{x}_0,\boldsymbol{x}_1] = \boldsymbol{C}[\boldsymbol{x}]=\begin{bmatrix} \mathrm{var}[\boldsymbol{x}_0] & \mathrm{cov}[\boldsymbol{x}_0,\boldsymbol{x}_1] \\ + \mathrm{cov}[\boldsymbol{x}_1,\boldsymbol{x}_0] & \mathrm{var}[\boldsymbol{x}_1] \\ + \end{bmatrix}, $$ -

    If we now multiply from the right with \( \boldsymbol{U} \) (using the orthogonality of \( \boldsymbol{U} \)) we get

    -$$ -\left(\boldsymbol{X}\boldsymbol{X}^T\right)\boldsymbol{U}=\boldsymbol{U}\boldsymbol{\Sigma}\boldsymbol{\Sigma}^T. -$$ +

    where we wrote $$\boldsymbol{C}[\boldsymbol{x}_0,\boldsymbol{x}_1] = \boldsymbol{C}[\boldsymbol{x}]$$ to indicate that this is the covariance of the vectors \( \boldsymbol{x} \) of the design/feature matrix \( \boldsymbol{X} \).

    -

    This means the vectors \( \boldsymbol{u}_i \) of the orthogonal matrix \( \boldsymbol{U} \) are the eigenvectors of the matrix \( \boldsymbol{X}\boldsymbol{X}^T \) -with eigenvalues given by the singular values squared, that is -

    -$$ -\left(\boldsymbol{X}\boldsymbol{X}^T\right)\boldsymbol{u}_i=\boldsymbol{u}_i\sigma_i^2. -$$ - -

    Important note: we have defined our design matrix \( \boldsymbol{X} \) to be an -\( n\times p \) matrix. In most supervised learning cases we have that \( n -\ge p \), and quite often we have \( n >> p \). For linear algebra based methods like ordinary least squares or Ridge regression, this leads to a matrix \( \boldsymbol{X}^T\boldsymbol{X} \) which is small and thereby easier to handle from a computational point of view (in terms of number of floating point operations). -

    - -

    In our lectures, the number of columns will -always refer to the number of features in our data set, while the -number of rows represents the number of data inputs. Note that in -other texts you may find the opposite notation. This has consequences -for the definition of for example the covariance matrix and its relation to the SVD. -

    +

    It is easy to generalize this to a matrix \( \boldsymbol{X}\in {\mathbb{R}}^{n\times p} \).

    @@ -431,9 +380,6 @@ for the definition of for example the covariance matrix and its relation to the

  • 61
  • 62
  • 63
    • -
  • 64
    • -
  • ...
    • -
  • 72
  • »
    • diff --git a/doc/pub/week35/html/._week35-bs055.html b/doc/pub/week35/html/._week35-bs055.html index e42c6c51e..e4a7a81a2 100644 --- a/doc/pub/week35/html/._week35-bs055.html +++ b/doc/pub/week35/html/._week35-bs055.html @@ -38,11 +38,7 @@ doconce format html week35.do.txt --html_style=bootstrap --pygments_html_style=d {'highest level': 2, 'sections': [('Plans for week 35', 2, None, 'plans-for-week-35'), ('Reading recommendations:', 3, None, 'reading-recommendations'), - ('For exercise sessions: Why Linear Regression (aka Ordinary ' - 'Least Squares and family), repeat from last week', - 2, - None, - 'for-exercise-sessions-why-linear-regression-aka-ordinary-least-squares-and-family-repeat-from-last-week'), + ('Reminder from last week', 2, None, 'reminder-from-last-week'), ('The equations for ordinary least squares', 2, None, @@ -122,32 +118,6 @@ doconce format html week35.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'testing-the-means-squared-error-as-function-of-complexity'), - ('More preprocessing examples, two-dimensional example, the ' - 'Franke function', - 2, - None, - 'more-preprocessing-examples-two-dimensional-example-the-franke-function'), - ('To think about, first part', - 2, - None, - 'to-think-about-first-part'), - ('More thinking', 2, None, 'more-thinking'), - ('Still thinking', 2, None, 'still-thinking'), - ('What does centering (subtracting the mean values) mean ' - 'mathematically?', - 2, - None, - 'what-does-centering-subtracting-the-mean-values-mean-mathematically'), - ('Further Manipulations', 2, None, 'further-manipulations'), - ('Wrapping it up', 2, None, 'wrapping-it-up'), - ('Linear Regression code, Intercept handling first', - 2, - None, - 'linear-regression-code-intercept-handling-first'), - ('Material for lecture Monday, August 26', - 2, - None, - 'material-for-lecture-monday-august-26'), ('Mathematical Interpretation of Ordinary Least Squares', 2, None, @@ -279,7 +249,7 @@ MathJax.Hub.Config({ @@ -360,32 +321,39 @@ MathJax.Hub.Config({

     

     

     

    -

    Meet the Covariance Matrix

    - -

    Before we move on to a discussion of Ridge and Lasso regression, we want to show an important example of the above.

    - -

    We have already noted that the matrix \( \boldsymbol{X}^T\boldsymbol{X} \) in ordinary -least squares is proportional to the second derivative of the cost -function, that is we have -

    +

    Linking with the SVD

    +

    We saw earlier that

    $$ -\frac{\partial^2 C(\boldsymbol{\beta})}{\partial \boldsymbol{\beta}\partial \boldsymbol{\beta}^T} =\frac{2}{n}\boldsymbol{X}^T\boldsymbol{X}. +\boldsymbol{X}^T\boldsymbol{X}=\boldsymbol{V}\boldsymbol{\Sigma}^T\boldsymbol{U}^T\boldsymbol{U}\boldsymbol{\Sigma}\boldsymbol{V}^T=\boldsymbol{V}\boldsymbol{\Sigma}^T\boldsymbol{\Sigma}\boldsymbol{V}^T. $$ -

    This quantity defines was what is called the Hessian matrix (the second derivative of a function we want to optimize).

    - -

    The Hessian matrix plays an important role and is defined in this course as

    - +

    Since the matrices here have dimension \( p\times p \), with \( p \) corresponding to the singular values, we defined earlier the matrix

    $$ -\boldsymbol{H}=\boldsymbol{X}^T\boldsymbol{X}. +\boldsymbol{\Sigma}^T\boldsymbol{\Sigma} = \begin{bmatrix} \tilde{\boldsymbol{\Sigma}} & \boldsymbol{0}\\ \end{bmatrix}\begin{bmatrix} \tilde{\boldsymbol{\Sigma}} \\ \boldsymbol{0}\\ \end{bmatrix}, +$$ + +

    where the tilde-matrix \( \tilde{\boldsymbol{\Sigma}} \) is a matrix of dimension \( p\times p \) containing only the singular values \( \sigma_i \), that is

    + +$$ +\tilde{\boldsymbol{\Sigma}}=\begin{bmatrix} \sigma_0 & 0 & 0 & \dots & 0 & 0 \\ + 0 & \sigma_1 & 0 & \dots & 0 & 0 \\ + 0 & 0 & \sigma_2 & \dots & 0 & 0 \\ + 0 & 0 & 0 & \dots & \sigma_{p-2} & 0 \\ + 0 & 0 & 0 & \dots & 0 & \sigma_{p-1} \\ +\end{bmatrix}, +$$ + +

    meaning we can write

    +$$ +\boldsymbol{X}^T\boldsymbol{X}=\boldsymbol{V}\tilde{\boldsymbol{\Sigma}}^2\boldsymbol{V}^T. +$$ + +

    Multiplying from the right with \( \boldsymbol{V} \) (using the orthogonality of \( \boldsymbol{V} \)) we get

    +$$ +\left(\boldsymbol{X}^T\boldsymbol{X}\right)\boldsymbol{V}=\boldsymbol{V}\tilde{\boldsymbol{\Sigma}}^2. $$ -

    The Hessian matrix for ordinary least squares is also proportional to -the covariance matrix. This means also that we can use the SVD to find -the eigenvalues of the covariance matrix and the Hessian matrix in -terms of the singular values. Let us develop these arguments, as they will play an important role in our machine learning studies. -

    @@ -409,10 +377,6 @@ terms of the singular values. Let us develop these arguments, as they will pla

  • 61
  • 62
  • 63
    • -
  • 64
    • -
  • 65
    • -
  • ...
    • -
  • 72
  • »
    • diff --git a/doc/pub/week35/html/._week35-bs056.html b/doc/pub/week35/html/._week35-bs056.html index 9a995f789..9702ce2cc 100644 --- a/doc/pub/week35/html/._week35-bs056.html +++ b/doc/pub/week35/html/._week35-bs056.html @@ -38,11 +38,7 @@ doconce format html week35.do.txt --html_style=bootstrap --pygments_html_style=d {'highest level': 2, 'sections': [('Plans for week 35', 2, None, 'plans-for-week-35'), ('Reading recommendations:', 3, None, 'reading-recommendations'), - ('For exercise sessions: Why Linear Regression (aka Ordinary ' - 'Least Squares and family), repeat from last week', - 2, - None, - 'for-exercise-sessions-why-linear-regression-aka-ordinary-least-squares-and-family-repeat-from-last-week'), + ('Reminder from last week', 2, None, 'reminder-from-last-week'), ('The equations for ordinary least squares', 2, None, @@ -122,32 +118,6 @@ doconce format html week35.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'testing-the-means-squared-error-as-function-of-complexity'), - ('More preprocessing examples, two-dimensional example, the ' - 'Franke function', - 2, - None, - 'more-preprocessing-examples-two-dimensional-example-the-franke-function'), - ('To think about, first part', - 2, - None, - 'to-think-about-first-part'), - ('More thinking', 2, None, 'more-thinking'), - ('Still thinking', 2, None, 'still-thinking'), - ('What does centering (subtracting the mean values) mean ' - 'mathematically?', - 2, - None, - 'what-does-centering-subtracting-the-mean-values-mean-mathematically'), - ('Further Manipulations', 2, None, 'further-manipulations'), - ('Wrapping it up', 2, None, 'wrapping-it-up'), - ('Linear Regression code, Intercept handling first', - 2, - None, - 'linear-regression-code-intercept-handling-first'), - ('Material for lecture Monday, August 26', - 2, - None, - 'material-for-lecture-monday-august-26'), ('Mathematical Interpretation of Ordinary Least Squares', 2, None, @@ -279,7 +249,7 @@ MathJax.Hub.Config({ @@ -360,46 +321,44 @@ MathJax.Hub.Config({

     

     

     

    -

    Introducing the Covariance and Correlation functions

    +

    What does it mean?

    -

    Before we discuss the link between for example Ridge regression and the singular value decomposition, we need to remind ourselves about -the definition of the covariance and the correlation function. These are quantities that play a central role in machine learning methods. +

    This means the vectors \( \boldsymbol{v}_i \) of the orthogonal matrix \( \boldsymbol{V} \) +are the eigenvectors of the matrix \( \boldsymbol{X}^T\boldsymbol{X} \) with eigenvalues +given by the singular values squared, that is

    -

    Suppose we have defined two vectors -\( \hat{x} \) and \( \hat{y} \) with \( n \) elements each. The covariance matrix \( \boldsymbol{C} \) is defined as +$$ +\left(\boldsymbol{X}^T\boldsymbol{X}\right)\boldsymbol{v}_i=\boldsymbol{v}_i\sigma_i^2. +$$ + +

    In other words, each non-zero singular value of \( \boldsymbol{X} \) is a positive +square root of an eigenvalue of \( \boldsymbol{X}^T\boldsymbol{X} \). It means also that +the columns of \( \boldsymbol{V} \) are the eigenvectors of +\( \boldsymbol{X}^T\boldsymbol{X} \). Since we have ordered the singular values of +\( \boldsymbol{X} \) in a descending order, it means that the column vectors +\( \boldsymbol{v}_i \) are hierarchically ordered by how much correlation they +encode from the columns of \( \boldsymbol{X} \).

    + +

    Note that these are also the eigenvectors and eigenvalues of the +Hessian matrix. Note also that the Hessian matrix we are discussing here is from a cost function defined by the mean squared error only. +

    + +

    If we now recall the definition of the covariance matrix (not using +Bessel's correction) we have +

    + $$ -\boldsymbol{C}[\boldsymbol{x},\boldsymbol{y}] = \begin{bmatrix} \mathrm{cov}[\boldsymbol{x},\boldsymbol{x}] & \mathrm{cov}[\boldsymbol{x},\boldsymbol{y}] \\ - \mathrm{cov}[\boldsymbol{y},\boldsymbol{x}] & \mathrm{cov}[\boldsymbol{y},\boldsymbol{y}] \\ - \end{bmatrix}, +\boldsymbol{C}[\boldsymbol{X}]=\frac{1}{n}\boldsymbol{X}^T\boldsymbol{X}, $$ -

    where for example

    -$$ -\mathrm{cov}[\boldsymbol{x},\boldsymbol{y}] =\frac{1}{n} \sum_{i=0}^{n-1}(x_i- \overline{x})(y_i- \overline{y}). -$$ - -

    With this definition and recalling that the variance is defined as

    -$$ -\mathrm{var}[\boldsymbol{x}]=\frac{1}{n} \sum_{i=0}^{n-1}(x_i- \overline{x})^2, -$$ - -

    we can rewrite the covariance matrix as

    -$$ -\boldsymbol{C}[\boldsymbol{x},\boldsymbol{y}] = \begin{bmatrix} \mathrm{var}[\boldsymbol{x}] & \mathrm{cov}[\boldsymbol{x},\boldsymbol{y}] \\ - \mathrm{cov}[\boldsymbol{x},\boldsymbol{y}] & \mathrm{var}[\boldsymbol{y}] \\ - \end{bmatrix}. -$$ - -

    Note: we have used \( 1/n \) in the above definitions of the sample variance and covariance. We assume then that we can calculate the exact mean value. -What you will find in essentially all statistics texts are equations -with a factor \( 1/(n-1) \). This is called Bessel's correction. This -method corrects the bias in the estimation of the population variance -and covariance. It also partially corrects the bias in the estimation -of the population standard deviation. If you use a library like -Scikit-Learn or nunmpy's function to calculate the covariance, this -quantity will be computed with a factor \( 1/(n-1) \). +

    meaning that every squared non-singular value of \( \boldsymbol{X} \) divided by \( n \) ( +the number of samples) are the eigenvalues of the covariance +matrix. Every singular value of \( \boldsymbol{X} \) is thus a positive square +root of an eigenvalue of \( \boldsymbol{X}^T\boldsymbol{X} \). If the matrix \( \boldsymbol{X} \) is +self-adjoint, the singular values of \( \boldsymbol{X} \) are equal to the +absolute value of the eigenvalues of \( \boldsymbol{X} \).

    @@ -423,11 +382,6 @@ quantity will be computed with a factor \( 1/(n-1) \).

  • 61
  • 62
  • 63
    • -
  • 64
    • -
  • 65
    • -
  • 66
    • -
  • ...
    • -
  • 72
  • »
    • diff --git a/doc/pub/week35/html/._week35-bs057.html b/doc/pub/week35/html/._week35-bs057.html index aa0b78f33..c33d870d6 100644 --- a/doc/pub/week35/html/._week35-bs057.html +++ b/doc/pub/week35/html/._week35-bs057.html @@ -38,11 +38,7 @@ doconce format html week35.do.txt --html_style=bootstrap --pygments_html_style=d {'highest level': 2, 'sections': [('Plans for week 35', 2, None, 'plans-for-week-35'), ('Reading recommendations:', 3, None, 'reading-recommendations'), - ('For exercise sessions: Why Linear Regression (aka Ordinary ' - 'Least Squares and family), repeat from last week', - 2, - None, - 'for-exercise-sessions-why-linear-regression-aka-ordinary-least-squares-and-family-repeat-from-last-week'), + ('Reminder from last week', 2, None, 'reminder-from-last-week'), ('The equations for ordinary least squares', 2, None, @@ -122,32 +118,6 @@ doconce format html week35.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'testing-the-means-squared-error-as-function-of-complexity'), - ('More preprocessing examples, two-dimensional example, the ' - 'Franke function', - 2, - None, - 'more-preprocessing-examples-two-dimensional-example-the-franke-function'), - ('To think about, first part', - 2, - None, - 'to-think-about-first-part'), - ('More thinking', 2, None, 'more-thinking'), - ('Still thinking', 2, None, 'still-thinking'), - ('What does centering (subtracting the mean values) mean ' - 'mathematically?', - 2, - None, - 'what-does-centering-subtracting-the-mean-values-mean-mathematically'), - ('Further Manipulations', 2, None, 'further-manipulations'), - ('Wrapping it up', 2, None, 'wrapping-it-up'), - ('Linear Regression code, Intercept handling first', - 2, - None, - 'linear-regression-code-intercept-handling-first'), - ('Material for lecture Monday, August 26', - 2, - None, - 'material-for-lecture-monday-august-26'), ('Mathematical Interpretation of Ordinary Least Squares', 2, None, @@ -279,7 +249,7 @@ MathJax.Hub.Config({ @@ -360,33 +321,40 @@ MathJax.Hub.Config({

     

     

     

    -

    Covariance and Correlation Matrix

    +

    And finally \( \boldsymbol{X}\boldsymbol{X}^T \)

    -

    The covariance takes values between zero and infinity and may thus -lead to problems with loss of numerical precision for particularly -large values. It is common to scale the covariance matrix by -introducing instead the correlation matrix defined via the so-called -correlation function +

    For \( \boldsymbol{X}\boldsymbol{X}^T \) we found

    + +$$ +\boldsymbol{X}\boldsymbol{X}^T=\boldsymbol{U}\boldsymbol{\Sigma}\boldsymbol{V}^T\boldsymbol{V}\boldsymbol{\Sigma}^T\boldsymbol{U}^T=\boldsymbol{U}\boldsymbol{\Sigma}^T\boldsymbol{\Sigma}\boldsymbol{U}^T. +$$ + +

    Since the matrices here have dimension \( n\times n \), we have

    +$$ +\boldsymbol{\Sigma}\boldsymbol{\Sigma}^T = \begin{bmatrix} \tilde{\boldsymbol{\Sigma}} \\ \boldsymbol{0}\\ \end{bmatrix}\begin{bmatrix} \tilde{\boldsymbol{\Sigma}} \boldsymbol{0}\\ \end{bmatrix}=\begin{bmatrix} \tilde{\boldsymbol{\Sigma}} & \boldsymbol{0} \\ \boldsymbol{0} & \boldsymbol{0}\\ \end{bmatrix}, +$$ + +

    leading to

    +$$ +\boldsymbol{X}\boldsymbol{X}^T=\boldsymbol{U}\begin{bmatrix} \tilde{\boldsymbol{\Sigma}} & \boldsymbol{0} \\ \boldsymbol{0} & \boldsymbol{0}\\ \end{bmatrix}\boldsymbol{U}^T. +$$ + +

    Multiplying with \( \boldsymbol{U} \) from the right gives us the eigenvalue problem

    +$$ +(\boldsymbol{X}\boldsymbol{X}^T)\boldsymbol{U}=\boldsymbol{U}\begin{bmatrix} \tilde{\boldsymbol{\Sigma}} & \boldsymbol{0} \\ \boldsymbol{0} & \boldsymbol{0}\\ \end{bmatrix}. +$$ + +

    It means that the eigenvalues of \( \boldsymbol{X}\boldsymbol{X}^T \) are again given by +the non-zero singular values plus now a series of zeros. The column +vectors of \( \boldsymbol{U} \) are the eigenvectors of \( \boldsymbol{X}\boldsymbol{X}^T \) and +measure how much correlations are contained in the rows of \( \boldsymbol{X} \).

    -$$ -\mathrm{corr}[\boldsymbol{x},\boldsymbol{y}]=\frac{\mathrm{cov}[\boldsymbol{x},\boldsymbol{y}]}{\sqrt{\mathrm{var}[\boldsymbol{x}] \mathrm{var}[\boldsymbol{y}]}}. -$$ - -

    The correlation function is then given by values \( \mathrm{corr}[\boldsymbol{x},\boldsymbol{y}] -\in [-1,1] \). This avoids eventual problems with too large values. We -can then define the correlation matrix for the two vectors \( \boldsymbol{x} \) -and \( \boldsymbol{y} \) as +

    Since we will mainly be interested in the correlations among the features +of our data (the columns of \( \boldsymbol{X} \), the quantity of interest for us are the non-zero singular +values and the column vectors of \( \boldsymbol{V} \).

    -$$ -\boldsymbol{K}[\boldsymbol{x},\boldsymbol{y}] = \begin{bmatrix} 1 & \mathrm{corr}[\boldsymbol{x},\boldsymbol{y}] \\ - \mathrm{corr}[\boldsymbol{y},\boldsymbol{x}] & 1 \\ - \end{bmatrix}, -$$ - -

    In the above example this is the function we constructed using pandas.

    -

    diff --git a/doc/pub/week35/html/._week35-bs058.html b/doc/pub/week35/html/._week35-bs058.html index 226a56a10..1db5e819c 100644 --- a/doc/pub/week35/html/._week35-bs058.html +++ b/doc/pub/week35/html/._week35-bs058.html @@ -38,11 +38,7 @@ doconce format html week35.do.txt --html_style=bootstrap --pygments_html_style=d {'highest level': 2, 'sections': [('Plans for week 35', 2, None, 'plans-for-week-35'), ('Reading recommendations:', 3, None, 'reading-recommendations'), - ('For exercise sessions: Why Linear Regression (aka Ordinary ' - 'Least Squares and family), repeat from last week', - 2, - None, - 'for-exercise-sessions-why-linear-regression-aka-ordinary-least-squares-and-family-repeat-from-last-week'), + ('Reminder from last week', 2, None, 'reminder-from-last-week'), ('The equations for ordinary least squares', 2, None, @@ -122,32 +118,6 @@ doconce format html week35.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'testing-the-means-squared-error-as-function-of-complexity'), - ('More preprocessing examples, two-dimensional example, the ' - 'Franke function', - 2, - None, - 'more-preprocessing-examples-two-dimensional-example-the-franke-function'), - ('To think about, first part', - 2, - None, - 'to-think-about-first-part'), - ('More thinking', 2, None, 'more-thinking'), - ('Still thinking', 2, None, 'still-thinking'), - ('What does centering (subtracting the mean values) mean ' - 'mathematically?', - 2, - None, - 'what-does-centering-subtracting-the-mean-values-mean-mathematically'), - ('Further Manipulations', 2, None, 'further-manipulations'), - ('Wrapping it up', 2, None, 'wrapping-it-up'), - ('Linear Regression code, Intercept handling first', - 2, - None, - 'linear-regression-code-intercept-handling-first'), - ('Material for lecture Monday, August 26', - 2, - None, - 'material-for-lecture-monday-august-26'), ('Mathematical Interpretation of Ordinary Least Squares', 2, None, @@ -279,7 +249,7 @@ MathJax.Hub.Config({ @@ -360,63 +321,57 @@ MathJax.Hub.Config({

     

     

     

    -

    Correlation Function and Design/Feature Matrix

    +

    Ridge and LASSO Regression

    -

    In our derivation of the various regression algorithms like Ordinary Least Squares or Ridge regression -we defined the design/feature matrix \( \boldsymbol{X} \) as +

    Let us remind ourselves about the expression for the standard Mean Squared Error (MSE) which we used to define our cost function and the equations for the ordinary least squares (OLS) method, that is +our optimization problem is +

    +$$ +{\displaystyle \min_{\boldsymbol{\beta}\in {\mathbb{R}}^{p}}}\frac{1}{n}\left\{\left(\boldsymbol{y}-\boldsymbol{X}\boldsymbol{\beta}\right)^T\left(\boldsymbol{y}-\boldsymbol{X}\boldsymbol{\beta}\right)\right\}. +$$ + +

    or we can state it as

    +$$ +{\displaystyle \min_{\boldsymbol{\beta}\in +{\mathbb{R}}^{p}}}\frac{1}{n}\sum_{i=0}^{n-1}\left(y_i-\tilde{y}_i\right)^2=\frac{1}{n}\vert\vert \boldsymbol{y}-\boldsymbol{X}\boldsymbol{\beta}\vert\vert_2^2, +$$ + +

    where we have used the definition of a norm-2 vector, that is

    +$$ +\vert\vert \boldsymbol{x}\vert\vert_2 = \sqrt{\sum_i x_i^2}. +$$ + +

    By minimizing the above equation with respect to the parameters +\( \boldsymbol{\beta} \) we could then obtain an analytical expression for the +parameters \( \boldsymbol{\beta} \). We can add a regularization parameter \( \lambda \) by +defining a new cost function to be optimized, that is

    $$ -\boldsymbol{X}=\begin{bmatrix} -x_{0,0} & x_{0,1} & x_{0,2}& \dots & \dots x_{0,p-1}\\ -x_{1,0} & x_{1,1} & x_{1,2}& \dots & \dots x_{1,p-1}\\ -x_{2,0} & x_{2,1} & x_{2,2}& \dots & \dots x_{2,p-1}\\ -\dots & \dots & \dots & \dots \dots & \dots \\ -x_{n-2,0} & x_{n-2,1} & x_{n-2,2}& \dots & \dots x_{n-2,p-1}\\ -x_{n-1,0} & x_{n-1,1} & x_{n-1,2}& \dots & \dots x_{n-1,p-1}\\ -\end{bmatrix}, +{\displaystyle \min_{\boldsymbol{\beta}\in +{\mathbb{R}}^{p}}}\frac{1}{n}\vert\vert \boldsymbol{y}-\boldsymbol{X}\boldsymbol{\beta}\vert\vert_2^2+\lambda\vert\vert \boldsymbol{\beta}\vert\vert_2^2 $$ -

    with \( \boldsymbol{X}\in {\mathbb{R}}^{n\times p} \), with the predictors/features \( p \) refering to the column numbers and the -entries \( n \) being the row elements. -We can rewrite the design/feature matrix in terms of its column vectors as -

    -$$ -\boldsymbol{X}=\begin{bmatrix} \boldsymbol{x}_0 & \boldsymbol{x}_1 & \boldsymbol{x}_2 & \dots & \dots & \boldsymbol{x}_{p-1}\end{bmatrix}, -$$ - -

    with a given vector

    -$$ -\boldsymbol{x}_i^T = \begin{bmatrix}x_{0,i} & x_{1,i} & x_{2,i}& \dots & \dots x_{n-1,i}\end{bmatrix}. -$$ - -

    With these definitions, we can now rewrite our \( 2\times 2 \) -correlation/covariance matrix in terms of a moe general design/feature -matrix \( \boldsymbol{X}\in {\mathbb{R}}^{n\times p} \). This leads to a \( p\times p \) -covariance matrix for the vectors \( \boldsymbol{x}_i \) with \( i=0,1,\dots,p-1 \) +

    which leads to the Ridge regression minimization problem where we +require that \( \vert\vert \boldsymbol{\beta}\vert\vert_2^2\le t \), where \( t \) is +a finite number larger than zero. By defining

    $$ -\boldsymbol{C}[\boldsymbol{x}] = \begin{bmatrix} -\mathrm{var}[\boldsymbol{x}_0] & \mathrm{cov}[\boldsymbol{x}_0,\boldsymbol{x}_1] & \mathrm{cov}[\boldsymbol{x}_0,\boldsymbol{x}_2] & \dots & \dots & \mathrm{cov}[\boldsymbol{x}_0,\boldsymbol{x}_{p-1}]\\ -\mathrm{cov}[\boldsymbol{x}_1,\boldsymbol{x}_0] & \mathrm{var}[\boldsymbol{x}_1] & \mathrm{cov}[\boldsymbol{x}_1,\boldsymbol{x}_2] & \dots & \dots & \mathrm{cov}[\boldsymbol{x}_1,\boldsymbol{x}_{p-1}]\\ -\mathrm{cov}[\boldsymbol{x}_2,\boldsymbol{x}_0] & \mathrm{cov}[\boldsymbol{x}_2,\boldsymbol{x}_1] & \mathrm{var}[\boldsymbol{x}_2] & \dots & \dots & \mathrm{cov}[\boldsymbol{x}_2,\boldsymbol{x}_{p-1}]\\ -\dots & \dots & \dots & \dots & \dots & \dots \\ -\dots & \dots & \dots & \dots & \dots & \dots \\ -\mathrm{cov}[\boldsymbol{x}_{p-1},\boldsymbol{x}_0] & \mathrm{cov}[\boldsymbol{x}_{p-1},\boldsymbol{x}_1] & \mathrm{cov}[\boldsymbol{x}_{p-1},\boldsymbol{x}_{2}] & \dots & \dots & \mathrm{var}[\boldsymbol{x}_{p-1}]\\ -\end{bmatrix}, +C(\boldsymbol{X},\boldsymbol{\beta})=\frac{1}{n}\vert\vert \boldsymbol{y}-\boldsymbol{X}\boldsymbol{\beta}\vert\vert_2^2+\lambda\vert\vert \boldsymbol{\beta}\vert\vert_1, $$ -

    and the correlation matrix

    +

    we have a new optimization equation

    $$ -\boldsymbol{K}[\boldsymbol{x}] = \begin{bmatrix} -1 & \mathrm{corr}[\boldsymbol{x}_0,\boldsymbol{x}_1] & \mathrm{corr}[\boldsymbol{x}_0,\boldsymbol{x}_2] & \dots & \dots & \mathrm{corr}[\boldsymbol{x}_0,\boldsymbol{x}_{p-1}]\\ -\mathrm{corr}[\boldsymbol{x}_1,\boldsymbol{x}_0] & 1 & \mathrm{corr}[\boldsymbol{x}_1,\boldsymbol{x}_2] & \dots & \dots & \mathrm{corr}[\boldsymbol{x}_1,\boldsymbol{x}_{p-1}]\\ -\mathrm{corr}[\boldsymbol{x}_2,\boldsymbol{x}_0] & \mathrm{corr}[\boldsymbol{x}_2,\boldsymbol{x}_1] & 1 & \dots & \dots & \mathrm{corr}[\boldsymbol{x}_2,\boldsymbol{x}_{p-1}]\\ -\dots & \dots & \dots & \dots & \dots & \dots \\ -\dots & \dots & \dots & \dots & \dots & \dots \\ -\mathrm{corr}[\boldsymbol{x}_{p-1},\boldsymbol{x}_0] & \mathrm{corr}[\boldsymbol{x}_{p-1},\boldsymbol{x}_1] & \mathrm{corr}[\boldsymbol{x}_{p-1},\boldsymbol{x}_{2}] & \dots & \dots & 1\\ -\end{bmatrix}, +{\displaystyle \min_{\boldsymbol{\beta}\in +{\mathbb{R}}^{p}}}\frac{1}{n}\vert\vert \boldsymbol{y}-\boldsymbol{X}\boldsymbol{\beta}\vert\vert_2^2+\lambda\vert\vert \boldsymbol{\beta}\vert\vert_1 +$$ + +

    which leads to Lasso regression. Lasso stands for least absolute shrinkage and selection operator.

    + +

    Here we have defined the norm-1 as

    +$$ +\vert\vert \boldsymbol{x}\vert\vert_1 = \sum_i \vert x_i\vert. $$ @@ -439,13 +394,6 @@ $$
  • 61
  • 62
  • 63
    • -
  • 64
    • -
  • 65
    • -
  • 66
    • -
  • 67
    • -
  • 68
    • -
  • ...
    • -
  • 72
  • »
    • diff --git a/doc/pub/week35/html/._week35-bs059.html b/doc/pub/week35/html/._week35-bs059.html index c471e93ce..02cdd228c 100644 --- a/doc/pub/week35/html/._week35-bs059.html +++ b/doc/pub/week35/html/._week35-bs059.html @@ -38,11 +38,7 @@ doconce format html week35.do.txt --html_style=bootstrap --pygments_html_style=d {'highest level': 2, 'sections': [('Plans for week 35', 2, None, 'plans-for-week-35'), ('Reading recommendations:', 3, None, 'reading-recommendations'), - ('For exercise sessions: Why Linear Regression (aka Ordinary ' - 'Least Squares and family), repeat from last week', - 2, - None, - 'for-exercise-sessions-why-linear-regression-aka-ordinary-least-squares-and-family-repeat-from-last-week'), + ('Reminder from last week', 2, None, 'reminder-from-last-week'), ('The equations for ordinary least squares', 2, None, @@ -122,32 +118,6 @@ doconce format html week35.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'testing-the-means-squared-error-as-function-of-complexity'), - ('More preprocessing examples, two-dimensional example, the ' - 'Franke function', - 2, - None, - 'more-preprocessing-examples-two-dimensional-example-the-franke-function'), - ('To think about, first part', - 2, - None, - 'to-think-about-first-part'), - ('More thinking', 2, None, 'more-thinking'), - ('Still thinking', 2, None, 'still-thinking'), - ('What does centering (subtracting the mean values) mean ' - 'mathematically?', - 2, - None, - 'what-does-centering-subtracting-the-mean-values-mean-mathematically'), - ('Further Manipulations', 2, None, 'further-manipulations'), - ('Wrapping it up', 2, None, 'wrapping-it-up'), - ('Linear Regression code, Intercept handling first', - 2, - None, - 'linear-regression-code-intercept-handling-first'), - ('Material for lecture Monday, August 26', - 2, - None, - 'material-for-lecture-monday-august-26'), ('Mathematical Interpretation of Ordinary Least Squares', 2, None, @@ -279,7 +249,7 @@ MathJax.Hub.Config({ @@ -360,61 +321,67 @@ MathJax.Hub.Config({

     

     

     

    -

    Covariance Matrix Examples

    +

    Deriving the Ridge Regression Equations

    -

    The Numpy function np.cov calculates the covariance elements using -the factor \( 1/(n-1) \) instead of \( 1/n \) since it assumes we do not have -the exact mean values. The following simple function uses the -np.vstack function which takes each vector of dimension \( 1\times n \) -and produces a \( 2\times n \) matrix \( \boldsymbol{W} \) -

    +

    Using the matrix-vector expression for Ridge regression and dropping the parameter \( 1/n \) in front of the standard means squared error equation, we have

    -

    Note that this assumes you have the features as the rows, and the inputs as columns, that is

    $$ -\boldsymbol{W} = \begin{bmatrix} x_0 & x_1 & x_2 & \dots & x_{n-2} & x_{n-1} \\ - y_0 & y_1 & y_2 & \dots & y_{n-2} & y_{n-1} \\ - \end{bmatrix}, +C(\boldsymbol{X},\boldsymbol{\beta})=\left\{(\boldsymbol{y}-\boldsymbol{X}\boldsymbol{\beta})^T(\boldsymbol{y}-\boldsymbol{X}\boldsymbol{\beta})\right\}+\lambda\boldsymbol{\beta}^T\boldsymbol{\beta}, $$ -

    which in turn is converted into into the \( 2\times 2 \) covariance matrix -\( \boldsymbol{C} \) via the Numpy function np.cov(). We note that we can also calculate -the mean value of each set of samples \( \boldsymbol{x} \) etc using the Numpy -function np.mean(x). We can also extract the eigenvalues of the -covariance matrix through the np.linalg.eig() function. +

    and +taking the derivatives with respect to \( \boldsymbol{\beta} \) we obtain then +a slightly modified matrix inversion problem which for finite values +of \( \lambda \) does not suffer from singularity problems. We obtain +the optimal parameters +

    +$$ +\hat{\boldsymbol{\beta}}_{\mathrm{Ridge}} = \left(\boldsymbol{X}^T\boldsymbol{X}+\lambda\boldsymbol{I}\right)^{-1}\boldsymbol{X}^T\boldsymbol{y}, +$$ + +

    with \( \boldsymbol{I} \) being a \( p\times p \) identity matrix with the constraint that

    + +$$ +\sum_{i=0}^{p-1} \beta_i^2 \leq t, +$$ + +

    with \( t \) a finite positive number.

    + +

    If we keep the \( 1/n \) factor, the equation for the optimal \( \beta \) changes to

    +$$ +\hat{\boldsymbol{\beta}}_{\mathrm{Ridge}} = \left(\boldsymbol{X}^T\boldsymbol{X}+n\lambda\boldsymbol{I}\right)^{-1}\boldsymbol{X}^T\boldsymbol{y}. +$$ + +

    In many textbooks the \( 1/n \) term is often omitted. Note that a library like Scikit-Learn does not include the \( 1/n \) factor in the setup of the cost function.

    + +

    When we compare this with the ordinary least squares result we have

    +$$ +\hat{\boldsymbol{\beta}}_{\mathrm{OLS}} = \left(\boldsymbol{X}^T\boldsymbol{X}\right)^{-1}\boldsymbol{X}^T\boldsymbol{y}, +$$ + +

    which can lead to singular matrices. However, with the SVD, we can always compute the inverse of the matrix \( \boldsymbol{X}^T\boldsymbol{X} \).

    + +

    We see that Ridge regression is nothing but the standard OLS with a +modified diagonal term added to \( \boldsymbol{X}^T\boldsymbol{X} \). The consequences, in +particular for our discussion of the bias-variance tradeoff are rather +interesting. We will see that for specific values of \( \lambda \), we may +even reduce the variance of the optimal parameters \( \boldsymbol{\beta} \). These topics and other related ones, will be discussed after the more linear algebra oriented analysis here.

    +

    Using our insights about the SVD of the design matrix \( \boldsymbol{X} \) +We have already analyzed the OLS solutions in terms of the eigenvectors (the columns) of the right singular value matrix \( \boldsymbol{U} \) as +

    +$$ +\tilde{\boldsymbol{y}}_{\mathrm{OLS}}=\boldsymbol{X}\boldsymbol{\beta} =\boldsymbol{U}\boldsymbol{U}^T\boldsymbol{y}. +$$ - -
    -
    -
    -
    -
    - 
    # Importing various packages
-import numpy as np
-n = 100
-x = np.random.normal(size=n)
-print(np.mean(x))
-y = 4+3*x+np.random.normal(size=n)
-print(np.mean(y))
-W = np.vstack((x, y))
-C = np.cov(W)
-print(C)
-
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    +

    For Ridge regression this becomes

    +$$ +\tilde{\boldsymbol{y}}_{\mathrm{Ridge}}=\boldsymbol{X}\boldsymbol{\beta}_{\mathrm{Ridge}} = \boldsymbol{U\Sigma V^T}\left(\boldsymbol{V}\boldsymbol{\Sigma}^2\boldsymbol{V}^T+\lambda\boldsymbol{I} \right)^{-1}(\boldsymbol{U\Sigma V^T})^T\boldsymbol{y}=\sum_{j=0}^{p-1}\boldsymbol{u}_j\boldsymbol{u}_j^T\frac{\sigma_j^2}{\sigma_j^2+\lambda}\boldsymbol{y}, +$$ + +

    with the vectors \( \boldsymbol{u}_j \) being the columns of \( \boldsymbol{U} \) from the SVD of the matrix \( \boldsymbol{X} \).

    @@ -434,14 +401,6 @@ C = np.c

  • 61
  • 62
  • 63
    • -
  • 64
    • -
  • 65
    • -
  • 66
    • -
  • 67
    • -
  • 68
    • -
  • 69
    • -
  • ...
    • -
  • 72
  • »
    • diff --git a/doc/pub/week35/html/._week35-bs060.html b/doc/pub/week35/html/._week35-bs060.html index 588934e48..3002a4c79 100644 --- a/doc/pub/week35/html/._week35-bs060.html +++ b/doc/pub/week35/html/._week35-bs060.html @@ -38,11 +38,7 @@ doconce format html week35.do.txt --html_style=bootstrap --pygments_html_style=d {'highest level': 2, 'sections': [('Plans for week 35', 2, None, 'plans-for-week-35'), ('Reading recommendations:', 3, None, 'reading-recommendations'), - ('For exercise sessions: Why Linear Regression (aka Ordinary ' - 'Least Squares and family), repeat from last week', - 2, - None, - 'for-exercise-sessions-why-linear-regression-aka-ordinary-least-squares-and-family-repeat-from-last-week'), + ('Reminder from last week', 2, None, 'reminder-from-last-week'), ('The equations for ordinary least squares', 2, None, @@ -122,32 +118,6 @@ doconce format html week35.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'testing-the-means-squared-error-as-function-of-complexity'), - ('More preprocessing examples, two-dimensional example, the ' - 'Franke function', - 2, - None, - 'more-preprocessing-examples-two-dimensional-example-the-franke-function'), - ('To think about, first part', - 2, - None, - 'to-think-about-first-part'), - ('More thinking', 2, None, 'more-thinking'), - ('Still thinking', 2, None, 'still-thinking'), - ('What does centering (subtracting the mean values) mean ' - 'mathematically?', - 2, - None, - 'what-does-centering-subtracting-the-mean-values-mean-mathematically'), - ('Further Manipulations', 2, None, 'further-manipulations'), - ('Wrapping it up', 2, None, 'wrapping-it-up'), - ('Linear Regression code, Intercept handling first', - 2, - None, - 'linear-regression-code-intercept-handling-first'), - ('Material for lecture Monday, August 26', - 2, - None, - 'material-for-lecture-monday-august-26'), ('Mathematical Interpretation of Ordinary Least Squares', 2, None, @@ -279,7 +249,7 @@ MathJax.Hub.Config({ @@ -360,64 +321,22 @@ MathJax.Hub.Config({

     

     

     

    -

    Correlation Matrix

    +

    Interpreting the Ridge results

    -

    The previous example can be converted into the correlation matrix by -simply scaling the matrix elements with the variances. We should also -subtract the mean values for each column. This leads to the following -code which sets up the correlations matrix for the previous example in -a more brute force way. Here we scale the mean values for each column of the design matrix, calculate the relevant mean values and variances and then finally set up the \( 2\times 2 \) correlation matrix (since we have only two vectors). +

    Since \( \lambda \geq 0 \), it means that compared to OLS, we have

    + +$$ +\frac{\sigma_j^2}{\sigma_j^2+\lambda} \leq 1. +$$ + +

    Ridge regression finds the coordinates of \( \boldsymbol{y} \) with respect to the +orthonormal basis \( \boldsymbol{U} \), it then shrinks the coordinates by +\( \frac{\sigma_j^2}{\sigma_j^2+\lambda} \). Recall that the SVD has +eigenvalues ordered in a descending way, that is \( \sigma_i \geq +\sigma_{i+1} \).

    - - -
    -
    -
    -
    -
    - 
    import numpy as np
-n = 100
-# define two vectors                                                                                           
-x = np.random.random(size=n)
-y = 4+3*x+np.random.normal(size=n)
-#scaling the x and y vectors                                                                                   
-x = x - np.mean(x)
-y = y - np.mean(y)
-variance_x = np.sum(x@x)/n
-variance_y = np.sum(y@y)/n
-print(variance_x)
-print(variance_y)
-cov_xy = np.sum(x@y)/n
-cov_xx = np.sum(x@x)/n
-cov_yy = np.sum(y@y)/n
-C = np.zeros((2,2))
-C[0,0]= cov_xx/variance_x
-C[1,1]= cov_yy/variance_y
-C[0,1]= cov_xy/np.sqrt(variance_y*variance_x)
-C[1,0]= C[0,1]
-print(C)
-
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    - -

    We see that the matrix elements along the diagonal are one as they -should be and that the matrix is symmetric. Furthermore, diagonalizing -this matrix we easily see that it is a positive definite matrix. -

    - -

    The above procedure with numpy can be made more compact if we use pandas.

    +

    For small eigenvalues \( \sigma_i \) it means that their contributions become less important, a fact which can be used to reduce the number of degrees of freedom. More about this when we have covered the material on a statistical interpretation of various linear regression methods.

    @@ -436,15 +355,6 @@ this matrix we easily see that it is a positive definite matrix.

  • 61
  • 62
  • 63
    • -
  • 64
    • -
  • 65
    • -
  • 66
    • -
  • 67
    • -
  • 68
    • -
  • 69
    • -
  • 70
    • -
  • ...
    • -
  • 72
  • »
    • diff --git a/doc/pub/week35/html/._week35-bs061.html b/doc/pub/week35/html/._week35-bs061.html index 1cdf88f6d..d1fb26f51 100644 --- a/doc/pub/week35/html/._week35-bs061.html +++ b/doc/pub/week35/html/._week35-bs061.html @@ -38,11 +38,7 @@ doconce format html week35.do.txt --html_style=bootstrap --pygments_html_style=d {'highest level': 2, 'sections': [('Plans for week 35', 2, None, 'plans-for-week-35'), ('Reading recommendations:', 3, None, 'reading-recommendations'), - ('For exercise sessions: Why Linear Regression (aka Ordinary ' - 'Least Squares and family), repeat from last week', - 2, - None, - 'for-exercise-sessions-why-linear-regression-aka-ordinary-least-squares-and-family-repeat-from-last-week'), + ('Reminder from last week', 2, None, 'reminder-from-last-week'), ('The equations for ordinary least squares', 2, None, @@ -122,32 +118,6 @@ doconce format html week35.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'testing-the-means-squared-error-as-function-of-complexity'), - ('More preprocessing examples, two-dimensional example, the ' - 'Franke function', - 2, - None, - 'more-preprocessing-examples-two-dimensional-example-the-franke-function'), - ('To think about, first part', - 2, - None, - 'to-think-about-first-part'), - ('More thinking', 2, None, 'more-thinking'), - ('Still thinking', 2, None, 'still-thinking'), - ('What does centering (subtracting the mean values) mean ' - 'mathematically?', - 2, - None, - 'what-does-centering-subtracting-the-mean-values-mean-mathematically'), - ('Further Manipulations', 2, None, 'further-manipulations'), - ('Wrapping it up', 2, None, 'wrapping-it-up'), - ('Linear Regression code, Intercept handling first', - 2, - None, - 'linear-regression-code-intercept-handling-first'), - ('Material for lecture Monday, August 26', - 2, - None, - 'material-for-lecture-monday-august-26'), ('Mathematical Interpretation of Ordinary Least Squares', 2, None, @@ -279,7 +249,7 @@ MathJax.Hub.Config({ @@ -360,46 +321,35 @@ MathJax.Hub.Config({

     

     

     

    -

    Correlation Matrix with Pandas

    +

    More interpretations

    -

    We whow here how we can set up the correlation matrix using pandas, as done in this simple code

    +

    For the sake of simplicity, let us assume that the design matrix is orthonormal, that is

    - -
    -
    -
    -
    -
    - 
    import numpy as np
-import pandas as pd
-n = 10
-x = np.random.normal(size=n)
-x = x - np.mean(x)
-y = 4+3*x+np.random.normal(size=n)
-y = y - np.mean(y)
-# Note that we transpose the matrix in order to stay with our ordering n x p
-X = (np.vstack((x, y))).T
-print(X)
-Xpd = pd.DataFrame(X)
-print(Xpd)
-correlation_matrix = Xpd.corr()
-print(correlation_matrix)
-
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    +$$ +\boldsymbol{X}^T\boldsymbol{X}=(\boldsymbol{X}^T\boldsymbol{X})^{-1} =\boldsymbol{I}. +$$ -

    We expand this model to the Franke function discussed above.

    +

    In this case the standard OLS results in

    +$$ +\boldsymbol{\beta}^{\mathrm{OLS}} = \boldsymbol{X}^T\boldsymbol{y}=\sum_{i=0}^{n-1}\boldsymbol{u}_i\boldsymbol{u}_i^T\boldsymbol{y}, +$$ + +

    and

    + +$$ +\boldsymbol{\beta}^{\mathrm{Ridge}} = \left(\boldsymbol{I}+\lambda\boldsymbol{I}\right)^{-1}\boldsymbol{X}^T\boldsymbol{y}=\left(1+\lambda\right)^{-1}\boldsymbol{\beta}^{\mathrm{OLS}}, +$$ + +

    that is the Ridge estimator scales the OLS estimator by the inverse of a factor \( 1+\lambda \), and +the Ridge estimator converges to zero when the hyperparameter goes to +infinity. +

    + +

    We will come back to more interpreations after we have gone through some of the statistical analysis part.

    + +

    For more discussions of Ridge and Lasso regression, Wessel van Wieringen's article is highly recommended. +Similarly, Mehta et al's article is also recommended. +

    @@ -417,16 +367,6 @@ correlation_matrix = Xpd61

  • 62
  • 63
    • -
  • 64
    • -
  • 65
    • -
  • 66
    • -
  • 67
    • -
  • 68
    • -
  • 69
    • -
  • 70
    • -
  • 71
    • -
  • ...
    • -
  • 72
  • »
    • diff --git a/doc/pub/week35/html/._week35-bs062.html b/doc/pub/week35/html/._week35-bs062.html index be0d99b5c..318c1563e 100644 --- a/doc/pub/week35/html/._week35-bs062.html +++ b/doc/pub/week35/html/._week35-bs062.html @@ -38,11 +38,7 @@ doconce format html week35.do.txt --html_style=bootstrap --pygments_html_style=d {'highest level': 2, 'sections': [('Plans for week 35', 2, None, 'plans-for-week-35'), ('Reading recommendations:', 3, None, 'reading-recommendations'), - ('For exercise sessions: Why Linear Regression (aka Ordinary ' - 'Least Squares and family), repeat from last week', - 2, - None, - 'for-exercise-sessions-why-linear-regression-aka-ordinary-least-squares-and-family-repeat-from-last-week'), + ('Reminder from last week', 2, None, 'reminder-from-last-week'), ('The equations for ordinary least squares', 2, None, @@ -122,32 +118,6 @@ doconce format html week35.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'testing-the-means-squared-error-as-function-of-complexity'), - ('More preprocessing examples, two-dimensional example, the ' - 'Franke function', - 2, - None, - 'more-preprocessing-examples-two-dimensional-example-the-franke-function'), - ('To think about, first part', - 2, - None, - 'to-think-about-first-part'), - ('More thinking', 2, None, 'more-thinking'), - ('Still thinking', 2, None, 'still-thinking'), - ('What does centering (subtracting the mean values) mean ' - 'mathematically?', - 2, - None, - 'what-does-centering-subtracting-the-mean-values-mean-mathematically'), - ('Further Manipulations', 2, None, 'further-manipulations'), - ('Wrapping it up', 2, None, 'wrapping-it-up'), - ('Linear Regression code, Intercept handling first', - 2, - None, - 'linear-regression-code-intercept-handling-first'), - ('Material for lecture Monday, August 26', - 2, - None, - 'material-for-lecture-monday-august-26'), ('Mathematical Interpretation of Ordinary Least Squares', 2, None, @@ -279,7 +249,7 @@ MathJax.Hub.Config({ @@ -360,83 +321,36 @@ MathJax.Hub.Config({

     

     

     

    -

    Correlation Matrix with Pandas and the Franke function

    +

    Deriving the Lasso Regression Equations

    +

    Using the matrix-vector expression for Lasso regression, we have the following cost function

    - -
    -
    -
    -
    -
    - 
    # Common imports
-import numpy as np
-import pandas as pd
+$$
+C(\boldsymbol{X},\boldsymbol{\beta})=\frac{1}{n}\left\{(\boldsymbol{y}-\boldsymbol{X}\boldsymbol{\beta})^T(\boldsymbol{y}-\boldsymbol{X}\boldsymbol{\beta})\right\}+\lambda\vert\vert\boldsymbol{\beta}\vert\vert_1,
+$$
 
+
    Taking the derivative with respect to \( \boldsymbol{\beta} \) and recalling that the derivative of the absolute value is (we drop the boldfaced vector symbol for simplicty)
    
+$$
+\frac{d \vert \beta\vert}{d \beta}=\mathrm{sgn}(\beta)=\left\{\begin{array}{cc} 1 & \beta > 0 \\-1 & \beta < 0, \end{array}\right.
+$$
 
-def FrankeFunction(x,y):
-	term1 = 0.75*np.exp(-(0.25*(9*x-2)**2) - 0.25*((9*y-2)**2))
-	term2 = 0.75*np.exp(-((9*x+1)**2)/49.0 - 0.1*(9*y+1))
-	term3 = 0.5*np.exp(-(9*x-7)**2/4.0 - 0.25*((9*y-3)**2))
-	term4 = -0.2*np.exp(-(9*x-4)**2 - (9*y-7)**2)
-	return term1 + term2 + term3 + term4
+
    we have that the derivative of the cost function is
    
 
+$$
+\frac{\partial C(\boldsymbol{X},\boldsymbol{\beta})}{\partial \boldsymbol{\beta}}=-\frac{2}{n}\boldsymbol{X}^T(\boldsymbol{y}-\boldsymbol{X}\boldsymbol{\beta})+\lambda sgn(\boldsymbol{\beta})=0,
+$$
 
-def create_X(x, y, n ):
-	if len(x.shape) > 1:
-		x = np.ravel(x)
-		y = np.ravel(y)
+
    and reordering we have
    
+$$
+\boldsymbol{X}^T\boldsymbol{X}\boldsymbol{\beta}+\frac{n}{2}\lambda sgn(\boldsymbol{\beta})=2\boldsymbol{X}^T\boldsymbol{y}.
+$$
 
-	N = len(x)
-	l = int((n+1)*(n+2)/2)		# Number of elements in beta
-	X = np.ones((N,l))
+
    We can redefine \( \lambda \) to absorb the constant \( n/2 \) and we rewrite the last equation as
    
+$$
+\boldsymbol{X}^T\boldsymbol{X}\boldsymbol{\beta}+\lambda sgn(\boldsymbol{\beta})=2\boldsymbol{X}^T\boldsymbol{y}.
+$$
 
-	for i in range(1,n+1):
-		q = int((i)*(i+1)/2)
-		for k in range(i+1):
-			X[:,q+k] = (x**(i-k))*(y**k)
-
-	return X
-
-
-# Making meshgrid of datapoints and compute Franke's function
-n = 4
-N = 100
-x = np.sort(np.random.uniform(0, 1, N))
-y = np.sort(np.random.uniform(0, 1, N))
-z = FrankeFunction(x, y)
-X = create_X(x, y, n=n)    
-
-Xpd = pd.DataFrame(X)
-# subtract the mean values and set up the covariance matrix
-Xpd = Xpd - Xpd.mean()
-covariance_matrix = Xpd.cov()
-print(covariance_matrix)
-
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    - -

    We note here that the covariance is zero for the first rows and -columns since all matrix elements in the design matrix were set to one -(we are fitting the function in terms of a polynomial of degree \( n \)). -

    - -

    This means that the variance for these elements will be zero and will -cause problems when we set up the correlation matrix. We can simply -drop these elements and construct a correlation -matrix without these elements. -

    +

    This equation does not lead to a nice analytical equation as in either Ridge regression or ordinary least squares. This equation can however be solved by using standard convex optimization algorithms using for example the Python package CVXOPT. We will discuss this later.

    @@ -453,16 +367,6 @@ matrix without these elements.

  • 61
  • 62
  • 63
    • -
  • 64
    • -
  • 65
    • -
  • 66
    • -
  • 67
    • -
  • 68
    • -
  • 69
    • -
  • 70
    • -
  • 71
    • -
  • 72
    • -
  • »
    • diff --git a/doc/pub/week35/html/week35-bs.html b/doc/pub/week35/html/week35-bs.html index 2f2b84c35..6714a5fc6 100644 --- a/doc/pub/week35/html/week35-bs.html +++ b/doc/pub/week35/html/week35-bs.html @@ -38,11 +38,7 @@ doconce format html week35.do.txt --html_style=bootstrap --pygments_html_style=d {'highest level': 2, 'sections': [('Plans for week 35', 2, None, 'plans-for-week-35'), ('Reading recommendations:', 3, None, 'reading-recommendations'), - ('For exercise sessions: Why Linear Regression (aka Ordinary ' - 'Least Squares and family), repeat from last week', - 2, - None, - 'for-exercise-sessions-why-linear-regression-aka-ordinary-least-squares-and-family-repeat-from-last-week'), + ('Reminder from last week', 2, None, 'reminder-from-last-week'), ('The equations for ordinary least squares', 2, None, @@ -122,32 +118,6 @@ doconce format html week35.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'testing-the-means-squared-error-as-function-of-complexity'), - ('More preprocessing examples, two-dimensional example, the ' - 'Franke function', - 2, - None, - 'more-preprocessing-examples-two-dimensional-example-the-franke-function'), - ('To think about, first part', - 2, - None, - 'to-think-about-first-part'), - ('More thinking', 2, None, 'more-thinking'), - ('Still thinking', 2, None, 'still-thinking'), - ('What does centering (subtracting the mean values) mean ' - 'mathematically?', - 2, - None, - 'what-does-centering-subtracting-the-mean-values-mean-mathematically'), - ('Further Manipulations', 2, None, 'further-manipulations'), - ('Wrapping it up', 2, None, 'wrapping-it-up'), - ('Linear Regression code, Intercept handling first', - 2, - None, - 'linear-regression-code-intercept-handling-first'), - ('Material for lecture Monday, August 26', - 2, - None, - 'material-for-lecture-monday-august-26'), ('Mathematical Interpretation of Ordinary Least Squares', 2, None, @@ -279,7 +249,7 @@ MathJax.Hub.Config({ @@ -400,7 +361,7 @@ MathJax.Hub.Config({
  • 9
  • 10
  • ...
    • -
  • 72
    • +
  • 63
  • »
    • diff --git a/doc/pub/week35/html/week35-reveal.html b/doc/pub/week35/html/week35-reveal.html index 2ec7b0b82..0d940c371 100644 --- a/doc/pub/week35/html/week35-reveal.html +++ b/doc/pub/week35/html/week35-reveal.html @@ -207,17 +207,17 @@ MathJax.Hub.Config({

    Reading recommendations:

      -

    1. These lecture notes -
      2. -

    2. Video for exercises week 35
      3. -

    3. Goodfellow, Bengio and Courville, Deep Learning, chapter 2 on linear algebra and sections 3.1-3.10 on elements of statistics (background)
      4. -

    4. Raschka et al on preprocessing of data, relevant for exercise 3 this week, see chapter 4.
      5. -

    5. For exercise 1 of week 35, the book by A. Aldo Faisal, Cheng Soon Ong, and Marc Peter Deisenroth on the Mathematics of Machine Learning, may be very relevant. In particular chapter 5 at URL"https://mml-book.github.io/" (section 5.5 on derivatives) is very useful for exercise 1 this coming week.
      6. +

    6. These lecture notes + +
      7. +

    7. Goodfellow, Bengio and Courville, Deep Learning, chapter 2 on linear algebra and sections 3.1-3.10 on elements of statistics (background)
      8. +

    8. Raschka et al on preprocessing of data, relevant for exercise 3 this week, see chapter 4.
      9. +

    9. For exercise 1 of week 35, the book by A. Aldo Faisal, Cheng Soon Ong, and Marc Peter Deisenroth on the Mathematics of Machine Learning, may be very relevant. In particular chapter 5 at URL"https://mml-book.github.io/" (section 5.5 on derivatives) is very useful for exercise 1 this coming week.
    -

    For exercise sessions: Why Linear Regression (aka Ordinary Least Squares and family), repeat from last week

    +

    Reminder from last week

    We need first a reminder from last week about linear regression.

    @@ -770,9 +770,9 @@ next week) with the variance of the optimal parameters important role in optmization algorithms and Principal Component Analysis as a way to reduce the dimensionality of a machine learning/data analysis problem. +v +Linear algebra question: Can we use the Hessian matrix to say something about properties of the cost function (our optmization problem)? (hint: think about convex or concave problems and how to relate these to a matrix!).

    - -

    Linear algebra question: Can we use the Hessian matrix to say something about properties of the cost function (our optmization problem)? (hint: think about convex or concave problems and how to relate these to a matrix!).

    @@ -1421,513 +1421,6 @@ plt.show()
    -
    -

    More preprocessing examples, two-dimensional example, the Franke function

    - - - -
    -
    -
    -
    -
    - 
    # Common imports
-import os
-import numpy as np
-import pandas as pd
-import matplotlib.pyplot as plt
-import sklearn.linear_model as skl
-from sklearn.metrics import mean_squared_error
-from sklearn.model_selection import  train_test_split
-from sklearn.preprocessing import MinMaxScaler, StandardScaler, Normalizer
-
-# Where to save the figures and data files
-PROJECT_ROOT_DIR = "Results"
-FIGURE_ID = "Results/FigureFiles"
-DATA_ID = "DataFiles/"
-
-if not os.path.exists(PROJECT_ROOT_DIR):
-    os.mkdir(PROJECT_ROOT_DIR)
-
-if not os.path.exists(FIGURE_ID):
-    os.makedirs(FIGURE_ID)
-
-if not os.path.exists(DATA_ID):
-    os.makedirs(DATA_ID)
-
-def image_path(fig_id):
-    return os.path.join(FIGURE_ID, fig_id)
-
-def data_path(dat_id):
-    return os.path.join(DATA_ID, dat_id)
-
-def save_fig(fig_id):
-    plt.savefig(image_path(fig_id) + ".png", format='png')
-
-
-def FrankeFunction(x,y):
-	term1 = 0.75*np.exp(-(0.25*(9*x-2)**2) - 0.25*((9*y-2)**2))
-	term2 = 0.75*np.exp(-((9*x+1)**2)/49.0 - 0.1*(9*y+1))
-	term3 = 0.5*np.exp(-(9*x-7)**2/4.0 - 0.25*((9*y-3)**2))
-	term4 = -0.2*np.exp(-(9*x-4)**2 - (9*y-7)**2)
-	return term1 + term2 + term3 + term4
-
-
-def create_X(x, y, n ):
-	if len(x.shape) > 1:
-		x = np.ravel(x)
-		y = np.ravel(y)
-
-	N = len(x)
-	l = int((n+1)*(n+2)/2)		# Number of elements in beta
-	X = np.ones((N,l))
-
-	for i in range(1,n+1):
-		q = int((i)*(i+1)/2)
-		for k in range(i+1):
-			X[:,q+k] = (x**(i-k))*(y**k)
-
-	return X
-
-
-# Making meshgrid of datapoints and compute Franke's function
-n = 5
-N = 1000
-x = np.sort(np.random.uniform(0, 1, N))
-y = np.sort(np.random.uniform(0, 1, N))
-z = FrankeFunction(x, y)
-X = create_X(x, y, n=n)    
-# split in training and test data
-X_train, X_test, y_train, y_test = train_test_split(X,z,test_size=0.2)
-
-
-clf = skl.LinearRegression().fit(X_train, y_train)
-
-# The mean squared error and R2 score
-print("MSE before scaling: {:.2f}".format(mean_squared_error(clf.predict(X_test), y_test)))
-print("R2 score before scaling {:.2f}".format(clf.score(X_test,y_test)))
-
-scaler = StandardScaler()
-scaler.fit(X_train)
-X_train_scaled = scaler.transform(X_train)
-X_test_scaled = scaler.transform(X_test)
-
-print("Feature min values before scaling:\n {}".format(X_train.min(axis=0)))
-print("Feature max values before scaling:\n {}".format(X_train.max(axis=0)))
-
-print("Feature min values after scaling:\n {}".format(X_train_scaled.min(axis=0)))
-print("Feature max values after scaling:\n {}".format(X_train_scaled.max(axis=0)))
-
-clf = skl.LinearRegression().fit(X_train_scaled, y_train)
-
-
-print("MSE after  scaling: {:.2f}".format(mean_squared_error(clf.predict(X_test_scaled), y_test)))
-print("R2 score for  scaled data: {:.2f}".format(clf.score(X_test_scaled,y_test)))
-
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    - -
    -

    To think about, first part

    - -

    When you are comparing your own code with for example Scikit-Learn's -library, there are some technicalities to keep in mind. The examples -here demonstrate some of these aspects with potential pitfalls. -

    - -

    The discussion here focuses on the role of the intercept, how we can -set up the design matrix, what scaling we should use and other topics -which tend confuse us. -

    - -

    The intercept can be interpreted as the expected value of our -target/output variables when all other predictors are set to zero. -Thus, if we cannot assume that the expected outputs/targets are zero -when all predictors are zero (the columns in the design matrix), it -may be a bad idea to implement a model which penalizes the intercept. -Furthermore, in for example Ridge and Lasso regression (to be discussed in moe detail next week), the default solutions -from the library Scikit-Learn (when not shrinking \( \beta_0 \)) for the unknown parameters -\( \boldsymbol{\beta} \), are derived under the assumption that both \( \boldsymbol{y} \) and -\( \boldsymbol{X} \) are zero centered, that is we subtract the mean values. -

    -
    - -
    -

    More thinking

    - -

    If our predictors represent different scales, then it is important to -standardize the design matrix \( \boldsymbol{X} \) by subtracting the mean of each -column from the corresponding column and dividing the column with its -standard deviation. Most machine learning libraries do this as a default. This means that if you compare your code with the results from a given library, -the results may differ. -

    - -

    The -Standadscaler -function in Scikit-Learn does this for us. For the data sets we -have been studying in our various examples, the data are in many cases -already scaled and there is no need to scale them. You as a user of different machine learning algorithms, should always perform a -survey of your data, with a critical assessment of them in case you need to scale the data. -

    - -

    If you need to scale the data, not doing so will give an unfair -penalization of the parameters since their magnitude depends on the -scale of their corresponding predictor. -

    - -

    Suppose as an example that you -you have an input variable given by the heights of different persons. -Human height might be measured in inches or meters or -kilometers. If measured in kilometers, a standard linear regression -model with this predictor would probably give a much bigger -coefficient term, than if measured in millimeters. -This can clearly lead to problems in evaluating the cost/loss functions. -

    -
    - -
    -

    Still thinking

    - -

    Keep in mind that when you transform your data set before training a model, the same transformation needs to be done -on your eventual new data set before making a prediction. If we translate this into a Python code, it would could be implemented as follows -(note that the lines are commented since the model function has not been defined) -

    - - -
    -
    -
    -
    -
    - 
    #Model training, we compute the mean value of y and X
-y_train_mean = np.mean(y_train)
-X_train_mean = np.mean(X_train,axis=0)
-X_train = X_train - X_train_mean
-y_train = y_train - y_train_mean
-
-# The we fit our model with the training data
-#trained_model = some_model.fit(X_train,y_train)
-
-
-#Model prediction, we need also to transform our data set used for the prediction.
-X_test = X_test - X_train_mean #Use mean from training data
-#y_pred = trained_model(X_test)
-y_pred = y_pred + y_train_mean
-
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    - -
    -

    What does centering (subtracting the mean values) mean mathematically?

    - -

    Let us try to understand what this may imply mathematically when we -subtract the mean values, also known as zero centering. For -simplicity, we will focus on ordinary regression, as done in the above example. -

    - -

    The cost/loss function for regression is

    -

     
    -$$ -C(\beta_0, \beta_1, ... , \beta_{p-1}) = \frac{1}{n}\sum_{i=0}^{n} \left(y_i - \beta_0 - \sum_{j=1}^{p-1} X_{ij}\beta_j\right)^2,. -$$ -

     
    - -

    Recall also that we use the squared value since this leads to an increase of the penalty for higher differences between predicted and output/target values.

    - -

    What we have done is to single out the \( \beta_0 \) term in the definition of the mean squared error (MSE). -The design matrix -\( X \) does in this case not contain any intercept column. -When we take the derivative with respect to \( \beta_0 \), we want the derivative to obey -

    -

     
    -$$ -\frac{\partial C}{\partial \beta_j} = 0, -$$ -

     
    - -

    for all \( j \). For \( \beta_0 \) we have

    - -

     
    -$$ -\frac{\partial C}{\partial \beta_0} = -\frac{2}{n}\sum_{i=0}^{n-1} \left(y_i - \beta_0 - \sum_{j=1}^{p-1} X_{ij} \beta_j\right). -$$ -

     
    - -

    Multiplying away the constant \( 2/n \), we obtain

    -

     
    -$$ -\sum_{i=0}^{n-1} \beta_0 = \sum_{i=0}^{n-1}y_i - \sum_{i=0}^{n-1} \sum_{j=1}^{p-1} X_{ij} \beta_j. -$$ -

     
    -

    - -
    -

    Further Manipulations

    - -

    Let us special first to the case where we have only two parameters \( \beta_0 \) and \( \beta_1 \). -Our result for \( \beta_0 \) simplifies then to -

    -

     
    -$$ -n\beta_0 = \sum_{i=0}^{n-1}y_i - \sum_{i=0}^{n-1} X_{i1} \beta_1. -$$ -

     
    - -

    We obtain then

    -

     
    -$$ -\beta_0 = \frac{1}{n}\sum_{i=0}^{n-1}y_i - \beta_1\frac{1}{n}\sum_{i=0}^{n-1} X_{i1}. -$$ -

     
    - -

    If we define

    -

     
    -$$ -\mu_1=\frac{1}{n}\sum_{i=0}^{n-1} (X_{i1}, -$$ -

     
    - -

    and if we define the mean value of the outputs as

    -

     
    -$$ -\mu_y=\frac{1}{n}\sum_{i=0}^{n-1}y_i, -$$ -

     
    - -

    we have

    -

     
    -$$ -\beta_0 = \mu_y - \beta_1\mu_{1}. -$$ -

     
    - -

    In the general case, that is we have more parameters than \( \beta_0 \) and \( \beta_1 \), we have

    -

     
    -$$ -\beta_0 = \frac{1}{n}\sum_{i=0}^{n-1}y_i - \frac{1}{n}\sum_{i=0}^{n-1}\sum_{j=1}^{p-1} X_{ij}\beta_j. -$$ -

     
    - -

    Replacing \( y_i \) with \( y_i - y_i - \overline{\boldsymbol{y}} \) and centering also our design matrix results in a cost function (in vector-matrix disguise)

    -

     
    -$$ -C(\boldsymbol{\beta}) = (\boldsymbol{\tilde{y}} - \tilde{X}\boldsymbol{\beta})^T(\boldsymbol{\tilde{y}} - \tilde{X}\boldsymbol{\beta}). -$$ -

     
    -

    - -
    -

    Wrapping it up

    - -

    If we minimize with respect to \( \boldsymbol{\beta} \) we have then

    - -

     
    -$$ -\hat{\boldsymbol{\beta}} = (\tilde{X}^T\tilde{X})^{-1}\tilde{X}^T\boldsymbol{\tilde{y}}, -$$ -

     
    - -

    where \( \boldsymbol{\tilde{y}} = \boldsymbol{y} - \overline{\boldsymbol{y}} \) -and \( \tilde{X}_{ij} = X_{ij} - \frac{1}{n}\sum_{k=0}^{n-1}X_{kj} \). -

    - -

    For Ridge regression we need to add \( \lambda \boldsymbol{\beta}^T\boldsymbol{\beta} \) to the cost function and get then

    -

     
    -$$ -\hat{\boldsymbol{\beta}} = (\tilde{X}^T\tilde{X} + \lambda I)^{-1}\tilde{X}^T\boldsymbol{\tilde{y}}. -$$ -

     
    - -

    What does this mean? And why do we insist on all this? Let us look at some examples.

    -
    - -
    -

    Linear Regression code, Intercept handling first

    - -

    This code shows a simple first-order fit to a data set using the above transformed data, where we consider the role of the intercept first, by either excluding it or including it (code example thanks to Øyvind Sigmundson Schøyen). Here our scaling of the data is done by subtracting the mean values only. -Note also that we do not split the data into training and test. -

    - - - -
    -
    -
    -
    -
    - 
    import numpy as np
-import matplotlib.pyplot as plt
-
-from sklearn.linear_model import LinearRegression
-
-
-np.random.seed(2021)
-
-def MSE(y_data,y_model):
-    n = np.size(y_model)
-    return np.sum((y_data-y_model)**2)/n
-
-
-def fit_beta(X, y):
-    return np.linalg.pinv(X.T @ X) @ X.T @ y
-
-
-true_beta = [2, 0.5, 3.7]
-
-x = np.linspace(0, 1, 11)
-y = np.sum(
-    np.asarray([x ** p * b for p, b in enumerate(true_beta)]), axis=0
-) + 0.1 * np.random.normal(size=len(x))
-
-degree = 3
-X = np.zeros((len(x), degree))
-
-# Include the intercept in the design matrix
-for p in range(degree):
-    X[:, p] = x ** p
-
-beta = fit_beta(X, y)
-
-# Intercept is included in the design matrix
-skl = LinearRegression(fit_intercept=False).fit(X, y)
-
-print(f"True beta: {true_beta}")
-print(f"Fitted beta: {beta}")
-print(f"Sklearn fitted beta: {skl.coef_}")
-ypredictOwn = X @ beta
-ypredictSKL = skl.predict(X)
-print(f"MSE with intercept column")
-print(MSE(y,ypredictOwn))
-print(f"MSE with intercept column from SKL")
-print(MSE(y,ypredictSKL))
-
-
-plt.figure()
-plt.scatter(x, y, label="Data")
-plt.plot(x, X @ beta, label="Fit")
-plt.plot(x, skl.predict(X), label="Sklearn (fit_intercept=False)")
-
-
-# Do not include the intercept in the design matrix
-X = np.zeros((len(x), degree - 1))
-
-for p in range(degree - 1):
-    X[:, p] = x ** (p + 1)
-
-# Intercept is not included in the design matrix
-skl = LinearRegression(fit_intercept=True).fit(X, y)
-
-# Use centered values for X and y when computing coefficients
-y_offset = np.average(y, axis=0)
-X_offset = np.average(X, axis=0)
-
-beta = fit_beta(X - X_offset, y - y_offset)
-intercept = np.mean(y_offset - X_offset @ beta)
-
-print(f"Manual intercept: {intercept}")
-print(f"Fitted beta (wiothout intercept): {beta}")
-print(f"Sklearn intercept: {skl.intercept_}")
-print(f"Sklearn fitted beta (without intercept): {skl.coef_}")
-ypredictOwn = X @ beta
-ypredictSKL = skl.predict(X)
-print(f"MSE with Manual intercept")
-print(MSE(y,ypredictOwn+intercept))
-print(f"MSE with Sklearn intercept")
-print(MSE(y,ypredictSKL))
-
-plt.plot(x, X @ beta + intercept, "--", label="Fit (manual intercept)")
-plt.plot(x, skl.predict(X), "--", label="Sklearn (fit_intercept=True)")
-plt.grid()
-plt.legend()
-
-plt.show()
-
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    - -

    The intercept is the value of our output/target variable -when all our features are zero and our function crosses the \( y \)-axis (for a one-dimensional case). -

    - -

    Printing the MSE, we see first that both methods give the same MSE, as -they should. However, when we move to for example Ridge regression (discussed next week), -the way we treat the intercept may give a larger or smaller MSE, -meaning that the MSE can be penalized by the value of the -intercept. Not including the intercept in the fit, means that the -regularization term does not include \( \beta_0 \). For different values -of \( \lambda \), this may lead to differing MSE values. -

    - -

    To remind the reader, the regularization term, with the intercept in Ridge regression is given by

    -

     
    -$$ -\lambda \vert\vert \boldsymbol{\beta} \vert\vert_2^2 = \lambda \sum_{j=0}^{p-1}\beta_j^2, -$$ -

     
    - -

    but when we take out the intercept, this equation becomes

    -

     
    -$$ -\lambda \vert\vert \boldsymbol{\beta} \vert\vert_2^2 = \lambda \sum_{j=1}^{p-1}\beta_j^2. -$$ -

     
    - -

    For Lasso regression we have

    -

     
    -$$ -\lambda \vert\vert \boldsymbol{\beta} \vert\vert_1 = \lambda \sum_{j=1}^{p-1}\vert\beta_j\vert. -$$ -

     
    - -

    It means that, when scaling the design matrix and the outputs/targets, -by subtracting the mean values, we have an optimization problem which -is not penalized by the intercept. The MSE value can then be smaller -since it focuses only on the remaining quantities. If we however bring -back the intercept, we will get an MSE which then contains the -intercept. This becomes more important when we discuss Ridge and Lasso -regression next week. -

    -
    - -
    -

    Material for lecture Monday, August 26

    -
    -

    Mathematical Interpretation of Ordinary Least Squares

    diff --git a/doc/pub/week35/html/week35-solarized.html b/doc/pub/week35/html/week35-solarized.html index 34aa29a21..b458df946 100644 --- a/doc/pub/week35/html/week35-solarized.html +++ b/doc/pub/week35/html/week35-solarized.html @@ -65,11 +65,7 @@ div.toc p,a { {'highest level': 2, 'sections': [('Plans for week 35', 2, None, 'plans-for-week-35'), ('Reading recommendations:', 3, None, 'reading-recommendations'), - ('For exercise sessions: Why Linear Regression (aka Ordinary ' - 'Least Squares and family), repeat from last week', - 2, - None, - 'for-exercise-sessions-why-linear-regression-aka-ordinary-least-squares-and-family-repeat-from-last-week'), + ('Reminder from last week', 2, None, 'reminder-from-last-week'), ('The equations for ordinary least squares', 2, None, @@ -149,32 +145,6 @@ div.toc p,a { 2, None, 'testing-the-means-squared-error-as-function-of-complexity'), - ('More preprocessing examples, two-dimensional example, the ' - 'Franke function', - 2, - None, - 'more-preprocessing-examples-two-dimensional-example-the-franke-function'), - ('To think about, first part', - 2, - None, - 'to-think-about-first-part'), - ('More thinking', 2, None, 'more-thinking'), - ('Still thinking', 2, None, 'still-thinking'), - ('What does centering (subtracting the mean values) mean ' - 'mathematically?', - 2, - None, - 'what-does-centering-subtracting-the-mean-values-mean-mathematically'), - ('Further Manipulations', 2, None, 'further-manipulations'), - ('Wrapping it up', 2, None, 'wrapping-it-up'), - ('Linear Regression code, Intercept handling first', - 2, - None, - 'linear-regression-code-intercept-handling-first'), - ('Material for lecture Monday, August 26', - 2, - None, - 'material-for-lecture-monday-august-26'), ('Mathematical Interpretation of Ordinary Least Squares', 2, None, @@ -323,15 +293,15 @@ MathJax.Hub.Config({

    Reading recommendations:

      -
    1. These lecture notes -
      2. -
    2. Video for exercises week 35
      3. -
    3. Goodfellow, Bengio and Courville, Deep Learning, chapter 2 on linear algebra and sections 3.1-3.10 on elements of statistics (background)
      4. -
    4. Raschka et al on preprocessing of data, relevant for exercise 3 this week, see chapter 4.
      5. -
    5. For exercise 1 of week 35, the book by A. Aldo Faisal, Cheng Soon Ong, and Marc Peter Deisenroth on the Mathematics of Machine Learning, may be very relevant. In particular chapter 5 at URL"https://mml-book.github.io/" (section 5.5 on derivatives) is very useful for exercise 1 this coming week.
      6. +
    6. These lecture notes + +
      7. +
    7. Goodfellow, Bengio and Courville, Deep Learning, chapter 2 on linear algebra and sections 3.1-3.10 on elements of statistics (background)
      8. +
    8. Raschka et al on preprocessing of data, relevant for exercise 3 this week, see chapter 4.
      9. +
    9. For exercise 1 of week 35, the book by A. Aldo Faisal, Cheng Soon Ong, and Marc Peter Deisenroth on the Mathematics of Machine Learning, may be very relevant. In particular chapter 5 at URL"https://mml-book.github.io/" (section 5.5 on derivatives) is very useful for exercise 1 this coming week.










    -

    For exercise sessions: Why Linear Regression (aka Ordinary Least Squares and family), repeat from last week

    +

    Reminder from last week

    We need first a reminder from last week about linear regression.

    @@ -784,10 +754,10 @@ next week) with the variance of the optimal parameters important role in optmization algorithms and Principal Component Analysis as a way to reduce the dimensionality of a machine learning/data analysis problem. +v +Linear algebra question: Can we use the Hessian matrix to say something about properties of the cost function (our optmization problem)? (hint: think about convex or concave problems and how to relate these to a matrix!).

    -

    Linear algebra question: Can we use the Hessian matrix to say something about properties of the cost function (our optmization problem)? (hint: think about convex or concave problems and how to relate these to a matrix!).

    -









    Interpretations and optimizing our parameters

    @@ -1416,476 +1386,6 @@ plt.show() -









    -

    More preprocessing examples, two-dimensional example, the Franke function

    - - - -
    -
    -
    -
    -
    - 
    # Common imports
-import os
-import numpy as np
-import pandas as pd
-import matplotlib.pyplot as plt
-import sklearn.linear_model as skl
-from sklearn.metrics import mean_squared_error
-from sklearn.model_selection import  train_test_split
-from sklearn.preprocessing import MinMaxScaler, StandardScaler, Normalizer
-
-# Where to save the figures and data files
-PROJECT_ROOT_DIR = "Results"
-FIGURE_ID = "Results/FigureFiles"
-DATA_ID = "DataFiles/"
-
-if not os.path.exists(PROJECT_ROOT_DIR):
-    os.mkdir(PROJECT_ROOT_DIR)
-
-if not os.path.exists(FIGURE_ID):
-    os.makedirs(FIGURE_ID)
-
-if not os.path.exists(DATA_ID):
-    os.makedirs(DATA_ID)
-
-def image_path(fig_id):
-    return os.path.join(FIGURE_ID, fig_id)
-
-def data_path(dat_id):
-    return os.path.join(DATA_ID, dat_id)
-
-def save_fig(fig_id):
-    plt.savefig(image_path(fig_id) + ".png", format='png')
-
-
-def FrankeFunction(x,y):
-	term1 = 0.75*np.exp(-(0.25*(9*x-2)**2) - 0.25*((9*y-2)**2))
-	term2 = 0.75*np.exp(-((9*x+1)**2)/49.0 - 0.1*(9*y+1))
-	term3 = 0.5*np.exp(-(9*x-7)**2/4.0 - 0.25*((9*y-3)**2))
-	term4 = -0.2*np.exp(-(9*x-4)**2 - (9*y-7)**2)
-	return term1 + term2 + term3 + term4
-
-
-def create_X(x, y, n ):
-	if len(x.shape) > 1:
-		x = np.ravel(x)
-		y = np.ravel(y)
-
-	N = len(x)
-	l = int((n+1)*(n+2)/2)		# Number of elements in beta
-	X = np.ones((N,l))
-
-	for i in range(1,n+1):
-		q = int((i)*(i+1)/2)
-		for k in range(i+1):
-			X[:,q+k] = (x**(i-k))*(y**k)
-
-	return X
-
-
-# Making meshgrid of datapoints and compute Franke's function
-n = 5
-N = 1000
-x = np.sort(np.random.uniform(0, 1, N))
-y = np.sort(np.random.uniform(0, 1, N))
-z = FrankeFunction(x, y)
-X = create_X(x, y, n=n)    
-# split in training and test data
-X_train, X_test, y_train, y_test = train_test_split(X,z,test_size=0.2)
-
-
-clf = skl.LinearRegression().fit(X_train, y_train)
-
-# The mean squared error and R2 score
-print("MSE before scaling: {:.2f}".format(mean_squared_error(clf.predict(X_test), y_test)))
-print("R2 score before scaling {:.2f}".format(clf.score(X_test,y_test)))
-
-scaler = StandardScaler()
-scaler.fit(X_train)
-X_train_scaled = scaler.transform(X_train)
-X_test_scaled = scaler.transform(X_test)
-
-print("Feature min values before scaling:\n {}".format(X_train.min(axis=0)))
-print("Feature max values before scaling:\n {}".format(X_train.max(axis=0)))
-
-print("Feature min values after scaling:\n {}".format(X_train_scaled.min(axis=0)))
-print("Feature max values after scaling:\n {}".format(X_train_scaled.max(axis=0)))
-
-clf = skl.LinearRegression().fit(X_train_scaled, y_train)
-
-
-print("MSE after  scaling: {:.2f}".format(mean_squared_error(clf.predict(X_test_scaled), y_test)))
-print("R2 score for  scaled data: {:.2f}".format(clf.score(X_test_scaled,y_test)))
-
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    - - -









    -

    To think about, first part

    - -

    When you are comparing your own code with for example Scikit-Learn's -library, there are some technicalities to keep in mind. The examples -here demonstrate some of these aspects with potential pitfalls. -

    - -

    The discussion here focuses on the role of the intercept, how we can -set up the design matrix, what scaling we should use and other topics -which tend confuse us. -

    - -

    The intercept can be interpreted as the expected value of our -target/output variables when all other predictors are set to zero. -Thus, if we cannot assume that the expected outputs/targets are zero -when all predictors are zero (the columns in the design matrix), it -may be a bad idea to implement a model which penalizes the intercept. -Furthermore, in for example Ridge and Lasso regression (to be discussed in moe detail next week), the default solutions -from the library Scikit-Learn (when not shrinking \( \beta_0 \)) for the unknown parameters -\( \boldsymbol{\beta} \), are derived under the assumption that both \( \boldsymbol{y} \) and -\( \boldsymbol{X} \) are zero centered, that is we subtract the mean values. -

    - -









    -

    More thinking

    - -

    If our predictors represent different scales, then it is important to -standardize the design matrix \( \boldsymbol{X} \) by subtracting the mean of each -column from the corresponding column and dividing the column with its -standard deviation. Most machine learning libraries do this as a default. This means that if you compare your code with the results from a given library, -the results may differ. -

    - -

    The -Standadscaler -function in Scikit-Learn does this for us. For the data sets we -have been studying in our various examples, the data are in many cases -already scaled and there is no need to scale them. You as a user of different machine learning algorithms, should always perform a -survey of your data, with a critical assessment of them in case you need to scale the data. -

    - -

    If you need to scale the data, not doing so will give an unfair -penalization of the parameters since their magnitude depends on the -scale of their corresponding predictor. -

    - -

    Suppose as an example that you -you have an input variable given by the heights of different persons. -Human height might be measured in inches or meters or -kilometers. If measured in kilometers, a standard linear regression -model with this predictor would probably give a much bigger -coefficient term, than if measured in millimeters. -This can clearly lead to problems in evaluating the cost/loss functions. -

    - -









    -

    Still thinking

    - -

    Keep in mind that when you transform your data set before training a model, the same transformation needs to be done -on your eventual new data set before making a prediction. If we translate this into a Python code, it would could be implemented as follows -(note that the lines are commented since the model function has not been defined) -

    - - -
    -
    -
    -
    -
    - 
    #Model training, we compute the mean value of y and X
-y_train_mean = np.mean(y_train)
-X_train_mean = np.mean(X_train,axis=0)
-X_train = X_train - X_train_mean
-y_train = y_train - y_train_mean
-
-# The we fit our model with the training data
-#trained_model = some_model.fit(X_train,y_train)
-
-
-#Model prediction, we need also to transform our data set used for the prediction.
-X_test = X_test - X_train_mean #Use mean from training data
-#y_pred = trained_model(X_test)
-y_pred = y_pred + y_train_mean
-
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    - - -









    -

    What does centering (subtracting the mean values) mean mathematically?

    - -

    Let us try to understand what this may imply mathematically when we -subtract the mean values, also known as zero centering. For -simplicity, we will focus on ordinary regression, as done in the above example. -

    - -

    The cost/loss function for regression is

    -$$ -C(\beta_0, \beta_1, ... , \beta_{p-1}) = \frac{1}{n}\sum_{i=0}^{n} \left(y_i - \beta_0 - \sum_{j=1}^{p-1} X_{ij}\beta_j\right)^2,. -$$ - -

    Recall also that we use the squared value since this leads to an increase of the penalty for higher differences between predicted and output/target values.

    - -

    What we have done is to single out the \( \beta_0 \) term in the definition of the mean squared error (MSE). -The design matrix -\( X \) does in this case not contain any intercept column. -When we take the derivative with respect to \( \beta_0 \), we want the derivative to obey -

    -$$ -\frac{\partial C}{\partial \beta_j} = 0, -$$ - -

    for all \( j \). For \( \beta_0 \) we have

    - -$$ -\frac{\partial C}{\partial \beta_0} = -\frac{2}{n}\sum_{i=0}^{n-1} \left(y_i - \beta_0 - \sum_{j=1}^{p-1} X_{ij} \beta_j\right). -$$ - -

    Multiplying away the constant \( 2/n \), we obtain

    -$$ -\sum_{i=0}^{n-1} \beta_0 = \sum_{i=0}^{n-1}y_i - \sum_{i=0}^{n-1} \sum_{j=1}^{p-1} X_{ij} \beta_j. -$$ - - -









    -

    Further Manipulations

    - -

    Let us special first to the case where we have only two parameters \( \beta_0 \) and \( \beta_1 \). -Our result for \( \beta_0 \) simplifies then to -

    -$$ -n\beta_0 = \sum_{i=0}^{n-1}y_i - \sum_{i=0}^{n-1} X_{i1} \beta_1. -$$ - -

    We obtain then

    -$$ -\beta_0 = \frac{1}{n}\sum_{i=0}^{n-1}y_i - \beta_1\frac{1}{n}\sum_{i=0}^{n-1} X_{i1}. -$$ - -

    If we define

    -$$ -\mu_1=\frac{1}{n}\sum_{i=0}^{n-1} (X_{i1}, -$$ - -

    and if we define the mean value of the outputs as

    -$$ -\mu_y=\frac{1}{n}\sum_{i=0}^{n-1}y_i, -$$ - -

    we have

    -$$ -\beta_0 = \mu_y - \beta_1\mu_{1}. -$$ - -

    In the general case, that is we have more parameters than \( \beta_0 \) and \( \beta_1 \), we have

    -$$ -\beta_0 = \frac{1}{n}\sum_{i=0}^{n-1}y_i - \frac{1}{n}\sum_{i=0}^{n-1}\sum_{j=1}^{p-1} X_{ij}\beta_j. -$$ - -

    Replacing \( y_i \) with \( y_i - y_i - \overline{\boldsymbol{y}} \) and centering also our design matrix results in a cost function (in vector-matrix disguise)

    -$$ -C(\boldsymbol{\beta}) = (\boldsymbol{\tilde{y}} - \tilde{X}\boldsymbol{\beta})^T(\boldsymbol{\tilde{y}} - \tilde{X}\boldsymbol{\beta}). -$$ - - -









    -

    Wrapping it up

    - -

    If we minimize with respect to \( \boldsymbol{\beta} \) we have then

    - -$$ -\hat{\boldsymbol{\beta}} = (\tilde{X}^T\tilde{X})^{-1}\tilde{X}^T\boldsymbol{\tilde{y}}, -$$ - -

    where \( \boldsymbol{\tilde{y}} = \boldsymbol{y} - \overline{\boldsymbol{y}} \) -and \( \tilde{X}_{ij} = X_{ij} - \frac{1}{n}\sum_{k=0}^{n-1}X_{kj} \). -

    - -

    For Ridge regression we need to add \( \lambda \boldsymbol{\beta}^T\boldsymbol{\beta} \) to the cost function and get then

    -$$ -\hat{\boldsymbol{\beta}} = (\tilde{X}^T\tilde{X} + \lambda I)^{-1}\tilde{X}^T\boldsymbol{\tilde{y}}. -$$ - -

    What does this mean? And why do we insist on all this? Let us look at some examples.

    - -









    -

    Linear Regression code, Intercept handling first

    - -

    This code shows a simple first-order fit to a data set using the above transformed data, where we consider the role of the intercept first, by either excluding it or including it (code example thanks to Øyvind Sigmundson Schøyen). Here our scaling of the data is done by subtracting the mean values only. -Note also that we do not split the data into training and test. -

    - - - -
    -
    -
    -
    -
    - 
    import numpy as np
-import matplotlib.pyplot as plt
-
-from sklearn.linear_model import LinearRegression
-
-
-np.random.seed(2021)
-
-def MSE(y_data,y_model):
-    n = np.size(y_model)
-    return np.sum((y_data-y_model)**2)/n
-
-
-def fit_beta(X, y):
-    return np.linalg.pinv(X.T @ X) @ X.T @ y
-
-
-true_beta = [2, 0.5, 3.7]
-
-x = np.linspace(0, 1, 11)
-y = np.sum(
-    np.asarray([x ** p * b for p, b in enumerate(true_beta)]), axis=0
-) + 0.1 * np.random.normal(size=len(x))
-
-degree = 3
-X = np.zeros((len(x), degree))
-
-# Include the intercept in the design matrix
-for p in range(degree):
-    X[:, p] = x ** p
-
-beta = fit_beta(X, y)
-
-# Intercept is included in the design matrix
-skl = LinearRegression(fit_intercept=False).fit(X, y)
-
-print(f"True beta: {true_beta}")
-print(f"Fitted beta: {beta}")
-print(f"Sklearn fitted beta: {skl.coef_}")
-ypredictOwn = X @ beta
-ypredictSKL = skl.predict(X)
-print(f"MSE with intercept column")
-print(MSE(y,ypredictOwn))
-print(f"MSE with intercept column from SKL")
-print(MSE(y,ypredictSKL))
-
-
-plt.figure()
-plt.scatter(x, y, label="Data")
-plt.plot(x, X @ beta, label="Fit")
-plt.plot(x, skl.predict(X), label="Sklearn (fit_intercept=False)")
-
-
-# Do not include the intercept in the design matrix
-X = np.zeros((len(x), degree - 1))
-
-for p in range(degree - 1):
-    X[:, p] = x ** (p + 1)
-
-# Intercept is not included in the design matrix
-skl = LinearRegression(fit_intercept=True).fit(X, y)
-
-# Use centered values for X and y when computing coefficients
-y_offset = np.average(y, axis=0)
-X_offset = np.average(X, axis=0)
-
-beta = fit_beta(X - X_offset, y - y_offset)
-intercept = np.mean(y_offset - X_offset @ beta)
-
-print(f"Manual intercept: {intercept}")
-print(f"Fitted beta (wiothout intercept): {beta}")
-print(f"Sklearn intercept: {skl.intercept_}")
-print(f"Sklearn fitted beta (without intercept): {skl.coef_}")
-ypredictOwn = X @ beta
-ypredictSKL = skl.predict(X)
-print(f"MSE with Manual intercept")
-print(MSE(y,ypredictOwn+intercept))
-print(f"MSE with Sklearn intercept")
-print(MSE(y,ypredictSKL))
-
-plt.plot(x, X @ beta + intercept, "--", label="Fit (manual intercept)")
-plt.plot(x, skl.predict(X), "--", label="Sklearn (fit_intercept=True)")
-plt.grid()
-plt.legend()
-
-plt.show()
-
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    - -

    The intercept is the value of our output/target variable -when all our features are zero and our function crosses the \( y \)-axis (for a one-dimensional case). -

    - -

    Printing the MSE, we see first that both methods give the same MSE, as -they should. However, when we move to for example Ridge regression (discussed next week), -the way we treat the intercept may give a larger or smaller MSE, -meaning that the MSE can be penalized by the value of the -intercept. Not including the intercept in the fit, means that the -regularization term does not include \( \beta_0 \). For different values -of \( \lambda \), this may lead to differing MSE values. -

    - -

    To remind the reader, the regularization term, with the intercept in Ridge regression is given by

    -$$ -\lambda \vert\vert \boldsymbol{\beta} \vert\vert_2^2 = \lambda \sum_{j=0}^{p-1}\beta_j^2, -$$ - -

    but when we take out the intercept, this equation becomes

    -$$ -\lambda \vert\vert \boldsymbol{\beta} \vert\vert_2^2 = \lambda \sum_{j=1}^{p-1}\beta_j^2. -$$ - -

    For Lasso regression we have

    -$$ -\lambda \vert\vert \boldsymbol{\beta} \vert\vert_1 = \lambda \sum_{j=1}^{p-1}\vert\beta_j\vert. -$$ - -

    It means that, when scaling the design matrix and the outputs/targets, -by subtracting the mean values, we have an optimization problem which -is not penalized by the intercept. The MSE value can then be smaller -since it focuses only on the remaining quantities. If we however bring -back the intercept, we will get an MSE which then contains the -intercept. This becomes more important when we discuss Ridge and Lasso -regression next week. -

    - -









    -

    Material for lecture Monday, August 26

    -









    Mathematical Interpretation of Ordinary Least Squares

    diff --git a/doc/pub/week35/html/week35.html b/doc/pub/week35/html/week35.html index 7c15b13e0..4cefeb9d7 100644 --- a/doc/pub/week35/html/week35.html +++ b/doc/pub/week35/html/week35.html @@ -142,11 +142,7 @@ div.toc p,a { {'highest level': 2, 'sections': [('Plans for week 35', 2, None, 'plans-for-week-35'), ('Reading recommendations:', 3, None, 'reading-recommendations'), - ('For exercise sessions: Why Linear Regression (aka Ordinary ' - 'Least Squares and family), repeat from last week', - 2, - None, - 'for-exercise-sessions-why-linear-regression-aka-ordinary-least-squares-and-family-repeat-from-last-week'), + ('Reminder from last week', 2, None, 'reminder-from-last-week'), ('The equations for ordinary least squares', 2, None, @@ -226,32 +222,6 @@ div.toc p,a { 2, None, 'testing-the-means-squared-error-as-function-of-complexity'), - ('More preprocessing examples, two-dimensional example, the ' - 'Franke function', - 2, - None, - 'more-preprocessing-examples-two-dimensional-example-the-franke-function'), - ('To think about, first part', - 2, - None, - 'to-think-about-first-part'), - ('More thinking', 2, None, 'more-thinking'), - ('Still thinking', 2, None, 'still-thinking'), - ('What does centering (subtracting the mean values) mean ' - 'mathematically?', - 2, - None, - 'what-does-centering-subtracting-the-mean-values-mean-mathematically'), - ('Further Manipulations', 2, None, 'further-manipulations'), - ('Wrapping it up', 2, None, 'wrapping-it-up'), - ('Linear Regression code, Intercept handling first', - 2, - None, - 'linear-regression-code-intercept-handling-first'), - ('Material for lecture Monday, August 26', - 2, - None, - 'material-for-lecture-monday-august-26'), ('Mathematical Interpretation of Ordinary Least Squares', 2, None, @@ -400,15 +370,15 @@ MathJax.Hub.Config({

    Reading recommendations:

      -
    1. These lecture notes -
      2. -
    2. Video for exercises week 35
      3. -
    3. Goodfellow, Bengio and Courville, Deep Learning, chapter 2 on linear algebra and sections 3.1-3.10 on elements of statistics (background)
      4. -
    4. Raschka et al on preprocessing of data, relevant for exercise 3 this week, see chapter 4.
      5. -
    5. For exercise 1 of week 35, the book by A. Aldo Faisal, Cheng Soon Ong, and Marc Peter Deisenroth on the Mathematics of Machine Learning, may be very relevant. In particular chapter 5 at URL"https://mml-book.github.io/" (section 5.5 on derivatives) is very useful for exercise 1 this coming week.
      6. +
    6. These lecture notes + +
      7. +
    7. Goodfellow, Bengio and Courville, Deep Learning, chapter 2 on linear algebra and sections 3.1-3.10 on elements of statistics (background)
      8. +
    8. Raschka et al on preprocessing of data, relevant for exercise 3 this week, see chapter 4.
      9. +
    9. For exercise 1 of week 35, the book by A. Aldo Faisal, Cheng Soon Ong, and Marc Peter Deisenroth on the Mathematics of Machine Learning, may be very relevant. In particular chapter 5 at URL"https://mml-book.github.io/" (section 5.5 on derivatives) is very useful for exercise 1 this coming week.










    -

    For exercise sessions: Why Linear Regression (aka Ordinary Least Squares and family), repeat from last week

    +

    Reminder from last week

    We need first a reminder from last week about linear regression.

    @@ -861,10 +831,10 @@ next week) with the variance of the optimal parameters important role in optmization algorithms and Principal Component Analysis as a way to reduce the dimensionality of a machine learning/data analysis problem. +v +Linear algebra question: Can we use the Hessian matrix to say something about properties of the cost function (our optmization problem)? (hint: think about convex or concave problems and how to relate these to a matrix!).

    -

    Linear algebra question: Can we use the Hessian matrix to say something about properties of the cost function (our optmization problem)? (hint: think about convex or concave problems and how to relate these to a matrix!).

    -









    Interpretations and optimizing our parameters

    @@ -1493,476 +1463,6 @@ plt.show() -









    -

    More preprocessing examples, two-dimensional example, the Franke function

    - - - -
    -
    -
    -
    -
    - 
    # Common imports
-import os
-import numpy as np
-import pandas as pd
-import matplotlib.pyplot as plt
-import sklearn.linear_model as skl
-from sklearn.metrics import mean_squared_error
-from sklearn.model_selection import  train_test_split
-from sklearn.preprocessing import MinMaxScaler, StandardScaler, Normalizer
-
-# Where to save the figures and data files
-PROJECT_ROOT_DIR = "Results"
-FIGURE_ID = "Results/FigureFiles"
-DATA_ID = "DataFiles/"
-
-if not os.path.exists(PROJECT_ROOT_DIR):
-    os.mkdir(PROJECT_ROOT_DIR)
-
-if not os.path.exists(FIGURE_ID):
-    os.makedirs(FIGURE_ID)
-
-if not os.path.exists(DATA_ID):
-    os.makedirs(DATA_ID)
-
-def image_path(fig_id):
-    return os.path.join(FIGURE_ID, fig_id)
-
-def data_path(dat_id):
-    return os.path.join(DATA_ID, dat_id)
-
-def save_fig(fig_id):
-    plt.savefig(image_path(fig_id) + ".png", format='png')
-
-
-def FrankeFunction(x,y):
-	term1 = 0.75*np.exp(-(0.25*(9*x-2)**2) - 0.25*((9*y-2)**2))
-	term2 = 0.75*np.exp(-((9*x+1)**2)/49.0 - 0.1*(9*y+1))
-	term3 = 0.5*np.exp(-(9*x-7)**2/4.0 - 0.25*((9*y-3)**2))
-	term4 = -0.2*np.exp(-(9*x-4)**2 - (9*y-7)**2)
-	return term1 + term2 + term3 + term4
-
-
-def create_X(x, y, n ):
-	if len(x.shape) > 1:
-		x = np.ravel(x)
-		y = np.ravel(y)
-
-	N = len(x)
-	l = int((n+1)*(n+2)/2)		# Number of elements in beta
-	X = np.ones((N,l))
-
-	for i in range(1,n+1):
-		q = int((i)*(i+1)/2)
-		for k in range(i+1):
-			X[:,q+k] = (x**(i-k))*(y**k)
-
-	return X
-
-
-# Making meshgrid of datapoints and compute Franke's function
-n = 5
-N = 1000
-x = np.sort(np.random.uniform(0, 1, N))
-y = np.sort(np.random.uniform(0, 1, N))
-z = FrankeFunction(x, y)
-X = create_X(x, y, n=n)    
-# split in training and test data
-X_train, X_test, y_train, y_test = train_test_split(X,z,test_size=0.2)
-
-
-clf = skl.LinearRegression().fit(X_train, y_train)
-
-# The mean squared error and R2 score
-print("MSE before scaling: {:.2f}".format(mean_squared_error(clf.predict(X_test), y_test)))
-print("R2 score before scaling {:.2f}".format(clf.score(X_test,y_test)))
-
-scaler = StandardScaler()
-scaler.fit(X_train)
-X_train_scaled = scaler.transform(X_train)
-X_test_scaled = scaler.transform(X_test)
-
-print("Feature min values before scaling:\n {}".format(X_train.min(axis=0)))
-print("Feature max values before scaling:\n {}".format(X_train.max(axis=0)))
-
-print("Feature min values after scaling:\n {}".format(X_train_scaled.min(axis=0)))
-print("Feature max values after scaling:\n {}".format(X_train_scaled.max(axis=0)))
-
-clf = skl.LinearRegression().fit(X_train_scaled, y_train)
-
-
-print("MSE after  scaling: {:.2f}".format(mean_squared_error(clf.predict(X_test_scaled), y_test)))
-print("R2 score for  scaled data: {:.2f}".format(clf.score(X_test_scaled,y_test)))
-
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    - - -









    -

    To think about, first part

    - -

    When you are comparing your own code with for example Scikit-Learn's -library, there are some technicalities to keep in mind. The examples -here demonstrate some of these aspects with potential pitfalls. -

    - -

    The discussion here focuses on the role of the intercept, how we can -set up the design matrix, what scaling we should use and other topics -which tend confuse us. -

    - -

    The intercept can be interpreted as the expected value of our -target/output variables when all other predictors are set to zero. -Thus, if we cannot assume that the expected outputs/targets are zero -when all predictors are zero (the columns in the design matrix), it -may be a bad idea to implement a model which penalizes the intercept. -Furthermore, in for example Ridge and Lasso regression (to be discussed in moe detail next week), the default solutions -from the library Scikit-Learn (when not shrinking \( \beta_0 \)) for the unknown parameters -\( \boldsymbol{\beta} \), are derived under the assumption that both \( \boldsymbol{y} \) and -\( \boldsymbol{X} \) are zero centered, that is we subtract the mean values. -

    - -









    -

    More thinking

    - -

    If our predictors represent different scales, then it is important to -standardize the design matrix \( \boldsymbol{X} \) by subtracting the mean of each -column from the corresponding column and dividing the column with its -standard deviation. Most machine learning libraries do this as a default. This means that if you compare your code with the results from a given library, -the results may differ. -

    - -

    The -Standadscaler -function in Scikit-Learn does this for us. For the data sets we -have been studying in our various examples, the data are in many cases -already scaled and there is no need to scale them. You as a user of different machine learning algorithms, should always perform a -survey of your data, with a critical assessment of them in case you need to scale the data. -

    - -

    If you need to scale the data, not doing so will give an unfair -penalization of the parameters since their magnitude depends on the -scale of their corresponding predictor. -

    - -

    Suppose as an example that you -you have an input variable given by the heights of different persons. -Human height might be measured in inches or meters or -kilometers. If measured in kilometers, a standard linear regression -model with this predictor would probably give a much bigger -coefficient term, than if measured in millimeters. -This can clearly lead to problems in evaluating the cost/loss functions. -

    - -









    -

    Still thinking

    - -

    Keep in mind that when you transform your data set before training a model, the same transformation needs to be done -on your eventual new data set before making a prediction. If we translate this into a Python code, it would could be implemented as follows -(note that the lines are commented since the model function has not been defined) -

    - - -
    -
    -
    -
    -
    - 
    #Model training, we compute the mean value of y and X
-y_train_mean = np.mean(y_train)
-X_train_mean = np.mean(X_train,axis=0)
-X_train = X_train - X_train_mean
-y_train = y_train - y_train_mean
-
-# The we fit our model with the training data
-#trained_model = some_model.fit(X_train,y_train)
-
-
-#Model prediction, we need also to transform our data set used for the prediction.
-X_test = X_test - X_train_mean #Use mean from training data
-#y_pred = trained_model(X_test)
-y_pred = y_pred + y_train_mean
-
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    - - -









    -

    What does centering (subtracting the mean values) mean mathematically?

    - -

    Let us try to understand what this may imply mathematically when we -subtract the mean values, also known as zero centering. For -simplicity, we will focus on ordinary regression, as done in the above example. -

    - -

    The cost/loss function for regression is

    -$$ -C(\beta_0, \beta_1, ... , \beta_{p-1}) = \frac{1}{n}\sum_{i=0}^{n} \left(y_i - \beta_0 - \sum_{j=1}^{p-1} X_{ij}\beta_j\right)^2,. -$$ - -

    Recall also that we use the squared value since this leads to an increase of the penalty for higher differences between predicted and output/target values.

    - -

    What we have done is to single out the \( \beta_0 \) term in the definition of the mean squared error (MSE). -The design matrix -\( X \) does in this case not contain any intercept column. -When we take the derivative with respect to \( \beta_0 \), we want the derivative to obey -

    -$$ -\frac{\partial C}{\partial \beta_j} = 0, -$$ - -

    for all \( j \). For \( \beta_0 \) we have

    - -$$ -\frac{\partial C}{\partial \beta_0} = -\frac{2}{n}\sum_{i=0}^{n-1} \left(y_i - \beta_0 - \sum_{j=1}^{p-1} X_{ij} \beta_j\right). -$$ - -

    Multiplying away the constant \( 2/n \), we obtain

    -$$ -\sum_{i=0}^{n-1} \beta_0 = \sum_{i=0}^{n-1}y_i - \sum_{i=0}^{n-1} \sum_{j=1}^{p-1} X_{ij} \beta_j. -$$ - - -









    -

    Further Manipulations

    - -

    Let us special first to the case where we have only two parameters \( \beta_0 \) and \( \beta_1 \). -Our result for \( \beta_0 \) simplifies then to -

    -$$ -n\beta_0 = \sum_{i=0}^{n-1}y_i - \sum_{i=0}^{n-1} X_{i1} \beta_1. -$$ - -

    We obtain then

    -$$ -\beta_0 = \frac{1}{n}\sum_{i=0}^{n-1}y_i - \beta_1\frac{1}{n}\sum_{i=0}^{n-1} X_{i1}. -$$ - -

    If we define

    -$$ -\mu_1=\frac{1}{n}\sum_{i=0}^{n-1} (X_{i1}, -$$ - -

    and if we define the mean value of the outputs as

    -$$ -\mu_y=\frac{1}{n}\sum_{i=0}^{n-1}y_i, -$$ - -

    we have

    -$$ -\beta_0 = \mu_y - \beta_1\mu_{1}. -$$ - -

    In the general case, that is we have more parameters than \( \beta_0 \) and \( \beta_1 \), we have

    -$$ -\beta_0 = \frac{1}{n}\sum_{i=0}^{n-1}y_i - \frac{1}{n}\sum_{i=0}^{n-1}\sum_{j=1}^{p-1} X_{ij}\beta_j. -$$ - -

    Replacing \( y_i \) with \( y_i - y_i - \overline{\boldsymbol{y}} \) and centering also our design matrix results in a cost function (in vector-matrix disguise)

    -$$ -C(\boldsymbol{\beta}) = (\boldsymbol{\tilde{y}} - \tilde{X}\boldsymbol{\beta})^T(\boldsymbol{\tilde{y}} - \tilde{X}\boldsymbol{\beta}). -$$ - - -









    -

    Wrapping it up

    - -

    If we minimize with respect to \( \boldsymbol{\beta} \) we have then

    - -$$ -\hat{\boldsymbol{\beta}} = (\tilde{X}^T\tilde{X})^{-1}\tilde{X}^T\boldsymbol{\tilde{y}}, -$$ - -

    where \( \boldsymbol{\tilde{y}} = \boldsymbol{y} - \overline{\boldsymbol{y}} \) -and \( \tilde{X}_{ij} = X_{ij} - \frac{1}{n}\sum_{k=0}^{n-1}X_{kj} \). -

    - -

    For Ridge regression we need to add \( \lambda \boldsymbol{\beta}^T\boldsymbol{\beta} \) to the cost function and get then

    -$$ -\hat{\boldsymbol{\beta}} = (\tilde{X}^T\tilde{X} + \lambda I)^{-1}\tilde{X}^T\boldsymbol{\tilde{y}}. -$$ - -

    What does this mean? And why do we insist on all this? Let us look at some examples.

    - -









    -

    Linear Regression code, Intercept handling first

    - -

    This code shows a simple first-order fit to a data set using the above transformed data, where we consider the role of the intercept first, by either excluding it or including it (code example thanks to Øyvind Sigmundson Schøyen). Here our scaling of the data is done by subtracting the mean values only. -Note also that we do not split the data into training and test. -

    - - - -
    -
    -
    -
    -
    - 
    import numpy as np
-import matplotlib.pyplot as plt
-
-from sklearn.linear_model import LinearRegression
-
-
-np.random.seed(2021)
-
-def MSE(y_data,y_model):
-    n = np.size(y_model)
-    return np.sum((y_data-y_model)**2)/n
-
-
-def fit_beta(X, y):
-    return np.linalg.pinv(X.T @ X) @ X.T @ y
-
-
-true_beta = [2, 0.5, 3.7]
-
-x = np.linspace(0, 1, 11)
-y = np.sum(
-    np.asarray([x ** p * b for p, b in enumerate(true_beta)]), axis=0
-) + 0.1 * np.random.normal(size=len(x))
-
-degree = 3
-X = np.zeros((len(x), degree))
-
-# Include the intercept in the design matrix
-for p in range(degree):
-    X[:, p] = x ** p
-
-beta = fit_beta(X, y)
-
-# Intercept is included in the design matrix
-skl = LinearRegression(fit_intercept=False).fit(X, y)
-
-print(f"True beta: {true_beta}")
-print(f"Fitted beta: {beta}")
-print(f"Sklearn fitted beta: {skl.coef_}")
-ypredictOwn = X @ beta
-ypredictSKL = skl.predict(X)
-print(f"MSE with intercept column")
-print(MSE(y,ypredictOwn))
-print(f"MSE with intercept column from SKL")
-print(MSE(y,ypredictSKL))
-
-
-plt.figure()
-plt.scatter(x, y, label="Data")
-plt.plot(x, X @ beta, label="Fit")
-plt.plot(x, skl.predict(X), label="Sklearn (fit_intercept=False)")
-
-
-# Do not include the intercept in the design matrix
-X = np.zeros((len(x), degree - 1))
-
-for p in range(degree - 1):
-    X[:, p] = x ** (p + 1)
-
-# Intercept is not included in the design matrix
-skl = LinearRegression(fit_intercept=True).fit(X, y)
-
-# Use centered values for X and y when computing coefficients
-y_offset = np.average(y, axis=0)
-X_offset = np.average(X, axis=0)
-
-beta = fit_beta(X - X_offset, y - y_offset)
-intercept = np.mean(y_offset - X_offset @ beta)
-
-print(f"Manual intercept: {intercept}")
-print(f"Fitted beta (wiothout intercept): {beta}")
-print(f"Sklearn intercept: {skl.intercept_}")
-print(f"Sklearn fitted beta (without intercept): {skl.coef_}")
-ypredictOwn = X @ beta
-ypredictSKL = skl.predict(X)
-print(f"MSE with Manual intercept")
-print(MSE(y,ypredictOwn+intercept))
-print(f"MSE with Sklearn intercept")
-print(MSE(y,ypredictSKL))
-
-plt.plot(x, X @ beta + intercept, "--", label="Fit (manual intercept)")
-plt.plot(x, skl.predict(X), "--", label="Sklearn (fit_intercept=True)")
-plt.grid()
-plt.legend()
-
-plt.show()
-
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    - -

    The intercept is the value of our output/target variable -when all our features are zero and our function crosses the \( y \)-axis (for a one-dimensional case). -

    - -

    Printing the MSE, we see first that both methods give the same MSE, as -they should. However, when we move to for example Ridge regression (discussed next week), -the way we treat the intercept may give a larger or smaller MSE, -meaning that the MSE can be penalized by the value of the -intercept. Not including the intercept in the fit, means that the -regularization term does not include \( \beta_0 \). For different values -of \( \lambda \), this may lead to differing MSE values. -

    - -

    To remind the reader, the regularization term, with the intercept in Ridge regression is given by

    -$$ -\lambda \vert\vert \boldsymbol{\beta} \vert\vert_2^2 = \lambda \sum_{j=0}^{p-1}\beta_j^2, -$$ - -

    but when we take out the intercept, this equation becomes

    -$$ -\lambda \vert\vert \boldsymbol{\beta} \vert\vert_2^2 = \lambda \sum_{j=1}^{p-1}\beta_j^2. -$$ - -

    For Lasso regression we have

    -$$ -\lambda \vert\vert \boldsymbol{\beta} \vert\vert_1 = \lambda \sum_{j=1}^{p-1}\vert\beta_j\vert. -$$ - -

    It means that, when scaling the design matrix and the outputs/targets, -by subtracting the mean values, we have an optimization problem which -is not penalized by the intercept. The MSE value can then be smaller -since it focuses only on the remaining quantities. If we however bring -back the intercept, we will get an MSE which then contains the -intercept. This becomes more important when we discuss Ridge and Lasso -regression next week. -

    - -









    -

    Material for lecture Monday, August 26

    -









    Mathematical Interpretation of Ordinary Least Squares

    diff --git a/doc/pub/week35/ipynb/ipynb-week35-src.tar.gz b/doc/pub/week35/ipynb/ipynb-week35-src.tar.gz index cf7fae483..bdcd9a505 100644 Binary files a/doc/pub/week35/ipynb/ipynb-week35-src.tar.gz and b/doc/pub/week35/ipynb/ipynb-week35-src.tar.gz differ diff --git a/doc/pub/week35/ipynb/week35.ipynb b/doc/pub/week35/ipynb/week35.ipynb index a70b96605..95e79be49 100644 --- a/doc/pub/week35/ipynb/week35.ipynb +++ b/doc/pub/week35/ipynb/week35.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "markdown", - "id": "ccdb6611", + "id": "f59bef82", "metadata": { "editable": true }, @@ -14,7 +14,7 @@ }, { "cell_type": "markdown", - "id": "c0afc18d", + "id": "d23d7811", "metadata": { "editable": true }, @@ -27,7 +27,7 @@ }, { "cell_type": "markdown", - "id": "b22575d0", + "id": "82fad38e", "metadata": { "editable": true }, @@ -49,7 +49,7 @@ }, { "cell_type": "markdown", - "id": "9e866536", + "id": "380a8031", "metadata": { "editable": true }, @@ -58,24 +58,23 @@ "\n", "1. These lecture notes\n", "\n", + "\n", "\n", - "2. [Video for exercises week 35](https://youtu.be/yiY0OltU1s8)\n", + "2. Goodfellow, Bengio and Courville, Deep Learning, chapter 2 on linear algebra and sections 3.1-3.10 on elements of statistics (background)\n", "\n", - "3. Goodfellow, Bengio and Courville, Deep Learning, chapter 2 on linear algebra and sections 3.1-3.10 on elements of statistics (background)\n", + "3. Raschka et al on preprocessing of data, relevant for exercise 3 this week, see chapter 4.\n", "\n", - "4. Raschka et al on preprocessing of data, relevant for exercise 3 this week, see chapter 4.\n", - "\n", - "5. For exercise 1 of week 35, the book by A. Aldo Faisal, Cheng Soon Ong, and Marc Peter Deisenroth on the Mathematics of Machine Learning, may be very relevant. In particular chapter 5 at URL\"https://mml-book.github.io/\" (section 5.5 on derivatives) is very useful for exercise 1 this coming week." + "4. For exercise 1 of week 35, the book by A. Aldo Faisal, Cheng Soon Ong, and Marc Peter Deisenroth on the Mathematics of Machine Learning, may be very relevant. In particular chapter 5 at URL\"https://mml-book.github.io/\" (section 5.5 on derivatives) is very useful for exercise 1 this coming week." ] }, { "cell_type": "markdown", - "id": "6b978003", + "id": "fc0a0056", "metadata": { "editable": true }, "source": [ - "## For exercise sessions: Why Linear Regression (aka Ordinary Least Squares and family), repeat from last week\n", + "## Reminder from last week\n", "\n", "We need first a reminder from last week about linear regression. \n", "\n", @@ -104,7 +103,7 @@ }, { "cell_type": "markdown", - "id": "a0d86e28", + "id": "5ff8abac", "metadata": { "editable": true }, @@ -120,7 +119,7 @@ }, { "cell_type": "markdown", - "id": "caa94e34", + "id": "df0716ae", "metadata": { "editable": true }, @@ -132,7 +131,7 @@ }, { "cell_type": "markdown", - "id": "9ab865ac", + "id": "72d9a463", "metadata": { "editable": true }, @@ -142,7 +141,7 @@ }, { "cell_type": "markdown", - "id": "8427a47a", + "id": "c81a5718", "metadata": { "editable": true }, @@ -154,7 +153,7 @@ }, { "cell_type": "markdown", - "id": "0def21bc", + "id": "4d357994", "metadata": { "editable": true }, @@ -175,7 +174,7 @@ }, { "cell_type": "markdown", - "id": "712bdbe0", + "id": "cf4d9cee", "metadata": { "editable": true }, @@ -187,7 +186,7 @@ }, { "cell_type": "markdown", - "id": "e6575f91", + "id": "c5ad4e38", "metadata": { "editable": true }, @@ -200,7 +199,7 @@ }, { "cell_type": "markdown", - "id": "e2d98dfc", + "id": "297ced52", "metadata": { "editable": true }, @@ -212,7 +211,7 @@ }, { "cell_type": "markdown", - "id": "af6c27fd", + "id": "0f7fefee", "metadata": { "editable": true }, @@ -224,7 +223,7 @@ }, { "cell_type": "markdown", - "id": "f4d57fd0", + "id": "bd8dc4f2", "metadata": { "editable": true }, @@ -234,7 +233,7 @@ }, { "cell_type": "markdown", - "id": "04daf870", + "id": "da82ea39", "metadata": { "editable": true }, @@ -246,7 +245,7 @@ }, { "cell_type": "markdown", - "id": "41869142", + "id": "23e19e04", "metadata": { "editable": true }, @@ -259,7 +258,7 @@ }, { "cell_type": "markdown", - "id": "03f5c547", + "id": "24f8e2c6", "metadata": { "editable": true }, @@ -271,7 +270,7 @@ }, { "cell_type": "markdown", - "id": "993ecba7", + "id": "2f9e0a7f", "metadata": { "editable": true }, @@ -281,7 +280,7 @@ }, { "cell_type": "markdown", - "id": "e3760c66", + "id": "62eff411", "metadata": { "editable": true }, @@ -293,7 +292,7 @@ }, { "cell_type": "markdown", - "id": "0f9d59e3", + "id": "a70a3f6f", "metadata": { "editable": true }, @@ -305,7 +304,7 @@ }, { "cell_type": "markdown", - "id": "b71b58be", + "id": "ca289772", "metadata": { "editable": true }, @@ -316,7 +315,7 @@ }, { "cell_type": "markdown", - "id": "31c1886e", + "id": "a8312deb", "metadata": { "editable": true }, @@ -328,7 +327,7 @@ }, { "cell_type": "markdown", - "id": "5b6afd25", + "id": "df1f3210", "metadata": { "editable": true }, @@ -347,7 +346,7 @@ }, { "cell_type": "markdown", - "id": "4caf3af4", + "id": "20dd8ffe", "metadata": { "editable": true }, @@ -360,7 +359,7 @@ }, { "cell_type": "markdown", - "id": "d94842ef", + "id": "f5acd1e3", "metadata": { "editable": true }, @@ -370,7 +369,7 @@ }, { "cell_type": "markdown", - "id": "d3e2283c", + "id": "f5f153bf", "metadata": { "editable": true }, @@ -382,7 +381,7 @@ }, { "cell_type": "markdown", - "id": "4567bd5b", + "id": "bc60661c", "metadata": { "editable": true }, @@ -392,7 +391,7 @@ }, { "cell_type": "markdown", - "id": "7f4ddaf5", + "id": "ad0c51c8", "metadata": { "editable": true }, @@ -404,7 +403,7 @@ }, { "cell_type": "markdown", - "id": "efc79f6f", + "id": "299217eb", "metadata": { "editable": true }, @@ -414,7 +413,7 @@ }, { "cell_type": "markdown", - "id": "f3801564", + "id": "37c4f3d2", "metadata": { "editable": true }, @@ -426,7 +425,7 @@ }, { "cell_type": "markdown", - "id": "b8a9a05d", + "id": "0aa3bf78", "metadata": { "editable": true }, @@ -437,7 +436,7 @@ }, { "cell_type": "markdown", - "id": "30015d33", + "id": "c7cd2c51", "metadata": { "editable": true }, @@ -449,7 +448,7 @@ }, { "cell_type": "markdown", - "id": "2fd1d34c", + "id": "8188af0b", "metadata": { "editable": true }, @@ -459,7 +458,7 @@ }, { "cell_type": "markdown", - "id": "b9e1f046", + "id": "bb587a70", "metadata": { "editable": true }, @@ -471,7 +470,7 @@ }, { "cell_type": "markdown", - "id": "663f8d76", + "id": "a0facf6c", "metadata": { "editable": true }, @@ -481,7 +480,7 @@ }, { "cell_type": "markdown", - "id": "ae070f3e", + "id": "a5128b98", "metadata": { "editable": true }, @@ -493,7 +492,7 @@ }, { "cell_type": "markdown", - "id": "45e15b96", + "id": "25915602", "metadata": { "editable": true }, @@ -513,7 +512,7 @@ }, { "cell_type": "markdown", - "id": "6409f80d", + "id": "a47041a7", "metadata": { "editable": true }, @@ -540,7 +539,7 @@ }, { "cell_type": "markdown", - "id": "b429c42f", + "id": "94f82a14", "metadata": { "editable": true }, @@ -552,7 +551,7 @@ }, { "cell_type": "markdown", - "id": "eb3e30bd", + "id": "c10b4b3f", "metadata": { "editable": true }, @@ -564,7 +563,7 @@ }, { "cell_type": "markdown", - "id": "17c17d6b", + "id": "a2ff2fae", "metadata": { "editable": true }, @@ -580,7 +579,7 @@ }, { "cell_type": "markdown", - "id": "47710d84", + "id": "35369c98", "metadata": { "editable": true }, @@ -598,7 +597,7 @@ }, { "cell_type": "markdown", - "id": "762fb3c5", + "id": "c3cb1b39", "metadata": { "editable": true }, @@ -610,7 +609,7 @@ }, { "cell_type": "markdown", - "id": "75064f38", + "id": "78fec5b2", "metadata": { "editable": true }, @@ -622,7 +621,7 @@ }, { "cell_type": "markdown", - "id": "d9635eb5", + "id": "c2cb8b77", "metadata": { "editable": true }, @@ -633,7 +632,7 @@ }, { "cell_type": "markdown", - "id": "d6400e4e", + "id": "672e3ddc", "metadata": { "editable": true }, @@ -645,7 +644,7 @@ }, { "cell_type": "markdown", - "id": "97392f70", + "id": "3d652847", "metadata": { "editable": true }, @@ -655,7 +654,7 @@ }, { "cell_type": "markdown", - "id": "c15005c1", + "id": "e4d1006a", "metadata": { "editable": true }, @@ -667,7 +666,7 @@ }, { "cell_type": "markdown", - "id": "710024a8", + "id": "27e8e9da", "metadata": { "editable": true }, @@ -681,7 +680,7 @@ }, { "cell_type": "markdown", - "id": "506c8710", + "id": "f1b4e3e2", "metadata": { "editable": true }, @@ -693,7 +692,7 @@ }, { "cell_type": "markdown", - "id": "87a4cf3a", + "id": "d247b10c", "metadata": { "editable": true }, @@ -705,7 +704,7 @@ }, { "cell_type": "markdown", - "id": "9980c663", + "id": "9d9b5cae", "metadata": { "editable": true }, @@ -717,7 +716,7 @@ }, { "cell_type": "markdown", - "id": "e24bb7fe", + "id": "28bc97b3", "metadata": { "editable": true }, @@ -727,7 +726,7 @@ }, { "cell_type": "markdown", - "id": "d1e362e8", + "id": "9bd668ca", "metadata": { "editable": true }, @@ -739,7 +738,7 @@ }, { "cell_type": "markdown", - "id": "11e6b29e", + "id": "1f072069", "metadata": { "editable": true }, @@ -751,7 +750,7 @@ }, { "cell_type": "markdown", - "id": "0ca6324b", + "id": "468e0aaf", "metadata": { "editable": true }, @@ -763,7 +762,7 @@ }, { "cell_type": "markdown", - "id": "61ea5a33", + "id": "df18a7bc", "metadata": { "editable": true }, @@ -777,7 +776,7 @@ }, { "cell_type": "markdown", - "id": "eeaff9a1", + "id": "7e778a20", "metadata": { "editable": true }, @@ -789,7 +788,7 @@ }, { "cell_type": "markdown", - "id": "c4b52288", + "id": "7b09debd", "metadata": { "editable": true }, @@ -801,7 +800,7 @@ }, { "cell_type": "markdown", - "id": "57f3e850", + "id": "bf4df408", "metadata": { "editable": true }, @@ -813,7 +812,7 @@ }, { "cell_type": "markdown", - "id": "43813bc8", + "id": "9527ed06", "metadata": { "editable": true }, @@ -823,7 +822,7 @@ }, { "cell_type": "markdown", - "id": "003314c4", + "id": "4b2b0605", "metadata": { "editable": true }, @@ -835,7 +834,7 @@ }, { "cell_type": "markdown", - "id": "5d2571d7", + "id": "8a355b26", "metadata": { "editable": true }, @@ -845,7 +844,7 @@ }, { "cell_type": "markdown", - "id": "4b7551cd", + "id": "ac595cbf", "metadata": { "editable": true }, @@ -857,7 +856,7 @@ }, { "cell_type": "markdown", - "id": "ac7c4c4f", + "id": "823f52d0", "metadata": { "editable": true }, @@ -867,7 +866,7 @@ }, { "cell_type": "markdown", - "id": "6300ae4e", + "id": "a90148b7", "metadata": { "editable": true }, @@ -879,7 +878,7 @@ }, { "cell_type": "markdown", - "id": "40f7b541", + "id": "d923e6f1", "metadata": { "editable": true }, @@ -891,7 +890,7 @@ }, { "cell_type": "markdown", - "id": "8ad576c7", + "id": "1106be29", "metadata": { "editable": true }, @@ -903,7 +902,7 @@ }, { "cell_type": "markdown", - "id": "32da6c26", + "id": "5ed90732", "metadata": { "editable": true }, @@ -918,7 +917,7 @@ }, { "cell_type": "markdown", - "id": "ee25da9a", + "id": "d516517b", "metadata": { "editable": true }, @@ -930,7 +929,7 @@ }, { "cell_type": "markdown", - "id": "0402d646", + "id": "c1379242", "metadata": { "editable": true }, @@ -940,7 +939,7 @@ }, { "cell_type": "markdown", - "id": "9cc8d0a5", + "id": "3724106d", "metadata": { "editable": true }, @@ -952,7 +951,7 @@ }, { "cell_type": "markdown", - "id": "69596f47", + "id": "e085c2ec", "metadata": { "editable": true }, @@ -962,7 +961,7 @@ }, { "cell_type": "markdown", - "id": "dbd25ca3", + "id": "01fa0478", "metadata": { "editable": true }, @@ -974,7 +973,7 @@ }, { "cell_type": "markdown", - "id": "b150ad39", + "id": "8c75a8cd", "metadata": { "editable": true }, @@ -984,7 +983,7 @@ }, { "cell_type": "markdown", - "id": "1d92dcb9", + "id": "e3e275e6", "metadata": { "editable": true }, @@ -996,7 +995,7 @@ }, { "cell_type": "markdown", - "id": "cb0832ad", + "id": "ab39269e", "metadata": { "editable": true }, @@ -1008,7 +1007,7 @@ }, { "cell_type": "markdown", - "id": "7fbab436", + "id": "ce0b7d4b", "metadata": { "editable": true }, @@ -1020,7 +1019,7 @@ }, { "cell_type": "markdown", - "id": "3c0f2a23", + "id": "31fbfca8", "metadata": { "editable": true }, @@ -1030,7 +1029,7 @@ }, { "cell_type": "markdown", - "id": "1e133836", + "id": "f3506f4e", "metadata": { "editable": true }, @@ -1042,7 +1041,7 @@ }, { "cell_type": "markdown", - "id": "b89c0c35", + "id": "6f709ae5", "metadata": { "editable": true }, @@ -1055,7 +1054,7 @@ }, { "cell_type": "markdown", - "id": "aa3f9fa4", + "id": "02fba45f", "metadata": { "editable": true }, @@ -1067,7 +1066,7 @@ }, { "cell_type": "markdown", - "id": "cf9fbd7b", + "id": "af072826", "metadata": { "editable": true }, @@ -1077,7 +1076,7 @@ }, { "cell_type": "markdown", - "id": "8bd036ea", + "id": "35947153", "metadata": { "editable": true }, @@ -1089,7 +1088,7 @@ }, { "cell_type": "markdown", - "id": "7ea6a503", + "id": "84e2032b", "metadata": { "editable": true }, @@ -1099,7 +1098,7 @@ }, { "cell_type": "markdown", - "id": "000c0787", + "id": "46895a0e", "metadata": { "editable": true }, @@ -1111,7 +1110,7 @@ }, { "cell_type": "markdown", - "id": "e536562e", + "id": "b2ffaf06", "metadata": { "editable": true }, @@ -1121,7 +1120,7 @@ }, { "cell_type": "markdown", - "id": "8c1b5fac", + "id": "1f591640", "metadata": { "editable": true }, @@ -1133,7 +1132,7 @@ }, { "cell_type": "markdown", - "id": "27ad9067", + "id": "0322fd59", "metadata": { "editable": true }, @@ -1143,7 +1142,7 @@ }, { "cell_type": "markdown", - "id": "5994a47a", + "id": "1e03255f", "metadata": { "editable": true }, @@ -1155,7 +1154,7 @@ }, { "cell_type": "markdown", - "id": "75f3bf54", + "id": "0a1403f8", "metadata": { "editable": true }, @@ -1165,7 +1164,7 @@ }, { "cell_type": "markdown", - "id": "734ecffe", + "id": "41a42e64", "metadata": { "editable": true }, @@ -1177,7 +1176,7 @@ }, { "cell_type": "markdown", - "id": "5c93ba79", + "id": "e68e4f65", "metadata": { "editable": true }, @@ -1189,7 +1188,7 @@ }, { "cell_type": "markdown", - "id": "5725f528", + "id": "9f4ecb7d", "metadata": { "editable": true }, @@ -1201,7 +1200,7 @@ }, { "cell_type": "markdown", - "id": "e19c9cc8", + "id": "d050fc18", "metadata": { "editable": true }, @@ -1213,7 +1212,7 @@ }, { "cell_type": "markdown", - "id": "26a3b695", + "id": "67837a5c", "metadata": { "editable": true }, @@ -1225,7 +1224,7 @@ }, { "cell_type": "markdown", - "id": "2f62389a", + "id": "562ad5bd", "metadata": { "editable": true }, @@ -1237,7 +1236,7 @@ }, { "cell_type": "markdown", - "id": "ecca56f6", + "id": "bf5751b0", "metadata": { "editable": true }, @@ -1253,7 +1252,7 @@ }, { "cell_type": "markdown", - "id": "746c5a8e", + "id": "6f1a224c", "metadata": { "editable": true }, @@ -1265,7 +1264,7 @@ }, { "cell_type": "markdown", - "id": "2a8ed441", + "id": "4c9387ca", "metadata": { "editable": true }, @@ -1275,7 +1274,7 @@ }, { "cell_type": "markdown", - "id": "484f17f7", + "id": "e55cd91c", "metadata": { "editable": true }, @@ -1287,7 +1286,7 @@ }, { "cell_type": "markdown", - "id": "0540ffa6", + "id": "5834e282", "metadata": { "editable": true }, @@ -1299,13 +1298,13 @@ "important role in optmization algorithms and Principal Component\n", "Analysis as a way to reduce the dimensionality of a machine learning/data analysis\n", "problem.\n", - "\n", + "v\n", "**Linear algebra question:** Can we use the Hessian matrix to say something about properties of the cost function (our optmization problem)? (hint: think about convex or concave problems and how to relate these to a matrix!)." ] }, { "cell_type": "markdown", - "id": "ac84b0c2", + "id": "d79e83c1", "metadata": { "editable": true }, @@ -1317,7 +1316,7 @@ }, { "cell_type": "markdown", - "id": "6b63dd02", + "id": "435ad3ea", "metadata": { "editable": true }, @@ -1329,7 +1328,7 @@ }, { "cell_type": "markdown", - "id": "8d59c3d7", + "id": "3d0c550b", "metadata": { "editable": true }, @@ -1339,7 +1338,7 @@ }, { "cell_type": "markdown", - "id": "e9865bd6", + "id": "1317ebae", "metadata": { "editable": true }, @@ -1351,7 +1350,7 @@ }, { "cell_type": "markdown", - "id": "8849bfb1", + "id": "c8c5a477", "metadata": { "editable": true }, @@ -1361,7 +1360,7 @@ }, { "cell_type": "markdown", - "id": "c195eaa5", + "id": "93d276b1", "metadata": { "editable": true }, @@ -1373,7 +1372,7 @@ }, { "cell_type": "markdown", - "id": "f6c368ec", + "id": "b292af90", "metadata": { "editable": true }, @@ -1383,7 +1382,7 @@ }, { "cell_type": "markdown", - "id": "08c3a03d", + "id": "e8b71fe3", "metadata": { "editable": true }, @@ -1397,7 +1396,7 @@ }, { "cell_type": "markdown", - "id": "4a2f8c99", + "id": "b5b88839", "metadata": { "editable": true }, @@ -1409,7 +1408,7 @@ }, { "cell_type": "markdown", - "id": "daa921af", + "id": "4b65ef7c", "metadata": { "editable": true }, @@ -1420,7 +1419,7 @@ }, { "cell_type": "markdown", - "id": "7461171f", + "id": "36fa0c5d", "metadata": { "editable": true }, @@ -1433,7 +1432,7 @@ { "cell_type": "code", "execution_count": 1, - "id": "ec8b4ca9", + "id": "5bfc7faa", "metadata": { "collapsed": false, "editable": true @@ -1460,7 +1459,7 @@ }, { "cell_type": "markdown", - "id": "a269c12b", + "id": "07764fc2", "metadata": { "editable": true }, @@ -1471,7 +1470,7 @@ { "cell_type": "code", "execution_count": 2, - "id": "5b539f21", + "id": "f299da23", "metadata": { "collapsed": false, "editable": true @@ -1484,7 +1483,7 @@ }, { "cell_type": "markdown", - "id": "4b50e82e", + "id": "839633de", "metadata": { "editable": true }, @@ -1498,7 +1497,7 @@ { "cell_type": "code", "execution_count": 3, - "id": "82a7d7a7", + "id": "abfc6c01", "metadata": { "collapsed": false, "editable": true @@ -1511,7 +1510,7 @@ }, { "cell_type": "markdown", - "id": "09d21070", + "id": "aff33949", "metadata": { "editable": true }, @@ -1522,7 +1521,7 @@ { "cell_type": "code", "execution_count": 4, - "id": "6628a95e", + "id": "67e83121", "metadata": { "collapsed": false, "editable": true @@ -1534,7 +1533,7 @@ }, { "cell_type": "markdown", - "id": "5df2096d", + "id": "14a294e9", "metadata": { "editable": true }, @@ -1545,7 +1544,7 @@ { "cell_type": "code", "execution_count": 5, - "id": "aeb595a0", + "id": "8d80a11c", "metadata": { "collapsed": false, "editable": true @@ -1561,7 +1560,7 @@ }, { "cell_type": "markdown", - "id": "9d806838", + "id": "fd9c909e", "metadata": { "editable": true }, @@ -1572,7 +1571,7 @@ { "cell_type": "code", "execution_count": 6, - "id": "f3ca06ef", + "id": "fcd82857", "metadata": { "collapsed": false, "editable": true @@ -1586,7 +1585,7 @@ }, { "cell_type": "markdown", - "id": "2dabf13f", + "id": "a91adf8f", "metadata": { "editable": true }, @@ -1607,7 +1606,7 @@ }, { "cell_type": "markdown", - "id": "57182c7a", + "id": "24f5afb4", "metadata": { "editable": true }, @@ -1618,7 +1617,7 @@ { "cell_type": "code", "execution_count": 7, - "id": "0739e111", + "id": "093bd1f1", "metadata": { "collapsed": false, "editable": true @@ -1671,7 +1670,7 @@ }, { "cell_type": "markdown", - "id": "04163e48", + "id": "08a5d650", "metadata": { "editable": true }, @@ -1682,7 +1681,7 @@ { "cell_type": "code", "execution_count": 8, - "id": "6ea4cac2", + "id": "26c93a00", "metadata": { "collapsed": false, "editable": true @@ -1707,7 +1706,7 @@ }, { "cell_type": "markdown", - "id": "5d634795", + "id": "0b3c6835", "metadata": { "editable": true }, @@ -1719,7 +1718,7 @@ }, { "cell_type": "markdown", - "id": "3c329913", + "id": "7eae01b1", "metadata": { "editable": true }, @@ -1748,7 +1747,7 @@ }, { "cell_type": "markdown", - "id": "94989a6b", + "id": "f4576ab0", "metadata": { "editable": true }, @@ -1773,7 +1772,7 @@ }, { "cell_type": "markdown", - "id": "4029aff8", + "id": "04b757cc", "metadata": { "editable": true }, @@ -1793,7 +1792,7 @@ }, { "cell_type": "markdown", - "id": "9cc2644b", + "id": "e96e0ec7", "metadata": { "editable": true }, @@ -1820,7 +1819,7 @@ }, { "cell_type": "markdown", - "id": "56108c6e", + "id": "b4e7944d", "metadata": { "editable": true }, @@ -1833,7 +1832,7 @@ }, { "cell_type": "markdown", - "id": "61adc18f", + "id": "385fbd02", "metadata": { "editable": true }, @@ -1845,7 +1844,7 @@ }, { "cell_type": "markdown", - "id": "03a0eed0", + "id": "65176139", "metadata": { "editable": true }, @@ -1856,7 +1855,7 @@ }, { "cell_type": "markdown", - "id": "79e0276b", + "id": "bc2bda95", "metadata": { "editable": true }, @@ -1872,7 +1871,7 @@ { "cell_type": "code", "execution_count": 9, - "id": "98326073", + "id": "f741746f", "metadata": { "collapsed": false, "editable": true @@ -1906,7 +1905,7 @@ }, { "cell_type": "markdown", - "id": "ba7e794f", + "id": "91b66d01", "metadata": { "editable": true }, @@ -1916,7 +1915,7 @@ }, { "cell_type": "markdown", - "id": "082c0ac7", + "id": "376037d2", "metadata": { "editable": true }, @@ -1931,7 +1930,7 @@ }, { "cell_type": "markdown", - "id": "bf91fffc", + "id": "3b6fa136", "metadata": { "editable": true }, @@ -1943,7 +1942,7 @@ }, { "cell_type": "markdown", - "id": "aad33bb5", + "id": "f1087613", "metadata": { "editable": true }, @@ -1953,7 +1952,7 @@ }, { "cell_type": "markdown", - "id": "07fba26a", + "id": "be2cd7be", "metadata": { "editable": true }, @@ -1969,7 +1968,7 @@ { "cell_type": "code", "execution_count": 10, - "id": "69c063ec", + "id": "0ac0921a", "metadata": { "collapsed": false, "editable": true @@ -1986,7 +1985,7 @@ }, { "cell_type": "markdown", - "id": "f3abf824", + "id": "700d2c70", "metadata": { "editable": true }, @@ -1999,7 +1998,7 @@ { "cell_type": "code", "execution_count": 11, - "id": "aba89c8a", + "id": "3556cc72", "metadata": { "collapsed": false, "editable": true @@ -2046,753 +2045,7 @@ }, { "cell_type": "markdown", - "id": "15cbce1e", - "metadata": { - "editable": true - }, - "source": [ - "## More preprocessing examples, two-dimensional example, the Franke function" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "id": "c4c90f56", - "metadata": { - "collapsed": false, - "editable": true - }, - "outputs": [], - "source": [ - "# Common imports\n", - "import os\n", - "import numpy as np\n", - "import pandas as pd\n", - "import matplotlib.pyplot as plt\n", - "import sklearn.linear_model as skl\n", - "from sklearn.metrics import mean_squared_error\n", - "from sklearn.model_selection import train_test_split\n", - "from sklearn.preprocessing import MinMaxScaler, StandardScaler, Normalizer\n", - "\n", - "# Where to save the figures and data files\n", - "PROJECT_ROOT_DIR = \"Results\"\n", - "FIGURE_ID = \"Results/FigureFiles\"\n", - "DATA_ID = \"DataFiles/\"\n", - "\n", - "if not os.path.exists(PROJECT_ROOT_DIR):\n", - " os.mkdir(PROJECT_ROOT_DIR)\n", - "\n", - "if not os.path.exists(FIGURE_ID):\n", - " os.makedirs(FIGURE_ID)\n", - "\n", - "if not os.path.exists(DATA_ID):\n", - " os.makedirs(DATA_ID)\n", - "\n", - "def image_path(fig_id):\n", - " return os.path.join(FIGURE_ID, fig_id)\n", - "\n", - "def data_path(dat_id):\n", - " return os.path.join(DATA_ID, dat_id)\n", - "\n", - "def save_fig(fig_id):\n", - " plt.savefig(image_path(fig_id) + \".png\", format='png')\n", - "\n", - "\n", - "def FrankeFunction(x,y):\n", - "\tterm1 = 0.75*np.exp(-(0.25*(9*x-2)**2) - 0.25*((9*y-2)**2))\n", - "\tterm2 = 0.75*np.exp(-((9*x+1)**2)/49.0 - 0.1*(9*y+1))\n", - "\tterm3 = 0.5*np.exp(-(9*x-7)**2/4.0 - 0.25*((9*y-3)**2))\n", - "\tterm4 = -0.2*np.exp(-(9*x-4)**2 - (9*y-7)**2)\n", - "\treturn term1 + term2 + term3 + term4\n", - "\n", - "\n", - "def create_X(x, y, n ):\n", - "\tif len(x.shape) > 1:\n", - "\t\tx = np.ravel(x)\n", - "\t\ty = np.ravel(y)\n", - "\n", - "\tN = len(x)\n", - "\tl = int((n+1)*(n+2)/2)\t\t# Number of elements in beta\n", - "\tX = np.ones((N,l))\n", - "\n", - "\tfor i in range(1,n+1):\n", - "\t\tq = int((i)*(i+1)/2)\n", - "\t\tfor k in range(i+1):\n", - "\t\t\tX[:,q+k] = (x**(i-k))*(y**k)\n", - "\n", - "\treturn X\n", - "\n", - "\n", - "# Making meshgrid of datapoints and compute Franke's function\n", - "n = 5\n", - "N = 1000\n", - "x = np.sort(np.random.uniform(0, 1, N))\n", - "y = np.sort(np.random.uniform(0, 1, N))\n", - "z = FrankeFunction(x, y)\n", - "X = create_X(x, y, n=n) \n", - "# split in training and test data\n", - "X_train, X_test, y_train, y_test = train_test_split(X,z,test_size=0.2)\n", - "\n", - "\n", - "clf = skl.LinearRegression().fit(X_train, y_train)\n", - "\n", - "# The mean squared error and R2 score\n", - "print(\"MSE before scaling: {:.2f}\".format(mean_squared_error(clf.predict(X_test), y_test)))\n", - "print(\"R2 score before scaling {:.2f}\".format(clf.score(X_test,y_test)))\n", - "\n", - "scaler = StandardScaler()\n", - "scaler.fit(X_train)\n", - "X_train_scaled = scaler.transform(X_train)\n", - "X_test_scaled = scaler.transform(X_test)\n", - "\n", - "print(\"Feature min values before scaling:\\n {}\".format(X_train.min(axis=0)))\n", - "print(\"Feature max values before scaling:\\n {}\".format(X_train.max(axis=0)))\n", - "\n", - "print(\"Feature min values after scaling:\\n {}\".format(X_train_scaled.min(axis=0)))\n", - "print(\"Feature max values after scaling:\\n {}\".format(X_train_scaled.max(axis=0)))\n", - "\n", - "clf = skl.LinearRegression().fit(X_train_scaled, y_train)\n", - "\n", - "\n", - "print(\"MSE after scaling: {:.2f}\".format(mean_squared_error(clf.predict(X_test_scaled), y_test)))\n", - "print(\"R2 score for scaled data: {:.2f}\".format(clf.score(X_test_scaled,y_test)))" - ] - }, - { - "cell_type": "markdown", - "id": "5239bbb2", - "metadata": { - "editable": true - }, - "source": [ - "## To think about, first part\n", - "\n", - "When you are comparing your own code with for example **Scikit-Learn**'s\n", - "library, there are some technicalities to keep in mind. The examples\n", - "here demonstrate some of these aspects with potential pitfalls.\n", - "\n", - "The discussion here focuses on the role of the intercept, how we can\n", - "set up the design matrix, what scaling we should use and other topics\n", - "which tend confuse us.\n", - "\n", - "The intercept can be interpreted as the expected value of our\n", - "target/output variables when all other predictors are set to zero.\n", - "Thus, if we cannot assume that the expected outputs/targets are zero\n", - "when all predictors are zero (the columns in the design matrix), it\n", - "may be a bad idea to implement a model which penalizes the intercept.\n", - "Furthermore, in for example Ridge and Lasso regression (to be discussed in moe detail next week), the default solutions\n", - "from the library **Scikit-Learn** (when not shrinking $\\beta_0$) for the unknown parameters\n", - "$\\boldsymbol{\\beta}$, are derived under the assumption that both $\\boldsymbol{y}$ and\n", - "$\\boldsymbol{X}$ are zero centered, that is we subtract the mean values." - ] - }, - { - "cell_type": "markdown", - "id": "906aa3e6", - "metadata": { - "editable": true - }, - "source": [ - "## More thinking\n", - "\n", - "If our predictors represent different scales, then it is important to\n", - "standardize the design matrix $\\boldsymbol{X}$ by subtracting the mean of each\n", - "column from the corresponding column and dividing the column with its\n", - "standard deviation. Most machine learning libraries do this as a default. This means that if you compare your code with the results from a given library,\n", - "the results may differ. \n", - "\n", - "The\n", - "[Standadscaler](https://scikit-learn.org/stable/modules/generated/sklearn.preprocessing.StandardScaler.html)\n", - "function in **Scikit-Learn** does this for us. For the data sets we\n", - "have been studying in our various examples, the data are in many cases\n", - "already scaled and there is no need to scale them. You as a user of different machine learning algorithms, should always perform a\n", - "survey of your data, with a critical assessment of them in case you need to scale the data.\n", - "\n", - "If you need to scale the data, not doing so will give an *unfair*\n", - "penalization of the parameters since their magnitude depends on the\n", - "scale of their corresponding predictor.\n", - "\n", - "Suppose as an example that you \n", - "you have an input variable given by the heights of different persons.\n", - "Human height might be measured in inches or meters or\n", - "kilometers. If measured in kilometers, a standard linear regression\n", - "model with this predictor would probably give a much bigger\n", - "coefficient term, than if measured in millimeters.\n", - "This can clearly lead to problems in evaluating the cost/loss functions." - ] - }, - { - "cell_type": "markdown", - "id": "58153d2d", - "metadata": { - "editable": true - }, - "source": [ - "## Still thinking\n", - "\n", - "Keep in mind that when you transform your data set before training a model, the same transformation needs to be done\n", - "on your eventual new data set before making a prediction. If we translate this into a Python code, it would could be implemented as follows\n", - "(note that the lines are commented since the model function has not been defined)" - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "id": "b0d28bb6", - "metadata": { - "collapsed": false, - "editable": true - }, - "outputs": [], - "source": [ - "#Model training, we compute the mean value of y and X\n", - "y_train_mean = np.mean(y_train)\n", - "X_train_mean = np.mean(X_train,axis=0)\n", - "X_train = X_train - X_train_mean\n", - "y_train = y_train - y_train_mean\n", - "\n", - "# The we fit our model with the training data\n", - "#trained_model = some_model.fit(X_train,y_train)\n", - "\n", - "\n", - "#Model prediction, we need also to transform our data set used for the prediction.\n", - "X_test = X_test - X_train_mean #Use mean from training data\n", - "#y_pred = trained_model(X_test)\n", - "y_pred = y_pred + y_train_mean" - ] - }, - { - "cell_type": "markdown", - "id": "7a20664f", - "metadata": { - "editable": true - }, - "source": [ - "## What does centering (subtracting the mean values) mean mathematically?\n", - "\n", - "Let us try to understand what this may imply mathematically when we\n", - "subtract the mean values, also known as *zero centering*. For\n", - "simplicity, we will focus on ordinary regression, as done in the above example.\n", - "\n", - "The cost/loss function for regression is" - ] - }, - { - "cell_type": "markdown", - "id": "bdfa73a1", - "metadata": { - "editable": true - }, - "source": [ - "$$\n", - "C(\\beta_0, \\beta_1, ... , \\beta_{p-1}) = \\frac{1}{n}\\sum_{i=0}^{n} \\left(y_i - \\beta_0 - \\sum_{j=1}^{p-1} X_{ij}\\beta_j\\right)^2,.\n", - "$$" - ] - }, - { - "cell_type": "markdown", - "id": "3ddce267", - "metadata": { - "editable": true - }, - "source": [ - "Recall also that we use the squared value since this leads to an increase of the penalty for higher differences between predicted and output/target values.\n", - "\n", - "What we have done is to single out the $\\beta_0$ term in the definition of the mean squared error (MSE).\n", - "The design matrix\n", - "$X$ does in this case not contain any intercept column.\n", - "When we take the derivative with respect to $\\beta_0$, we want the derivative to obey" - ] - }, - { - "cell_type": "markdown", - "id": "ae248818", - "metadata": { - "editable": true - }, - "source": [ - "$$\n", - "\\frac{\\partial C}{\\partial \\beta_j} = 0,\n", - "$$" - ] - }, - { - "cell_type": "markdown", - "id": "4b619661", - "metadata": { - "editable": true - }, - "source": [ - "for all $j$. For $\\beta_0$ we have" - ] - }, - { - "cell_type": "markdown", - "id": "18485c0e", - "metadata": { - "editable": true - }, - "source": [ - "$$\n", - "\\frac{\\partial C}{\\partial \\beta_0} = -\\frac{2}{n}\\sum_{i=0}^{n-1} \\left(y_i - \\beta_0 - \\sum_{j=1}^{p-1} X_{ij} \\beta_j\\right).\n", - "$$" - ] - }, - { - "cell_type": "markdown", - "id": "9c22d445", - "metadata": { - "editable": true - }, - "source": [ - "Multiplying away the constant $2/n$, we obtain" - ] - }, - { - "cell_type": "markdown", - "id": "77ddf7e3", - "metadata": { - "editable": true - }, - "source": [ - "$$\n", - "\\sum_{i=0}^{n-1} \\beta_0 = \\sum_{i=0}^{n-1}y_i - \\sum_{i=0}^{n-1} \\sum_{j=1}^{p-1} X_{ij} \\beta_j.\n", - "$$" - ] - }, - { - "cell_type": "markdown", - "id": "602a1c3a", - "metadata": { - "editable": true - }, - "source": [ - "## Further Manipulations\n", - "\n", - "Let us special first to the case where we have only two parameters $\\beta_0$ and $\\beta_1$.\n", - "Our result for $\\beta_0$ simplifies then to" - ] - }, - { - "cell_type": "markdown", - "id": "5eae89e5", - "metadata": { - "editable": true - }, - "source": [ - "$$\n", - "n\\beta_0 = \\sum_{i=0}^{n-1}y_i - \\sum_{i=0}^{n-1} X_{i1} \\beta_1.\n", - "$$" - ] - }, - { - "cell_type": "markdown", - "id": "0d58bcd1", - "metadata": { - "editable": true - }, - "source": [ - "We obtain then" - ] - }, - { - "cell_type": "markdown", - "id": "2fa1a7c8", - "metadata": { - "editable": true - }, - "source": [ - "$$\n", - "\\beta_0 = \\frac{1}{n}\\sum_{i=0}^{n-1}y_i - \\beta_1\\frac{1}{n}\\sum_{i=0}^{n-1} X_{i1}.\n", - "$$" - ] - }, - { - "cell_type": "markdown", - "id": "07f18ad5", - "metadata": { - "editable": true - }, - "source": [ - "If we define" - ] - }, - { - "cell_type": "markdown", - "id": "acb4f352", - "metadata": { - "editable": true - }, - "source": [ - "$$\n", - "\\mu_1=\\frac{1}{n}\\sum_{i=0}^{n-1} (X_{i1},\n", - "$$" - ] - }, - { - "cell_type": "markdown", - "id": "101abab9", - "metadata": { - "editable": true - }, - "source": [ - "and if we define the mean value of the outputs as" - ] - }, - { - "cell_type": "markdown", - "id": "9f0c7afc", - "metadata": { - "editable": true - }, - "source": [ - "$$\n", - "\\mu_y=\\frac{1}{n}\\sum_{i=0}^{n-1}y_i,\n", - "$$" - ] - }, - { - "cell_type": "markdown", - "id": "c26112a5", - "metadata": { - "editable": true - }, - "source": [ - "we have" - ] - }, - { - "cell_type": "markdown", - "id": "3dd97935", - "metadata": { - "editable": true - }, - "source": [ - "$$\n", - "\\beta_0 = \\mu_y - \\beta_1\\mu_{1}.\n", - "$$" - ] - }, - { - "cell_type": "markdown", - "id": "6ffc6800", - "metadata": { - "editable": true - }, - "source": [ - "In the general case, that is we have more parameters than $\\beta_0$ and $\\beta_1$, we have" - ] - }, - { - "cell_type": "markdown", - "id": "3005c990", - "metadata": { - "editable": true - }, - "source": [ - "$$\n", - "\\beta_0 = \\frac{1}{n}\\sum_{i=0}^{n-1}y_i - \\frac{1}{n}\\sum_{i=0}^{n-1}\\sum_{j=1}^{p-1} X_{ij}\\beta_j.\n", - "$$" - ] - }, - { - "cell_type": "markdown", - "id": "dd1a460b", - "metadata": { - "editable": true - }, - "source": [ - "Replacing $y_i$ with $y_i - y_i - \\overline{\\boldsymbol{y}}$ and centering also our design matrix results in a cost function (in vector-matrix disguise)" - ] - }, - { - "cell_type": "markdown", - "id": "40114117", - "metadata": { - "editable": true - }, - "source": [ - "$$\n", - "C(\\boldsymbol{\\beta}) = (\\boldsymbol{\\tilde{y}} - \\tilde{X}\\boldsymbol{\\beta})^T(\\boldsymbol{\\tilde{y}} - \\tilde{X}\\boldsymbol{\\beta}).\n", - "$$" - ] - }, - { - "cell_type": "markdown", - "id": "efcef87d", - "metadata": { - "editable": true - }, - "source": [ - "## Wrapping it up\n", - "\n", - "If we minimize with respect to $\\boldsymbol{\\beta}$ we have then" - ] - }, - { - "cell_type": "markdown", - "id": "e6f98099", - "metadata": { - "editable": true - }, - "source": [ - "$$\n", - "\\hat{\\boldsymbol{\\beta}} = (\\tilde{X}^T\\tilde{X})^{-1}\\tilde{X}^T\\boldsymbol{\\tilde{y}},\n", - "$$" - ] - }, - { - "cell_type": "markdown", - "id": "d24f866b", - "metadata": { - "editable": true - }, - "source": [ - "where $\\boldsymbol{\\tilde{y}} = \\boldsymbol{y} - \\overline{\\boldsymbol{y}}$\n", - "and $\\tilde{X}_{ij} = X_{ij} - \\frac{1}{n}\\sum_{k=0}^{n-1}X_{kj}$.\n", - "\n", - "For Ridge regression we need to add $\\lambda \\boldsymbol{\\beta}^T\\boldsymbol{\\beta}$ to the cost function and get then" - ] - }, - { - "cell_type": "markdown", - "id": "e161290b", - "metadata": { - "editable": true - }, - "source": [ - "$$\n", - "\\hat{\\boldsymbol{\\beta}} = (\\tilde{X}^T\\tilde{X} + \\lambda I)^{-1}\\tilde{X}^T\\boldsymbol{\\tilde{y}}.\n", - "$$" - ] - }, - { - "cell_type": "markdown", - "id": "93daa4eb", - "metadata": { - "editable": true - }, - "source": [ - "What does this mean? And why do we insist on all this? Let us look at some examples." - ] - }, - { - "cell_type": "markdown", - "id": "7807e41e", - "metadata": { - "editable": true - }, - "source": [ - "## Linear Regression code, Intercept handling first\n", - "\n", - "This code shows a simple first-order fit to a data set using the above transformed data, where we consider the role of the intercept first, by either excluding it or including it (*code example thanks to Øyvind Sigmundson Schøyen*). Here our scaling of the data is done by subtracting the mean values only.\n", - "Note also that we do not split the data into training and test." - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "id": "269cd6a7", - "metadata": { - "collapsed": false, - "editable": true - }, - "outputs": [], - "source": [ - "import numpy as np\n", - "import matplotlib.pyplot as plt\n", - "\n", - "from sklearn.linear_model import LinearRegression\n", - "\n", - "\n", - "np.random.seed(2021)\n", - "\n", - "def MSE(y_data,y_model):\n", - " n = np.size(y_model)\n", - " return np.sum((y_data-y_model)**2)/n\n", - "\n", - "\n", - "def fit_beta(X, y):\n", - " return np.linalg.pinv(X.T @ X) @ X.T @ y\n", - "\n", - "\n", - "true_beta = [2, 0.5, 3.7]\n", - "\n", - "x = np.linspace(0, 1, 11)\n", - "y = np.sum(\n", - " np.asarray([x ** p * b for p, b in enumerate(true_beta)]), axis=0\n", - ") + 0.1 * np.random.normal(size=len(x))\n", - "\n", - "degree = 3\n", - "X = np.zeros((len(x), degree))\n", - "\n", - "# Include the intercept in the design matrix\n", - "for p in range(degree):\n", - " X[:, p] = x ** p\n", - "\n", - "beta = fit_beta(X, y)\n", - "\n", - "# Intercept is included in the design matrix\n", - "skl = LinearRegression(fit_intercept=False).fit(X, y)\n", - "\n", - "print(f\"True beta: {true_beta}\")\n", - "print(f\"Fitted beta: {beta}\")\n", - "print(f\"Sklearn fitted beta: {skl.coef_}\")\n", - "ypredictOwn = X @ beta\n", - "ypredictSKL = skl.predict(X)\n", - "print(f\"MSE with intercept column\")\n", - "print(MSE(y,ypredictOwn))\n", - "print(f\"MSE with intercept column from SKL\")\n", - "print(MSE(y,ypredictSKL))\n", - "\n", - "\n", - "plt.figure()\n", - "plt.scatter(x, y, label=\"Data\")\n", - "plt.plot(x, X @ beta, label=\"Fit\")\n", - "plt.plot(x, skl.predict(X), label=\"Sklearn (fit_intercept=False)\")\n", - "\n", - "\n", - "# Do not include the intercept in the design matrix\n", - "X = np.zeros((len(x), degree - 1))\n", - "\n", - "for p in range(degree - 1):\n", - " X[:, p] = x ** (p + 1)\n", - "\n", - "# Intercept is not included in the design matrix\n", - "skl = LinearRegression(fit_intercept=True).fit(X, y)\n", - "\n", - "# Use centered values for X and y when computing coefficients\n", - "y_offset = np.average(y, axis=0)\n", - "X_offset = np.average(X, axis=0)\n", - "\n", - "beta = fit_beta(X - X_offset, y - y_offset)\n", - "intercept = np.mean(y_offset - X_offset @ beta)\n", - "\n", - "print(f\"Manual intercept: {intercept}\")\n", - "print(f\"Fitted beta (wiothout intercept): {beta}\")\n", - "print(f\"Sklearn intercept: {skl.intercept_}\")\n", - "print(f\"Sklearn fitted beta (without intercept): {skl.coef_}\")\n", - "ypredictOwn = X @ beta\n", - "ypredictSKL = skl.predict(X)\n", - "print(f\"MSE with Manual intercept\")\n", - "print(MSE(y,ypredictOwn+intercept))\n", - "print(f\"MSE with Sklearn intercept\")\n", - "print(MSE(y,ypredictSKL))\n", - "\n", - "plt.plot(x, X @ beta + intercept, \"--\", label=\"Fit (manual intercept)\")\n", - "plt.plot(x, skl.predict(X), \"--\", label=\"Sklearn (fit_intercept=True)\")\n", - "plt.grid()\n", - "plt.legend()\n", - "\n", - "plt.show()" - ] - }, - { - "cell_type": "markdown", - "id": "69344fc7", - "metadata": { - "editable": true - }, - "source": [ - "The intercept is the value of our output/target variable\n", - "when all our features are zero and our function crosses the $y$-axis (for a one-dimensional case). \n", - "\n", - "Printing the MSE, we see first that both methods give the same MSE, as\n", - "they should. However, when we move to for example Ridge regression (discussed next week),\n", - "the way we treat the intercept may give a larger or smaller MSE,\n", - "meaning that the MSE can be penalized by the value of the\n", - "intercept. Not including the intercept in the fit, means that the\n", - "regularization term does not include $\\beta_0$. For different values\n", - "of $\\lambda$, this may lead to differing MSE values. \n", - "\n", - "To remind the reader, the regularization term, with the intercept in Ridge regression is given by" - ] - }, - { - "cell_type": "markdown", - "id": "409afa98", - "metadata": { - "editable": true - }, - "source": [ - "$$\n", - "\\lambda \\vert\\vert \\boldsymbol{\\beta} \\vert\\vert_2^2 = \\lambda \\sum_{j=0}^{p-1}\\beta_j^2,\n", - "$$" - ] - }, - { - "cell_type": "markdown", - "id": "069a6d37", - "metadata": { - "editable": true - }, - "source": [ - "but when we take out the intercept, this equation becomes" - ] - }, - { - "cell_type": "markdown", - "id": "be8127e9", - "metadata": { - "editable": true - }, - "source": [ - "$$\n", - "\\lambda \\vert\\vert \\boldsymbol{\\beta} \\vert\\vert_2^2 = \\lambda \\sum_{j=1}^{p-1}\\beta_j^2.\n", - "$$" - ] - }, - { - "cell_type": "markdown", - "id": "81afd3d2", - "metadata": { - "editable": true - }, - "source": [ - "For Lasso regression we have" - ] - }, - { - "cell_type": "markdown", - "id": "3d305ea0", - "metadata": { - "editable": true - }, - "source": [ - "$$\n", - "\\lambda \\vert\\vert \\boldsymbol{\\beta} \\vert\\vert_1 = \\lambda \\sum_{j=1}^{p-1}\\vert\\beta_j\\vert.\n", - "$$" - ] - }, - { - "cell_type": "markdown", - "id": "46a309f4", - "metadata": { - "editable": true - }, - "source": [ - "It means that, when scaling the design matrix and the outputs/targets,\n", - "by subtracting the mean values, we have an optimization problem which\n", - "is not penalized by the intercept. The MSE value can then be smaller\n", - "since it focuses only on the remaining quantities. If we however bring\n", - "back the intercept, we will get an MSE which then contains the\n", - "intercept. This becomes more important when we discuss Ridge and Lasso\n", - "regression next week." - ] - }, - { - "cell_type": "markdown", - "id": "7784c499", - "metadata": { - "editable": true - }, - "source": [ - "## Material for lecture Monday, August 26" - ] - }, - { - "cell_type": "markdown", - "id": "9c88c4c0", + "id": "88222231", "metadata": { "editable": true }, @@ -2806,7 +2059,7 @@ }, { "cell_type": "markdown", - "id": "464f230e", + "id": "9fa5b4c6", "metadata": { "editable": true }, @@ -2818,7 +2071,7 @@ }, { "cell_type": "markdown", - "id": "08aad1db", + "id": "81bb7765", "metadata": { "editable": true }, @@ -2830,7 +2083,7 @@ }, { "cell_type": "markdown", - "id": "4d578f5f", + "id": "e6d5287b", "metadata": { "editable": true }, @@ -2842,7 +2095,7 @@ }, { "cell_type": "markdown", - "id": "de05c7a1", + "id": "c95f3611", "metadata": { "editable": true }, @@ -2852,7 +2105,7 @@ }, { "cell_type": "markdown", - "id": "f53694e6", + "id": "b688fc2e", "metadata": { "editable": true }, @@ -2864,7 +2117,7 @@ }, { "cell_type": "markdown", - "id": "4f5ea350", + "id": "3109192b", "metadata": { "editable": true }, @@ -2874,7 +2127,7 @@ }, { "cell_type": "markdown", - "id": "3bfa3efd", + "id": "833ca3e1", "metadata": { "editable": true }, @@ -2886,7 +2139,7 @@ }, { "cell_type": "markdown", - "id": "de3d124c", + "id": "f2f8408c", "metadata": { "editable": true }, @@ -2897,7 +2150,7 @@ }, { "cell_type": "markdown", - "id": "6e640f49", + "id": "b93b6e5d", "metadata": { "editable": true }, @@ -2909,7 +2162,7 @@ }, { "cell_type": "markdown", - "id": "5950f185", + "id": "71a3aeb6", "metadata": { "editable": true }, @@ -2921,7 +2174,7 @@ }, { "cell_type": "markdown", - "id": "1b3fc92e", + "id": "cd016113", "metadata": { "editable": true }, @@ -2931,7 +2184,7 @@ }, { "cell_type": "markdown", - "id": "de268fe9", + "id": "510e85fb", "metadata": { "editable": true }, @@ -2943,7 +2196,7 @@ }, { "cell_type": "markdown", - "id": "fa1d18aa", + "id": "b00cee57", "metadata": { "editable": true }, @@ -2955,7 +2208,7 @@ }, { "cell_type": "markdown", - "id": "ef568162", + "id": "9ee31587", "metadata": { "editable": true }, @@ -2965,7 +2218,7 @@ }, { "cell_type": "markdown", - "id": "a582cf65", + "id": "deca37bb", "metadata": { "editable": true }, @@ -2977,7 +2230,7 @@ }, { "cell_type": "markdown", - "id": "8f5f63b8", + "id": "836d9d6c", "metadata": { "editable": true }, @@ -2987,7 +2240,7 @@ }, { "cell_type": "markdown", - "id": "5a4ec385", + "id": "eee14cd2", "metadata": { "editable": true }, @@ -2999,7 +2252,7 @@ }, { "cell_type": "markdown", - "id": "31baddb6", + "id": "40906c45", "metadata": { "editable": true }, @@ -3009,7 +2262,7 @@ }, { "cell_type": "markdown", - "id": "e604e0a0", + "id": "09e6f2ed", "metadata": { "editable": true }, @@ -3049,7 +2302,7 @@ }, { "cell_type": "markdown", - "id": "a7efcc07", + "id": "ffd73218", "metadata": { "editable": true }, @@ -3066,7 +2319,7 @@ }, { "cell_type": "markdown", - "id": "9a63946b", + "id": "93e66912", "metadata": { "editable": true }, @@ -3089,7 +2342,7 @@ }, { "cell_type": "markdown", - "id": "91cd1899", + "id": "600d07ab", "metadata": { "editable": true }, @@ -3106,7 +2359,7 @@ }, { "cell_type": "markdown", - "id": "9f6ff71b", + "id": "46e320a9", "metadata": { "editable": true }, @@ -3125,7 +2378,7 @@ }, { "cell_type": "markdown", - "id": "9189b8e5", + "id": "b23338e4", "metadata": { "editable": true }, @@ -3136,7 +2389,7 @@ }, { "cell_type": "markdown", - "id": "e44ae7a3", + "id": "bc566714", "metadata": { "editable": true }, @@ -3148,7 +2401,7 @@ }, { "cell_type": "markdown", - "id": "479ed320", + "id": "ea1b8269", "metadata": { "editable": true }, @@ -3166,7 +2419,7 @@ }, { "cell_type": "markdown", - "id": "2e06a501", + "id": "9aee9220", "metadata": { "editable": true }, @@ -3182,7 +2435,7 @@ }, { "cell_type": "markdown", - "id": "123814d1", + "id": "61e7fbda", "metadata": { "editable": true }, @@ -3194,7 +2447,7 @@ }, { "cell_type": "markdown", - "id": "6e68558c", + "id": "afcc930d", "metadata": { "editable": true }, @@ -3204,7 +2457,7 @@ }, { "cell_type": "markdown", - "id": "2ed11f21", + "id": "6ed4a916", "metadata": { "editable": true }, @@ -3219,7 +2472,7 @@ }, { "cell_type": "markdown", - "id": "36909b98", + "id": "20949d31", "metadata": { "editable": true }, @@ -3231,7 +2484,7 @@ }, { "cell_type": "markdown", - "id": "fd68db68", + "id": "1ad7169f", "metadata": { "editable": true }, @@ -3241,7 +2494,7 @@ }, { "cell_type": "markdown", - "id": "eb87bdc3", + "id": "4d8a46e3", "metadata": { "editable": true }, @@ -3253,7 +2506,7 @@ }, { "cell_type": "markdown", - "id": "8f0dc332", + "id": "79a96264", "metadata": { "editable": true }, @@ -3263,7 +2516,7 @@ }, { "cell_type": "markdown", - "id": "8f7d733a", + "id": "0d30c476", "metadata": { "editable": true }, @@ -3275,7 +2528,7 @@ }, { "cell_type": "markdown", - "id": "64baeb5d", + "id": "d8320347", "metadata": { "editable": true }, @@ -3287,7 +2540,7 @@ }, { "cell_type": "markdown", - "id": "07f2b298", + "id": "54e7ef4a", "metadata": { "editable": true }, @@ -3302,7 +2555,7 @@ }, { "cell_type": "markdown", - "id": "1ccc8b83", + "id": "ea96bab9", "metadata": { "editable": true }, @@ -3313,7 +2566,7 @@ }, { "cell_type": "markdown", - "id": "7b475271", + "id": "8f610591", "metadata": { "editable": true }, @@ -3333,7 +2586,7 @@ }, { "cell_type": "markdown", - "id": "e6ece141", + "id": "3cbcadcc", "metadata": { "editable": true }, @@ -3345,7 +2598,7 @@ }, { "cell_type": "markdown", - "id": "1e833f59", + "id": "3d6fa2f6", "metadata": { "editable": true }, @@ -3355,7 +2608,7 @@ }, { "cell_type": "markdown", - "id": "0be33d71", + "id": "03adee16", "metadata": { "editable": true }, @@ -3367,7 +2620,7 @@ }, { "cell_type": "markdown", - "id": "d668cac1", + "id": "e2f52707", "metadata": { "editable": true }, @@ -3396,7 +2649,7 @@ }, { "cell_type": "markdown", - "id": "86b628d5", + "id": "143557e4", "metadata": { "editable": true }, @@ -3423,7 +2676,7 @@ }, { "cell_type": "markdown", - "id": "e6c9ce31", + "id": "8c5068c0", "metadata": { "editable": true }, @@ -3433,8 +2686,8 @@ }, { "cell_type": "code", - "execution_count": 15, - "id": "c6317c42", + "execution_count": 12, + "id": "090ea41c", "metadata": { "collapsed": false, "editable": true @@ -3474,7 +2727,7 @@ }, { "cell_type": "markdown", - "id": "df27fd06", + "id": "7d055231", "metadata": { "editable": true }, @@ -3491,7 +2744,7 @@ }, { "cell_type": "markdown", - "id": "c63a8b89", + "id": "5d1a15fe", "metadata": { "editable": true }, @@ -3514,7 +2767,7 @@ }, { "cell_type": "markdown", - "id": "79ae927b", + "id": "ee3105fb", "metadata": { "editable": true }, @@ -3528,7 +2781,7 @@ }, { "cell_type": "markdown", - "id": "79975321", + "id": "b62baed5", "metadata": { "editable": true }, @@ -3547,7 +2800,7 @@ }, { "cell_type": "markdown", - "id": "b257306d", + "id": "1c4c8788", "metadata": { "editable": true }, @@ -3557,7 +2810,7 @@ }, { "cell_type": "markdown", - "id": "c2bdaa0e", + "id": "a6c47646", "metadata": { "editable": true }, @@ -3569,7 +2822,7 @@ }, { "cell_type": "markdown", - "id": "45cfadf2", + "id": "d5903c9f", "metadata": { "editable": true }, @@ -3583,7 +2836,7 @@ }, { "cell_type": "markdown", - "id": "9c67eed3", + "id": "64e17d36", "metadata": { "editable": true }, @@ -3595,7 +2848,7 @@ }, { "cell_type": "markdown", - "id": "1774315a", + "id": "572c9d3f", "metadata": { "editable": true }, @@ -3605,7 +2858,7 @@ }, { "cell_type": "markdown", - "id": "b9f897de", + "id": "3f1a2cb9", "metadata": { "editable": true }, @@ -3617,7 +2870,7 @@ }, { "cell_type": "markdown", - "id": "f176c943", + "id": "41708552", "metadata": { "editable": true }, @@ -3634,7 +2887,7 @@ }, { "cell_type": "markdown", - "id": "d20b14e8", + "id": "97003366", "metadata": { "editable": true }, @@ -3644,7 +2897,7 @@ }, { "cell_type": "markdown", - "id": "0492da46", + "id": "3bb5d954", "metadata": { "editable": true }, @@ -3660,7 +2913,7 @@ }, { "cell_type": "markdown", - "id": "fcee2cab", + "id": "0c60d5d7", "metadata": { "editable": true }, @@ -3670,7 +2923,7 @@ }, { "cell_type": "markdown", - "id": "8f9bc482", + "id": "bcb087ce", "metadata": { "editable": true }, @@ -3686,7 +2939,7 @@ }, { "cell_type": "markdown", - "id": "f9bbf701", + "id": "b718bf55", "metadata": { "editable": true }, @@ -3696,7 +2949,7 @@ }, { "cell_type": "markdown", - "id": "3c3775bc", + "id": "3d4eef3e", "metadata": { "editable": true }, @@ -3712,7 +2965,7 @@ }, { "cell_type": "markdown", - "id": "2f934004", + "id": "9a61fbcc", "metadata": { "editable": true }, @@ -3722,7 +2975,7 @@ }, { "cell_type": "markdown", - "id": "32659f1f", + "id": "bc19b5f2", "metadata": { "editable": true }, @@ -3739,7 +2992,7 @@ }, { "cell_type": "markdown", - "id": "b3456630", + "id": "4449fbb0", "metadata": { "editable": true }, @@ -3751,7 +3004,7 @@ }, { "cell_type": "markdown", - "id": "16472f5e", + "id": "31bcf6bb", "metadata": { "editable": true }, @@ -3763,7 +3016,7 @@ }, { "cell_type": "markdown", - "id": "c840ac36", + "id": "844ac1e6", "metadata": { "editable": true }, @@ -3775,7 +3028,7 @@ }, { "cell_type": "markdown", - "id": "61672015", + "id": "22f3a41d", "metadata": { "editable": true }, @@ -3785,7 +3038,7 @@ }, { "cell_type": "markdown", - "id": "a4675701", + "id": "15d8fff5", "metadata": { "editable": true }, @@ -3797,7 +3050,7 @@ }, { "cell_type": "markdown", - "id": "10e8cee6", + "id": "430bf7ee", "metadata": { "editable": true }, @@ -3809,7 +3062,7 @@ }, { "cell_type": "markdown", - "id": "73b4187e", + "id": "720d4fb8", "metadata": { "editable": true }, @@ -3821,7 +3074,7 @@ }, { "cell_type": "markdown", - "id": "9bf0f81b", + "id": "8deee9d7", "metadata": { "editable": true }, @@ -3831,7 +3084,7 @@ }, { "cell_type": "markdown", - "id": "1a36a696", + "id": "897ea593", "metadata": { "editable": true }, @@ -3843,7 +3096,7 @@ }, { "cell_type": "markdown", - "id": "594232da", + "id": "5321f718", "metadata": { "editable": true }, @@ -3853,7 +3106,7 @@ }, { "cell_type": "markdown", - "id": "c997de31", + "id": "ba34da1a", "metadata": { "editable": true }, @@ -3865,7 +3118,7 @@ }, { "cell_type": "markdown", - "id": "50489dee", + "id": "0e8a413f", "metadata": { "editable": true }, @@ -3881,7 +3134,7 @@ }, { "cell_type": "markdown", - "id": "b49a358b", + "id": "dd064267", "metadata": { "editable": true }, @@ -3893,7 +3146,7 @@ }, { "cell_type": "markdown", - "id": "668564a0", + "id": "3a1bc23a", "metadata": { "editable": true }, @@ -3905,7 +3158,7 @@ }, { "cell_type": "markdown", - "id": "f1ad1def", + "id": "3bc120c3", "metadata": { "editable": true }, @@ -3915,7 +3168,7 @@ }, { "cell_type": "markdown", - "id": "9a987d59", + "id": "fa44e331", "metadata": { "editable": true }, @@ -3927,7 +3180,7 @@ }, { "cell_type": "markdown", - "id": "bb7ec9c8", + "id": "2f7d8c59", "metadata": { "editable": true }, @@ -3938,7 +3191,7 @@ }, { "cell_type": "markdown", - "id": "261faade", + "id": "e5c786e1", "metadata": { "editable": true }, @@ -3950,7 +3203,7 @@ }, { "cell_type": "markdown", - "id": "16b140aa", + "id": "d4c7d4c6", "metadata": { "editable": true }, @@ -3960,7 +3213,7 @@ }, { "cell_type": "markdown", - "id": "802b4d9a", + "id": "c8bf0455", "metadata": { "editable": true }, @@ -3972,7 +3225,7 @@ }, { "cell_type": "markdown", - "id": "e9eedf0d", + "id": "b8a2bdb3", "metadata": { "editable": true }, @@ -3982,7 +3235,7 @@ }, { "cell_type": "markdown", - "id": "079b5e73", + "id": "20e30049", "metadata": { "editable": true }, @@ -3994,7 +3247,7 @@ }, { "cell_type": "markdown", - "id": "62bf0334", + "id": "801ec139", "metadata": { "editable": true }, @@ -4005,7 +3258,7 @@ }, { "cell_type": "markdown", - "id": "d1cc04f6", + "id": "66a71bf6", "metadata": { "editable": true }, @@ -4017,7 +3270,7 @@ }, { "cell_type": "markdown", - "id": "52ee8a82", + "id": "975d23d9", "metadata": { "editable": true }, @@ -4035,7 +3288,7 @@ }, { "cell_type": "markdown", - "id": "fac21326", + "id": "93c0159c", "metadata": { "editable": true }, @@ -4051,7 +3304,7 @@ }, { "cell_type": "markdown", - "id": "a0735dee", + "id": "29add97b", "metadata": { "editable": true }, @@ -4063,7 +3316,7 @@ }, { "cell_type": "markdown", - "id": "43d1c941", + "id": "b17c198b", "metadata": { "editable": true }, @@ -4075,7 +3328,7 @@ }, { "cell_type": "markdown", - "id": "3b13ad47", + "id": "ea1d3d0b", "metadata": { "editable": true }, @@ -4087,7 +3340,7 @@ }, { "cell_type": "markdown", - "id": "ab30b507", + "id": "a169ccb3", "metadata": { "editable": true }, @@ -4100,7 +3353,7 @@ }, { "cell_type": "markdown", - "id": "15358e5b", + "id": "c90bce62", "metadata": { "editable": true }, @@ -4116,7 +3369,7 @@ }, { "cell_type": "markdown", - "id": "f096b326", + "id": "3b26baba", "metadata": { "editable": true }, @@ -4130,7 +3383,7 @@ }, { "cell_type": "markdown", - "id": "34123b65", + "id": "8331409e", "metadata": { "editable": true }, @@ -4140,7 +3393,7 @@ }, { "cell_type": "markdown", - "id": "1a76a2d1", + "id": "4d391ba3", "metadata": { "editable": true }, @@ -4152,7 +3405,7 @@ }, { "cell_type": "markdown", - "id": "09a6204c", + "id": "46947320", "metadata": { "editable": true }, @@ -4162,7 +3415,7 @@ }, { "cell_type": "markdown", - "id": "728d0a62", + "id": "acf0ace2", "metadata": { "editable": true }, @@ -4174,7 +3427,7 @@ }, { "cell_type": "markdown", - "id": "40c723e8", + "id": "02563a13", "metadata": { "editable": true }, @@ -4184,7 +3437,7 @@ }, { "cell_type": "markdown", - "id": "7d62d8dc", + "id": "349de4e6", "metadata": { "editable": true }, @@ -4198,7 +3451,7 @@ }, { "cell_type": "markdown", - "id": "a632eae9", + "id": "39e116df", "metadata": { "editable": true }, @@ -4215,7 +3468,7 @@ }, { "cell_type": "markdown", - "id": "dff41734", + "id": "30d2f2c3", "metadata": { "editable": true }, @@ -4231,7 +3484,7 @@ }, { "cell_type": "markdown", - "id": "a30f12ee", + "id": "7e5a597b", "metadata": { "editable": true }, @@ -4243,7 +3496,7 @@ }, { "cell_type": "markdown", - "id": "c8f85f57", + "id": "4514d138", "metadata": { "editable": true }, @@ -4256,7 +3509,7 @@ }, { "cell_type": "markdown", - "id": "342d3a4b", + "id": "581de71b", "metadata": { "editable": true }, @@ -4270,7 +3523,7 @@ }, { "cell_type": "markdown", - "id": "956a5ff2", + "id": "7eade20d", "metadata": { "editable": true }, @@ -4280,7 +3533,7 @@ }, { "cell_type": "markdown", - "id": "df74ff0c", + "id": "164ca354", "metadata": { "editable": true }, @@ -4293,7 +3546,7 @@ }, { "cell_type": "markdown", - "id": "9f03f75a", + "id": "f82f15ec", "metadata": { "editable": true }, @@ -4312,7 +3565,7 @@ }, { "cell_type": "markdown", - "id": "911429b6", + "id": "e9ecfbe0", "metadata": { "editable": true }, @@ -4324,7 +3577,7 @@ }, { "cell_type": "markdown", - "id": "ccebbb7c", + "id": "69aac1cd", "metadata": { "editable": true }, @@ -4336,7 +3589,7 @@ }, { "cell_type": "markdown", - "id": "a1432d81", + "id": "2055a5b8", "metadata": { "editable": true }, @@ -4346,7 +3599,7 @@ }, { "cell_type": "markdown", - "id": "3de8dbb1", + "id": "927516d0", "metadata": { "editable": true }, @@ -4358,7 +3611,7 @@ }, { "cell_type": "markdown", - "id": "a14bc1c1", + "id": "a58a52b4", "metadata": { "editable": true }, @@ -4371,7 +3624,7 @@ }, { "cell_type": "markdown", - "id": "7734ece5", + "id": "819a1ff5", "metadata": { "editable": true }, @@ -4390,7 +3643,7 @@ }, { "cell_type": "markdown", - "id": "1ff5ff00", + "id": "24d43e16", "metadata": { "editable": true }, @@ -4400,7 +3653,7 @@ }, { "cell_type": "markdown", - "id": "67dff48d", + "id": "7eca0f06", "metadata": { "editable": true }, @@ -4419,7 +3672,7 @@ }, { "cell_type": "markdown", - "id": "68f55d91", + "id": "04bf3889", "metadata": { "editable": true }, @@ -4437,7 +3690,7 @@ }, { "cell_type": "markdown", - "id": "21044b05", + "id": "c1f805e1", "metadata": { "editable": true }, @@ -4451,7 +3704,7 @@ }, { "cell_type": "markdown", - "id": "b86a31e2", + "id": "c696df78", "metadata": { "editable": true }, @@ -4465,8 +3718,8 @@ }, { "cell_type": "code", - "execution_count": 16, - "id": "f742bbbe", + "execution_count": 13, + "id": "5f05dc7e", "metadata": { "collapsed": false, "editable": true @@ -4487,7 +3740,7 @@ }, { "cell_type": "markdown", - "id": "229f0242", + "id": "2d99bd39", "metadata": { "editable": true }, @@ -4503,8 +3756,8 @@ }, { "cell_type": "code", - "execution_count": 17, - "id": "1a54cd71", + "execution_count": 14, + "id": "a40890e3", "metadata": { "collapsed": false, "editable": true @@ -4536,7 +3789,7 @@ }, { "cell_type": "markdown", - "id": "128631d4", + "id": "c031063b", "metadata": { "editable": true }, @@ -4550,7 +3803,7 @@ }, { "cell_type": "markdown", - "id": "10135197", + "id": "f441b1cc", "metadata": { "editable": true }, @@ -4562,8 +3815,8 @@ }, { "cell_type": "code", - "execution_count": 18, - "id": "6c1e5bcd", + "execution_count": 15, + "id": "e1485588", "metadata": { "collapsed": false, "editable": true @@ -4588,7 +3841,7 @@ }, { "cell_type": "markdown", - "id": "0255aadb", + "id": "e5210041", "metadata": { "editable": true }, @@ -4598,7 +3851,7 @@ }, { "cell_type": "markdown", - "id": "de21328e", + "id": "d97cf3fc", "metadata": { "editable": true }, @@ -4608,8 +3861,8 @@ }, { "cell_type": "code", - "execution_count": 19, - "id": "d97c5a59", + "execution_count": 16, + "id": "19805d36", "metadata": { "collapsed": false, "editable": true @@ -4663,7 +3916,7 @@ }, { "cell_type": "markdown", - "id": "3e0b0a2a", + "id": "f7c176cb", "metadata": { "editable": true }, @@ -4680,7 +3933,7 @@ }, { "cell_type": "markdown", - "id": "aca67321", + "id": "99748b66", "metadata": { "editable": true }, @@ -4692,7 +3945,7 @@ }, { "cell_type": "markdown", - "id": "c8762693", + "id": "146a709a", "metadata": { "editable": true }, @@ -4704,7 +3957,7 @@ }, { "cell_type": "markdown", - "id": "c48c1c5d", + "id": "7bb735a1", "metadata": { "editable": true }, @@ -4714,7 +3967,7 @@ }, { "cell_type": "markdown", - "id": "1eefd09a", + "id": "f28204bc", "metadata": { "editable": true }, @@ -4731,7 +3984,7 @@ }, { "cell_type": "markdown", - "id": "4b9792f7", + "id": "d7632836", "metadata": { "editable": true }, @@ -4741,7 +3994,7 @@ }, { "cell_type": "markdown", - "id": "8378b52f", + "id": "3b66313c", "metadata": { "editable": true }, @@ -4756,7 +4009,7 @@ }, { "cell_type": "markdown", - "id": "cc3bc66b", + "id": "7970a23a", "metadata": { "editable": true }, @@ -4766,7 +4019,7 @@ }, { "cell_type": "markdown", - "id": "ba5c5ea3", + "id": "20f1bfc3", "metadata": { "editable": true }, @@ -4780,7 +4033,7 @@ }, { "cell_type": "markdown", - "id": "8c4f734d", + "id": "5314cc68", "metadata": { "editable": true }, @@ -4792,7 +4045,7 @@ }, { "cell_type": "markdown", - "id": "10582364", + "id": "df38126e", "metadata": { "editable": true }, @@ -4804,7 +4057,7 @@ }, { "cell_type": "markdown", - "id": "607183ba", + "id": "c0b851ee", "metadata": { "editable": true }, @@ -4816,7 +4069,7 @@ }, { "cell_type": "markdown", - "id": "a5f5f092", + "id": "1f7f4a30", "metadata": { "editable": true }, @@ -4826,7 +4079,7 @@ }, { "cell_type": "markdown", - "id": "88c7c660", + "id": "096824a1", "metadata": { "editable": true }, @@ -4838,7 +4091,7 @@ }, { "cell_type": "markdown", - "id": "72675923", + "id": "54e04c69", "metadata": { "editable": true }, @@ -4848,7 +4101,7 @@ }, { "cell_type": "markdown", - "id": "e7d861e7", + "id": "7e8d11af", "metadata": { "editable": true }, @@ -4865,7 +4118,7 @@ }, { "cell_type": "markdown", - "id": "792ec8f7", + "id": "54c1dc98", "metadata": { "editable": true }, @@ -4875,7 +4128,7 @@ }, { "cell_type": "markdown", - "id": "b73b5ba6", + "id": "db51e12a", "metadata": { "editable": true }, @@ -4887,7 +4140,7 @@ }, { "cell_type": "markdown", - "id": "d34bc354", + "id": "683fad63", "metadata": { "editable": true }, @@ -4897,7 +4150,7 @@ }, { "cell_type": "markdown", - "id": "5d26f548", + "id": "1b731b71", "metadata": { "editable": true }, @@ -4909,7 +4162,7 @@ }, { "cell_type": "markdown", - "id": "92a468b3", + "id": "2e3d21a5", "metadata": { "editable": true }, @@ -4923,7 +4176,7 @@ }, { "cell_type": "markdown", - "id": "ccfca507", + "id": "5c62c361", "metadata": { "editable": true }, @@ -4935,7 +4188,7 @@ }, { "cell_type": "markdown", - "id": "2a460d9b", + "id": "64b8d908", "metadata": { "editable": true }, @@ -4957,7 +4210,7 @@ }, { "cell_type": "markdown", - "id": "fd63a3b1", + "id": "fd0e56a2", "metadata": { "editable": true }, @@ -4969,7 +4222,7 @@ }, { "cell_type": "markdown", - "id": "c85c9d8f", + "id": "1b0aa1e6", "metadata": { "editable": true }, @@ -4984,7 +4237,7 @@ }, { "cell_type": "markdown", - "id": "c55bb3f4", + "id": "6924b66b", "metadata": { "editable": true }, @@ -4996,7 +4249,7 @@ }, { "cell_type": "markdown", - "id": "e88b4176", + "id": "9996cc20", "metadata": { "editable": true }, @@ -5008,7 +4261,7 @@ }, { "cell_type": "markdown", - "id": "a4cc1963", + "id": "586c291e", "metadata": { "editable": true }, @@ -5018,7 +4271,7 @@ }, { "cell_type": "markdown", - "id": "4d8e14c3", + "id": "0b9dd539", "metadata": { "editable": true }, @@ -5030,7 +4283,7 @@ }, { "cell_type": "markdown", - "id": "919bd695", + "id": "a8794b80", "metadata": { "editable": true }, @@ -5040,7 +4293,7 @@ }, { "cell_type": "markdown", - "id": "bfbd9dc0", + "id": "045c8882", "metadata": { "editable": true }, @@ -5052,7 +4305,7 @@ }, { "cell_type": "markdown", - "id": "607d8f20", + "id": "e28bdf2f", "metadata": { "editable": true }, @@ -5062,7 +4315,7 @@ }, { "cell_type": "markdown", - "id": "2040b8d3", + "id": "f3c04273", "metadata": { "editable": true }, @@ -5074,7 +4327,7 @@ }, { "cell_type": "markdown", - "id": "5a40fa05", + "id": "9a0b68ea", "metadata": { "editable": true }, @@ -5091,7 +4344,7 @@ }, { "cell_type": "markdown", - "id": "14afd189", + "id": "5eed2425", "metadata": { "editable": true }, @@ -5104,7 +4357,7 @@ }, { "cell_type": "markdown", - "id": "e4db62b7", + "id": "9dab0c45", "metadata": { "editable": true }, @@ -5116,7 +4369,7 @@ }, { "cell_type": "markdown", - "id": "9d806d37", + "id": "5644b71a", "metadata": { "editable": true }, @@ -5126,7 +4379,7 @@ }, { "cell_type": "markdown", - "id": "8b1b8a23", + "id": "e4639fb2", "metadata": { "editable": true }, @@ -5139,7 +4392,7 @@ }, { "cell_type": "markdown", - "id": "270b762f", + "id": "5fee1177", "metadata": { "editable": true }, @@ -5149,7 +4402,7 @@ }, { "cell_type": "markdown", - "id": "700590d4", + "id": "d536a108", "metadata": { "editable": true }, @@ -5161,7 +4414,7 @@ }, { "cell_type": "markdown", - "id": "ad0a6eb4", + "id": "33e4ced5", "metadata": { "editable": true }, @@ -5174,7 +4427,7 @@ }, { "cell_type": "markdown", - "id": "ac7b1a5a", + "id": "88d83772", "metadata": { "editable": true }, @@ -5187,7 +4440,7 @@ }, { "cell_type": "markdown", - "id": "5307736f", + "id": "ec444bdc", "metadata": { "editable": true }, @@ -5199,7 +4452,7 @@ }, { "cell_type": "markdown", - "id": "db64760a", + "id": "968537e5", "metadata": { "editable": true }, @@ -5211,7 +4464,7 @@ }, { "cell_type": "markdown", - "id": "a9620053", + "id": "fd16ef5d", "metadata": { "editable": true }, @@ -5221,7 +4474,7 @@ }, { "cell_type": "markdown", - "id": "04ddb81a", + "id": "ee9bafbd", "metadata": { "editable": true }, @@ -5234,7 +4487,7 @@ }, { "cell_type": "markdown", - "id": "1c39709c", + "id": "cf86de9f", "metadata": { "editable": true }, @@ -5246,7 +4499,7 @@ }, { "cell_type": "markdown", - "id": "708b19ff", + "id": "d65614ad", "metadata": { "editable": true }, @@ -5258,7 +4511,7 @@ }, { "cell_type": "markdown", - "id": "98b61f5f", + "id": "ca3686ce", "metadata": { "editable": true }, @@ -5270,7 +4523,7 @@ }, { "cell_type": "markdown", - "id": "5cdd331c", + "id": "aeec861a", "metadata": { "editable": true }, @@ -5282,7 +4535,7 @@ }, { "cell_type": "markdown", - "id": "836c07ae", + "id": "c0cbdf16", "metadata": { "editable": true }, @@ -5296,7 +4549,7 @@ }, { "cell_type": "markdown", - "id": "96314f8f", + "id": "f3ce9172", "metadata": { "editable": true }, @@ -5308,7 +4561,7 @@ }, { "cell_type": "markdown", - "id": "89bf0fe8", + "id": "63fb91f4", "metadata": { "editable": true }, @@ -5318,7 +4571,7 @@ }, { "cell_type": "markdown", - "id": "c2931d72", + "id": "e2e09d2c", "metadata": { "editable": true }, @@ -5330,7 +4583,7 @@ }, { "cell_type": "markdown", - "id": "69908b86", + "id": "51621114", "metadata": { "editable": true }, @@ -5342,7 +4595,7 @@ }, { "cell_type": "markdown", - "id": "25950d7d", + "id": "78f80a63", "metadata": { "editable": true }, @@ -5354,7 +4607,7 @@ }, { "cell_type": "markdown", - "id": "59bde830", + "id": "ac75ee51", "metadata": { "editable": true }, @@ -5366,7 +4619,7 @@ }, { "cell_type": "markdown", - "id": "5dfd96b8", + "id": "474894b7", "metadata": { "editable": true }, @@ -5378,7 +4631,7 @@ }, { "cell_type": "markdown", - "id": "3f29b030", + "id": "254701bf", "metadata": { "editable": true }, @@ -5397,7 +4650,7 @@ }, { "cell_type": "markdown", - "id": "353c06ce", + "id": "31fe4fa7", "metadata": { "editable": true }, @@ -5409,7 +4662,7 @@ }, { "cell_type": "markdown", - "id": "ac24388f", + "id": "3a9c5112", "metadata": { "editable": true }, @@ -5419,7 +4672,7 @@ }, { "cell_type": "markdown", - "id": "c81762ec", + "id": "cd0353f6", "metadata": { "editable": true }, @@ -5431,7 +4684,7 @@ }, { "cell_type": "markdown", - "id": "e5bae424", + "id": "dc9874e9", "metadata": { "editable": true }, @@ -5441,7 +4694,7 @@ }, { "cell_type": "markdown", - "id": "7454857b", + "id": "11386feb", "metadata": { "editable": true }, @@ -5453,7 +4706,7 @@ }, { "cell_type": "markdown", - "id": "a527a05c", + "id": "affaaaf6", "metadata": { "editable": true }, @@ -5465,7 +4718,7 @@ }, { "cell_type": "markdown", - "id": "a20b11c6", + "id": "1a888182", "metadata": { "editable": true }, @@ -5481,7 +4734,7 @@ }, { "cell_type": "markdown", - "id": "8ef02ab4", + "id": "86273ea0", "metadata": { "editable": true }, @@ -5493,7 +4746,7 @@ }, { "cell_type": "markdown", - "id": "a184eb42", + "id": "2d4aaac8", "metadata": { "editable": true }, @@ -5505,7 +4758,7 @@ }, { "cell_type": "markdown", - "id": "63716f8a", + "id": "2bfc281a", "metadata": { "editable": true }, @@ -5515,7 +4768,7 @@ }, { "cell_type": "markdown", - "id": "ddee3add", + "id": "2040777d", "metadata": { "editable": true }, @@ -5527,7 +4780,7 @@ }, { "cell_type": "markdown", - "id": "ec699ec4", + "id": "a63d3173", "metadata": { "editable": true }, @@ -5537,7 +4790,7 @@ }, { "cell_type": "markdown", - "id": "53487beb", + "id": "271e2323", "metadata": { "editable": true }, @@ -5549,7 +4802,7 @@ }, { "cell_type": "markdown", - "id": "65ad10aa", + "id": "9e2436e2", "metadata": { "editable": true }, @@ -5566,7 +4819,7 @@ }, { "cell_type": "markdown", - "id": "cce028b9", + "id": "17a55a55", "metadata": { "editable": true }, @@ -5578,7 +4831,7 @@ }, { "cell_type": "markdown", - "id": "269d3567", + "id": "c313de2e", "metadata": { "editable": true }, @@ -5590,7 +4843,7 @@ }, { "cell_type": "markdown", - "id": "836b95d5", + "id": "a8f03e7f", "metadata": { "editable": true }, @@ -5600,7 +4853,7 @@ }, { "cell_type": "markdown", - "id": "4a32c585", + "id": "359569a7", "metadata": { "editable": true }, @@ -5612,7 +4865,7 @@ }, { "cell_type": "markdown", - "id": "2012f091", + "id": "76d42b3d", "metadata": { "editable": true }, @@ -5622,7 +4875,7 @@ }, { "cell_type": "markdown", - "id": "c9765df7", + "id": "4bb63551", "metadata": { "editable": true }, @@ -5634,7 +4887,7 @@ }, { "cell_type": "markdown", - "id": "e2029e3a", + "id": "d5eb25ad", "metadata": { "editable": true }, @@ -5644,7 +4897,7 @@ }, { "cell_type": "markdown", - "id": "e9cfccc3", + "id": "38afb347", "metadata": { "editable": true }, @@ -5656,7 +4909,7 @@ }, { "cell_type": "markdown", - "id": "7c423543", + "id": "687738a0", "metadata": { "editable": true }, @@ -5666,7 +4919,7 @@ }, { "cell_type": "markdown", - "id": "53133e77", + "id": "06c513a7", "metadata": { "editable": true }, @@ -5678,7 +4931,7 @@ }, { "cell_type": "markdown", - "id": "a9f47795", + "id": "88794c16", "metadata": { "editable": true }, diff --git a/doc/src/week35/week35.do.txt b/doc/src/week35/week35.do.txt index fae0ed508..e8ffa19c5 100644 --- a/doc/src/week35/week35.do.txt +++ b/doc/src/week35/week35.do.txt @@ -16,19 +16,19 @@ o Monday: Ridge and Lasso regression and Singular Value Decomposition === Reading recommendations: === - o These lecture notes +o These lecture notes # o "Video of lecture":"https://youtu.be/VKakN-e4aUA" # o "Video for exercises week 35":"https://youtu.be/yiY0OltU1s8" - o Goodfellow, Bengio and Courville, Deep Learning, chapter 2 on linear algebra and sections 3.1-3.10 on elements of statistics (background) - o Raschka et al on preprocessing of data, relevant for exercise 3 this week, see chapter 4. - o For exercise 1 of week 35, the book by A. Aldo Faisal, Cheng Soon Ong, and Marc Peter Deisenroth on the Mathematics of Machine Learning, may be very relevant. In particular chapter 5 at URL"https://mml-book.github.io/" (section 5.5 on derivatives) is very useful for exercise 1 this coming week. +o Goodfellow, Bengio and Courville, Deep Learning, chapter 2 on linear algebra and sections 3.1-3.10 on elements of statistics (background) +o Raschka et al on preprocessing of data, relevant for exercise 3 this week, see chapter 4. +o For exercise 1 of week 35, the book by A. Aldo Faisal, Cheng Soon Ong, and Marc Peter Deisenroth on the Mathematics of Machine Learning, may be very relevant. In particular chapter 5 at URL"https://mml-book.github.io/" (section 5.5 on derivatives) is very useful for exercise 1 this coming week. !split -===== For exercise sessions: Why Linear Regression (aka Ordinary Least Squares and family), repeat from last week ===== +===== Reminder from last week ===== We need first a reminder from last week about linear regression. @@ -532,7 +532,7 @@ $\hat{\bm{\beta}}$. Furthermore, we will see later this week that it is important role in optmization algorithms and Principal Component Analysis as a way to reduce the dimensionality of a machine learning/data analysis problem. - +v _Linear algebra question:_ Can we use the Hessian matrix to say something about properties of the cost function (our optmization problem)? (hint: think about convex or concave problems and how to relate these to a matrix!). !split @@ -945,445 +945,6 @@ plt.show() -!split -===== More preprocessing examples, two-dimensional example, the Franke function ===== - -!bc pycod -# Common imports -import os -import numpy as np -import pandas as pd -import matplotlib.pyplot as plt -import sklearn.linear_model as skl -from sklearn.metrics import mean_squared_error -from sklearn.model_selection import train_test_split -from sklearn.preprocessing import MinMaxScaler, StandardScaler, Normalizer - -# Where to save the figures and data files -PROJECT_ROOT_DIR = "Results" -FIGURE_ID = "Results/FigureFiles" -DATA_ID = "DataFiles/" - -if not os.path.exists(PROJECT_ROOT_DIR): - os.mkdir(PROJECT_ROOT_DIR) - -if not os.path.exists(FIGURE_ID): - os.makedirs(FIGURE_ID) - -if not os.path.exists(DATA_ID): - os.makedirs(DATA_ID) - -def image_path(fig_id): - return os.path.join(FIGURE_ID, fig_id) - -def data_path(dat_id): - return os.path.join(DATA_ID, dat_id) - -def save_fig(fig_id): - plt.savefig(image_path(fig_id) + ".png", format='png') - - -def FrankeFunction(x,y): - term1 = 0.75*np.exp(-(0.25*(9*x-2)**2) - 0.25*((9*y-2)**2)) - term2 = 0.75*np.exp(-((9*x+1)**2)/49.0 - 0.1*(9*y+1)) - term3 = 0.5*np.exp(-(9*x-7)**2/4.0 - 0.25*((9*y-3)**2)) - term4 = -0.2*np.exp(-(9*x-4)**2 - (9*y-7)**2) - return term1 + term2 + term3 + term4 - - -def create_X(x, y, n ): - if len(x.shape) > 1: - x = np.ravel(x) - y = np.ravel(y) - - N = len(x) - l = int((n+1)*(n+2)/2) # Number of elements in beta - X = np.ones((N,l)) - - for i in range(1,n+1): - q = int((i)*(i+1)/2) - for k in range(i+1): - X[:,q+k] = (x**(i-k))*(y**k) - - return X - - -# Making meshgrid of datapoints and compute Franke's function -n = 5 -N = 1000 -x = np.sort(np.random.uniform(0, 1, N)) -y = np.sort(np.random.uniform(0, 1, N)) -z = FrankeFunction(x, y) -X = create_X(x, y, n=n) -# split in training and test data -X_train, X_test, y_train, y_test = train_test_split(X,z,test_size=0.2) - - -clf = skl.LinearRegression().fit(X_train, y_train) - -# The mean squared error and R2 score -print("MSE before scaling: {:.2f}".format(mean_squared_error(clf.predict(X_test), y_test))) -print("R2 score before scaling {:.2f}".format(clf.score(X_test,y_test))) - -scaler = StandardScaler() -scaler.fit(X_train) -X_train_scaled = scaler.transform(X_train) -X_test_scaled = scaler.transform(X_test) - -print("Feature min values before scaling:\n {}".format(X_train.min(axis=0))) -print("Feature max values before scaling:\n {}".format(X_train.max(axis=0))) - -print("Feature min values after scaling:\n {}".format(X_train_scaled.min(axis=0))) -print("Feature max values after scaling:\n {}".format(X_train_scaled.max(axis=0))) - -clf = skl.LinearRegression().fit(X_train_scaled, y_train) - - -print("MSE after scaling: {:.2f}".format(mean_squared_error(clf.predict(X_test_scaled), y_test))) -print("R2 score for scaled data: {:.2f}".format(clf.score(X_test_scaled,y_test))) - -!ec - - - - -!split -===== To think about, first part ===== - -When you are comparing your own code with for example _Scikit-Learn_'s -library, there are some technicalities to keep in mind. The examples -here demonstrate some of these aspects with potential pitfalls. - -The discussion here focuses on the role of the intercept, how we can -set up the design matrix, what scaling we should use and other topics -which tend confuse us. - - - -The intercept can be interpreted as the expected value of our -target/output variables when all other predictors are set to zero. -Thus, if we cannot assume that the expected outputs/targets are zero -when all predictors are zero (the columns in the design matrix), it -may be a bad idea to implement a model which penalizes the intercept. -Furthermore, in for example Ridge and Lasso regression (to be discussed in moe detail next week), the default solutions -from the library _Scikit-Learn_ (when not shrinking $\beta_0$) for the unknown parameters -$\bm{\beta}$, are derived under the assumption that both $\bm{y}$ and -$\bm{X}$ are zero centered, that is we subtract the mean values. - - -!split -===== More thinking ===== - - -If our predictors represent different scales, then it is important to -standardize the design matrix $\bm{X}$ by subtracting the mean of each -column from the corresponding column and dividing the column with its -standard deviation. Most machine learning libraries do this as a default. This means that if you compare your code with the results from a given library, -the results may differ. - -The -"Standadscaler":"https://scikit-learn.org/stable/modules/generated/sklearn.preprocessing.StandardScaler.html" -function in _Scikit-Learn_ does this for us. For the data sets we -have been studying in our various examples, the data are in many cases -already scaled and there is no need to scale them. You as a user of different machine learning algorithms, should always perform a -survey of your data, with a critical assessment of them in case you need to scale the data. - -If you need to scale the data, not doing so will give an *unfair* -penalization of the parameters since their magnitude depends on the -scale of their corresponding predictor. - -Suppose as an example that you -you have an input variable given by the heights of different persons. -Human height might be measured in inches or meters or -kilometers. If measured in kilometers, a standard linear regression -model with this predictor would probably give a much bigger -coefficient term, than if measured in millimeters. -This can clearly lead to problems in evaluating the cost/loss functions. - - -!split -===== Still thinking ===== - -Keep in mind that when you transform your data set before training a model, the same transformation needs to be done -on your eventual new data set before making a prediction. If we translate this into a Python code, it would could be implemented as follows -(note that the lines are commented since the model function has not been defined) -!bc pycod -#Model training, we compute the mean value of y and X -y_train_mean = np.mean(y_train) -X_train_mean = np.mean(X_train,axis=0) -X_train = X_train - X_train_mean -y_train = y_train - y_train_mean - -# The we fit our model with the training data -#trained_model = some_model.fit(X_train,y_train) - - -#Model prediction, we need also to transform our data set used for the prediction. -X_test = X_test - X_train_mean #Use mean from training data -#y_pred = trained_model(X_test) -y_pred = y_pred + y_train_mean -!ec - - -!split -===== What does centering (subtracting the mean values) mean mathematically? ===== - - -Let us try to understand what this may imply mathematically when we -subtract the mean values, also known as *zero centering*. For -simplicity, we will focus on ordinary regression, as done in the above example. - -The cost/loss function for regression is -!bt -\[ -C(\beta_0, \beta_1, ... , \beta_{p-1}) = \frac{1}{n}\sum_{i=0}^{n} \left(y_i - \beta_0 - \sum_{j=1}^{p-1} X_{ij}\beta_j\right)^2,. -\] -!et -Recall also that we use the squared value since this leads to an increase of the penalty for higher differences between predicted and output/target values. - -What we have done is to single out the $\beta_0$ term in the definition of the mean squared error (MSE). -The design matrix -$X$ does in this case not contain any intercept column. -When we take the derivative with respect to $\beta_0$, we want the derivative to obey -!bt -\[ -\frac{\partial C}{\partial \beta_j} = 0, -\] -!et - -for all $j$. For $\beta_0$ we have - -!bt -\[ -\frac{\partial C}{\partial \beta_0} = -\frac{2}{n}\sum_{i=0}^{n-1} \left(y_i - \beta_0 - \sum_{j=1}^{p-1} X_{ij} \beta_j\right). -\] -!et -Multiplying away the constant $2/n$, we obtain -!bt -\[ -\sum_{i=0}^{n-1} \beta_0 = \sum_{i=0}^{n-1}y_i - \sum_{i=0}^{n-1} \sum_{j=1}^{p-1} X_{ij} \beta_j. -\] -!et - -!split -===== Further Manipulations ===== - - -Let us special first to the case where we have only two parameters $\beta_0$ and $\beta_1$. -Our result for $\beta_0$ simplifies then to -!bt -\[ -n\beta_0 = \sum_{i=0}^{n-1}y_i - \sum_{i=0}^{n-1} X_{i1} \beta_1. -\] -!et -We obtain then -!bt -\[ -\beta_0 = \frac{1}{n}\sum_{i=0}^{n-1}y_i - \beta_1\frac{1}{n}\sum_{i=0}^{n-1} X_{i1}. -\] -!et -If we define -!bt -\[ -\mu_1=\frac{1}{n}\sum_{i=0}^{n-1} (X_{i1}, -\] -!et -and if we define the mean value of the outputs as -!bt -\[ -\mu_y=\frac{1}{n}\sum_{i=0}^{n-1}y_i, -\] -!et -we have -!bt -\[ -\beta_0 = \mu_y - \beta_1\mu_{1}. -\] -!et -In the general case, that is we have more parameters than $\beta_0$ and $\beta_1$, we have -!bt -\[ -\beta_0 = \frac{1}{n}\sum_{i=0}^{n-1}y_i - \frac{1}{n}\sum_{i=0}^{n-1}\sum_{j=1}^{p-1} X_{ij}\beta_j. -\] -!et - - - -Replacing $y_i$ with $y_i - y_i - \overline{\bm{y}}$ and centering also our design matrix results in a cost function (in vector-matrix disguise) -!bt -\[ -C(\boldsymbol{\beta}) = (\boldsymbol{\tilde{y}} - \tilde{X}\boldsymbol{\beta})^T(\boldsymbol{\tilde{y}} - \tilde{X}\boldsymbol{\beta}). -\] -!et - -!split -===== Wrapping it up ===== - -If we minimize with respect to $\bm{\beta}$ we have then - -!bt -\[ -\hat{\bm{\beta}} = (\tilde{X}^T\tilde{X})^{-1}\tilde{X}^T\boldsymbol{\tilde{y}}, -\] -!et - -where $\boldsymbol{\tilde{y}} = \boldsymbol{y} - \overline{\bm{y}}$ -and $\tilde{X}_{ij} = X_{ij} - \frac{1}{n}\sum_{k=0}^{n-1}X_{kj}$. - -For Ridge regression we need to add $\lambda \boldsymbol{\beta}^T\boldsymbol{\beta}$ to the cost function and get then -!bt -\[ -\hat{\bm{\beta}} = (\tilde{X}^T\tilde{X} + \lambda I)^{-1}\tilde{X}^T\boldsymbol{\tilde{y}}. -\] -!et - -What does this mean? And why do we insist on all this? Let us look at some examples. - - - -!split -===== Linear Regression code, Intercept handling first ===== - -This code shows a simple first-order fit to a data set using the above transformed data, where we consider the role of the intercept first, by either excluding it or including it (*code example thanks to Øyvind Sigmundson Schøyen*). Here our scaling of the data is done by subtracting the mean values only. -Note also that we do not split the data into training and test. - -!bc pycod -import numpy as np -import matplotlib.pyplot as plt - -from sklearn.linear_model import LinearRegression - - -np.random.seed(2021) - -def MSE(y_data,y_model): - n = np.size(y_model) - return np.sum((y_data-y_model)**2)/n - - -def fit_beta(X, y): - return np.linalg.pinv(X.T @ X) @ X.T @ y - - -true_beta = [2, 0.5, 3.7] - -x = np.linspace(0, 1, 11) -y = np.sum( - np.asarray([x ** p * b for p, b in enumerate(true_beta)]), axis=0 -) + 0.1 * np.random.normal(size=len(x)) - -degree = 3 -X = np.zeros((len(x), degree)) - -# Include the intercept in the design matrix -for p in range(degree): - X[:, p] = x ** p - -beta = fit_beta(X, y) - -# Intercept is included in the design matrix -skl = LinearRegression(fit_intercept=False).fit(X, y) - -print(f"True beta: {true_beta}") -print(f"Fitted beta: {beta}") -print(f"Sklearn fitted beta: {skl.coef_}") -ypredictOwn = X @ beta -ypredictSKL = skl.predict(X) -print(f"MSE with intercept column") -print(MSE(y,ypredictOwn)) -print(f"MSE with intercept column from SKL") -print(MSE(y,ypredictSKL)) - - -plt.figure() -plt.scatter(x, y, label="Data") -plt.plot(x, X @ beta, label="Fit") -plt.plot(x, skl.predict(X), label="Sklearn (fit_intercept=False)") - - -# Do not include the intercept in the design matrix -X = np.zeros((len(x), degree - 1)) - -for p in range(degree - 1): - X[:, p] = x ** (p + 1) - -# Intercept is not included in the design matrix -skl = LinearRegression(fit_intercept=True).fit(X, y) - -# Use centered values for X and y when computing coefficients -y_offset = np.average(y, axis=0) -X_offset = np.average(X, axis=0) - -beta = fit_beta(X - X_offset, y - y_offset) -intercept = np.mean(y_offset - X_offset @ beta) - -print(f"Manual intercept: {intercept}") -print(f"Fitted beta (wiothout intercept): {beta}") -print(f"Sklearn intercept: {skl.intercept_}") -print(f"Sklearn fitted beta (without intercept): {skl.coef_}") -ypredictOwn = X @ beta -ypredictSKL = skl.predict(X) -print(f"MSE with Manual intercept") -print(MSE(y,ypredictOwn+intercept)) -print(f"MSE with Sklearn intercept") -print(MSE(y,ypredictSKL)) - -plt.plot(x, X @ beta + intercept, "--", label="Fit (manual intercept)") -plt.plot(x, skl.predict(X), "--", label="Sklearn (fit_intercept=True)") -plt.grid() -plt.legend() - -plt.show() - -!ec - -The intercept is the value of our output/target variable -when all our features are zero and our function crosses the $y$-axis (for a one-dimensional case). - -Printing the MSE, we see first that both methods give the same MSE, as -they should. However, when we move to for example Ridge regression (discussed next week), -the way we treat the intercept may give a larger or smaller MSE, -meaning that the MSE can be penalized by the value of the -intercept. Not including the intercept in the fit, means that the -regularization term does not include $\beta_0$. For different values -of $\lambda$, this may lead to differing MSE values. - -To remind the reader, the regularization term, with the intercept in Ridge regression is given by -!bt -\[ -\lambda \vert\vert \bm{\beta} \vert\vert_2^2 = \lambda \sum_{j=0}^{p-1}\beta_j^2, -\] -!et -but when we take out the intercept, this equation becomes -!bt -\[ -\lambda \vert\vert \bm{\beta} \vert\vert_2^2 = \lambda \sum_{j=1}^{p-1}\beta_j^2. -\] -!et - -For Lasso regression we have -!bt -\[ -\lambda \vert\vert \bm{\beta} \vert\vert_1 = \lambda \sum_{j=1}^{p-1}\vert\beta_j\vert. -\] -!et - -It means that, when scaling the design matrix and the outputs/targets, -by subtracting the mean values, we have an optimization problem which -is not penalized by the intercept. The MSE value can then be smaller -since it focuses only on the remaining quantities. If we however bring -back the intercept, we will get an MSE which then contains the -intercept. This becomes more important when we discuss Ridge and Lasso -regression next week. - - - - - - -!split -===== Material for lecture Monday, August 26 ===== - !split ===== Mathematical Interpretation of Ordinary Least Squares =====