From edfdf77b5cd5b6cf46c21734c2ac5775a81a9e83 Mon Sep 17 00:00:00 2001 From: Morten Hjorth-Jensen Date: Mon, 16 Sep 2024 13:01:42 +0200 Subject: [PATCH] update --- doc/pub/week38/html/._week38-bs001.html | 4 +- doc/pub/week38/html/week38-reveal.html | 5 +- doc/pub/week38/html/week38-solarized.html | 4 +- doc/pub/week38/html/week38.html | 4 +- doc/pub/week38/ipynb/ipynb-week38-src.tar.gz | Bin 192 -> 191 bytes doc/pub/week38/ipynb/week38.ipynb | 1101 ++++++++++++------ doc/src/week38/week38.do.txt | 2 +- 7 files changed, 737 insertions(+), 383 deletions(-) diff --git a/doc/pub/week38/html/._week38-bs001.html b/doc/pub/week38/html/._week38-bs001.html index 9864dbaf6..f7f68de23 100644 --- a/doc/pub/week38/html/._week38-bs001.html +++ b/doc/pub/week38/html/._week38-bs001.html @@ -328,8 +328,8 @@ MathJax.Hub.Config({ diff --git a/doc/pub/week38/html/week38-reveal.html b/doc/pub/week38/html/week38-reveal.html index 95a59798f..29701aea5 100644 --- a/doc/pub/week38/html/week38-reveal.html +++ b/doc/pub/week38/html/week38-reveal.html @@ -204,8 +204,9 @@ MathJax.Hub.Config({

  • Logistic regression as our first encounter of classification methods. From binary cases to several categories.
    • -

  • Start gradient and optimization methods - +

  • Start gradient and optimization methods
    • + +

  • Video of lecture
    • diff --git a/doc/pub/week38/html/week38-solarized.html b/doc/pub/week38/html/week38-solarized.html index c00d535ef..86a59b657 100644 --- a/doc/pub/week38/html/week38-solarized.html +++ b/doc/pub/week38/html/week38-solarized.html @@ -288,8 +288,8 @@ MathJax.Hub.Config({

    diff --git a/doc/pub/week38/html/week38.html b/doc/pub/week38/html/week38.html index e07d0c203..faa966ec0 100644 --- a/doc/pub/week38/html/week38.html +++ b/doc/pub/week38/html/week38.html @@ -365,8 +365,8 @@ MathJax.Hub.Config({

    diff --git a/doc/pub/week38/ipynb/ipynb-week38-src.tar.gz b/doc/pub/week38/ipynb/ipynb-week38-src.tar.gz index 6f56bfd7e08aa6f1c140fe278f3fbed079cd0bca..cf1c07b76a4a931a73b7fbda073a669f8bdfa2a3 100644 GIT binary patch literal 191 zcmV;w06_mAiwFP-5a?zA1MSaC3c@fD2H>uHia9|^`gAMUg$qH%3#4RfV{KBC6z%Qp z19YXhDI(LhkTi~A%q&f&B;j07C{2LIX%-SX7C>l}(8vR_ z+(|E;u}lxAG}RfULHTZO7%R&Ud*)Z*nSbI~2?N`GZ!0B1DvO0uHQW&EXcdUImj!Tz tW^94RYbP`UlRYrbg%wJ&i{EN>(yTFo|LbQQ$8nsmJpez5K?wi|006fUQF#CW literal 192 zcmV;x06+g9iwFQ2$mV7M1MSbv3c@f92XN1Oiafz+ZMS(9+`)q&;tO;vb9JtrZHMmd z-3RDN@iIi{@A4-kgyfK}H@h@+cej`gAtdG;hD_pUOtPHn5v37O9w#*7oG=Q6vWSrd zkoit}X{{5cKcz0us4S{?bNyIReAqL+0?+&t$5L9@?mJg%1xh>2w65TWSg}\n", @@ -12,8 +14,10 @@ }, { "cell_type": "markdown", - "id": "93b0d36c", - "metadata": {}, + "id": "b457168e", + "metadata": { + "editable": true + }, "source": [ "# Week 38: Logistic Regression and Optimization\n", "**Morten Hjorth-Jensen**, Department of Physics and Center for Computing in Science Education, University of Oslo and Department of Physics and Astronomy and Facility for Rare Isotope Beams, Michigan State University\n", @@ -23,8 +27,10 @@ }, { "cell_type": "markdown", - "id": "42899962", - "metadata": {}, + "id": "ed45b565", + "metadata": { + "editable": true + }, "source": [ "## Plans for week 38, lecture Monday September 16\n", "\n", @@ -33,14 +39,17 @@ " * Logistic regression as our first encounter of classification methods. From binary cases to several categories.\n", "\n", " * Start gradient and optimization methods\n", - "\n", + "\n", + " * [Video of lecture](https://youtu.be/c9DIfNHy2ks)\n", "" ] }, { "cell_type": "markdown", - "id": "4175fb49", - "metadata": {}, + "id": "81f6434c", + "metadata": { + "editable": true + }, "source": [ "## Suggested reading and videos\n", " * Readings and Videos:\n", @@ -60,8 +69,10 @@ }, { "cell_type": "markdown", - "id": "cd73082e", - "metadata": {}, + "id": "f6be4e35", + "metadata": { + "editable": true + }, "source": [ "## Plans for the lab sessions\n", "\n", @@ -78,16 +89,20 @@ }, { "cell_type": "markdown", - "id": "84c47adf", - "metadata": {}, + "id": "98962200", + "metadata": { + "editable": true + }, "source": [ "## Material for lecture Thursday September 21" ] }, { "cell_type": "markdown", - "id": "1299fcbc", - "metadata": {}, + "id": "21256bbb", + "metadata": { + "editable": true + }, "source": [ "## Logistic Regression\n", "\n", @@ -106,8 +121,10 @@ }, { "cell_type": "markdown", - "id": "e70ead5f", - "metadata": {}, + "id": "23a19970", + "metadata": { + "editable": true + }, "source": [ "## Classification problems\n", "\n", @@ -130,8 +147,10 @@ }, { "cell_type": "markdown", - "id": "fa1b67be", - "metadata": {}, + "id": "ba4f386f", + "metadata": { + "editable": true + }, "source": [ "## Optimization and Deep learning\n", "\n", @@ -152,8 +171,10 @@ }, { "cell_type": "markdown", - "id": "7c54a4ff", - "metadata": {}, + "id": "90553621", + "metadata": { + "editable": true + }, "source": [ "## Basics\n", "\n", @@ -175,8 +196,10 @@ }, { "cell_type": "markdown", - "id": "838d316e", - "metadata": {}, + "id": "717baa58", + "metadata": { + "editable": true + }, "source": [ "$$\n", "y_i = \\begin{bmatrix} 0 & \\mathrm{no}\\\\ 1 & \\mathrm{yes} \\end{bmatrix}.\n", @@ -185,8 +208,10 @@ }, { "cell_type": "markdown", - "id": "285bf3a2", - "metadata": {}, + "id": "1a961a80", + "metadata": { + "editable": true + }, "source": [ "## Linear classifier\n", "\n", @@ -201,8 +226,10 @@ }, { "cell_type": "markdown", - "id": "6b03bb50", - "metadata": {}, + "id": "f571bcc7", + "metadata": { + "editable": true + }, "source": [ "\n", "
    \n", @@ -217,8 +244,10 @@ }, { "cell_type": "markdown", - "id": "f153c89c", - "metadata": {}, + "id": "324469c5", + "metadata": { + "editable": true + }, "source": [ "where $\\boldsymbol{y}$ is a vector representing the possible outcomes, $\\boldsymbol{X}$ is our\n", "$n\\times p$ design matrix and $\\boldsymbol{\\beta}$ represents our estimators/predictors." @@ -226,8 +255,10 @@ }, { "cell_type": "markdown", - "id": "487ff8fe", - "metadata": {}, + "id": "f30341e8", + "metadata": { + "editable": true + }, "source": [ "## Some selected properties\n", "\n", @@ -251,8 +282,10 @@ }, { "cell_type": "markdown", - "id": "e64f34ad", - "metadata": {}, + "id": "65dc36a2", + "metadata": { + "editable": true + }, "source": [ "## Simple example\n", "\n", @@ -262,8 +295,11 @@ { "cell_type": "code", "execution_count": 1, - "id": "eb389ab6", - "metadata": {}, + "id": "d2bd5570", + "metadata": { + "collapsed": false, + "editable": true + }, "outputs": [], "source": [ "%matplotlib inline\n", @@ -326,8 +362,10 @@ }, { "cell_type": "markdown", - "id": "d0f9afe8", - "metadata": {}, + "id": "a7f6517f", + "metadata": { + "editable": true + }, "source": [ "## Plotting the mean value for each group\n", "\n", @@ -337,8 +375,11 @@ { "cell_type": "code", "execution_count": 2, - "id": "499179b1", - "metadata": {}, + "id": "b1bc9bfb", + "metadata": { + "collapsed": false, + "editable": true + }, "outputs": [], "source": [ "agegroupmean = np.array([0.1, 0.133, 0.250, 0.333, 0.462, 0.625, 0.765, 0.800])\n", @@ -353,8 +394,10 @@ }, { "cell_type": "markdown", - "id": "cf814d3f", - "metadata": {}, + "id": "48ac2243", + "metadata": { + "editable": true + }, "source": [ "We are now trying to find a function $f(y\\vert x)$, that is a function which gives us an expected value for the output $y$ with a given input $x$.\n", "In standard linear regression with a linear dependence on $x$, we would write this in terms of our model" @@ -362,8 +405,10 @@ }, { "cell_type": "markdown", - "id": "28658b61", - "metadata": {}, + "id": "0dfc8ed6", + "metadata": { + "editable": true + }, "source": [ "$$\n", "f(y_i\\vert x_i)=\\beta_0+\\beta_1 x_i.\n", @@ -372,8 +417,10 @@ }, { "cell_type": "markdown", - "id": "dec92579", - "metadata": {}, + "id": "ad7bcee4", + "metadata": { + "editable": true + }, "source": [ "This expression implies however that $f(y_i\\vert x_i)$ could take any\n", "value from minus infinity to plus infinity. If we however let\n", @@ -389,8 +436,10 @@ }, { "cell_type": "markdown", - "id": "92ad450b", - "metadata": {}, + "id": "046ee488", + "metadata": { + "editable": true + }, "source": [ "## The logistic function\n", "\n", @@ -409,8 +458,10 @@ }, { "cell_type": "markdown", - "id": "35cf0753", - "metadata": {}, + "id": "9c57ceb4", + "metadata": { + "editable": true + }, "source": [ "$$\n", "p(t) = \\frac{1}{1+\\mathrm \\exp{-t}}=\\frac{\\exp{t}}{1+\\mathrm \\exp{t}}.\n", @@ -419,16 +470,20 @@ }, { "cell_type": "markdown", - "id": "fd212f79", - "metadata": {}, + "id": "1a9e2480", + "metadata": { + "editable": true + }, "source": [ "Note that $1-p(t)= p(-t)$." ] }, { "cell_type": "markdown", - "id": "34d6199f", - "metadata": {}, + "id": "56c784c4", + "metadata": { + "editable": true + }, "source": [ "## Examples of likelihood functions used in logistic regression and nueral networks\n", "\n", @@ -438,8 +493,11 @@ { "cell_type": "code", "execution_count": 3, - "id": "174df351", - "metadata": {}, + "id": "b7b68582", + "metadata": { + "collapsed": false, + "editable": true + }, "outputs": [], "source": [ "\"\"\"The sigmoid function (or the logistic curve) is a\n", @@ -500,8 +558,10 @@ }, { "cell_type": "markdown", - "id": "89fdf010", - "metadata": {}, + "id": "a0379ac2", + "metadata": { + "editable": true + }, "source": [ "## Two parameters\n", "\n", @@ -510,8 +570,10 @@ }, { "cell_type": "markdown", - "id": "6216d0ba", - "metadata": {}, + "id": "a246601b", + "metadata": { + "editable": true + }, "source": [ "$$\n", "\\begin{align*}\n", @@ -523,8 +585,10 @@ }, { "cell_type": "markdown", - "id": "daa87f9a", - "metadata": {}, + "id": "a673db4a", + "metadata": { + "editable": true + }, "source": [ "where $\\boldsymbol{\\beta}$ are the weights we wish to extract from data, in our case $\\beta_0$ and $\\beta_1$. \n", "\n", @@ -533,8 +597,10 @@ }, { "cell_type": "markdown", - "id": "85088741", - "metadata": {}, + "id": "415e506b", + "metadata": { + "editable": true + }, "source": [ "$$\n", "p(y_i=0\\vert x_i, \\boldsymbol{\\beta}) = 1-p(y_i=1\\vert x_i, \\boldsymbol{\\beta}).\n", @@ -543,8 +609,10 @@ }, { "cell_type": "markdown", - "id": "035508f1", - "metadata": {}, + "id": "5816c4a0", + "metadata": { + "editable": true + }, "source": [ "## Maximum likelihood\n", "\n", @@ -558,8 +626,10 @@ }, { "cell_type": "markdown", - "id": "8972c037", - "metadata": {}, + "id": "6c2f4da8", + "metadata": { + "editable": true + }, "source": [ "$$\n", "\\begin{align*}\n", @@ -570,16 +640,20 @@ }, { "cell_type": "markdown", - "id": "3eac3b65", - "metadata": {}, + "id": "9b5f3e01", + "metadata": { + "editable": true + }, "source": [ "from which we obtain the log-likelihood and our **cost/loss** function" ] }, { "cell_type": "markdown", - "id": "133efea3", - "metadata": {}, + "id": "2da0f8f4", + "metadata": { + "editable": true + }, "source": [ "$$\n", "\\mathcal{C}(\\boldsymbol{\\beta}) = \\sum_{i=1}^n \\left( y_i\\log{p(y_i=1|x_i,\\boldsymbol{\\beta})} + (1-y_i)\\log\\left[1-p(y_i=1|x_i,\\boldsymbol{\\beta}))\\right]\\right).\n", @@ -588,8 +662,10 @@ }, { "cell_type": "markdown", - "id": "566aabdb", - "metadata": {}, + "id": "c931e332", + "metadata": { + "editable": true + }, "source": [ "## The cost function rewritten\n", "\n", @@ -598,8 +674,10 @@ }, { "cell_type": "markdown", - "id": "4cf34420", - "metadata": {}, + "id": "552463f5", + "metadata": { + "editable": true + }, "source": [ "$$\n", "\\mathcal{C}(\\boldsymbol{\\beta}) = \\sum_{i=1}^n \\left(y_i(\\beta_0+\\beta_1x_i) -\\log{(1+\\exp{(\\beta_0+\\beta_1x_i)})}\\right).\n", @@ -608,8 +686,10 @@ }, { "cell_type": "markdown", - "id": "90ccf024", - "metadata": {}, + "id": "e92007bd", + "metadata": { + "editable": true + }, "source": [ "The maximum likelihood estimator is defined as the set of parameters that maximize the log-likelihood where we maximize with respect to $\\beta$.\n", "Since the cost (error) function is just the negative log-likelihood, for logistic regression we have that" @@ -617,8 +697,10 @@ }, { "cell_type": "markdown", - "id": "824ccda5", - "metadata": {}, + "id": "bb909028", + "metadata": { + "editable": true + }, "source": [ "$$\n", "\\mathcal{C}(\\boldsymbol{\\beta})=-\\sum_{i=1}^n \\left(y_i(\\beta_0+\\beta_1x_i) -\\log{(1+\\exp{(\\beta_0+\\beta_1x_i)})}\\right).\n", @@ -627,8 +709,10 @@ }, { "cell_type": "markdown", - "id": "bbfb14d3", - "metadata": {}, + "id": "2cd2edcb", + "metadata": { + "editable": true + }, "source": [ "This equation is known in statistics as the **cross entropy**. Finally, we note that just as in linear regression, \n", "in practice we often supplement the cross-entropy with additional regularization terms, usually $L_1$ and $L_2$ regularization as we did for Ridge and Lasso regression." @@ -636,8 +720,10 @@ }, { "cell_type": "markdown", - "id": "bfc7e1be", - "metadata": {}, + "id": "4aeaf578", + "metadata": { + "editable": true + }, "source": [ "## Minimizing the cross entropy\n", "\n", @@ -650,8 +736,10 @@ }, { "cell_type": "markdown", - "id": "bae6c599", - "metadata": {}, + "id": "b2015302", + "metadata": { + "editable": true + }, "source": [ "$$\n", "\\frac{\\partial \\mathcal{C}(\\boldsymbol{\\beta})}{\\partial \\beta_0} = -\\sum_{i=1}^n \\left(y_i -\\frac{\\exp{(\\beta_0+\\beta_1x_i)}}{1+\\exp{(\\beta_0+\\beta_1x_i)}}\\right),\n", @@ -660,16 +748,20 @@ }, { "cell_type": "markdown", - "id": "550a7b2b", - "metadata": {}, + "id": "6825d45d", + "metadata": { + "editable": true + }, "source": [ "and" ] }, { "cell_type": "markdown", - "id": "5b072ab8", - "metadata": {}, + "id": "6e5613b3", + "metadata": { + "editable": true + }, "source": [ "$$\n", "\\frac{\\partial \\mathcal{C}(\\boldsymbol{\\beta})}{\\partial \\beta_1} = -\\sum_{i=1}^n \\left(y_ix_i -x_i\\frac{\\exp{(\\beta_0+\\beta_1x_i)}}{1+\\exp{(\\beta_0+\\beta_1x_i)}}\\right).\n", @@ -678,8 +770,10 @@ }, { "cell_type": "markdown", - "id": "9ed08b1d", - "metadata": {}, + "id": "3bba9707", + "metadata": { + "editable": true + }, "source": [ "## A more compact expression\n", "\n", @@ -691,8 +785,10 @@ }, { "cell_type": "markdown", - "id": "11b32bd3", - "metadata": {}, + "id": "9ddaf67d", + "metadata": { + "editable": true + }, "source": [ "$$\n", "\\frac{\\partial \\mathcal{C}(\\boldsymbol{\\beta})}{\\partial \\boldsymbol{\\beta}} = -\\boldsymbol{X}^T\\left(\\boldsymbol{y}-\\boldsymbol{p}\\right).\n", @@ -701,8 +797,10 @@ }, { "cell_type": "markdown", - "id": "47dec6ef", - "metadata": {}, + "id": "3bc329a8", + "metadata": { + "editable": true + }, "source": [ "If we in addition define a diagonal matrix $\\boldsymbol{W}$ with elements \n", "$p(y_i\\vert x_i,\\boldsymbol{\\beta})(1-p(y_i\\vert x_i,\\boldsymbol{\\beta})$, we can obtain a compact expression of the second derivative as" @@ -710,8 +808,10 @@ }, { "cell_type": "markdown", - "id": "43e12319", - "metadata": {}, + "id": "0e96d262", + "metadata": { + "editable": true + }, "source": [ "$$\n", "\\frac{\\partial^2 \\mathcal{C}(\\boldsymbol{\\beta})}{\\partial \\boldsymbol{\\beta}\\partial \\boldsymbol{\\beta}^T} = \\boldsymbol{X}^T\\boldsymbol{W}\\boldsymbol{X}.\n", @@ -720,8 +820,10 @@ }, { "cell_type": "markdown", - "id": "a0c3198a", - "metadata": {}, + "id": "29ae3912", + "metadata": { + "editable": true + }, "source": [ "## Extending to more predictors\n", "\n", @@ -730,8 +832,10 @@ }, { "cell_type": "markdown", - "id": "fb7b011d", - "metadata": {}, + "id": "38b173e7", + "metadata": { + "editable": true + }, "source": [ "$$\n", "\\log{ \\frac{p(\\boldsymbol{\\beta}\\boldsymbol{x})}{1-p(\\boldsymbol{\\beta}\\boldsymbol{x})}} = \\beta_0+\\beta_1x_1+\\beta_2x_2+\\dots+\\beta_px_p.\n", @@ -740,16 +844,20 @@ }, { "cell_type": "markdown", - "id": "acfaf754", - "metadata": {}, + "id": "f97ea2a7", + "metadata": { + "editable": true + }, "source": [ "Here we defined $\\boldsymbol{x}=[1,x_1,x_2,\\dots,x_p]$ and $\\boldsymbol{\\beta}=[\\beta_0, \\beta_1, \\dots, \\beta_p]$ leading to" ] }, { "cell_type": "markdown", - "id": "33e112ba", - "metadata": {}, + "id": "5bf299f4", + "metadata": { + "editable": true + }, "source": [ "$$\n", "p(\\boldsymbol{\\beta}\\boldsymbol{x})=\\frac{ \\exp{(\\beta_0+\\beta_1x_1+\\beta_2x_2+\\dots+\\beta_px_p)}}{1+\\exp{(\\beta_0+\\beta_1x_1+\\beta_2x_2+\\dots+\\beta_px_p)}}.\n", @@ -758,8 +866,10 @@ }, { "cell_type": "markdown", - "id": "75173b91", - "metadata": {}, + "id": "8d432ac3", + "metadata": { + "editable": true + }, "source": [ "## Including more classes\n", "\n", @@ -770,8 +880,10 @@ }, { "cell_type": "markdown", - "id": "60ea4955", - "metadata": {}, + "id": "c51817f0", + "metadata": { + "editable": true + }, "source": [ "$$\n", "\\log{\\frac{p(C=1\\vert x)}{p(K\\vert x)}} = \\beta_{10}+\\beta_{11}x_1,\n", @@ -780,16 +892,20 @@ }, { "cell_type": "markdown", - "id": "2ac0c3fe", - "metadata": {}, + "id": "8805fe2c", + "metadata": { + "editable": true + }, "source": [ "and" ] }, { "cell_type": "markdown", - "id": "c5633fe2", - "metadata": {}, + "id": "5021714e", + "metadata": { + "editable": true + }, "source": [ "$$\n", "\\log{\\frac{p(C=2\\vert x)}{p(K\\vert x)}} = \\beta_{20}+\\beta_{21}x_1,\n", @@ -798,16 +914,20 @@ }, { "cell_type": "markdown", - "id": "af63c7fc", - "metadata": {}, + "id": "9c0444b7", + "metadata": { + "editable": true + }, "source": [ "and so on till the class $C=K-1$ class" ] }, { "cell_type": "markdown", - "id": "68713442", - "metadata": {}, + "id": "88aa1709", + "metadata": { + "editable": true + }, "source": [ "$$\n", "\\log{\\frac{p(C=K-1\\vert x)}{p(K\\vert x)}} = \\beta_{(K-1)0}+\\beta_{(K-1)1}x_1,\n", @@ -816,8 +936,10 @@ }, { "cell_type": "markdown", - "id": "ae89a4ec", - "metadata": {}, + "id": "fce1f271", + "metadata": { + "editable": true + }, "source": [ "and the model is specified in term of $K-1$ so-called log-odds or\n", "**logit** transformations." @@ -825,8 +947,10 @@ }, { "cell_type": "markdown", - "id": "dc86b762", - "metadata": {}, + "id": "337c6786", + "metadata": { + "editable": true + }, "source": [ "## More classes\n", "\n", @@ -846,8 +970,10 @@ }, { "cell_type": "markdown", - "id": "4109330d", - "metadata": {}, + "id": "68e07c53", + "metadata": { + "editable": true + }, "source": [ "$$\n", "p(C=k\\vert \\mathbf {x} )=\\frac{\\exp{(\\beta_{k0}+\\beta_{k1}x_1)}}{1+\\sum_{l=1}^{K-1}\\exp{(\\beta_{l0}+\\beta_{l1}x_1)}}.\n", @@ -856,16 +982,20 @@ }, { "cell_type": "markdown", - "id": "dcf3f4c4", - "metadata": {}, + "id": "fbaa4072", + "metadata": { + "editable": true + }, "source": [ "It is easy to extend to more predictors. The final class is" ] }, { "cell_type": "markdown", - "id": "148d45e3", - "metadata": {}, + "id": "e562f2df", + "metadata": { + "editable": true + }, "source": [ "$$\n", "p(C=K\\vert \\mathbf {x} )=\\frac{1}{1+\\sum_{l=1}^{K-1}\\exp{(\\beta_{l0}+\\beta_{l1}x_1)}},\n", @@ -874,8 +1004,10 @@ }, { "cell_type": "markdown", - "id": "5c082186", - "metadata": {}, + "id": "1694e3d5", + "metadata": { + "editable": true + }, "source": [ "and they sum to one. Our earlier discussions were all specialized to\n", "the case with two classes only. It is easy to see from the above that\n", @@ -889,16 +1021,20 @@ }, { "cell_type": "markdown", - "id": "d5068fec", - "metadata": {}, + "id": "8f97950e", + "metadata": { + "editable": true + }, "source": [ "## Friday September 23" ] }, { "cell_type": "markdown", - "id": "d33abe73", - "metadata": {}, + "id": "b6a2bc72", + "metadata": { + "editable": true + }, "source": [ "## Searching for Optimal Regularization Parameters $\\lambda$\n", "\n", @@ -914,8 +1050,11 @@ { "cell_type": "code", "execution_count": 4, - "id": "d445125f", - "metadata": {}, + "id": "bc64603e", + "metadata": { + "collapsed": false, + "editable": true + }, "outputs": [], "source": [ "import numpy as np\n", @@ -966,8 +1105,10 @@ }, { "cell_type": "markdown", - "id": "3ad4a826", - "metadata": {}, + "id": "e6ea8584", + "metadata": { + "editable": true + }, "source": [ "Here we have performed a rather data greedy calculation as function of the regularization parameter $\\lambda$. There is no resampling here. The latter can easily be added by employing the function **RidgeCV** instead of just calling the **Ridge** function. For **RidgeCV** we need to pass the array of $\\lambda$ values.\n", "By inspecting the figure we can in turn determine which is the optimal regularization parameter.\n", @@ -976,8 +1117,10 @@ }, { "cell_type": "markdown", - "id": "ff84b2e7", - "metadata": {}, + "id": "00ccedeb", + "metadata": { + "editable": true + }, "source": [ "## Grid Search\n", "\n", @@ -989,8 +1132,11 @@ { "cell_type": "code", "execution_count": 5, - "id": "4b834234", - "metadata": {}, + "id": "179881ac", + "metadata": { + "collapsed": false, + "editable": true + }, "outputs": [], "source": [ "import numpy as np\n", @@ -1039,8 +1185,10 @@ }, { "cell_type": "markdown", - "id": "7514764b", - "metadata": {}, + "id": "16775a21", + "metadata": { + "editable": true + }, "source": [ "By default the grid search function includes cross validation with\n", "five folds. The [Scikit-Learn\n", @@ -1052,8 +1200,10 @@ }, { "cell_type": "markdown", - "id": "7991ea9f", - "metadata": {}, + "id": "ad4157f8", + "metadata": { + "editable": true + }, "source": [ "## Randomized Grid Search\n", "\n", @@ -1070,8 +1220,11 @@ { "cell_type": "code", "execution_count": 6, - "id": "ae4f99a1", - "metadata": {}, + "id": "76a1871a", + "metadata": { + "collapsed": false, + "editable": true + }, "outputs": [], "source": [ "import numpy as np\n", @@ -1121,8 +1274,10 @@ }, { "cell_type": "markdown", - "id": "91278700", - "metadata": {}, + "id": "daad0489", + "metadata": { + "editable": true + }, "source": [ "## Wisconsin Cancer Data\n", "\n", @@ -1134,8 +1289,11 @@ { "cell_type": "code", "execution_count": 7, - "id": "8f006bb8", - "metadata": {}, + "id": "fe6ea6c7", + "metadata": { + "collapsed": false, + "editable": true + }, "outputs": [], "source": [ "import matplotlib.pyplot as plt\n", @@ -1158,8 +1316,10 @@ }, { "cell_type": "markdown", - "id": "015f8534", - "metadata": {}, + "id": "d2b782f7", + "metadata": { + "editable": true + }, "source": [ "## Using the correlation matrix\n", "\n", @@ -1170,8 +1330,11 @@ { "cell_type": "code", "execution_count": 8, - "id": "fa6d3421", - "metadata": {}, + "id": "9a9334b6", + "metadata": { + "collapsed": false, + "editable": true + }, "outputs": [], "source": [ "import matplotlib.pyplot as plt\n", @@ -1212,8 +1375,10 @@ }, { "cell_type": "markdown", - "id": "49ff3f87", - "metadata": {}, + "id": "a775d363", + "metadata": { + "editable": true + }, "source": [ "## Discussing the correlation data\n", "\n", @@ -1235,8 +1400,11 @@ { "cell_type": "code", "execution_count": 9, - "id": "3e800820", - "metadata": {}, + "id": "6466bcf7", + "metadata": { + "collapsed": false, + "editable": true + }, "outputs": [], "source": [ "cancerpd = pd.DataFrame(cancer.data, columns=cancer.feature_names)" @@ -1244,8 +1412,10 @@ }, { "cell_type": "markdown", - "id": "9f831f22", - "metadata": {}, + "id": "7d127c92", + "metadata": { + "editable": true + }, "source": [ "and then" ] @@ -1253,8 +1423,11 @@ { "cell_type": "code", "execution_count": 10, - "id": "e663ae38", - "metadata": {}, + "id": "66bf9f55", + "metadata": { + "collapsed": false, + "editable": true + }, "outputs": [], "source": [ "correlation_matrix = cancerpd.corr().round(1)" @@ -1262,8 +1435,10 @@ }, { "cell_type": "markdown", - "id": "737193a2", - "metadata": {}, + "id": "740664e0", + "metadata": { + "editable": true + }, "source": [ "Diagonalizing this matrix we can in turn say something about which\n", "features are of relevance and which are not. This leads us to\n", @@ -1273,8 +1448,10 @@ }, { "cell_type": "markdown", - "id": "25104113", - "metadata": {}, + "id": "73f7d88c", + "metadata": { + "editable": true + }, "source": [ "## Other measures in classification studies: Cancer Data again" ] @@ -1282,8 +1459,11 @@ { "cell_type": "code", "execution_count": 11, - "id": "6783b76e", - "metadata": {}, + "id": "29598664", + "metadata": { + "collapsed": false, + "editable": true + }, "outputs": [], "source": [ "import matplotlib.pyplot as plt\n", @@ -1322,8 +1502,10 @@ }, { "cell_type": "markdown", - "id": "bdb4928a", - "metadata": {}, + "id": "d10e35ad", + "metadata": { + "editable": true + }, "source": [ "## Optimization, the central part of any Machine Learning algortithm\n", "\n", @@ -1341,8 +1523,10 @@ }, { "cell_type": "markdown", - "id": "8b73ff85", - "metadata": {}, + "id": "868f4fc9", + "metadata": { + "editable": true + }, "source": [ "## Revisiting our Logistic Regression case\n", "\n", @@ -1356,8 +1540,10 @@ }, { "cell_type": "markdown", - "id": "2b771369", - "metadata": {}, + "id": "e40c6036", + "metadata": { + "editable": true + }, "source": [ "$$\n", "\\begin{align*}\n", @@ -1369,16 +1555,20 @@ }, { "cell_type": "markdown", - "id": "10c01ae1", - "metadata": {}, + "id": "ebc39f54", + "metadata": { + "editable": true + }, "source": [ "where $\\boldsymbol{\\beta}$ are the weights we wish to extract from data, in our case $\\beta_0$ and $\\beta_1$." ] }, { "cell_type": "markdown", - "id": "b205858e", - "metadata": {}, + "id": "cdc96f44", + "metadata": { + "editable": true + }, "source": [ "## The equations to solve\n", "\n", @@ -1391,8 +1581,10 @@ }, { "cell_type": "markdown", - "id": "5f48f306", - "metadata": {}, + "id": "efe00ed3", + "metadata": { + "editable": true + }, "source": [ "$$\n", "\\frac{\\partial \\mathcal{C}(\\boldsymbol{\\beta})}{\\partial \\boldsymbol{\\beta}} = -\\boldsymbol{X}^T\\left(\\boldsymbol{y}-\\boldsymbol{p}\\right).\n", @@ -1401,8 +1593,10 @@ }, { "cell_type": "markdown", - "id": "deff0dec", - "metadata": {}, + "id": "d5f729c5", + "metadata": { + "editable": true + }, "source": [ "If we in addition define a diagonal matrix $\\boldsymbol{W}$ with elements \n", "$p(y_i\\vert x_i,\\boldsymbol{\\beta})(1-p(y_i\\vert x_i,\\boldsymbol{\\beta})$, we can obtain a compact expression of the second derivative as" @@ -1410,8 +1604,10 @@ }, { "cell_type": "markdown", - "id": "4a3eee34", - "metadata": {}, + "id": "1a41bf0f", + "metadata": { + "editable": true + }, "source": [ "$$\n", "\\frac{\\partial^2 \\mathcal{C}(\\boldsymbol{\\beta})}{\\partial \\boldsymbol{\\beta}\\partial \\boldsymbol{\\beta}^T} = \\boldsymbol{X}^T\\boldsymbol{W}\\boldsymbol{X}.\n", @@ -1420,16 +1616,20 @@ }, { "cell_type": "markdown", - "id": "ac2953d0", - "metadata": {}, + "id": "8b64c820", + "metadata": { + "editable": true + }, "source": [ "This defines what is called the Hessian matrix." ] }, { "cell_type": "markdown", - "id": "fcacb96c", - "metadata": {}, + "id": "76cee9c1", + "metadata": { + "editable": true + }, "source": [ "## Solving using Newton-Raphson's method\n", "\n", @@ -1440,8 +1640,10 @@ }, { "cell_type": "markdown", - "id": "d8946939", - "metadata": {}, + "id": "43c91975", + "metadata": { + "editable": true + }, "source": [ "$$\n", "\\boldsymbol{\\beta}^{\\mathrm{new}} = \\boldsymbol{\\beta}^{\\mathrm{old}}-\\left(\\frac{\\partial^2 \\mathcal{C}(\\boldsymbol{\\beta})}{\\partial \\boldsymbol{\\beta}\\partial \\boldsymbol{\\beta}^T}\\right)^{-1}_{\\boldsymbol{\\beta}^{\\mathrm{old}}}\\times \\left(\\frac{\\partial \\mathcal{C}(\\boldsymbol{\\beta})}{\\partial \\boldsymbol{\\beta}}\\right)_{\\boldsymbol{\\beta}^{\\mathrm{old}}},\n", @@ -1450,16 +1652,20 @@ }, { "cell_type": "markdown", - "id": "1cb3f7cd", - "metadata": {}, + "id": "b550e27c", + "metadata": { + "editable": true + }, "source": [ "or in matrix form as" ] }, { "cell_type": "markdown", - "id": "5707329b", - "metadata": {}, + "id": "3512606f", + "metadata": { + "editable": true + }, "source": [ "$$\n", "\\boldsymbol{\\beta}^{\\mathrm{new}} = \\boldsymbol{\\beta}^{\\mathrm{old}}-\\left(\\boldsymbol{X}^T\\boldsymbol{W}\\boldsymbol{X} \\right)^{-1}\\times \\left(-\\boldsymbol{X}^T(\\boldsymbol{y}-\\boldsymbol{p}) \\right)_{\\boldsymbol{\\beta}^{\\mathrm{old}}}.\n", @@ -1468,8 +1674,10 @@ }, { "cell_type": "markdown", - "id": "996511e9", - "metadata": {}, + "id": "ed7113e2", + "metadata": { + "editable": true + }, "source": [ "The right-hand side is computed with the old values of $\\beta$. \n", "\n", @@ -1478,8 +1686,10 @@ }, { "cell_type": "markdown", - "id": "a35fcbf1", - "metadata": {}, + "id": "5c94e88a", + "metadata": { + "editable": true + }, "source": [ "## Brief reminder on Newton-Raphson's method\n", "\n", @@ -1496,8 +1706,10 @@ }, { "cell_type": "markdown", - "id": "080ea026", - "metadata": {}, + "id": "f0f54976", + "metadata": { + "editable": true + }, "source": [ "## The equations\n", "\n", @@ -1510,8 +1722,10 @@ }, { "cell_type": "markdown", - "id": "49636667", - "metadata": {}, + "id": "818e74ab", + "metadata": { + "editable": true + }, "source": [ "\n", "
    \n", @@ -1524,8 +1738,10 @@ }, { "cell_type": "markdown", - "id": "3bd16905", - "metadata": {}, + "id": "6fd49143", + "metadata": { + "editable": true + }, "source": [ "For small enough values of the function and for well-behaved\n", "functions, the terms beyond linear are unimportant, hence we obtain" @@ -1533,8 +1749,10 @@ }, { "cell_type": "markdown", - "id": "68cc347e", - "metadata": {}, + "id": "60172ae6", + "metadata": { + "editable": true + }, "source": [ "$$\n", "f(x)+(s-x)f'(x)\\approx 0,\n", @@ -1543,16 +1761,20 @@ }, { "cell_type": "markdown", - "id": "3ba3e954", - "metadata": {}, + "id": "8348ee17", + "metadata": { + "editable": true + }, "source": [ "yielding" ] }, { "cell_type": "markdown", - "id": "18a71133", - "metadata": {}, + "id": "b40896a1", + "metadata": { + "editable": true + }, "source": [ "$$\n", "s\\approx x-\\frac{f(x)}{f'(x)}.\n", @@ -1561,16 +1783,20 @@ }, { "cell_type": "markdown", - "id": "cebe1598", - "metadata": {}, + "id": "d316c0f2", + "metadata": { + "editable": true + }, "source": [ "Having in mind an iterative procedure, it is natural to start iterating with" ] }, { "cell_type": "markdown", - "id": "262fc2e7", - "metadata": {}, + "id": "c901548d", + "metadata": { + "editable": true + }, "source": [ "$$\n", "x_{n+1}=x_n-\\frac{f(x_n)}{f'(x_n)}.\n", @@ -1579,8 +1805,10 @@ }, { "cell_type": "markdown", - "id": "d8ac2615", - "metadata": {}, + "id": "1ed9ffca", + "metadata": { + "editable": true + }, "source": [ "## Simple geometric interpretation\n", "\n", @@ -1599,8 +1827,10 @@ }, { "cell_type": "markdown", - "id": "1637b673", - "metadata": {}, + "id": "bca207b1", + "metadata": { + "editable": true + }, "source": [ "## Extending to more than one variable\n", "\n", @@ -1610,8 +1840,10 @@ }, { "cell_type": "markdown", - "id": "efb3a153", - "metadata": {}, + "id": "7bb057bf", + "metadata": { + "editable": true + }, "source": [ "$$\n", "\\begin{array}{cc} f_1(x_1,x_2) &=0\\\\\n", @@ -1621,16 +1853,20 @@ }, { "cell_type": "markdown", - "id": "cd1d0b0b", - "metadata": {}, + "id": "46c3fe1d", + "metadata": { + "editable": true + }, "source": [ "which we Taylor expand to obtain" ] }, { "cell_type": "markdown", - "id": "1409701b", - "metadata": {}, + "id": "4a1099a8", + "metadata": { + "editable": true + }, "source": [ "$$\n", "\\begin{array}{cc} 0=f_1(x_1+h_1,x_2+h_2)=&f_1(x_1,x_2)+h_1\n", @@ -1645,16 +1881,20 @@ }, { "cell_type": "markdown", - "id": "969cdccb", - "metadata": {}, + "id": "34ca9747", + "metadata": { + "editable": true + }, "source": [ "Defining the Jacobian matrix ${\\bf \\boldsymbol{J}}$ we have" ] }, { "cell_type": "markdown", - "id": "1d26f373", - "metadata": {}, + "id": "28a1eba3", + "metadata": { + "editable": true + }, "source": [ "$$\n", "{\\bf \\boldsymbol{J}}=\\left( \\begin{array}{cc}\n", @@ -1666,16 +1906,20 @@ }, { "cell_type": "markdown", - "id": "b6fa2c0e", - "metadata": {}, + "id": "25968453", + "metadata": { + "editable": true + }, "source": [ "we can rephrase Newton's method as" ] }, { "cell_type": "markdown", - "id": "c26cca81", - "metadata": {}, + "id": "358899db", + "metadata": { + "editable": true + }, "source": [ "$$\n", "\\left(\\begin{array}{c} x_1^{n+1} \\\\ x_2^{n+1} \\end{array} \\right)=\n", @@ -1686,16 +1930,20 @@ }, { "cell_type": "markdown", - "id": "e6813f1b", - "metadata": {}, + "id": "6e508f2f", + "metadata": { + "editable": true + }, "source": [ "where we have defined" ] }, { "cell_type": "markdown", - "id": "f53cce6d", - "metadata": {}, + "id": "66139dbd", + "metadata": { + "editable": true + }, "source": [ "$$\n", "\\left(\\begin{array}{c} h_1^{n} \\\\ h_2^{n} \\end{array} \\right)=\n", @@ -1706,8 +1954,10 @@ }, { "cell_type": "markdown", - "id": "47a097ab", - "metadata": {}, + "id": "8dce9074", + "metadata": { + "editable": true + }, "source": [ "We need thus to compute the inverse of the Jacobian matrix and it\n", "is to understand that difficulties may\n", @@ -1719,8 +1969,10 @@ }, { "cell_type": "markdown", - "id": "682046e4", - "metadata": {}, + "id": "689e37e5", + "metadata": { + "editable": true + }, "source": [ "## Steepest descent\n", "\n", @@ -1734,8 +1986,10 @@ }, { "cell_type": "markdown", - "id": "91f0fc21", - "metadata": {}, + "id": "cdb3d691", + "metadata": { + "editable": true + }, "source": [ "$$\n", "\\mathbf{x}_{k+1} = \\mathbf{x}_k - \\gamma_k \\nabla F(\\mathbf{x}_k),\n", @@ -1744,8 +1998,10 @@ }, { "cell_type": "markdown", - "id": "7e415ba2", - "metadata": {}, + "id": "6817476f", + "metadata": { + "editable": true + }, "source": [ "with $\\gamma_k > 0$.\n", "\n", @@ -1756,8 +2012,10 @@ }, { "cell_type": "markdown", - "id": "d67a88be", - "metadata": {}, + "id": "25b1271d", + "metadata": { + "editable": true + }, "source": [ "## More on Steepest descent\n", "\n", @@ -1769,8 +2027,10 @@ }, { "cell_type": "markdown", - "id": "ee068302", - "metadata": {}, + "id": "61814aee", + "metadata": { + "editable": true + }, "source": [ "$$\n", "\\mathbf{x}_{k+1} = \\mathbf{x}_k - \\gamma_k \\nabla F(\\mathbf{x}_k), \\ \\ k \\geq 0.\n", @@ -1779,8 +2039,10 @@ }, { "cell_type": "markdown", - "id": "d63dc655", - "metadata": {}, + "id": "8a8c5bd6", + "metadata": { + "editable": true + }, "source": [ "The parameter $\\gamma_k$ is often referred to as the step length or\n", "the learning rate within the context of Machine Learning." @@ -1788,8 +2050,10 @@ }, { "cell_type": "markdown", - "id": "fc1fc57d", - "metadata": {}, + "id": "42b71504", + "metadata": { + "editable": true + }, "source": [ "## The ideal\n", "\n", @@ -1814,8 +2078,10 @@ }, { "cell_type": "markdown", - "id": "b97eed70", - "metadata": {}, + "id": "671b2192", + "metadata": { + "editable": true + }, "source": [ "## The sensitiveness of the gradient descent\n", "\n", @@ -1834,8 +2100,10 @@ }, { "cell_type": "markdown", - "id": "c35084e9", - "metadata": {}, + "id": "3fe798d9", + "metadata": { + "editable": true + }, "source": [ "## Convex functions\n", "\n", @@ -1854,8 +2122,10 @@ }, { "cell_type": "markdown", - "id": "a11b1893", - "metadata": {}, + "id": "f316a962", + "metadata": { + "editable": true + }, "source": [ "## Convex function\n", "\n", @@ -1864,8 +2134,10 @@ }, { "cell_type": "markdown", - "id": "f54f8d27", - "metadata": {}, + "id": "97f9a464", + "metadata": { + "editable": true + }, "source": [ "## Conditions on convex functions\n", "\n", @@ -1899,8 +2171,10 @@ }, { "cell_type": "markdown", - "id": "a8d7420c", - "metadata": {}, + "id": "25b328f5", + "metadata": { + "editable": true + }, "source": [ "## More on convex functions\n", "\n", @@ -1925,8 +2199,10 @@ }, { "cell_type": "markdown", - "id": "877d661b", - "metadata": {}, + "id": "de65b0c2", + "metadata": { + "editable": true + }, "source": [ "## Some simple problems\n", "\n", @@ -1953,8 +2229,10 @@ }, { "cell_type": "markdown", - "id": "74823208", - "metadata": {}, + "id": "c3dc52a8", + "metadata": { + "editable": true + }, "source": [ "## Revisiting our first homework\n", "\n", @@ -1975,8 +2253,11 @@ { "cell_type": "code", "execution_count": 12, - "id": "2bafe07c", - "metadata": {}, + "id": "3f5d367b", + "metadata": { + "collapsed": false, + "editable": true + }, "outputs": [], "source": [ "x = 2*np.random.rand(m,1)\n", @@ -1985,8 +2266,10 @@ }, { "cell_type": "markdown", - "id": "a18bdeb4", - "metadata": {}, + "id": "5f4a843c", + "metadata": { + "editable": true + }, "source": [ "with $x_i \\in [0,1] $ is chosen randomly using a uniform distribution. Additionally we have a stochastic noise chosen according to a normal distribution $\\cal {N}(0,1)$. \n", "The linear regression model is given by" @@ -1994,8 +2277,10 @@ }, { "cell_type": "markdown", - "id": "16415b8c", - "metadata": {}, + "id": "3aee82dd", + "metadata": { + "editable": true + }, "source": [ "$$\n", "h_\\beta(x) = \\boldsymbol{y} = \\beta_0 + \\beta_1 x,\n", @@ -2004,16 +2289,20 @@ }, { "cell_type": "markdown", - "id": "dc273c04", - "metadata": {}, + "id": "727e0a46", + "metadata": { + "editable": true + }, "source": [ "such that" ] }, { "cell_type": "markdown", - "id": "4332c289", - "metadata": {}, + "id": "18f89452", + "metadata": { + "editable": true + }, "source": [ "$$\n", "\\boldsymbol{y}_i = \\beta_0 + \\beta_1 x_i.\n", @@ -2022,8 +2311,10 @@ }, { "cell_type": "markdown", - "id": "e96ba3ef", - "metadata": {}, + "id": "86dd5168", + "metadata": { + "editable": true + }, "source": [ "## Gradient descent example\n", "\n", @@ -2034,8 +2325,10 @@ }, { "cell_type": "markdown", - "id": "deb14fe7", - "metadata": {}, + "id": "3bae9989", + "metadata": { + "editable": true + }, "source": [ "$$\n", "X \\equiv \\begin{bmatrix}\n", @@ -2048,16 +2341,20 @@ }, { "cell_type": "markdown", - "id": "e468ef7d", - "metadata": {}, + "id": "1ac737bb", + "metadata": { + "editable": true + }, "source": [ "The cost/loss/risk function is given by (" ] }, { "cell_type": "markdown", - "id": "97154cec", - "metadata": {}, + "id": "22a7b230", + "metadata": { + "editable": true + }, "source": [ "$$\n", "C(\\beta) = \\frac{1}{n}||X\\beta-\\mathbf{y}||_{2}^{2} = \\frac{1}{n}\\sum_{i=1}^{100}\\left[ (\\beta_0 + \\beta_1 x_i)^2 - 2 y_i (\\beta_0 + \\beta_1 x_i) + y_i^2\\right]\n", @@ -2066,16 +2363,20 @@ }, { "cell_type": "markdown", - "id": "6e30b09a", - "metadata": {}, + "id": "4041ae93", + "metadata": { + "editable": true + }, "source": [ "and we want to find $\\beta$ such that $C(\\beta)$ is minimized." ] }, { "cell_type": "markdown", - "id": "d5dc7377", - "metadata": {}, + "id": "9c8a529f", + "metadata": { + "editable": true + }, "source": [ "## The derivative of the cost/loss function\n", "\n", @@ -2084,8 +2385,10 @@ }, { "cell_type": "markdown", - "id": "49d2621f", - "metadata": {}, + "id": "feea5c49", + "metadata": { + "editable": true + }, "source": [ "$$\n", "\\nabla_{\\beta} C(\\beta) = \\frac{2}{n}\\begin{bmatrix} \\sum_{i=1}^{100} \\left(\\beta_0+\\beta_1x_i-y_i\\right) \\\\\n", @@ -2096,16 +2399,20 @@ }, { "cell_type": "markdown", - "id": "f2950e6b", - "metadata": {}, + "id": "32d8a938", + "metadata": { + "editable": true + }, "source": [ "where $X$ is the design matrix defined above." ] }, { "cell_type": "markdown", - "id": "00be7184", - "metadata": {}, + "id": "f944ed82", + "metadata": { + "editable": true + }, "source": [ "## The Hessian matrix\n", "The Hessian matrix of $C(\\beta)$ is given by" @@ -2113,8 +2420,10 @@ }, { "cell_type": "markdown", - "id": "97b443f2", - "metadata": {}, + "id": "60b063df", + "metadata": { + "editable": true + }, "source": [ "$$\n", "\\boldsymbol{H} \\equiv \\begin{bmatrix}\n", @@ -2126,16 +2435,20 @@ }, { "cell_type": "markdown", - "id": "0676ebaf", - "metadata": {}, + "id": "bfdd0de8", + "metadata": { + "editable": true + }, "source": [ "This result implies that $C(\\beta)$ is a convex function since the matrix $X^T X$ always is positive semi-definite." ] }, { "cell_type": "markdown", - "id": "4ad98c2d", - "metadata": {}, + "id": "b70a644b", + "metadata": { + "editable": true + }, "source": [ "## Simple program\n", "\n", @@ -2144,8 +2457,10 @@ }, { "cell_type": "markdown", - "id": "7405f7a9", - "metadata": {}, + "id": "4935f300", + "metadata": { + "editable": true + }, "source": [ "$$\n", "\\beta_{k+1} = \\beta_k - \\gamma \\nabla_\\beta C(\\beta_k), \\ k=0,1,\\cdots\n", @@ -2154,8 +2469,10 @@ }, { "cell_type": "markdown", - "id": "ca70147e", - "metadata": {}, + "id": "d9fe6992", + "metadata": { + "editable": true + }, "source": [ "We can use the expression we computed for the gradient and let use a\n", "$\\beta_0$ be chosen randomly and let $\\gamma = 0.001$. Stop iterating\n", @@ -2167,8 +2484,10 @@ }, { "cell_type": "markdown", - "id": "d07fdcda", - "metadata": {}, + "id": "195b0044", + "metadata": { + "editable": true + }, "source": [ "## Gradient Descent Example\n", "\n", @@ -2178,8 +2497,11 @@ { "cell_type": "code", "execution_count": 13, - "id": "2483d42a", - "metadata": {}, + "id": "36bad401", + "metadata": { + "collapsed": false, + "editable": true + }, "outputs": [], "source": [ "\n", @@ -2232,8 +2554,10 @@ }, { "cell_type": "markdown", - "id": "6071973b", - "metadata": {}, + "id": "6341effe", + "metadata": { + "editable": true + }, "source": [ "## And a corresponding example using **scikit-learn**" ] @@ -2241,8 +2565,11 @@ { "cell_type": "code", "execution_count": 14, - "id": "f31a67da", - "metadata": {}, + "id": "b1114145", + "metadata": { + "collapsed": false, + "editable": true + }, "outputs": [], "source": [ "# Importing various packages\n", @@ -2265,8 +2592,10 @@ }, { "cell_type": "markdown", - "id": "c6ae27ed", - "metadata": {}, + "id": "448e4708", + "metadata": { + "editable": true + }, "source": [ "## Gradient descent and Ridge\n", "\n", @@ -2275,8 +2604,10 @@ }, { "cell_type": "markdown", - "id": "55efcb99", - "metadata": {}, + "id": "613cc32f", + "metadata": { + "editable": true + }, "source": [ "$$\n", "C_{\\text{ridge}}(\\beta) = \\frac{1}{n}||X\\beta -\\mathbf{y}||^2 + \\lambda ||\\beta||^2, \\ \\lambda \\geq 0.\n", @@ -2285,16 +2616,20 @@ }, { "cell_type": "markdown", - "id": "aedf6cb2", - "metadata": {}, + "id": "727dbcda", + "metadata": { + "editable": true + }, "source": [ "In order to minimize $C_{\\text{ridge}}(\\beta)$ using GD we adjust the gradient as follows" ] }, { "cell_type": "markdown", - "id": "7bb7c811", - "metadata": {}, + "id": "5bdb0b5e", + "metadata": { + "editable": true + }, "source": [ "$$\n", "\\nabla_\\beta C_{\\text{ridge}}(\\beta) = \\frac{2}{n}\\begin{bmatrix} \\sum_{i=1}^{100} \\left(\\beta_0+\\beta_1x_i-y_i\\right) \\\\\n", @@ -2305,16 +2640,20 @@ }, { "cell_type": "markdown", - "id": "42227947", - "metadata": {}, + "id": "7a76e64c", + "metadata": { + "editable": true + }, "source": [ "We can easily extend our program to minimize $C_{\\text{ridge}}(\\beta)$ using gradient descent and compare with the analytical solution given by" ] }, { "cell_type": "markdown", - "id": "9c164deb", - "metadata": {}, + "id": "30ec5b3a", + "metadata": { + "editable": true + }, "source": [ "$$\n", "\\beta_{\\text{ridge}} = \\left(X^T X + n\\lambda I_{2 \\times 2} \\right)^{-1} X^T \\mathbf{y}.\n", @@ -2323,8 +2662,10 @@ }, { "cell_type": "markdown", - "id": "eb38ae48", - "metadata": {}, + "id": "9d138a4f", + "metadata": { + "editable": true + }, "source": [ "## The Hessian matrix for Ridge Regression\n", "The Hessian matrix of Ridge Regression for our simple example is given by" @@ -2332,8 +2673,10 @@ }, { "cell_type": "markdown", - "id": "a6fcf56c", - "metadata": {}, + "id": "2d932b23", + "metadata": { + "editable": true + }, "source": [ "$$\n", "\\boldsymbol{H} \\equiv \\begin{bmatrix}\n", @@ -2345,8 +2688,10 @@ }, { "cell_type": "markdown", - "id": "3a2b5ae4", - "metadata": {}, + "id": "297ec387", + "metadata": { + "editable": true + }, "source": [ "This implies that the Hessian matrix is positive definite, hence the stationary point is a\n", "minimum.\n", @@ -2357,8 +2702,10 @@ }, { "cell_type": "markdown", - "id": "1a7eb7e6", - "metadata": {}, + "id": "67746899", + "metadata": { + "editable": true + }, "source": [ "## Program example for gradient descent with Ridge Regression" ] @@ -2366,8 +2713,11 @@ { "cell_type": "code", "execution_count": 15, - "id": "ccb7a9b1", - "metadata": {}, + "id": "241b8563", + "metadata": { + "collapsed": false, + "editable": true + }, "outputs": [], "source": [ "from random import random, seed\n", @@ -2424,8 +2774,10 @@ }, { "cell_type": "markdown", - "id": "95a71108", - "metadata": {}, + "id": "0fbba954", + "metadata": { + "editable": true + }, "source": [ "## Using gradient descent methods, limitations\n", "\n", @@ -2444,8 +2796,10 @@ }, { "cell_type": "markdown", - "id": "33f25532", - "metadata": {}, + "id": "1da57290", + "metadata": { + "editable": true + }, "source": [ "## Challenge yourself the coming weekend\n", "\n", @@ -2454,16 +2808,20 @@ }, { "cell_type": "markdown", - "id": "13ee68fe", - "metadata": {}, + "id": "4ec72b3c", + "metadata": { + "editable": true + }, "source": [ "## Lab session: Material from last week and relevant for the first project" ] }, { "cell_type": "markdown", - "id": "e3e607f1", - "metadata": {}, + "id": "4c6beaee", + "metadata": { + "editable": true + }, "source": [ "## Various steps in cross-validation\n", "\n", @@ -2485,8 +2843,10 @@ }, { "cell_type": "markdown", - "id": "42b264a0", - "metadata": {}, + "id": "9faaa3b4", + "metadata": { + "editable": true + }, "source": [ "## How to set up the cross-validation for Ridge and/or Lasso\n", "\n", @@ -2499,8 +2859,10 @@ }, { "cell_type": "markdown", - "id": "ff85b5e7", - "metadata": {}, + "id": "1404fbc3", + "metadata": { + "editable": true + }, "source": [ "$$\n", "\\begin{align*}\n", @@ -2513,8 +2875,10 @@ }, { "cell_type": "markdown", - "id": "5d0533bb", - "metadata": {}, + "id": "40db0350", + "metadata": { + "editable": true + }, "source": [ "* Evaluate the prediction performance of these models on the test set by $C[y_i, \\boldsymbol{X}_{i, \\ast}; \\boldsymbol{\\beta}_{-i}(\\lambda), \\boldsymbol{\\sigma}_{-i}^2(\\lambda)]$. Or, by the prediction error $|y_i - \\boldsymbol{X}_{i, \\ast} \\boldsymbol{\\beta}_{-i}(\\lambda)|$, the relative error, the error squared or the R2 score function.\n", "\n", @@ -2525,8 +2889,10 @@ }, { "cell_type": "markdown", - "id": "3ab3921a", - "metadata": {}, + "id": "6da0d307", + "metadata": { + "editable": true + }, "source": [ "## Cross-validation in brief\n", "\n", @@ -2551,8 +2917,10 @@ }, { "cell_type": "markdown", - "id": "0595f2f0", - "metadata": {}, + "id": "13daa87d", + "metadata": { + "editable": true + }, "source": [ "## Code Example for Cross-validation and $k$-fold Cross-validation\n", "\n", @@ -2562,8 +2930,11 @@ { "cell_type": "code", "execution_count": 16, - "id": "2902bb22", - "metadata": {}, + "id": "6cc85c1c", + "metadata": { + "collapsed": false, + "editable": true + }, "outputs": [], "source": [ "import numpy as np\n", @@ -2658,25 +3029,7 @@ ] } ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.9.18" - } - }, + "metadata": {}, "nbformat": 4, "nbformat_minor": 5 } diff --git a/doc/src/week38/week38.do.txt b/doc/src/week38/week38.do.txt index d1427ede4..9099a9b37 100644 --- a/doc/src/week38/week38.do.txt +++ b/doc/src/week38/week38.do.txt @@ -11,7 +11,7 @@ DATE: September 16-20, 2024 !bblock Material for the lecture on Monday September 16 * Logistic regression as our first encounter of classification methods. From binary cases to several categories. * Start gradient and optimization methods -# * "Video of lecture":"https://youtu.be/cJrSwRsVSGM" + * "Video of lecture":"https://youtu.be/c9DIfNHy2ks" # * Whiteboard notes at URL:"https://github.com/CompPhysics/MachineLearning/blob/master/doc/HandWrittenNotes/2023/NotesSep21.pdf" !eblock