update

doc/LectureNotes/_build/html/week47.html

@@ -55,7 +55,7 @@ const thebe_selector_output = ".output, .cell_output"
     <script defer="defer" src="https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-mml-chtml.js"></script>
     <link rel="index" title="Index" href="genindex.html" />
     <link rel="search" title="Search" href="search.html" />
-    <link rel="next" title="Project 1 on Machine Learning, deadline October 9 (midnight), 2023" href="project1.html" />
+    <link rel="next" title="Exercise week 47" href="exercisesweek47.html" />
     <link rel="prev" title="Week 46: Decision Trees, Ensemble methods and Random Forests" href="week46.html" />
     <meta name="viewport" content="width=device-width, initial-scale=1" />
     <meta name="docsearch:language" content="None">
@@ -363,6 +363,11 @@ const thebe_selector_output = ".output, .cell_output"
    Week 47: From Decision Trees to Ensemble Methods, Random Forests and Boosting Methods   and Summary of Course
   </a>
  </li>
+ <li class="toctree-l1">
+  <a class="reference internal" href="exercisesweek47.html">
+   Exercise week 47
+  </a>
+ </li>
 </ul>
 <p aria-level="2" class="caption" role="heading">
  <span class="caption-text">
@@ -380,11 +385,6 @@ const thebe_selector_output = ".output, .cell_output"
    Project 2 on Machine Learning, deadline November 17 (Midnight)
   </a>
  </li>
- <li class="toctree-l1">
-  <a class="reference internal" href="project3.html">
-   Project 3 on Machine Learning, deadline December 18 (midnight), 2023
-  </a>
- </li>
 </ul>
 
     </div>
@@ -1294,6 +1294,8 @@ doconce format html week47.do.txt --no_mako -->
 <li><p>Readings and Videos:</p>
 <ul>
 <li><p>These lecture notes</p></li>
+<li><p><a class="reference external" href="https://youtu.be/SpWXsvn5I9E">Video of Lecture</a></p></li>
+<li><p><a class="reference external" href="https://github.com/CompPhysics/MachineLearning/blob/master/doc/HandWrittenNotes/2023/NotesNov23.pdf">Whiteboard notes</a></p></li>
 <li><p><a class="reference external" href="https://www.youtube.com/watch?v=RmajweUFKvM&amp;ab_channel=Simplilearn">Video on Decision trees</a></p></li>
 <li><p><a class="reference external" href="https://www.youtube.com/watch?v=wPqtzj5VZus&amp;ab_channel=H2O.ai">Video on boosting methods by Hastie</a></p></li>
 <li><p><a class="reference external" href="https://www.youtube.com/watch?v=LsK-xG1cLYA">Video on AdaBoost</a></p></li>
@@ -1347,13 +1349,44 @@ a decision tree wth different depths and perform a bootstrap aggregate (in this
 <div class="cell_input docutils container">
 <div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="o">%</span><span class="k">matplotlib</span> inline
 
-
+<span class="c1"># Common imports</span>
 <span class="kn">import</span> <span class="nn">matplotlib.pyplot</span> <span class="k">as</span> <span class="nn">plt</span>
 <span class="kn">import</span> <span class="nn">numpy</span> <span class="k">as</span> <span class="nn">np</span>
 <span class="kn">from</span> <span class="nn">sklearn.model_selection</span> <span class="kn">import</span> <span class="n">train_test_split</span>
 <span class="kn">from</span> <span class="nn">sklearn.pipeline</span> <span class="kn">import</span> <span class="n">make_pipeline</span>
 <span class="kn">from</span> <span class="nn">sklearn.utils</span> <span class="kn">import</span> <span class="n">resample</span>
 <span class="kn">from</span> <span class="nn">sklearn.tree</span> <span class="kn">import</span> <span class="n">DecisionTreeRegressor</span>
+<span class="kn">import</span> <span class="nn">pandas</span> <span class="k">as</span> <span class="nn">pd</span>
+<span class="kn">from</span> <span class="nn">sklearn.tree</span> <span class="kn">import</span> <span class="n">DecisionTreeClassifier</span>
+<span class="kn">from</span> <span class="nn">sklearn.model_selection</span> <span class="kn">import</span> <span class="n">train_test_split</span>
+<span class="kn">from</span> <span class="nn">sklearn.preprocessing</span> <span class="kn">import</span> <span class="n">StandardScaler</span><span class="p">,</span> <span class="n">OneHotEncoder</span>
+<span class="kn">from</span> <span class="nn">sklearn.compose</span> <span class="kn">import</span> <span class="n">ColumnTransformer</span>
+<span class="kn">from</span> <span class="nn">IPython.display</span> <span class="kn">import</span> <span class="n">Image</span> 
+<span class="kn">import</span> <span class="nn">os</span>
+
+<span class="c1"># Where to save the figures and data files</span>
+<span class="n">PROJECT_ROOT_DIR</span> <span class="o">=</span> <span class="s2">&quot;Results&quot;</span>
+<span class="n">FIGURE_ID</span> <span class="o">=</span> <span class="s2">&quot;Results/FigureFiles&quot;</span>
+<span class="n">DATA_ID</span> <span class="o">=</span> <span class="s2">&quot;DataFiles/&quot;</span>
+
+<span class="k">if</span> <span class="ow">not</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">exists</span><span class="p">(</span><span class="n">PROJECT_ROOT_DIR</span><span class="p">):</span>
+    <span class="n">os</span><span class="o">.</span><span class="n">mkdir</span><span class="p">(</span><span class="n">PROJECT_ROOT_DIR</span><span class="p">)</span>
+
+<span class="k">if</span> <span class="ow">not</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">exists</span><span class="p">(</span><span class="n">FIGURE_ID</span><span class="p">):</span>
+    <span class="n">os</span><span class="o">.</span><span class="n">makedirs</span><span class="p">(</span><span class="n">FIGURE_ID</span><span class="p">)</span>
+
+<span class="k">if</span> <span class="ow">not</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">exists</span><span class="p">(</span><span class="n">DATA_ID</span><span class="p">):</span>
+    <span class="n">os</span><span class="o">.</span><span class="n">makedirs</span><span class="p">(</span><span class="n">DATA_ID</span><span class="p">)</span>
+
+<span class="k">def</span> <span class="nf">image_path</span><span class="p">(</span><span class="n">fig_id</span><span class="p">):</span>
+    <span class="k">return</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">FIGURE_ID</span><span class="p">,</span> <span class="n">fig_id</span><span class="p">)</span>
+
+<span class="k">def</span> <span class="nf">data_path</span><span class="p">(</span><span class="n">dat_id</span><span class="p">):</span>
+    <span class="k">return</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">DATA_ID</span><span class="p">,</span> <span class="n">dat_id</span><span class="p">)</span>
+
+<span class="k">def</span> <span class="nf">save_fig</span><span class="p">(</span><span class="n">fig_id</span><span class="p">):</span>
+    <span class="n">plt</span><span class="o">.</span><span class="n">savefig</span><span class="p">(</span><span class="n">image_path</span><span class="p">(</span><span class="n">fig_id</span><span class="p">)</span> <span class="o">+</span> <span class="s2">&quot;.png&quot;</span><span class="p">,</span> <span class="nb">format</span><span class="o">=</span><span class="s1">&#39;png&#39;</span><span class="p">)</span>
+
 
 <span class="n">n</span> <span class="o">=</span> <span class="mi">100</span>
 <span class="n">n_boostraps</span> <span class="o">=</span> <span class="mi">100</span>
@@ -1410,55 +1443,44 @@ a decision tree wth different depths and perform a bootstrap aggregate (in this
 </div>
 <div class="cell_output docutils container">
 <div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>Polynomial degree: 1
-Error: 0.06535721144629267
-Bias^2: 0.05063935014924984
-Var: 0.014717861297042837
-0.06535721144629267 &gt;= 0.05063935014924984 + 0.014717861297042837 = 0.06535721144629268
+Error: 0.05485044745873867
+Bias^2: 0.05363014989229746
+Var: 0.0012202975664411882
+0.05485044745873867 &gt;= 0.05363014989229746 + 0.0012202975664411882 = 0.05485044745873865
 Polynomial degree: 2
-Error: 0.057050038714429464
-Bias^2: 0.04640211541749118
-Var: 0.010647923296938294
-0.057050038714429464 &gt;= 0.04640211541749118 + 0.010647923296938294 = 0.05705003871442947
+Error: 0.04754825003279861
+Bias^2: 0.0362312015777108
+Var: 0.01131704845508782
+0.04754825003279861 &gt;= 0.0362312015777108 + 0.01131704845508782 = 0.04754825003279862
 Polynomial degree: 3
-Error: 0.027176144692975295
-Bias^2: 0.01985410516551995
-Var: 0.007322039527455341
-0.027176144692975295 &gt;= 0.01985410516551995 + 0.007322039527455341 = 0.02717614469297529
+Error: 0.028256917047283964
+Bias^2: 0.019709199043491926
+Var: 0.008547718003792035
+0.028256917047283964 &gt;= 0.019709199043491926 + 0.008547718003792035 = 0.02825691704728396
 Polynomial degree: 4
-Error: 0.016797223248922855
-Bias^2: 0.010349011563455255
-Var: 0.006448211685467599
-0.016797223248922855 &gt;= 0.010349011563455255 + 0.006448211685467599 = 0.016797223248922855
+Error: 0.02417252675174287
+Bias^2: 0.016541517177965183
+Var: 0.007631009573777696
+0.02417252675174287 &gt;= 0.016541517177965183 + 0.007631009573777696 = 0.02417252675174288
 Polynomial degree: 5
-Error: 0.01554799732002449
-Bias^2: 0.009843577857448754
-Var: 0.005704419462575732
-0.01554799732002449 &gt;= 0.009843577857448754 + 0.005704419462575732 = 0.015547997320024485
+Error: 0.020350773309798075
+Bias^2: 0.013742894355267554
+Var: 0.006607878954530523
+0.020350773309798075 &gt;= 0.013742894355267554 + 0.006607878954530523 = 0.02035077330979808
 Polynomial degree: 6
-Error: 0.017779706159045054
-Bias^2: 0.011376853216444322
-Var: 0.0064028529426007315
-0.017779706159045054 &gt;= 0.011376853216444322 + 0.0064028529426007315 = 0.017779706159045054
+Error: 0.019509108923639135
+Bias^2: 0.01312013610582818
+Var: 0.006388972817810939
+0.019509108923639135 &gt;= 0.01312013610582818 + 0.006388972817810939 = 0.01950910892363912
 Polynomial degree: 7
-Error: 0.018425115690829077
-Bias^2: 0.011789112484359082
-Var: 0.006636003206469997
-0.018425115690829077 &gt;= 0.011789112484359082 + 0.006636003206469997 = 0.01842511569082908
-Simple tree: 0.3722291337559605
+Error: 0.020056743323946562
+Bias^2: 0.012815095479733507
+Var: 0.007241647844213045
+0.020056743323946562 &gt;= 0.012815095479733507 + 0.007241647844213045 = 0.020056743323946552
+Simple tree: 0.5601973572808581
 </pre></div>
 </div>
-<div class="output traceback highlight-ipythontb notranslate"><div class="highlight"><pre><span></span><span class="gt">---------------------------------------------------------------------------</span>
-<span class="ne">NameError</span><span class="g g-Whitespace">                                 </span>Traceback (most recent call last)
-<span class="nn">Input In [1],</span> in <span class="ni">&lt;cell line: 59&gt;</span><span class="nt">()</span>
-<span class="g g-Whitespace">     </span><span class="mi">57</span> <span class="n">plt</span><span class="o">.</span><span class="n">plot</span><span class="p">(</span><span class="n">polydegree</span><span class="p">,</span> <span class="n">variance</span><span class="p">,</span> <span class="n">label</span><span class="o">=</span><span class="s1">&#39;Variance&#39;</span><span class="p">)</span>
-<span class="g g-Whitespace">     </span><span class="mi">58</span> <span class="n">plt</span><span class="o">.</span><span class="n">legend</span><span class="p">()</span>
-<span class="ne">---&gt; </span><span class="mi">59</span> <span class="n">save_fig</span><span class="p">(</span><span class="s2">&quot;baggingboot&quot;</span><span class="p">)</span>
-<span class="g g-Whitespace">     </span><span class="mi">60</span> <span class="n">plt</span><span class="o">.</span><span class="n">show</span><span class="p">()</span>
-
-<span class="ne">NameError</span>: name &#39;save_fig&#39; is not defined
-</pre></div>
-</div>
-<img alt="_images/week47_6_2.png" src="_images/week47_6_2.png" />
+<img alt="_images/week47_6_1.png" src="_images/week47_6_1.png" />
 </div>
 </div>
 </div>
@@ -1580,6 +1602,23 @@ this setting.</p>
 </pre></div>
 </div>
 </div>
+<div class="cell_output docutils container">
+<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>(426, 30)
+(143, 30)
+Test set accuracy Logistic Regression with scaled data: 0.96
+Test set accuracy SVM with scaled data: 0.96
+Test set accuracy with Decision Trees and scaled data: 0.87
+</pre></div>
+</div>
+<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>[0.93333333 0.73333333 0.93333333 1.         1.         0.92857143
+ 1.         0.92857143 0.92857143 1.        ]
+Test set accuracy with Random Forests and scaled data: 0.98
+</pre></div>
+</div>
+<img alt="_images/week47_12_2.png" src="_images/week47_12_2.png" />
+<img alt="_images/week47_12_3.png" src="_images/week47_12_3.png" />
+<img alt="_images/week47_12_4.png" src="_images/week47_12_4.png" />
+</div>
 </div>
 <p>Recall that the cumulative gains curve shows the percentage of the
 overall number of cases in a given category <em>gained</em> by targeting a
@@ -1611,6 +1650,11 @@ discrimination threshold is varied. It plots the true positive rate against the
 </pre></div>
 </div>
 </div>
+<div class="cell_output docutils container">
+<div class="output text_plain highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>0.9790209790209791
+</pre></div>
+</div>
+</div>
 </div>
 </div>
 <div class="section" id="boosting-a-bird-s-eye-view">
@@ -1866,6 +1910,11 @@ observations that are missed in the previous iterations.</p>
 </pre></div>
 </div>
 </div>
+<div class="cell_output docutils container">
+<img alt="_images/week47_73_0.png" src="_images/week47_73_0.png" />
+<img alt="_images/week47_73_1.png" src="_images/week47_73_1.png" />
+<img alt="_images/week47_73_2.png" src="_images/week47_73_2.png" />
+</div>
 </div>
 </div>
 <div class="section" id="gradient-boosting-basics-with-steepest-descent-functional-gradient-descent">
@@ -1987,6 +2036,48 @@ C(\boldsymbol{y},\boldsymbol{f})=\sum_{i=0}^{n-1}(y_i-f(x_i))^2.
 </pre></div>
 </div>
 </div>
+<div class="cell_output docutils container">
+<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>Max depth: 1
+Error: 0.5069860830630872
+Bias^2: 0.28951491799791806
+Var: 0.21747116506516934
+0.5069860830630872 &gt;= 0.28951491799791806 + 0.21747116506516934 = 0.5069860830630875
+Max depth: 2
+Error: 0.5222413718621189
+Bias^2: 0.28962869031035515
+Var: 0.23261268155176382
+0.5222413718621189 &gt;= 0.28962869031035515 + 0.23261268155176382 = 0.5222413718621189
+Max depth: 3
+Error: 0.522240032475565
+Bias^2: 0.2896287710119233
+Var: 0.2326112614636416
+0.522240032475565 &gt;= 0.2896287710119233 + 0.2326112614636416 = 0.5222400324755649
+Max depth: 4
+Error: 0.5222400329453616
+Bias^2: 0.28962877060331055
+Var: 0.2326112623420511
+0.5222400329453616 &gt;= 0.28962877060331055 + 0.2326112623420511 = 0.5222400329453616
+Max depth: 5
+Error: 0.5222400329453616
+Bias^2: 0.28962877060331055
+Var: 0.2326112623420511
+0.5222400329453616 &gt;= 0.28962877060331055 + 0.2326112623420511 = 0.5222400329453616
+</pre></div>
+</div>
+<div class="output stderr highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>/Users/mhjensen/miniforge3/envs/myenv/lib/python3.9/site-packages/sklearn/ensemble/_gb.py:494: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples, ), for example using ravel().
+  y = column_or_1d(y, warn=True)
+/Users/mhjensen/miniforge3/envs/myenv/lib/python3.9/site-packages/sklearn/ensemble/_gb.py:494: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples, ), for example using ravel().
+  y = column_or_1d(y, warn=True)
+/Users/mhjensen/miniforge3/envs/myenv/lib/python3.9/site-packages/sklearn/ensemble/_gb.py:494: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples, ), for example using ravel().
+  y = column_or_1d(y, warn=True)
+/Users/mhjensen/miniforge3/envs/myenv/lib/python3.9/site-packages/sklearn/ensemble/_gb.py:494: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples, ), for example using ravel().
+  y = column_or_1d(y, warn=True)
+/Users/mhjensen/miniforge3/envs/myenv/lib/python3.9/site-packages/sklearn/ensemble/_gb.py:494: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples, ), for example using ravel().
+  y = column_or_1d(y, warn=True)
+</pre></div>
+</div>
+<img alt="_images/week47_92_2.png" src="_images/week47_92_2.png" />
+</div>
 </div>
 </div>
 <div class="section" id="gradient-boosting-classification-example">
@@ -2036,6 +2127,20 @@ C(\boldsymbol{y},\boldsymbol{f})=\sum_{i=0}^{n-1}(y_i-f(x_i))^2.
 </pre></div>
 </div>
 </div>
+<div class="cell_output docutils container">
+<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>(426, 30)
+(143, 30)
+</pre></div>
+</div>
+<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>[0.93333333 0.93333333 0.8        0.85714286 1.         0.92857143
+ 1.         0.92857143 0.92857143 0.92857143]
+Test set accuracy with Gradient boosting and scaled data: 0.97
+</pre></div>
+</div>
+<img alt="_images/week47_94_2.png" src="_images/week47_94_2.png" />
+<img alt="_images/week47_94_3.png" src="_images/week47_94_3.png" />
+<img alt="_images/week47_94_4.png" src="_images/week47_94_4.png" />
+</div>
 </div>
 </div>
 <div class="section" id="xgboost-extreme-gradient-boosting">
@@ -2099,6 +2204,93 @@ sketch for efficient proposal calculation. It introduces a novel sparsity-aware
 </pre></div>
 </div>
 </div>
+<div class="cell_output docutils container">
+<div class="output stderr highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>/Users/mhjensen/miniforge3/envs/myenv/lib/python3.9/site-packages/xgboost/compat.py:36: FutureWarning: pandas.Int64Index is deprecated and will be removed from pandas in a future version. Use pandas.Index with the appropriate dtype instead.
+  from pandas import MultiIndex, Int64Index
+</pre></div>
+</div>
+<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>[15:38:29] WARNING: /var/folders/nz/j6p8yfhx1mv_0grj5xl4650h0000gp/T/abs_eek2t0c4ro/croots/recipe/xgboost-split_1659548960591/work/src/learner.cc:576: 
+Parameters: { &quot;colsaobjective&quot; } might not be used.
+
+  This could be a false alarm, with some parameters getting used by language bindings but
+  then being mistakenly passed down to XGBoost core, or some parameter actually being used
+  but getting flagged wrongly here. Please open an issue if you find any such cases.
+
+
+Max depth: 0
+Error: 0.35587778675776993
+Bias^2: 0.35587778675776993
+Var: 0.0
+0.35587778675776993 &gt;= 0.35587778675776993 + 0.0 = 0.35587778675776993
+[15:38:29] WARNING: /var/folders/nz/j6p8yfhx1mv_0grj5xl4650h0000gp/T/abs_eek2t0c4ro/croots/recipe/xgboost-split_1659548960591/work/src/learner.cc:576: 
+Parameters: { &quot;colsaobjective&quot; } might not be used.
+
+  This could be a false alarm, with some parameters getting used by language bindings but
+  then being mistakenly passed down to XGBoost core, or some parameter actually being used
+  but getting flagged wrongly here. Please open an issue if you find any such cases.
+
+
+Max depth: 1
+Error: 0.3001669239476101
+Bias^2: 0.2731899981798276
+Var: 0.0269769337028265
+0.3001669239476101 &gt;= 0.2731899981798276 + 0.0269769337028265 = 0.3001669318826541
+[15:38:29] WARNING: /var/folders/nz/j6p8yfhx1mv_0grj5xl4650h0000gp/T/abs_eek2t0c4ro/croots/recipe/xgboost-split_1659548960591/work/src/learner.cc:576: 
+Parameters: { &quot;colsaobjective&quot; } might not be used.
+
+  This could be a false alarm, with some parameters getting used by language bindings but
+  then being mistakenly passed down to XGBoost core, or some parameter actually being used
+  but getting flagged wrongly here. Please open an issue if you find any such cases.
+
+
+Max depth: 2
+Error: 0.30000279381576256
+Bias^2: 0.2711099029541577
+Var: 0.02889288030564785
+0.30000279381576256 &gt;= 0.2711099029541577 + 0.02889288030564785 = 0.30000278325980556
+[15:38:29] WARNING: /var/folders/nz/j6p8yfhx1mv_0grj5xl4650h0000gp/T/abs_eek2t0c4ro/croots/recipe/xgboost-split_1659548960591/work/src/learner.cc:576: 
+Parameters: { &quot;colsaobjective&quot; } might not be used.
+
+  This could be a false alarm, with some parameters getting used by language bindings but
+  then being mistakenly passed down to XGBoost core, or some parameter actually being used
+  but getting flagged wrongly here. Please open an issue if you find any such cases.
+
+
+Max depth: 3
+Error: 0.2999692169766251
+Bias^2: 0.2710765113417533
+Var: 0.028892725706100464
+0.2999692169766251 &gt;= 0.2710765113417533 + 0.028892725706100464 = 0.2999692370478538
+[15:38:29] WARNING: /var/folders/nz/j6p8yfhx1mv_0grj5xl4650h0000gp/T/abs_eek2t0c4ro/croots/recipe/xgboost-split_1659548960591/work/src/learner.cc:576: 
+Parameters: { &quot;colsaobjective&quot; } might not be used.
+
+  This could be a false alarm, with some parameters getting used by language bindings but
+  then being mistakenly passed down to XGBoost core, or some parameter actually being used
+  but getting flagged wrongly here. Please open an issue if you find any such cases.
+
+
+Max depth: 4
+Error: 0.2999728858924457
+Bias^2: 0.2710867359084896
+Var: 0.02888614870607853
+0.2999728858924457 &gt;= 0.2710867359084896 + 0.02888614870607853 = 0.2999728846145681
+[15:38:29] WARNING: /var/folders/nz/j6p8yfhx1mv_0grj5xl4650h0000gp/T/abs_eek2t0c4ro/croots/recipe/xgboost-split_1659548960591/work/src/learner.cc:576: 
+Parameters: { &quot;colsaobjective&quot; } might not be used.
+
+  This could be a false alarm, with some parameters getting used by language bindings but
+  then being mistakenly passed down to XGBoost core, or some parameter actually being used
+  but getting flagged wrongly here. Please open an issue if you find any such cases.
+
+
+Max depth: 5
+Error: 0.29998782047173667
+Bias^2: 0.2711004934923823
+Var: 0.02888733707368374
+0.29998782047173667 &gt;= 0.2711004934923823 + 0.02888733707368374 = 0.29998783056606604
+</pre></div>
+</div>
+<img alt="_images/week47_97_2.png" src="_images/week47_97_2.png" />
+</div>
 </div>
 </div>
 <div class="section" id="xgboost-on-the-cancer-data">
@@ -2160,6 +2352,23 @@ sketch for efficient proposal calculation. It introduces a novel sparsity-aware
 </pre></div>
 </div>
 </div>
+<div class="cell_output docutils container">
+<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>(426, 30)
+(143, 30)
+[15:38:29] WARNING: /var/folders/nz/j6p8yfhx1mv_0grj5xl4650h0000gp/T/abs_eek2t0c4ro/croots/recipe/xgboost-split_1659548960591/work/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective &#39;binary:logistic&#39; was changed from &#39;error&#39; to &#39;logloss&#39;. Explicitly set eval_metric if you&#39;d like to restore the old behavior.
+Test set accuracy with Gradient Boosting and scaled data: 1.00
+</pre></div>
+</div>
+<div class="output stderr highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>/Users/mhjensen/miniforge3/envs/myenv/lib/python3.9/site-packages/xgboost/sklearn.py:1224: UserWarning: The use of label encoder in XGBClassifier is deprecated and will be removed in a future release. To remove this warning, do the following: 1) Pass option use_label_encoder=False when constructing XGBClassifier object; and 2) Encode your labels (y) as integers starting with 0, i.e. 0, 1, 2, ..., [num_class - 1].
+  warnings.warn(label_encoder_deprecation_msg, UserWarning)
+</pre></div>
+</div>
+<img alt="_images/week47_99_2.png" src="_images/week47_99_2.png" />
+<img alt="_images/week47_99_3.png" src="_images/week47_99_3.png" />
+<img alt="_images/week47_99_4.png" src="_images/week47_99_4.png" />
+<img alt="_images/week47_99_5.png" src="_images/week47_99_5.png" />
+<img alt="_images/week47_99_6.png" src="_images/week47_99_6.png" />
+</div>
 </div>
 </div>
 <div class="section" id="summary-of-course">
@@ -2406,7 +2615,7 @@ set of hyperparameters and regularization methods.</p>
 <div class="section" id="other-courses-on-data-science-and-machine-learning-at-uio">
 <h2>Other courses on Data science and Machine Learning  at UiO<a class="headerlink" href="#other-courses-on-data-science-and-machine-learning-at-uio" title="Permalink to this headline">¶</a></h2>
 <ol class="simple">
-<li><p><a class="reference external" href="https://www.uio.no/studier/emner/matnat/fys/FYS5429/index-eng.html">FYS5429 Advanced Machine Learning and Data Analysis for the Physical Sciences</a></p></li>
+<li><p><a class="reference external" href="https://www.uio.no/studier/emner/matnat/fys/FYS5429/index-eng.html">FYS5429 Advanced Machine Learning and Data Analysis for the Physical Sciences</a>. Discussed deep learning and generative deep learning.</p></li>
 <li><p><a class="reference external" href="https://www.uio.no/studier/emner/matnat/fys/FYS5419/index-eng.html">FYS5419 Quantum Computing and Quantum Machine Learning</a></p></li>
 <li><p><a class="reference external" href="http://www.uio.no/studier/emner/matnat/math/STK2100/index-eng.html">STK2100 Machine learning and statistical methods for prediction and classification</a>.</p></li>
 <li><p><a class="reference external" href="https://www.uio.no/studier/emner/matnat/ifi/IN3050/index-eng.html">IN3050/IN4050 Introduction to Artificial Intelligence and Machine Learning</a>. Introductory course in machine learning and AI with an algorithmic approach.</p></li>
@@ -2439,6 +2648,7 @@ networks have been proposed, such as</p>
 <li><p>Convolutional neural networks, which are mostly used in image and video data processing, and have also been applied to sequential data such as text processing;</p></li>
 <li><p>Recurrent neural networks, which can process sequential data of variable length and have been widely used in natural language understanding and speech processing;</p></li>
 <li><p>Encoder-decoder framework, which is mostly used for image or sequence generation, such as machine translation, text summarization, and image captioning.</p></li>
+<li><p><strong>Generative deep learning</strong>! Recent textbook by David Foster (and obviously many other ones) at <a class="reference external" href="https://www.oreilly.com/library/view/generative-deep-learning/9781492041931/">https://www.oreilly.com/library/view/generative-deep-learning/9781492041931/</a>”</p></li>
 </ol>
 </div>
 <div class="section" id="types-of-machine-learning-a-repetition">
@@ -2468,7 +2678,7 @@ One of the major reasons is that they can be stacked layer-wise to build deep ne
 </div>
 <div class="section" id="boltzmann-machines">
 <h2>Boltzmann Machines<a class="headerlink" href="#boltzmann-machines" title="Permalink to this headline">¶</a></h2>
-<p>Why use a generative model rather than the more well known discriminative deep neural networks (DNN)?</p>
+<p>Why use a generative model rather than the more well known discriminative deep neural networks (DNN)?  <strong>Simplest approach to generative deep learning</strong>.</p>
 <ul class="simple">
 <li><p>Discriminitave methods have several limitations: They are mainly supervised learning methods, thus requiring labeled data. And there are tasks they cannot accomplish, like drawing new examples from an unknown probability distribution.</p></li>
 <li><p>A generative model can learn to represent and sample from a probability distribution. The core idea is to learn a parametric model of the probability distribution from which the training data was drawn. As an example</p></li>
@@ -2915,10 +3125,10 @@ topics. Together, we will not just predict the future, but create it.</p>
             <p class="prev-next-title">Week 46: Decision Trees, Ensemble methods  and Random Forests</p>
         </div>
     </a>
-    <a class='right-next' id="next-link" href="project1.html" title="next page">
+    <a class='right-next' id="next-link" href="exercisesweek47.html" title="next page">
     <div class="prev-next-info">
         <p class="prev-next-subtitle">next</p>
-        <p class="prev-next-title">Project 1 on Machine Learning, deadline October 9 (midnight), 2023</p>
+        <p class="prev-next-title">Exercise week 47</p>
     </div>
     <i class="fas fa-angle-right"></i>
     </a>

doc/LectureNotes/_build/jupyter_execute/week47.py

@@ -28,6 +28,10 @@
 # 
 #     * These lecture notes
 # 
+#     * [Video of Lecture](https://youtu.be/SpWXsvn5I9E)
+# 
+#     * [Whiteboard notes](https://github.com/CompPhysics/MachineLearning/blob/master/doc/HandWrittenNotes/2023/NotesNov23.pdf)
+# 
 #     * [Video on Decision trees](https://www.youtube.com/watch?v=RmajweUFKvM&ab_channel=Simplilearn)
 # 
 #     * [Video on boosting methods by Hastie](https://www.youtube.com/watch?v=wPqtzj5VZus&ab_channel=H2O.ai)
@@ -86,13 +90,44 @@
 
 get_ipython().run_line_magic('matplotlib', 'inline')
 
-
+# Common imports
 import matplotlib.pyplot as plt
 import numpy as np
 from sklearn.model_selection import train_test_split
 from sklearn.pipeline import make_pipeline
 from sklearn.utils import resample
 from sklearn.tree import DecisionTreeRegressor
+import pandas as pd
+from sklearn.tree import DecisionTreeClassifier
+from sklearn.model_selection import train_test_split
+from sklearn.preprocessing import StandardScaler, OneHotEncoder
+from sklearn.compose import ColumnTransformer
+from IPython.display import Image 
+import os
+
+# Where to save the figures and data files
+PROJECT_ROOT_DIR = "Results"
+FIGURE_ID = "Results/FigureFiles"
+DATA_ID = "DataFiles/"
+
+if not os.path.exists(PROJECT_ROOT_DIR):
+    os.mkdir(PROJECT_ROOT_DIR)
+
+if not os.path.exists(FIGURE_ID):
+    os.makedirs(FIGURE_ID)
+
+if not os.path.exists(DATA_ID):
+    os.makedirs(DATA_ID)
+
+def image_path(fig_id):
+    return os.path.join(FIGURE_ID, fig_id)
+
+def data_path(dat_id):
+    return os.path.join(DATA_ID, dat_id)
+
+def save_fig(fig_id):
+    plt.savefig(image_path(fig_id) + ".png", format='png')
+
 
 n = 100
 n_boostraps = 100
@@ -1154,7 +1189,7 @@ plt.show()
 
 # ## Other courses on Data science and Machine Learning  at UiO
 # 
-# 1. [FYS5429 Advanced Machine Learning and Data Analysis for the Physical Sciences](https://www.uio.no/studier/emner/matnat/fys/FYS5429/index-eng.html)
+# 1. [FYS5429 Advanced Machine Learning and Data Analysis for the Physical Sciences](https://www.uio.no/studier/emner/matnat/fys/FYS5429/index-eng.html). Discussed deep learning and generative deep learning.
 # 
 # 2. [FYS5419 Quantum Computing and Quantum Machine Learning](https://www.uio.no/studier/emner/matnat/fys/FYS5419/index-eng.html)
 # 
@@ -1195,6 +1230,8 @@ plt.show()
 # 2. Recurrent neural networks, which can process sequential data of variable length and have been widely used in natural language understanding and speech processing;
 # 
 # 3. Encoder-decoder framework, which is mostly used for image or sequence generation, such as machine translation, text summarization, and image captioning.
+# 
+# 4. **Generative deep learning**! Recent textbook by David Foster (and obviously many other ones) at <https://www.oreilly.com/library/view/generative-deep-learning/9781492041931/>"
 
 # ## Types of Machine Learning, a repetition
 # 
@@ -1228,7 +1265,7 @@ plt.show()
 
 # ## Boltzmann Machines
 # 
-# Why use a generative model rather than the more well known discriminative deep neural networks (DNN)? 
+# Why use a generative model rather than the more well known discriminative deep neural networks (DNN)?  **Simplest approach to generative deep learning**.
 # 
 # * Discriminitave methods have several limitations: They are mainly supervised learning methods, thus requiring labeled data. And there are tasks they cannot accomplish, like drawing new examples from an unknown probability distribution.
 #