typos
This commit is contained in:
Morten Hjorth-Jensen
@@ -38,6 +38,18 @@ doconce format html week48.do.txt --html_style=bootstrap --pygments_html_style=d
|
||||
{'highest level': 2,
|
||||
'sections': [('Overview of week 48', 2, None, 'overview-of-week-48'),
|
||||
('Plan for week 47', 2, None, 'plan-for-week-47'),
|
||||
('Random Forest Algorithm, reminder from last week',
|
||||
2,
|
||||
None,
|
||||
'random-forest-algorithm-reminder-from-last-week'),
|
||||
('Random Forests Compared with other Methods on the Cancer Data',
|
||||
2,
|
||||
None,
|
||||
'random-forests-compared-with-other-methods-on-the-cancer-data'),
|
||||
('Compare Bagging on Trees with Random Forests',
|
||||
2,
|
||||
None,
|
||||
'compare-bagging-on-trees-with-random-forests'),
|
||||
("Boosting, a Bird's Eye View",
|
||||
2,
|
||||
None,
|
||||
@@ -267,74 +279,77 @@ MathJax.Hub.Config({
|
||||
<ul class="dropdown-menu">
|
||||
<!-- navigation toc: --> <li><a href="._week48-bs001.html#overview-of-week-48" style="font-size: 80%;">Overview of week 48</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week48-bs002.html#plan-for-week-47" style="font-size: 80%;">Plan for week 47</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week48-bs003.html#boosting-a-bird-s-eye-view" style="font-size: 80%;">Boosting, a Bird's Eye View</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week48-bs004.html#what-is-boosting-additive-modelling-iterative-fitting" style="font-size: 80%;">What is boosting? Additive Modelling/Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week48-bs005.html#iterative-fitting-regression-and-squared-error-cost-function" style="font-size: 80%;">Iterative Fitting, Regression and Squared-error Cost Function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week48-bs006.html#squared-error-example-and-iterative-fitting" style="font-size: 80%;">Squared-Error Example and Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week48-bs007.html#iterative-fitting-classification-and-adaboost" style="font-size: 80%;">Iterative Fitting, Classification and AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week48-bs008.html#adaptive-boosting-adaboost" style="font-size: 80%;">Adaptive Boosting, AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week48-bs009.html#building-up-adaboost" style="font-size: 80%;">Building up AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week48-bs010.html#adaptive-boosting-adaboost-basic-algorithm" style="font-size: 80%;">Adaptive boosting: AdaBoost, Basic Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week48-bs011.html#basic-steps-of-adaboost" style="font-size: 80%;">Basic Steps of AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week48-bs012.html#adaboost-examples" style="font-size: 80%;">AdaBoost Examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week48-bs013.html#making-an-adaboost-code-yourself" style="font-size: 80%;">Making an ADAboost code yourself</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week48-bs014.html#gradient-boosting-basics-with-steepest-descent-functional-gradient-descent" style="font-size: 80%;">Gradient boosting: Basics with Steepest Descent/Functional Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week48-bs015.html#the-squared-error-again-steepest-descent" style="font-size: 80%;">The Squared-Error again! Steepest Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week48-bs016.html#steepest-descent-example" style="font-size: 80%;">Steepest Descent Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week48-bs017.html#gradient-boosting-algorithm" style="font-size: 80%;">Gradient Boosting, algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week48-bs018.html#gradient-boosting-examples-of-regression" style="font-size: 80%;">Gradient Boosting, Examples of Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week48-bs019.html#gradient-boosting-classification-example" style="font-size: 80%;">Gradient Boosting, Classification Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week48-bs020.html#xgboost-extreme-gradient-boosting" style="font-size: 80%;">XGBoost: Extreme Gradient Boosting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week48-bs021.html#regression-case" style="font-size: 80%;">Regression Case</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week48-bs022.html#xgboost-on-the-cancer-data" style="font-size: 80%;">Xgboost on the Cancer Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week48-bs023.html#gradient-boosting-making-our-own-code-for-a-regression-case" style="font-size: 80%;">Gradient boosting, making our own code for a regression case</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week48-bs024.html#summary-of-course" style="font-size: 80%;">Summary of course</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week48-bs025.html#what-me-worry-no-final-exam-in-this-course" style="font-size: 80%;">What? Me worry? No final exam in this course!</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week48-bs026.html#topics-we-have-covered-this-year" style="font-size: 80%;">Topics we have covered this year</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week48-bs027.html#statistical-analysis-and-optimization-of-data" style="font-size: 80%;">Statistical analysis and optimization of data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week48-bs028.html#machine-learning" style="font-size: 80%;">Machine learning</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week48-bs029.html#learning-outcomes-and-overarching-aims-of-this-course" style="font-size: 80%;">Learning outcomes and overarching aims of this course</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week48-bs030.html#perspective-on-machine-learning" style="font-size: 80%;">Perspective on Machine Learning</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week48-bs031.html#machine-learning-research" style="font-size: 80%;">Machine Learning Research</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week48-bs032.html#starting-your-machine-learning-project" style="font-size: 80%;">Starting your Machine Learning Project</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week48-bs033.html#choose-a-model-and-algorithm" style="font-size: 80%;">Choose a Model and Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week48-bs034.html#preparing-your-data" style="font-size: 80%;">Preparing Your Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week48-bs035.html#which-activation-and-weights-to-choose-in-neural-networks" style="font-size: 80%;">Which activation and weights to choose in neural networks</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week48-bs036.html#optimization-methods-and-hyperparameters" style="font-size: 80%;">Optimization Methods and Hyperparameters</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week48-bs037.html#resampling" style="font-size: 80%;">Resampling</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week48-bs038.html#other-courses-on-data-science-and-machine-learning-at-uio" style="font-size: 80%;">Other courses on Data science and Machine Learning at UiO</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week48-bs039.html#additional-courses-of-interest" style="font-size: 80%;">Additional courses of interest</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week48-bs040.html#what-s-the-future-like" style="font-size: 80%;">What's the future like?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week48-bs041.html#types-of-machine-learning-a-repetition" style="font-size: 80%;">Types of Machine Learning, a repetition</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week48-bs042.html#why-boltzmann-machines" style="font-size: 80%;">Why Boltzmann machines?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week48-bs043.html#boltzmann-machines" style="font-size: 80%;">Boltzmann Machines</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week48-bs044.html#some-similarities-and-differences-from-dnns" style="font-size: 80%;">Some similarities and differences from DNNs</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week48-bs045.html#boltzmann-machines-bm" style="font-size: 80%;">Boltzmann machines (BM)</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week48-bs046.html#a-standard-bm-setup" style="font-size: 80%;">A standard BM setup</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week48-bs047.html#the-structure-of-the-rbm-network" style="font-size: 80%;">The structure of the RBM network</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week48-bs048.html#the-network" style="font-size: 80%;">The network</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week48-bs049.html#goals" style="font-size: 80%;">Goals</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week48-bs050.html#joint-distribution" style="font-size: 80%;">Joint distribution</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week48-bs051.html#network-elements-the-energy-function" style="font-size: 80%;">Network Elements, the energy function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week48-bs052.html#defining-different-types-of-rbms" style="font-size: 80%;">Defining different types of RBMs</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week48-bs053.html#more-about-rbms" style="font-size: 80%;">More about RBMs</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week48-bs054.html#autoencoders-overarching-view" style="font-size: 80%;">Autoencoders: Overarching view</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week48-bs055.html#bayesian-machine-learning" style="font-size: 80%;">Bayesian Machine Learning</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week48-bs056.html#reinforcement-learning" style="font-size: 80%;">Reinforcement Learning</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week48-bs057.html#transfer-learning" style="font-size: 80%;">Transfer learning</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week48-bs058.html#adversarial-learning" style="font-size: 80%;">Adversarial learning</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week48-bs059.html#dual-learning" style="font-size: 80%;">Dual learning</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week48-bs060.html#distributed-machine-learning" style="font-size: 80%;">Distributed machine learning</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week48-bs061.html#meta-learning" style="font-size: 80%;">Meta learning</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week48-bs062.html#the-challenges-facing-machine-learning" style="font-size: 80%;">The Challenges Facing Machine Learning</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week48-bs063.html#explainable-machine-learning" style="font-size: 80%;">Explainable machine learning</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week48-bs064.html#quantum-machine-learning" style="font-size: 80%;">Quantum machine learning</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week48-bs065.html#quantum-machine-learning-algorithms-based-on-linear-algebra" style="font-size: 80%;">Quantum machine learning algorithms based on linear algebra</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week48-bs066.html#quantum-reinforcement-learning" style="font-size: 80%;">Quantum reinforcement learning</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week48-bs067.html#quantum-deep-learning" style="font-size: 80%;">Quantum deep learning</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week48-bs068.html#social-machine-learning" style="font-size: 80%;">Social machine learning</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week48-bs069.html#the-last-words" style="font-size: 80%;">The last words?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week48-bs070.html#best-wishes-to-you-all-and-thanks-so-much-for-your-heroic-efforts-this-semester" style="font-size: 80%;">Best wishes to you all and thanks so much for your heroic efforts this semester</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week48-bs003.html#random-forest-algorithm-reminder-from-last-week" style="font-size: 80%;">Random Forest Algorithm, reminder from last week</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week48-bs004.html#random-forests-compared-with-other-methods-on-the-cancer-data" style="font-size: 80%;">Random Forests Compared with other Methods on the Cancer Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week48-bs005.html#compare-bagging-on-trees-with-random-forests" style="font-size: 80%;">Compare Bagging on Trees with Random Forests</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week48-bs006.html#boosting-a-bird-s-eye-view" style="font-size: 80%;">Boosting, a Bird's Eye View</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week48-bs007.html#what-is-boosting-additive-modelling-iterative-fitting" style="font-size: 80%;">What is boosting? Additive Modelling/Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week48-bs008.html#iterative-fitting-regression-and-squared-error-cost-function" style="font-size: 80%;">Iterative Fitting, Regression and Squared-error Cost Function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week48-bs009.html#squared-error-example-and-iterative-fitting" style="font-size: 80%;">Squared-Error Example and Iterative Fitting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week48-bs010.html#iterative-fitting-classification-and-adaboost" style="font-size: 80%;">Iterative Fitting, Classification and AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week48-bs011.html#adaptive-boosting-adaboost" style="font-size: 80%;">Adaptive Boosting, AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week48-bs012.html#building-up-adaboost" style="font-size: 80%;">Building up AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week48-bs013.html#adaptive-boosting-adaboost-basic-algorithm" style="font-size: 80%;">Adaptive boosting: AdaBoost, Basic Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week48-bs014.html#basic-steps-of-adaboost" style="font-size: 80%;">Basic Steps of AdaBoost</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week48-bs015.html#adaboost-examples" style="font-size: 80%;">AdaBoost Examples</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week48-bs016.html#making-an-adaboost-code-yourself" style="font-size: 80%;">Making an ADAboost code yourself</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week48-bs017.html#gradient-boosting-basics-with-steepest-descent-functional-gradient-descent" style="font-size: 80%;">Gradient boosting: Basics with Steepest Descent/Functional Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week48-bs018.html#the-squared-error-again-steepest-descent" style="font-size: 80%;">The Squared-Error again! Steepest Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week48-bs019.html#steepest-descent-example" style="font-size: 80%;">Steepest Descent Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week48-bs020.html#gradient-boosting-algorithm" style="font-size: 80%;">Gradient Boosting, algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week48-bs021.html#gradient-boosting-examples-of-regression" style="font-size: 80%;">Gradient Boosting, Examples of Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week48-bs022.html#gradient-boosting-classification-example" style="font-size: 80%;">Gradient Boosting, Classification Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week48-bs023.html#xgboost-extreme-gradient-boosting" style="font-size: 80%;">XGBoost: Extreme Gradient Boosting</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week48-bs024.html#regression-case" style="font-size: 80%;">Regression Case</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week48-bs025.html#xgboost-on-the-cancer-data" style="font-size: 80%;">Xgboost on the Cancer Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week48-bs026.html#gradient-boosting-making-our-own-code-for-a-regression-case" style="font-size: 80%;">Gradient boosting, making our own code for a regression case</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week48-bs027.html#summary-of-course" style="font-size: 80%;">Summary of course</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week48-bs028.html#what-me-worry-no-final-exam-in-this-course" style="font-size: 80%;">What? Me worry? No final exam in this course!</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week48-bs029.html#topics-we-have-covered-this-year" style="font-size: 80%;">Topics we have covered this year</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week48-bs030.html#statistical-analysis-and-optimization-of-data" style="font-size: 80%;">Statistical analysis and optimization of data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week48-bs031.html#machine-learning" style="font-size: 80%;">Machine learning</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week48-bs032.html#learning-outcomes-and-overarching-aims-of-this-course" style="font-size: 80%;">Learning outcomes and overarching aims of this course</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week48-bs033.html#perspective-on-machine-learning" style="font-size: 80%;">Perspective on Machine Learning</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week48-bs034.html#machine-learning-research" style="font-size: 80%;">Machine Learning Research</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week48-bs035.html#starting-your-machine-learning-project" style="font-size: 80%;">Starting your Machine Learning Project</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week48-bs036.html#choose-a-model-and-algorithm" style="font-size: 80%;">Choose a Model and Algorithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week48-bs037.html#preparing-your-data" style="font-size: 80%;">Preparing Your Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week48-bs038.html#which-activation-and-weights-to-choose-in-neural-networks" style="font-size: 80%;">Which activation and weights to choose in neural networks</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week48-bs039.html#optimization-methods-and-hyperparameters" style="font-size: 80%;">Optimization Methods and Hyperparameters</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week48-bs040.html#resampling" style="font-size: 80%;">Resampling</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week48-bs041.html#other-courses-on-data-science-and-machine-learning-at-uio" style="font-size: 80%;">Other courses on Data science and Machine Learning at UiO</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week48-bs042.html#additional-courses-of-interest" style="font-size: 80%;">Additional courses of interest</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week48-bs043.html#what-s-the-future-like" style="font-size: 80%;">What's the future like?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week48-bs044.html#types-of-machine-learning-a-repetition" style="font-size: 80%;">Types of Machine Learning, a repetition</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week48-bs045.html#why-boltzmann-machines" style="font-size: 80%;">Why Boltzmann machines?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week48-bs046.html#boltzmann-machines" style="font-size: 80%;">Boltzmann Machines</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week48-bs047.html#some-similarities-and-differences-from-dnns" style="font-size: 80%;">Some similarities and differences from DNNs</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week48-bs048.html#boltzmann-machines-bm" style="font-size: 80%;">Boltzmann machines (BM)</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week48-bs049.html#a-standard-bm-setup" style="font-size: 80%;">A standard BM setup</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week48-bs050.html#the-structure-of-the-rbm-network" style="font-size: 80%;">The structure of the RBM network</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week48-bs051.html#the-network" style="font-size: 80%;">The network</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week48-bs052.html#goals" style="font-size: 80%;">Goals</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week48-bs053.html#joint-distribution" style="font-size: 80%;">Joint distribution</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week48-bs054.html#network-elements-the-energy-function" style="font-size: 80%;">Network Elements, the energy function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week48-bs055.html#defining-different-types-of-rbms" style="font-size: 80%;">Defining different types of RBMs</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week48-bs056.html#more-about-rbms" style="font-size: 80%;">More about RBMs</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week48-bs057.html#autoencoders-overarching-view" style="font-size: 80%;">Autoencoders: Overarching view</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week48-bs058.html#bayesian-machine-learning" style="font-size: 80%;">Bayesian Machine Learning</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week48-bs059.html#reinforcement-learning" style="font-size: 80%;">Reinforcement Learning</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week48-bs060.html#transfer-learning" style="font-size: 80%;">Transfer learning</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week48-bs061.html#adversarial-learning" style="font-size: 80%;">Adversarial learning</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week48-bs062.html#dual-learning" style="font-size: 80%;">Dual learning</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week48-bs063.html#distributed-machine-learning" style="font-size: 80%;">Distributed machine learning</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week48-bs064.html#meta-learning" style="font-size: 80%;">Meta learning</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week48-bs065.html#the-challenges-facing-machine-learning" style="font-size: 80%;">The Challenges Facing Machine Learning</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week48-bs066.html#explainable-machine-learning" style="font-size: 80%;">Explainable machine learning</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week48-bs067.html#quantum-machine-learning" style="font-size: 80%;">Quantum machine learning</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week48-bs068.html#quantum-machine-learning-algorithms-based-on-linear-algebra" style="font-size: 80%;">Quantum machine learning algorithms based on linear algebra</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week48-bs069.html#quantum-reinforcement-learning" style="font-size: 80%;">Quantum reinforcement learning</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week48-bs070.html#quantum-deep-learning" style="font-size: 80%;">Quantum deep learning</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week48-bs071.html#social-machine-learning" style="font-size: 80%;">Social machine learning</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week48-bs072.html#the-last-words" style="font-size: 80%;">The last words?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week48-bs073.html#best-wishes-to-you-all-and-thanks-so-much-for-your-heroic-efforts-this-semester" style="font-size: 80%;">Best wishes to you all and thanks so much for your heroic efforts this semester</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -386,7 +401,7 @@ MathJax.Hub.Config({
|
||||
<li><a href="._week48-bs008.html">9</a></li>
|
||||
<li><a href="._week48-bs009.html">10</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._week48-bs070.html">71</a></li>
|
||||
<li><a href="._week48-bs073.html">74</a></li>
|
||||
<li><a href="._week48-bs001.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -209,7 +209,7 @@ MathJax.Hub.Config({
|
||||
|
||||
<p><li> Lab sessions at usual times.</li>
|
||||
|
||||
<p><li> For the week of December 2-6, lab sessions atart at 10am and end 4pm, room FØ434, Tuesday and Wednesday</li>
|
||||
<p><li> For the week of December 2-6, lab sessions start at 10am and end at 4pm, room FØ434, Tuesday and Wednesday</li>
|
||||
</ul>
|
||||
</div>
|
||||
|
||||
@@ -222,8 +222,8 @@ MathJax.Hub.Config({
|
||||
<p><li> Summary of course</li>
|
||||
<p><li> Readings and Videos:
|
||||
<ol type="a"></li>
|
||||
<p><li> These lecture notes at <a href="https://github.com/CompPhysics/MachineLearning/blob/master/doc/pub/week47/ipynb/week48.ipynb" target="_blank"><tt>https://github.com/CompPhysics/MachineLearning/blob/master/doc/pub/week47/ipynb/week48.ipynb</tt></a></li>
|
||||
<p><li> See also lecture notes from week 47 at <a href="https://github.com/CompPhysics/MachineLearning/blob/master/doc/pub/week46/ipynb/week47.ipynb" target="_blank"><tt>https://github.com/CompPhysics/MachineLearning/blob/master/doc/pub/week46/ipynb/week47.ipynb</tt></a>. The lecture on Monday starts with a repetition on AdaBoost before we move over to gradient boosting with examples
|
||||
<p><li> These lecture notes at <a href="https://github.com/CompPhysics/MachineLearning/blob/master/doc/pub/week48/ipynb/week48.ipynb" target="_blank"><tt>https://github.com/CompPhysics/MachineLearning/blob/master/doc/pub/week48/ipynb/week48.ipynb</tt></a></li>
|
||||
<p><li> See also lecture notes from week 47 at <a href="https://github.com/CompPhysics/MachineLearning/blob/master/doc/pub/week47/ipynb/week47.ipynb" target="_blank"><tt>https://github.com/CompPhysics/MachineLearning/blob/master/doc/pub/week47/ipynb/week47.ipynb</tt></a>. The lecture on Monday starts with a repetition on AdaBoost before we move over to gradient boosting with examples
|
||||
<!-- o Video of lecture at <a href="https://youtu.be/RIHzmLv05DA" target="_blank"><tt>https://youtu.be/RIHzmLv05DA</tt></a> -->
|
||||
<!-- o Whiteboard notes at <a href="https://github.com/CompPhysics/MachineLearning/blob/master/doc/HandWrittenNotes/2024/NotesNovember25.pdf" target="_blank"><tt>https://github.com/CompPhysics/MachineLearning/blob/master/doc/HandWrittenNotes/2024/NotesNovember25.pdf</tt></a> --></li>
|
||||
<p><li> Video on Decision trees <a href="https://www.youtube.com/watch?v=RmajweUFKvM&ab_channel=Simplilearn" target="_blank"><tt>https://www.youtube.com/watch?v=RmajweUFKvM&ab_channel=Simplilearn</tt></a></li>
|
||||
@@ -237,6 +237,183 @@ MathJax.Hub.Config({
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section>
|
||||
<h2 id="random-forest-algorithm-reminder-from-last-week">Random Forest Algorithm, reminder from last week </h2>
|
||||
|
||||
<p>The algorithm described here can be applied to both classification and regression problems.</p>
|
||||
|
||||
<p>We will grow of forest of say \( B \) trees.</p>
|
||||
<ol>
|
||||
<p><li> For \( b=1:B \)
|
||||
<ol type="a"></li>
|
||||
<p><li> Draw a bootstrap sample from the training data organized in our \( \boldsymbol{X} \) matrix.</li>
|
||||
<p><li> We grow then a random forest tree \( T_b \) based on the bootstrapped data by repeating the steps outlined till we reach the maximum node size is reached</li>
|
||||
<ol>
|
||||
|
||||
<p><li> we select \( m \le p \) variables at random from the \( p \) predictors/features</li>
|
||||
|
||||
<p><li> pick the best split point among the \( m \) features using for example the CART algorithm and create a new node</li>
|
||||
|
||||
<p><li> split the node into daughter nodes</li>
|
||||
</ol>
|
||||
<p>
|
||||
</ol>
|
||||
<p>
|
||||
<p><li> Output then the ensemble of trees \( \{T_b\}_1^{B} \) and make predictions for either a regression type of problem or a classification type of problem.</li>
|
||||
</ol>
|
||||
</section>
|
||||
|
||||
<section>
|
||||
<h2 id="random-forests-compared-with-other-methods-on-the-cancer-data">Random Forests Compared with other Methods on the Cancer Data </h2>
|
||||
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="cell border-box-sizing code_cell rendered">
|
||||
<div class="input">
|
||||
<div class="inner_cell">
|
||||
<div class="input_area">
|
||||
<div class="highlight" style="background: #eeeedd">
|
||||
<pre style="font-size: 80%; line-height: 125%;"><span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">matplotlib.pyplot</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">plt</span>
|
||||
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">numpy</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">np</span>
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.model_selection</span> <span style="color: #8B008B; font-weight: bold">import</span> train_test_split
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.datasets</span> <span style="color: #8B008B; font-weight: bold">import</span> load_breast_cancer
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.svm</span> <span style="color: #8B008B; font-weight: bold">import</span> SVC
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.linear_model</span> <span style="color: #8B008B; font-weight: bold">import</span> LogisticRegression
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.tree</span> <span style="color: #8B008B; font-weight: bold">import</span> DecisionTreeClassifier
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.ensemble</span> <span style="color: #8B008B; font-weight: bold">import</span> BaggingClassifier
|
||||
|
||||
<span style="color: #228B22"># Load the data</span>
|
||||
cancer = load_breast_cancer()
|
||||
|
||||
X_train, X_test, y_train, y_test = train_test_split(cancer.data,cancer.target,random_state=<span style="color: #B452CD">0</span>)
|
||||
<span style="color: #658b00">print</span>(X_train.shape)
|
||||
<span style="color: #658b00">print</span>(X_test.shape)
|
||||
<span style="color: #228B22">#define methods</span>
|
||||
<span style="color: #228B22"># Logistic Regression</span>
|
||||
logreg = LogisticRegression(solver=<span style="color: #CD5555">'lbfgs'</span>)
|
||||
<span style="color: #228B22"># Support vector machine</span>
|
||||
svm = SVC(gamma=<span style="color: #CD5555">'auto'</span>, C=<span style="color: #B452CD">100</span>)
|
||||
<span style="color: #228B22"># Decision Trees</span>
|
||||
deep_tree_clf = DecisionTreeClassifier(max_depth=<span style="color: #8B008B; font-weight: bold">None</span>)
|
||||
<span style="color: #228B22">#Scale the data</span>
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.preprocessing</span> <span style="color: #8B008B; font-weight: bold">import</span> StandardScaler
|
||||
scaler = StandardScaler()
|
||||
scaler.fit(X_train)
|
||||
X_train_scaled = scaler.transform(X_train)
|
||||
X_test_scaled = scaler.transform(X_test)
|
||||
<span style="color: #228B22"># Logistic Regression</span>
|
||||
logreg.fit(X_train_scaled, y_train)
|
||||
<span style="color: #658b00">print</span>(<span style="color: #CD5555">"Test set accuracy Logistic Regression with scaled data: {:.2f}"</span>.format(logreg.score(X_test_scaled,y_test)))
|
||||
<span style="color: #228B22"># Support Vector Machine</span>
|
||||
svm.fit(X_train_scaled, y_train)
|
||||
<span style="color: #658b00">print</span>(<span style="color: #CD5555">"Test set accuracy SVM with scaled data: {:.2f}"</span>.format(logreg.score(X_test_scaled,y_test)))
|
||||
<span style="color: #228B22"># Decision Trees</span>
|
||||
deep_tree_clf.fit(X_train_scaled, y_train)
|
||||
<span style="color: #658b00">print</span>(<span style="color: #CD5555">"Test set accuracy with Decision Trees and scaled data: {:.2f}"</span>.format(deep_tree_clf.score(X_test_scaled,y_test)))
|
||||
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.ensemble</span> <span style="color: #8B008B; font-weight: bold">import</span> RandomForestClassifier
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.preprocessing</span> <span style="color: #8B008B; font-weight: bold">import</span> LabelEncoder
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.model_selection</span> <span style="color: #8B008B; font-weight: bold">import</span> cross_validate
|
||||
<span style="color: #228B22"># Data set not specificied</span>
|
||||
<span style="color: #228B22">#Instantiate the model with 500 trees and entropy as splitting criteria</span>
|
||||
Random_Forest_model = RandomForestClassifier(n_estimators=<span style="color: #B452CD">500</span>,criterion=<span style="color: #CD5555">"entropy"</span>)
|
||||
Random_Forest_model.fit(X_train_scaled, y_train)
|
||||
<span style="color: #228B22">#Cross validation</span>
|
||||
accuracy = cross_validate(Random_Forest_model,X_test_scaled,y_test,cv=<span style="color: #B452CD">10</span>)[<span style="color: #CD5555">'test_score'</span>]
|
||||
<span style="color: #658b00">print</span>(accuracy)
|
||||
<span style="color: #658b00">print</span>(<span style="color: #CD5555">"Test set accuracy with Random Forests and scaled data: {:.2f}"</span>.format(Random_Forest_model.score(X_test_scaled,y_test)))
|
||||
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">scikitplot</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">skplt</span>
|
||||
y_pred = Random_Forest_model.predict(X_test_scaled)
|
||||
skplt.metrics.plot_confusion_matrix(y_test, y_pred, normalize=<span style="color: #8B008B; font-weight: bold">True</span>)
|
||||
plt.show()
|
||||
y_probas = Random_Forest_model.predict_proba(X_test_scaled)
|
||||
skplt.metrics.plot_roc(y_test, y_probas)
|
||||
plt.show()
|
||||
skplt.metrics.plot_cumulative_gain(y_test, y_probas)
|
||||
plt.show()
|
||||
</pre>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="output_wrapper">
|
||||
<div class="output">
|
||||
<div class="output_area">
|
||||
<div class="output_subarea output_stream output_stdout output_text">
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<p>Recall that the cumulative gains curve shows the percentage of the
|
||||
overall number of cases in a given category <em>gained</em> by targeting a
|
||||
percentage of the total number of cases.
|
||||
</p>
|
||||
|
||||
<p>Similarly, the receiver operating characteristic curve, or ROC curve,
|
||||
displays the diagnostic ability of a binary classifier system as its
|
||||
discrimination threshold is varied. It plots the true positive rate against the false positive rate.
|
||||
</p>
|
||||
</section>
|
||||
|
||||
<section>
|
||||
<h2 id="compare-bagging-on-trees-with-random-forests">Compare Bagging on Trees with Random Forests </h2>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="cell border-box-sizing code_cell rendered">
|
||||
<div class="input">
|
||||
<div class="inner_cell">
|
||||
<div class="input_area">
|
||||
<div class="highlight" style="background: #eeeedd">
|
||||
<pre style="font-size: 80%; line-height: 125%;">bag_clf = BaggingClassifier(
|
||||
DecisionTreeClassifier(splitter=<span style="color: #CD5555">"random"</span>, max_leaf_nodes=<span style="color: #B452CD">16</span>, random_state=<span style="color: #B452CD">42</span>),
|
||||
n_estimators=<span style="color: #B452CD">500</span>, max_samples=<span style="color: #B452CD">1.0</span>, bootstrap=<span style="color: #8B008B; font-weight: bold">True</span>, n_jobs=-<span style="color: #B452CD">1</span>, random_state=<span style="color: #B452CD">42</span>)
|
||||
</pre>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="output_wrapper">
|
||||
<div class="output">
|
||||
<div class="output_area">
|
||||
<div class="output_subarea output_stream output_stdout output_text">
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="cell border-box-sizing code_cell rendered">
|
||||
<div class="input">
|
||||
<div class="inner_cell">
|
||||
<div class="input_area">
|
||||
<div class="highlight" style="background: #eeeedd">
|
||||
<pre style="font-size: 80%; line-height: 125%;">bag_clf.fit(X_train, y_train)
|
||||
y_pred = bag_clf.predict(X_test)
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.ensemble</span> <span style="color: #8B008B; font-weight: bold">import</span> RandomForestClassifier
|
||||
rnd_clf = RandomForestClassifier(n_estimators=<span style="color: #B452CD">500</span>, max_leaf_nodes=<span style="color: #B452CD">16</span>, n_jobs=-<span style="color: #B452CD">1</span>, random_state=<span style="color: #B452CD">42</span>)
|
||||
rnd_clf.fit(X_train, y_train)
|
||||
y_pred_rf = rnd_clf.predict(X_test)
|
||||
np.sum(y_pred == y_pred_rf) / <span style="color: #658b00">len</span>(y_pred)
|
||||
</pre>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="output_wrapper">
|
||||
<div class="output">
|
||||
<div class="output_area">
|
||||
<div class="output_subarea output_stream output_stdout output_text">
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section>
|
||||
<h2 id="boosting-a-bird-s-eye-view">Boosting, a Bird's Eye View </h2>
|
||||
|
||||
|
||||
@@ -65,6 +65,18 @@ div.toc p,a {
|
||||
{'highest level': 2,
|
||||
'sections': [('Overview of week 48', 2, None, 'overview-of-week-48'),
|
||||
('Plan for week 47', 2, None, 'plan-for-week-47'),
|
||||
('Random Forest Algorithm, reminder from last week',
|
||||
2,
|
||||
None,
|
||||
'random-forest-algorithm-reminder-from-last-week'),
|
||||
('Random Forests Compared with other Methods on the Cancer Data',
|
||||
2,
|
||||
None,
|
||||
'random-forests-compared-with-other-methods-on-the-cancer-data'),
|
||||
('Compare Bagging on Trees with Random Forests',
|
||||
2,
|
||||
None,
|
||||
'compare-bagging-on-trees-with-random-forests'),
|
||||
("Boosting, a Bird's Eye View",
|
||||
2,
|
||||
None,
|
||||
@@ -309,7 +321,7 @@ MathJax.Hub.Config({
|
||||
<li> Work and Discussion of project 3</li>
|
||||
<li> Last weekly exercise</li>
|
||||
<li> Lab sessions at usual times.</li>
|
||||
<li> For the week of December 2-6, lab sessions atart at 10am and end 4pm, room FØ434, Tuesday and Wednesday</li>
|
||||
<li> For the week of December 2-6, lab sessions start at 10am and end at 4pm, room FØ434, Tuesday and Wednesday</li>
|
||||
</ul>
|
||||
</div>
|
||||
|
||||
@@ -322,8 +334,8 @@ MathJax.Hub.Config({
|
||||
<li> Summary of course</li>
|
||||
<li> Readings and Videos:
|
||||
<ol type="a"></li>
|
||||
<li> These lecture notes at <a href="https://github.com/CompPhysics/MachineLearning/blob/master/doc/pub/week47/ipynb/week48.ipynb" target="_blank"><tt>https://github.com/CompPhysics/MachineLearning/blob/master/doc/pub/week47/ipynb/week48.ipynb</tt></a></li>
|
||||
<li> See also lecture notes from week 47 at <a href="https://github.com/CompPhysics/MachineLearning/blob/master/doc/pub/week46/ipynb/week47.ipynb" target="_blank"><tt>https://github.com/CompPhysics/MachineLearning/blob/master/doc/pub/week46/ipynb/week47.ipynb</tt></a>. The lecture on Monday starts with a repetition on AdaBoost before we move over to gradient boosting with examples
|
||||
<li> These lecture notes at <a href="https://github.com/CompPhysics/MachineLearning/blob/master/doc/pub/week48/ipynb/week48.ipynb" target="_blank"><tt>https://github.com/CompPhysics/MachineLearning/blob/master/doc/pub/week48/ipynb/week48.ipynb</tt></a></li>
|
||||
<li> See also lecture notes from week 47 at <a href="https://github.com/CompPhysics/MachineLearning/blob/master/doc/pub/week47/ipynb/week47.ipynb" target="_blank"><tt>https://github.com/CompPhysics/MachineLearning/blob/master/doc/pub/week47/ipynb/week47.ipynb</tt></a>. The lecture on Monday starts with a repetition on AdaBoost before we move over to gradient boosting with examples
|
||||
<!-- o Video of lecture at <a href="https://youtu.be/RIHzmLv05DA" target="_blank"><tt>https://youtu.be/RIHzmLv05DA</tt></a> -->
|
||||
<!-- o Whiteboard notes at <a href="https://github.com/CompPhysics/MachineLearning/blob/master/doc/HandWrittenNotes/2024/NotesNovember25.pdf" target="_blank"><tt>https://github.com/CompPhysics/MachineLearning/blob/master/doc/HandWrittenNotes/2024/NotesNovember25.pdf</tt></a> --></li>
|
||||
<li> Video on Decision trees <a href="https://www.youtube.com/watch?v=RmajweUFKvM&ab_channel=Simplilearn" target="_blank"><tt>https://www.youtube.com/watch?v=RmajweUFKvM&ab_channel=Simplilearn</tt></a></li>
|
||||
@@ -336,6 +348,175 @@ MathJax.Hub.Config({
|
||||
</div>
|
||||
|
||||
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
<h2 id="random-forest-algorithm-reminder-from-last-week">Random Forest Algorithm, reminder from last week </h2>
|
||||
|
||||
<p>The algorithm described here can be applied to both classification and regression problems.</p>
|
||||
|
||||
<p>We will grow of forest of say \( B \) trees.</p>
|
||||
<ol>
|
||||
<li> For \( b=1:B \)
|
||||
<ol type="a"></li>
|
||||
<li> Draw a bootstrap sample from the training data organized in our \( \boldsymbol{X} \) matrix.</li>
|
||||
<li> We grow then a random forest tree \( T_b \) based on the bootstrapped data by repeating the steps outlined till we reach the maximum node size is reached</li>
|
||||
<ol>
|
||||
<li> we select \( m \le p \) variables at random from the \( p \) predictors/features</li>
|
||||
<li> pick the best split point among the \( m \) features using for example the CART algorithm and create a new node</li>
|
||||
<li> split the node into daughter nodes</li>
|
||||
</ol>
|
||||
</ol>
|
||||
<li> Output then the ensemble of trees \( \{T_b\}_1^{B} \) and make predictions for either a regression type of problem or a classification type of problem.</li>
|
||||
</ol>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
<h2 id="random-forests-compared-with-other-methods-on-the-cancer-data">Random Forests Compared with other Methods on the Cancer Data </h2>
|
||||
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="cell border-box-sizing code_cell rendered">
|
||||
<div class="input">
|
||||
<div class="inner_cell">
|
||||
<div class="input_area">
|
||||
<div class="highlight" style="background: #eeeedd">
|
||||
<pre style="line-height: 125%;"><span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">matplotlib.pyplot</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">plt</span>
|
||||
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">numpy</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">np</span>
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.model_selection</span> <span style="color: #8B008B; font-weight: bold">import</span> train_test_split
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.datasets</span> <span style="color: #8B008B; font-weight: bold">import</span> load_breast_cancer
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.svm</span> <span style="color: #8B008B; font-weight: bold">import</span> SVC
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.linear_model</span> <span style="color: #8B008B; font-weight: bold">import</span> LogisticRegression
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.tree</span> <span style="color: #8B008B; font-weight: bold">import</span> DecisionTreeClassifier
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.ensemble</span> <span style="color: #8B008B; font-weight: bold">import</span> BaggingClassifier
|
||||
|
||||
<span style="color: #228B22"># Load the data</span>
|
||||
cancer = load_breast_cancer()
|
||||
|
||||
X_train, X_test, y_train, y_test = train_test_split(cancer.data,cancer.target,random_state=<span style="color: #B452CD">0</span>)
|
||||
<span style="color: #658b00">print</span>(X_train.shape)
|
||||
<span style="color: #658b00">print</span>(X_test.shape)
|
||||
<span style="color: #228B22">#define methods</span>
|
||||
<span style="color: #228B22"># Logistic Regression</span>
|
||||
logreg = LogisticRegression(solver=<span style="color: #CD5555">'lbfgs'</span>)
|
||||
<span style="color: #228B22"># Support vector machine</span>
|
||||
svm = SVC(gamma=<span style="color: #CD5555">'auto'</span>, C=<span style="color: #B452CD">100</span>)
|
||||
<span style="color: #228B22"># Decision Trees</span>
|
||||
deep_tree_clf = DecisionTreeClassifier(max_depth=<span style="color: #8B008B; font-weight: bold">None</span>)
|
||||
<span style="color: #228B22">#Scale the data</span>
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.preprocessing</span> <span style="color: #8B008B; font-weight: bold">import</span> StandardScaler
|
||||
scaler = StandardScaler()
|
||||
scaler.fit(X_train)
|
||||
X_train_scaled = scaler.transform(X_train)
|
||||
X_test_scaled = scaler.transform(X_test)
|
||||
<span style="color: #228B22"># Logistic Regression</span>
|
||||
logreg.fit(X_train_scaled, y_train)
|
||||
<span style="color: #658b00">print</span>(<span style="color: #CD5555">"Test set accuracy Logistic Regression with scaled data: {:.2f}"</span>.format(logreg.score(X_test_scaled,y_test)))
|
||||
<span style="color: #228B22"># Support Vector Machine</span>
|
||||
svm.fit(X_train_scaled, y_train)
|
||||
<span style="color: #658b00">print</span>(<span style="color: #CD5555">"Test set accuracy SVM with scaled data: {:.2f}"</span>.format(logreg.score(X_test_scaled,y_test)))
|
||||
<span style="color: #228B22"># Decision Trees</span>
|
||||
deep_tree_clf.fit(X_train_scaled, y_train)
|
||||
<span style="color: #658b00">print</span>(<span style="color: #CD5555">"Test set accuracy with Decision Trees and scaled data: {:.2f}"</span>.format(deep_tree_clf.score(X_test_scaled,y_test)))
|
||||
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.ensemble</span> <span style="color: #8B008B; font-weight: bold">import</span> RandomForestClassifier
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.preprocessing</span> <span style="color: #8B008B; font-weight: bold">import</span> LabelEncoder
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.model_selection</span> <span style="color: #8B008B; font-weight: bold">import</span> cross_validate
|
||||
<span style="color: #228B22"># Data set not specificied</span>
|
||||
<span style="color: #228B22">#Instantiate the model with 500 trees and entropy as splitting criteria</span>
|
||||
Random_Forest_model = RandomForestClassifier(n_estimators=<span style="color: #B452CD">500</span>,criterion=<span style="color: #CD5555">"entropy"</span>)
|
||||
Random_Forest_model.fit(X_train_scaled, y_train)
|
||||
<span style="color: #228B22">#Cross validation</span>
|
||||
accuracy = cross_validate(Random_Forest_model,X_test_scaled,y_test,cv=<span style="color: #B452CD">10</span>)[<span style="color: #CD5555">'test_score'</span>]
|
||||
<span style="color: #658b00">print</span>(accuracy)
|
||||
<span style="color: #658b00">print</span>(<span style="color: #CD5555">"Test set accuracy with Random Forests and scaled data: {:.2f}"</span>.format(Random_Forest_model.score(X_test_scaled,y_test)))
|
||||
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">scikitplot</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">skplt</span>
|
||||
y_pred = Random_Forest_model.predict(X_test_scaled)
|
||||
skplt.metrics.plot_confusion_matrix(y_test, y_pred, normalize=<span style="color: #8B008B; font-weight: bold">True</span>)
|
||||
plt.show()
|
||||
y_probas = Random_Forest_model.predict_proba(X_test_scaled)
|
||||
skplt.metrics.plot_roc(y_test, y_probas)
|
||||
plt.show()
|
||||
skplt.metrics.plot_cumulative_gain(y_test, y_probas)
|
||||
plt.show()
|
||||
</pre>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="output_wrapper">
|
||||
<div class="output">
|
||||
<div class="output_area">
|
||||
<div class="output_subarea output_stream output_stdout output_text">
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<p>Recall that the cumulative gains curve shows the percentage of the
|
||||
overall number of cases in a given category <em>gained</em> by targeting a
|
||||
percentage of the total number of cases.
|
||||
</p>
|
||||
|
||||
<p>Similarly, the receiver operating characteristic curve, or ROC curve,
|
||||
displays the diagnostic ability of a binary classifier system as its
|
||||
discrimination threshold is varied. It plots the true positive rate against the false positive rate.
|
||||
</p>
|
||||
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
<h2 id="compare-bagging-on-trees-with-random-forests">Compare Bagging on Trees with Random Forests </h2>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="cell border-box-sizing code_cell rendered">
|
||||
<div class="input">
|
||||
<div class="inner_cell">
|
||||
<div class="input_area">
|
||||
<div class="highlight" style="background: #eeeedd">
|
||||
<pre style="line-height: 125%;">bag_clf = BaggingClassifier(
|
||||
DecisionTreeClassifier(splitter=<span style="color: #CD5555">"random"</span>, max_leaf_nodes=<span style="color: #B452CD">16</span>, random_state=<span style="color: #B452CD">42</span>),
|
||||
n_estimators=<span style="color: #B452CD">500</span>, max_samples=<span style="color: #B452CD">1.0</span>, bootstrap=<span style="color: #8B008B; font-weight: bold">True</span>, n_jobs=-<span style="color: #B452CD">1</span>, random_state=<span style="color: #B452CD">42</span>)
|
||||
</pre>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="output_wrapper">
|
||||
<div class="output">
|
||||
<div class="output_area">
|
||||
<div class="output_subarea output_stream output_stdout output_text">
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="cell border-box-sizing code_cell rendered">
|
||||
<div class="input">
|
||||
<div class="inner_cell">
|
||||
<div class="input_area">
|
||||
<div class="highlight" style="background: #eeeedd">
|
||||
<pre style="line-height: 125%;">bag_clf.fit(X_train, y_train)
|
||||
y_pred = bag_clf.predict(X_test)
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.ensemble</span> <span style="color: #8B008B; font-weight: bold">import</span> RandomForestClassifier
|
||||
rnd_clf = RandomForestClassifier(n_estimators=<span style="color: #B452CD">500</span>, max_leaf_nodes=<span style="color: #B452CD">16</span>, n_jobs=-<span style="color: #B452CD">1</span>, random_state=<span style="color: #B452CD">42</span>)
|
||||
rnd_clf.fit(X_train, y_train)
|
||||
y_pred_rf = rnd_clf.predict(X_test)
|
||||
np.sum(y_pred == y_pred_rf) / <span style="color: #658b00">len</span>(y_pred)
|
||||
</pre>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="output_wrapper">
|
||||
<div class="output">
|
||||
<div class="output_area">
|
||||
<div class="output_subarea output_stream output_stdout output_text">
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
<h2 id="boosting-a-bird-s-eye-view">Boosting, a Bird's Eye View </h2>
|
||||
|
||||
|
||||
@@ -142,6 +142,18 @@ div.toc p,a {
|
||||
{'highest level': 2,
|
||||
'sections': [('Overview of week 48', 2, None, 'overview-of-week-48'),
|
||||
('Plan for week 47', 2, None, 'plan-for-week-47'),
|
||||
('Random Forest Algorithm, reminder from last week',
|
||||
2,
|
||||
None,
|
||||
'random-forest-algorithm-reminder-from-last-week'),
|
||||
('Random Forests Compared with other Methods on the Cancer Data',
|
||||
2,
|
||||
None,
|
||||
'random-forests-compared-with-other-methods-on-the-cancer-data'),
|
||||
('Compare Bagging on Trees with Random Forests',
|
||||
2,
|
||||
None,
|
||||
'compare-bagging-on-trees-with-random-forests'),
|
||||
("Boosting, a Bird's Eye View",
|
||||
2,
|
||||
None,
|
||||
@@ -386,7 +398,7 @@ MathJax.Hub.Config({
|
||||
<li> Work and Discussion of project 3</li>
|
||||
<li> Last weekly exercise</li>
|
||||
<li> Lab sessions at usual times.</li>
|
||||
<li> For the week of December 2-6, lab sessions atart at 10am and end 4pm, room FØ434, Tuesday and Wednesday</li>
|
||||
<li> For the week of December 2-6, lab sessions start at 10am and end at 4pm, room FØ434, Tuesday and Wednesday</li>
|
||||
</ul>
|
||||
</div>
|
||||
|
||||
@@ -399,8 +411,8 @@ MathJax.Hub.Config({
|
||||
<li> Summary of course</li>
|
||||
<li> Readings and Videos:
|
||||
<ol type="a"></li>
|
||||
<li> These lecture notes at <a href="https://github.com/CompPhysics/MachineLearning/blob/master/doc/pub/week47/ipynb/week48.ipynb" target="_blank"><tt>https://github.com/CompPhysics/MachineLearning/blob/master/doc/pub/week47/ipynb/week48.ipynb</tt></a></li>
|
||||
<li> See also lecture notes from week 47 at <a href="https://github.com/CompPhysics/MachineLearning/blob/master/doc/pub/week46/ipynb/week47.ipynb" target="_blank"><tt>https://github.com/CompPhysics/MachineLearning/blob/master/doc/pub/week46/ipynb/week47.ipynb</tt></a>. The lecture on Monday starts with a repetition on AdaBoost before we move over to gradient boosting with examples
|
||||
<li> These lecture notes at <a href="https://github.com/CompPhysics/MachineLearning/blob/master/doc/pub/week48/ipynb/week48.ipynb" target="_blank"><tt>https://github.com/CompPhysics/MachineLearning/blob/master/doc/pub/week48/ipynb/week48.ipynb</tt></a></li>
|
||||
<li> See also lecture notes from week 47 at <a href="https://github.com/CompPhysics/MachineLearning/blob/master/doc/pub/week47/ipynb/week47.ipynb" target="_blank"><tt>https://github.com/CompPhysics/MachineLearning/blob/master/doc/pub/week47/ipynb/week47.ipynb</tt></a>. The lecture on Monday starts with a repetition on AdaBoost before we move over to gradient boosting with examples
|
||||
<!-- o Video of lecture at <a href="https://youtu.be/RIHzmLv05DA" target="_blank"><tt>https://youtu.be/RIHzmLv05DA</tt></a> -->
|
||||
<!-- o Whiteboard notes at <a href="https://github.com/CompPhysics/MachineLearning/blob/master/doc/HandWrittenNotes/2024/NotesNovember25.pdf" target="_blank"><tt>https://github.com/CompPhysics/MachineLearning/blob/master/doc/HandWrittenNotes/2024/NotesNovember25.pdf</tt></a> --></li>
|
||||
<li> Video on Decision trees <a href="https://www.youtube.com/watch?v=RmajweUFKvM&ab_channel=Simplilearn" target="_blank"><tt>https://www.youtube.com/watch?v=RmajweUFKvM&ab_channel=Simplilearn</tt></a></li>
|
||||
@@ -413,6 +425,175 @@ MathJax.Hub.Config({
|
||||
</div>
|
||||
|
||||
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
<h2 id="random-forest-algorithm-reminder-from-last-week">Random Forest Algorithm, reminder from last week </h2>
|
||||
|
||||
<p>The algorithm described here can be applied to both classification and regression problems.</p>
|
||||
|
||||
<p>We will grow of forest of say \( B \) trees.</p>
|
||||
<ol>
|
||||
<li> For \( b=1:B \)
|
||||
<ol type="a"></li>
|
||||
<li> Draw a bootstrap sample from the training data organized in our \( \boldsymbol{X} \) matrix.</li>
|
||||
<li> We grow then a random forest tree \( T_b \) based on the bootstrapped data by repeating the steps outlined till we reach the maximum node size is reached</li>
|
||||
<ol>
|
||||
<li> we select \( m \le p \) variables at random from the \( p \) predictors/features</li>
|
||||
<li> pick the best split point among the \( m \) features using for example the CART algorithm and create a new node</li>
|
||||
<li> split the node into daughter nodes</li>
|
||||
</ol>
|
||||
</ol>
|
||||
<li> Output then the ensemble of trees \( \{T_b\}_1^{B} \) and make predictions for either a regression type of problem or a classification type of problem.</li>
|
||||
</ol>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
<h2 id="random-forests-compared-with-other-methods-on-the-cancer-data">Random Forests Compared with other Methods on the Cancer Data </h2>
|
||||
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="cell border-box-sizing code_cell rendered">
|
||||
<div class="input">
|
||||
<div class="inner_cell">
|
||||
<div class="input_area">
|
||||
<div class="highlight" style="background: #f8f8f8">
|
||||
<pre style="line-height: 125%;"><span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">matplotlib.pyplot</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">plt</span>
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">numpy</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">np</span>
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.model_selection</span> <span style="color: #008000; font-weight: bold">import</span> train_test_split
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.datasets</span> <span style="color: #008000; font-weight: bold">import</span> load_breast_cancer
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.svm</span> <span style="color: #008000; font-weight: bold">import</span> SVC
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.linear_model</span> <span style="color: #008000; font-weight: bold">import</span> LogisticRegression
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.tree</span> <span style="color: #008000; font-weight: bold">import</span> DecisionTreeClassifier
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.ensemble</span> <span style="color: #008000; font-weight: bold">import</span> BaggingClassifier
|
||||
|
||||
<span style="color: #408080; font-style: italic"># Load the data</span>
|
||||
cancer <span style="color: #666666">=</span> load_breast_cancer()
|
||||
|
||||
X_train, X_test, y_train, y_test <span style="color: #666666">=</span> train_test_split(cancer<span style="color: #666666">.</span>data,cancer<span style="color: #666666">.</span>target,random_state<span style="color: #666666">=0</span>)
|
||||
<span style="color: #008000">print</span>(X_train<span style="color: #666666">.</span>shape)
|
||||
<span style="color: #008000">print</span>(X_test<span style="color: #666666">.</span>shape)
|
||||
<span style="color: #408080; font-style: italic">#define methods</span>
|
||||
<span style="color: #408080; font-style: italic"># Logistic Regression</span>
|
||||
logreg <span style="color: #666666">=</span> LogisticRegression(solver<span style="color: #666666">=</span><span style="color: #BA2121">'lbfgs'</span>)
|
||||
<span style="color: #408080; font-style: italic"># Support vector machine</span>
|
||||
svm <span style="color: #666666">=</span> SVC(gamma<span style="color: #666666">=</span><span style="color: #BA2121">'auto'</span>, C<span style="color: #666666">=100</span>)
|
||||
<span style="color: #408080; font-style: italic"># Decision Trees</span>
|
||||
deep_tree_clf <span style="color: #666666">=</span> DecisionTreeClassifier(max_depth<span style="color: #666666">=</span><span style="color: #008000; font-weight: bold">None</span>)
|
||||
<span style="color: #408080; font-style: italic">#Scale the data</span>
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.preprocessing</span> <span style="color: #008000; font-weight: bold">import</span> StandardScaler
|
||||
scaler <span style="color: #666666">=</span> StandardScaler()
|
||||
scaler<span style="color: #666666">.</span>fit(X_train)
|
||||
X_train_scaled <span style="color: #666666">=</span> scaler<span style="color: #666666">.</span>transform(X_train)
|
||||
X_test_scaled <span style="color: #666666">=</span> scaler<span style="color: #666666">.</span>transform(X_test)
|
||||
<span style="color: #408080; font-style: italic"># Logistic Regression</span>
|
||||
logreg<span style="color: #666666">.</span>fit(X_train_scaled, y_train)
|
||||
<span style="color: #008000">print</span>(<span style="color: #BA2121">"Test set accuracy Logistic Regression with scaled data: </span><span style="color: #BB6688; font-weight: bold">{:.2f}</span><span style="color: #BA2121">"</span><span style="color: #666666">.</span>format(logreg<span style="color: #666666">.</span>score(X_test_scaled,y_test)))
|
||||
<span style="color: #408080; font-style: italic"># Support Vector Machine</span>
|
||||
svm<span style="color: #666666">.</span>fit(X_train_scaled, y_train)
|
||||
<span style="color: #008000">print</span>(<span style="color: #BA2121">"Test set accuracy SVM with scaled data: </span><span style="color: #BB6688; font-weight: bold">{:.2f}</span><span style="color: #BA2121">"</span><span style="color: #666666">.</span>format(logreg<span style="color: #666666">.</span>score(X_test_scaled,y_test)))
|
||||
<span style="color: #408080; font-style: italic"># Decision Trees</span>
|
||||
deep_tree_clf<span style="color: #666666">.</span>fit(X_train_scaled, y_train)
|
||||
<span style="color: #008000">print</span>(<span style="color: #BA2121">"Test set accuracy with Decision Trees and scaled data: </span><span style="color: #BB6688; font-weight: bold">{:.2f}</span><span style="color: #BA2121">"</span><span style="color: #666666">.</span>format(deep_tree_clf<span style="color: #666666">.</span>score(X_test_scaled,y_test)))
|
||||
|
||||
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.ensemble</span> <span style="color: #008000; font-weight: bold">import</span> RandomForestClassifier
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.preprocessing</span> <span style="color: #008000; font-weight: bold">import</span> LabelEncoder
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.model_selection</span> <span style="color: #008000; font-weight: bold">import</span> cross_validate
|
||||
<span style="color: #408080; font-style: italic"># Data set not specificied</span>
|
||||
<span style="color: #408080; font-style: italic">#Instantiate the model with 500 trees and entropy as splitting criteria</span>
|
||||
Random_Forest_model <span style="color: #666666">=</span> RandomForestClassifier(n_estimators<span style="color: #666666">=500</span>,criterion<span style="color: #666666">=</span><span style="color: #BA2121">"entropy"</span>)
|
||||
Random_Forest_model<span style="color: #666666">.</span>fit(X_train_scaled, y_train)
|
||||
<span style="color: #408080; font-style: italic">#Cross validation</span>
|
||||
accuracy <span style="color: #666666">=</span> cross_validate(Random_Forest_model,X_test_scaled,y_test,cv<span style="color: #666666">=10</span>)[<span style="color: #BA2121">'test_score'</span>]
|
||||
<span style="color: #008000">print</span>(accuracy)
|
||||
<span style="color: #008000">print</span>(<span style="color: #BA2121">"Test set accuracy with Random Forests and scaled data: </span><span style="color: #BB6688; font-weight: bold">{:.2f}</span><span style="color: #BA2121">"</span><span style="color: #666666">.</span>format(Random_Forest_model<span style="color: #666666">.</span>score(X_test_scaled,y_test)))
|
||||
|
||||
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">scikitplot</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">skplt</span>
|
||||
y_pred <span style="color: #666666">=</span> Random_Forest_model<span style="color: #666666">.</span>predict(X_test_scaled)
|
||||
skplt<span style="color: #666666">.</span>metrics<span style="color: #666666">.</span>plot_confusion_matrix(y_test, y_pred, normalize<span style="color: #666666">=</span><span style="color: #008000; font-weight: bold">True</span>)
|
||||
plt<span style="color: #666666">.</span>show()
|
||||
y_probas <span style="color: #666666">=</span> Random_Forest_model<span style="color: #666666">.</span>predict_proba(X_test_scaled)
|
||||
skplt<span style="color: #666666">.</span>metrics<span style="color: #666666">.</span>plot_roc(y_test, y_probas)
|
||||
plt<span style="color: #666666">.</span>show()
|
||||
skplt<span style="color: #666666">.</span>metrics<span style="color: #666666">.</span>plot_cumulative_gain(y_test, y_probas)
|
||||
plt<span style="color: #666666">.</span>show()
|
||||
</pre>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="output_wrapper">
|
||||
<div class="output">
|
||||
<div class="output_area">
|
||||
<div class="output_subarea output_stream output_stdout output_text">
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<p>Recall that the cumulative gains curve shows the percentage of the
|
||||
overall number of cases in a given category <em>gained</em> by targeting a
|
||||
percentage of the total number of cases.
|
||||
</p>
|
||||
|
||||
<p>Similarly, the receiver operating characteristic curve, or ROC curve,
|
||||
displays the diagnostic ability of a binary classifier system as its
|
||||
discrimination threshold is varied. It plots the true positive rate against the false positive rate.
|
||||
</p>
|
||||
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
<h2 id="compare-bagging-on-trees-with-random-forests">Compare Bagging on Trees with Random Forests </h2>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="cell border-box-sizing code_cell rendered">
|
||||
<div class="input">
|
||||
<div class="inner_cell">
|
||||
<div class="input_area">
|
||||
<div class="highlight" style="background: #f8f8f8">
|
||||
<pre style="line-height: 125%;">bag_clf <span style="color: #666666">=</span> BaggingClassifier(
|
||||
DecisionTreeClassifier(splitter<span style="color: #666666">=</span><span style="color: #BA2121">"random"</span>, max_leaf_nodes<span style="color: #666666">=16</span>, random_state<span style="color: #666666">=42</span>),
|
||||
n_estimators<span style="color: #666666">=500</span>, max_samples<span style="color: #666666">=1.0</span>, bootstrap<span style="color: #666666">=</span><span style="color: #008000; font-weight: bold">True</span>, n_jobs<span style="color: #666666">=-1</span>, random_state<span style="color: #666666">=42</span>)
|
||||
</pre>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="output_wrapper">
|
||||
<div class="output">
|
||||
<div class="output_area">
|
||||
<div class="output_subarea output_stream output_stdout output_text">
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="cell border-box-sizing code_cell rendered">
|
||||
<div class="input">
|
||||
<div class="inner_cell">
|
||||
<div class="input_area">
|
||||
<div class="highlight" style="background: #f8f8f8">
|
||||
<pre style="line-height: 125%;">bag_clf<span style="color: #666666">.</span>fit(X_train, y_train)
|
||||
y_pred <span style="color: #666666">=</span> bag_clf<span style="color: #666666">.</span>predict(X_test)
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.ensemble</span> <span style="color: #008000; font-weight: bold">import</span> RandomForestClassifier
|
||||
rnd_clf <span style="color: #666666">=</span> RandomForestClassifier(n_estimators<span style="color: #666666">=500</span>, max_leaf_nodes<span style="color: #666666">=16</span>, n_jobs<span style="color: #666666">=-1</span>, random_state<span style="color: #666666">=42</span>)
|
||||
rnd_clf<span style="color: #666666">.</span>fit(X_train, y_train)
|
||||
y_pred_rf <span style="color: #666666">=</span> rnd_clf<span style="color: #666666">.</span>predict(X_test)
|
||||
np<span style="color: #666666">.</span>sum(y_pred <span style="color: #666666">==</span> y_pred_rf) <span style="color: #666666">/</span> <span style="color: #008000">len</span>(y_pred)
|
||||
</pre>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="output_wrapper">
|
||||
<div class="output">
|
||||
<div class="output_area">
|
||||
<div class="output_subarea output_stream output_stdout output_text">
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
<h2 id="boosting-a-bird-s-eye-view">Boosting, a Bird's Eye View </h2>
|
||||
|
||||
|
||||
Binary file not shown.
+329
-159
File diff suppressed because it is too large
Load Diff
@@ -12,15 +12,15 @@ DATE: today
|
||||
* Work and Discussion of project 3
|
||||
* Last weekly exercise
|
||||
* Lab sessions at usual times.
|
||||
* For the week of December 2-6, lab sessions atart at 10am and end 4pm, room FØ434, Tuesday and Wednesday
|
||||
* For the week of December 2-6, lab sessions start at 10am and end at 4pm, room FØ434, Tuesday and Wednesday
|
||||
!eblock
|
||||
|
||||
!bblock Plans for the lecture Monday 25 November, with video suggestions etc
|
||||
o Boosting and gradient boosting and ensemble models
|
||||
o Summary of course
|
||||
o Readings and Videos:
|
||||
o These lecture notes at URL:"https://github.com/CompPhysics/MachineLearning/blob/master/doc/pub/week47/ipynb/week48.ipynb"
|
||||
o See also lecture notes from week 47 at URL:"https://github.com/CompPhysics/MachineLearning/blob/master/doc/pub/week46/ipynb/week47.ipynb". The lecture on Monday starts with a repetition on AdaBoost before we move over to gradient boosting with examples
|
||||
o These lecture notes at URL:"https://github.com/CompPhysics/MachineLearning/blob/master/doc/pub/week48/ipynb/week48.ipynb"
|
||||
o See also lecture notes from week 47 at URL:"https://github.com/CompPhysics/MachineLearning/blob/master/doc/pub/week47/ipynb/week47.ipynb". The lecture on Monday starts with a repetition on AdaBoost before we move over to gradient boosting with examples
|
||||
# o Video of lecture at URL:"https://youtu.be/RIHzmLv05DA"
|
||||
# o Whiteboard notes at URL:"https://github.com/CompPhysics/MachineLearning/blob/master/doc/HandWrittenNotes/2024/NotesNovember25.pdf"
|
||||
o Video on Decision trees URL:"https://www.youtube.com/watch?v=RmajweUFKvM&ab_channel=Simplilearn"
|
||||
@@ -31,6 +31,128 @@ o Readings and Videos:
|
||||
!eblock
|
||||
|
||||
|
||||
!split
|
||||
===== Random Forest Algorithm, reminder from last week =====
|
||||
|
||||
|
||||
The algorithm described here can be applied to both classification and regression problems.
|
||||
|
||||
We will grow of forest of say $B$ trees.
|
||||
o For $b=1:B$
|
||||
o Draw a bootstrap sample from the training data organized in our $\bm{X}$ matrix.
|
||||
o We grow then a random forest tree $T_b$ based on the bootstrapped data by repeating the steps outlined till we reach the maximum node size is reached
|
||||
o we select $m \le p$ variables at random from the $p$ predictors/features
|
||||
o pick the best split point among the $m$ features using for example the CART algorithm and create a new node
|
||||
o split the node into daughter nodes
|
||||
o Output then the ensemble of trees $\{T_b\}_1^{B}$ and make predictions for either a regression type of problem or a classification type of problem.
|
||||
|
||||
|
||||
|
||||
!split
|
||||
===== Random Forests Compared with other Methods on the Cancer Data =====
|
||||
|
||||
|
||||
!bc pycod
|
||||
import matplotlib.pyplot as plt
|
||||
import numpy as np
|
||||
from sklearn.model_selection import train_test_split
|
||||
from sklearn.datasets import load_breast_cancer
|
||||
from sklearn.svm import SVC
|
||||
from sklearn.linear_model import LogisticRegression
|
||||
from sklearn.tree import DecisionTreeClassifier
|
||||
from sklearn.ensemble import BaggingClassifier
|
||||
|
||||
# Load the data
|
||||
cancer = load_breast_cancer()
|
||||
|
||||
X_train, X_test, y_train, y_test = train_test_split(cancer.data,cancer.target,random_state=0)
|
||||
print(X_train.shape)
|
||||
print(X_test.shape)
|
||||
#define methods
|
||||
# Logistic Regression
|
||||
logreg = LogisticRegression(solver='lbfgs')
|
||||
# Support vector machine
|
||||
svm = SVC(gamma='auto', C=100)
|
||||
# Decision Trees
|
||||
deep_tree_clf = DecisionTreeClassifier(max_depth=None)
|
||||
#Scale the data
|
||||
from sklearn.preprocessing import StandardScaler
|
||||
scaler = StandardScaler()
|
||||
scaler.fit(X_train)
|
||||
X_train_scaled = scaler.transform(X_train)
|
||||
X_test_scaled = scaler.transform(X_test)
|
||||
# Logistic Regression
|
||||
logreg.fit(X_train_scaled, y_train)
|
||||
print("Test set accuracy Logistic Regression with scaled data: {:.2f}".format(logreg.score(X_test_scaled,y_test)))
|
||||
# Support Vector Machine
|
||||
svm.fit(X_train_scaled, y_train)
|
||||
print("Test set accuracy SVM with scaled data: {:.2f}".format(logreg.score(X_test_scaled,y_test)))
|
||||
# Decision Trees
|
||||
deep_tree_clf.fit(X_train_scaled, y_train)
|
||||
print("Test set accuracy with Decision Trees and scaled data: {:.2f}".format(deep_tree_clf.score(X_test_scaled,y_test)))
|
||||
|
||||
|
||||
from sklearn.ensemble import RandomForestClassifier
|
||||
from sklearn.preprocessing import LabelEncoder
|
||||
from sklearn.model_selection import cross_validate
|
||||
# Data set not specificied
|
||||
#Instantiate the model with 500 trees and entropy as splitting criteria
|
||||
Random_Forest_model = RandomForestClassifier(n_estimators=500,criterion="entropy")
|
||||
Random_Forest_model.fit(X_train_scaled, y_train)
|
||||
#Cross validation
|
||||
accuracy = cross_validate(Random_Forest_model,X_test_scaled,y_test,cv=10)['test_score']
|
||||
print(accuracy)
|
||||
print("Test set accuracy with Random Forests and scaled data: {:.2f}".format(Random_Forest_model.score(X_test_scaled,y_test)))
|
||||
|
||||
|
||||
import scikitplot as skplt
|
||||
y_pred = Random_Forest_model.predict(X_test_scaled)
|
||||
skplt.metrics.plot_confusion_matrix(y_test, y_pred, normalize=True)
|
||||
plt.show()
|
||||
y_probas = Random_Forest_model.predict_proba(X_test_scaled)
|
||||
skplt.metrics.plot_roc(y_test, y_probas)
|
||||
plt.show()
|
||||
skplt.metrics.plot_cumulative_gain(y_test, y_probas)
|
||||
plt.show()
|
||||
|
||||
!ec
|
||||
|
||||
|
||||
Recall that the cumulative gains curve shows the percentage of the
|
||||
overall number of cases in a given category *gained* by targeting a
|
||||
percentage of the total number of cases.
|
||||
|
||||
Similarly, the receiver operating characteristic curve, or ROC curve,
|
||||
displays the diagnostic ability of a binary classifier system as its
|
||||
discrimination threshold is varied. It plots the true positive rate against the false positive rate.
|
||||
|
||||
|
||||
!split
|
||||
===== Compare Bagging on Trees with Random Forests =====
|
||||
!bc pycod
|
||||
bag_clf = BaggingClassifier(
|
||||
DecisionTreeClassifier(splitter="random", max_leaf_nodes=16, random_state=42),
|
||||
n_estimators=500, max_samples=1.0, bootstrap=True, n_jobs=-1, random_state=42)
|
||||
!ec
|
||||
|
||||
|
||||
|
||||
!bc pycod
|
||||
bag_clf.fit(X_train, y_train)
|
||||
y_pred = bag_clf.predict(X_test)
|
||||
from sklearn.ensemble import RandomForestClassifier
|
||||
rnd_clf = RandomForestClassifier(n_estimators=500, max_leaf_nodes=16, n_jobs=-1, random_state=42)
|
||||
rnd_clf.fit(X_train, y_train)
|
||||
y_pred_rf = rnd_clf.predict(X_test)
|
||||
np.sum(y_pred == y_pred_rf) / len(y_pred)
|
||||
!ec
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
!split
|
||||
@@ -1539,3 +1661,4 @@ FIGURE: [figures/Nebbdyr2.png, width=500 frac=0.6]
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user