update week 39
This commit is contained in:
Morten Hjorth-Jensen
@@ -0,0 +1,201 @@
|
||||
TrueLabel,PredictedLabel
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,0
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
|
@@ -0,0 +1,301 @@
|
||||
TrueLabel,PredictedLabel
|
||||
0,0
|
||||
0,1
|
||||
0,0
|
||||
0,0
|
||||
0,1
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,1
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,1
|
||||
0,0
|
||||
0,2
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
1,1
|
||||
1,1
|
||||
1,0
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,0
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,0
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,0
|
||||
1,1
|
||||
1,1
|
||||
1,0
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
|
+161
-323
@@ -8,8 +8,8 @@ doconce format html week39.do.txt --html_style=bootstrap --pygments_html_style=d
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
||||
<meta name="generator" content="DocOnce: https://github.com/doconce/doconce/" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<meta name="description" content="Week 39: Optimization and Gradient Methods">
|
||||
<title>Week 39: Optimization and Gradient Methods</title>
|
||||
<meta name="description" content="Week 39: Resampling methods and logistic regression">
|
||||
<title>Week 39: Resampling methods and logistic regression</title>
|
||||
<!-- Bootstrap style: bootstrap -->
|
||||
<!-- doconce format html week39.do.txt --html_style=bootstrap --pygments_html_style=default --html_admon=bootstrap_panel --html_output=week39-bs --no_mako -->
|
||||
<link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
|
||||
@@ -36,20 +36,115 @@ doconce format html week39.do.txt --html_style=bootstrap --pygments_html_style=d
|
||||
|
||||
<!-- tocinfo
|
||||
{'highest level': 2,
|
||||
'sections': [('Plan for week 39, September 23-27, 2024',
|
||||
'sections': [('Plan for week 39, September 22-26, 2025',
|
||||
2,
|
||||
None,
|
||||
'plan-for-week-39-september-23-27-2024'),
|
||||
('Lecture Monday September 23',
|
||||
'plan-for-week-39-september-22-26-2025'),
|
||||
('Readings and Videos, resampling methods',
|
||||
2,
|
||||
None,
|
||||
'lecture-monday-september-23'),
|
||||
'readings-and-videos-resampling-methods'),
|
||||
('Readings and Videos, logistic regression',
|
||||
2,
|
||||
None,
|
||||
'readings-and-videos-logistic-regression'),
|
||||
('Lab sessions week 39', 2, None, 'lab-sessions-week-39'),
|
||||
('Lecture Monday September 23, Optimization, the central part of '
|
||||
'any Machine Learning algortithm',
|
||||
('Lecture material', 2, None, 'lecture-material'),
|
||||
('Resampling methods', 2, None, 'resampling-methods'),
|
||||
('Resampling approaches can be computationally expensive',
|
||||
2,
|
||||
None,
|
||||
'lecture-monday-september-23-optimization-the-central-part-of-any-machine-learning-algortithm'),
|
||||
'resampling-approaches-can-be-computationally-expensive'),
|
||||
('Why resampling methods ?', 2, None, 'why-resampling-methods'),
|
||||
('Statistical analysis', 2, None, 'statistical-analysis'),
|
||||
('Resampling methods', 2, None, 'resampling-methods'),
|
||||
('Resampling methods: Bootstrap',
|
||||
2,
|
||||
None,
|
||||
'resampling-methods-bootstrap'),
|
||||
('The bias-variance tradeoff',
|
||||
2,
|
||||
None,
|
||||
'the-bias-variance-tradeoff'),
|
||||
('A way to Read the Bias-Variance Tradeoff',
|
||||
2,
|
||||
None,
|
||||
'a-way-to-read-the-bias-variance-tradeoff'),
|
||||
('Understanding what happens',
|
||||
2,
|
||||
None,
|
||||
'understanding-what-happens'),
|
||||
('Summing up', 2, None, 'summing-up'),
|
||||
("Another Example from Scikit-Learn's Repository",
|
||||
2,
|
||||
None,
|
||||
'another-example-from-scikit-learn-s-repository'),
|
||||
('Various steps in cross-validation',
|
||||
2,
|
||||
None,
|
||||
'various-steps-in-cross-validation'),
|
||||
('Cross-validation in brief',
|
||||
2,
|
||||
None,
|
||||
'cross-validation-in-brief'),
|
||||
('Code Example for Cross-validation and $k$-fold '
|
||||
'Cross-validation',
|
||||
2,
|
||||
None,
|
||||
'code-example-for-cross-validation-and-k-fold-cross-validation'),
|
||||
('More examples on bootstrap and cross-validation and errors',
|
||||
2,
|
||||
None,
|
||||
'more-examples-on-bootstrap-and-cross-validation-and-errors'),
|
||||
('The same example but now with cross-validation',
|
||||
2,
|
||||
None,
|
||||
'the-same-example-but-now-with-cross-validation'),
|
||||
('Logistic Regression', 2, None, 'logistic-regression'),
|
||||
('Classification problems', 2, None, 'classification-problems'),
|
||||
('Optimization and Deep learning',
|
||||
2,
|
||||
None,
|
||||
'optimization-and-deep-learning'),
|
||||
('Basics', 2, None, 'basics'),
|
||||
('Linear classifier', 2, None, 'linear-classifier'),
|
||||
('Some selected properties', 2, None, 'some-selected-properties'),
|
||||
('Simple example', 2, None, 'simple-example'),
|
||||
('Plotting the mean value for each group',
|
||||
2,
|
||||
None,
|
||||
'plotting-the-mean-value-for-each-group'),
|
||||
('The logistic function', 2, None, 'the-logistic-function'),
|
||||
('Examples of likelihood functions used in logistic regression '
|
||||
'and nueral networks',
|
||||
2,
|
||||
None,
|
||||
'examples-of-likelihood-functions-used-in-logistic-regression-and-nueral-networks'),
|
||||
('Two parameters', 2, None, 'two-parameters'),
|
||||
('Maximum likelihood', 2, None, 'maximum-likelihood'),
|
||||
('The cost function rewritten',
|
||||
2,
|
||||
None,
|
||||
'the-cost-function-rewritten'),
|
||||
('Minimizing the cross entropy',
|
||||
2,
|
||||
None,
|
||||
'minimizing-the-cross-entropy'),
|
||||
('A more compact expression',
|
||||
2,
|
||||
None,
|
||||
'a-more-compact-expression'),
|
||||
('Extending to more predictors',
|
||||
2,
|
||||
None,
|
||||
'extending-to-more-predictors'),
|
||||
('Including more classes', 2, None, 'including-more-classes'),
|
||||
('More classes', 2, None, 'more-classes'),
|
||||
('Optimization, the central part of any Machine Learning '
|
||||
'algortithm',
|
||||
2,
|
||||
None,
|
||||
'optimization-the-central-part-of-any-machine-learning-algortithm'),
|
||||
('Revisiting our Logistic Regression case',
|
||||
2,
|
||||
None,
|
||||
@@ -59,225 +154,14 @@ doconce format html week39.do.txt --html_style=bootstrap --pygments_html_style=d
|
||||
2,
|
||||
None,
|
||||
'solving-using-newton-raphson-s-method'),
|
||||
("Brief reminder on Newton-Raphson's method",
|
||||
('Example code for Logistic Regression',
|
||||
2,
|
||||
None,
|
||||
'brief-reminder-on-newton-raphson-s-method'),
|
||||
('The equations', 2, None, 'the-equations'),
|
||||
('Simple geometric interpretation',
|
||||
2,
|
||||
'example-code-for-logistic-regression'),
|
||||
('Synthetic data generation',
|
||||
3,
|
||||
None,
|
||||
'simple-geometric-interpretation'),
|
||||
('Extending to more than one variable',
|
||||
2,
|
||||
None,
|
||||
'extending-to-more-than-one-variable'),
|
||||
('Steepest descent', 2, None, 'steepest-descent'),
|
||||
('More on Steepest descent', 2, None, 'more-on-steepest-descent'),
|
||||
('The ideal', 2, None, 'the-ideal'),
|
||||
('The sensitiveness of the gradient descent',
|
||||
2,
|
||||
None,
|
||||
'the-sensitiveness-of-the-gradient-descent'),
|
||||
('Convex functions', 2, None, 'convex-functions'),
|
||||
('Convex function', 2, None, 'convex-function'),
|
||||
('Conditions on convex functions',
|
||||
2,
|
||||
None,
|
||||
'conditions-on-convex-functions'),
|
||||
('More on convex functions', 2, None, 'more-on-convex-functions'),
|
||||
('Some simple problems', 2, None, 'some-simple-problems'),
|
||||
('Standard steepest descent',
|
||||
2,
|
||||
None,
|
||||
'standard-steepest-descent'),
|
||||
('Gradient method', 2, None, 'gradient-method'),
|
||||
('Steepest descent method', 2, None, 'steepest-descent-method'),
|
||||
('Steepest descent method', 2, None, 'steepest-descent-method'),
|
||||
('Final expressions', 2, None, 'final-expressions'),
|
||||
('Steepest descent example', 2, None, 'steepest-descent-example'),
|
||||
('Conjugate gradient method',
|
||||
2,
|
||||
None,
|
||||
'conjugate-gradient-method'),
|
||||
('Conjugate gradient method',
|
||||
2,
|
||||
None,
|
||||
'conjugate-gradient-method'),
|
||||
('Conjugate gradient method',
|
||||
2,
|
||||
None,
|
||||
'conjugate-gradient-method'),
|
||||
('Conjugate gradient method',
|
||||
2,
|
||||
None,
|
||||
'conjugate-gradient-method'),
|
||||
('Conjugate gradient method and iterations',
|
||||
2,
|
||||
None,
|
||||
'conjugate-gradient-method-and-iterations'),
|
||||
('Conjugate gradient method',
|
||||
2,
|
||||
None,
|
||||
'conjugate-gradient-method'),
|
||||
('Conjugate gradient method',
|
||||
2,
|
||||
None,
|
||||
'conjugate-gradient-method'),
|
||||
('Conjugate gradient method',
|
||||
2,
|
||||
None,
|
||||
'conjugate-gradient-method'),
|
||||
('Revisiting our first homework',
|
||||
2,
|
||||
None,
|
||||
'revisiting-our-first-homework'),
|
||||
('Gradient descent example', 2, None, 'gradient-descent-example'),
|
||||
('The derivative of the cost/loss function',
|
||||
2,
|
||||
None,
|
||||
'the-derivative-of-the-cost-loss-function'),
|
||||
('The Hessian matrix', 2, None, 'the-hessian-matrix'),
|
||||
('Simple program', 2, None, 'simple-program'),
|
||||
('Gradient Descent Example', 2, None, 'gradient-descent-example'),
|
||||
('And a corresponding example using _scikit-learn_',
|
||||
2,
|
||||
None,
|
||||
'and-a-corresponding-example-using-scikit-learn'),
|
||||
('Gradient descent and Ridge',
|
||||
2,
|
||||
None,
|
||||
'gradient-descent-and-ridge'),
|
||||
('The Hessian matrix for Ridge Regression',
|
||||
2,
|
||||
None,
|
||||
'the-hessian-matrix-for-ridge-regression'),
|
||||
('Program example for gradient descent with Ridge Regression',
|
||||
2,
|
||||
None,
|
||||
'program-example-for-gradient-descent-with-ridge-regression'),
|
||||
('Using gradient descent methods, limitations',
|
||||
2,
|
||||
None,
|
||||
'using-gradient-descent-methods-limitations'),
|
||||
('Improving gradient descent with momentum',
|
||||
2,
|
||||
None,
|
||||
'improving-gradient-descent-with-momentum'),
|
||||
('Same code but now with momentum gradient descent',
|
||||
2,
|
||||
None,
|
||||
'same-code-but-now-with-momentum-gradient-descent'),
|
||||
('Overview video on Stochastic Gradient Descent',
|
||||
2,
|
||||
None,
|
||||
'overview-video-on-stochastic-gradient-descent'),
|
||||
('Batches and mini-batches', 2, None, 'batches-and-mini-batches'),
|
||||
('Stochastic Gradient Descent (SGD)',
|
||||
2,
|
||||
None,
|
||||
'stochastic-gradient-descent-sgd'),
|
||||
('Stochastic Gradient Descent',
|
||||
2,
|
||||
None,
|
||||
'stochastic-gradient-descent'),
|
||||
('Computation of gradients', 2, None, 'computation-of-gradients'),
|
||||
('SGD example', 2, None, 'sgd-example'),
|
||||
('The gradient step', 2, None, 'the-gradient-step'),
|
||||
('Simple example code', 2, None, 'simple-example-code'),
|
||||
('When do we stop?', 2, None, 'when-do-we-stop'),
|
||||
('Slightly different approach',
|
||||
2,
|
||||
None,
|
||||
'slightly-different-approach'),
|
||||
('Time decay rate', 2, None, 'time-decay-rate'),
|
||||
('Code with a Number of Minibatches which varies',
|
||||
2,
|
||||
None,
|
||||
'code-with-a-number-of-minibatches-which-varies'),
|
||||
('Replace or not', 2, None, 'replace-or-not'),
|
||||
('Momentum based GD', 2, None, 'momentum-based-gd'),
|
||||
('More on momentum based approaches',
|
||||
2,
|
||||
None,
|
||||
'more-on-momentum-based-approaches'),
|
||||
('Momentum parameter', 2, None, 'momentum-parameter'),
|
||||
('Second moment of the gradient',
|
||||
2,
|
||||
None,
|
||||
'second-moment-of-the-gradient'),
|
||||
('RMS prop', 2, None, 'rms-prop'),
|
||||
('"ADAM optimizer":"https://arxiv.org/abs/1412.6980"',
|
||||
2,
|
||||
None,
|
||||
'adam-optimizer-https-arxiv-org-abs-1412-6980'),
|
||||
('Algorithms and codes for Adagrad, RMSprop and Adam',
|
||||
2,
|
||||
None,
|
||||
'algorithms-and-codes-for-adagrad-rmsprop-and-adam'),
|
||||
('Practical tips', 2, None, 'practical-tips'),
|
||||
('Automatic differentiation',
|
||||
2,
|
||||
None,
|
||||
'automatic-differentiation'),
|
||||
('Using autograd', 2, None, 'using-autograd'),
|
||||
('Autograd with more complicated functions',
|
||||
2,
|
||||
None,
|
||||
'autograd-with-more-complicated-functions'),
|
||||
('More complicated functions using the elements of their '
|
||||
'arguments directly',
|
||||
2,
|
||||
None,
|
||||
'more-complicated-functions-using-the-elements-of-their-arguments-directly'),
|
||||
('Functions using mathematical functions from Numpy',
|
||||
2,
|
||||
None,
|
||||
'functions-using-mathematical-functions-from-numpy'),
|
||||
('More autograd', 2, None, 'more-autograd'),
|
||||
('And with loops', 2, None, 'and-with-loops'),
|
||||
('Using recursion', 2, None, 'using-recursion'),
|
||||
('Unsupported functions', 2, None, 'unsupported-functions'),
|
||||
('The syntax a.dot(b) when finding the dot product',
|
||||
2,
|
||||
None,
|
||||
'the-syntax-a-dot-b-when-finding-the-dot-product'),
|
||||
('Using Autograd with OLS', 2, None, 'using-autograd-with-ols'),
|
||||
('Same code but now with momentum gradient descent',
|
||||
2,
|
||||
None,
|
||||
'same-code-but-now-with-momentum-gradient-descent'),
|
||||
("But none of these can compete with Newton's method",
|
||||
2,
|
||||
None,
|
||||
'but-none-of-these-can-compete-with-newton-s-method'),
|
||||
('Including Stochastic Gradient Descent with Autograd',
|
||||
2,
|
||||
None,
|
||||
'including-stochastic-gradient-descent-with-autograd'),
|
||||
('Same code but now with momentum gradient descent',
|
||||
2,
|
||||
None,
|
||||
'same-code-but-now-with-momentum-gradient-descent'),
|
||||
('Similar (second order function now) problem but now with '
|
||||
'AdaGrad',
|
||||
2,
|
||||
None,
|
||||
'similar-second-order-function-now-problem-but-now-with-adagrad'),
|
||||
('RMSprop for adaptive learning rate with Stochastic Gradient '
|
||||
'Descent',
|
||||
2,
|
||||
None,
|
||||
'rmsprop-for-adaptive-learning-rate-with-stochastic-gradient-descent'),
|
||||
('And finally "ADAM":"https://arxiv.org/pdf/1412.6980.pdf"',
|
||||
2,
|
||||
None,
|
||||
'and-finally-adam-https-arxiv-org-pdf-1412-6980-pdf'),
|
||||
('And Logistic Regression', 2, None, 'and-logistic-regression'),
|
||||
('Introducing "JAX":"https://jax.readthedocs.io/en/latest/"',
|
||||
2,
|
||||
None,
|
||||
'introducing-jax-https-jax-readthedocs-io-en-latest')]}
|
||||
'synthetic-data-generation')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -305,101 +189,58 @@ MathJax.Hub.Config({
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
</button>
|
||||
<a class="navbar-brand" href="week39-bs.html">Week 39: Optimization and Gradient Methods</a>
|
||||
<a class="navbar-brand" href="week39-bs.html">Week 39: Resampling methods and logistic regression</a>
|
||||
</div>
|
||||
<div class="navbar-collapse collapse navbar-responsive-collapse">
|
||||
<ul class="nav navbar-nav navbar-right">
|
||||
<li class="dropdown">
|
||||
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
|
||||
<ul class="dropdown-menu">
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs001.html#plan-for-week-39-september-23-27-2024" style="font-size: 80%;">Plan for week 39, September 23-27, 2024</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs002.html#lecture-monday-september-23" style="font-size: 80%;">Lecture Monday September 23</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs003.html#lab-sessions-week-39" style="font-size: 80%;">Lab sessions week 39</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs004.html#lecture-monday-september-23-optimization-the-central-part-of-any-machine-learning-algortithm" style="font-size: 80%;">Lecture Monday September 23, Optimization, the central part of any Machine Learning algortithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs005.html#revisiting-our-logistic-regression-case" style="font-size: 80%;">Revisiting our Logistic Regression case</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs006.html#the-equations-to-solve" style="font-size: 80%;">The equations to solve</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs007.html#solving-using-newton-raphson-s-method" style="font-size: 80%;">Solving using Newton-Raphson's method</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs008.html#brief-reminder-on-newton-raphson-s-method" style="font-size: 80%;">Brief reminder on Newton-Raphson's method</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs009.html#the-equations" style="font-size: 80%;">The equations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs010.html#simple-geometric-interpretation" style="font-size: 80%;">Simple geometric interpretation</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs011.html#extending-to-more-than-one-variable" style="font-size: 80%;">Extending to more than one variable</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs012.html#steepest-descent" style="font-size: 80%;">Steepest descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs013.html#more-on-steepest-descent" style="font-size: 80%;">More on Steepest descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs014.html#the-ideal" style="font-size: 80%;">The ideal</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs015.html#the-sensitiveness-of-the-gradient-descent" style="font-size: 80%;">The sensitiveness of the gradient descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs016.html#convex-functions" style="font-size: 80%;">Convex functions</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs017.html#convex-function" style="font-size: 80%;">Convex function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs018.html#conditions-on-convex-functions" style="font-size: 80%;">Conditions on convex functions</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs019.html#more-on-convex-functions" style="font-size: 80%;">More on convex functions</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs020.html#some-simple-problems" style="font-size: 80%;">Some simple problems</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs021.html#standard-steepest-descent" style="font-size: 80%;">Standard steepest descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs022.html#gradient-method" style="font-size: 80%;">Gradient method</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs024.html#steepest-descent-method" style="font-size: 80%;">Steepest descent method</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs024.html#steepest-descent-method" style="font-size: 80%;">Steepest descent method</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs025.html#final-expressions" style="font-size: 80%;">Final expressions</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs026.html#steepest-descent-example" style="font-size: 80%;">Steepest descent example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs034.html#conjugate-gradient-method" style="font-size: 80%;">Conjugate gradient method</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs034.html#conjugate-gradient-method" style="font-size: 80%;">Conjugate gradient method</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs034.html#conjugate-gradient-method" style="font-size: 80%;">Conjugate gradient method</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs034.html#conjugate-gradient-method" style="font-size: 80%;">Conjugate gradient method</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs031.html#conjugate-gradient-method-and-iterations" style="font-size: 80%;">Conjugate gradient method and iterations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs034.html#conjugate-gradient-method" style="font-size: 80%;">Conjugate gradient method</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs034.html#conjugate-gradient-method" style="font-size: 80%;">Conjugate gradient method</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs034.html#conjugate-gradient-method" style="font-size: 80%;">Conjugate gradient method</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs035.html#revisiting-our-first-homework" style="font-size: 80%;">Revisiting our first homework</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs040.html#gradient-descent-example" style="font-size: 80%;">Gradient descent example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs037.html#the-derivative-of-the-cost-loss-function" style="font-size: 80%;">The derivative of the cost/loss function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs038.html#the-hessian-matrix" style="font-size: 80%;">The Hessian matrix</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs039.html#simple-program" style="font-size: 80%;">Simple program</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs040.html#gradient-descent-example" style="font-size: 80%;">Gradient Descent Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs041.html#and-a-corresponding-example-using-scikit-learn" style="font-size: 80%;">And a corresponding example using <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs042.html#gradient-descent-and-ridge" style="font-size: 80%;">Gradient descent and Ridge</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs043.html#the-hessian-matrix-for-ridge-regression" style="font-size: 80%;">The Hessian matrix for Ridge Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs044.html#program-example-for-gradient-descent-with-ridge-regression" style="font-size: 80%;">Program example for gradient descent with Ridge Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs045.html#using-gradient-descent-methods-limitations" style="font-size: 80%;">Using gradient descent methods, limitations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs046.html#improving-gradient-descent-with-momentum" style="font-size: 80%;">Improving gradient descent with momentum</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs083.html#same-code-but-now-with-momentum-gradient-descent" style="font-size: 80%;">Same code but now with momentum gradient descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs048.html#overview-video-on-stochastic-gradient-descent" style="font-size: 80%;">Overview video on Stochastic Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs049.html#batches-and-mini-batches" style="font-size: 80%;">Batches and mini-batches</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs050.html#stochastic-gradient-descent-sgd" style="font-size: 80%;">Stochastic Gradient Descent (SGD)</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs051.html#stochastic-gradient-descent" style="font-size: 80%;">Stochastic Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs052.html#computation-of-gradients" style="font-size: 80%;">Computation of gradients</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs053.html#sgd-example" style="font-size: 80%;">SGD example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs054.html#the-gradient-step" style="font-size: 80%;">The gradient step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs055.html#simple-example-code" style="font-size: 80%;">Simple example code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs056.html#when-do-we-stop" style="font-size: 80%;">When do we stop?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs057.html#slightly-different-approach" style="font-size: 80%;">Slightly different approach</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs058.html#time-decay-rate" style="font-size: 80%;">Time decay rate</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs059.html#code-with-a-number-of-minibatches-which-varies" style="font-size: 80%;">Code with a Number of Minibatches which varies</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs060.html#replace-or-not" style="font-size: 80%;">Replace or not</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs061.html#momentum-based-gd" style="font-size: 80%;">Momentum based GD</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs062.html#more-on-momentum-based-approaches" style="font-size: 80%;">More on momentum based approaches</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs063.html#momentum-parameter" style="font-size: 80%;">Momentum parameter</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs064.html#second-moment-of-the-gradient" style="font-size: 80%;">Second moment of the gradient</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs065.html#rms-prop" style="font-size: 80%;">RMS prop</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs066.html#adam-optimizer-https-arxiv-org-abs-1412-6980" style="font-size: 80%;">"ADAM optimizer":"https://arxiv.org/abs/1412.6980"</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs067.html#algorithms-and-codes-for-adagrad-rmsprop-and-adam" style="font-size: 80%;">Algorithms and codes for Adagrad, RMSprop and Adam</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs068.html#practical-tips" style="font-size: 80%;">Practical tips</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs069.html#automatic-differentiation" style="font-size: 80%;">Automatic differentiation</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs070.html#using-autograd" style="font-size: 80%;">Using autograd</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs071.html#autograd-with-more-complicated-functions" style="font-size: 80%;">Autograd with more complicated functions</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs072.html#more-complicated-functions-using-the-elements-of-their-arguments-directly" style="font-size: 80%;">More complicated functions using the elements of their arguments directly</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs073.html#functions-using-mathematical-functions-from-numpy" style="font-size: 80%;">Functions using mathematical functions from Numpy</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs074.html#more-autograd" style="font-size: 80%;">More autograd</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs075.html#and-with-loops" style="font-size: 80%;">And with loops</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs076.html#using-recursion" style="font-size: 80%;">Using recursion</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs077.html#unsupported-functions" style="font-size: 80%;">Unsupported functions</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs078.html#the-syntax-a-dot-b-when-finding-the-dot-product" style="font-size: 80%;">The syntax a.dot(b) when finding the dot product</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs079.html#using-autograd-with-ols" style="font-size: 80%;">Using Autograd with OLS</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs083.html#same-code-but-now-with-momentum-gradient-descent" style="font-size: 80%;">Same code but now with momentum gradient descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs081.html#but-none-of-these-can-compete-with-newton-s-method" style="font-size: 80%;">But none of these can compete with Newton's method</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs082.html#including-stochastic-gradient-descent-with-autograd" style="font-size: 80%;">Including Stochastic Gradient Descent with Autograd</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs083.html#same-code-but-now-with-momentum-gradient-descent" style="font-size: 80%;">Same code but now with momentum gradient descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs084.html#similar-second-order-function-now-problem-but-now-with-adagrad" style="font-size: 80%;">Similar (second order function now) problem but now with AdaGrad</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs085.html#rmsprop-for-adaptive-learning-rate-with-stochastic-gradient-descent" style="font-size: 80%;">RMSprop for adaptive learning rate with Stochastic Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs086.html#and-finally-adam-https-arxiv-org-pdf-1412-6980-pdf" style="font-size: 80%;">And finally "ADAM":"https://arxiv.org/pdf/1412.6980.pdf"</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs087.html#and-logistic-regression" style="font-size: 80%;">And Logistic Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs088.html#introducing-jax-https-jax-readthedocs-io-en-latest" style="font-size: 80%;">Introducing "JAX":"https://jax.readthedocs.io/en/latest/"</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs001.html#plan-for-week-39-september-22-26-2025" style="font-size: 80%;"><b>Plan for week 39, September 22-26, 2025</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs002.html#readings-and-videos-resampling-methods" style="font-size: 80%;"><b>Readings and Videos, resampling methods</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs003.html#readings-and-videos-logistic-regression" style="font-size: 80%;"><b>Readings and Videos, logistic regression</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs004.html#lab-sessions-week-39" style="font-size: 80%;"><b>Lab sessions week 39</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs005.html#lecture-material" style="font-size: 80%;"><b>Lecture material</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs010.html#resampling-methods" style="font-size: 80%;"><b>Resampling methods</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs007.html#resampling-approaches-can-be-computationally-expensive" style="font-size: 80%;"><b>Resampling approaches can be computationally expensive</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs008.html#why-resampling-methods" style="font-size: 80%;"><b>Why resampling methods ?</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs009.html#statistical-analysis" style="font-size: 80%;"><b>Statistical analysis</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs010.html#resampling-methods" style="font-size: 80%;"><b>Resampling methods</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs011.html#resampling-methods-bootstrap" style="font-size: 80%;"><b>Resampling methods: Bootstrap</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs012.html#the-bias-variance-tradeoff" style="font-size: 80%;"><b>The bias-variance tradeoff</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs013.html#a-way-to-read-the-bias-variance-tradeoff" style="font-size: 80%;"><b>A way to Read the Bias-Variance Tradeoff</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs014.html#understanding-what-happens" style="font-size: 80%;"><b>Understanding what happens</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs015.html#summing-up" style="font-size: 80%;"><b>Summing up</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs016.html#another-example-from-scikit-learn-s-repository" style="font-size: 80%;"><b>Another Example from Scikit-Learn's Repository</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs017.html#various-steps-in-cross-validation" style="font-size: 80%;"><b>Various steps in cross-validation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs018.html#cross-validation-in-brief" style="font-size: 80%;"><b>Cross-validation in brief</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs019.html#code-example-for-cross-validation-and-k-fold-cross-validation" style="font-size: 80%;"><b>Code Example for Cross-validation and \( k \)-fold Cross-validation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs020.html#more-examples-on-bootstrap-and-cross-validation-and-errors" style="font-size: 80%;"><b>More examples on bootstrap and cross-validation and errors</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs021.html#the-same-example-but-now-with-cross-validation" style="font-size: 80%;"><b>The same example but now with cross-validation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs022.html#logistic-regression" style="font-size: 80%;"><b>Logistic Regression</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs023.html#classification-problems" style="font-size: 80%;"><b>Classification problems</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs024.html#optimization-and-deep-learning" style="font-size: 80%;"><b>Optimization and Deep learning</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs025.html#basics" style="font-size: 80%;"><b>Basics</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs026.html#linear-classifier" style="font-size: 80%;"><b>Linear classifier</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs027.html#some-selected-properties" style="font-size: 80%;"><b>Some selected properties</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs028.html#simple-example" style="font-size: 80%;"><b>Simple example</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs029.html#plotting-the-mean-value-for-each-group" style="font-size: 80%;"><b>Plotting the mean value for each group</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs030.html#the-logistic-function" style="font-size: 80%;"><b>The logistic function</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs031.html#examples-of-likelihood-functions-used-in-logistic-regression-and-nueral-networks" style="font-size: 80%;"><b>Examples of likelihood functions used in logistic regression and nueral networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs032.html#two-parameters" style="font-size: 80%;"><b>Two parameters</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs033.html#maximum-likelihood" style="font-size: 80%;"><b>Maximum likelihood</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs034.html#the-cost-function-rewritten" style="font-size: 80%;"><b>The cost function rewritten</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs035.html#minimizing-the-cross-entropy" style="font-size: 80%;"><b>Minimizing the cross entropy</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs036.html#a-more-compact-expression" style="font-size: 80%;"><b>A more compact expression</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs037.html#extending-to-more-predictors" style="font-size: 80%;"><b>Extending to more predictors</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs038.html#including-more-classes" style="font-size: 80%;"><b>Including more classes</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs039.html#more-classes" style="font-size: 80%;"><b>More classes</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs040.html#optimization-the-central-part-of-any-machine-learning-algortithm" style="font-size: 80%;"><b>Optimization, the central part of any Machine Learning algortithm</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs041.html#revisiting-our-logistic-regression-case" style="font-size: 80%;"><b>Revisiting our Logistic Regression case</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs042.html#the-equations-to-solve" style="font-size: 80%;"><b>The equations to solve</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs043.html#solving-using-newton-raphson-s-method" style="font-size: 80%;"><b>Solving using Newton-Raphson's method</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs044.html#example-code-for-logistic-regression" style="font-size: 80%;"><b>Example code for Logistic Regression</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs044.html#synthetic-data-generation" style="font-size: 80%;"> Synthetic data generation</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -413,19 +254,16 @@ MathJax.Hub.Config({
|
||||
<!-- ------------------- main content ---------------------- -->
|
||||
<div class="jumbotron">
|
||||
<center>
|
||||
<h1>Week 39: Optimization and Gradient Methods</h1>
|
||||
<h1>Week 39: Resampling methods and logistic regression</h1>
|
||||
</center> <!-- document title -->
|
||||
|
||||
<!-- author(s): Morten Hjorth-Jensen -->
|
||||
<center>
|
||||
<b>Morten Hjorth-Jensen</b> [1, 2]
|
||||
<b>Morten Hjorth-Jensen</b>
|
||||
</center>
|
||||
<!-- institution(s) -->
|
||||
<!-- institution -->
|
||||
<center>
|
||||
[1] <b>Department of Physics, University of Oslo</b>
|
||||
</center>
|
||||
<center>
|
||||
[2] <b>Department of Physics and Astronomy and Facility for Rare Isotope Beams, Michigan State University</b>
|
||||
<b>Department of Physics, University of Oslo</b>
|
||||
</center>
|
||||
<br>
|
||||
<center>
|
||||
@@ -454,7 +292,7 @@ MathJax.Hub.Config({
|
||||
<li><a href="._week39-bs008.html">9</a></li>
|
||||
<li><a href="._week39-bs009.html">10</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._week39-bs088.html">89</a></li>
|
||||
<li><a href="._week39-bs044.html">45</a></li>
|
||||
<li><a href="._week39-bs001.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
@@ -468,7 +306,7 @@ MathJax.Hub.Config({
|
||||
</footer>
|
||||
-->
|
||||
<center style="font-size:80%">
|
||||
<!-- copyright --> © 1999-2024, Morten Hjorth-Jensen. Released under CC Attribution-NonCommercial 4.0 license
|
||||
<!-- copyright --> © 1999-2025, Morten Hjorth-Jensen. Released under CC Attribution-NonCommercial 4.0 license
|
||||
</center>
|
||||
</body>
|
||||
</html>
|
||||
|
||||
+1717
-3520
File diff suppressed because it is too large
Load Diff
+1747
-3546
File diff suppressed because it is too large
Load Diff
+1747
-3546
File diff suppressed because it is too large
Load Diff
Binary file not shown.
+2224
-4111
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,100 @@
|
||||
1, 21, 1, 0
|
||||
2, 23, 1, 0
|
||||
3, 25, 1, 1
|
||||
4, 29, 1, 0
|
||||
5, 21, 1, 0
|
||||
6, 24, 1, 0
|
||||
7, 27, 1, 0
|
||||
8, 29, 1, 0
|
||||
9, 28, 1, 0
|
||||
10, 26, 1, 0
|
||||
11, 30, 2, 0
|
||||
12, 31, 2, 0
|
||||
13, 31, 2, 0
|
||||
14, 31, 2, 1
|
||||
15, 32, 2, 0
|
||||
16, 34, 2, 0
|
||||
17, 34, 2, 0
|
||||
18, 31, 2, 0
|
||||
19, 32, 2, 0
|
||||
20, 32, 2, 0
|
||||
21, 33, 2, 0
|
||||
22, 34, 2, 0
|
||||
23, 31, 2, 1
|
||||
24, 30, 2, 0
|
||||
25, 33, 2, 0
|
||||
26, 36, 3, 1
|
||||
27, 35, 3, 0
|
||||
28, 35, 3, 0
|
||||
29, 38, 3, 0
|
||||
30, 37, 3, 1
|
||||
31, 36, 3, 0
|
||||
32, 35, 3, 0
|
||||
33, 39, 3, 0
|
||||
34, 39, 3, 0
|
||||
35, 38, 3, 1
|
||||
36, 37, 3, 0
|
||||
37, 37, 3, 0
|
||||
38, 40, 4, 0
|
||||
39, 41, 4, 1
|
||||
40, 44, 4, 0
|
||||
41, 44, 4, 0
|
||||
42, 43, 4, 1
|
||||
43, 42, 4, 0
|
||||
44, 41, 4, 0
|
||||
45, 40, 4, 1
|
||||
46, 42, 4, 0
|
||||
47, 42, 4, 0
|
||||
48, 43, 4, 0
|
||||
49, 44, 4, 1
|
||||
50, 44, 4, 0
|
||||
51, 42, 4, 0
|
||||
52, 41, 4, 1
|
||||
53, 45, 5, 0
|
||||
54, 45, 5, 1
|
||||
55, 49, 5, 0
|
||||
56, 48, 5, 1
|
||||
57, 47, 5, 0
|
||||
58, 49, 5, 1
|
||||
59, 46, 5, 1
|
||||
60, 45, 5, 0
|
||||
61, 49, 5, 1
|
||||
62, 48, 5, 0
|
||||
63, 47, 5, 1
|
||||
64, 46, 5, 0
|
||||
65, 47, 5, 0
|
||||
66, 50, 6, 1
|
||||
67, 51, 6, 1
|
||||
68, 51, 6, 0
|
||||
69, 54, 6, 1
|
||||
70, 53, 6, 1
|
||||
71, 51, 6, 0
|
||||
72, 52, 6, 1
|
||||
73, 54, 6, 0
|
||||
74, 55, 7, 1
|
||||
75, 56, 7, 1
|
||||
76, 58, 7, 0
|
||||
77, 59, 7, 1
|
||||
78, 59, 7, 1
|
||||
79, 58, 7, 0
|
||||
80, 55, 7, 1
|
||||
81, 56, 7, 1
|
||||
82, 57, 7, 1
|
||||
83, 58, 7, 1
|
||||
84, 59, 7, 0
|
||||
85, 55, 7, 1
|
||||
86, 56, 7, 1
|
||||
87, 57, 7, 1
|
||||
88, 58, 7, 0
|
||||
89, 59, 7, 1
|
||||
90, 56, 7, 1
|
||||
91, 60, 8, 1
|
||||
92, 65, 8, 1
|
||||
93, 67, 8, 1
|
||||
94, 66, 8, 0
|
||||
95, 63, 8, 1
|
||||
96, 61, 8, 1
|
||||
97, 69, 8, 1
|
||||
98, 65, 8, 1
|
||||
99, 64, 8, 1
|
||||
100, 63, 8, 0
|
||||
|
@@ -0,0 +1,201 @@
|
||||
TrueLabel,PredictedLabel
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,0
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
|
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,80 @@
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
|
||||
# Load the dataset (ensure the CSV file is in the working directory)
|
||||
df = pd.read_csv('creditcard.csv')
|
||||
|
||||
# Preprocess the data
|
||||
df = df.drop('ID', axis=1) # drop ID column
|
||||
# Rename the target column for convenience (optional)
|
||||
df = df.rename(columns={'default payment next month': 'default'})
|
||||
# Separate features and target
|
||||
X = df.drop('default', axis=1).values # features (shape: [30000, 23])
|
||||
y = df['default'].values # target (shape: [30000,])
|
||||
|
||||
# Standardize features (zero mean, unit std dev)
|
||||
X_mean = X.mean(axis=0)
|
||||
X_std = X.std(axis=0)
|
||||
X = (X - X_mean) / X_std
|
||||
|
||||
# Split into training and test sets (80/20 split)
|
||||
m = X.shape[0]
|
||||
train_size = int(0.8 * m)
|
||||
indices = np.random.permutation(m) # shuffle indices for randomness
|
||||
train_idx, test_idx = indices[:train_size], indices[train_size:]
|
||||
X_train, X_test = X[train_idx], X[test_idx]
|
||||
y_train, y_test = y[train_idx], y[test_idx]
|
||||
|
||||
# Add intercept term (bias) to features by adding a column of 1s
|
||||
X_train = np.hstack([np.ones((X_train.shape[0], 1)), X_train])
|
||||
X_test = np.hstack([np.ones((X_test.shape[0], 1)), X_test])
|
||||
|
||||
# Initialize logistic regression parameters
|
||||
n_features = X_train.shape[1] # number of features including bias
|
||||
theta = np.zeros(n_features) # model weights (initialized to 0)
|
||||
|
||||
# Set training hyperparameters
|
||||
learning_rate = 0.1
|
||||
epochs = 500
|
||||
|
||||
# Training loop (gradient descent)
|
||||
m_train = X_train.shape[0]
|
||||
for epoch in range(epochs):
|
||||
# Compute predictions (sigmoid of linear combination)
|
||||
z = X_train.dot(theta) # linear combination
|
||||
predictions = 1 / (1 + np.exp(-z)) # sigmoid function
|
||||
|
||||
# Compute the gradient of loss w.r.t. theta
|
||||
error = predictions - y_train # vector of (pred - true) for each example
|
||||
grad = (X_train.T.dot(error)) / m_train # gradient vector
|
||||
|
||||
# Update weights
|
||||
theta -= learning_rate * grad
|
||||
|
||||
# (Optional) compute and print loss every 50 epochs for monitoring
|
||||
if epoch % 50 == 0:
|
||||
# Binary cross-entropy loss
|
||||
loss = -np.mean(y_train * np.log(predictions + 1e-15) +
|
||||
(1 - y_train) * np.log(1 - predictions + 1e-15))
|
||||
print(f"Epoch {epoch}: Training loss = {loss:.4f}")
|
||||
|
||||
# 5. Evaluate the model on training and test data
|
||||
# Predict probabilities for test set and classify as 1 if sigmoid >= 0.5
|
||||
# (We can equivalently check linear term >= 0, since sigma(x)>=0.5 iff x>=0)
|
||||
train_prob = 1 / (1 + np.exp(-X_train.dot(theta)))
|
||||
test_prob = 1 / (1 + np.exp(-X_test.dot(theta)))
|
||||
train_pred = (train_prob >= 0.5).astype(int)
|
||||
test_pred = (test_prob >= 0.5).astype(int)
|
||||
|
||||
# Calculate accuracy
|
||||
train_accuracy = (train_pred == y_train).mean()
|
||||
test_accuracy = (test_pred == y_test).mean()
|
||||
|
||||
# Calculate final loss on training set for reference
|
||||
final_loss = -np.mean(y_train * np.log(train_prob + 1e-15) +
|
||||
(1 - y_train) * np.log(1 - train_prob + 1e-15))
|
||||
|
||||
# Print performance metrics
|
||||
print(f"Final Training Loss: {final_loss:.4f}")
|
||||
print(f"Training Accuracy: {train_accuracy * 100:.2f}%")
|
||||
print(f"Test Accuracy: {test_accuracy * 100:.2f}%")
|
||||
@@ -0,0 +1,163 @@
|
||||
TITLE: Week 40: Credit card example
|
||||
AUTHOR: Morten Hjorth-Jensen {copyright, 1999-present|CC BY-NC} at Department of Physics, University of Oslo
|
||||
DATE: Week 40
|
||||
|
||||
|
||||
!split
|
||||
===== Example case: Logistic Regression on Taiwan Credit Card Default Dataset =====
|
||||
|
||||
|
||||
|
||||
=== Dataset Overview and Preparation ===
|
||||
|
||||
|
||||
The Default of Credit Card Clients dataset (Taiwan credit card default
|
||||
data) contains 30,000 instances and 23 features (plus an ID and a
|
||||
binary default indicator). The target variable is _default payment
|
||||
next month_ (1 = default, 0 = no default) . Key characteristics of the
|
||||
data include:
|
||||
|
||||
o Instances: 30,000 card clients (each a unique ID)
|
||||
o Features: 23 predictive features (mostly numeric, including demographics, credit limit, payment history, bill amounts, etc.)
|
||||
o Target: Binary default indicator for the next month (Yes=1, No=0)
|
||||
o Missing Values: None – the dataset has no missing values recorded
|
||||
o Class Distribution: 22.1% of clients (6,636) defaulted vs 77.9% non-default (i.e. the data is imbalanced about 1:3).
|
||||
o Basic Stats: Average credit limit is around NT$167,500, and mean age is ~35.5 years . Bill statement amounts are on the order of tens of thousands NT (mean ≈ 40k), while monthly payment amounts average a few thousand NT (mean ≈ 5k) .
|
||||
|
||||
|
||||
=== Feature details ===
|
||||
|
||||
All features are numeric. Some are categorical in nature (encoded as
|
||||
integers): e.g. SEX (1=male, 2=female), EDUCATION (1=graduate,
|
||||
2=university, 3=high school, 4=others, with 5-6 as unknown) and
|
||||
MARRIAGE (1=married, 2=single, 3=others) . Others are continuous
|
||||
(credit amount, bill amounts, payments) or ordinal (e.g. $PAY_0 – PAY_6$
|
||||
repayment status, where -1 = pay duly, 1 = 1 month delay, 2 = 2 months
|
||||
delay, etc.) .
|
||||
|
||||
=== Preprocessing steps ===
|
||||
|
||||
For binary classification, we prepare the data as follows:
|
||||
|
||||
o Remove the ID column (identifier not useful for prediction).
|
||||
o Use the 23 remaining features as $X$ and the _default payment next month_ as the binary target $y$.
|
||||
o The categorical features (SEX, EDUCATION, MARRIAGE) are already integer-encoded; for simplicity, we will use them as-is (one could one-hot encode these, but it’s not strictly necessary for logistic regression).
|
||||
o Standardize the feature values – i.e. scale each feature to zero mean and unit variance. This helps the gradient descent in logistic regression converge faster.
|
||||
o Split the dataset into a training set and testing set (we’ll use an 80/20 split). The model will be trained on the training set and evaluated on the test set to gauge performance on unseen data.
|
||||
|
||||
|
||||
!split
|
||||
===== Implementing Logistic Regression =====
|
||||
|
||||
|
||||
We will implement a logistic regression classifier using only NumPy,
|
||||
without any machine learning libraries. Logistic regression computes a
|
||||
weighted linear combination of features (plus a bias term) and applies
|
||||
a sigmoid function $\sigma(z) = 1/(1+e^{-z})$ to produce a probability
|
||||
between 0 and 1. We train the model by minimizing the logistic loss
|
||||
(binary cross-entropy) using gradient descent.
|
||||
|
||||
Below is a single Python script that loads and preprocesses the data,
|
||||
then defines and trains the logistic regression model using gradient
|
||||
descent. It also outputs the model’s performance metrics (accuracy and
|
||||
loss). The file can be downloaded from the UCI database at URL:"https://archive.ics.uci.edu/dataset/350/default+of+credit+card+clients"
|
||||
|
||||
!bc pycod
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
|
||||
# Load the dataset (ensure the CSV file is in the working directory)
|
||||
df = pd.read_csv('credit_card_clients.csv')
|
||||
|
||||
# Preprocess the data
|
||||
df = df.drop('ID', axis=1) # drop ID column
|
||||
# Rename the target column for convenience (optional)
|
||||
df = df.rename(columns={'default payment next month': 'default'})
|
||||
# Separate features and target
|
||||
X = df.drop('default', axis=1).values # features (shape: [30000, 23])
|
||||
y = df['default'].values # target (shape: [30000,])
|
||||
|
||||
# Standardize features (zero mean, unit std dev)
|
||||
X_mean = X.mean(axis=0)
|
||||
X_std = X.std(axis=0)
|
||||
X = (X - X_mean) / X_std
|
||||
|
||||
# Split into training and test sets (80/20 split)
|
||||
m = X.shape[0]
|
||||
train_size = int(0.8 * m)
|
||||
indices = np.random.permutation(m) # shuffle indices for randomness
|
||||
train_idx, test_idx = indices[:train_size], indices[train_size:]
|
||||
X_train, X_test = X[train_idx], X[test_idx]
|
||||
y_train, y_test = y[train_idx], y[test_idx]
|
||||
|
||||
# Add intercept term (bias) to features by adding a column of 1s
|
||||
X_train = np.hstack([np.ones((X_train.shape[0], 1)), X_train])
|
||||
X_test = np.hstack([np.ones((X_test.shape[0], 1)), X_test])
|
||||
|
||||
# Initialize logistic regression parameters
|
||||
n_features = X_train.shape[1] # number of features including bias
|
||||
theta = np.zeros(n_features) # model weights (initialized to 0)
|
||||
|
||||
# Set training hyperparameters
|
||||
learning_rate = 0.1
|
||||
epochs = 500
|
||||
|
||||
# Training loop (gradient descent)
|
||||
m_train = X_train.shape[0]
|
||||
for epoch in range(epochs):
|
||||
# Compute predictions (sigmoid of linear combination)
|
||||
z = X_train.dot(theta) # linear combination
|
||||
predictions = 1 / (1 + np.exp(-z)) # sigmoid function
|
||||
|
||||
# Compute the gradient of loss w.r.t. theta
|
||||
error = predictions - y_train # vector of (pred - true) for each example
|
||||
grad = (X_train.T.dot(error)) / m_train # gradient vector
|
||||
|
||||
# Update weights
|
||||
theta -= learning_rate * grad
|
||||
|
||||
# (Optional) compute and print loss every 50 epochs for monitoring
|
||||
if epoch % 50 == 0:
|
||||
# Binary cross-entropy loss
|
||||
loss = -np.mean(y_train * np.log(predictions + 1e-15) +
|
||||
(1 - y_train) * np.log(1 - predictions + 1e-15))
|
||||
print(f"Epoch {epoch}: Training loss = {loss:.4f}")
|
||||
|
||||
# 5. Evaluate the model on training and test data
|
||||
# Predict probabilities for test set and classify as 1 if sigmoid >= 0.5
|
||||
# (We can equivalently check linear term >= 0, since sigma(x)>=0.5 iff x>=0)
|
||||
train_prob = 1 / (1 + np.exp(-X_train.dot(theta)))
|
||||
test_prob = 1 / (1 + np.exp(-X_test.dot(theta)))
|
||||
train_pred = (train_prob >= 0.5).astype(int)
|
||||
test_pred = (test_prob >= 0.5).astype(int)
|
||||
|
||||
# Calculate accuracy
|
||||
train_accuracy = (train_pred == y_train).mean()
|
||||
test_accuracy = (test_pred == y_test).mean()
|
||||
|
||||
# Calculate final loss on training set for reference
|
||||
final_loss = -np.mean(y_train * np.log(train_prob + 1e-15) +
|
||||
(1 - y_train) * np.log(1 - train_prob + 1e-15))
|
||||
|
||||
# Print performance metrics
|
||||
print(f"Final Training Loss: {final_loss:.4f}")
|
||||
print(f"Training Accuracy: {train_accuracy * 100:.2f}%")
|
||||
print(f"Test Accuracy: {test_accuracy * 100:.2f}%")
|
||||
!ec
|
||||
|
||||
|
||||
Explanation: We initialize the weight vector theta (including a bias term) to zeros. In each training epoch, we compute the logistic predictions for all training examples and then update the weights using the gradient of the cross-entropy loss. The learning rate and number of epochs are set to fixed values (which can be tuned). We periodically print the training loss to ensure the model is learning (decreasing loss). After training, we obtain predicted probabilities on the train and test sets, convert them to class labels (threshold 0.5), and compute the accuracy.
|
||||
|
||||
!split
|
||||
===== Model performance and results =====
|
||||
|
||||
|
||||
After training, the code prints out the final loss and accuracies. You should observe that the training and test accuracy are around 78–80% for this logistic model.
|
||||
|
||||
This performance is in line with expectations. The dataset’s class imbalance means that always predicting “no default” would already achieve ~77.9% accuracy . Our logistic regression does slightly better, around 79–81% accuracy, by identifying some of the default cases correctly. (In one reference implementation, logistic regression achieved ~81.3% accuracy on a test set .)
|
||||
|
||||
The model’s loss converges to a reasonable value, indicating the gradient descent optimization was successful.
|
||||
|
||||
Note: The relatively modest improvement over the baseline is due to the imbalance and the limited complexity of a linear model. In practice, one could improve default prediction by using more complex models or addressing the imbalance (e.g. with oversampling or adjusting the classification threshold). Nonetheless, the above example demonstrates the complete process: data loading, preprocessing, model training, and evaluation, fulfilling the binary classification task on the Taiwan credit card default dataset.
|
||||
|
||||
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 1.1 MiB |
Binary file not shown.
|
After Width: | Height: | Size: 179 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 156 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 33 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 183 KiB |
@@ -0,0 +1,301 @@
|
||||
TrueLabel,PredictedLabel
|
||||
0,0
|
||||
0,1
|
||||
0,0
|
||||
0,0
|
||||
0,1
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,1
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,1
|
||||
0,0
|
||||
0,2
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
0,0
|
||||
1,1
|
||||
1,1
|
||||
1,0
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,0
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,0
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
1,0
|
||||
1,1
|
||||
1,1
|
||||
1,0
|
||||
1,1
|
||||
1,1
|
||||
1,1
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
2,2
|
||||
|
+874
-734
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user