diff --git a/doc/pub/week47/html/._week47-bs000.html b/doc/pub/week47/html/._week47-bs000.html
index e96c3a089..f6f7353f5 100644
--- a/doc/pub/week47/html/._week47-bs000.html
+++ b/doc/pub/week47/html/._week47-bs000.html
@@ -53,31 +53,30 @@ Automatically generated HTML file from DocOnce source
('Getting into the details', 2, None, '___sec9'),
('First attempt at a minimization approach', 2, None, '___sec10'),
('Solving the equations', 2, None, '___sec11'),
- ('Code Example', 2, None, '___sec12'),
- ('Problems with the Simpler Approach', 2, None, '___sec13'),
- ('A better approach', 2, None, '___sec14'),
+ ('Can we code this?', 2, None, '___sec12'),
+ ('A better approach', 2, None, '___sec13'),
('A quick Reminder on Lagrangian Multipliers',
2,
None,
- '___sec15'),
- ('Adding the Multiplier', 2, None, '___sec16'),
- ('Setting up the Problem', 2, None, '___sec17'),
- ('The problem to solve', 2, None, '___sec18'),
- ('The last steps', 2, None, '___sec19'),
- ('A soft classifier', 2, None, '___sec20'),
- ('Soft optmization problem', 2, None, '___sec21'),
- ('Kernels and non-linearity', 2, None, '___sec22'),
- ('The equations', 2, None, '___sec23'),
- ('The problem to solve', 2, None, '___sec24'),
- ("Different kernels and Mercer's theorem", 2, None, '___sec25'),
- ('The moons example', 2, None, '___sec26'),
+ '___sec14'),
+ ('Adding the Multiplier', 2, None, '___sec15'),
+ ('Setting up the Problem', 2, None, '___sec16'),
+ ('The problem to solve', 2, None, '___sec17'),
+ ('The last steps', 2, None, '___sec18'),
+ ('A soft classifier', 2, None, '___sec19'),
+ ('Soft optmization problem', 2, None, '___sec20'),
+ ('Kernels and non-linearity', 2, None, '___sec21'),
+ ('The equations', 2, None, '___sec22'),
+ ('The problem to solve', 2, None, '___sec23'),
+ ("Different kernels and Mercer's theorem", 2, None, '___sec24'),
+ ('The moons example', 2, None, '___sec25'),
('Mathematical optimization of convex functions',
2,
None,
- '___sec27'),
- ('How do we solve these problems?', 2, None, '___sec28'),
- ('A simple example', 2, None, '___sec29'),
- ('Back to the more realistic cases', 2, None, '___sec30')]}
+ '___sec26'),
+ ('How do we solve these problems?', 2, None, '___sec27'),
+ ('A simple example', 2, None, '___sec28'),
+ ('Back to the more realistic cases', 2, None, '___sec29')]}
end of tocinfo -->
@@ -127,25 +126,24 @@ MathJax.Hub.Config({
Getting into the details
First attempt at a minimization approach
Solving the equations
- Code Example
- Problems with the Simpler Approach
- A better approach
- A quick Reminder on Lagrangian Multipliers
- Adding the Multiplier
- Setting up the Problem
- The problem to solve
- The last steps
- A soft classifier
- Soft optmization problem
- Kernels and non-linearity
- The equations
- The problem to solve
- Different kernels and Mercer's theorem
- The moons example
- Mathematical optimization of convex functions
- How do we solve these problems?
- A simple example
- Back to the more realistic cases
+ Can we code this?
+ A better approach
+ A quick Reminder on Lagrangian Multipliers
+ Adding the Multiplier
+ Setting up the Problem
+ The problem to solve
+ The last steps
+ A soft classifier
+ Soft optmization problem
+ Kernels and non-linearity
+ The equations
+ The problem to solve
+ Different kernels and Mercer's theorem
+ The moons example
+ Mathematical optimization of convex functions
+ How do we solve these problems?
+ A simple example
+ Back to the more realistic cases
@@ -204,7 +202,7 @@ MathJax.Hub.Config({
9
10
...
- 32
+ 31
»
diff --git a/doc/pub/week47/html/._week47-bs001.html b/doc/pub/week47/html/._week47-bs001.html
index 91fe9b148..5ad61894f 100644
--- a/doc/pub/week47/html/._week47-bs001.html
+++ b/doc/pub/week47/html/._week47-bs001.html
@@ -53,31 +53,30 @@ Automatically generated HTML file from DocOnce source
('Getting into the details', 2, None, '___sec9'),
('First attempt at a minimization approach', 2, None, '___sec10'),
('Solving the equations', 2, None, '___sec11'),
- ('Code Example', 2, None, '___sec12'),
- ('Problems with the Simpler Approach', 2, None, '___sec13'),
- ('A better approach', 2, None, '___sec14'),
+ ('Can we code this?', 2, None, '___sec12'),
+ ('A better approach', 2, None, '___sec13'),
('A quick Reminder on Lagrangian Multipliers',
2,
None,
- '___sec15'),
- ('Adding the Multiplier', 2, None, '___sec16'),
- ('Setting up the Problem', 2, None, '___sec17'),
- ('The problem to solve', 2, None, '___sec18'),
- ('The last steps', 2, None, '___sec19'),
- ('A soft classifier', 2, None, '___sec20'),
- ('Soft optmization problem', 2, None, '___sec21'),
- ('Kernels and non-linearity', 2, None, '___sec22'),
- ('The equations', 2, None, '___sec23'),
- ('The problem to solve', 2, None, '___sec24'),
- ("Different kernels and Mercer's theorem", 2, None, '___sec25'),
- ('The moons example', 2, None, '___sec26'),
+ '___sec14'),
+ ('Adding the Multiplier', 2, None, '___sec15'),
+ ('Setting up the Problem', 2, None, '___sec16'),
+ ('The problem to solve', 2, None, '___sec17'),
+ ('The last steps', 2, None, '___sec18'),
+ ('A soft classifier', 2, None, '___sec19'),
+ ('Soft optmization problem', 2, None, '___sec20'),
+ ('Kernels and non-linearity', 2, None, '___sec21'),
+ ('The equations', 2, None, '___sec22'),
+ ('The problem to solve', 2, None, '___sec23'),
+ ("Different kernels and Mercer's theorem", 2, None, '___sec24'),
+ ('The moons example', 2, None, '___sec25'),
('Mathematical optimization of convex functions',
2,
None,
- '___sec27'),
- ('How do we solve these problems?', 2, None, '___sec28'),
- ('A simple example', 2, None, '___sec29'),
- ('Back to the more realistic cases', 2, None, '___sec30')]}
+ '___sec26'),
+ ('How do we solve these problems?', 2, None, '___sec27'),
+ ('A simple example', 2, None, '___sec28'),
+ ('Back to the more realistic cases', 2, None, '___sec29')]}
end of tocinfo -->
@@ -127,25 +126,24 @@ MathJax.Hub.Config({
Getting into the details
First attempt at a minimization approach
Solving the equations
- Code Example
- Problems with the Simpler Approach
- A better approach
- A quick Reminder on Lagrangian Multipliers
- Adding the Multiplier
- Setting up the Problem
- The problem to solve
- The last steps
- A soft classifier
- Soft optmization problem
- Kernels and non-linearity
- The equations
- The problem to solve
- Different kernels and Mercer's theorem
- The moons example
- Mathematical optimization of convex functions
- How do we solve these problems?
- A simple example
- Back to the more realistic cases
+ Can we code this?
+ A better approach
+ A quick Reminder on Lagrangian Multipliers
+ Adding the Multiplier
+ Setting up the Problem
+ The problem to solve
+ The last steps
+ A soft classifier
+ Soft optmization problem
+ Kernels and non-linearity
+ The equations
+ The problem to solve
+ Different kernels and Mercer's theorem
+ The moons example
+ Mathematical optimization of convex functions
+ How do we solve these problems?
+ A simple example
+ Back to the more realistic cases
@@ -190,7 +188,7 @@ Geron's chapter 5. Chapter 12 (sections 12.1-12.3 are the most relevant ones) o
10
11
...
- 32
+ 31
»
diff --git a/doc/pub/week47/html/._week47-bs002.html b/doc/pub/week47/html/._week47-bs002.html
index dfb2ca5de..76db46a10 100644
--- a/doc/pub/week47/html/._week47-bs002.html
+++ b/doc/pub/week47/html/._week47-bs002.html
@@ -53,31 +53,30 @@ Automatically generated HTML file from DocOnce source
('Getting into the details', 2, None, '___sec9'),
('First attempt at a minimization approach', 2, None, '___sec10'),
('Solving the equations', 2, None, '___sec11'),
- ('Code Example', 2, None, '___sec12'),
- ('Problems with the Simpler Approach', 2, None, '___sec13'),
- ('A better approach', 2, None, '___sec14'),
+ ('Can we code this?', 2, None, '___sec12'),
+ ('A better approach', 2, None, '___sec13'),
('A quick Reminder on Lagrangian Multipliers',
2,
None,
- '___sec15'),
- ('Adding the Multiplier', 2, None, '___sec16'),
- ('Setting up the Problem', 2, None, '___sec17'),
- ('The problem to solve', 2, None, '___sec18'),
- ('The last steps', 2, None, '___sec19'),
- ('A soft classifier', 2, None, '___sec20'),
- ('Soft optmization problem', 2, None, '___sec21'),
- ('Kernels and non-linearity', 2, None, '___sec22'),
- ('The equations', 2, None, '___sec23'),
- ('The problem to solve', 2, None, '___sec24'),
- ("Different kernels and Mercer's theorem", 2, None, '___sec25'),
- ('The moons example', 2, None, '___sec26'),
+ '___sec14'),
+ ('Adding the Multiplier', 2, None, '___sec15'),
+ ('Setting up the Problem', 2, None, '___sec16'),
+ ('The problem to solve', 2, None, '___sec17'),
+ ('The last steps', 2, None, '___sec18'),
+ ('A soft classifier', 2, None, '___sec19'),
+ ('Soft optmization problem', 2, None, '___sec20'),
+ ('Kernels and non-linearity', 2, None, '___sec21'),
+ ('The equations', 2, None, '___sec22'),
+ ('The problem to solve', 2, None, '___sec23'),
+ ("Different kernels and Mercer's theorem", 2, None, '___sec24'),
+ ('The moons example', 2, None, '___sec25'),
('Mathematical optimization of convex functions',
2,
None,
- '___sec27'),
- ('How do we solve these problems?', 2, None, '___sec28'),
- ('A simple example', 2, None, '___sec29'),
- ('Back to the more realistic cases', 2, None, '___sec30')]}
+ '___sec26'),
+ ('How do we solve these problems?', 2, None, '___sec27'),
+ ('A simple example', 2, None, '___sec28'),
+ ('Back to the more realistic cases', 2, None, '___sec29')]}
end of tocinfo -->
@@ -127,25 +126,24 @@ MathJax.Hub.Config({
Getting into the details
First attempt at a minimization approach
Solving the equations
- Code Example
- Problems with the Simpler Approach
- A better approach
- A quick Reminder on Lagrangian Multipliers
- Adding the Multiplier
- Setting up the Problem
- The problem to solve
- The last steps
- A soft classifier
- Soft optmization problem
- Kernels and non-linearity
- The equations
- The problem to solve
- Different kernels and Mercer's theorem
- The moons example
- Mathematical optimization of convex functions
- How do we solve these problems?
- A simple example
- Back to the more realistic cases
+ Can we code this?
+ A better approach
+ A quick Reminder on Lagrangian Multipliers
+ Adding the Multiplier
+ Setting up the Problem
+ The problem to solve
+ The last steps
+ A soft classifier
+ Soft optmization problem
+ Kernels and non-linearity
+ The equations
+ The problem to solve
+ Different kernels and Mercer's theorem
+ The moons example
+ Mathematical optimization of convex functions
+ How do we solve these problems?
+ A simple example
+ Back to the more realistic cases
@@ -184,7 +182,7 @@ We start with our final topic this semester, Support Vector Machines
11
12
...
- 32
+ 31
»
diff --git a/doc/pub/week47/html/._week47-bs003.html b/doc/pub/week47/html/._week47-bs003.html
index 6e19a08f3..4ab75b994 100644
--- a/doc/pub/week47/html/._week47-bs003.html
+++ b/doc/pub/week47/html/._week47-bs003.html
@@ -53,31 +53,30 @@ Automatically generated HTML file from DocOnce source
('Getting into the details', 2, None, '___sec9'),
('First attempt at a minimization approach', 2, None, '___sec10'),
('Solving the equations', 2, None, '___sec11'),
- ('Code Example', 2, None, '___sec12'),
- ('Problems with the Simpler Approach', 2, None, '___sec13'),
- ('A better approach', 2, None, '___sec14'),
+ ('Can we code this?', 2, None, '___sec12'),
+ ('A better approach', 2, None, '___sec13'),
('A quick Reminder on Lagrangian Multipliers',
2,
None,
- '___sec15'),
- ('Adding the Multiplier', 2, None, '___sec16'),
- ('Setting up the Problem', 2, None, '___sec17'),
- ('The problem to solve', 2, None, '___sec18'),
- ('The last steps', 2, None, '___sec19'),
- ('A soft classifier', 2, None, '___sec20'),
- ('Soft optmization problem', 2, None, '___sec21'),
- ('Kernels and non-linearity', 2, None, '___sec22'),
- ('The equations', 2, None, '___sec23'),
- ('The problem to solve', 2, None, '___sec24'),
- ("Different kernels and Mercer's theorem", 2, None, '___sec25'),
- ('The moons example', 2, None, '___sec26'),
+ '___sec14'),
+ ('Adding the Multiplier', 2, None, '___sec15'),
+ ('Setting up the Problem', 2, None, '___sec16'),
+ ('The problem to solve', 2, None, '___sec17'),
+ ('The last steps', 2, None, '___sec18'),
+ ('A soft classifier', 2, None, '___sec19'),
+ ('Soft optmization problem', 2, None, '___sec20'),
+ ('Kernels and non-linearity', 2, None, '___sec21'),
+ ('The equations', 2, None, '___sec22'),
+ ('The problem to solve', 2, None, '___sec23'),
+ ("Different kernels and Mercer's theorem", 2, None, '___sec24'),
+ ('The moons example', 2, None, '___sec25'),
('Mathematical optimization of convex functions',
2,
None,
- '___sec27'),
- ('How do we solve these problems?', 2, None, '___sec28'),
- ('A simple example', 2, None, '___sec29'),
- ('Back to the more realistic cases', 2, None, '___sec30')]}
+ '___sec26'),
+ ('How do we solve these problems?', 2, None, '___sec27'),
+ ('A simple example', 2, None, '___sec28'),
+ ('Back to the more realistic cases', 2, None, '___sec29')]}
end of tocinfo -->
@@ -127,25 +126,24 @@ MathJax.Hub.Config({
Getting into the details
First attempt at a minimization approach
Solving the equations
- Code Example
- Problems with the Simpler Approach
- A better approach
- A quick Reminder on Lagrangian Multipliers
- Adding the Multiplier
- Setting up the Problem
- The problem to solve
- The last steps
- A soft classifier
- Soft optmization problem
- Kernels and non-linearity
- The equations
- The problem to solve
- Different kernels and Mercer's theorem
- The moons example
- Mathematical optimization of convex functions
- How do we solve these problems?
- A simple example
- Back to the more realistic cases
+ Can we code this?
+ A better approach
+ A quick Reminder on Lagrangian Multipliers
+ Adding the Multiplier
+ Setting up the Problem
+ The problem to solve
+ The last steps
+ A soft classifier
+ Soft optmization problem
+ Kernels and non-linearity
+ The equations
+ The problem to solve
+ Different kernels and Mercer's theorem
+ The moons example
+ Mathematical optimization of convex functions
+ How do we solve these problems?
+ A simple example
+ Back to the more realistic cases
@@ -185,7 +183,7 @@ Friday's lecture is split in two parts. The first lecture is deveoted to a prese
12
13
...
- 32
+ 31
»
diff --git a/doc/pub/week47/html/._week47-bs004.html b/doc/pub/week47/html/._week47-bs004.html
index b49fab415..ae63e1737 100644
--- a/doc/pub/week47/html/._week47-bs004.html
+++ b/doc/pub/week47/html/._week47-bs004.html
@@ -53,31 +53,30 @@ Automatically generated HTML file from DocOnce source
('Getting into the details', 2, None, '___sec9'),
('First attempt at a minimization approach', 2, None, '___sec10'),
('Solving the equations', 2, None, '___sec11'),
- ('Code Example', 2, None, '___sec12'),
- ('Problems with the Simpler Approach', 2, None, '___sec13'),
- ('A better approach', 2, None, '___sec14'),
+ ('Can we code this?', 2, None, '___sec12'),
+ ('A better approach', 2, None, '___sec13'),
('A quick Reminder on Lagrangian Multipliers',
2,
None,
- '___sec15'),
- ('Adding the Multiplier', 2, None, '___sec16'),
- ('Setting up the Problem', 2, None, '___sec17'),
- ('The problem to solve', 2, None, '___sec18'),
- ('The last steps', 2, None, '___sec19'),
- ('A soft classifier', 2, None, '___sec20'),
- ('Soft optmization problem', 2, None, '___sec21'),
- ('Kernels and non-linearity', 2, None, '___sec22'),
- ('The equations', 2, None, '___sec23'),
- ('The problem to solve', 2, None, '___sec24'),
- ("Different kernels and Mercer's theorem", 2, None, '___sec25'),
- ('The moons example', 2, None, '___sec26'),
+ '___sec14'),
+ ('Adding the Multiplier', 2, None, '___sec15'),
+ ('Setting up the Problem', 2, None, '___sec16'),
+ ('The problem to solve', 2, None, '___sec17'),
+ ('The last steps', 2, None, '___sec18'),
+ ('A soft classifier', 2, None, '___sec19'),
+ ('Soft optmization problem', 2, None, '___sec20'),
+ ('Kernels and non-linearity', 2, None, '___sec21'),
+ ('The equations', 2, None, '___sec22'),
+ ('The problem to solve', 2, None, '___sec23'),
+ ("Different kernels and Mercer's theorem", 2, None, '___sec24'),
+ ('The moons example', 2, None, '___sec25'),
('Mathematical optimization of convex functions',
2,
None,
- '___sec27'),
- ('How do we solve these problems?', 2, None, '___sec28'),
- ('A simple example', 2, None, '___sec29'),
- ('Back to the more realistic cases', 2, None, '___sec30')]}
+ '___sec26'),
+ ('How do we solve these problems?', 2, None, '___sec27'),
+ ('A simple example', 2, None, '___sec28'),
+ ('Back to the more realistic cases', 2, None, '___sec29')]}
end of tocinfo -->
@@ -127,25 +126,24 @@ MathJax.Hub.Config({
Getting into the details
First attempt at a minimization approach
Solving the equations
- Code Example
- Problems with the Simpler Approach
- A better approach
- A quick Reminder on Lagrangian Multipliers
- Adding the Multiplier
- Setting up the Problem
- The problem to solve
- The last steps
- A soft classifier
- Soft optmization problem
- Kernels and non-linearity
- The equations
- The problem to solve
- Different kernels and Mercer's theorem
- The moons example
- Mathematical optimization of convex functions
- How do we solve these problems?
- A simple example
- Back to the more realistic cases
+ Can we code this?
+ A better approach
+ A quick Reminder on Lagrangian Multipliers
+ Adding the Multiplier
+ Setting up the Problem
+ The problem to solve
+ The last steps
+ A soft classifier
+ Soft optmization problem
+ Kernels and non-linearity
+ The equations
+ The problem to solve
+ Different kernels and Mercer's theorem
+ The moons example
+ Mathematical optimization of convex functions
+ How do we solve these problems?
+ A simple example
+ Back to the more realistic cases
@@ -196,7 +194,7 @@ Here are the various projects that will be presented during the first lecture (a
13
14
...
- 32
+ 31
»
diff --git a/doc/pub/week47/html/._week47-bs005.html b/doc/pub/week47/html/._week47-bs005.html
index caac38707..3becf03de 100644
--- a/doc/pub/week47/html/._week47-bs005.html
+++ b/doc/pub/week47/html/._week47-bs005.html
@@ -53,31 +53,30 @@ Automatically generated HTML file from DocOnce source
('Getting into the details', 2, None, '___sec9'),
('First attempt at a minimization approach', 2, None, '___sec10'),
('Solving the equations', 2, None, '___sec11'),
- ('Code Example', 2, None, '___sec12'),
- ('Problems with the Simpler Approach', 2, None, '___sec13'),
- ('A better approach', 2, None, '___sec14'),
+ ('Can we code this?', 2, None, '___sec12'),
+ ('A better approach', 2, None, '___sec13'),
('A quick Reminder on Lagrangian Multipliers',
2,
None,
- '___sec15'),
- ('Adding the Multiplier', 2, None, '___sec16'),
- ('Setting up the Problem', 2, None, '___sec17'),
- ('The problem to solve', 2, None, '___sec18'),
- ('The last steps', 2, None, '___sec19'),
- ('A soft classifier', 2, None, '___sec20'),
- ('Soft optmization problem', 2, None, '___sec21'),
- ('Kernels and non-linearity', 2, None, '___sec22'),
- ('The equations', 2, None, '___sec23'),
- ('The problem to solve', 2, None, '___sec24'),
- ("Different kernels and Mercer's theorem", 2, None, '___sec25'),
- ('The moons example', 2, None, '___sec26'),
+ '___sec14'),
+ ('Adding the Multiplier', 2, None, '___sec15'),
+ ('Setting up the Problem', 2, None, '___sec16'),
+ ('The problem to solve', 2, None, '___sec17'),
+ ('The last steps', 2, None, '___sec18'),
+ ('A soft classifier', 2, None, '___sec19'),
+ ('Soft optmization problem', 2, None, '___sec20'),
+ ('Kernels and non-linearity', 2, None, '___sec21'),
+ ('The equations', 2, None, '___sec22'),
+ ('The problem to solve', 2, None, '___sec23'),
+ ("Different kernels and Mercer's theorem", 2, None, '___sec24'),
+ ('The moons example', 2, None, '___sec25'),
('Mathematical optimization of convex functions',
2,
None,
- '___sec27'),
- ('How do we solve these problems?', 2, None, '___sec28'),
- ('A simple example', 2, None, '___sec29'),
- ('Back to the more realistic cases', 2, None, '___sec30')]}
+ '___sec26'),
+ ('How do we solve these problems?', 2, None, '___sec27'),
+ ('A simple example', 2, None, '___sec28'),
+ ('Back to the more realistic cases', 2, None, '___sec29')]}
end of tocinfo -->
@@ -127,25 +126,24 @@ MathJax.Hub.Config({
Getting into the details
First attempt at a minimization approach
Solving the equations
- Code Example
- Problems with the Simpler Approach
- A better approach
- A quick Reminder on Lagrangian Multipliers
- Adding the Multiplier
- Setting up the Problem
- The problem to solve
- The last steps
- A soft classifier
- Soft optmization problem
- Kernels and non-linearity
- The equations
- The problem to solve
- Different kernels and Mercer's theorem
- The moons example
- Mathematical optimization of convex functions
- How do we solve these problems?
- A simple example
- Back to the more realistic cases
+ Can we code this?
+ A better approach
+ A quick Reminder on Lagrangian Multipliers
+ Adding the Multiplier
+ Setting up the Problem
+ The problem to solve
+ The last steps
+ A soft classifier
+ Soft optmization problem
+ Kernels and non-linearity
+ The equations
+ The problem to solve
+ Different kernels and Mercer's theorem
+ The moons example
+ Mathematical optimization of convex functions
+ How do we solve these problems?
+ A simple example
+ Back to the more realistic cases
@@ -212,7 +210,7 @@ unlikely that we can separate classes easily by say straight lines.
14
15
...
- 32
+ 31
»
diff --git a/doc/pub/week47/html/._week47-bs006.html b/doc/pub/week47/html/._week47-bs006.html
index 8460a426e..eb9473a12 100644
--- a/doc/pub/week47/html/._week47-bs006.html
+++ b/doc/pub/week47/html/._week47-bs006.html
@@ -53,31 +53,30 @@ Automatically generated HTML file from DocOnce source
('Getting into the details', 2, None, '___sec9'),
('First attempt at a minimization approach', 2, None, '___sec10'),
('Solving the equations', 2, None, '___sec11'),
- ('Code Example', 2, None, '___sec12'),
- ('Problems with the Simpler Approach', 2, None, '___sec13'),
- ('A better approach', 2, None, '___sec14'),
+ ('Can we code this?', 2, None, '___sec12'),
+ ('A better approach', 2, None, '___sec13'),
('A quick Reminder on Lagrangian Multipliers',
2,
None,
- '___sec15'),
- ('Adding the Multiplier', 2, None, '___sec16'),
- ('Setting up the Problem', 2, None, '___sec17'),
- ('The problem to solve', 2, None, '___sec18'),
- ('The last steps', 2, None, '___sec19'),
- ('A soft classifier', 2, None, '___sec20'),
- ('Soft optmization problem', 2, None, '___sec21'),
- ('Kernels and non-linearity', 2, None, '___sec22'),
- ('The equations', 2, None, '___sec23'),
- ('The problem to solve', 2, None, '___sec24'),
- ("Different kernels and Mercer's theorem", 2, None, '___sec25'),
- ('The moons example', 2, None, '___sec26'),
+ '___sec14'),
+ ('Adding the Multiplier', 2, None, '___sec15'),
+ ('Setting up the Problem', 2, None, '___sec16'),
+ ('The problem to solve', 2, None, '___sec17'),
+ ('The last steps', 2, None, '___sec18'),
+ ('A soft classifier', 2, None, '___sec19'),
+ ('Soft optmization problem', 2, None, '___sec20'),
+ ('Kernels and non-linearity', 2, None, '___sec21'),
+ ('The equations', 2, None, '___sec22'),
+ ('The problem to solve', 2, None, '___sec23'),
+ ("Different kernels and Mercer's theorem", 2, None, '___sec24'),
+ ('The moons example', 2, None, '___sec25'),
('Mathematical optimization of convex functions',
2,
None,
- '___sec27'),
- ('How do we solve these problems?', 2, None, '___sec28'),
- ('A simple example', 2, None, '___sec29'),
- ('Back to the more realistic cases', 2, None, '___sec30')]}
+ '___sec26'),
+ ('How do we solve these problems?', 2, None, '___sec27'),
+ ('A simple example', 2, None, '___sec28'),
+ ('Back to the more realistic cases', 2, None, '___sec29')]}
end of tocinfo -->
@@ -127,25 +126,24 @@ MathJax.Hub.Config({
Getting into the details
First attempt at a minimization approach
Solving the equations
- Code Example
- Problems with the Simpler Approach
- A better approach
- A quick Reminder on Lagrangian Multipliers
- Adding the Multiplier
- Setting up the Problem
- The problem to solve
- The last steps
- A soft classifier
- Soft optmization problem
- Kernels and non-linearity
- The equations
- The problem to solve
- Different kernels and Mercer's theorem
- The moons example
- Mathematical optimization of convex functions
- How do we solve these problems?
- A simple example
- Back to the more realistic cases
+ Can we code this?
+ A better approach
+ A quick Reminder on Lagrangian Multipliers
+ Adding the Multiplier
+ Setting up the Problem
+ The problem to solve
+ The last steps
+ A soft classifier
+ Soft optmization problem
+ Kernels and non-linearity
+ The equations
+ The problem to solve
+ Different kernels and Mercer's theorem
+ The moons example
+ Mathematical optimization of convex functions
+ How do we solve these problems?
+ A simple example
+ Back to the more realistic cases
@@ -267,7 +265,7 @@ plt.show()
15
16
...
- 32
+ 31
»
diff --git a/doc/pub/week47/html/._week47-bs007.html b/doc/pub/week47/html/._week47-bs007.html
index c8f856dc1..3fcd27301 100644
--- a/doc/pub/week47/html/._week47-bs007.html
+++ b/doc/pub/week47/html/._week47-bs007.html
@@ -53,31 +53,30 @@ Automatically generated HTML file from DocOnce source
('Getting into the details', 2, None, '___sec9'),
('First attempt at a minimization approach', 2, None, '___sec10'),
('Solving the equations', 2, None, '___sec11'),
- ('Code Example', 2, None, '___sec12'),
- ('Problems with the Simpler Approach', 2, None, '___sec13'),
- ('A better approach', 2, None, '___sec14'),
+ ('Can we code this?', 2, None, '___sec12'),
+ ('A better approach', 2, None, '___sec13'),
('A quick Reminder on Lagrangian Multipliers',
2,
None,
- '___sec15'),
- ('Adding the Multiplier', 2, None, '___sec16'),
- ('Setting up the Problem', 2, None, '___sec17'),
- ('The problem to solve', 2, None, '___sec18'),
- ('The last steps', 2, None, '___sec19'),
- ('A soft classifier', 2, None, '___sec20'),
- ('Soft optmization problem', 2, None, '___sec21'),
- ('Kernels and non-linearity', 2, None, '___sec22'),
- ('The equations', 2, None, '___sec23'),
- ('The problem to solve', 2, None, '___sec24'),
- ("Different kernels and Mercer's theorem", 2, None, '___sec25'),
- ('The moons example', 2, None, '___sec26'),
+ '___sec14'),
+ ('Adding the Multiplier', 2, None, '___sec15'),
+ ('Setting up the Problem', 2, None, '___sec16'),
+ ('The problem to solve', 2, None, '___sec17'),
+ ('The last steps', 2, None, '___sec18'),
+ ('A soft classifier', 2, None, '___sec19'),
+ ('Soft optmization problem', 2, None, '___sec20'),
+ ('Kernels and non-linearity', 2, None, '___sec21'),
+ ('The equations', 2, None, '___sec22'),
+ ('The problem to solve', 2, None, '___sec23'),
+ ("Different kernels and Mercer's theorem", 2, None, '___sec24'),
+ ('The moons example', 2, None, '___sec25'),
('Mathematical optimization of convex functions',
2,
None,
- '___sec27'),
- ('How do we solve these problems?', 2, None, '___sec28'),
- ('A simple example', 2, None, '___sec29'),
- ('Back to the more realistic cases', 2, None, '___sec30')]}
+ '___sec26'),
+ ('How do we solve these problems?', 2, None, '___sec27'),
+ ('A simple example', 2, None, '___sec28'),
+ ('Back to the more realistic cases', 2, None, '___sec29')]}
end of tocinfo -->
@@ -127,25 +126,24 @@ MathJax.Hub.Config({
Getting into the details
First attempt at a minimization approach
Solving the equations
- Code Example
- Problems with the Simpler Approach
- A better approach
- A quick Reminder on Lagrangian Multipliers
- Adding the Multiplier
- Setting up the Problem
- The problem to solve
- The last steps
- A soft classifier
- Soft optmization problem
- Kernels and non-linearity
- The equations
- The problem to solve
- Different kernels and Mercer's theorem
- The moons example
- Mathematical optimization of convex functions
- How do we solve these problems?
- A simple example
- Back to the more realistic cases
+ Can we code this?
+ A better approach
+ A quick Reminder on Lagrangian Multipliers
+ Adding the Multiplier
+ Setting up the Problem
+ The problem to solve
+ The last steps
+ A soft classifier
+ Soft optmization problem
+ Kernels and non-linearity
+ The equations
+ The problem to solve
+ Different kernels and Mercer's theorem
+ The moons example
+ Mathematical optimization of convex functions
+ How do we solve these problems?
+ A simple example
+ Back to the more realistic cases
@@ -212,7 +210,7 @@ $$
16
17
...
- 32
+ 31
»
diff --git a/doc/pub/week47/html/._week47-bs008.html b/doc/pub/week47/html/._week47-bs008.html
index 348dc4b0c..88a771945 100644
--- a/doc/pub/week47/html/._week47-bs008.html
+++ b/doc/pub/week47/html/._week47-bs008.html
@@ -53,31 +53,30 @@ Automatically generated HTML file from DocOnce source
('Getting into the details', 2, None, '___sec9'),
('First attempt at a minimization approach', 2, None, '___sec10'),
('Solving the equations', 2, None, '___sec11'),
- ('Code Example', 2, None, '___sec12'),
- ('Problems with the Simpler Approach', 2, None, '___sec13'),
- ('A better approach', 2, None, '___sec14'),
+ ('Can we code this?', 2, None, '___sec12'),
+ ('A better approach', 2, None, '___sec13'),
('A quick Reminder on Lagrangian Multipliers',
2,
None,
- '___sec15'),
- ('Adding the Multiplier', 2, None, '___sec16'),
- ('Setting up the Problem', 2, None, '___sec17'),
- ('The problem to solve', 2, None, '___sec18'),
- ('The last steps', 2, None, '___sec19'),
- ('A soft classifier', 2, None, '___sec20'),
- ('Soft optmization problem', 2, None, '___sec21'),
- ('Kernels and non-linearity', 2, None, '___sec22'),
- ('The equations', 2, None, '___sec23'),
- ('The problem to solve', 2, None, '___sec24'),
- ("Different kernels and Mercer's theorem", 2, None, '___sec25'),
- ('The moons example', 2, None, '___sec26'),
+ '___sec14'),
+ ('Adding the Multiplier', 2, None, '___sec15'),
+ ('Setting up the Problem', 2, None, '___sec16'),
+ ('The problem to solve', 2, None, '___sec17'),
+ ('The last steps', 2, None, '___sec18'),
+ ('A soft classifier', 2, None, '___sec19'),
+ ('Soft optmization problem', 2, None, '___sec20'),
+ ('Kernels and non-linearity', 2, None, '___sec21'),
+ ('The equations', 2, None, '___sec22'),
+ ('The problem to solve', 2, None, '___sec23'),
+ ("Different kernels and Mercer's theorem", 2, None, '___sec24'),
+ ('The moons example', 2, None, '___sec25'),
('Mathematical optimization of convex functions',
2,
None,
- '___sec27'),
- ('How do we solve these problems?', 2, None, '___sec28'),
- ('A simple example', 2, None, '___sec29'),
- ('Back to the more realistic cases', 2, None, '___sec30')]}
+ '___sec26'),
+ ('How do we solve these problems?', 2, None, '___sec27'),
+ ('A simple example', 2, None, '___sec28'),
+ ('Back to the more realistic cases', 2, None, '___sec29')]}
end of tocinfo -->
@@ -127,25 +126,24 @@ MathJax.Hub.Config({
Getting into the details
First attempt at a minimization approach
Solving the equations
- Code Example
- Problems with the Simpler Approach
- A better approach
- A quick Reminder on Lagrangian Multipliers
- Adding the Multiplier
- Setting up the Problem
- The problem to solve
- The last steps
- A soft classifier
- Soft optmization problem
- Kernels and non-linearity
- The equations
- The problem to solve
- Different kernels and Mercer's theorem
- The moons example
- Mathematical optimization of convex functions
- How do we solve these problems?
- A simple example
- Back to the more realistic cases
+ Can we code this?
+ A better approach
+ A quick Reminder on Lagrangian Multipliers
+ Adding the Multiplier
+ Setting up the Problem
+ The problem to solve
+ The last steps
+ A soft classifier
+ Soft optmization problem
+ Kernels and non-linearity
+ The equations
+ The problem to solve
+ Different kernels and Mercer's theorem
+ The moons example
+ Mathematical optimization of convex functions
+ How do we solve these problems?
+ A simple example
+ Back to the more realistic cases
@@ -224,7 +222,7 @@ When we try to separate hyperplanes, if it exists, we can use it to construct a
17
18
...
- 32
+ 31
»
diff --git a/doc/pub/week47/html/._week47-bs009.html b/doc/pub/week47/html/._week47-bs009.html
index f1c2ecb0e..8375de71d 100644
--- a/doc/pub/week47/html/._week47-bs009.html
+++ b/doc/pub/week47/html/._week47-bs009.html
@@ -53,31 +53,30 @@ Automatically generated HTML file from DocOnce source
('Getting into the details', 2, None, '___sec9'),
('First attempt at a minimization approach', 2, None, '___sec10'),
('Solving the equations', 2, None, '___sec11'),
- ('Code Example', 2, None, '___sec12'),
- ('Problems with the Simpler Approach', 2, None, '___sec13'),
- ('A better approach', 2, None, '___sec14'),
+ ('Can we code this?', 2, None, '___sec12'),
+ ('A better approach', 2, None, '___sec13'),
('A quick Reminder on Lagrangian Multipliers',
2,
None,
- '___sec15'),
- ('Adding the Multiplier', 2, None, '___sec16'),
- ('Setting up the Problem', 2, None, '___sec17'),
- ('The problem to solve', 2, None, '___sec18'),
- ('The last steps', 2, None, '___sec19'),
- ('A soft classifier', 2, None, '___sec20'),
- ('Soft optmization problem', 2, None, '___sec21'),
- ('Kernels and non-linearity', 2, None, '___sec22'),
- ('The equations', 2, None, '___sec23'),
- ('The problem to solve', 2, None, '___sec24'),
- ("Different kernels and Mercer's theorem", 2, None, '___sec25'),
- ('The moons example', 2, None, '___sec26'),
+ '___sec14'),
+ ('Adding the Multiplier', 2, None, '___sec15'),
+ ('Setting up the Problem', 2, None, '___sec16'),
+ ('The problem to solve', 2, None, '___sec17'),
+ ('The last steps', 2, None, '___sec18'),
+ ('A soft classifier', 2, None, '___sec19'),
+ ('Soft optmization problem', 2, None, '___sec20'),
+ ('Kernels and non-linearity', 2, None, '___sec21'),
+ ('The equations', 2, None, '___sec22'),
+ ('The problem to solve', 2, None, '___sec23'),
+ ("Different kernels and Mercer's theorem", 2, None, '___sec24'),
+ ('The moons example', 2, None, '___sec25'),
('Mathematical optimization of convex functions',
2,
None,
- '___sec27'),
- ('How do we solve these problems?', 2, None, '___sec28'),
- ('A simple example', 2, None, '___sec29'),
- ('Back to the more realistic cases', 2, None, '___sec30')]}
+ '___sec26'),
+ ('How do we solve these problems?', 2, None, '___sec27'),
+ ('A simple example', 2, None, '___sec28'),
+ ('Back to the more realistic cases', 2, None, '___sec29')]}
end of tocinfo -->
@@ -127,25 +126,24 @@ MathJax.Hub.Config({
Getting into the details
First attempt at a minimization approach
Solving the equations
- Code Example
- Problems with the Simpler Approach
- A better approach
- A quick Reminder on Lagrangian Multipliers
- Adding the Multiplier
- Setting up the Problem
- The problem to solve
- The last steps
- A soft classifier
- Soft optmization problem
- Kernels and non-linearity
- The equations
- The problem to solve
- Different kernels and Mercer's theorem
- The moons example
- Mathematical optimization of convex functions
- How do we solve these problems?
- A simple example
- Back to the more realistic cases
+ Can we code this?
+ A better approach
+ A quick Reminder on Lagrangian Multipliers
+ Adding the Multiplier
+ Setting up the Problem
+ The problem to solve
+ The last steps
+ A soft classifier
+ Soft optmization problem
+ Kernels and non-linearity
+ The equations
+ The problem to solve
+ Different kernels and Mercer's theorem
+ The moons example
+ Mathematical optimization of convex functions
+ How do we solve these problems?
+ A simple example
+ Back to the more realistic cases
@@ -210,7 +208,7 @@ for our data sample.
18
19
...
- 32
+ 31
»
diff --git a/doc/pub/week47/html/._week47-bs010.html b/doc/pub/week47/html/._week47-bs010.html
index 3027ccbfa..c05fb0d4f 100644
--- a/doc/pub/week47/html/._week47-bs010.html
+++ b/doc/pub/week47/html/._week47-bs010.html
@@ -53,31 +53,30 @@ Automatically generated HTML file from DocOnce source
('Getting into the details', 2, None, '___sec9'),
('First attempt at a minimization approach', 2, None, '___sec10'),
('Solving the equations', 2, None, '___sec11'),
- ('Code Example', 2, None, '___sec12'),
- ('Problems with the Simpler Approach', 2, None, '___sec13'),
- ('A better approach', 2, None, '___sec14'),
+ ('Can we code this?', 2, None, '___sec12'),
+ ('A better approach', 2, None, '___sec13'),
('A quick Reminder on Lagrangian Multipliers',
2,
None,
- '___sec15'),
- ('Adding the Multiplier', 2, None, '___sec16'),
- ('Setting up the Problem', 2, None, '___sec17'),
- ('The problem to solve', 2, None, '___sec18'),
- ('The last steps', 2, None, '___sec19'),
- ('A soft classifier', 2, None, '___sec20'),
- ('Soft optmization problem', 2, None, '___sec21'),
- ('Kernels and non-linearity', 2, None, '___sec22'),
- ('The equations', 2, None, '___sec23'),
- ('The problem to solve', 2, None, '___sec24'),
- ("Different kernels and Mercer's theorem", 2, None, '___sec25'),
- ('The moons example', 2, None, '___sec26'),
+ '___sec14'),
+ ('Adding the Multiplier', 2, None, '___sec15'),
+ ('Setting up the Problem', 2, None, '___sec16'),
+ ('The problem to solve', 2, None, '___sec17'),
+ ('The last steps', 2, None, '___sec18'),
+ ('A soft classifier', 2, None, '___sec19'),
+ ('Soft optmization problem', 2, None, '___sec20'),
+ ('Kernels and non-linearity', 2, None, '___sec21'),
+ ('The equations', 2, None, '___sec22'),
+ ('The problem to solve', 2, None, '___sec23'),
+ ("Different kernels and Mercer's theorem", 2, None, '___sec24'),
+ ('The moons example', 2, None, '___sec25'),
('Mathematical optimization of convex functions',
2,
None,
- '___sec27'),
- ('How do we solve these problems?', 2, None, '___sec28'),
- ('A simple example', 2, None, '___sec29'),
- ('Back to the more realistic cases', 2, None, '___sec30')]}
+ '___sec26'),
+ ('How do we solve these problems?', 2, None, '___sec27'),
+ ('A simple example', 2, None, '___sec28'),
+ ('Back to the more realistic cases', 2, None, '___sec29')]}
end of tocinfo -->
@@ -127,25 +126,24 @@ MathJax.Hub.Config({
Getting into the details
First attempt at a minimization approach
Solving the equations
- Code Example
- Problems with the Simpler Approach
- A better approach
- A quick Reminder on Lagrangian Multipliers
- Adding the Multiplier
- Setting up the Problem
- The problem to solve
- The last steps
- A soft classifier
- Soft optmization problem
- Kernels and non-linearity
- The equations
- The problem to solve
- Different kernels and Mercer's theorem
- The moons example
- Mathematical optimization of convex functions
- How do we solve these problems?
- A simple example
- Back to the more realistic cases
+ Can we code this?
+ A better approach
+ A quick Reminder on Lagrangian Multipliers
+ Adding the Multiplier
+ Setting up the Problem
+ The problem to solve
+ The last steps
+ A soft classifier
+ Soft optmization problem
+ Kernels and non-linearity
+ The equations
+ The problem to solve
+ Different kernels and Mercer's theorem
+ The moons example
+ Mathematical optimization of convex functions
+ How do we solve these problems?
+ A simple example
+ Back to the more realistic cases
@@ -206,7 +204,7 @@ $$
19
20
...
- 32
+ 31
»
diff --git a/doc/pub/week47/html/._week47-bs011.html b/doc/pub/week47/html/._week47-bs011.html
index a8da297a2..3a8197e79 100644
--- a/doc/pub/week47/html/._week47-bs011.html
+++ b/doc/pub/week47/html/._week47-bs011.html
@@ -53,31 +53,30 @@ Automatically generated HTML file from DocOnce source
('Getting into the details', 2, None, '___sec9'),
('First attempt at a minimization approach', 2, None, '___sec10'),
('Solving the equations', 2, None, '___sec11'),
- ('Code Example', 2, None, '___sec12'),
- ('Problems with the Simpler Approach', 2, None, '___sec13'),
- ('A better approach', 2, None, '___sec14'),
+ ('Can we code this?', 2, None, '___sec12'),
+ ('A better approach', 2, None, '___sec13'),
('A quick Reminder on Lagrangian Multipliers',
2,
None,
- '___sec15'),
- ('Adding the Multiplier', 2, None, '___sec16'),
- ('Setting up the Problem', 2, None, '___sec17'),
- ('The problem to solve', 2, None, '___sec18'),
- ('The last steps', 2, None, '___sec19'),
- ('A soft classifier', 2, None, '___sec20'),
- ('Soft optmization problem', 2, None, '___sec21'),
- ('Kernels and non-linearity', 2, None, '___sec22'),
- ('The equations', 2, None, '___sec23'),
- ('The problem to solve', 2, None, '___sec24'),
- ("Different kernels and Mercer's theorem", 2, None, '___sec25'),
- ('The moons example', 2, None, '___sec26'),
+ '___sec14'),
+ ('Adding the Multiplier', 2, None, '___sec15'),
+ ('Setting up the Problem', 2, None, '___sec16'),
+ ('The problem to solve', 2, None, '___sec17'),
+ ('The last steps', 2, None, '___sec18'),
+ ('A soft classifier', 2, None, '___sec19'),
+ ('Soft optmization problem', 2, None, '___sec20'),
+ ('Kernels and non-linearity', 2, None, '___sec21'),
+ ('The equations', 2, None, '___sec22'),
+ ('The problem to solve', 2, None, '___sec23'),
+ ("Different kernels and Mercer's theorem", 2, None, '___sec24'),
+ ('The moons example', 2, None, '___sec25'),
('Mathematical optimization of convex functions',
2,
None,
- '___sec27'),
- ('How do we solve these problems?', 2, None, '___sec28'),
- ('A simple example', 2, None, '___sec29'),
- ('Back to the more realistic cases', 2, None, '___sec30')]}
+ '___sec26'),
+ ('How do we solve these problems?', 2, None, '___sec27'),
+ ('A simple example', 2, None, '___sec28'),
+ ('Back to the more realistic cases', 2, None, '___sec29')]}
end of tocinfo -->
@@ -127,25 +126,24 @@ MathJax.Hub.Config({
Getting into the details
First attempt at a minimization approach
Solving the equations
- Code Example
- Problems with the Simpler Approach
- A better approach
- A quick Reminder on Lagrangian Multipliers
- Adding the Multiplier
- Setting up the Problem
- The problem to solve
- The last steps
- A soft classifier
- Soft optmization problem
- Kernels and non-linearity
- The equations
- The problem to solve
- Different kernels and Mercer's theorem
- The moons example
- Mathematical optimization of convex functions
- How do we solve these problems?
- A simple example
- Back to the more realistic cases
+ Can we code this?
+ A better approach
+ A quick Reminder on Lagrangian Multipliers
+ Adding the Multiplier
+ Setting up the Problem
+ The problem to solve
+ The last steps
+ A soft classifier
+ Soft optmization problem
+ Kernels and non-linearity
+ The equations
+ The problem to solve
+ Different kernels and Mercer's theorem
+ The moons example
+ Mathematical optimization of convex functions
+ How do we solve these problems?
+ A simple example
+ Back to the more realistic cases
@@ -209,7 +207,7 @@ $$
20
21
...
- 32
+ 31
»
diff --git a/doc/pub/week47/html/._week47-bs012.html b/doc/pub/week47/html/._week47-bs012.html
index 5efebf7b3..64e370743 100644
--- a/doc/pub/week47/html/._week47-bs012.html
+++ b/doc/pub/week47/html/._week47-bs012.html
@@ -53,31 +53,30 @@ Automatically generated HTML file from DocOnce source
('Getting into the details', 2, None, '___sec9'),
('First attempt at a minimization approach', 2, None, '___sec10'),
('Solving the equations', 2, None, '___sec11'),
- ('Code Example', 2, None, '___sec12'),
- ('Problems with the Simpler Approach', 2, None, '___sec13'),
- ('A better approach', 2, None, '___sec14'),
+ ('Can we code this?', 2, None, '___sec12'),
+ ('A better approach', 2, None, '___sec13'),
('A quick Reminder on Lagrangian Multipliers',
2,
None,
- '___sec15'),
- ('Adding the Multiplier', 2, None, '___sec16'),
- ('Setting up the Problem', 2, None, '___sec17'),
- ('The problem to solve', 2, None, '___sec18'),
- ('The last steps', 2, None, '___sec19'),
- ('A soft classifier', 2, None, '___sec20'),
- ('Soft optmization problem', 2, None, '___sec21'),
- ('Kernels and non-linearity', 2, None, '___sec22'),
- ('The equations', 2, None, '___sec23'),
- ('The problem to solve', 2, None, '___sec24'),
- ("Different kernels and Mercer's theorem", 2, None, '___sec25'),
- ('The moons example', 2, None, '___sec26'),
+ '___sec14'),
+ ('Adding the Multiplier', 2, None, '___sec15'),
+ ('Setting up the Problem', 2, None, '___sec16'),
+ ('The problem to solve', 2, None, '___sec17'),
+ ('The last steps', 2, None, '___sec18'),
+ ('A soft classifier', 2, None, '___sec19'),
+ ('Soft optmization problem', 2, None, '___sec20'),
+ ('Kernels and non-linearity', 2, None, '___sec21'),
+ ('The equations', 2, None, '___sec22'),
+ ('The problem to solve', 2, None, '___sec23'),
+ ("Different kernels and Mercer's theorem", 2, None, '___sec24'),
+ ('The moons example', 2, None, '___sec25'),
('Mathematical optimization of convex functions',
2,
None,
- '___sec27'),
- ('How do we solve these problems?', 2, None, '___sec28'),
- ('A simple example', 2, None, '___sec29'),
- ('Back to the more realistic cases', 2, None, '___sec30')]}
+ '___sec26'),
+ ('How do we solve these problems?', 2, None, '___sec27'),
+ ('A simple example', 2, None, '___sec28'),
+ ('Back to the more realistic cases', 2, None, '___sec29')]}
end of tocinfo -->
@@ -127,25 +126,24 @@ MathJax.Hub.Config({
Getting into the details
First attempt at a minimization approach
Solving the equations
- Code Example
- Problems with the Simpler Approach
- A better approach
- A quick Reminder on Lagrangian Multipliers
- Adding the Multiplier
- Setting up the Problem
- The problem to solve
- The last steps
- A soft classifier
- Soft optmization problem
- Kernels and non-linearity
- The equations
- The problem to solve
- Different kernels and Mercer's theorem
- The moons example
- Mathematical optimization of convex functions
- How do we solve these problems?
- A simple example
- Back to the more realistic cases
+ Can we code this?
+ A better approach
+ A quick Reminder on Lagrangian Multipliers
+ Adding the Multiplier
+ Setting up the Problem
+ The problem to solve
+ The last steps
+ A soft classifier
+ Soft optmization problem
+ Kernels and non-linearity
+ The equations
+ The problem to solve
+ Different kernels and Mercer's theorem
+ The moons example
+ Mathematical optimization of convex functions
+ How do we solve these problems?
+ A simple example
+ Back to the more realistic cases
@@ -202,7 +200,7 @@ where \( \eta \) is our by now well-known learning rate.
21
22
...
- 32
+ 31
»
diff --git a/doc/pub/week47/html/._week47-bs013.html b/doc/pub/week47/html/._week47-bs013.html
index f4f20dafc..c44f2ad49 100644
--- a/doc/pub/week47/html/._week47-bs013.html
+++ b/doc/pub/week47/html/._week47-bs013.html
@@ -53,31 +53,30 @@ Automatically generated HTML file from DocOnce source
('Getting into the details', 2, None, '___sec9'),
('First attempt at a minimization approach', 2, None, '___sec10'),
('Solving the equations', 2, None, '___sec11'),
- ('Code Example', 2, None, '___sec12'),
- ('Problems with the Simpler Approach', 2, None, '___sec13'),
- ('A better approach', 2, None, '___sec14'),
+ ('Can we code this?', 2, None, '___sec12'),
+ ('A better approach', 2, None, '___sec13'),
('A quick Reminder on Lagrangian Multipliers',
2,
None,
- '___sec15'),
- ('Adding the Multiplier', 2, None, '___sec16'),
- ('Setting up the Problem', 2, None, '___sec17'),
- ('The problem to solve', 2, None, '___sec18'),
- ('The last steps', 2, None, '___sec19'),
- ('A soft classifier', 2, None, '___sec20'),
- ('Soft optmization problem', 2, None, '___sec21'),
- ('Kernels and non-linearity', 2, None, '___sec22'),
- ('The equations', 2, None, '___sec23'),
- ('The problem to solve', 2, None, '___sec24'),
- ("Different kernels and Mercer's theorem", 2, None, '___sec25'),
- ('The moons example', 2, None, '___sec26'),
+ '___sec14'),
+ ('Adding the Multiplier', 2, None, '___sec15'),
+ ('Setting up the Problem', 2, None, '___sec16'),
+ ('The problem to solve', 2, None, '___sec17'),
+ ('The last steps', 2, None, '___sec18'),
+ ('A soft classifier', 2, None, '___sec19'),
+ ('Soft optmization problem', 2, None, '___sec20'),
+ ('Kernels and non-linearity', 2, None, '___sec21'),
+ ('The equations', 2, None, '___sec22'),
+ ('The problem to solve', 2, None, '___sec23'),
+ ("Different kernels and Mercer's theorem", 2, None, '___sec24'),
+ ('The moons example', 2, None, '___sec25'),
('Mathematical optimization of convex functions',
2,
None,
- '___sec27'),
- ('How do we solve these problems?', 2, None, '___sec28'),
- ('A simple example', 2, None, '___sec29'),
- ('Back to the more realistic cases', 2, None, '___sec30')]}
+ '___sec26'),
+ ('How do we solve these problems?', 2, None, '___sec27'),
+ ('A simple example', 2, None, '___sec28'),
+ ('Back to the more realistic cases', 2, None, '___sec29')]}
end of tocinfo -->
@@ -127,25 +126,24 @@ MathJax.Hub.Config({
Getting into the details
First attempt at a minimization approach
Solving the equations
- Code Example
- Problems with the Simpler Approach
- A better approach
- A quick Reminder on Lagrangian Multipliers
- Adding the Multiplier
- Setting up the Problem
- The problem to solve
- The last steps
- A soft classifier
- Soft optmization problem
- Kernels and non-linearity
- The equations
- The problem to solve
- Different kernels and Mercer's theorem
- The moons example
- Mathematical optimization of convex functions
- How do we solve these problems?
- A simple example
- Back to the more realistic cases
+ Can we code this?
+ A better approach
+ A quick Reminder on Lagrangian Multipliers
+ Adding the Multiplier
+ Setting up the Problem
+ The problem to solve
+ The last steps
+ A soft classifier
+ Soft optmization problem
+ Kernels and non-linearity
+ The equations
+ The problem to solve
+ Different kernels and Mercer's theorem
+ The moons example
+ Mathematical optimization of convex functions
+ How do we solve these problems?
+ A simple example
+ Back to the more realistic cases
@@ -161,17 +159,27 @@ MathJax.Hub.Config({
-Code Example
+Can we code this?
The equations we discussed above can be coded rather easily (the
-framework is similar to what we developed for logistic
-regression). We are going to set up a simple case with two classes only and we want to find a line which separates them the best possible way.
-
+framework is similar to what we developed for logistic regression). We
+can set up a simple case with two classes only and we want to find a
+line which separates them the best possible way.
+
+
+There are however problems with this approach, although it looks
+pretty straightforward to implement. When running a code for such a
+case we can easily end up with many diffeent lines which separate the
+two classes.
+
+
+For small
+gaps between the entries, we may also end up needing many iterations
+before the solutions converge and if the data cannot be separated
+properly into two distinct classes, we may not experience a converge
+at all.
-
-
@@ -198,7 +206,7 @@ regression). We are going to set up a simple case with two classes only and we w
22
23
...
- 32
+ 31
»
diff --git a/doc/pub/week47/html/._week47-bs014.html b/doc/pub/week47/html/._week47-bs014.html
index 5ae161f80..2fbfa74c4 100644
--- a/doc/pub/week47/html/._week47-bs014.html
+++ b/doc/pub/week47/html/._week47-bs014.html
@@ -53,31 +53,30 @@ Automatically generated HTML file from DocOnce source
('Getting into the details', 2, None, '___sec9'),
('First attempt at a minimization approach', 2, None, '___sec10'),
('Solving the equations', 2, None, '___sec11'),
- ('Code Example', 2, None, '___sec12'),
- ('Problems with the Simpler Approach', 2, None, '___sec13'),
- ('A better approach', 2, None, '___sec14'),
+ ('Can we code this?', 2, None, '___sec12'),
+ ('A better approach', 2, None, '___sec13'),
('A quick Reminder on Lagrangian Multipliers',
2,
None,
- '___sec15'),
- ('Adding the Multiplier', 2, None, '___sec16'),
- ('Setting up the Problem', 2, None, '___sec17'),
- ('The problem to solve', 2, None, '___sec18'),
- ('The last steps', 2, None, '___sec19'),
- ('A soft classifier', 2, None, '___sec20'),
- ('Soft optmization problem', 2, None, '___sec21'),
- ('Kernels and non-linearity', 2, None, '___sec22'),
- ('The equations', 2, None, '___sec23'),
- ('The problem to solve', 2, None, '___sec24'),
- ("Different kernels and Mercer's theorem", 2, None, '___sec25'),
- ('The moons example', 2, None, '___sec26'),
+ '___sec14'),
+ ('Adding the Multiplier', 2, None, '___sec15'),
+ ('Setting up the Problem', 2, None, '___sec16'),
+ ('The problem to solve', 2, None, '___sec17'),
+ ('The last steps', 2, None, '___sec18'),
+ ('A soft classifier', 2, None, '___sec19'),
+ ('Soft optmization problem', 2, None, '___sec20'),
+ ('Kernels and non-linearity', 2, None, '___sec21'),
+ ('The equations', 2, None, '___sec22'),
+ ('The problem to solve', 2, None, '___sec23'),
+ ("Different kernels and Mercer's theorem", 2, None, '___sec24'),
+ ('The moons example', 2, None, '___sec25'),
('Mathematical optimization of convex functions',
2,
None,
- '___sec27'),
- ('How do we solve these problems?', 2, None, '___sec28'),
- ('A simple example', 2, None, '___sec29'),
- ('Back to the more realistic cases', 2, None, '___sec30')]}
+ '___sec26'),
+ ('How do we solve these problems?', 2, None, '___sec27'),
+ ('A simple example', 2, None, '___sec28'),
+ ('Back to the more realistic cases', 2, None, '___sec29')]}
end of tocinfo -->
@@ -127,25 +126,24 @@ MathJax.Hub.Config({
Getting into the details
First attempt at a minimization approach
Solving the equations
- Code Example
- Problems with the Simpler Approach
- A better approach
- A quick Reminder on Lagrangian Multipliers
- Adding the Multiplier
- Setting up the Problem
- The problem to solve
- The last steps
- A soft classifier
- Soft optmization problem
- Kernels and non-linearity
- The equations
- The problem to solve
- Different kernels and Mercer's theorem
- The moons example
- Mathematical optimization of convex functions
- How do we solve these problems?
- A simple example
- Back to the more realistic cases
+ Can we code this?
+ A better approach
+ A quick Reminder on Lagrangian Multipliers
+ Adding the Multiplier
+ Setting up the Problem
+ The problem to solve
+ The last steps
+ A soft classifier
+ Soft optmization problem
+ Kernels and non-linearity
+ The equations
+ The problem to solve
+ Different kernels and Mercer's theorem
+ The moons example
+ Mathematical optimization of convex functions
+ How do we solve these problems?
+ A simple example
+ Back to the more realistic cases
@@ -161,18 +159,44 @@ MathJax.Hub.Config({
-Problems with the Simpler Approach
+A better approach
-There are however problems with this approach, although it looks
-pretty straightforward to implement. When running the above code, we see that we can easily end up with many diffeent lines which separate the two classes.
+A better approach is rather to try to define a large margin between
+the two classes (if they are well separated from the beginning).
-For small
-gaps between the entries, we may also end up needing many iterations
-before the solutions converge and if the data cannot be separated
-properly into two distinct classes, we may not experience a converge
-at all.
+Thus, we wish to find a margin \( M \) with \( \boldsymbol{w} \) normalized to
+\( \vert\vert \boldsymbol{w}\vert\vert =1 \) subject to the condition
+
+$$
+y_i(\boldsymbol{w}^T\boldsymbol{x}_i+b) \geq M \hspace{0.1cm}\forall i=1,2,\dots, p.
+$$
+
+All points are thus at a signed distance from the decision boundary defined by the line \( L \). The parameters \( b \) and \( w_1 \) and \( w_2 \) define this line.
+
+
+We seek thus the largest value \( M \) defined by
+$$
+\frac{1}{\vert \vert \boldsymbol{w}\vert\vert}y_i(\boldsymbol{w}^T\boldsymbol{x}_i+b) \geq M \hspace{0.1cm}\forall i=1,2,\dots, n,
+$$
+
+or just
+$$
+y_i(\boldsymbol{w}^T\boldsymbol{x}_i+b) \geq M\vert \vert \boldsymbol{w}\vert\vert \hspace{0.1cm}\forall i.
+$$
+
+If we scale the equation so that \( \vert \vert \boldsymbol{w}\vert\vert = 1/M \), we have to find the minimum of
+\( \boldsymbol{w}^T\boldsymbol{w}=\vert \vert \boldsymbol{w}\vert\vert \) (the norm) subject to the condition
+$$
+y_i(\boldsymbol{w}^T\boldsymbol{x}_i+b) \geq 1 \hspace{0.1cm}\forall i.
+$$
+
+
+We have thus defined our margin as the invers of the norm of
+\( \boldsymbol{w} \). We want to minimize the norm in order to have a as large as
+possible margin \( M \). Before we proceed, we need to remind ourselves
+about Lagrangian multipliers.
@@ -200,7 +224,7 @@ at all.
23
24
...
- 32
+ 31
»
diff --git a/doc/pub/week47/html/._week47-bs015.html b/doc/pub/week47/html/._week47-bs015.html
index 884dee6de..35a39f87b 100644
--- a/doc/pub/week47/html/._week47-bs015.html
+++ b/doc/pub/week47/html/._week47-bs015.html
@@ -53,31 +53,30 @@ Automatically generated HTML file from DocOnce source
('Getting into the details', 2, None, '___sec9'),
('First attempt at a minimization approach', 2, None, '___sec10'),
('Solving the equations', 2, None, '___sec11'),
- ('Code Example', 2, None, '___sec12'),
- ('Problems with the Simpler Approach', 2, None, '___sec13'),
- ('A better approach', 2, None, '___sec14'),
+ ('Can we code this?', 2, None, '___sec12'),
+ ('A better approach', 2, None, '___sec13'),
('A quick Reminder on Lagrangian Multipliers',
2,
None,
- '___sec15'),
- ('Adding the Multiplier', 2, None, '___sec16'),
- ('Setting up the Problem', 2, None, '___sec17'),
- ('The problem to solve', 2, None, '___sec18'),
- ('The last steps', 2, None, '___sec19'),
- ('A soft classifier', 2, None, '___sec20'),
- ('Soft optmization problem', 2, None, '___sec21'),
- ('Kernels and non-linearity', 2, None, '___sec22'),
- ('The equations', 2, None, '___sec23'),
- ('The problem to solve', 2, None, '___sec24'),
- ("Different kernels and Mercer's theorem", 2, None, '___sec25'),
- ('The moons example', 2, None, '___sec26'),
+ '___sec14'),
+ ('Adding the Multiplier', 2, None, '___sec15'),
+ ('Setting up the Problem', 2, None, '___sec16'),
+ ('The problem to solve', 2, None, '___sec17'),
+ ('The last steps', 2, None, '___sec18'),
+ ('A soft classifier', 2, None, '___sec19'),
+ ('Soft optmization problem', 2, None, '___sec20'),
+ ('Kernels and non-linearity', 2, None, '___sec21'),
+ ('The equations', 2, None, '___sec22'),
+ ('The problem to solve', 2, None, '___sec23'),
+ ("Different kernels and Mercer's theorem", 2, None, '___sec24'),
+ ('The moons example', 2, None, '___sec25'),
('Mathematical optimization of convex functions',
2,
None,
- '___sec27'),
- ('How do we solve these problems?', 2, None, '___sec28'),
- ('A simple example', 2, None, '___sec29'),
- ('Back to the more realistic cases', 2, None, '___sec30')]}
+ '___sec26'),
+ ('How do we solve these problems?', 2, None, '___sec27'),
+ ('A simple example', 2, None, '___sec28'),
+ ('Back to the more realistic cases', 2, None, '___sec29')]}
end of tocinfo -->
@@ -127,25 +126,24 @@ MathJax.Hub.Config({
Getting into the details
First attempt at a minimization approach
Solving the equations
- Code Example
- Problems with the Simpler Approach
- A better approach
- A quick Reminder on Lagrangian Multipliers
- Adding the Multiplier
- Setting up the Problem
- The problem to solve
- The last steps
- A soft classifier
- Soft optmization problem
- Kernels and non-linearity
- The equations
- The problem to solve
- Different kernels and Mercer's theorem
- The moons example
- Mathematical optimization of convex functions
- How do we solve these problems?
- A simple example
- Back to the more realistic cases
+ Can we code this?
+ A better approach
+ A quick Reminder on Lagrangian Multipliers
+ Adding the Multiplier
+ Setting up the Problem
+ The problem to solve
+ The last steps
+ A soft classifier
+ Soft optmization problem
+ Kernels and non-linearity
+ The equations
+ The problem to solve
+ Different kernels and Mercer's theorem
+ The moons example
+ Mathematical optimization of convex functions
+ How do we solve these problems?
+ A simple example
+ Back to the more realistic cases
@@ -161,44 +159,52 @@ MathJax.Hub.Config({
-A better approach
+A quick Reminder on Lagrangian Multipliers
-A better approach is rather to try to define a large margin between
-the two classes (if they are well separated from the beginning).
+Consider a function of three independent variables \( f(x,y,z) \) . For the function \( f \) to be an
+extreme we have
+$$
+df=0.
+$$
+
+A necessary and sufficient condition is
+$$
+\frac{\partial f}{\partial x} =\frac{\partial f}{\partial y}=\frac{\partial f}{\partial z}=0,
+$$
+
+due to
+$$
+df = \frac{\partial f}{\partial x}dx+\frac{\partial f}{\partial y}dy+\frac{\partial f}{\partial z}dz.
+$$
+
+In many problems the variables \( x,y,z \) are often subject to constraints (such as those above for the margin)
+so that they are no longer all independent. It is possible at least in principle to use each
+constraint to eliminate one variable
+and to proceed with a new and smaller set of independent varables.
-Thus, we wish to find a margin \( M \) with \( \boldsymbol{w} \) normalized to
-\( \vert\vert \boldsymbol{w}\vert\vert =1 \) subject to the condition
-
+The use of so-called Lagrangian multipliers is an alternative technique when the elimination
+of variables is incovenient or undesirable. Assume that we have an equation of constraint on
+the variables \( x,y,z \)
$$
-y_i(\boldsymbol{w}^T\boldsymbol{x}_i+b) \geq M \hspace{0.1cm}\forall i=1,2,\dots, p.
+\phi(x,y,z) = 0,
$$
-All points are thus at a signed distance from the decision boundary defined by the line \( L \). The parameters \( b \) and \( w_1 \) and \( w_2 \) define this line.
-
-
-We seek thus the largest value \( M \) defined by
+ resulting in
$$
-\frac{1}{\vert \vert \boldsymbol{w}\vert\vert}y_i(\boldsymbol{w}^T\boldsymbol{x}_i+b) \geq M \hspace{0.1cm}\forall i=1,2,\dots, n,
+d\phi = \frac{\partial \phi}{\partial x}dx+\frac{\partial \phi}{\partial y}dy+\frac{\partial \phi}{\partial z}dz =0.
$$
-or just
+Now we cannot set anymore
$$
-y_i(\boldsymbol{w}^T\boldsymbol{x}_i+b) \geq M\vert \vert \boldsymbol{w}\vert\vert \hspace{0.1cm}\forall i.
+\frac{\partial f}{\partial x} =\frac{\partial f}{\partial y}=\frac{\partial f}{\partial z}=0,
$$
-If we scale the equation so that \( \vert \vert \boldsymbol{w}\vert\vert = 1/M \), we have to find the minimum of
-\( \boldsymbol{w}^T\boldsymbol{w}=\vert \vert \boldsymbol{w}\vert\vert \) (the norm) subject to the condition
-$$
-y_i(\boldsymbol{w}^T\boldsymbol{x}_i+b) \geq 1 \hspace{0.1cm}\forall i.
-$$
-
-
-We have thus defined our margin as the invers of the norm of
-\( \boldsymbol{w} \). We want to minimize the norm in order to have a as large as
-possible margin \( M \). Before we proceed, we need to remind ourselves
-about Lagrangian multipliers.
+if \( df=0 \) is wanted
+because there are now only two independent variables! Assume \( x \) and \( y \) are the independent
+variables.
+Then \( dz \) is no longer arbitrary.
@@ -226,7 +232,7 @@ about Lagrangian multipliers.
24
25
...
- 32
+ 31
»
diff --git a/doc/pub/week47/html/._week47-bs016.html b/doc/pub/week47/html/._week47-bs016.html
index 08c0a0ca8..e74b3a50f 100644
--- a/doc/pub/week47/html/._week47-bs016.html
+++ b/doc/pub/week47/html/._week47-bs016.html
@@ -53,31 +53,30 @@ Automatically generated HTML file from DocOnce source
('Getting into the details', 2, None, '___sec9'),
('First attempt at a minimization approach', 2, None, '___sec10'),
('Solving the equations', 2, None, '___sec11'),
- ('Code Example', 2, None, '___sec12'),
- ('Problems with the Simpler Approach', 2, None, '___sec13'),
- ('A better approach', 2, None, '___sec14'),
+ ('Can we code this?', 2, None, '___sec12'),
+ ('A better approach', 2, None, '___sec13'),
('A quick Reminder on Lagrangian Multipliers',
2,
None,
- '___sec15'),
- ('Adding the Multiplier', 2, None, '___sec16'),
- ('Setting up the Problem', 2, None, '___sec17'),
- ('The problem to solve', 2, None, '___sec18'),
- ('The last steps', 2, None, '___sec19'),
- ('A soft classifier', 2, None, '___sec20'),
- ('Soft optmization problem', 2, None, '___sec21'),
- ('Kernels and non-linearity', 2, None, '___sec22'),
- ('The equations', 2, None, '___sec23'),
- ('The problem to solve', 2, None, '___sec24'),
- ("Different kernels and Mercer's theorem", 2, None, '___sec25'),
- ('The moons example', 2, None, '___sec26'),
+ '___sec14'),
+ ('Adding the Multiplier', 2, None, '___sec15'),
+ ('Setting up the Problem', 2, None, '___sec16'),
+ ('The problem to solve', 2, None, '___sec17'),
+ ('The last steps', 2, None, '___sec18'),
+ ('A soft classifier', 2, None, '___sec19'),
+ ('Soft optmization problem', 2, None, '___sec20'),
+ ('Kernels and non-linearity', 2, None, '___sec21'),
+ ('The equations', 2, None, '___sec22'),
+ ('The problem to solve', 2, None, '___sec23'),
+ ("Different kernels and Mercer's theorem", 2, None, '___sec24'),
+ ('The moons example', 2, None, '___sec25'),
('Mathematical optimization of convex functions',
2,
None,
- '___sec27'),
- ('How do we solve these problems?', 2, None, '___sec28'),
- ('A simple example', 2, None, '___sec29'),
- ('Back to the more realistic cases', 2, None, '___sec30')]}
+ '___sec26'),
+ ('How do we solve these problems?', 2, None, '___sec27'),
+ ('A simple example', 2, None, '___sec28'),
+ ('Back to the more realistic cases', 2, None, '___sec29')]}
end of tocinfo -->
@@ -127,25 +126,24 @@ MathJax.Hub.Config({
Getting into the details
First attempt at a minimization approach
Solving the equations
- Code Example
- Problems with the Simpler Approach
- A better approach
- A quick Reminder on Lagrangian Multipliers
- Adding the Multiplier
- Setting up the Problem
- The problem to solve
- The last steps
- A soft classifier
- Soft optmization problem
- Kernels and non-linearity
- The equations
- The problem to solve
- Different kernels and Mercer's theorem
- The moons example
- Mathematical optimization of convex functions
- How do we solve these problems?
- A simple example
- Back to the more realistic cases
+ Can we code this?
+ A better approach
+ A quick Reminder on Lagrangian Multipliers
+ Adding the Multiplier
+ Setting up the Problem
+ The problem to solve
+ The last steps
+ A soft classifier
+ Soft optmization problem
+ Kernels and non-linearity
+ The equations
+ The problem to solve
+ Different kernels and Mercer's theorem
+ The moons example
+ Mathematical optimization of convex functions
+ How do we solve these problems?
+ A simple example
+ Back to the more realistic cases
@@ -161,53 +159,46 @@ MathJax.Hub.Config({
-A quick Reminder on Lagrangian Multipliers
+Adding the Multiplier
-Consider a function of three independent variables \( f(x,y,z) \) . For the function \( f \) to be an
-extreme we have
+However, we can add to
$$
-df=0.
+df = \frac{\partial f}{\partial x}dx+\frac{\partial f}{\partial y}dy+\frac{\partial f}{\partial z}dz,
$$
-A necessary and sufficient condition is
+a multiplum of \( d\phi \), viz. \( \lambda d\phi \), resulting in
$$
-\frac{\partial f}{\partial x} =\frac{\partial f}{\partial y}=\frac{\partial f}{\partial z}=0,
+df+\lambda d\phi = (\frac{\partial f}{\partial z}+\lambda
+\frac{\partial \phi}{\partial x})dx+(\frac{\partial f}{\partial y}+\lambda\frac{\partial \phi}{\partial y})dy+
+(\frac{\partial f}{\partial z}+\lambda\frac{\partial \phi}{\partial z})dz =0.
$$
-due to
+Our multiplier is chosen so that
$$
-df = \frac{\partial f}{\partial x}dx+\frac{\partial f}{\partial y}dy+\frac{\partial f}{\partial z}dz.
+\frac{\partial f}{\partial z}+\lambda\frac{\partial \phi}{\partial z} =0.
$$
-In many problems the variables \( x,y,z \) are often subject to constraints (such as those above for the margin)
-so that they are no longer all independent. It is possible at least in principle to use each
-constraint to eliminate one variable
-and to proceed with a new and smaller set of independent varables.
-
-The use of so-called Lagrangian multipliers is an alternative technique when the elimination
-of variables is incovenient or undesirable. Assume that we have an equation of constraint on
-the variables \( x,y,z \)
+We need to remember that we took \( dx \) and \( dy \) to be arbitrary and thus we must have
$$
-\phi(x,y,z) = 0,
+\frac{\partial f}{\partial x}+\lambda\frac{\partial \phi}{\partial x} =0,
$$
- resulting in
+and
$$
-d\phi = \frac{\partial \phi}{\partial x}dx+\frac{\partial \phi}{\partial y}dy+\frac{\partial \phi}{\partial z}dz =0.
+\frac{\partial f}{\partial y}+\lambda\frac{\partial \phi}{\partial y} =0.
$$
-Now we cannot set anymore
+When all these equations are satisfied, \( df=0 \). We have four unknowns, \( x,y,z \) and
+\( \lambda \). Actually we want only \( x,y,z \), \( \lambda \) needs not to be determined,
+it is therefore often called
+Lagrange's undetermined multiplier.
+If we have a set of constraints \( \phi_k \) we have the equations
$$
-\frac{\partial f}{\partial x} =\frac{\partial f}{\partial y}=\frac{\partial f}{\partial z}=0,
+\frac{\partial f}{\partial x_i}+\sum_k\lambda_k\frac{\partial \phi_k}{\partial x_i} =0.
$$
-if \( df=0 \) is wanted
-because there are now only two independent variables! Assume \( x \) and \( y \) are the independent
-variables.
-Then \( dz \) is no longer arbitrary.
-
@@ -234,7 +225,7 @@ Then \( dz \) is no longer arbitrary.
25
26
...
- 32
+ 31
»
diff --git a/doc/pub/week47/html/._week47-bs017.html b/doc/pub/week47/html/._week47-bs017.html
index 93c641fba..93f91e58f 100644
--- a/doc/pub/week47/html/._week47-bs017.html
+++ b/doc/pub/week47/html/._week47-bs017.html
@@ -53,31 +53,30 @@ Automatically generated HTML file from DocOnce source
('Getting into the details', 2, None, '___sec9'),
('First attempt at a minimization approach', 2, None, '___sec10'),
('Solving the equations', 2, None, '___sec11'),
- ('Code Example', 2, None, '___sec12'),
- ('Problems with the Simpler Approach', 2, None, '___sec13'),
- ('A better approach', 2, None, '___sec14'),
+ ('Can we code this?', 2, None, '___sec12'),
+ ('A better approach', 2, None, '___sec13'),
('A quick Reminder on Lagrangian Multipliers',
2,
None,
- '___sec15'),
- ('Adding the Multiplier', 2, None, '___sec16'),
- ('Setting up the Problem', 2, None, '___sec17'),
- ('The problem to solve', 2, None, '___sec18'),
- ('The last steps', 2, None, '___sec19'),
- ('A soft classifier', 2, None, '___sec20'),
- ('Soft optmization problem', 2, None, '___sec21'),
- ('Kernels and non-linearity', 2, None, '___sec22'),
- ('The equations', 2, None, '___sec23'),
- ('The problem to solve', 2, None, '___sec24'),
- ("Different kernels and Mercer's theorem", 2, None, '___sec25'),
- ('The moons example', 2, None, '___sec26'),
+ '___sec14'),
+ ('Adding the Multiplier', 2, None, '___sec15'),
+ ('Setting up the Problem', 2, None, '___sec16'),
+ ('The problem to solve', 2, None, '___sec17'),
+ ('The last steps', 2, None, '___sec18'),
+ ('A soft classifier', 2, None, '___sec19'),
+ ('Soft optmization problem', 2, None, '___sec20'),
+ ('Kernels and non-linearity', 2, None, '___sec21'),
+ ('The equations', 2, None, '___sec22'),
+ ('The problem to solve', 2, None, '___sec23'),
+ ("Different kernels and Mercer's theorem", 2, None, '___sec24'),
+ ('The moons example', 2, None, '___sec25'),
('Mathematical optimization of convex functions',
2,
None,
- '___sec27'),
- ('How do we solve these problems?', 2, None, '___sec28'),
- ('A simple example', 2, None, '___sec29'),
- ('Back to the more realistic cases', 2, None, '___sec30')]}
+ '___sec26'),
+ ('How do we solve these problems?', 2, None, '___sec27'),
+ ('A simple example', 2, None, '___sec28'),
+ ('Back to the more realistic cases', 2, None, '___sec29')]}
end of tocinfo -->
@@ -127,25 +126,24 @@ MathJax.Hub.Config({
Getting into the details
First attempt at a minimization approach
Solving the equations
- Code Example
- Problems with the Simpler Approach
- A better approach
- A quick Reminder on Lagrangian Multipliers
- Adding the Multiplier
- Setting up the Problem
- The problem to solve
- The last steps
- A soft classifier
- Soft optmization problem
- Kernels and non-linearity
- The equations
- The problem to solve
- Different kernels and Mercer's theorem
- The moons example
- Mathematical optimization of convex functions
- How do we solve these problems?
- A simple example
- Back to the more realistic cases
+ Can we code this?
+ A better approach
+ A quick Reminder on Lagrangian Multipliers
+ Adding the Multiplier
+ Setting up the Problem
+ The problem to solve
+ The last steps
+ A soft classifier
+ Soft optmization problem
+ Kernels and non-linearity
+ The equations
+ The problem to solve
+ Different kernels and Mercer's theorem
+ The moons example
+ Mathematical optimization of convex functions
+ How do we solve these problems?
+ A simple example
+ Back to the more realistic cases
@@ -161,45 +159,43 @@ MathJax.Hub.Config({
-Adding the Multiplier
+Setting up the Problem
+In order to solve the above problem, we define the following Lagrangian function to be minimized
+$$
+{\cal L}(\lambda,b,\boldsymbol{w})=\frac{1}{2}\boldsymbol{w}^T\boldsymbol{w}-\sum_{i=1}^n\lambda_i\left[y_i(\boldsymbol{w}^T\boldsymbol{x}_i+b)-1\right],
+$$
+
+where \( \lambda_i \) is a so-called Lagrange multiplier subject to the condition \( \lambda_i \geq 0 \).
-However, we can add to
+Taking the derivatives with respect to \( b \) and \( \boldsymbol{w} \) we obtain
$$
-df = \frac{\partial f}{\partial x}dx+\frac{\partial f}{\partial y}dy+\frac{\partial f}{\partial z}dz,
+\frac{\partial {\cal L}}{\partial b} = -\sum_{i} \lambda_iy_i=0,
$$
-a multiplum of \( d\phi \), viz. \( \lambda d\phi \), resulting in
+and
$$
-df+\lambda d\phi = (\frac{\partial f}{\partial z}+\lambda
-\frac{\partial \phi}{\partial x})dx+(\frac{\partial f}{\partial y}+\lambda\frac{\partial \phi}{\partial y})dy+
-(\frac{\partial f}{\partial z}+\lambda\frac{\partial \phi}{\partial z})dz =0.
+\frac{\partial {\cal L}}{\partial \boldsymbol{w}} = 0 = \boldsymbol{w}-\sum_{i} \lambda_iy_i\boldsymbol{x}_i.
$$
-Our multiplier is chosen so that
+Inserting these constraints into the equation for \( {\cal L} \) we obtain
$$
-\frac{\partial f}{\partial z}+\lambda\frac{\partial \phi}{\partial z} =0.
+{\cal L}=\sum_i\lambda_i-\frac{1}{2}\sum_{ij}^n\lambda_i\lambda_jy_iy_j\boldsymbol{x}_i^T\boldsymbol{x}_j,
$$
-
-We need to remember that we took \( dx \) and \( dy \) to be arbitrary and thus we must have
+subject to the constraints \( \lambda_i\geq 0 \) and \( \sum_i\lambda_iy_i=0 \).
+We must in addition satisfy the Karush-Kuhn-Tucker (KKT) condition
$$
-\frac{\partial f}{\partial x}+\lambda\frac{\partial \phi}{\partial x} =0,
+\lambda_i\left[y_i(\boldsymbol{w}^T\boldsymbol{x}_i+b) -1\right] \hspace{0.1cm}\forall i.
$$
-and
-$$
-\frac{\partial f}{\partial y}+\lambda\frac{\partial \phi}{\partial y} =0.
-$$
-When all these equations are satisfied, \( df=0 \). We have four unknowns, \( x,y,z \) and
-\( \lambda \). Actually we want only \( x,y,z \), \( \lambda \) needs not to be determined,
-it is therefore often called
-Lagrange's undetermined multiplier.
-If we have a set of constraints \( \phi_k \) we have the equations
-$$
-\frac{\partial f}{\partial x_i}+\sum_k\lambda_k\frac{\partial \phi_k}{\partial x_i} =0.
-$$
+
+- If \( \lambda_i > 0 \), then \( y_i(\boldsymbol{w}^T\boldsymbol{x}_i+b)=1 \) and we say that \( x_i \) is on the boundary.
+- If \( y_i(\boldsymbol{w}^T\boldsymbol{x}_i+b)> 1 \), we say \( x_i \) is not on the boundary and we set \( \lambda_i=0 \).
+
+
+When \( \lambda_i > 0 \), the vectors \( \boldsymbol{x}_i \) are called support vectors. They are the vectors closest to the line (or hyperplane) and define the margin \( M \).
@@ -227,7 +223,7 @@ $$
26
27
...
- 32
+ 31
»
diff --git a/doc/pub/week47/html/._week47-bs018.html b/doc/pub/week47/html/._week47-bs018.html
index f6fd45b55..e7ffc11a8 100644
--- a/doc/pub/week47/html/._week47-bs018.html
+++ b/doc/pub/week47/html/._week47-bs018.html
@@ -53,31 +53,30 @@ Automatically generated HTML file from DocOnce source
('Getting into the details', 2, None, '___sec9'),
('First attempt at a minimization approach', 2, None, '___sec10'),
('Solving the equations', 2, None, '___sec11'),
- ('Code Example', 2, None, '___sec12'),
- ('Problems with the Simpler Approach', 2, None, '___sec13'),
- ('A better approach', 2, None, '___sec14'),
+ ('Can we code this?', 2, None, '___sec12'),
+ ('A better approach', 2, None, '___sec13'),
('A quick Reminder on Lagrangian Multipliers',
2,
None,
- '___sec15'),
- ('Adding the Multiplier', 2, None, '___sec16'),
- ('Setting up the Problem', 2, None, '___sec17'),
- ('The problem to solve', 2, None, '___sec18'),
- ('The last steps', 2, None, '___sec19'),
- ('A soft classifier', 2, None, '___sec20'),
- ('Soft optmization problem', 2, None, '___sec21'),
- ('Kernels and non-linearity', 2, None, '___sec22'),
- ('The equations', 2, None, '___sec23'),
- ('The problem to solve', 2, None, '___sec24'),
- ("Different kernels and Mercer's theorem", 2, None, '___sec25'),
- ('The moons example', 2, None, '___sec26'),
+ '___sec14'),
+ ('Adding the Multiplier', 2, None, '___sec15'),
+ ('Setting up the Problem', 2, None, '___sec16'),
+ ('The problem to solve', 2, None, '___sec17'),
+ ('The last steps', 2, None, '___sec18'),
+ ('A soft classifier', 2, None, '___sec19'),
+ ('Soft optmization problem', 2, None, '___sec20'),
+ ('Kernels and non-linearity', 2, None, '___sec21'),
+ ('The equations', 2, None, '___sec22'),
+ ('The problem to solve', 2, None, '___sec23'),
+ ("Different kernels and Mercer's theorem", 2, None, '___sec24'),
+ ('The moons example', 2, None, '___sec25'),
('Mathematical optimization of convex functions',
2,
None,
- '___sec27'),
- ('How do we solve these problems?', 2, None, '___sec28'),
- ('A simple example', 2, None, '___sec29'),
- ('Back to the more realistic cases', 2, None, '___sec30')]}
+ '___sec26'),
+ ('How do we solve these problems?', 2, None, '___sec27'),
+ ('A simple example', 2, None, '___sec28'),
+ ('Back to the more realistic cases', 2, None, '___sec29')]}
end of tocinfo -->
@@ -127,25 +126,24 @@ MathJax.Hub.Config({
Getting into the details
First attempt at a minimization approach
Solving the equations
- Code Example
- Problems with the Simpler Approach
- A better approach
- A quick Reminder on Lagrangian Multipliers
- Adding the Multiplier
- Setting up the Problem
- The problem to solve
- The last steps
- A soft classifier
- Soft optmization problem
- Kernels and non-linearity
- The equations
- The problem to solve
- Different kernels and Mercer's theorem
- The moons example
- Mathematical optimization of convex functions
- How do we solve these problems?
- A simple example
- Back to the more realistic cases
+ Can we code this?
+ A better approach
+ A quick Reminder on Lagrangian Multipliers
+ Adding the Multiplier
+ Setting up the Problem
+ The problem to solve
+ The last steps
+ A soft classifier
+ Soft optmization problem
+ Kernels and non-linearity
+ The equations
+ The problem to solve
+ Different kernels and Mercer's theorem
+ The moons example
+ Mathematical optimization of convex functions
+ How do we solve these problems?
+ A simple example
+ Back to the more realistic cases
@@ -161,43 +159,26 @@ MathJax.Hub.Config({
-Setting up the Problem
-In order to solve the above problem, we define the following Lagrangian function to be minimized
-$$
-{\cal L}(\lambda,b,\boldsymbol{w})=\frac{1}{2}\boldsymbol{w}^T\boldsymbol{w}-\sum_{i=1}^n\lambda_i\left[y_i(\boldsymbol{w}^T\boldsymbol{x}_i+b)-1\right],
-$$
-
-where \( \lambda_i \) is a so-called Lagrange multiplier subject to the condition \( \lambda_i \geq 0 \).
+The problem to solve
-Taking the derivatives with respect to \( b \) and \( \boldsymbol{w} \) we obtain
-$$
-\frac{\partial {\cal L}}{\partial b} = -\sum_{i} \lambda_iy_i=0,
-$$
-
-and
-$$
-\frac{\partial {\cal L}}{\partial \boldsymbol{w}} = 0 = \boldsymbol{w}-\sum_{i} \lambda_iy_i\boldsymbol{x}_i.
-$$
-
-Inserting these constraints into the equation for \( {\cal L} \) we obtain
+We can rewrite
$$
{\cal L}=\sum_i\lambda_i-\frac{1}{2}\sum_{ij}^n\lambda_i\lambda_jy_iy_j\boldsymbol{x}_i^T\boldsymbol{x}_j,
$$
-subject to the constraints \( \lambda_i\geq 0 \) and \( \sum_i\lambda_iy_i=0 \).
-We must in addition satisfy the Karush-Kuhn-Tucker (KKT) condition
+and its constraints in terms of a matrix-vector problem where we minimize w.r.t. \( \lambda \) the following problem
$$
-\lambda_i\left[y_i(\boldsymbol{w}^T\boldsymbol{x}_i+b) -1\right] \hspace{0.1cm}\forall i.
+\frac{1}{2} \boldsymbol{\lambda}^T\begin{bmatrix} y_1y_1\boldsymbol{x}_1^T\boldsymbol{x}_1 & y_1y_2\boldsymbol{x}_1^T\boldsymbol{x}_2 & \dots & \dots & y_1y_n\boldsymbol{x}_1^T\boldsymbol{x}_n \\
+y_2y_1\boldsymbol{x}_2^T\boldsymbol{x}_1 & y_2y_2\boldsymbol{x}_2^T\boldsymbol{x}_2 & \dots & \dots & y_1y_n\boldsymbol{x}_2^T\boldsymbol{x}_n \\
+\dots & \dots & \dots & \dots & \dots \\
+\dots & \dots & \dots & \dots & \dots \\
+y_ny_1\boldsymbol{x}_n^T\boldsymbol{x}_1 & y_ny_2\boldsymbol{x}_n^T\boldsymbol{x}_2 & \dots & \dots & y_ny_n\boldsymbol{x}_n^T\boldsymbol{x}_n \\
+\end{bmatrix}\boldsymbol{\lambda}-\mathbb{1}\boldsymbol{\lambda},
$$
-
-
-- If \( \lambda_i > 0 \), then \( y_i(\boldsymbol{w}^T\boldsymbol{x}_i+b)=1 \) and we say that \( x_i \) is on the boundary.
-- If \( y_i(\boldsymbol{w}^T\boldsymbol{x}_i+b)> 1 \), we say \( x_i \) is not on the boundary and we set \( \lambda_i=0 \).
-
-
-When \( \lambda_i > 0 \), the vectors \( \boldsymbol{x}_i \) are called support vectors. They are the vectors closest to the line (or hyperplane) and define the margin \( M \).
+subject to \( \boldsymbol{y}^T\boldsymbol{\lambda}=0 \). Here we defined the vectors \( \boldsymbol{\lambda} =[\lambda_1,\lambda_2,\dots,\lambda_n] \) and
+\( \boldsymbol{y}=[y_1,y_2,\dots,y_n] \).
@@ -225,7 +206,7 @@ When \( \lambda_i > 0 \), the vectors \( \boldsymbol{x}_i \) are called support
27
28
...
- 32
+ 31
»
diff --git a/doc/pub/week47/html/._week47-bs019.html b/doc/pub/week47/html/._week47-bs019.html
index 43a03c61f..7d24fb9f2 100644
--- a/doc/pub/week47/html/._week47-bs019.html
+++ b/doc/pub/week47/html/._week47-bs019.html
@@ -53,31 +53,30 @@ Automatically generated HTML file from DocOnce source
('Getting into the details', 2, None, '___sec9'),
('First attempt at a minimization approach', 2, None, '___sec10'),
('Solving the equations', 2, None, '___sec11'),
- ('Code Example', 2, None, '___sec12'),
- ('Problems with the Simpler Approach', 2, None, '___sec13'),
- ('A better approach', 2, None, '___sec14'),
+ ('Can we code this?', 2, None, '___sec12'),
+ ('A better approach', 2, None, '___sec13'),
('A quick Reminder on Lagrangian Multipliers',
2,
None,
- '___sec15'),
- ('Adding the Multiplier', 2, None, '___sec16'),
- ('Setting up the Problem', 2, None, '___sec17'),
- ('The problem to solve', 2, None, '___sec18'),
- ('The last steps', 2, None, '___sec19'),
- ('A soft classifier', 2, None, '___sec20'),
- ('Soft optmization problem', 2, None, '___sec21'),
- ('Kernels and non-linearity', 2, None, '___sec22'),
- ('The equations', 2, None, '___sec23'),
- ('The problem to solve', 2, None, '___sec24'),
- ("Different kernels and Mercer's theorem", 2, None, '___sec25'),
- ('The moons example', 2, None, '___sec26'),
+ '___sec14'),
+ ('Adding the Multiplier', 2, None, '___sec15'),
+ ('Setting up the Problem', 2, None, '___sec16'),
+ ('The problem to solve', 2, None, '___sec17'),
+ ('The last steps', 2, None, '___sec18'),
+ ('A soft classifier', 2, None, '___sec19'),
+ ('Soft optmization problem', 2, None, '___sec20'),
+ ('Kernels and non-linearity', 2, None, '___sec21'),
+ ('The equations', 2, None, '___sec22'),
+ ('The problem to solve', 2, None, '___sec23'),
+ ("Different kernels and Mercer's theorem", 2, None, '___sec24'),
+ ('The moons example', 2, None, '___sec25'),
('Mathematical optimization of convex functions',
2,
None,
- '___sec27'),
- ('How do we solve these problems?', 2, None, '___sec28'),
- ('A simple example', 2, None, '___sec29'),
- ('Back to the more realistic cases', 2, None, '___sec30')]}
+ '___sec26'),
+ ('How do we solve these problems?', 2, None, '___sec27'),
+ ('A simple example', 2, None, '___sec28'),
+ ('Back to the more realistic cases', 2, None, '___sec29')]}
end of tocinfo -->
@@ -127,25 +126,24 @@ MathJax.Hub.Config({
Getting into the details
First attempt at a minimization approach
Solving the equations
- Code Example
- Problems with the Simpler Approach
- A better approach
- A quick Reminder on Lagrangian Multipliers
- Adding the Multiplier
- Setting up the Problem
- The problem to solve
- The last steps
- A soft classifier
- Soft optmization problem
- Kernels and non-linearity
- The equations
- The problem to solve
- Different kernels and Mercer's theorem
- The moons example
- Mathematical optimization of convex functions
- How do we solve these problems?
- A simple example
- Back to the more realistic cases
+ Can we code this?
+ A better approach
+ A quick Reminder on Lagrangian Multipliers
+ Adding the Multiplier
+ Setting up the Problem
+ The problem to solve
+ The last steps
+ A soft classifier
+ Soft optmization problem
+ Kernels and non-linearity
+ The equations
+ The problem to solve
+ Different kernels and Mercer's theorem
+ The moons example
+ Mathematical optimization of convex functions
+ How do we solve these problems?
+ A simple example
+ Back to the more realistic cases
@@ -161,26 +159,36 @@ MathJax.Hub.Config({
-The problem to solve
+The last steps
-We can rewrite
+Solving the above problem, yields the values of \( \lambda_i \).
+To find the coefficients of your hyperplane we need simply to compute
$$
-{\cal L}=\sum_i\lambda_i-\frac{1}{2}\sum_{ij}^n\lambda_i\lambda_jy_iy_j\boldsymbol{x}_i^T\boldsymbol{x}_j,
+\boldsymbol{w}=\sum_{i} \lambda_iy_i\boldsymbol{x}_i.
$$
-and its constraints in terms of a matrix-vector problem where we minimize w.r.t. \( \lambda \) the following problem
+With our vector \( \boldsymbol{w} \) we can in turn find the value of the intercept \( b \) (here in two dimensions) via
$$
-\frac{1}{2} \boldsymbol{\lambda}^T\begin{bmatrix} y_1y_1\boldsymbol{x}_1^T\boldsymbol{x}_1 & y_1y_2\boldsymbol{x}_1^T\boldsymbol{x}_2 & \dots & \dots & y_1y_n\boldsymbol{x}_1^T\boldsymbol{x}_n \\
-y_2y_1\boldsymbol{x}_2^T\boldsymbol{x}_1 & y_2y_2\boldsymbol{x}_2^T\boldsymbol{x}_2 & \dots & \dots & y_1y_n\boldsymbol{x}_2^T\boldsymbol{x}_n \\
-\dots & \dots & \dots & \dots & \dots \\
-\dots & \dots & \dots & \dots & \dots \\
-y_ny_1\boldsymbol{x}_n^T\boldsymbol{x}_1 & y_ny_2\boldsymbol{x}_n^T\boldsymbol{x}_2 & \dots & \dots & y_ny_n\boldsymbol{x}_n^T\boldsymbol{x}_n \\
-\end{bmatrix}\boldsymbol{\lambda}-\mathbb{1}\boldsymbol{\lambda},
+y_i(\boldsymbol{w}^T\boldsymbol{x}_i+b)=1,
$$
-subject to \( \boldsymbol{y}^T\boldsymbol{\lambda}=0 \). Here we defined the vectors \( \boldsymbol{\lambda} =[\lambda_1,\lambda_2,\dots,\lambda_n] \) and
-\( \boldsymbol{y}=[y_1,y_2,\dots,y_n] \).
+resulting in
+$$
+b = \frac{1}{y_i}-\boldsymbol{w}^T\boldsymbol{x}_i,
+$$
+
+or if we write it out in terms of the support vectors only, with \( N_s \) being their number, we have
+$$
+b = \frac{1}{N_s}\sum_{j\in N_s}\left(y_j-\sum_{i=1}^n\lambda_iy_i\boldsymbol{x}_i^T\boldsymbol{x}_j\right).
+$$
+
+With our hyperplane coefficients we can use our classifier to assign any observation by simply using
+$$
+y_i = \mathrm{sign}(\boldsymbol{w}^T\boldsymbol{x}_i+b).
+$$
+
+Below we discuss how to find the optimal values of \( \lambda_i \). Before we proceed however, we discuss now the so-called soft classifier.
@@ -208,7 +216,7 @@ subject to \( \boldsymbol{y}^T\boldsymbol{\lambda}=0 \). Here we defined the vec
28
29
...
- 32
+ 31
»
diff --git a/doc/pub/week47/html/._week47-bs020.html b/doc/pub/week47/html/._week47-bs020.html
index 6ec6015e8..973a3b8c8 100644
--- a/doc/pub/week47/html/._week47-bs020.html
+++ b/doc/pub/week47/html/._week47-bs020.html
@@ -53,31 +53,30 @@ Automatically generated HTML file from DocOnce source
('Getting into the details', 2, None, '___sec9'),
('First attempt at a minimization approach', 2, None, '___sec10'),
('Solving the equations', 2, None, '___sec11'),
- ('Code Example', 2, None, '___sec12'),
- ('Problems with the Simpler Approach', 2, None, '___sec13'),
- ('A better approach', 2, None, '___sec14'),
+ ('Can we code this?', 2, None, '___sec12'),
+ ('A better approach', 2, None, '___sec13'),
('A quick Reminder on Lagrangian Multipliers',
2,
None,
- '___sec15'),
- ('Adding the Multiplier', 2, None, '___sec16'),
- ('Setting up the Problem', 2, None, '___sec17'),
- ('The problem to solve', 2, None, '___sec18'),
- ('The last steps', 2, None, '___sec19'),
- ('A soft classifier', 2, None, '___sec20'),
- ('Soft optmization problem', 2, None, '___sec21'),
- ('Kernels and non-linearity', 2, None, '___sec22'),
- ('The equations', 2, None, '___sec23'),
- ('The problem to solve', 2, None, '___sec24'),
- ("Different kernels and Mercer's theorem", 2, None, '___sec25'),
- ('The moons example', 2, None, '___sec26'),
+ '___sec14'),
+ ('Adding the Multiplier', 2, None, '___sec15'),
+ ('Setting up the Problem', 2, None, '___sec16'),
+ ('The problem to solve', 2, None, '___sec17'),
+ ('The last steps', 2, None, '___sec18'),
+ ('A soft classifier', 2, None, '___sec19'),
+ ('Soft optmization problem', 2, None, '___sec20'),
+ ('Kernels and non-linearity', 2, None, '___sec21'),
+ ('The equations', 2, None, '___sec22'),
+ ('The problem to solve', 2, None, '___sec23'),
+ ("Different kernels and Mercer's theorem", 2, None, '___sec24'),
+ ('The moons example', 2, None, '___sec25'),
('Mathematical optimization of convex functions',
2,
None,
- '___sec27'),
- ('How do we solve these problems?', 2, None, '___sec28'),
- ('A simple example', 2, None, '___sec29'),
- ('Back to the more realistic cases', 2, None, '___sec30')]}
+ '___sec26'),
+ ('How do we solve these problems?', 2, None, '___sec27'),
+ ('A simple example', 2, None, '___sec28'),
+ ('Back to the more realistic cases', 2, None, '___sec29')]}
end of tocinfo -->
@@ -127,25 +126,24 @@ MathJax.Hub.Config({
Getting into the details
First attempt at a minimization approach
Solving the equations
- Code Example
- Problems with the Simpler Approach
- A better approach
- A quick Reminder on Lagrangian Multipliers
- Adding the Multiplier
- Setting up the Problem
- The problem to solve
- The last steps
- A soft classifier
- Soft optmization problem
- Kernels and non-linearity
- The equations
- The problem to solve
- Different kernels and Mercer's theorem
- The moons example
- Mathematical optimization of convex functions
- How do we solve these problems?
- A simple example
- Back to the more realistic cases
+ Can we code this?
+ A better approach
+ A quick Reminder on Lagrangian Multipliers
+ Adding the Multiplier
+ Setting up the Problem
+ The problem to solve
+ The last steps
+ A soft classifier
+ Soft optmization problem
+ Kernels and non-linearity
+ The equations
+ The problem to solve
+ Different kernels and Mercer's theorem
+ The moons example
+ Mathematical optimization of convex functions
+ How do we solve these problems?
+ A simple example
+ Back to the more realistic cases
@@ -161,36 +159,37 @@ MathJax.Hub.Config({
-The last steps
+A soft classifier
-Solving the above problem, yields the values of \( \lambda_i \).
-To find the coefficients of your hyperplane we need simply to compute
-$$
-\boldsymbol{w}=\sum_{i} \lambda_iy_i\boldsymbol{x}_i.
-$$
+Till now, the margin is strictly defined by the support vectors. This defines what is called a hard classifier, that is the margins are well defined.
-With our vector \( \boldsymbol{w} \) we can in turn find the value of the intercept \( b \) (here in two dimensions) via
+
+Suppose now that classes overlap in feature space, as shown in the
+figure here. One way to deal with this problem before we define the
+so-called kernel approach, is to allow a kind of slack in the sense
+that we allow some points to be on the wrong side of the margin.
+
+
+We introduce thus the so-called slack variables \( \boldsymbol{\xi} =[\xi_1,x_2,\dots,x_n] \) and
+modify our previous equation
$$
y_i(\boldsymbol{w}^T\boldsymbol{x}_i+b)=1,
$$
-resulting in
+to
$$
-b = \frac{1}{y_i}-\boldsymbol{w}^T\boldsymbol{x}_i,
+y_i(\boldsymbol{w}^T\boldsymbol{x}_i+b)=1-\xi_i,
$$
-or if we write it out in terms of the support vectors only, with \( N_s \) being their number, we have
-$$
-b = \frac{1}{N_s}\sum_{j\in N_s}\left(y_j-\sum_{i=1}^n\lambda_iy_i\boldsymbol{x}_i^T\boldsymbol{x}_j\right).
-$$
+with the requirement \( \xi_i\geq 0 \). The total violation is now \( \sum_i\xi \).
+The value \( \xi_i \) in the constraint the last constraint corresponds to the amount by which the prediction
+\( y_i(\boldsymbol{w}^T\boldsymbol{x}_i+b)=1 \) is on the wrong side of its margin. Hence by bounding the sum \( \sum_i \xi_i \),
+we bound the total amount by which predictions fall on the wrong side of their margins.
-With our hyperplane coefficients we can use our classifier to assign any observation by simply using
-$$
-y_i = \mathrm{sign}(\boldsymbol{w}^T\boldsymbol{x}_i+b).
-$$
-
-Below we discuss how to find the optimal values of \( \lambda_i \). Before we proceed however, we discuss now the so-called soft classifier.
+
+Misclassifications occur when \( \xi_i > 1 \). Thus bounding the total sum by some value \( C \) bounds in turn the total number of
+misclassifications.
@@ -218,7 +217,7 @@ Below we discuss how to find the optimal values of \( \lambda_i \). Before we pr
29
30
...
- 32
+ 31
»
diff --git a/doc/pub/week47/html/._week47-bs021.html b/doc/pub/week47/html/._week47-bs021.html
index a7de379d9..bf1e412a3 100644
--- a/doc/pub/week47/html/._week47-bs021.html
+++ b/doc/pub/week47/html/._week47-bs021.html
@@ -53,31 +53,30 @@ Automatically generated HTML file from DocOnce source
('Getting into the details', 2, None, '___sec9'),
('First attempt at a minimization approach', 2, None, '___sec10'),
('Solving the equations', 2, None, '___sec11'),
- ('Code Example', 2, None, '___sec12'),
- ('Problems with the Simpler Approach', 2, None, '___sec13'),
- ('A better approach', 2, None, '___sec14'),
+ ('Can we code this?', 2, None, '___sec12'),
+ ('A better approach', 2, None, '___sec13'),
('A quick Reminder on Lagrangian Multipliers',
2,
None,
- '___sec15'),
- ('Adding the Multiplier', 2, None, '___sec16'),
- ('Setting up the Problem', 2, None, '___sec17'),
- ('The problem to solve', 2, None, '___sec18'),
- ('The last steps', 2, None, '___sec19'),
- ('A soft classifier', 2, None, '___sec20'),
- ('Soft optmization problem', 2, None, '___sec21'),
- ('Kernels and non-linearity', 2, None, '___sec22'),
- ('The equations', 2, None, '___sec23'),
- ('The problem to solve', 2, None, '___sec24'),
- ("Different kernels and Mercer's theorem", 2, None, '___sec25'),
- ('The moons example', 2, None, '___sec26'),
+ '___sec14'),
+ ('Adding the Multiplier', 2, None, '___sec15'),
+ ('Setting up the Problem', 2, None, '___sec16'),
+ ('The problem to solve', 2, None, '___sec17'),
+ ('The last steps', 2, None, '___sec18'),
+ ('A soft classifier', 2, None, '___sec19'),
+ ('Soft optmization problem', 2, None, '___sec20'),
+ ('Kernels and non-linearity', 2, None, '___sec21'),
+ ('The equations', 2, None, '___sec22'),
+ ('The problem to solve', 2, None, '___sec23'),
+ ("Different kernels and Mercer's theorem", 2, None, '___sec24'),
+ ('The moons example', 2, None, '___sec25'),
('Mathematical optimization of convex functions',
2,
None,
- '___sec27'),
- ('How do we solve these problems?', 2, None, '___sec28'),
- ('A simple example', 2, None, '___sec29'),
- ('Back to the more realistic cases', 2, None, '___sec30')]}
+ '___sec26'),
+ ('How do we solve these problems?', 2, None, '___sec27'),
+ ('A simple example', 2, None, '___sec28'),
+ ('Back to the more realistic cases', 2, None, '___sec29')]}
end of tocinfo -->
@@ -127,25 +126,24 @@ MathJax.Hub.Config({
Getting into the details
First attempt at a minimization approach
Solving the equations
- Code Example
- Problems with the Simpler Approach
- A better approach
- A quick Reminder on Lagrangian Multipliers
- Adding the Multiplier
- Setting up the Problem
- The problem to solve
- The last steps
- A soft classifier
- Soft optmization problem
- Kernels and non-linearity
- The equations
- The problem to solve
- Different kernels and Mercer's theorem
- The moons example
- Mathematical optimization of convex functions
- How do we solve these problems?
- A simple example
- Back to the more realistic cases
+ Can we code this?
+ A better approach
+ A quick Reminder on Lagrangian Multipliers
+ Adding the Multiplier
+ Setting up the Problem
+ The problem to solve
+ The last steps
+ A soft classifier
+ Soft optmization problem
+ Kernels and non-linearity
+ The equations
+ The problem to solve
+ Different kernels and Mercer's theorem
+ The moons example
+ Mathematical optimization of convex functions
+ How do we solve these problems?
+ A simple example
+ Back to the more realistic cases
@@ -161,37 +159,56 @@ MathJax.Hub.Config({
-A soft classifier
+Soft optmization problem
-Till now, the margin is strictly defined by the support vectors. This defines what is called a hard classifier, that is the margins are well defined.
+This has in turn the consequences that we change our optmization problem to finding the minimum of
+$$
+{\cal L}=\frac{1}{2}\boldsymbol{w}^T\boldsymbol{w}-\sum_{i=1}^n\lambda_i\left[y_i(\boldsymbol{w}^T\boldsymbol{x}_i+b)-(1-\xi_)\right]+C\sum_{i=1}^n\xi_i-\sum_{i=1}^n\gamma_i\xi_i,
+$$
+
+subject to
+$$
+y_i(\boldsymbol{w}^T\boldsymbol{x}_i+b)=1-\xi_i \hspace{0.1cm}\forall i,
+$$
+
+with the requirement \( \xi_i\geq 0 \).
-Suppose now that classes overlap in feature space, as shown in the
-figure here. One way to deal with this problem before we define the
-so-called kernel approach, is to allow a kind of slack in the sense
-that we allow some points to be on the wrong side of the margin.
-
-
-We introduce thus the so-called slack variables \( \boldsymbol{\xi} =[\xi_1,x_2,\dots,x_n] \) and
-modify our previous equation
+Taking the derivatives with respect to \( b \) and \( \boldsymbol{w} \) we obtain
$$
-y_i(\boldsymbol{w}^T\boldsymbol{x}_i+b)=1,
+\frac{\partial {\cal L}}{\partial b} = -\sum_{i} \lambda_iy_i=0,
$$
-to
+and
$$
-y_i(\boldsymbol{w}^T\boldsymbol{x}_i+b)=1-\xi_i,
+\frac{\partial {\cal L}}{\partial \boldsymbol{w}} = 0 = \boldsymbol{w}-\sum_{i} \lambda_iy_i\boldsymbol{x}_i,
$$
-with the requirement \( \xi_i\geq 0 \). The total violation is now \( \sum_i\xi \).
-The value \( \xi_i \) in the constraint the last constraint corresponds to the amount by which the prediction
-\( y_i(\boldsymbol{w}^T\boldsymbol{x}_i+b)=1 \) is on the wrong side of its margin. Hence by bounding the sum \( \sum_i \xi_i \),
-we bound the total amount by which predictions fall on the wrong side of their margins.
+and
+$$
+\lambda_i = C-\gamma_i \hspace{0.1cm}\forall i.
+$$
-
-Misclassifications occur when \( \xi_i > 1 \). Thus bounding the total sum by some value \( C \) bounds in turn the total number of
-misclassifications.
+Inserting these constraints into the equation for \( {\cal L} \) we obtain the same equation as before
+$$
+{\cal L}=\sum_i\lambda_i-\frac{1}{2}\sum_{ij}^n\lambda_i\lambda_jy_iy_j\boldsymbol{x}_i^T\boldsymbol{x}_j,
+$$
+
+but now subject to the constraints \( \lambda_i\geq 0 \), \( \sum_i\lambda_iy_i=0 \) and \( 0\leq\lambda_i \leq C \).
+We must in addition satisfy the Karush-Kuhn-Tucker condition which now reads
+$$
+\lambda_i\left[y_i(\boldsymbol{w}^T\boldsymbol{x}_i+b) -(1-\xi_)\right]=0 \hspace{0.1cm}\forall i,
+$$
+
+$$
+\gamma_i\xi_i = 0,
+$$
+
+and
+$$
+y_i(\boldsymbol{w}^T\boldsymbol{x}_i+b) -(1-\xi_) \geq 0 \hspace{0.1cm}\forall i.
+$$
@@ -218,8 +235,6 @@ misclassifications.
29
30
31
- ...
- 32
»
diff --git a/doc/pub/week47/html/._week47-bs022.html b/doc/pub/week47/html/._week47-bs022.html
index e22e4cf7a..ba925b75a 100644
--- a/doc/pub/week47/html/._week47-bs022.html
+++ b/doc/pub/week47/html/._week47-bs022.html
@@ -53,31 +53,30 @@ Automatically generated HTML file from DocOnce source
('Getting into the details', 2, None, '___sec9'),
('First attempt at a minimization approach', 2, None, '___sec10'),
('Solving the equations', 2, None, '___sec11'),
- ('Code Example', 2, None, '___sec12'),
- ('Problems with the Simpler Approach', 2, None, '___sec13'),
- ('A better approach', 2, None, '___sec14'),
+ ('Can we code this?', 2, None, '___sec12'),
+ ('A better approach', 2, None, '___sec13'),
('A quick Reminder on Lagrangian Multipliers',
2,
None,
- '___sec15'),
- ('Adding the Multiplier', 2, None, '___sec16'),
- ('Setting up the Problem', 2, None, '___sec17'),
- ('The problem to solve', 2, None, '___sec18'),
- ('The last steps', 2, None, '___sec19'),
- ('A soft classifier', 2, None, '___sec20'),
- ('Soft optmization problem', 2, None, '___sec21'),
- ('Kernels and non-linearity', 2, None, '___sec22'),
- ('The equations', 2, None, '___sec23'),
- ('The problem to solve', 2, None, '___sec24'),
- ("Different kernels and Mercer's theorem", 2, None, '___sec25'),
- ('The moons example', 2, None, '___sec26'),
+ '___sec14'),
+ ('Adding the Multiplier', 2, None, '___sec15'),
+ ('Setting up the Problem', 2, None, '___sec16'),
+ ('The problem to solve', 2, None, '___sec17'),
+ ('The last steps', 2, None, '___sec18'),
+ ('A soft classifier', 2, None, '___sec19'),
+ ('Soft optmization problem', 2, None, '___sec20'),
+ ('Kernels and non-linearity', 2, None, '___sec21'),
+ ('The equations', 2, None, '___sec22'),
+ ('The problem to solve', 2, None, '___sec23'),
+ ("Different kernels and Mercer's theorem", 2, None, '___sec24'),
+ ('The moons example', 2, None, '___sec25'),
('Mathematical optimization of convex functions',
2,
None,
- '___sec27'),
- ('How do we solve these problems?', 2, None, '___sec28'),
- ('A simple example', 2, None, '___sec29'),
- ('Back to the more realistic cases', 2, None, '___sec30')]}
+ '___sec26'),
+ ('How do we solve these problems?', 2, None, '___sec27'),
+ ('A simple example', 2, None, '___sec28'),
+ ('Back to the more realistic cases', 2, None, '___sec29')]}
end of tocinfo -->
@@ -127,25 +126,24 @@ MathJax.Hub.Config({
Getting into the details
First attempt at a minimization approach
Solving the equations
- Code Example
- Problems with the Simpler Approach
- A better approach
- A quick Reminder on Lagrangian Multipliers
- Adding the Multiplier
- Setting up the Problem
- The problem to solve
- The last steps
- A soft classifier
- Soft optmization problem
- Kernels and non-linearity
- The equations
- The problem to solve
- Different kernels and Mercer's theorem
- The moons example
- Mathematical optimization of convex functions
- How do we solve these problems?
- A simple example
- Back to the more realistic cases
+ Can we code this?
+ A better approach
+ A quick Reminder on Lagrangian Multipliers
+ Adding the Multiplier
+ Setting up the Problem
+ The problem to solve
+ The last steps
+ A soft classifier
+ Soft optmization problem
+ Kernels and non-linearity
+ The equations
+ The problem to solve
+ Different kernels and Mercer's theorem
+ The moons example
+ Mathematical optimization of convex functions
+ How do we solve these problems?
+ A simple example
+ Back to the more realistic cases
@@ -161,57 +159,76 @@ MathJax.Hub.Config({
-Soft optmization problem
+Kernels and non-linearity
-This has in turn the consequences that we change our optmization problem to finding the minimum of
-$$
-{\cal L}=\frac{1}{2}\boldsymbol{w}^T\boldsymbol{w}-\sum_{i=1}^n\lambda_i\left[y_i(\boldsymbol{w}^T\boldsymbol{x}_i+b)-(1-\xi_)\right]+C\sum_{i=1}^n\xi_i-\sum_{i=1}^n\gamma_i\xi_i,
-$$
-
-subject to
-$$
-y_i(\boldsymbol{w}^T\boldsymbol{x}_i+b)=1-\xi_i \hspace{0.1cm}\forall i,
-$$
-
-with the requirement \( \xi_i\geq 0 \).
+The cases we have studied till now, were all characterized by two classes
+with a close to linear separability. The classifiers we have described
+so far find linear boundaries in our input feature space. It is
+possible to make our procedure more flexible by exploring the feature
+space using other basis expansions such as higher-order polynomials,
+wavelets, splines etc.
-Taking the derivatives with respect to \( b \) and \( \boldsymbol{w} \) we obtain
-$$
-\frac{\partial {\cal L}}{\partial b} = -\sum_{i} \lambda_iy_i=0,
-$$
+If our feature space is not easy to separate, as shown in the figure
+here, we can achieve a better separation by introducing more complex
+basis functions. The ideal would be, as shown in the next figure, to, via a specific transformation to
+obtain a separation between the classes which is almost linear.
-and
-$$
-\frac{\partial {\cal L}}{\partial \boldsymbol{w}} = 0 = \boldsymbol{w}-\sum_{i} \lambda_iy_i\boldsymbol{x}_i,
-$$
+
+The change of basis, from \( x\rightarrow z=\phi(x) \) leads to the same type of equations to be solved, except that
+we need to introduce for example a polynomial transformation to a two-dimensional training set.
-and
-$$
-\lambda_i = C-\gamma_i \hspace{0.1cm}\forall i.
-$$
+
-Inserting these constraints into the equation for \( {\cal L} \) we obtain the same equation as before
-$$
-{\cal L}=\sum_i\lambda_i-\frac{1}{2}\sum_{ij}^n\lambda_i\lambda_jy_iy_j\boldsymbol{x}_i^T\boldsymbol{x}_j,
-$$
+
+
import numpy as np
+import os
-but now subject to the constraints \( \lambda_i\geq 0 \), \( \sum_i\lambda_iy_i=0 \) and \( 0\leq\lambda_i \leq C \).
-We must in addition satisfy the Karush-Kuhn-Tucker condition which now reads
-$$
-\lambda_i\left[y_i(\boldsymbol{w}^T\boldsymbol{x}_i+b) -(1-\xi_)\right]=0 \hspace{0.1cm}\forall i,
-$$
+np.random.seed(42)
-$$
-\gamma_i\xi_i = 0,
-$$
+# To plot pretty figures
+import matplotlib
+import matplotlib.pyplot as plt
+plt.rcParams['axes.labelsize'] = 14
+plt.rcParams['xtick.labelsize'] = 12
+plt.rcParams['ytick.labelsize'] = 12
-and
-$$
-y_i(\boldsymbol{w}^T\boldsymbol{x}_i+b) -(1-\xi_) \geq 0 \hspace{0.1cm}\forall i.
-$$
+from sklearn.svm import SVC
+from sklearn import datasets
+
+
+
+X1D = np.linspace(-4, 4, 9).reshape(-1, 1)
+X2D = np.c_[X1D, X1D**2]
+y = np.array([0, 0, 1, 1, 1, 1, 1, 0, 0])
+
+plt.figure(figsize=(11, 4))
+
+plt.subplot(121)
+plt.grid(True, which='both')
+plt.axhline(y=0, color='k')
+plt.plot(X1D[:, 0][y==0], np.zeros(4), "bs")
+plt.plot(X1D[:, 0][y==1], np.zeros(5), "g^")
+plt.gca().get_yaxis().set_ticks([])
+plt.xlabel(r"$x_1$", fontsize=20)
+plt.axis([-4.5, 4.5, -0.2, 0.2])
+
+plt.subplot(122)
+plt.grid(True, which='both')
+plt.axhline(y=0, color='k')
+plt.axvline(x=0, color='k')
+plt.plot(X2D[:, 0][y==0], X2D[:, 1][y==0], "bs")
+plt.plot(X2D[:, 0][y==1], X2D[:, 1][y==1], "g^")
+plt.xlabel(r"$x_1$", fontsize=20)
+plt.ylabel(r"$x_2$", fontsize=20, rotation=0)
+plt.gca().get_yaxis().set_ticks([0, 4, 8, 12, 16])
+plt.plot([-4.5, 4.5], [6.5, 6.5], "r--", linewidth=3)
+plt.axis([-4.5, 4.5, -1, 17])
+plt.subplots_adjust(right=1)
+plt.show()
+
@@ -236,7 +253,6 @@ $$
29
30
31
- 32
»
diff --git a/doc/pub/week47/html/._week47-bs023.html b/doc/pub/week47/html/._week47-bs023.html
index 67433da6e..d1f76b67f 100644
--- a/doc/pub/week47/html/._week47-bs023.html
+++ b/doc/pub/week47/html/._week47-bs023.html
@@ -53,31 +53,30 @@ Automatically generated HTML file from DocOnce source
('Getting into the details', 2, None, '___sec9'),
('First attempt at a minimization approach', 2, None, '___sec10'),
('Solving the equations', 2, None, '___sec11'),
- ('Code Example', 2, None, '___sec12'),
- ('Problems with the Simpler Approach', 2, None, '___sec13'),
- ('A better approach', 2, None, '___sec14'),
+ ('Can we code this?', 2, None, '___sec12'),
+ ('A better approach', 2, None, '___sec13'),
('A quick Reminder on Lagrangian Multipliers',
2,
None,
- '___sec15'),
- ('Adding the Multiplier', 2, None, '___sec16'),
- ('Setting up the Problem', 2, None, '___sec17'),
- ('The problem to solve', 2, None, '___sec18'),
- ('The last steps', 2, None, '___sec19'),
- ('A soft classifier', 2, None, '___sec20'),
- ('Soft optmization problem', 2, None, '___sec21'),
- ('Kernels and non-linearity', 2, None, '___sec22'),
- ('The equations', 2, None, '___sec23'),
- ('The problem to solve', 2, None, '___sec24'),
- ("Different kernels and Mercer's theorem", 2, None, '___sec25'),
- ('The moons example', 2, None, '___sec26'),
+ '___sec14'),
+ ('Adding the Multiplier', 2, None, '___sec15'),
+ ('Setting up the Problem', 2, None, '___sec16'),
+ ('The problem to solve', 2, None, '___sec17'),
+ ('The last steps', 2, None, '___sec18'),
+ ('A soft classifier', 2, None, '___sec19'),
+ ('Soft optmization problem', 2, None, '___sec20'),
+ ('Kernels and non-linearity', 2, None, '___sec21'),
+ ('The equations', 2, None, '___sec22'),
+ ('The problem to solve', 2, None, '___sec23'),
+ ("Different kernels and Mercer's theorem", 2, None, '___sec24'),
+ ('The moons example', 2, None, '___sec25'),
('Mathematical optimization of convex functions',
2,
None,
- '___sec27'),
- ('How do we solve these problems?', 2, None, '___sec28'),
- ('A simple example', 2, None, '___sec29'),
- ('Back to the more realistic cases', 2, None, '___sec30')]}
+ '___sec26'),
+ ('How do we solve these problems?', 2, None, '___sec27'),
+ ('A simple example', 2, None, '___sec28'),
+ ('Back to the more realistic cases', 2, None, '___sec29')]}
end of tocinfo -->
@@ -127,25 +126,24 @@ MathJax.Hub.Config({
Getting into the details
First attempt at a minimization approach
Solving the equations
- Code Example
- Problems with the Simpler Approach
- A better approach
- A quick Reminder on Lagrangian Multipliers
- Adding the Multiplier
- Setting up the Problem
- The problem to solve
- The last steps
- A soft classifier
- Soft optmization problem
- Kernels and non-linearity
- The equations
- The problem to solve
- Different kernels and Mercer's theorem
- The moons example
- Mathematical optimization of convex functions
- How do we solve these problems?
- A simple example
- Back to the more realistic cases
+ Can we code this?
+ A better approach
+ A quick Reminder on Lagrangian Multipliers
+ Adding the Multiplier
+ Setting up the Problem
+ The problem to solve
+ The last steps
+ A soft classifier
+ Soft optmization problem
+ Kernels and non-linearity
+ The equations
+ The problem to solve
+ Different kernels and Mercer's theorem
+ The moons example
+ Mathematical optimization of convex functions
+ How do we solve these problems?
+ A simple example
+ Back to the more realistic cases
@@ -161,76 +159,48 @@ MathJax.Hub.Config({
-Kernels and non-linearity
+The equations
-The cases we have studied till now, were all characterized by two classes
-with a close to linear separability. The classifiers we have described
-so far find linear boundaries in our input feature space. It is
-possible to make our procedure more flexible by exploring the feature
-space using other basis expansions such as higher-order polynomials,
-wavelets, splines etc.
+Suppose we define a polynomial transformation of degree two only (we continue to live in a plane with \( x_i \) and \( y_i \) as variables)
+$$
+z = \phi(x_i) =\left(x_i^2, y_i^2, \sqrt{2}x_iy_i\right).
+$$
-If our feature space is not easy to separate, as shown in the figure
-here, we can achieve a better separation by introducing more complex
-basis functions. The ideal would be, as shown in the next figure, to, via a specific transformation to
-obtain a separation between the classes which is almost linear.
+With our new basis, the equations we solved earlier are basically the same, that is we have now (without the slack option for simplicity)
+$$
+{\cal L}=\sum_i\lambda_i-\frac{1}{2}\sum_{ij}^n\lambda_i\lambda_jy_iy_j\boldsymbol{z}_i^T\boldsymbol{z}_j,
+$$
+
+subject to the constraints \( \lambda_i\geq 0 \), \( \sum_i\lambda_iy_i=0 \), and for the support vectors
+$$
+y_i(\boldsymbol{w}^T\boldsymbol{z}_i+b)= 1 \hspace{0.1cm}\forall i,
+$$
+
+from which we also find \( b \).
+To compute \( \boldsymbol{z}_i^T\boldsymbol{z}_j \) we define the kernel \( K(\boldsymbol{x}_i,\boldsymbol{x}_j) \) as
+$$
+K(\boldsymbol{x}_i,\boldsymbol{x}_j)=\boldsymbol{z}_i^T\boldsymbol{z}_j= \phi(\boldsymbol{x}_i)^T\phi(\boldsymbol{x}_j).
+$$
+
+For the above example, the kernel reads
+$$
+K(\boldsymbol{x}_i,\boldsymbol{x}_j)=[x_i^2, y_i^2, \sqrt{2}x_iy_i]^T\begin{bmatrix} x_j^2 \\ y_j^2 \\ \sqrt{2}x_jy_j \end{bmatrix}=x_i^2x_j^2+2x_ix_jy_iy_j+y_i^2y_j^2.
+$$
-The change of basis, from \( x\rightarrow z=\phi(x) \) leads to the same type of equations to be solved, except that
-we need to introduce for example a polynomial transformation to a two-dimensional training set.
+We note that this is nothing but the dot product of the two original
+vectors \( (\boldsymbol{x}_i^T\boldsymbol{x}_j)^2 \). Instead of thus computing the
+product in the Lagrangian of \( \boldsymbol{z}_i^T\boldsymbol{z}_j \) we simply compute
+the dot product \( (\boldsymbol{x}_i^T\boldsymbol{x}_j)^2 \).
+This leads to the so-called
+kernel trick and the result leads to the same as if we went through
+the trouble of performing the transformation
+\( \phi(\boldsymbol{x}_i)^T\phi(\boldsymbol{x}_j) \) during the SVM calculations.
-
-
import numpy as np
-import os
-
-np.random.seed(42)
-
-# To plot pretty figures
-import matplotlib
-import matplotlib.pyplot as plt
-plt.rcParams['axes.labelsize'] = 14
-plt.rcParams['xtick.labelsize'] = 12
-plt.rcParams['ytick.labelsize'] = 12
-
-
-from sklearn.svm import SVC
-from sklearn import datasets
-
-
-
-X1D = np.linspace(-4, 4, 9).reshape(-1, 1)
-X2D = np.c_[X1D, X1D**2]
-y = np.array([0, 0, 1, 1, 1, 1, 1, 0, 0])
-
-plt.figure(figsize=(11, 4))
-
-plt.subplot(121)
-plt.grid(True, which='both')
-plt.axhline(y=0, color='k')
-plt.plot(X1D[:, 0][y==0], np.zeros(4), "bs")
-plt.plot(X1D[:, 0][y==1], np.zeros(5), "g^")
-plt.gca().get_yaxis().set_ticks([])
-plt.xlabel(r"$x_1$", fontsize=20)
-plt.axis([-4.5, 4.5, -0.2, 0.2])
-
-plt.subplot(122)
-plt.grid(True, which='both')
-plt.axhline(y=0, color='k')
-plt.axvline(x=0, color='k')
-plt.plot(X2D[:, 0][y==0], X2D[:, 1][y==0], "bs")
-plt.plot(X2D[:, 0][y==1], X2D[:, 1][y==1], "g^")
-plt.xlabel(r"$x_1$", fontsize=20)
-plt.ylabel(r"$x_2$", fontsize=20, rotation=0)
-plt.gca().get_yaxis().set_ticks([0, 4, 8, 12, 16])
-plt.plot([-4.5, 4.5], [6.5, 6.5], "r--", linewidth=3)
-plt.axis([-4.5, 4.5, -1, 17])
-plt.subplots_adjust(right=1)
-plt.show()
-
@@ -254,7 +224,6 @@ plt.show()
29
30
31
- 32
»
diff --git a/doc/pub/week47/html/._week47-bs024.html b/doc/pub/week47/html/._week47-bs024.html
index b33a5f4b5..2f88196bb 100644
--- a/doc/pub/week47/html/._week47-bs024.html
+++ b/doc/pub/week47/html/._week47-bs024.html
@@ -53,31 +53,30 @@ Automatically generated HTML file from DocOnce source
('Getting into the details', 2, None, '___sec9'),
('First attempt at a minimization approach', 2, None, '___sec10'),
('Solving the equations', 2, None, '___sec11'),
- ('Code Example', 2, None, '___sec12'),
- ('Problems with the Simpler Approach', 2, None, '___sec13'),
- ('A better approach', 2, None, '___sec14'),
+ ('Can we code this?', 2, None, '___sec12'),
+ ('A better approach', 2, None, '___sec13'),
('A quick Reminder on Lagrangian Multipliers',
2,
None,
- '___sec15'),
- ('Adding the Multiplier', 2, None, '___sec16'),
- ('Setting up the Problem', 2, None, '___sec17'),
- ('The problem to solve', 2, None, '___sec18'),
- ('The last steps', 2, None, '___sec19'),
- ('A soft classifier', 2, None, '___sec20'),
- ('Soft optmization problem', 2, None, '___sec21'),
- ('Kernels and non-linearity', 2, None, '___sec22'),
- ('The equations', 2, None, '___sec23'),
- ('The problem to solve', 2, None, '___sec24'),
- ("Different kernels and Mercer's theorem", 2, None, '___sec25'),
- ('The moons example', 2, None, '___sec26'),
+ '___sec14'),
+ ('Adding the Multiplier', 2, None, '___sec15'),
+ ('Setting up the Problem', 2, None, '___sec16'),
+ ('The problem to solve', 2, None, '___sec17'),
+ ('The last steps', 2, None, '___sec18'),
+ ('A soft classifier', 2, None, '___sec19'),
+ ('Soft optmization problem', 2, None, '___sec20'),
+ ('Kernels and non-linearity', 2, None, '___sec21'),
+ ('The equations', 2, None, '___sec22'),
+ ('The problem to solve', 2, None, '___sec23'),
+ ("Different kernels and Mercer's theorem", 2, None, '___sec24'),
+ ('The moons example', 2, None, '___sec25'),
('Mathematical optimization of convex functions',
2,
None,
- '___sec27'),
- ('How do we solve these problems?', 2, None, '___sec28'),
- ('A simple example', 2, None, '___sec29'),
- ('Back to the more realistic cases', 2, None, '___sec30')]}
+ '___sec26'),
+ ('How do we solve these problems?', 2, None, '___sec27'),
+ ('A simple example', 2, None, '___sec28'),
+ ('Back to the more realistic cases', 2, None, '___sec29')]}
end of tocinfo -->
@@ -127,25 +126,24 @@ MathJax.Hub.Config({
Getting into the details
First attempt at a minimization approach
Solving the equations
- Code Example
- Problems with the Simpler Approach
- A better approach
- A quick Reminder on Lagrangian Multipliers
- Adding the Multiplier
- Setting up the Problem
- The problem to solve
- The last steps
- A soft classifier
- Soft optmization problem
- Kernels and non-linearity
- The equations
- The problem to solve
- Different kernels and Mercer's theorem
- The moons example
- Mathematical optimization of convex functions
- How do we solve these problems?
- A simple example
- Back to the more realistic cases
+ Can we code this?
+ A better approach
+ A quick Reminder on Lagrangian Multipliers
+ Adding the Multiplier
+ Setting up the Problem
+ The problem to solve
+ The last steps
+ A soft classifier
+ Soft optmization problem
+ Kernels and non-linearity
+ The equations
+ The problem to solve
+ Different kernels and Mercer's theorem
+ The moons example
+ Mathematical optimization of convex functions
+ How do we solve these problems?
+ A simple example
+ Back to the more realistic cases
@@ -161,47 +159,38 @@ MathJax.Hub.Config({
-The equations
+The problem to solve
+Using our definition of the kernel We can rewrite again the Lagrangian
+$$
+{\cal L}=\sum_i\lambda_i-\frac{1}{2}\sum_{ij}^n\lambda_i\lambda_jy_iy_j\boldsymbol{x}_i^T\boldsymbol{z}_j,
+$$
+
+subject to the constraints \( \lambda_i\geq 0 \), \( \sum_i\lambda_iy_i=0 \) in terms of a convex optimization problem
+$$
+\frac{1}{2} \boldsymbol{\lambda}^T\begin{bmatrix} y_1y_1K(\boldsymbol{x}_1,\boldsymbol{x}_1) & y_1y_2K(\boldsymbol{x}_1,\boldsymbol{x}_2) & \dots & \dots & y_1y_nK(\boldsymbol{x}_1,\boldsymbol{x}_n) \\
+y_2y_1K(\boldsymbol{x}_2,\boldsymbol{x}_1) & y_2y_2(\boldsymbol{x}_2,\boldsymbol{x}_2) & \dots & \dots & y_1y_nK(\boldsymbol{x}_2,\boldsymbol{x}_n) \\
+\dots & \dots & \dots & \dots & \dots \\
+\dots & \dots & \dots & \dots & \dots \\
+y_ny_1K(\boldsymbol{x}_n,\boldsymbol{x}_1) & y_ny_2K(\boldsymbol{x}_n\boldsymbol{x}_2) & \dots & \dots & y_ny_nK(\boldsymbol{x}_n,\boldsymbol{x}_n) \\
+\end{bmatrix}\boldsymbol{\lambda}-\mathbb{1}\boldsymbol{\lambda},
+$$
+
+subject to \( \boldsymbol{y}^T\boldsymbol{\lambda}=0 \). Here we defined the vectors \( \boldsymbol{\lambda} =[\lambda_1,\lambda_2,\dots,\lambda_n] \) and
+\( \boldsymbol{y}=[y_1,y_2,\dots,y_n] \).
+If we add the slack constants this leads to the additional constraint \( 0\leq \lambda_i \leq C \).
-Suppose we define a polynomial transformation of degree two only (we continue to live in a plane with \( x_i \) and \( y_i \) as variables)
+We can rewrite this (see the solutions below) in terms of a convex optimization problem of the type
$$
-z = \phi(x_i) =\left(x_i^2, y_i^2, \sqrt{2}x_iy_i\right).
+\begin{align*}
+ &\mathrm{min}_{\lambda}\hspace{0.2cm} \frac{1}{2}\boldsymbol{\lambda}^T\boldsymbol{P}\boldsymbol{\lambda}+\boldsymbol{q}^T\boldsymbol{\lambda},\\ \nonumber
+ &\mathrm{subject\hspace{0.1cm}to} \hspace{0.2cm} \boldsymbol{G}\boldsymbol{\lambda} \preceq \boldsymbol{h} \hspace{0.2cm} \wedge \boldsymbol{A}\boldsymbol{\lambda}=f.
+\end{align*}
$$
-
-With our new basis, the equations we solved earlier are basically the same, that is we have now (without the slack option for simplicity)
-$$
-{\cal L}=\sum_i\lambda_i-\frac{1}{2}\sum_{ij}^n\lambda_i\lambda_jy_iy_j\boldsymbol{z}_i^T\boldsymbol{z}_j,
-$$
-
-subject to the constraints \( \lambda_i\geq 0 \), \( \sum_i\lambda_iy_i=0 \), and for the support vectors
-$$
-y_i(\boldsymbol{w}^T\boldsymbol{z}_i+b)= 1 \hspace{0.1cm}\forall i,
-$$
-
-from which we also find \( b \).
-To compute \( \boldsymbol{z}_i^T\boldsymbol{z}_j \) we define the kernel \( K(\boldsymbol{x}_i,\boldsymbol{x}_j) \) as
-$$
-K(\boldsymbol{x}_i,\boldsymbol{x}_j)=\boldsymbol{z}_i^T\boldsymbol{z}_j= \phi(\boldsymbol{x}_i)^T\phi(\boldsymbol{x}_j).
-$$
-
-For the above example, the kernel reads
-$$
-K(\boldsymbol{x}_i,\boldsymbol{x}_j)=[x_i^2, y_i^2, \sqrt{2}x_iy_i]^T\begin{bmatrix} x_j^2 \\ y_j^2 \\ \sqrt{2}x_jy_j \end{bmatrix}=x_i^2x_j^2+2x_ix_jy_iy_j+y_i^2y_j^2.
-$$
-
-
-We note that this is nothing but the dot product of the two original
-vectors \( (\boldsymbol{x}_i^T\boldsymbol{x}_j)^2 \). Instead of thus computing the
-product in the Lagrangian of \( \boldsymbol{z}_i^T\boldsymbol{z}_j \) we simply compute
-the dot product \( (\boldsymbol{x}_i^T\boldsymbol{x}_j)^2 \).
-
-
-This leads to the so-called
-kernel trick and the result leads to the same as if we went through
-the trouble of performing the transformation
-\( \phi(\boldsymbol{x}_i)^T\phi(\boldsymbol{x}_j) \) during the SVM calculations.
+Below we discuss how to solve these equations. Here we note that the matrix \( \boldsymbol{P} \) has matrix elements \( p_{ij}=y_iy_jK(\boldsymbol{x}_i,\boldsymbol{x}_j) \).
+Given a kernel \( K \) and the targets \( y_i \) this matrix is easy to set up. The constraint \( \boldsymbol{y}^T\boldsymbol{\lambda}=0 \) leads to \( f=0 \) and \( \boldsymbol{A}=\boldsymbol{y} \). How to set up the matrix \( \boldsymbol{G} \) is discussed later. Here note that the inequalities \( 0\leq \lambda_i \leq C \) can be split up into
+\( 0\leq \lambda_i \) and \( \lambda_i \leq C \). These two inequalities define then the matrix \( \boldsymbol{G} \) and the vector \( \boldsymbol{h} \).
@@ -225,7 +214,6 @@ the trouble of performing the transformation
29
30
31
- 32
»
diff --git a/doc/pub/week47/html/._week47-bs025.html b/doc/pub/week47/html/._week47-bs025.html
index b109dfad6..cc06971f2 100644
--- a/doc/pub/week47/html/._week47-bs025.html
+++ b/doc/pub/week47/html/._week47-bs025.html
@@ -53,31 +53,30 @@ Automatically generated HTML file from DocOnce source
('Getting into the details', 2, None, '___sec9'),
('First attempt at a minimization approach', 2, None, '___sec10'),
('Solving the equations', 2, None, '___sec11'),
- ('Code Example', 2, None, '___sec12'),
- ('Problems with the Simpler Approach', 2, None, '___sec13'),
- ('A better approach', 2, None, '___sec14'),
+ ('Can we code this?', 2, None, '___sec12'),
+ ('A better approach', 2, None, '___sec13'),
('A quick Reminder on Lagrangian Multipliers',
2,
None,
- '___sec15'),
- ('Adding the Multiplier', 2, None, '___sec16'),
- ('Setting up the Problem', 2, None, '___sec17'),
- ('The problem to solve', 2, None, '___sec18'),
- ('The last steps', 2, None, '___sec19'),
- ('A soft classifier', 2, None, '___sec20'),
- ('Soft optmization problem', 2, None, '___sec21'),
- ('Kernels and non-linearity', 2, None, '___sec22'),
- ('The equations', 2, None, '___sec23'),
- ('The problem to solve', 2, None, '___sec24'),
- ("Different kernels and Mercer's theorem", 2, None, '___sec25'),
- ('The moons example', 2, None, '___sec26'),
+ '___sec14'),
+ ('Adding the Multiplier', 2, None, '___sec15'),
+ ('Setting up the Problem', 2, None, '___sec16'),
+ ('The problem to solve', 2, None, '___sec17'),
+ ('The last steps', 2, None, '___sec18'),
+ ('A soft classifier', 2, None, '___sec19'),
+ ('Soft optmization problem', 2, None, '___sec20'),
+ ('Kernels and non-linearity', 2, None, '___sec21'),
+ ('The equations', 2, None, '___sec22'),
+ ('The problem to solve', 2, None, '___sec23'),
+ ("Different kernels and Mercer's theorem", 2, None, '___sec24'),
+ ('The moons example', 2, None, '___sec25'),
('Mathematical optimization of convex functions',
2,
None,
- '___sec27'),
- ('How do we solve these problems?', 2, None, '___sec28'),
- ('A simple example', 2, None, '___sec29'),
- ('Back to the more realistic cases', 2, None, '___sec30')]}
+ '___sec26'),
+ ('How do we solve these problems?', 2, None, '___sec27'),
+ ('A simple example', 2, None, '___sec28'),
+ ('Back to the more realistic cases', 2, None, '___sec29')]}
end of tocinfo -->
@@ -127,25 +126,24 @@ MathJax.Hub.Config({
Getting into the details
First attempt at a minimization approach
Solving the equations
- Code Example
- Problems with the Simpler Approach
- A better approach
- A quick Reminder on Lagrangian Multipliers
- Adding the Multiplier
- Setting up the Problem
- The problem to solve
- The last steps
- A soft classifier
- Soft optmization problem
- Kernels and non-linearity
- The equations
- The problem to solve
- Different kernels and Mercer's theorem
- The moons example
- Mathematical optimization of convex functions
- How do we solve these problems?
- A simple example
- Back to the more realistic cases
+ Can we code this?
+ A better approach
+ A quick Reminder on Lagrangian Multipliers
+ Adding the Multiplier
+ Setting up the Problem
+ The problem to solve
+ The last steps
+ A soft classifier
+ Soft optmization problem
+ Kernels and non-linearity
+ The equations
+ The problem to solve
+ Different kernels and Mercer's theorem
+ The moons example
+ Mathematical optimization of convex functions
+ How do we solve these problems?
+ A simple example
+ Back to the more realistic cases
@@ -161,38 +159,40 @@ MathJax.Hub.Config({
-The problem to solve
-Using our definition of the kernel We can rewrite again the Lagrangian
-$$
-{\cal L}=\sum_i\lambda_i-\frac{1}{2}\sum_{ij}^n\lambda_i\lambda_jy_iy_j\boldsymbol{x}_i^T\boldsymbol{z}_j,
-$$
-
-subject to the constraints \( \lambda_i\geq 0 \), \( \sum_i\lambda_iy_i=0 \) in terms of a convex optimization problem
-$$
-\frac{1}{2} \boldsymbol{\lambda}^T\begin{bmatrix} y_1y_1K(\boldsymbol{x}_1,\boldsymbol{x}_1) & y_1y_2K(\boldsymbol{x}_1,\boldsymbol{x}_2) & \dots & \dots & y_1y_nK(\boldsymbol{x}_1,\boldsymbol{x}_n) \\
-y_2y_1K(\boldsymbol{x}_2,\boldsymbol{x}_1) & y_2y_2(\boldsymbol{x}_2,\boldsymbol{x}_2) & \dots & \dots & y_1y_nK(\boldsymbol{x}_2,\boldsymbol{x}_n) \\
-\dots & \dots & \dots & \dots & \dots \\
-\dots & \dots & \dots & \dots & \dots \\
-y_ny_1K(\boldsymbol{x}_n,\boldsymbol{x}_1) & y_ny_2K(\boldsymbol{x}_n\boldsymbol{x}_2) & \dots & \dots & y_ny_nK(\boldsymbol{x}_n,\boldsymbol{x}_n) \\
-\end{bmatrix}\boldsymbol{\lambda}-\mathbb{1}\boldsymbol{\lambda},
-$$
-
-subject to \( \boldsymbol{y}^T\boldsymbol{\lambda}=0 \). Here we defined the vectors \( \boldsymbol{\lambda} =[\lambda_1,\lambda_2,\dots,\lambda_n] \) and
-\( \boldsymbol{y}=[y_1,y_2,\dots,y_n] \).
-If we add the slack constants this leads to the additional constraint \( 0\leq \lambda_i \leq C \).
+Different kernels and Mercer's theorem
-We can rewrite this (see the solutions below) in terms of a convex optimization problem of the type
+There are several popular kernels being used. These are
+
+
+- Linear: \( K(\boldsymbol{x},\boldsymbol{y})=\boldsymbol{x}^T\boldsymbol{y} \),
+- Polynomial: \( K(\boldsymbol{x},\boldsymbol{y})=(\boldsymbol{x}^T\boldsymbol{y}+\gamma)^d \),
+- Gaussian Radial Basis Function: \( K(\boldsymbol{x},\boldsymbol{y})=\exp{\left(-\gamma\vert\vert\boldsymbol{x}-\boldsymbol{y}\vert\vert^2\right)} \),
+- Tanh: \( K(\boldsymbol{x},\boldsymbol{y})=\tanh{(\boldsymbol{x}^T\boldsymbol{y}+\gamma)} \),
+
+
+and many other ones.
+
+
+An important theorem for us is Mercer's
+theorem. The
+theorem states that if a kernel function \( K \) is symmetric, continuous
+and leads to a positive semi-definite matrix \( \boldsymbol{P} \) then there
+exists a function \( \phi \) that maps \( \boldsymbol{x}_i \) and \( \boldsymbol{x}_j \) into
+another space (possibly with much higher dimensions) such that
+
$$
-\begin{align*}
- &\mathrm{min}_{\lambda}\hspace{0.2cm} \frac{1}{2}\boldsymbol{\lambda}^T\boldsymbol{P}\boldsymbol{\lambda}+\boldsymbol{q}^T\boldsymbol{\lambda},\\ \nonumber
- &\mathrm{subject\hspace{0.1cm}to} \hspace{0.2cm} \boldsymbol{G}\boldsymbol{\lambda} \preceq \boldsymbol{h} \hspace{0.2cm} \wedge \boldsymbol{A}\boldsymbol{\lambda}=f.
-\end{align*}
+K(\boldsymbol{x}_i,\boldsymbol{x}_j)=\phi(\boldsymbol{x}_i)^T\phi(\boldsymbol{x}_j).
$$
-Below we discuss how to solve these equations. Here we note that the matrix \( \boldsymbol{P} \) has matrix elements \( p_{ij}=y_iy_jK(\boldsymbol{x}_i,\boldsymbol{x}_j) \).
-Given a kernel \( K \) and the targets \( y_i \) this matrix is easy to set up. The constraint \( \boldsymbol{y}^T\boldsymbol{\lambda}=0 \) leads to \( f=0 \) and \( \boldsymbol{A}=\boldsymbol{y} \). How to set up the matrix \( \boldsymbol{G} \) is discussed later. Here note that the inequalities \( 0\leq \lambda_i \leq C \) can be split up into
-\( 0\leq \lambda_i \) and \( \lambda_i \leq C \). These two inequalities define then the matrix \( \boldsymbol{G} \) and the vector \( \boldsymbol{h} \).
+
+So you can use \( K \) as a kernel since you know \( \phi \) exists, even if
+you don’t know what \( \phi \) is.
+
+
+Note that some frequently used kernels (such as the Sigmoid kernel)
+don’t respect all of Mercer’s conditions, yet they generally work well
+in practice.
@@ -215,7 +215,6 @@ Given a kernel \( K \) and the targets \( y_i \) this matrix is easy to set up.
29
30
31
- 32
»
diff --git a/doc/pub/week47/html/._week47-bs026.html b/doc/pub/week47/html/._week47-bs026.html
index 43a5e09c6..2b1ec9580 100644
--- a/doc/pub/week47/html/._week47-bs026.html
+++ b/doc/pub/week47/html/._week47-bs026.html
@@ -53,31 +53,30 @@ Automatically generated HTML file from DocOnce source
('Getting into the details', 2, None, '___sec9'),
('First attempt at a minimization approach', 2, None, '___sec10'),
('Solving the equations', 2, None, '___sec11'),
- ('Code Example', 2, None, '___sec12'),
- ('Problems with the Simpler Approach', 2, None, '___sec13'),
- ('A better approach', 2, None, '___sec14'),
+ ('Can we code this?', 2, None, '___sec12'),
+ ('A better approach', 2, None, '___sec13'),
('A quick Reminder on Lagrangian Multipliers',
2,
None,
- '___sec15'),
- ('Adding the Multiplier', 2, None, '___sec16'),
- ('Setting up the Problem', 2, None, '___sec17'),
- ('The problem to solve', 2, None, '___sec18'),
- ('The last steps', 2, None, '___sec19'),
- ('A soft classifier', 2, None, '___sec20'),
- ('Soft optmization problem', 2, None, '___sec21'),
- ('Kernels and non-linearity', 2, None, '___sec22'),
- ('The equations', 2, None, '___sec23'),
- ('The problem to solve', 2, None, '___sec24'),
- ("Different kernels and Mercer's theorem", 2, None, '___sec25'),
- ('The moons example', 2, None, '___sec26'),
+ '___sec14'),
+ ('Adding the Multiplier', 2, None, '___sec15'),
+ ('Setting up the Problem', 2, None, '___sec16'),
+ ('The problem to solve', 2, None, '___sec17'),
+ ('The last steps', 2, None, '___sec18'),
+ ('A soft classifier', 2, None, '___sec19'),
+ ('Soft optmization problem', 2, None, '___sec20'),
+ ('Kernels and non-linearity', 2, None, '___sec21'),
+ ('The equations', 2, None, '___sec22'),
+ ('The problem to solve', 2, None, '___sec23'),
+ ("Different kernels and Mercer's theorem", 2, None, '___sec24'),
+ ('The moons example', 2, None, '___sec25'),
('Mathematical optimization of convex functions',
2,
None,
- '___sec27'),
- ('How do we solve these problems?', 2, None, '___sec28'),
- ('A simple example', 2, None, '___sec29'),
- ('Back to the more realistic cases', 2, None, '___sec30')]}
+ '___sec26'),
+ ('How do we solve these problems?', 2, None, '___sec27'),
+ ('A simple example', 2, None, '___sec28'),
+ ('Back to the more realistic cases', 2, None, '___sec29')]}
end of tocinfo -->
@@ -127,25 +126,24 @@ MathJax.Hub.Config({
Getting into the details
First attempt at a minimization approach
Solving the equations
- Code Example
- Problems with the Simpler Approach
- A better approach
- A quick Reminder on Lagrangian Multipliers
- Adding the Multiplier
- Setting up the Problem
- The problem to solve
- The last steps
- A soft classifier
- Soft optmization problem
- Kernels and non-linearity
- The equations
- The problem to solve
- Different kernels and Mercer's theorem
- The moons example
- Mathematical optimization of convex functions
- How do we solve these problems?
- A simple example
- Back to the more realistic cases
+ Can we code this?
+ A better approach
+ A quick Reminder on Lagrangian Multipliers
+ Adding the Multiplier
+ Setting up the Problem
+ The problem to solve
+ The last steps
+ A soft classifier
+ Soft optmization problem
+ Kernels and non-linearity
+ The equations
+ The problem to solve
+ Different kernels and Mercer's theorem
+ The moons example
+ Mathematical optimization of convex functions
+ How do we solve these problems?
+ A simple example
+ Back to the more realistic cases
@@ -161,41 +159,199 @@ MathJax.Hub.Config({
-Different kernels and Mercer's theorem
-
+The moons example
-There are several popular kernels being used. These are
-
-- Linear: \( K(\boldsymbol{x},\boldsymbol{y})=\boldsymbol{x}^T\boldsymbol{y} \),
-- Polynomial: \( K(\boldsymbol{x},\boldsymbol{y})=(\boldsymbol{x}^T\boldsymbol{y}+\gamma)^d \),
-- Gaussian Radial Basis Function: \( K(\boldsymbol{x},\boldsymbol{y})=\exp{\left(-\gamma\vert\vert\boldsymbol{x}-\boldsymbol{y}\vert\vert^2\right)} \),
-- Tanh: \( K(\boldsymbol{x},\boldsymbol{y})=\tanh{(\boldsymbol{x}^T\boldsymbol{y}+\gamma)} \),
-
+
+from __future__ import division, print_function, unicode_literals
-and many other ones.
+import numpy as np
+np.random.seed(42)
-
-An important theorem for us is Mercer's
-theorem. The
-theorem states that if a kernel function \( K \) is symmetric, continuous
-and leads to a positive semi-definite matrix \( \boldsymbol{P} \) then there
-exists a function \( \phi \) that maps \( \boldsymbol{x}_i \) and \( \boldsymbol{x}_j \) into
-another space (possibly with much higher dimensions) such that
+import matplotlib
+import matplotlib.pyplot as plt
+plt.rcParams['axes.labelsize'] = 14
+plt.rcParams['xtick.labelsize'] = 12
+plt.rcParams['ytick.labelsize'] = 12
-$$
-K(\boldsymbol{x}_i,\boldsymbol{x}_j)=\phi(\boldsymbol{x}_i)^T\phi(\boldsymbol{x}_j).
-$$
-
-So you can use \( K \) as a kernel since you know \( \phi \) exists, even if
-you don’t know what \( \phi \) is.
+from sklearn.svm import SVC
+from sklearn import datasets
-
-Note that some frequently used kernels (such as the Sigmoid kernel)
-don’t respect all of Mercer’s conditions, yet they generally work well
-in practice.
+
+from sklearn.pipeline import Pipeline
+from sklearn.preprocessing import StandardScaler
+from sklearn.svm import LinearSVC
+
+
+from sklearn.datasets import make_moons
+X, y = make_moons(n_samples=100, noise=0.15, random_state=42)
+
+def plot_dataset(X, y, axes):
+ plt.plot(X[:, 0][y==0], X[:, 1][y==0], "bs")
+ plt.plot(X[:, 0][y==1], X[:, 1][y==1], "g^")
+ plt.axis(axes)
+ plt.grid(True, which='both')
+ plt.xlabel(r"$x_1$", fontsize=20)
+ plt.ylabel(r"$x_2$", fontsize=20, rotation=0)
+
+plot_dataset(X, y, [-1.5, 2.5, -1, 1.5])
+plt.show()
+
+from sklearn.datasets import make_moons
+from sklearn.pipeline import Pipeline
+from sklearn.preprocessing import PolynomialFeatures
+
+polynomial_svm_clf = Pipeline([
+ ("poly_features", PolynomialFeatures(degree=3)),
+ ("scaler", StandardScaler()),
+ ("svm_clf", LinearSVC(C=10, loss="hinge", random_state=42))
+ ])
+
+polynomial_svm_clf.fit(X, y)
+
+def plot_predictions(clf, axes):
+ x0s = np.linspace(axes[0], axes[1], 100)
+ x1s = np.linspace(axes[2], axes[3], 100)
+ x0, x1 = np.meshgrid(x0s, x1s)
+ X = np.c_[x0.ravel(), x1.ravel()]
+ y_pred = clf.predict(X).reshape(x0.shape)
+ y_decision = clf.decision_function(X).reshape(x0.shape)
+ plt.contourf(x0, x1, y_pred, cmap=plt.cm.brg, alpha=0.2)
+ plt.contourf(x0, x1, y_decision, cmap=plt.cm.brg, alpha=0.1)
+
+plot_predictions(polynomial_svm_clf, [-1.5, 2.5, -1, 1.5])
+plot_dataset(X, y, [-1.5, 2.5, -1, 1.5])
+
+plt.show()
+
+
+from sklearn.svm import SVC
+
+poly_kernel_svm_clf = Pipeline([
+ ("scaler", StandardScaler()),
+ ("svm_clf", SVC(kernel="poly", degree=3, coef0=1, C=5))
+ ])
+poly_kernel_svm_clf.fit(X, y)
+
+poly100_kernel_svm_clf = Pipeline([
+ ("scaler", StandardScaler()),
+ ("svm_clf", SVC(kernel="poly", degree=10, coef0=100, C=5))
+ ])
+poly100_kernel_svm_clf.fit(X, y)
+
+plt.figure(figsize=(11, 4))
+
+plt.subplot(121)
+plot_predictions(poly_kernel_svm_clf, [-1.5, 2.5, -1, 1.5])
+plot_dataset(X, y, [-1.5, 2.5, -1, 1.5])
+plt.title(r"$d=3, r=1, C=5$", fontsize=18)
+
+plt.subplot(122)
+plot_predictions(poly100_kernel_svm_clf, [-1.5, 2.5, -1, 1.5])
+plot_dataset(X, y, [-1.5, 2.5, -1, 1.5])
+plt.title(r"$d=10, r=100, C=5$", fontsize=18)
+
+plt.show()
+
+def gaussian_rbf(x, landmark, gamma):
+ return np.exp(-gamma * np.linalg.norm(x - landmark, axis=1)**2)
+
+gamma = 0.3
+
+x1s = np.linspace(-4.5, 4.5, 200).reshape(-1, 1)
+x2s = gaussian_rbf(x1s, -2, gamma)
+x3s = gaussian_rbf(x1s, 1, gamma)
+
+XK = np.c_[gaussian_rbf(X1D, -2, gamma), gaussian_rbf(X1D, 1, gamma)]
+yk = np.array([0, 0, 1, 1, 1, 1, 1, 0, 0])
+
+plt.figure(figsize=(11, 4))
+
+plt.subplot(121)
+plt.grid(True, which='both')
+plt.axhline(y=0, color='k')
+plt.scatter(x=[-2, 1], y=[0, 0], s=150, alpha=0.5, c="red")
+plt.plot(X1D[:, 0][yk==0], np.zeros(4), "bs")
+plt.plot(X1D[:, 0][yk==1], np.zeros(5), "g^")
+plt.plot(x1s, x2s, "g--")
+plt.plot(x1s, x3s, "b:")
+plt.gca().get_yaxis().set_ticks([0, 0.25, 0.5, 0.75, 1])
+plt.xlabel(r"$x_1$", fontsize=20)
+plt.ylabel(r"Similarity", fontsize=14)
+plt.annotate(r'$\mathbf{x}$',
+ xy=(X1D[3, 0], 0),
+ xytext=(-0.5, 0.20),
+ ha="center",
+ arrowprops=dict(facecolor='black', shrink=0.1),
+ fontsize=18,
+ )
+plt.text(-2, 0.9, "$x_2$", ha="center", fontsize=20)
+plt.text(1, 0.9, "$x_3$", ha="center", fontsize=20)
+plt.axis([-4.5, 4.5, -0.1, 1.1])
+
+plt.subplot(122)
+plt.grid(True, which='both')
+plt.axhline(y=0, color='k')
+plt.axvline(x=0, color='k')
+plt.plot(XK[:, 0][yk==0], XK[:, 1][yk==0], "bs")
+plt.plot(XK[:, 0][yk==1], XK[:, 1][yk==1], "g^")
+plt.xlabel(r"$x_2$", fontsize=20)
+plt.ylabel(r"$x_3$ ", fontsize=20, rotation=0)
+plt.annotate(r'$\phi\left(\mathbf{x}\right)$',
+ xy=(XK[3, 0], XK[3, 1]),
+ xytext=(0.65, 0.50),
+ ha="center",
+ arrowprops=dict(facecolor='black', shrink=0.1),
+ fontsize=18,
+ )
+plt.plot([-0.1, 1.1], [0.57, -0.1], "r--", linewidth=3)
+plt.axis([-0.1, 1.1, -0.1, 1.1])
+
+plt.subplots_adjust(right=1)
+
+plt.show()
+
+
+x1_example = X1D[3, 0]
+for landmark in (-2, 1):
+ k = gaussian_rbf(np.array([[x1_example]]), np.array([[landmark]]), gamma)
+ print("Phi({}, {}) = {}".format(x1_example, landmark, k))
+
+rbf_kernel_svm_clf = Pipeline([
+ ("scaler", StandardScaler()),
+ ("svm_clf", SVC(kernel="rbf", gamma=5, C=0.001))
+ ])
+rbf_kernel_svm_clf.fit(X, y)
+
+
+from sklearn.svm import SVC
+
+gamma1, gamma2 = 0.1, 5
+C1, C2 = 0.001, 1000
+hyperparams = (gamma1, C1), (gamma1, C2), (gamma2, C1), (gamma2, C2)
+
+svm_clfs = []
+for gamma, C in hyperparams:
+ rbf_kernel_svm_clf = Pipeline([
+ ("scaler", StandardScaler()),
+ ("svm_clf", SVC(kernel="rbf", gamma=gamma, C=C))
+ ])
+ rbf_kernel_svm_clf.fit(X, y)
+ svm_clfs.append(rbf_kernel_svm_clf)
+
+plt.figure(figsize=(11, 7))
+
+for i, svm_clf in enumerate(svm_clfs):
+ plt.subplot(221 + i)
+ plot_predictions(svm_clf, [-1.5, 2.5, -1, 1.5])
+ plot_dataset(X, y, [-1.5, 2.5, -1, 1.5])
+ gamma, C = hyperparams[i]
+ plt.title(r"$\gamma = {}, C = {}$".format(gamma, C), fontsize=16)
+
+plt.show()
+
@@ -216,7 +372,6 @@ in practice.
29
30
31
- 32
»
diff --git a/doc/pub/week47/html/._week47-bs027.html b/doc/pub/week47/html/._week47-bs027.html
index 4891a99d3..3aac06e09 100644
--- a/doc/pub/week47/html/._week47-bs027.html
+++ b/doc/pub/week47/html/._week47-bs027.html
@@ -53,31 +53,30 @@ Automatically generated HTML file from DocOnce source
('Getting into the details', 2, None, '___sec9'),
('First attempt at a minimization approach', 2, None, '___sec10'),
('Solving the equations', 2, None, '___sec11'),
- ('Code Example', 2, None, '___sec12'),
- ('Problems with the Simpler Approach', 2, None, '___sec13'),
- ('A better approach', 2, None, '___sec14'),
+ ('Can we code this?', 2, None, '___sec12'),
+ ('A better approach', 2, None, '___sec13'),
('A quick Reminder on Lagrangian Multipliers',
2,
None,
- '___sec15'),
- ('Adding the Multiplier', 2, None, '___sec16'),
- ('Setting up the Problem', 2, None, '___sec17'),
- ('The problem to solve', 2, None, '___sec18'),
- ('The last steps', 2, None, '___sec19'),
- ('A soft classifier', 2, None, '___sec20'),
- ('Soft optmization problem', 2, None, '___sec21'),
- ('Kernels and non-linearity', 2, None, '___sec22'),
- ('The equations', 2, None, '___sec23'),
- ('The problem to solve', 2, None, '___sec24'),
- ("Different kernels and Mercer's theorem", 2, None, '___sec25'),
- ('The moons example', 2, None, '___sec26'),
+ '___sec14'),
+ ('Adding the Multiplier', 2, None, '___sec15'),
+ ('Setting up the Problem', 2, None, '___sec16'),
+ ('The problem to solve', 2, None, '___sec17'),
+ ('The last steps', 2, None, '___sec18'),
+ ('A soft classifier', 2, None, '___sec19'),
+ ('Soft optmization problem', 2, None, '___sec20'),
+ ('Kernels and non-linearity', 2, None, '___sec21'),
+ ('The equations', 2, None, '___sec22'),
+ ('The problem to solve', 2, None, '___sec23'),
+ ("Different kernels and Mercer's theorem", 2, None, '___sec24'),
+ ('The moons example', 2, None, '___sec25'),
('Mathematical optimization of convex functions',
2,
None,
- '___sec27'),
- ('How do we solve these problems?', 2, None, '___sec28'),
- ('A simple example', 2, None, '___sec29'),
- ('Back to the more realistic cases', 2, None, '___sec30')]}
+ '___sec26'),
+ ('How do we solve these problems?', 2, None, '___sec27'),
+ ('A simple example', 2, None, '___sec28'),
+ ('Back to the more realistic cases', 2, None, '___sec29')]}
end of tocinfo -->
@@ -127,25 +126,24 @@ MathJax.Hub.Config({
Getting into the details
First attempt at a minimization approach
Solving the equations
- Code Example
- Problems with the Simpler Approach
- A better approach
- A quick Reminder on Lagrangian Multipliers
- Adding the Multiplier
- Setting up the Problem
- The problem to solve
- The last steps
- A soft classifier
- Soft optmization problem
- Kernels and non-linearity
- The equations
- The problem to solve
- Different kernels and Mercer's theorem
- The moons example
- Mathematical optimization of convex functions
- How do we solve these problems?
- A simple example
- Back to the more realistic cases
+ Can we code this?
+ A better approach
+ A quick Reminder on Lagrangian Multipliers
+ Adding the Multiplier
+ Setting up the Problem
+ The problem to solve
+ The last steps
+ A soft classifier
+ Soft optmization problem
+ Kernels and non-linearity
+ The equations
+ The problem to solve
+ Different kernels and Mercer's theorem
+ The moons example
+ Mathematical optimization of convex functions
+ How do we solve these problems?
+ A simple example
+ Back to the more realistic cases
@@ -161,199 +159,28 @@ MathJax.Hub.Config({
-The moons example
+Mathematical optimization of convex functions
+
+A mathematical (quadratic) optimization problem, or just optimization problem, has the form
+$$
+\begin{align*}
+ &\mathrm{min}_{\lambda}\hspace{0.2cm} \frac{1}{2}\boldsymbol{\lambda}^T\boldsymbol{P}\boldsymbol{\lambda}+\boldsymbol{q}^T\boldsymbol{\lambda},\\ \nonumber
+ &\mathrm{subject\hspace{0.1cm}to} \hspace{0.2cm} \boldsymbol{G}\boldsymbol{\lambda} \preceq \boldsymbol{h} \wedge \boldsymbol{A}\boldsymbol{\lambda}=f.
+\end{align*}
+$$
-
-
from __future__ import division, print_function, unicode_literals
+subject to some constraints for say a selected set \( i=1,2,\dots, n \).
+In our case we are optimizing with respect to the Lagrangian multipliers \( \lambda_i \), and the
+vector \( \boldsymbol{\lambda}=[\lambda_1, \lambda_2,\dots, \lambda_n] \) is the optimization variable we are dealing with.
-import numpy as np
-np.random.seed(42)
+
+In our case we are particularly interested in a class of optimization problems called convex optmization problems.
+In our discussion on gradient descent methods we discussed at length the definition of a convex function.
-import matplotlib
-import matplotlib.pyplot as plt
-plt.rcParams['axes.labelsize'] = 14
-plt.rcParams['xtick.labelsize'] = 12
-plt.rcParams['ytick.labelsize'] = 12
+
+Convex optimization problems play a central role in applied mathematics and we recommend strongly Boyd and Vandenberghe's text on the topics.
-
-from sklearn.svm import SVC
-from sklearn import datasets
-
-
-
-from sklearn.pipeline import Pipeline
-from sklearn.preprocessing import StandardScaler
-from sklearn.svm import LinearSVC
-
-
-from sklearn.datasets import make_moons
-X, y = make_moons(n_samples=100, noise=0.15, random_state=42)
-
-def plot_dataset(X, y, axes):
- plt.plot(X[:, 0][y==0], X[:, 1][y==0], "bs")
- plt.plot(X[:, 0][y==1], X[:, 1][y==1], "g^")
- plt.axis(axes)
- plt.grid(True, which='both')
- plt.xlabel(r"$x_1$", fontsize=20)
- plt.ylabel(r"$x_2$", fontsize=20, rotation=0)
-
-plot_dataset(X, y, [-1.5, 2.5, -1, 1.5])
-plt.show()
-
-from sklearn.datasets import make_moons
-from sklearn.pipeline import Pipeline
-from sklearn.preprocessing import PolynomialFeatures
-
-polynomial_svm_clf = Pipeline([
- ("poly_features", PolynomialFeatures(degree=3)),
- ("scaler", StandardScaler()),
- ("svm_clf", LinearSVC(C=10, loss="hinge", random_state=42))
- ])
-
-polynomial_svm_clf.fit(X, y)
-
-def plot_predictions(clf, axes):
- x0s = np.linspace(axes[0], axes[1], 100)
- x1s = np.linspace(axes[2], axes[3], 100)
- x0, x1 = np.meshgrid(x0s, x1s)
- X = np.c_[x0.ravel(), x1.ravel()]
- y_pred = clf.predict(X).reshape(x0.shape)
- y_decision = clf.decision_function(X).reshape(x0.shape)
- plt.contourf(x0, x1, y_pred, cmap=plt.cm.brg, alpha=0.2)
- plt.contourf(x0, x1, y_decision, cmap=plt.cm.brg, alpha=0.1)
-
-plot_predictions(polynomial_svm_clf, [-1.5, 2.5, -1, 1.5])
-plot_dataset(X, y, [-1.5, 2.5, -1, 1.5])
-
-plt.show()
-
-
-from sklearn.svm import SVC
-
-poly_kernel_svm_clf = Pipeline([
- ("scaler", StandardScaler()),
- ("svm_clf", SVC(kernel="poly", degree=3, coef0=1, C=5))
- ])
-poly_kernel_svm_clf.fit(X, y)
-
-poly100_kernel_svm_clf = Pipeline([
- ("scaler", StandardScaler()),
- ("svm_clf", SVC(kernel="poly", degree=10, coef0=100, C=5))
- ])
-poly100_kernel_svm_clf.fit(X, y)
-
-plt.figure(figsize=(11, 4))
-
-plt.subplot(121)
-plot_predictions(poly_kernel_svm_clf, [-1.5, 2.5, -1, 1.5])
-plot_dataset(X, y, [-1.5, 2.5, -1, 1.5])
-plt.title(r"$d=3, r=1, C=5$", fontsize=18)
-
-plt.subplot(122)
-plot_predictions(poly100_kernel_svm_clf, [-1.5, 2.5, -1, 1.5])
-plot_dataset(X, y, [-1.5, 2.5, -1, 1.5])
-plt.title(r"$d=10, r=100, C=5$", fontsize=18)
-
-plt.show()
-
-def gaussian_rbf(x, landmark, gamma):
- return np.exp(-gamma * np.linalg.norm(x - landmark, axis=1)**2)
-
-gamma = 0.3
-
-x1s = np.linspace(-4.5, 4.5, 200).reshape(-1, 1)
-x2s = gaussian_rbf(x1s, -2, gamma)
-x3s = gaussian_rbf(x1s, 1, gamma)
-
-XK = np.c_[gaussian_rbf(X1D, -2, gamma), gaussian_rbf(X1D, 1, gamma)]
-yk = np.array([0, 0, 1, 1, 1, 1, 1, 0, 0])
-
-plt.figure(figsize=(11, 4))
-
-plt.subplot(121)
-plt.grid(True, which='both')
-plt.axhline(y=0, color='k')
-plt.scatter(x=[-2, 1], y=[0, 0], s=150, alpha=0.5, c="red")
-plt.plot(X1D[:, 0][yk==0], np.zeros(4), "bs")
-plt.plot(X1D[:, 0][yk==1], np.zeros(5), "g^")
-plt.plot(x1s, x2s, "g--")
-plt.plot(x1s, x3s, "b:")
-plt.gca().get_yaxis().set_ticks([0, 0.25, 0.5, 0.75, 1])
-plt.xlabel(r"$x_1$", fontsize=20)
-plt.ylabel(r"Similarity", fontsize=14)
-plt.annotate(r'$\mathbf{x}$',
- xy=(X1D[3, 0], 0),
- xytext=(-0.5, 0.20),
- ha="center",
- arrowprops=dict(facecolor='black', shrink=0.1),
- fontsize=18,
- )
-plt.text(-2, 0.9, "$x_2$", ha="center", fontsize=20)
-plt.text(1, 0.9, "$x_3$", ha="center", fontsize=20)
-plt.axis([-4.5, 4.5, -0.1, 1.1])
-
-plt.subplot(122)
-plt.grid(True, which='both')
-plt.axhline(y=0, color='k')
-plt.axvline(x=0, color='k')
-plt.plot(XK[:, 0][yk==0], XK[:, 1][yk==0], "bs")
-plt.plot(XK[:, 0][yk==1], XK[:, 1][yk==1], "g^")
-plt.xlabel(r"$x_2$", fontsize=20)
-plt.ylabel(r"$x_3$ ", fontsize=20, rotation=0)
-plt.annotate(r'$\phi\left(\mathbf{x}\right)$',
- xy=(XK[3, 0], XK[3, 1]),
- xytext=(0.65, 0.50),
- ha="center",
- arrowprops=dict(facecolor='black', shrink=0.1),
- fontsize=18,
- )
-plt.plot([-0.1, 1.1], [0.57, -0.1], "r--", linewidth=3)
-plt.axis([-0.1, 1.1, -0.1, 1.1])
-
-plt.subplots_adjust(right=1)
-
-plt.show()
-
-
-x1_example = X1D[3, 0]
-for landmark in (-2, 1):
- k = gaussian_rbf(np.array([[x1_example]]), np.array([[landmark]]), gamma)
- print("Phi({}, {}) = {}".format(x1_example, landmark, k))
-
-rbf_kernel_svm_clf = Pipeline([
- ("scaler", StandardScaler()),
- ("svm_clf", SVC(kernel="rbf", gamma=5, C=0.001))
- ])
-rbf_kernel_svm_clf.fit(X, y)
-
-
-from sklearn.svm import SVC
-
-gamma1, gamma2 = 0.1, 5
-C1, C2 = 0.001, 1000
-hyperparams = (gamma1, C1), (gamma1, C2), (gamma2, C1), (gamma2, C2)
-
-svm_clfs = []
-for gamma, C in hyperparams:
- rbf_kernel_svm_clf = Pipeline([
- ("scaler", StandardScaler()),
- ("svm_clf", SVC(kernel="rbf", gamma=gamma, C=C))
- ])
- rbf_kernel_svm_clf.fit(X, y)
- svm_clfs.append(rbf_kernel_svm_clf)
-
-plt.figure(figsize=(11, 7))
-
-for i, svm_clf in enumerate(svm_clfs):
- plt.subplot(221 + i)
- plot_predictions(svm_clf, [-1.5, 2.5, -1, 1.5])
- plot_dataset(X, y, [-1.5, 2.5, -1, 1.5])
- gamma, C = hyperparams[i]
- plt.title(r"$\gamma = {}, C = {}$".format(gamma, C), fontsize=16)
-
-plt.show()
-
@@ -373,7 +200,6 @@ plt.show()
29
30
31
- 32
»
diff --git a/doc/pub/week47/html/._week47-bs028.html b/doc/pub/week47/html/._week47-bs028.html
index da0f5666a..ca5315deb 100644
--- a/doc/pub/week47/html/._week47-bs028.html
+++ b/doc/pub/week47/html/._week47-bs028.html
@@ -53,31 +53,30 @@ Automatically generated HTML file from DocOnce source
('Getting into the details', 2, None, '___sec9'),
('First attempt at a minimization approach', 2, None, '___sec10'),
('Solving the equations', 2, None, '___sec11'),
- ('Code Example', 2, None, '___sec12'),
- ('Problems with the Simpler Approach', 2, None, '___sec13'),
- ('A better approach', 2, None, '___sec14'),
+ ('Can we code this?', 2, None, '___sec12'),
+ ('A better approach', 2, None, '___sec13'),
('A quick Reminder on Lagrangian Multipliers',
2,
None,
- '___sec15'),
- ('Adding the Multiplier', 2, None, '___sec16'),
- ('Setting up the Problem', 2, None, '___sec17'),
- ('The problem to solve', 2, None, '___sec18'),
- ('The last steps', 2, None, '___sec19'),
- ('A soft classifier', 2, None, '___sec20'),
- ('Soft optmization problem', 2, None, '___sec21'),
- ('Kernels and non-linearity', 2, None, '___sec22'),
- ('The equations', 2, None, '___sec23'),
- ('The problem to solve', 2, None, '___sec24'),
- ("Different kernels and Mercer's theorem", 2, None, '___sec25'),
- ('The moons example', 2, None, '___sec26'),
+ '___sec14'),
+ ('Adding the Multiplier', 2, None, '___sec15'),
+ ('Setting up the Problem', 2, None, '___sec16'),
+ ('The problem to solve', 2, None, '___sec17'),
+ ('The last steps', 2, None, '___sec18'),
+ ('A soft classifier', 2, None, '___sec19'),
+ ('Soft optmization problem', 2, None, '___sec20'),
+ ('Kernels and non-linearity', 2, None, '___sec21'),
+ ('The equations', 2, None, '___sec22'),
+ ('The problem to solve', 2, None, '___sec23'),
+ ("Different kernels and Mercer's theorem", 2, None, '___sec24'),
+ ('The moons example', 2, None, '___sec25'),
('Mathematical optimization of convex functions',
2,
None,
- '___sec27'),
- ('How do we solve these problems?', 2, None, '___sec28'),
- ('A simple example', 2, None, '___sec29'),
- ('Back to the more realistic cases', 2, None, '___sec30')]}
+ '___sec26'),
+ ('How do we solve these problems?', 2, None, '___sec27'),
+ ('A simple example', 2, None, '___sec28'),
+ ('Back to the more realistic cases', 2, None, '___sec29')]}
end of tocinfo -->
@@ -127,25 +126,24 @@ MathJax.Hub.Config({
Getting into the details
First attempt at a minimization approach
Solving the equations
- Code Example
- Problems with the Simpler Approach
- A better approach
- A quick Reminder on Lagrangian Multipliers
- Adding the Multiplier
- Setting up the Problem
- The problem to solve
- The last steps
- A soft classifier
- Soft optmization problem
- Kernels and non-linearity
- The equations
- The problem to solve
- Different kernels and Mercer's theorem
- The moons example
- Mathematical optimization of convex functions
- How do we solve these problems?
- A simple example
- Back to the more realistic cases
+ Can we code this?
+ A better approach
+ A quick Reminder on Lagrangian Multipliers
+ Adding the Multiplier
+ Setting up the Problem
+ The problem to solve
+ The last steps
+ A soft classifier
+ Soft optmization problem
+ Kernels and non-linearity
+ The equations
+ The problem to solve
+ Different kernels and Mercer's theorem
+ The moons example
+ Mathematical optimization of convex functions
+ How do we solve these problems?
+ A simple example
+ Back to the more realistic cases
@@ -161,27 +159,28 @@ MathJax.Hub.Config({
-Mathematical optimization of convex functions
+How do we solve these problems?
-A mathematical (quadratic) optimization problem, or just optimization problem, has the form
-$$
-\begin{align*}
- &\mathrm{min}_{\lambda}\hspace{0.2cm} \frac{1}{2}\boldsymbol{\lambda}^T\boldsymbol{P}\boldsymbol{\lambda}+\boldsymbol{q}^T\boldsymbol{\lambda},\\ \nonumber
- &\mathrm{subject\hspace{0.1cm}to} \hspace{0.2cm} \boldsymbol{G}\boldsymbol{\lambda} \preceq \boldsymbol{h} \wedge \boldsymbol{A}\boldsymbol{\lambda}=f.
-\end{align*}
-$$
-
-subject to some constraints for say a selected set \( i=1,2,\dots, n \).
-In our case we are optimizing with respect to the Lagrangian multipliers \( \lambda_i \), and the
-vector \( \boldsymbol{\lambda}=[\lambda_1, \lambda_2,\dots, \lambda_n] \) is the optimization variable we are dealing with.
+If we use Python as programming language and wish to venture beyond
+scikit-learn, tensorflow and similar software which makes our
+lives so much easier, we need to dive into the wonderful world of
+quadratic programming. We can, if we wish, solve the minimization
+problem using say standard gradient methods or conjugate gradient
+methods. However, these methods tend to exhibit a rather slow
+converge. So, welcome to the promised land of quadratic programming.
-In our case we are particularly interested in a class of optimization problems called convex optmization problems.
-In our discussion on gradient descent methods we discussed at length the definition of a convex function.
+The functions we need are contained in the quadratic programming package CVXOPT and we need to import it together with numpy as
-Convex optimization problems play a central role in applied mathematics and we recommend strongly Boyd and Vandenberghe's text on the topics.
+
+
+
import numpy
+import cvxopt
+
+This will make our life much easier. You don't need t write your own optimizer.
@@ -201,7 +200,6 @@ Convex optimization problems play a central role in applied mathematics and we r
29
30
31
- 32
»
diff --git a/doc/pub/week47/html/._week47-bs029.html b/doc/pub/week47/html/._week47-bs029.html
index ca9d4386f..d58fbe6e1 100644
--- a/doc/pub/week47/html/._week47-bs029.html
+++ b/doc/pub/week47/html/._week47-bs029.html
@@ -53,31 +53,30 @@ Automatically generated HTML file from DocOnce source
('Getting into the details', 2, None, '___sec9'),
('First attempt at a minimization approach', 2, None, '___sec10'),
('Solving the equations', 2, None, '___sec11'),
- ('Code Example', 2, None, '___sec12'),
- ('Problems with the Simpler Approach', 2, None, '___sec13'),
- ('A better approach', 2, None, '___sec14'),
+ ('Can we code this?', 2, None, '___sec12'),
+ ('A better approach', 2, None, '___sec13'),
('A quick Reminder on Lagrangian Multipliers',
2,
None,
- '___sec15'),
- ('Adding the Multiplier', 2, None, '___sec16'),
- ('Setting up the Problem', 2, None, '___sec17'),
- ('The problem to solve', 2, None, '___sec18'),
- ('The last steps', 2, None, '___sec19'),
- ('A soft classifier', 2, None, '___sec20'),
- ('Soft optmization problem', 2, None, '___sec21'),
- ('Kernels and non-linearity', 2, None, '___sec22'),
- ('The equations', 2, None, '___sec23'),
- ('The problem to solve', 2, None, '___sec24'),
- ("Different kernels and Mercer's theorem", 2, None, '___sec25'),
- ('The moons example', 2, None, '___sec26'),
+ '___sec14'),
+ ('Adding the Multiplier', 2, None, '___sec15'),
+ ('Setting up the Problem', 2, None, '___sec16'),
+ ('The problem to solve', 2, None, '___sec17'),
+ ('The last steps', 2, None, '___sec18'),
+ ('A soft classifier', 2, None, '___sec19'),
+ ('Soft optmization problem', 2, None, '___sec20'),
+ ('Kernels and non-linearity', 2, None, '___sec21'),
+ ('The equations', 2, None, '___sec22'),
+ ('The problem to solve', 2, None, '___sec23'),
+ ("Different kernels and Mercer's theorem", 2, None, '___sec24'),
+ ('The moons example', 2, None, '___sec25'),
('Mathematical optimization of convex functions',
2,
None,
- '___sec27'),
- ('How do we solve these problems?', 2, None, '___sec28'),
- ('A simple example', 2, None, '___sec29'),
- ('Back to the more realistic cases', 2, None, '___sec30')]}
+ '___sec26'),
+ ('How do we solve these problems?', 2, None, '___sec27'),
+ ('A simple example', 2, None, '___sec28'),
+ ('Back to the more realistic cases', 2, None, '___sec29')]}
end of tocinfo -->
@@ -127,25 +126,24 @@ MathJax.Hub.Config({
Getting into the details
First attempt at a minimization approach
Solving the equations
- Code Example
- Problems with the Simpler Approach
- A better approach
- A quick Reminder on Lagrangian Multipliers
- Adding the Multiplier
- Setting up the Problem
- The problem to solve
- The last steps
- A soft classifier
- Soft optmization problem
- Kernels and non-linearity
- The equations
- The problem to solve
- Different kernels and Mercer's theorem
- The moons example
- Mathematical optimization of convex functions
- How do we solve these problems?
- A simple example
- Back to the more realistic cases
+ Can we code this?
+ A better approach
+ A quick Reminder on Lagrangian Multipliers
+ Adding the Multiplier
+ Setting up the Problem
+ The problem to solve
+ The last steps
+ A soft classifier
+ Soft optmization problem
+ Kernels and non-linearity
+ The equations
+ The problem to solve
+ Different kernels and Mercer's theorem
+ The moons example
+ Mathematical optimization of convex functions
+ How do we solve these problems?
+ A simple example
+ Back to the more realistic cases
@@ -161,29 +159,71 @@ MathJax.Hub.Config({
-How do we solve these problems?
+A simple example
-If we use Python as programming language and wish to venture beyond
-scikit-learn, tensorflow and similar software which makes our
-lives so much easier, we need to dive into the wonderful world of
-quadratic programming. We can, if we wish, solve the minimization
-problem using say standard gradient methods or conjugate gradient
-methods. However, these methods tend to exhibit a rather slow
-converge. So, welcome to the promised land of quadratic programming.
+We remind ourselves about the general problem we want to solve
+$$
+\begin{align*}
+ &\mathrm{min}_{x}\hspace{0.2cm} \frac{1}{2}\boldsymbol{x}^T\boldsymbol{P}\boldsymbol{x}+\boldsymbol{q}^T\boldsymbol{x},\\ \nonumber
+ &\mathrm{subject\hspace{0.1cm} to} \hspace{0.2cm} \boldsymbol{G}\boldsymbol{x} \preceq \boldsymbol{h} \wedge \boldsymbol{A}\boldsymbol{x}=f.
+\end{align*}
+$$
-The functions we need are contained in the quadratic programming package CVXOPT and we need to import it together with numpy as
+Let us show how to perform the optmization using a simple case. Assume we want to optimize the following problem
+$$
+\begin{align*}
+ &\mathrm{min}_{x}\hspace{0.2cm} \frac{1}{2}x^2+5x+3y \\ \nonumber
+ &\mathrm{subject to} \\ \nonumber
+ &x, y \geq 0 \\ \nonumber
+ &x+3y \geq 15 \\ \nonumber
+ &2x+5y \leq 100 \\ \nonumber
+ &3x+4y \leq 80. \\ \nonumber
+\end{align*}
+$$
+The minimization problem can be rewritten in terms of vectors and matrices as (with \( x \) and \( y \) being the unknowns)
+$$
+\frac{1}{2}\begin{bmatrix} x\\ y \end{bmatrix}^T \begin{bmatrix} 1 & 0\\ 0 & 0 \end{bmatrix} \begin{bmatrix} x \\ y \end{bmatrix} + \begin{bmatrix}3\\ 4 \end{bmatrix}^T \begin{bmatrix}x \\ y \end{bmatrix}.
+$$
+
+Similarly, we can now set up the inequalities (we need to change \( \geq \) to \( \leq \) by multiplying with \( -1 \) on bot sides) as the following matrix-vector equation
+$$
+\begin{bmatrix} -1 & 0 \\ 0 & -1 \\ -1 & -3 \\ 2 & 5 \\ 3 & 4\end{bmatrix}\begin{bmatrix} x \\ y\end{bmatrix} \preceq \begin{bmatrix}0 \\ 0\\ -15 \\ 100 \\ 80\end{bmatrix}.
+$$
+
+We have collapsed all the inequalities into a single matrix \( \boldsymbol{G} \). We see also that our matrix
+$$
+\boldsymbol{P} =\begin{bmatrix} 1 & 0\\ 0 & 0 \end{bmatrix}
+$$
+
+is clearly positive semi-definite (all eigenvalues larger or equal zero).
+Finally, the vector \( \boldsymbol{h} \) is defined as
+$$
+\boldsymbol{h} = \begin{bmatrix}0 \\ 0\\ -15 \\ 100 \\ 80\end{bmatrix}.
+$$
+
+
+Since we don't have any equalities the matrix \( \boldsymbol{A} \) is set to zero
+The following code solves the equations for us
-
import numpy
-import cvxopt
+# Import the necessary packages
+import numpy
+from cvxopt import matrix
+from cvxopt import solvers
+P = matrix(numpy.diag([1,0]), tc=’d’)
+q = matrix(numpy.array([3,4]), tc=’d’)
+G = matrix(numpy.array([[-1,0],[0,-1],[-1,-3],[2,5],[3,4]]), tc=’d’)
+h = matrix(numpy.array([0,0,-15,100,80]), tc=’d’)
+# Construct the QP, invoke solver
+sol = solvers.qp(P,q,G,h)
+# Extract optimal value and solution
+sol[’x’]
+sol[’primal objective’]
-This will make our life much easier. You don't need t write your own optimizer.
-
@@ -201,7 +241,6 @@ This will make our life much easier. You don't need t write your own optimizer.
29
30
31
-
32
»
diff --git a/doc/pub/week47/html/._week47-bs030.html b/doc/pub/week47/html/._week47-bs030.html
index 1a7eb37b6..796dc2e6f 100644
--- a/doc/pub/week47/html/._week47-bs030.html
+++ b/doc/pub/week47/html/._week47-bs030.html
@@ -53,31 +53,30 @@ Automatically generated HTML file from DocOnce source
('Getting into the details', 2, None, '___sec9'),
('First attempt at a minimization approach', 2, None, '___sec10'),
('Solving the equations', 2, None, '___sec11'),
- ('Code Example', 2, None, '___sec12'),
- ('Problems with the Simpler Approach', 2, None, '___sec13'),
- ('A better approach', 2, None, '___sec14'),
+ ('Can we code this?', 2, None, '___sec12'),
+ ('A better approach', 2, None, '___sec13'),
('A quick Reminder on Lagrangian Multipliers',
2,
None,
- '___sec15'),
- ('Adding the Multiplier', 2, None, '___sec16'),
- ('Setting up the Problem', 2, None, '___sec17'),
- ('The problem to solve', 2, None, '___sec18'),
- ('The last steps', 2, None, '___sec19'),
- ('A soft classifier', 2, None, '___sec20'),
- ('Soft optmization problem', 2, None, '___sec21'),
- ('Kernels and non-linearity', 2, None, '___sec22'),
- ('The equations', 2, None, '___sec23'),
- ('The problem to solve', 2, None, '___sec24'),
- ("Different kernels and Mercer's theorem", 2, None, '___sec25'),
- ('The moons example', 2, None, '___sec26'),
+ '___sec14'),
+ ('Adding the Multiplier', 2, None, '___sec15'),
+ ('Setting up the Problem', 2, None, '___sec16'),
+ ('The problem to solve', 2, None, '___sec17'),
+ ('The last steps', 2, None, '___sec18'),
+ ('A soft classifier', 2, None, '___sec19'),
+ ('Soft optmization problem', 2, None, '___sec20'),
+ ('Kernels and non-linearity', 2, None, '___sec21'),
+ ('The equations', 2, None, '___sec22'),
+ ('The problem to solve', 2, None, '___sec23'),
+ ("Different kernels and Mercer's theorem", 2, None, '___sec24'),
+ ('The moons example', 2, None, '___sec25'),
('Mathematical optimization of convex functions',
2,
None,
- '___sec27'),
- ('How do we solve these problems?', 2, None, '___sec28'),
- ('A simple example', 2, None, '___sec29'),
- ('Back to the more realistic cases', 2, None, '___sec30')]}
+ '___sec26'),
+ ('How do we solve these problems?', 2, None, '___sec27'),
+ ('A simple example', 2, None, '___sec28'),
+ ('Back to the more realistic cases', 2, None, '___sec29')]}
end of tocinfo -->
@@ -127,25 +126,24 @@ MathJax.Hub.Config({
Getting into the details
First attempt at a minimization approach
Solving the equations
-
Code Example
-
Problems with the Simpler Approach
-
A better approach
-
A quick Reminder on Lagrangian Multipliers
-
Adding the Multiplier
-
Setting up the Problem
-
The problem to solve
-
The last steps
-
A soft classifier
-
Soft optmization problem
-
Kernels and non-linearity
-
The equations
-
The problem to solve
-
Different kernels and Mercer's theorem
-
The moons example
-
Mathematical optimization of convex functions
-
How do we solve these problems?
-
A simple example
-
Back to the more realistic cases
+
Can we code this?
+
A better approach
+
A quick Reminder on Lagrangian Multipliers
+
Adding the Multiplier
+
Setting up the Problem
+
The problem to solve
+
The last steps
+
A soft classifier
+
Soft optmization problem
+
Kernels and non-linearity
+
The equations
+
The problem to solve
+
Different kernels and Mercer's theorem
+
The moons example
+
Mathematical optimization of convex functions
+
How do we solve these problems?
+
A simple example
+
Back to the more realistic cases
@@ -161,72 +159,25 @@ MathJax.Hub.Config({
-
A simple example
+
Back to the more realistic cases
-We remind ourselves about the general problem we want to solve
+We are now ready to return to our setup of the optmization problem for a more realistic case. Introducing the slack parameter \( C \) we have
$$
-\begin{align*}
- &\mathrm{min}_{x}\hspace{0.2cm} \frac{1}{2}\boldsymbol{x}^T\boldsymbol{P}\boldsymbol{x}+\boldsymbol{q}^T\boldsymbol{x},\\ \nonumber
- &\mathrm{subject\hspace{0.1cm} to} \hspace{0.2cm} \boldsymbol{G}\boldsymbol{x} \preceq \boldsymbol{h} \wedge \boldsymbol{A}\boldsymbol{x}=f.
-\end{align*}
+\frac{1}{2} \boldsymbol{\lambda}^T\begin{bmatrix} y_1y_1K(\boldsymbol{x}_1,\boldsymbol{x}_1) & y_1y_2K(\boldsymbol{x}_1,\boldsymbol{x}_2) & \dots & \dots & y_1y_nK(\boldsymbol{x}_1,\boldsymbol{x}_n) \\
+y_2y_1K(\boldsymbol{x}_2,\boldsymbol{x}_1) & y_2y_2K(\boldsymbol{x}_2,\boldsymbol{x}_2) & \dots & \dots & y_1y_nK(\boldsymbol{x}_2,\boldsymbol{x}_n) \\
+\dots & \dots & \dots & \dots & \dots \\
+\dots & \dots & \dots & \dots & \dots \\
+y_ny_1K(\boldsymbol{x}_n,\boldsymbol{x}_1) & y_ny_2K(\boldsymbol{x}_n\boldsymbol{x}_2) & \dots & \dots & y_ny_nK(\boldsymbol{x}_n,\boldsymbol{x}_n) \\
+\end{bmatrix}\boldsymbol{\lambda}-\mathbb{I}\boldsymbol{\lambda},
$$
+subject to \( \boldsymbol{y}^T\boldsymbol{\lambda}=0 \). Here we defined the vectors \( \boldsymbol{\lambda} =[\lambda_1,\lambda_2,\dots,\lambda_n] \) and
+\( \boldsymbol{y}=[y_1,y_2,\dots,y_n] \).
+With the slack constants this leads to the additional constraint \( 0\leq \lambda_i \leq C \).
+
-Let us show how to perform the optmization using a simple case. Assume we want to optimize the following problem
-$$
-\begin{align*}
- &\mathrm{min}_{x}\hspace{0.2cm} \frac{1}{2}x^2+5x+3y \\ \nonumber
- &\mathrm{subject to} \\ \nonumber
- &x, y \geq 0 \\ \nonumber
- &x+3y \geq 15 \\ \nonumber
- &2x+5y \leq 100 \\ \nonumber
- &3x+4y \leq 80. \\ \nonumber
-\end{align*}
-$$
-The minimization problem can be rewritten in terms of vectors and matrices as (with \( x \) and \( y \) being the unknowns)
-$$
-\frac{1}{2}\begin{bmatrix} x\\ y \end{bmatrix}^T \begin{bmatrix} 1 & 0\\ 0 & 0 \end{bmatrix} \begin{bmatrix} x \\ y \end{bmatrix} + \begin{bmatrix}3\\ 4 \end{bmatrix}^T \begin{bmatrix}x \\ y \end{bmatrix}.
-$$
-
-Similarly, we can now set up the inequalities (we need to change \( \geq \) to \( \leq \) by multiplying with \( -1 \) on bot sides) as the following matrix-vector equation
-$$
-\begin{bmatrix} -1 & 0 \\ 0 & -1 \\ -1 & -3 \\ 2 & 5 \\ 3 & 4\end{bmatrix}\begin{bmatrix} x \\ y\end{bmatrix} \preceq \begin{bmatrix}0 \\ 0\\ -15 \\ 100 \\ 80\end{bmatrix}.
-$$
-
-We have collapsed all the inequalities into a single matrix \( \boldsymbol{G} \). We see also that our matrix
-$$
-\boldsymbol{P} =\begin{bmatrix} 1 & 0\\ 0 & 0 \end{bmatrix}
-$$
-
-is clearly positive semi-definite (all eigenvalues larger or equal zero).
-Finally, the vector \( \boldsymbol{h} \) is defined as
-$$
-\boldsymbol{h} = \begin{bmatrix}0 \\ 0\\ -15 \\ 100 \\ 80\end{bmatrix}.
-$$
-
-
-Since we don't have any equalities the matrix \( \boldsymbol{A} \) is set to zero
-The following code solves the equations for us
-
-
-
-
# Import the necessary packages
-import numpy
-from cvxopt import matrix
-from cvxopt import solvers
-P = matrix(numpy.diag([1,0]), tc=’d’)
-q = matrix(numpy.array([3,4]), tc=’d’)
-G = matrix(numpy.array([[-1,0],[0,-1],[-1,-3],[2,5],[3,4]]), tc=’d’)
-h = matrix(numpy.array([0,0,-15,100,80]), tc=’d’)
-# Construct the QP, invoke solver
-sol = solvers.qp(P,q,G,h)
-# Extract optimal value and solution
-sol[’x’]
-sol[’primal objective’]
-
-
diff --git a/doc/pub/week47/html/week47-bs.html b/doc/pub/week47/html/week47-bs.html
index e96c3a089..f6f7353f5 100644
--- a/doc/pub/week47/html/week47-bs.html
+++ b/doc/pub/week47/html/week47-bs.html
@@ -53,31 +53,30 @@ Automatically generated HTML file from DocOnce source
('Getting into the details', 2, None, '___sec9'),
('First attempt at a minimization approach', 2, None, '___sec10'),
('Solving the equations', 2, None, '___sec11'),
- ('Code Example', 2, None, '___sec12'),
- ('Problems with the Simpler Approach', 2, None, '___sec13'),
- ('A better approach', 2, None, '___sec14'),
+ ('Can we code this?', 2, None, '___sec12'),
+ ('A better approach', 2, None, '___sec13'),
('A quick Reminder on Lagrangian Multipliers',
2,
None,
- '___sec15'),
- ('Adding the Multiplier', 2, None, '___sec16'),
- ('Setting up the Problem', 2, None, '___sec17'),
- ('The problem to solve', 2, None, '___sec18'),
- ('The last steps', 2, None, '___sec19'),
- ('A soft classifier', 2, None, '___sec20'),
- ('Soft optmization problem', 2, None, '___sec21'),
- ('Kernels and non-linearity', 2, None, '___sec22'),
- ('The equations', 2, None, '___sec23'),
- ('The problem to solve', 2, None, '___sec24'),
- ("Different kernels and Mercer's theorem", 2, None, '___sec25'),
- ('The moons example', 2, None, '___sec26'),
+ '___sec14'),
+ ('Adding the Multiplier', 2, None, '___sec15'),
+ ('Setting up the Problem', 2, None, '___sec16'),
+ ('The problem to solve', 2, None, '___sec17'),
+ ('The last steps', 2, None, '___sec18'),
+ ('A soft classifier', 2, None, '___sec19'),
+ ('Soft optmization problem', 2, None, '___sec20'),
+ ('Kernels and non-linearity', 2, None, '___sec21'),
+ ('The equations', 2, None, '___sec22'),
+ ('The problem to solve', 2, None, '___sec23'),
+ ("Different kernels and Mercer's theorem", 2, None, '___sec24'),
+ ('The moons example', 2, None, '___sec25'),
('Mathematical optimization of convex functions',
2,
None,
- '___sec27'),
- ('How do we solve these problems?', 2, None, '___sec28'),
- ('A simple example', 2, None, '___sec29'),
- ('Back to the more realistic cases', 2, None, '___sec30')]}
+ '___sec26'),
+ ('How do we solve these problems?', 2, None, '___sec27'),
+ ('A simple example', 2, None, '___sec28'),
+ ('Back to the more realistic cases', 2, None, '___sec29')]}
end of tocinfo -->
@@ -127,25 +126,24 @@ MathJax.Hub.Config({
Getting into the details
First attempt at a minimization approach
Solving the equations
-
Code Example
-
Problems with the Simpler Approach
-
A better approach
-
A quick Reminder on Lagrangian Multipliers
-
Adding the Multiplier
-
Setting up the Problem
-
The problem to solve
-
The last steps
-
A soft classifier
-
Soft optmization problem
-
Kernels and non-linearity
-
The equations
-
The problem to solve
-
Different kernels and Mercer's theorem
-
The moons example
-
Mathematical optimization of convex functions
-
How do we solve these problems?
-
A simple example
-
Back to the more realistic cases
+
Can we code this?
+
A better approach
+
A quick Reminder on Lagrangian Multipliers
+
Adding the Multiplier
+
Setting up the Problem
+
The problem to solve
+
The last steps
+
A soft classifier
+
Soft optmization problem
+
Kernels and non-linearity
+
The equations
+
The problem to solve
+
Different kernels and Mercer's theorem
+
The moons example
+
Mathematical optimization of convex functions
+
How do we solve these problems?
+
A simple example
+
Back to the more realistic cases
@@ -204,7 +202,7 @@ MathJax.Hub.Config({
9
10
...
-
32
+
31
»
diff --git a/doc/pub/week47/html/week47-reveal.html b/doc/pub/week47/html/week47-reveal.html
index b29673c88..992d61b64 100644
--- a/doc/pub/week47/html/week47-reveal.html
+++ b/doc/pub/week47/html/week47-reveal.html
@@ -524,26 +524,19 @@ where \( \eta \) is our by now well-known learning rate.
-Code Example
+Can we code this?
The equations we discussed above can be coded rather easily (the
-framework is similar to what we developed for logistic
-regression). We are going to set up a simple case with two classes only and we want to find a line which separates them the best possible way.
-
-
-
-
-
-
-
-
-Problems with the Simpler Approach
+framework is similar to what we developed for logistic regression). We
+can set up a simple case with two classes only and we want to find a
+line which separates them the best possible way.
There are however problems with this approach, although it looks
-pretty straightforward to implement. When running the above code, we see that we can easily end up with many diffeent lines which separate the two classes.
+pretty straightforward to implement. When running a code for such a
+case we can easily end up with many diffeent lines which separate the
+two classes.
For small
@@ -555,7 +548,7 @@ at all.
-A better approach
+A better approach
A better approach is rather to try to define a large margin between
@@ -605,7 +598,7 @@ about Lagrangian multipliers.
-A quick Reminder on Lagrangian Multipliers
+A quick Reminder on Lagrangian Multipliers
Consider a function of three independent variables \( f(x,y,z) \) . For the function \( f \) to be an
@@ -667,7 +660,7 @@ Then \( dz \) is no longer arbitrary.
-Adding the Multiplier
+Adding the Multiplier
However, we can add to
@@ -722,7 +715,7 @@ $$
-Setting up the Problem
+Setting up the Problem
In order to solve the above problem, we define the following Lagrangian function to be minimized
$$
@@ -774,7 +767,7 @@ When \( \lambda_i > 0 \), the vectors \( \boldsymbol{x}_i \) are called support
-The problem to solve
+The problem to solve
We can rewrite
@@ -802,7 +795,7 @@ subject to \( \boldsymbol{y}^T\boldsymbol{\lambda}=0 \). Here we defined the vec
-The last steps
+The last steps
Solving the above problem, yields the values of \( \lambda_i \).
@@ -846,7 +839,7 @@ Below we discuss how to find the optimal values of \( \lambda_i \). Before we pr
-A soft classifier
+A soft classifier
Till now, the margin is strictly defined by the support vectors. This defines what is called a hard classifier, that is the margins are well defined.
@@ -885,7 +878,7 @@ misclassifications.
-Soft optmization problem
+Soft optmization problem
This has in turn the consequences that we change our optmization problem to finding the minimum of
@@ -957,7 +950,7 @@ $$
-Kernels and non-linearity
+Kernels and non-linearity
The cases we have studied till now, were all characterized by two classes
@@ -1031,7 +1024,7 @@ plt.show()
-The equations
+The equations
Suppose we define a polynomial transformation of degree two only (we continue to live in a plane with \( x_i \) and \( y_i \) as variables)
@@ -1086,7 +1079,7 @@ the trouble of performing the transformation
-The problem to solve
+The problem to solve
Using our definition of the kernel We can rewrite again the Lagrangian
$$
@@ -1128,7 +1121,7 @@ Given a kernel \( K \) and the targets \( y_i \) this matrix is easy to set up.
-Different kernels and Mercer's theorem
+Different kernels and Mercer's theorem
There are several popular kernels being used. These are
@@ -1169,7 +1162,7 @@ in practice.
-The moons example
+The moons example
@@ -1366,7 +1359,7 @@ plt.show()
-Mathematical optimization of convex functions
+Mathematical optimization of convex functions
A mathematical (quadratic) optimization problem, or just optimization problem, has the form
@@ -1393,7 +1386,7 @@ Convex optimization problems play a central role in applied mathematics and we r
-How do we solve these problems?
+How do we solve these problems?
If we use Python as programming language and wish to venture beyond
@@ -1419,7 +1412,7 @@ This will make our life much easier. You don't need t write your own optimizer.
-A simple example
+A simple example
We remind ourselves about the general problem we want to solve
@@ -1500,7 +1493,7 @@ sol[’primal obj
-Back to the more realistic cases
+Back to the more realistic cases
We are now ready to return to our setup of the optmization problem for a more realistic case. Introducing the slack parameter \( C \) we have
diff --git a/doc/pub/week47/html/week47-solarized.html b/doc/pub/week47/html/week47-solarized.html
index d63bf6582..c22722275 100644
--- a/doc/pub/week47/html/week47-solarized.html
+++ b/doc/pub/week47/html/week47-solarized.html
@@ -47,31 +47,30 @@ div { text-align: justify; text-justify: inter-word; }
('Getting into the details', 2, None, '___sec9'),
('First attempt at a minimization approach', 2, None, '___sec10'),
('Solving the equations', 2, None, '___sec11'),
- ('Code Example', 2, None, '___sec12'),
- ('Problems with the Simpler Approach', 2, None, '___sec13'),
- ('A better approach', 2, None, '___sec14'),
+ ('Can we code this?', 2, None, '___sec12'),
+ ('A better approach', 2, None, '___sec13'),
('A quick Reminder on Lagrangian Multipliers',
2,
None,
- '___sec15'),
- ('Adding the Multiplier', 2, None, '___sec16'),
- ('Setting up the Problem', 2, None, '___sec17'),
- ('The problem to solve', 2, None, '___sec18'),
- ('The last steps', 2, None, '___sec19'),
- ('A soft classifier', 2, None, '___sec20'),
- ('Soft optmization problem', 2, None, '___sec21'),
- ('Kernels and non-linearity', 2, None, '___sec22'),
- ('The equations', 2, None, '___sec23'),
- ('The problem to solve', 2, None, '___sec24'),
- ("Different kernels and Mercer's theorem", 2, None, '___sec25'),
- ('The moons example', 2, None, '___sec26'),
+ '___sec14'),
+ ('Adding the Multiplier', 2, None, '___sec15'),
+ ('Setting up the Problem', 2, None, '___sec16'),
+ ('The problem to solve', 2, None, '___sec17'),
+ ('The last steps', 2, None, '___sec18'),
+ ('A soft classifier', 2, None, '___sec19'),
+ ('Soft optmization problem', 2, None, '___sec20'),
+ ('Kernels and non-linearity', 2, None, '___sec21'),
+ ('The equations', 2, None, '___sec22'),
+ ('The problem to solve', 2, None, '___sec23'),
+ ("Different kernels and Mercer's theorem", 2, None, '___sec24'),
+ ('The moons example', 2, None, '___sec25'),
('Mathematical optimization of convex functions',
2,
None,
- '___sec27'),
- ('How do we solve these problems?', 2, None, '___sec28'),
- ('A simple example', 2, None, '___sec29'),
- ('Back to the more realistic cases', 2, None, '___sec30')]}
+ '___sec26'),
+ ('How do we solve these problems?', 2, None, '___sec27'),
+ ('A simple example', 2, None, '___sec28'),
+ ('Back to the more realistic cases', 2, None, '___sec29')]}
end of tocinfo -->
@@ -452,25 +451,19 @@ where \( \eta \) is our by now well-known learning rate.
-
Code Example
+Can we code this?
The equations we discussed above can be coded rather easily (the
-framework is similar to what we developed for logistic
-regression). We are going to set up a simple case with two classes only and we want to find a line which separates them the best possible way.
-
-
-
-
-
-
-
-
Problems with the Simpler Approach
+framework is similar to what we developed for logistic regression). We
+can set up a simple case with two classes only and we want to find a
+line which separates them the best possible way.
There are however problems with this approach, although it looks
-pretty straightforward to implement. When running the above code, we see that we can easily end up with many diffeent lines which separate the two classes.
+pretty straightforward to implement. When running a code for such a
+case we can easily end up with many diffeent lines which separate the
+two classes.
For small
@@ -482,7 +475,7 @@ at all.
-
A better approach
+A better approach
A better approach is rather to try to define a large margin between
@@ -524,7 +517,7 @@ about Lagrangian multipliers.
-
A quick Reminder on Lagrangian Multipliers
+A quick Reminder on Lagrangian Multipliers
Consider a function of three independent variables \( f(x,y,z) \) . For the function \( f \) to be an
@@ -574,7 +567,7 @@ Then \( dz \) is no longer arbitrary.
-
Adding the Multiplier
+Adding the Multiplier
However, we can add to
@@ -617,7 +610,7 @@ $$
-
Setting up the Problem
+Setting up the Problem
In order to solve the above problem, we define the following Lagrangian function to be minimized
$$
{\cal L}(\lambda,b,\boldsymbol{w})=\frac{1}{2}\boldsymbol{w}^T\boldsymbol{w}-\sum_{i=1}^n\lambda_i\left[y_i(\boldsymbol{w}^T\boldsymbol{x}_i+b)-1\right],
@@ -658,7 +651,7 @@ When \( \lambda_i > 0 \), the vectors \( \boldsymbol{x}_i \) are called support
-
The problem to solve
+The problem to solve
We can rewrite
@@ -682,7 +675,7 @@ subject to \( \boldsymbol{y}^T\boldsymbol{\lambda}=0 \). Here we defined the vec
-
The last steps
+The last steps
Solving the above problem, yields the values of \( \lambda_i \).
@@ -716,7 +709,7 @@ Below we discuss how to find the optimal values of \( \lambda_i \). Before we pr
-
A soft classifier
+A soft classifier
Till now, the margin is strictly defined by the support vectors. This defines what is called a hard classifier, that is the margins are well defined.
@@ -751,7 +744,7 @@ misclassifications.
-
Soft optmization problem
+Soft optmization problem
This has in turn the consequences that we change our optmization problem to finding the minimum of
@@ -805,7 +798,7 @@ $$
-
Kernels and non-linearity
+Kernels and non-linearity
The cases we have studied till now, were all characterized by two classes
@@ -878,7 +871,7 @@ plt.show()
-
The equations
+The equations
Suppose we define a polynomial transformation of degree two only (we continue to live in a plane with \( x_i \) and \( y_i \) as variables)
@@ -923,7 +916,7 @@ the trouble of performing the transformation
-
The problem to solve
+The problem to solve
Using our definition of the kernel We can rewrite again the Lagrangian
$$
{\cal L}=\sum_i\lambda_i-\frac{1}{2}\sum_{ij}^n\lambda_i\lambda_jy_iy_j\boldsymbol{x}_i^T\boldsymbol{z}_j,
@@ -959,7 +952,7 @@ Given a kernel \( K \) and the targets \( y_i \) this matrix is easy to set up.
-
Different kernels and Mercer's theorem
+Different kernels and Mercer's theorem
There are several popular kernels being used. These are
@@ -997,7 +990,7 @@ in practice.
-
The moons example
+The moons example
@@ -1193,7 +1186,7 @@ plt.show()
-
Mathematical optimization of convex functions
+Mathematical optimization of convex functions
A mathematical (quadratic) optimization problem, or just optimization problem, has the form
@@ -1218,7 +1211,7 @@ Convex optimization problems play a central role in applied mathematics and we r
-
How do we solve these problems?
+How do we solve these problems?
If we use Python as programming language and wish to venture beyond
@@ -1244,7 +1237,7 @@ This will make our life much easier. You don't need t write your own optimizer.
-
A simple example
+A simple example
We remind ourselves about the general problem we want to solve
@@ -1312,7 +1305,7 @@ sol[’primal obj
-
Back to the more realistic cases
+Back to the more realistic cases
We are now ready to return to our setup of the optmization problem for a more realistic case. Introducing the slack parameter \( C \) we have
diff --git a/doc/pub/week47/html/week47.html b/doc/pub/week47/html/week47.html
index 922be221c..5a755bc43 100644
--- a/doc/pub/week47/html/week47.html
+++ b/doc/pub/week47/html/week47.html
@@ -52,31 +52,30 @@ div { text-align: justify; text-justify: inter-word; }
('Getting into the details', 2, None, '___sec9'),
('First attempt at a minimization approach', 2, None, '___sec10'),
('Solving the equations', 2, None, '___sec11'),
- ('Code Example', 2, None, '___sec12'),
- ('Problems with the Simpler Approach', 2, None, '___sec13'),
- ('A better approach', 2, None, '___sec14'),
+ ('Can we code this?', 2, None, '___sec12'),
+ ('A better approach', 2, None, '___sec13'),
('A quick Reminder on Lagrangian Multipliers',
2,
None,
- '___sec15'),
- ('Adding the Multiplier', 2, None, '___sec16'),
- ('Setting up the Problem', 2, None, '___sec17'),
- ('The problem to solve', 2, None, '___sec18'),
- ('The last steps', 2, None, '___sec19'),
- ('A soft classifier', 2, None, '___sec20'),
- ('Soft optmization problem', 2, None, '___sec21'),
- ('Kernels and non-linearity', 2, None, '___sec22'),
- ('The equations', 2, None, '___sec23'),
- ('The problem to solve', 2, None, '___sec24'),
- ("Different kernels and Mercer's theorem", 2, None, '___sec25'),
- ('The moons example', 2, None, '___sec26'),
+ '___sec14'),
+ ('Adding the Multiplier', 2, None, '___sec15'),
+ ('Setting up the Problem', 2, None, '___sec16'),
+ ('The problem to solve', 2, None, '___sec17'),
+ ('The last steps', 2, None, '___sec18'),
+ ('A soft classifier', 2, None, '___sec19'),
+ ('Soft optmization problem', 2, None, '___sec20'),
+ ('Kernels and non-linearity', 2, None, '___sec21'),
+ ('The equations', 2, None, '___sec22'),
+ ('The problem to solve', 2, None, '___sec23'),
+ ("Different kernels and Mercer's theorem", 2, None, '___sec24'),
+ ('The moons example', 2, None, '___sec25'),
('Mathematical optimization of convex functions',
2,
None,
- '___sec27'),
- ('How do we solve these problems?', 2, None, '___sec28'),
- ('A simple example', 2, None, '___sec29'),
- ('Back to the more realistic cases', 2, None, '___sec30')]}
+ '___sec26'),
+ ('How do we solve these problems?', 2, None, '___sec27'),
+ ('A simple example', 2, None, '___sec28'),
+ ('Back to the more realistic cases', 2, None, '___sec29')]}
end of tocinfo -->
@@ -457,25 +456,19 @@ where \( \eta \) is our by now well-known learning rate.
-
Code Example
+Can we code this?
The equations we discussed above can be coded rather easily (the
-framework is similar to what we developed for logistic
-regression). We are going to set up a simple case with two classes only and we want to find a line which separates them the best possible way.
-
-
-
-
-
-
-
-
Problems with the Simpler Approach
+framework is similar to what we developed for logistic regression). We
+can set up a simple case with two classes only and we want to find a
+line which separates them the best possible way.
There are however problems with this approach, although it looks
-pretty straightforward to implement. When running the above code, we see that we can easily end up with many diffeent lines which separate the two classes.
+pretty straightforward to implement. When running a code for such a
+case we can easily end up with many diffeent lines which separate the
+two classes.
For small
@@ -487,7 +480,7 @@ at all.
-
A better approach
+A better approach
A better approach is rather to try to define a large margin between
@@ -529,7 +522,7 @@ about Lagrangian multipliers.
-
A quick Reminder on Lagrangian Multipliers
+A quick Reminder on Lagrangian Multipliers
Consider a function of three independent variables \( f(x,y,z) \) . For the function \( f \) to be an
@@ -579,7 +572,7 @@ Then \( dz \) is no longer arbitrary.
-
Adding the Multiplier
+Adding the Multiplier
However, we can add to
@@ -622,7 +615,7 @@ $$
-
Setting up the Problem
+Setting up the Problem
In order to solve the above problem, we define the following Lagrangian function to be minimized
$$
{\cal L}(\lambda,b,\boldsymbol{w})=\frac{1}{2}\boldsymbol{w}^T\boldsymbol{w}-\sum_{i=1}^n\lambda_i\left[y_i(\boldsymbol{w}^T\boldsymbol{x}_i+b)-1\right],
@@ -663,7 +656,7 @@ When \( \lambda_i > 0 \), the vectors \( \boldsymbol{x}_i \) are called support
-
The problem to solve
+The problem to solve
We can rewrite
@@ -687,7 +680,7 @@ subject to \( \boldsymbol{y}^T\boldsymbol{\lambda}=0 \). Here we defined the vec
-
The last steps
+The last steps
Solving the above problem, yields the values of \( \lambda_i \).
@@ -721,7 +714,7 @@ Below we discuss how to find the optimal values of \( \lambda_i \). Before we pr
-
A soft classifier
+A soft classifier
Till now, the margin is strictly defined by the support vectors. This defines what is called a hard classifier, that is the margins are well defined.
@@ -756,7 +749,7 @@ misclassifications.
-
Soft optmization problem
+Soft optmization problem
This has in turn the consequences that we change our optmization problem to finding the minimum of
@@ -810,7 +803,7 @@ $$
-
Kernels and non-linearity
+Kernels and non-linearity
The cases we have studied till now, were all characterized by two classes
@@ -883,7 +876,7 @@ plt.show()
-
The equations
+The equations
Suppose we define a polynomial transformation of degree two only (we continue to live in a plane with \( x_i \) and \( y_i \) as variables)
@@ -928,7 +921,7 @@ the trouble of performing the transformation
-
The problem to solve
+The problem to solve
Using our definition of the kernel We can rewrite again the Lagrangian
$$
{\cal L}=\sum_i\lambda_i-\frac{1}{2}\sum_{ij}^n\lambda_i\lambda_jy_iy_j\boldsymbol{x}_i^T\boldsymbol{z}_j,
@@ -964,7 +957,7 @@ Given a kernel \( K \) and the targets \( y_i \) this matrix is easy to set up.
-
Different kernels and Mercer's theorem
+Different kernels and Mercer's theorem
There are several popular kernels being used. These are
@@ -1002,7 +995,7 @@ in practice.
-
The moons example
+The moons example
@@ -1198,7 +1191,7 @@ plt.show()
-
Mathematical optimization of convex functions
+Mathematical optimization of convex functions
A mathematical (quadratic) optimization problem, or just optimization problem, has the form
@@ -1223,7 +1216,7 @@ Convex optimization problems play a central role in applied mathematics and we r
-
How do we solve these problems?
+How do we solve these problems?
If we use Python as programming language and wish to venture beyond
@@ -1249,7 +1242,7 @@ This will make our life much easier. You don't need t write your own optimizer.
-
A simple example
+A simple example
We remind ourselves about the general problem we want to solve
@@ -1317,7 +1310,7 @@ sol[’primal objective’]
-
Back to the more realistic cases
+Back to the more realistic cases
We are now ready to return to our setup of the optmization problem for a more realistic case. Introducing the slack parameter \( C \) we have
diff --git a/doc/pub/week47/ipynb/ipynb-week47-src.tar.gz b/doc/pub/week47/ipynb/ipynb-week47-src.tar.gz
index 1dc28a641..4cc101cd8 100644
Binary files a/doc/pub/week47/ipynb/ipynb-week47-src.tar.gz and b/doc/pub/week47/ipynb/ipynb-week47-src.tar.gz differ
diff --git a/doc/pub/week47/ipynb/week47.ipynb b/doc/pub/week47/ipynb/week47.ipynb
index 7760a5e3c..e1dcc93ce 100644
--- a/doc/pub/week47/ipynb/week47.ipynb
+++ b/doc/pub/week47/ipynb/week47.ipynb
@@ -463,22 +463,18 @@
"where $\\eta$ is our by now well-known learning rate. \n",
"\n",
"\n",
- "## Code Example\n",
+ "## Can we code this?\n",
"\n",
"The equations we discussed above can be coded rather easily (the\n",
- "framework is similar to what we developed for logistic\n",
- "regression). We are going to set up a simple case with two classes only and we want to find a line which separates them the best possible way."
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "## Problems with the Simpler Approach\n",
+ "framework is similar to what we developed for logistic regression). We\n",
+ "can set up a simple case with two classes only and we want to find a\n",
+ "line which separates them the best possible way.\n",
"\n",
"\n",
"There are however problems with this approach, although it looks\n",
- "pretty straightforward to implement. When running the above code, we see that we can easily end up with many diffeent lines which separate the two classes.\n",
+ "pretty straightforward to implement. When running a code for such a\n",
+ "case we can easily end up with many diffeent lines which separate the\n",
+ "two classes.\n",
"\n",
"\n",
"For small\n",
diff --git a/doc/src/week47/week47.do.txt b/doc/src/week47/week47.do.txt
index af70b14bd..76026cf62 100644
--- a/doc/src/week47/week47.do.txt
+++ b/doc/src/week47/week47.do.txt
@@ -313,21 +313,18 @@ where $\eta$ is our by now well-known learning rate.
!split
-===== Code Example =====
+===== Can we code this? =====
The equations we discussed above can be coded rather easily (the
-framework is similar to what we developed for logistic
-regression). We are going to set up a simple case with two classes only and we want to find a line which separates them the best possible way.
-!bc pycod
-
-!ec
-
-!split
-===== Problems with the Simpler Approach =====
+framework is similar to what we developed for logistic regression). We
+can set up a simple case with two classes only and we want to find a
+line which separates them the best possible way.
There are however problems with this approach, although it looks
-pretty straightforward to implement. When running the above code, we see that we can easily end up with many diffeent lines which separate the two classes.
+pretty straightforward to implement. When running a code for such a
+case we can easily end up with many diffeent lines which separate the
+two classes.
For small