317 lines
18 KiB
HTML
317 lines
18 KiB
HTML
<!--
|
|
Automatically generated HTML file from DocOnce source
|
|
(https://github.com/hplgit/doconce/)
|
|
-->
|
|
<html>
|
|
<head>
|
|
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
|
<meta name="generator" content="DocOnce: https://github.com/hplgit/doconce/" />
|
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
|
<meta name="description" content="Week 48: Support Vector Machines and Summary of course">
|
|
|
|
<title>Week 48: Support Vector Machines and Summary of course</title>
|
|
|
|
<!-- Bootstrap style: bootstrap -->
|
|
<link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
|
|
<!-- not necessary
|
|
<link href="https://netdna.bootstrapcdn.com/font-awesome/4.0.3/css/font-awesome.css" rel="stylesheet">
|
|
-->
|
|
|
|
<style type="text/css">
|
|
|
|
/* Add scrollbar to dropdown menus in bootstrap navigation bar */
|
|
.dropdown-menu {
|
|
height: auto;
|
|
max-height: 400px;
|
|
overflow-x: hidden;
|
|
}
|
|
|
|
/* Adds an invisible element before each target to offset for the navigation
|
|
bar */
|
|
.anchor::before {
|
|
content:"";
|
|
display:block;
|
|
height:50px; /* fixed header height for style bootstrap */
|
|
margin:-50px 0 0; /* negative fixed header height */
|
|
}
|
|
</style>
|
|
|
|
|
|
</head>
|
|
|
|
<!-- tocinfo
|
|
{'highest level': 2,
|
|
'sections': [('Overview of week 48', 2, None, '___sec0'),
|
|
('Thursday', 2, None, '___sec1'),
|
|
('Friday', 2, None, '___sec2'),
|
|
('Support Vector Machines, overarching aims', 2, None, '___sec3'),
|
|
('Kernels and non-linearity', 2, None, '___sec4'),
|
|
('The equations', 2, None, '___sec5'),
|
|
('The problem to solve', 2, None, '___sec6'),
|
|
("Different kernels and Mercer's theorem", 2, None, '___sec7'),
|
|
('The moons example', 2, None, '___sec8'),
|
|
('Mathematical optimization of convex functions',
|
|
2,
|
|
None,
|
|
'___sec9'),
|
|
('How do we solve these problems?', 2, None, '___sec10'),
|
|
('A simple example', 2, None, '___sec11'),
|
|
('Back to the more realistic cases', 2, None, '___sec12'),
|
|
('Summary of course', 2, None, '___sec13'),
|
|
('What? Me worry? No final exam in this course!',
|
|
2,
|
|
None,
|
|
'___sec14'),
|
|
('Topics we have covered this year', 2, None, '___sec15'),
|
|
('Statistical analysis and optimization of data',
|
|
2,
|
|
None,
|
|
'___sec16'),
|
|
('Machine learning', 2, None, '___sec17'),
|
|
('Learning outcomes and overarching aims of this course',
|
|
2,
|
|
None,
|
|
'___sec18'),
|
|
('Perspective on Machine Learning', 2, None, '___sec19'),
|
|
('Machine Learning Research', 2, None, '___sec20'),
|
|
('Starting your Machine Learning Project', 2, None, '___sec21'),
|
|
('Choose a Model and Algorithm', 2, None, '___sec22'),
|
|
('Preparing Your Data', 2, None, '___sec23'),
|
|
('Which Activation and Weights to Choose in Neural Networks',
|
|
2,
|
|
None,
|
|
'___sec24'),
|
|
('Optimization Methods and Hyperparameters', 2, None, '___sec25'),
|
|
('Resampling', 2, None, '___sec26'),
|
|
('Other courses on Data science and Machine Learning at UiO',
|
|
2,
|
|
None,
|
|
'___sec27'),
|
|
('Additional courses of interest', 2, None, '___sec28'),
|
|
("What's the future like?", 2, None, '___sec29'),
|
|
('Types of Machine Learning, a repetition', 2, None, '___sec30'),
|
|
('Why Boltzmann machines?', 2, None, '___sec31'),
|
|
('Boltzmann Machines', 2, None, '___sec32'),
|
|
('Some similarities and differences from DNNs',
|
|
2,
|
|
None,
|
|
'___sec33'),
|
|
('Boltzmann machines (BM)', 2, None, '___sec34'),
|
|
('A standard BM setup', 2, None, '___sec35'),
|
|
('The structure of the RBM network', 2, None, '___sec36'),
|
|
('The network', 2, None, '___sec37'),
|
|
('Goals', 2, None, '___sec38'),
|
|
('Joint distribution', 2, None, '___sec39'),
|
|
('Network Elements, the energy function', 2, None, '___sec40'),
|
|
('Defining different types of RBMs', 2, None, '___sec41'),
|
|
('More about RBMs', 2, None, '___sec42'),
|
|
('Autoencoders: Overarching view', 2, None, '___sec43'),
|
|
('Bayesian Machine Learning', 2, None, '___sec44'),
|
|
('Reinforcement Learning', 2, None, '___sec45'),
|
|
('Transfer learning', 2, None, '___sec46'),
|
|
('Adversarial learning', 2, None, '___sec47'),
|
|
('Dual learning', 2, None, '___sec48'),
|
|
('Distributed machine learning', 2, None, '___sec49'),
|
|
('Meta learning', 2, None, '___sec50'),
|
|
('The Challenges Facing Machine Learning', 2, None, '___sec51'),
|
|
('Explainable machine learning', 2, None, '___sec52'),
|
|
('Quantum machine learning', 2, None, '___sec53'),
|
|
('Quantum machine learning algorithms based on linear algebra',
|
|
2,
|
|
None,
|
|
'___sec54'),
|
|
('Quantum reinforcement learning', 2, None, '___sec55'),
|
|
('Quantum deep learning', 2, None, '___sec56'),
|
|
('Social machine learning', 2, None, '___sec57'),
|
|
('The last words?', 2, None, '___sec58'),
|
|
('Best wishes to you all and thanks so much for your heroic '
|
|
'efforts this semester',
|
|
2,
|
|
None,
|
|
'___sec59')]}
|
|
end of tocinfo -->
|
|
|
|
<body>
|
|
|
|
|
|
|
|
<script type="text/x-mathjax-config">
|
|
MathJax.Hub.Config({
|
|
TeX: {
|
|
equationNumbers: { autoNumber: "none" },
|
|
extensions: ["AMSmath.js", "AMSsymbols.js", "autobold.js", "color.js"]
|
|
}
|
|
});
|
|
</script>
|
|
<script type="text/javascript" async
|
|
src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
|
|
</script>
|
|
|
|
|
|
|
|
|
|
<!-- Bootstrap navigation bar -->
|
|
<div class="navbar navbar-default navbar-fixed-top">
|
|
<div class="navbar-header">
|
|
<button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-responsive-collapse">
|
|
<span class="icon-bar"></span>
|
|
<span class="icon-bar"></span>
|
|
<span class="icon-bar"></span>
|
|
</button>
|
|
<a class="navbar-brand" href="week48-bs.html">Week 48: Support Vector Machines and Summary of course</a>
|
|
</div>
|
|
|
|
<div class="navbar-collapse collapse navbar-responsive-collapse">
|
|
<ul class="nav navbar-nav navbar-right">
|
|
<li class="dropdown">
|
|
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
|
|
<ul class="dropdown-menu">
|
|
<!-- navigation toc: --> <li><a href="._week48-bs001.html#___sec0" style="font-size: 80%;">Overview of week 48</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week48-bs002.html#___sec1" style="font-size: 80%;">Thursday</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week48-bs003.html#___sec2" style="font-size: 80%;">Friday</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week48-bs004.html#___sec3" style="font-size: 80%;">Support Vector Machines, overarching aims</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week48-bs005.html#___sec4" style="font-size: 80%;">Kernels and non-linearity</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week48-bs006.html#___sec5" style="font-size: 80%;">The equations</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week48-bs007.html#___sec6" style="font-size: 80%;">The problem to solve</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week48-bs008.html#___sec7" style="font-size: 80%;">Different kernels and Mercer's theorem</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week48-bs009.html#___sec8" style="font-size: 80%;">The moons example</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week48-bs010.html#___sec9" style="font-size: 80%;">Mathematical optimization of convex functions</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week48-bs011.html#___sec10" style="font-size: 80%;">How do we solve these problems?</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week48-bs012.html#___sec11" style="font-size: 80%;">A simple example</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week48-bs013.html#___sec12" style="font-size: 80%;">Back to the more realistic cases</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week48-bs014.html#___sec13" style="font-size: 80%;">Summary of course</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week48-bs015.html#___sec14" style="font-size: 80%;">What? Me worry? No final exam in this course!</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week48-bs016.html#___sec15" style="font-size: 80%;">Topics we have covered this year</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week48-bs017.html#___sec16" style="font-size: 80%;">Statistical analysis and optimization of data</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week48-bs018.html#___sec17" style="font-size: 80%;">Machine learning</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week48-bs019.html#___sec18" style="font-size: 80%;">Learning outcomes and overarching aims of this course</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week48-bs020.html#___sec19" style="font-size: 80%;">Perspective on Machine Learning</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week48-bs021.html#___sec20" style="font-size: 80%;">Machine Learning Research</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week48-bs022.html#___sec21" style="font-size: 80%;">Starting your Machine Learning Project</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week48-bs023.html#___sec22" style="font-size: 80%;">Choose a Model and Algorithm</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week48-bs024.html#___sec23" style="font-size: 80%;">Preparing Your Data</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week48-bs025.html#___sec24" style="font-size: 80%;">Which Activation and Weights to Choose in Neural Networks</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week48-bs026.html#___sec25" style="font-size: 80%;">Optimization Methods and Hyperparameters</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week48-bs027.html#___sec26" style="font-size: 80%;">Resampling</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week48-bs028.html#___sec27" style="font-size: 80%;">Other courses on Data science and Machine Learning at UiO</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week48-bs029.html#___sec28" style="font-size: 80%;">Additional courses of interest</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week48-bs030.html#___sec29" style="font-size: 80%;">What's the future like?</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week48-bs031.html#___sec30" style="font-size: 80%;">Types of Machine Learning, a repetition</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week48-bs032.html#___sec31" style="font-size: 80%;">Why Boltzmann machines?</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week48-bs033.html#___sec32" style="font-size: 80%;">Boltzmann Machines</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week48-bs034.html#___sec33" style="font-size: 80%;">Some similarities and differences from DNNs</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week48-bs035.html#___sec34" style="font-size: 80%;">Boltzmann machines (BM)</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week48-bs036.html#___sec35" style="font-size: 80%;">A standard BM setup</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week48-bs037.html#___sec36" style="font-size: 80%;">The structure of the RBM network</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week48-bs038.html#___sec37" style="font-size: 80%;">The network</a></li>
|
|
<!-- navigation toc: --> <li><a href="#___sec38" style="font-size: 80%;">Goals</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week48-bs040.html#___sec39" style="font-size: 80%;">Joint distribution</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week48-bs041.html#___sec40" style="font-size: 80%;">Network Elements, the energy function</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week48-bs042.html#___sec41" style="font-size: 80%;">Defining different types of RBMs</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week48-bs043.html#___sec42" style="font-size: 80%;">More about RBMs</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week48-bs044.html#___sec43" style="font-size: 80%;">Autoencoders: Overarching view</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week48-bs045.html#___sec44" style="font-size: 80%;">Bayesian Machine Learning</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week48-bs046.html#___sec45" style="font-size: 80%;">Reinforcement Learning</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week48-bs047.html#___sec46" style="font-size: 80%;">Transfer learning</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week48-bs048.html#___sec47" style="font-size: 80%;">Adversarial learning</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week48-bs049.html#___sec48" style="font-size: 80%;">Dual learning</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week48-bs050.html#___sec49" style="font-size: 80%;">Distributed machine learning</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week48-bs051.html#___sec50" style="font-size: 80%;">Meta learning</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week48-bs052.html#___sec51" style="font-size: 80%;">The Challenges Facing Machine Learning</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week48-bs053.html#___sec52" style="font-size: 80%;">Explainable machine learning</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week48-bs054.html#___sec53" style="font-size: 80%;">Quantum machine learning</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week48-bs055.html#___sec54" style="font-size: 80%;">Quantum machine learning algorithms based on linear algebra</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week48-bs056.html#___sec55" style="font-size: 80%;">Quantum reinforcement learning</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week48-bs057.html#___sec56" style="font-size: 80%;">Quantum deep learning</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week48-bs058.html#___sec57" style="font-size: 80%;">Social machine learning</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week48-bs059.html#___sec58" style="font-size: 80%;">The last words?</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week48-bs060.html#___sec59" style="font-size: 80%;">Best wishes to you all and thanks so much for your heroic efforts this semester</a></li>
|
|
|
|
</ul>
|
|
</li>
|
|
</ul>
|
|
</div>
|
|
</div>
|
|
</div> <!-- end of navigation bar -->
|
|
|
|
<div class="container">
|
|
|
|
<p> </p><p> </p><p> </p> <!-- add vertical space -->
|
|
|
|
<a name="part0039"></a>
|
|
<!-- !split -->
|
|
|
|
<h2 id="___sec38" class="anchor">Goals </h2>
|
|
|
|
<p>
|
|
The goal of the hidden layer is to increase the model's expressive power. We encode complex interactions between visible variables by introducing additional, hidden variables that interact with visible degrees of freedom in a simple manner, yet still reproduce the complex correlations between visible degrees in the data once marginalized over (integrated out).
|
|
|
|
<p>
|
|
Examples of this trick being employed in for example physics:
|
|
|
|
<ol>
|
|
<li> The Hubbard-Stratonovich transformation</li>
|
|
<li> The introduction of ghost fields in gauge theory</li>
|
|
<li> Shadow wave functions in Quantum Monte Carlo simulations</li>
|
|
</ol>
|
|
|
|
<b>The network parameters, to be optimized/learned</b>:
|
|
|
|
<ol>
|
|
<li> \( \mathbf{a} \) represents the visible bias, a vector of same length as \( \mathbf{x} \).</li>
|
|
<li> \( \mathbf{b} \) represents the hidden bias, a vector of same lenght as \( \mathbf{h} \).</li>
|
|
<li> \( W \) represents the interaction weights, a matrix of size \( M\times N \).</li>
|
|
</ol>
|
|
|
|
<p>
|
|
<!-- navigation buttons at the bottom of the page -->
|
|
<ul class="pagination">
|
|
<li><a href="._week48-bs038.html">«</a></li>
|
|
<li><a href="._week48-bs000.html">1</a></li>
|
|
<li><a href="">...</a></li>
|
|
<li><a href="._week48-bs031.html">32</a></li>
|
|
<li><a href="._week48-bs032.html">33</a></li>
|
|
<li><a href="._week48-bs033.html">34</a></li>
|
|
<li><a href="._week48-bs034.html">35</a></li>
|
|
<li><a href="._week48-bs035.html">36</a></li>
|
|
<li><a href="._week48-bs036.html">37</a></li>
|
|
<li><a href="._week48-bs037.html">38</a></li>
|
|
<li><a href="._week48-bs038.html">39</a></li>
|
|
<li class="active"><a href="._week48-bs039.html">40</a></li>
|
|
<li><a href="._week48-bs040.html">41</a></li>
|
|
<li><a href="._week48-bs041.html">42</a></li>
|
|
<li><a href="._week48-bs042.html">43</a></li>
|
|
<li><a href="._week48-bs043.html">44</a></li>
|
|
<li><a href="._week48-bs044.html">45</a></li>
|
|
<li><a href="._week48-bs045.html">46</a></li>
|
|
<li><a href="._week48-bs046.html">47</a></li>
|
|
<li><a href="._week48-bs047.html">48</a></li>
|
|
<li><a href="._week48-bs048.html">49</a></li>
|
|
<li><a href="">...</a></li>
|
|
<li><a href="._week48-bs060.html">61</a></li>
|
|
<li><a href="._week48-bs040.html">»</a></li>
|
|
</ul>
|
|
<!-- ------------------- end of main content --------------- -->
|
|
|
|
</div> <!-- end container -->
|
|
<!-- include javascript, jQuery *first* -->
|
|
<script src="https://ajax.googleapis.com/ajax/libs/jquery/1.10.2/jquery.min.js"></script>
|
|
<script src="https://netdna.bootstrapcdn.com/bootstrap/3.0.0/js/bootstrap.min.js"></script>
|
|
|
|
<!-- Bootstrap footer
|
|
<footer>
|
|
<a href="http://..."><img width="250" align=right src="http://..."></a>
|
|
</footer>
|
|
-->
|
|
|
|
|
|
<center style="font-size:80%">
|
|
<!-- copyright only on the titlepage -->
|
|
</center>
|
|
|
|
|
|
</body>
|
|
</html>
|
|
|
|
|