update on log reg
This commit is contained in:
mhjensen
@@ -55,7 +55,17 @@ Automatically generated HTML file from DocOnce source
|
||||
('Including more classes', 2, None, '___sec12'),
|
||||
('The Softmax function', 2, None, '___sec13'),
|
||||
('A _scikit-learn_ example', 2, None, '___sec14'),
|
||||
('A simple classification problem', 2, None, '___sec15')]}
|
||||
('A simple classification problem', 2, None, '___sec15'),
|
||||
('The two-dimensional Ising model, Predicting phase transition '
|
||||
'of the two-dimensional Ising model',
|
||||
2,
|
||||
None,
|
||||
'___sec16'),
|
||||
('Reading in the data', 2, None, '___sec17'),
|
||||
('Logistic regression', 2, None, '___sec18'),
|
||||
('Exploring the logistic regression', 2, None, '___sec19'),
|
||||
('Accuracy of a classification model', 2, None, '___sec20'),
|
||||
('Analyzing the results', 2, None, '___sec21')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -109,6 +119,12 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs014.html#___sec13" style="font-size: 80%;">The Softmax function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs015.html#___sec14" style="font-size: 80%;">A <b>scikit-learn</b> example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs016.html#___sec15" style="font-size: 80%;">A simple classification problem</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs017.html#___sec16" style="font-size: 80%;">The two-dimensional Ising model, Predicting phase transition of the two-dimensional Ising model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs018.html#___sec17" style="font-size: 80%;">Reading in the data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs019.html#___sec18" style="font-size: 80%;">Logistic regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs020.html#___sec19" style="font-size: 80%;">Exploring the logistic regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs021.html#___sec20" style="font-size: 80%;">Accuracy of a classification model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs022.html#___sec21" style="font-size: 80%;">Analyzing the results</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -160,7 +176,7 @@ MathJax.Hub.Config({
|
||||
<li><a href="._LogReg-bs008.html">9</a></li>
|
||||
<li><a href="._LogReg-bs009.html">10</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._LogReg-bs016.html">17</a></li>
|
||||
<li><a href="._LogReg-bs022.html">23</a></li>
|
||||
<li><a href="._LogReg-bs001.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -55,7 +55,17 @@ Automatically generated HTML file from DocOnce source
|
||||
('Including more classes', 2, None, '___sec12'),
|
||||
('The Softmax function', 2, None, '___sec13'),
|
||||
('A _scikit-learn_ example', 2, None, '___sec14'),
|
||||
('A simple classification problem', 2, None, '___sec15')]}
|
||||
('A simple classification problem', 2, None, '___sec15'),
|
||||
('The two-dimensional Ising model, Predicting phase transition '
|
||||
'of the two-dimensional Ising model',
|
||||
2,
|
||||
None,
|
||||
'___sec16'),
|
||||
('Reading in the data', 2, None, '___sec17'),
|
||||
('Logistic regression', 2, None, '___sec18'),
|
||||
('Exploring the logistic regression', 2, None, '___sec19'),
|
||||
('Accuracy of a classification model', 2, None, '___sec20'),
|
||||
('Analyzing the results', 2, None, '___sec21')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -109,6 +119,12 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs014.html#___sec13" style="font-size: 80%;">The Softmax function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs015.html#___sec14" style="font-size: 80%;">A <b>scikit-learn</b> example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs016.html#___sec15" style="font-size: 80%;">A simple classification problem</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs017.html#___sec16" style="font-size: 80%;">The two-dimensional Ising model, Predicting phase transition of the two-dimensional Ising model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs018.html#___sec17" style="font-size: 80%;">Reading in the data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs019.html#___sec18" style="font-size: 80%;">Logistic regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs020.html#___sec19" style="font-size: 80%;">Exploring the logistic regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs021.html#___sec20" style="font-size: 80%;">Accuracy of a classification model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs022.html#___sec21" style="font-size: 80%;">Analyzing the results</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -176,7 +192,7 @@ failure etc.
|
||||
<li><a href="._LogReg-bs009.html">10</a></li>
|
||||
<li><a href="._LogReg-bs010.html">11</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._LogReg-bs016.html">17</a></li>
|
||||
<li><a href="._LogReg-bs022.html">23</a></li>
|
||||
<li><a href="._LogReg-bs002.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -55,7 +55,17 @@ Automatically generated HTML file from DocOnce source
|
||||
('Including more classes', 2, None, '___sec12'),
|
||||
('The Softmax function', 2, None, '___sec13'),
|
||||
('A _scikit-learn_ example', 2, None, '___sec14'),
|
||||
('A simple classification problem', 2, None, '___sec15')]}
|
||||
('A simple classification problem', 2, None, '___sec15'),
|
||||
('The two-dimensional Ising model, Predicting phase transition '
|
||||
'of the two-dimensional Ising model',
|
||||
2,
|
||||
None,
|
||||
'___sec16'),
|
||||
('Reading in the data', 2, None, '___sec17'),
|
||||
('Logistic regression', 2, None, '___sec18'),
|
||||
('Exploring the logistic regression', 2, None, '___sec19'),
|
||||
('Accuracy of a classification model', 2, None, '___sec20'),
|
||||
('Analyzing the results', 2, None, '___sec21')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -109,6 +119,12 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs014.html#___sec13" style="font-size: 80%;">The Softmax function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs015.html#___sec14" style="font-size: 80%;">A <b>scikit-learn</b> example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs016.html#___sec15" style="font-size: 80%;">A simple classification problem</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs017.html#___sec16" style="font-size: 80%;">The two-dimensional Ising model, Predicting phase transition of the two-dimensional Ising model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs018.html#___sec17" style="font-size: 80%;">Reading in the data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs019.html#___sec18" style="font-size: 80%;">Logistic regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs020.html#___sec19" style="font-size: 80%;">Exploring the logistic regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs021.html#___sec20" style="font-size: 80%;">Accuracy of a classification model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs022.html#___sec21" style="font-size: 80%;">Analyzing the results</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -155,7 +171,7 @@ models, as we will see later.
|
||||
<li><a href="._LogReg-bs010.html">11</a></li>
|
||||
<li><a href="._LogReg-bs011.html">12</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._LogReg-bs016.html">17</a></li>
|
||||
<li><a href="._LogReg-bs022.html">23</a></li>
|
||||
<li><a href="._LogReg-bs003.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -55,7 +55,17 @@ Automatically generated HTML file from DocOnce source
|
||||
('Including more classes', 2, None, '___sec12'),
|
||||
('The Softmax function', 2, None, '___sec13'),
|
||||
('A _scikit-learn_ example', 2, None, '___sec14'),
|
||||
('A simple classification problem', 2, None, '___sec15')]}
|
||||
('A simple classification problem', 2, None, '___sec15'),
|
||||
('The two-dimensional Ising model, Predicting phase transition '
|
||||
'of the two-dimensional Ising model',
|
||||
2,
|
||||
None,
|
||||
'___sec16'),
|
||||
('Reading in the data', 2, None, '___sec17'),
|
||||
('Logistic regression', 2, None, '___sec18'),
|
||||
('Exploring the logistic regression', 2, None, '___sec19'),
|
||||
('Accuracy of a classification model', 2, None, '___sec20'),
|
||||
('Analyzing the results', 2, None, '___sec21')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -109,6 +119,12 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs014.html#___sec13" style="font-size: 80%;">The Softmax function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs015.html#___sec14" style="font-size: 80%;">A <b>scikit-learn</b> example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs016.html#___sec15" style="font-size: 80%;">A simple classification problem</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs017.html#___sec16" style="font-size: 80%;">The two-dimensional Ising model, Predicting phase transition of the two-dimensional Ising model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs018.html#___sec17" style="font-size: 80%;">Reading in the data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs019.html#___sec18" style="font-size: 80%;">Logistic regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs020.html#___sec19" style="font-size: 80%;">Exploring the logistic regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs021.html#___sec20" style="font-size: 80%;">Accuracy of a classification model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs022.html#___sec21" style="font-size: 80%;">Analyzing the results</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -163,7 +179,7 @@ $$
|
||||
<li><a href="._LogReg-bs011.html">12</a></li>
|
||||
<li><a href="._LogReg-bs012.html">13</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._LogReg-bs016.html">17</a></li>
|
||||
<li><a href="._LogReg-bs022.html">23</a></li>
|
||||
<li><a href="._LogReg-bs004.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -55,7 +55,17 @@ Automatically generated HTML file from DocOnce source
|
||||
('Including more classes', 2, None, '___sec12'),
|
||||
('The Softmax function', 2, None, '___sec13'),
|
||||
('A _scikit-learn_ example', 2, None, '___sec14'),
|
||||
('A simple classification problem', 2, None, '___sec15')]}
|
||||
('A simple classification problem', 2, None, '___sec15'),
|
||||
('The two-dimensional Ising model, Predicting phase transition '
|
||||
'of the two-dimensional Ising model',
|
||||
2,
|
||||
None,
|
||||
'___sec16'),
|
||||
('Reading in the data', 2, None, '___sec17'),
|
||||
('Logistic regression', 2, None, '___sec18'),
|
||||
('Exploring the logistic regression', 2, None, '___sec19'),
|
||||
('Accuracy of a classification model', 2, None, '___sec20'),
|
||||
('Analyzing the results', 2, None, '___sec21')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -109,6 +119,12 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs014.html#___sec13" style="font-size: 80%;">The Softmax function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs015.html#___sec14" style="font-size: 80%;">A <b>scikit-learn</b> example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs016.html#___sec15" style="font-size: 80%;">A simple classification problem</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs017.html#___sec16" style="font-size: 80%;">The two-dimensional Ising model, Predicting phase transition of the two-dimensional Ising model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs018.html#___sec17" style="font-size: 80%;">Reading in the data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs019.html#___sec18" style="font-size: 80%;">Logistic regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs020.html#___sec19" style="font-size: 80%;">Exploring the logistic regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs021.html#___sec20" style="font-size: 80%;">Accuracy of a classification model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs022.html#___sec21" style="font-size: 80%;">Analyzing the results</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -162,7 +178,7 @@ where \( \hat{y} \) is a vector representing the possible outcomes, \( \hat{X} \
|
||||
<li><a href="._LogReg-bs012.html">13</a></li>
|
||||
<li><a href="._LogReg-bs013.html">14</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._LogReg-bs016.html">17</a></li>
|
||||
<li><a href="._LogReg-bs022.html">23</a></li>
|
||||
<li><a href="._LogReg-bs005.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -55,7 +55,17 @@ Automatically generated HTML file from DocOnce source
|
||||
('Including more classes', 2, None, '___sec12'),
|
||||
('The Softmax function', 2, None, '___sec13'),
|
||||
('A _scikit-learn_ example', 2, None, '___sec14'),
|
||||
('A simple classification problem', 2, None, '___sec15')]}
|
||||
('A simple classification problem', 2, None, '___sec15'),
|
||||
('The two-dimensional Ising model, Predicting phase transition '
|
||||
'of the two-dimensional Ising model',
|
||||
2,
|
||||
None,
|
||||
'___sec16'),
|
||||
('Reading in the data', 2, None, '___sec17'),
|
||||
('Logistic regression', 2, None, '___sec18'),
|
||||
('Exploring the logistic regression', 2, None, '___sec19'),
|
||||
('Accuracy of a classification model', 2, None, '___sec20'),
|
||||
('Analyzing the results', 2, None, '___sec21')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -109,6 +119,12 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs014.html#___sec13" style="font-size: 80%;">The Softmax function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs015.html#___sec14" style="font-size: 80%;">A <b>scikit-learn</b> example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs016.html#___sec15" style="font-size: 80%;">A simple classification problem</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs017.html#___sec16" style="font-size: 80%;">The two-dimensional Ising model, Predicting phase transition of the two-dimensional Ising model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs018.html#___sec17" style="font-size: 80%;">Reading in the data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs019.html#___sec18" style="font-size: 80%;">Logistic regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs020.html#___sec19" style="font-size: 80%;">Exploring the logistic regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs021.html#___sec20" style="font-size: 80%;">Accuracy of a classification model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs022.html#___sec21" style="font-size: 80%;">Analyzing the results</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -169,7 +185,7 @@ The code for plotting the perceptron can be seen here. This si nothing but the s
|
||||
<li><a href="._LogReg-bs013.html">14</a></li>
|
||||
<li><a href="._LogReg-bs014.html">15</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._LogReg-bs016.html">17</a></li>
|
||||
<li><a href="._LogReg-bs022.html">23</a></li>
|
||||
<li><a href="._LogReg-bs006.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -55,7 +55,17 @@ Automatically generated HTML file from DocOnce source
|
||||
('Including more classes', 2, None, '___sec12'),
|
||||
('The Softmax function', 2, None, '___sec13'),
|
||||
('A _scikit-learn_ example', 2, None, '___sec14'),
|
||||
('A simple classification problem', 2, None, '___sec15')]}
|
||||
('A simple classification problem', 2, None, '___sec15'),
|
||||
('The two-dimensional Ising model, Predicting phase transition '
|
||||
'of the two-dimensional Ising model',
|
||||
2,
|
||||
None,
|
||||
'___sec16'),
|
||||
('Reading in the data', 2, None, '___sec17'),
|
||||
('Logistic regression', 2, None, '___sec18'),
|
||||
('Exploring the logistic regression', 2, None, '___sec19'),
|
||||
('Accuracy of a classification model', 2, None, '___sec20'),
|
||||
('Analyzing the results', 2, None, '___sec21')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -109,6 +119,12 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs014.html#___sec13" style="font-size: 80%;">The Softmax function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs015.html#___sec14" style="font-size: 80%;">A <b>scikit-learn</b> example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs016.html#___sec15" style="font-size: 80%;">A simple classification problem</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs017.html#___sec16" style="font-size: 80%;">The two-dimensional Ising model, Predicting phase transition of the two-dimensional Ising model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs018.html#___sec17" style="font-size: 80%;">Reading in the data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs019.html#___sec18" style="font-size: 80%;">Logistic regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs020.html#___sec19" style="font-size: 80%;">Exploring the logistic regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs021.html#___sec20" style="font-size: 80%;">Accuracy of a classification model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs022.html#___sec21" style="font-size: 80%;">Analyzing the results</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -170,7 +186,7 @@ The following code plots the logistic function.
|
||||
<li><a href="._LogReg-bs014.html">15</a></li>
|
||||
<li><a href="._LogReg-bs015.html">16</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._LogReg-bs016.html">17</a></li>
|
||||
<li><a href="._LogReg-bs022.html">23</a></li>
|
||||
<li><a href="._LogReg-bs007.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -55,7 +55,17 @@ Automatically generated HTML file from DocOnce source
|
||||
('Including more classes', 2, None, '___sec12'),
|
||||
('The Softmax function', 2, None, '___sec13'),
|
||||
('A _scikit-learn_ example', 2, None, '___sec14'),
|
||||
('A simple classification problem', 2, None, '___sec15')]}
|
||||
('A simple classification problem', 2, None, '___sec15'),
|
||||
('The two-dimensional Ising model, Predicting phase transition '
|
||||
'of the two-dimensional Ising model',
|
||||
2,
|
||||
None,
|
||||
'___sec16'),
|
||||
('Reading in the data', 2, None, '___sec17'),
|
||||
('Logistic regression', 2, None, '___sec18'),
|
||||
('Exploring the logistic regression', 2, None, '___sec19'),
|
||||
('Accuracy of a classification model', 2, None, '___sec20'),
|
||||
('Analyzing the results', 2, None, '___sec21')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -109,6 +119,12 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs014.html#___sec13" style="font-size: 80%;">The Softmax function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs015.html#___sec14" style="font-size: 80%;">A <b>scikit-learn</b> example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs016.html#___sec15" style="font-size: 80%;">A simple classification problem</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs017.html#___sec16" style="font-size: 80%;">The two-dimensional Ising model, Predicting phase transition of the two-dimensional Ising model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs018.html#___sec17" style="font-size: 80%;">Reading in the data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs019.html#___sec18" style="font-size: 80%;">Logistic regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs020.html#___sec19" style="font-size: 80%;">Exploring the logistic regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs021.html#___sec20" style="font-size: 80%;">Accuracy of a classification model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs022.html#___sec21" style="font-size: 80%;">Analyzing the results</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -165,6 +181,8 @@ $$
|
||||
<li><a href="._LogReg-bs014.html">15</a></li>
|
||||
<li><a href="._LogReg-bs015.html">16</a></li>
|
||||
<li><a href="._LogReg-bs016.html">17</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._LogReg-bs022.html">23</a></li>
|
||||
<li><a href="._LogReg-bs008.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -55,7 +55,17 @@ Automatically generated HTML file from DocOnce source
|
||||
('Including more classes', 2, None, '___sec12'),
|
||||
('The Softmax function', 2, None, '___sec13'),
|
||||
('A _scikit-learn_ example', 2, None, '___sec14'),
|
||||
('A simple classification problem', 2, None, '___sec15')]}
|
||||
('A simple classification problem', 2, None, '___sec15'),
|
||||
('The two-dimensional Ising model, Predicting phase transition '
|
||||
'of the two-dimensional Ising model',
|
||||
2,
|
||||
None,
|
||||
'___sec16'),
|
||||
('Reading in the data', 2, None, '___sec17'),
|
||||
('Logistic regression', 2, None, '___sec18'),
|
||||
('Exploring the logistic regression', 2, None, '___sec19'),
|
||||
('Accuracy of a classification model', 2, None, '___sec20'),
|
||||
('Analyzing the results', 2, None, '___sec21')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -109,6 +119,12 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs014.html#___sec13" style="font-size: 80%;">The Softmax function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs015.html#___sec14" style="font-size: 80%;">A <b>scikit-learn</b> example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs016.html#___sec15" style="font-size: 80%;">A simple classification problem</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs017.html#___sec16" style="font-size: 80%;">The two-dimensional Ising model, Predicting phase transition of the two-dimensional Ising model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs018.html#___sec17" style="font-size: 80%;">Reading in the data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs019.html#___sec18" style="font-size: 80%;">Logistic regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs020.html#___sec19" style="font-size: 80%;">Exploring the logistic regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs021.html#___sec20" style="font-size: 80%;">Accuracy of a classification model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs022.html#___sec21" style="font-size: 80%;">Analyzing the results</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -166,6 +182,9 @@ $$
|
||||
<li><a href="._LogReg-bs014.html">15</a></li>
|
||||
<li><a href="._LogReg-bs015.html">16</a></li>
|
||||
<li><a href="._LogReg-bs016.html">17</a></li>
|
||||
<li><a href="._LogReg-bs017.html">18</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._LogReg-bs022.html">23</a></li>
|
||||
<li><a href="._LogReg-bs009.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -55,7 +55,17 @@ Automatically generated HTML file from DocOnce source
|
||||
('Including more classes', 2, None, '___sec12'),
|
||||
('The Softmax function', 2, None, '___sec13'),
|
||||
('A _scikit-learn_ example', 2, None, '___sec14'),
|
||||
('A simple classification problem', 2, None, '___sec15')]}
|
||||
('A simple classification problem', 2, None, '___sec15'),
|
||||
('The two-dimensional Ising model, Predicting phase transition '
|
||||
'of the two-dimensional Ising model',
|
||||
2,
|
||||
None,
|
||||
'___sec16'),
|
||||
('Reading in the data', 2, None, '___sec17'),
|
||||
('Logistic regression', 2, None, '___sec18'),
|
||||
('Exploring the logistic regression', 2, None, '___sec19'),
|
||||
('Accuracy of a classification model', 2, None, '___sec20'),
|
||||
('Analyzing the results', 2, None, '___sec21')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -109,6 +119,12 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs014.html#___sec13" style="font-size: 80%;">The Softmax function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs015.html#___sec14" style="font-size: 80%;">A <b>scikit-learn</b> example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs016.html#___sec15" style="font-size: 80%;">A simple classification problem</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs017.html#___sec16" style="font-size: 80%;">The two-dimensional Ising model, Predicting phase transition of the two-dimensional Ising model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs018.html#___sec17" style="font-size: 80%;">Reading in the data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs019.html#___sec18" style="font-size: 80%;">Logistic regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs020.html#___sec19" style="font-size: 80%;">Exploring the logistic regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs021.html#___sec20" style="font-size: 80%;">Accuracy of a classification model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs022.html#___sec21" style="font-size: 80%;">Analyzing the results</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -164,6 +180,10 @@ in practice we often supplement the cross-entropy with additional regularization
|
||||
<li><a href="._LogReg-bs014.html">15</a></li>
|
||||
<li><a href="._LogReg-bs015.html">16</a></li>
|
||||
<li><a href="._LogReg-bs016.html">17</a></li>
|
||||
<li><a href="._LogReg-bs017.html">18</a></li>
|
||||
<li><a href="._LogReg-bs018.html">19</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._LogReg-bs022.html">23</a></li>
|
||||
<li><a href="._LogReg-bs010.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -55,7 +55,17 @@ Automatically generated HTML file from DocOnce source
|
||||
('Including more classes', 2, None, '___sec12'),
|
||||
('The Softmax function', 2, None, '___sec13'),
|
||||
('A _scikit-learn_ example', 2, None, '___sec14'),
|
||||
('A simple classification problem', 2, None, '___sec15')]}
|
||||
('A simple classification problem', 2, None, '___sec15'),
|
||||
('The two-dimensional Ising model, Predicting phase transition '
|
||||
'of the two-dimensional Ising model',
|
||||
2,
|
||||
None,
|
||||
'___sec16'),
|
||||
('Reading in the data', 2, None, '___sec17'),
|
||||
('Logistic regression', 2, None, '___sec18'),
|
||||
('Exploring the logistic regression', 2, None, '___sec19'),
|
||||
('Accuracy of a classification model', 2, None, '___sec20'),
|
||||
('Analyzing the results', 2, None, '___sec21')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -109,6 +119,12 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs014.html#___sec13" style="font-size: 80%;">The Softmax function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs015.html#___sec14" style="font-size: 80%;">A <b>scikit-learn</b> example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs016.html#___sec15" style="font-size: 80%;">A simple classification problem</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs017.html#___sec16" style="font-size: 80%;">The two-dimensional Ising model, Predicting phase transition of the two-dimensional Ising model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs018.html#___sec17" style="font-size: 80%;">Reading in the data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs019.html#___sec18" style="font-size: 80%;">Logistic regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs020.html#___sec19" style="font-size: 80%;">Exploring the logistic regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs021.html#___sec20" style="font-size: 80%;">Accuracy of a classification model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs022.html#___sec21" style="font-size: 80%;">Analyzing the results</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -165,6 +181,11 @@ $$
|
||||
<li><a href="._LogReg-bs014.html">15</a></li>
|
||||
<li><a href="._LogReg-bs015.html">16</a></li>
|
||||
<li><a href="._LogReg-bs016.html">17</a></li>
|
||||
<li><a href="._LogReg-bs017.html">18</a></li>
|
||||
<li><a href="._LogReg-bs018.html">19</a></li>
|
||||
<li><a href="._LogReg-bs019.html">20</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._LogReg-bs022.html">23</a></li>
|
||||
<li><a href="._LogReg-bs011.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -55,7 +55,17 @@ Automatically generated HTML file from DocOnce source
|
||||
('Including more classes', 2, None, '___sec12'),
|
||||
('The Softmax function', 2, None, '___sec13'),
|
||||
('A _scikit-learn_ example', 2, None, '___sec14'),
|
||||
('A simple classification problem', 2, None, '___sec15')]}
|
||||
('A simple classification problem', 2, None, '___sec15'),
|
||||
('The two-dimensional Ising model, Predicting phase transition '
|
||||
'of the two-dimensional Ising model',
|
||||
2,
|
||||
None,
|
||||
'___sec16'),
|
||||
('Reading in the data', 2, None, '___sec17'),
|
||||
('Logistic regression', 2, None, '___sec18'),
|
||||
('Exploring the logistic regression', 2, None, '___sec19'),
|
||||
('Accuracy of a classification model', 2, None, '___sec20'),
|
||||
('Analyzing the results', 2, None, '___sec21')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -109,6 +119,12 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs014.html#___sec13" style="font-size: 80%;">The Softmax function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs015.html#___sec14" style="font-size: 80%;">A <b>scikit-learn</b> example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs016.html#___sec15" style="font-size: 80%;">A simple classification problem</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs017.html#___sec16" style="font-size: 80%;">The two-dimensional Ising model, Predicting phase transition of the two-dimensional Ising model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs018.html#___sec17" style="font-size: 80%;">Reading in the data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs019.html#___sec18" style="font-size: 80%;">Logistic regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs020.html#___sec19" style="font-size: 80%;">Exploring the logistic regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs021.html#___sec20" style="font-size: 80%;">Accuracy of a classification model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs022.html#___sec21" style="font-size: 80%;">Analyzing the results</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -165,6 +181,12 @@ $$
|
||||
<li><a href="._LogReg-bs014.html">15</a></li>
|
||||
<li><a href="._LogReg-bs015.html">16</a></li>
|
||||
<li><a href="._LogReg-bs016.html">17</a></li>
|
||||
<li><a href="._LogReg-bs017.html">18</a></li>
|
||||
<li><a href="._LogReg-bs018.html">19</a></li>
|
||||
<li><a href="._LogReg-bs019.html">20</a></li>
|
||||
<li><a href="._LogReg-bs020.html">21</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._LogReg-bs022.html">23</a></li>
|
||||
<li><a href="._LogReg-bs012.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -55,7 +55,17 @@ Automatically generated HTML file from DocOnce source
|
||||
('Including more classes', 2, None, '___sec12'),
|
||||
('The Softmax function', 2, None, '___sec13'),
|
||||
('A _scikit-learn_ example', 2, None, '___sec14'),
|
||||
('A simple classification problem', 2, None, '___sec15')]}
|
||||
('A simple classification problem', 2, None, '___sec15'),
|
||||
('The two-dimensional Ising model, Predicting phase transition '
|
||||
'of the two-dimensional Ising model',
|
||||
2,
|
||||
None,
|
||||
'___sec16'),
|
||||
('Reading in the data', 2, None, '___sec17'),
|
||||
('Logistic regression', 2, None, '___sec18'),
|
||||
('Exploring the logistic regression', 2, None, '___sec19'),
|
||||
('Accuracy of a classification model', 2, None, '___sec20'),
|
||||
('Analyzing the results', 2, None, '___sec21')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -109,6 +119,12 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs014.html#___sec13" style="font-size: 80%;">The Softmax function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs015.html#___sec14" style="font-size: 80%;">A <b>scikit-learn</b> example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs016.html#___sec15" style="font-size: 80%;">A simple classification problem</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs017.html#___sec16" style="font-size: 80%;">The two-dimensional Ising model, Predicting phase transition of the two-dimensional Ising model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs018.html#___sec17" style="font-size: 80%;">Reading in the data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs019.html#___sec18" style="font-size: 80%;">Logistic regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs020.html#___sec19" style="font-size: 80%;">Exploring the logistic regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs021.html#___sec20" style="font-size: 80%;">Accuracy of a classification model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs022.html#___sec21" style="font-size: 80%;">Analyzing the results</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -157,6 +173,13 @@ $$
|
||||
<li><a href="._LogReg-bs014.html">15</a></li>
|
||||
<li><a href="._LogReg-bs015.html">16</a></li>
|
||||
<li><a href="._LogReg-bs016.html">17</a></li>
|
||||
<li><a href="._LogReg-bs017.html">18</a></li>
|
||||
<li><a href="._LogReg-bs018.html">19</a></li>
|
||||
<li><a href="._LogReg-bs019.html">20</a></li>
|
||||
<li><a href="._LogReg-bs020.html">21</a></li>
|
||||
<li><a href="._LogReg-bs021.html">22</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._LogReg-bs022.html">23</a></li>
|
||||
<li><a href="._LogReg-bs013.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -55,7 +55,17 @@ Automatically generated HTML file from DocOnce source
|
||||
('Including more classes', 2, None, '___sec12'),
|
||||
('The Softmax function', 2, None, '___sec13'),
|
||||
('A _scikit-learn_ example', 2, None, '___sec14'),
|
||||
('A simple classification problem', 2, None, '___sec15')]}
|
||||
('A simple classification problem', 2, None, '___sec15'),
|
||||
('The two-dimensional Ising model, Predicting phase transition '
|
||||
'of the two-dimensional Ising model',
|
||||
2,
|
||||
None,
|
||||
'___sec16'),
|
||||
('Reading in the data', 2, None, '___sec17'),
|
||||
('Logistic regression', 2, None, '___sec18'),
|
||||
('Exploring the logistic regression', 2, None, '___sec19'),
|
||||
('Accuracy of a classification model', 2, None, '___sec20'),
|
||||
('Analyzing the results', 2, None, '___sec21')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -109,6 +119,12 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs014.html#___sec13" style="font-size: 80%;">The Softmax function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs015.html#___sec14" style="font-size: 80%;">A <b>scikit-learn</b> example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs016.html#___sec15" style="font-size: 80%;">A simple classification problem</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs017.html#___sec16" style="font-size: 80%;">The two-dimensional Ising model, Predicting phase transition of the two-dimensional Ising model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs018.html#___sec17" style="font-size: 80%;">Reading in the data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs019.html#___sec18" style="font-size: 80%;">Logistic regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs020.html#___sec19" style="font-size: 80%;">Exploring the logistic regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs021.html#___sec20" style="font-size: 80%;">Accuracy of a classification model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs022.html#___sec21" style="font-size: 80%;">Analyzing the results</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -163,6 +179,12 @@ and the model is specified in term of \( K-1 \) so-called log-odds or <b>logit</
|
||||
<li><a href="._LogReg-bs014.html">15</a></li>
|
||||
<li><a href="._LogReg-bs015.html">16</a></li>
|
||||
<li><a href="._LogReg-bs016.html">17</a></li>
|
||||
<li><a href="._LogReg-bs017.html">18</a></li>
|
||||
<li><a href="._LogReg-bs018.html">19</a></li>
|
||||
<li><a href="._LogReg-bs019.html">20</a></li>
|
||||
<li><a href="._LogReg-bs020.html">21</a></li>
|
||||
<li><a href="._LogReg-bs021.html">22</a></li>
|
||||
<li><a href="._LogReg-bs022.html">23</a></li>
|
||||
<li><a href="._LogReg-bs014.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -55,7 +55,17 @@ Automatically generated HTML file from DocOnce source
|
||||
('Including more classes', 2, None, '___sec12'),
|
||||
('The Softmax function', 2, None, '___sec13'),
|
||||
('A _scikit-learn_ example', 2, None, '___sec14'),
|
||||
('A simple classification problem', 2, None, '___sec15')]}
|
||||
('A simple classification problem', 2, None, '___sec15'),
|
||||
('The two-dimensional Ising model, Predicting phase transition '
|
||||
'of the two-dimensional Ising model',
|
||||
2,
|
||||
None,
|
||||
'___sec16'),
|
||||
('Reading in the data', 2, None, '___sec17'),
|
||||
('Logistic regression', 2, None, '___sec18'),
|
||||
('Exploring the logistic regression', 2, None, '___sec19'),
|
||||
('Accuracy of a classification model', 2, None, '___sec20'),
|
||||
('Analyzing the results', 2, None, '___sec21')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -109,6 +119,12 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="#___sec13" style="font-size: 80%;">The Softmax function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs015.html#___sec14" style="font-size: 80%;">A <b>scikit-learn</b> example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs016.html#___sec15" style="font-size: 80%;">A simple classification problem</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs017.html#___sec16" style="font-size: 80%;">The two-dimensional Ising model, Predicting phase transition of the two-dimensional Ising model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs018.html#___sec17" style="font-size: 80%;">Reading in the data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs019.html#___sec18" style="font-size: 80%;">Logistic regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs020.html#___sec19" style="font-size: 80%;">Exploring the logistic regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs021.html#___sec20" style="font-size: 80%;">Accuracy of a classification model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs022.html#___sec21" style="font-size: 80%;">Analyzing the results</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -172,6 +188,12 @@ Newton's method and gradient descent methods are discussed in the material on <a
|
||||
<li class="active"><a href="._LogReg-bs014.html">15</a></li>
|
||||
<li><a href="._LogReg-bs015.html">16</a></li>
|
||||
<li><a href="._LogReg-bs016.html">17</a></li>
|
||||
<li><a href="._LogReg-bs017.html">18</a></li>
|
||||
<li><a href="._LogReg-bs018.html">19</a></li>
|
||||
<li><a href="._LogReg-bs019.html">20</a></li>
|
||||
<li><a href="._LogReg-bs020.html">21</a></li>
|
||||
<li><a href="._LogReg-bs021.html">22</a></li>
|
||||
<li><a href="._LogReg-bs022.html">23</a></li>
|
||||
<li><a href="._LogReg-bs015.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -55,7 +55,17 @@ Automatically generated HTML file from DocOnce source
|
||||
('Including more classes', 2, None, '___sec12'),
|
||||
('The Softmax function', 2, None, '___sec13'),
|
||||
('A _scikit-learn_ example', 2, None, '___sec14'),
|
||||
('A simple classification problem', 2, None, '___sec15')]}
|
||||
('A simple classification problem', 2, None, '___sec15'),
|
||||
('The two-dimensional Ising model, Predicting phase transition '
|
||||
'of the two-dimensional Ising model',
|
||||
2,
|
||||
None,
|
||||
'___sec16'),
|
||||
('Reading in the data', 2, None, '___sec17'),
|
||||
('Logistic regression', 2, None, '___sec18'),
|
||||
('Exploring the logistic regression', 2, None, '___sec19'),
|
||||
('Accuracy of a classification model', 2, None, '___sec20'),
|
||||
('Analyzing the results', 2, None, '___sec21')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -109,6 +119,12 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs014.html#___sec13" style="font-size: 80%;">The Softmax function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec14" style="font-size: 80%;">A <b>scikit-learn</b> example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs016.html#___sec15" style="font-size: 80%;">A simple classification problem</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs017.html#___sec16" style="font-size: 80%;">The two-dimensional Ising model, Predicting phase transition of the two-dimensional Ising model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs018.html#___sec17" style="font-size: 80%;">Reading in the data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs019.html#___sec18" style="font-size: 80%;">Logistic regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs020.html#___sec19" style="font-size: 80%;">Exploring the logistic regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs021.html#___sec20" style="font-size: 80%;">Accuracy of a classification model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs022.html#___sec21" style="font-size: 80%;">Analyzing the results</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -165,6 +181,12 @@ plt<span style="color: #666666">.</span>show()
|
||||
<li><a href="._LogReg-bs014.html">15</a></li>
|
||||
<li class="active"><a href="._LogReg-bs015.html">16</a></li>
|
||||
<li><a href="._LogReg-bs016.html">17</a></li>
|
||||
<li><a href="._LogReg-bs017.html">18</a></li>
|
||||
<li><a href="._LogReg-bs018.html">19</a></li>
|
||||
<li><a href="._LogReg-bs019.html">20</a></li>
|
||||
<li><a href="._LogReg-bs020.html">21</a></li>
|
||||
<li><a href="._LogReg-bs021.html">22</a></li>
|
||||
<li><a href="._LogReg-bs022.html">23</a></li>
|
||||
<li><a href="._LogReg-bs016.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -55,7 +55,17 @@ Automatically generated HTML file from DocOnce source
|
||||
('Including more classes', 2, None, '___sec12'),
|
||||
('The Softmax function', 2, None, '___sec13'),
|
||||
('A _scikit-learn_ example', 2, None, '___sec14'),
|
||||
('A simple classification problem', 2, None, '___sec15')]}
|
||||
('A simple classification problem', 2, None, '___sec15'),
|
||||
('The two-dimensional Ising model, Predicting phase transition '
|
||||
'of the two-dimensional Ising model',
|
||||
2,
|
||||
None,
|
||||
'___sec16'),
|
||||
('Reading in the data', 2, None, '___sec17'),
|
||||
('Logistic regression', 2, None, '___sec18'),
|
||||
('Exploring the logistic regression', 2, None, '___sec19'),
|
||||
('Accuracy of a classification model', 2, None, '___sec20'),
|
||||
('Analyzing the results', 2, None, '___sec21')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -109,6 +119,12 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs014.html#___sec13" style="font-size: 80%;">The Softmax function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs015.html#___sec14" style="font-size: 80%;">A <b>scikit-learn</b> example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec15" style="font-size: 80%;">A simple classification problem</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs017.html#___sec16" style="font-size: 80%;">The two-dimensional Ising model, Predicting phase transition of the two-dimensional Ising model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs018.html#___sec17" style="font-size: 80%;">Reading in the data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs019.html#___sec18" style="font-size: 80%;">Logistic regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs020.html#___sec19" style="font-size: 80%;">Exploring the logistic regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs021.html#___sec20" style="font-size: 80%;">Accuracy of a classification model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs022.html#___sec21" style="font-size: 80%;">Analyzing the results</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -179,7 +195,6 @@ MathJax.Hub.Config({
|
||||
main()
|
||||
</pre></div>
|
||||
<p>
|
||||
|
||||
<p>
|
||||
<!-- navigation buttons at the bottom of the page -->
|
||||
<ul class="pagination">
|
||||
@@ -195,6 +210,13 @@ MathJax.Hub.Config({
|
||||
<li><a href="._LogReg-bs014.html">15</a></li>
|
||||
<li><a href="._LogReg-bs015.html">16</a></li>
|
||||
<li class="active"><a href="._LogReg-bs016.html">17</a></li>
|
||||
<li><a href="._LogReg-bs017.html">18</a></li>
|
||||
<li><a href="._LogReg-bs018.html">19</a></li>
|
||||
<li><a href="._LogReg-bs019.html">20</a></li>
|
||||
<li><a href="._LogReg-bs020.html">21</a></li>
|
||||
<li><a href="._LogReg-bs021.html">22</a></li>
|
||||
<li><a href="._LogReg-bs022.html">23</a></li>
|
||||
<li><a href="._LogReg-bs017.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
|
||||
@@ -0,0 +1,247 @@
|
||||
<!--
|
||||
Automatically generated HTML file from DocOnce source
|
||||
(https://github.com/hplgit/doconce/)
|
||||
-->
|
||||
<html>
|
||||
<head>
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
||||
<meta name="generator" content="DocOnce: https://github.com/hplgit/doconce/" />
|
||||
<meta name="description" content="Data Analysis and Machine Learning: Logistic Regression">
|
||||
|
||||
<title>Data Analysis and Machine Learning: Logistic Regression</title>
|
||||
|
||||
<!-- Bootstrap style: bootstrap -->
|
||||
<link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
|
||||
<!-- not necessary
|
||||
<link href="https://netdna.bootstrapcdn.com/font-awesome/4.0.3/css/font-awesome.css" rel="stylesheet">
|
||||
-->
|
||||
|
||||
<style type="text/css">
|
||||
|
||||
/* Add scrollbar to dropdown menus in bootstrap navigation bar */
|
||||
.dropdown-menu {
|
||||
height: auto;
|
||||
max-height: 400px;
|
||||
overflow-x: hidden;
|
||||
}
|
||||
|
||||
/* Adds an invisible element before each target to offset for the navigation
|
||||
bar */
|
||||
.anchor::before {
|
||||
content:"";
|
||||
display:block;
|
||||
height:50px; /* fixed header height for style bootstrap */
|
||||
margin:-50px 0 0; /* negative fixed header height */
|
||||
}
|
||||
</style>
|
||||
|
||||
|
||||
</head>
|
||||
|
||||
<!-- tocinfo
|
||||
{'highest level': 2,
|
||||
'sections': [('Logistic Regression', 2, None, '___sec0'),
|
||||
('Optimization and Deep learning', 2, None, '___sec1'),
|
||||
('Basics', 2, None, '___sec2'),
|
||||
('Linear classifier', 2, None, '___sec3'),
|
||||
('Some selected properties', 2, None, '___sec4'),
|
||||
('The logistic function', 2, None, '___sec5'),
|
||||
('Two parameters', 2, None, '___sec6'),
|
||||
('Maximum likelihood', 2, None, '___sec7'),
|
||||
('The cost function rewritten', 2, None, '___sec8'),
|
||||
('Minimizing the cross entropy', 2, None, '___sec9'),
|
||||
('A more compact expression', 2, None, '___sec10'),
|
||||
('Extending to more predictors', 2, None, '___sec11'),
|
||||
('Including more classes', 2, None, '___sec12'),
|
||||
('The Softmax function', 2, None, '___sec13'),
|
||||
('A _scikit-learn_ example', 2, None, '___sec14'),
|
||||
('A simple classification problem', 2, None, '___sec15'),
|
||||
('The two-dimensional Ising model, Predicting phase transition '
|
||||
'of the two-dimensional Ising model',
|
||||
2,
|
||||
None,
|
||||
'___sec16'),
|
||||
('Reading in the data', 2, None, '___sec17'),
|
||||
('Logistic regression', 2, None, '___sec18'),
|
||||
('Exploring the logistic regression', 2, None, '___sec19'),
|
||||
('Accuracy of a classification model', 2, None, '___sec20'),
|
||||
('Analyzing the results', 2, None, '___sec21')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
|
||||
|
||||
|
||||
<script type="text/x-mathjax-config">
|
||||
MathJax.Hub.Config({
|
||||
TeX: {
|
||||
equationNumbers: { autoNumber: "none" },
|
||||
extensions: ["AMSmath.js", "AMSsymbols.js", "autobold.js", "color.js"]
|
||||
}
|
||||
});
|
||||
</script>
|
||||
<script type="text/javascript" async
|
||||
src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
|
||||
</script>
|
||||
|
||||
|
||||
|
||||
|
||||
<!-- Bootstrap navigation bar -->
|
||||
<div class="navbar navbar-default navbar-fixed-top">
|
||||
<div class="navbar-header">
|
||||
<button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-responsive-collapse">
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
</button>
|
||||
<a class="navbar-brand" href="LogReg-bs.html">Data Analysis and Machine Learning: Logistic Regression</a>
|
||||
</div>
|
||||
|
||||
<div class="navbar-collapse collapse navbar-responsive-collapse">
|
||||
<ul class="nav navbar-nav navbar-right">
|
||||
<li class="dropdown">
|
||||
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
|
||||
<ul class="dropdown-menu">
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs001.html#___sec0" style="font-size: 80%;">Logistic Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs002.html#___sec1" style="font-size: 80%;">Optimization and Deep learning</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs003.html#___sec2" style="font-size: 80%;">Basics</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs004.html#___sec3" style="font-size: 80%;">Linear classifier</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs005.html#___sec4" style="font-size: 80%;">Some selected properties</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs006.html#___sec5" style="font-size: 80%;">The logistic function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs007.html#___sec6" style="font-size: 80%;">Two parameters</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs008.html#___sec7" style="font-size: 80%;">Maximum likelihood</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs009.html#___sec8" style="font-size: 80%;">The cost function rewritten</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs010.html#___sec9" style="font-size: 80%;">Minimizing the cross entropy</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs011.html#___sec10" style="font-size: 80%;">A more compact expression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs012.html#___sec11" style="font-size: 80%;">Extending to more predictors</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs013.html#___sec12" style="font-size: 80%;">Including more classes</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs014.html#___sec13" style="font-size: 80%;">The Softmax function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs015.html#___sec14" style="font-size: 80%;">A <b>scikit-learn</b> example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs016.html#___sec15" style="font-size: 80%;">A simple classification problem</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec16" style="font-size: 80%;">The two-dimensional Ising model, Predicting phase transition of the two-dimensional Ising model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs018.html#___sec17" style="font-size: 80%;">Reading in the data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs019.html#___sec18" style="font-size: 80%;">Logistic regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs020.html#___sec19" style="font-size: 80%;">Exploring the logistic regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs021.html#___sec20" style="font-size: 80%;">Accuracy of a classification model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs022.html#___sec21" style="font-size: 80%;">Analyzing the results</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
</div> <!-- end of navigation bar -->
|
||||
|
||||
<div class="container">
|
||||
|
||||
<p> </p><p> </p><p> </p> <!-- add vertical space -->
|
||||
|
||||
<a name="part0017"></a>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="___sec16" class="anchor">The two-dimensional Ising model, Predicting phase transition of the two-dimensional Ising model </h2>
|
||||
|
||||
<p>
|
||||
The Hamiltonian of the two-dimensional Ising model without an external field for a constant coupling constant \( J \) is given by
|
||||
$$
|
||||
\begin{align}
|
||||
H = -J \sum_{\langle ij\rangle} S_i S_j,
|
||||
\tag{2}
|
||||
\end{align}
|
||||
$$
|
||||
|
||||
where \( S_i \in \{-1, 1\} \) and \( \langle ij \rangle \) signifies that we only iterate over the nearest neighbors in the lattice. We will be looking at a system of \( L = 40 \) spins in each dimension, i.e., \( L^2 = 1600 \) spins in total. Opposed to the one-dimensional Ising model we will get a phase transition from an <b>ordered</b> phase to a <b>disordered</b> phase at the critical temperature
|
||||
|
||||
$$
|
||||
\begin{align}
|
||||
\frac{T_c}{J} = \frac{2}{\log\left(1 + \sqrt{2}\right)} \approx 2.26,
|
||||
\tag{3}
|
||||
\end{align}
|
||||
$$
|
||||
|
||||
as shown by Lars Onsager.
|
||||
|
||||
<p>
|
||||
Here we use <b>logistic regression</b> to predict when a phase transition
|
||||
occurs. The data we will look at is a set of spin configurations,
|
||||
i.e., individual lattices with spins, labeled <b>ordered</b> <code>1</code> or
|
||||
<b>disordered</b> <code>0</code>. Our job is to build a model which will take in a
|
||||
spin configuration and predict whether or not the spin configuration
|
||||
constitutes an ordered or a disordered phase. To achieve this we will
|
||||
represent the lattices as flattened arrays with \( 1600 \) elements
|
||||
instead of a matrix of \( 40 \times 40 \) elements. As an extra test of
|
||||
the performance of the algorithms we will divide the dataset into
|
||||
three pieces. We will do a conventional train-test-split on a
|
||||
combination of totally ordered and totally disordered phases. The
|
||||
remaining "critical-like" states will be used as test data which we
|
||||
hope the model will be able to make good extrapolated predictions on.
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span><span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">pickle</span>
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">os</span>
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">glob</span>
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">numpy</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">np</span>
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">pandas</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">pd</span>
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">matplotlib.pyplot</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">plt</span>
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">seaborn</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">sns</span>
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">sklearn.model_selection</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">skms</span>
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">sklearn.linear_model</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">skl</span>
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">sklearn.metrics</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">skm</span>
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">tqdm</span>
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">copy</span>
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">time</span>
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">IPython.display</span> <span style="color: #008000; font-weight: bold">import</span> display
|
||||
|
||||
<span style="color: #666666">%</span>matplotlib inline
|
||||
|
||||
sns<span style="color: #666666">.</span>set(color_codes<span style="color: #666666">=</span><span style="color: #008000">True</span>)
|
||||
</pre></div>
|
||||
<p>
|
||||
<p>
|
||||
<!-- navigation buttons at the bottom of the page -->
|
||||
<ul class="pagination">
|
||||
<li><a href="._LogReg-bs016.html">«</a></li>
|
||||
<li><a href="._LogReg-bs000.html">1</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._LogReg-bs009.html">10</a></li>
|
||||
<li><a href="._LogReg-bs010.html">11</a></li>
|
||||
<li><a href="._LogReg-bs011.html">12</a></li>
|
||||
<li><a href="._LogReg-bs012.html">13</a></li>
|
||||
<li><a href="._LogReg-bs013.html">14</a></li>
|
||||
<li><a href="._LogReg-bs014.html">15</a></li>
|
||||
<li><a href="._LogReg-bs015.html">16</a></li>
|
||||
<li><a href="._LogReg-bs016.html">17</a></li>
|
||||
<li class="active"><a href="._LogReg-bs017.html">18</a></li>
|
||||
<li><a href="._LogReg-bs018.html">19</a></li>
|
||||
<li><a href="._LogReg-bs019.html">20</a></li>
|
||||
<li><a href="._LogReg-bs020.html">21</a></li>
|
||||
<li><a href="._LogReg-bs021.html">22</a></li>
|
||||
<li><a href="._LogReg-bs022.html">23</a></li>
|
||||
<li><a href="._LogReg-bs018.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
</div> <!-- end container -->
|
||||
<!-- include javascript, jQuery *first* -->
|
||||
<script src="https://ajax.googleapis.com/ajax/libs/jquery/1.10.2/jquery.min.js"></script>
|
||||
<script src="https://netdna.bootstrapcdn.com/bootstrap/3.0.0/js/bootstrap.min.js"></script>
|
||||
|
||||
<!-- Bootstrap footer
|
||||
<footer>
|
||||
<a href="http://..."><img width="250" align=right src="http://..."></a>
|
||||
</footer>
|
||||
-->
|
||||
|
||||
|
||||
<center style="font-size:80%">
|
||||
<!-- copyright only on the titlepage -->
|
||||
</center>
|
||||
|
||||
|
||||
</body>
|
||||
</html>
|
||||
|
||||
|
||||
@@ -0,0 +1,234 @@
|
||||
<!--
|
||||
Automatically generated HTML file from DocOnce source
|
||||
(https://github.com/hplgit/doconce/)
|
||||
-->
|
||||
<html>
|
||||
<head>
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
||||
<meta name="generator" content="DocOnce: https://github.com/hplgit/doconce/" />
|
||||
<meta name="description" content="Data Analysis and Machine Learning: Logistic Regression">
|
||||
|
||||
<title>Data Analysis and Machine Learning: Logistic Regression</title>
|
||||
|
||||
<!-- Bootstrap style: bootstrap -->
|
||||
<link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
|
||||
<!-- not necessary
|
||||
<link href="https://netdna.bootstrapcdn.com/font-awesome/4.0.3/css/font-awesome.css" rel="stylesheet">
|
||||
-->
|
||||
|
||||
<style type="text/css">
|
||||
|
||||
/* Add scrollbar to dropdown menus in bootstrap navigation bar */
|
||||
.dropdown-menu {
|
||||
height: auto;
|
||||
max-height: 400px;
|
||||
overflow-x: hidden;
|
||||
}
|
||||
|
||||
/* Adds an invisible element before each target to offset for the navigation
|
||||
bar */
|
||||
.anchor::before {
|
||||
content:"";
|
||||
display:block;
|
||||
height:50px; /* fixed header height for style bootstrap */
|
||||
margin:-50px 0 0; /* negative fixed header height */
|
||||
}
|
||||
</style>
|
||||
|
||||
|
||||
</head>
|
||||
|
||||
<!-- tocinfo
|
||||
{'highest level': 2,
|
||||
'sections': [('Logistic Regression', 2, None, '___sec0'),
|
||||
('Optimization and Deep learning', 2, None, '___sec1'),
|
||||
('Basics', 2, None, '___sec2'),
|
||||
('Linear classifier', 2, None, '___sec3'),
|
||||
('Some selected properties', 2, None, '___sec4'),
|
||||
('The logistic function', 2, None, '___sec5'),
|
||||
('Two parameters', 2, None, '___sec6'),
|
||||
('Maximum likelihood', 2, None, '___sec7'),
|
||||
('The cost function rewritten', 2, None, '___sec8'),
|
||||
('Minimizing the cross entropy', 2, None, '___sec9'),
|
||||
('A more compact expression', 2, None, '___sec10'),
|
||||
('Extending to more predictors', 2, None, '___sec11'),
|
||||
('Including more classes', 2, None, '___sec12'),
|
||||
('The Softmax function', 2, None, '___sec13'),
|
||||
('A _scikit-learn_ example', 2, None, '___sec14'),
|
||||
('A simple classification problem', 2, None, '___sec15'),
|
||||
('The two-dimensional Ising model, Predicting phase transition '
|
||||
'of the two-dimensional Ising model',
|
||||
2,
|
||||
None,
|
||||
'___sec16'),
|
||||
('Reading in the data', 2, None, '___sec17'),
|
||||
('Logistic regression', 2, None, '___sec18'),
|
||||
('Exploring the logistic regression', 2, None, '___sec19'),
|
||||
('Accuracy of a classification model', 2, None, '___sec20'),
|
||||
('Analyzing the results', 2, None, '___sec21')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
|
||||
|
||||
|
||||
<script type="text/x-mathjax-config">
|
||||
MathJax.Hub.Config({
|
||||
TeX: {
|
||||
equationNumbers: { autoNumber: "none" },
|
||||
extensions: ["AMSmath.js", "AMSsymbols.js", "autobold.js", "color.js"]
|
||||
}
|
||||
});
|
||||
</script>
|
||||
<script type="text/javascript" async
|
||||
src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
|
||||
</script>
|
||||
|
||||
|
||||
|
||||
|
||||
<!-- Bootstrap navigation bar -->
|
||||
<div class="navbar navbar-default navbar-fixed-top">
|
||||
<div class="navbar-header">
|
||||
<button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-responsive-collapse">
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
</button>
|
||||
<a class="navbar-brand" href="LogReg-bs.html">Data Analysis and Machine Learning: Logistic Regression</a>
|
||||
</div>
|
||||
|
||||
<div class="navbar-collapse collapse navbar-responsive-collapse">
|
||||
<ul class="nav navbar-nav navbar-right">
|
||||
<li class="dropdown">
|
||||
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
|
||||
<ul class="dropdown-menu">
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs001.html#___sec0" style="font-size: 80%;">Logistic Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs002.html#___sec1" style="font-size: 80%;">Optimization and Deep learning</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs003.html#___sec2" style="font-size: 80%;">Basics</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs004.html#___sec3" style="font-size: 80%;">Linear classifier</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs005.html#___sec4" style="font-size: 80%;">Some selected properties</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs006.html#___sec5" style="font-size: 80%;">The logistic function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs007.html#___sec6" style="font-size: 80%;">Two parameters</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs008.html#___sec7" style="font-size: 80%;">Maximum likelihood</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs009.html#___sec8" style="font-size: 80%;">The cost function rewritten</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs010.html#___sec9" style="font-size: 80%;">Minimizing the cross entropy</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs011.html#___sec10" style="font-size: 80%;">A more compact expression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs012.html#___sec11" style="font-size: 80%;">Extending to more predictors</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs013.html#___sec12" style="font-size: 80%;">Including more classes</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs014.html#___sec13" style="font-size: 80%;">The Softmax function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs015.html#___sec14" style="font-size: 80%;">A <b>scikit-learn</b> example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs016.html#___sec15" style="font-size: 80%;">A simple classification problem</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs017.html#___sec16" style="font-size: 80%;">The two-dimensional Ising model, Predicting phase transition of the two-dimensional Ising model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec17" style="font-size: 80%;">Reading in the data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs019.html#___sec18" style="font-size: 80%;">Logistic regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs020.html#___sec19" style="font-size: 80%;">Exploring the logistic regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs021.html#___sec20" style="font-size: 80%;">Accuracy of a classification model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs022.html#___sec21" style="font-size: 80%;">Analyzing the results</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
</div> <!-- end of navigation bar -->
|
||||
|
||||
<div class="container">
|
||||
|
||||
<p> </p><p> </p><p> </p> <!-- add vertical space -->
|
||||
|
||||
<a name="part0018"></a>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="___sec17" class="anchor">Reading in the data </h2>
|
||||
|
||||
<p>
|
||||
Using the data from <a href="https://physics.bu.edu/~pankajm/ML-Review-Datasets/isingMC/" target="_self">Mehta et al.</a> (specifically the two datasets named <code>Ising2DFM_reSample_L40_T=All.pkl</code> and <code>Ising2DFM_reSample_L40_T=All_labels.pkl</code>) we have to unpack the data into numpy arrays.
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span>filenames <span style="color: #666666">=</span> glob<span style="color: #666666">.</span>glob(os<span style="color: #666666">.</span>path<span style="color: #666666">.</span>join(<span style="color: #BA2121">".."</span>, <span style="color: #BA2121">"dat"</span>, <span style="color: #BA2121">"*"</span>))
|
||||
label_filename <span style="color: #666666">=</span> <span style="color: #008000">list</span>(<span style="color: #008000">filter</span>(<span style="color: #008000; font-weight: bold">lambda</span> x: <span style="color: #BA2121">"label"</span> <span style="color: #AA22FF; font-weight: bold">in</span> x, filenames))[<span style="color: #666666">0</span>]
|
||||
dat_filename <span style="color: #666666">=</span> <span style="color: #008000">list</span>(<span style="color: #008000">filter</span>(<span style="color: #008000; font-weight: bold">lambda</span> x: <span style="color: #BA2121">"label"</span> <span style="color: #AA22FF; font-weight: bold">not</span> <span style="color: #AA22FF; font-weight: bold">in</span> x, filenames))[<span style="color: #666666">0</span>]
|
||||
|
||||
<span style="color: #408080; font-style: italic"># Read in the labels</span>
|
||||
<span style="color: #008000; font-weight: bold">with</span> <span style="color: #008000">open</span>(label_filename, <span style="color: #BA2121">"rb"</span>) <span style="color: #008000; font-weight: bold">as</span> f:
|
||||
labels <span style="color: #666666">=</span> pickle<span style="color: #666666">.</span>load(f)
|
||||
|
||||
<span style="color: #408080; font-style: italic"># Read in the corresponding configurations</span>
|
||||
<span style="color: #008000; font-weight: bold">with</span> <span style="color: #008000">open</span>(dat_filename, <span style="color: #BA2121">"rb"</span>) <span style="color: #008000; font-weight: bold">as</span> f:
|
||||
data <span style="color: #666666">=</span> np<span style="color: #666666">.</span>unpackbits(pickle<span style="color: #666666">.</span>load(f))<span style="color: #666666">.</span>reshape(<span style="color: #666666">-1</span>, <span style="color: #666666">1600</span>)<span style="color: #666666">.</span>astype(<span style="color: #BA2121">"int"</span>)
|
||||
|
||||
<span style="color: #408080; font-style: italic"># Set spin-down to -1</span>
|
||||
data[data <span style="color: #666666">==</span> <span style="color: #666666">0</span>] <span style="color: #666666">=</span> <span style="color: #666666">-1</span>
|
||||
</pre></div>
|
||||
<p>
|
||||
This dataset consists of \( 10000 \) samples, i.e., \( 10000 \) spin
|
||||
configurations with \( 40 \times 40 \) spins each, for \( 16 \) temperatures
|
||||
between \( 0.25 \) to \( 4.0 \). Next we create a train/test-split and keep
|
||||
the data in the critical phase as a separate dataset for
|
||||
extrapolation-testing.
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span><span style="color: #408080; font-style: italic"># Set up slices of the dataset</span>
|
||||
ordered <span style="color: #666666">=</span> <span style="color: #008000">slice</span>(<span style="color: #666666">0</span>, <span style="color: #666666">70000</span>)
|
||||
critical <span style="color: #666666">=</span> <span style="color: #008000">slice</span>(<span style="color: #666666">70000</span>, <span style="color: #666666">100000</span>)
|
||||
disordered <span style="color: #666666">=</span> <span style="color: #008000">slice</span>(<span style="color: #666666">100000</span>, <span style="color: #666666">160000</span>)
|
||||
|
||||
X_train, X_test, y_train, y_test <span style="color: #666666">=</span> skms<span style="color: #666666">.</span>train_test_split(
|
||||
np<span style="color: #666666">.</span>concatenate((data[ordered], data[disordered])),
|
||||
np<span style="color: #666666">.</span>concatenate((labels[ordered], labels[disordered])),
|
||||
test_size<span style="color: #666666">=0.95</span>
|
||||
)
|
||||
</pre></div>
|
||||
<p>
|
||||
Using a small training set yields a better accuracy. This will be discussed in the end.
|
||||
|
||||
<p>
|
||||
<p>
|
||||
<!-- navigation buttons at the bottom of the page -->
|
||||
<ul class="pagination">
|
||||
<li><a href="._LogReg-bs017.html">«</a></li>
|
||||
<li><a href="._LogReg-bs000.html">1</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._LogReg-bs010.html">11</a></li>
|
||||
<li><a href="._LogReg-bs011.html">12</a></li>
|
||||
<li><a href="._LogReg-bs012.html">13</a></li>
|
||||
<li><a href="._LogReg-bs013.html">14</a></li>
|
||||
<li><a href="._LogReg-bs014.html">15</a></li>
|
||||
<li><a href="._LogReg-bs015.html">16</a></li>
|
||||
<li><a href="._LogReg-bs016.html">17</a></li>
|
||||
<li><a href="._LogReg-bs017.html">18</a></li>
|
||||
<li class="active"><a href="._LogReg-bs018.html">19</a></li>
|
||||
<li><a href="._LogReg-bs019.html">20</a></li>
|
||||
<li><a href="._LogReg-bs020.html">21</a></li>
|
||||
<li><a href="._LogReg-bs021.html">22</a></li>
|
||||
<li><a href="._LogReg-bs022.html">23</a></li>
|
||||
<li><a href="._LogReg-bs019.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
</div> <!-- end container -->
|
||||
<!-- include javascript, jQuery *first* -->
|
||||
<script src="https://ajax.googleapis.com/ajax/libs/jquery/1.10.2/jquery.min.js"></script>
|
||||
<script src="https://netdna.bootstrapcdn.com/bootstrap/3.0.0/js/bootstrap.min.js"></script>
|
||||
|
||||
<!-- Bootstrap footer
|
||||
<footer>
|
||||
<a href="http://..."><img width="250" align=right src="http://..."></a>
|
||||
</footer>
|
||||
-->
|
||||
|
||||
|
||||
<center style="font-size:80%">
|
||||
<!-- copyright only on the titlepage -->
|
||||
</center>
|
||||
|
||||
|
||||
</body>
|
||||
</html>
|
||||
|
||||
|
||||
@@ -0,0 +1,234 @@
|
||||
<!--
|
||||
Automatically generated HTML file from DocOnce source
|
||||
(https://github.com/hplgit/doconce/)
|
||||
-->
|
||||
<html>
|
||||
<head>
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
||||
<meta name="generator" content="DocOnce: https://github.com/hplgit/doconce/" />
|
||||
<meta name="description" content="Data Analysis and Machine Learning: Logistic Regression">
|
||||
|
||||
<title>Data Analysis and Machine Learning: Logistic Regression</title>
|
||||
|
||||
<!-- Bootstrap style: bootstrap -->
|
||||
<link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
|
||||
<!-- not necessary
|
||||
<link href="https://netdna.bootstrapcdn.com/font-awesome/4.0.3/css/font-awesome.css" rel="stylesheet">
|
||||
-->
|
||||
|
||||
<style type="text/css">
|
||||
|
||||
/* Add scrollbar to dropdown menus in bootstrap navigation bar */
|
||||
.dropdown-menu {
|
||||
height: auto;
|
||||
max-height: 400px;
|
||||
overflow-x: hidden;
|
||||
}
|
||||
|
||||
/* Adds an invisible element before each target to offset for the navigation
|
||||
bar */
|
||||
.anchor::before {
|
||||
content:"";
|
||||
display:block;
|
||||
height:50px; /* fixed header height for style bootstrap */
|
||||
margin:-50px 0 0; /* negative fixed header height */
|
||||
}
|
||||
</style>
|
||||
|
||||
|
||||
</head>
|
||||
|
||||
<!-- tocinfo
|
||||
{'highest level': 2,
|
||||
'sections': [('Logistic Regression', 2, None, '___sec0'),
|
||||
('Optimization and Deep learning', 2, None, '___sec1'),
|
||||
('Basics', 2, None, '___sec2'),
|
||||
('Linear classifier', 2, None, '___sec3'),
|
||||
('Some selected properties', 2, None, '___sec4'),
|
||||
('The logistic function', 2, None, '___sec5'),
|
||||
('Two parameters', 2, None, '___sec6'),
|
||||
('Maximum likelihood', 2, None, '___sec7'),
|
||||
('The cost function rewritten', 2, None, '___sec8'),
|
||||
('Minimizing the cross entropy', 2, None, '___sec9'),
|
||||
('A more compact expression', 2, None, '___sec10'),
|
||||
('Extending to more predictors', 2, None, '___sec11'),
|
||||
('Including more classes', 2, None, '___sec12'),
|
||||
('The Softmax function', 2, None, '___sec13'),
|
||||
('A _scikit-learn_ example', 2, None, '___sec14'),
|
||||
('A simple classification problem', 2, None, '___sec15'),
|
||||
('The two-dimensional Ising model, Predicting phase transition '
|
||||
'of the two-dimensional Ising model',
|
||||
2,
|
||||
None,
|
||||
'___sec16'),
|
||||
('Reading in the data', 2, None, '___sec17'),
|
||||
('Logistic regression', 2, None, '___sec18'),
|
||||
('Exploring the logistic regression', 2, None, '___sec19'),
|
||||
('Accuracy of a classification model', 2, None, '___sec20'),
|
||||
('Analyzing the results', 2, None, '___sec21')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
|
||||
|
||||
|
||||
<script type="text/x-mathjax-config">
|
||||
MathJax.Hub.Config({
|
||||
TeX: {
|
||||
equationNumbers: { autoNumber: "none" },
|
||||
extensions: ["AMSmath.js", "AMSsymbols.js", "autobold.js", "color.js"]
|
||||
}
|
||||
});
|
||||
</script>
|
||||
<script type="text/javascript" async
|
||||
src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
|
||||
</script>
|
||||
|
||||
|
||||
|
||||
|
||||
<!-- Bootstrap navigation bar -->
|
||||
<div class="navbar navbar-default navbar-fixed-top">
|
||||
<div class="navbar-header">
|
||||
<button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-responsive-collapse">
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
</button>
|
||||
<a class="navbar-brand" href="LogReg-bs.html">Data Analysis and Machine Learning: Logistic Regression</a>
|
||||
</div>
|
||||
|
||||
<div class="navbar-collapse collapse navbar-responsive-collapse">
|
||||
<ul class="nav navbar-nav navbar-right">
|
||||
<li class="dropdown">
|
||||
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
|
||||
<ul class="dropdown-menu">
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs001.html#___sec0" style="font-size: 80%;">Logistic Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs002.html#___sec1" style="font-size: 80%;">Optimization and Deep learning</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs003.html#___sec2" style="font-size: 80%;">Basics</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs004.html#___sec3" style="font-size: 80%;">Linear classifier</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs005.html#___sec4" style="font-size: 80%;">Some selected properties</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs006.html#___sec5" style="font-size: 80%;">The logistic function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs007.html#___sec6" style="font-size: 80%;">Two parameters</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs008.html#___sec7" style="font-size: 80%;">Maximum likelihood</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs009.html#___sec8" style="font-size: 80%;">The cost function rewritten</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs010.html#___sec9" style="font-size: 80%;">Minimizing the cross entropy</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs011.html#___sec10" style="font-size: 80%;">A more compact expression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs012.html#___sec11" style="font-size: 80%;">Extending to more predictors</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs013.html#___sec12" style="font-size: 80%;">Including more classes</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs014.html#___sec13" style="font-size: 80%;">The Softmax function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs015.html#___sec14" style="font-size: 80%;">A <b>scikit-learn</b> example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs016.html#___sec15" style="font-size: 80%;">A simple classification problem</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs017.html#___sec16" style="font-size: 80%;">The two-dimensional Ising model, Predicting phase transition of the two-dimensional Ising model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs018.html#___sec17" style="font-size: 80%;">Reading in the data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec18" style="font-size: 80%;">Logistic regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs020.html#___sec19" style="font-size: 80%;">Exploring the logistic regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs021.html#___sec20" style="font-size: 80%;">Accuracy of a classification model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs022.html#___sec21" style="font-size: 80%;">Analyzing the results</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
</div> <!-- end of navigation bar -->
|
||||
|
||||
<div class="container">
|
||||
|
||||
<p> </p><p> </p><p> </p> <!-- add vertical space -->
|
||||
|
||||
<a name="part0019"></a>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="___sec18" class="anchor">Logistic regression </h2>
|
||||
|
||||
<p>
|
||||
Logistic regression is a linear model for classification. Recalling
|
||||
the cost function for ordinary least squares with both L2 (ridge) and
|
||||
L1 (LASSO) penalties we will see that the logistic cost function is
|
||||
very similar. In OLS we wish to predict a continuous variable
|
||||
\( \hat{y} \) using
|
||||
$$
|
||||
\begin{align}
|
||||
\hat{y} = X\omega,
|
||||
\tag{4}
|
||||
\end{align}
|
||||
$$
|
||||
|
||||
<p>
|
||||
where \( X \in \mathbb{R}^{n \times p} \) is the input data and \( \omega^{p
|
||||
\times d} \) are the weights of the regression. In a classification
|
||||
setting (binary classification in our situation) we are interested in
|
||||
a positive or negative answer. We can thus define either answer to be
|
||||
above or below some threshold. But, in order to limit the size of the
|
||||
answer and also to get a probability interpretation on how sure we are
|
||||
for either answer we can compute the sigmoid function of OLS. That is,
|
||||
|
||||
$$
|
||||
\begin{align}
|
||||
f(X\omega) = \frac{1}{1 + \exp(-X\omega)}.
|
||||
\tag{5}
|
||||
\end{align}
|
||||
$$
|
||||
|
||||
We are thus interested in minizming the following cost function
|
||||
$$
|
||||
\begin{align}
|
||||
C(X, \omega) = \sum_{i = 1}^n \left\{
|
||||
- y_i\log\left( f(x_i^T\omega) \right)
|
||||
- (1 - y_i)\log\left[1 - f(x_i^T\omega)\right]
|
||||
\right\},
|
||||
\tag{6}
|
||||
\end{align}
|
||||
$$
|
||||
|
||||
<p>
|
||||
where we will restrict ourselves to a value for \( f(z) \) as the sigmoid
|
||||
described above. We can also tack on a L2 (Ridge) or L1 (LASSO)
|
||||
penalization to this cost function in the same manner we did for
|
||||
linear regression.
|
||||
|
||||
<p>
|
||||
<p>
|
||||
<!-- navigation buttons at the bottom of the page -->
|
||||
<ul class="pagination">
|
||||
<li><a href="._LogReg-bs018.html">«</a></li>
|
||||
<li><a href="._LogReg-bs000.html">1</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._LogReg-bs011.html">12</a></li>
|
||||
<li><a href="._LogReg-bs012.html">13</a></li>
|
||||
<li><a href="._LogReg-bs013.html">14</a></li>
|
||||
<li><a href="._LogReg-bs014.html">15</a></li>
|
||||
<li><a href="._LogReg-bs015.html">16</a></li>
|
||||
<li><a href="._LogReg-bs016.html">17</a></li>
|
||||
<li><a href="._LogReg-bs017.html">18</a></li>
|
||||
<li><a href="._LogReg-bs018.html">19</a></li>
|
||||
<li class="active"><a href="._LogReg-bs019.html">20</a></li>
|
||||
<li><a href="._LogReg-bs020.html">21</a></li>
|
||||
<li><a href="._LogReg-bs021.html">22</a></li>
|
||||
<li><a href="._LogReg-bs022.html">23</a></li>
|
||||
<li><a href="._LogReg-bs020.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
</div> <!-- end container -->
|
||||
<!-- include javascript, jQuery *first* -->
|
||||
<script src="https://ajax.googleapis.com/ajax/libs/jquery/1.10.2/jquery.min.js"></script>
|
||||
<script src="https://netdna.bootstrapcdn.com/bootstrap/3.0.0/js/bootstrap.min.js"></script>
|
||||
|
||||
<!-- Bootstrap footer
|
||||
<footer>
|
||||
<a href="http://..."><img width="250" align=right src="http://..."></a>
|
||||
</footer>
|
||||
-->
|
||||
|
||||
|
||||
<center style="font-size:80%">
|
||||
<!-- copyright only on the titlepage -->
|
||||
</center>
|
||||
|
||||
|
||||
</body>
|
||||
</html>
|
||||
|
||||
|
||||
@@ -0,0 +1,233 @@
|
||||
<!--
|
||||
Automatically generated HTML file from DocOnce source
|
||||
(https://github.com/hplgit/doconce/)
|
||||
-->
|
||||
<html>
|
||||
<head>
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
||||
<meta name="generator" content="DocOnce: https://github.com/hplgit/doconce/" />
|
||||
<meta name="description" content="Data Analysis and Machine Learning: Logistic Regression">
|
||||
|
||||
<title>Data Analysis and Machine Learning: Logistic Regression</title>
|
||||
|
||||
<!-- Bootstrap style: bootstrap -->
|
||||
<link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
|
||||
<!-- not necessary
|
||||
<link href="https://netdna.bootstrapcdn.com/font-awesome/4.0.3/css/font-awesome.css" rel="stylesheet">
|
||||
-->
|
||||
|
||||
<style type="text/css">
|
||||
|
||||
/* Add scrollbar to dropdown menus in bootstrap navigation bar */
|
||||
.dropdown-menu {
|
||||
height: auto;
|
||||
max-height: 400px;
|
||||
overflow-x: hidden;
|
||||
}
|
||||
|
||||
/* Adds an invisible element before each target to offset for the navigation
|
||||
bar */
|
||||
.anchor::before {
|
||||
content:"";
|
||||
display:block;
|
||||
height:50px; /* fixed header height for style bootstrap */
|
||||
margin:-50px 0 0; /* negative fixed header height */
|
||||
}
|
||||
</style>
|
||||
|
||||
|
||||
</head>
|
||||
|
||||
<!-- tocinfo
|
||||
{'highest level': 2,
|
||||
'sections': [('Logistic Regression', 2, None, '___sec0'),
|
||||
('Optimization and Deep learning', 2, None, '___sec1'),
|
||||
('Basics', 2, None, '___sec2'),
|
||||
('Linear classifier', 2, None, '___sec3'),
|
||||
('Some selected properties', 2, None, '___sec4'),
|
||||
('The logistic function', 2, None, '___sec5'),
|
||||
('Two parameters', 2, None, '___sec6'),
|
||||
('Maximum likelihood', 2, None, '___sec7'),
|
||||
('The cost function rewritten', 2, None, '___sec8'),
|
||||
('Minimizing the cross entropy', 2, None, '___sec9'),
|
||||
('A more compact expression', 2, None, '___sec10'),
|
||||
('Extending to more predictors', 2, None, '___sec11'),
|
||||
('Including more classes', 2, None, '___sec12'),
|
||||
('The Softmax function', 2, None, '___sec13'),
|
||||
('A _scikit-learn_ example', 2, None, '___sec14'),
|
||||
('A simple classification problem', 2, None, '___sec15'),
|
||||
('The two-dimensional Ising model, Predicting phase transition '
|
||||
'of the two-dimensional Ising model',
|
||||
2,
|
||||
None,
|
||||
'___sec16'),
|
||||
('Reading in the data', 2, None, '___sec17'),
|
||||
('Logistic regression', 2, None, '___sec18'),
|
||||
('Exploring the logistic regression', 2, None, '___sec19'),
|
||||
('Accuracy of a classification model', 2, None, '___sec20'),
|
||||
('Analyzing the results', 2, None, '___sec21')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
|
||||
|
||||
|
||||
<script type="text/x-mathjax-config">
|
||||
MathJax.Hub.Config({
|
||||
TeX: {
|
||||
equationNumbers: { autoNumber: "none" },
|
||||
extensions: ["AMSmath.js", "AMSsymbols.js", "autobold.js", "color.js"]
|
||||
}
|
||||
});
|
||||
</script>
|
||||
<script type="text/javascript" async
|
||||
src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
|
||||
</script>
|
||||
|
||||
|
||||
|
||||
|
||||
<!-- Bootstrap navigation bar -->
|
||||
<div class="navbar navbar-default navbar-fixed-top">
|
||||
<div class="navbar-header">
|
||||
<button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-responsive-collapse">
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
</button>
|
||||
<a class="navbar-brand" href="LogReg-bs.html">Data Analysis and Machine Learning: Logistic Regression</a>
|
||||
</div>
|
||||
|
||||
<div class="navbar-collapse collapse navbar-responsive-collapse">
|
||||
<ul class="nav navbar-nav navbar-right">
|
||||
<li class="dropdown">
|
||||
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
|
||||
<ul class="dropdown-menu">
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs001.html#___sec0" style="font-size: 80%;">Logistic Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs002.html#___sec1" style="font-size: 80%;">Optimization and Deep learning</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs003.html#___sec2" style="font-size: 80%;">Basics</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs004.html#___sec3" style="font-size: 80%;">Linear classifier</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs005.html#___sec4" style="font-size: 80%;">Some selected properties</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs006.html#___sec5" style="font-size: 80%;">The logistic function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs007.html#___sec6" style="font-size: 80%;">Two parameters</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs008.html#___sec7" style="font-size: 80%;">Maximum likelihood</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs009.html#___sec8" style="font-size: 80%;">The cost function rewritten</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs010.html#___sec9" style="font-size: 80%;">Minimizing the cross entropy</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs011.html#___sec10" style="font-size: 80%;">A more compact expression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs012.html#___sec11" style="font-size: 80%;">Extending to more predictors</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs013.html#___sec12" style="font-size: 80%;">Including more classes</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs014.html#___sec13" style="font-size: 80%;">The Softmax function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs015.html#___sec14" style="font-size: 80%;">A <b>scikit-learn</b> example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs016.html#___sec15" style="font-size: 80%;">A simple classification problem</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs017.html#___sec16" style="font-size: 80%;">The two-dimensional Ising model, Predicting phase transition of the two-dimensional Ising model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs018.html#___sec17" style="font-size: 80%;">Reading in the data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs019.html#___sec18" style="font-size: 80%;">Logistic regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec19" style="font-size: 80%;">Exploring the logistic regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs021.html#___sec20" style="font-size: 80%;">Accuracy of a classification model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs022.html#___sec21" style="font-size: 80%;">Analyzing the results</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
</div> <!-- end of navigation bar -->
|
||||
|
||||
<div class="container">
|
||||
|
||||
<p> </p><p> </p><p> </p> <!-- add vertical space -->
|
||||
|
||||
<a name="part0020"></a>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="___sec19" class="anchor">Exploring the logistic regression </h2>
|
||||
|
||||
<p>
|
||||
The penalization factor \( \lambda \) is inverted in the case of the
|
||||
logistic regression model we use. We will explore several values of
|
||||
\( \lambda \) using both L1 and L2 penalization. We do this using a grid
|
||||
search over different parameters and run a 3-fold cross validation for
|
||||
each configuration. In other words, we fit a model 3 times for each
|
||||
configuration of the hyper parameters.
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span>lambdas <span style="color: #666666">=</span> np<span style="color: #666666">.</span>logspace(<span style="color: #666666">-7</span>, <span style="color: #666666">-1</span>, <span style="color: #666666">7</span>)
|
||||
|
||||
param_grid <span style="color: #666666">=</span> {
|
||||
<span style="color: #BA2121">"C"</span>: <span style="color: #008000">list</span>(<span style="color: #666666">1.0/</span>lambdas),
|
||||
<span style="color: #BA2121">"penalty"</span>: [<span style="color: #BA2121">"l1"</span>, <span style="color: #BA2121">"l2"</span>]
|
||||
}
|
||||
clf <span style="color: #666666">=</span> skms<span style="color: #666666">.</span>GridSearchCV(
|
||||
skl<span style="color: #666666">.</span>LogisticRegression(),
|
||||
param_grid<span style="color: #666666">=</span>param_grid,
|
||||
n_jobs<span style="color: #666666">=-1</span>,
|
||||
return_train_score<span style="color: #666666">=</span><span style="color: #008000">True</span>
|
||||
)
|
||||
t0 <span style="color: #666666">=</span> time<span style="color: #666666">.</span>time()
|
||||
clf<span style="color: #666666">.</span>fit(X_train, y_train)
|
||||
t1 <span style="color: #666666">=</span> time<span style="color: #666666">.</span>time()
|
||||
|
||||
<span style="color: #008000; font-weight: bold">print</span> (
|
||||
<span style="color: #BA2121">"Time spent fitting GridSearchCV(LogisticRegression): {0:.3f} sec"</span><span style="color: #666666">.</span>format(
|
||||
t1 <span style="color: #666666">-</span> t0
|
||||
)
|
||||
)
|
||||
</pre></div>
|
||||
<p>
|
||||
We can see that logistic regression is quite slow and using the grid
|
||||
search and cross validation results in quite a heavy
|
||||
computation. Below we show the results of the different
|
||||
configurations.
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span>logreg_df <span style="color: #666666">=</span> pd<span style="color: #666666">.</span>DataFrame(clf<span style="color: #666666">.</span>cv_results_)
|
||||
|
||||
display(logreg_df)
|
||||
</pre></div>
|
||||
<p>
|
||||
<p>
|
||||
<!-- navigation buttons at the bottom of the page -->
|
||||
<ul class="pagination">
|
||||
<li><a href="._LogReg-bs019.html">«</a></li>
|
||||
<li><a href="._LogReg-bs000.html">1</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._LogReg-bs012.html">13</a></li>
|
||||
<li><a href="._LogReg-bs013.html">14</a></li>
|
||||
<li><a href="._LogReg-bs014.html">15</a></li>
|
||||
<li><a href="._LogReg-bs015.html">16</a></li>
|
||||
<li><a href="._LogReg-bs016.html">17</a></li>
|
||||
<li><a href="._LogReg-bs017.html">18</a></li>
|
||||
<li><a href="._LogReg-bs018.html">19</a></li>
|
||||
<li><a href="._LogReg-bs019.html">20</a></li>
|
||||
<li class="active"><a href="._LogReg-bs020.html">21</a></li>
|
||||
<li><a href="._LogReg-bs021.html">22</a></li>
|
||||
<li><a href="._LogReg-bs022.html">23</a></li>
|
||||
<li><a href="._LogReg-bs021.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
</div> <!-- end container -->
|
||||
<!-- include javascript, jQuery *first* -->
|
||||
<script src="https://ajax.googleapis.com/ajax/libs/jquery/1.10.2/jquery.min.js"></script>
|
||||
<script src="https://netdna.bootstrapcdn.com/bootstrap/3.0.0/js/bootstrap.min.js"></script>
|
||||
|
||||
<!-- Bootstrap footer
|
||||
<footer>
|
||||
<a href="http://..."><img width="250" align=right src="http://..."></a>
|
||||
</footer>
|
||||
-->
|
||||
|
||||
|
||||
<center style="font-size:80%">
|
||||
<!-- copyright only on the titlepage -->
|
||||
</center>
|
||||
|
||||
|
||||
</body>
|
||||
</html>
|
||||
|
||||
|
||||
@@ -0,0 +1,232 @@
|
||||
<!--
|
||||
Automatically generated HTML file from DocOnce source
|
||||
(https://github.com/hplgit/doconce/)
|
||||
-->
|
||||
<html>
|
||||
<head>
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
||||
<meta name="generator" content="DocOnce: https://github.com/hplgit/doconce/" />
|
||||
<meta name="description" content="Data Analysis and Machine Learning: Logistic Regression">
|
||||
|
||||
<title>Data Analysis and Machine Learning: Logistic Regression</title>
|
||||
|
||||
<!-- Bootstrap style: bootstrap -->
|
||||
<link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
|
||||
<!-- not necessary
|
||||
<link href="https://netdna.bootstrapcdn.com/font-awesome/4.0.3/css/font-awesome.css" rel="stylesheet">
|
||||
-->
|
||||
|
||||
<style type="text/css">
|
||||
|
||||
/* Add scrollbar to dropdown menus in bootstrap navigation bar */
|
||||
.dropdown-menu {
|
||||
height: auto;
|
||||
max-height: 400px;
|
||||
overflow-x: hidden;
|
||||
}
|
||||
|
||||
/* Adds an invisible element before each target to offset for the navigation
|
||||
bar */
|
||||
.anchor::before {
|
||||
content:"";
|
||||
display:block;
|
||||
height:50px; /* fixed header height for style bootstrap */
|
||||
margin:-50px 0 0; /* negative fixed header height */
|
||||
}
|
||||
</style>
|
||||
|
||||
|
||||
</head>
|
||||
|
||||
<!-- tocinfo
|
||||
{'highest level': 2,
|
||||
'sections': [('Logistic Regression', 2, None, '___sec0'),
|
||||
('Optimization and Deep learning', 2, None, '___sec1'),
|
||||
('Basics', 2, None, '___sec2'),
|
||||
('Linear classifier', 2, None, '___sec3'),
|
||||
('Some selected properties', 2, None, '___sec4'),
|
||||
('The logistic function', 2, None, '___sec5'),
|
||||
('Two parameters', 2, None, '___sec6'),
|
||||
('Maximum likelihood', 2, None, '___sec7'),
|
||||
('The cost function rewritten', 2, None, '___sec8'),
|
||||
('Minimizing the cross entropy', 2, None, '___sec9'),
|
||||
('A more compact expression', 2, None, '___sec10'),
|
||||
('Extending to more predictors', 2, None, '___sec11'),
|
||||
('Including more classes', 2, None, '___sec12'),
|
||||
('The Softmax function', 2, None, '___sec13'),
|
||||
('A _scikit-learn_ example', 2, None, '___sec14'),
|
||||
('A simple classification problem', 2, None, '___sec15'),
|
||||
('The two-dimensional Ising model, Predicting phase transition '
|
||||
'of the two-dimensional Ising model',
|
||||
2,
|
||||
None,
|
||||
'___sec16'),
|
||||
('Reading in the data', 2, None, '___sec17'),
|
||||
('Logistic regression', 2, None, '___sec18'),
|
||||
('Exploring the logistic regression', 2, None, '___sec19'),
|
||||
('Accuracy of a classification model', 2, None, '___sec20'),
|
||||
('Analyzing the results', 2, None, '___sec21')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
|
||||
|
||||
|
||||
<script type="text/x-mathjax-config">
|
||||
MathJax.Hub.Config({
|
||||
TeX: {
|
||||
equationNumbers: { autoNumber: "none" },
|
||||
extensions: ["AMSmath.js", "AMSsymbols.js", "autobold.js", "color.js"]
|
||||
}
|
||||
});
|
||||
</script>
|
||||
<script type="text/javascript" async
|
||||
src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
|
||||
</script>
|
||||
|
||||
|
||||
|
||||
|
||||
<!-- Bootstrap navigation bar -->
|
||||
<div class="navbar navbar-default navbar-fixed-top">
|
||||
<div class="navbar-header">
|
||||
<button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-responsive-collapse">
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
</button>
|
||||
<a class="navbar-brand" href="LogReg-bs.html">Data Analysis and Machine Learning: Logistic Regression</a>
|
||||
</div>
|
||||
|
||||
<div class="navbar-collapse collapse navbar-responsive-collapse">
|
||||
<ul class="nav navbar-nav navbar-right">
|
||||
<li class="dropdown">
|
||||
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
|
||||
<ul class="dropdown-menu">
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs001.html#___sec0" style="font-size: 80%;">Logistic Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs002.html#___sec1" style="font-size: 80%;">Optimization and Deep learning</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs003.html#___sec2" style="font-size: 80%;">Basics</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs004.html#___sec3" style="font-size: 80%;">Linear classifier</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs005.html#___sec4" style="font-size: 80%;">Some selected properties</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs006.html#___sec5" style="font-size: 80%;">The logistic function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs007.html#___sec6" style="font-size: 80%;">Two parameters</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs008.html#___sec7" style="font-size: 80%;">Maximum likelihood</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs009.html#___sec8" style="font-size: 80%;">The cost function rewritten</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs010.html#___sec9" style="font-size: 80%;">Minimizing the cross entropy</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs011.html#___sec10" style="font-size: 80%;">A more compact expression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs012.html#___sec11" style="font-size: 80%;">Extending to more predictors</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs013.html#___sec12" style="font-size: 80%;">Including more classes</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs014.html#___sec13" style="font-size: 80%;">The Softmax function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs015.html#___sec14" style="font-size: 80%;">A <b>scikit-learn</b> example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs016.html#___sec15" style="font-size: 80%;">A simple classification problem</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs017.html#___sec16" style="font-size: 80%;">The two-dimensional Ising model, Predicting phase transition of the two-dimensional Ising model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs018.html#___sec17" style="font-size: 80%;">Reading in the data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs019.html#___sec18" style="font-size: 80%;">Logistic regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs020.html#___sec19" style="font-size: 80%;">Exploring the logistic regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec20" style="font-size: 80%;">Accuracy of a classification model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs022.html#___sec21" style="font-size: 80%;">Analyzing the results</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
</div> <!-- end of navigation bar -->
|
||||
|
||||
<div class="container">
|
||||
|
||||
<p> </p><p> </p><p> </p> <!-- add vertical space -->
|
||||
|
||||
<a name="part0021"></a>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="___sec20" class="anchor">Accuracy of a classification model </h2>
|
||||
|
||||
<p>
|
||||
To determine how well a classification model is performing we count
|
||||
the number of correctly labeled classes and divide by the number of
|
||||
classes in total. The accuracy is thus given by
|
||||
|
||||
$$
|
||||
\begin{align}
|
||||
a(y, \hat{y}) = \frac{1}{n}\sum_{i = 1}^{n} I(y_i = \hat{y}_i),
|
||||
\tag{7}
|
||||
\end{align}
|
||||
$$
|
||||
|
||||
<p>
|
||||
where \( I(y_i = \hat{y}_i) \) is the indicator function given by
|
||||
|
||||
$$
|
||||
\begin{align}
|
||||
I(x = y) = \begin{cases}
|
||||
1 & x = y,
|
||||
\tag{8}\\
|
||||
0 & x \neq y.
|
||||
\end{cases}
|
||||
\tag{9}
|
||||
\end{align}
|
||||
$$
|
||||
|
||||
<p>
|
||||
This is the accuracy provided by Scikit-learn when using <b>sklearn.metrics.accuracyscore</b>.
|
||||
|
||||
<p>
|
||||
Below we compute the accuracy of the best fit model on the training data (which should give a good accuracy), the test data (which has not been shown to the model) and the critical data (completely new data that needs to be extrapolated).
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span>train_accuracy <span style="color: #666666">=</span> skm<span style="color: #666666">.</span>accuracy_score(y_train, clf<span style="color: #666666">.</span>predict(X_train))
|
||||
test_accuracy <span style="color: #666666">=</span> skm<span style="color: #666666">.</span>accuracy_score(y_test, clf<span style="color: #666666">.</span>predict(X_test))
|
||||
critical_accuracy <span style="color: #666666">=</span> skm<span style="color: #666666">.</span>accuracy_score(labels[critical], clf<span style="color: #666666">.</span>predict(data[critical]))
|
||||
|
||||
<span style="color: #008000; font-weight: bold">print</span> (<span style="color: #BA2121">"Accuracy on train data: {0}"</span><span style="color: #666666">.</span>format(train_accuracy))
|
||||
<span style="color: #008000; font-weight: bold">print</span> (<span style="color: #BA2121">"Accuracy on test data: {0}"</span><span style="color: #666666">.</span>format(test_accuracy))
|
||||
<span style="color: #008000; font-weight: bold">print</span> (<span style="color: #BA2121">"Accuracy on critical data: {0}"</span><span style="color: #666666">.</span>format(critical_accuracy))
|
||||
</pre></div>
|
||||
<p>
|
||||
We can see that we get quite good accuracy on the training data, but gradually worsening accuracy on the test and critical data.
|
||||
|
||||
<p>
|
||||
<p>
|
||||
<!-- navigation buttons at the bottom of the page -->
|
||||
<ul class="pagination">
|
||||
<li><a href="._LogReg-bs020.html">«</a></li>
|
||||
<li><a href="._LogReg-bs000.html">1</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._LogReg-bs013.html">14</a></li>
|
||||
<li><a href="._LogReg-bs014.html">15</a></li>
|
||||
<li><a href="._LogReg-bs015.html">16</a></li>
|
||||
<li><a href="._LogReg-bs016.html">17</a></li>
|
||||
<li><a href="._LogReg-bs017.html">18</a></li>
|
||||
<li><a href="._LogReg-bs018.html">19</a></li>
|
||||
<li><a href="._LogReg-bs019.html">20</a></li>
|
||||
<li><a href="._LogReg-bs020.html">21</a></li>
|
||||
<li class="active"><a href="._LogReg-bs021.html">22</a></li>
|
||||
<li><a href="._LogReg-bs022.html">23</a></li>
|
||||
<li><a href="._LogReg-bs022.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
</div> <!-- end container -->
|
||||
<!-- include javascript, jQuery *first* -->
|
||||
<script src="https://ajax.googleapis.com/ajax/libs/jquery/1.10.2/jquery.min.js"></script>
|
||||
<script src="https://netdna.bootstrapcdn.com/bootstrap/3.0.0/js/bootstrap.min.js"></script>
|
||||
|
||||
<!-- Bootstrap footer
|
||||
<footer>
|
||||
<a href="http://..."><img width="250" align=right src="http://..."></a>
|
||||
</footer>
|
||||
-->
|
||||
|
||||
|
||||
<center style="font-size:80%">
|
||||
<!-- copyright only on the titlepage -->
|
||||
</center>
|
||||
|
||||
|
||||
</body>
|
||||
</html>
|
||||
|
||||
|
||||
@@ -0,0 +1,242 @@
|
||||
<!--
|
||||
Automatically generated HTML file from DocOnce source
|
||||
(https://github.com/hplgit/doconce/)
|
||||
-->
|
||||
<html>
|
||||
<head>
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
||||
<meta name="generator" content="DocOnce: https://github.com/hplgit/doconce/" />
|
||||
<meta name="description" content="Data Analysis and Machine Learning: Logistic Regression">
|
||||
|
||||
<title>Data Analysis and Machine Learning: Logistic Regression</title>
|
||||
|
||||
<!-- Bootstrap style: bootstrap -->
|
||||
<link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
|
||||
<!-- not necessary
|
||||
<link href="https://netdna.bootstrapcdn.com/font-awesome/4.0.3/css/font-awesome.css" rel="stylesheet">
|
||||
-->
|
||||
|
||||
<style type="text/css">
|
||||
|
||||
/* Add scrollbar to dropdown menus in bootstrap navigation bar */
|
||||
.dropdown-menu {
|
||||
height: auto;
|
||||
max-height: 400px;
|
||||
overflow-x: hidden;
|
||||
}
|
||||
|
||||
/* Adds an invisible element before each target to offset for the navigation
|
||||
bar */
|
||||
.anchor::before {
|
||||
content:"";
|
||||
display:block;
|
||||
height:50px; /* fixed header height for style bootstrap */
|
||||
margin:-50px 0 0; /* negative fixed header height */
|
||||
}
|
||||
</style>
|
||||
|
||||
|
||||
</head>
|
||||
|
||||
<!-- tocinfo
|
||||
{'highest level': 2,
|
||||
'sections': [('Logistic Regression', 2, None, '___sec0'),
|
||||
('Optimization and Deep learning', 2, None, '___sec1'),
|
||||
('Basics', 2, None, '___sec2'),
|
||||
('Linear classifier', 2, None, '___sec3'),
|
||||
('Some selected properties', 2, None, '___sec4'),
|
||||
('The logistic function', 2, None, '___sec5'),
|
||||
('Two parameters', 2, None, '___sec6'),
|
||||
('Maximum likelihood', 2, None, '___sec7'),
|
||||
('The cost function rewritten', 2, None, '___sec8'),
|
||||
('Minimizing the cross entropy', 2, None, '___sec9'),
|
||||
('A more compact expression', 2, None, '___sec10'),
|
||||
('Extending to more predictors', 2, None, '___sec11'),
|
||||
('Including more classes', 2, None, '___sec12'),
|
||||
('The Softmax function', 2, None, '___sec13'),
|
||||
('A _scikit-learn_ example', 2, None, '___sec14'),
|
||||
('A simple classification problem', 2, None, '___sec15'),
|
||||
('The two-dimensional Ising model, Predicting phase transition '
|
||||
'of the two-dimensional Ising model',
|
||||
2,
|
||||
None,
|
||||
'___sec16'),
|
||||
('Reading in the data', 2, None, '___sec17'),
|
||||
('Logistic regression', 2, None, '___sec18'),
|
||||
('Exploring the logistic regression', 2, None, '___sec19'),
|
||||
('Accuracy of a classification model', 2, None, '___sec20'),
|
||||
('Analyzing the results', 2, None, '___sec21')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
|
||||
|
||||
|
||||
<script type="text/x-mathjax-config">
|
||||
MathJax.Hub.Config({
|
||||
TeX: {
|
||||
equationNumbers: { autoNumber: "none" },
|
||||
extensions: ["AMSmath.js", "AMSsymbols.js", "autobold.js", "color.js"]
|
||||
}
|
||||
});
|
||||
</script>
|
||||
<script type="text/javascript" async
|
||||
src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
|
||||
</script>
|
||||
|
||||
|
||||
|
||||
|
||||
<!-- Bootstrap navigation bar -->
|
||||
<div class="navbar navbar-default navbar-fixed-top">
|
||||
<div class="navbar-header">
|
||||
<button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-responsive-collapse">
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
</button>
|
||||
<a class="navbar-brand" href="LogReg-bs.html">Data Analysis and Machine Learning: Logistic Regression</a>
|
||||
</div>
|
||||
|
||||
<div class="navbar-collapse collapse navbar-responsive-collapse">
|
||||
<ul class="nav navbar-nav navbar-right">
|
||||
<li class="dropdown">
|
||||
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
|
||||
<ul class="dropdown-menu">
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs001.html#___sec0" style="font-size: 80%;">Logistic Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs002.html#___sec1" style="font-size: 80%;">Optimization and Deep learning</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs003.html#___sec2" style="font-size: 80%;">Basics</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs004.html#___sec3" style="font-size: 80%;">Linear classifier</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs005.html#___sec4" style="font-size: 80%;">Some selected properties</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs006.html#___sec5" style="font-size: 80%;">The logistic function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs007.html#___sec6" style="font-size: 80%;">Two parameters</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs008.html#___sec7" style="font-size: 80%;">Maximum likelihood</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs009.html#___sec8" style="font-size: 80%;">The cost function rewritten</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs010.html#___sec9" style="font-size: 80%;">Minimizing the cross entropy</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs011.html#___sec10" style="font-size: 80%;">A more compact expression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs012.html#___sec11" style="font-size: 80%;">Extending to more predictors</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs013.html#___sec12" style="font-size: 80%;">Including more classes</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs014.html#___sec13" style="font-size: 80%;">The Softmax function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs015.html#___sec14" style="font-size: 80%;">A <b>scikit-learn</b> example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs016.html#___sec15" style="font-size: 80%;">A simple classification problem</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs017.html#___sec16" style="font-size: 80%;">The two-dimensional Ising model, Predicting phase transition of the two-dimensional Ising model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs018.html#___sec17" style="font-size: 80%;">Reading in the data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs019.html#___sec18" style="font-size: 80%;">Logistic regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs020.html#___sec19" style="font-size: 80%;">Exploring the logistic regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs021.html#___sec20" style="font-size: 80%;">Accuracy of a classification model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec21" style="font-size: 80%;">Analyzing the results</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
</div> <!-- end of navigation bar -->
|
||||
|
||||
<div class="container">
|
||||
|
||||
<p> </p><p> </p><p> </p> <!-- add vertical space -->
|
||||
|
||||
<a name="part0022"></a>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="___sec21" class="anchor">Analyzing the results </h2>
|
||||
|
||||
<p>
|
||||
Below we show a different metric for determining the quality of our
|
||||
model, namely the <b>reciever operating characteristic</b> (ROC). The ROC
|
||||
curve tells us how well the model correctly classifies the different
|
||||
labels. We plot the <b>true positive rate</b> (the rate of predicted
|
||||
positive classes that are positive) versus the <b>false positive rate</b>
|
||||
(the rate of predicted positive classes that are negative). The ROC
|
||||
curve is built by computing the true positive rate and the false
|
||||
positive rate for varying <b>thresholds</b>, i.e, which probability we
|
||||
should acredit a certain class.
|
||||
|
||||
<p>
|
||||
By computing the <b>area under the curve</b> (AUC) of the ROC curve we get an estimate of how well our model is performing. Pure guessing will get an AUC of \( 0.5 \). A perfect score will get an AUC of \( 1.0 \).
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span>fig <span style="color: #666666">=</span> plt<span style="color: #666666">.</span>figure(figsize<span style="color: #666666">=</span>(<span style="color: #666666">20</span>, <span style="color: #666666">14</span>))
|
||||
|
||||
<span style="color: #008000; font-weight: bold">for</span> (_X, _y), label <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">zip</span>(
|
||||
[
|
||||
(X_train, y_train),
|
||||
(X_test, y_test),
|
||||
(data[critical], labels[critical])
|
||||
],
|
||||
[<span style="color: #BA2121">"Train"</span>, <span style="color: #BA2121">"Test"</span>, <span style="color: #BA2121">"Critical"</span>]
|
||||
):
|
||||
proba <span style="color: #666666">=</span> clf<span style="color: #666666">.</span>predict_proba(_X)
|
||||
fpr, tpr, _ <span style="color: #666666">=</span> skm<span style="color: #666666">.</span>roc_curve(_y, proba[:, <span style="color: #666666">1</span>])
|
||||
roc_auc <span style="color: #666666">=</span> skm<span style="color: #666666">.</span>auc(fpr, tpr)
|
||||
|
||||
<span style="color: #008000; font-weight: bold">print</span> (<span style="color: #BA2121">"LogisticRegression AUC ({0}): {1}"</span><span style="color: #666666">.</span>format(label, roc_auc))
|
||||
|
||||
plt<span style="color: #666666">.</span>plot(fpr, tpr, label<span style="color: #666666">=</span><span style="color: #BA2121">"{0} (AUC = {1})"</span><span style="color: #666666">.</span>format(label, roc_auc), linewidth<span style="color: #666666">=4.0</span>)
|
||||
|
||||
plt<span style="color: #666666">.</span>plot([<span style="color: #666666">0</span>, <span style="color: #666666">1</span>], [<span style="color: #666666">0</span>, <span style="color: #666666">1</span>], <span style="color: #BA2121">"--"</span>, label<span style="color: #666666">=</span><span style="color: #BA2121">"Guessing (AUC = 0.5)"</span>, linewidth<span style="color: #666666">=4.0</span>)
|
||||
|
||||
plt<span style="color: #666666">.</span>title(<span style="color: #BA2121">r"The ROC curve for LogisticRegression"</span>, fontsize<span style="color: #666666">=18</span>)
|
||||
plt<span style="color: #666666">.</span>xlabel(<span style="color: #BA2121">r"False positive rate"</span>, fontsize<span style="color: #666666">=18</span>)
|
||||
plt<span style="color: #666666">.</span>ylabel(<span style="color: #BA2121">r"True positive rate"</span>, fontsize<span style="color: #666666">=18</span>)
|
||||
plt<span style="color: #666666">.</span>axis([<span style="color: #666666">-0.01</span>, <span style="color: #666666">1.01</span>, <span style="color: #666666">-0.01</span>, <span style="color: #666666">1.01</span>])
|
||||
plt<span style="color: #666666">.</span>xticks(fontsize<span style="color: #666666">=18</span>)
|
||||
plt<span style="color: #666666">.</span>yticks(fontsize<span style="color: #666666">=18</span>)
|
||||
plt<span style="color: #666666">.</span>legend(loc<span style="color: #666666">=</span><span style="color: #BA2121">"best"</span>, fontsize<span style="color: #666666">=18</span>)
|
||||
plt<span style="color: #666666">.</span>show()
|
||||
</pre></div>
|
||||
<p>
|
||||
We can see that this plot of the ROC looks very strange. This tells us
|
||||
that logistic regression is quite inept at predicting the Ising model
|
||||
transition and is therefore highly non-linear. The ROC curve for the
|
||||
training data looks quite good, but as the testing data is so far off
|
||||
we see that we are dealing with an overfit model.
|
||||
|
||||
<p>
|
||||
A previous run with \( 50\% \) of the data used for training yielded a
|
||||
worse performance than using a smaller training set. This again gives
|
||||
confidence to the fact that logistic regression is not able to
|
||||
correctly fit the Ising model as it is not a linear model.
|
||||
|
||||
<p>
|
||||
<!-- navigation buttons at the bottom of the page -->
|
||||
<ul class="pagination">
|
||||
<li><a href="._LogReg-bs021.html">«</a></li>
|
||||
<li><a href="._LogReg-bs000.html">1</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._LogReg-bs014.html">15</a></li>
|
||||
<li><a href="._LogReg-bs015.html">16</a></li>
|
||||
<li><a href="._LogReg-bs016.html">17</a></li>
|
||||
<li><a href="._LogReg-bs017.html">18</a></li>
|
||||
<li><a href="._LogReg-bs018.html">19</a></li>
|
||||
<li><a href="._LogReg-bs019.html">20</a></li>
|
||||
<li><a href="._LogReg-bs020.html">21</a></li>
|
||||
<li><a href="._LogReg-bs021.html">22</a></li>
|
||||
<li class="active"><a href="._LogReg-bs022.html">23</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
</div> <!-- end container -->
|
||||
<!-- include javascript, jQuery *first* -->
|
||||
<script src="https://ajax.googleapis.com/ajax/libs/jquery/1.10.2/jquery.min.js"></script>
|
||||
<script src="https://netdna.bootstrapcdn.com/bootstrap/3.0.0/js/bootstrap.min.js"></script>
|
||||
|
||||
<!-- Bootstrap footer
|
||||
<footer>
|
||||
<a href="http://..."><img width="250" align=right src="http://..."></a>
|
||||
</footer>
|
||||
-->
|
||||
|
||||
|
||||
<center style="font-size:80%">
|
||||
<!-- copyright only on the titlepage -->
|
||||
</center>
|
||||
|
||||
|
||||
</body>
|
||||
</html>
|
||||
|
||||
|
||||
@@ -55,7 +55,17 @@ Automatically generated HTML file from DocOnce source
|
||||
('Including more classes', 2, None, '___sec12'),
|
||||
('The Softmax function', 2, None, '___sec13'),
|
||||
('A _scikit-learn_ example', 2, None, '___sec14'),
|
||||
('A simple classification problem', 2, None, '___sec15')]}
|
||||
('A simple classification problem', 2, None, '___sec15'),
|
||||
('The two-dimensional Ising model, Predicting phase transition '
|
||||
'of the two-dimensional Ising model',
|
||||
2,
|
||||
None,
|
||||
'___sec16'),
|
||||
('Reading in the data', 2, None, '___sec17'),
|
||||
('Logistic regression', 2, None, '___sec18'),
|
||||
('Exploring the logistic regression', 2, None, '___sec19'),
|
||||
('Accuracy of a classification model', 2, None, '___sec20'),
|
||||
('Analyzing the results', 2, None, '___sec21')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -109,6 +119,12 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs014.html#___sec13" style="font-size: 80%;">The Softmax function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs015.html#___sec14" style="font-size: 80%;">A <b>scikit-learn</b> example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs016.html#___sec15" style="font-size: 80%;">A simple classification problem</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs017.html#___sec16" style="font-size: 80%;">The two-dimensional Ising model, Predicting phase transition of the two-dimensional Ising model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs018.html#___sec17" style="font-size: 80%;">Reading in the data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs019.html#___sec18" style="font-size: 80%;">Logistic regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs020.html#___sec19" style="font-size: 80%;">Exploring the logistic regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs021.html#___sec20" style="font-size: 80%;">Accuracy of a classification model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs022.html#___sec21" style="font-size: 80%;">Analyzing the results</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -160,7 +176,7 @@ MathJax.Hub.Config({
|
||||
<li><a href="._LogReg-bs008.html">9</a></li>
|
||||
<li><a href="._LogReg-bs009.html">10</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._LogReg-bs016.html">17</a></li>
|
||||
<li><a href="._LogReg-bs022.html">23</a></li>
|
||||
<li><a href="._LogReg-bs001.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -618,6 +618,351 @@ plt.show()
|
||||
</section>
|
||||
|
||||
|
||||
<section>
|
||||
<h2 id="___sec16">The two-dimensional Ising model, Predicting phase transition of the two-dimensional Ising model </h2>
|
||||
|
||||
<p>
|
||||
The Hamiltonian of the two-dimensional Ising model without an external field for a constant coupling constant \( J \) is given by
|
||||
<p> <br>
|
||||
$$
|
||||
\begin{align}
|
||||
H = -J \sum_{\langle ij\rangle} S_i S_j,
|
||||
\tag{2}
|
||||
\end{align}
|
||||
$$
|
||||
<p> <br>
|
||||
|
||||
where \( S_i \in \{-1, 1\} \) and \( \langle ij \rangle \) signifies that we only iterate over the nearest neighbors in the lattice. We will be looking at a system of \( L = 40 \) spins in each dimension, i.e., \( L^2 = 1600 \) spins in total. Opposed to the one-dimensional Ising model we will get a phase transition from an <b>ordered</b> phase to a <b>disordered</b> phase at the critical temperature
|
||||
|
||||
<p> <br>
|
||||
$$
|
||||
\begin{align}
|
||||
\frac{T_c}{J} = \frac{2}{\log\left(1 + \sqrt{2}\right)} \approx 2.26,
|
||||
\tag{3}
|
||||
\end{align}
|
||||
$$
|
||||
<p> <br>
|
||||
|
||||
as shown by Lars Onsager.
|
||||
|
||||
<p>
|
||||
Here we use <b>logistic regression</b> to predict when a phase transition
|
||||
occurs. The data we will look at is a set of spin configurations,
|
||||
i.e., individual lattices with spins, labeled <b>ordered</b> <code>1</code> or
|
||||
<b>disordered</b> <code>0</code>. Our job is to build a model which will take in a
|
||||
spin configuration and predict whether or not the spin configuration
|
||||
constitutes an ordered or a disordered phase. To achieve this we will
|
||||
represent the lattices as flattened arrays with \( 1600 \) elements
|
||||
instead of a matrix of \( 40 \times 40 \) elements. As an extra test of
|
||||
the performance of the algorithms we will divide the dataset into
|
||||
three pieces. We will do a conventional train-test-split on a
|
||||
combination of totally ordered and totally disordered phases. The
|
||||
remaining "critical-like" states will be used as test data which we
|
||||
hope the model will be able to make good extrapolated predictions on.
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="highlight" style="background: #eeeedd"><pre style="font-size: 80%; line-height: 125%"><span></span><span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">pickle</span>
|
||||
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">os</span>
|
||||
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">glob</span>
|
||||
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">numpy</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">np</span>
|
||||
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">pandas</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">pd</span>
|
||||
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">matplotlib.pyplot</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">plt</span>
|
||||
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">seaborn</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">sns</span>
|
||||
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">sklearn.model_selection</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">skms</span>
|
||||
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">sklearn.linear_model</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">skl</span>
|
||||
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">sklearn.metrics</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">skm</span>
|
||||
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">tqdm</span>
|
||||
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">copy</span>
|
||||
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">time</span>
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">IPython.display</span> <span style="color: #8B008B; font-weight: bold">import</span> display
|
||||
|
||||
%matplotlib inline
|
||||
|
||||
sns.set(color_codes=<span style="color: #658b00">True</span>)
|
||||
</pre></div>
|
||||
</section>
|
||||
|
||||
|
||||
<section>
|
||||
<h2 id="___sec17">Reading in the data </h2>
|
||||
|
||||
<p>
|
||||
Using the data from <a href="https://physics.bu.edu/~pankajm/ML-Review-Datasets/isingMC/" target="_blank">Mehta et al.</a> (specifically the two datasets named <code>Ising2DFM_reSample_L40_T=All.pkl</code> and <code>Ising2DFM_reSample_L40_T=All_labels.pkl</code>) we have to unpack the data into numpy arrays.
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="highlight" style="background: #eeeedd"><pre style="font-size: 80%; line-height: 125%"><span></span>filenames = glob.glob(os.path.join(<span style="color: #CD5555">".."</span>, <span style="color: #CD5555">"dat"</span>, <span style="color: #CD5555">"*"</span>))
|
||||
label_filename = <span style="color: #658b00">list</span>(<span style="color: #658b00">filter</span>(<span style="color: #8B008B; font-weight: bold">lambda</span> x: <span style="color: #CD5555">"label"</span> <span style="color: #8B008B">in</span> x, filenames))[<span style="color: #B452CD">0</span>]
|
||||
dat_filename = <span style="color: #658b00">list</span>(<span style="color: #658b00">filter</span>(<span style="color: #8B008B; font-weight: bold">lambda</span> x: <span style="color: #CD5555">"label"</span> <span style="color: #8B008B">not</span> <span style="color: #8B008B">in</span> x, filenames))[<span style="color: #B452CD">0</span>]
|
||||
|
||||
<span style="color: #228B22"># Read in the labels</span>
|
||||
<span style="color: #8B008B; font-weight: bold">with</span> <span style="color: #658b00">open</span>(label_filename, <span style="color: #CD5555">"rb"</span>) <span style="color: #8B008B; font-weight: bold">as</span> f:
|
||||
labels = pickle.load(f)
|
||||
|
||||
<span style="color: #228B22"># Read in the corresponding configurations</span>
|
||||
<span style="color: #8B008B; font-weight: bold">with</span> <span style="color: #658b00">open</span>(dat_filename, <span style="color: #CD5555">"rb"</span>) <span style="color: #8B008B; font-weight: bold">as</span> f:
|
||||
data = np.unpackbits(pickle.load(f)).reshape(-<span style="color: #B452CD">1</span>, <span style="color: #B452CD">1600</span>).astype(<span style="color: #CD5555">"int"</span>)
|
||||
|
||||
<span style="color: #228B22"># Set spin-down to -1</span>
|
||||
data[data == <span style="color: #B452CD">0</span>] = -<span style="color: #B452CD">1</span>
|
||||
</pre></div>
|
||||
<p>
|
||||
This dataset consists of \( 10000 \) samples, i.e., \( 10000 \) spin
|
||||
configurations with \( 40 \times 40 \) spins each, for \( 16 \) temperatures
|
||||
between \( 0.25 \) to \( 4.0 \). Next we create a train/test-split and keep
|
||||
the data in the critical phase as a separate dataset for
|
||||
extrapolation-testing.
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="highlight" style="background: #eeeedd"><pre style="font-size: 80%; line-height: 125%"><span></span><span style="color: #228B22"># Set up slices of the dataset</span>
|
||||
ordered = <span style="color: #658b00">slice</span>(<span style="color: #B452CD">0</span>, <span style="color: #B452CD">70000</span>)
|
||||
critical = <span style="color: #658b00">slice</span>(<span style="color: #B452CD">70000</span>, <span style="color: #B452CD">100000</span>)
|
||||
disordered = <span style="color: #658b00">slice</span>(<span style="color: #B452CD">100000</span>, <span style="color: #B452CD">160000</span>)
|
||||
|
||||
X_train, X_test, y_train, y_test = skms.train_test_split(
|
||||
np.concatenate((data[ordered], data[disordered])),
|
||||
np.concatenate((labels[ordered], labels[disordered])),
|
||||
test_size=<span style="color: #B452CD">0.95</span>
|
||||
)
|
||||
</pre></div>
|
||||
<p>
|
||||
Using a small training set yields a better accuracy. This will be discussed in the end.
|
||||
</section>
|
||||
|
||||
|
||||
<section>
|
||||
<h2 id="___sec18">Logistic regression </h2>
|
||||
|
||||
<p>
|
||||
Logistic regression is a linear model for classification. Recalling
|
||||
the cost function for ordinary least squares with both L2 (ridge) and
|
||||
L1 (LASSO) penalties we will see that the logistic cost function is
|
||||
very similar. In OLS we wish to predict a continuous variable
|
||||
\( \hat{y} \) using
|
||||
<p> <br>
|
||||
$$
|
||||
\begin{align}
|
||||
\hat{y} = X\omega,
|
||||
\tag{4}
|
||||
\end{align}
|
||||
$$
|
||||
<p> <br>
|
||||
|
||||
<p>
|
||||
where \( X \in \mathbb{R}^{n \times p} \) is the input data and \( \omega^{p
|
||||
\times d} \) are the weights of the regression. In a classification
|
||||
setting (binary classification in our situation) we are interested in
|
||||
a positive or negative answer. We can thus define either answer to be
|
||||
above or below some threshold. But, in order to limit the size of the
|
||||
answer and also to get a probability interpretation on how sure we are
|
||||
for either answer we can compute the sigmoid function of OLS. That is,
|
||||
|
||||
<p> <br>
|
||||
$$
|
||||
\begin{align}
|
||||
f(X\omega) = \frac{1}{1 + \exp(-X\omega)}.
|
||||
\tag{5}
|
||||
\end{align}
|
||||
$$
|
||||
<p> <br>
|
||||
|
||||
We are thus interested in minizming the following cost function
|
||||
<p> <br>
|
||||
$$
|
||||
\begin{align}
|
||||
C(X, \omega) = \sum_{i = 1}^n \left\{
|
||||
- y_i\log\left( f(x_i^T\omega) \right)
|
||||
- (1 - y_i)\log\left[1 - f(x_i^T\omega)\right]
|
||||
\right\},
|
||||
\tag{6}
|
||||
\end{align}
|
||||
$$
|
||||
<p> <br>
|
||||
|
||||
<p>
|
||||
where we will restrict ourselves to a value for \( f(z) \) as the sigmoid
|
||||
described above. We can also tack on a L2 (Ridge) or L1 (LASSO)
|
||||
penalization to this cost function in the same manner we did for
|
||||
linear regression.
|
||||
</section>
|
||||
|
||||
|
||||
<section>
|
||||
<h2 id="___sec19">Exploring the logistic regression </h2>
|
||||
|
||||
<p>
|
||||
The penalization factor \( \lambda \) is inverted in the case of the
|
||||
logistic regression model we use. We will explore several values of
|
||||
\( \lambda \) using both L1 and L2 penalization. We do this using a grid
|
||||
search over different parameters and run a 3-fold cross validation for
|
||||
each configuration. In other words, we fit a model 3 times for each
|
||||
configuration of the hyper parameters.
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="highlight" style="background: #eeeedd"><pre style="font-size: 80%; line-height: 125%"><span></span>lambdas = np.logspace(-<span style="color: #B452CD">7</span>, -<span style="color: #B452CD">1</span>, <span style="color: #B452CD">7</span>)
|
||||
|
||||
param_grid = {
|
||||
<span style="color: #CD5555">"C"</span>: <span style="color: #658b00">list</span>(<span style="color: #B452CD">1.0</span>/lambdas),
|
||||
<span style="color: #CD5555">"penalty"</span>: [<span style="color: #CD5555">"l1"</span>, <span style="color: #CD5555">"l2"</span>]
|
||||
}
|
||||
clf = skms.GridSearchCV(
|
||||
skl.LogisticRegression(),
|
||||
param_grid=param_grid,
|
||||
n_jobs=-<span style="color: #B452CD">1</span>,
|
||||
return_train_score=<span style="color: #658b00">True</span>
|
||||
)
|
||||
t0 = time.time()
|
||||
clf.fit(X_train, y_train)
|
||||
t1 = time.time()
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">print</span> (
|
||||
<span style="color: #CD5555">"Time spent fitting GridSearchCV(LogisticRegression): {0:.3f} sec"</span>.format(
|
||||
t1 - t0
|
||||
)
|
||||
)
|
||||
</pre></div>
|
||||
<p>
|
||||
We can see that logistic regression is quite slow and using the grid
|
||||
search and cross validation results in quite a heavy
|
||||
computation. Below we show the results of the different
|
||||
configurations.
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="highlight" style="background: #eeeedd"><pre style="font-size: 80%; line-height: 125%"><span></span>logreg_df = pd.DataFrame(clf.cv_results_)
|
||||
|
||||
display(logreg_df)
|
||||
</pre></div>
|
||||
</section>
|
||||
|
||||
|
||||
<section>
|
||||
<h2 id="___sec20">Accuracy of a classification model </h2>
|
||||
|
||||
<p>
|
||||
To determine how well a classification model is performing we count
|
||||
the number of correctly labeled classes and divide by the number of
|
||||
classes in total. The accuracy is thus given by
|
||||
|
||||
<p> <br>
|
||||
$$
|
||||
\begin{align}
|
||||
a(y, \hat{y}) = \frac{1}{n}\sum_{i = 1}^{n} I(y_i = \hat{y}_i),
|
||||
\tag{7}
|
||||
\end{align}
|
||||
$$
|
||||
<p> <br>
|
||||
|
||||
<p>
|
||||
where \( I(y_i = \hat{y}_i) \) is the indicator function given by
|
||||
|
||||
<p> <br>
|
||||
$$
|
||||
\begin{align}
|
||||
I(x = y) = \begin{cases}
|
||||
1 & x = y,
|
||||
\tag{8}\\
|
||||
0 & x \neq y.
|
||||
\end{cases}
|
||||
\tag{9}
|
||||
\end{align}
|
||||
$$
|
||||
<p> <br>
|
||||
|
||||
<p>
|
||||
This is the accuracy provided by Scikit-learn when using <b>sklearn.metrics.accuracyscore</b>.
|
||||
|
||||
<p>
|
||||
Below we compute the accuracy of the best fit model on the training data (which should give a good accuracy), the test data (which has not been shown to the model) and the critical data (completely new data that needs to be extrapolated).
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="highlight" style="background: #eeeedd"><pre style="font-size: 80%; line-height: 125%"><span></span>train_accuracy = skm.accuracy_score(y_train, clf.predict(X_train))
|
||||
test_accuracy = skm.accuracy_score(y_test, clf.predict(X_test))
|
||||
critical_accuracy = skm.accuracy_score(labels[critical], clf.predict(data[critical]))
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">print</span> (<span style="color: #CD5555">"Accuracy on train data: {0}"</span>.format(train_accuracy))
|
||||
<span style="color: #8B008B; font-weight: bold">print</span> (<span style="color: #CD5555">"Accuracy on test data: {0}"</span>.format(test_accuracy))
|
||||
<span style="color: #8B008B; font-weight: bold">print</span> (<span style="color: #CD5555">"Accuracy on critical data: {0}"</span>.format(critical_accuracy))
|
||||
</pre></div>
|
||||
<p>
|
||||
We can see that we get quite good accuracy on the training data, but gradually worsening accuracy on the test and critical data.
|
||||
</section>
|
||||
|
||||
|
||||
<section>
|
||||
<h2 id="___sec21">Analyzing the results </h2>
|
||||
|
||||
<p>
|
||||
Below we show a different metric for determining the quality of our
|
||||
model, namely the <b>reciever operating characteristic</b> (ROC). The ROC
|
||||
curve tells us how well the model correctly classifies the different
|
||||
labels. We plot the <b>true positive rate</b> (the rate of predicted
|
||||
positive classes that are positive) versus the <b>false positive rate</b>
|
||||
(the rate of predicted positive classes that are negative). The ROC
|
||||
curve is built by computing the true positive rate and the false
|
||||
positive rate for varying <b>thresholds</b>, i.e, which probability we
|
||||
should acredit a certain class.
|
||||
|
||||
<p>
|
||||
By computing the <b>area under the curve</b> (AUC) of the ROC curve we get an estimate of how well our model is performing. Pure guessing will get an AUC of \( 0.5 \). A perfect score will get an AUC of \( 1.0 \).
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="highlight" style="background: #eeeedd"><pre style="font-size: 80%; line-height: 125%"><span></span>fig = plt.figure(figsize=(<span style="color: #B452CD">20</span>, <span style="color: #B452CD">14</span>))
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">for</span> (_X, _y), label <span style="color: #8B008B">in</span> <span style="color: #658b00">zip</span>(
|
||||
[
|
||||
(X_train, y_train),
|
||||
(X_test, y_test),
|
||||
(data[critical], labels[critical])
|
||||
],
|
||||
[<span style="color: #CD5555">"Train"</span>, <span style="color: #CD5555">"Test"</span>, <span style="color: #CD5555">"Critical"</span>]
|
||||
):
|
||||
proba = clf.predict_proba(_X)
|
||||
fpr, tpr, _ = skm.roc_curve(_y, proba[:, <span style="color: #B452CD">1</span>])
|
||||
roc_auc = skm.auc(fpr, tpr)
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">print</span> (<span style="color: #CD5555">"LogisticRegression AUC ({0}): {1}"</span>.format(label, roc_auc))
|
||||
|
||||
plt.plot(fpr, tpr, label=<span style="color: #CD5555">"{0} (AUC = {1})"</span>.format(label, roc_auc), linewidth=<span style="color: #B452CD">4.0</span>)
|
||||
|
||||
plt.plot([<span style="color: #B452CD">0</span>, <span style="color: #B452CD">1</span>], [<span style="color: #B452CD">0</span>, <span style="color: #B452CD">1</span>], <span style="color: #CD5555">"--"</span>, label=<span style="color: #CD5555">"Guessing (AUC = 0.5)"</span>, linewidth=<span style="color: #B452CD">4.0</span>)
|
||||
|
||||
plt.title(<span style="color: #CD5555">r"The ROC curve for LogisticRegression"</span>, fontsize=<span style="color: #B452CD">18</span>)
|
||||
plt.xlabel(<span style="color: #CD5555">r"False positive rate"</span>, fontsize=<span style="color: #B452CD">18</span>)
|
||||
plt.ylabel(<span style="color: #CD5555">r"True positive rate"</span>, fontsize=<span style="color: #B452CD">18</span>)
|
||||
plt.axis([-<span style="color: #B452CD">0.01</span>, <span style="color: #B452CD">1.01</span>, -<span style="color: #B452CD">0.01</span>, <span style="color: #B452CD">1.01</span>])
|
||||
plt.xticks(fontsize=<span style="color: #B452CD">18</span>)
|
||||
plt.yticks(fontsize=<span style="color: #B452CD">18</span>)
|
||||
plt.legend(loc=<span style="color: #CD5555">"best"</span>, fontsize=<span style="color: #B452CD">18</span>)
|
||||
plt.show()
|
||||
</pre></div>
|
||||
<p>
|
||||
We can see that this plot of the ROC looks very strange. This tells us
|
||||
that logistic regression is quite inept at predicting the Ising model
|
||||
transition and is therefore highly non-linear. The ROC curve for the
|
||||
training data looks quite good, but as the testing data is so far off
|
||||
we see that we are dealing with an overfit model.
|
||||
|
||||
<p>
|
||||
A previous run with \( 50\% \) of the data used for training yielded a
|
||||
worse performance than using a smaller training set. This again gives
|
||||
confidence to the fact that logistic regression is not able to
|
||||
correctly fit the Ising model as it is not a linear model.
|
||||
</section>
|
||||
|
||||
|
||||
|
||||
</div> <!-- class="slides" -->
|
||||
</div> <!-- class="reveal" -->
|
||||
|
||||
@@ -49,7 +49,17 @@ div { text-align: justify; text-justify: inter-word; }
|
||||
('Including more classes', 2, None, '___sec12'),
|
||||
('The Softmax function', 2, None, '___sec13'),
|
||||
('A _scikit-learn_ example', 2, None, '___sec14'),
|
||||
('A simple classification problem', 2, None, '___sec15')]}
|
||||
('A simple classification problem', 2, None, '___sec15'),
|
||||
('The two-dimensional Ising model, Predicting phase transition '
|
||||
'of the two-dimensional Ising model',
|
||||
2,
|
||||
None,
|
||||
'___sec16'),
|
||||
('Reading in the data', 2, None, '___sec17'),
|
||||
('Logistic regression', 2, None, '___sec18'),
|
||||
('Exploring the logistic regression', 2, None, '___sec19'),
|
||||
('Accuracy of a classification model', 2, None, '___sec20'),
|
||||
('Analyzing the results', 2, None, '___sec21')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -510,6 +520,333 @@ plt.show()
|
||||
main()
|
||||
</pre></div>
|
||||
<p>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="___sec16">The two-dimensional Ising model, Predicting phase transition of the two-dimensional Ising model </h2>
|
||||
|
||||
<p>
|
||||
The Hamiltonian of the two-dimensional Ising model without an external field for a constant coupling constant \( J \) is given by
|
||||
$$
|
||||
\begin{align}
|
||||
H = -J \sum_{\langle ij\rangle} S_i S_j,
|
||||
\label{_auto2}
|
||||
\end{align}
|
||||
$$
|
||||
|
||||
where \( S_i \in \{-1, 1\} \) and \( \langle ij \rangle \) signifies that we only iterate over the nearest neighbors in the lattice. We will be looking at a system of \( L = 40 \) spins in each dimension, i.e., \( L^2 = 1600 \) spins in total. Opposed to the one-dimensional Ising model we will get a phase transition from an <b>ordered</b> phase to a <b>disordered</b> phase at the critical temperature
|
||||
|
||||
$$
|
||||
\begin{align}
|
||||
\frac{T_c}{J} = \frac{2}{\log\left(1 + \sqrt{2}\right)} \approx 2.26,
|
||||
\label{_auto3}
|
||||
\end{align}
|
||||
$$
|
||||
|
||||
as shown by Lars Onsager.
|
||||
|
||||
<p>
|
||||
Here we use <b>logistic regression</b> to predict when a phase transition
|
||||
occurs. The data we will look at is a set of spin configurations,
|
||||
i.e., individual lattices with spins, labeled <b>ordered</b> <code>1</code> or
|
||||
<b>disordered</b> <code>0</code>. Our job is to build a model which will take in a
|
||||
spin configuration and predict whether or not the spin configuration
|
||||
constitutes an ordered or a disordered phase. To achieve this we will
|
||||
represent the lattices as flattened arrays with \( 1600 \) elements
|
||||
instead of a matrix of \( 40 \times 40 \) elements. As an extra test of
|
||||
the performance of the algorithms we will divide the dataset into
|
||||
three pieces. We will do a conventional train-test-split on a
|
||||
combination of totally ordered and totally disordered phases. The
|
||||
remaining "critical-like" states will be used as test data which we
|
||||
hope the model will be able to make good extrapolated predictions on.
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="highlight" style="background: #eeeedd"><pre style="line-height: 125%"><span></span><span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">pickle</span>
|
||||
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">os</span>
|
||||
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">glob</span>
|
||||
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">numpy</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">np</span>
|
||||
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">pandas</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">pd</span>
|
||||
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">matplotlib.pyplot</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">plt</span>
|
||||
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">seaborn</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">sns</span>
|
||||
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">sklearn.model_selection</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">skms</span>
|
||||
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">sklearn.linear_model</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">skl</span>
|
||||
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">sklearn.metrics</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">skm</span>
|
||||
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">tqdm</span>
|
||||
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">copy</span>
|
||||
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">time</span>
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">IPython.display</span> <span style="color: #8B008B; font-weight: bold">import</span> display
|
||||
|
||||
%matplotlib inline
|
||||
|
||||
sns.set(color_codes=<span style="color: #658b00">True</span>)
|
||||
</pre></div>
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec17">Reading in the data </h2>
|
||||
|
||||
<p>
|
||||
Using the data from <a href="https://physics.bu.edu/~pankajm/ML-Review-Datasets/isingMC/" target="_blank">Mehta et al.</a> (specifically the two datasets named <code>Ising2DFM_reSample_L40_T=All.pkl</code> and <code>Ising2DFM_reSample_L40_T=All_labels.pkl</code>) we have to unpack the data into numpy arrays.
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="highlight" style="background: #eeeedd"><pre style="line-height: 125%"><span></span>filenames = glob.glob(os.path.join(<span style="color: #CD5555">".."</span>, <span style="color: #CD5555">"dat"</span>, <span style="color: #CD5555">"*"</span>))
|
||||
label_filename = <span style="color: #658b00">list</span>(<span style="color: #658b00">filter</span>(<span style="color: #8B008B; font-weight: bold">lambda</span> x: <span style="color: #CD5555">"label"</span> <span style="color: #8B008B">in</span> x, filenames))[<span style="color: #B452CD">0</span>]
|
||||
dat_filename = <span style="color: #658b00">list</span>(<span style="color: #658b00">filter</span>(<span style="color: #8B008B; font-weight: bold">lambda</span> x: <span style="color: #CD5555">"label"</span> <span style="color: #8B008B">not</span> <span style="color: #8B008B">in</span> x, filenames))[<span style="color: #B452CD">0</span>]
|
||||
|
||||
<span style="color: #228B22"># Read in the labels</span>
|
||||
<span style="color: #8B008B; font-weight: bold">with</span> <span style="color: #658b00">open</span>(label_filename, <span style="color: #CD5555">"rb"</span>) <span style="color: #8B008B; font-weight: bold">as</span> f:
|
||||
labels = pickle.load(f)
|
||||
|
||||
<span style="color: #228B22"># Read in the corresponding configurations</span>
|
||||
<span style="color: #8B008B; font-weight: bold">with</span> <span style="color: #658b00">open</span>(dat_filename, <span style="color: #CD5555">"rb"</span>) <span style="color: #8B008B; font-weight: bold">as</span> f:
|
||||
data = np.unpackbits(pickle.load(f)).reshape(-<span style="color: #B452CD">1</span>, <span style="color: #B452CD">1600</span>).astype(<span style="color: #CD5555">"int"</span>)
|
||||
|
||||
<span style="color: #228B22"># Set spin-down to -1</span>
|
||||
data[data == <span style="color: #B452CD">0</span>] = -<span style="color: #B452CD">1</span>
|
||||
</pre></div>
|
||||
<p>
|
||||
This dataset consists of \( 10000 \) samples, i.e., \( 10000 \) spin
|
||||
configurations with \( 40 \times 40 \) spins each, for \( 16 \) temperatures
|
||||
between \( 0.25 \) to \( 4.0 \). Next we create a train/test-split and keep
|
||||
the data in the critical phase as a separate dataset for
|
||||
extrapolation-testing.
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="highlight" style="background: #eeeedd"><pre style="line-height: 125%"><span></span><span style="color: #228B22"># Set up slices of the dataset</span>
|
||||
ordered = <span style="color: #658b00">slice</span>(<span style="color: #B452CD">0</span>, <span style="color: #B452CD">70000</span>)
|
||||
critical = <span style="color: #658b00">slice</span>(<span style="color: #B452CD">70000</span>, <span style="color: #B452CD">100000</span>)
|
||||
disordered = <span style="color: #658b00">slice</span>(<span style="color: #B452CD">100000</span>, <span style="color: #B452CD">160000</span>)
|
||||
|
||||
X_train, X_test, y_train, y_test = skms.train_test_split(
|
||||
np.concatenate((data[ordered], data[disordered])),
|
||||
np.concatenate((labels[ordered], labels[disordered])),
|
||||
test_size=<span style="color: #B452CD">0.95</span>
|
||||
)
|
||||
</pre></div>
|
||||
<p>
|
||||
Using a small training set yields a better accuracy. This will be discussed in the end.
|
||||
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec18">Logistic regression </h2>
|
||||
|
||||
<p>
|
||||
Logistic regression is a linear model for classification. Recalling
|
||||
the cost function for ordinary least squares with both L2 (ridge) and
|
||||
L1 (LASSO) penalties we will see that the logistic cost function is
|
||||
very similar. In OLS we wish to predict a continuous variable
|
||||
\( \hat{y} \) using
|
||||
$$
|
||||
\begin{align}
|
||||
\hat{y} = X\omega,
|
||||
\label{_auto4}
|
||||
\end{align}
|
||||
$$
|
||||
|
||||
<p>
|
||||
where \( X \in \mathbb{R}^{n \times p} \) is the input data and \( \omega^{p
|
||||
\times d} \) are the weights of the regression. In a classification
|
||||
setting (binary classification in our situation) we are interested in
|
||||
a positive or negative answer. We can thus define either answer to be
|
||||
above or below some threshold. But, in order to limit the size of the
|
||||
answer and also to get a probability interpretation on how sure we are
|
||||
for either answer we can compute the sigmoid function of OLS. That is,
|
||||
|
||||
$$
|
||||
\begin{align}
|
||||
f(X\omega) = \frac{1}{1 + \exp(-X\omega)}.
|
||||
\label{_auto5}
|
||||
\end{align}
|
||||
$$
|
||||
|
||||
We are thus interested in minizming the following cost function
|
||||
$$
|
||||
\begin{align}
|
||||
C(X, \omega) = \sum_{i = 1}^n \left\{
|
||||
- y_i\log\left( f(x_i^T\omega) \right)
|
||||
- (1 - y_i)\log\left[1 - f(x_i^T\omega)\right]
|
||||
\right\},
|
||||
\label{_auto6}
|
||||
\end{align}
|
||||
$$
|
||||
|
||||
<p>
|
||||
where we will restrict ourselves to a value for \( f(z) \) as the sigmoid
|
||||
described above. We can also tack on a L2 (Ridge) or L1 (LASSO)
|
||||
penalization to this cost function in the same manner we did for
|
||||
linear regression.
|
||||
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec19">Exploring the logistic regression </h2>
|
||||
|
||||
<p>
|
||||
The penalization factor \( \lambda \) is inverted in the case of the
|
||||
logistic regression model we use. We will explore several values of
|
||||
\( \lambda \) using both L1 and L2 penalization. We do this using a grid
|
||||
search over different parameters and run a 3-fold cross validation for
|
||||
each configuration. In other words, we fit a model 3 times for each
|
||||
configuration of the hyper parameters.
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="highlight" style="background: #eeeedd"><pre style="line-height: 125%"><span></span>lambdas = np.logspace(-<span style="color: #B452CD">7</span>, -<span style="color: #B452CD">1</span>, <span style="color: #B452CD">7</span>)
|
||||
|
||||
param_grid = {
|
||||
<span style="color: #CD5555">"C"</span>: <span style="color: #658b00">list</span>(<span style="color: #B452CD">1.0</span>/lambdas),
|
||||
<span style="color: #CD5555">"penalty"</span>: [<span style="color: #CD5555">"l1"</span>, <span style="color: #CD5555">"l2"</span>]
|
||||
}
|
||||
clf = skms.GridSearchCV(
|
||||
skl.LogisticRegression(),
|
||||
param_grid=param_grid,
|
||||
n_jobs=-<span style="color: #B452CD">1</span>,
|
||||
return_train_score=<span style="color: #658b00">True</span>
|
||||
)
|
||||
t0 = time.time()
|
||||
clf.fit(X_train, y_train)
|
||||
t1 = time.time()
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">print</span> (
|
||||
<span style="color: #CD5555">"Time spent fitting GridSearchCV(LogisticRegression): {0:.3f} sec"</span>.format(
|
||||
t1 - t0
|
||||
)
|
||||
)
|
||||
</pre></div>
|
||||
<p>
|
||||
We can see that logistic regression is quite slow and using the grid
|
||||
search and cross validation results in quite a heavy
|
||||
computation. Below we show the results of the different
|
||||
configurations.
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="highlight" style="background: #eeeedd"><pre style="line-height: 125%"><span></span>logreg_df = pd.DataFrame(clf.cv_results_)
|
||||
|
||||
display(logreg_df)
|
||||
</pre></div>
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec20">Accuracy of a classification model </h2>
|
||||
|
||||
<p>
|
||||
To determine how well a classification model is performing we count
|
||||
the number of correctly labeled classes and divide by the number of
|
||||
classes in total. The accuracy is thus given by
|
||||
|
||||
$$
|
||||
\begin{align}
|
||||
a(y, \hat{y}) = \frac{1}{n}\sum_{i = 1}^{n} I(y_i = \hat{y}_i),
|
||||
\label{_auto7}
|
||||
\end{align}
|
||||
$$
|
||||
|
||||
<p>
|
||||
where \( I(y_i = \hat{y}_i) \) is the indicator function given by
|
||||
|
||||
$$
|
||||
\begin{align}
|
||||
I(x = y) = \begin{cases}
|
||||
1 & x = y,
|
||||
\label{_auto8}\\
|
||||
0 & x \neq y.
|
||||
\end{cases}
|
||||
\label{_auto9}
|
||||
\end{align}
|
||||
$$
|
||||
|
||||
<p>
|
||||
This is the accuracy provided by Scikit-learn when using <b>sklearn.metrics.accuracyscore</b>.
|
||||
|
||||
<p>
|
||||
Below we compute the accuracy of the best fit model on the training data (which should give a good accuracy), the test data (which has not been shown to the model) and the critical data (completely new data that needs to be extrapolated).
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="highlight" style="background: #eeeedd"><pre style="line-height: 125%"><span></span>train_accuracy = skm.accuracy_score(y_train, clf.predict(X_train))
|
||||
test_accuracy = skm.accuracy_score(y_test, clf.predict(X_test))
|
||||
critical_accuracy = skm.accuracy_score(labels[critical], clf.predict(data[critical]))
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">print</span> (<span style="color: #CD5555">"Accuracy on train data: {0}"</span>.format(train_accuracy))
|
||||
<span style="color: #8B008B; font-weight: bold">print</span> (<span style="color: #CD5555">"Accuracy on test data: {0}"</span>.format(test_accuracy))
|
||||
<span style="color: #8B008B; font-weight: bold">print</span> (<span style="color: #CD5555">"Accuracy on critical data: {0}"</span>.format(critical_accuracy))
|
||||
</pre></div>
|
||||
<p>
|
||||
We can see that we get quite good accuracy on the training data, but gradually worsening accuracy on the test and critical data.
|
||||
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec21">Analyzing the results </h2>
|
||||
|
||||
<p>
|
||||
Below we show a different metric for determining the quality of our
|
||||
model, namely the <b>reciever operating characteristic</b> (ROC). The ROC
|
||||
curve tells us how well the model correctly classifies the different
|
||||
labels. We plot the <b>true positive rate</b> (the rate of predicted
|
||||
positive classes that are positive) versus the <b>false positive rate</b>
|
||||
(the rate of predicted positive classes that are negative). The ROC
|
||||
curve is built by computing the true positive rate and the false
|
||||
positive rate for varying <b>thresholds</b>, i.e, which probability we
|
||||
should acredit a certain class.
|
||||
|
||||
<p>
|
||||
By computing the <b>area under the curve</b> (AUC) of the ROC curve we get an estimate of how well our model is performing. Pure guessing will get an AUC of \( 0.5 \). A perfect score will get an AUC of \( 1.0 \).
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="highlight" style="background: #eeeedd"><pre style="line-height: 125%"><span></span>fig = plt.figure(figsize=(<span style="color: #B452CD">20</span>, <span style="color: #B452CD">14</span>))
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">for</span> (_X, _y), label <span style="color: #8B008B">in</span> <span style="color: #658b00">zip</span>(
|
||||
[
|
||||
(X_train, y_train),
|
||||
(X_test, y_test),
|
||||
(data[critical], labels[critical])
|
||||
],
|
||||
[<span style="color: #CD5555">"Train"</span>, <span style="color: #CD5555">"Test"</span>, <span style="color: #CD5555">"Critical"</span>]
|
||||
):
|
||||
proba = clf.predict_proba(_X)
|
||||
fpr, tpr, _ = skm.roc_curve(_y, proba[:, <span style="color: #B452CD">1</span>])
|
||||
roc_auc = skm.auc(fpr, tpr)
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">print</span> (<span style="color: #CD5555">"LogisticRegression AUC ({0}): {1}"</span>.format(label, roc_auc))
|
||||
|
||||
plt.plot(fpr, tpr, label=<span style="color: #CD5555">"{0} (AUC = {1})"</span>.format(label, roc_auc), linewidth=<span style="color: #B452CD">4.0</span>)
|
||||
|
||||
plt.plot([<span style="color: #B452CD">0</span>, <span style="color: #B452CD">1</span>], [<span style="color: #B452CD">0</span>, <span style="color: #B452CD">1</span>], <span style="color: #CD5555">"--"</span>, label=<span style="color: #CD5555">"Guessing (AUC = 0.5)"</span>, linewidth=<span style="color: #B452CD">4.0</span>)
|
||||
|
||||
plt.title(<span style="color: #CD5555">r"The ROC curve for LogisticRegression"</span>, fontsize=<span style="color: #B452CD">18</span>)
|
||||
plt.xlabel(<span style="color: #CD5555">r"False positive rate"</span>, fontsize=<span style="color: #B452CD">18</span>)
|
||||
plt.ylabel(<span style="color: #CD5555">r"True positive rate"</span>, fontsize=<span style="color: #B452CD">18</span>)
|
||||
plt.axis([-<span style="color: #B452CD">0.01</span>, <span style="color: #B452CD">1.01</span>, -<span style="color: #B452CD">0.01</span>, <span style="color: #B452CD">1.01</span>])
|
||||
plt.xticks(fontsize=<span style="color: #B452CD">18</span>)
|
||||
plt.yticks(fontsize=<span style="color: #B452CD">18</span>)
|
||||
plt.legend(loc=<span style="color: #CD5555">"best"</span>, fontsize=<span style="color: #B452CD">18</span>)
|
||||
plt.show()
|
||||
</pre></div>
|
||||
<p>
|
||||
We can see that this plot of the ROC looks very strange. This tells us
|
||||
that logistic regression is quite inept at predicting the Ising model
|
||||
transition and is therefore highly non-linear. The ROC curve for the
|
||||
training data looks quite good, but as the testing data is so far off
|
||||
we see that we are dealing with an overfit model.
|
||||
|
||||
<p>
|
||||
A previous run with \( 50\% \) of the data used for training yielded a
|
||||
worse performance than using a smaller training set. This again gives
|
||||
confidence to the fact that logistic regression is not able to
|
||||
correctly fit the Ising model as it is not a linear model.
|
||||
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
|
||||
@@ -54,7 +54,17 @@ div { text-align: justify; text-justify: inter-word; }
|
||||
('Including more classes', 2, None, '___sec12'),
|
||||
('The Softmax function', 2, None, '___sec13'),
|
||||
('A _scikit-learn_ example', 2, None, '___sec14'),
|
||||
('A simple classification problem', 2, None, '___sec15')]}
|
||||
('A simple classification problem', 2, None, '___sec15'),
|
||||
('The two-dimensional Ising model, Predicting phase transition '
|
||||
'of the two-dimensional Ising model',
|
||||
2,
|
||||
None,
|
||||
'___sec16'),
|
||||
('Reading in the data', 2, None, '___sec17'),
|
||||
('Logistic regression', 2, None, '___sec18'),
|
||||
('Exploring the logistic regression', 2, None, '___sec19'),
|
||||
('Accuracy of a classification model', 2, None, '___sec20'),
|
||||
('Analyzing the results', 2, None, '___sec21')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -515,6 +525,333 @@ plt<span style="color: #666666">.</span>show()
|
||||
main()
|
||||
</pre></div>
|
||||
<p>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="___sec16">The two-dimensional Ising model, Predicting phase transition of the two-dimensional Ising model </h2>
|
||||
|
||||
<p>
|
||||
The Hamiltonian of the two-dimensional Ising model without an external field for a constant coupling constant \( J \) is given by
|
||||
$$
|
||||
\begin{align}
|
||||
H = -J \sum_{\langle ij\rangle} S_i S_j,
|
||||
\label{_auto2}
|
||||
\end{align}
|
||||
$$
|
||||
|
||||
where \( S_i \in \{-1, 1\} \) and \( \langle ij \rangle \) signifies that we only iterate over the nearest neighbors in the lattice. We will be looking at a system of \( L = 40 \) spins in each dimension, i.e., \( L^2 = 1600 \) spins in total. Opposed to the one-dimensional Ising model we will get a phase transition from an <b>ordered</b> phase to a <b>disordered</b> phase at the critical temperature
|
||||
|
||||
$$
|
||||
\begin{align}
|
||||
\frac{T_c}{J} = \frac{2}{\log\left(1 + \sqrt{2}\right)} \approx 2.26,
|
||||
\label{_auto3}
|
||||
\end{align}
|
||||
$$
|
||||
|
||||
as shown by Lars Onsager.
|
||||
|
||||
<p>
|
||||
Here we use <b>logistic regression</b> to predict when a phase transition
|
||||
occurs. The data we will look at is a set of spin configurations,
|
||||
i.e., individual lattices with spins, labeled <b>ordered</b> <code>1</code> or
|
||||
<b>disordered</b> <code>0</code>. Our job is to build a model which will take in a
|
||||
spin configuration and predict whether or not the spin configuration
|
||||
constitutes an ordered or a disordered phase. To achieve this we will
|
||||
represent the lattices as flattened arrays with \( 1600 \) elements
|
||||
instead of a matrix of \( 40 \times 40 \) elements. As an extra test of
|
||||
the performance of the algorithms we will divide the dataset into
|
||||
three pieces. We will do a conventional train-test-split on a
|
||||
combination of totally ordered and totally disordered phases. The
|
||||
remaining "critical-like" states will be used as test data which we
|
||||
hope the model will be able to make good extrapolated predictions on.
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span><span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">pickle</span>
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">os</span>
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">glob</span>
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">numpy</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">np</span>
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">pandas</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">pd</span>
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">matplotlib.pyplot</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">plt</span>
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">seaborn</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">sns</span>
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">sklearn.model_selection</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">skms</span>
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">sklearn.linear_model</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">skl</span>
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">sklearn.metrics</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">skm</span>
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">tqdm</span>
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">copy</span>
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">time</span>
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">IPython.display</span> <span style="color: #008000; font-weight: bold">import</span> display
|
||||
|
||||
<span style="color: #666666">%</span>matplotlib inline
|
||||
|
||||
sns<span style="color: #666666">.</span>set(color_codes<span style="color: #666666">=</span><span style="color: #008000">True</span>)
|
||||
</pre></div>
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec17">Reading in the data </h2>
|
||||
|
||||
<p>
|
||||
Using the data from <a href="https://physics.bu.edu/~pankajm/ML-Review-Datasets/isingMC/" target="_blank">Mehta et al.</a> (specifically the two datasets named <code>Ising2DFM_reSample_L40_T=All.pkl</code> and <code>Ising2DFM_reSample_L40_T=All_labels.pkl</code>) we have to unpack the data into numpy arrays.
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span>filenames <span style="color: #666666">=</span> glob<span style="color: #666666">.</span>glob(os<span style="color: #666666">.</span>path<span style="color: #666666">.</span>join(<span style="color: #BA2121">".."</span>, <span style="color: #BA2121">"dat"</span>, <span style="color: #BA2121">"*"</span>))
|
||||
label_filename <span style="color: #666666">=</span> <span style="color: #008000">list</span>(<span style="color: #008000">filter</span>(<span style="color: #008000; font-weight: bold">lambda</span> x: <span style="color: #BA2121">"label"</span> <span style="color: #AA22FF; font-weight: bold">in</span> x, filenames))[<span style="color: #666666">0</span>]
|
||||
dat_filename <span style="color: #666666">=</span> <span style="color: #008000">list</span>(<span style="color: #008000">filter</span>(<span style="color: #008000; font-weight: bold">lambda</span> x: <span style="color: #BA2121">"label"</span> <span style="color: #AA22FF; font-weight: bold">not</span> <span style="color: #AA22FF; font-weight: bold">in</span> x, filenames))[<span style="color: #666666">0</span>]
|
||||
|
||||
<span style="color: #408080; font-style: italic"># Read in the labels</span>
|
||||
<span style="color: #008000; font-weight: bold">with</span> <span style="color: #008000">open</span>(label_filename, <span style="color: #BA2121">"rb"</span>) <span style="color: #008000; font-weight: bold">as</span> f:
|
||||
labels <span style="color: #666666">=</span> pickle<span style="color: #666666">.</span>load(f)
|
||||
|
||||
<span style="color: #408080; font-style: italic"># Read in the corresponding configurations</span>
|
||||
<span style="color: #008000; font-weight: bold">with</span> <span style="color: #008000">open</span>(dat_filename, <span style="color: #BA2121">"rb"</span>) <span style="color: #008000; font-weight: bold">as</span> f:
|
||||
data <span style="color: #666666">=</span> np<span style="color: #666666">.</span>unpackbits(pickle<span style="color: #666666">.</span>load(f))<span style="color: #666666">.</span>reshape(<span style="color: #666666">-1</span>, <span style="color: #666666">1600</span>)<span style="color: #666666">.</span>astype(<span style="color: #BA2121">"int"</span>)
|
||||
|
||||
<span style="color: #408080; font-style: italic"># Set spin-down to -1</span>
|
||||
data[data <span style="color: #666666">==</span> <span style="color: #666666">0</span>] <span style="color: #666666">=</span> <span style="color: #666666">-1</span>
|
||||
</pre></div>
|
||||
<p>
|
||||
This dataset consists of \( 10000 \) samples, i.e., \( 10000 \) spin
|
||||
configurations with \( 40 \times 40 \) spins each, for \( 16 \) temperatures
|
||||
between \( 0.25 \) to \( 4.0 \). Next we create a train/test-split and keep
|
||||
the data in the critical phase as a separate dataset for
|
||||
extrapolation-testing.
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span><span style="color: #408080; font-style: italic"># Set up slices of the dataset</span>
|
||||
ordered <span style="color: #666666">=</span> <span style="color: #008000">slice</span>(<span style="color: #666666">0</span>, <span style="color: #666666">70000</span>)
|
||||
critical <span style="color: #666666">=</span> <span style="color: #008000">slice</span>(<span style="color: #666666">70000</span>, <span style="color: #666666">100000</span>)
|
||||
disordered <span style="color: #666666">=</span> <span style="color: #008000">slice</span>(<span style="color: #666666">100000</span>, <span style="color: #666666">160000</span>)
|
||||
|
||||
X_train, X_test, y_train, y_test <span style="color: #666666">=</span> skms<span style="color: #666666">.</span>train_test_split(
|
||||
np<span style="color: #666666">.</span>concatenate((data[ordered], data[disordered])),
|
||||
np<span style="color: #666666">.</span>concatenate((labels[ordered], labels[disordered])),
|
||||
test_size<span style="color: #666666">=0.95</span>
|
||||
)
|
||||
</pre></div>
|
||||
<p>
|
||||
Using a small training set yields a better accuracy. This will be discussed in the end.
|
||||
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec18">Logistic regression </h2>
|
||||
|
||||
<p>
|
||||
Logistic regression is a linear model for classification. Recalling
|
||||
the cost function for ordinary least squares with both L2 (ridge) and
|
||||
L1 (LASSO) penalties we will see that the logistic cost function is
|
||||
very similar. In OLS we wish to predict a continuous variable
|
||||
\( \hat{y} \) using
|
||||
$$
|
||||
\begin{align}
|
||||
\hat{y} = X\omega,
|
||||
\label{_auto4}
|
||||
\end{align}
|
||||
$$
|
||||
|
||||
<p>
|
||||
where \( X \in \mathbb{R}^{n \times p} \) is the input data and \( \omega^{p
|
||||
\times d} \) are the weights of the regression. In a classification
|
||||
setting (binary classification in our situation) we are interested in
|
||||
a positive or negative answer. We can thus define either answer to be
|
||||
above or below some threshold. But, in order to limit the size of the
|
||||
answer and also to get a probability interpretation on how sure we are
|
||||
for either answer we can compute the sigmoid function of OLS. That is,
|
||||
|
||||
$$
|
||||
\begin{align}
|
||||
f(X\omega) = \frac{1}{1 + \exp(-X\omega)}.
|
||||
\label{_auto5}
|
||||
\end{align}
|
||||
$$
|
||||
|
||||
We are thus interested in minizming the following cost function
|
||||
$$
|
||||
\begin{align}
|
||||
C(X, \omega) = \sum_{i = 1}^n \left\{
|
||||
- y_i\log\left( f(x_i^T\omega) \right)
|
||||
- (1 - y_i)\log\left[1 - f(x_i^T\omega)\right]
|
||||
\right\},
|
||||
\label{_auto6}
|
||||
\end{align}
|
||||
$$
|
||||
|
||||
<p>
|
||||
where we will restrict ourselves to a value for \( f(z) \) as the sigmoid
|
||||
described above. We can also tack on a L2 (Ridge) or L1 (LASSO)
|
||||
penalization to this cost function in the same manner we did for
|
||||
linear regression.
|
||||
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec19">Exploring the logistic regression </h2>
|
||||
|
||||
<p>
|
||||
The penalization factor \( \lambda \) is inverted in the case of the
|
||||
logistic regression model we use. We will explore several values of
|
||||
\( \lambda \) using both L1 and L2 penalization. We do this using a grid
|
||||
search over different parameters and run a 3-fold cross validation for
|
||||
each configuration. In other words, we fit a model 3 times for each
|
||||
configuration of the hyper parameters.
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span>lambdas <span style="color: #666666">=</span> np<span style="color: #666666">.</span>logspace(<span style="color: #666666">-7</span>, <span style="color: #666666">-1</span>, <span style="color: #666666">7</span>)
|
||||
|
||||
param_grid <span style="color: #666666">=</span> {
|
||||
<span style="color: #BA2121">"C"</span>: <span style="color: #008000">list</span>(<span style="color: #666666">1.0/</span>lambdas),
|
||||
<span style="color: #BA2121">"penalty"</span>: [<span style="color: #BA2121">"l1"</span>, <span style="color: #BA2121">"l2"</span>]
|
||||
}
|
||||
clf <span style="color: #666666">=</span> skms<span style="color: #666666">.</span>GridSearchCV(
|
||||
skl<span style="color: #666666">.</span>LogisticRegression(),
|
||||
param_grid<span style="color: #666666">=</span>param_grid,
|
||||
n_jobs<span style="color: #666666">=-1</span>,
|
||||
return_train_score<span style="color: #666666">=</span><span style="color: #008000">True</span>
|
||||
)
|
||||
t0 <span style="color: #666666">=</span> time<span style="color: #666666">.</span>time()
|
||||
clf<span style="color: #666666">.</span>fit(X_train, y_train)
|
||||
t1 <span style="color: #666666">=</span> time<span style="color: #666666">.</span>time()
|
||||
|
||||
<span style="color: #008000; font-weight: bold">print</span> (
|
||||
<span style="color: #BA2121">"Time spent fitting GridSearchCV(LogisticRegression): {0:.3f} sec"</span><span style="color: #666666">.</span>format(
|
||||
t1 <span style="color: #666666">-</span> t0
|
||||
)
|
||||
)
|
||||
</pre></div>
|
||||
<p>
|
||||
We can see that logistic regression is quite slow and using the grid
|
||||
search and cross validation results in quite a heavy
|
||||
computation. Below we show the results of the different
|
||||
configurations.
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span>logreg_df <span style="color: #666666">=</span> pd<span style="color: #666666">.</span>DataFrame(clf<span style="color: #666666">.</span>cv_results_)
|
||||
|
||||
display(logreg_df)
|
||||
</pre></div>
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec20">Accuracy of a classification model </h2>
|
||||
|
||||
<p>
|
||||
To determine how well a classification model is performing we count
|
||||
the number of correctly labeled classes and divide by the number of
|
||||
classes in total. The accuracy is thus given by
|
||||
|
||||
$$
|
||||
\begin{align}
|
||||
a(y, \hat{y}) = \frac{1}{n}\sum_{i = 1}^{n} I(y_i = \hat{y}_i),
|
||||
\label{_auto7}
|
||||
\end{align}
|
||||
$$
|
||||
|
||||
<p>
|
||||
where \( I(y_i = \hat{y}_i) \) is the indicator function given by
|
||||
|
||||
$$
|
||||
\begin{align}
|
||||
I(x = y) = \begin{cases}
|
||||
1 & x = y,
|
||||
\label{_auto8}\\
|
||||
0 & x \neq y.
|
||||
\end{cases}
|
||||
\label{_auto9}
|
||||
\end{align}
|
||||
$$
|
||||
|
||||
<p>
|
||||
This is the accuracy provided by Scikit-learn when using <b>sklearn.metrics.accuracyscore</b>.
|
||||
|
||||
<p>
|
||||
Below we compute the accuracy of the best fit model on the training data (which should give a good accuracy), the test data (which has not been shown to the model) and the critical data (completely new data that needs to be extrapolated).
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span>train_accuracy <span style="color: #666666">=</span> skm<span style="color: #666666">.</span>accuracy_score(y_train, clf<span style="color: #666666">.</span>predict(X_train))
|
||||
test_accuracy <span style="color: #666666">=</span> skm<span style="color: #666666">.</span>accuracy_score(y_test, clf<span style="color: #666666">.</span>predict(X_test))
|
||||
critical_accuracy <span style="color: #666666">=</span> skm<span style="color: #666666">.</span>accuracy_score(labels[critical], clf<span style="color: #666666">.</span>predict(data[critical]))
|
||||
|
||||
<span style="color: #008000; font-weight: bold">print</span> (<span style="color: #BA2121">"Accuracy on train data: {0}"</span><span style="color: #666666">.</span>format(train_accuracy))
|
||||
<span style="color: #008000; font-weight: bold">print</span> (<span style="color: #BA2121">"Accuracy on test data: {0}"</span><span style="color: #666666">.</span>format(test_accuracy))
|
||||
<span style="color: #008000; font-weight: bold">print</span> (<span style="color: #BA2121">"Accuracy on critical data: {0}"</span><span style="color: #666666">.</span>format(critical_accuracy))
|
||||
</pre></div>
|
||||
<p>
|
||||
We can see that we get quite good accuracy on the training data, but gradually worsening accuracy on the test and critical data.
|
||||
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec21">Analyzing the results </h2>
|
||||
|
||||
<p>
|
||||
Below we show a different metric for determining the quality of our
|
||||
model, namely the <b>reciever operating characteristic</b> (ROC). The ROC
|
||||
curve tells us how well the model correctly classifies the different
|
||||
labels. We plot the <b>true positive rate</b> (the rate of predicted
|
||||
positive classes that are positive) versus the <b>false positive rate</b>
|
||||
(the rate of predicted positive classes that are negative). The ROC
|
||||
curve is built by computing the true positive rate and the false
|
||||
positive rate for varying <b>thresholds</b>, i.e, which probability we
|
||||
should acredit a certain class.
|
||||
|
||||
<p>
|
||||
By computing the <b>area under the curve</b> (AUC) of the ROC curve we get an estimate of how well our model is performing. Pure guessing will get an AUC of \( 0.5 \). A perfect score will get an AUC of \( 1.0 \).
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span>fig <span style="color: #666666">=</span> plt<span style="color: #666666">.</span>figure(figsize<span style="color: #666666">=</span>(<span style="color: #666666">20</span>, <span style="color: #666666">14</span>))
|
||||
|
||||
<span style="color: #008000; font-weight: bold">for</span> (_X, _y), label <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">zip</span>(
|
||||
[
|
||||
(X_train, y_train),
|
||||
(X_test, y_test),
|
||||
(data[critical], labels[critical])
|
||||
],
|
||||
[<span style="color: #BA2121">"Train"</span>, <span style="color: #BA2121">"Test"</span>, <span style="color: #BA2121">"Critical"</span>]
|
||||
):
|
||||
proba <span style="color: #666666">=</span> clf<span style="color: #666666">.</span>predict_proba(_X)
|
||||
fpr, tpr, _ <span style="color: #666666">=</span> skm<span style="color: #666666">.</span>roc_curve(_y, proba[:, <span style="color: #666666">1</span>])
|
||||
roc_auc <span style="color: #666666">=</span> skm<span style="color: #666666">.</span>auc(fpr, tpr)
|
||||
|
||||
<span style="color: #008000; font-weight: bold">print</span> (<span style="color: #BA2121">"LogisticRegression AUC ({0}): {1}"</span><span style="color: #666666">.</span>format(label, roc_auc))
|
||||
|
||||
plt<span style="color: #666666">.</span>plot(fpr, tpr, label<span style="color: #666666">=</span><span style="color: #BA2121">"{0} (AUC = {1})"</span><span style="color: #666666">.</span>format(label, roc_auc), linewidth<span style="color: #666666">=4.0</span>)
|
||||
|
||||
plt<span style="color: #666666">.</span>plot([<span style="color: #666666">0</span>, <span style="color: #666666">1</span>], [<span style="color: #666666">0</span>, <span style="color: #666666">1</span>], <span style="color: #BA2121">"--"</span>, label<span style="color: #666666">=</span><span style="color: #BA2121">"Guessing (AUC = 0.5)"</span>, linewidth<span style="color: #666666">=4.0</span>)
|
||||
|
||||
plt<span style="color: #666666">.</span>title(<span style="color: #BA2121">r"The ROC curve for LogisticRegression"</span>, fontsize<span style="color: #666666">=18</span>)
|
||||
plt<span style="color: #666666">.</span>xlabel(<span style="color: #BA2121">r"False positive rate"</span>, fontsize<span style="color: #666666">=18</span>)
|
||||
plt<span style="color: #666666">.</span>ylabel(<span style="color: #BA2121">r"True positive rate"</span>, fontsize<span style="color: #666666">=18</span>)
|
||||
plt<span style="color: #666666">.</span>axis([<span style="color: #666666">-0.01</span>, <span style="color: #666666">1.01</span>, <span style="color: #666666">-0.01</span>, <span style="color: #666666">1.01</span>])
|
||||
plt<span style="color: #666666">.</span>xticks(fontsize<span style="color: #666666">=18</span>)
|
||||
plt<span style="color: #666666">.</span>yticks(fontsize<span style="color: #666666">=18</span>)
|
||||
plt<span style="color: #666666">.</span>legend(loc<span style="color: #666666">=</span><span style="color: #BA2121">"best"</span>, fontsize<span style="color: #666666">=18</span>)
|
||||
plt<span style="color: #666666">.</span>show()
|
||||
</pre></div>
|
||||
<p>
|
||||
We can see that this plot of the ROC looks very strange. This tells us
|
||||
that logistic regression is quite inept at predicting the Ising model
|
||||
transition and is therefore highly non-linear. The ROC curve for the
|
||||
training data looks quite good, but as the testing data is so far off
|
||||
we see that we are dealing with an overfit model.
|
||||
|
||||
<p>
|
||||
A previous run with \( 50\% \) of the data used for training yielded a
|
||||
worse performance than using a smaller training set. This again gives
|
||||
confidence to the fact that logistic regression is not able to
|
||||
correctly fit the Ising model as it is not a linear model.
|
||||
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
|
||||
@@ -616,6 +616,489 @@
|
||||
"if __name__ == \"__main__\":\n",
|
||||
" main()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"<!-- !split -->\n",
|
||||
"## The two-dimensional Ising model, Predicting phase transition of the two-dimensional Ising model\n",
|
||||
"\n",
|
||||
"The Hamiltonian of the two-dimensional Ising model without an external field for a constant coupling constant $J$ is given by"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"<!-- Equation labels as ordinary links -->\n",
|
||||
"<div id=\"_auto2\"></div>\n",
|
||||
"\n",
|
||||
"$$\n",
|
||||
"\\begin{equation}\n",
|
||||
" H = -J \\sum_{\\langle ij\\rangle} S_i S_j,\n",
|
||||
"\\label{_auto2} \\tag{2}\n",
|
||||
"\\end{equation}\n",
|
||||
"$$"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"where $S_i \\in \\{-1, 1\\}$ and $\\langle ij \\rangle$ signifies that we only iterate over the nearest neighbors in the lattice. We will be looking at a system of $L = 40$ spins in each dimension, i.e., $L^2 = 1600$ spins in total. Opposed to the one-dimensional Ising model we will get a phase transition from an **ordered** phase to a **disordered** phase at the critical temperature"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"<!-- Equation labels as ordinary links -->\n",
|
||||
"<div id=\"_auto3\"></div>\n",
|
||||
"\n",
|
||||
"$$\n",
|
||||
"\\begin{equation}\n",
|
||||
" \\frac{T_c}{J} = \\frac{2}{\\log\\left(1 + \\sqrt{2}\\right)} \\approx 2.26,\n",
|
||||
"\\label{_auto3} \\tag{3}\n",
|
||||
"\\end{equation}\n",
|
||||
"$$"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"as shown by Lars Onsager.\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"Here we use **logistic regression** to predict when a phase transition\n",
|
||||
"occurs. The data we will look at is a set of spin configurations,\n",
|
||||
"i.e., individual lattices with spins, labeled **ordered** `1` or\n",
|
||||
"**disordered** `0`. Our job is to build a model which will take in a\n",
|
||||
"spin configuration and predict whether or not the spin configuration\n",
|
||||
"constitutes an ordered or a disordered phase. To achieve this we will\n",
|
||||
"represent the lattices as flattened arrays with $1600$ elements\n",
|
||||
"instead of a matrix of $40 \\times 40$ elements. As an extra test of\n",
|
||||
"the performance of the algorithms we will divide the dataset into\n",
|
||||
"three pieces. We will do a conventional train-test-split on a\n",
|
||||
"combination of totally ordered and totally disordered phases. The\n",
|
||||
"remaining \"critical-like\" states will be used as test data which we\n",
|
||||
"hope the model will be able to make good extrapolated predictions on."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 3,
|
||||
"metadata": {
|
||||
"collapsed": false
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import pickle\n",
|
||||
"import os\n",
|
||||
"import glob\n",
|
||||
"import numpy as np\n",
|
||||
"import pandas as pd\n",
|
||||
"import matplotlib.pyplot as plt\n",
|
||||
"import seaborn as sns\n",
|
||||
"import sklearn.model_selection as skms\n",
|
||||
"import sklearn.linear_model as skl\n",
|
||||
"import sklearn.metrics as skm\n",
|
||||
"import tqdm\n",
|
||||
"import copy\n",
|
||||
"import time\n",
|
||||
"from IPython.display import display\n",
|
||||
"\n",
|
||||
"%matplotlib inline\n",
|
||||
"\n",
|
||||
"sns.set(color_codes=True)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Reading in the data\n",
|
||||
"\n",
|
||||
"Using the data from [Mehta et al.](https://physics.bu.edu/~pankajm/ML-Review-Datasets/isingMC/) (specifically the two datasets named `Ising2DFM_reSample_L40_T=All.pkl` and `Ising2DFM_reSample_L40_T=All_labels.pkl`) we have to unpack the data into numpy arrays."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 4,
|
||||
"metadata": {
|
||||
"collapsed": false
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"filenames = glob.glob(os.path.join(\"..\", \"dat\", \"*\"))\n",
|
||||
"label_filename = list(filter(lambda x: \"label\" in x, filenames))[0]\n",
|
||||
"dat_filename = list(filter(lambda x: \"label\" not in x, filenames))[0]\n",
|
||||
"\n",
|
||||
"# Read in the labels\n",
|
||||
"with open(label_filename, \"rb\") as f:\n",
|
||||
" labels = pickle.load(f)\n",
|
||||
"\n",
|
||||
"# Read in the corresponding configurations\n",
|
||||
"with open(dat_filename, \"rb\") as f:\n",
|
||||
" data = np.unpackbits(pickle.load(f)).reshape(-1, 1600).astype(\"int\")\n",
|
||||
"\n",
|
||||
"# Set spin-down to -1\n",
|
||||
"data[data == 0] = -1"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"This dataset consists of $10000$ samples, i.e., $10000$ spin\n",
|
||||
"configurations with $40 \\times 40$ spins each, for $16$ temperatures\n",
|
||||
"between $0.25$ to $4.0$. Next we create a train/test-split and keep\n",
|
||||
"the data in the critical phase as a separate dataset for\n",
|
||||
"extrapolation-testing."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 5,
|
||||
"metadata": {
|
||||
"collapsed": false
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Set up slices of the dataset\n",
|
||||
"ordered = slice(0, 70000)\n",
|
||||
"critical = slice(70000, 100000)\n",
|
||||
"disordered = slice(100000, 160000)\n",
|
||||
"\n",
|
||||
"X_train, X_test, y_train, y_test = skms.train_test_split(\n",
|
||||
" np.concatenate((data[ordered], data[disordered])),\n",
|
||||
" np.concatenate((labels[ordered], labels[disordered])),\n",
|
||||
" test_size=0.95\n",
|
||||
")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"Using a small training set yields a better accuracy. This will be discussed in the end.\n",
|
||||
"\n",
|
||||
"## Logistic regression\n",
|
||||
"\n",
|
||||
"Logistic regression is a linear model for classification. Recalling\n",
|
||||
"the cost function for ordinary least squares with both L2 (ridge) and\n",
|
||||
"L1 (LASSO) penalties we will see that the logistic cost function is\n",
|
||||
"very similar. In OLS we wish to predict a continuous variable\n",
|
||||
"$\\hat{y}$ using"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"<!-- Equation labels as ordinary links -->\n",
|
||||
"<div id=\"_auto4\"></div>\n",
|
||||
"\n",
|
||||
"$$\n",
|
||||
"\\begin{equation}\n",
|
||||
" \\hat{y} = X\\omega,\n",
|
||||
"\\label{_auto4} \\tag{4}\n",
|
||||
"\\end{equation}\n",
|
||||
"$$"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"where $X \\in \\mathbb{R}^{n \\times p}$ is the input data and $\\omega^{p\n",
|
||||
"\\times d}$ are the weights of the regression. In a classification\n",
|
||||
"setting (binary classification in our situation) we are interested in\n",
|
||||
"a positive or negative answer. We can thus define either answer to be\n",
|
||||
"above or below some threshold. But, in order to limit the size of the\n",
|
||||
"answer and also to get a probability interpretation on how sure we are\n",
|
||||
"for either answer we can compute the sigmoid function of OLS. That is,"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"<!-- Equation labels as ordinary links -->\n",
|
||||
"<div id=\"_auto5\"></div>\n",
|
||||
"\n",
|
||||
"$$\n",
|
||||
"\\begin{equation}\n",
|
||||
" f(X\\omega) = \\frac{1}{1 + \\exp(-X\\omega)}.\n",
|
||||
"\\label{_auto5} \\tag{5}\n",
|
||||
"\\end{equation}\n",
|
||||
"$$"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"We are thus interested in minizming the following cost function"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"<!-- Equation labels as ordinary links -->\n",
|
||||
"<div id=\"_auto6\"></div>\n",
|
||||
"\n",
|
||||
"$$\n",
|
||||
"\\begin{equation}\n",
|
||||
" C(X, \\omega) = \\sum_{i = 1}^n \\left\\{\n",
|
||||
" - y_i\\log\\left( f(x_i^T\\omega) \\right)\n",
|
||||
" - (1 - y_i)\\log\\left[1 - f(x_i^T\\omega)\\right]\n",
|
||||
" \\right\\},\n",
|
||||
"\\label{_auto6} \\tag{6}\n",
|
||||
"\\end{equation}\n",
|
||||
"$$"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"where we will restrict ourselves to a value for $f(z)$ as the sigmoid\n",
|
||||
"described above. We can also tack on a L2 (Ridge) or L1 (LASSO)\n",
|
||||
"penalization to this cost function in the same manner we did for\n",
|
||||
"linear regression.\n",
|
||||
"\n",
|
||||
"## Exploring the logistic regression\n",
|
||||
"\n",
|
||||
"The penalization factor $\\lambda$ is inverted in the case of the\n",
|
||||
"logistic regression model we use. We will explore several values of\n",
|
||||
"$\\lambda$ using both L1 and L2 penalization. We do this using a grid\n",
|
||||
"search over different parameters and run a 3-fold cross validation for\n",
|
||||
"each configuration. In other words, we fit a model 3 times for each\n",
|
||||
"configuration of the hyper parameters."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 6,
|
||||
"metadata": {
|
||||
"collapsed": false
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"lambdas = np.logspace(-7, -1, 7)\n",
|
||||
"\n",
|
||||
"param_grid = {\n",
|
||||
" \"C\": list(1.0/lambdas),\n",
|
||||
" \"penalty\": [\"l1\", \"l2\"]\n",
|
||||
"}\n",
|
||||
"clf = skms.GridSearchCV(\n",
|
||||
" skl.LogisticRegression(),\n",
|
||||
" param_grid=param_grid,\n",
|
||||
" n_jobs=-1,\n",
|
||||
" return_train_score=True\n",
|
||||
")\n",
|
||||
"t0 = time.time()\n",
|
||||
"clf.fit(X_train, y_train)\n",
|
||||
"t1 = time.time()\n",
|
||||
"\n",
|
||||
"print (\n",
|
||||
" \"Time spent fitting GridSearchCV(LogisticRegression): {0:.3f} sec\".format(\n",
|
||||
" t1 - t0\n",
|
||||
" )\n",
|
||||
")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"We can see that logistic regression is quite slow and using the grid\n",
|
||||
"search and cross validation results in quite a heavy\n",
|
||||
"computation. Below we show the results of the different\n",
|
||||
"configurations."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 7,
|
||||
"metadata": {
|
||||
"collapsed": false
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"logreg_df = pd.DataFrame(clf.cv_results_)\n",
|
||||
"\n",
|
||||
"display(logreg_df)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Accuracy of a classification model\n",
|
||||
"\n",
|
||||
"To determine how well a classification model is performing we count\n",
|
||||
"the number of correctly labeled classes and divide by the number of\n",
|
||||
"classes in total. The accuracy is thus given by"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"<!-- Equation labels as ordinary links -->\n",
|
||||
"<div id=\"_auto7\"></div>\n",
|
||||
"\n",
|
||||
"$$\n",
|
||||
"\\begin{equation}\n",
|
||||
" a(y, \\hat{y}) = \\frac{1}{n}\\sum_{i = 1}^{n} I(y_i = \\hat{y}_i),\n",
|
||||
"\\label{_auto7} \\tag{7}\n",
|
||||
"\\end{equation}\n",
|
||||
"$$"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"where $I(y_i = \\hat{y}_i)$ is the indicator function given by"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"<!-- Equation labels as ordinary links -->\n",
|
||||
"<div id=\"_auto8\"></div>\n",
|
||||
"\n",
|
||||
"$$\n",
|
||||
"\\begin{equation}\n",
|
||||
" I(x = y) = \\begin{cases}\n",
|
||||
" 1 x = y, \n",
|
||||
"\\label{_auto8} \\tag{8}\n",
|
||||
"\\end{equation}\n",
|
||||
"$$"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"<!-- Equation labels as ordinary links -->\n",
|
||||
"<div id=\"_auto9\"></div>\n",
|
||||
"\n",
|
||||
"$$\n",
|
||||
"\\begin{equation} \n",
|
||||
" 0 x \\neq y.\n",
|
||||
" \\end{cases}\n",
|
||||
"\\label{_auto9} \\tag{9}\n",
|
||||
"\\end{equation}\n",
|
||||
"$$"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"This is the accuracy provided by Scikit-learn when using **sklearn.metrics.accuracyscore**.\n",
|
||||
"\n",
|
||||
"Below we compute the accuracy of the best fit model on the training data (which should give a good accuracy), the test data (which has not been shown to the model) and the critical data (completely new data that needs to be extrapolated)."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 8,
|
||||
"metadata": {
|
||||
"collapsed": false
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"train_accuracy = skm.accuracy_score(y_train, clf.predict(X_train))\n",
|
||||
"test_accuracy = skm.accuracy_score(y_test, clf.predict(X_test))\n",
|
||||
"critical_accuracy = skm.accuracy_score(labels[critical], clf.predict(data[critical]))\n",
|
||||
"\n",
|
||||
"print (\"Accuracy on train data: {0}\".format(train_accuracy))\n",
|
||||
"print (\"Accuracy on test data: {0}\".format(test_accuracy))\n",
|
||||
"print (\"Accuracy on critical data: {0}\".format(critical_accuracy))"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"We can see that we get quite good accuracy on the training data, but gradually worsening accuracy on the test and critical data.\n",
|
||||
"\n",
|
||||
"## Analyzing the results\n",
|
||||
"\n",
|
||||
"Below we show a different metric for determining the quality of our\n",
|
||||
"model, namely the **reciever operating characteristic** (ROC). The ROC\n",
|
||||
"curve tells us how well the model correctly classifies the different\n",
|
||||
"labels. We plot the **true positive rate** (the rate of predicted\n",
|
||||
"positive classes that are positive) versus the **false positive rate**\n",
|
||||
"(the rate of predicted positive classes that are negative). The ROC\n",
|
||||
"curve is built by computing the true positive rate and the false\n",
|
||||
"positive rate for varying **thresholds**, i.e, which probability we\n",
|
||||
"should acredit a certain class.\n",
|
||||
"\n",
|
||||
"By computing the **area under the curve** (AUC) of the ROC curve we get an estimate of how well our model is performing. Pure guessing will get an AUC of $0.5$. A perfect score will get an AUC of $1.0$."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 9,
|
||||
"metadata": {
|
||||
"collapsed": false
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"fig = plt.figure(figsize=(20, 14))\n",
|
||||
"\n",
|
||||
"for (_X, _y), label in zip(\n",
|
||||
" [\n",
|
||||
" (X_train, y_train),\n",
|
||||
" (X_test, y_test),\n",
|
||||
" (data[critical], labels[critical])\n",
|
||||
" ],\n",
|
||||
" [\"Train\", \"Test\", \"Critical\"]\n",
|
||||
"):\n",
|
||||
" proba = clf.predict_proba(_X)\n",
|
||||
" fpr, tpr, _ = skm.roc_curve(_y, proba[:, 1])\n",
|
||||
" roc_auc = skm.auc(fpr, tpr)\n",
|
||||
"\n",
|
||||
" print (\"LogisticRegression AUC ({0}): {1}\".format(label, roc_auc))\n",
|
||||
"\n",
|
||||
" plt.plot(fpr, tpr, label=\"{0} (AUC = {1})\".format(label, roc_auc), linewidth=4.0)\n",
|
||||
"\n",
|
||||
"plt.plot([0, 1], [0, 1], \"--\", label=\"Guessing (AUC = 0.5)\", linewidth=4.0)\n",
|
||||
"\n",
|
||||
"plt.title(r\"The ROC curve for LogisticRegression\", fontsize=18)\n",
|
||||
"plt.xlabel(r\"False positive rate\", fontsize=18)\n",
|
||||
"plt.ylabel(r\"True positive rate\", fontsize=18)\n",
|
||||
"plt.axis([-0.01, 1.01, -0.01, 1.01])\n",
|
||||
"plt.xticks(fontsize=18)\n",
|
||||
"plt.yticks(fontsize=18)\n",
|
||||
"plt.legend(loc=\"best\", fontsize=18)\n",
|
||||
"plt.show()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"We can see that this plot of the ROC looks very strange. This tells us\n",
|
||||
"that logistic regression is quite inept at predicting the Ising model\n",
|
||||
"transition and is therefore highly non-linear. The ROC curve for the\n",
|
||||
"training data looks quite good, but as the testing data is so far off\n",
|
||||
"we see that we are dealing with an overfit model.\n",
|
||||
"\n",
|
||||
"A previous run with $50\\%$ of the data used for training yielded a\n",
|
||||
"worse performance than using a smaller training set. This again gives\n",
|
||||
"confidence to the fact that logistic regression is not able to\n",
|
||||
"correctly fit the Ising model as it is not a linear model."
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {},
|
||||
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -388,3 +388,292 @@ def main():
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
!ec
|
||||
|
||||
!split
|
||||
===== The two-dimensional Ising model, Predicting phase transition of the two-dimensional Ising model =====
|
||||
|
||||
The Hamiltonian of the two-dimensional Ising model without an external field for a constant coupling constant $J$ is given by
|
||||
!bt
|
||||
\begin{align}
|
||||
H = -J \sum_{\langle ij\rangle} S_i S_j,
|
||||
\end{align}
|
||||
!et
|
||||
where $S_i \in \{-1, 1\}$ and $\langle ij \rangle$ signifies that we only iterate over the nearest neighbors in the lattice. We will be looking at a system of $L = 40$ spins in each dimension, i.e., $L^2 = 1600$ spins in total. Opposed to the one-dimensional Ising model we will get a phase transition from an _ordered_ phase to a _disordered_ phase at the critical temperature
|
||||
|
||||
!bt
|
||||
\begin{align}
|
||||
\frac{T_c}{J} = \frac{2}{\log\left(1 + \sqrt{2}\right)} \approx 2.26,
|
||||
\end{align}
|
||||
!et
|
||||
as shown by Lars Onsager.
|
||||
|
||||
|
||||
Here we use _logistic regression_ to predict when a phase transition
|
||||
occurs. The data we will look at is a set of spin configurations,
|
||||
i.e., individual lattices with spins, labeled _ordered_ `1` or
|
||||
_disordered_ `0`. Our job is to build a model which will take in a
|
||||
spin configuration and predict whether or not the spin configuration
|
||||
constitutes an ordered or a disordered phase. To achieve this we will
|
||||
represent the lattices as flattened arrays with $1600$ elements
|
||||
instead of a matrix of $40 \times 40$ elements. As an extra test of
|
||||
the performance of the algorithms we will divide the dataset into
|
||||
three pieces. We will do a conventional train-test-split on a
|
||||
combination of totally ordered and totally disordered phases. The
|
||||
remaining "critical-like" states will be used as test data which we
|
||||
hope the model will be able to make good extrapolated predictions on.
|
||||
|
||||
|
||||
!bc pycod
|
||||
import pickle
|
||||
import os
|
||||
import glob
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
import matplotlib.pyplot as plt
|
||||
import seaborn as sns
|
||||
import sklearn.model_selection as skms
|
||||
import sklearn.linear_model as skl
|
||||
import sklearn.metrics as skm
|
||||
import tqdm
|
||||
import copy
|
||||
import time
|
||||
from IPython.display import display
|
||||
|
||||
%matplotlib inline
|
||||
|
||||
sns.set(color_codes=True)
|
||||
!ec
|
||||
|
||||
!split
|
||||
===== Reading in the data =====
|
||||
|
||||
Using the data from "Mehta et al.":"https://physics.bu.edu/~pankajm/ML-Review-Datasets/isingMC/" (specifically the two datasets named `Ising2DFM_reSample_L40_T=All.pkl` and `Ising2DFM_reSample_L40_T=All_labels.pkl`) we have to unpack the data into numpy arrays.
|
||||
|
||||
|
||||
!bc pycod
|
||||
filenames = glob.glob(os.path.join("..", "dat", "*"))
|
||||
label_filename = list(filter(lambda x: "label" in x, filenames))[0]
|
||||
dat_filename = list(filter(lambda x: "label" not in x, filenames))[0]
|
||||
|
||||
# Read in the labels
|
||||
with open(label_filename, "rb") as f:
|
||||
labels = pickle.load(f)
|
||||
|
||||
# Read in the corresponding configurations
|
||||
with open(dat_filename, "rb") as f:
|
||||
data = np.unpackbits(pickle.load(f)).reshape(-1, 1600).astype("int")
|
||||
|
||||
# Set spin-down to -1
|
||||
data[data == 0] = -1
|
||||
!ec
|
||||
|
||||
This dataset consists of $10000$ samples, i.e., $10000$ spin
|
||||
configurations with $40 \times 40$ spins each, for $16$ temperatures
|
||||
between $0.25$ to $4.0$. Next we create a train/test-split and keep
|
||||
the data in the critical phase as a separate dataset for
|
||||
extrapolation-testing.
|
||||
|
||||
|
||||
!bc pycod
|
||||
# Set up slices of the dataset
|
||||
ordered = slice(0, 70000)
|
||||
critical = slice(70000, 100000)
|
||||
disordered = slice(100000, 160000)
|
||||
|
||||
X_train, X_test, y_train, y_test = skms.train_test_split(
|
||||
np.concatenate((data[ordered], data[disordered])),
|
||||
np.concatenate((labels[ordered], labels[disordered])),
|
||||
test_size=0.95
|
||||
)
|
||||
!ec
|
||||
|
||||
Using a small training set yields a better accuracy. This will be discussed in the end.
|
||||
|
||||
!split
|
||||
===== Logistic regression =====
|
||||
|
||||
Logistic regression is a linear model for classification. Recalling
|
||||
the cost function for ordinary least squares with both L2 (ridge) and
|
||||
L1 (LASSO) penalties we will see that the logistic cost function is
|
||||
very similar. In OLS we wish to predict a continuous variable
|
||||
$\hat{y}$ using
|
||||
!bt
|
||||
\begin{align}
|
||||
\hat{y} = X\omega,
|
||||
\end{align}
|
||||
!et
|
||||
|
||||
where $X \in \mathbb{R}^{n \times p}$ is the input data and $\omega^{p
|
||||
\times d}$ are the weights of the regression. In a classification
|
||||
setting (binary classification in our situation) we are interested in
|
||||
a positive or negative answer. We can thus define either answer to be
|
||||
above or below some threshold. But, in order to limit the size of the
|
||||
answer and also to get a probability interpretation on how sure we are
|
||||
for either answer we can compute the sigmoid function of OLS. That is,
|
||||
|
||||
!bt
|
||||
\begin{align}
|
||||
f(X\omega) = \frac{1}{1 + \exp(-X\omega)}.
|
||||
\end{align}
|
||||
!et
|
||||
We are thus interested in minizming the following cost function
|
||||
!bt
|
||||
\begin{align}
|
||||
C(X, \omega) = \sum_{i = 1}^n \left\{
|
||||
- y_i\log\left( f(x_i^T\omega) \right)
|
||||
- (1 - y_i)\log\left[1 - f(x_i^T\omega)\right]
|
||||
\right\},
|
||||
\end{align}
|
||||
!et
|
||||
|
||||
where we will restrict ourselves to a value for $f(z)$ as the sigmoid
|
||||
described above. We can also tack on a L2 (Ridge) or L1 (LASSO)
|
||||
penalization to this cost function in the same manner we did for
|
||||
linear regression.
|
||||
|
||||
!split
|
||||
===== Exploring the logistic regression =====
|
||||
|
||||
The penalization factor $\lambda$ is inverted in the case of the
|
||||
logistic regression model we use. We will explore several values of
|
||||
$\lambda$ using both L1 and L2 penalization. We do this using a grid
|
||||
search over different parameters and run a 3-fold cross validation for
|
||||
each configuration. In other words, we fit a model 3 times for each
|
||||
configuration of the hyper parameters.
|
||||
|
||||
|
||||
!bc pycod
|
||||
lambdas = np.logspace(-7, -1, 7)
|
||||
|
||||
param_grid = {
|
||||
"C": list(1.0/lambdas),
|
||||
"penalty": ["l1", "l2"]
|
||||
}
|
||||
clf = skms.GridSearchCV(
|
||||
skl.LogisticRegression(),
|
||||
param_grid=param_grid,
|
||||
n_jobs=-1,
|
||||
return_train_score=True
|
||||
)
|
||||
t0 = time.time()
|
||||
clf.fit(X_train, y_train)
|
||||
t1 = time.time()
|
||||
|
||||
print (
|
||||
"Time spent fitting GridSearchCV(LogisticRegression): {0:.3f} sec".format(
|
||||
t1 - t0
|
||||
)
|
||||
)
|
||||
!ec
|
||||
|
||||
We can see that logistic regression is quite slow and using the grid
|
||||
search and cross validation results in quite a heavy
|
||||
computation. Below we show the results of the different
|
||||
configurations.
|
||||
|
||||
|
||||
!bc pycod
|
||||
logreg_df = pd.DataFrame(clf.cv_results_)
|
||||
|
||||
display(logreg_df)
|
||||
!ec
|
||||
|
||||
!split
|
||||
===== Accuracy of a classification model =====
|
||||
|
||||
To determine how well a classification model is performing we count
|
||||
the number of correctly labeled classes and divide by the number of
|
||||
classes in total. The accuracy is thus given by
|
||||
|
||||
!bt
|
||||
\begin{align}
|
||||
a(y, \hat{y}) = \frac{1}{n}\sum_{i = 1}^{n} I(y_i = \hat{y}_i),
|
||||
\end{align}
|
||||
!et
|
||||
|
||||
where $I(y_i = \hat{y}_i)$ is the indicator function given by
|
||||
|
||||
!bt
|
||||
\begin{align}
|
||||
I(x = y) = \begin{cases}
|
||||
1 & x = y, \\
|
||||
0 & x \neq y.
|
||||
\end{cases}
|
||||
\end{align}
|
||||
!et
|
||||
|
||||
This is the accuracy provided by Scikit-learn when using _sklearn.metrics.accuracyscore_.
|
||||
|
||||
Below we compute the accuracy of the best fit model on the training data (which should give a good accuracy), the test data (which has not been shown to the model) and the critical data (completely new data that needs to be extrapolated).
|
||||
|
||||
|
||||
!bc pycod
|
||||
train_accuracy = skm.accuracy_score(y_train, clf.predict(X_train))
|
||||
test_accuracy = skm.accuracy_score(y_test, clf.predict(X_test))
|
||||
critical_accuracy = skm.accuracy_score(labels[critical], clf.predict(data[critical]))
|
||||
|
||||
print ("Accuracy on train data: {0}".format(train_accuracy))
|
||||
print ("Accuracy on test data: {0}".format(test_accuracy))
|
||||
print ("Accuracy on critical data: {0}".format(critical_accuracy))
|
||||
!ec
|
||||
|
||||
We can see that we get quite good accuracy on the training data, but gradually worsening accuracy on the test and critical data.
|
||||
|
||||
!split
|
||||
===== Analyzing the results =====
|
||||
|
||||
Below we show a different metric for determining the quality of our
|
||||
model, namely the _reciever operating characteristic_ (ROC). The ROC
|
||||
curve tells us how well the model correctly classifies the different
|
||||
labels. We plot the _true positive rate_ (the rate of predicted
|
||||
positive classes that are positive) versus the _false positive rate_
|
||||
(the rate of predicted positive classes that are negative). The ROC
|
||||
curve is built by computing the true positive rate and the false
|
||||
positive rate for varying _thresholds_, i.e, which probability we
|
||||
should acredit a certain class.
|
||||
|
||||
By computing the _area under the curve_ (AUC) of the ROC curve we get an estimate of how well our model is performing. Pure guessing will get an AUC of $0.5$. A perfect score will get an AUC of $1.0$.
|
||||
|
||||
|
||||
!bc pycod
|
||||
fig = plt.figure(figsize=(20, 14))
|
||||
|
||||
for (_X, _y), label in zip(
|
||||
[
|
||||
(X_train, y_train),
|
||||
(X_test, y_test),
|
||||
(data[critical], labels[critical])
|
||||
],
|
||||
["Train", "Test", "Critical"]
|
||||
):
|
||||
proba = clf.predict_proba(_X)
|
||||
fpr, tpr, _ = skm.roc_curve(_y, proba[:, 1])
|
||||
roc_auc = skm.auc(fpr, tpr)
|
||||
|
||||
print ("LogisticRegression AUC ({0}): {1}".format(label, roc_auc))
|
||||
|
||||
plt.plot(fpr, tpr, label="{0} (AUC = {1})".format(label, roc_auc), linewidth=4.0)
|
||||
|
||||
plt.plot([0, 1], [0, 1], "--", label="Guessing (AUC = 0.5)", linewidth=4.0)
|
||||
|
||||
plt.title(r"The ROC curve for LogisticRegression", fontsize=18)
|
||||
plt.xlabel(r"False positive rate", fontsize=18)
|
||||
plt.ylabel(r"True positive rate", fontsize=18)
|
||||
plt.axis([-0.01, 1.01, -0.01, 1.01])
|
||||
plt.xticks(fontsize=18)
|
||||
plt.yticks(fontsize=18)
|
||||
plt.legend(loc="best", fontsize=18)
|
||||
plt.show()
|
||||
!ec
|
||||
|
||||
We can see that this plot of the ROC looks very strange. This tells us
|
||||
that logistic regression is quite inept at predicting the Ising model
|
||||
transition and is therefore highly non-linear. The ROC curve for the
|
||||
training data looks quite good, but as the testing data is so far off
|
||||
we see that we are dealing with an overfit model.
|
||||
|
||||
A previous run with $50\%$ of the data used for training yielded a
|
||||
worse performance than using a smaller training set. This again gives
|
||||
confidence to the fact that logistic regression is not able to
|
||||
correctly fit the Ising model as it is not a linear model.
|
||||
|
||||
|
||||
Reference in New Issue
Block a user