redundant code lines
This commit is contained in:
Morten Hjorth-Jensen
@@ -1050,8 +1050,6 @@ print(X_test.shape)
|
||||
# Logistic Regression
|
||||
logreg = LogisticRegression(solver='lbfgs')
|
||||
logreg.fit(X_train, y_train)
|
||||
print("Test set accuracy with Logistic Regression: {:.2f}".format(logreg.score(X_test,y_test)))
|
||||
|
||||
|
||||
from sklearn.preprocessing import LabelEncoder
|
||||
from sklearn.model_selection import cross_validate
|
||||
@@ -1060,7 +1058,6 @@ accuracy = cross_validate(logreg,X_test,y_test,cv=10)['test_score']
|
||||
print(accuracy)
|
||||
print("Test set accuracy with Logistic Regression: {:.2f}".format(logreg.score(X_test,y_test)))
|
||||
|
||||
|
||||
import scikitplot as skplt
|
||||
y_pred = logreg.predict(X_test)
|
||||
skplt.metrics.plot_confusion_matrix(y_test, y_pred, normalize=True)
|
||||
@@ -1527,374 +1524,6 @@ Using the definition of convexity, try to show that a function satisfying the pr
|
||||
|
||||
|
||||
|
||||
!split
|
||||
===== Standard steepest descent =====
|
||||
|
||||
|
||||
Before we proceed, we would like to discuss the approach called the
|
||||
_standard Steepest descent_ (different from the above steepest descent discussion), which again leads to us having to be able
|
||||
to compute a matrix. It belongs to the class of Conjugate Gradient methods (CG).
|
||||
|
||||
"The success of the CG method":"https://www.cs.cmu.edu/~quake-papers/painless-conjugate-gradient.pdf"
|
||||
for finding solutions of non-linear problems is based on the theory
|
||||
of conjugate gradients for linear systems of equations. It belongs to
|
||||
the class of iterative methods for solving problems from linear
|
||||
algebra of the type
|
||||
!bt
|
||||
\begin{equation*}
|
||||
\bm{A}\bm{x} = \bm{b}.
|
||||
\end{equation*}
|
||||
!et
|
||||
|
||||
In the iterative process we end up with a problem like
|
||||
|
||||
!bt
|
||||
\begin{equation*}
|
||||
\bm{r}= \bm{b}-\bm{A}\bm{x},
|
||||
\end{equation*}
|
||||
!et
|
||||
where $\bm{r}$ is the so-called residual or error in the iterative process.
|
||||
|
||||
When we have found the exact solution, $\bm{r}=0$.
|
||||
|
||||
!split
|
||||
===== Gradient method =====
|
||||
|
||||
The residual is zero when we reach the minimum of the quadratic equation
|
||||
!bt
|
||||
\begin{equation*}
|
||||
P(\bm{x})=\frac{1}{2}\bm{x}^T\bm{A}\bm{x} - \bm{x}^T\bm{b},
|
||||
\end{equation*}
|
||||
!et
|
||||
|
||||
with the constraint that the matrix $\bm{A}$ is positive definite and
|
||||
symmetric. This defines also the Hessian and we want it to be positive definite.
|
||||
|
||||
|
||||
!split
|
||||
===== Steepest descent method =====
|
||||
|
||||
We denote the initial guess for $\bm{x}$ as $\bm{x}_0$.
|
||||
We can assume without loss of generality that
|
||||
!bt
|
||||
\begin{equation*}
|
||||
\bm{x}_0=0,
|
||||
\end{equation*}
|
||||
!et
|
||||
or consider the system
|
||||
!bt
|
||||
\begin{equation*}
|
||||
\bm{A}\bm{z} = \bm{b}-\bm{A}\bm{x}_0,
|
||||
\end{equation*}
|
||||
!et
|
||||
instead.
|
||||
|
||||
|
||||
!split
|
||||
===== Steepest descent method =====
|
||||
!bblock
|
||||
One can show that the solution $\bm{x}$ is also the unique minimizer of the quadratic form
|
||||
!bt
|
||||
\begin{equation*}
|
||||
f(\bm{x}) = \frac{1}{2}\bm{x}^T\bm{A}\bm{x} - \bm{x}^T \bm{x} , \quad \bm{x}\in\mathbf{R}^n.
|
||||
\end{equation*}
|
||||
!et
|
||||
This suggests taking the first basis vector $\bm{r}_1$ (see below for definition)
|
||||
to be the gradient of $f$ at $\bm{x}=\bm{x}_0$,
|
||||
which equals
|
||||
!bt
|
||||
\begin{equation*}
|
||||
\bm{A}\bm{x}_0-\bm{b},
|
||||
\end{equation*}
|
||||
!et
|
||||
and
|
||||
$\bm{x}_0=0$ it is equal $-\bm{b}$.
|
||||
|
||||
!eblock
|
||||
|
||||
!split
|
||||
===== Final expressions =====
|
||||
!bblock
|
||||
We can compute the residual iteratively as
|
||||
!bt
|
||||
\begin{equation*}
|
||||
\bm{r}_{k+1}=\bm{b}-\bm{A}\bm{x}_{k+1},
|
||||
\end{equation*}
|
||||
!et
|
||||
which equals
|
||||
!bt
|
||||
\begin{equation*}
|
||||
\bm{b}-\bm{A}(\bm{x}_k+\alpha_k\bm{r}_k),
|
||||
\end{equation*}
|
||||
!et
|
||||
or
|
||||
!bt
|
||||
\begin{equation*}
|
||||
(\bm{b}-\bm{A}\bm{x}_k)-\alpha_k\bm{A}\bm{r}_k,
|
||||
\end{equation*}
|
||||
!et
|
||||
which gives
|
||||
|
||||
!bt
|
||||
\[
|
||||
\alpha_k = \frac{\bm{r}_k^T\bm{r}_k}{\bm{r}_k^T\bm{A}\bm{r}_k}
|
||||
\]
|
||||
!et
|
||||
leading to the iterative scheme
|
||||
!bt
|
||||
\begin{equation*}
|
||||
\bm{x}_{k+1}=\bm{x}_k-\alpha_k\bm{r}_{k},
|
||||
\end{equation*}
|
||||
!et
|
||||
!eblock
|
||||
|
||||
|
||||
|
||||
!split
|
||||
===== Steepest descent example =====
|
||||
|
||||
!bc pycod
|
||||
import numpy as np
|
||||
import numpy.linalg as la
|
||||
|
||||
import scipy.optimize as sopt
|
||||
|
||||
import matplotlib.pyplot as pt
|
||||
from mpl_toolkits.mplot3d import axes3d
|
||||
|
||||
def f(x):
|
||||
return x[0]**2 + 3.0*x[1]**2
|
||||
|
||||
def df(x):
|
||||
return np.array([2*x[0], 6*x[1]])
|
||||
|
||||
fig = pt.figure()
|
||||
ax = fig.gca(projection="3d")
|
||||
|
||||
xmesh, ymesh = np.mgrid[-3:3:50j,-3:3:50j]
|
||||
fmesh = f(np.array([xmesh, ymesh]))
|
||||
ax.plot_surface(xmesh, ymesh, fmesh)
|
||||
!ec
|
||||
And then as countor plot
|
||||
!bc pycod
|
||||
pt.axis("equal")
|
||||
pt.contour(xmesh, ymesh, fmesh)
|
||||
guesses = [np.array([2, 2./5])]
|
||||
!ec
|
||||
Find guesses
|
||||
!bc pycod
|
||||
x = guesses[-1]
|
||||
s = -df(x)
|
||||
!ec
|
||||
Run it!
|
||||
!bc pycod
|
||||
def f1d(alpha):
|
||||
return f(x + alpha*s)
|
||||
|
||||
alpha_opt = sopt.golden(f1d)
|
||||
next_guess = x + alpha_opt * s
|
||||
guesses.append(next_guess)
|
||||
print(next_guess)
|
||||
!ec
|
||||
What happened?
|
||||
!bc pycod
|
||||
pt.axis("equal")
|
||||
pt.contour(xmesh, ymesh, fmesh, 50)
|
||||
it_array = np.array(guesses)
|
||||
pt.plot(it_array.T[0], it_array.T[1], "x-")
|
||||
!ec
|
||||
|
||||
Note that we did only one iteration here. We can easily add more using our previous guesses.
|
||||
|
||||
!split
|
||||
===== Conjugate gradient method =====
|
||||
!bblock
|
||||
In the CG method we define so-called conjugate directions and two vectors
|
||||
$\bm{s}$ and $\bm{t}$
|
||||
are said to be
|
||||
conjugate if
|
||||
!bt
|
||||
\begin{equation*}
|
||||
\bm{s}^T\bm{A}\bm{t}= 0.
|
||||
\end{equation*}
|
||||
!et
|
||||
The philosophy of the CG method is to perform searches in various conjugate directions
|
||||
of our vectors $\bm{x}_i$ obeying the above criterion, namely
|
||||
!bt
|
||||
\begin{equation*}
|
||||
\bm{x}_i^T\bm{A}\bm{x}_j= 0.
|
||||
\end{equation*}
|
||||
!et
|
||||
Two vectors are conjugate if they are orthogonal with respect to
|
||||
this inner product. Being conjugate is a symmetric relation: if $\bm{s}$ is conjugate to $\bm{t}$, then $\bm{t}$ is conjugate to $\bm{s}$.
|
||||
!eblock
|
||||
|
||||
!split
|
||||
===== Conjugate gradient method =====
|
||||
!bblock
|
||||
An example is given by the eigenvectors of the matrix
|
||||
!bt
|
||||
\begin{equation*}
|
||||
\bm{v}_i^T\bm{A}\bm{v}_j= \lambda\bm{v}_i^T\bm{v}_j,
|
||||
\end{equation*}
|
||||
!et
|
||||
which is zero unless $i=j$.
|
||||
!eblock
|
||||
|
||||
|
||||
!split
|
||||
===== Conjugate gradient method =====
|
||||
!bblock
|
||||
Assume now that we have a symmetric positive-definite matrix $\bm{A}$ of size
|
||||
$n\times n$. At each iteration $i+1$ we obtain the conjugate direction of a vector
|
||||
!bt
|
||||
\begin{equation*}
|
||||
\bm{x}_{i+1}=\bm{x}_{i}+\alpha_i\bm{p}_{i}.
|
||||
\end{equation*}
|
||||
!et
|
||||
We assume that $\bm{p}_{i}$ is a sequence of $n$ mutually conjugate directions.
|
||||
Then the $\bm{p}_{i}$ form a basis of $R^n$ and we can expand the solution
|
||||
$ \bm{A}\bm{x} = \bm{b}$ in this basis, namely
|
||||
|
||||
!bt
|
||||
\begin{equation*}
|
||||
\bm{x} = \sum^{n}_{i=1} \alpha_i \bm{p}_i.
|
||||
\end{equation*}
|
||||
!et
|
||||
!eblock
|
||||
|
||||
!split
|
||||
===== Conjugate gradient method =====
|
||||
!bblock
|
||||
The coefficients are given by
|
||||
!bt
|
||||
\begin{equation*}
|
||||
\mathbf{A}\mathbf{x} = \sum^{n}_{i=1} \alpha_i \mathbf{A} \mathbf{p}_i = \mathbf{b}.
|
||||
\end{equation*}
|
||||
!et
|
||||
Multiplying with $\bm{p}_k^T$ from the left gives
|
||||
|
||||
!bt
|
||||
\begin{equation*}
|
||||
\bm{p}_k^T \bm{A}\bm{x} = \sum^{n}_{i=1} \alpha_i\bm{p}_k^T \bm{A}\bm{p}_i= \bm{p}_k^T \bm{b},
|
||||
\end{equation*}
|
||||
!et
|
||||
and we can define the coefficients $\alpha_k$ as
|
||||
|
||||
!bt
|
||||
\begin{equation*}
|
||||
\alpha_k = \frac{\bm{p}_k^T \bm{b}}{\bm{p}_k^T \bm{A} \bm{p}_k}
|
||||
\end{equation*}
|
||||
!et
|
||||
!eblock
|
||||
|
||||
!split
|
||||
===== Conjugate gradient method and iterations =====
|
||||
!bblock
|
||||
|
||||
If we choose the conjugate vectors $\bm{p}_k$ carefully,
|
||||
then we may not need all of them to obtain a good approximation to the solution
|
||||
$\bm{x}$.
|
||||
We want to regard the conjugate gradient method as an iterative method.
|
||||
This will us to solve systems where $n$ is so large that the direct
|
||||
method would take too much time.
|
||||
|
||||
We denote the initial guess for $\bm{x}$ as $\bm{x}_0$.
|
||||
We can assume without loss of generality that
|
||||
!bt
|
||||
\begin{equation*}
|
||||
\bm{x}_0=0,
|
||||
\end{equation*}
|
||||
!et
|
||||
or consider the system
|
||||
!bt
|
||||
\begin{equation*}
|
||||
\bm{A}\bm{z} = \bm{b}-\bm{A}\bm{x}_0,
|
||||
\end{equation*}
|
||||
!et
|
||||
instead.
|
||||
!eblock
|
||||
|
||||
|
||||
!split
|
||||
===== Conjugate gradient method =====
|
||||
!bblock
|
||||
One can show that the solution $\bm{x}$ is also the unique minimizer of the quadratic form
|
||||
!bt
|
||||
\begin{equation*}
|
||||
f(\bm{x}) = \frac{1}{2}\bm{x}^T\bm{A}\bm{x} - \bm{x}^T \bm{x} , \quad \bm{x}\in\mathbf{R}^n.
|
||||
\end{equation*}
|
||||
!et
|
||||
This suggests taking the first basis vector $\bm{p}_1$
|
||||
to be the gradient of $f$ at $\bm{x}=\bm{x}_0$,
|
||||
which equals
|
||||
!bt
|
||||
\begin{equation*}
|
||||
\bm{A}\bm{x}_0-\bm{b},
|
||||
\end{equation*}
|
||||
!et
|
||||
and
|
||||
$\bm{x}_0=0$ it is equal $-\bm{b}$.
|
||||
The other vectors in the basis will be conjugate to the gradient,
|
||||
hence the name conjugate gradient method.
|
||||
!eblock
|
||||
|
||||
|
||||
!split
|
||||
===== Conjugate gradient method =====
|
||||
!bblock
|
||||
Let $\bm{r}_k$ be the residual at the $k$-th step:
|
||||
!bt
|
||||
\begin{equation*}
|
||||
\bm{r}_k=\bm{b}-\bm{A}\bm{x}_k.
|
||||
\end{equation*}
|
||||
!et
|
||||
Note that $\bm{r}_k$ is the negative gradient of $f$ at
|
||||
$\bm{x}=\bm{x}_k$,
|
||||
so the gradient descent method would be to move in the direction $\bm{r}_k$.
|
||||
Here, we insist that the directions $\bm{p}_k$ are conjugate to each other,
|
||||
so we take the direction closest to the gradient $\bm{r}_k$
|
||||
under the conjugacy constraint.
|
||||
This gives the following expression
|
||||
!bt
|
||||
\begin{equation*}
|
||||
\bm{p}_{k+1}=\bm{r}_k-\frac{\bm{p}_k^T \bm{A}\bm{r}_k}{\bm{p}_k^T\bm{A}\bm{p}_k} \bm{p}_k.
|
||||
\end{equation*}
|
||||
!et
|
||||
!eblock
|
||||
|
||||
!split
|
||||
===== Conjugate gradient method =====
|
||||
!bblock
|
||||
We can also compute the residual iteratively as
|
||||
!bt
|
||||
\begin{equation*}
|
||||
\bm{r}_{k+1}=\bm{b}-\bm{A}\bm{x}_{k+1},
|
||||
\end{equation*}
|
||||
!et
|
||||
which equals
|
||||
!bt
|
||||
\begin{equation*}
|
||||
\bm{b}-\bm{A}(\bm{x}_k+\alpha_k\bm{p}_k),
|
||||
\end{equation*}
|
||||
!et
|
||||
or
|
||||
!bt
|
||||
\begin{equation*}
|
||||
(\bm{b}-\bm{A}\bm{x}_k)-\alpha_k\bm{A}\bm{p}_k,
|
||||
\end{equation*}
|
||||
!et
|
||||
which gives
|
||||
|
||||
!bt
|
||||
\begin{equation*}
|
||||
\bm{r}_{k+1}=\bm{r}_k-\bm{A}\bm{p}_{k},
|
||||
\end{equation*}
|
||||
!et
|
||||
!eblock
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
!split
|
||||
|
||||
Reference in New Issue
Block a user