+In addition to the plot of the features, we study now also the covariance (or rather the correlation matrix). +We use also Pandas to compute the correlation matrix.
@@ -152,16 +155,19 @@ MathJax.Hub.Config({
from sklearn.datasets import load_breast_cancer
from sklearn.linear_model import LogisticRegression
cancer = load_breast_cancer()
+import pandas as pd
+# Making a data frame
+cancerpd = pd.DataFrame(cancer.data, columns=cancer.feature_names)
fig, axes = plt.subplots(15,2,figsize=(10,20))
-male = cancer.data[cancer.target == 0]
-bene = cancer.data[cancer.target == 1]
+malignant = cancer.data[cancer.target == 0]
+benign = cancer.data[cancer.target == 1]
ax = axes.ravel()
for i in range(30):
_, bins = np.histogram(cancer.data[:,i], bins =50)
- ax[i].hist(male[:,i], bins = bins, alpha = 0.5)
- ax[i].hist(bene[:,i], bins = bins, alpha = 0.5)
+ ax[i].hist(malignant[:,i], bins = bins, alpha = 0.5)
+ ax[i].hist(benign[:,i], bins = bins, alpha = 0.5)
ax[i].set_title(cancer.feature_names[i])
ax[i].set_yticks(())
ax[0].set_xlabel("Feature magnitude")
@@ -170,19 +176,27 @@ ax[0].le
fig.tight_layout()
plt.show()
-# Set up training data
+import seaborn as sns
+correlation_matrix = cancerpd.corr().round(1)
+# use the heatmap function from seaborn to plot the correlation matrix
+# annot = True to print the values inside the square
+sns.heatmap(data=correlation_matrix, annot=True)
+plt.show()
+#split into train and test and then scale thereafter
X_train, X_test, y_train, y_test = train_test_split(cancer.data,cancer.target,random_state=0)
-# Perform Logistic Regression
+print(X_train.shape)
+print(X_test.shape)
+
logreg = LogisticRegression()
logreg.fit(X_train, y_train)
-print("Test set accuracy: {:.2f}".format(logreg.score(X_test,y_test)))
+print("Test set accuracy from Logistic Regression: {:.2f}".format(logreg.score(X_test,y_test)))
-# Scale data
-from sklearn.preprocessing import StandardScaler
+from sklearn.preprocessing import MinMaxScaler, StandardScaler
scaler = StandardScaler()
scaler.fit(X_train)
X_train_scaled = scaler.transform(X_train)
X_test_scaled = scaler.transform(X_test)
+
logreg.fit(X_train_scaled, y_train)
print("Test set accuracy scaled data: {:.2f}".format(logreg.score(X_test_scaled,y_test)))
diff --git a/doc/pub/DimRed/html/DimRed-reveal.html b/doc/pub/DimRed/html/DimRed-reveal.html
index b6b87f510..c8b2d434c 100644
--- a/doc/pub/DimRed/html/DimRed-reveal.html
+++ b/doc/pub/DimRed/html/DimRed-reveal.html
@@ -420,6 +420,9 @@ logreg.fit(X_train_scaled, y_train)
+In addition to the plot of the features, we study now also the covariance (or rather the correlation matrix).
+We use also Pandas to compute the correlation matrix.
@@ -429,16 +432,19 @@ logreg.fit(X_train_scaled, y_train)
from sklearn.datasets import load_breast_cancer
from sklearn.linear_model import LogisticRegression
cancer = load_breast_cancer()
+import pandas as pd
+# Making a data frame
+cancerpd = pd.DataFrame(cancer.data, columns=cancer.feature_names)
fig, axes = plt.subplots(15,2,figsize=(10,20))
-male = cancer.data[cancer.target == 0]
-bene = cancer.data[cancer.target == 1]
+malignant = cancer.data[cancer.target == 0]
+benign = cancer.data[cancer.target == 1]
ax = axes.ravel()
for i in range(30):
_, bins = np.histogram(cancer.data[:,i], bins =50)
- ax[i].hist(male[:,i], bins = bins, alpha = 0.5)
- ax[i].hist(bene[:,i], bins = bins, alpha = 0.5)
+ ax[i].hist(malignant[:,i], bins = bins, alpha = 0.5)
+ ax[i].hist(benign[:,i], bins = bins, alpha = 0.5)
ax[i].set_title(cancer.feature_names[i])
ax[i].set_yticks(())
ax[0].set_xlabel("Feature magnitude")
@@ -447,19 +453,27 @@ ax[0].legend([&
fig.tight_layout()
plt.show()
-# Set up training data
+import seaborn as sns
+correlation_matrix = cancerpd.corr().round(1)
+# use the heatmap function from seaborn to plot the correlation matrix
+# annot = True to print the values inside the square
+sns.heatmap(data=correlation_matrix, annot=True)
+plt.show()
+#split into train and test and then scale thereafter
X_train, X_test, y_train, y_test = train_test_split(cancer.data,cancer.target,random_state=0)
-# Perform Logistic Regression
+print(X_train.shape)
+print(X_test.shape)
+
logreg = LogisticRegression()
logreg.fit(X_train, y_train)
-print("Test set accuracy: {:.2f}".format(logreg.score(X_test,y_test)))
+print("Test set accuracy from Logistic Regression: {:.2f}".format(logreg.score(X_test,y_test)))
-# Scale data
-from sklearn.preprocessing import StandardScaler
+from sklearn.preprocessing import MinMaxScaler, StandardScaler
scaler = StandardScaler()
scaler.fit(X_train)
X_train_scaled = scaler.transform(X_train)
X_test_scaled = scaler.transform(X_test)
+
logreg.fit(X_train_scaled, y_train)
print("Test set accuracy scaled data: {:.2f}".format(logreg.score(X_test_scaled,y_test)))
diff --git a/doc/pub/DimRed/html/DimRed-solarized.html b/doc/pub/DimRed/html/DimRed-solarized.html
index d722569c4..bca2cf49d 100644
--- a/doc/pub/DimRed/html/DimRed-solarized.html
+++ b/doc/pub/DimRed/html/DimRed-solarized.html
@@ -403,6 +403,9 @@ logreg.fit(X_train_scaled, y_train)
+In addition to the plot of the features, we study now also the covariance (or rather the correlation matrix).
+We use also Pandas to compute the correlation matrix.
@@ -412,16 +415,19 @@ logreg.fit(X_train_scaled, y_train)
from sklearn.datasets import load_breast_cancer
from sklearn.linear_model import LogisticRegression
cancer = load_breast_cancer()
+import pandas as pd
+# Making a data frame
+cancerpd = pd.DataFrame(cancer.data, columns=cancer.feature_names)
fig, axes = plt.subplots(15,2,figsize=(10,20))
-male = cancer.data[cancer.target == 0]
-bene = cancer.data[cancer.target == 1]
+malignant = cancer.data[cancer.target == 0]
+benign = cancer.data[cancer.target == 1]
ax = axes.ravel()
for i in range(30):
_, bins = np.histogram(cancer.data[:,i], bins =50)
- ax[i].hist(male[:,i], bins = bins, alpha = 0.5)
- ax[i].hist(bene[:,i], bins = bins, alpha = 0.5)
+ ax[i].hist(malignant[:,i], bins = bins, alpha = 0.5)
+ ax[i].hist(benign[:,i], bins = bins, alpha = 0.5)
ax[i].set_title(cancer.feature_names[i])
ax[i].set_yticks(())
ax[0].set_xlabel("Feature magnitude")
@@ -430,19 +436,27 @@ ax[0].legend([&
fig.tight_layout()
plt.show()
-# Set up training data
+import seaborn as sns
+correlation_matrix = cancerpd.corr().round(1)
+# use the heatmap function from seaborn to plot the correlation matrix
+# annot = True to print the values inside the square
+sns.heatmap(data=correlation_matrix, annot=True)
+plt.show()
+#split into train and test and then scale thereafter
X_train, X_test, y_train, y_test = train_test_split(cancer.data,cancer.target,random_state=0)
-# Perform Logistic Regression
+print(X_train.shape)
+print(X_test.shape)
+
logreg = LogisticRegression()
logreg.fit(X_train, y_train)
-print("Test set accuracy: {:.2f}".format(logreg.score(X_test,y_test)))
+print("Test set accuracy from Logistic Regression: {:.2f}".format(logreg.score(X_test,y_test)))
-# Scale data
-from sklearn.preprocessing import StandardScaler
+from sklearn.preprocessing import MinMaxScaler, StandardScaler
scaler = StandardScaler()
scaler.fit(X_train)
X_train_scaled = scaler.transform(X_train)
X_test_scaled = scaler.transform(X_test)
+
logreg.fit(X_train_scaled, y_train)
print("Test set accuracy scaled data: {:.2f}".format(logreg.score(X_test_scaled,y_test)))
diff --git a/doc/pub/DimRed/html/DimRed.html b/doc/pub/DimRed/html/DimRed.html
index aa6733c50..53607192d 100644
--- a/doc/pub/DimRed/html/DimRed.html
+++ b/doc/pub/DimRed/html/DimRed.html
@@ -408,6 +408,9 @@ logreg.fit(X_train_scaled, y_train)
+In addition to the plot of the features, we study now also the covariance (or rather the correlation matrix).
+We use also Pandas to compute the correlation matrix.
@@ -417,16 +420,19 @@ logreg.fit(X_train_scaled, y_train)
from sklearn.datasets import load_breast_cancer
from sklearn.linear_model import LogisticRegression
cancer = load_breast_cancer()
+import pandas as pd
+# Making a data frame
+cancerpd = pd.DataFrame(cancer.data, columns=cancer.feature_names)
fig, axes = plt.subplots(15,2,figsize=(10,20))
-male = cancer.data[cancer.target == 0]
-bene = cancer.data[cancer.target == 1]
+malignant = cancer.data[cancer.target == 0]
+benign = cancer.data[cancer.target == 1]
ax = axes.ravel()
for i in range(30):
_, bins = np.histogram(cancer.data[:,i], bins =50)
- ax[i].hist(male[:,i], bins = bins, alpha = 0.5)
- ax[i].hist(bene[:,i], bins = bins, alpha = 0.5)
+ ax[i].hist(malignant[:,i], bins = bins, alpha = 0.5)
+ ax[i].hist(benign[:,i], bins = bins, alpha = 0.5)
ax[i].set_title(cancer.feature_names[i])
ax[i].set_yticks(())
ax[0].set_xlabel("Feature magnitude")
@@ -435,19 +441,27 @@ ax[0].le
fig.tight_layout()
plt.show()
-# Set up training data
+import seaborn as sns
+correlation_matrix = cancerpd.corr().round(1)
+# use the heatmap function from seaborn to plot the correlation matrix
+# annot = True to print the values inside the square
+sns.heatmap(data=correlation_matrix, annot=True)
+plt.show()
+#split into train and test and then scale thereafter
X_train, X_test, y_train, y_test = train_test_split(cancer.data,cancer.target,random_state=0)
-# Perform Logistic Regression
+print(X_train.shape)
+print(X_test.shape)
+
logreg = LogisticRegression()
logreg.fit(X_train, y_train)
-print("Test set accuracy: {:.2f}".format(logreg.score(X_test,y_test)))
+print("Test set accuracy from Logistic Regression: {:.2f}".format(logreg.score(X_test,y_test)))
-# Scale data
-from sklearn.preprocessing import StandardScaler
+from sklearn.preprocessing import MinMaxScaler, StandardScaler
scaler = StandardScaler()
scaler.fit(X_train)
X_train_scaled = scaler.transform(X_train)
X_test_scaled = scaler.transform(X_test)
+
logreg.fit(X_train_scaled, y_train)
print("Test set accuracy scaled data: {:.2f}".format(logreg.score(X_test_scaled,y_test)))
diff --git a/doc/pub/DimRed/ipynb/DimRed.ipynb b/doc/pub/DimRed/ipynb/DimRed.ipynb
index c77b0241e..8bf7e3023 100644
--- a/doc/pub/DimRed/ipynb/DimRed.ipynb
+++ b/doc/pub/DimRed/ipynb/DimRed.ipynb
@@ -283,7 +283,10 @@
"cell_type": "markdown",
"metadata": {},
"source": [
- "## Why should we think of reducing the dimensionality"
+ "## Why should we think of reducing the dimensionality\n",
+ "\n",
+ "In addition to the plot of the features, we study now also the covariance (or rather the correlation matrix).\n",
+ "We use also **Pandas** to compute the correlation matrix."
]
},
{
@@ -300,16 +303,19 @@
"from sklearn.datasets import load_breast_cancer\n",
"from sklearn.linear_model import LogisticRegression\n",
"cancer = load_breast_cancer()\n",
+ "import pandas as pd\n",
+ "# Making a data frame\n",
+ "cancerpd = pd.DataFrame(cancer.data, columns=cancer.feature_names)\n",
"\n",
"fig, axes = plt.subplots(15,2,figsize=(10,20))\n",
- "male = cancer.data[cancer.target == 0]\n",
- "bene = cancer.data[cancer.target == 1]\n",
+ "malignant = cancer.data[cancer.target == 0]\n",
+ "benign = cancer.data[cancer.target == 1]\n",
"ax = axes.ravel()\n",
"\n",
"for i in range(30):\n",
" _, bins = np.histogram(cancer.data[:,i], bins =50)\n",
- " ax[i].hist(male[:,i], bins = bins, alpha = 0.5)\n",
- " ax[i].hist(bene[:,i], bins = bins, alpha = 0.5)\n",
+ " ax[i].hist(malignant[:,i], bins = bins, alpha = 0.5)\n",
+ " ax[i].hist(benign[:,i], bins = bins, alpha = 0.5)\n",
" ax[i].set_title(cancer.feature_names[i])\n",
" ax[i].set_yticks(())\n",
"ax[0].set_xlabel(\"Feature magnitude\")\n",
@@ -318,19 +324,27 @@
"fig.tight_layout()\n",
"plt.show()\n",
"\n",
- "# Set up training data\n",
+ "import seaborn as sns\n",
+ "correlation_matrix = cancerpd.corr().round(1)\n",
+ "# use the heatmap function from seaborn to plot the correlation matrix\n",
+ "# annot = True to print the values inside the square\n",
+ "sns.heatmap(data=correlation_matrix, annot=True)\n",
+ "plt.show()\n",
+ "#split into train and test and then scale thereafter\n",
"X_train, X_test, y_train, y_test = train_test_split(cancer.data,cancer.target,random_state=0)\n",
- "# Perform Logistic Regression \n",
+ "print(X_train.shape)\n",
+ "print(X_test.shape)\n",
+ "\n",
"logreg = LogisticRegression()\n",
"logreg.fit(X_train, y_train)\n",
- "print(\"Test set accuracy: {:.2f}\".format(logreg.score(X_test,y_test)))\n",
+ "print(\"Test set accuracy from Logistic Regression: {:.2f}\".format(logreg.score(X_test,y_test)))\n",
"\n",
- "# Scale data\n",
- "from sklearn.preprocessing import StandardScaler\n",
+ "from sklearn.preprocessing import MinMaxScaler, StandardScaler\n",
"scaler = StandardScaler()\n",
"scaler.fit(X_train)\n",
"X_train_scaled = scaler.transform(X_train)\n",
"X_test_scaled = scaler.transform(X_test)\n",
+ "\n",
"logreg.fit(X_train_scaled, y_train)\n",
"print(\"Test set accuracy scaled data: {:.2f}\".format(logreg.score(X_test_scaled,y_test)))"
]
diff --git a/doc/pub/DimRed/ipynb/ipynb-DimRed-src.tar.gz b/doc/pub/DimRed/ipynb/ipynb-DimRed-src.tar.gz
index 1c12d631a..927742fa6 100644
Binary files a/doc/pub/DimRed/ipynb/ipynb-DimRed-src.tar.gz and b/doc/pub/DimRed/ipynb/ipynb-DimRed-src.tar.gz differ
diff --git a/doc/pub/DimRed/pdf/DimRed-minted.pdf b/doc/pub/DimRed/pdf/DimRed-minted.pdf
index 35d6907b0..906b00f9f 100644
Binary files a/doc/pub/DimRed/pdf/DimRed-minted.pdf and b/doc/pub/DimRed/pdf/DimRed-minted.pdf differ
diff --git a/doc/src/DimRed/DimRed.do.txt b/doc/src/DimRed/DimRed.do.txt
index 4fa0e2360..26cdddd72 100644
--- a/doc/src/DimRed/DimRed.do.txt
+++ b/doc/src/DimRed/DimRed.do.txt
@@ -251,6 +251,8 @@ print("Test set accuracy scaled data: {:.2f}".format(logreg.score(X_test_scaled,
!split
===== Why should we think of reducing the dimensionality =====
+In addition to the plot of the features, we study now also the covariance (or rather the correlation matrix).
+We use also _Pandas_ to compute the correlation matrix.
!bc pycod
import matplotlib.pyplot as plt
import numpy as np
@@ -258,16 +260,19 @@ from sklearn.model_selection import train_test_split
from sklearn.datasets import load_breast_cancer
from sklearn.linear_model import LogisticRegression
cancer = load_breast_cancer()
+import pandas as pd
+# Making a data frame
+cancerpd = pd.DataFrame(cancer.data, columns=cancer.feature_names)
fig, axes = plt.subplots(15,2,figsize=(10,20))
-male = cancer.data[cancer.target == 0]
-bene = cancer.data[cancer.target == 1]
+malignant = cancer.data[cancer.target == 0]
+benign = cancer.data[cancer.target == 1]
ax = axes.ravel()
for i in range(30):
_, bins = np.histogram(cancer.data[:,i], bins =50)
- ax[i].hist(male[:,i], bins = bins, alpha = 0.5)
- ax[i].hist(bene[:,i], bins = bins, alpha = 0.5)
+ ax[i].hist(malignant[:,i], bins = bins, alpha = 0.5)
+ ax[i].hist(benign[:,i], bins = bins, alpha = 0.5)
ax[i].set_title(cancer.feature_names[i])
ax[i].set_yticks(())
ax[0].set_xlabel("Feature magnitude")
@@ -276,22 +281,34 @@ ax[0].legend(["Malignant", "Benign"], loc ="best")
fig.tight_layout()
plt.show()
-# Set up training data
+import seaborn as sns
+correlation_matrix = cancerpd.corr().round(1)
+# use the heatmap function from seaborn to plot the correlation matrix
+# annot = True to print the values inside the square
+sns.heatmap(data=correlation_matrix, annot=True)
+plt.show()
+#split into train and test and then scale thereafter
X_train, X_test, y_train, y_test = train_test_split(cancer.data,cancer.target,random_state=0)
-# Perform Logistic Regression
+print(X_train.shape)
+print(X_test.shape)
+
logreg = LogisticRegression()
logreg.fit(X_train, y_train)
-print("Test set accuracy: {:.2f}".format(logreg.score(X_test,y_test)))
+print("Test set accuracy from Logistic Regression: {:.2f}".format(logreg.score(X_test,y_test)))
-# Scale data
-from sklearn.preprocessing import StandardScaler
+from sklearn.preprocessing import MinMaxScaler, StandardScaler
scaler = StandardScaler()
scaler.fit(X_train)
X_train_scaled = scaler.transform(X_train)
X_test_scaled = scaler.transform(X_test)
+
logreg.fit(X_train_scaled, y_train)
print("Test set accuracy scaled data: {:.2f}".format(logreg.score(X_test_scaled,y_test)))
+
+
+
+
!ec
diff --git a/doc/src/DimRed/PCAcancer.py b/doc/src/DimRed/PCAcancer.py
new file mode 100644
index 000000000..b4983907e
--- /dev/null
+++ b/doc/src/DimRed/PCAcancer.py
@@ -0,0 +1,53 @@
+import matplotlib.pyplot as plt
+import numpy as np
+from sklearn.model_selection import train_test_split
+from sklearn.datasets import load_breast_cancer
+from sklearn.linear_model import LogisticRegression
+cancer = load_breast_cancer()
+import pandas as pd
+
+cancerpd = pd.DataFrame(cancer.data, columns=cancer.feature_names)
+
+fig, axes = plt.subplots(15,2,figsize=(10,20))
+malignant = cancer.data[cancer.target == 0]
+benign = cancer.data[cancer.target == 1]
+ax = axes.ravel()
+
+for i in range(30):
+ _, bins = np.histogram(cancer.data[:,i], bins =50)
+ ax[i].hist(malignant[:,i], bins = bins, alpha = 0.5)
+ ax[i].hist(benign[:,i], bins = bins, alpha = 0.5)
+ ax[i].set_title(cancer.feature_names[i])
+ ax[i].set_yticks(())
+ax[0].set_xlabel("Feature magnitude")
+ax[0].set_ylabel("Frequency")
+ax[0].legend(["Malignant", "Benign"], loc ="best")
+fig.tight_layout()
+plt.show()
+
+import seaborn as sns
+correlation_matrix = cancerpd.corr().round(1)
+# use the heatmap function from seaborn to plot the correlation matrix
+# annot = True to print the values inside the square
+sns.heatmap(data=correlation_matrix, annot=True)
+plt.show()
+X_train, X_test, y_train, y_test = train_test_split(cancer.data,cancer.target,random_state=0)
+print(X_train.shape)
+print(X_test.shape)
+
+logreg = LogisticRegression()
+logreg.fit(X_train, y_train)
+print("Test set accuracy from Logistic Regression: {:.2f}".format(logreg.score(X_test,y_test)))
+
+from sklearn.preprocessing import MinMaxScaler, StandardScaler
+scaler = StandardScaler()
+scaler.fit(X_train)
+X_train_scaled = scaler.transform(X_train)
+X_test_scaled = scaler.transform(X_test)
+
+logreg.fit(X_train_scaled, y_train)
+print("Test set accuracy scaled data: {:.2f}".format(logreg.score(X_test_scaled,y_test)))
+
+
+
+
Why should we think of reducing the dimensionality
+Why should we think of reducing the dimensionality
+Why should we think of reducing the dimensionality
+