79 KiB
79 KiB
In [1]:
import numpy as np
class LogisticRegression:
"""
Logistic Regression for binary and multiclass classification.
"""
def __init__(self, lr=0.01, epochs=1000, fit_intercept=True, verbose=False):
self.lr = lr # Learning rate for gradient descent
self.epochs = epochs # Number of iterations
self.fit_intercept = fit_intercept # Whether to add intercept (bias)
self.verbose = verbose # Print loss during training if True
self.weights = None
self.multi_class = False # Will be determined at fit time
def _add_intercept(self, X):
"""Add intercept term (column of ones) to feature matrix."""
intercept = np.ones((X.shape[0], 1))
return np.concatenate((intercept, X), axis=1)
def _sigmoid(self, z):
"""Sigmoid function for binary logistic."""
return 1 / (1 + np.exp(-z))
def _softmax(self, Z):
"""Softmax function for multiclass logistic."""
exp_Z = np.exp(Z - np.max(Z, axis=1, keepdims=True))
return exp_Z / np.sum(exp_Z, axis=1, keepdims=True)
def fit(self, X, y):
"""
Train the logistic regression model using gradient descent.
Supports binary (sigmoid) and multiclass (softmax) based on y.
"""
X = np.array(X)
y = np.array(y)
n_samples, n_features = X.shape
# Add intercept if needed
if self.fit_intercept:
X = self._add_intercept(X)
n_features += 1
# Determine classes and mode (binary vs multiclass)
unique_classes = np.unique(y)
if len(unique_classes) > 2:
self.multi_class = True
else:
self.multi_class = False
# ----- Multiclass case -----
if self.multi_class:
n_classes = len(unique_classes)
# Map original labels to 0...n_classes-1
class_to_index = {c: idx for idx, c in enumerate(unique_classes)}
y_indices = np.array([class_to_index[c] for c in y])
# Initialize weight matrix (features x classes)
self.weights = np.zeros((n_features, n_classes))
# One-hot encode y
Y_onehot = np.zeros((n_samples, n_classes))
Y_onehot[np.arange(n_samples), y_indices] = 1
# Gradient descent
for epoch in range(self.epochs):
scores = X.dot(self.weights) # Linear scores (n_samples x n_classes)
probs = self._softmax(scores) # Probabilities (n_samples x n_classes)
# Compute gradient (features x classes)
gradient = (1 / n_samples) * X.T.dot(probs - Y_onehot)
# Update weights
self.weights -= self.lr * gradient
if self.verbose and epoch % 100 == 0:
# Compute current loss (categorical cross-entropy)
loss = -np.sum(Y_onehot * np.log(probs + 1e-15)) / n_samples
print(f"[Epoch {epoch}] Multiclass loss: {loss:.4f}")
# ----- Binary case -----
else:
# Convert y to 0/1 if not already
if not np.array_equal(unique_classes, [0, 1]):
# Map the two classes to 0 and 1
class0, class1 = unique_classes
y_binary = np.where(y == class1, 1, 0)
else:
y_binary = y.copy().astype(int)
# Initialize weights vector (features,)
self.weights = np.zeros(n_features)
# Gradient descent
for epoch in range(self.epochs):
linear_model = X.dot(self.weights) # (n_samples,)
probs = self._sigmoid(linear_model) # (n_samples,)
# Gradient for binary cross-entropy
gradient = (1 / n_samples) * X.T.dot(probs - y_binary)
self.weights -= self.lr * gradient
if self.verbose and epoch % 100 == 0:
# Compute binary cross-entropy loss
loss = -np.mean(
y_binary * np.log(probs + 1e-15) +
(1 - y_binary) * np.log(1 - probs + 1e-15)
)
print(f"[Epoch {epoch}] Binary loss: {loss:.4f}")
def predict_prob(self, X):
"""
Compute probability estimates. Returns a 1D array for binary or
a 2D array (n_samples x n_classes) for multiclass.
"""
X = np.array(X)
# Add intercept if the model used it
if self.fit_intercept:
X = self._add_intercept(X)
scores = X.dot(self.weights)
if self.multi_class:
return self._softmax(scores)
else:
return self._sigmoid(scores)
def predict(self, X):
"""
Predict class labels for samples in X.
Returns integer class labels (0,1 for binary, or 0...C-1 for multiclass).
"""
probs = self.predict_prob(X)
if self.multi_class:
# Choose class with highest probability
return np.argmax(probs, axis=1)
else:
# Threshold at 0.5 for binary
return (probs >= 0.5).astype(int)In [2]:
# Evaluation Metrics
#We define helper functions for accuracy and cross-entropy loss. Accuracy is the fraction of correct predictions . For loss, we compute the appropriate cross-entropy:
def accuracy_score(y_true, y_pred):
"""Accuracy = (# correct predictions) / (total samples)."""
y_true = np.array(y_true)
y_pred = np.array(y_pred)
return np.mean(y_true == y_pred)
def binary_cross_entropy(y_true, y_prob):
"""
Binary cross-entropy loss.
y_true: true binary labels (0 or 1), y_prob: predicted probabilities for class 1.
"""
y_true = np.array(y_true)
y_prob = np.clip(np.array(y_prob), 1e-15, 1-1e-15)
return -np.mean(y_true * np.log(y_prob) + (1 - y_true) * np.log(1 - y_prob))
def categorical_cross_entropy(y_true, y_prob):
"""
Categorical cross-entropy loss for multiclass.
y_true: true labels (0...C-1), y_prob: array of predicted probabilities (n_samples x C).
"""
y_true = np.array(y_true, dtype=int)
y_prob = np.clip(np.array(y_prob), 1e-15, 1-1e-15)
# One-hot encode true labels
n_samples, n_classes = y_prob.shape
one_hot = np.zeros_like(y_prob)
one_hot[np.arange(n_samples), y_true] = 1
# Compute cross-entropy
loss_vec = -np.sum(one_hot * np.log(y_prob), axis=1)
return np.mean(loss_vec)In [3]:
import numpy as np
def generate_binary_data(n_samples=100, n_features=2, random_state=None):
"""
Generate synthetic binary classification data.
Returns (X, y) where X is (n_samples x n_features), y in {0,1}.
"""
rng = np.random.RandomState(random_state)
# Half samples for class 0, half for class 1
n0 = n_samples // 2
n1 = n_samples - n0
# Class 0 around mean -2, class 1 around +2
mean0 = -2 * np.ones(n_features)
mean1 = 2 * np.ones(n_features)
X0 = rng.randn(n0, n_features) + mean0
X1 = rng.randn(n1, n_features) + mean1
X = np.vstack((X0, X1))
y = np.array([0]*n0 + [1]*n1)
return X, y
def generate_multiclass_data(n_samples=150, n_features=2, n_classes=3, random_state=None):
"""
Generate synthetic multiclass data with n_classes Gaussian clusters.
"""
rng = np.random.RandomState(random_state)
X = []
y = []
samples_per_class = n_samples // n_classes
for cls in range(n_classes):
# Random cluster center for each class
center = rng.uniform(-5, 5, size=n_features)
Xi = rng.randn(samples_per_class, n_features) + center
yi = [cls] * samples_per_class
X.append(Xi)
y.extend(yi)
X = np.vstack(X)
y = np.array(y)
return X, y
# Generate and test on binary data
X_bin, y_bin = generate_binary_data(n_samples=200, n_features=2, random_state=42)
model_bin = LogisticRegression(lr=0.1, epochs=1000)
model_bin.fit(X_bin, y_bin)
y_prob_bin = model_bin.predict_prob(X_bin) # probabilities for class 1
y_pred_bin = model_bin.predict(X_bin) # predicted classes 0 or 1
acc_bin = accuracy_score(y_bin, y_pred_bin)
loss_bin = binary_cross_entropy(y_bin, y_prob_bin)
print(f"Binary Classification - Accuracy: {acc_bin:.2f}, Cross-Entropy Loss: {loss_bin:.2f}")
#For multiclass:
# Generate and test on multiclass data
X_multi, y_multi = generate_multiclass_data(n_samples=300, n_features=2, n_classes=3, random_state=1)
model_multi = LogisticRegression(lr=0.1, epochs=1000)
model_multi.fit(X_multi, y_multi)
y_prob_multi = model_multi.predict_prob(X_multi) # (n_samples x 3) probabilities
y_pred_multi = model_multi.predict(X_multi) # predicted labels 0,1,2
acc_multi = accuracy_score(y_multi, y_pred_multi)
loss_multi = categorical_cross_entropy(y_multi, y_prob_multi)
print(f"Multiclass Classification - Accuracy: {acc_multi:.2f}, Cross-Entropy Loss: {loss_multi:.2f}")
# CSV Export
import csv
# Export binary results
with open('binary_results.csv', mode='w', newline='') as f:
writer = csv.writer(f)
writer.writerow(["TrueLabel", "PredictedLabel"])
for true, pred in zip(y_bin, y_pred_bin):
writer.writerow([true, pred])
# Export multiclass results
with open('multiclass_results.csv', mode='w', newline='') as f:
writer = csv.writer(f)
writer.writerow(["TrueLabel", "PredictedLabel"])
for true, pred in zip(y_multi, y_pred_multi):
writer.writerow([true, pred])In [4]:
%matplotlib inline
import matplotlib.pyplot as plt
import numpy as np
from sklearn.model_selection import train_test_split
from sklearn.datasets import load_breast_cancer
from sklearn.linear_model import LogisticRegression
# Load the data
cancer = load_breast_cancer()
X_train, X_test, y_train, y_test = train_test_split(cancer.data,cancer.target,random_state=0)
print(X_train.shape)
print(X_test.shape)
# Logistic Regression
logreg = LogisticRegression(solver='lbfgs')
logreg.fit(X_train, y_train)
print("Test set accuracy with Logistic Regression: {:.2f}".format(logreg.score(X_test,y_test)))In [5]:
import matplotlib.pyplot as plt
import numpy as np
from sklearn.model_selection import train_test_split
from sklearn.datasets import load_breast_cancer
from sklearn.linear_model import LogisticRegression
cancer = load_breast_cancer()
import pandas as pd
# Making a data frame
cancerpd = pd.DataFrame(cancer.data, columns=cancer.feature_names)
fig, axes = plt.subplots(15,2,figsize=(10,20))
malignant = cancer.data[cancer.target == 0]
benign = cancer.data[cancer.target == 1]
ax = axes.ravel()
for i in range(30):
_, bins = np.histogram(cancer.data[:,i], bins =50)
ax[i].hist(malignant[:,i], bins = bins, alpha = 0.5)
ax[i].hist(benign[:,i], bins = bins, alpha = 0.5)
ax[i].set_title(cancer.feature_names[i])
ax[i].set_yticks(())
ax[0].set_xlabel("Feature magnitude")
ax[0].set_ylabel("Frequency")
ax[0].legend(["Malignant", "Benign"], loc ="best")
fig.tight_layout()
plt.show()
import seaborn as sns
correlation_matrix = cancerpd.corr().round(1)
# use the heatmap function from seaborn to plot the correlation matrix
# annot = True to print the values inside the square
plt.figure(figsize=(15,8))
sns.heatmap(data=correlation_matrix, annot=True)
plt.show()In [6]:
cancerpd = pd.DataFrame(cancer.data, columns=cancer.feature_names)In [7]:
correlation_matrix = cancerpd.corr().round(1)In [8]:
import matplotlib.pyplot as plt
import numpy as np
from sklearn.model_selection import train_test_split
from sklearn.datasets import load_breast_cancer
from sklearn.linear_model import LogisticRegression
# Load the data
cancer = load_breast_cancer()
X_train, X_test, y_train, y_test = train_test_split(cancer.data,cancer.target,random_state=0)
print(X_train.shape)
print(X_test.shape)
# Logistic Regression
logreg = LogisticRegression(solver='lbfgs')
logreg.fit(X_train, y_train)
from sklearn.preprocessing import LabelEncoder
from sklearn.model_selection import cross_validate
#Cross validation
accuracy = cross_validate(logreg,X_test,y_test,cv=10)['test_score']
print(accuracy)
print("Test set accuracy with Logistic Regression: {:.2f}".format(logreg.score(X_test,y_test)))
import scikitplot as skplt
y_pred = logreg.predict(X_test)
skplt.metrics.plot_confusion_matrix(y_test, y_pred, normalize=True)
plt.show()
y_probas = logreg.predict_proba(X_test)
skplt.metrics.plot_roc(y_test, y_probas)
plt.show()
skplt.metrics.plot_cumulative_gain(y_test, y_probas)
plt.show()Illustration of a single perceptron model and a multi-perceptron model
Figure 1: In a) we show a single perceptron model while in b) we dispay a network with two hidden layers, an input layer and an output layer.
In [9]:
"""
Simple code that tests XOR, OR and AND gates with linear regression
"""
import numpy as np
# Design matrix
X = np.array([ [1, 0, 0], [1, 0, 1], [1, 1, 0],[1, 1, 1]],dtype=np.float64)
print(f"The X.TX matrix:{X.T @ X}")
Xinv = np.linalg.pinv(X.T @ X)
print(f"The invers of X.TX matrix:{Xinv}")
# The XOR gate
yXOR = np.array( [ 0, 1 ,1, 0])
ThetaXOR = Xinv @ X.T @ yXOR
print(f"The values of theta for the XOR gate:{ThetaXOR}")
print(f"The linear regression prediction for the XOR gate:{X @ ThetaXOR}")
# The OR gate
yOR = np.array( [ 0, 1 ,1, 1])
ThetaOR = Xinv @ X.T @ yOR
print(f"The values of theta for the OR gate:{ThetaOR}")
print(f"The linear regression prediction for the OR gate:{X @ ThetaOR}")
# The OR gate
yAND = np.array( [ 0, 0 ,0, 1])
ThetaAND = Xinv @ X.T @ yAND
print(f"The values of theta for the AND gate:{ThetaAND}")
print(f"The linear regression prediction for the AND gate:{X @ ThetaAND}")In [10]:
"""
Simple code that tests XOR and OR gates with linear regression
and logistic regression
"""
import matplotlib.pyplot as plt
from sklearn.linear_model import LogisticRegression
import numpy as np
# Design matrix
X = np.array([ [1, 0, 0], [1, 0, 1], [1, 1, 0],[1, 1, 1]],dtype=np.float64)
print(f"The X.TX matrix:{X.T @ X}")
Xinv = np.linalg.pinv(X.T @ X)
print(f"The invers of X.TX matrix:{Xinv}")
# The XOR gate
yXOR = np.array( [ 0, 1 ,1, 0])
ThetaXOR = Xinv @ X.T @ yXOR
print(f"The values of theta for the XOR gate:{ThetaXOR}")
print(f"The linear regression prediction for the XOR gate:{X @ ThetaXOR}")
# The OR gate
yOR = np.array( [ 0, 1 ,1, 1])
ThetaOR = Xinv @ X.T @ yOR
print(f"The values of theta for the OR gate:{ThetaOR}")
print(f"The linear regression prediction for the OR gate:{X @ ThetaOR}")
# The OR gate
yAND = np.array( [ 0, 0 ,0, 1])
ThetaAND = Xinv @ X.T @ yAND
print(f"The values of theta for the AND gate:{ThetaAND}")
print(f"The linear regression prediction for the AND gate:{X @ ThetaAND}")
# Now we change to logistic regression
# Logistic Regression
logreg = LogisticRegression()
logreg.fit(X, yOR)
print("Test set accuracy with Logistic Regression for OR gate: {:.2f}".format(logreg.score(X,yOR)))
logreg.fit(X, yXOR)
print("Test set accuracy with Logistic Regression for XOR gate: {:.2f}".format(logreg.score(X,yXOR)))
logreg.fit(X, yAND)
print("Test set accuracy with Logistic Regression for AND gate: {:.2f}".format(logreg.score(X,yAND)))In [11]:
# and now neural networks with Scikit-Learn and the XOR
from sklearn.neural_network import MLPClassifier
from sklearn.datasets import make_classification
X, yXOR = make_classification(n_samples=100, random_state=1)
FFNN = MLPClassifier(random_state=1, max_iter=300).fit(X, yXOR)
FFNN.predict_proba(X)
print(f"Test set accuracy with Feed Forward Neural Network for XOR gate:{FFNN.score(X, yXOR)}")Warning:
Output truncated. This notebook contains too many cells to display efficiently.