import numpy as np class LogisticRegression: """ Logistic Regression for binary and multiclass classification. """ def __init__(self, lr=0.01, epochs=1000, fit_intercept=True, verbose=False): self.lr = lr # Learning rate for gradient descent self.epochs = epochs # Number of iterations self.fit_intercept = fit_intercept # Whether to add intercept (bias) self.verbose = verbose # Print loss during training if True self.weights = None self.multi_class = False # Will be determined at fit time def _add_intercept(self, X): """Add intercept term (column of ones) to feature matrix.""" intercept = np.ones((X.shape[0], 1)) return np.concatenate((intercept, X), axis=1) def _sigmoid(self, z): """Sigmoid function for binary logistic.""" return 1 / (1 + np.exp(-z)) def _softmax(self, Z): """Softmax function for multiclass logistic.""" exp_Z = np.exp(Z - np.max(Z, axis=1, keepdims=True)) return exp_Z / np.sum(exp_Z, axis=1, keepdims=True) def fit(self, X, y): """ Train the logistic regression model using gradient descent. Supports binary (sigmoid) and multiclass (softmax) based on y. """ X = np.array(X) y = np.array(y) n_samples, n_features = X.shape # Add intercept if needed if self.fit_intercept: X = self._add_intercept(X) n_features += 1 # Determine classes and mode (binary vs multiclass) unique_classes = np.unique(y) if len(unique_classes) > 2: self.multi_class = True else: self.multi_class = False # ----- Multiclass case ----- if self.multi_class: n_classes = len(unique_classes) # Map original labels to 0...n_classes-1 class_to_index = {c: idx for idx, c in enumerate(unique_classes)} y_indices = np.array([class_to_index[c] for c in y]) # Initialize weight matrix (features x classes) self.weights = np.zeros((n_features, n_classes)) # One-hot encode y Y_onehot = np.zeros((n_samples, n_classes)) Y_onehot[np.arange(n_samples), y_indices] = 1 # Gradient descent for epoch in range(self.epochs): scores = X.dot(self.weights) # Linear scores (n_samples x n_classes) probs = self._softmax(scores) # Probabilities (n_samples x n_classes) # Compute gradient (features x classes) gradient = (1 / n_samples) * X.T.dot(probs - Y_onehot) # Update weights self.weights -= self.lr * gradient if self.verbose and epoch % 100 == 0: # Compute current loss (categorical cross-entropy) loss = -np.sum(Y_onehot * np.log(probs + 1e-15)) / n_samples print(f"[Epoch {epoch}] Multiclass loss: {loss:.4f}") # ----- Binary case ----- else: # Convert y to 0/1 if not already if not np.array_equal(unique_classes, [0, 1]): # Map the two classes to 0 and 1 class0, class1 = unique_classes y_binary = np.where(y == class1, 1, 0) else: y_binary = y.copy().astype(int) # Initialize weights vector (features,) self.weights = np.zeros(n_features) # Gradient descent for epoch in range(self.epochs): linear_model = X.dot(self.weights) # (n_samples,) probs = self._sigmoid(linear_model) # (n_samples,) # Gradient for binary cross-entropy gradient = (1 / n_samples) * X.T.dot(probs - y_binary) self.weights -= self.lr * gradient if self.verbose and epoch % 100 == 0: # Compute binary cross-entropy loss loss = -np.mean( y_binary * np.log(probs + 1e-15) + (1 - y_binary) * np.log(1 - probs + 1e-15) ) print(f"[Epoch {epoch}] Binary loss: {loss:.4f}") def predict_prob(self, X): """ Compute probability estimates. Returns a 1D array for binary or a 2D array (n_samples x n_classes) for multiclass. """ X = np.array(X) # Add intercept if the model used it if self.fit_intercept: X = self._add_intercept(X) scores = X.dot(self.weights) if self.multi_class: return self._softmax(scores) else: return self._sigmoid(scores) def predict(self, X): """ Predict class labels for samples in X. Returns integer class labels (0,1 for binary, or 0...C-1 for multiclass). """ probs = self.predict_prob(X) if self.multi_class: # Choose class with highest probability return np.argmax(probs, axis=1) else: # Threshold at 0.5 for binary return (probs >= 0.5).astype(int) """ The class implements the sigmoid and softmax internally. During fit(), we check the number of classes: if more than 2, we set self.multi_class=True and perform multinomial logistic regression. We one-hot encode the target vector and update a weight matrix with softmax probabilities. Otherwise, we do standard binary logistic regression, converting labels to 0/1 if needed and updating a weight vector. In both cases we use batch gradient descent on the cross-entropy loss (we add a small epsilon 1e-15 to logs for numerical stability). Progress (loss) can be printed if verbose=True. """ # Evaluation Metrics #We define helper functions for accuracy and cross-entropy loss. Accuracy is the fraction of correct predictions . For loss, we compute the appropriate cross-entropy: def accuracy_score(y_true, y_pred): """Accuracy = (# correct predictions) / (total samples).""" y_true = np.array(y_true) y_pred = np.array(y_pred) return np.mean(y_true == y_pred) def binary_cross_entropy(y_true, y_prob): """ Binary cross-entropy loss. y_true: true binary labels (0 or 1), y_prob: predicted probabilities for class 1. """ y_true = np.array(y_true) y_prob = np.clip(np.array(y_prob), 1e-15, 1-1e-15) return -np.mean(y_true * np.log(y_prob) + (1 - y_true) * np.log(1 - y_prob)) def categorical_cross_entropy(y_true, y_prob): """ Categorical cross-entropy loss for multiclass. y_true: true labels (0...C-1), y_prob: array of predicted probabilities (n_samples x C). """ y_true = np.array(y_true, dtype=int) y_prob = np.clip(np.array(y_prob), 1e-15, 1-1e-15) # One-hot encode true labels n_samples, n_classes = y_prob.shape one_hot = np.zeros_like(y_prob) one_hot[np.arange(n_samples), y_true] = 1 # Compute cross-entropy loss_vec = -np.sum(one_hot * np.log(y_prob), axis=1) return np.mean(loss_vec) """ Synthetic data generation Binary classification data: Create two Gaussian clusters in 2D. For example, class 0 around mean [-2,-2] and class 1 around [2,2]. Multiclass data: Create several Gaussian clusters (one per class) spread out in feature space. """ import numpy as np def generate_binary_data(n_samples=100, n_features=2, random_state=None): """ Generate synthetic binary classification data. Returns (X, y) where X is (n_samples x n_features), y in {0,1}. """ rng = np.random.RandomState(random_state) # Half samples for class 0, half for class 1 n0 = n_samples // 2 n1 = n_samples - n0 # Class 0 around mean -2, class 1 around +2 mean0 = -2 * np.ones(n_features) mean1 = 2 * np.ones(n_features) X0 = rng.randn(n0, n_features) + mean0 X1 = rng.randn(n1, n_features) + mean1 X = np.vstack((X0, X1)) y = np.array([0]*n0 + [1]*n1) return X, y def generate_multiclass_data(n_samples=150, n_features=2, n_classes=3, random_state=None): """ Generate synthetic multiclass data with n_classes Gaussian clusters. """ rng = np.random.RandomState(random_state) X = [] y = [] samples_per_class = n_samples // n_classes for cls in range(n_classes): # Random cluster center for each class center = rng.uniform(-5, 5, size=n_features) Xi = rng.randn(samples_per_class, n_features) + center yi = [cls] * samples_per_class X.append(Xi) y.extend(yi) X = np.vstack(X) y = np.array(y) return X, y # Generate and test on binary data X_bin, y_bin = generate_binary_data(n_samples=200, n_features=2, random_state=42) model_bin = LogisticRegression(lr=0.1, epochs=1000) model_bin.fit(X_bin, y_bin) y_prob_bin = model_bin.predict_prob(X_bin) # probabilities for class 1 y_pred_bin = model_bin.predict(X_bin) # predicted classes 0 or 1 acc_bin = accuracy_score(y_bin, y_pred_bin) loss_bin = binary_cross_entropy(y_bin, y_prob_bin) print(f"Binary Classification - Accuracy: {acc_bin:.2f}, Cross-Entropy Loss: {loss_bin:.2f}") #For multiclass: # Generate and test on multiclass data X_multi, y_multi = generate_multiclass_data(n_samples=300, n_features=2, n_classes=3, random_state=1) model_multi = LogisticRegression(lr=0.1, epochs=1000) model_multi.fit(X_multi, y_multi) y_prob_multi = model_multi.predict_prob(X_multi) # (n_samples x 3) probabilities y_pred_multi = model_multi.predict(X_multi) # predicted labels 0,1,2 acc_multi = accuracy_score(y_multi, y_pred_multi) loss_multi = categorical_cross_entropy(y_multi, y_prob_multi) print(f"Multiclass Classification - Accuracy: {acc_multi:.2f}, Cross-Entropy Loss: {loss_multi:.2f}") # CSV Export import csv # Export binary results with open('binary_results.csv', mode='w', newline='') as f: writer = csv.writer(f) writer.writerow(["TrueLabel", "PredictedLabel"]) for true, pred in zip(y_bin, y_pred_bin): writer.writerow([true, pred]) # Export multiclass results with open('multiclass_results.csv', mode='w', newline='') as f: writer = csv.writer(f) writer.writerow(["TrueLabel", "PredictedLabel"]) for true, pred in zip(y_multi, y_pred_multi): writer.writerow([true, pred])