update week 39

doc/src/week39/DataFiles/chddata.csv

@@ -0,0 +1,100 @@
+1,    21,      1,         0
+2,    23,      1,         0
+3,    25,      1,         1
+4,    29,      1,         0
+5,    21,      1,         0
+6,    24,      1,         0
+7,    27,      1,         0
+8,    29,      1,         0
+9,    28,      1,         0
+10,   26,      1,         0
+11,   30,      2,         0
+12,   31,      2,         0 
+13,   31,      2,         0 
+14,   31,      2,         1
+15,   32,      2,         0 
+16,   34,      2,         0
+17,   34,      2,         0 
+18,   31,      2,         0 
+19,   32,      2,         0  
+20,   32,      2,         0 
+21,   33,      2,         0 
+22,   34,      2,         0 
+23,   31,      2,         1 
+24,   30,      2,         0 
+25,   33,      2,         0
+26,   36,      3,         1 
+27,   35,      3,         0 
+28,   35,      3,         0 
+29,   38,      3,         0 
+30,   37,      3,         1 
+31,   36,      3,         0 
+32,   35,      3,         0 
+33,   39,      3,         0 
+34,   39,      3,         0 
+35,   38,      3,         1 
+36,   37,      3,         0 
+37,   37,      3,         0 
+38,   40,      4,         0 
+39,   41,      4,         1
+40,   44,      4,         0 
+41,   44,      4,         0 
+42,   43,      4,         1 
+43,   42,      4,         0 
+44,   41,      4,         0 
+45,   40,      4,         1 
+46,   42,      4,         0 
+47,   42,      4,         0 
+48,   43,      4,         0 
+49,   44,      4,         1 
+50,   44,      4,         0 
+51,   42,      4,         0 
+52,   41,      4,         1 
+53,   45,      5,         0 
+54,   45,      5,         1 
+55,   49,      5,         0 
+56,   48,      5,         1 
+57,   47,      5,         0 
+58,   49,      5,         1 
+59,   46,      5,         1 
+60,   45,      5,         0 
+61,   49,      5,         1 
+62,   48,      5,         0 
+63,   47,      5,         1 
+64,   46,      5,         0 
+65,   47,      5,         0 
+66,   50,      6,         1 
+67,   51,      6,         1 
+68,   51,      6,         0 
+69,   54,      6,         1 
+70,   53,      6,         1 
+71,   51,      6,         0 
+72,   52,      6,         1 
+73,   54,      6,         0 
+74,   55,      7,         1 
+75,   56,      7,         1 
+76,   58,      7,         0 
+77,   59,      7,         1 
+78,   59,      7,         1 
+79,   58,      7,         0 
+80,   55,      7,         1 
+81,   56,      7,         1 
+82,   57,      7,         1 
+83,   58,      7,         1 
+84,   59,      7,         0 
+85,   55,      7,         1 
+86,   56,      7,         1 
+87,   57,      7,         1 
+88,   58,      7,         0 
+89,   59,      7,         1 
+90,   56,      7,         1 
+91,   60,      8,         1 
+92,   65,      8,         1 
+93,   67,      8,         1 
+94,   66,      8,         0 
+95,   63,      8,         1 
+96,   61,      8,         1 
+97,   69,      8,         1 
+98,   65,      8,         1 
+99,   64,      8,         1 
+100,  63,      8,        0 

doc/src/week39/binary_results.csv

@@ -0,0 +1,201 @@
+TrueLabel,PredictedLabel
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+1,1
+1,1
+1,1
+1,1
+1,1
+1,1
+1,1
+1,1
+1,1
+1,1
+1,1
+1,1
+1,1
+1,1
+1,1
+1,1
+1,1
+1,1
+1,1
+1,1
+1,1
+1,1
+1,1
+1,1
+1,1
+1,1
+1,1
+1,1
+1,1
+1,1
+1,1
+1,0
+1,1
+1,1
+1,1
+1,1
+1,1
+1,1
+1,1
+1,1
+1,1
+1,1
+1,1
+1,1
+1,1
+1,1
+1,1
+1,1
+1,1
+1,1
+1,1
+1,1
+1,1
+1,1
+1,1
+1,1
+1,1
+1,1
+1,1
+1,1
+1,1
+1,1
+1,1
+1,1
+1,1
+1,1
+1,1
+1,1
+1,1
+1,1
+1,1
+1,1
+1,1
+1,1
+1,1
+1,1
+1,1
+1,1
+1,1
+1,1
+1,1
+1,1
+1,1
+1,1
+1,1
+1,1
+1,1
+1,1
+1,1
+1,1
+1,1
+1,1
+1,1
+1,1
+1,1
+1,1
+1,1
+1,1
+1,1
+1,1

doc/src/week39/codes/logreg.py

@@ -0,0 +1,80 @@
+import numpy as np
+import pandas as pd
+
+# Load the dataset (ensure the CSV file is in the working directory)
+df = pd.read_csv('creditcard.csv')
+
+# Preprocess the data
+df = df.drop('ID', axis=1)  # drop ID column
+# Rename the target column for convenience (optional)
+df = df.rename(columns={'default payment next month': 'default'})
+# Separate features and target
+X = df.drop('default', axis=1).values  # features (shape: [30000, 23])
+y = df['default'].values              # target (shape: [30000,])
+
+# Standardize features (zero mean, unit std dev)
+X_mean = X.mean(axis=0)
+X_std = X.std(axis=0)
+X = (X - X_mean) / X_std
+
+# Split into training and test sets (80/20 split)
+m = X.shape[0]
+train_size = int(0.8 * m)
+indices = np.random.permutation(m)  # shuffle indices for randomness
+train_idx, test_idx = indices[:train_size], indices[train_size:]
+X_train, X_test = X[train_idx], X[test_idx]
+y_train, y_test = y[train_idx], y[test_idx]
+
+# Add intercept term (bias) to features by adding a column of 1s
+X_train = np.hstack([np.ones((X_train.shape[0], 1)), X_train])
+X_test  = np.hstack([np.ones((X_test.shape[0], 1)), X_test])
+
+# Initialize logistic regression parameters
+n_features = X_train.shape[1]  # number of features including bias
+theta = np.zeros(n_features)   # model weights (initialized to 0)
+
+# Set training hyperparameters
+learning_rate = 0.1
+epochs = 500
+
+# Training loop (gradient descent)
+m_train = X_train.shape[0]
+for epoch in range(epochs):
+    # Compute predictions (sigmoid of linear combination)
+    z = X_train.dot(theta)                   # linear combination
+    predictions = 1 / (1 + np.exp(-z))       # sigmoid function
+    
+    # Compute the gradient of loss w.r.t. theta
+    error = predictions - y_train            # vector of (pred - true) for each example
+    grad = (X_train.T.dot(error)) / m_train  # gradient vector
+    
+    # Update weights
+    theta -= learning_rate * grad
+    
+    # (Optional) compute and print loss every 50 epochs for monitoring
+    if epoch % 50 == 0:
+        # Binary cross-entropy loss
+        loss = -np.mean(y_train * np.log(predictions + 1e-15) + 
+                        (1 - y_train) * np.log(1 - predictions + 1e-15))
+        print(f"Epoch {epoch}: Training loss = {loss:.4f}")
+
+# 5. Evaluate the model on training and test data
+# Predict probabilities for test set and classify as 1 if sigmoid >= 0.5
+# (We can equivalently check linear term >= 0, since sigma(x)>=0.5 iff x>=0)
+train_prob = 1 / (1 + np.exp(-X_train.dot(theta)))
+test_prob  = 1 / (1 + np.exp(-X_test.dot(theta)))
+train_pred = (train_prob >= 0.5).astype(int)
+test_pred  = (test_prob  >= 0.5).astype(int)
+
+# Calculate accuracy
+train_accuracy = (train_pred == y_train).mean()
+test_accuracy = (test_pred == y_test).mean()
+
+# Calculate final loss on training set for reference
+final_loss = -np.mean(y_train * np.log(train_prob + 1e-15) + 
+                      (1 - y_train) * np.log(1 - train_prob + 1e-15))
+
+# Print performance metrics
+print(f"Final Training Loss: {final_loss:.4f}")
+print(f"Training Accuracy: {train_accuracy * 100:.2f}%")
+print(f"Test Accuracy: {test_accuracy * 100:.2f}%")

doc/src/week39/creditcard.do.txt

@@ -0,0 +1,163 @@
+TITLE: Week 40: Credit card example
+AUTHOR: Morten Hjorth-Jensen {copyright, 1999-present|CC BY-NC} at Department of Physics, University of Oslo
+DATE: Week 40
+
+
+!split
+===== Example case: Logistic Regression on Taiwan Credit Card Default Dataset =====
+
+
+
+=== Dataset Overview and Preparation ===
+
+
+The Default of Credit Card Clients dataset (Taiwan credit card default
+data) contains 30,000 instances and 23 features (plus an ID and a
+binary default indicator). The target variable is _default payment
+next month_ (1 = default, 0 = no default) . Key characteristics of the
+data include:
+
+o Instances: 30,000 card clients (each a unique ID) 
+o Features: 23 predictive features (mostly numeric, including demographics, credit limit, payment history, bill amounts, etc.) 
+o Target: Binary default indicator for the next month (Yes=1, No=0) 
+o Missing Values: None – the dataset has no missing values recorded 
+o Class Distribution: 22.1% of clients (6,636) defaulted vs 77.9% non-default  (i.e. the data is imbalanced about 1:3).
+o Basic Stats: Average credit limit is around NT$167,500, and mean age is ~35.5 years . Bill statement amounts are on the order of tens of thousands NT (mean ≈ 40k), while monthly payment amounts average a few thousand NT (mean ≈ 5k) .
+
+
+=== Feature details ===
+
+All features are numeric. Some are categorical in nature (encoded as
+integers): e.g. SEX (1=male, 2=female), EDUCATION (1=graduate,
+2=university, 3=high school, 4=others, with 5-6 as unknown) and
+MARRIAGE (1=married, 2=single, 3=others) . Others are continuous
+(credit amount, bill amounts, payments) or ordinal (e.g. $PAY_0 – PAY_6$
+repayment status, where -1 = pay duly, 1 = 1 month delay, 2 = 2 months
+delay, etc.) .
+
+=== Preprocessing steps ===
+
+For binary classification, we prepare the data as follows:
+
+o Remove the ID column (identifier not useful for prediction).
+o Use the 23 remaining features as $X$ and the _default payment next month_ as the binary target $y$.
+o The categorical features (SEX, EDUCATION, MARRIAGE) are already integer-encoded; for simplicity, we will use them as-is (one could one-hot encode these, but it’s not strictly necessary for logistic regression).
+o Standardize the feature values – i.e. scale each feature to zero mean and unit variance. This helps the gradient descent in logistic regression converge faster.
+o Split the dataset into a training set and testing set (we’ll use an 80/20 split). The model will be trained on the training set and evaluated on the test set to gauge performance on unseen data.
+
+
+!split
+===== Implementing Logistic Regression =====
+
+
+We will implement a logistic regression classifier using only NumPy,
+without any machine learning libraries. Logistic regression computes a
+weighted linear combination of features (plus a bias term) and applies
+a sigmoid function $\sigma(z) = 1/(1+e^{-z})$ to produce a probability
+between 0 and 1. We train the model by minimizing the logistic loss
+(binary cross-entropy) using gradient descent.
+
+Below is a single Python script that loads and preprocesses the data,
+then defines and trains the logistic regression model using gradient
+descent. It also outputs the model’s performance metrics (accuracy and
+loss). The file can be downloaded from the UCI database at URL:"https://archive.ics.uci.edu/dataset/350/default+of+credit+card+clients"
+
+!bc pycod
+import numpy as np
+import pandas as pd
+
+# Load the dataset (ensure the CSV file is in the working directory)
+df = pd.read_csv('credit_card_clients.csv')
+
+# Preprocess the data
+df = df.drop('ID', axis=1)  # drop ID column
+# Rename the target column for convenience (optional)
+df = df.rename(columns={'default payment next month': 'default'})
+# Separate features and target
+X = df.drop('default', axis=1).values  # features (shape: [30000, 23])
+y = df['default'].values              # target (shape: [30000,])
+
+# Standardize features (zero mean, unit std dev)
+X_mean = X.mean(axis=0)
+X_std = X.std(axis=0)
+X = (X - X_mean) / X_std
+
+# Split into training and test sets (80/20 split)
+m = X.shape[0]
+train_size = int(0.8 * m)
+indices = np.random.permutation(m)  # shuffle indices for randomness
+train_idx, test_idx = indices[:train_size], indices[train_size:]
+X_train, X_test = X[train_idx], X[test_idx]
+y_train, y_test = y[train_idx], y[test_idx]
+
+# Add intercept term (bias) to features by adding a column of 1s
+X_train = np.hstack([np.ones((X_train.shape[0], 1)), X_train])
+X_test  = np.hstack([np.ones((X_test.shape[0], 1)), X_test])
+
+# Initialize logistic regression parameters
+n_features = X_train.shape[1]  # number of features including bias
+theta = np.zeros(n_features)   # model weights (initialized to 0)
+
+# Set training hyperparameters
+learning_rate = 0.1
+epochs = 500
+
+# Training loop (gradient descent)
+m_train = X_train.shape[0]
+for epoch in range(epochs):
+    # Compute predictions (sigmoid of linear combination)
+    z = X_train.dot(theta)                   # linear combination
+    predictions = 1 / (1 + np.exp(-z))       # sigmoid function
+    
+    # Compute the gradient of loss w.r.t. theta
+    error = predictions - y_train            # vector of (pred - true) for each example
+    grad = (X_train.T.dot(error)) / m_train  # gradient vector
+    
+    # Update weights
+    theta -= learning_rate * grad
+    
+    # (Optional) compute and print loss every 50 epochs for monitoring
+    if epoch % 50 == 0:
+        # Binary cross-entropy loss
+        loss = -np.mean(y_train * np.log(predictions + 1e-15) + 
+                        (1 - y_train) * np.log(1 - predictions + 1e-15))
+        print(f"Epoch {epoch}: Training loss = {loss:.4f}")
+
+# 5. Evaluate the model on training and test data
+# Predict probabilities for test set and classify as 1 if sigmoid >= 0.5
+# (We can equivalently check linear term >= 0, since sigma(x)>=0.5 iff x>=0)
+train_prob = 1 / (1 + np.exp(-X_train.dot(theta)))
+test_prob  = 1 / (1 + np.exp(-X_test.dot(theta)))
+train_pred = (train_prob >= 0.5).astype(int)
+test_pred  = (test_prob  >= 0.5).astype(int)
+
+# Calculate accuracy
+train_accuracy = (train_pred == y_train).mean()
+test_accuracy = (test_pred == y_test).mean()
+
+# Calculate final loss on training set for reference
+final_loss = -np.mean(y_train * np.log(train_prob + 1e-15) + 
+                      (1 - y_train) * np.log(1 - train_prob + 1e-15))
+
+# Print performance metrics
+print(f"Final Training Loss: {final_loss:.4f}")
+print(f"Training Accuracy: {train_accuracy * 100:.2f}%")
+print(f"Test Accuracy: {test_accuracy * 100:.2f}%")
+!ec
+
+
+Explanation: We initialize the weight vector theta (including a bias term) to zeros. In each training epoch, we compute the logistic predictions for all training examples and then update the weights using the gradient of the cross-entropy loss. The learning rate and number of epochs are set to fixed values (which can be tuned). We periodically print the training loss to ensure the model is learning (decreasing loss). After training, we obtain predicted probabilities on the train and test sets, convert them to class labels (threshold 0.5), and compute the accuracy.
+
+!split
+===== Model performance and results =====
+
+
+After training, the code prints out the final loss and accuracies. You should observe that the training and test accuracy are around 78–80% for this logistic model. 
+
+This performance is in line with expectations. The dataset’s class imbalance means that always predicting “no default” would already achieve ~77.9% accuracy . Our logistic regression does slightly better, around 79–81% accuracy, by identifying some of the default cases correctly. (In one reference implementation, logistic regression achieved ~81.3% accuracy on a test set .)
+
+The model’s loss converges to a reasonable value, indicating the gradient descent optimization was successful.
+
+Note: The relatively modest improvement over the baseline is due to the imbalance and the limited complexity of a linear model. In practice, one could improve default prediction by using more complex models or addressing the imbalance (e.g. with oversampling or adjusting the classification threshold). Nonetheless, the above example demonstrates the complete process: data loading, preprocessing, model training, and evaluation, fulfilling the binary classification task on the Taiwan credit card default dataset.
+
+

doc/src/week39/multiclass_results.csv

@@ -0,0 +1,301 @@
+TrueLabel,PredictedLabel
+0,0
+0,1
+0,0
+0,0
+0,1
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,1
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,1
+0,0
+0,2
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+1,1
+1,1
+1,0
+1,1
+1,1
+1,1
+1,1
+1,1
+1,1
+1,1
+1,0
+1,1
+1,1
+1,1
+1,1
+1,1
+1,1
+1,1
+1,1
+1,1
+1,1
+1,1
+1,1
+1,1
+1,0
+1,1
+1,1
+1,1
+1,1
+1,1
+1,1
+1,1
+1,1
+1,1
+1,1
+1,1
+1,1
+1,1
+1,1
+1,1
+1,1
+1,1
+1,1
+1,1
+1,1
+1,1
+1,1
+1,1
+1,1
+1,1
+1,1
+1,1
+1,1
+1,1
+1,1
+1,1
+1,1
+1,1
+1,1
+1,1
+1,1
+1,1
+1,1
+1,1
+1,1
+1,1
+1,1
+1,1
+1,1
+1,1
+1,1
+1,1
+1,1
+1,1
+1,1
+1,1
+1,1
+1,1
+1,1
+1,1
+1,1
+1,1
+1,1
+1,1
+1,1
+1,1
+1,1
+1,1
+1,1
+1,1
+1,1
+1,1
+1,1
+1,0
+1,1
+1,1
+1,0
+1,1
+1,1
+1,1
+2,2
+2,2
+2,2
+2,2
+2,2
+2,2
+2,2
+2,2
+2,2
+2,2
+2,2
+2,2
+2,2
+2,2
+2,2
+2,2
+2,2
+2,2
+2,2
+2,2
+2,2
+2,2
+2,2
+2,2
+2,2
+2,2
+2,2
+2,2
+2,2
+2,2
+2,2
+2,2
+2,2
+2,2
+2,2
+2,2
+2,2
+2,2
+2,2
+2,2
+2,2
+2,2
+2,2
+2,2
+2,2
+2,2
+2,2
+2,2
+2,2
+2,2
+2,2
+2,2
+2,2
+2,2
+2,2
+2,2
+2,2
+2,2
+2,2
+2,2
+2,2
+2,2
+2,2
+2,2
+2,2
+2,2
+2,2
+2,2
+2,2
+2,2
+2,2
+2,2
+2,2
+2,2
+2,2
+2,2
+2,2
+2,2
+2,2
+2,2
+2,2
+2,2
+2,2
+2,2
+2,2
+2,2
+2,2
+2,2
+2,2
+2,2
+2,2
+2,2
+2,2
+2,2
+2,2
+2,2
+2,2
+2,2
+2,2
+2,2