Start coding of solutions

- Implement OLS and Ridge as well as python structure - Start with gradient descent - Create plots for the above mentioned - Create a basic structure for the python source - Install pre commit hooks to check and format the code
2025-09-08 13:28:06 +02:00
parent 2395d64ca8
commit 1a0324d473
15 changed files with 785 additions and 0 deletions
@@ -0,0 +1,82 @@
+import numpy as np
+from sklearn.preprocessing import StandardScaler  # type: ignore
+from sklearn.metrics import mean_squared_error, r2_score  # type: ignore
+
+
+def polynomial_features(x: np.ndarray, p: int, intercept: bool = True) -> np.ndarray:
+    """Generates a design matrix with polynomial features up to degree p.
+    Args:
+        x: The input data vector of shape (n_samples,).
+        p: The maximum polynomial degree.
+        intercept: Whether to include the intercept term (degree 0).
+    Returns:
+        The design matrix of shape (n_samples, p + 1) if intercept is True,
+        else (n_samples, p).
+    """
+    if intercept:
+        P = np.arange(p + 1)
+    else:
+        P = np.arange(1, p + 1)
+    X = np.power(x[:, np.newaxis], P)
+    return X
+
+
+def runge_function(x: np.ndarray) -> np.ndarray:
+    """Computes the Runge function values for the input x.
+    Args:
+        x: The input data vector of shape (n_samples,).
+    Returns:
+        The Runge function values of shape (n_samples,).
+    """
+    return 1 / (1 + 25 * np.square(x))
+
+
+def scale_data(
+    X: np.ndarray, X_test: np.ndarray | None = None
+) -> tuple[np.ndarray, np.ndarray | None]:
+    """Scales the input data using StandardScaler.
+    Args:
+        X: The training data matrix of shape (n_samples, n_features).
+        X_test: The test data matrix of shape (n_samples_test, n_features) or None.
+    Returns:
+        A tuple containing the scaled training data and the scaled test data (or None if X_test is None).
+    """
+    scaler = StandardScaler()
+    single_feature = False
+    if len(X.shape) == 1:
+        X = X.reshape(-1, 1)
+        single_feature = True
+    if X_test is not None and single_feature:
+        X_test = X_test.reshape(-1, 1)
+    X_scaled = scaler.fit_transform(X)
+    if single_feature:
+        X_scaled = X_scaled.ravel()
+    if X_test is not None:
+        X_test_scaled = scaler.transform(X_test)
+        if single_feature:
+            X_test_scaled = X_test_scaled.ravel()
+        return X_scaled, X_test_scaled
+    return X_scaled, None
+
+
+def noise_data(y: np.ndarray, noise_level: float = 1.0) -> np.ndarray:
+    """Adds Gaussian noise to the target data.
+    Args:
+        y: The target vector of shape (n_samples,).
+        noise_level: The standard deviation of the Gaussian noise.
+    Returns:
+        The noisy target vector of shape (n_samples,).
+    """
+    noise = np.random.normal(0, noise_level, size=y.shape)
+    return y + noise
+
+
+def evaluate_model(y_true: np.ndarray, y_pred: np.ndarray) -> tuple[float, float]:
+    """Evaluates the model using Mean Squared Error (MSE) and R^2 score.
+    Args:
+        y_true: The true target vector of shape (n_samples,).
+        y_pred: The predicted target vector of shape (n_samples,).
+    Returns:
+        A tuple containing the MSE and R^2 score.
+    """
+    return mean_squared_error(y_true, y_pred), r2_score(y_true, y_pred)
@@ -0,0 +1,60 @@
+import numpy as np
+
+
+def Ridge_parameters(X: np.ndarray, y: np.ndarray, lam: float) -> np.ndarray:
+    """Computes the Ridge regression parameters.
+    Args:
+        X: The input data matrix of shape (n_samples, n_features).
+        y: The target vector of shape (n_samples,).
+        lam: The regularization parameter (lambda).
+    Returns:
+        The Ridge regression parameters of shape (n_features,).
+    """
+    # Assumes X is scaled and has no intercept column
+    return np.linalg.inv(X.T @ X + lam * np.eye(X.shape[1])) @ X.T @ y
+
+
+def OLS_parameters(X: np.ndarray, y: np.ndarray) -> np.ndarray:
+    """Computes the Ordinary Least Squares (OLS) regression parameters.
+    Args:
+        X: The input data matrix of shape (n_samples, n_features).
+        y: The target vector of shape (n_samples,).
+    Returns:
+        The OLS regression parameters of shape (n_features,).
+    """
+    return Ridge_parameters(X, y, lam=0.0)  # OLS is Ridge with lambda=0
+
+
+def gradient_descent(
+    X, y, cost_func, grad_cost_func, eta=0.1, num_iters=1000, **kwargs
+):
+    # Initialize weights
+    theta = np.zeros(X.shape[1])
+    # Store cost history
+    cost_history = np.zeros(num_iters)
+    for t in range(num_iters):
+        # Compute cost
+        cost_history[t] = cost_func(X, y, theta, **kwargs)
+        # Compute gradient
+        grad = grad_cost_func(X, y, theta, **kwargs)
+        # Update weights
+        theta -= eta * grad
+    return theta, cost_history
+
+
+def OLS_cost_func(X, y, theta):
+    error = (X @ theta) - y
+    return 0.5 * np.mean(error**2)
+
+
+def OLS_grad_cost_func(X, y, theta):
+    error = (X @ theta) - y
+    return (X.T @ error) / len(y)
+
+
+def Ridge_cost_func(X, y, theta, lam):
+    return OLS_cost_func(X, y, theta) + 0.5 * lam * np.sum(np.square(theta))
+
+
+def Ridge_grad_cost_func(X, y, theta, lam):
+    return OLS_grad_cost_func(X, y, theta) + lam * theta
@@ -0,0 +1,133 @@
+import matplotlib.pyplot as plt
+from matplotlib.colors import SymLogNorm
+import numpy as np
+
+FIG_WIDTH = 6
+
+
+def get_rc_params():
+    colors = ["FF220C", "70D6FF", "8AAA79", "666370", "1C1F33"]
+    rcParams = plt.rcParams
+    # Use LaTeX for rendering
+    # Setup fonts
+    rcParams["text.usetex"] = True
+    rcParams["font.family"] = "serif"
+    rcParams["font.size"] = 12
+    rcParams["axes.labelsize"] = 12
+    rcParams["axes.titlesize"] = 12
+    rcParams["legend.fontsize"] = 10
+    rcParams["xtick.labelsize"] = 10
+    rcParams["ytick.labelsize"] = 10
+    # Figure size and resolution
+    rcParams["figure.figsize"] = (FIG_WIDTH, 4)
+    rcParams["figure.dpi"] = 300
+    # Use colors from the palette
+    rcParams["axes.prop_cycle"] = plt.cycler(color=[f"#{color}" for color in colors])
+    # Grid
+    rcParams["axes.grid"] = True
+    rcParams["grid.alpha"] = 0.5
+    rcParams["grid.linestyle"] = "--"
+    # Point ticks to the inside of the axes
+    rcParams["xtick.direction"] = "in"
+    rcParams["ytick.direction"] = "in"
+    rcParams["xtick.top"] = True
+    rcParams["ytick.right"] = True
+    return rcParams
+
+
+def set_rc_params():
+    plt.rcParams.update(get_rc_params())
+
+
+set_rc_params()
+
+
+def get_figsize(rel_height: float = 2 / 3) -> tuple[float, float]:
+    """Returns a figure size tuple based on the global FIG_WIDTH and a relative height.
+    Args:
+        rel_height: The relative height of the figure compared to FIG_WIDTH.
+    Returns:
+        A tuple (width, height) for the figure size.
+    """
+    return (FIG_WIDTH, FIG_WIDTH * rel_height)
+
+
+def mse_r2_plot(
+    polynomial_degrees: np.ndarray,
+    train_mse_list: list[float],
+    mse_list: list[float],
+    train_r2_list: list[float],
+    r2_list: list[float],
+    labels: dict[str, str] | None = None,
+) -> tuple[plt.Figure, tuple[plt.Axes, plt.Axes]]:
+    std_labels = {
+        "set1": "Train Set",
+        "set2": "Test Set",
+        "xlabel": "Polynomial Degree",
+        "ylabel1": "Mean Squared Error",
+        "ylabel2": "$R^2$ Score",
+    }
+
+    if labels is None:
+        labels = std_labels
+    else:
+        for key in std_labels:
+            if key not in labels:
+                labels[key] = std_labels[key]
+
+    fig, (ax1, ax2) = plt.subplots(1, 2, figsize=get_figsize(0.5))
+    ax1.plot(polynomial_degrees, train_mse_list, marker="o", label=labels["set1"])
+    ax1.plot(polynomial_degrees, mse_list, marker="o", label=labels["set2"])
+    ax1.legend()
+    ax1.set_xlabel(labels["xlabel"])
+    ax1.set_ylabel(labels["ylabel1"])
+
+    ax2.plot(polynomial_degrees, train_r2_list, marker="o", label=labels["set1"])
+    ax2.plot(polynomial_degrees, r2_list, marker="o", label=labels["set2"])
+    ax2.legend()
+    ax2.set_xlabel(labels["xlabel"])
+    ax2.set_ylabel(labels["ylabel2"])
+
+    fig.tight_layout()
+    return fig, (ax1, ax2)
+
+
+def parameter_plot(
+    polynomial_degrees: np.ndarray,
+    beta_OLS_list: list[np.ndarray],
+    labels: dict[str, str] | None = None,
+) -> tuple[plt.Figure, plt.Axes]:
+    std_labels = {
+        "xlabel": "Polynomial Degree of Fit",
+        "ylabel": "Coefficient Index $i$",
+        "cbar": "Coefficient Value",
+    }
+
+    if labels is None:
+        labels = std_labels
+    else:
+        for key in std_labels:
+            if key not in labels:
+                labels[key] = std_labels[key]
+
+    fig, ax = plt.subplots(figsize=get_figsize(0.5))
+    beta_OLS = np.zeros((beta_OLS_list[-1].shape[0], len(polynomial_degrees)))
+    beta_OLS[:] = np.nan
+    for i in range(len(beta_OLS_list)):
+        for j in range(len(beta_OLS_list[i])):
+            beta_OLS[j, i] = beta_OLS_list[i][j]
+    cmap = plt.get_cmap("coolwarm")
+    norm = SymLogNorm(
+        linthresh=1e-3, vmin=np.nanmin(beta_OLS), vmax=np.nanmax(beta_OLS)
+    )
+    im = ax.imshow(beta_OLS, aspect="auto", cmap=cmap, origin="lower", norm=norm)
+    cbar = fig.colorbar(im, ax=ax)
+    cbar.set_label(labels["cbar"])
+    ax.set_xlabel(labels["xlabel"])
+    ax.set_ylabel(labels["ylabel"])
+    ax.set_xticks(np.arange(0, len(polynomial_degrees), step=3))
+    ax.set_xticklabels(polynomial_degrees[::3])
+    ax.set_yticks(np.arange(0, beta_OLS.shape[0], 5))
+    ax.set_yticklabels(np.arange(1, beta_OLS.shape[0] + 1, step=5))
+    fig.tight_layout()
+    return fig, ax