Start coding of solutions

- Implement OLS and Ridge as well as python structure - Start with gradient descent - Create plots for the above mentioned - Create a basic structure for the python source - Install pre commit hooks to check and format the code
2025-09-08 13:28:06 +02:00
parent 2395d64ca8
commit 1a0324d473
15 changed files with 785 additions and 0 deletions
@@ -0,0 +1,24 @@
+repos:
+  # Ruff via uv
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.6.9  # pick the latest release
+    hooks:
+      - id: ruff
+        args: [--fix]        # optional: auto-fix issues
+      - id: ruff-format      # black-compatible formatter
+
+  # Mypy via uv
+  - repo: local
+    hooks:
+      - id: mypy
+        name: mypy (via uv)
+        entry: uv run mypy
+        language: system
+        types: [python]
+        pass_filenames: false
+        args:
+          - --allow-redefinition
+          - --disable-error-code=import-untyped
+          - --ignore-missing-imports
+          - .
+
@@ -7,8 +7,12 @@ requires-python = ">=3.13"
 dependencies = [
    "ipykernel>=6.30.1",
    "matplotlib>=3.10.6",
+    "mypy>=1.17.1",
    "numpy>=2.3.2",
    "pandas>=2.3.2",
    "scikit-learn>=1.7.1",
    "scipy>=1.16.1",
 ]
+
+[tool.mypy]
+mypy_path = "src"
@@ -0,0 +1,82 @@
+import numpy as np
+from sklearn.preprocessing import StandardScaler  # type: ignore
+from sklearn.metrics import mean_squared_error, r2_score  # type: ignore
+
+
+def polynomial_features(x: np.ndarray, p: int, intercept: bool = True) -> np.ndarray:
+    """Generates a design matrix with polynomial features up to degree p.
+    Args:
+        x: The input data vector of shape (n_samples,).
+        p: The maximum polynomial degree.
+        intercept: Whether to include the intercept term (degree 0).
+    Returns:
+        The design matrix of shape (n_samples, p + 1) if intercept is True,
+        else (n_samples, p).
+    """
+    if intercept:
+        P = np.arange(p + 1)
+    else:
+        P = np.arange(1, p + 1)
+    X = np.power(x[:, np.newaxis], P)
+    return X
+
+
+def runge_function(x: np.ndarray) -> np.ndarray:
+    """Computes the Runge function values for the input x.
+    Args:
+        x: The input data vector of shape (n_samples,).
+    Returns:
+        The Runge function values of shape (n_samples,).
+    """
+    return 1 / (1 + 25 * np.square(x))
+
+
+def scale_data(
+    X: np.ndarray, X_test: np.ndarray | None = None
+) -> tuple[np.ndarray, np.ndarray | None]:
+    """Scales the input data using StandardScaler.
+    Args:
+        X: The training data matrix of shape (n_samples, n_features).
+        X_test: The test data matrix of shape (n_samples_test, n_features) or None.
+    Returns:
+        A tuple containing the scaled training data and the scaled test data (or None if X_test is None).
+    """
+    scaler = StandardScaler()
+    single_feature = False
+    if len(X.shape) == 1:
+        X = X.reshape(-1, 1)
+        single_feature = True
+    if X_test is not None and single_feature:
+        X_test = X_test.reshape(-1, 1)
+    X_scaled = scaler.fit_transform(X)
+    if single_feature:
+        X_scaled = X_scaled.ravel()
+    if X_test is not None:
+        X_test_scaled = scaler.transform(X_test)
+        if single_feature:
+            X_test_scaled = X_test_scaled.ravel()
+        return X_scaled, X_test_scaled
+    return X_scaled, None
+
+
+def noise_data(y: np.ndarray, noise_level: float = 1.0) -> np.ndarray:
+    """Adds Gaussian noise to the target data.
+    Args:
+        y: The target vector of shape (n_samples,).
+        noise_level: The standard deviation of the Gaussian noise.
+    Returns:
+        The noisy target vector of shape (n_samples,).
+    """
+    noise = np.random.normal(0, noise_level, size=y.shape)
+    return y + noise
+
+
+def evaluate_model(y_true: np.ndarray, y_pred: np.ndarray) -> tuple[float, float]:
+    """Evaluates the model using Mean Squared Error (MSE) and R^2 score.
+    Args:
+        y_true: The true target vector of shape (n_samples,).
+        y_pred: The predicted target vector of shape (n_samples,).
+    Returns:
+        A tuple containing the MSE and R^2 score.
+    """
+    return mean_squared_error(y_true, y_pred), r2_score(y_true, y_pred)
@@ -0,0 +1,60 @@
+import numpy as np
+
+
+def Ridge_parameters(X: np.ndarray, y: np.ndarray, lam: float) -> np.ndarray:
+    """Computes the Ridge regression parameters.
+    Args:
+        X: The input data matrix of shape (n_samples, n_features).
+        y: The target vector of shape (n_samples,).
+        lam: The regularization parameter (lambda).
+    Returns:
+        The Ridge regression parameters of shape (n_features,).
+    """
+    # Assumes X is scaled and has no intercept column
+    return np.linalg.inv(X.T @ X + lam * np.eye(X.shape[1])) @ X.T @ y
+
+
+def OLS_parameters(X: np.ndarray, y: np.ndarray) -> np.ndarray:
+    """Computes the Ordinary Least Squares (OLS) regression parameters.
+    Args:
+        X: The input data matrix of shape (n_samples, n_features).
+        y: The target vector of shape (n_samples,).
+    Returns:
+        The OLS regression parameters of shape (n_features,).
+    """
+    return Ridge_parameters(X, y, lam=0.0)  # OLS is Ridge with lambda=0
+
+
+def gradient_descent(
+    X, y, cost_func, grad_cost_func, eta=0.1, num_iters=1000, **kwargs
+):
+    # Initialize weights
+    theta = np.zeros(X.shape[1])
+    # Store cost history
+    cost_history = np.zeros(num_iters)
+    for t in range(num_iters):
+        # Compute cost
+        cost_history[t] = cost_func(X, y, theta, **kwargs)
+        # Compute gradient
+        grad = grad_cost_func(X, y, theta, **kwargs)
+        # Update weights
+        theta -= eta * grad
+    return theta, cost_history
+
+
+def OLS_cost_func(X, y, theta):
+    error = (X @ theta) - y
+    return 0.5 * np.mean(error**2)
+
+
+def OLS_grad_cost_func(X, y, theta):
+    error = (X @ theta) - y
+    return (X.T @ error) / len(y)
+
+
+def Ridge_cost_func(X, y, theta, lam):
+    return OLS_cost_func(X, y, theta) + 0.5 * lam * np.sum(np.square(theta))
+
+
+def Ridge_grad_cost_func(X, y, theta, lam):
+    return OLS_grad_cost_func(X, y, theta) + lam * theta
@@ -0,0 +1,133 @@
+import matplotlib.pyplot as plt
+from matplotlib.colors import SymLogNorm
+import numpy as np
+
+FIG_WIDTH = 6
+
+
+def get_rc_params():
+    colors = ["FF220C", "70D6FF", "8AAA79", "666370", "1C1F33"]
+    rcParams = plt.rcParams
+    # Use LaTeX for rendering
+    # Setup fonts
+    rcParams["text.usetex"] = True
+    rcParams["font.family"] = "serif"
+    rcParams["font.size"] = 12
+    rcParams["axes.labelsize"] = 12
+    rcParams["axes.titlesize"] = 12
+    rcParams["legend.fontsize"] = 10
+    rcParams["xtick.labelsize"] = 10
+    rcParams["ytick.labelsize"] = 10
+    # Figure size and resolution
+    rcParams["figure.figsize"] = (FIG_WIDTH, 4)
+    rcParams["figure.dpi"] = 300
+    # Use colors from the palette
+    rcParams["axes.prop_cycle"] = plt.cycler(color=[f"#{color}" for color in colors])
+    # Grid
+    rcParams["axes.grid"] = True
+    rcParams["grid.alpha"] = 0.5
+    rcParams["grid.linestyle"] = "--"
+    # Point ticks to the inside of the axes
+    rcParams["xtick.direction"] = "in"
+    rcParams["ytick.direction"] = "in"
+    rcParams["xtick.top"] = True
+    rcParams["ytick.right"] = True
+    return rcParams
+
+
+def set_rc_params():
+    plt.rcParams.update(get_rc_params())
+
+
+set_rc_params()
+
+
+def get_figsize(rel_height: float = 2 / 3) -> tuple[float, float]:
+    """Returns a figure size tuple based on the global FIG_WIDTH and a relative height.
+    Args:
+        rel_height: The relative height of the figure compared to FIG_WIDTH.
+    Returns:
+        A tuple (width, height) for the figure size.
+    """
+    return (FIG_WIDTH, FIG_WIDTH * rel_height)
+
+
+def mse_r2_plot(
+    polynomial_degrees: np.ndarray,
+    train_mse_list: list[float],
+    mse_list: list[float],
+    train_r2_list: list[float],
+    r2_list: list[float],
+    labels: dict[str, str] | None = None,
+) -> tuple[plt.Figure, tuple[plt.Axes, plt.Axes]]:
+    std_labels = {
+        "set1": "Train Set",
+        "set2": "Test Set",
+        "xlabel": "Polynomial Degree",
+        "ylabel1": "Mean Squared Error",
+        "ylabel2": "$R^2$ Score",
+    }
+
+    if labels is None:
+        labels = std_labels
+    else:
+        for key in std_labels:
+            if key not in labels:
+                labels[key] = std_labels[key]
+
+    fig, (ax1, ax2) = plt.subplots(1, 2, figsize=get_figsize(0.5))
+    ax1.plot(polynomial_degrees, train_mse_list, marker="o", label=labels["set1"])
+    ax1.plot(polynomial_degrees, mse_list, marker="o", label=labels["set2"])
+    ax1.legend()
+    ax1.set_xlabel(labels["xlabel"])
+    ax1.set_ylabel(labels["ylabel1"])
+
+    ax2.plot(polynomial_degrees, train_r2_list, marker="o", label=labels["set1"])
+    ax2.plot(polynomial_degrees, r2_list, marker="o", label=labels["set2"])
+    ax2.legend()
+    ax2.set_xlabel(labels["xlabel"])
+    ax2.set_ylabel(labels["ylabel2"])
+
+    fig.tight_layout()
+    return fig, (ax1, ax2)
+
+
+def parameter_plot(
+    polynomial_degrees: np.ndarray,
+    beta_OLS_list: list[np.ndarray],
+    labels: dict[str, str] | None = None,
+) -> tuple[plt.Figure, plt.Axes]:
+    std_labels = {
+        "xlabel": "Polynomial Degree of Fit",
+        "ylabel": "Coefficient Index $i$",
+        "cbar": "Coefficient Value",
+    }
+
+    if labels is None:
+        labels = std_labels
+    else:
+        for key in std_labels:
+            if key not in labels:
+                labels[key] = std_labels[key]
+
+    fig, ax = plt.subplots(figsize=get_figsize(0.5))
+    beta_OLS = np.zeros((beta_OLS_list[-1].shape[0], len(polynomial_degrees)))
+    beta_OLS[:] = np.nan
+    for i in range(len(beta_OLS_list)):
+        for j in range(len(beta_OLS_list[i])):
+            beta_OLS[j, i] = beta_OLS_list[i][j]
+    cmap = plt.get_cmap("coolwarm")
+    norm = SymLogNorm(
+        linthresh=1e-3, vmin=np.nanmin(beta_OLS), vmax=np.nanmax(beta_OLS)
+    )
+    im = ax.imshow(beta_OLS, aspect="auto", cmap=cmap, origin="lower", norm=norm)
+    cbar = fig.colorbar(im, ax=ax)
+    cbar.set_label(labels["cbar"])
+    ax.set_xlabel(labels["xlabel"])
+    ax.set_ylabel(labels["ylabel"])
+    ax.set_xticks(np.arange(0, len(polynomial_degrees), step=3))
+    ax.set_xticklabels(polynomial_degrees[::3])
+    ax.set_yticks(np.arange(0, beta_OLS.shape[0], 5))
+    ax.set_yticklabels(np.arange(1, beta_OLS.shape[0] + 1, step=5))
+    fig.tight_layout()
+    return fig, ax
@@ -414,6 +414,41 @@ wheels = [
    { url = "https://files.pythonhosted.org/packages/8f/8e/9ad090d3553c280a8060fbf6e24dc1c0c29704ee7d1c372f0c174aa59285/matplotlib_inline-0.1.7-py3-none-any.whl", hash = "sha256:df192d39a4ff8f21b1895d72e6a13f5fcc5099f00fa84384e0ea28c2cc0653ca", size = 9899, upload-time = "2024-04-15T13:44:43.265Z" },
 ]

+[[package]]
+name = "mypy"
+version = "1.17.1"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "mypy-extensions" },
+    { name = "pathspec" },
+    { name = "typing-extensions" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/8e/22/ea637422dedf0bf36f3ef238eab4e455e2a0dcc3082b5cc067615347ab8e/mypy-1.17.1.tar.gz", hash = "sha256:25e01ec741ab5bb3eec8ba9cdb0f769230368a22c959c4937360efb89b7e9f01", size = 3352570, upload-time = "2025-07-31T07:54:19.204Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/5b/82/aec2fc9b9b149f372850291827537a508d6c4d3664b1750a324b91f71355/mypy-1.17.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:93378d3203a5c0800c6b6d850ad2f19f7a3cdf1a3701d3416dbf128805c6a6a7", size = 11075338, upload-time = "2025-07-31T07:53:38.873Z" },
+    { url = "https://files.pythonhosted.org/packages/07/ac/ee93fbde9d2242657128af8c86f5d917cd2887584cf948a8e3663d0cd737/mypy-1.17.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:15d54056f7fe7a826d897789f53dd6377ec2ea8ba6f776dc83c2902b899fee81", size = 10113066, upload-time = "2025-07-31T07:54:14.707Z" },
+    { url = "https://files.pythonhosted.org/packages/5a/68/946a1e0be93f17f7caa56c45844ec691ca153ee8b62f21eddda336a2d203/mypy-1.17.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:209a58fed9987eccc20f2ca94afe7257a8f46eb5df1fb69958650973230f91e6", size = 11875473, upload-time = "2025-07-31T07:53:14.504Z" },
+    { url = "https://files.pythonhosted.org/packages/9f/0f/478b4dce1cb4f43cf0f0d00fba3030b21ca04a01b74d1cd272a528cf446f/mypy-1.17.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:099b9a5da47de9e2cb5165e581f158e854d9e19d2e96b6698c0d64de911dd849", size = 12744296, upload-time = "2025-07-31T07:53:03.896Z" },
+    { url = "https://files.pythonhosted.org/packages/ca/70/afa5850176379d1b303f992a828de95fc14487429a7139a4e0bdd17a8279/mypy-1.17.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:fa6ffadfbe6994d724c5a1bb6123a7d27dd68fc9c059561cd33b664a79578e14", size = 12914657, upload-time = "2025-07-31T07:54:08.576Z" },
+    { url = "https://files.pythonhosted.org/packages/53/f9/4a83e1c856a3d9c8f6edaa4749a4864ee98486e9b9dbfbc93842891029c2/mypy-1.17.1-cp313-cp313-win_amd64.whl", hash = "sha256:9a2b7d9180aed171f033c9f2fc6c204c1245cf60b0cb61cf2e7acc24eea78e0a", size = 9593320, upload-time = "2025-07-31T07:53:01.341Z" },
+    { url = "https://files.pythonhosted.org/packages/38/56/79c2fac86da57c7d8c48622a05873eaab40b905096c33597462713f5af90/mypy-1.17.1-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:15a83369400454c41ed3a118e0cc58bd8123921a602f385cb6d6ea5df050c733", size = 11040037, upload-time = "2025-07-31T07:54:10.942Z" },
+    { url = "https://files.pythonhosted.org/packages/4d/c3/adabe6ff53638e3cad19e3547268482408323b1e68bf082c9119000cd049/mypy-1.17.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:55b918670f692fc9fba55c3298d8a3beae295c5cded0a55dccdc5bbead814acd", size = 10131550, upload-time = "2025-07-31T07:53:41.307Z" },
+    { url = "https://files.pythonhosted.org/packages/b8/c5/2e234c22c3bdeb23a7817af57a58865a39753bde52c74e2c661ee0cfc640/mypy-1.17.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:62761474061feef6f720149d7ba876122007ddc64adff5ba6f374fda35a018a0", size = 11872963, upload-time = "2025-07-31T07:53:16.878Z" },
+    { url = "https://files.pythonhosted.org/packages/ab/26/c13c130f35ca8caa5f2ceab68a247775648fdcd6c9a18f158825f2bc2410/mypy-1.17.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c49562d3d908fd49ed0938e5423daed8d407774a479b595b143a3d7f87cdae6a", size = 12710189, upload-time = "2025-07-31T07:54:01.962Z" },
+    { url = "https://files.pythonhosted.org/packages/82/df/c7d79d09f6de8383fe800521d066d877e54d30b4fb94281c262be2df84ef/mypy-1.17.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:397fba5d7616a5bc60b45c7ed204717eaddc38f826e3645402c426057ead9a91", size = 12900322, upload-time = "2025-07-31T07:53:10.551Z" },
+    { url = "https://files.pythonhosted.org/packages/b8/98/3d5a48978b4f708c55ae832619addc66d677f6dc59f3ebad71bae8285ca6/mypy-1.17.1-cp314-cp314-win_amd64.whl", hash = "sha256:9d6b20b97d373f41617bd0708fd46aa656059af57f2ef72aa8c7d6a2b73b74ed", size = 9751879, upload-time = "2025-07-31T07:52:56.683Z" },
+    { url = "https://files.pythonhosted.org/packages/1d/f3/8fcd2af0f5b806f6cf463efaffd3c9548a28f84220493ecd38d127b6b66d/mypy-1.17.1-py3-none-any.whl", hash = "sha256:a9f52c0351c21fe24c21d8c0eb1f62967b262d6729393397b6f443c3b773c3b9", size = 2283411, upload-time = "2025-07-31T07:53:24.664Z" },
+]
+
+[[package]]
+name = "mypy-extensions"
+version = "1.1.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/a2/6e/371856a3fb9d31ca8dac321cda606860fa4548858c0cc45d9d1d4ca2628b/mypy_extensions-1.1.0.tar.gz", hash = "sha256:52e68efc3284861e772bbcd66823fde5ae21fd2fdb51c62a211403730b916558", size = 6343, upload-time = "2025-04-22T14:54:24.164Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/79/7b/2c79738432f5c924bef5071f933bcc9efd0473bac3b4aa584a6f7c1c8df8/mypy_extensions-1.1.0-py3-none-any.whl", hash = "sha256:1be4cccdb0f2482337c4743e60421de3a356cd97508abadd57d47403e94f5505", size = 4963, upload-time = "2025-04-22T14:54:22.983Z" },
+]
+
 [[package]]
 name = "nest-asyncio"
 version = "1.6.0"
@@ -520,6 +555,15 @@ wheels = [
    { url = "https://files.pythonhosted.org/packages/16/32/f8e3c85d1d5250232a5d3477a2a28cc291968ff175caeadaf3cc19ce0e4a/parso-0.8.5-py2.py3-none-any.whl", hash = "sha256:646204b5ee239c396d040b90f9e272e9a8017c630092bf59980beb62fd033887", size = 106668, upload-time = "2025-08-23T15:15:25.663Z" },
 ]

+[[package]]
+name = "pathspec"
+version = "0.12.1"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/ca/bc/f35b8446f4531a7cb215605d100cd88b7ac6f44ab3fc94870c120ab3adbf/pathspec-0.12.1.tar.gz", hash = "sha256:a482d51503a1ab33b1c67a6c3813a26953dbdc71c31dacaef9a838c4e29f5712", size = 51043, upload-time = "2023-12-10T22:30:45Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/cc/20/ff623b09d963f88bfde16306a54e12ee5ea43e9b597108672ff3a408aad6/pathspec-0.12.1-py3-none-any.whl", hash = "sha256:a0d503e138a4c123b27490a4f7beda6a01c6f288df0e4a8b79c7eb0dc7b4cc08", size = 31191, upload-time = "2023-12-10T22:30:43.14Z" },
+]
+
 [[package]]
 name = "pexpect"
 version = "4.9.0"
@@ -603,6 +647,7 @@ source = { virtual = "." }
 dependencies = [
    { name = "ipykernel" },
    { name = "matplotlib" },
+    { name = "mypy" },
    { name = "numpy" },
    { name = "pandas" },
    { name = "scikit-learn" },
@@ -613,6 +658,7 @@ dependencies = [
 requires-dist = [
    { name = "ipykernel", specifier = ">=6.30.1" },
    { name = "matplotlib", specifier = ">=3.10.6" },
+    { name = "mypy", specifier = ">=1.17.1" },
    { name = "numpy", specifier = ">=2.3.2" },
    { name = "pandas", specifier = ">=2.3.2" },
    { name = "scikit-learn", specifier = ">=1.7.1" },
@@ -899,6 +945,15 @@ wheels = [
    { url = "https://files.pythonhosted.org/packages/00/c0/8f5d070730d7836adc9c9b6408dec68c6ced86b304a9b26a14df072a6e8c/traitlets-5.14.3-py3-none-any.whl", hash = "sha256:b74e89e397b1ed28cc831db7aea759ba6640cb3de13090ca145426688ff1ac4f", size = 85359, upload-time = "2024-04-19T11:11:46.763Z" },
 ]

+[[package]]
+name = "typing-extensions"
+version = "4.15.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/72/94/1a15dd82efb362ac84269196e94cf00f187f7ed21c242792a923cdb1c61f/typing_extensions-4.15.0.tar.gz", hash = "sha256:0cea48d173cc12fa28ecabc3b837ea3cf6f38c6d1136f85cbaaf598984861466", size = 109391, upload-time = "2025-08-25T13:49:26.313Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/18/67/36e9267722cc04a6b9f15c7f3441c2363321a3ea07da7ae0c0707beb2a9c/typing_extensions-4.15.0-py3-none-any.whl", hash = "sha256:f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548", size = 44614, upload-time = "2025-08-25T13:49:24.86Z" },
+]
+
 [[package]]
 name = "tzdata"
 version = "2025.2"