Move Gradient Descent to a Class based architecture to make use of inheritance and precomputations.

.pre-commit-config.yaml

@@ -22,3 +22,10 @@ repos:
           - --ignore-missing-imports
           - .
 
+  # Strip Jupyter outputs
+  - repo: https://github.com/kynan/nbstripout
+    rev: 0.7.1
+    hooks:
+      - id: nbstripout
+        name: Strip Jupyter notebook outputs
+        files: \.ipynb$

src/optimizers.py

@@ -25,36 +25,101 @@ def OLS_parameters(X: np.ndarray, y: np.ndarray) -> np.ndarray:
     return Ridge_parameters(X, y, lam=0.0)  # OLS is Ridge with lambda=0
 
 
-def gradient_descent(
+class GradientDescent:
-    X, y, cost_func, grad_cost_func, eta=0.1, num_iters=1000, **kwargs
+    def __init__(
-):
+        self, *args, learning_rate: float = 0.1, num_iterations: int = 1000, **kwargs
-    # Initialize weights
+    ):
-    theta = np.zeros(X.shape[1])
+        """Gradient Descent Class
-    # Store cost history
+
-    cost_history = np.zeros(num_iters)
+        Args:
-    for t in range(num_iters):
+            learning_rate (float, optional): Learning rate used for step updates. Defaults to 0.1.
-        # Compute cost
+            num_iterations (int, optional): Number of iterations for gradient descent. Defaults to 1000.
-        cost_history[t] = cost_func(X, y, theta, **kwargs)
+        """
-        # Compute gradient
+        self.cost_history = np.zeros(num_iterations)
-        grad = grad_cost_func(X, y, theta, **kwargs)
+        self.learning_rate = learning_rate
-        # Update weights
+        self.num_iterations = num_iterations
-        theta -= eta * grad
+
-    return theta, cost_history
+    def fit(self, X: np.ndarray, y: np.ndarray) -> np.ndarray:
+        """Returns the optimal solution for an optimization problem using the gradient descent
+
+        Args:
+            X (np.ndarray): X values
+            y (np.ndarray): y values
+
+        Returns:
+            np.ndarray: Optimal parameters
+        """
+        self.X = X
+        self.y = y
+        self.theta = np.zeros(X.shape[1])
+        self._precomp()
+        for t in range(self.num_iterations):
+            self._comp_step()
+            self.cost_history[t] = self._compute_cost()
+            self._update_theta()
+        return self.theta
+
+    def _precomp(self):
+        pass
+
+    def _comp_step(self):
+        pass
+
+    def _compute_cost(self):
+        pass
+
+    def _update_theta(self):
+        pass
 
 
-def OLS_cost_func(X, y, theta):
+class OLSGradientDescent(GradientDescent):
-    error = (X @ theta) - y
+    def _precomp(self):
-    return 0.5 * np.mean(error**2)
+        self.XTX = self.X.T @ self.X
+        self.XTy = self.X.T @ self.y
+        self.n = len(self.y)
+
+    def _comp_step(self):
+        self.err = self.X @ self.theta - self.y
+
+    def _compute_cost(self) -> float:
+        return 0.5 * np.mean(np.square(self.err))
+
+    def _compute_grad(self) -> np.ndarray:
+        return (self.XTX @ self.theta - self.XTy) / self.n
+
+    def _update_theta(self):
+        self.theta -= self.learning_rate * self._compute_grad()
 
 
-def OLS_grad_cost_func(X, y, theta):
+class RidgeGradientDescent(OLSGradientDescent):
-    error = (X @ theta) - y
+    def __init__(self, *args, lam: float = 0.1, **kwargs):
-    return (X.T @ error) / len(y)
+        super().__init__(*args, **kwargs)
+        self.lam = lam
+
+    def _compute_cost(self) -> float:
+        return super()._compute_cost() + 0.5 * self.lam * np.sum(np.square(self.theta))
+
+    def _compute_grad(self) -> np.ndarray:
+        return super()._compute_grad() + self.lam * self.theta
 
 
-def Ridge_cost_func(X, y, theta, lam):
+class OLSMomentum(OLSGradientDescent):
-    return OLS_cost_func(X, y, theta) + 0.5 * lam * np.sum(np.square(theta))
+    def __init__(self, *args, delta: float = 1.0, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.delta = delta
+
+    def _precomp(self):
+        self.last_theta = np.zeros_like(self.theta)
+        return super()._precomp()
+
+    def _update_theta(self):
+        v = self.delta * (
+            self.theta - self.last_theta - self.learning_rate * self._compute_grad()
+        )
+        self.last_theta = self.theta.copy()
+        self.theta += v
 
 
-def Ridge_grad_cost_func(X, y, theta, lam):
+class RidgeMomentum(OLSMomentum, RidgeGradientDescent):
-    return OLS_grad_cost_func(X, y, theta) + lam * theta
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)