Show unregularized solution(s) in plot

regularization.py

@@ -129,6 +129,24 @@ class Regularization:
         ]
         loss_levels.reverse()
 
+        try:
+            unregularized_w = np.linalg.solve(X.T @ X, X.T @ y)
+        except np.linalg.LinAlgError:
+            # the solutions are on a line
+            eig_vals, eig_vectors = np.linalg.eigh(X.T @ X)
+            line_direction = eig_vectors[:, np.argmin(eig_vals)]
+            m = line_direction[1] / line_direction[0]
+
+            candidate_w = np.linalg.lstsq(X, y, rcond=None)[0]
+            b = candidate_w[1] - m * candidate_w[0]
+
+            unregularized_w1 = np.linspace(w1_range[0], w1_range[1], num_dots)
+            unregularized_w2 = m * unregularized_w1 + b
+            unregularized_w = np.stack((unregularized_w1, unregularized_w2), axis=-1)
+
+            mask = (unregularized_w2 <= w2_range[1]) & (unregularized_w2 >= w2_range[0])
+            unregularized_w = unregularized_w[mask]
+
         if plot_path:
             if loss_type == "l2":
                 path_w = l2_loss_regularization_path(y, X, regularization_type=reg_type)
@@ -159,6 +177,7 @@ class Regularization:
             reg_values, 
             loss_levels, 
             reg_levels,
+            unregularized_w,
             path_w,
         )
         
@@ -170,6 +189,7 @@ class Regularization:
         reg_values: np.ndarray, 
         loss_levels: list, 
         reg_levels: list,
+        unregularized_w: np.ndarray,
         path_w: np.ndarray | None,
     ):
         fig, ax = plt.subplots(figsize=(8, 8))
@@ -189,6 +209,12 @@ class Regularization:
         cs2 = ax.contour(W1, W2, losses, levels=loss_levels, colors=colors[::-1])
         ax.clabel(cs2, inline=True, fontsize=8)
 
+        # unregularized solution
+        if unregularized_w.ndim == 1:
+            ax.plot(unregularized_w[0], unregularized_w[1], "bx", markersize=5, label="unregularized solution")
+        else:
+            ax.plot(unregularized_w[:, 0], unregularized_w[:, 1], "b-", label="unregularized solution")
+
         # regularization path
         if path_w is not None:
             ax.plot(path_w[:, 0], path_w[:, 1], "r-")
@@ -197,9 +223,20 @@ class Regularization:
         loss_line = mlines.Line2D([], [], color='black', linestyle='-', label='loss')
         reg_line = mlines.Line2D([], [], color='black', linestyle='--', label='regularization')
         handles = [loss_line, reg_line]
+        
         if path_w is not None:
             path_line = mlines.Line2D([], [], color='red', linestyle='-', label='regularization path')
             handles.append(path_line)
+
+        if unregularized_w.ndim == 1:
+            handles.append(
+                mlines.Line2D([], [], color='blue', marker='x', linestyle='None', label='unregularized solution')
+            )
+        else:
+            handles.append(
+                mlines.Line2D([], [], color='blue', linestyle='-', label='unregularized solution')
+            )
+
         ax.legend(handles=handles)
 
         ax.grid(True)