Add application file

app.py

@@ -0,0 +1,57 @@
+import gradio as gr
+import numpy as np
+from linear_regression import LinearRegression
+
+def transform_space(X, degree):
+    X_temp = X[:]
+    for d in range(2, degree + 1):
+        X_temp = np.concatenate(((X[:, 0] ** d).reshape(-1, 1), X_temp), axis=1)
+    return X_temp
+
+def prepare_data(num_points=100, degree=1, noise=10):
+    X = np.linspace(-2, 4, num_points)
+    X = X.reshape(-1, 1)
+    coef = []
+    for d in range(degree):
+        coef.append(np.random.uniform(0, 8))
+    coef.append(np.random.uniform(0, 10))
+    coef = np.array(coef)
+    X_transform = transform_space(X, degree)
+    ones = np.ones((X.shape[0], 1))
+    X_transform = np.concatenate((X_transform, ones), axis=1)
+    y = X_transform.dot(coef).reshape((num_points, 1)) + np.random.uniform(1, noise, (num_points, 1))
+    return X, X_transform[:, :-1], y
+
+def create_examples():
+    linear_X, linear_X_transform, linear_y = prepare_data(num_points=100, degree=1, noise=10)
+    polynomial2_X, polynomial2_X_transform, polynomial2_y = prepare_data(num_points=100, degree=2, noise=10)
+    polynomial3_X, polynomial3_X_transform, polynomial3_y = prepare_data(num_points=100, degree=3, noise=20)
+
+    LRModel = LinearRegression(alpha=0.05, epochs=1000, lambda_=0.01, do_visualize=True)
+    LRModel.train(linear_X_transform, linear_y)
+    LRModel.create_gif(linear_X, linear_X_transform, linear_y, "linear_regression_1.gif")
+
+    LR2Model = LinearRegression(alpha=0.05, epochs=1000, lambda_=0.01, do_visualize=True)
+    LR2Model.train(polynomial2_X_transform, polynomial2_y)
+    LR2Model.create_gif(polynomial2_X, polynomial2_X_transform, polynomial2_y, "linear_regression_2.gif")
+
+    LR3Model = LinearRegression(alpha=0.001, epochs=1000, lambda_=0.01, do_visualize=True)
+    LR3Model.train(polynomial3_X_transform, polynomial3_y)
+    LR3Model.create_gif(polynomial3_X, polynomial3_X_transform, polynomial3_y, "linear_regression_3.gif")
+
+# create_examples()
+
+def visualize(choice):
+    if choice == "Linear":
+        return "linear_regression_1.gif"
+    elif choice == "Polynomial":
+        return "linear_regression_2.gif"
+    else:
+        return "linear_regression_3.gif"
+
+iface = gr.Interface(visualize, 
+                     inputs=[          
+                        gr.Dropdown(choices=["Linear", "Polynomial 2 degree", "Polynomial 3 degree"], value="Linear")
+                     ], 
+                     outputs=gr.Image().style(full_width=True, height="600"))
+iface.launch()

linear_regression.py

@@ -0,0 +1,122 @@
+"""
+Author: Giang Tran
+Email: giangtran240896@gmail.com
+Docs: https://giangtran.me/machine-learning/linear-regression
+"""
+import sys
+sys.path.append("..")
+import numpy as np
+import copy
+import imageio
+import io
+import matplotlib.pyplot as plt
+
+class LinearRegression:
+
+    def __init__(self, alpha, epochs=1000, lambda_=0.1, do_visualize=False):
+        self.alpha = alpha
+        self.epochs = epochs
+        self.lambda_ = lambda_
+        self.w = None
+        self.b = None
+        self.do_visualize = do_visualize
+        self.vis_elems = {
+            "loss": [],
+            "iteration": [],
+            "weight": [],
+            "bias": []
+        }
+
+    def _standardize(self, X, y):
+        x_mean = np.mean(X, axis=0)
+        x_std = np.std(X, axis=0)
+        print(x_mean, x_std)
+        y_mean = np.mean(y)
+        y_std = np.std(y)
+        print(y_mean, y_std)
+        return ((X - x_mean)/x_std, x_mean, x_std), ((y - y_mean) / y_std, y_mean, y_std)
+
+    def _hypothesis(self, X, w, b):
+        return np.dot(X, w) + b
+
+    def _mse_loss(self, X, y_hat, y):
+        m = y.shape[0]
+        return np.sum((y_hat - y)**2)/(2*m) + self.lambda_*np.linalg.norm(self.w, 2)**2 / (2*m)
+
+    def _gradient(self, X, y_hat, y):
+        m = X.shape[0]
+        return 1/m * np.dot(X.T, y_hat - y) + (self.lambda_/m*self.w)
+
+    def _gradient_bias(self, y_hat, y):
+        m = y.shape[0]
+        return 1/m * np.sum(y_hat - y)  
+    
+    def _train_one_epoch(self, X_train, y_train, e):
+        y_hat = self.predict(X_train)
+        loss = self._mse_loss(X_train, y_hat, y_train)
+        print("Loss at epoch %s: %f" % (e, loss))
+        w_grad = self._gradient(X_train, y_hat, y_train)
+        b_grad = self._gradient_bias(y_hat, y_train)
+        self._update_params(w_grad, b_grad)
+        w_grad_norm = np.linalg.norm(w_grad, 2)
+        return loss, w_grad_norm
+
+    def _train(self, X_train, y_train):
+        prev_loss = 0
+        for e in range(self.epochs):
+            loss, w_grad_norm = self._train_one_epoch(X_train, y_train, e)
+            if abs(prev_loss - loss) < 0.001:
+                break
+            prev_loss = loss
+            if self.do_visualize and e % 5 == 0:
+                self.vis_elems["loss"].append(loss)
+                self.vis_elems["iteration"].append(e)
+                self.vis_elems["weight"].append(copy.deepcopy(self.w))
+                self.vis_elems["bias"].append(copy.deepcopy(self.b))
+                prev_loss = loss
+
+            if w_grad_norm < 1e-4:
+                break
+
+    def _update_params(self, w_grad, b_grad):
+        self.w -= self.alpha*w_grad
+        self.b -= self.alpha*b_grad
+
+    def _plot(self, w, b, loss, iteration, X, X_transform, y):
+        y_plot = self._hypothesis(X_transform, w, b)
+        plt.figure(0, figsize=(6, 6))
+        plt.clf()
+        plt.title("Loss: " + str(loss))
+        plt.scatter(X[:, 0], y, color='r')
+        label = "Iteration: " + str(iteration)
+        for ind, w in enumerate(w):
+            label += "\nTheta %s: %.2f" % (ind+1, w)
+        label += "\nBias: %.2f" % b
+        plt.plot(X, y_plot, '-', label=label)
+        plt.legend()
+        img_buf = io.BytesIO()
+        plt.savefig(img_buf, format='png')
+        img_buf.seek(0)
+        return imageio.imread(img_buf)
+
+    def create_gif(self, X, X_transform, y, file_name):
+        imgs = []
+        for l, i, w, b in zip(self.vis_elems["loss"], self.vis_elems["iteration"], self.vis_elems["weight"], self.vis_elems["bias"]):
+            imgs.append(self._plot(w, b, l, i, X, X_transform, y))
+        imageio.mimsave(file_name, imgs, fps=5)
+
+    def train(self, X_train, y_train):
+        self.w = np.random.uniform(size=(X_train.shape[1], 1))
+        self.b = np.random.randint(low=20, high=50)
+        self._train(X_train, y_train)
+
+    def predict(self, X_test, w=None, b=None):
+        assert X_test.shape[1] == self.w.shape[0], "Incorrect shape."
+        pred = self._hypothesis(X_test, self.w, self.b)
+        return pred
+
+
+    def r2_score(self, y_hat, y_test):
+        total_sum_squares = np.sum((y_test - np.mean(y_test))**2)
+        residual_sum_squares = np.sum((y_test - y_hat)**2)
+        return 1 - residual_sum_squares/total_sum_squares