Luggage Bags Cost Prediction — Gradient Descent
By: Nithin Rajan
Tags: Python, Linear Regression, Gradient Descent, Sklearn, Machine Learning

import math

import numpy as np

import pandas as pd

import matplotlib.pyplot as plt

import seaborn as sns

sns.set(rc={'figure.figsize':(11.7,8.27)})

from sklearn import preprocessing

from sklearn.preprocessing import normalize

from sklearn.metrics import mean_absolute_error, mean_squared_error, r2_score

from sklearn.model_selection import train_test_split

from sklearn.linear_model import SGDRegressor

from sklearn import linear_model # Reading CSV Input file and storing as pandas dataframe object

data = pd.read_csv('Data_miniproject.csv')

data.columns Index(['Cost', 'Weight', 'Weight1', 'Length', 'Height', 'Width'], dtype='object') # Checking top 5 rows for data

data.head(5)

EDA

# checking the data type and wthether it will accept null or not

data.info() <class 'pandas.core.frame.DataFrame'>

RangeIndex: 159 entries, 0 to 158

Data columns (total 6 columns):

# Column Non-Null Count Dtype

--- ------ -------------- -----

0 Cost 159 non-null float64

1 Weight 159 non-null float64

2 Weight1 159 non-null float64

3 Length 159 non-null float64

4 Height 159 non-null float64

5 Width 159 non-null float64

dtypes: float64(6)

memory usage: 7.6 KB # Generate descriptive statistics.

# Checking Min, Max, Mean, Std for all the columns

data.describe(percentiles=[.10, .25, .5, .75])

Check if there is any Null Values

# checking column wise null values

data.isnull().any() Cost False

Weight False

Weight1 False

Length False

Height False

Width False

dtype: bool # checking column wise null value count

data.isnull().any()

data.isnull().sum() Cost 0

Weight 0

Weight1 0

Length 0

Height 0

Width 0

dtype: int64 # checking overall null value count

data.isnull().sum().sum() 0 sns.jointplot(x="Length", y="Cost",kind="reg", data=data) <seaborn.axisgrid.JointGrid at 0x7f66b74c6210>

sns.jointplot(x="Height", y="Cost",kind="reg",data=data) <seaborn.axisgrid.JointGrid at 0x7f66b513c310>

# checking the frequency distribution

data.hist(figsize=(10,10)) array([[<matplotlib.axes._subplots.AxesSubplot object at 0x7f66b7521dd0>,

<matplotlib.axes._subplots.AxesSubplot object at 0x7f66b4f85650>],

[<matplotlib.axes._subplots.AxesSubplot object at 0x7f66b4f37e50>,

<matplotlib.axes._subplots.AxesSubplot object at 0x7f66b4ef9690>],

[<matplotlib.axes._subplots.AxesSubplot object at 0x7f66b4eace90>,

<matplotlib.axes._subplots.AxesSubplot object at 0x7f66b4e6c6d0>]],

dtype=object)

# For each pair of features (columns) in the dataset, we can visualize the scatter plot for each pair

# along with the feature’s histogram along the diagonal

sns.pairplot(data) <seaborn.axisgrid.PairGrid at 0x7f66b4b66250>

Removing Outliers

# From the descriptive statistics, cost min is 0, so checking how many cost rows are zero

data[data['Cost']==0]

# Removing rows where cost is 0.

data.drop(data[data['Cost']==0].index,inplace=True) # Finding outliers using boxplot

melted_df = pd.melt(data.drop('Cost', axis=1))

sns.boxplot(x="variable", y="value", data=melted_df) <matplotlib.axes._subplots.AxesSubplot at 0x7f66a79206d0>

data[(data['Weight'] >= 55)&(data['Length']>=64)]

# From the boxplot, dotted points are outliers, so removing these rows

data.drop(data[(data['Weight'] >= 55)&(data['Length']>=64)].index, inplace=True)

data.shape (155, 6) groups = melted_df.groupby("variable")

for name, group in groups:

plt.plot(group["variable"], group["value"], marker="o", label=name)

sns.scatterplot(data=data.drop('Cost', axis=1)) <matplotlib.axes._subplots.AxesSubplot at 0x7f66a5b37d10>

print(data.drop('Cost',axis=1).cumsum()) Weight Weight1 Length Height Width

0 23.2 25.4 30.0 11.5200 4.0200

1 47.2 51.7 61.2 24.0000 8.3256

2 71.1 78.2 92.3 36.3778 13.0217

3 97.4 107.2 125.8 49.1078 17.4772

4 123.9 136.2 159.8 61.5518 22.6112

.. ... ... ... ... ...

154 3932.5 4259.5 4687.6 1379.3888 672.7894

155 3944.2 4271.9 4701.1 1381.8188 674.0584

156 3956.3 4284.9 4714.9 1384.0958 675.3142

157 3969.5 4299.2 4730.1 1386.9686 677.3814

158 3983.3 4314.2 4746.3 1389.9008 679.2606



[155 rows x 5 columns] # plotting the cummulative values for all the rows

data.drop('Cost',axis=1).cumsum().plot() <matplotlib.axes._subplots.AxesSubplot at 0x7f66a5ab6e90>

Correlations

corr = data.corr()

sns.heatmap(corr, xticklabels=corr.columns, yticklabels=corr.columns, annot=True) <matplotlib.axes._subplots.AxesSubplot at 0x7f66a58ff150>

# displaying the top 12 highly correlated column mappings

print(data.corr().abs().unstack().sort_values(ascending = False).nlargest(25)) Width Width 1.000000

Height Height 1.000000

Weight Weight 1.000000

Weight1 Weight1 1.000000

Length Length 1.000000

Cost Cost 1.000000

Weight1 Weight 0.999417

Weight Weight1 0.999417

Weight1 Length 0.993026

Length Weight1 0.993026

Weight Length 0.990580

Length Weight 0.990580

Width Cost 0.924029

Cost Width 0.924029

Length 0.907373

Length Cost 0.907373

Cost Weight1 0.899734

Weight1 Cost 0.899734

Width Length 0.896793

Length Width 0.896793

Weight Cost 0.895740

Cost Weight 0.895740

Width Weight1 0.895324

Weight1 Width 0.895324

Weight Width 0.888883

dtype: float64 ## Weight1 Weight 0.999516

## Since weight and weight 1 has higher co-relation dropping 1 column

## Weight1 Cost has higher coreleation(0.899734) than Weight Cost(0.895740), So keeping Weight1 and dropping Weight

data.drop('Weight', axis=1,inplace=True)

data.head(3)

Features Scaling

#data = normalize(data, axis=0)

# Scaling the features by using min max scaling

minmax_scale = preprocessing.MinMaxScaler().fit(data.iloc[:,1:])

data.iloc[:,1:] = minmax_scale.transform(data.iloc[:,1:]) data.head(3)

X = data.iloc[:,1:].values

y = data.iloc[:,0:1].values # Splitting the dataset by train and test

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.25, random_state=42) learning_rate = 0.5

max_iteration = 5000



s_learning_rate = 0.05

s_max_iteration = 1000



mb_learning_rate = 0.05

mb_max_iteration = 10000

batch_size = 30 theta = np.zeros((data.shape[1], 1))

s_theta = np.zeros((data.shape[1], 1))

mb_theta = np.zeros((data.shape[1], 1)) class GradientDescent:

''' Gradient Descent class which contains batch, mini-batch and stochastic gradient '''



def __init__(self):

''' Constructor of Gradient Descent'''

print('GradientDescent Class initiated')



def get_metrics(self, x, y, theta):

''' To calculate the metrics like MAE, MSE, RMSE, R2 '''

y_pred = [(theta[1] * x1) + (theta[2] * x2)+ (theta[3] * x3)+ (theta[4] * x4) + theta[0]

for x1,x2,x3,x4 in x]

mae = mean_absolute_error(y,y_pred)

mse = mean_squared_error(y,y_pred)

metrics = "MAE = {0}, MSE = {1}, RMSE = {2}, R2 = {3}".format(mae, mse, math.sqrt(mse), r2_score(y,y_pred))

return metrics



def h (self, theta, X) :

''' hypotheseis function to calculate the equation '''

tempX = np.ones((X.shape[0], X.shape[1] + 1))

tempX[:,1:] = X

res = np.matmul(tempX, theta)

return res



def loss (self, theta, X, Y) :

''' Loss function to find the difference between actual and predicted values '''

return np.average(np.square(Y - self.h(theta, X))) / 2



def gradient(self, theta, X, Y):

tempX = np.ones((X.shape[0], X.shape[1] + 1))

tempX[:,1:] = X

d_theta = - np.average((Y - self.h(theta, X)) * tempX, axis= 0)

d_theta = d_theta.reshape((d_theta.shape[0], 1))

return d_theta



def batch_gradient_descent (self, theta, X, Y, learning_rate, max_iteration, gap):

''' Gradient or Batch function to calculate the gradient for the whole dataset '''

cost = np.zeros(max_iteration)

for i in range(max_iteration):

d_theta = self.gradient (theta, X, Y)

theta = theta - learning_rate * d_theta

cost[i] = self.loss(theta, X, Y)

if i % gap == 0 :

print ('iteration : ', i, ' loss : ', self.loss(theta, X, Y))

return theta, cost



def minibatch_gradient_descent (self, theta, X, Y, learning_rate, max_iteration, batch_size, gap):

''' Mini Batch Gradient Function to calculate the gradient by splitting the dataset by multiple batches '''

cost = np.zeros(max_iteration)

for i in range(max_iteration) :

for j in range(0, X.shape[0], batch_size):

d_theta = self.gradient(theta, X[j:j+batch_size,:], Y[j:j+batch_size,:])

theta = theta - learning_rate * d_theta

cost[i] = self.loss(theta, X, Y)

if i % gap == 0 :

print ('iteration : ', i, ' loss : ', self.loss(theta, X, Y))

return theta, cost



def stochastic_gradient_descent (self, theta, X, Y, learning_rate, max_iteration, gap):

''' Stochastic Gradient Function to calculate the gradient for all rows by looping the dataset '''

cost = np.zeros(max_iteration)

for i in range(max_iteration) :

for j in range(X.shape[0]):

d_theta = self.gradient(theta, X[j,:].reshape(1, X.shape[1]), Y[j,:].reshape(1, 1))

theta = theta - learning_rate * d_theta

cost[i] = self.loss(theta, X, Y)

if i % gap == 0 :

print ('iteration : ', i, ' loss : ', self.loss(theta, X, Y))

return theta, cost

Batch Gradient Descent

gradient_obj = GradientDescent() GradientDescent Class initiated theta, cost = gradient_obj.batch_gradient_descent(theta, X_train, y_train, learning_rate, max_iteration, 1000)

print(theta) iteration : 0 loss : 26337.652117479473

iteration : 1000 loss : 5610.284986828074

iteration : 2000 loss : 5594.9578737015345

iteration : 3000 loss : 5581.183507278225

iteration : 4000 loss : 5568.280949897843

[[-227.20878731]

[ 609.04391977]

[ -19.20415027]

[ 205.60804752]

[ 584.45550559]] #plot the cost

fig, ax = plt.subplots()

ax.plot(np.arange(max_iteration), cost, 'r')

ax.legend(loc='upper right', labels=['batch gradient descent'])

ax.set_xlabel('Iterations')

ax.set_ylabel('Cost')

ax.set_title('Batch Gradient Descent - Error vs. Training Epoch', fontsize=16, fontweight='bold')

plt.show()

print('Batch Gradient Descent')

print(gradient_obj.get_metrics(X_train,y_train,theta ))

print(gradient_obj.get_metrics(X_test,y_test,theta )) Batch Gradient Descent

MAE = 88.79709921177003, MSE = 11112.410017110149, RMSE = 105.41541641102665, R2 = 0.8859422707853275

MAE = 77.27555335889485, MSE = 9723.52560637414, RMSE = 98.60793886079426, R2 = 0.9118690650458746

Mini Gradient Descent

mb_theta, mb_cost = gradient_obj.minibatch_gradient_descent (mb_theta, X_train, y_train,mb_learning_rate, mb_max_iteration, batch_size, 1000)

print(theta) iteration : 0 loss : 68343.90926706592

iteration : 1000 loss : 5643.550424686975

iteration : 2000 loss : 5617.635873635339

iteration : 3000 loss : 5609.310338702081

iteration : 4000 loss : 5603.001382493756

iteration : 5000 loss : 5597.081134387655

iteration : 6000 loss : 5591.351527799153

iteration : 7000 loss : 5585.779399027338

iteration : 8000 loss : 5580.35487657774

iteration : 9000 loss : 5575.072604021426

[[-227.20878731]

[ 609.04391977]

[ -19.20415027]

[ 205.60804752]

[ 584.45550559]] #plot the cost

fig, ax = plt.subplots()

ax.plot(np.arange(mb_max_iteration), mb_cost, 'r')

ax.legend(loc='upper right', labels=['mini batch gradient descent'])

ax.set_xlabel('Iterations')

ax.set_ylabel('Cost')

ax.set_title('Mini Batch Gradient Descent - Error vs. Training Epoch', fontsize=16, fontweight='bold')

plt.show()

print('Mini Batch Gradient Descent')

print(gradient_obj.get_metrics(X_train,y_train,mb_theta ))

print(gradient_obj.get_metrics(X_test,y_test,mb_theta )) Mini Batch Gradient Descent

MAE = 88.66784872143079, MSE = 11139.867009657384, RMSE = 105.54556840368706, R2 = 0.8856604523304488

MAE = 77.49573516074169, MSE = 9840.907489095162, RMSE = 99.20134822216461, R2 = 0.9108051530977127

Stochastic Gradient Descent

s_theta, s_cost = gradient_obj.stochastic_gradient_descent(s_theta, X_train, y_train, s_learning_rate, s_max_iteration, 1000) iteration : 0 loss : 13402.40743175595 #plot the cost

fig, ax = plt.subplots()

ax.plot(np.arange(s_max_iteration), s_cost, 'r')

ax.legend(loc='upper right', labels=['stochastic gradient descent'])

ax.set_xlabel('Iterations')

ax.set_ylabel('Cost')

ax.set_title('Stochastic Gradient Descent Error vs. Training Epoch', fontsize=16, fontweight='bold')

plt.show()

print('Stochastic Gradient Descent')

print(gradient_obj.get_metrics(X_train,y_train,s_theta ))

print(gradient_obj.get_metrics(X_test,y_test,s_theta )) Stochastic Gradient Descent

MAE = 84.55370657249655, MSE = 11392.599861030192, RMSE = 106.73612256883887, R2 = 0.8830664034174635

MAE = 79.3819756016456, MSE = 10935.721474038075, RMSE = 104.57399999061944, R2 = 0.9008821083092442

sklearn SGDRegressor

sgd_regressor = SGDRegressor(max_iter=10000, tol=0.005)

sgd_regressor.fit(X_train, y_train.ravel())

print(sgd_regressor.coef_)

print(sgd_regressor.intercept_)



y_pred = sgd_regressor.predict(X_test)

MAE = mean_absolute_error(y_test,y_pred)

MSE = mean_squared_error(y_test, y_pred)

RMSE = math.sqrt(MSE)

print('sklearn SGDRegressor')

print("MAE = {0}, MSE = {1}, RMSE = {2}, R2 = {3}".format(MAE, MSE, math.sqrt(MSE), sgd_regressor.score(X_train, y_train))) [359.50742321 316.35108074 225.46361317 481.28038367]

[-228.71501955]

sklearn SGDRegressor

MAE = 75.52972443435446, MSE = 9571.755566316218, RMSE = 97.83534926761502, R2 = 0.8828668103782972

sklearn linear regression