Kaggle competition: Google Analytics Customer Revenue Prediction
submission.to_csv('submission.csv', index = False )<import_modules>
InteractiveShell.ast_node_interactivity='all'
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
from math import cos, sin<load_from_csv>
def load_dataset(path,nrows=None):
cols=['channelGrouping', 'date', 'device', 'fullVisitorId', 'geoNetwork',
'socialEngagementType', 'totals', 'trafficSource', 'visitId','customDimensions',
'visitNumber', 'visitStartTime']
json_cols=['device','geoNetwork', 'totals', 'trafficSource']
df = pd.read_csv(path,
converters={column: json.loads for column in json_cols},
dtype={'fullVisitorId': 'str'},
nrows=nrows, usecols=cols)
 for column in json_cols:
column_as_df = json_normalize(df[column])
 column_as_df.columns = [f"{column}.{subcolumn}" for subcolumn in column_as_df.columns]
df = df.drop(column, axis=1 ).merge(column_as_df, right_index=True, left_index=True)
 return df
train_df = pd.read_csv('.. /input/covid19-global-forecasting-week-1/train.csv')
 test_df = pd.read_csv('.. /input/covid19-global-forecasting-week-1/test.csv')
 df = [train_df, test_df]<count_missing_values>
%%time
train=load_dataset('.. /input/train_v2.csv' )
train_df.isnull().sum()<define_variables>
%%time
test=load_dataset('.. /input/test_v2.csv' )
arr2 = [i for i in range(163)]<define_variables>
for cols in train.columns:
if(train[cols].nunique() == 1):
train[cols].value_counts()
del train[cols]
for cols in test.columns:
if(test[cols].nunique() == 1):
del test[cols]
train['set']='Train'
test['set']='Test'
data= pd.concat([train,test],axis=0 )
country_dict = dict(zip(arr1, arr2))<data_type_conversions>
data.drop('customDimensions',inplace=True,axis=1)
 data.head(5 )
for dataset in df:
dataset['Country_int'] = dataset['Country/Region'].map(country_dict ).astype(int )<data_type_conversions>
for col in data.columns:
if data[col].isnull().sum() > 0:
rate = data[col].isnull().sum() * 100 / data.shape[0]
print(f'Column {col} has {rate:.4f}% missing values.')
 if data[col].dtype == 'object':
if(data[col] == 'not available in demo dataset' ).sum() > 0:
rate =(data[col] == 'not available in demo dataset' ).sum() * 100 / data.shape[0]
print(f'Column {col} has {rate:.4f}% values not available in dataset.' )
for dataset in df:
dataset['Date'] = pd.to_datetime(dataset['Date'] )<feature_engineering>
data['date']=data['date'].apply(lambda x: pd.to_datetime(str(x),format='%Y%m%d'))
data['year']=data['date'].dt.year
data['month']=data['date'].dt.month
data['day']=data['date'].dt.day
data['weekday']=data['date'].dt.weekday
for dataset in df:
dataset['Day'] = dataset.Date.apply(lambda x: x.day)
 dataset['Month'] = dataset.Date.apply(lambda x: x.month )<count_unique_values>
not_null=pd.notnull(data['totals.transactionRevenue'] ).sum()
print(f'No of records generating the transactions : {not_null}')
 null_val=round(pd.isnull(data['totals.transactionRevenue'] ).sum() /data.shape[0],4)
 print(f'No of records with Null transaction : {null_val}')
 data['totals.transactionRevenue']=data['totals.transactionRevenue'].fillna('0' )
for j in train_df['Country/Region'].unique() :
l = train_df[(train_df['Country/Region'] == j)]
len_state = len(l['Province/State'].unique())
 if len_state != 1:
arr_state1 = l['Province/State'].unique()
arr_state2 = [i for i in range(1,len_state+1)]
state_dict = dict(zip(arr_state1, arr_state2))
for dataset in df:
dataset['Province/State'] = dataset['Province/State'].map(state_dict )<feature_engineering>
data.columns
del data['visitId']
del data['visitNumber']
for dataset in df:
dataset['X_Cord'] =(dataset['Lat'].apply(cos)) *(dataset['Long'].apply(cos))
dataset['Y_Cord'] =(dataset['Lat'].apply(cos)) *(dataset['Long'].apply(sin))<data_type_conversions>
data['totals.hits']=data['totals.hits'].fillna(1)
 data['totals.hits']=data['totals.hits'].astype('Int64' )
for dataset in df:
dataset['Province/State'] = dataset['Province/State'].astype(int )<data_type_conversions>
data['totals.pageviews']=data['totals.pageviews'].fillna(1)
 data['totals.pageviews']=data['totals.pageviews'].astype('Int64')
 data['totals.pageviews'].dtypes
train_df['ConfirmedCases'] = train_df['ConfirmedCases'].astype(int)
 train_df['Fatalities'] = train_df['Fatalities'].astype(int )<import_modules>
data['totals.totalTransactionRevenue']=data['totals.totalTransactionRevenue'].fillna(0)
 data['totals.totalTransactionRevenue']=data['totals.totalTransactionRevenue'].astype('Int64')
 data.groupby('channelGrouping')['totals.totalTransactionRevenue'].mean().plot(kind='bar',color='orange' )
import seaborn as sns<prepare_x_and_y>
data['trafficSource.adwordsClickInfo.adNetworkType'].value_counts() /data.shape[0]
data.groupby(['trafficSource.adwordsClickInfo.adNetworkType','channelGrouping'])['totals.transactionRevenue'].mean()
X = train_df.drop(['Id', 'Country/Region', 'Lat', 'Long', 'Date', 'ConfirmedCases', 'Fatalities', 'Province/State'], axis=1)
 y1 = train_df['ConfirmedCases']
y2 = train_df['Fatalities']
X_test = test_df.drop(['ForecastId', 'Country/Region', 'Lat', 'Long', 'Date', 'Province/State'], axis=1 )<import_modules>
data['trafficSource.adwordsClickInfo.adNetworkType']=data['trafficSource.adwordsClickInfo.adNetworkType'].fillna('No Network' )
from sklearn.linear_model import LinearRegression
from sklearn.neighbors import KNeighborsRegressor
from sklearn.tree import DecisionTreeRegressor
from sklearn.ensemble import RandomForestRegressor
from xgboost import XGBRegressor
from sklearn.model_selection import GridSearchCV<train_model>
del data['trafficSource.adwordsClickInfo.gclId']
data['trafficSource.adwordsClickInfo.adNetworkType'].value_counts()
data['trafficSource.adwordsClickInfo.page']=data['trafficSource.adwordsClickInfo.page'].fillna('99' )
lin_reg_regressor = LinearRegression()
lin_reg_regressor.fit(X, y1)
 lin_reg_regressor.score(X, y1 )<train_on_grid>
key_word=pd.DataFrame(data.groupby(['trafficSource.keyword'])['totals.transactionRevenue'].mean() ).reset_index()
key_word=key_word.sort_values(by='totals.transactionRevenue',ascending=False)
KNN_regressor = KNeighborsRegressor()
KNN_regressor.fit(X, y1)
 KNN_regressor.score(X, y1 )<train_on_grid>
data['trafficSource.referralPath']=data['trafficSource.referralPath'].fillna('No Reference Path' )
tree_regressor = DecisionTreeRegressor()
tree_regressor.fit(X, y1)
 tree_regressor.score(X, y1 )<train_model>
def process_device_b(data1):
i = str(data1)
 if '_' in i:
return 'other'
elif '.' in i:
return 'other'
elif len(i)>25:
return 'other'
else:
return i
data['device.browser']=data['device.browser'].apply(process_device_b )
Random_Forest_regressor = RandomForestRegressor()
Random_Forest_regressor.fit(X, y1)
 Random_Forest_regressor.score(X, y1 )<train_on_grid>
def key_word(key):
low_key=str(key ).lower()
if 'goo' in low_key or 'google' in low_key or 'googl' in low_key or 'stor' in low_key:
return 'google'
if 'merchandi' in low_key:
return 'merchandise'
if 'youtube' in low_key or 'yo' in low_key or 'yuotub' in low_key or 'yuo tub' in low_key or 'tub' in low_key or 'yutob' in low_key:
return 'youtube'
if bool(re.search(r'\d', low_key)) :
return 'other'
if 'tshirt' in low_key or 't shirt' in low_key or 't-shirt' in low_key:
return 'tshirt'
if 'shirts' in low_key or 'shirt' in low_key:
return 'shirts'
if 'lamp' in low_key or 'lamps' in low_key:
return 'lamps'
if 'apparel' in low_key:
return 'apparel'
if len(low_key)>28:
return 'other'
if 'note' in low_key:
return 'notebook'
if 'android' in low_key:
return 'android'
else:
return low_key
data['trafficSource.keyword']=data['trafficSource.keyword'].apply(key_word)
XGB_regressor = XGBRegressor()
XGB_regressor.fit(X, y1)
 XGB_regressor.score(X, y1 )<define_search_space>
def traffic_feature(source_value):
if 'google' in source_value:
return 'google'
elif 'yahoo' in source_value:
return 'yahoo'
elif 'facebook' in source_value:
return 'facebook'
elif 'youtube' in source_value:
return 'youtube'
elif 'pinterest' in source_value:
return 'pinterest'
elif 'baidu' in source_value:
return 'baidu'
elif 'reddit' in source_value:
return 'reddit'
elif 'wow' in source_value:
return 'wow'
elif len(source_value)>20:
return 'other'
elif 'bing' in source_value:
return 'bing'
else:
return source_value
data['trafficSource.source']=data['trafficSource.source'].apply(traffic_feature)
param_test = {
'max_depth': range(3, 12, 2),
'min_child_weight': range(1, 6, 2),
'gamma': [i/10.0 for i in range(0,5)],
'learning_rate': [0.1, 0.2, 0.3],
'n_estimators': [100, 400, 600, 900, 1100]
}<choose_model_class>
data.groupby(['geoNetwork.networkDomain','channelGrouping'])['channelGrouping'].count()
gsearch = GridSearchCV(estimator = XGBRegressor() ,
param_grid = param_test,
scoring='neg_root_mean_squared_error',
n_jobs=-1,
cv=5 )<find_best_params>
drop_cols = ['trafficSource.referralPath', 'trafficSource.adContent', 'trafficSource.adwordsClickInfo.slot', 'trafficSource.adwordsClickInfo.page',
'trafficSource.adwordsClickInfo.adNetworkType','visitStartTime','device.isMobile']
gsearch.best_estimator_<train_model>
for i, t in data.dtypes.iteritems() :
if t == object:
data[i] = pd.factorize(data[i])[0]
XGB_regressor=XGBRegressor(base_score=0.5, colsample_bylevel=1,
colsample_bynode=1, colsample_bytree=1, gamma=0.0, gpu_id=-1,
importance_type='gain', learning_rate=0.1, max_delta_step=0, max_depth=3,
min_child_weight=1, n_estimators=100, n_jobs=0, num_parallel_tree=1,
objective='reg:squarederror', random_state=0, reg_alpha=0,
reg_lambda=1, scale_pos_weight=1, subsample=1)
 XGB_regressor.fit(X, y1)
 XGB_regressor.score(X, y1 )<predict_on_test>
train_x=train_df.iloc[:,:-1]
train_y=train_df.iloc[:,-1]
test_x=test_df.iloc[:,:-1]
test_y=test_df.iloc[:,-1]
ConfirmedCasesPred = XGB_regressor.predict(X_test)
 ConfirmedCasesPred = pd.DataFrame(ConfirmedCasesPred, columns=['ConfirmedCases'] )<train_model>
param = { 'objective':'regression',
'max_depth': -1,
'learning_rate':0.005,
"min_child_samples":40,
"boosting":"gbdt",
"feature_fraction":0.8,
"bagging_freq":1,
"bagging_fraction":0.8 ,
"bagging_seed": 3,
"metric": 'rmse',
"lambda_l1": 1,
'lambda_l2': 1,
"verbosity": -1}
XGB_regressor.fit(X, y2)
 XGB_regressor.score(X, y2 )<predict_on_test>
train_y = train_y.apply(lambda x: np.log(x)if x > 0 else x )
FatalitiesPred = XGB_regressor.predict(X_test)
 FatalitiesPred = pd.DataFrame(FatalitiesPred, columns=['Fatalities'] )<create_dataframe>
folds = KFold(n_splits=5, shuffle=True, random_state=15)
 oof = np.zeros(len(train_df))
predictions = np.zeros(len(test_df))
start = time.time()
features = list(train_df.columns)
 feature_importance_df = pd.DataFrame()
for fold_,(trn_idx, val_idx)in enumerate(folds.split(train_x.values, train_y.values)) :
trn_data = lgb.Dataset(train_x.iloc[trn_idx], label=train_y.iloc[trn_idx])
 val_data = lgb.Dataset(train_x.iloc[val_idx], label=train_y.iloc[val_idx])
 num_round = 10000
clf = lgb.train(param, trn_data, num_round, valid_sets = [trn_data, val_data], verbose_eval=500 ,early_stopping_rounds = 500)
 oof[val_idx] = clf.predict(train_x.iloc[val_idx].values, num_iteration=clf.best_iteration)
 fold_importance_df = pd.Series()
fold_importance_df["feature"] = features
fold_importance_df["importance"] = clf.feature_importance()
fold_importance_df["fold"] = fold_ + 1
feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)
 predictions += clf.predict(test_x.values, num_iteration=clf.best_iteration)/ folds.n_splits
ForecastId = test_df.ForecastId
ForecastId = pd.DataFrame(ForecastId )<concatenate>
print("CV score: {:<8.5f}".format(mean_squared_error(oof, train_y)**0.5))
pred_file = pd.concat([ForecastId, ConfirmedCasesPred, FatalitiesPred], axis=1 )<save_to_csv>
submission = pd.DataFrame()
submission['fullVisitorId'] = test['fullVisitorId']
submission['PredictedLogRevenue'] = predictions
submit_file = submission[['fullVisitorId', 'PredictedLogRevenue']].groupby('fullVisitorId' ).sum().reset_index()
submit_file.to_csv('submit_group.csv',index=False)