Kaggle competition: Google Analytics Customer Revenue Prediction
pred_file.to_csv('submission.csv', index=False )<load_from_csv>
def load_df(csv_path, nrows=None):
USE_COLUMNS = [
'channelGrouping', 'date', 'device', 'fullVisitorId', 'geoNetwork',
'socialEngagementType', 'totals', 'trafficSource', 'visitId',
'visitNumber', 'visitStartTime', 'customDimensions'
]
JSON_COLUMNS = ['device', 'geoNetwork', 'totals', 'trafficSource']
df = pd.read_csv(csv_path,
converters={column: json.loads for column in JSON_COLUMNS},
dtype={'fullVisitorId': 'str'},
nrows=nrows, usecols=USE_COLUMNS)
 for column in JSON_COLUMNS:
column_as_df = json_normalize(df[column])
 column_as_df.columns = [f"{column}.{subcolumn}" for subcolumn in column_as_df.columns]
df = df.drop(column, axis=1 ).merge(column_as_df, right_index=True, left_index=True)
 print(f"Loaded {os.path.basename(csv_path)}.Shape: {df.shape}")
 return df
x=pd.read_csv('/kaggle/input/covid19-global-forecasting-week-1/train.csv')
 x_test=pd.read_csv('/kaggle/input/covid19-global-forecasting-week-1/test.csv' )<count_missing_values>
%%time
train_df = load_df(".. /input/train_v2.csv")
 test_df = load_df(".. /input/test_v2.csv" )
x.isnull().sum()<split>
train_df = train_df.drop(labels=['trafficSource.campaignCode'],axis=1)
 train_df = train_df.drop(labels=['customDimensions','totals.totalTransactionRevenue'],axis=1)
 test_df = test_df.drop(labels=['customDimensions','totals.transactionRevenue','totals.totalTransactionRevenue'],axis=1 )
x1=x.iloc[:,3:5]
x_test2=x_test.iloc[:,3:5]<count_missing_values>
train_df['date']= pd.to_datetime(train_df['date'],format='%Y%m%d')
 test_df['date']= pd.to_datetime(test_df['date'],format='%Y%m%d')
 train_df.head()
x_test.isnull().sum()<import_modules>
train_df['totals.transactionRevenue'] = train_df['totals.transactionRevenue'].astype('float')
 print('Total records: ', len(train_df), 'customers doesnt contribute for revenue: ',train_df['totals.transactionRevenue'].isna().sum() , \
'customers contributing revenue: ',len(train_df)- train_df['totals.transactionRevenue'].isna().sum())
from fancyimpute import IterativeImputer as MICE<define_variables>
print('Features with one unique values are :')
 print(train_df.columns[train_df.nunique() ==1])
 new_df = train_df.drop(labels=train_df.columns[train_df.nunique() ==1],axis=1)
 test_df = test_df.drop(labels=train_df.columns[train_df.nunique() ==1],axis=1 )
x1_c = list(x1 )<prepare_x_and_y>
new_df.isna().sum()
y=MICE()<create_dataframe>
new_df1 = new_df.drop(labels=['geoNetwork.region','geoNetwork.networkDomain','geoNetwork.metro','geoNetwork.continent','geoNetwork.country','visitId','visitStartTime'],axis=1)
 test_df1 = test_df.drop(labels=['geoNetwork.region','geoNetwork.networkDomain','geoNetwork.metro','geoNetwork.continent','geoNetwork.country','visitId','visitStartTime'],axis=1)
x2 = pd.DataFrame(y.fit_transform(x1))<create_dataframe>
new_df1 = new_df1.drop(labels=['date','device.isMobile'], axis=1)
 test_df1 = test_df1.drop(labels=['date','device.isMobile'], axis=1 )
x_test3=pd.DataFrame(y.transform(x_test2))<rename_columns>
new_df1 = new_df1.drop(labels=['trafficSource.adwordsClickInfo.adNetworkType', 'trafficSource.adwordsClickInfo.gclId', 'trafficSource.adwordsClickInfo.page', 'trafficSource.adwordsClickInfo.slot', 'trafficSource.medium', 'trafficSource.medium', 'trafficSource.referralPath','trafficSource.source'],axis=1)
 test_df1 = test_df1.drop(labels=['trafficSource.adwordsClickInfo.adNetworkType', 'trafficSource.adwordsClickInfo.gclId', 'trafficSource.adwordsClickInfo.page', 'trafficSource.adwordsClickInfo.slot', 'trafficSource.medium', 'trafficSource.medium', 'trafficSource.referralPath','trafficSource.source'],axis=1)
 print(new_df1.shape)
 new_df1.head()
x2.columns=x1_c
x_test3.columns=x1_c<count_missing_values>
new_df1.fillna('0',inplace=True)
 test_df1.fillna('0',inplace=True )
x2.isnull().sum()<count_missing_values>
new_df1.isna().sum()
x_test3.isnull().sum()<feature_engineering>
new_df1['trafficSource.adContent'].fillna('Noadcontent',inplace=True)
 new_df1['trafficSource.keyword'].fillna('NA',inplace=True)
 test_df1['trafficSource.adContent'].fillna('Noadcontent',inplace=True)
 test_df1['trafficSource.keyword'].fillna('NA',inplace=True)
 for columns in ['totals.sessionQualityDim','totals.timeOnSite','totals.transactions']:
new_df1[columns].fillna('0',inplace=True)
 new_df1[columns] = new_df1[columns].astype('int')
 test_df1[columns].fillna('0',inplace=True)
 test_df1[columns] = test_df1[columns].astype('int' )
x['Lat']=x2['Lat']
x_test['Lat']=x_test3['Lat']<feature_engineering>
def convert_category_todummies(df,field):
dummy_df = pd.get_dummies(df[field])
 df = pd.concat([df,dummy_df],axis=1)
 df.drop(labels=[field],axis=1,inplace=True)
 return df
x['Long']=x2['Long']
x_test['Long']=x_test3['Long']<count_missing_values>
def convert_category_tolevel(df,field):
df[field],index = pd.factorize(df[field])
 return df
x.isnull().sum()<data_type_conversions>
scaler1 = MinMaxScaler()
merged_df['totals.transactionRevenue'] = np.log1p(merged_df['totals.transactionRevenue'])
 new_df1 = merged_df.iloc[:train_size]
test_df1 = merged_df[train_size:]
test_df1 = test_df1.drop(labels=['totals.transactionRevenue'],axis=1)
 print(new_df1.shape, test_df1.shape )
x['Date']=pd.to_datetime(x['Date'])
 x_test['Date']=pd.to_datetime(x_test['Date'] )<import_modules>
del train_df, new_df, test_df,merged_df, transaction_df, nontransaction_df, plot_df
gc.collect()
from datetime import date<define_variables>
train_x = new_df1.groupby(by=['fullVisitorId'] ).mean()
del new_df1
gc.collect()
z=x['Date']-mindate<feature_engineering>
train_y = train_x['totals.transactionRevenue']
train_x = train_x.drop(labels=['totals.transactionRevenue'],axis=1)
 scaled_x = scaler1.fit_transform(train_x.values)
 train_x = pd.DataFrame(scaled_x, columns=train_x.columns )
for i in range(len(z)) :
z[i]=int(str(z[i] ).split() [0] )<feature_engineering>
X_train, X_val, y_train, y_val = train_test_split(train_x, train_y, test_size=0.15, random_state=1)
 lgb_train_data = lgb.Dataset(X_train, label=y_train)
 lgb_val_data = lgb.Dataset(X_val, label=y_val)
 params = {
"objective" : "regression",
"metric" : "rmse",
"max_depth" : 10,
"boosting" : "gbdt",
"learning_rate" : 0.0025,
"bagging_fraction" : 0.7,
"feature_fraction" : 0.7,
"bagging_frequency" : 6,
"bagging_seed" : 42,
"seed": 42}
model = lgb.train(params, lgb_train_data,
num_boost_round=5000,
valid_sets=[lgb_train_data, lgb_val_data],
early_stopping_rounds=100,
verbose_eval=500 )
x['Date']=z<define_variables>
test_x = test_df1.groupby(by=['fullVisitorId'] ).mean()
visitorid = test_x.index
scaled_test = scaler1.transform(test_x)
 test_x = pd.DataFrame(scaled_test,columns=test_x.columns )
c=x_test['Date']-mindate<feature_engineering>
y_pred_train = model.predict(X_train, num_iteration=model.best_iteration)
 y_pred_val = model.predict(X_val, num_iteration=model.best_iteration)
 y_pred_submit = model.predict(test_x, num_iteration=model.best_iteration)
 print(f"LGBM: RMSE val: {np.sqrt(mean_squared_error(y_val, y_pred_val)) } - RMSE train: {np.sqrt(mean_squared_error(y_train, y_pred_train)) }" )
for i in range(len(c)) :
c[i]=int(str(c[i] ).split() [0] )<feature_engineering>
x_test['Date']=c<data_type_conversions>
x['Date']=x['Date'].astype(int )<data_type_conversions>
inputs = Input(shape=(1,70))
x = Bidirectional(LSTM(200,recurrent_dropout=0.2, kernel_initializer='lecun_normal', return_sequences=True))(inputs)
 x = Bidirectional(LSTM(120,recurrent_dropout=0.2, kernel_initializer='lecun_normal'))(x)
 x = Dense(50, activation='sigmoid' )(x)
 x = Dropout(0.1 )(x)
 x = Dense(20,activation='elu' )(x)
 output = Dense(1,activation='linear' )(x)
 model2 = Model(inputs=inputs, outputs=output)
 model2.compile(loss='mse', optimizer='adam')
 model2.fit(X_train, y_train, epochs=4, batch_size=64, validation_data=(X_val, y_val), verbose=1, shuffle=False )
x_test['Date']=x_test['Date'].astype(int )<split>
test_x = test_x.values
test_x = test_x.reshape(( test_x.shape[0],1,test_x.shape[1]))
y_pred_train = model2.predict(X_train)
 y_pred_val = model2.predict(X_val)
 y_pred_submit = model2.predict(test_x)
 print(f"LSTM: RMSE val: {np.sqrt(mean_squared_error(y_val, y_pred_val)) } - RMSE train: {np.sqrt(mean_squared_error(y_train, y_pred_train)) }" )
<data_type_conversions><EOS> 
submission = pd.DataFrame({'fullVisitorId':visitorid,'PredictedLogRevenue':np.squeeze(y_pred_submit)})
 submission['fullVisitorId']= submission['fullVisitorId'].astype(str)
 submission['PredictedLogRevenue']=submission['PredictedLogRevenue'].apply(lambda x: 0 if x<0 else x)
 submission.to_csv('submission2.csv',index=False )