Kaggle competition: Google Analytics Customer Revenue Prediction
<SOS> metric: RMSE Kaggle data source: google-analytics-customer-revenue-prediction<split>
%matplotlib inline
py.init_notebook_mode(connected=True)
 gc.enable()
pd.options.mode.chained_assignment = None
pd.options.display.max_columns = 999
print(os.listdir(".. /input"))
y2_val=x.iloc[:,-1]<data_type_conversions>
json_data=["device","geoNetwork","totals","trafficSource"]
gc.enable()
features = ['channelGrouping', 'date', 'fullVisitorId', 'visitId',\
'visitNumber', 'visitStartTime', 'device_browser',\
'device_deviceCategory', 'device_isMobile', 'device_operatingSystem',\
'geoNetwork_city', 'geoNetwork_continent', 'geoNetwork_country',\
'geoNetwork_metro', 'geoNetwork_networkDomain', 'geoNetwork_region',\
'geoNetwork_subContinent', 'totals_bounces', 'totals_hits',\
'totals_newVisits', 'totals_pageviews', 'totals_transactionRevenue',\
'trafficSource_adContent', 'trafficSource_campaign',\
'trafficSource_isTrueDirect', 'trafficSource_keyword',\
'trafficSource_medium', 'trafficSource_referralPath',\
'trafficSource_source']
def load_df(csv_path='.. /input/train_v2.csv'):
JSON_COLUMNS = ['device', 'geoNetwork', 'totals', 'trafficSource']
ans = pd.DataFrame()
dfs = pd.read_csv(csv_path, sep=',',
converters={column: json.loads for column in JSON_COLUMNS},
dtype={'fullVisitorId': 'str'},
chunksize = 100000)
 for df in dfs:
df.reset_index(drop = True,inplace = True)
 for column in JSON_COLUMNS:
column_as_df = json_normalize(df[column])
 column_as_df.columns = [f"{column}_{subcolumn}" for subcolumn in column_as_df.columns]
df = df.drop(column, axis=1 ).merge(column_as_df, right_index=True, left_index=True)
 print(f"Loaded {os.path.basename(csv_path)}.Shape: {df.shape}")
 use_df = df[features]
del df
gc.collect()
ans = pd.concat([ans, use_df], axis = 0 ).reset_index(drop = True)
 print(ans.shape)
 return ans
train = load_df()
train.shape
y2_val=y2_val.astype(int )<drop_column>
test=load_df(csv_path='.. /input/test_v2.csv' )
x_val=x.drop(['ConfirmedCases','Fatalities'],axis=1 )<import_modules>
train["totals_transactionRevenue"].fillna(0, inplace=True)
 train_y = train["totals_transactionRevenue"].values
import seaborn as sns;<count_missing_values>
num_cols = ["totals_hits", "totals_pageviews", "visitNumber", "visitStartTime", 'totals_bounces', 'totals_newVisits']
for col in num_cols:
train[col] = train[col].astype(float)
 test[col] = test[col].astype(float)
 excluded_features = [
'date', 'fullVisitorId', 'sessionId', 'totals.transactionRevenue',
'visitId', 'visitStartTime', 'nb_sessions', 'max_visits','visit_time','totals_visits'
]
categorical_features = [
_f for _f in train.columns
if(_f not in excluded_features)&(train[_f].dtype == 'object')
 ]
categorical_features
z=x_val['Province/State'].isna() 
<count_missing_values>
train_val=np.log1p(train_y )
z1=x_test['Province/State'].isna()<feature_engineering>
for f in categorical_features:
train[f], indexer = pd.factorize(train[f])
 test[f] = indexer.get_indexer(test[f])
 train.head()
for i in range(len(z1)) :
if z1[i]==True:
x_test['Province/State'][i]=x_test['Country/Region'][i]<count_missing_values>
train["totals_transactionRevenue"].fillna(0, inplace=True)
 train_y = train["totals_transactionRevenue"]
train_val=np.log1p(train_y )
x_test.isna().sum()<filter>
train_x,test_x,val_x,val_y=train_test_split(train,train_val,test_size=0.2, random_state=42)
for i in range(len(z)) :
if z[i]==True:
x_val['Province/State'][i]=x_val['Country/Region'][i]<count_missing_values>
trn_x=train_x[categorical_features+num_cols]
tst_x=test_x[categorical_features+num_cols]
x_val.isnull().sum()<drop_column>
def run_lgb(train_X, train_y, val_X, val_y, test_X):
params = {
"objective" : "regression",
"metric" : "rmse",
"num_leaves" : 30,
"min_child_samples" : 100,
"learning_rate" : 0.1,
"bagging_fraction" : 0.7,
"feature_fraction" : 0.5,
"bagging_frequency" : 5,
"bagging_seed" : 2018,
"verbosity" : -1
}
lgtrain = lgb.Dataset(train_X, label=train_y)
 lgval = lgb.Dataset(val_X, label=val_y)
 model = lgb.train(params, lgtrain, 1000, valid_sets=[lgval], early_stopping_rounds=100, verbose_eval=100)
 pred_test_y = model.predict(test_X, num_iteration=model.best_iteration)
 pred_val_y = model.predict(val_X, num_iteration=model.best_iteration)
 return pred_test_y, model, pred_val_y
pred_test, model, pred_val = run_lgb(trn_x, val_x,tst_x, val_y, test_X )
x.drop(['ConfirmedCases','Fatalities'],axis=1,inplace=True )<count_missing_values>
pred_val[pred_val<0] = 0
val_pred_df = pd.DataFrame({"fullVisitorId":test_x["fullVisitorId"].values})
 val_pred_df["transactionRevenue"] = test_x["totals_transactionRevenue"].values
val_pred_df["PredictedRevenue"] = np.expm1(pred_val)
 val_pred_df = val_pred_df.groupby("fullVisitorId")["transactionRevenue", "PredictedRevenue"].sum().reset_index()
print(np.sqrt(metrics.mean_squared_error(np.log1p(val_pred_df["transactionRevenue"].values), np.log1p(val_pred_df["PredictedRevenue"].values))))
<categorify><EOS> 
submission = pd.DataFrame({"fullVisitorId":test["fullVisitorId"].values})
 pred_test[pred_test<0] = 0
submission["PredictedLogRevenue"] = np.expm1(pred_test)
 submission = submission.groupby("fullVisitorId")["PredictedLogRevenue"].sum().reset_index()
submission.columns = ["fullVisitorId", "PredictedLogRevenue"]
submission["PredictedLogRevenue"] = np.log1p(submission["PredictedLogRevenue"])
 submission.to_csv("predicted_values.csv", index=False )