Kaggle competition: COVID19 Global Forecasting (Week 1)
xgb_params = {
'objective': 'reg:linear',
'booster': 'gbtree',
'learning_rate': 0.02,
'max_depth': 22,
'min_child_weight': 57,
'gamma' : 1.45,
'alpha': 0.0,
'lambda': 0.0,
'subsample': 0.67,
'colsample_bytree': 0.054,
'colsample_bylevel': 0.50,
'n_jobs': -1,
'random_state': 456
}<prepare_x_and_y>
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
%%time
folds = get_folds(df=full_data[['totals.pageviews']].reset_index() , n_splits=5)
 oof_preds = np.zeros(full_data.shape[0])
 oof_preds1 = np.zeros(full_data.shape[0])
 both_oof = np.zeros(full_data.shape[0])
 sub_preds = np.zeros(sub_full_data.shape[0])
 vis_importances = pd.DataFrame()
for fold_,(trn_, val_)in enumerate(tqdm_notebook(folds)) :
trn_x, trn_y = full_data.iloc[trn_], trn_user_target['target'].iloc[trn_]
val_x, val_y = full_data.iloc[val_], trn_user_target['target'].iloc[val_]
reg = lgb.LGBMRegressor(
 num_leaves=31,
learning_rate=0.03,
n_estimators=100,
subsample=.9,
colsample_bytree=.9,
random_state=1 
)
 reg.fit(
 trn_x, np.log1p(trn_y),
eval_set=[(trn_x, np.log1p(trn_y)) ,(val_x, np.log1p(val_y)) ],
eval_names=['TRAIN', 'VALID'],
early_stopping_rounds=50,
eval_metric='rmse',
verbose=100 
)
 imp_df = pd.DataFrame()
imp_df['feature'] = trn_x.columns
imp_df['gain'] = reg.booster_.feature_importance(importance_type='gain')
 imp_df['fold'] = fold_ + 1
vis_importances = pd.concat([vis_importances, imp_df], axis=0, sort=False)
 oof_preds[val_] = reg.predict(val_x, num_iteration=reg.best_iteration_)
 oof_preds[oof_preds < 0] = 0
both_oof[val_] = oof_preds[val_]
_preds = reg.predict(sub_full_data[full_data.columns], num_iteration=reg.best_iteration_)
 _preds[_preds < 0] = 0
sub_preds +=(_preds / len(folds))
mean_squared_error(np.log1p(trn_user_target['target']), oof_preds)**.5<compute_test_metric>
features = pd.read_csv('.. /input/covid19-global-forecasting-week-1/train.csv')
 test_features = pd.read_csv('.. /input/covid19-global-forecasting-week-1/test.csv' )
mean_squared_error(np.log1p(trn_user_target['target']), oof_preds)**.5<save_to_csv>
features.isnull().sum()
sub_full_data['PredictedLogRevenue'] = sub_preds
idx = list(set(sub_full_data.index ).intersection(set(full_data.index)))
 a = full_data.loc[idx]['Revenue']
diff = np.expm1(a[a>-10])- trn_user_target['target'].loc[idx][a>-10] / 5e5
diff[diff < 0] = 0
sub_full_data['PredictedLogRevenue'].loc[diff.index] = np.log1p(diff * 5e5)
 sub_full_data[['PredictedLogRevenue']].to_csv('new_test_leak_xgb.csv', index=True )<set_options>
test_features.isnull().sum()
warnings.simplefilter("ignore")
 pd.set_option("display.max_rows", 1000, "display.max_columns", 1000)
 pd.options.mode.chained_assignment = None
def load_df(csv_path='.. /input/ga-customer-revenue-prediction/train.csv', nrows=None):
JSON_COLUMNS = ['device', 'geoNetwork', 'totals', 'trafficSource']
df = pd.read_csv(csv_path,
converters={column: json.loads for column in JSON_COLUMNS},
dtype={'fullVisitorId': 'str'},
nrows=nrows)
 for column in JSON_COLUMNS:
column_as_df = json_normalize(df[column])
 column_as_df.columns = [f"{column}.{subcolumn}" for subcolumn in column_as_df.columns]
df = df.drop(column, axis=1 ).merge(column_as_df, right_index=True, left_index=True)
 print(f"Loaded {os.path.basename(csv_path)}.Shape: {df.shape}")
 return df
train = load_df()
test = load_df(".. /input/ga-customer-revenue-prediction/test.csv")
 train_store_1 = pd.read_csv('.. /input/exported-google-analytics-data/Train_external_data.csv', low_memory=False, skiprows=6, dtype={"Client Id":'str'})
 train_store_2 = pd.read_csv('.. /input/exported-google-analytics-data/Train_external_data_2.csv', low_memory=False, skiprows=6, dtype={"Client Id":'str'})
 test_store_1 = pd.read_csv('.. /input/exported-google-analytics-data/Test_external_data.csv', low_memory=False, skiprows=6, dtype={"Client Id":'str'})
 test_store_2 = pd.read_csv('.. /input/exported-google-analytics-data/Test_external_data_2.csv', low_memory=False, skiprows=6, dtype={"Client Id":'str'})
 for df in [train_store_1, train_store_2, test_store_1, test_store_2]:
df["visitId"] = df["Client Id"].apply(lambda x: x.split('.', 1)[1] ).astype(str)
 train_exdata = pd.concat([train_store_1, train_store_2], sort=False)
 test_exdata = pd.concat([test_store_1, test_store_2], sort=False)
 for df in [train, test]:
df["visitId"] = df["visitId"].astype(str)
 train_new = train.merge(train_exdata, how="left", on="visitId")
 test_new = test.merge(test_exdata, how="left", on="visitId")
 for df in [train_new, test_new]:
df.drop("Client Id", 1, inplace=True)
 for df in [train_new, test_new]:
df["Revenue"].fillna('$', inplace=True)
 df["Revenue"] = df["Revenue"].apply(lambda x: x.replace('$', '' ).replace(',', ''))
df["Revenue"] = pd.to_numeric(df["Revenue"], errors="coerce")
 df["Revenue"].fillna(0.0, inplace=True)
 for df in [train_new, test_new]:
df["Sessions"] = df["Sessions"].fillna(0)
 df["Avg.Session Duration"] = df["Avg.Session Duration"].fillna(0)
 df["Bounce Rate"] = df["Bounce Rate"].fillna(0)
 df["Revenue"] = df["Revenue"].fillna(0)
 df["Transactions"] = df["Transactions"].fillna(0)
 df["Goal Conversion Rate"] = df["Goal Conversion Rate"].fillna(0)
 df['trafficSource.adContent'].fillna('N/A', inplace=True)
 df['trafficSource.adwordsClickInfo.slot'].fillna('N/A', inplace=True)
 df['trafficSource.adwordsClickInfo.page'].fillna(0.0, inplace=True)
 df['trafficSource.adwordsClickInfo.isVideoAd'].fillna('N/A', inplace=True)
 df['trafficSource.adwordsClickInfo.adNetworkType'].fillna('N/A', inplace=True)
 df['trafficSource.adwordsClickInfo.gclId'].fillna('N/A', inplace=True)
 df['trafficSource.isTrueDirect'].fillna('N/A', inplace=True)
 df['trafficSource.referralPath'].fillna('N/A', inplace=True)
 df['trafficSource.keyword'].fillna('N/A', inplace=True)
 df['totals.bounces'].fillna(0.0, inplace=True)
 df['totals.newVisits'].fillna(0.0, inplace=True)
 df['totals.pageviews'].fillna(0.0, inplace=True)
 del train
del test
train = train_new
test = test_new
del train_new
del test_new
for df in [train, test]:
df['date_new'] = pd.to_datetime(df['visitStartTime'], unit='s')
 df['date'] = df['date_new'].dt.date
df['sess_date_dow'] = df['date_new'].dt.dayofweek
df['sess_date_hours'] = df['date_new'].dt.hour
df['sess_month'] = df['date_new'].dt.month
const_cols = [col for col in train.columns if len(train[col].unique())==1]
for df in [train, test]:
df.drop(const_cols, 1, inplace=True)
 train.drop('trafficSource.campaignCode', 1, inplace=True)
 def get_folds(df=None, n_splits=5):

 unique_vis = np.array(sorted(df['fullVisitorId'].unique()))
folds = GroupKFold(n_splits=n_splits)
 fold_ids = []
ids = np.arange(df.shape[0])
 for trn_vis, val_vis in folds.split(X=unique_vis, y=unique_vis, groups=unique_vis):
fold_ids.append(
 [
ids[df['fullVisitorId'].isin(unique_vis[trn_vis])],
ids[df['fullVisitorId'].isin(unique_vis[val_vis])]
] 
)
 return fold_ids
y_reg = train['totals.transactionRevenue'].fillna(0)
 del train['totals.transactionRevenue']
if 'totals.transactionRevenue' in test.columns:
del test['totals.transactionRevenue']
excluded_features = [
'date', 'date_new', 'fullVisitorId', 'sessionId', 'totals.transactionRevenue',
'visitId', 'visitStartTime'
]
categorical_features = [
_f for _f in train.columns
if(_f not in excluded_features)&(train[_f].dtype == 'object')
 ]
for f in categorical_features:
train[f], indexer = pd.factorize(train[f])
 test[f] = indexer.get_indexer(test[f])
 for df in [train, test]:
df['hits/pageviews'] =(df["totals.pageviews"]/(df["totals.hits"])).apply(lambda x: 0 if np.isinf(x)else x)
 df['is_high_hits'] = np.logical_or(df["totals.hits"]>4,df["totals.pageviews"]>4 ).astype(np.int32)
 df["Revenue"] = np.log1p(df["Revenue"])
 warnings.simplefilter(action='ignore', category=FutureWarning)
 folds = get_folds(df=train, n_splits=2)
 y_reg = y_reg.astype(float)
 train_features = [_f for _f in train.columns if _f not in excluded_features]
print(train_features)
 importances = pd.DataFrame()
oof_reg_preds = np.zeros(train.shape[0])
 sub_reg_preds = np.zeros(test.shape[0])
 for fold_,(trn_, val_)in enumerate(folds):
trn_x, trn_y = train[train_features].iloc[trn_], y_reg.iloc[trn_]
val_x, val_y = train[train_features].iloc[val_], y_reg.iloc[val_]
reg = lgb.LGBMRegressor(
 num_leaves=31,
learning_rate=0.03,
n_estimators=1000,
subsample=.9,
colsample_bytree=.9,
random_state=1 
)
 reg.fit(
 trn_x, np.log1p(trn_y),
eval_set=[(val_x, np.log1p(val_y)) ],
early_stopping_rounds=100,
verbose=100,
eval_metric='rmse' 
)
 imp_df = pd.DataFrame()
imp_df['feature'] = train_features
imp_df['gain'] = reg.booster_.feature_importance(importance_type='gain')
 imp_df['fold'] = fold_ + 1
importances = pd.concat([importances, imp_df], axis=0, sort=False)
 oof_reg_preds[val_] = reg.predict(val_x, num_iteration=reg.best_iteration_)
 oof_reg_preds[oof_reg_preds < 0] = 0
_preds = reg.predict(test[train_features], num_iteration=reg.best_iteration_)
 _preds[_preds < 0] = 0
sub_reg_preds += np.expm1(_preds)/ len(folds)
 print(mean_squared_error(np.log1p(y_reg), oof_reg_preds)**.5)
 train['predictions'] = np.expm1(oof_reg_preds)
 test['predictions'] = sub_reg_preds
trn_data = train[train_features + ['fullVisitorId']].groupby('fullVisitorId' ).mean()
trn_pred_list = train[['fullVisitorId', 'predictions']].groupby('fullVisitorId')\ 
.apply(lambda df: list(df.predictions)) \ 
.apply(lambda x: {'pred_'+str(i): pred for i, pred in enumerate(x)})
 trn_all_predictions = pd.DataFrame(list(trn_pred_list.values), index=trn_data.index)
 trn_feats = trn_all_predictions.columns
with warnings.catch_warnings() :
warnings.simplefilter("ignore")
 trn_all_predictions['t_mean'] = np.log1p(trn_all_predictions[trn_feats].mean(axis=1))
trn_all_predictions['t_median'] = np.log1p(trn_all_predictions[trn_feats].median(axis=1))
trn_all_predictions['t_sum_log'] = np.log1p(trn_all_predictions[trn_feats] ).sum(axis=1)
 trn_all_predictions['t_sum_act'] = np.log1p(trn_all_predictions[trn_feats].fillna(0 ).sum(axis=1))
trn_all_predictions['t_nb_sess'] = trn_all_predictions[trn_feats].isnull().sum(axis=1)
 full_data = pd.concat([trn_data, trn_all_predictions], axis=1)
 del trn_data, trn_all_predictions
gc.collect()
print(full_data.shape)
 sub_pred_list = test[['fullVisitorId', 'predictions']].groupby('fullVisitorId')\ 
.apply(lambda df: list(df.predictions)) \ 
.apply(lambda x: {'pred_'+str(i): pred for i, pred in enumerate(x)})
 sub_data = test[train_features + ['fullVisitorId']].groupby('fullVisitorId' ).mean()
sub_all_predictions = pd.DataFrame(list(sub_pred_list.values), index=sub_data.index)
 with warnings.catch_warnings() :
warnings.simplefilter("ignore")
 for f in trn_feats:
if f not in sub_all_predictions.columns:
sub_all_predictions[f] = np.nan
sub_all_predictions['t_mean'] = np.log1p(sub_all_predictions[trn_feats].mean(axis=1))
sub_all_predictions['t_median'] = np.log1p(sub_all_predictions[trn_feats].median(axis=1))
sub_all_predictions['t_sum_log'] = np.log1p(sub_all_predictions[trn_feats] ).sum(axis=1)
 sub_all_predictions['t_sum_act'] = np.log1p(sub_all_predictions[trn_feats].fillna(0 ).sum(axis=1))
sub_all_predictions['t_nb_sess'] = sub_all_predictions[trn_feats].isnull().sum(axis=1)
 sub_full_data = pd.concat([sub_data, sub_all_predictions], axis=1)
 del sub_data, sub_all_predictions
gc.collect()
print(sub_full_data.shape)
 train['target'] = y_reg
trn_user_target = train[['fullVisitorId', 'target']].groupby('fullVisitorId' ).sum()
warnings.simplefilter(action='ignore', category=FutureWarning)
 folds = get_folds(df=full_data[['totals.pageviews']].reset_index() , n_splits=2)
 oof_preds = np.zeros(full_data.shape[0])
 sub_preds = np.zeros(sub_full_data.shape[0])
 vis_importances = pd.DataFrame()
for fold_,(trn_, val_)in enumerate(folds):
trn_x, trn_y = full_data.iloc[trn_], trn_user_target['target'].iloc[trn_]
val_x, val_y = full_data.iloc[val_], trn_user_target['target'].iloc[val_]
reg = lgb.LGBMRegressor(
 num_leaves=31,
learning_rate=0.03,
n_estimators=1000,
subsample=.9,
colsample_bytree=.9,
random_state=1 
)
 reg.fit(
 trn_x, np.log1p(trn_y),
eval_set=[(trn_x, np.log1p(trn_y)) ,(val_x, np.log1p(val_y)) ],
eval_names=['TRAIN', 'VALID'],
early_stopping_rounds=100,
eval_metric='rmse',
verbose=100 
)
 imp_df = pd.DataFrame()
imp_df['feature'] = trn_x.columns
imp_df['gain'] = reg.booster_.feature_importance(importance_type='gain')
 imp_df['fold'] = fold_ + 1
vis_importances = pd.concat([vis_importances, imp_df], axis=0, sort=False)
 oof_preds[val_] = reg.predict(val_x, num_iteration=reg.best_iteration_)
 oof_preds[oof_preds < 0] = 0
_preds = reg.predict(sub_full_data[full_data.columns], num_iteration=reg.best_iteration_)
 _preds[_preds < 0] = 0
sub_preds += _preds / len(folds)
 print(mean_squared_error(np.log1p(trn_user_target['target']), oof_preds)**.5)
 sub_full_data['PredictedLogRevenue'] = sub_preds
sub_full_data[['PredictedLogRevenue']].to_csv('submission_new.csv', index=True )<set_options>
features["Date"] = features["Date"].apply(lambda x: x.replace("-",""))
features["Date"] = features["Date"].astype(int)
 features.head()
warnings.simplefilter('error', SettingWithCopyWarning)
 gc.enable()<load_from_csv>
test_features["Date"] = test_features["Date"].apply(lambda x: x.replace("-",""))
test_features["Date"] = test_features["Date"].astype(int)
 test_features.head()
train = pd.read_csv('.. /input/create-extracted-json-fields-dataset/extracted_fields_train.gz',
dtype={'date': str, 'fullVisitorId': str, 'sessionId':str}, nrows=None)
 test = pd.read_csv('.. /input/create-extracted-json-fields-dataset/extracted_fields_test.gz',
dtype={'date': str, 'fullVisitorId': str, 'sessionId':str}, nrows=None )<prepare_x_and_y>
train1_x = features[['Lat','Long','Date']]
train1_y = features[['ConfirmedCases']]
test1_x = test_features[['Lat','Long','Date']]
y_reg = train['totals.transactionRevenue'].fillna(0)
 del train['totals.transactionRevenue']
if 'totals.transactionRevenue' in test.columns:
del test['totals.transactionRevenue']<data_type_conversions>
model = RandomForestClassifier(max_depth=200, random_state=1)
 model.fit(train1_x, train1_y)
 conf_cases = model.predict(test1_x )
train['target'] = y_reg
for df in [train, test]:
df['vis_date'] = pd.to_datetime(df['visitStartTime'], unit='s')
 df['sess_date_dow'] = df['vis_date'].dt.dayofweek
df['sess_date_hours'] = df['vis_date'].dt.hour
df['sess_date_dom'] = df['vis_date'].dt.day
df.sort_values(['fullVisitorId', 'vis_date'], ascending=True, inplace=True)
 df['next_session_1'] =(
 df['vis_date'] - df[['fullVisitorId', 'vis_date']].groupby('fullVisitorId')['vis_date'].shift(1)
 ).astype(np.int64)// 1e9 // 60 // 60
df['next_session_2'] =(
 df['vis_date'] - df[['fullVisitorId', 'vis_date']].groupby('fullVisitorId')['vis_date'].shift(-1)
 ).astype(np.int64)// 1e9 // 60 // 60
df['number_of_pageviews'] = df['date'].map(
 df[['date', 'totals.pageviews']].groupby('date')['totals.pageviews'].sum() 
)
 df['ratio_pageviews'] = df['totals.pageviews'] / df['number_of_pageviews']
df['number_of_sessions'] = df['date'].map(
 df[['date']].groupby('date' ).size() 
)
 y_reg = train['target']
del train['target']<define_variables>
train2_x = features[['Lat','Long','Date']]
train2_y = features[['Fatalities']]
test2_x = test_features[['Lat','Long','Date']]
mobile_browsers = ['amazon', 'android', 'blackberry', 'browser', 'chrome', 'in-app', 'iphone', 'konqueror', 'lunascape',
'mini', 'mozilla', 'netscape', 'nokia', 'playstation', 'puffin', 'samsung']
browsers = ['chrome', 'coc coc', 'edge', 'firefox', 'internet explorer', 'iron', 'maxthon', 'opera', 'safari']
key_sources = ['ad', 'amazon', 'arstechnica', 'ask', 'baidu', 'baidu', 'bing', 'businessinsider', 'dealspotr',
'dfa', 'direct', 'duckduckgo', 'edu', 'facebook', 'feedly', 'flipboard', 'github', 'golang', 'google',
'gophergala', 'linkedin', 'lunametrics', 'mail', 'mdn', 'messenger', 'metrics', 'msn', 'mysearch',
'outlook', 'partners', 'phandroid', 'pinterest', 'qiita', 'reddit', 'sashihara', 'search', 'seroundtable',
'siliconvalley', 'sm.cn', 'sogou', 'squishable', 't-online.de', 't.co', 'twitter', 'vk.com', 'wow',
'yahoo', 'yandex', 'youtube']
def browser_mapping(x):
if x in browsers:
return x
elif any([word in x for word in mobile_browsers]):
return 'mobile_browser'
elif '(not set)' in x:
return 'nan'
else:
return 'others'
def adcontents_mapping(x):
if 'google' in x:
return 'google'
elif '(not set)' in x or 'nan' in x:
return 'nan'
elif 'ad' in x:
return 'ad'
else:
return 'others'
def source_mapping(x):
for word in key_sources:
if word in x:
return word
if '(not set)' in x or 'nan' in x:
return 'nan'
else:
return 'others'
train['device.browser'] = train['device.browser'].map(lambda x:browser_mapping(str(x ).lower())).astype('str')
 train['trafficSource.adContent'] = train['trafficSource.adContent'].map(lambda x:adcontents_mapping(str(x ).lower())).astype('str')
 train['trafficSource.source'] = train['trafficSource.source'].map(lambda x:source_mapping(str(x ).lower())).astype('str')
 test['device.browser'] = test['device.browser'].map(lambda x:browser_mapping(str(x ).lower())).astype('str')
 test['trafficSource.adContent'] = test['trafficSource.adContent'].map(lambda x:adcontents_mapping(str(x ).lower())).astype('str')
 test['trafficSource.source'] = test['trafficSource.source'].map(lambda x:source_mapping(str(x ).lower())).astype('str')
 def process_device(data_df):
print("process device...")
 data_df['source.country'] = data_df['trafficSource.source'] + '_' + data_df['geoNetwork.country']
data_df['campaign.medium'] = data_df['trafficSource.campaign'] + '_' + data_df['trafficSource.medium']
data_df['browser.category'] = data_df['device.browser'] + '_' + data_df['device.deviceCategory']
data_df['browser.os'] = data_df['device.browser'] + '_' + data_df['device.operatingSystem']
return data_df
train = process_device(train)
 test = process_device(test)
 def custom(data):
print('custom.. ')
 data['device_deviceCategory_channelGrouping'] = data['device.deviceCategory'] + "_" + data['channelGrouping']
data['channelGrouping_browser'] = data['device.browser'] + "_" + data['channelGrouping']
data['channelGrouping_OS'] = data['device.operatingSystem'] + "_" + data['channelGrouping']
for i in ['geoNetwork.city', 'geoNetwork.continent', 'geoNetwork.country','geoNetwork.metro', 'geoNetwork.networkDomain', 'geoNetwork.region','geoNetwork.subContinent']:
for j in ['device.browser','device.deviceCategory', 'device.operatingSystem', 'trafficSource.source']:
data[i + "_" + j] = data[i] + "_" + data[j]
data['content.source'] = data['trafficSource.adContent'] + "_" + data['source.country']
data['medium.source'] = data['trafficSource.medium'] + "_" + data['source.country']
return data
train = custom(train)
 test = custom(test )<define_variables>
model = RandomForestClassifier(max_depth=200, random_state=1)
 model.fit(train2_x, train2_y)
 fatalities = model.predict(test2_x )
excluded_features = [
'date', 'fullVisitorId', 'sessionId', 'totals.transactionRevenue',
'visitId', 'visitStartTime', 'vis_date', 'nb_sessions', 'max_visits'
]
categorical_features = [
_f for _f in train.columns
if(_f not in excluded_features)&(train[_f].dtype == 'object')
 ]<feature_engineering>
Sub = pd.read_csv('.. /input/covid19-global-forecasting-week-1/submission.csv')
 output = pd.DataFrame({
'ForecastId': Sub['ForecastId'],
'ConfirmedCases': conf_cases,
'Fatalities': fatalities
} )
<prepare_x_and_y><EOS> 
output.to_csv('submission.csv', index=False )