Kaggle competition: COVID19 Global Forecasting (Week 1)
sub_full_data['PredictedLogRevenue'] = sub_preds
sub_full_data[['PredictedLogRevenue']].to_csv('new_test.csv', index=True )<set_options>
warnings.simplefilter('error', SettingWithCopyWarning)
 gc.enable()
%matplotlib inline<load_from_csv>
df = pd.read_csv("/kaggle/input/covid19-global-forecasting-week-1/train.csv",index_col='Id')
 test = pd.read_csv("/kaggle/input/covid19-global-forecasting-week-1/test.csv")
 e2= set(df['Country/Region'])
 df['ConfirmedCases']=df['ConfirmedCases'].astype(int)
 df['Fatalities']=df['Fatalities'].astype(int)
 df["Date"] = pd.to_datetime(df["Date"])
 df['Weekday']= df.apply(lambda row: row["Date"].weekday() ,axis=1)
 df["Weekday"] =(df["Weekday"] < 5 ).astype(int)
 print(df.shape)
train = pd.read_csv('.. /input/create-extracted-json-fields-dataset/extracted_fields_train.gz',
dtype={'date': str, 'fullVisitorId': str, 'sessionId':str}, nrows=None)
 test = pd.read_csv('.. /input/create-extracted-json-fields-dataset/extracted_fields_test.gz',
dtype={'date': str, 'fullVisitorId': str, 'sessionId':str}, nrows=None)
 train.shape, test.shape<prepare_x_and_y>
df['ConfirmedCases']=df['ConfirmedCases'].astype(int)
 df['Fatalities']=df['Fatalities'].astype(int)
 df["Date"] = pd.to_datetime(df["Date"])
 df['Weekday']= df.apply(lambda row: row["Date"].weekday() ,axis=1)
 df["Weekday"] =(df["Weekday"] < 5 ).astype(int )
y_reg = train['totals.transactionRevenue'].fillna(0)
 del train['totals.transactionRevenue']
if 'totals.transactionRevenue' in test.columns:
del test['totals.transactionRevenue']<feature_engineering>
df.isnull().sum()
for df in [train, test]:
df['date'] = pd.to_datetime(df['visitStartTime'], unit='s')
 df['sess_date_dow'] = df['date'].dt.dayofweek
df['sess_date_hours'] = df['date'].dt.hour
df['sess_date_dom'] = df['date'].dt.day<categorify>
df.groupby(['Date','Country/Region'] ).first()
def browser_mapping(x):
browsers = ['chrome','safari','firefox','internet explorer','edge','opera','coc coc','maxthon','iron']
if x in browsers:
return x.lower()
elif('android' in x)or('samsung' in x)or('mini' in x)or('iphone' in x)or('in-app' in x)or('playstation' in x):
return 'mobile browser'
elif('mozilla' in x)or('chrome' in x)or('blackberry' in x)or('nokia' in x)or('browser' in x)or('amazon' in x):
return 'mobile browser'
elif('lunascape' in x)or('netscape' in x)or('blackberry' in x)or('konqueror' in x)or('puffin' in x)or('amazon' in x):
return 'mobile browser'
elif '(not set)' in x:
return x
else:
return 'others'
def adcontents_mapping(x):
if('google' in x):
return 'google'
elif('placement' in x)|('placememnt' in x):
return 'placement'
elif '(not set)' in x or 'nan' in x:
return x
elif 'ad' in x:
return 'ad'
else:
return 'others'
def source_mapping(x):
if('google' in x):
return 'google'
elif('youtube' in x):
return 'youtube'
elif '(not set)' in x or 'nan' in x:
return x
elif 'yahoo' in x:
return 'yahoo'
elif 'facebook' in x:
return 'facebook'
elif 'reddit' in x:
return 'reddit'
elif 'bing' in x:
return 'bing'
elif 'quora' in x:
return 'quora'
elif 'outlook' in x:
return 'outlook'
elif 'linkedin' in x:
return 'linkedin'
elif 'pinterest' in x:
return 'pinterest'
elif 'ask' in x:
return 'ask'
elif 'siliconvalley' in x:
return 'siliconvalley'
elif 'lunametrics' in x:
return 'lunametrics'
elif 'amazon' in x:
return 'amazon'
elif 'mysearch' in x:
return 'mysearch'
elif 'qiita' in x:
return 'qiita'
elif 'messenger' in x:
return 'messenger'
elif 'twitter' in x:
return 'twitter'
elif 't.co' in x:
return 't.co'
elif 'vk.com' in x:
return 'vk.com'
elif 'search' in x:
return 'search'
elif 'edu' in x:
return 'edu'
elif 'mail' in x:
return 'mail'
elif 'ad' in x:
return 'ad'
elif 'golang' in x:
return 'golang'
elif 'direct' in x:
return 'direct'
elif 'dealspotr' in x:
return 'dealspotr'
elif 'sashihara' in x:
return 'sashihara'
elif 'phandroid' in x:
return 'phandroid'
elif 'baidu' in x:
return 'baidu'
elif 'mdn' in x:
return 'mdn'
elif 'duckduckgo' in x:
return 'duckduckgo'
elif 'seroundtable' in x:
return 'seroundtable'
elif 'metrics' in x:
return 'metrics'
elif 'sogou' in x:
return 'sogou'
elif 'businessinsider' in x:
return 'businessinsider'
elif 'github' in x:
return 'github'
elif 'gophergala' in x:
return 'gophergala'
elif 'yandex' in x:
return 'yandex'
elif 'msn' in x:
return 'msn'
elif 'dfa' in x:
return 'dfa'
elif '(not set)' in x:
return '(not set)'
elif 'feedly' in x:
return 'feedly'
elif 'arstechnica' in x:
return 'arstechnica'
elif 'squishable' in x:
return 'squishable'
elif 'flipboard' in x:
return 'flipboard'
elif 't-online.de' in x:
return 't-online.de'
elif 'sm.cn' in x:
return 'sm.cn'
elif 'wow' in x:
return 'wow'
elif 'baidu' in x:
return 'baidu'
elif 'partners' in x:
return 'partners'
else:
return 'others'
train['device.browser'] = train['device.browser'].map(lambda x:browser_mapping(str(x ).lower())).astype('str')
 train['trafficSource.adContent'] = train['trafficSource.adContent'].map(lambda x:adcontents_mapping(str(x ).lower())).astype('str')
 train['trafficSource.source'] = train['trafficSource.source'].map(lambda x:source_mapping(str(x ).lower())).astype('str')
 test['device.browser'] = test['device.browser'].map(lambda x:browser_mapping(str(x ).lower())).astype('str')
 test['trafficSource.adContent'] = test['trafficSource.adContent'].map(lambda x:adcontents_mapping(str(x ).lower())).astype('str')
 test['trafficSource.source'] = test['trafficSource.source'].map(lambda x:source_mapping(str(x ).lower())).astype('str')
 def process_device(data_df):
print("process device...")
 data_df['source.country'] = data_df['trafficSource.source'] + '_' + data_df['geoNetwork.country']
data_df['campaign.medium'] = data_df['trafficSource.campaign'] + '_' + data_df['trafficSource.medium']
data_df['browser.category'] = data_df['device.browser'] + '_' + data_df['device.deviceCategory']
data_df['browser.os'] = data_df['device.browser'] + '_' + data_df['device.operatingSystem']
return data_df
train = process_device(train)
 test = process_device(test)
 def custom(data):
print('custom.. ')
 data['device_deviceCategory_channelGrouping'] = data['device.deviceCategory'] + "_" + data['channelGrouping']
data['channelGrouping_browser'] = data['device.browser'] + "_" + data['channelGrouping']
data['channelGrouping_OS'] = data['device.operatingSystem'] + "_" + data['channelGrouping']
for i in ['geoNetwork.city', 'geoNetwork.continent', 'geoNetwork.country','geoNetwork.metro', 'geoNetwork.networkDomain', 'geoNetwork.region','geoNetwork.subContinent']:
for j in ['device.browser','device.deviceCategory', 'device.operatingSystem', 'trafficSource.source']:
data[i + "_" + j] = data[i] + "_" + data[j]
data['content.source'] = data['trafficSource.adContent'] + "_" + data['source.country']
data['medium.source'] = data['trafficSource.medium'] + "_" + data['source.country']
return data
train = custom(train)
 test = custom(test )<define_variables>
df_public = df[df["Date"]<"2020-03-12"]
df_public.groupby(['Date','Lat','Long'], as_index=False ).agg({'ConfirmedCases': 'sum', 'Fatalities': 'sum', 'Weekday': 'first'} )
excluded_features = [
'date', 'fullVisitorId', 'sessionId', 'totals.transactionRevenue',
'visitId', 'visitStartTime'
]
categorical_features = [
_f for _f in train.columns
if(_f not in excluded_features)&(train[_f].dtype == 'object')
 ]<feature_engineering>
def preprocessing(dataframe):
z=dataframe['Date']-df['Date'].min()
for i in z.index:
z[i]=int(str(z[i] ).split() [0])
 data=dataframe
x =data[['Lat', 'Long', 'Date','Weekday']]
y1 = data[['ConfirmedCases']]
y2 = data[['Fatalities']]
x_test = test[['Lat', 'Long', 'Date']]
x_test["Date"] = pd.to_datetime(x_test["Date"])
 return z,x,y1,y2
for f in categorical_features:
train[f], indexer = pd.factorize(train[f])
 test[f] = indexer.get_indexer(test[f] )<train_model>
z=df_public['Date']-df_public['Date'].min()
for i in z.index:
z[i]=int(str(z[i] ).split() [0] )
params={'learning_rate': 0.03,
'objective':'regression',
'metric':'rmse',
'num_leaves': 31,
'verbose': 1,
"subsample": 0.99,
"colsample_bytree": 0.99,
"random_state":42,
'max_depth': 15,
'lambda_l2': 0.02085548700474218,
'lambda_l1': 0.004107624022751344,
'bagging_fraction': 0.7934712636944741,
'feature_fraction': 0.686612409641711,
'min_child_samples': 21
} 
<train_model>
data=df_public
x =data[['Lat', 'Long', 'Date','Weekday']]
y1 = data[['ConfirmedCases']]
y2 = data[['Fatalities']]
x_test = test[['Lat', 'Long', 'Date']]
x_test["Date"] = pd.to_datetime(x_test["Date"] )
', params)
 Best num_boost_round:', len(cv_results['rmse-mean'])) 
<prepare_x_and_y>
x_test["Weekday"]= x_test.apply(lambda row: row["Date"].weekday() ,axis=1)
 x_test["Weekday"] =(x_test["Weekday"] < 5 ).astype(int )
folds = get_folds(df=train, n_splits=5)
 train_features = [_f for _f in train.columns if _f not in excluded_features]
importances = pd.DataFrame()
oof_reg_preds = np.zeros(train.shape[0])
 sub_reg_preds = np.zeros(test.shape[0])
 for fold_,(trn_, val_)in enumerate(folds):
print("Fold:",fold_)
 trn_x, trn_y = train[train_features].iloc[trn_], y_reg.iloc[trn_]
val_x, val_y = train[train_features].iloc[val_], y_reg.iloc[val_]
reg = lgb.LGBMRegressor(**params,
n_estimators=1000 
)
 reg.fit(
 trn_x, np.log1p(trn_y),
eval_set=[(val_x, np.log1p(val_y)) ],
early_stopping_rounds=50,
verbose=100,
eval_metric='rmse' 
)
 imp_df = pd.DataFrame()
imp_df['feature'] = train_features
imp_df['gain'] = reg.booster_.feature_importance(importance_type='gain')
 imp_df['fold'] = fold_ + 1
importances = pd.concat([importances, imp_df], axis=0, sort=False)
 oof_reg_preds[val_] = reg.predict(val_x, num_iteration=reg.best_iteration_)
 oof_reg_preds[oof_reg_preds < 0] = 0
_preds = reg.predict(test[train_features], num_iteration=reg.best_iteration_)
 _preds[_preds < 0] = 0
sub_reg_preds += np.expm1(_preds)/ len(folds)
 mean_squared_error(np.log1p(y_reg), oof_reg_preds)**.5<prepare_output>
c=z.max() +1
y=x_test['Date']-x_test['Date'].min()
for i in y.index:
y[i]=int(str(y[i] ).split() [0])+c
train['predictions'] = np.expm1(oof_reg_preds)
 test['predictions'] = sub_reg_preds<groupby>
x['Date']=z
x_test['Date']=y
trn_data = train[train_features + ['fullVisitorId']].groupby('fullVisitorId' ).mean()<groupby>
x_test2=x_test.drop(['Weekday'],axis=1)
 x2=x.drop(['Weekday'],axis=1 )
%%time
trn_pred_list = train[['fullVisitorId', 'predictions']].groupby('fullVisitorId')\ 
.apply(lambda df: list(df.predictions)) \ 
.apply(lambda x: {'pred_'+str(i): pred for i, pred in enumerate(x)} )<prepare_output>
from sklearn.tree import DecisionTreeClassifier
from sklearn.neighbors import KNeighborsClassifier
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import KFold, cross_val_score, train_test_split
trn_all_predictions = pd.DataFrame(list(trn_pred_list.values), index=trn_data.index)
 trn_feats = trn_all_predictions.columns
trn_all_predictions['t_mean'] = np.log1p(trn_all_predictions[trn_feats].mean(axis=1))
trn_all_predictions['t_median'] = np.log1p(trn_all_predictions[trn_feats].median(axis=1))
trn_all_predictions['t_sum_log'] = np.log1p(trn_all_predictions[trn_feats] ).sum(axis=1)
 trn_all_predictions['t_sum_act'] = np.log1p(trn_all_predictions[trn_feats].fillna(0 ).sum(axis=1))
trn_all_predictions['t_nb_sess'] = trn_all_predictions[trn_feats].isnull().sum(axis=1)
 full_data = pd.concat([trn_data, trn_all_predictions], axis=1)
 del trn_data, trn_all_predictions
gc.collect()
full_data.shape<groupby>
scaler = StandardScaler()
X_scaled = scaler.fit_transform(x)
 x_test_scaled = scaler.transform(x_test)
 X_train, X_test, y_train, y_test = train_test_split(X_scaled, y1, test_size=0.2, random_state=44 )
%%time
sub_pred_list = test[['fullVisitorId', 'predictions']].groupby('fullVisitorId')\ 
.apply(lambda df: list(df.predictions)) \ 
.apply(lambda x: {'pred_'+str(i): pred for i, pred in enumerate(x)} )<prepare_output>
models = []
models.append(( "RF",RandomForestClassifier()))
models.append(( "Dtree",DecisionTreeClassifier()))
models.append(( "KNN",KNeighborsClassifier()))
sub_data = test[train_features + ['fullVisitorId']].groupby('fullVisitorId' ).mean()
sub_all_predictions = pd.DataFrame(list(sub_pred_list.values), index=sub_data.index)
 for f in trn_feats:
if f not in sub_all_predictions.columns:
sub_all_predictions[f] = np.nan
sub_all_predictions['t_mean'] = np.log1p(sub_all_predictions[trn_feats].mean(axis=1))
sub_all_predictions['t_median'] = np.log1p(sub_all_predictions[trn_feats].median(axis=1))
sub_all_predictions['t_sum_log'] = np.log1p(sub_all_predictions[trn_feats] ).sum(axis=1)
 sub_all_predictions['t_sum_act'] = np.log1p(sub_all_predictions[trn_feats].fillna(0 ).sum(axis=1))
sub_all_predictions['t_nb_sess'] = sub_all_predictions[trn_feats].isnull().sum(axis=1)
 sub_full_data = pd.concat([sub_data, sub_all_predictions], axis=1)
 del sub_data, sub_all_predictions
gc.collect()
sub_full_data.shape<groupby>
for name,model in models:
kfold = KFold(n_splits=2, random_state=22)
 cv_result = cross_val_score(model,X_train,y_train, cv = kfold,scoring = "accuracy")
 print(name, cv_result )
train['target'] = y_reg
trn_user_target = train[['fullVisitorId', 'target']].groupby('fullVisitorId' ).sum()<train_model>
X_train, X_test, y_train, y_test = train_test_split(X_scaled, y2, test_size=0.2, random_state=44 )
params={'learning_rate': 0.05,
'objective':'regression',
'metric':'rmse',
'num_leaves': 200,
'verbose': 1,
"subsample": 0.99,
"colsample_bytree": 0.99,
"random_state":33,
'max_depth': 14,
'lambda_l2': 0.02085548700474218,
'lambda_l1': 0.004107624022751344,
'bagging_fraction': 0.7934712636944741,
'feature_fraction': 0.686612409641711,
'min_child_samples': 21
}
', params)
 Best num_boost_round:', len(cv_results['rmse-mean'])) 
<prepare_x_and_y>
for name,model in models:
kfold = KFold(n_splits=2, random_state=22)
 cv_result = cross_val_score(model,X_train,y_train, cv = kfold,scoring = "accuracy")
 print(name, cv_result )
folds = get_folds(df=full_data[['totals.pageviews']].reset_index() , n_splits=5)
 oof_preds = np.zeros(full_data.shape[0])
 sub_preds = np.zeros(sub_full_data.shape[0])
 vis_importances = pd.DataFrame()
for fold_,(trn_, val_)in enumerate(folds):
trn_x, trn_y = full_data.iloc[trn_], trn_user_target['target'].iloc[trn_]
val_x, val_y = full_data.iloc[val_], trn_user_target['target'].iloc[val_]
reg = lgb.LGBMRegressor(**params,
n_estimators=1500, 
)
 reg.fit(
 trn_x, np.log1p(trn_y),
eval_set=[(trn_x, np.log1p(trn_y)) ,(val_x, np.log1p(val_y)) ],
eval_names=['TRAIN', 'VALID'],
early_stopping_rounds=50,
eval_metric='rmse',
verbose=100 
)
 imp_df = pd.DataFrame()
imp_df['feature'] = trn_x.columns
imp_df['gain'] = reg.booster_.feature_importance(importance_type='gain')
 imp_df['fold'] = fold_ + 1
vis_importances = pd.concat([vis_importances, imp_df], axis=0, sort=False)
 oof_preds[val_] = reg.predict(val_x, num_iteration=reg.best_iteration_)
 oof_preds[oof_preds < 0] = 0
_preds = reg.predict(sub_full_data[full_data.columns], num_iteration=reg.best_iteration_)
 _preds[_preds < 0] = 0
sub_preds += _preds / len(folds)
 mean_squared_error(np.log1p(trn_user_target['target']), oof_preds)**.5<save_to_csv>
z,x,y1,y2 = preprocessing(df )
sub_full_data['PredictedLogRevenue'] = sub_preds
sub_full_data[['PredictedLogRevenue']].to_csv('new_test.csv', index=True )<set_options>
x['Date']=z
warnings.simplefilter('error', SettingWithCopyWarning)
 gc.enable()
%matplotlib inline<load_from_csv>
print("-----------Confirmed------
")
 do_your_thing(x,y1)
 print("-----------Fatality------
")
 do_your_thing(x,y2 )
def load_data() :
train = pd.read_csv('.. /input/create-extracted-json-fields-dataset/extracted_fields_train.gz',
dtype={'date': str, 'fullVisitorId': str, 'sessionId':str}, nrows=None)
 test = pd.read_csv('.. /input/create-extracted-json-fields-dataset/extracted_fields_test.gz',
dtype={'date': str, 'fullVisitorId': str, 'sessionId':str}, nrows=None)
 return train,test
train, test = load_data()
print("Train Shape:", train.shape)
 print("Test Shape:",test.shape )<prepare_x_and_y>
kfold = KFold(n_splits=2, shuffle = True, random_state = 0)
 scaler = StandardScaler()
X_scaled = scaler.fit_transform(x)
 X_train1, X_test1, y_train1, y_test1 = train_test_split(X_scaled, y1, test_size=0.2, random_state=44)
 best_score = 0
for n_estimators in [90, 100, 110, 120, 200]:
for max_features in [0.6, 0.7, 0.8, 0.9, 1]:
print(n_estimators,"--",max_features,"--",'
')
 rf = RandomForestClassifier(n_estimators = n_estimators, criterion = 'entropy', max_features = max_features)
 scores = cross_val_score(rf, X_train1, y_train1, cv=kfold)
 score = np.mean(scores)
 print("Score is ",score,"
")
 if score > best_score:
best_score = score
best_parameters = {'n_estimators': n_estimators, 'max_features': max_features,'criterion':'entropy'}
print("BREACHED!!-->",best_parameters,"
" )
y_reg = train['totals.transactionRevenue'].fillna(0)
 del train['totals.transactionRevenue']
if 'totals.transactionRevenue' in test.columns:
del test['totals.transactionRevenue']<feature_engineering>
print("Params: ", best_parameters )
for df in [train, test]:
df['date'] = pd.to_datetime(df['visitStartTime'], unit='s')
 df['sess_date_dow'] = df['date'].dt.dayofweek
df['sess_date_hours'] = df['date'].dt.hour
df['sess_date_dom'] = df['date'].dt.day<categorify>
scaler = StandardScaler()
X_scaled = scaler.fit_transform(x)
def browser_mapping(x):
browsers = ['chrome','safari','firefox','internet explorer','edge','opera','coc coc','maxthon','iron']
if x in browsers:
return x.lower()
elif('android' in x)or('samsung' in x)or('mini' in x)or('iphone' in x)or('in-app' in x)or('playstation' in x):
return 'mobile browser'
elif('mozilla' in x)or('chrome' in x)or('blackberry' in x)or('nokia' in x)or('browser' in x)or('amazon' in x):
return 'mobile browser'
elif('lunascape' in x)or('netscape' in x)or('blackberry' in x)or('konqueror' in x)or('puffin' in x)or('amazon' in x):
return 'mobile browser'
elif '(not set)' in x:
return x
else:
return 'others'
def adcontents_mapping(x):
if('google' in x):
return 'google'
elif('placement' in x)|('placememnt' in x):
return 'placement'
elif '(not set)' in x or 'nan' in x:
return x
elif 'ad' in x:
return 'ad'
else:
return 'others'
def source_mapping(x):
if('google' in x):
return 'google'
elif('youtube' in x):
return 'youtube'
elif '(not set)' in x or 'nan' in x:
return x
elif 'yahoo' in x:
return 'yahoo'
elif 'facebook' in x:
return 'facebook'
elif 'reddit' in x:
return 'reddit'
elif 'bing' in x:
return 'bing'
elif 'quora' in x:
return 'quora'
elif 'outlook' in x:
return 'outlook'
elif 'linkedin' in x:
return 'linkedin'
elif 'pinterest' in x:
return 'pinterest'
elif 'ask' in x:
return 'ask'
elif 'siliconvalley' in x:
return 'siliconvalley'
elif 'lunametrics' in x:
return 'lunametrics'
elif 'amazon' in x:
return 'amazon'
elif 'mysearch' in x:
return 'mysearch'
elif 'qiita' in x:
return 'qiita'
elif 'messenger' in x:
return 'messenger'
elif 'twitter' in x:
return 'twitter'
elif 't.co' in x:
return 't.co'
elif 'vk.com' in x:
return 'vk.com'
elif 'search' in x:
return 'search'
elif 'edu' in x:
return 'edu'
elif 'mail' in x:
return 'mail'
elif 'ad' in x:
return 'ad'
elif 'golang' in x:
return 'golang'
elif 'direct' in x:
return 'direct'
elif 'dealspotr' in x:
return 'dealspotr'
elif 'sashihara' in x:
return 'sashihara'
elif 'phandroid' in x:
return 'phandroid'
elif 'baidu' in x:
return 'baidu'
elif 'mdn' in x:
return 'mdn'
elif 'duckduckgo' in x:
return 'duckduckgo'
elif 'seroundtable' in x:
return 'seroundtable'
elif 'metrics' in x:
return 'metrics'
elif 'sogou' in x:
return 'sogou'
elif 'businessinsider' in x:
return 'businessinsider'
elif 'github' in x:
return 'github'
elif 'gophergala' in x:
return 'gophergala'
elif 'yandex' in x:
return 'yandex'
elif 'msn' in x:
return 'msn'
elif 'dfa' in x:
return 'dfa'
elif '(not set)' in x:
return '(not set)'
elif 'feedly' in x:
return 'feedly'
elif 'arstechnica' in x:
return 'arstechnica'
elif 'squishable' in x:
return 'squishable'
elif 'flipboard' in x:
return 'flipboard'
elif 't-online.de' in x:
return 't-online.de'
elif 'sm.cn' in x:
return 'sm.cn'
elif 'wow' in x:
return 'wow'
elif 'baidu' in x:
return 'baidu'
elif 'partners' in x:
return 'partners'
else:
return 'others'
train['device.browser'] = train['device.browser'].map(lambda x:browser_mapping(str(x ).lower())).astype('str')
 train['trafficSource.adContent'] = train['trafficSource.adContent'].map(lambda x:adcontents_mapping(str(x ).lower())).astype('str')
 train['trafficSource.source'] = train['trafficSource.source'].map(lambda x:source_mapping(str(x ).lower())).astype('str')
 test['device.browser'] = test['device.browser'].map(lambda x:browser_mapping(str(x ).lower())).astype('str')
 test['trafficSource.adContent'] = test['trafficSource.adContent'].map(lambda x:adcontents_mapping(str(x ).lower())).astype('str')
 test['trafficSource.source'] = test['trafficSource.source'].map(lambda x:source_mapping(str(x ).lower())).astype('str')
 def process_device(data_df):
print("process device...")
 data_df['source.country'] = data_df['trafficSource.source'] + '_' + data_df['geoNetwork.country']
data_df['campaign.medium'] = data_df['trafficSource.campaign'] + '_' + data_df['trafficSource.medium']
data_df['browser.category'] = data_df['device.browser'] + '_' + data_df['device.deviceCategory']
data_df['browser.os'] = data_df['device.browser'] + '_' + data_df['device.operatingSystem']
return data_df
train = process_device(train)
 test = process_device(test)
 def custom(data):
print('custom.. ')
 data['device_deviceCategory_channelGrouping'] = data['device.deviceCategory'] + "_" + data['channelGrouping']
data['channelGrouping_browser'] = data['device.browser'] + "_" + data['channelGrouping']
data['channelGrouping_OS'] = data['device.operatingSystem'] + "_" + data['channelGrouping']
for i in ['geoNetwork.city', 'geoNetwork.continent', 'geoNetwork.country','geoNetwork.metro', 'geoNetwork.networkDomain', 'geoNetwork.region','geoNetwork.subContinent']:
for j in ['device.browser','device.deviceCategory', 'device.operatingSystem', 'trafficSource.source']:
data[i + "_" + j] = data[i] + "_" + data[j]
data['content.source'] = data['trafficSource.adContent'] + "_" + data['source.country']
data['medium.source'] = data['trafficSource.medium'] + "_" + data['source.country']
return data
train = custom(train)
 test = custom(test )<define_variables>
rf = RandomForestClassifier(**best_parameters)
 rf.fit(X_scaled,y1 )
excluded_features = [
'date', 'fullVisitorId', 'sessionId', 'totals.transactionRevenue',
'visitId', 'visitStartTime'
]
categorical_features = [
_f for _f in train.columns
if(_f not in excluded_features)&(train[_f].dtype == 'object')
 ]<feature_engineering>
test_scaled = scaler.transform(x_test )
for f in categorical_features:
train[f], indexer = pd.factorize(train[f])
 test[f] = indexer.get_indexer(test[f] )<split>
y_pred_confirmed = rf.predict(test_scaled )
train_features = [_f for _f in train.columns if _f not in excluded_features]
X_train, y_train = train[train_features], y_reg<init_hyperparams>
predictions = pd.DataFrame({'ForecastId':test['ForecastId'],'ConfirmedCases':y_pred_confirmed})
 predictions.head()
params={'learning_rate': 0.01,
'objective':'regression',
'metric':'rmse',
'num_leaves': 31,
'verbose': 1,
'bagging_fraction': 0.9,
'feature_fraction': 0.9,
"random_state":42,
'max_depth': 5,
"bagging_seed" : 42,
"verbosity" : -1,
"bagging_frequency" : 5,
'lambda_l2': 0.5,
'lambda_l1': 0.5,
'min_child_samples': 36
}
xgb_params = {
'objective': 'reg:linear',
'booster': 'gbtree',
'learning_rate': 0.02,
'max_depth': 22,
'min_child_weight': 57,
'gamma' : 1.45,
'alpha': 0.0,
'lambda': 0.0,
'subsample': 0.67,
'colsample_bytree': 0.054,
'colsample_bylevel': 0.50,
'n_jobs': -1,
'random_state': 456
}
cat_param = {
'learning_rate' :0.03,
'depth' :10,
'eval_metric' :'RMSE',
'od_type' :'Iter',
'metric_period ' : 50,
'od_wait' : 20,
'seed' : 42
}<prepare_x_and_y>
print("Params: ", best_parameters )
folds = get_folds(df=train, n_splits=5)
 train_features = [_f for _f in train.columns if _f not in excluded_features]
importances = pd.DataFrame()
oof_reg_preds = np.zeros(train.shape[0])
 oof_reg_preds1 = np.zeros(train.shape[0])
 oof_reg_preds2 = np.zeros(train.shape[0])
 merge_pred = np.zeros(train.shape[0])
 sub_reg_preds = np.zeros(test.shape[0])
 for fold_,(trn_, val_)in enumerate(folds):
print("Fold:",fold_)
 trn_x, trn_y = train[train_features].iloc[trn_], y_reg.iloc[trn_]
val_x, val_y = train[train_features].iloc[val_], y_reg.iloc[val_]
reg = lgb.LGBMRegressor(**params,n_estimators=1100)
 xgb = XGBRegressor(**xgb_params, n_estimators=1000)
 cat = CatBoostRegressor(iterations=1000,learning_rate=0.03,
depth=10,
eval_metric='RMSE',
random_seed = 42,
bagging_temperature = 0.2,
od_type='Iter',
metric_period = 50,
od_wait=20)
 print("-"* 20 + "LightGBM Training" + "-"* 20)
 reg.fit(trn_x, np.log1p(trn_y),eval_set=[(val_x, np.log1p(val_y)) ],early_stopping_rounds=50,verbose=100,eval_metric='rmse')
 print("-"* 20 + "XGboost Training" + "-"* 20)
 xgb.fit(trn_x, np.log1p(trn_y),eval_set=[(val_x, np.log1p(val_y)) ],early_stopping_rounds=50,eval_metric='rmse',verbose=100)
 print("-"* 20 + "Catboost Training" + "-"* 20)
 cat.fit(trn_x, np.log1p(trn_y), eval_set=[(val_x, np.log1p(val_y)) ],early_stopping_rounds=50,use_best_model=True,verbose=100)
 imp_df = pd.DataFrame()
imp_df['feature'] = train_features
imp_df['gain_reg'] = reg.booster_.feature_importance(importance_type='gain')
 imp_df['fold'] = fold_ + 1
importances = pd.concat([importances, imp_df], axis=0, sort=False)
 oof_reg_preds[val_] = reg.predict(val_x, num_iteration=reg.best_iteration_)
 oof_reg_preds[oof_reg_preds < 0] = 0
lgb_preds = reg.predict(test[train_features], num_iteration=reg.best_iteration_)
 lgb_preds[lgb_preds < 0] = 0
oof_reg_preds1[val_] = xgb.predict(val_x)
 oof_reg_preds1[oof_reg_preds1 < 0] = 0
xgb_preds = xgb.predict(test[train_features])
 xgb_preds[xgb_preds < 0] = 0
oof_reg_preds2[val_] = cat.predict(val_x)
 oof_reg_preds1[oof_reg_preds2 < 0] = 0
cat_preds = cat.predict(test[train_features])
 cat_preds[xgb_preds < 0] = 0
merge_pred[val_] = oof_reg_preds[val_] * 0.6 + oof_reg_preds1[val_] * 0.3 +oof_reg_preds2[val_] * 0.1
sub_reg_preds += np.expm1(_preds)/ len(folds)
 sub_preds +=(lgb_preds / len(folds)) * 0.6 +(xgb_preds / len(folds)) * 0.3 +(cat_preds / len(folds)) * 0.1
print("LGBM ", mean_squared_error(np.log1p(trn_user_target['target']), oof_reg_preds)**.5)
 print("XGBoost ", mean_squared_error(np.log1p(trn_user_target['target']), oof_reg_preds1)**.5)
 print("CatBoost ", mean_squared_error(np.log1p(trn_user_target['target']), oof_reg_preds2)**.5 )<prepare_output>
scaler = StandardScaler()
X_scaled = scaler.fit_transform(x)
 rf = RandomForestClassifier(**best_parameters)
 rf.fit(X_scaled,y2 )
train['predictions'] = np.expm1(oof_reg_preds)
 test['predictions'] = sub_reg_preds<groupby>
test_scaled = scaler.transform(x_test )
trn_data = train[train_features + ['fullVisitorId']].groupby('fullVisitorId' ).mean()<groupby>
y_pred_fatal = rf.predict(test_scaled )
%%time
trn_pred_list = train[['fullVisitorId', 'predictions']].groupby('fullVisitorId')\ 
.apply(lambda df: list(df.predictions)) \ 
.apply(lambda x: {'pred_'+str(i): pred for i, pred in enumerate(x)} )<prepare_output>
predictions = pd.DataFrame({'ForecastId':test['ForecastId'],'ConfirmedCases':y_pred_confirmed,'Fatalities':y_pred_fatal})
 predictions.head()
trn_all_predictions = pd.DataFrame(list(trn_pred_list.values), index=trn_data.index)
 trn_feats = trn_all_predictions.columns
trn_all_predictions['t_mean'] = np.log1p(trn_all_predictions[trn_feats].mean(axis=1))
trn_all_predictions['t_median'] = np.log1p(trn_all_predictions[trn_feats].median(axis=1))
trn_all_predictions['t_sum_log'] = np.log1p(trn_all_predictions[trn_feats] ).sum(axis=1)
 trn_all_predictions['t_sum_act'] = np.log1p(trn_all_predictions[trn_feats].fillna(0 ).sum(axis=1))
trn_all_predictions['t_nb_sess'] = trn_all_predictions[trn_feats].isnull().sum(axis=1)
 full_data = pd.concat([trn_data, trn_all_predictions], axis=1)
 del trn_data, trn_all_predictions
gc.collect()
full_data.shape<groupby>
')
 ')
%%time
sub_pred_list = test[['fullVisitorId', 'predictions']].groupby('fullVisitorId')\ 
.apply(lambda df: list(df.predictions)) \ 
.apply(lambda x: {'pred_'+str(i): pred for i, pred in enumerate(x)} )<prepare_output>
<groupby><EOS> 
predictions.to_csv('submission.csv', header=True, index=False )