Kaggle competition: COVID19 Global Forecasting (Week 1)
for to_del in ["date", "sessionId", "visitId", "day"]:
del train[to_del]
del test[to_del]<define_variables>
train = pd.read_csv("/kaggle/input/covid19-global-forecasting-week-1/train.csv" )
excluded = ['date', 'fullVisitorId', 'sessionId', 'totals.transactionRevenue', 'visitId', 'visitStartTime', "month", "help"]
cat_cols = [f for f in train.columns if(train[f].dtype == 'int64' and f not in excluded)]
real_cols = [f for f in train.columns if(not f in cat_cols and f not in excluded)]<compute_test_metric>
client = bigquery.Client()
dataset_ref = client.dataset("noaa_gsod", project="bigquery-public-data")
 dataset = client.get_dataset(dataset_ref)
 tables = list(client.list_tables(dataset))
table_ref = dataset_ref.table("stations")
 table = client.get_table(table_ref)
 stations_df = client.list_rows(table ).to_dataframe()
table_ref = dataset_ref.table("gsod2020")
 table = client.get_table(table_ref)
 twenty_twenty_df = client.list_rows(table ).to_dataframe()
stations_df['STN'] = stations_df['usaf'] + '-' + stations_df['wban']
twenty_twenty_df['STN'] = twenty_twenty_df['stn'] + '-' + twenty_twenty_df['wban']
cols_1 = ['STN', 'mo', 'da', 'temp', 'min', 'max', 'stp', 'wdsp', 'prcp', 'fog']
cols_2 = ['STN', 'country', 'state', 'call', 'lat', 'lon', 'elev']
weather_df = twenty_twenty_df[cols_1].join(stations_df[cols_2].set_index('STN'), on='STN')
 weather_df.tail(10 )
def score(data, y):
validation_res = pd.DataFrame(
 {"fullVisitorId": data["fullVisitorId"].values,
"transactionRevenue": data["totals.transactionRevenue"].values,
"predictedRevenue": np.expm1(y)})
 validation_res = validation_res.groupby("fullVisitorId")["transactionRevenue", "predictedRevenue"].sum().reset_index()
return np.sqrt(mean_squared_error(np.log1p(validation_res["transactionRevenue"].values),
np.log1p(validation_res["predictedRevenue"].values)) )<categorify>
weather_df['day_from_jan_first'] =(weather_df['da'].apply(int)
 + 31*(weather_df['mo']=='02')
 + 60*(weather_df['mo']=='03')
 + 91*(weather_df['mo']=='04')
)
 mo = train['Date'].apply(lambda x: x[5:7])
 da = train['Date'].apply(lambda x: x[8:10])
 train['day_from_jan_first'] =(da.apply(int)
 + 31*(mo=='02')
 + 60*(mo=='03')
 + 91*(mo=='04')
)
 C = []
for j in train.index:
df = train.iloc[j:(j+1)]
mat = cdist(df[['Lat','Long', 'day_from_jan_first']],
weather_df[['lat','lon', 'day_from_jan_first']],
metric='euclidean')
 new_df = pd.DataFrame(mat, index=df.Id, columns=weather_df.index)
 arr = new_df.values
new_close = np.where(arr == np.nanmin(arr, axis=1)[:,None],new_df.columns,False)
 L = [i[i.astype(bool)].tolist() [0] for i in new_close]
C.append(L[0])
 train['closest_station'] = C
train = train.set_index('closest_station' ).join(weather_df[['temp', 'min', 'max', 'stp', 'wdsp', 'prcp', 'fog']], ).reset_index().drop(['index'], axis=1)
 train.sort_values(by=['Id'], inplace=True)
 train.head()
class KFoldValidation() :
def __init__(self, data, n_splits=5):
unique_vis = np.array(sorted(data['fullVisitorId'].astype(str ).unique()))
folds = GroupKFold(n_splits)
 ids = np.arange(data.shape[0])
 self.fold_ids = []
for trn_vis, val_vis in folds.split(X=unique_vis, y=unique_vis, groups=unique_vis):
self.fold_ids.append([
ids[data['fullVisitorId'].astype(str ).isin(unique_vis[trn_vis])],
ids[data['fullVisitorId'].astype(str ).isin(unique_vis[val_vis])]
])
 def validate(self, train, test, features, model, name="", prepare_stacking=False,
fit_params={"early_stopping_rounds": 50, "verbose": 100, "eval_metric": "rmse"}):
model.FI = pd.DataFrame(index=features)
 full_score = 0
if prepare_stacking:
test[name] = 0
train[name] = np.NaN
for fold_id,(trn, val)in enumerate(self.fold_ids):
devel = train[features].iloc[trn]
y_devel = np.log1p(train["totals.transactionRevenue"].iloc[trn])
 valid = train[features].iloc[val]
y_valid = np.log1p(train["totals.transactionRevenue"].iloc[val])
 print("Fold ", fold_id, ":")
 model.fit(devel, y_devel, eval_set=[(valid, y_valid)], **fit_params)
 if len(model.feature_importances_)== len(features):
model.FI['fold' + str(fold_id)] = model.feature_importances_ / model.feature_importances_.sum()
predictions = model.predict(valid)
 predictions[predictions < 0] = 0
print("Fold ", fold_id, " error: ", mean_squared_error(y_valid, predictions)**0.5)
 fold_score = score(train.iloc[val], predictions)
 full_score += fold_score / len(self.fold_ids)
 print("Fold ", fold_id, " score: ", fold_score)
 if prepare_stacking:
train[name].iloc[val] = predictions
test_predictions = model.predict(test[features])
 test_predictions[test_predictions < 0] = 0
test[name] += test_predictions / len(self.fold_ids)
 print("Final score: ", full_score)
 return full_score<choose_model_class>
test = pd.read_csv("/kaggle/input/covid19-global-forecasting-week-1/test.csv" )
Kfolder = KFoldValidation(train )<choose_model_class>
weather_df['day_from_jan_first'] =(weather_df['da'].apply(int)
 + 31*(weather_df['mo']=='02')
 + 60*(weather_df['mo']=='03')
 + 91*(weather_df['mo']=='04')
)
 mo = test['Date'].apply(lambda x: x[5:7])
 da = test['Date'].apply(lambda x: x[8:10])
 test['day_from_jan_first'] =(da.apply(int)
 + 31*(mo=='02')
 + 60*(mo=='03')
 + 91*(mo=='04')
)
 C = []
for j in test.index:
df = test.iloc[j:(j+1)]
mat = cdist(df[['Lat','Long', 'day_from_jan_first']],
weather_df[['lat','lon', 'day_from_jan_first']],
metric='euclidean')
 new_df = pd.DataFrame(mat, index=df.ForecastId, columns=weather_df.index)
 arr = new_df.values
new_close = np.where(arr == np.nanmin(arr, axis=1)[:,None],new_df.columns,False)
 L = [i[i.astype(bool)].tolist() [0] for i in new_close]
C.append(L[0])
 test['closest_station'] = C
test = test.set_index('closest_station' ).join(weather_df[['temp', 'min', 'max', 'stp', 'wdsp', 'prcp', 'fog']], ).reset_index().drop(['index'], axis=1)
 test.sort_values(by=['ForecastId'], inplace=True)
 test.head()
lgbmodel = lgb.LGBMRegressor(n_estimators=1000, objective="regression", metric="rmse", num_leaves=31, min_child_samples=100,
learning_rate=0.03, bagging_fraction=0.7, feature_fraction=0.5, bagging_frequency=5,
bagging_seed=2019, subsample=.9, colsample_bytree=.9, use_best_model=True )<train_model>
train["wdsp"] = pd.to_numeric(train["wdsp"])
 test["wdsp"] = pd.to_numeric(test["wdsp"])
 train["fog"] = pd.to_numeric(train["fog"])
 test["fog"] = pd.to_numeric(test["fog"] )
Kfolder.validate(train, test, real_cols + cat_cols, lgbmodel, "lgbpred", prepare_stacking=True )<create_dataframe>
X_train = train.drop(["Fatalities", "ConfirmedCases", "Id"], axis=1)
 X_test = test.drop(["ForecastId"], axis=1 )
def create_user_df(df):
agg_data = df[real_cols + cat_cols + ['fullVisitorId']].groupby('fullVisitorId' ).mean()
pred_list = df[['fullVisitorId', 'lgbpred']].groupby('fullVisitorId' ).apply(lambda visitor_df: list(visitor_df.lgbpred)) \ 
.apply(lambda x: {'pred_'+str(i): pred for i, pred in enumerate(x)})
 all_predictions = pd.DataFrame(list(pred_list.values), index=agg_data.index)
 feats = all_predictions.columns
all_predictions['t_mean'] = all_predictions.mean(axis=1)
 all_predictions['t_median'] = all_predictions.median(axis=1)
 all_predictions['t_sum_log'] = all_predictions.sum(axis=1)
 all_predictions['t_sum_act'] = all_predictions.fillna(0 ).sum(axis=1)
 all_predictions['t_nb_sess'] = all_predictions.isnull().sum(axis=1)
 full_data = pd.concat([agg_data, all_predictions], axis=1 ).astype(float)
 full_data['fullVisitorId'] = full_data.index
del agg_data, all_predictions
gc.collect()
return full_data<create_dataframe>
X_train['Date']= pd.to_datetime(X_train['Date'])
 X_test['Date']= pd.to_datetime(X_test['Date'] )
user_train = create_user_df(train)
 user_test = create_user_df(test )<feature_engineering>
X_train = X_train.set_index(['Date'])
 X_test = X_test.set_index(['Date'] )
features = list(user_train.columns)[:-1]
user_train["totals.transactionRevenue"] = train[['fullVisitorId', 'totals.transactionRevenue']].groupby('fullVisitorId' ).sum()<feature_engineering>
def create_time_features(df):

 df['date'] = df.index
df['hour'] = df['date'].dt.hour
df['dayofweek'] = df['date'].dt.dayofweek
df['quarter'] = df['date'].dt.quarter
df['month'] = df['date'].dt.month
df['year'] = df['date'].dt.year
df['dayofyear'] = df['date'].dt.dayofyear
df['dayofmonth'] = df['date'].dt.day
df['weekofyear'] = df['date'].dt.weekofyear
X = df[['hour','dayofweek','quarter','month','year',
'dayofyear','dayofmonth','weekofyear']]
return X
for f in features:
if f not in user_test.columns:
user_test[f] = np.nan<choose_model_class>
create_time_features(X_train)
 create_time_features(X_test )
Kfolder = KFoldValidation(user_train )<choose_model_class>
X_train.drop("date", axis=1, inplace=True)
 X_test.drop("date", axis=1, inplace=True )
lgbmodel = lgb.LGBMRegressor(n_estimators=1000, objective="regression", metric="rmse", num_leaves=31, min_child_samples=100,
learning_rate=0.03, bagging_fraction=0.7, feature_fraction=0.5, bagging_frequency=5,
bagging_seed=2019, subsample=.9, colsample_bytree=.9,
use_best_model=True )<train_model>
X_train = pd.concat([X_train,pd.get_dummies(X_train['Province/State'], prefix='ps')],axis=1)
 X_train.drop(['Province/State'],axis=1, inplace=True)
 X_test = pd.concat([X_test,pd.get_dummies(X_test['Province/State'], prefix='ps')],axis=1)
 X_test.drop(['Province/State'],axis=1, inplace=True)
 X_train = pd.concat([X_train,pd.get_dummies(X_train['Country/Region'], prefix='cr')],axis=1)
 X_train.drop(['Country/Region'],axis=1, inplace=True)
 X_test = pd.concat([X_test,pd.get_dummies(X_test['Country/Region'], prefix='cr')],axis=1)
 X_test.drop(['Country/Region'],axis=1, inplace=True )
Kfolder.validate(user_train, user_test, features, lgbmodel, name="lgbfinal", prepare_stacking=True )<choose_model_class>
Y1= train["ConfirmedCases"]
xgbmodel = xgb.XGBRegressor(max_depth=22, learning_rate=0.02, n_estimators=1000,
objective='reg:linear', gamma=1.45, seed=2019, silent=False,
subsample=0.67, colsample_bytree=0.054, colsample_bylevel=0.50 )<train_model>
Y2 = train["Fatalities"]
Kfolder.validate(user_train, user_test, features, xgbmodel, name="xgbfinal", prepare_stacking=True )<choose_model_class>
model = RandomForestClassifier(bootstrap=True,max_depth=None, max_features='auto', max_leaf_nodes=None,
n_estimators=150, random_state=None, n_jobs=1, verbose=0)
 model.fit(X_train,Y1)
 pred1 = model.predict(X_test)
 pred1 = pd.DataFrame(pred1)
 pred1.columns = ["ConfirmedCases_prediction"]
catmodel = cat.CatBoostRegressor(iterations=500, learning_rate=0.2, depth=5, random_seed=2019 )<train_model>
model = RandomForestClassifier(bootstrap=True,max_depth=None, max_features='auto', max_leaf_nodes=None,
n_estimators=150, random_state=None, n_jobs=1, verbose=0)
 model.fit(X_train,Y2)
 pred2 = model.predict(X_test)
 pred2 = pd.DataFrame(pred2)
 pred2.columns = ["Death_prediction"]
Kfolder.validate(user_train, user_test, features, catmodel, name="catfinal", prepare_stacking=True,
fit_params={"use_best_model": True, "verbose": 100} )<compute_test_metric>
data_submission = pd.read_csv("/kaggle/input/covid19-global-forecasting-week-1/submission.csv")
 data_submission.columns
sub_new = data_submission[["ForecastId"]]
user_train['PredictedLogRevenue'] = 0.4 * user_train["lgbfinal"] + \
0.2 * user_train["xgbfinal"] + \
0.4 * user_train["catfinal"]
score(user_train, user_train.PredictedLogRevenue )<save_to_csv>
concat = pd.concat([pred1,pred2,sub_new],axis=1)
 concat.head()
concat.columns = ['ConfirmedCases', 'Fatalities', 'ForecastId']
concat = concat[['ForecastId','ConfirmedCases', 'Fatalities']]
user_test['PredictedLogRevenue'] = 0.4 * user_test["lgbfinal"] + 0.4 * user_test["catfinal"] + 0.2 * user_test["xgbfinal"]
user_test[['PredictedLogRevenue']].to_csv('leaky submission.csv', index=True )<categorify>
concat["ConfirmedCases"] = concat["ConfirmedCases"].astype(int)
 concat["Fatalities"] = concat["Fatalities"].astype(int )
def browser_mapping(x):
browsers = ['chrome','safari','firefox','internet explorer','edge','opera','coc coc','maxthon','iron']
if x in browsers:
return x.lower()
elif('android' in x)or('samsung' in x)or('mini' in x)or('iphone' in x)or('in-app' in x)or('playstation' in x):
return 'mobile browser'
elif('mozilla' in x)or('chrome' in x)or('blackberry' in x)or('nokia' in x)or('browser' in x)or('amazon' in x):
return 'mobile browser'
elif('lunascape' in x)or('netscape' in x)or('blackberry' in x)or('konqueror' in x)or('puffin' in x)or('amazon' in x):
return 'mobile browser'
elif '(not set)' in x:
return x
else:
return 'others'
def adcontents_mapping(x):
if('google' in x):
return 'google'
elif('placement' in x)|('placememnt' in x):
return 'placement'
elif '(not set)' in x or 'nan' in x:
return x
elif 'ad' in x:
return 'ad'
else:
return 'others'
def source_mapping(x):
if('google' in x):
return 'google'
elif('youtube' in x):
return 'youtube'
elif '(not set)' in x or 'nan' in x:
return x
elif 'yahoo' in x:
return 'yahoo'
elif 'facebook' in x:
return 'facebook'
elif 'reddit' in x:
return 'reddit'
elif 'bing' in x:
return 'bing'
elif 'quora' in x:
return 'quora'
elif 'outlook' in x:
return 'outlook'
elif 'linkedin' in x:
return 'linkedin'
elif 'pinterest' in x:
return 'pinterest'
elif 'ask' in x:
return 'ask'
elif 'siliconvalley' in x:
return 'siliconvalley'
elif 'lunametrics' in x:
return 'lunametrics'
elif 'amazon' in x:
return 'amazon'
elif 'mysearch' in x:
return 'mysearch'
elif 'qiita' in x:
return 'qiita'
elif 'messenger' in x:
return 'messenger'
elif 'twitter' in x:
return 'twitter'
elif 't.co' in x:
return 't.co'
elif 'vk.com' in x:
return 'vk.com'
elif 'search' in x:
return 'search'
elif 'edu' in x:
return 'edu'
elif 'mail' in x:
return 'mail'
elif 'ad' in x:
return 'ad'
elif 'golang' in x:
return 'golang'
elif 'direct' in x:
return 'direct'
elif 'dealspotr' in x:
return 'dealspotr'
elif 'sashihara' in x:
return 'sashihara'
elif 'phandroid' in x:
return 'phandroid'
elif 'baidu' in x:
return 'baidu'
elif 'mdn' in x:
return 'mdn'
elif 'duckduckgo' in x:
return 'duckduckgo'
elif 'seroundtable' in x:
return 'seroundtable'
elif 'metrics' in x:
return 'metrics'
elif 'sogou' in x:
return 'sogou'
elif 'businessinsider' in x:
return 'businessinsider'
elif 'github' in x:
return 'github'
elif 'gophergala' in x:
return 'gophergala'
elif 'yandex' in x:
return 'yandex'
elif 'msn' in x:
return 'msn'
elif 'dfa' in x:
return 'dfa'
elif '(not set)' in x:
return '(not set)'
elif 'feedly' in x:
return 'feedly'
elif 'arstechnica' in x:
return 'arstechnica'
elif 'squishable' in x:
return 'squishable'
elif 'flipboard' in x:
return 'flipboard'
elif 't-online.de' in x:
return 't-online.de'
elif 'sm.cn' in x:
return 'sm.cn'
elif 'wow' in x:
return 'wow'
elif 'baidu' in x:
return 'baidu'
elif 'partners' in x:
return 'partners'
else:
return 'others'
def make_mapping(data_df):
data_df['device.browser'] = data_df['device.browser'].map(lambda x:browser_mapping(str(x ).lower())).astype('str')
 data_df['trafficSource.adContent'] = data_df['trafficSource.adContent'].map(lambda x:adcontents_mapping(str(x ).lower())).astype('str')
 data_df['trafficSource.source'] = data_df['trafficSource.source'].map(lambda x:source_mapping(str(x ).lower())).astype('str')
 return data_df
def process_device(data_df):
print("process device...")
 data_df['source.country'] = data_df['trafficSource.source'] + '_' + data_df['geoNetwork.country']
data_df['campaign.medium'] = data_df['trafficSource.campaign'] + '_' + data_df['trafficSource.medium']
data_df['browser.category'] = data_df['device.browser'] + '_' + data_df['device.deviceCategory']
data_df['browser.os'] = data_df['device.browser'] + '_' + data_df['device.operatingSystem']
return data_df
def custom(data):
print('custom.. ')
 data['device_deviceCategory_channelGrouping'] = data['device.deviceCategory'] + "_" + data['channelGrouping']
data['channelGrouping_browser'] = data['device.browser'] + "_" + data['channelGrouping']
data['channelGrouping_OS'] = data['device.operatingSystem'] + "_" + data['channelGrouping']
for i in ['geoNetwork.city', 'geoNetwork.continent', 'geoNetwork.country','geoNetwork.metro', 'geoNetwork.networkDomain', 'geoNetwork.region','geoNetwork.subContinent']:
for j in ['device.browser','device.deviceCategory', 'device.operatingSystem', 'trafficSource.source']:
data[i + "_" + j] = data[i] + "_" + data[j]
data['content.source'] = data['trafficSource.adContent'] + "_" + data['source.country']
data['medium.source'] = data['trafficSource.medium'] + "_" + data['source.country']
return data
def process(data):
return custom(process_device(make_mapping(data)))
<define_variables>
concat.to_csv("submission.csv",index=False )