Kaggle competition: COVID19 Global Forecasting (Week 3)
<SOS> metric: MCRMSLE Kaggle data source: covid19-global-forecasting-week-3<feature_engineering>
import pandas as pd
import datetime
import lightgbm as lgb
import numpy as np
from sklearn import preprocessing
df_train['boilerplate'].replace(to_replace=r'"title":', value="",inplace=True,regex=True)
 df_train['boilerplate'].replace(to_replace=r'"url":',value="",inplace=True,regex=True)
 df_train['boilerplate'].replace(to_replace=r'{|}',value="",inplace=True,regex=True)
 df_train['boilerplate']=df_train['boilerplate'].str.lower()
df_test['boilerplate'].replace(to_replace=r'"title":', value="",inplace=True,regex=True)
 df_test['boilerplate'].replace(to_replace=r'"url":',value="",inplace=True,regex=True)
 df_test['boilerplate'].replace(to_replace=r'{|}',value="",inplace=True,regex=True)
 df_test['boilerplate']=df_test['boilerplate'].str.lower()<load_pretrained>
train = pd.read_csv(".. /input/covid19-global-forecasting-week-3/train.csv")
 test = pd.read_csv(".. /input/covid19-global-forecasting-week-3/test.csv")
 sub = pd.read_csv(".. /input/covid19-global-forecasting-week-3/submission.csv" )
tokenizer = AutoTokenizer.from_pretrained('bert-base-uncased' )<prepare_x_and_y>
train = train.append(test[test['Date']>'2020-04-07'] )
SEQ_length=512
Xids=np.zeros(( df_train.shape[0],SEQ_length))
Xmask=np.zeros(( df_train.shape[0],SEQ_length))
y=np.zeros(( df_train.shape[0],1))
Xids_test=np.zeros(( df_test.shape[0],SEQ_length))
Xmask_test=np.zeros(( df_test.shape[0],SEQ_length))
Xids<categorify>
train['Date'] = pd.to_datetime(train['Date'], format='%Y-%m-%d' )
for i,sequence in enumerate(df_train['boilerplate']):
tokens=tokenizer.encode_plus(sequence,max_length=SEQ_length,padding='max_length',add_special_tokens=True,
truncation=True,return_token_type_ids=False,return_attention_mask=True,
return_tensors='tf')
 Xids[i,:],Xmask[i,:],y[i,0]=tokens['input_ids'],tokens['attention_mask'],df_train.loc[i,'label']
for i,sequence in enumerate(df_test['boilerplate']):
tokens=tokenizer.encode_plus(sequence,max_length=SEQ_length,padding='max_length',add_special_tokens=True,
truncation=True,return_token_type_ids=False,return_attention_mask=True,
return_tensors='tf')
 Xids_test[i,:],Xmask_test[i,:]=tokens['input_ids'],tokens['attention_mask']<set_options>
train['day_dist'] = train['Date']-train['Date'].min()
tf.config.get_visible_devices()<categorify>
train['day_dist'] = train['day_dist'].dt.days
dataset=tf.data.Dataset.from_tensor_slices(( Xids,Xmask,y))
def map_func(input_ids,mask,labels):
return {'input_ids':input_ids,'attention_mask':mask},labels
dataset=dataset.map(map_func)
 dataset=dataset.shuffle(100000 ).batch(32 ).prefetch(1000)
 DS_size=len(list(dataset))
train=dataset.take(round(DS_size*0.85))
val=dataset.skip(round(DS_size*0.85))<categorify>
cat_cols = train.dtypes[train.dtypes=='object'].keys()
cat_cols
dataset_test=tf.data.Dataset.from_tensor_slices(( Xids_test,Xmask_test))
def map_func(input_ids,mask):
return {'input_ids':input_ids,'attention_mask':mask}
dataset_test=dataset_test.map(map_func)
 dataset_test=dataset_test.batch(32 ).prefetch(1000 )<choose_model_class>
for cat_col in cat_cols:
train[cat_col].fillna('no_value', inplace = True )
distil_bert = 'distilbert-base-uncased'
config = DistilBertConfig(dropout=0.2, attention_dropout=0.2)
 config.output_hidden_states = False
transformer_model = TFDistilBertModel.from_pretrained(distil_bert, config = config)
 input_ids_in = tf.keras.layers.Input(shape=(SEQ_length,), name='input_ids', dtype='int32')
 input_masks_in = tf.keras.layers.Input(shape=(SEQ_length,), name='attention_mask', dtype='int32')
 embedding_layer = transformer_model(input_ids_in, attention_mask=input_masks_in)[0]
X = tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(50, return_sequences=True, dropout=0.1, recurrent_dropout=0.1))(embedding_layer)
 X = tf.keras.layers.GlobalMaxPool1D()(X)
 X = tf.keras.layers.Dense(50, activation='relu' )(X)
 X = tf.keras.layers.Dropout(0.2 )(X)
 X = tf.keras.layers.Dense(1, activation='sigmoid' )(X)
 model = tf.keras.Model(inputs=[input_ids_in, input_masks_in], outputs = X)
 for layer in model.layers[:3]:
layer.trainable = False<choose_model_class>
train['place'] = train['Province_State']+'_'+train['Country_Region']
model.compile(loss=tf.keras.losses.BinaryCrossentropy() ,
optimizer='adam',metrics=[tf.keras.metrics.AUC() ,tf.keras.metrics.Precision() ,tf.keras.metrics.Recall()
] )<train_model>
cat_cols = train.dtypes[train.dtypes=='object'].keys()
cat_cols
history=model.fit(train,validation_data=val,epochs=3 )<save_to_csv>
for cat_col in ['place']:
le = preprocessing.LabelEncoder()
le.fit(train[cat_col])
 train[cat_col]=le.transform(train[cat_col] )
predictions=model.predict(dataset_test)
 df_test['label']=predictions
df_test.to_csv('submission.csv',columns=['urlid','label'],index=False )<categorify>
drop_cols = ['Id','ForecastId', 'ConfirmedCases','Date', 'Fatalities',
'day_dist', 'Province_State', 'Country_Region']
input_x=tf.data.Dataset.from_tensor_slices(( Xids,Xmask,y))
def map_func(input_ids,mask,labels):
return {'input_ids':input_ids,'attention_mask':mask}
input_x=input_x.map(map_func)
 input_x=input_x.shuffle(100000 ).batch(32 ).prefetch(1000)
 y_true = y<predict_on_test>
val = train[(train['Date']>='2020-03-20')&(train['Id'].isnull() ==False)]
y_pred=model.predict(dataset)
 y_pred<prepare_output>
y_ft = train["Fatalities"]
y_val_ft = val["Fatalities"]
y_cc = train["ConfirmedCases"]
y_val_cc = val["ConfirmedCases"]
y_pred = np.round(y_pred)
 y_pred<compute_test_metric>
def rmsle(y_true, y_pred):
return np.sqrt(np.mean(np.power(np.log1p(y_pred)- np.log1p(y_true), 2)) )
print(metrics.classification_report(y_true, y_pred))<import_modules>
def mape(y_true, y_pred):
return np.mean(np.abs(y_pred -y_true)*100/(y_true+1))
import os
from pathlib import Path
import numpy as np
import pandas as pd
from sklearn.preprocessing import MinMaxScaler
from sklearn.model_selection import cross_validate, GridSearchCV
from sklearn.pipeline import make_pipeline
from sklearn.ensemble import RandomForestClassifier, VotingClassifier
from sklearn.svm import SVC
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import accuracy_score<define_variables>
dates = dates[dates>'2020-04-07']
FILEDIR = Path('/kaggle/input/google-cloud-ncaa-march-madness-2020-division-1-mens-tournament' )<load_from_csv>
params = {
"objective": "regression",
"boosting": 'gbdt',
"num_leaves": 1280,
"learning_rate": 0.05,
"feature_fraction": 0.9,
"reg_lambda": 2,
"metric": "rmse",
'min_data_in_leaf':20
}
sub = pd.read_csv(FILEDIR / 'MSampleSubmissionStage1_2020.csv', usecols=['ID'])
 id_splited = sub['ID'].str.split('_', expand=True ).astype(int ).rename(columns={0: 'Season', 1: 'Team1', 2: 'Team2'})
 sub = pd.concat([sub, id_splited], axis=1 ).set_index(['Season', 'Team1', 'Team2'] ).sort_index()<count_duplicates>
train[train['Date']==date]
tourney_teams = {}
tourney_teams_all = set()
for season in sub.index.get_level_values('Season' ).drop_duplicates() :
tourney_teams[season] = set()
tourney_teams[season].update(sub.loc[season].index.get_level_values('Team1'))
tourney_teams[season].update(sub.loc[season].index.get_level_values('Team2'))
tourney_teams_all.update(tourney_teams[season])
 {k: len(v)for k, v in tourney_teams.items() }<load_from_csv>
test[test['Country_Region']=='Italy']
conferences = pd.read_csv(FILEDIR / 'MDataFiles_Stage1/MTeamConferences.csv')
 conferences = pd.concat(
 [conferences.query('Season == @season and TeamID in @teams')for season, teams in tourney_teams.items() ])
 conferences = conferences.set_index(['Season', 'TeamID'] ).sort_index()<load_from_csv>
test[(test['Country_Region']=='China')&(test['Province_State']=='Zhejiang')]
coaches = pd.read_csv(FILEDIR / 'MDataFiles_Stage1/MTeamCoaches.csv')
 coaches = pd.concat(
 [coaches.query('Season == @season and TeamID in @team')for season, team in tourney_teams.items() ])
 coaches = coaches[coaches['LastDayNum'] == 154].set_index(['Season', 'TeamID'] ).sort_index() [['CoachName']]<load_from_csv>
test[test['Country_Region']=='Italy']
teams = pd.read_csv(FILEDIR / 'MDataFiles_Stage1/MTeams.csv', usecols=['TeamID', 'FirstD1Season'])
 teams['FirstD1Season'] = 2020 - teams['FirstD1Season']
teams = pd.concat(
 [teams.query('TeamID in @team' ).assign(Season=season)for season, team in tourney_teams.items() ])
 teams = teams.set_index(['Season', 'TeamID'] ).sort_index()<load_from_csv>
train_sub = pd.read_csv(".. /input/covid19-global-forecasting-week-3/train.csv" )
seeds = pd.read_csv(FILEDIR / 'MDataFiles_Stage1/MNCAATourneySeeds.csv')
 seeds = pd.concat(
 [seeds.query('Season == @season and TeamID in @teams')for season, teams in tourney_teams.items() ])
 seeds = seeds.set_index(['Season', 'TeamID'] ).sort_index()
seeds['Region'] = seeds['Seed'].str[0]
seeds['Number'] = seeds['Seed'].str[1:3].astype(int)
 del seeds['Seed']<load_from_csv>
test = pd.merge(test,train_sub[['Province_State','Country_Region',
'Date','ConfirmedCases',
'Fatalities']],
on=['Province_State','Country_Region',
'Date'], how='left' )
regular = pd.read_csv(FILEDIR / 'MDataFiles_Stage1/MRegularSeasonDetailedResults.csv')
 regular = regular.drop(columns=['DayNum', 'LTeamID'])
 regular = pd.concat(
 [regular.query('Season == @season and WTeamID in @teams')for season, teams in tourney_teams.items() ])
 regular = regular.groupby(['Season', 'WTeamID'] ).sum()
regular = regular.rename_axis(index=['Season', 'TeamID'] )<concatenate>
test.loc[test['ConfirmedCases_x'].isnull() ==True]
ctcsr = pd.concat([coaches, teams, conferences, seeds, regular], axis=1 )<load_from_csv>
test.loc[test['ConfirmedCases_x'].isnull() ==True, 'ConfirmedCases_x'] =test.loc[test['ConfirmedCases_x'].isnull() ==True, 'ConfirmedCases_y']
result = pd.read_csv(FILEDIR / 'MDataFiles_Stage1/MNCAATourneyCompactResults.csv')
 result = result[result['Season'] >= 2015].set_index(['Season', 'WTeamID', 'LTeamID'] )<concatenate>
test.loc[test['Fatalities_x'].isnull() ==True, 'Fatalities_x'] = test.loc[test['Fatalities_x'].isnull() ==True, 'Fatalities_y']
merged_teams = pd.concat(
 [ctcsr.loc[[(season, wteam),(season, lteam)], :] for season, wteam, lteam, in result.index])
 team1 = merged_teams.iloc[::2, :].reset_index('TeamID')
 team2 = merged_teams.iloc[1::2, :].reset_index('TeamID')
 merged_teams = pd.concat([
pd.concat([team1.add_suffix('1'), team2.add_suffix('2')], axis=1 ).assign(Res=1),
pd.concat([team2.add_suffix('1'), team1.add_suffix('2')], axis=1 ).assign(Res=0),
] ).reset_index().set_index(['Season', 'TeamID1', 'TeamID2'] ).sort_index()<categorify>
last_amount = test.loc[(test['Country_Region']=='Italy')&(test['Date']=='2020-04-07'),'ConfirmedCases_x']
last_fat = test.loc[(test['Country_Region']=='Italy')&(test['Date']=='2020-04-07'),'Fatalities_x']
x_columns = merged_teams.columns[merged_teams.columns != 'Res']
X = merged_teams[x_columns]
for column in X.select_dtypes(include='number'):
X[column] = MinMaxScaler().fit_transform(X[column].to_numpy().reshape(-1,1))
X = pd.get_dummies(X, columns=x_columns[X.dtypes == 'object'] )<prepare_x_and_y>
i = 0
k = 30
y = merged_teams['Res']<define_search_space>
test.loc[(test['Country_Region']=='Italy')]
clfs = {}
clfs['SVC'] = {
'instance': SVC(probability=True),
'params': [
{'kernel': ['linear'], 'C': [0.01, 0.05, 0.1, 0.5, 1]},
{'kernel': ['rbf'], 'C': [1, 10, 50, 100, 250], 'gamma': [0.1, 0.2, 0.3]}
]
}
clfs['RandomForestClassifier'] = {
'instance': RandomForestClassifier(n_jobs=-1),
'params': {
'n_estimators': [25, 50, 100],
'criterion': ['gini', 'entropy'],
'max_depth': [10, 25, 50, None]
}
}
clfs['LogisticRegression'] = {
'instance': LogisticRegression(max_iter=500, n_jobs=-1),
'params': [
{'penalty': ['l2'], 'C': [0.1, 0.5, 1, 5, 10]},
{'penalty': ['l1'], 'solver': ['liblinear', 'saga'], 'C': [0.1, 0.5, 1, 5, 10]},
{'penalty': ['elasticnet'], 'C': [0.1, 0.5, 1, 5, 10], 'l1_ratio': [0.1, 0.3, 0.5, 0.7, 0.9]}
]
}<train_on_grid>
for date in dates:
k = k-1
i = i+1
test.loc[(test['Country_Region']=='Italy')&(test['Date']==date),
'ConfirmedCases_x']=last_amount.values[0] + i*(5000-(100*i))
test.loc[(test['Country_Region']=='Italy')&(test['Date']==date),
'Fatalities_x'] = last_fat.values[0]+i*(800-(10*i))
for clf_name, clf in clfs.items() :
print('<{}>'.format(clf_name))
print(' training...'.format(clf_name))
gs = GridSearchCV(clf['instance'], param_grid=clf['params'], cv=5, n_jobs=-1)
 gs.fit(X, y)
 clfs[clf_name]['best_estimator'] = gs.best_estimator_
print(' best_score: {:.3f}'.format(gs.best_score_))
print(' best_params: {}'.format(gs.best_params_))<train_on_grid>
test.loc[(test['Country_Region']=='Spain')]
vote = VotingClassifier(
 estimators=[(clf_name, clf['best_estimator'])for clf_name, clf in clfs.items() ],
voting='soft',
n_jobs=-1 
)
 vote.fit(X, y)
 vote.estimators_<compute_test_metric>
last_amount = test.loc[(test['Country_Region']=='Spain')&(test['Date']=='2020-04-07'),'ConfirmedCases_x']
last_fat = test.loc[(test['Country_Region']=='Spain')&(test['Date']=='2020-04-07'),'Fatalities_x']
i = 0
k = 30
for date in dates:
k = k-1
i = i+1
test.loc[(test['Country_Region']=='Spain')&(test['Date']==date),
'ConfirmedCases_x']=last_amount.values[0] + i*(5000-(100*i))
test.loc[(test['Country_Region']=='Spain')&(test['Date']==date),
'Fatalities_x'] = last_fat.values[0]+i*(800-(10*i))
for clf_name, clf in clfs.items() :
score = accuracy_score(y, clf['best_estimator'].predict(X))
print(clf_name, score)
 print('Vote', accuracy_score(y, vote.predict(X)) )<predict_on_test>
last_amount = test.loc[(test['Country_Region']=='China')&(test['Province_State']!='Hubei')&(test['Date']=='2020-04-07'),'ConfirmedCases_x']
last_fat = test.loc[(test['Country_Region']=='China')&(test['Province_State']!='Hubei')&(test['Date']=='2020-04-07'),'Fatalities_x']
predict_proba = pd.DataFrame(
 {clf_name: clf['best_estimator'].predict_proba(X)[:, 1] for clf_name, clf in clfs.items() },
index=X.index)
 predict_proba['Vote'] = vote.predict_proba(X)[:, 1]
_ = predict_proba.plot(kind='hist', bins=50, grid=True, alpha=0.5, figsize=(16,8))<save_to_csv>
i = 0
k = 30
for date in dates:
k = k-1
i = i+1
test.loc[(test['Country_Region']=='China')&(test['Province_State']!='Hubei')&(test['Date']==date),
'Fatalities_x']= last_fat.values
test.loc[(test['Country_Region']=='China')&(test['Province_State']!='Hubei')&(test['Date']==date),
'ConfirmedCases_x']= last_amount.values + i
columns = predict_proba.columns
for column in columns:
sub[column] = 0.5
mask = [idx for idx in sub.index if idx in X.index]
sub.loc[mask, columns] = predict_proba.loc[mask, columns]
for column in columns:
sub[['ID', column]].rename(columns={column: 'pred'} ).to_csv('predict_proba-{}.csv'.format(column), index=False )<save_to_csv>
last_amount = test.loc[(test['Country_Region']=='China')&(test['Province_State']=='Hubei')&(test['Date']=='2020-04-07'),'ConfirmedCases_x']
last_fat = test.loc[(test['Country_Region']=='China')&(test['Province_State']=='Hubei')&(test['Date']=='2020-04-07'),'Fatalities_x']
predict = pd.DataFrame(
 {clf_name: clf['best_estimator'].predict(X)for clf_name, clf in clfs.items() },
index=X.index)
 predict['Vote'] = vote.predict(X)
 columns = predict.columns
for column in columns:
sub[column] = 0.5
mask = [idx for idx in sub.index if idx in X.index]
sub.loc[mask, columns] = predict.loc[mask, columns]
for column in columns:
sub[['ID', column]].rename(columns={column: 'pred'} ).to_csv('predict-{}.csv'.format(column), index=False )<load_from_csv>
k=30
i=0
for date in dates:
k = k-1
i = i+1
test.loc[(test['Country_Region']=='China')&(test['Province_State']=='Hubei')&(test['Date']==date),'ConfirmedCases_x']= last_amount.values[0]
test.loc[(test['Country_Region']=='China')&(test['Province_State']=='Hubei')&(test['Date']==date),'Fatalities_x']= last_fat.values[0] + i
target_name = 'predict_proba-RandomForestClassifier.csv'
new_name = 'final-submission.csv'
shutil.copy(target_name, new_name )<import_modules>
sub = test[['ForecastId','ConfirmedCases_x','Fatalities_x']]
import pandas as pd
import numpy as np
from sklearn.linear_model import LogisticRegression
import matplotlib.pyplot as plt
from sklearn.utils import shuffle
from sklearn.model_selection import GridSearchCV
from sklearn.model_selection import KFold
import lightgbm as lgb
import xgboost as xgb
from xgboost import XGBClassifier
import gc
import matplotlib.pyplot as plt
from sklearn import preprocessing
from sklearn.model_selection import RandomizedSearchCV
from sklearn.ensemble import RandomForestRegressor<load_from_csv>
sub.columns=['ForecastId','ConfirmedCases','Fatalities']
Tourney_Compact_Results = pd.read_csv('.. /input/google-cloud-ncaa-march-madness-2020-division-1-mens-tournament/MDataFiles_Stage1/MNCAATourneyCompactResults.csv')
 Tourney_Seeds = pd.read_csv('.. /input/google-cloud-ncaa-march-madness-2020-division-1-mens-tournament/MDataFiles_Stage1/MNCAATourneySeeds.csv' )<load_from_csv>
sub.loc[sub['ConfirmedCases']<0,'ConfirmedCases']=0
RegularSeason_Compact_Results = pd.read_csv('.. /input/google-cloud-ncaa-march-madness-2020-division-1-mens-tournament/MDataFiles_Stage1/MRegularSeasonCompactResults.csv')
 MSeasons = pd.read_csv('.. /input/google-cloud-ncaa-march-madness-2020-division-1-mens-tournament/MDataFiles_Stage1/MSeasons.csv')
 MTeams=pd.read_csv('.. /input/google-cloud-ncaa-march-madness-2020-division-1-mens-tournament/MDataFiles_Stage1/MTeams.csv' )<merge>
sub.loc[sub['Fatalities']<0, 'Fatalities']=0
Tourney_Results_Compact=pd.merge(Tourney_Compact_Results, Tourney_Seeds, left_on=['Season', 'WTeamID'], right_on=['Season', 'TeamID'], how='left')
 Tourney_Results_Compact.rename(columns={'Seed':'WinningSeed'},inplace=True)
 Tourney_Results_Compact=Tourney_Results_Compact.drop(['TeamID'],axis=1)
 Tourney_Results_Compact = pd.merge(Tourney_Results_Compact, Tourney_Seeds, left_on=['Season', 'LTeamID'], right_on=['Season', 'TeamID'], how='left')
 Tourney_Results_Compact.rename(columns={'Seed':'LoosingSeed'}, inplace=True)
 Tourney_Results_Compact=Tourney_Results_Compact.drop(['TeamID','NumOT','WLoc'],axis=1)
 Tourney_Results_Compact<drop_column>
sub.to_csv('submission.csv',index=False )