Kaggle competition: COVID19 Global Forecasting (Week 2)
<SOS> metric: MCRMSLE Kaggle data source: covid19-global-forecasting-(week-2)<find_best_params>
pd.options.display.max_rows = 999
pd.options.display.max_columns = 999
train_file = ".. /input/covid19-global-forecasting-week-2/train.csv"
test_file = ".. /input/covid19-global-forecasting-week-2/test.csv"
sub_file = ".. /input/covid19-global-forecasting-week-2/submission.csv"
Optimistic=True
grid.best_params_<predict_on_test>
df = pd.read_csv(train_file)
lr = LogisticRegression(C=1.0,max_iter=10,multi_class='multinomial',penalty='l2',solver='sag')
 lr.fit(X_train,y_train)
 predictions = lr.predict(X_test)
 print(classification_report(y_test,predictions))<drop_column>
loc_group = ["Province_State", "Country_Region"]
def preprocess(df,datecol):
df["Date"] = df[datecol].astype("datetime64[ms]")
 for col in loc_group:
df[col].fillna("none", inplace=True)
 return df
df= preprocess(df,"Date" )
id_ = test_data['id']
test_data = test_data.drop(['id'],axis=1 )<categorify>
TRAIN_X = datetime.strptime('20-03-11', '%y-%m-%d' )
test_data = pd.get_dummies(test_data, columns = ['color'] )<predict_on_test>
TEST_END = df["Date"].max()
TRAIN_LAST = datetime.strptime('20-03-23', '%y-%m-%d')
 TRAIN_FIRST = df["Date"].max() - timedelta(days=5 )
result = pd.DataFrame(lr.predict(test_data))
result.columns=['type']<categorify>
df=df.sort_values(['Province_State','Country_Region','Date'] )
result[result['type']==0]='Ghoul'
result[result['type']==1]='Goblin'
result[result['type']==2]='Ghost'
result<concatenate>
TARGETS = ["ConfirmedCases", "Fatalities"]
for col in TARGETS:
df["prev_{}".format(col)] = df[col].shift()
df[f'delta_{col}'] = df[col]-df[f'prev_{col}']
df[f'rolling_{col}'] = df[f'delta_{col}'].rolling(window=3 ).mean()
df[f'div_rolling_{col}'] =(df[f'rolling_{col}']-df[f'rolling_{col}'].shift())/df[f'rolling_{col}'].shift()
df['div_rolling_ConfirmedCases']=df['div_rolling_ConfirmedCases'].replace([np.inf, -np.inf], np.nan ).fillna(0)
 df['div_rolling_Fatalities']=df['div_rolling_Fatalities'].replace([np.inf, -np.inf], np.nan ).fillna(0 )
result = pd.concat([id_,result],axis=1 )<save_to_csv>
df[df['Province_State']=="New York"]
result.to_csv('result.csv',index=False )<load_from_csv>
df = df[df.Date>df.Date.min() + timedelta(days=3)]
train = pd.read_csv("/kaggle/input/data-ghoul/train.csv")
 test = pd.read_csv("/kaggle/input/data-ghoul/test.csv")
 IDtest = test["id"]<define_variables>
dfSmall = df[df.rolling_ConfirmedCases>100].reset_index()
def detect_outliers(df,n,features):
outlier_indices = []
for col in features:
Q1 = np.percentile(df[col], 25)
 Q3 = np.percentile(df[col],75)
 IQR = Q3 - Q1
outlier_step = 1.5 * IQR
outlier_list_col = df[(df[col] < Q1 - outlier_step)|(df[col] > Q3 + outlier_step)].index
outlier_indices.extend(outlier_list_col)
 outlier_indices = Counter(outlier_indices)
 multiple_outliers = list(k for k, v in outlier_indices.items() if v > n)
 return multiple_outliers
Outliers_to_drop = detect_outliers(train,2,["bone_length","rotting_flesh","hair_length","has_soul"] )<filter>
MaxDate = dfSmall.loc[dfSmall.groupby(loc_group)['rolling_ConfirmedCases'].idxmax() ][['Province_State','Country_Region','Date']].reset_index()
train.loc[Outliers_to_drop]<concatenate>
df = df.merge(MaxDate, how = 'left',on=['Province_State','Country_Region'],suffixes=['','_Max'] )
train_len = len(train)
 dataset = pd.concat(objs=[train, test], axis=0 ).reset_index(drop=True )<count_missing_values>
MinDate = dfSmall.loc[dfSmall.groupby(loc_group)['rolling_ConfirmedCases'].idxmin() ][['Province_State','Country_Region','Date']].reset_index()
dataset = dataset.fillna(np.nan)
 dataset.isnull().sum()<count_missing_values>
df = df.merge(MinDate, how = 'left',on=['Province_State','Country_Region'],suffixes=['','_Min'] )
train.isnull().sum()<count_missing_values>
df['Date_Max'] = df['Date_Max'].fillna(TEST_END)
train.isnull().sum()<categorify>
df['Date_Min'] =df["Date_Max"] - timedelta(days=5 )
train["type_int"] = train["type"].replace({
"Ghoul":1,
"Goblin":2,
"Ghost":3
} )<categorify>
df[['Date_Max','Date_Min']].drop_duplicates()
dataset = pd.get_dummies(dataset, columns=["color"] )<categorify>
df['Days'] =(df['Date']-df['Date_Min'].min() ).dt.days.astype('int16' )
dataset["type"] = dataset["type"].replace({
"Ghoul":1,
"Goblin":2,
"Ghost":3
} )<split>
df['div_rolling_ConfirmedCases'] = np.clip(df['div_rolling_ConfirmedCases'],0,1)
 df3=df[(df['Date']>df['Date_Min'])&(df['Date']<=df['Date_Max'])].groupby(loc_group)['div_rolling_ConfirmedCases'].mean().reset_index()
df3.columns = ['Province_State','Country_Region','Coeff']
train = dataset[:train_len]
test = dataset[train_len:]
test.drop(["type", "id"],axis = 1,inplace=True )<prepare_x_and_y>
df['div_rolling_Fatalities'] = np.clip(df['div_rolling_Fatalities'],0,1)
 df4=df[(df['Date']>df['Date_Min'])&(df['Date']<=(df['Date_Max'])) ].groupby(loc_group)['div_rolling_Fatalities'].mean().reset_index()
df4.columns = ['Province_State','Country_Region','Death_Coeff']
train["type"] = train["type"].astype(int)
 Y_train = train["type"]
X_train = train.drop(["type", "id"], axis = 1 )<choose_model_class>
df4[df4['Country_Region']=='Turkey']
kfold = StratifiedKFold(n_splits=10 )<choose_model_class>
random_state = 2
classifiers = []
classifiers.append(SVC(random_state=random_state))
classifiers.append(DecisionTreeClassifier(random_state=random_state))
classifiers.append(AdaBoostClassifier(DecisionTreeClassifier(random_state=random_state),random_state=random_state,learning_rate=0.1))
classifiers.append(RandomForestClassifier(random_state=random_state))
classifiers.append(ExtraTreesClassifier(random_state=random_state))
classifiers.append(GradientBoostingClassifier(random_state=random_state))
classifiers.append(MLPClassifier(random_state=random_state))
classifiers.append(KNeighborsClassifier())
 classifiers.append(LogisticRegression(random_state = random_state))
classifiers.append(LinearDiscriminantAnalysis())
 cv_results = []
for classifier in classifiers :
cv_results.append(cross_val_score(classifier, X_train, y = Y_train, scoring = "accuracy", cv = kfold, n_jobs=4))
cv_means = []
cv_std = []
for cv_result in cv_results:
cv_means.append(cv_result.mean())
 cv_std.append(cv_result.std())
 cv_res = pd.DataFrame({"CrossValMeans":cv_means,"CrossValerrors": cv_std,"Algorithm":["SVC","DecisionTree","AdaBoost",
"RandomForest","ExtraTrees","GradientBoosting","MultipleLayerPerceptron","KNeighboors","LogisticRegression","LinearDiscriminantAnalysis"]} )<import_modules>
if Optimistic:
df['Check'] = df["Date"].max() + timedelta(days=7)
 else:
df['Check'] = df["Date"].max() + timedelta(days=14)
 df['Date_Max'] = np.where(df['Date_Max']>df["Date"].max() - timedelta(days=1),df["Check"],df['Date_Max'])
classifiers<train_on_grid>
test_df = preprocess(pd.read_csv(test_file),"Date")
 df = df.merge(test_df,how='outer', on =['Province_State','Country_Region','Date'])
 df=df.sort_values(['Province_State','Country_Region','Date'])
 df['Date_Max']=df.groupby(loc_group)['Date_Max'].fillna(method='ffill')
 if Optimistic:
df['Death_Date_Max'] = df['Date_Max']+ timedelta(days=5)
 else:
df['Death_Date_Max'] = df['Date_Max']+ timedelta(days=10 )
LDA = LinearDiscriminantAnalysis()
ex_param_grid = {"n_components": [None, 1, 2, 3, 4]}
gsLDA = GridSearchCV(LDA,param_grid = ex_param_grid, cv=kfold, scoring="accuracy", n_jobs= 4, verbose = 1)
 gsLDA.fit(X_train,Y_train)
 LDA_best = gsLDA.best_estimator_
gsLDA.best_score_<train_on_grid>
df = df.merge(df3,how='left',on=['Province_State','Country_Region'])
 df = df.merge(df4,how='left',on=['Province_State','Country_Region'] )
GBC = GradientBoostingClassifier()
gb_param_grid = {'loss' : ["deviance"],
'n_estimators' : [100,200,300],
'learning_rate': [0.1, 0.05, 0.01],
'max_depth': [4, 8],
'min_samples_leaf': [100,150],
'max_features': [0.3, 0.1]
}
gsGBC = GridSearchCV(GBC,param_grid = gb_param_grid, cv=kfold, scoring="accuracy", n_jobs= 4, verbose = 1)
 gsGBC.fit(X_train,Y_train)
 GBC_best = gsGBC.best_estimator_
gsGBC.best_score_<train_on_grid>
df['Coeff']=df.groupby(loc_group)['Coeff'].fillna(method='ffill')
 df['Coeff'] = np.where(df['Coeff'].isna() ,0,df['Coeff'])
 df['Coeff'] = np.clip(df['Coeff'],0.05,0.75)
 df['Death_Coeff']=df.groupby(loc_group)['Death_Coeff'].fillna(method='ffill')
 df['Death_Coeff'] = np.where(df['Death_Coeff'].isna() ,0,df['Death_Coeff'])
 df['Death_Coeff'] = np.clip(df['Death_Coeff'],0.05,0.75)
 df['Death_Coeff'] = np.where(df['Death_Coeff']>df['Coeff'],df['Coeff'],df['Death_Coeff'])
 df['Coeff'] = np.where(df.Date <= df['Date_Max'],df['Coeff'],-0.7*df['Coeff'])
 df['Death_Coeff'] = np.where(df.Date <= df['Death_Date_Max'],df['Death_Coeff'],-0.7*df['Death_Coeff'] )
MLP = MLPClassifier()
ex_param_grid = {
'hidden_layer_sizes': [(50,50,50),(50,100,50),(100,)],
'activation': ['tanh', 'relu'],
'solver': ['sgd', 'adam'],
'alpha': [0.0001, 0.05],
'learning_rate': ['constant','adaptive'],
}
gsMLP = GridSearchCV(MLP, param_grid = ex_param_grid, cv=kfold, scoring="accuracy", n_jobs= 4, verbose = 1)
 gsMLP.fit(X_train,Y_train)
 MLP_best = gsMLP.best_estimator_
gsMLP.best_score_<train_on_grid>
list1 =[]
list2=[]
pred = 0
change = 0
pred_death = 0
change_death = 0
for index, row in df.iterrows() :
if(row['Date']<=TRAIN_LAST)|(row['Date']==TRAIN_FIRST):
change =(1+row['Coeff'])*row['rolling_ConfirmedCases']
pred = row['ConfirmedCases']-change
change_death =(1+row['Death_Coeff'])*row['rolling_Fatalities']
pred_death = row['Fatalities']-change_death
else:
change = change*(1+row['Coeff'])
 change_death = change_death*(1+row['Death_Coeff'])
 if row['Date']==TEST_END:
change =(1+row['Coeff'])*row['rolling_ConfirmedCases']
pred = row['ConfirmedCases']-change
change_death =(1+row['Death_Coeff'])*row['rolling_Fatalities']
pred_death = row['Fatalities']-change_death
if change<0:
change =0
pred = pred+change
list1.append(pred)
 if change_death<0:
change_death =0
pred_death = pred_death+change_death
if pred_death>pred*0.1:
pred_death = pred*0.1
list2.append(pred_death)
 df['pred_CC']= list1
df['pred_Fatalities']= list2
LRC = LogisticRegression()
ex_param_grid = {
'C': [0.001, 0.01, 0.1, 1, 10, 100, 1000]
}
gsLRC = GridSearchCV(LRC, param_grid = ex_param_grid, cv=kfold, scoring="accuracy", n_jobs= 4, verbose = 1)
 gsLRC.fit(X_train,Y_train)
 LRC_best = gsLRC.best_estimator_
gsLRC.best_score_<train_model>
df.to_csv("df2.csv" )
votingC = VotingClassifier(estimators=[('lda', LDA_best),('gbc', GBC_best), 
('mlp', MLP_best),('lrc',LRC_best)], voting='soft', n_jobs=4)
 votingC = votingC.fit(X_train, Y_train )<save_to_csv>
df.groupby('Date')[['ConfirmedCases','Fatalities','pred_CC','pred_Fatalities']].sum().to_csv('sum.csv' )
test_type = pd.Series(votingC.predict(test), name="Type")
 test_type = test_type.replace({
1:"Ghoul",
2:"Goblin",
3:"Ghost"
})
 results = pd.concat([IDtest,test_type],axis=1)
 results.to_csv("voting_prediction.csv",index=False )<set_options>
df[df['Province_State']=='New York'][['Date','ConfirmedCases','Fatalities','pred_CC','pred_Fatalities','Coeff','Death_Coeff','rolling_ConfirmedCases','Date_Max','Death_Date_Max']]
%matplotlib inline
warnings.filterwarnings("ignore" )<load_from_csv>
df['Error_CC']= np.where(df['ConfirmedCases']==0,0,(abs(( df['pred_CC']-df['ConfirmedCases'])/df['ConfirmedCases'])))
 df['Error_Death']= np.where(df['Fatalities']==0,0,abs(( df['pred_Fatalities']-df['Fatalities'])/df['Fatalities']))
df.groupby('Date')[['Error_CC','Error_Death']].sum().to_csv('error.csv' )
train = pd.read_csv(".. /input/train.csv")
 test = pd.read_csv(".. /input/test.csv" )<load_from_csv>
df[df['Country_Region']=='Spain'][['Date','ConfirmedCases','Fatalities','pred_CC','pred_Fatalities','Coeff','Death_Coeff','rolling_ConfirmedCases','Date_Max','Death_Date_Max']]
train = pd.read_csv(".. /input/train.csv")
 test = pd.read_csv(".. /input/test.csv" )<drop_column>
sub_df = pd.read_csv(sub_file)
train.drop("id", axis=1, inplace=True )<drop_column>
sub_df = sub_df.merge(df[['ForecastId','pred_CC','pred_Fatalities']],how='left',on='ForecastId' )
train.drop("color", axis=1, inplace=True)
 test.drop("color", axis=1, inplace=True )<prepare_x_and_y>
sub_df['ConfirmedCases']= sub_df['pred_CC'].astype('int')
 sub_df['Fatalities']= sub_df['pred_Fatalities'].astype('int')
 sub_df = sub_df.drop(['pred_CC','pred_Fatalities'],axis=1 )
<normalization><EOS> 
sub_df.to_csv("submission.csv", index=False )