Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<data_type_conversions>
print(os.listdir(".. /input"))
%matplotlib inline
mpl.rc('axes', labelsize=14)
 mpl.rc('xtick', labelsize=12)
 mpl.rc('ytick', labelsize=12)
 warnings.filterwarnings("ignore" )
train_df["Date"] = pd.to_datetime(train_df["Date"])
 test_df["Date"] = pd.to_datetime(test_df["Date"] )<feature_engineering>
train=pd.read_csv('.. /input/train.csv' )
train_df["NewCases"] = train_df.groupby(["Country_Region", "Province_State"])["ConfirmedCases"].diff(periods=1)
 train_df["NewCases"] = train_df["NewCases"].fillna(0)
 train_df["NewCases"] = np.where(train_df["NewCases"] < 0, 0, train_df["NewCases"])
 train_df["NewFatalities"] = train_df.groupby(["Country_Region", "Province_State"])["Fatalities"].diff(periods=1)
 train_df["NewFatalities"] = train_df["NewFatalities"].fillna(0)
 train_df["NewFatalities"] = np.where(train_df["NewFatalities"] < 0, 0, train_df["NewFatalities"] )<feature_engineering>
test=pd.read_csv('.. /input/test.csv' )
train_df["NewCases"] = np.log(train_df["NewCases"] + 1)
 train_df["NewFatalities"] = np.log(train_df["NewFatalities"] + 1 )<categorify>
train.isnull().sum()
def preprocess_train(n_prev, n_next):
df = train_df.copy()
input_feats, output_feats = [], []
for i in range(1, n_prev+1):
for feat in ["NewCases", "NewFatalities"]:
df["{}_prev_{}".format(feat, i)] = df.groupby(["Country_Region", "Province_State"])[feat].shift(i)
 input_feats.append("{}_prev_{}".format(feat, i))
output_feats.extend(["NewCases", "NewFatalities"])
 for i in range(1, n_next):
for feat in ["NewCases", "NewFatalities"]:
df["{}_next_{}".format(feat, i)] = df.groupby(["Country_Region", "Province_State"])[feat].shift(-i)
 output_feats.append("{}_next_{}".format(feat, i))
df.dropna(inplace=True)
 const_df = pd.get_dummies(df[["Province_State", "Country_Region"]], drop_first=True)
 time_df = df[input_feats]
time_df = time_df.values.reshape(( df.shape[0],-1,2))
output_df = df[output_feats]
return const_df, time_df, output_df<sort_values>
train.isnull().sum()
def preprocess_test(n_prev):
input_feats = []
append_df = pd.concat([train_df, test_df[test_df["Date"] == train_df["Date"].max() + timedelta(days=1)]])
 append_df.sort_values(["Country_Region", "Province_State", "Date"], ascending=[True, True, True], inplace=True)
 for i in range(1, n_prev+1):
for feat in ["NewCases", "NewFatalities"]:
append_df["{}_prev_{}".format(feat, i)] = append_df.groupby(["Country_Region", "Province_State"])[feat].shift(i)
 input_feats.append("{}_prev_{}".format(feat, i))
append_df = append_df[append_df["ForecastId"].notnull() ]
const_df = pd.get_dummies(append_df[["Province_State", "Country_Region"]], drop_first=True)
 time_df = append_df[input_feats]
time_df = time_df.values.reshape(( append_df.shape[0],-1,2))
return const_df, time_df<define_variables>
test.isnull().sum()
n_next =(test_df["Date"].max() - train_df["Date"].max() ).days
n_next<train_model>
test.isnull().sum()
const_df, time_df, output_df = preprocess_train(n_next, n_next )<split>
train['Survived'].value_counts()
const_test_df, time_test_df = preprocess_test(n_next )<import_modules>
train['Pclass'].value_counts()
from keras.models import Model
from keras import layers
from keras import Input<choose_model_class>
train[['Pclass', 'Survived']].groupby(['Pclass'], as_index=False ).mean()
time_input = Input(shape=(time_df.shape[1], time_df.shape[2]))
lstm = layers.LSTM(32 )(time_input)
 const_input = Input(shape=(const_df.shape[1],))
combine = layers.concatenate([lstm, const_input], axis=-1)
 output = layers.Dense(output_df.shape[1], activation='relu' )(combine)
 model = Model([time_input, const_input], output)
 model.compile(optimizer='adam',
loss='mean_squared_error')
 model.summary()<train_model>
train['Sex'].value_counts()
model.fit([time_df, const_df], output_df, epochs=300, batch_size=128 )<predict_on_test>
train[['Sex', 'Survived']].groupby(['Sex'], as_index=False ).mean()
output = model.predict([time_test_df, const_test_df])
 output.shape<feature_engineering>
train.groupby('Pclass' ).apply(lambda x:x.groupby('Sex')['Survived'].mean() ).style.background_gradient(cmap='cool' )
sub_test_df = test_df[test_df["Date"] > train_df["Date"].max() ]
sub_test_df = pd.concat([sub_test_df,
pd.DataFrame(output.reshape(( -1, 2)) , columns=["NewCases", "NewFatalities"], index=sub_test_df.index)],
axis=1)
 sub_test_df["NewCases"] = np.exp(sub_test_df["NewCases"])- 1
sub_test_df["NewFatalities"] = np.exp(sub_test_df["NewFatalities"])- 1
sub_test_df<merge>
train['Cabin'].nunique()
fixed_test_df = test_df[test_df["Date"] <= train_df["Date"].max() ].merge(train_df[train_df["Date"] >= test_df["Date"].min() ][["Province_State","Country_Region", "Date", "ConfirmedCases", "Fatalities"]],
how="left", on=["Province_State","Country_Region", "Date"])
 fixed_test_df<concatenate>
train['Ticket'].nunique()
predict_df = pd.concat([sub_test_df, fixed_test_df] ).sort_values(["Country_Region", "Province_State", "Date"],
ascending=[True, True, True])
 predict_df<feature_engineering>
train['PassengerId'].nunique()
predict_df = predict_df.reset_index()
for i in range(len(predict_df)) :
if pd.isnull(predict_df.iloc[i]["ConfirmedCases"]):
predict_df.loc[i, "ConfirmedCases"] = predict_df.iloc[i - 1]["ConfirmedCases"] + predict_df.iloc[i]["NewCases"]
if pd.isnull(predict_df.iloc[i]["Fatalities"]):
predict_df.loc[i, "Fatalities"] = predict_df.iloc[i - 1]["Fatalities"] + predict_df.iloc[i]["NewFatalities"]
predict_df<save_to_csv>
train['Embarked'].value_counts()
predict_df[["ForecastId", "ConfirmedCases", "Fatalities"]].to_csv("submission.csv", index=False )<set_options>
train['Embarked'].fillna('S', inplace=True )
%matplotlib inline<set_options>
train['Embarked'].value_counts()
plt.style.use('default')
 warnings.filterwarnings('ignore')
<load_from_csv>
train['FamilySize'] = train['SibSp'] + train['Parch']
test['FamilySize'] = test['SibSp'] + test['Parch']
train=pd.read_csv('/kaggle/input/covid19-global-forecasting-week-4/train.csv')
 test=pd.read_csv('/kaggle/input/covid19-global-forecasting-week-4/test.csv')
 train.head()<data_type_conversions>
from sklearn.preprocessing import LabelEncoder
pd.concat([round(100*train[catcols].isnull().sum() /train.shape[0],2 ).to_frame('train'),
round(100*test[catcols].isnull().sum() /test.shape[0],2 ).to_frame('test')],axis=1 )<feature_engineering>
test['Fare'].fillna(test['Fare'].median() , inplace = True)
 train['FareCategory'] = pd.qcut(train['Fare'], 5)
 test['FareCategory'] = pd.qcut(test['Fare'], 5 )
print(f'TRAIN -> date_min= {train["Date"].min() } ; date_max= {train["Date"].max() }')
 print(f'TEST -> date_min= {test["Date"].min() } ; date_max= {test["Date"].max() }' )<count_unique_values>
train['Fare_Code'] = LabelEncoder().fit_transform(train['FareCategory'] )
len(set(train.Date.unique())& set(test.Date.unique()))<data_type_conversions>
test['Fare_Code'] = LabelEncoder().fit_transform(test['FareCategory'] )
def prepare_features(data):
data['Province_State']=data['Province_State'].str.lower()
data['Country_Region']=data['Country_Region'].str.lower()
data['UnkownProvince_State']=data['Province_State'].isnull().astype(int)
 data.fillna({'Province_State':''},inplace=True)
 data['Province_State']=data['Province_State'].apply(lambda x: ''.join([ch for ch in x if ch.isalpha() ]))
data['Country_Region']=data['Country_Region'].apply(lambda x: ''.join([ch for ch in x if ch.isalpha() ]))
data['Country_RegionFreq']=data['Country_Region'].map(data['Country_Region'].value_counts(1 ).to_dict())
 return data
catcols.append('UnkownProvince_State' )<categorify>
train['Title'] = 0
for salut in train:
train['Title'] = train.Name.str.extract('([A-Za-z]+)\.')
 test['Title'] = 0
for salut in test:
test['Title'] = test.Name.str.extract('([A-Za-z]+)\.' )
train=prepare_features(train)
 test=prepare_features(test)
 train.head(3 )<count_unique_values>
mapping = {'Mlle': 'Miss',
'Major': 'Mr',
'Col': 'Mr',
'Sir': 'Mr',
'Don': 'Mr',
'Mme': 'Miss',
'Jonkheer': 'Mr',
'Lady': 'Mrs',
'Capt': 'Mr',
'Countess': 'Mrs',
'Ms': 'Miss',
'Dona': 'Mrs'
}
train.replace({'Title': mapping}, inplace=True)
 test.replace({'Title': mapping}, inplace=True )
train['Country_Region'].nunique()<load_from_csv>
data_df = train.append(test)
 titles = ['Dr', 'Master', 'Miss', 'Mr', 'Mrs', 'Rev']
for title in titles:
age_to_impute = data_df.groupby('Title')['Age'].median() [titles.index(title)]
data_df.loc[(data_df['Age'].isnull())&(data_df['Title'] == title), 'Age'] = age_to_impute
cols_to_keep=['Country Name','Population ages 0-14, total', 'Population ages 15-64, female','Population ages 15-64, male',
'Population ages 15-64, total','Population ages 65 and above, total',
'Population ages 80 and above, female(% of female population)',
'Population ages 80 and above, male(% of male population)','Population, male',
'Population, total','Rural population(% of total population)']
add_inf=pd.read_csv('/kaggle/input/world-population-and-development-indicators/data.csv')[cols_to_keep]
add_inf.head(3 )<categorify>
train['Age'] = data_df['Age'][:891]
test['Age'] = data_df['Age'][891:]
add_inf['Country Name'].replace({'bahamasthe':'bahamas','bruneidarussalam':'bahamas','czechrepublic':'czechia',
'congodemrep':'congokinshasa','congorep':'congobrazzaville','egyptarabrep':'egypt',
'gambiathe':'gambia','iranislamicrep':'iran','korearep':'koreasouth','unitedstates':'us',
'kyrgyzrepublic':'kyrgyzstan','russianfederation':'russia','stkittsandnevis':'saintkittsandnevis',
'stlucia':'saintlucia','stvincentandthegrenadines':'saintvincentandthegrenadines',
'slovakrepublic':'slovakia','syrianarabrepublic':'syria','venezuelarb':'venezuela'},inplace=True)
 print('us' in add_inf['Country Name'].unique())
 add_inf.drop_duplicates(subset=['Country Name'],inplace=True)
 print(add_inf['Country Name'].nunique() )<count_unique_values>
plt.style.use('default' )
print('From train:{} ; From add_inf:{} ; intersection:{}'.format(train['Country_Region'].nunique() ,add_inf['Country Name'].nunique() ,
len(set(train['Country_Region'])& set(add_inf['Country Name']))))
print('From test:{} ; From add_inf:{} ; intersection:{}'.format(test['Country_Region'].nunique() ,add_inf['Country Name'].nunique() ,
len(set(test['Country_Region'])& set(add_inf['Country Name']))))<categorify>
train['AgeCategory'] = pd.qcut(train['Age'], 4)
 test['AgeCategory'] = pd.qcut(test['Age'], 4 )
def scale_popultaion_features(data):
data['Population ages 0-14, total'].replace(0,data['Population ages 0-14, total'].mode() ,inplace=True)
 data['Population ages 15-64, total'].replace(0,data['Population ages 15-64, total'].mode() ,inplace=True)
 data['Population ages 65 and above, total'].replace(0,data['Population ages 65 and above, total'].mode() ,inplace=True)
 data['Population, total'].replace(0,data['Population, total'].mode() ,inplace=True)
 data['Population ages 15-64, male']=data['Population ages 15-64, male']/data['Population ages 15-64, total']
data['Population ages 15-64, female']=data['Population ages 15-64, female']/data['Population ages 15-64, total']
data['Population ages 0-14, total']=data['Population ages 0-14, total']/data['Population, total']
data['Population ages 15-64, total']=data['Population ages 15-64, total']/data['Population, total']
data['Population ages 65 and above, total']=data['Population ages 65 and above, total']/data['Population, total']
data['Population, male']=data['Population, male']/data['Population, total']
pcq_features=[col for col in data.columns if '%' in col]
print(pcq_features)
 data[pcq_features]=.01*data[pcq_features]
data.drop(columns=['Population, total'],inplace=True)
 data.fillna(data.median().to_dict() ,inplace=True)
 return data<merge>
train['Age_Code'] = LabelEncoder().fit_transform(train['AgeCategory'])
 test['Age_Code'] = LabelEncoder().fit_transform(test['AgeCategory'] )
print('train shape: {} ; test shape: {}'.format(train.shape,test.shape))
train=train.merge(add_inf,left_on='Country_Region',right_on='Country Name',how='left')
 test=test.merge(add_inf,left_on='Country_Region',right_on='Country Name',how='left')
 print('train shape: {} ; test shape: {}'.format(train.shape,test.shape))
train.head()<define_variables>
data_df['Last_Name'] = data_df['Name'].apply(lambda x: str.split(x, ",")[0])
 DEFAULT_SURVIVAL_VALUE = 0.5
data_df['Family_Survival'] = DEFAULT_SURVIVAL_VALUE
for grp, grp_df in data_df[['Survived','Name', 'Last_Name', 'Fare', 'Ticket', 'PassengerId',
'SibSp', 'Parch', 'Age', 'Cabin']].groupby(['Last_Name', 'Fare']):
if(len(grp_df)!= 1):
for ind, row in grp_df.iterrows() :
smax = grp_df.drop(ind)['Survived'].max()
smin = grp_df.drop(ind)['Survived'].min()
passID = row['PassengerId']
if(smax == 1.0):
data_df.loc[data_df['PassengerId'] == passID, 'Family_Survival'] = 1
elif(smin==0.0):
data_df.loc[data_df['PassengerId'] == passID, 'Family_Survival'] = 0
print("Number of passengers with family survival information:",
data_df.loc[data_df['Family_Survival']!=0.5].shape[0] )
cols_to_keep=['country', 'total_covid_19_tests',
'total_covid_19_tests_per_million_people',
'inform_risk', 'inform_p2p_hazard_and_exposure_dimension',
'people_using_at_least_basic_sanitation_services',
'inform_vulnerability', 'inform_health_conditions',
'inform_epidemic_vulnerability', 'mortality_rate_under_5',
'prevalence_of_undernourishment', 'inform_lack_of_coping_capacity',
'inform_access_to_healthcare',
'inform_epidemic_lack_of_coping_capacity', 'physicians_density',
'current_health_expenditure_per_capita',
'maternal_mortality_ratio', 'entry_date', 'category',
'measure', 'global-school-closures']
who_data=pd.read_csv('/kaggle/input/whodata/WHO data.csv')[cols_to_keep]
who_data.drop_duplicates(subset=['country'],inplace=True)
 print(who_data.shape)
 who_data.head(2 )<categorify>
for _, grp_df in data_df.groupby('Ticket'):
if(len(grp_df)!= 1):
for ind, row in grp_df.iterrows() :
if(row['Family_Survival'] == 0)|(row['Family_Survival']== 0.5):
smax = grp_df.drop(ind)['Survived'].max()
smin = grp_df.drop(ind)['Survived'].min()
passID = row['PassengerId']
if(smax == 1.0):
data_df.loc[data_df['PassengerId'] == passID, 'Family_Survival'] = 1
elif(smin==0.0):
data_df.loc[data_df['PassengerId'] == passID, 'Family_Survival'] = 0
print("Number of passenger with family/group survival information: "
+str(data_df[data_df['Family_Survival']!=0.5].shape[0]))
train['Family_Survival'] = data_df['Family_Survival'][:891]
test['Family_Survival'] = data_df['Family_Survival'][891:]
who_data['country'].replace({'capeverde':'caboverde','czechrepublic':'czechia','myanmar':'burma',
'congodemrep':'congokinshasa','congorep':'congobrazzaville','guinea':'guineabissau',
'swaziland':'eswatini','southkorea':'koreasouth','macedonia':'northmacedonia',
'timor':'timorleste','unitedstates':'us','unitedstatesvirginislands':'us',
'vatican':'holysee','palestine':'westbankandgaza'},inplace=True)
 print(who_data['country'].nunique())
 who_data.drop_duplicates(subset=['country'],inplace=True )<count_unique_values>
train['Sex'].replace(['male','female'],[0,1],inplace=True)
 test['Sex'].replace(['male','female'],[0,1],inplace=True )
print('From train:{} ; From who-data:{} ; intersection:{}'.format(train['Country_Region'].nunique() ,who_data['country'].nunique() ,
len(set(train['Country_Region'])& set(who_data['country']))))
print('From test:{} ; From who-data:{} ; intersection:{}'.format(test['Country_Region'].nunique() ,who_data['country'].nunique() ,
len(set(test['Country_Region'])& set(who_data['country']))))<count_missing_values>
drop_elements = ['PassengerId', 'Name', 'SibSp', 'Parch','Ticket', 'Cabin', 'FareCategory', 'AgeCategory','Age', 'Fare', 'Title', 'Embarked']
who_data.select_dtypes(include='number' ).isnull().sum() /who_data.shape[0]<merge>
train = train.drop(drop_elements, axis=1)
 test = test.drop(drop_elements, axis=1 )
print('train shape: {} ; test shape: {}'.format(train.shape,test.shape))
train=train.merge(who_data,left_on='Country_Region',right_on='country',how='left')
 test=test.merge(who_data,left_on='Country_Region',right_on='country',how='left')
 train.fillna(train.quantile (.15 ).to_dict() ,inplace=True)
 test.fillna(train.quantile (.15 ).to_dict() ,inplace=True)
 print('train shape: {} ; test shape: {}'.format(train.shape,test.shape))
train.head(3 )<data_type_conversions>
y = train['Survived']
train['Date'] = pd.to_datetime(train['Date'], format = '%Y-%m-%d')
 test['Date'] = pd.to_datetime(test['Date'], format = '%Y-%m-%d' )<feature_engineering>
X = train[train.columns[1:]]
def create_date_features(df):
df['day'] = df['Date'].dt.day
df['month'] = df['Date'].dt.month
df['dayofweek'] = df['Date'].dt.dayofweek
df['dayofyear'] = df['Date'].dt.dayofyear
df['weekofyear'] = df['Date'].dt.weekofyear
df['Date_day_month'] = df['Date'].dt.strftime("%m%d" ).astype(int)
 return df
train=create_date_features(train)
 test=create_date_features(test)
 test.head(3 )<drop_column>
from sklearn.compose import ColumnTransformer
from sklearn.pipeline import Pipeline
from sklearn.model_selection import train_test_split, GridSearchCV
from sklearn.impute import SimpleImputer
from sklearn.preprocessing import StandardScaler
train['Province_State']=train['Country_Region']+' '+train['Province_State']
test['Province_State']=test['Country_Region']+' '+test['Province_State']
train['Province_State']=train['Province_State'].str.replace(' ','')
 test['Province_State']=test['Province_State'].str.replace(' ','')
 train.drop(columns=['Country Name','Country_Region','country'],inplace=True)
 test.drop(columns=['Country Name','Country_Region','country'],inplace=True)
 train.head(2 )<data_type_conversions>
all_features = ['Pclass', 'Sex', 'FamilySize', 'Fare_Code', 'Age_Code', 'Family_Survival']
train['measures_applied']=(train.Date.dt.strftime('%Y-%m-%d')>=train['entry_date'] ).astype(int)
 test['measures_applied']=(test.Date.dt.strftime('%Y-%m-%d')>=test['entry_date'] ).astype(int)
 train.drop(columns=['entry_date'],inplace=True)
 test.drop(columns=['entry_date'],inplace=True)
 train.head(3 )<categorify>
all_preprocess = ColumnTransformer(
 transformers = [ 
('allfeatures', all_transformer, all_features),
]
)
train=pd.get_dummies(columns=['category','measure','global-school-closures'],data=train)
 test=pd.get_dummies(columns=['category','measure','global-school-closures'],data=test)
 train.head(3 )<count_unique_values>
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, stratify=train['Survived'] )
s=test.select_dtypes(include='number' ).nunique()
binary_features=s[s==2].index.values
binary_features<categorify>
from sklearn.linear_model import LogisticRegression
from sklearn.ensemble import RandomForestClassifier
from sklearn.svm import SVC
from sklearn.neighbors import KNeighborsClassifier
from sklearn.linear_model import SGDClassifier
from sklearn.model_selection import cross_val_score
train.fillna({col:0 for col in binary_features},inplace=True)
 test.fillna({col:0 for col in binary_features},inplace=True)
 filling_dict=train.median().to_dict()
train.fillna(filling_dict,inplace=True)
 test.fillna(filling_dict,inplace=True)
 train.head(2 )<prepare_x_and_y>
classifiers = [
LogisticRegression(random_state=42),
RandomForestClassifier(random_state=42),
SVC(random_state=42),
KNeighborsClassifier() ,
SGDClassifier(random_state=42),
]
out = pd.DataFrame({'ForecastId': [], 'ConfirmedCases': [], 'Fatalities': []})
 for province in tqdm(train['Province_State'].unique()):
train_province = train.loc[(train['Province_State'] == province)].copy()
y_Conf_true = train_province['ConfirmedCases']
y_Fat_true = train_province['Fatalities']
X_train_prov = train_province[features]
test_province = test[(test['Province_State']== province)].copy()
X_test_Id = test_province['ForecastId']
X_test_prov = test_province[features]
reg_Conf = XGBRegressor(n_estimators=1000)
 reg_Conf.fit(X_train_prov, y_Conf_true)
 y_Conf_pred = reg_Conf.predict(X_test_prov)
 reg_Fat = XGBRegressor(n_estimators=1000)
 reg_Fat.fit(X_train_prov, y_Fat_true)
 y_Fat_pred = reg_Fat.predict(X_test_prov)
 predictions = pd.DataFrame({'ForecastId': X_test_Id, 'ConfirmedCases': y_Conf_pred, 'Fatalities': y_Fat_pred})
 out = pd.concat([out, predictions], axis=0)
 out['ForecastId']=out['ForecastId'].astype(int )<save_to_csv>
first_round_scores = {}
for classifier in classifiers:
pipe = Pipeline(steps=[('preprocessor', all_preprocess), 
('classifier', classifier)])
 pipe.fit(X_train, y_train)
 print(classifier)
 score = pipe.score(X_test, y_test)
 first_round_scores[classifier.__class__.__name__[:10]] = score
print("model score: %.3f" % score )
out.to_csv('submission.csv',index=False )<set_options>
grid_scores = {}
log_clf = Pipeline(steps=[('preprocessor', all_preprocess), 
('classifier', LogisticRegression(random_state=42)) ])
 log_param_grid = {
'classifier__C': [0.01, 0.1, 1.0, 10],
'classifier__solver' : ['liblinear','lbfgs','sag', 'saga'],
'classifier__max_iter' : [500],
}
log_grid_search = GridSearchCV(log_clf, log_param_grid, cv=10, iid=True)
 log_grid_search.fit(X_train, y_train)
 print('Logistic Regression - grid_search.best_params_ and best_scores_', log_grid_search.best_params_, log_grid_search.best_score_)
 log_model_score = log_grid_search.score(X_test, y_test)
 print("Logistic Regression - model score: ", log_model_score)
 grid_scores['log'] = log_model_score
%matplotlib inline<compute_test_metric>
rf_clf = Pipeline(steps=[('preprocessor', all_preprocess), 
('classifier', RandomForestClassifier(random_state=42)) ])
 rf_param_grid = {
'classifier__n_estimators' : [50, 100],
'classifier__max_features' : [2, 3],
'classifier__criterion' : ['gini', 'entropy']
}
rf_grid_search = GridSearchCV(rf_clf, rf_param_grid, cv=10, iid=True)
 rf_grid_search.fit(X_train, y_train)
 print('Random Forest grid_search.best_params_ and best_scores_', rf_grid_search.best_params_, rf_grid_search.best_score_)
 rf_model_score = rf_grid_search.score(X_test, y_test)
 print("Random Forest - model score: ", rf_model_score)
 grid_scores['rf'] = rf_model_score
def dS_dt(S, I, R_t, T_inf):
return -(R_t / T_inf)* I * S
def dE_dt(S, E, I, R_t, T_inf, T_inc):
return(R_t / T_inf)* I * S -(T_inc**-1)* E
def dI_dt(I, E, T_inc, T_inf):
return(T_inc**-1)* E -(T_inf**-1)* I
def dR_dt(I, T_inf):
return(T_inf**-1)* I
def SEIR_model(t, y, R_t, T_inf, T_inc):
if callable(R_t):
reproduction = R_t(t)
 else:
reproduction = R_t
S, E, I, R = y
S_out = dS_dt(S, I, reproduction, T_inf)
 E_out = dE_dt(S, E, I, reproduction, T_inf, T_inc)
 I_out = dI_dt(I, E, T_inc, T_inf)
 R_out = dR_dt(I, T_inf)
 return [S_out, E_out, I_out, R_out]<load_from_csv>
svm_clf = Pipeline(steps=[('preprocessor', all_preprocess), 
('classifier', SVC(random_state=42)) ])
 svm_param_grid = [
{'classifier__kernel': ['linear'], 'classifier__C': [10., 30., 100., 300.]},
{'classifier__kernel': ['rbf'], 'classifier__C': [1.0, 3.0, 10., 30., 100., 300.],
'classifier__gamma': [0.01, 0.03, 0.1, 0.3, 1.0]},
]
svm_grid_search = GridSearchCV(svm_clf, svm_param_grid, cv=10, iid=True)
 svm_grid_search.fit(X_train, y_train)
 print('SVM grid_search.best_params_ and best_scores_', svm_grid_search.best_params_, svm_grid_search.best_score_)
 svm_model_score = svm_grid_search.score(X_test, y_test)
 print("SVM - model score: ", svm_model_score)
 grid_scores['svm'] = svm_model_score
train = pd.read_csv('.. /input/covid19-global-forecasting-week-4/train.csv')
 test = pd.read_csv('.. /input/covid19-global-forecasting-week-4/test.csv')
 train['Date_datetime'] = train['Date'].apply(lambda x:(datetime.datetime.strptime(x, '%Y-%m-%d')) )<load_from_csv>
knn_clf = Pipeline(steps=[('preprocessor', all_preprocess), 
('classifier', KNeighborsClassifier())])
 knn_param_grid = {
'classifier__n_neighbors': [5, 6, 7, 8, 9, 10, 11, 12, 14, 16, 18, 20, 22, 24, 26 ],
'classifier__weights': ['uniform', 'distance' ],
'classifier__leaf_size': list(range(1,50,5)) ,
}
knn_grid_search = GridSearchCV(knn_clf, knn_param_grid, cv=10, iid=True, scoring ='roc_auc')
 knn_grid_search.fit(X_train, y_train)
 print('KNN grid_search.best_params_ and best_scores_', knn_grid_search.best_params_, knn_grid_search.best_score_)
 knn_model_score = knn_grid_search.score(X_test, y_test)
 print("KNN - model score: ", knn_model_score)
 grid_scores['knn'] = knn_model_score
pop_info = pd.read_csv('/kaggle/input/covid19-population-data/population_data.csv')
 country_pop = pop_info.query('Type == "Country/Region"')
 province_pop = pop_info.query('Type == "Province/State"')
 country_lookup = dict(zip(country_pop['Name'], country_pop['Population']))
province_lookup = dict(zip(province_pop['Name'], province_pop['Population']))<define_variables>
sgd_clf = Pipeline(steps=[('preprocessor', all_preprocess), 
('classifier', SGDClassifier(random_state=42)) ])
 sgd_param_grid = {
'classifier__max_iter': [100, 200],
'classifier__alpha': [0.0001, 0.001, 0.01, 0.1],
}
sgd_grid_search = GridSearchCV(sgd_clf, sgd_param_grid, cv=10, iid=True)
 sgd_grid_search.fit(X_train, y_train)
 print('SGD grid_search.best_params_ and best_scores_', sgd_grid_search.best_params_, sgd_grid_search.best_score_)
 sgd_model_score = sgd_grid_search.score(X_test, y_test)
 print("SGD - model score: ", sgd_model_score)
 grid_scores['sgd'] = sgd_model_score
Country = 'Hubei'
N = pop_info[pop_info['Name']==Country]['Population'].tolist() [0]
train_loc = train[train['Country_Region']==Country].query('ConfirmedCases > 0')
 if len(train_loc)==0:
train_loc = train[train['Province_State']==Country].query('ConfirmedCases > 0')
 n_infected = train_loc['ConfirmedCases'].iloc[0]
max_days = len(train_loc)
 s =(N - n_infected)/ N
e = 0.
 i = n_infected / N
r = 0.
 T_inc = 5.2
T_inf = 2.9
R_0 = 3.954
sol = solve_ivp(SEIR_model, [0, max_days], [s, e, i, r], args=(R_0, T_inf, T_inc),
t_eval=np.arange(max_days))
plot_model_and_predict(train_loc, N, sol, title = 'SEIR Model(without intervention)' )<import_modules>
final_pipe = Pipeline(steps=[('preprocessor', all_preprocess)] )
from scipy.optimize import minimize
from sklearn.metrics import mean_squared_log_error, mean_squared_error<compute_train_metric>
X_final_processed = final_pipe.fit_transform(X )
def eval_model_const(params, data, population, return_solution=False, forecast_days=0):
R_0, cfr = params
N = population
n_infected = data['ConfirmedCases'].iloc[0]
max_days = len(data)+ forecast_days
s, e, i, r =(N - n_infected)/ N, 0, n_infected / N, 0
def time_varying_reproduction(t):
if t > 80:
return R_0 * 0.5
else:
return R_0
sol = solve_ivp(SEIR_model, [0, max_days], [s, e, i, r], args=(time_varying_reproduction, T_inf, T_inc),
t_eval=np.arange(0, max_days))
sus, exp, inf, rec = sol.y
y_pred_cases = np.clip(( inf + rec)* N ,0,np.inf)
 y_true_cases = data['ConfirmedCases'].values
y_pred_fat = np.clip(rec*N* cfr, 0, np.inf)
 y_true_fat = data['Fatalities'].values
optim_days = min(20, len(data))
weights = 1 / np.arange(1, optim_days+1)[::-1]
msle_cases = mean_squared_log_error(y_true_cases[-optim_days:], y_pred_cases[-optim_days:], weights)
 msle_fat = mean_squared_log_error(y_true_fat[-optim_days:], y_pred_fat[-optim_days:], weights)
 msle_final = np.mean([msle_cases, msle_fat])
 if return_solution:
return msle_final, sol
else:
return msle_final<compute_train_metric>
test_final_processed = final_pipe.transform(test )
def eval_model_decay(params, data, population, return_solution=False, forecast_days=0):
R_0, cfr, k, L = params
N = population
n_infected = data['ConfirmedCases'].iloc[0]
max_days = len(data)+ forecast_days
s, e, i, r =(N - n_infected)/ N, 0, n_infected / N, 0
def time_varying_reproduction(t):
return R_0 /(1 +(t/L)**k)
 sol = solve_ivp(SEIR_model, [0, max_days], [s, e, i, r], args=(time_varying_reproduction, T_inf, T_inc),
t_eval=np.arange(0, max_days))
sus, exp, inf, rec = sol.y
y_pred_cases = np.clip(( inf + rec)* N ,0,np.inf)
 y_true_cases = data['ConfirmedCases'].values
y_pred_fat = np.clip(rec*N* cfr, 0, np.inf)
 y_true_fat = data['Fatalities'].values
optim_days = min(20, len(data))
weights = 1 / np.arange(1, optim_days+1)[::-1]
msle_cases = mean_squared_log_error(y_true_cases[-optim_days:], y_pred_cases[-optim_days:], weights)
 msle_fat = mean_squared_log_error(y_true_fat[-optim_days:], y_pred_fat[-optim_days:], weights)
 msle_final = np.mean([msle_cases, msle_fat])
 if return_solution:
return msle_final, sol
else:
return msle_final<train_model>
knn_hyperparameters = {
'n_neighbors': [6, 7, 8, 9, 10, 11, 12, 14, 16, 18, 20, 22],
'algorithm' : ['auto'],
'weights': ['uniform', 'distance'],
'leaf_size': list(range(1,50,5)) ,
}
gd=GridSearchCV(estimator = KNeighborsClassifier() , param_grid = knn_hyperparameters,
cv=10, scoring = "roc_auc")
 gd.fit(X_final_processed, y)
 print(gd.best_score_)
 print(gd.best_estimator_ )
def fit_model_new(data, area_name, initial_guess=[2.2, 0.02, 2, 50],
bounds=(( 1, 20),(0, 0.15),(1, 3),(1, 100)) , make_plot=True, decay_mode = None):
if area_name in ['France']:
train = data.query('ConfirmedCases > 0' ).copy() [:-1]
else:
train = data.query('ConfirmedCases > 0' ).copy()
train_data = train
if len(train_data)== 0:
result_zero = np.zeros(( 43))
return pd.DataFrame({'ConfirmedCases':result_zero,'Fatalities':result_zero}), 0
try:
population = pop_info[pop_info['Name']==area_name]['Population'].tolist() [0]
except IndexError:
print('country not in population set, '+str(area_name))
population = 1000000
if area_name == 'US':
population = 327200000
if area_name == 'Global':
population = 7744240900
cases_per_million = train_data['ConfirmedCases'].max() * 10**6 / population
n_infected = train_data['ConfirmedCases'].iloc[0]
if cases_per_million < 1:
population = population/100
res_const = minimize(eval_model_const, [2.2, 0.02], bounds=(( 1, 20),(0, 0.15)) ,
args=(train_data, population, False),
method='L-BFGS-B')
 res_decay = minimize(eval_model_decay, initial_guess, bounds=bounds,
args=(train_data, population, False),
method='L-BFGS-B')
 test_end = datetime.datetime.strptime('2020-05-14','%Y-%m-%d')
 test_start = datetime.datetime.strptime('2020-04-02','%Y-%m-%d')
 train_test = data[data.Date_datetime>=test_start]
test_period =(test_end - test_start ).days
train_max = train_data.Date_datetime.max()
train_min = train_data.Date_datetime.min()
add_date = 0
delta_days =(test_end - train_max ).days
train_add_time=[]
if train_min > test_start:
add_date =(train_min-test_start ).days
last = train_min-timedelta(add_date)
 train_add_time = np.arange(last, train_min, dtype='datetime64[D]' ).tolist()
train_add_time = pd.to_datetime(train_add_time)
 dates_all = train_add_time.append(pd.to_datetime(np.arange(train_min, test_end+timedelta(1), dtype='datetime64[D]')))
 else:
dates_all = pd.to_datetime(np.arange(train_min, test_end+timedelta(1), dtype='datetime64[D]'))
if decay_mode is None:
if res_const.fun < res_decay.fun :
msle, sol = eval_model_const(res_const.x, train_data, population, True, delta_days+add_date)
 res = res_const
else:
msle, sol = eval_model_decay(res_decay.x, train_data, population, True, delta_days+add_date)
 res = res_decay
R_0, cfr, k, L = res.x
else:
if decay_mode =='day_decay':
msle, sol = eval_model_const(res_const.x, train_data, population, True, delta_days+add_date)
 res = res_const
else:
msle, sol = eval_model_decay(res_decay.x, train_data, population, True, delta_days+add_date)
 res = res_decay
R_0, cfr, k, L = res.x
sus, exp, inf, rec = sol.y
y_pred = pd.DataFrame({
'ConfirmedCases': cumsum_signal(np.diff(( inf + rec)* population, prepend=n_infected ).cumsum()),
'Fatalities': cumsum_signal(( np.clip(rec * population * res.x[1], 0, np.inf)).tolist())
 })
 y_pred_valid = y_pred.iloc[:len(train_data)]
y_pred_test = pd.concat([train_test[['ConfirmedCases', 'Fatalities']],y_pred.iloc[-(delta_days):]], ignore_index=True)
 y_true_valid = train_data[['ConfirmedCases', 'Fatalities']]
valid_msle_cases = mean_squared_log_error(y_true_valid['ConfirmedCases'], y_pred_valid['ConfirmedCases'])
 valid_msle_fat = mean_squared_log_error(y_true_valid['Fatalities'], y_pred_valid['Fatalities'])
 valid_msle = np.mean([valid_msle_cases, valid_msle_fat])
 if make_plot:
if len(res.x)<=2:
print(f'Validation MSLE: {valid_msle:0.5f}, using intervention days decay, Reproduction number(R0): {res.x[0]:0.5f}, Fatal rate : {res.x[1]:0.5f}')
 else:
print(f'Validation MSLE: {valid_msle:0.5f}, using Hill decay, Reproduction number(R0): {res.x[0]:0.5f}, Fatal rate : {res.x[1]:0.5f}, K : {res.x[2]:0.5f}, L: {res.x[3]:0.5f}')
 f = plt.figure(figsize=(16,5))
ax = f.add_subplot(1,2,1)
 ax.plot(exp, 'y', label='Exposed');
ax.plot(inf, 'r', label='Infected');
ax.plot(rec, 'c', label='Recovered/deceased');
plt.title('SEIR Model Trends')
 plt.xlabel("Days", fontsize=10);
plt.ylabel("Fraction of population", fontsize=10);
plt.legend(loc='best');
ax2 = f.add_subplot(1,2,2)
 xaxis = train_data['Date_datetime'].tolist()
xaxis = dates.date2num(xaxis)
 hfmt = dates.DateFormatter('%m
%d')
 ax2.xaxis.set_major_formatter(hfmt)
 ax2.plot(np.array(train_data['Date_datetime'], dtype='datetime64[D]'),train_data['ConfirmedCases'],label='Confirmed Cases(train)', c='g')
 ax2.plot(np.array(train_data['Date_datetime'], dtype='datetime64[D]'), y_pred['ConfirmedCases'][:len(train_data)],label='Cumulative modeled infections', c='r')
 plt.title('Real ConfirmedCase and Predict ConfirmedCase')
 plt.legend(loc='best');
plt.show()
if len(res.x)>2:
msle, sol = eval_model_decay(res.x, train_data, population, True, 300)
 else:
msle, sol = eval_model_const(res.x, train_data, population, True, 300)
 sus, exp, inf, rec = sol.y
y_pred = pd.DataFrame({
'ConfirmedCases': cumsum_signal(np.diff(( inf + rec)* population, prepend=n_infected ).cumsum()),
'Fatalities': cumsum_signal(np.clip(rec, 0, np.inf)* population * res.x[1])
 })
 start = train_min
end = start + timedelta(len(y_pred))
time_array = np.arange(start, end, dtype='datetime64[D]')
 max_day = numpy.where(inf == numpy.amax(inf)) [0][0]
where_time = time_array[max_day]
pred_max_day = y_pred['ConfirmedCases'][max_day]
xy_show_max_estimation =(where_time, max_day)
 con = y_pred['ConfirmedCases']
fat = y_pred['Fatalities']
max_day_con = numpy.where(con == numpy.amax(con)) [0][0]
max_day_fat = numpy.where(fat == numpy.amax(fat)) [0][0]
max_con = numpy.amax(con)
 max_fat = numpy.amax(fat)
 where_time_con = time_array[len(time_array)-50]
xy_show_max_estimation_confirmed =(where_time_con, max_con)
 fig = go.Figure()
fig.add_trace(go.Scatter(x=time_array, y=y_pred['ConfirmedCases'].astype(int),
mode='lines',
line = dict(color='red'),
name='Estimation Confirmed Case Start from '+ str(start.date())+ ' to ' +str(end.date())))
 fig.add_trace(go.Scatter(x=time_array, y=y_pred['Fatalities'].astype(int),
mode='lines',
line = dict(color='yellow'),
name='Estimation Fatalities Start from '+ str(start.date())+ ' to ' +str(end.date())))
 fig.add_trace(go.Scatter(x=time_array[:len(train)], y=train['ConfirmedCases'],
mode='lines',
name='Confirmed case until '+ str(train_max.date()),line = dict(color='green', width=4)))
 fig.add_trace(go.Scatter(x=time_array[:len(train)], y=train['Fatalities'],
mode='lines',
name='Fatalities case until '+ str(train_max.date()),line = dict(color='blue', width=4)))
 fig.add_annotation(
 x=where_time_con,
y=max_con-(max_con/30),
showarrow=False,
text="Estimate Max Case around:" +str(int(max_con)) ,
font=dict(
 color="Blue",
size=15 
))
fig.add_annotation(
 x=where_time_con,
y=max_fat-(max_fat/30),
showarrow=False,
text="Estimate Max death around:" +str(int(max_fat)) ,
font=dict(
 color="Blue",
size=15 
))
fig.add_annotation(
 x=time_array[len(train)-1],
y=train['ConfirmedCases'].tolist() [-1],
showarrow=True,
text=f"Real Max ConfirmedCase: " +str(int(train['ConfirmedCases'].tolist() [-1])))
 fig.add_annotation(
 x=time_array[len(train)-1],
y=train['Fatalities'].tolist() [-1],
showarrow=True,
text=f"Real Max Fatalities: " +str(int(train['Fatalities'].tolist() [-1])))
 fig.add_annotation(
 x=where_time,
y=pred_max_day,
text='Infect start decrease from: ' + str(where_time))
fig.update_layout(title='Estimate Confirmed Case ,'+area_name+' Total population ='+ str(int(population)) , legend_orientation="h")
 fig.show()
return y_pred_test, valid_msle<feature_engineering>
gd.best_estimator_.fit(X_final_processed, y)
 y_pred = gd.best_estimator_.predict(test_final_processed )
country = 'Taiwan*'
if country not in train['Country_Region'].unique() :
country_pd_train = train[train['Province_State']==country]
else:
country_pd_train = train[train['Country_Region']==country]
a,b = fit_model_new(country_pd_train,country,make_plot=True )<split>
knn = KNeighborsClassifier(algorithm='auto', leaf_size=26, metric='minkowski',
metric_params=None, n_jobs=None, n_neighbors=6, p=2,
weights='uniform')
 knn.fit(X_final_processed, y)
 y_pred = knn.predict(test_final_processed )
country = 'Japan'
if country not in train['Country_Region'].unique() :
country_pd_train = train[train['Province_State']==country]
else:
country_pd_train = train[train['Country_Region']==country]
a,b = fit_model_new(country_pd_train,country,make_plot=True )<split>
submission = pd.DataFrame(pd.read_csv(".. /input/test.csv")['PassengerId'])
 submission['Survived'] = y_pred
submission.to_csv("submission.csv", index = False )