Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<count_unique_values>
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
latest_data = country_grouped_covid['ObservationDate'] == '04/13/2020'
country_data = country_grouped_covid[latest_data]
country_list = country_data['Country/Region'].unique()
print("The total number of countries with COVID-19 Confirmed cases = {}".format(country_list.size))<feature_engineering>
train_data = pd.read_csv('.. /input/titanic/train.csv')
 test_data = pd.read_csv('.. /input/titanic/test.csv')
 train = train_data.copy()
test = test_data.copy()
py.init_notebook_mode(connected=True)
 formated_gdf = covid_cases.groupby(['ObservationDate', 'Country/Region'])['Confirmed', 'Deaths', 'Recovered'].max()
formated_gdf = formated_gdf.reset_index()
formated_gdf['Date'] = pd.to_datetime(formated_gdf['ObservationDate'])
 formated_gdf['Date'] = formated_gdf['Date'].dt.strftime('%m/%d/%Y')
 formated_gdf['log_ConfirmedCases'] = np.log(formated_gdf.Confirmed + 1)
 formated_gdf['log_Fatalities'] = np.log(formated_gdf.Deaths + 1 )<feature_engineering>
train.drop(['PassengerId'], axis=1, inplace=True)
 test.drop(['PassengerId'], axis=1, inplace=True)
 pred = train['Survived']
py.init_notebook_mode(connected=True)
 formated_gdf = covid_cases.groupby(['ObservationDate', 'Country/Region'])['Confirmed', 'Deaths', 'Recovered'].max()
formated_gdf = formated_gdf.reset_index()
formated_gdf['Date'] = pd.to_datetime(formated_gdf['ObservationDate'])
 formated_gdf['Date'] = formated_gdf['Date'].dt.strftime('%m/%d/%Y')
 formated_gdf['log_ConfirmedCases'] = np.log(formated_gdf.Confirmed + 1)
 formated_gdf['log_Fatalities'] = np.log(formated_gdf.Deaths + 1 )<load_from_csv>
train.isnull().sum()
folder_name = '/kaggle/input/covcsd-covid19-countries-statistical-dataset/'
file_type = 'csv'
seperator =','
dataframe = pd.concat([pd.read_csv(f, sep=seperator)for f in glob.glob(folder_name + "/*."+file_type)],ignore_index=True,sort=False )<create_dataframe>
train.isnull().sum()
covid_data = dataframe[['Date', 'State', 'Country', 'Cumulative_cases', 'Cumulative_death',
'Daily_cases', 'Daily_death', 'Latitude', 'Longitude', 'Temperature',
'Min_temperature', 'Max_temperature', 'Wind_speed', 'Precipitation',
'Fog_Presence', 'Population', 'Population Density/km', 'Median_Age',
'Sex_Ratio', 'Age%_65+', 'Hospital Beds/1000', 'Available Beds/1000',
'Confirmed Cases/1000', 'Lung Patients(F)', 'Lung Patients(M)',
'Life Expectancy(M)', 'Life Expectancy(F)', 'Total_tests_conducted',
'Out_Travels(mill.) ', 'In_travels(mill.) ', 'Domestic_Travels(mill.) ']]<feature_engineering>
test.isnull().sum()
training_data['Country_Region'] = training_data['Country_Region'] + ' ' + training_data['Province_State']
testing_data['Country_Region'] = testing_data['Country_Region'] + ' ' + testing_data['Province_State']
del training_data['Province_State']
del testing_data['Province_State']
def split_date(date):
date = date.split('-')
 date[0] = int(date[0])
 if(date[1][0] == '0'):
date[1] = int(date[1][1])
 else:
date[1] = int(date[1])
 if(date[2][0] == '0'):
date[2] = int(date[2][1])
 else:
date[2] = int(date[2])
 return date
training_data.Date = training_data.Date.apply(split_date)
 testing_data.Date = testing_data.Date.apply(split_date )<feature_engineering>
test.isnull().sum()
year = []
month = []
day = []
for i in training_data.Date:
year.append(i[0])
 month.append(i[1])
 day.append(i[2])
 training_data['Year'] = year
training_data['Month'] = month
training_data['Day'] = day
del training_data['Date']<feature_engineering>
train['Age'].fillna(train['Age'].quantile(0.5), inplace=True)
 test['Age'].fillna(test['Age'].quantile(0.5), inplace=True )
year = []
month = []
day = []
for i in testing_data.Date:
year.append(i[0])
 month.append(i[1])
 day.append(i[2])
 testing_data['Year'] = year
testing_data['Month'] = month
testing_data['Day'] = day
del testing_data['Date']
del training_data['Id']
del testing_data['ForecastId']
del testing_data['Year']
del training_data['Year']<drop_column>
train['Embarked'].fillna('S', inplace=True)
 test['Embarked'].fillna('S', inplace=True )
latest_data = covid_data['Date'] == '30-03-2020'
country_data_detailed = covid_data[latest_data]
country_data_detailed.drop(['Daily_cases','Daily_death','Latitude','Longitude'],axis=1,inplace=True)
 country_data_detailed.head(3 )<categorify>
test['Fare'].fillna(test['Fare'].quantile(0.5), inplace=True )
country_data_detailed.replace('Not Reported',np.nan,inplace=True)
 country_data_detailed.replace('N/A',np.nan,inplace=True)
 country_data_detailed.head(3 )<data_type_conversions>
sex1 = pd.get_dummies(train['Sex'])
 sex2 = pd.get_dummies(test['Sex'] )
country_data_detailed['Lung Patients(F)'].replace('Not reported',np.nan,inplace=True)
 country_data_detailed['Lung Patients(F)'] = country_data_detailed['Lung Patients(F)'].astype("float" )<load_from_csv>
train.drop(['Sex'], axis=1, inplace=True)
 test.drop(['Sex'], axis=1, inplace=True)
 train = pd.concat([train, sex1], axis=1)
 test = pd.concat([test, sex2], axis=1 )
temperature_data = pd.read_csv('/kaggle/input/covcsd-covid19-countries-statistical-dataset/temperature_data.csv')
 temperature_data.head()<compute_train_metric>
train.drop(['female'], axis=1, inplace=True)
 test.drop(['female'], axis=1, inplace=True )
sample = temperature_dataset['Temperature'].sample(n=250)
 test = temperature_dataset['Temperature']
stat, p = ttest_ind(sample, test)
 print('Statistics=%.3f, p=%.3f' %(stat, p))<normalization>
embark1 = pd.get_dummies(train['Embarked'])
 embark2 = pd.get_dummies(test['Embarked'])
 train.drop(['Embarked'], axis=1, inplace=True)
 test.drop(['Embarked'], axis=1, inplace=True)
 train = pd.concat([train, embark1], axis=1)
 test = pd.concat([test, embark2], axis=1 )
training_data['ConfirmedCases'] = training_data['ConfirmedCases'].apply(int)
 training_data['Fatalities'] = training_data['Fatalities'].apply(int)
 cases = training_data.ConfirmedCases
fatalities = training_data.Fatalities
del training_data['ConfirmedCases']
del training_data['Fatalities']
lb = LabelEncoder()
training_data['Country_Region'] = lb.fit_transform(training_data['Country_Region'])
 testing_data['Country_Region'] = lb.transform(testing_data['Country_Region'])
 scaler = MinMaxScaler()
x_train = scaler.fit_transform(training_data.values)
 x_test = scaler.transform(testing_data.values )<train_model>
def family(x):
if x['SibSp'] + x['Parch'] > 1:
return 1
else:
return 0
train['Family'] = train.apply(family, axis=1)
 test['Family'] = test.apply(family, axis=1 )
rf = XGBRegressor(n_estimators = 1500 , max_depth = 15, learning_rate=0.1)
 rf.fit(x_train,cases)
 cases_pred = rf.predict(x_test)
 rf = XGBRegressor(n_estimators = 1500 , max_depth = 15, learning_rate=0.1)
 rf.fit(x_train,fatalities)
 fatalities_pred = rf.predict(x_test )<feature_engineering>
train.drop(['SibSp','Parch'], axis=1, inplace=True)
 test.drop(['SibSp','Parch'], axis=1, inplace=True )
cases_pred = np.around(cases_pred)
 fatalities_pred = np.around(fatalities_pred)
 cases_pred[cases_pred < 0] = 0
fatalities_pred[fatalities_pred < 0] = 0<load_from_csv>
train['Cabin'] = pd.Series(i[0] if not pd.isnull(i)else 'X' for i in train['Cabin'])
 test['Cabin'] = pd.Series(i[0] if not pd.isnull(i)else 'X' for i in test['Cabin'] )
submission_dataset = pd.read_csv(".. /input/covid19-global-forecasting-week-4/submission.csv")
 submission_dataset['ConfirmedCases'] = cases_pred
submission_dataset['Fatalities'] = fatalities_pred
submission_dataset.head()<save_to_csv>
train['Cabin'] = train['Cabin'].map({
'X': 0,
'A': 1,
'B': 2,
'C': 3,
'D': 4,
'E': 5,
'F': 6,
'G': 7,
'T': 0
})
 train['Cabin'] = train['Cabin'].astype(int)
 test['Cabin'] = test['Cabin'].map({
'X': 0,
'A': 1,
'B': 2,
'C': 3,
'D': 4,
'E': 5,
'F': 6,
'G': 7,
'T': 0
})
 test['Cabin'] = test['Cabin'].astype(int )
submission_dataset.to_csv("submission.csv" , index = False )<save_to_csv>
train_title = [i.split(",")[1].split(".")[0].strip() for i in train["Name"]]
train["Title"] = pd.Series(train_title)
 test_title = [i.split(",")[1].split(".")[0].strip() for i in test["Name"]]
test["Title"] = pd.Series(test_title )
submission_dataset.to_csv("submission.csv" , index = False )<set_options>
train = train.drop(['Name'], axis = 1)
 test = test.drop(['Name'], axis = 1 )
plotly.offline.init_notebook_mode()
%matplotlib inline
def RMSLE(pred,actual):
return np.sqrt(np.mean(np.power(( np.log(pred+1)-np.log(actual+1)) ,2)) )<set_options>
train["Title"] = train["Title"].replace(['Lady', 'the Countess','Countess','Capt', 'Col','Don', 'Dr', 'Major', 'Rev', 'Sir', 'Jonkheer', 'Dona'], 'Rare')
 train["Title"] = train["Title"].map({"Master":0, "Miss":1, "Ms" : 1 , "Mme":1, "Mlle":1, "Mrs":1, "Mr":2, "Rare":3})
 train["Title"] = train["Title"].astype(int)
 test["Title"] = test["Title"].replace(['Lady', 'the Countess','Countess','Capt', 'Col','Don', 'Dr', 'Major', 'Rev', 'Sir', 'Jonkheer', 'Dona'], 'Rare')
 test["Title"] = test["Title"].map({"Master":0, "Miss":1, "Ms" : 1 , "Mme":1, "Mlle":1, "Mrs":1, "Mr":2, "Rare":3})
 test["Title"] = test["Title"].astype(int )
warnings.filterwarnings("ignore")
 %matplotlib inline
%config InlineBackend.figure_format = 'retina' 
<load_from_csv>
Ticket1 = []
for i in list(train.Ticket):
if not i.isdigit() :
Ticket1.append(i.replace(".","" ).replace("/","" ).strip().split(' ')[0])
 else:
Ticket1.append("X")
 train["Ticket"] = Ticket1
Ticket2 = []
for j in list(test.Ticket):
if not j.isdigit() :
Ticket2.append(j.replace(".","" ).replace("/","" ).strip().split(' ')[0])
 else:
Ticket2.append("X")
 test["Ticket"] = Ticket2
train = pd.read_csv("/kaggle/input/covid19-global-forecasting-week-4/train.csv")
 test = pd.read_csv("/kaggle/input/covid19-global-forecasting-week-4/test.csv")
 region_metadata = pd.read_csv("/kaggle/input/covid19-forecasting-metadata/region_metadata.csv")
 region_date_metadata = pd.read_csv("/kaggle/input/covid19-forecasting-metadata/region_date_metadata.csv" )<merge>
train= pd.get_dummies(train, columns = ["Ticket"], prefix="T")
 test = pd.get_dummies(test, columns = ["Ticket"], prefix="T" )
train = train.merge(test[["ForecastId", "Province_State", "Country_Region", "Date"]], on = ["Province_State", "Country_Region", "Date"], how = "left")
 display(train.head())
 test = test[~test.Date.isin(train.Date.unique())]
display(test.head())
 df = pd.concat([train, test], sort = False)
 df.head()<categorify>
train = train.drop(['T_SP','T_SOP','T_Fa','T_LINE','T_SWPP','T_SCOW','T_PPP','T_AS','T_CASOTON'],axis = 1)
 test = test.drop(['T_SCA3','T_STONOQ','T_AQ4','T_A','T_LP','T_AQ3'],axis = 1 )
df["geo"] = df.Country_Region.astype(str)+ ": " + df.Province_State.astype(str)
 df.loc[df.Province_State.isna() , "geo"] = df[df.Province_State.isna() ].Country_Region
df.ConfirmedCases = df.groupby("geo")["ConfirmedCases"].cummax()
df.Fatalities = df.groupby("geo")["Fatalities"].cummax()
df = df.merge(region_metadata, on = ["Country_Region", "Province_State"])
 df = df.merge(region_date_metadata, on = ["Country_Region", "Province_State", "Date"], how = "left")
 df.continent = LabelEncoder().fit_transform(df.continent)
 df.Date = pd.to_datetime(df.Date, format = "%Y-%m-%d")
 df.sort_values(["geo", "Date"], inplace = True)
 df.head()<categorify>
train.drop(['Survived'],axis=1,inplace=True )
DAYS_SINCE_CASES = [1, 10, 50, 100, 500, 1000, 5000, 10000]
min_date_train = np.min(df[~df.Id.isna() ].Date)
 max_date_train = np.max(df[~df.Id.isna() ].Date)
 min_date_test = np.min(df[~df.ForecastId.isna() ].Date)
 max_date_test = np.max(df[~df.ForecastId.isna() ].Date)
 n_dates_test = len(df[~df.ForecastId.isna() ].Date.unique())
 print("Train date range:", str(min_date_train), " - ", str(max_date_train))
print("Test date range:", str(min_date_test), " - ", str(max_date_test))
for lag in range(1, 41):
df[f"lag_{lag}_cc"] = df.groupby("geo")["ConfirmedCases"].shift(lag)
 df[f"lag_{lag}_ft"] = df.groupby("geo")["Fatalities"].shift(lag)
 df[f"lag_{lag}_rc"] = df.groupby("geo")["Recoveries"].shift(lag)
 for case in DAYS_SINCE_CASES:
df = df.merge(df[df.ConfirmedCases >= case].groupby("geo")["Date"].min().reset_index().rename(columns = {"Date": f"case_{case}_date"}), on = "geo", how = "left" )<feature_engineering>
print('Train:')
 print(train.isnull().sum())
 print('Test:')
 print(test.isnull().sum() )
def prepare_features(df, gap):
df["perc_1_ac"] =(df[f"lag_{gap}_cc"] - df[f"lag_{gap}_ft"] - df[f"lag_{gap}_rc"])/ df[f"lag_{gap}_cc"]
df["perc_1_cc"] = df[f"lag_{gap}_cc"] / df.population
df["diff_1_cc"] = df[f"lag_{gap}_cc"] - df[f"lag_{gap + 1}_cc"]
df["diff_2_cc"] = df[f"lag_{gap + 1}_cc"] - df[f"lag_{gap + 2}_cc"]
df["diff_3_cc"] = df[f"lag_{gap + 2}_cc"] - df[f"lag_{gap + 3}_cc"]
df["diff_1_ft"] = df[f"lag_{gap}_ft"] - df[f"lag_{gap + 1}_ft"]
df["diff_2_ft"] = df[f"lag_{gap + 1}_ft"] - df[f"lag_{gap + 2}_ft"]
df["diff_3_ft"] = df[f"lag_{gap + 2}_ft"] - df[f"lag_{gap + 3}_ft"]
df["diff_123_cc"] =(df[f"lag_{gap}_cc"] - df[f"lag_{gap + 3}_cc"])/ 3
df["diff_123_ft"] =(df[f"lag_{gap}_ft"] - df[f"lag_{gap + 3}_ft"])/ 3
df["diff_change_1_cc"] = df.diff_1_cc / df.diff_2_cc
df["diff_change_2_cc"] = df.diff_2_cc / df.diff_3_cc
df["diff_change_1_ft"] = df.diff_1_ft / df.diff_2_ft
df["diff_change_2_ft"] = df.diff_2_ft / df.diff_3_ft
df["diff_change_12_cc"] =(df.diff_change_1_cc + df.diff_change_2_cc)/ 2
df["diff_change_12_ft"] =(df.diff_change_1_ft + df.diff_change_2_ft)/ 2
df["change_1_cc"] = df[f"lag_{gap}_cc"] / df[f"lag_{gap + 1}_cc"]
df["change_2_cc"] = df[f"lag_{gap + 1}_cc"] / df[f"lag_{gap + 2}_cc"]
df["change_3_cc"] = df[f"lag_{gap + 2}_cc"] / df[f"lag_{gap + 3}_cc"]
df["change_1_ft"] = df[f"lag_{gap}_ft"] / df[f"lag_{gap + 1}_ft"]
df["change_2_ft"] = df[f"lag_{gap + 1}_ft"] / df[f"lag_{gap + 2}_ft"]
df["change_3_ft"] = df[f"lag_{gap + 2}_ft"] / df[f"lag_{gap + 3}_ft"]
df["change_123_cc"] = df[f"lag_{gap}_cc"] / df[f"lag_{gap + 3}_cc"]
df["change_123_ft"] = df[f"lag_{gap}_ft"] / df[f"lag_{gap + 3}_ft"]
for case in DAYS_SINCE_CASES:
df[f"days_since_{case}_case"] =(df[f"case_{case}_date"] - df.Date ).astype("timedelta64[D]")
 df.loc[df[f"days_since_{case}_case"] < gap, f"days_since_{case}_case"] = np.nan
df["country_flag"] = df.Province_State.isna().astype(int)
 df["density"] = df.population / df.area
df["target_cc"] = np.log1p(df.ConfirmedCases)- np.log1p(df[f"lag_{gap}_cc"])
 df["target_ft"] = np.log1p(df.Fatalities)- np.log1p(df[f"lag_{gap}_ft"])
 features = [
f"lag_{gap}_cc",
f"lag_{gap}_ft",
f"lag_{gap}_rc",
"perc_1_ac",
"perc_1_cc",
"diff_1_cc",
"diff_2_cc",
"diff_3_cc",
"diff_1_ft",
"diff_2_ft",
"diff_3_ft",
"diff_123_cc",
"diff_123_ft",
"diff_change_1_cc",
"diff_change_2_cc",
"diff_change_1_ft",
"diff_change_2_ft",
"diff_change_12_cc",
"diff_change_12_ft",
"change_1_cc",
"change_2_cc",
"change_3_cc",
"change_1_ft",
"change_2_ft",
"change_3_ft",
"change_123_cc",
"change_123_ft",
"days_since_1_case",
"days_since_10_case",
"days_since_50_case",
"days_since_100_case",
"days_since_500_case",
"days_since_1000_case",
"days_since_5000_case",
"days_since_10000_case",
"country_flag",
"lat",
"lon",
"continent",
"population",
"area",
"density",
"target_cc",
"target_ft"
]
return df[features]<predict_on_test>
scaler = StandardScaler()
train2 = scaler.fit_transform(train)
 test2 = scaler.fit_transform(test )
def build_predict_lgbm(df_train, df_test, gap):
df_train.dropna(subset = ["target_cc", "target_ft", f"lag_{gap}_cc", f"lag_{gap}_ft"], inplace = True)
 target_cc = df_train.target_cc
target_ft = df_train.target_ft
test_lag_cc = df_test[f"lag_{gap}_cc"].values
test_lag_ft = df_test[f"lag_{gap}_ft"].values
df_train.drop(["target_cc", "target_ft"], axis = 1, inplace = True)
 df_test.drop(["target_cc", "target_ft"], axis = 1, inplace = True)
 categorical_features = ["continent"]
dtrain_cc = lgb.Dataset(df_train, label = target_cc, categorical_feature = categorical_features)
 dtrain_ft = lgb.Dataset(df_train, label = target_ft, categorical_feature = categorical_features)
 model_cc = lgb.train(LGB_PARAMS, train_set = dtrain_cc, num_boost_round = 200)
 model_ft = lgb.train(LGB_PARAMS, train_set = dtrain_ft, num_boost_round = 200)
 y_pred_cc = np.expm1(model_cc.predict(df_test, num_boost_round = 200)+ np.log1p(test_lag_cc))
y_pred_ft = np.expm1(model_ft.predict(df_test, num_boost_round = 200)+ np.log1p(test_lag_ft))
return y_pred_cc, y_pred_ft, model_cc, model_ft<predict_on_test>
KFold_Score = pd.DataFrame()
classifiers = ['Linear SVM', 'Radial SVM', 'LogisticRegression',
'RandomForestClassifier', 'AdaBoostClassifier',
'XGBoostClassifier', 'KNeighborsClassifier','GradientBoostingClassifier']
models = [svm.SVC(kernel='linear'),
svm.SVC(kernel='rbf'),
LogisticRegression(max_iter = 1000),
RandomForestClassifier(n_estimators=200, random_state=0),
AdaBoostClassifier(random_state = 0),
xgb.XGBClassifier(n_estimators=100),
KNeighborsClassifier() ,
GradientBoostingClassifier(random_state=0)
 ]
j = 0
for i in models:
model = i
cv = KFold(n_splits=5, random_state=0, shuffle=True)
 KFold_Score[classifiers[j]] =(cross_val_score(model, train2, np.ravel(pred), scoring = 'accuracy', cv=cv))
j = j+1
def predict_mad(df_test, gap, val = False):
df_test["avg_diff_cc"] =(df_test[f"lag_{gap}_cc"] - df_test[f"lag_{gap + 3}_cc"])/ 3
df_test["avg_diff_ft"] =(df_test[f"lag_{gap}_ft"] - df_test[f"lag_{gap + 3}_ft"])/ 3
if val:
y_pred_cc = df_test[f"lag_{gap}_cc"] + gap * df_test.avg_diff_cc -(1 - MAD_FACTOR)* df_test.avg_diff_cc * np.sum([x for x in range(gap)])/ VAL_DAYS
y_pred_ft = df_test[f"lag_{gap}_ft"] + gap * df_test.avg_diff_ft -(1 - MAD_FACTOR)* df_test.avg_diff_ft * np.sum([x for x in range(gap)])/ VAL_DAYS
else:
y_pred_cc = df_test[f"lag_{gap}_cc"] + gap * df_test.avg_diff_cc -(1 - MAD_FACTOR)* df_test.avg_diff_cc * np.sum([x for x in range(gap)])/ n_dates_test
y_pred_ft = df_test[f"lag_{gap}_ft"] + gap * df_test.avg_diff_ft -(1 - MAD_FACTOR)* df_test.avg_diff_ft * np.sum([x for x in range(gap)])/ n_dates_test
return y_pred_cc, y_pred_ft<init_hyperparams>
mean = pd.DataFrame(KFold_Score.mean() , index= classifiers)
 KFold_Score = pd.concat([KFold_Score,mean.T])
 KFold_Score.index=['Fold 1','Fold 2','Fold 3','Fold 4','Fold 5','Mean']
KFold_Score.T.sort_values(by=['Mean'], ascending = False )
SEED = 24
LGB_PARAMS = {"objective": "regression",
"num_leaves": 5,
"learning_rate": 0.013,
"bagging_fraction": 0.91,
"feature_fraction": 0.81,
"reg_alpha": 0.13,
"reg_lambda": 0.13,
"metric": "rmse",
"seed": SEED
}
VAL_DAYS = 7
MAD_FACTOR = 0.5<split>
col_name1[0],col_name1[2] = col_name1[2],col_name1[0]
col_name2[0],col_name2[2] = col_name2[2],col_name2[0]
df_train = df[~df.Id.isna() ]
df_test_full = df[~df.ForecastId.isna() ]<feature_engineering>
train_new = train[col_name1]
test_new = test[col_name2]
df_preds_val = []
df_preds_test = []
for date in df_test_full.Date.unique() :
print("[INFO] Date:", date)
 if date in df_train.Date.values:
df_pred_test = df_test_full.loc[df_test_full.Date == date, ["ForecastId", "ConfirmedCases", "Fatalities"]].rename(columns = {"ConfirmedCases": "ConfirmedCases_test", "Fatalities": "Fatalities_test"})
 else:
df_test = df_test_full[df_test_full.Date == date]
gap =(pd.Timestamp(date)- max_date_train ).days
if gap <= VAL_DAYS:
val_date = max_date_train - pd.Timedelta(VAL_DAYS, "D")+ pd.Timedelta(gap, "D")
 df_build = df_train[df_train.Date < val_date]
df_val = df_train[df_train.Date == val_date]
X_build = prepare_features(df_build, gap)
 X_val = prepare_features(df_val, gap)
 y_val_cc_lgb, y_val_ft_lgb, _, _ = build_predict_lgbm(X_build, X_val, gap)
 y_val_cc_mad, y_val_ft_mad = predict_mad(df_val, gap, val = True)
 df_pred_val = pd.DataFrame({"Id": df_val.Id.values,
"ConfirmedCases_val_lgb": y_val_cc_lgb,
"Fatalities_val_lgb": y_val_ft_lgb,
"ConfirmedCases_val_mad": y_val_cc_mad,
"Fatalities_val_mad": y_val_ft_mad,
})
 df_preds_val.append(df_pred_val)
 X_train = prepare_features(df_train, gap)
 X_test = prepare_features(df_test, gap)
 y_test_cc_lgb, y_test_ft_lgb, model_cc, model_ft = build_predict_lgbm(X_train, X_test, gap)
 y_test_cc_mad, y_test_ft_mad = predict_mad(df_test, gap)
 if gap == 1:
model_1_cc = model_cc
model_1_ft = model_ft
features_1 = X_train.columns.values
elif gap == 14:
model_14_cc = model_cc
model_14_ft = model_ft
features_14 = X_train.columns.values
elif gap == 28:
model_28_cc = model_cc
model_28_ft = model_ft
features_28 = X_train.columns.values
df_pred_test = pd.DataFrame({"ForecastId": df_test.ForecastId.values,
"ConfirmedCases_test_lgb": y_test_cc_lgb,
"Fatalities_test_lgb": y_test_ft_lgb,
"ConfirmedCases_test_mad": y_test_cc_mad,
"Fatalities_test_mad": y_test_ft_mad,
})
 df_preds_test.append(df_pred_test )<merge>
train_new = train_new.drop(['Cabin'],axis = 1)
 test_new = test_new.drop(['Cabin'],axis = 1 )
df = df.merge(pd.concat(df_preds_val, sort = False), on = "Id", how = "left")
 df = df.merge(pd.concat(df_preds_test, sort = False), on = "ForecastId", how = "left")
 rmsle_cc_lgb = np.sqrt(mean_squared_error(np.log1p(df[~df.ConfirmedCases_val_lgb.isna() ].ConfirmedCases), np.log1p(df[~df.ConfirmedCases_val_lgb.isna() ].ConfirmedCases_val_lgb)))
 rmsle_ft_lgb = np.sqrt(mean_squared_error(np.log1p(df[~df.Fatalities_val_lgb.isna() ].Fatalities), np.log1p(df[~df.Fatalities_val_lgb.isna() ].Fatalities_val_lgb)))
 rmsle_cc_mad = np.sqrt(mean_squared_error(np.log1p(df[~df.ConfirmedCases_val_mad.isna() ].ConfirmedCases), np.log1p(df[~df.ConfirmedCases_val_mad.isna() ].ConfirmedCases_val_mad)))
 rmsle_ft_mad = np.sqrt(mean_squared_error(np.log1p(df[~df.Fatalities_val_mad.isna() ].Fatalities), np.log1p(df[~df.Fatalities_val_mad.isna() ].Fatalities_val_mad)))
 print("LGB CC RMSLE Val of", VAL_DAYS, "days for CC:", round(rmsle_cc_lgb, 2))
print("LGB FT RMSLE Val of", VAL_DAYS, "days for FT:", round(rmsle_ft_lgb, 2))
print("LGB Overall RMSLE Val of", VAL_DAYS, "days:", round(( rmsle_cc_lgb + rmsle_ft_lgb)/ 2, 2))
print("MAD CC RMSLE Val of", VAL_DAYS, "days for CC:", round(rmsle_cc_mad, 2))
print("MAD FT RMSLE Val of", VAL_DAYS, "days for FT:", round(rmsle_ft_mad, 2))
print("MAD Overall RMSLE Val of", VAL_DAYS, "days:", round(( rmsle_cc_mad + rmsle_ft_mad)/ 2, 2))<feature_engineering>
sc = StandardScaler()
train3 = sc.fit_transform(train_new)
 test3 = sc.transform(test_new )
test = df.loc[~df.ForecastId.isna() , ["ForecastId", "Country_Region", "Province_State", "Date",
"ConfirmedCases_test", "ConfirmedCases_test_lgb", "ConfirmedCases_test_mad",
"Fatalities_test", "Fatalities_test_lgb", "Fatalities_test_mad"]].reset_index()
test["ConfirmedCases"] = 0.3 * test.ConfirmedCases_test_lgb + 0.7 * test.ConfirmedCases_test_mad
test["Fatalities"] = 0.25 * test.Fatalities_test_lgb + 0.75 * test.Fatalities_test_mad
test.loc[test.Country_Region.isin(["China", "US", "Diamond Princess"]), "ConfirmedCases"] = test[test.Country_Region.isin(["China", "US", "Diamond Princess"])].ConfirmedCases_test_mad.values
test.loc[test.Country_Region.isin(["China", "US", "Diamond Princess"]), "Fatalities"] = test[test.Country_Region.isin(["China", "US", "Diamond Princess"])].Fatalities_test_mad.values
test.loc[test.Date.isin(df_train.Date.values), "ConfirmedCases"] = test[test.Date.isin(df_train.Date.values)].ConfirmedCases_test.values
test.loc[test.Date.isin(df_train.Date.values), "Fatalities"] = test[test.Date.isin(df_train.Date.values)].Fatalities_test.values
sub0 = test[["ForecastId", "ConfirmedCases", "Fatalities"]]
sub0.ForecastId = sub0.ForecastId.astype(int)
 sub0.head()<load_from_csv>
clf = RandomForestClassifier(random_state=0 )
test = pd.read_csv(".. /input/covid19-global-forecasting-week-4/test.csv")
 train = pd.read_csv(".. /input/covid19-global-forecasting-week-4/train.csv")
 train['Province_State'].fillna('', inplace=True)
 test['Province_State'].fillna('', inplace=True)
 train['Date'] = pd.to_datetime(train['Date'])
 test['Date'] = pd.to_datetime(test['Date'])
 train = train.sort_values(['Country_Region','Province_State','Date'])
 test = test.sort_values(['Country_Region','Province_State','Date'] )<feature_engineering>
param_grid={
'n_estimators': [200,300],
'max_features': ['auto', 'sqrt'],
'max_depth': [6,7,8],
'criterion':['gini','entropy']
}
FirstDate = train.groupby('Country_Region' ).min() ['Date'].unique() [0]
train['Last Confirm'] = train['ConfirmedCases'].shift(1)
 while train[(train['Last Confirm'] > train['ConfirmedCases'])&(train['Date'] > FirstDate)].shape[0] > 0:
train['Last Confirm'] = train['ConfirmedCases'].shift(1)
 train['Last Fatalities'] = train['Fatalities'].shift(1)
 train.loc[(train['Last Confirm'] > train['ConfirmedCases'])&(train['Date'] > FirstDate),'ConfirmedCases'] = train.loc[(train['Last Confirm'] > train['ConfirmedCases'])&(train['Date'] > FirstDate),'Last Confirm']
train.loc[(train['Last Fatalities'] > train['Fatalities'])&(train['Date'] > FirstDate),'Fatalities'] = train.loc[(train['Last Fatalities'] > train['Fatalities'])&(train['Date'] > FirstDate),'Last Fatalities']
train['Last Confirm'] = train['ConfirmedCases'].shift(1)
 train['Last Fatalities'] = train['Fatalities'].shift(1 )<import_modules>
CV_clf = GridSearchCV(estimator=clf, param_grid=param_grid, cv=5)
 CV_clf.fit(train3, pred)
 CV_clf.best_params_
from statsmodels.tsa.statespace.sarimax import SARIMAX
from statsmodels.tsa.arima_model import ARIMA<feature_engineering>
clf1 = RandomForestClassifier(random_state=0, n_estimators=200, criterion='gini', max_features='auto', max_depth=8)
 clf1.fit(train3, pred )
feature_day = [1,20,50,100,200,500,1000]
def CreateInput(data):
feature = []
for day in feature_day:
data.loc[:,'Number day from ' + str(day)+ ' case'] = 0
if(train[(train['Country_Region'] == country)&(train['Province_State'] == province)&(train['ConfirmedCases'] < day)]['Date'].count() > 0):
fromday = train[(train['Country_Region'] == country)&(train['Province_State'] == province)&(train['ConfirmedCases'] < day)]['Date'].max()
else:
fromday = train[(train['Country_Region'] == country)&(train['Province_State'] == province)]['Date'].min()
for i in range(0, len(data)) :
if(data['Date'].iloc[i] > fromday):
day_denta = data['Date'].iloc[i] - fromday
data['Number day from ' + str(day)+ ' case'].iloc[i] = day_denta.days
feature = feature + ['Number day from ' + str(day)+ ' case']
return data[feature]
pred_data_all = pd.DataFrame()
with tqdm(total=len(train['Country_Region'].unique())) as pbar:
for country in train['Country_Region'].unique() :
for province in train[(train['Country_Region'] == country)]['Province_State'].unique() :
with warnings.catch_warnings() :
warnings.filterwarnings("ignore")
 df_train = train[(train['Country_Region'] == country)&(train['Province_State'] == province)]
df_test = test[(test['Country_Region'] == country)&(test['Province_State'] == province)]
X_train = CreateInput(df_train)
 y_train_confirmed = df_train['ConfirmedCases'].ravel()
y_train_fatalities = df_train['Fatalities'].ravel()
X_pred = CreateInput(df_test)
 feature_use = X_pred.columns[0]
for i in range(X_pred.shape[1] - 1,0,-1):
if(X_pred.iloc[0,i] > 0):
feature_use = X_pred.columns[i]
break
idx = X_train[X_train[feature_use] == 0].shape[0]
adjusted_X_train = X_train[idx:][feature_use].values.reshape(-1, 1)
 adjusted_y_train_confirmed = y_train_confirmed[idx:]
adjusted_y_train_fatalities = y_train_fatalities[idx:]
idx = X_pred[X_pred[feature_use] == 0].shape[0]
adjusted_X_pred = X_pred[idx:][feature_use].values.reshape(-1, 1)
 pred_data = test[(test['Country_Region'] == country)&(test['Province_State'] == province)]
max_train_date = train[(train['Country_Region'] == country)&(train['Province_State'] == province)]['Date'].max()
min_test_date = pred_data['Date'].min()
model = SARIMAX(adjusted_y_train_confirmed, order=(1,1,0),
measurement_error=True ).fit(disp=False)
 y_hat_confirmed = model.forecast(pred_data[pred_data['Date'] > max_train_date].shape[0])
 y_train_confirmed = train[(train['Country_Region'] == country)&(train['Province_State'] == province)&(train['Date'] >= min_test_date)]['ConfirmedCases'].values
y_hat_confirmed = np.concatenate(( y_train_confirmed,y_hat_confirmed), axis = 0)
 model = SARIMAX(adjusted_y_train_fatalities, order=(1,1,0),
measurement_error=True ).fit(disp=False)
 y_hat_fatalities = model.forecast(pred_data[pred_data['Date'] > max_train_date].shape[0])
 y_train_fatalities = train[(train['Country_Region'] == country)&(train['Province_State'] == province)&(train['Date'] >= min_test_date)]['Fatalities'].values
y_hat_fatalities = np.concatenate(( y_train_fatalities,y_hat_fatalities), axis = 0)
 pred_data['ConfirmedCases_hat'] = y_hat_confirmed
pred_data['Fatalities_hat'] = y_hat_fatalities
pred_data_all = pred_data_all.append(pred_data)
 pbar.update(1)
 df_val = pd.merge(pred_data_all,train[['Date','Country_Region','Province_State','ConfirmedCases','Fatalities']],on=['Date','Country_Region','Province_State'], how='left')
 df_val.loc[df_val['Fatalities_hat'] < 0,'Fatalities_hat'] = 0
df_val.loc[df_val['ConfirmedCases_hat'] < 0,'ConfirmedCases_hat'] = 0
df_val_3 = df_val.copy()<rename_columns>
pred3 = clf1.predict(test3 )
sub1 = df_val_3
submission = sub1[['ForecastId','ConfirmedCases_hat','Fatalities_hat']]
submission.columns = ['ForecastId','ConfirmedCases','Fatalities']<save_to_csv>
pred_test = pred3
output = pd.DataFrame({
'PassengerId': test_data.PassengerId,
'Survived': pred_test
})
 output.to_csv('./submission.csv', index=False )