Kaggle competition: COVID19 Global Forecasting (Week 2)
data.loc[(( data['age'] > 19)|(data['age'] < 7)) &(data['rez_esc'].isnull()), 'rez_esc'] = 0
data['rez_esc-missing'] = data['rez_esc'].isnull()<feature_engineering>
pd.set_option("display.max_columns", 200)
 pd.set_option("display.max_rows", 200 )
data.loc[data['rez_esc'] > 5, 'rez_esc'] = 5<train_model>
country_info = pd.read_csv("/kaggle/input/countryinfo/covid19countryinfo.csv")
 country_info = country_info.rename({"region": "state"}, axis=1)
 country_info.loc[country_info["state"].isna() , "state"] = "Unknown"
country_info = country_info.drop([col for col in country_info.columns if "Unnamed" in col], axis=1)
 country_info["pop"] = country_info["pop"].str.replace(',', '' ).astype(float)
 pollution = pd.read_csv("/kaggle/input/pollution-by-country-for-covid19-analysis/region_pollution.csv")
 pollution = pollution.rename({"Region": "country",
"Outdoor Pollution(deaths per 100000)": "outdoor_pol",
"Indoor Pollution(deaths per 100000)": "indoor_pol"}, axis=1)
 economy = pd.read_csv("/kaggle/input/the-economic-freedom-index/economic_freedom_index2019_data.csv", engine='python')
 economy_cols = [col for col in economy.columns if economy[col].dtype == "float64"] + ["Country"]
economy = economy[economy_cols]
economy = economy.rename({"Country": "country"}, axis=1)
 def append_external_data(df):
df = pd.merge(df, country_info, on=["country", "state"], how="left")
 df = pd.merge(df, pollution, on="country", how="left")
 df = pd.merge(df, economy, on="country", how="left")
 return df
id_ = ['Id', 'idhogar', 'Target']<define_variables>
country_info["publicplace"] = np.where(country_info["publicplace"].str.contains("/"), country_info["publicplace"], np.nan )
ind_bool = ['v18q', 'dis', 'male', 'female', 'estadocivil1', 'estadocivil2', 'estadocivil3',
'estadocivil4', 'estadocivil5', 'estadocivil6', 'estadocivil7',
'parentesco1', 'parentesco2', 'parentesco3', 'parentesco4', 'parentesco5',
'parentesco6', 'parentesco7', 'parentesco8', 'parentesco9', 'parentesco10',
'parentesco11', 'parentesco12', 'instlevel1', 'instlevel2', 'instlevel3',
'instlevel4', 'instlevel5', 'instlevel6', 'instlevel7', 'instlevel8',
'instlevel9', 'mobilephone', 'rez_esc-missing']
ind_ordered = ['rez_esc', 'escolari', 'age']<define_variables>
country_info["publicplace"].value_counts()
hh_bool = ['hacdor', 'hacapo', 'v14a', 'refrig', 'paredblolad', 'paredzocalo',
'paredpreb','pisocemento', 'pareddes', 'paredmad',
'paredzinc', 'paredfibras', 'paredother', 'pisomoscer', 'pisoother',
'pisonatur', 'pisonotiene', 'pisomadera',
'techozinc', 'techoentrepiso', 'techocane', 'techootro', 'cielorazo',
'abastaguadentro', 'abastaguafuera', 'abastaguano',
'public', 'planpri', 'noelec', 'coopele', 'sanitario1',
'sanitario2', 'sanitario3', 'sanitario5', 'sanitario6',
'energcocinar1', 'energcocinar2', 'energcocinar3', 'energcocinar4',
'elimbasu1', 'elimbasu2', 'elimbasu3', 'elimbasu4',
'elimbasu5', 'elimbasu6', 'epared1', 'epared2', 'epared3',
'etecho1', 'etecho2', 'etecho3', 'eviv1', 'eviv2', 'eviv3',
'tipovivi1', 'tipovivi2', 'tipovivi3', 'tipovivi4', 'tipovivi5',
'computer', 'television', 'lugar1', 'lugar2', 'lugar3',
'lugar4', 'lugar5', 'lugar6', 'area1', 'area2', 'v2a1-missing']
hh_ordered = [ 'rooms', 'r4h1', 'r4h2', 'r4h3', 'r4m1','r4m2','r4m3', 'r4t1', 'r4t2',
'r4t3', 'v18q1', 'tamhog','tamviv','hhsize','hogar_nin',
'hogar_adul','hogar_mayor','hogar_total', 'bedrooms', 'qmobilephone']
hh_cont = ['v2a1', 'dependency', 'edjefe', 'edjefa', 'meaneduc', 'overcrowding']<define_variables>
list_rel_columns = ['state', 'country', 'pop', 'tests',
'testpop', 'density', 'medianage', 'urbanpop', 'quarantine', 'schools',
'publicplace', 'gatheringlimit', 'gathering', 'nonessential',
'hospibed', 'smokers', 'sex0', 'sex14', 'sex25', 'sex54', 'sex64',
'sex65plus', 'sexratio', 'lung', 'femalelung', 'malelung', 'gdp2019',
'healthexp', 'healthperpop', 'fertility']
sqr_ = ['SQBescolari', 'SQBage', 'SQBhogar_total', 'SQBedjefe',
'SQBhogar_nin', 'SQBovercrowding', 'SQBdependency', 'SQBmeaned', 'agesq']<define_variables>
country_info = country_info[list_rel_columns]
x = ind_bool + ind_ordered + id_ + hh_bool + hh_ordered + hh_cont + sqr_
print('There are no repeats: ', np.all(np.array(list(Counter(x ).values())) == 1))
print('We covered every variable: ', len(x)== data.shape[1] )<filter>
country_info.loc[country_info["country"] == "China"]
corr_matrix.loc[corr_matrix['tamhog'].abs() > 0.9, corr_matrix['tamhog'].abs() > 0.9]<drop_column>
def aggregate_label(df):
country_df = df[["country", "Date", "ConfirmedCases", "Fatalities"]].groupby(["country", "Date"], as_index=False ).sum()
country_df = country_df.rename({"ConfirmedCases": "country_cases", "Fatalities": "country_fatalities"}, axis=1)
 df = pd.merge(df, country_df, on=["country", "Date"], how="left")
 return df
heads = heads.drop(columns = ['tamhog', 'hogar_total', 'r4t3'] )<feature_engineering>
def calculate_days_since_event(df, feature_name, casualties, casualties_amount, groupby=["country"]):
cases_df = df.loc[df[casualties] > casualties_amount][groupby + ["Date"]].groupby(groupby, as_index=False ).min()
cases_df = cases_df.rename({"Date": "relevant_date"}, axis=1)
 df = pd.merge(df, cases_df, on=groupby, how="left")
 df[feature_name] =(pd.to_datetime(df["Date"])- pd.to_datetime(df["relevant_date"])).dt.days
df.loc[df[feature_name] < 0, feature_name] = 0
df = df.drop("relevant_date", axis = 1)
 return df
elec = []
for i, row in heads.iterrows() :
if row['noelec'] == 1:
elec.append(0)
 elif row['coopele'] == 1:
elec.append(1)
 elif row['public'] == 1:
elec.append(2)
 elif row['planpri'] == 1:
elec.append(3)
 else:
elec.append(np.nan)
 heads['elec'] = elec
heads['elec-missing'] = heads['elec'].isnull() 
<groupby>
def generate_time_features(df):
df = calculate_days_since_event(df, "days_from_first_death", "Fatalities", 0, ["country"])
 df = calculate_days_since_event(df, "days_from_first_case", "ConfirmedCases", 0, ["country"])
 df = calculate_days_since_event(df, "days_from_first_case_province", "ConfirmedCases", 0, ["country", "state"])
 df = calculate_days_since_event(df, "days_from_first_death_province", "Fatalities", 0, ["country", "state"])
 df = calculate_days_since_event(df, "days_from_centenary_case", "ConfirmedCases", 99, ["country"])
 df = calculate_days_since_event(df, "days_from_centenary_case_province", "Fatalities", 99, ["country", "state"])
 df = calculate_days_since_event(df, "days_from_centenary_daily_cases_province", "ConfirmedCases_daily", 99, ["country", "state"])
 df = calculate_days_since_event(df, "days_from_centenary_daily_cases", "ConfirmedCases_daily", 99, ["country"])
 df["days_from_first_ever_case"] =(pd.to_datetime(df["Date"])- pd.to_datetime("2019-12-01")).dt.days
df.loc[df["days_from_first_ever_case"] < 0, "days_from_first_ever_case"] = 0
df["days_from_quarantine"] =(pd.to_datetime(df["Date"])- pd.to_datetime(df["quarantine"])).dt.days
df["days_from_quarantine"].fillna(0)
 df.loc[df["days_from_quarantine"] < 30, "days_from_quarantine"] = 0
df.loc[df["days_from_quarantine"] >= 30, "days_from_quarantine"] = 1
df["days_from_school"] =(pd.to_datetime(df["Date"])- pd.to_datetime(df["schools"])).dt.days
df["days_from_school"].fillna(df["days_from_quarantine"])
 df.loc[df["days_from_school"] < 30, "days_from_school"] = 0
df.loc[df["days_from_school"] >= 30, "days_from_school"] = 1
df["days_from_publicplace"] =(pd.to_datetime(df["Date"])- pd.to_datetime(df["publicplace"])).dt.days
df["days_from_publicplace"].fillna(df["days_from_quarantine"])
 df.loc[df["days_from_publicplace"] < 30, "days_from_publicplace"] = 0
df.loc[df["days_from_publicplace"] >= 30, "days_from_publicplace"] = 1
df["days_from_gathering"] =(pd.to_datetime(df["Date"])- pd.to_datetime(df["gathering"])).dt.days
df["days_from_gathering"].fillna(df["days_from_quarantine"])
 df.loc[df["days_from_gathering"] < 30, "days_from_gathering"] = 0
df.loc[df["days_from_gathering"] >= 30, "days_from_gathering"] = 1
df["days_from_nonessential"] =(pd.to_datetime(df["Date"])- pd.to_datetime(df["nonessential"])).dt.days
df["days_from_nonessential"].fillna(df["days_from_quarantine"])
 df.loc[df["days_from_nonessential"] < 30, "days_from_nonessential"] = 0
df.loc[df["days_from_nonessential"] >= 30, "days_from_nonessential"] = 1
return df
heads = heads.drop(columns = 'area2')
 heads.groupby('area1')['Target'].value_counts(normalize = True)
<drop_column>
def generate_ar_features(df, group_by_cols, value_cols):
diff_df = df.groupby(group_by_cols)[value_cols].diff().fillna(0)
 diff_df.columns = [col + "_daily" for col in value_cols]
value_cols += [col + "_daily" for col in value_cols]
df = pd.concat([df, diff_df], axis=1)
 pct_df = df.groupby(group_by_cols)[value_cols].pct_change().fillna(0)
 pct_df.columns = [col + "_pct_change" for col in value_cols]
value_cols += [col + "_pct_change" for col in value_cols]
df = pd.concat([df, pct_df], axis=1)
 yesterday_df = df.groupby(group_by_cols)[value_cols].shift()
value_cols = [col + "_yesterday" for col in value_cols]
yesterday_df.columns = value_cols
df = pd.concat([df, yesterday_df], axis=1)
 three_days_avg = df.groupby(group_by_cols)[value_cols].rolling(3 ).mean()
three_days_avg = three_days_avg.reset_index() [value_cols]
three_days_avg.columns = [col + "_3_day_avg" for col in three_days_avg.columns]
df = pd.concat([df, three_days_avg], axis=1)
 seven_days_avg = df.groupby(group_by_cols)[value_cols].rolling(7 ).mean()
seven_days_avg = seven_days_avg.reset_index() [value_cols]
seven_days_avg.columns = [col + "_7_day_avg" for col in seven_days_avg.columns]
df = pd.concat([df, seven_days_avg], axis=1)
 df = df.replace([np.inf, -np.inf], 0)
 return df
heads['roof'] = np.argmax(np.array(heads[['etecho1', 'etecho2', 'etecho3']]),
axis = 1)
 heads = heads.drop(columns = ['etecho1', 'etecho2', 'etecho3'])
 heads['floor'] = np.argmax(np.array(heads[['eviv1', 'eviv2', 'eviv3']]),
axis = 1)
 heads = heads.drop(columns = ['eviv1', 'eviv2', 'eviv3'] )<feature_engineering>
def generate_features(df):
group_by_cols = ["state","country"]
value_cols = ["ConfirmedCases", "Fatalities", "country_cases", "country_fatalities"]
df = aggregate_label(df)
 df = append_external_data(df)
 df = generate_ar_features(df, group_by_cols, value_cols)
 df = generate_time_features(df)
 df["dow"] = pd.to_datetime(df["Date"] ).dt.dayofweek
df.loc[df["ConfirmedCases_yesterday"]<0, "ConfirmedCases_yesterday"] = 0
df.loc[df["Fatalities_yesterday"]<0, "Fatalities_yesterday"] = 0
return df
heads['warning'] = 1 *(heads['sanitario1'] + 
(heads['elec'] == 0)+
heads['pisonotiene'] +
heads['abastaguano'] + 
(heads['cielorazo'] == 0)) 
<feature_engineering>
train = pd.read_csv("/kaggle/input/covid19-global-forecasting-week-2/train.csv")
 train = train.rename({"Province_State": "state", "Country_Region": "country"}, axis=1)
 train.loc[train["state"].isna() , "state"] = "Unknown"
train = generate_features(train)
 print(train["Date"].min() , "-", train["Date"].max())
 train.loc[train["country"] == "Italy"].tail()
heads['phones-per-capita'] = heads['qmobilephone'] / heads['tamviv']
heads['tablets-per-capita'] = heads['v18q1'] / heads['tamviv']
heads['rooms-per-capita'] = heads['rooms'] / heads['tamviv']
heads['rent-per-capita'] = heads['v2a1'] / heads['tamviv']<drop_column>
train.loc[train["state"] == "Hubei"]
ind = ind.drop(columns = 'male' )<groupby>
test = pd.read_csv("/kaggle/input/covid19-global-forecasting-week-2/test.csv")
 test = test.rename({"Province_State": "state", "Country_Region": "country"}, axis=1)
 test.loc[test["state"].isna() , "state"] = "Unknown"
print(test["Date"].min() , "-", test["Date"].max())
 test.head()
range_ = lambda x: x.max() - x.min()
range_.__name__ = 'range_'
ind_agg = ind.drop(columns = 'Target' ).groupby('idhogar' ).agg(['min', 'max', 'sum', 'count', 'std', range_])
 ind_agg.head()<rename_columns>
train.loc[train["Date"]<"2020-03-22", "split"] = "train"
train.loc[train["Date"]>="2020-03-22", "split"] = "test"
new_col = []
for c in ind_agg.columns.levels[0]:
for stat in ind_agg.columns.levels[1]:
new_col.append(f'{c}-{stat}')
 ind_agg.columns = new_col
ind_agg.head()<merge>
import lightgbm as lgb
from sklearn.metrics import mean_squared_error
ind_agg = ind_agg.drop(columns = to_drop)
 ind_feats = list(ind_agg.columns)
 final = heads.merge(ind_agg, on = 'idhogar', how = 'left')
 print('Final features shape: ', final.shape )<import_modules>
def train_model(df, label, base_label, features=features, **kwargs):
X_train = df.loc[df["split"] == "train"][features]
y_train = np.log(df.loc[df["split"] == "train"][label] + 1)
 b_train = np.log(df.loc[df["split"] == "train"][base_label] + 1)
 X_test = df.loc[df["split"] == "test"][features]
y_test = np.log(df.loc[df["split"] == "test", label] + 1)
 b_test = np.log(df.loc[df["split"] == "test", base_label] + 1)
 print(kwargs)
 model = lgb.LGBMRegressor(**kwargs)
 model.fit(X_train, y_train, init_score = b_train)
 y_pred = model.predict(X_test)
 print(np.sqrt(mean_squared_error(y_test, y_pred + b_test)))
 return model
scorer = make_scorer(f1_score, greater_is_better=True, average = 'macro' )<data_type_conversions>
lgb_model_cases = train_model(train, "ConfirmedCases", "ConfirmedCases_yesterday",
max_depth=5,
colsample_bytree=0.8,
learning_rate=0.1,
n_estimators=500,
subsample=0.8 )
train_labels = np.array(list(final[final['Target'].notnull() ]['Target'].astype(np.uint8)))
 train_set = final[final['Target'].notnull() ].drop(columns = ['Id', 'idhogar', 'Target'])
 test_set = final[final['Target'].isnull() ].drop(columns = ['Id', 'idhogar', 'Target'])
 submission_base = test[['Id', 'idhogar']].copy()<categorify>
lgb_model_fatalities = train_model(train, "Fatalities", "Fatalities_yesterday", features,
max_depth=5,
colsample_bytree=0.8,
learning_rate=0.1,
n_estimators=500,
subsample=0.8
)
features = list(train_set.columns)
 pipeline = Pipeline([('imputer', Imputer(strategy = 'median')) , 
('scaler', MinMaxScaler())])
 train_set = pipeline.fit_transform(train_set)
 test_set = pipeline.transform(test_set )<compute_train_metric>
X_train = train[features]
y_train = np.log(train["ConfirmedCases"] + 1)
 b_train = np.log(train["ConfirmedCases_yesterday"] + 1)
 cases_model = lgb.LGBMRegressor(max_depth=5,
colsample_bytree=0.8,
learning_rate=0.1,
n_estimators=500,
subsample=0.8 
)
 cases_model.fit(X_train, y_train, init_score = b_train)
 X_train = train[features]
y_train = np.log(train["Fatalities"] + 1)
 b_train = np.log(train["Fatalities_yesterday"] + 1)
 fatalities_model = lgb.LGBMRegressor(max_depth=5,
colsample_bytree=0.8,
learning_rate=0.1,
n_estimators=500,
subsample=0.8)
 fatalities_model.fit(X_train, y_train, init_score = b_train )
model = RandomForestClassifier(n_estimators=100, random_state=10,
n_jobs = -1)
 cv_score = cross_val_score(model, train_set, train_labels, cv = 10, scoring = scorer)
 print(f'10 Fold Cross Validation F1 Score = {round(cv_score.mean() , 4)} with std = {round(cv_score.std() , 4)}' )<train_model>
base_df = pd.read_csv("/kaggle/input/covid19-global-forecasting-week-2/train.csv")
 base_df = base_df.rename({"Province_State": "state", "Country_Region": "country"}, axis=1)
 base_df.loc[base_df["state"].isna() , "state"] = "Unknown"
scoring_dates = test["Date"].unique()
model.fit(train_set, train_labels)
 feature_importances = pd.DataFrame({'feature': features, 'importance': model.feature_importances_})
 feature_importances.head()<import_modules>
from datetime import datetime as dt, timedelta
from sklearn.svm import LinearSVC
from sklearn.naive_bayes import GaussianNB
from sklearn.neural_network import MLPClassifier
from sklearn.linear_model import LogisticRegressionCV, RidgeClassifierCV
from sklearn.discriminant_analysis import LinearDiscriminantAnalysis
from sklearn.neighbors import KNeighborsClassifier<compute_train_metric>
pred_df = pd.DataFrame(columns=base_df.columns)
 for date in scoring_dates.tolist() :
print(date)
 new_df = base_df.loc[base_df["Date"] < date].copy()
curr_date_df = test.loc[test["Date"] == date].copy()
curr_date_df["ConfirmedCases"] = 0
curr_date_df["Fatalities"] = 0
new_df = new_df.append(curr_date_df ).reset_index(drop=True)
 new_df = generate_features(new_df)
 new_df[features] = new_df[features]
predictions = cases_model.predict(new_df[features])+ np.log(new_df["ConfirmedCases_yesterday"] + 1)
 new_df["predicted_cases"] = round(np.maximum(np.exp(predictions)- 1, new_df["ConfirmedCases_yesterday"]))
predictions = fatalities_model.predict(new_df[features])+ np.log(new_df["Fatalities_yesterday"] + 1)
 new_df["predicted_fatalities"] = np.maximum(np.exp(predictions)- 1, new_df["Fatalities_yesterday"])
 new_df["predicted_fatalities"] = round(np.minimum(new_df["predicted_fatalities"], new_df["predicted_cases"]*0.2))
new_df.loc[new_df["Date"] == date, "ConfirmedCases"] = new_df.loc[new_df["Date"] == date, "predicted_cases"]
new_df.loc[new_df["Date"] == date, "Fatalities"] = new_df.loc[new_df["Date"] == date, "predicted_fatalities"]
pred_df = pred_df.append(new_df.loc[new_df["Date"] == date][pred_df.columns.tolist() ])
 if date not in base_df["Date"].unique() :
base_df = base_df.append(new_df.loc[new_df["Date"] == date][base_df.columns.tolist() ] )
model_results = cv_model(train_set, train_labels, LinearSVC() ,
'LSVC', model_results )<compute_test_metric>
pred_df.loc[pred_df["state"] == "Hubei"]
model_results = cv_model(train_set, train_labels,
GaussianNB() , 'GNB', model_results )<choose_model_class>
pred_df.loc[pred_df["country"] == "Italy"]
model_results = cv_model(train_set, train_labels,
MLPClassifier(hidden_layer_sizes=(32, 64, 128, 64, 32)) ,
'MLP', model_results )<compute_test_metric>
pred_df.loc[pred_df["country"] == "Israel"]
model_results = cv_model(train_set, train_labels,
LinearDiscriminantAnalysis() ,
'LDA', model_results )<compute_train_metric>
pred_df.loc[pred_df["country"] == "Argentina"]
model_results = cv_model(train_set, train_labels,
RidgeClassifierCV() , 'RIDGE', model_results )<find_best_params>
pred_df.loc[pred_df["country"] == "Uruguay"]
for n in [5, 10, 20]:
print(f'
KNN with {n} neighbors
')
 model_results = cv_model(train_set, train_labels,
KNeighborsClassifier(n_neighbors = n),
f'knn-{n}', model_results )<choose_model_class>
test = pd.merge(test, pred_df[["state", "country", "Date", "ConfirmedCases", "Fatalities"]], on=["state", "country", "Date"], how="left" )
model_results = cv_model(train_set, train_labels,
ExtraTreesClassifier(n_estimators = 100, random_state = 10),
'EXT', model_results )<compute_train_metric>
test[["ForecastId", "ConfirmedCases", "Fatalities"]].to_csv("submission.csv", index=False )