Kaggle competition: Competição DSA de Machine Learning
<SOS> metric: cateagorizationaccuracy Kaggle data source: competicao-dsa-machine-learning-jan-2019<save_to_csv>
warnings.filterwarnings('ignore')
 %matplotlib inline
daily_log_deaths = train_clean.pivot('Location', 'Date', 'LogFatalities' ).reset_index()
daily_log_deaths = daily_log_deaths.sort_values(TRAIN_END, ascending=False)
 daily_log_deaths.to_csv('daily_log_deaths.csv', index=False)
 for i, d in tqdm(enumerate(pd.date_range(add_days(TRAIN_END, 1), add_days(TEST_END, 1)))) :
new_day = str(d ).split(' ')[0]
last_day = dt.datetime.strptime(new_day, DATEFORMAT)- dt.timedelta(days=1)
 last_day = last_day.strftime(DATEFORMAT)
 for loc in death_deltas.Location:
death_delta = death_deltas.loc[death_deltas.Location == loc, 'DELTA'].values[0]
daily_log_deaths.loc[daily_log_deaths.Location == loc, new_day] = daily_log_deaths.loc[daily_log_deaths.Location == loc, last_day] + \
death_delta * DECAY ** i<feature_engineering>
dataset_treino = pd.read_csv(".. /input/dataset_treino.csv", index_col = "Order")
 dataset_teste = pd.read_csv(".. /input/dataset_teste.csv", index_col = "OrderId" )
confirmed = []
fatalities = []
for id, d, loc in tqdm(test[['ForecastId', 'Date', 'Location']].values):
c = to_exp(daily_log_confirmed.loc[daily_log_confirmed.Location == loc, d].values[0])
 f = to_exp(daily_log_deaths.loc[daily_log_deaths.Location == loc, d].values[0])
 confirmed.append(c)
 fatalities.append(f )<prepare_output>
dataset_treino = dataset_treino.drop(["Property Id","Property Name","Parent Property Id",
"Parent Property Name", "NYC Borough, Block and Lot(BBL)self-reported",
"NYC Building Identification Number(BIN)","Postal Code",
"Street Number","Street Name",
"Borough", "Address 1(self-reported)", 'Address 2'],
axis = 1 )
my_submission = test.copy()
my_submission['ConfirmedCases'] = confirmed
my_submission['Fatalities'] = fatalities
my_submission.shape
my_submission.head() 
<save_to_csv>
dataset_treino = dataset_treino.drop(["2nd Largest Property Use Type",
"2nd Largest Property Use - Gross Floor Area(ft²)","3rd Largest Property Use Type",
"3rd Largest Property Use Type - Gross Floor Area(ft²)", "Fuel Oil
"Fuel Oil
"Diesel
my_submission[[
'ForecastId', 'ConfirmedCases', 'Fatalities'
]].to_csv('submission.csv', index=False)
 print(DECAY)
 my_submission.head()
my_submission.tail()
my_submission.shape<train_model>
dataset_treino = dataset_treino.drop(["Largest Property Use Type","List of All Property Use Types at Property"], axis = 1 )
end = dt.datetime.now()
print('Finished', end,(end - start ).seconds, 's' )<set_options>
dataset_treino['BBL - 10 digits'] = dataset_treino['BBL - 10 digits'].astype('category')
 dataset_treino['BBL - 10 digits'] = dataset_treino['BBL - 10 digits'].apply(lambda x: x[0] ).astype('category')
 dataset_teste['BBL - 10 digits'] = dataset_teste['BBL - 10 digits'].astype('category')
 dataset_teste['BBL - 10 digits'] = dataset_teste['BBL - 10 digits'].apply(lambda x: x[0] ).astype('category' )
plotly.offline.init_notebook_mode()
%matplotlib inline
def RMSLE(pred,actual):
return np.sqrt(np.mean(np.power(( np.log(pred+1)-np.log(actual+1)) ,2)) )<set_options>
dataset_treino = dataset_treino.where(( pd.notnull(dataset_treino)) , 0.001)
 d = {'Not Available': 0.001}
df = dataset_treino.replace(d)
 dataset_teste = dataset_teste.where(( pd.notnull(dataset_teste)) , 0.001)
 df_teste = dataset_teste.replace(d )
warnings.filterwarnings("ignore")
 %matplotlib inline
%config InlineBackend.figure_format = 'retina' 
<load_from_csv>
colunas = ["Year Built", "Site EUI(kBtu/ft²)","Weather Normalized Site EUI(kBtu/ft²)",
"Weather Normalized Site Electricity Intensity(kWh/ft²)","Weather Normalized Site Natural Gas Intensity(therms/ft²)",
"Natural Gas Use(kBtu)","Weather Normalized Site Electricity(kWh)","Total GHG Emissions(Metric Tons CO2e)",
"Direct GHG Emissions(Metric Tons CO2e)","Indirect GHG Emissions(Metric Tons CO2e)",
"Water Use(All Water Sources )(kgal)","Water Intensity(All Water Sources )(gal/ft²)","Source EUI(kBtu/ft²)",
"Weather Normalized Site Natural Gas Use(therms)", "Electricity Use - Grid Purchase(kBtu)",
"Natural Gas Use(kBtu)", "Weather Normalized Site Natural Gas Use(therms)","Weather Normalized Source EUI(kBtu/ft²)",
"DOF Gross Floor Area","Number of Buildings - Self-reported", "Census Tract",
"Largest Property Use Type - Gross Floor Area(ft²)","Property GFA - Self-Reported(ft²)","Occupancy",
"Council District","Community Board"]
array = df[colunas].values
array_teste = df_teste[colunas].values
train = pd.read_csv("/kaggle/input/covid19-global-forecasting-week-3/train.csv")
 test = pd.read_csv("/kaggle/input/covid19-global-forecasting-week-3/test.csv")
 region_metadata = pd.read_csv("/kaggle/input/covid19-forecasting-metadata/region_metadata.csv")
 region_date_metadata = pd.read_csv("/kaggle/input/covid19-forecasting-metadata/region_date_metadata.csv" )<merge>
array = array.astype(float)
 array_teste = array_teste.astype(float )
train = train.merge(test[["ForecastId", "Province_State", "Country_Region", "Date"]], on = ["Province_State", "Country_Region", "Date"], how = "left")
 display(train.head())
 test = test[~test.Date.isin(train.Date.unique())]
display(test.head())
 df = pd.concat([train, test], sort = False)
 df.head()<categorify>
imp = SimpleImputer(missing_values = 0.001, strategy = 'median' ).fit(array)
 dataset_treino[colunas] = imp.transform(array)
 dataset_teste[colunas] = imp.transform(array_teste )
df["geo"] = df.Country_Region.astype(str)+ ": " + df.Province_State.astype(str)
 df.loc[df.Province_State.isna() , "geo"] = df[df.Province_State.isna() ].Country_Region
df.ConfirmedCases = df.groupby("geo")["ConfirmedCases"].cummax()
df.Fatalities = df.groupby("geo")["Fatalities"].cummax()
df = df.merge(region_metadata, on = ["Country_Region", "Province_State"])
 df = df.merge(region_date_metadata, on = ["Country_Region", "Province_State", "Date"], how = "left")
 df.continent = LabelEncoder().fit_transform(df.continent)
 df.Date = pd.to_datetime(df.Date, format = "%Y-%m-%d")
 df.sort_values(["geo", "Date"], inplace = True)
 df.head()<categorify>
dataset_treino = dataset_treino.drop(["DOF Gross Floor Area", "Largest Property Use Type - Gross Floor Area(ft²)",
"Weather Normalized Site EUI(kBtu/ft²)","Number of Buildings - Self-reported",
"Occupancy","Electricity Use - Grid Purchase(kBtu)",
"Weather Normalized Site Electricity(kWh)","Indirect GHG Emissions(Metric Tons CO2e)",
'Property GFA - Self-Reported(ft²)',"Water Use(All Water Sources )(kgal)",
"Water Intensity(All Water Sources )(gal/ft²)","Latitude","Longitude",
"Community Board", "Council District","Census Tract"],
axis = 1 )
DAYS_SINCE_CASES = [1, 10, 50, 100, 500, 1000, 5000, 10000]
min_date_train = np.min(df[~df.Id.isna() ].Date)
 max_date_train = np.max(df[~df.Id.isna() ].Date)
 min_date_test = np.min(df[~df.ForecastId.isna() ].Date)
 max_date_test = np.max(df[~df.ForecastId.isna() ].Date)
 n_dates_test = len(df[~df.ForecastId.isna() ].Date.unique())
 print("Train date range:", str(min_date_train), " - ", str(max_date_train))
print("Test date range:", str(min_date_test), " - ", str(max_date_test))
for lag in range(1, 41):
df[f"lag_{lag}_cc"] = df.groupby("geo")["ConfirmedCases"].shift(lag)
 df[f"lag_{lag}_ft"] = df.groupby("geo")["Fatalities"].shift(lag)
 df[f"lag_{lag}_rc"] = df.groupby("geo")["Recoveries"].shift(lag)
 for case in DAYS_SINCE_CASES:
df = df.merge(df[df.ConfirmedCases >= case].groupby("geo")["Date"].min().reset_index().rename(columns = {"Date": f"case_{case}_date"}), on = "geo", how = "left" )<feature_engineering>
colunas_num = ["ENERGY STAR Score", "Site EUI(kBtu/ft²)", "Year Built",
"Weather Normalized Site Electricity Intensity(kWh/ft²)", "Weather Normalized Site Natural Gas Intensity(therms/ft²)",
"Natural Gas Use(kBtu)", "Weather Normalized Site Natural Gas Use(therms)", "Total GHG Emissions(Metric Tons CO2e)",
"Direct GHG Emissions(Metric Tons CO2e)", "Source EUI(kBtu/ft²)", "Weather Normalized Source EUI(kBtu/ft²)"]
vif = pd.DataFrame()
vif["VIF Factor"] = [variance_inflation_factor(dataset_treino[colunas_num].values, i)for i in range(dataset_treino[colunas_num].shape[1])]
vif["features"] = dataset_treino[colunas_num].columns
vif
def prepare_features(df, gap):
df["perc_1_ac"] =(df[f"lag_{gap}_cc"] - df[f"lag_{gap}_ft"] - df[f"lag_{gap}_rc"])/ df[f"lag_{gap}_cc"]
df["perc_1_cc"] = df[f"lag_{gap}_cc"] / df.population
df["diff_1_cc"] = df[f"lag_{gap}_cc"] - df[f"lag_{gap + 1}_cc"]
df["diff_2_cc"] = df[f"lag_{gap + 1}_cc"] - df[f"lag_{gap + 2}_cc"]
df["diff_3_cc"] = df[f"lag_{gap + 2}_cc"] - df[f"lag_{gap + 3}_cc"]
df["diff_1_ft"] = df[f"lag_{gap}_ft"] - df[f"lag_{gap + 1}_ft"]
df["diff_2_ft"] = df[f"lag_{gap + 1}_ft"] - df[f"lag_{gap + 2}_ft"]
df["diff_3_ft"] = df[f"lag_{gap + 2}_ft"] - df[f"lag_{gap + 3}_ft"]
df["diff_123_cc"] =(df[f"lag_{gap}_cc"] - df[f"lag_{gap + 3}_cc"])/ 3
df["diff_123_ft"] =(df[f"lag_{gap}_ft"] - df[f"lag_{gap + 3}_ft"])/ 3
df["diff_change_1_cc"] = df.diff_1_cc / df.diff_2_cc
df["diff_change_2_cc"] = df.diff_2_cc / df.diff_3_cc
df["diff_change_1_ft"] = df.diff_1_ft / df.diff_2_ft
df["diff_change_2_ft"] = df.diff_2_ft / df.diff_3_ft
df["diff_change_12_cc"] =(df.diff_change_1_cc + df.diff_change_2_cc)/ 2
df["diff_change_12_ft"] =(df.diff_change_1_ft + df.diff_change_2_ft)/ 2
df["change_1_cc"] = df[f"lag_{gap}_cc"] / df[f"lag_{gap + 1}_cc"]
df["change_2_cc"] = df[f"lag_{gap + 1}_cc"] / df[f"lag_{gap + 2}_cc"]
df["change_3_cc"] = df[f"lag_{gap + 2}_cc"] / df[f"lag_{gap + 3}_cc"]
df["change_1_ft"] = df[f"lag_{gap}_ft"] / df[f"lag_{gap + 1}_ft"]
df["change_2_ft"] = df[f"lag_{gap + 1}_ft"] / df[f"lag_{gap + 2}_ft"]
df["change_3_ft"] = df[f"lag_{gap + 2}_ft"] / df[f"lag_{gap + 3}_ft"]
df["change_123_cc"] = df[f"lag_{gap}_cc"] / df[f"lag_{gap + 3}_cc"]
df["change_123_ft"] = df[f"lag_{gap}_ft"] / df[f"lag_{gap + 3}_ft"]
for case in DAYS_SINCE_CASES:
df[f"days_since_{case}_case"] =(df[f"case_{case}_date"] - df.Date ).astype("timedelta64[D]")
 df.loc[df[f"days_since_{case}_case"] < gap, f"days_since_{case}_case"] = np.nan
df["country_flag"] = df.Province_State.isna().astype(int)
 df["density"] = df.population / df.area
df["target_cc"] = np.log1p(df.ConfirmedCases)- np.log1p(df[f"lag_{gap}_cc"])
 df["target_ft"] = np.log1p(df.Fatalities)- np.log1p(df[f"lag_{gap}_ft"])
 features = [
f"lag_{gap}_cc",
f"lag_{gap}_ft",
f"lag_{gap}_rc",
"perc_1_ac",
"perc_1_cc",
"diff_1_cc",
"diff_2_cc",
"diff_3_cc",
"diff_1_ft",
"diff_2_ft",
"diff_3_ft",
"diff_123_cc",
"diff_123_ft",
"diff_change_1_cc",
"diff_change_2_cc",
"diff_change_1_ft",
"diff_change_2_ft",
"diff_change_12_cc",
"diff_change_12_ft",
"change_1_cc",
"change_2_cc",
"change_3_cc",
"change_1_ft",
"change_2_ft",
"change_3_ft",
"change_123_cc",
"change_123_ft",
"days_since_1_case",
"days_since_10_case",
"days_since_50_case",
"days_since_100_case",
"days_since_500_case",
"days_since_1000_case",
"days_since_5000_case",
"days_since_10000_case",
"country_flag",
"lat",
"lon",
"continent",
"population",
"area",
"density",
"target_cc",
"target_ft"
]
return df[features]<predict_on_test>
dataset_treino = dataset_treino.drop(["Source EUI(kBtu/ft²)", "Weather Normalized Source EUI(kBtu/ft²)","Natural Gas Use(kBtu)",
"Year Built","Direct GHG Emissions(Metric Tons CO2e)"],
axis = 1 )
def build_predict_lgbm(df_train, df_test, gap):
df_train.dropna(subset = ["target_cc", "target_ft", f"lag_{gap}_cc", f"lag_{gap}_ft"], inplace = True)
 target_cc = df_train.target_cc
target_ft = df_train.target_ft
test_lag_cc = df_test[f"lag_{gap}_cc"].values
test_lag_ft = df_test[f"lag_{gap}_ft"].values
df_train.drop(["target_cc", "target_ft"], axis = 1, inplace = True)
 df_test.drop(["target_cc", "target_ft"], axis = 1, inplace = True)
 categorical_features = ["continent"]
dtrain_cc = lgb.Dataset(df_train, label = target_cc, categorical_feature = categorical_features)
 dtrain_ft = lgb.Dataset(df_train, label = target_ft, categorical_feature = categorical_features)
 model_cc = lgb.train(LGB_PARAMS, train_set = dtrain_cc, num_boost_round = 200)
 model_ft = lgb.train(LGB_PARAMS, train_set = dtrain_ft, num_boost_round = 200)
 y_pred_cc = np.expm1(model_cc.predict(df_test, num_boost_round = 200)+ np.log1p(test_lag_cc))
y_pred_ft = np.expm1(model_ft.predict(df_test, num_boost_round = 200)+ np.log1p(test_lag_ft))
return y_pred_cc, y_pred_ft, model_cc, model_ft<predict_on_test>
colunas_num = ["ENERGY STAR Score", "Site EUI(kBtu/ft²)",
"Weather Normalized Site Electricity Intensity(kWh/ft²)", "Weather Normalized Site Natural Gas Intensity(therms/ft²)",
"Weather Normalized Site Natural Gas Use(therms)", "Total GHG Emissions(Metric Tons CO2e)"]
vif = pd.DataFrame()
vif["VIF Factor"] = [variance_inflation_factor(dataset_treino[colunas_num].values, i)for i in range(dataset_treino[colunas_num].shape[1])]
vif["features"] = dataset_treino[colunas_num].columns
vif
def predict_mad(df_test, gap, val = False):
df_test["avg_diff_cc"] =(df_test[f"lag_{gap}_cc"] - df_test[f"lag_{gap + 3}_cc"])/ 3
df_test["avg_diff_ft"] =(df_test[f"lag_{gap}_ft"] - df_test[f"lag_{gap + 3}_ft"])/ 3
if val:
y_pred_cc = df_test[f"lag_{gap}_cc"] + gap * df_test.avg_diff_cc -(1 - MAD_FACTOR)* df_test.avg_diff_cc * np.sum([x for x in range(gap)])/ VAL_DAYS
y_pred_ft = df_test[f"lag_{gap}_ft"] + gap * df_test.avg_diff_ft -(1 - MAD_FACTOR)* df_test.avg_diff_ft * np.sum([x for x in range(gap)])/ VAL_DAYS
else:
y_pred_cc = df_test[f"lag_{gap}_cc"] + gap * df_test.avg_diff_cc -(1 - MAD_FACTOR)* df_test.avg_diff_cc * np.sum([x for x in range(gap)])/ n_dates_test
y_pred_ft = df_test[f"lag_{gap}_ft"] + gap * df_test.avg_diff_ft -(1 - MAD_FACTOR)* df_test.avg_diff_ft * np.sum([x for x in range(gap)])/ n_dates_test
return y_pred_cc, y_pred_ft<init_hyperparams>
coluna_out = "Weather Normalized Site Natural Gas Use(therms)"
mean = np.mean(dataset_treino[coluna_out])
 sd = np.std(dataset_treino[coluna_out])
 out = mean + 2*sd
dataset_treino = dataset_treino.drop(dataset_treino[dataset_treino[coluna_out] > out].index)
 coluna_out = "Weather Normalized Site Natural Gas Intensity(therms/ft²)"
mean = np.mean(dataset_treino[coluna_out])
 sd = np.std(dataset_treino[coluna_out])
 out = mean + 2*sd
dataset_treino = dataset_treino.drop(dataset_treino[dataset_treino[coluna_out] > out].index)
 coluna_out = "Weather Normalized Site Electricity Intensity(kWh/ft²)"
mean = np.mean(dataset_treino[coluna_out])
 sd = np.std(dataset_treino[coluna_out])
 out = mean + 2*sd
dataset_treino = dataset_treino.drop(dataset_treino[dataset_treino[coluna_out] > out].index)
 coluna_out = "Site EUI(kBtu/ft²)"
mean = np.mean(dataset_treino[coluna_out])
 sd = np.std(dataset_treino[coluna_out])
 out = mean + 2*sd
dataset_treino = dataset_treino.drop(dataset_treino[dataset_treino[coluna_out] > out].index)
 coluna_out = "Total GHG Emissions(Metric Tons CO2e)"
mean = np.mean(dataset_treino[coluna_out])
 sd = np.std(dataset_treino[coluna_out])
 out = mean + 2*sd
dataset_treino = dataset_treino.drop(dataset_treino[dataset_treino[coluna_out] > out].index )
SEED = 23
LGB_PARAMS = {"objective": "regression",
"num_leaves": 5,
"learning_rate": 0.013,
"bagging_fraction": 0.91,
"feature_fraction": 0.81,
"reg_alpha": 0.13,
"reg_lambda": 0.13,
"metric": "rmse",
"seed": SEED
}
VAL_DAYS = 7
MAD_FACTOR = 0.5<split>
coluna_cat = ["Primary Property Type - Self Selected", "Metered Areas(Energy)", 'BBL - 10 digits','Water Required?']
dummies = pd.get_dummies(dataset_treino[coluna_cat])
 dummies_teste = pd.get_dummies(dataset_teste[coluna_cat])
 final_train, final_test = dummies.align(dummies_teste,join='left',axis=1 )
df_train = df[~df.Id.isna() ]
df_test_full = df[~df.ForecastId.isna() ]<feature_engineering>
coluna_num = ["Site EUI(kBtu/ft²)",
"Weather Normalized Site Electricity Intensity(kWh/ft²)", "Weather Normalized Site Natural Gas Intensity(therms/ft²)",
"Weather Normalized Site Natural Gas Use(therms)", "Total GHG Emissions(Metric Tons CO2e)"]
target = "ENERGY STAR Score"
df_preds_val = []
df_preds_test = []
for date in df_test_full.Date.unique() :
print("[INFO] Date:", date)
 if date in df_train.Date.values:
df_pred_test = df_test_full.loc[df_test_full.Date == date, ["ForecastId", "ConfirmedCases", "Fatalities"]].rename(columns = {"ConfirmedCases": "ConfirmedCases_test", "Fatalities": "Fatalities_test"})
 else:
df_test = df_test_full[df_test_full.Date == date]
gap =(pd.Timestamp(date)- max_date_train ).days
if gap <= VAL_DAYS:
val_date = max_date_train - pd.Timedelta(VAL_DAYS, "D")+ pd.Timedelta(gap, "D")
 df_build = df_train[df_train.Date < val_date]
df_val = df_train[df_train.Date == val_date]
X_build = prepare_features(df_build, gap)
 X_val = prepare_features(df_val, gap)
 y_val_cc_lgb, y_val_ft_lgb, _, _ = build_predict_lgbm(X_build, X_val, gap)
 y_val_cc_mad, y_val_ft_mad = predict_mad(df_val, gap, val = True)
 df_pred_val = pd.DataFrame({"Id": df_val.Id.values,
"ConfirmedCases_val_lgb": y_val_cc_lgb,
"Fatalities_val_lgb": y_val_ft_lgb,
"ConfirmedCases_val_mad": y_val_cc_mad,
"Fatalities_val_mad": y_val_ft_mad,
})
 df_preds_val.append(df_pred_val)
 X_train = prepare_features(df_train, gap)
 X_test = prepare_features(df_test, gap)
 y_test_cc_lgb, y_test_ft_lgb, model_cc, model_ft = build_predict_lgbm(X_train, X_test, gap)
 y_test_cc_mad, y_test_ft_mad = predict_mad(df_test, gap)
 if gap == 1:
model_1_cc = model_cc
model_1_ft = model_ft
features_1 = X_train.columns.values
elif gap == 14:
model_14_cc = model_cc
model_14_ft = model_ft
features_14 = X_train.columns.values
elif gap == 28:
model_28_cc = model_cc
model_28_ft = model_ft
features_28 = X_train.columns.values
df_pred_test = pd.DataFrame({"ForecastId": df_test.ForecastId.values,
"ConfirmedCases_test_lgb": y_test_cc_lgb,
"Fatalities_test_lgb": y_test_ft_lgb,
"ConfirmedCases_test_mad": y_test_cc_mad,
"Fatalities_test_mad": y_test_ft_mad,
})
 df_preds_test.append(df_pred_test )<merge>
X = dataset_treino[coluna_num].values
Y_target = dataset_treino[target].values
X_teste = dataset_teste[coluna_num].values
scaler = MinMaxScaler(feature_range =(-1, 1)).fit(X)
 rescaledX = scaler.transform(X)
 rescaledX_teste = scaler.transform(X_teste )
df = df.merge(pd.concat(df_preds_val, sort = False), on = "Id", how = "left")
 df = df.merge(pd.concat(df_preds_test, sort = False), on = "ForecastId", how = "left")
 rmsle_cc_lgb = np.sqrt(mean_squared_error(np.log1p(df[~df.ConfirmedCases_val_lgb.isna() ].ConfirmedCases), np.log1p(df[~df.ConfirmedCases_val_lgb.isna() ].ConfirmedCases_val_lgb)))
 rmsle_ft_lgb = np.sqrt(mean_squared_error(np.log1p(df[~df.Fatalities_val_lgb.isna() ].Fatalities), np.log1p(df[~df.Fatalities_val_lgb.isna() ].Fatalities_val_lgb)))
 rmsle_cc_mad = np.sqrt(mean_squared_error(np.log1p(df[~df.ConfirmedCases_val_mad.isna() ].ConfirmedCases), np.log1p(df[~df.ConfirmedCases_val_mad.isna() ].ConfirmedCases_val_mad)))
 rmsle_ft_mad = np.sqrt(mean_squared_error(np.log1p(df[~df.Fatalities_val_mad.isna() ].Fatalities), np.log1p(df[~df.Fatalities_val_mad.isna() ].Fatalities_val_mad)))
 print("LGB CC RMSLE Val of", VAL_DAYS, "days for CC:", round(rmsle_cc_lgb, 2))
print("LGB FT RMSLE Val of", VAL_DAYS, "days for FT:", round(rmsle_ft_lgb, 2))
print("LGB Overall RMSLE Val of", VAL_DAYS, "days:", round(( rmsle_cc_lgb + rmsle_ft_lgb)/ 2, 2))
print("MAD CC RMSLE Val of", VAL_DAYS, "days for CC:", round(rmsle_cc_mad, 2))
print("MAD FT RMSLE Val of", VAL_DAYS, "days for FT:", round(rmsle_ft_mad, 2))
print("MAD Overall RMSLE Val of", VAL_DAYS, "days:", round(( rmsle_cc_mad + rmsle_ft_mad)/ 2, 2))<feature_engineering>
data_modelo = dataset_treino[coluna_num].copy()
data_modelo[coluna_num] = rescaledX
data_modelo = data_modelo.join(final_train)
 data_modelo_teste = dataset_teste[coluna_num].copy()
data_modelo_teste[coluna_num] = rescaledX_teste
data_modelo_teste = data_modelo_teste.join(final_test )
test = df.loc[~df.ForecastId.isna() , ["ForecastId", "Country_Region", "Province_State", "Date",
"ConfirmedCases_test", "ConfirmedCases_test_lgb", "ConfirmedCases_test_mad",
"Fatalities_test", "Fatalities_test_lgb", "Fatalities_test_mad"]].reset_index()
test["ConfirmedCases"] = 0.3 * test.ConfirmedCases_test_lgb + 0.7 * test.ConfirmedCases_test_mad
test["Fatalities"] = 0.25 * test.Fatalities_test_lgb + 0.75 * test.Fatalities_test_mad
test.loc[test.Country_Region.isin(["China", "US", "Diamond Princess"]), "ConfirmedCases"] = test[test.Country_Region.isin(["China", "US", "Diamond Princess"])].ConfirmedCases_test_mad.values
test.loc[test.Country_Region.isin(["China", "US", "Diamond Princess"]), "Fatalities"] = test[test.Country_Region.isin(["China", "US", "Diamond Princess"])].Fatalities_test_mad.values
test.loc[test.Date.isin(df_train.Date.values), "ConfirmedCases"] = test[test.Date.isin(df_train.Date.values)].ConfirmedCases_test.values
test.loc[test.Date.isin(df_train.Date.values), "Fatalities"] = test[test.Date.isin(df_train.Date.values)].Fatalities_test.values
sub0 = test[["ForecastId", "ConfirmedCases", "Fatalities"]]
sub0.ForecastId = sub0.ForecastId.astype(int)
 sub0.head()<load_from_csv>
data_modelo_teste[np.isnan(data_modelo_teste)] = 0
test = pd.read_csv(".. /input/covid19-global-forecasting-week-3/test.csv")
 train = pd.read_csv(".. /input/covid19-global-forecasting-week-3/train.csv")
 train['Province_State'].fillna('', inplace=True)
 test['Province_State'].fillna('', inplace=True)
 train['Date'] = pd.to_datetime(train['Date'])
 test['Date'] = pd.to_datetime(test['Date'])
 train = train.sort_values(['Country_Region','Province_State','Date'])
 test = test.sort_values(['Country_Region','Province_State','Date'] )<feature_engineering>
X_treino = data_modelo.values
Y_treino = Y_target
FirstDate = train.groupby('Country_Region' ).min() ['Date'].unique() [0]
train['Last Confirm'] = train['ConfirmedCases'].shift(1)
 while train[(train['Last Confirm'] > train['ConfirmedCases'])&(train['Date'] > FirstDate)].shape[0] > 0:
train['Last Confirm'] = train['ConfirmedCases'].shift(1)
 train['Last Fatalities'] = train['Fatalities'].shift(1)
 train.loc[(train['Last Confirm'] > train['ConfirmedCases'])&(train['Date'] > FirstDate),'ConfirmedCases'] = train.loc[(train['Last Confirm'] > train['ConfirmedCases'])&(train['Date'] > FirstDate),'Last Confirm']
train.loc[(train['Last Fatalities'] > train['Fatalities'])&(train['Date'] > FirstDate),'Fatalities'] = train.loc[(train['Last Fatalities'] > train['Fatalities'])&(train['Date'] > FirstDate),'Last Fatalities']
train['Last Confirm'] = train['ConfirmedCases'].shift(1)
 train['Last Fatalities'] = train['Fatalities'].shift(1 )<import_modules>
modelos = []
modelos.append(( 'GBM', GradientBoostingRegressor(n_estimators = 270, loss = 'huber', max_depth = 4,
learning_rate = 0.05, subsample = 0.7)))
 modelos.append(( 'XG', XGBRegressor(n_estimators = 300, max_depth = 4, learning_rate = 0.05,
gamma = 0.3, subsample= 0.75, min_child_weight = 0.5)))
 resultados = []
nomes = []
for nome, modelo in modelos:
kfold = model_selection.KFold(50, True, random_state = 7)
 cross_val_result = model_selection.cross_val_score(modelo,
X_treino,
Y_treino,
cv = kfold, scoring = 'neg_mean_absolute_error')
 resultados.append(cross_val_result)
 nomes.append(nome)
 texto = "%s: %f(%f)" %(nome, cross_val_result.mean() , cross_val_result.std())
 print(texto )
from statsmodels.tsa.statespace.sarimax import SARIMAX
from statsmodels.tsa.arima_model import ARIMA<feature_engineering>
model_gbm = GradientBoostingRegressor(n_estimators = 270, loss = 'huber', max_depth = 4,
learning_rate = 0.05, subsample = 0.7)
 model_gbm.fit(X_treino,Y_treino)
 previsoes_final_gbm = model_gbm.predict(data_modelo_teste.values )
feature_day = [1,20,50,100,200,500,1000]
def CreateInput(data):
feature = []
for day in feature_day:
data.loc[:,'Number day from ' + str(day)+ ' case'] = 0
if(train[(train['Country_Region'] == country)&(train['Province_State'] == province)&(train['ConfirmedCases'] < day)]['Date'].count() > 0):
fromday = train[(train['Country_Region'] == country)&(train['Province_State'] == province)&(train['ConfirmedCases'] < day)]['Date'].max()
else:
fromday = train[(train['Country_Region'] == country)&(train['Province_State'] == province)]['Date'].min()
for i in range(0, len(data)) :
if(data['Date'].iloc[i] > fromday):
day_denta = data['Date'].iloc[i] - fromday
data['Number day from ' + str(day)+ ' case'].iloc[i] = day_denta.days
feature = feature + ['Number day from ' + str(day)+ ' case']
return data[feature]
pred_data_all = pd.DataFrame()
with tqdm(total=len(train['Country_Region'].unique())) as pbar:
for country in train['Country_Region'].unique() :
for province in train[(train['Country_Region'] == country)]['Province_State'].unique() :
with warnings.catch_warnings() :
warnings.filterwarnings("ignore")
 df_train = train[(train['Country_Region'] == country)&(train['Province_State'] == province)]
df_test = test[(test['Country_Region'] == country)&(test['Province_State'] == province)]
X_train = CreateInput(df_train)
 y_train_confirmed = df_train['ConfirmedCases'].ravel()
y_train_fatalities = df_train['Fatalities'].ravel()
X_pred = CreateInput(df_test)
 feature_use = X_pred.columns[0]
for i in range(X_pred.shape[1] - 1,0,-1):
if(X_pred.iloc[0,i] > 0):
feature_use = X_pred.columns[i]
break
idx = X_train[X_train[feature_use] == 0].shape[0]
adjusted_X_train = X_train[idx:][feature_use].values.reshape(-1, 1)
 adjusted_y_train_confirmed = y_train_confirmed[idx:]
adjusted_y_train_fatalities = y_train_fatalities[idx:]
idx = X_pred[X_pred[feature_use] == 0].shape[0]
adjusted_X_pred = X_pred[idx:][feature_use].values.reshape(-1, 1)
 pred_data = test[(test['Country_Region'] == country)&(test['Province_State'] == province)]
max_train_date = train[(train['Country_Region'] == country)&(train['Province_State'] == province)]['Date'].max()
min_test_date = pred_data['Date'].min()
model = SARIMAX(adjusted_y_train_confirmed, order=(1,1,0),
measurement_error=True ).fit(disp=False)
 y_hat_confirmed = model.forecast(pred_data[pred_data['Date'] > max_train_date].shape[0])
 y_train_confirmed = train[(train['Country_Region'] == country)&(train['Province_State'] == province)&(train['Date'] >= min_test_date)]['ConfirmedCases'].values
y_hat_confirmed = np.concatenate(( y_train_confirmed,y_hat_confirmed), axis = 0)
 model = SARIMAX(adjusted_y_train_fatalities, order=(1,1,0),
measurement_error=True ).fit(disp=False)
 y_hat_fatalities = model.forecast(pred_data[pred_data['Date'] > max_train_date].shape[0])
 y_train_fatalities = train[(train['Country_Region'] == country)&(train['Province_State'] == province)&(train['Date'] >= min_test_date)]['Fatalities'].values
y_hat_fatalities = np.concatenate(( y_train_fatalities,y_hat_fatalities), axis = 0)
 pred_data['ConfirmedCases_hat'] = y_hat_confirmed
pred_data['Fatalities_hat'] = y_hat_fatalities
pred_data_all = pred_data_all.append(pred_data)
 pbar.update(1)
 df_val = pd.merge(pred_data_all,train[['Date','Country_Region','Province_State','ConfirmedCases','Fatalities']],on=['Date','Country_Region','Province_State'], how='left')
 df_val.loc[df_val['Fatalities_hat'] < 0,'Fatalities_hat'] = 0
df_val.loc[df_val['ConfirmedCases_hat'] < 0,'ConfirmedCases_hat'] = 0
df_val_3 = df_val.copy()<rename_columns>
for i in range(0,len(previsoes_final_gbm)) :
if previsoes_final_gbm[i] > 100:
previsoes_final_gbm[i] = 100
sub1 = df_val_3
submission = sub1[['ForecastId','ConfirmedCases_hat','Fatalities_hat']]
submission.columns = ['ForecastId','ConfirmedCases','Fatalities']<save_to_csv>
for i in range(0,len(previsoes_final_gbm)) :
if previsoes_final_gbm[i] < 1:
previsoes_final_gbm[i] = 1
TARGETS = ["ConfirmedCases", "Fatalities"]
sub_df = sub0.copy()
for t in TARGETS:
sub_df[t] = np.expm1(np.log1p(submission[t].values)*0.4 + np.log1p(sub0[t].values)*0.6)
 sub_df.to_csv("submission.csv", index=False )<count_missing_values>
previsoes_final = previsoes_final_gbm.round(0 ).astype('int' )
sub0.isna().sum()<load_from_csv>
dataset_test_submission = pd.read_csv(".. /input/dataset_teste.csv", index_col = 'Property Id')
 dataset_test_submission['score'] = previsoes_final
Submission = dataset_test_submission['score']
Submission = pd.DataFrame(Submission);Submission.head(10 )
<prepare_x_and_y><EOS> 
Submission.to_csv('Submission.csv', header = True )