Kaggle competition: M5 Forecasting - Accuracy
<SOS> metric: M5_WRMSSE Kaggle data source: m5-forecasting-accuracy<define_variables>
LoanType = CategoricalDtype(["Cash loans", "Revolving loans", "Consumer loans"], False)
 HouseType = CategoricalDtype(["block of flats", "terraced house", "specific housing"])
 WeekDayType = CategoricalDtype(['SUNDAY', 'MONDAY', 'TUESDAY', 'WEDNESDAY', 'THURSDAY', 'FRIDAY', 'SATURDAY'], True)
 YesNoType = CategoricalDtype(["N", "Y"], True)
 LoanStatusType = CategoricalDtype(['Approved', 'Refused', 'Canceled', 'Unused offer'], False)
 EducationType = CategoricalDtype(["Incomplete higher", "Higher education", "Lower secondary",
"Secondary / secondary special", "Academic degree"], True)
 FamilyType = CategoricalDtype(["Civil marriage", "Married", "Separated", "Single / not married", "Widow"], False)
 HousingType = CategoricalDtype(["House / apartment", "Rented apartment", "With parents",
"Municipal apartment", "Office apartment", "Co-op apartment"], False)
 IncomeType = CategoricalDtype(["Unemployed", "Student", "State servant", "Working", "Commercial associate",
"Businessman", "Maternity leave", "Pensioner"], False)
 AccompanyType = CategoricalDtype(["Unaccompanied", "Spouse", "partner", "Family", "Children",
"Group of people", "Other_A", "Other_B"], False)
 OccupationType = CategoricalDtype(["Accountants", "Cleaning staff", "Cooking staff", "Core staff",
"Drivers", "HR staff", "High skill tech staff", "IT staff", "Laborers",
"Low-skill Laborers", "Managers", "Medicine staff",
"Private service staff", "Realty agents", "Sales staff", "Secretaries",
"Security staff", "Waiters/barmen staff"], False)
 CreditStatusType = CategoricalDtype(["Closed", "Active", "Sold", "Bad debt", "Signed"], True)
 BureauBalanceStatusType = CategoricalDtype(["C", *map(str, range(6)) ], True )<define_variables>
warnings.simplefilter(action='ignore')
def load_app_data(train_only=False):
gender_type = CategoricalDtype(["M", "F"], False)
 yes_no_type2 = CategoricalDtype(["No", "Yes"], True)
 wall_material_type = CategoricalDtype(["Stone, brick", "Wooden", "Block", "Panel",
"Monolithic", "Mixed", "Others"], False)
 fondkapremont_type = CategoricalDtype(["reg oper account", "org spec account",
"reg oper spec account", "not specified"], False)
 col_types = {
"SK_ID_CURR": np.uint32, "TARGET": np.bool, "CODE_GENDER": gender_type, "NAME_CONTRACT_TYPE": LoanType,
"FLAG_OWN_CAR": YesNoType, "FLAG_OWN_REALTY": YesNoType, "CNT_CHILDREN": np.uint8,
"AMT_INCOME_TOTAL": np.float32, "AMT_CREDIT": np.float32,
"AMT_ANNUITY": np.float32, "AMT_GOODS_PRICE": np.float32,
"NAME_TYPE_SUITE": AccompanyType, "NAME_EDUCATION_TYPE": EducationType, "NAME_INCOME_TYPE": IncomeType,
"NAME_FAMILY_STATUS": FamilyType, "NAME_HOUSING_TYPE": HousingType,
"REGION_POPULATION_RELATIVE": np.float32,
"REGION_RATING_CLIENT": np.uint8, "REGION_RATING_CLIENT_W_CITY": np.uint8,
"WEEKDAY_APPR_PROCESS_START": WeekDayType, "HOUR_APPR_PROCESS_START": np.uint8,
"DAYS_EMPLOYED": np.float32, "DAYS_BIRTH": np.int32,
"DAYS_REGISTRATION": np.float32, "DAYS_ID_PUBLISH": np.float32,
"OWN_CAR_AGE": np.float16,
"EXT_SOURCE_1": np.float32, "EXT_SOURCE_2": np.float32, "EXT_SOURCE_3": np.float32,
"FLAG_MOBIL": np.bool, "FLAG_EMP_PHONE": np.bool, "FLAG_WORK_PHONE": np.bool,
"FLAG_CONT_MOBILE": np.bool, "FLAG_PHONE": np.bool, "FLAG_EMAIL": np.bool,
"OCCUPATION_TYPE": OccupationType, "CNT_FAM_MEMBERS": np.float16,
"REG_REGION_NOT_LIVE_REGION": np.bool, "REG_REGION_NOT_WORK_REGION": np.bool,
"LIVE_REGION_NOT_WORK_REGION": np.bool, "REG_CITY_NOT_LIVE_CITY": np.bool,
"REG_CITY_NOT_WORK_CITY": np.bool, "LIVE_CITY_NOT_WORK_CITY": np.bool,
"ORGANIZATION_TYPE": "category",
"OBS_30_CNT_SOCIAL_CIRCLE": np.float16, "DEF_30_CNT_SOCIAL_CIRCLE": np.float16,
"OBS_60_CNT_SOCIAL_CIRCLE": np.float16, "DEF_60_CNT_SOCIAL_CIRCLE": np.float16,
"DAYS_LAST_PHONE_CHANGE": np.float32,
"AMT_REQ_CREDIT_BUREAU_HOUR": np.float16, "AMT_REQ_CREDIT_BUREAU_DAY": np.float16,
"AMT_REQ_CREDIT_BUREAU_WEEK": np.float16, "AMT_REQ_CREDIT_BUREAU_MON": np.float16,
"AMT_REQ_CREDIT_BUREAU_QRT": np.float16, "AMT_REQ_CREDIT_BUREAU_YEAR": np.float16,
"HOUSETYPE_MODE": HouseType, "FONDKAPREMONT_MODE": fondkapremont_type,
"WALLSMATERIAL_MODE": wall_material_type, "EMERGENCYSTATE_MODE": yes_no_type2
}
float_housing_columns = [
"APARTMENTS_AVG", "BASEMENTAREA_AVG", "YEARS_BEGINEXPLUATATION_AVG",
"YEARS_BUILD_AVG", "COMMONAREA_AVG", "ELEVATORS_AVG", "ENTRANCES_AVG",
"FLOORSMAX_AVG", "FLOORSMIN_AVG", "LANDAREA_AVG",
"LIVINGAPARTMENTS_AVG", "LIVINGAREA_AVG", "NONLIVINGAPARTMENTS_AVG", "NONLIVINGAREA_AVG",
"APARTMENTS_MODE", "BASEMENTAREA_MODE", "YEARS_BEGINEXPLUATATION_MODE",
"YEARS_BUILD_MODE", "COMMONAREA_MODE", "ELEVATORS_MODE", "ENTRANCES_MODE",
"FLOORSMAX_MODE", "FLOORSMIN_MODE", "LANDAREA_MODE",
"LIVINGAPARTMENTS_MODE", "LIVINGAREA_MODE", "NONLIVINGAPARTMENTS_MODE",
"NONLIVINGAREA_MODE", "TOTALAREA_MODE",
"APARTMENTS_MEDI", "BASEMENTAREA_MEDI", "YEARS_BEGINEXPLUATATION_MEDI",
"YEARS_BUILD_MEDI", "COMMONAREA_MEDI", "ELEVATORS_MEDI", "ENTRANCES_MEDI",
"FLOORSMAX_MEDI", "FLOORSMIN_MEDI", "LANDAREA_MEDI", "LIVINGAPARTMENTS_MEDI",
"LIVINGAREA_MEDI", "NONLIVINGAPARTMENTS_MEDI", "NONLIVINGAREA_MEDI",
]
col_types.update(( "FLAG_DOCUMENT_" + str(v), np.bool)for v in range(2, 22))
col_types.update(( col, np.float16)for col in float_housing_columns)
 replace_dict = {"DAYS_EMPLOYED": {365243: np.nan}, "DAYS_LAST_PHONE_CHANGE": {0: np.nan}}
na_values = ["XNA", "Unknown"]
df_app_train = pd.read_csv(PATH_APP_TRAIN, index_col=0, na_values=na_values, dtype=col_types)
 df_app_train.replace(replace_dict, inplace=True)
 assert df_app_train.shape ==(307511, 121)
 if train_only:
return df_app_train
else:
df_app_test = pd.read_csv(PATH_APP_TEST, index_col=0, na_values=na_values, dtype=col_types)
 df_app_test.replace(replace_dict, inplace=True)
 assert df_app_test.shape ==(48744, 120)
 return df_app_train, df_app_test
def load_prev_app_data() :
interest_types = CategoricalDtype(["low_action", "low_normal", "middle", "high"], True)
 col_types = {
"SK_ID_PREV": np.uint32, "SK_ID_CURR": np.uint32, "NAME_CONTRACT_TYPE": LoanType,
"AMT_ANNUITY": np.float32, "AMT_APPLICATION": np.float32, "AMT_CREDIT": np.float32,
"AMT_DOWN_PAYMENT": np.float32, "AMT_GOODS_PRICE": np.float32,
"WEEKDAY_APPR_PROCESS_START": WeekDayType, "HOUR_APPR_PROCESS_START": np.uint8,
"FLAG_LAST_APPL_PER_CONTRACT": YesNoType, "NFLAG_LAST_APPL_IN_DAY": np.bool,
"RATE_DOWN_PAYMENT": np.float32, "RATE_INTEREST_PRIMARY": np.float32, "RATE_INTEREST_PRIVILEGED": np.float32,
"NAME_CASH_LOAN_PURPOSE": "category", "NAME_CONTRACT_STATUS": LoanStatusType,
"DAYS_DECISION": np.int16, "NAME_PAYMENT_TYPE": "category", "CODE_REJECT_REASON": "category",
"NAME_TYPE_SUITE": "category", "NAME_CLIENT_TYPE": "category", "NAME_GOODS_CATEGORY": "category",
"NAME_PORTFOLIO": "category", "NAME_PRODUCT_TYPE": "category", "CHANNEL_TYPE": "category",
"SELLERPLACE_AREA": np.float32, "NAME_SELLER_INDUSTRY": "category", "CNT_PAYMENT": np.float16,
"NAME_YIELD_GROUP": interest_types, "PRODUCT_COMBINATION": "category",
"DAYS_FIRST_DRAWING": np.float32, "DAYS_FIRST_DUE": np.float32,
"DAYS_LAST_DUE_1ST_VERSION": np.float32, "DAYS_LAST_DUE": np.float32, "DAYS_TERMINATION": np.float32,
"NFLAG_INSURED_ON_APPROVAL": np.float16
}
d = {365243: np.nan}
replace_dict = {"SELLERPLACE_AREA": {-1: np.nan},
"DAYS_FIRST_DRAWING": d, "DAYS_FIRST_DUE": d,
"DAYS_LAST_DUE_1ST_VERSION": d, "DAYS_LAST_DUE": d, "DAYS_TERMINATION": d}
df_prev_app = pd.read_csv(PATH_PRE_APP, na_values=["XNA", ""], index_col=0,
keep_default_na=False, dtype=col_types)
 df_prev_app.replace(replace_dict, inplace=True)
 assert df_prev_app.shape ==(1670214, 36), f"Incorrect shape for df_prev_app: {df_prev_app.shape}"
return df_prev_app
def load_install_payments(load_processed=True):
col_types = {
"SK_ID_PREV": np.uint32, "SK_ID_CURR": np.uint32,
"NUM_INSTALMENT_VERSION": np.uint8, "NUM_INSTALMENT_NUMBER": np.uint16,
"DAYS_INSTALMENT": np.int16, "DAYS_ENTRY_PAYMENT": np.float16,
"AMT_INSTALMENT": np.float32, "AMT_PAYMENT": np.float32,
"NUM_PAYMENTS": np.uint16, "AMT_OVERDUE": np.float32, "AMT_DPD30": np.float32
}
if load_processed:
return pd.read_csv(PATH_INST_PAY_PROCESSED, dtype=col_types)
 else:
df = pd.read_csv(PATH_INST_PAY, dtype=col_types)
 assert df.shape ==(13605401, 8)
 return df
def load_pos_balance() :
col_types = {
"SK_ID_PREV": np.uint32, "SK_ID_CURR": np.uint32, "MONTHS_BALANCE": np.int16,
"CNT_INSTALMENT": np.float16, "CNT_INSTALMENT_FUTURE": np.float16,
"NAME_CONTRACT_STATUS": CreditStatusType,
"SK_DPD": np.uint16, "SK_DPD_DEF": np.uint16
}
df = pd.read_csv(PATH_POS_BALANCE, na_values=["XNA"], dtype=col_types)
 assert df.shape ==(10001358, 8)
 return df
def load_credit_balance_data() :
col_types = {
"SK_ID_PREV": np.uint32, "SK_ID_CURR": np.uint32, "MONTHS_BALANCE": np.int16,
"AMT_BALANCE": np.float32, "AMT_CREDIT_LIMIT_ACTUAL": np.float32,
"AMT_DRAWINGS_ATM_CURRENT": np.float32, "AMT_DRAWINGS_CURRENT": np.float32,
"AMT_DRAWINGS_OTHER_CURRENT": np.float32, "AMT_DRAWINGS_POS_CURRENT": np.float32,
"AMT_INST_MIN_REGULARITY": np.float32, "AMT_PAYMENT_CURRENT": np.float32,
"AMT_PAYMENT_TOTAL_CURRENT": np.float32, "AMT_RECEIVABLE_PRINCIPAL": np.float32,
"AMT_RECIVABLE": np.float32, "AMT_TOTAL_RECEIVABLE": np.float32,
"CNT_DRAWINGS_ATM_CURRENT": np.float16, "CNT_DRAWINGS_CURRENT": np.float16,
"CNT_DRAWINGS_OTHER_CURRENT": np.float16, "CNT_DRAWINGS_POS_CURRENT": np.float16,
"CNT_INSTALMENT_MATURE_CUM": np.float16, "NAME_CONTRACT_STATUS": "category",
"SK_DPD": np.uint16, "SK_DPD_DEF": np.uint16
}
df = pd.read_csv(PATH_CREDIT_BALANCE, dtype=col_types)
 assert df.shape ==(3840312, 23)
 return df
def load_bureau() :
currency_types = CategoricalDtype(["currency " + str(v)for v in range(1, 5)], False)
 col_types = {
"SK_ID_BUREAU": np.uint32, "SK_ID_CURR": np.uint32, "CREDIT_ACTIVE": CreditStatusType,
"CREDIT_CURRENCY": currency_types, "DAYS_CREDIT": np.float16, "CREDIT_DAY_OVERDUE": np.uint16,
"DAYS_ENDDATE_FACT": np.float16, "DAYS_CREDIT_ENDDATE": np.float16,
"AMT_CREDIT_MAX_OVERDUE": np.float32, "CNT_CREDIT_PROLONG": np.uint8,
"AMT_CREDIT_SUM": np.float32, "AMT_CREDIT_SUM_DEBT": np.float32, "AMT_CREDIT_SUM_LIMIT": np.float32,
"AMT_CREDIT_SUM_OVERDUE": np.float32, "AMT_ANNUITY": np.float32,
"CREDIT_TYPE": "category"
}
df = pd.read_csv(PATH_BUREAU, index_col=1, dtype=col_types)
 assert df.shape ==(1716428, 16)
 return df
def load_bureau_balance() :
col_types = {
"SK_ID_BUREAU": np.uint32, "MONTHS_BALANCE": np.int16, "STATUS": BureauBalanceStatusType
}
df = pd.read_csv(PATH_BUREAU_BALANCE, na_values=["X"], dtype=col_types)
 assert df.shape ==(27299925, 3)
 return df<categorify>
nbeats_pred01_df = pd.read_csv('.. /input/previous-results/nbeats_toplvl_forecasts1.csv')
 nbeats_pred02_df = pd.read_csv('.. /input/previous-results/nbeats_toplvl_forecasts2.csv')
def flatten_agg_df_columns(df_agg, prefix=None):
if prefix is None:
df_agg.columns = ['_'.join([c0, c1.upper() ])for c0, c1 in df_agg.columns]
else:
df_agg.columns = ['_'.join([prefix, c0, c1.upper() ])for c0, c1 in df_agg.columns]
return df_agg
def clean_inst_pay(df_inst_pay):
print("Cleaning installment payments data")
 df_inst_pay["DAYS_ENTRY_PAYMENT"].fillna(0, inplace=True)
 df_inst_pay["AMT_PAYMENT"].fillna(-1, inplace=True)
 df_inst_pay_valid_filter =(df_inst_pay["AMT_PAYMENT"] > 0)|(df_inst_pay["AMT_INSTALMENT"] > 0)
 print(f"Remove {(~df_inst_pay_valid_filter ).sum() :d} invalid records.")
 df_inst_pay_group = df_inst_pay[df_inst_pay_valid_filter].groupby(["SK_ID_PREV", "NUM_INSTALMENT_NUMBER",
"DAYS_ENTRY_PAYMENT", "AMT_PAYMENT"])
 del df_inst_pay_valid_filter
print("Aggregate multiple installments for one payment")
 df_inst_pay_group_cnt = df_inst_pay_group.size()
df_inst_agg = flatten_agg_df_columns(df_inst_pay_group.agg({
"SK_ID_CURR": ["min", "max"],
"NUM_INSTALMENT_VERSION": ["max", "nunique"],
"DAYS_INSTALMENT": ["min", "max"],
"AMT_INSTALMENT": ["min", "max", "sum"]
}))
del df_inst_pay_group
print("Processing 1")
 assert(df_inst_agg["SK_ID_CURR_MIN"] == df_inst_agg["SK_ID_CURR_MAX"] ).all(axis=None), "Inconsistent SK_ID_CURR"
df_inst_pay_processed = pd.DataFrame(index=df_inst_agg.index)
 df_inst_pay_processed["SK_ID_CURR"] = df_inst_agg["SK_ID_CURR_MIN"]
df_inst_pay_group_cnt_distict = df_inst_agg["NUM_INSTALMENT_VERSION_NUNIQUE"]
df_inst_pay_group_check =(( df_inst_pay_group_cnt == 2)| 
(df_inst_pay_group_cnt_distict == 1))
assert df_inst_pay_group_check.all(axis=None)
 del df_inst_pay_group_cnt, df_inst_pay_group_check
df_inst_pay_processed["NUM_INSTALMENT_VERSION"] = df_inst_agg["NUM_INSTALMENT_VERSION_MAX"]
assert(df_inst_agg["DAYS_INSTALMENT_MIN"] == df_inst_agg["DAYS_INSTALMENT_MAX"] ).all(axis=None)
 df_inst_pay_processed["DAYS_INSTALMENT"] = df_inst_agg["DAYS_INSTALMENT_MIN"]
df_agg_filter =(df_inst_pay_group_cnt_distict == 2)
 assert(df_agg_filter |(df_inst_agg["AMT_INSTALMENT_MIN"] == df_inst_agg["AMT_INSTALMENT_MAX"])).all(axis=None)
 df_inst_pay_processed["AMT_INSTALMENT"] = df_inst_agg["AMT_INSTALMENT_MIN"]
df_inst_pay_processed.loc[df_agg_filter, "AMT_INSTALMENT"] = df_inst_agg["AMT_INSTALMENT_SUM"]
print("%d payments aggregated" % df_agg_filter.sum())
 del df_inst_pay_group_cnt_distict, df_agg_filter
df_inst_pay_processed.reset_index(inplace=True)
 df_inst_pay_processed["DAYS_ENTRY_PAYMENT"] = df_inst_pay_processed["DAYS_ENTRY_PAYMENT"].astype(np.float16,
copy=False)
 df_inst_pay_processed["AMT_PAYMENT"] = df_inst_pay_processed["AMT_PAYMENT"].astype(np.float32, copy=False)
 df_inst_pay_processed["AMT_PAYMENT"].replace(-1, -np.inf, inplace=True)
 assert(( df_inst_pay_processed["AMT_PAYMENT"] >= 0)| 
(df_inst_pay_processed["DAYS_ENTRY_PAYMENT"] == 0)).all(axis=None)
 df_diff_entry_offset = df_inst_pay_processed["DAYS_ENTRY_PAYMENT"] - df_inst_pay_processed["DAYS_INSTALMENT"]
df_inst_pay_processed["AMT_DUE_PAYMENT"] =(np.fmax(df_inst_pay_processed["AMT_PAYMENT"], 0)* 
(df_diff_entry_offset <= 0))
df_inst_pay_processed["AMT_DUE30_PAYMENT"] =(np.fmax(df_inst_pay_processed["AMT_PAYMENT"], 0)* 
(df_diff_entry_offset <= 30))
df_inst_pay_group = df_inst_pay_processed.groupby(["SK_ID_PREV", "NUM_INSTALMENT_NUMBER", "NUM_INSTALMENT_VERSION"])
 del df_diff_entry_offset, df_inst_pay_processed, df_inst_agg
print("Aggregate multiple payments for one installment")
 df_inst_pay_group_cnt = df_inst_pay_group.size()
df_inst_agg = flatten_agg_df_columns(df_inst_pay_group.agg({
"SK_ID_CURR": ["min", "max"],
"DAYS_INSTALMENT": ["min", "max"],
"DAYS_ENTRY_PAYMENT": ["min", "max"],
"AMT_INSTALMENT": ["min", "max", "sum"],
"AMT_PAYMENT": ["sum"],
"AMT_DUE_PAYMENT": ["sum"],
"AMT_DUE30_PAYMENT": ["sum"]
}, skipna=False))
del df_inst_pay_group
print("Finish aggregations")
 gc.collect()
print("Processing 2")
 assert(df_inst_agg["SK_ID_CURR_MIN"] == df_inst_agg["SK_ID_CURR_MAX"] ).all() , "Inconsistent SK_ID_CURR"
df_inst_pay_processed = pd.DataFrame(index=df_inst_agg.index)
 df_inst_pay_processed["SK_ID_CURR"] = df_inst_agg["SK_ID_CURR_MIN"]
assert(df_inst_agg["DAYS_INSTALMENT_MIN"] ==
df_inst_agg["DAYS_INSTALMENT_MAX"] ).all(axis=None), "Inconsistent DAYS_INSTALMENT"
df_inst_pay_processed["DAYS_INSTALMENT"] = df_inst_agg["DAYS_INSTALMENT_MIN"]
df_inst_pay_processed["DAYS_FIRST_PAYMENT"] = df_inst_agg["DAYS_ENTRY_PAYMENT_MIN"].replace(0, np.nan)
 df_inst_pay_processed["DAYS_LAST_PAYMENT"] = df_inst_agg["DAYS_ENTRY_PAYMENT_MAX"].replace(0, np.nan)
 assert(df_inst_agg["AMT_INSTALMENT_MIN"] == df_inst_agg["AMT_INSTALMENT_MAX"] ).all(axis=None)
 df_inst_pay_processed["AMT_INSTALMENT"] = df_inst_agg["AMT_INSTALMENT_MIN"]
df_inst_pay_invalid_filter =(df_inst_agg["AMT_PAYMENT_SUM"] < 0)
 assert(( ~df_inst_pay_invalid_filter)|(df_inst_pay_group_cnt == 1)).all(axis=None)
 df_inst_pay_processed["AMT_PAYMENT"] = df_inst_agg["AMT_PAYMENT_SUM"]
df_inst_pay_processed.loc[df_inst_pay_invalid_filter, "AMT_PAYMENT"] = np.nan
assert(df_inst_pay_processed["AMT_PAYMENT"] != 0 ).all(axis=None)
 df_inst_pay_invalid_filter = df_inst_pay_processed["AMT_PAYMENT"].isnull()
df_inst_pay_processed["NUM_PAYMENTS"] = df_inst_pay_group_cnt.astype(np.uint16)
 df_inst_pay_processed.loc[df_inst_pay_invalid_filter, "NUM_PAYMENTS"] = np.uint16(0)
 print("%d installments aggregated" %(df_inst_pay_group_cnt > 1 ).sum())
 del df_inst_pay_group_cnt, df_inst_pay_invalid_filter
df_inst_pay_processed["AMT_OVERDUE"] = np.fmax(df_inst_pay_processed["AMT_INSTALMENT"] -
df_inst_agg["AMT_DUE_PAYMENT_SUM"], 0)
 df_inst_pay_processed["AMT_OVERDUE"] *=(df_inst_pay_processed["AMT_OVERDUE"] >= 0.01)
 df_inst_pay_processed["AMT_DPD30"] = np.fmax(df_inst_pay_processed["AMT_INSTALMENT"] -
df_inst_agg["AMT_DUE30_PAYMENT_SUM"], 0)
 df_inst_pay_processed["AMT_DPD30"] *=(df_inst_pay_processed["AMT_DPD30"] >= 0.01)
 df_inst_pay_processed["AMT_UNPAID"] = np.fmax(df_inst_pay_processed["AMT_INSTALMENT"] -
df_inst_pay_processed["AMT_PAYMENT"].fillna(0), 0)
 df_inst_pay_processed["AMT_UNPAID"] *=(df_inst_pay_processed["AMT_UNPAID"] >= 0.01)
 df_inst_pay_processed.reset_index(inplace=True)
 del df_inst_agg
print("Finish processing")
 output_columns = ["SK_ID_PREV", "SK_ID_CURR", "NUM_INSTALMENT_VERSION", "NUM_INSTALMENT_NUMBER",
"DAYS_INSTALMENT", "DAYS_FIRST_PAYMENT", "DAYS_LAST_PAYMENT", "NUM_PAYMENTS",
"AMT_INSTALMENT", "AMT_PAYMENT", "AMT_OVERDUE", "AMT_DPD30", "AMT_UNPAID"]
df_inst_pay_processed.drop(df_inst_pay_processed.columns.drop(output_columns), axis=1, inplace=True)
 print("Finish cleaning installment payment data")
 return df_inst_pay_processed
def merge_payment_info() :
df_inst_pay = load_install_payments()
df_pos = load_pos_balance()
df_inst_pay["MONTHS_BALANCE"] = df_inst_pay["DAYS_INSTALMENT"] // 30
df_pos.set_index(["SK_ID_PREV", "MONTHS_BALANCE"], inplace=True, drop=True)
 df_merged = df_inst_pay.join(df_pos, on=["SK_ID_PREV", "MONTHS_BALANCE"], how="outer", rsuffix="_POS")
 t = df_merged[(df_merged.NAME_CONTRACT_STATUS != "Active")& df_merged.SK_ID_CURR.notnull() ]
return t<create_dataframe>
BUILD_ENSEMBLE = True
def group_values(col_orig, new_col_values):
return pd.DataFrame({col: col_orig.isin(values)for col, values in new_col_values})
 def get_preprocessed_bureau_data() :
df_bureau = load_bureau()
df_bureau_balance = load_bureau_balance()
df_bureau_balance["STATUS"] = df_bureau_balance["STATUS"].cat.codes - 1
df_bureau_balance["HAS_OVERDUE"] = df_bureau_balance["STATUS"] > 0
df_bureau_balance_agg = df_bureau_balance.groupby("SK_ID_BUREAU" ).agg({
"HAS_OVERDUE": ["sum"],
"STATUS": ["max"]
})
 df_bureau_balance_agg.columns = ["CNT_OVERDUE", "MAX_OVERDUE_TIME"]
df_bureau = df_bureau.join(df_bureau_balance_agg)
 df_bureau["CNT_OVERDUE"].where(df_bureau["CNT_OVERDUE"] > 0,
df_bureau["AMT_CREDIT_MAX_OVERDUE"].gt(0 ).astype(np.uint8, copy=False),
inplace=True)
 df_bureau.loc[df_bureau["DAYS_ENDDATE_FACT"].isna() &(df_bureau["AMT_CREDIT_SUM_DEBT"] == 0)& 
(df_bureau["DAYS_CREDIT_ENDDATE"] < 0), "DAYS_ENDDATE_FACT"] = df_bureau["DAYS_CREDIT_ENDDATE"]
df_bureau["DURATION"] = df_bureau["DAYS_ENDDATE_FACT"].fillna(0)- df_bureau["DAYS_CREDIT"]
df_bureau["CLOSE_DURATION_RATIO"] =(( df_bureau["DAYS_ENDDATE_FACT"] - df_bureau["DAYS_CREDIT"])/ 
(df_bureau["DAYS_CREDIT_ENDDATE"] - df_bureau["DAYS_CREDIT"]))
bureau_groups = df_bureau.groupby("SK_ID_CURR")
 df_bureau_agg = flatten_agg_df_columns(bureau_groups.agg({
"CNT_OVERDUE": ["sum"],
"DURATION": ["sum"],
"AMT_CREDIT_MAX_OVERDUE": ["max"],
"AMT_CREDIT_SUM_OVERDUE": ["max", "sum"],
"DAYS_CREDIT": ["min", "max"],
"CREDIT_DAY_OVERDUE": ["max", "mean"],
"MAX_OVERDUE_TIME": ["max"],
"AMT_CREDIT_SUM": ["max", "mean", "sum"],
"AMT_CREDIT_SUM_DEBT": ["max", "mean", "sum"],
"CNT_CREDIT_PROLONG": ["max"],
"DURATION": ["sum"],
"DAYS_ENDDATE_FACT": ["max"],
"CLOSE_DURATION_RATIO": ["min", "max", "mean"]
}), "BUREAU")
 df_bureau_agg["BUREAU_OVERDUE_FREQ"] = df_bureau_agg["BUREAU_CNT_OVERDUE_SUM"] / df_bureau_agg["BUREAU_DURATION_SUM"]
df_bureau_agg["BUREAU_LOAN_CNT"] = bureau_groups.size()
bureau_active_groups = df_bureau[df_bureau.CREDIT_ACTIVE == "Active"].groupby("SK_ID_CURR")
 df_bureau_active_agg = flatten_agg_df_columns(bureau_active_groups.agg({
"DAYS_CREDIT_ENDDATE": ["max"],
"AMT_ANNUITY": ["sum"]
}), "BUREAU_ACTIVE")
 df_bureau_agg = df_bureau_agg.join(df_bureau_active_agg, how="left")
 df_bureau_agg.fillna({
"BUREAU_AMT_CREDIT_MAX_OVERDUE_MAX": 0,
"BUREAU_AMT_CREDIT_SUM_DEBT_SUM": 0,
"BUREAU_ACTIVE_AMT_CREDIT_SUM_DEBT_SUM": 0,
}, inplace=True)
 del df_bureau, df_bureau_balance
return df_bureau_agg<prepare_x_and_y>
if BUILD_ENSEMBLE:
pred_01_df = pd.read_csv('.. /input/previous-results/90_submission_v1.csv')
 pred_02_df = pd.read_csv('.. /input/previous-results/93_submission_v2.csv')
 pred_03_df = pd.read_csv('.. /input/previous-results/95_submission_v3.csv')
 pred_04_df = pd.read_csv('.. /input/previous-results/97_submission_v4.csv')
 pred_05_df = pd.read_csv('.. /input/previous-results/99_submission_v5.csv')
 avg_pred =(np.array(pred_01_df.values[:,1:])
 + np.array(pred_02_df.values[:,1:])
 + np.array(pred_03_df.values[:,1:])
 + np.array(pred_04_df.values[:,1:])
 + np.array(pred_05_df.values[:,1:])
)/5.0
valid_pred_df = pd.DataFrame(avg_pred, columns=pred_01_df.columns[1:])
 submission_pred_df = pd.concat([pred_01_df['id'],valid_pred_df],axis=1)
 else:
print('Should not submit single distibution')
def get_preprocessed_data(impute=True, random_seed=None):
print("Reading data")
 df_app_train, df_app_test = load_app_data()
print("Finish reading data")
 print("Preprocess training data")
 df_bureau_agg = None
df_prev_app_agg = None
df_bureau_agg = get_preprocessed_bureau_data()
print("Finish preprocessing bureau data")
 df_prev_app_agg = get_preprocessed_previous_app_data(True, True)
 print("Finish preprocessing previous application data")
 df_app_train = shuffle(df_app_train, random_state=random_seed)
 transformers = dict()
X_train = preprocess_app(transformers, df_app_train, df_bureau_agg, df_prev_app_agg, impute=impute)
 y_train = df_app_train["TARGET"]
print("Finish preprocessing application data")
 print("Preprocess test data")
 X_test = preprocess_app(transformers, df_app_test, df_bureau_agg, df_prev_app_agg, impute=impute)
 print("Training data shape:", X_train.shape)
 X_train.info(verbose=5)
 return X_train, y_train, X_test
X_train, y_train, X_test = get_preprocessed_data(PERFORM_IMPUTATION, RANDOM_SEED )<choose_model_class>
validation_gt_data = pd.read_csv('.. /input/m5-forecasting-accuracy/sales_train_evaluation.csv')
 validation_gt_data['id'] = validation_gt_data['id'].str.replace('_evaluation','_validation')
 validation_gt_data = validation_gt_data.drop(['item_id','dept_id','cat_id','store_id','state_id'],axis=1)
 validation_gt_data = pd.concat([validation_gt_data[['id']],validation_gt_data.iloc[:,-28:]],axis=1)
 validation_gt_data.columns=submission_pred_df.columns.values
def build_model_and_classify(X_train, y_train, X_test=None, classifier="xgb", early_stopping=True,
cv=CROSS_VALIDATION_FOLD, tune_param=False, random_seed=None,
output_path="submission.csv"):
print("Initializing classifier")
 if classifier == "xgb":
if tune_param:
clf = XGBClassifier(seed=random_seed, tree_method="gpu_hist")
 else:
clf = XGBClassifier(n_estimators=2000, min_child_weight=32, max_depth=6,
min_split_loss=0.08, learning_rate=0.1,
reg_lambda=1, reg_alpha=0.6,
scale_pos_weight=3,
seed=random_seed, tree_method="gpu_hist")
 elif classifier == "lgbm":
clf = LGBMClassifier(
 boosting_type="goss", n_estimators=1000, learning_rate=0.00513,
num_leaves=54, max_depth=10, subsample_for_bin=240000,
reg_alpha=0.436193, reg_lambda=0.479169,
min_split_gain=0.025, colsample_bytree=0.52,
subsample=1,
silent=-1, verbose=-1 
)
 else:
raise ValueError("Unsupported classifier: " + classifier)
 if early_stopping:
fit_orig = clf.fit
VALIDATION_SIZE = 0.1
def fit_wrapped(X, y, **kwargs):
X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=VALIDATION_SIZE)
 fit_orig(X_train, y_train, eval_set=[(X_val, y_val)],
early_stopping_rounds=50, eval_metric="auc")
 clf.fit = fit_wrapped
if cv > 0:
if tune_param:
print("Tuning classifier parameters")
 params = {
"n_estimators": [200, 400],
"max_depth": [6, 8],
"max_leaves": [64, 128],
"min_child_weight": [24, 32],
"min_split_loss": [0.02 *(1 << n)for n in range(6)],
"learning_rate": [0.06 + 0.02 *(1 << n)for n in range(1, 4)],
"reg_lambda": [0.2 * n for n in range(6)],
"reg_alpha": [0.2 * n for n in range(6)],
"scale_pos_weight": [1, 3, 5]
}
model_selection_task = RandomizedSearchCV(clf, params, n_iter=50, scoring="roc_auc",
cv=cv, n_jobs=5, verbose=5)
 global results
results = model_selection_task.fit(X_train, y_train)
 print(f"Best score: {results.best_score_}")
 print(f"Best parameters: {results.best_params_}")
 print(f"CV results: {results.cv_results_}")
 clf = results.best_estimator_
else:
print(f"Perform {cv:d}-fold cross validation")
 score_val = sum(cross_val_score(clf, X_train, y_train,
cv=cv, scoring="roc_auc", verbose=10, n_jobs=5)) / cv
print("Validation AUC: %.6f" % score_val)
 else:
test_size = 0.1
print(f"Perform hold-out validation(Test size: {test_size:.0%})")
 X_train, X_val, y_train, y_val = train_test_split(X_train, y_train, test_size=test_size, random_state=random_seed)
 clf.fit(X_train, y_train)
 prob_val = clf.predict_proba(X_val)[:, 1]
score_val = roc_auc_score(y_val, prob_val)
 print("Validation AUC: %.6f" % score_val)
 if X_test is None:
return
print("Training classifier")
 clf.fit(X_train, y_train)
 print("Classify training and test set")
 test_prob_df = pd.DataFrame(clf.predict_proba(X_test)[:, 1], index=X_test.index, columns=["TARGET"])
 test_prob_df.to_csv(output_path)
 print("Finished classifying" )<train_model>
submission_pred_df = pd.concat([validation_gt_data, submission_pred_df.iloc[30490:,:]],axis=0 ).reset_index(drop=True)
 submission_pred_df
build_model_and_classify(X_train, y_train, X_test, "lgbm", cv=CROSS_VALIDATION_FOLD, tune_param=True,
random_seed=RANDOM_SEED, output_path="submission_lgbm.csv" )<train_model>
build_model_and_classify(X_train, y_train, X_test, tune_param=True, random_seed=RANDOM_SEED,
output_path="submission_xgb.csv" )<prepare_output>
name_cols = bottom_lvl_pred_df.id.str.split(pat='_',expand=True)
 name_cols['dept_id']=name_cols[0]+'_'+name_cols[1]
name_cols['store_id']=name_cols[3]+'_'+name_cols[4]
name_cols = name_cols.rename(columns={0: "cat_id", 3: "state_id"})
 name_cols = name_cols.drop([1,2,4,5],axis=1)
 bottom_lvl_pred_df = pd.concat([name_cols,bottom_lvl_pred_df],axis=1 )
df_result = pd.DataFrame.from_records(results.cv_results_["params"])
 df_result["rank"] = results.cv_results_["rank_test_score"]
df_result["score"] = results.cv_results_["mean_test_score"]
df_result.sort_values(by="rank", inplace=True)
 df_result<sort_values>
cat_cols = ['id', 'dept_id', 'cat_id', 'store_id', 'state_id']
ts_cols = [col for col in bottom_lvl_pred_df.columns if col not in cat_cols]
ts_dict = {t: int(t[1:])for t in ts_cols}
print(' unique forecasts: %i' % bottom_lvl_pred_df.shape[0])
 for col in cat_cols:
print(' N_unique %s: %i' %(col, bottom_lvl_pred_df[col].nunique()))
for c in df_result.columns[:-2]:
print(df_result.groupby(c)["score"].mean().sort_values() )<load_from_csv>
all_sales = pd.DataFrame(bottom_lvl_pred_df[ts_cols].sum() ).transpose()
all_sales['id_str'] = 'all'
all_sales = all_sales[ ['id_str'] + [c for c in all_sales if c not in ['id_str']] ]
sub_path = ".. /input/statoil-iceberg-submissions"
all_files = os.listdir(sub_path)
 outs = [pd.read_csv(os.path.join(sub_path, f), index_col=0)for f in all_files]
concat_sub = pd.concat(outs, axis=1)
 cols = list(map(lambda x: "is_iceberg_" + str(x), range(len(concat_sub.columns))))
concat_sub.columns = cols
concat_sub.reset_index(inplace=True)
 concat_sub.head() 
<feature_engineering>
state_sales = bottom_lvl_pred_df.groupby('state_id',as_index=False)[ts_cols].sum()
state_sales['id_str'] = state_sales['state_id']
state_sales = state_sales[ ['id_str'] + [c for c in state_sales if c not in ['id_str']] ]
state_sales = state_sales.drop(['state_id'],axis=1 )
concat_sub['is_iceberg_max'] = concat_sub.iloc[:, 1:6].max(axis=1)
 concat_sub['is_iceberg_min'] = concat_sub.iloc[:, 1:6].min(axis=1)
 concat_sub['is_iceberg_mean'] = concat_sub.iloc[:, 1:6].mean(axis=1)
 concat_sub['is_iceberg_median'] = concat_sub.iloc[:, 1:6].median(axis=1 )<define_variables>
store_sales = bottom_lvl_pred_df.groupby('store_id',as_index=False)[ts_cols].sum()
store_sales['id_str'] = store_sales['store_id']
store_sales = store_sales[ ['id_str'] + [c for c in store_sales if c not in ['id_str']] ]
store_sales = store_sales.drop(['store_id'],axis=1 )
cutoff_lo = 0.8
cutoff_hi = 0.2<save_to_csv>
cat_sales = bottom_lvl_pred_df.groupby('cat_id',as_index=False)[ts_cols].sum()
cat_sales['id_str'] = cat_sales['cat_id']
cat_sales = cat_sales[ ['id_str'] + [c for c in cat_sales if c not in ['id_str']] ]
cat_sales = cat_sales.drop(['cat_id'],axis=1)
concat_sub['is_iceberg'] = concat_sub['is_iceberg_mean']
concat_sub[['id', 'is_iceberg']].to_csv('stack_mean.csv',
index=False, float_format='%.6f' )<save_to_csv>
dept_sales = bottom_lvl_pred_df.groupby('dept_id',as_index=False)[ts_cols].sum()
dept_sales['id_str'] = dept_sales['dept_id']
dept_sales = dept_sales[ ['id_str'] + [c for c in dept_sales if c not in ['id_str']] ]
dept_sales = dept_sales.drop(['dept_id'],axis=1 )
concat_sub['is_iceberg'] = concat_sub['is_iceberg_median']
concat_sub[['id', 'is_iceberg']].to_csv('stack_median.csv',
index=False, float_format='%.6f' )<save_to_csv>
all_pred_agg = pd.concat([all_sales,state_sales,store_sales,cat_sales,dept_sales],ignore_index=True)
concat_sub['is_iceberg'] = np.where(np.all(concat_sub.iloc[:,1:6] > cutoff_lo, axis=1), 1,
np.where(np.all(concat_sub.iloc[:,1:6] < cutoff_hi, axis=1),
0, concat_sub['is_iceberg_median']))
concat_sub[['id', 'is_iceberg']].to_csv('stack_pushout_median.csv',
index=False, float_format='%.6f' )<feature_engineering>
metrics_df = nbeats_pred01_df[['id_str']]
error =(np.array(all_pred_agg.values[:,1:])- np.array(nbeats_pred01_df.values[:,1:]))
successive_diff = np.diff(nbeats_pred01_df.values[:,1:])** 2
denom = successive_diff.mean(1)
 num = error.mean(1)**2
rmsse = num / denom
metrics_df['rmsse'] = rmsse
metrics_df['mean_error'] = error.mean(1)
 metrics_df['mean_abs_error'] = np.absolute(error ).mean(1)
 squared_error = error **2
mean_squ_err = np.array(squared_error.mean(1), dtype=np.float64)
 metrics_df['rmse'] = np.sqrt(mean_squ_err)
 metrics_df
concat_sub['is_iceberg'] = np.where(np.all(concat_sub.iloc[:,1:6] > cutoff_lo, axis=1),
concat_sub['is_iceberg_max'],
np.where(np.all(concat_sub.iloc[:,1:6] < cutoff_hi, axis=1),
concat_sub['is_iceberg_min'],
concat_sub['is_iceberg_mean']))
concat_sub[['id', 'is_iceberg']].to_csv('stack_minmax_mean.csv',
index=False, float_format='%.6f' )<feature_engineering>
metrics_df = nbeats_pred02_df[['id_str']]
error =(np.array(all_pred_agg.values[:,1:])- np.array(nbeats_pred02_df.values[:,1:]))
successive_diff = np.diff(nbeats_pred01_df.values[:,1:])** 2
denom = successive_diff.mean(1)
 num = error.mean(1)**2
rmsse = num / denom
metrics_df['rmsse'] = rmsse
metrics_df['mean_error'] = error.mean(1)
 metrics_df['mean_abs_error'] = np.absolute(error ).mean(1)
 squared_error = error **2
mean_squ_err = np.array(squared_error.mean(1), dtype=np.float64)
 metrics_df['rmse'] = np.sqrt(mean_squ_err)
 metrics_df
<load_from_csv><EOS> 
submission_pred_df.to_csv('m5-final-submission.csv', index=False)