Kaggle competition: COVID19 Global Forecasting (Week 2)
<SOS> metric: MCRMSLE Kaggle data source: covid19-global-forecasting-(week-2)<categorify>
import os
import numpy as np
import pandas as pd
from scipy.optimize.minpack import curve_fit
gc.enable()
buro_bal = pd.read_csv('.. /input/bureau_balance.csv')
 print('Buro bal shape : ', buro_bal.shape)
 print('transform to dummies')
 buro_bal = pd.concat([buro_bal, pd.get_dummies(buro_bal.STATUS, prefix='buro_bal_status')], axis=1 ).drop('STATUS', axis=1)
 print('Counting buros')
 buro_counts = buro_bal[['SK_ID_BUREAU', 'MONTHS_BALANCE']].groupby('SK_ID_BUREAU' ).count()
buro_bal['buro_count'] = buro_bal['SK_ID_BUREAU'].map(buro_counts['MONTHS_BALANCE'])
 print('averaging buro bal')
 avg_buro_bal = buro_bal.groupby('SK_ID_BUREAU' ).mean()
avg_buro_bal.columns = ['avg_buro_' + f_ for f_ in avg_buro_bal.columns]
del buro_bal
gc.collect()
print('Read Bureau')
 buro = pd.read_csv('.. /input/bureau.csv')
 print('Go to dummies')
 buro_credit_active_dum = pd.get_dummies(buro.CREDIT_ACTIVE, prefix='ca_')
 buro_credit_currency_dum = pd.get_dummies(buro.CREDIT_CURRENCY, prefix='cu_')
 buro_credit_type_dum = pd.get_dummies(buro.CREDIT_TYPE, prefix='ty_')
 buro_full = pd.concat([buro, buro_credit_active_dum, buro_credit_currency_dum, buro_credit_type_dum], axis=1)
 del buro_credit_active_dum, buro_credit_currency_dum, buro_credit_type_dum
gc.collect()
print('Merge with buro avg')
 buro_full = buro_full.merge(right=avg_buro_bal.reset_index() , how='left', on='SK_ID_BUREAU', suffixes=('', '_bur_bal'))
print('Counting buro per SK_ID_CURR')
 nb_bureau_per_curr = buro_full[['SK_ID_CURR', 'SK_ID_BUREAU']].groupby('SK_ID_CURR' ).count()
buro_full['SK_ID_BUREAU'] = buro_full['SK_ID_CURR'].map(nb_bureau_per_curr['SK_ID_BUREAU'])
 print('Averaging bureau')
 avg_buro = buro_full.groupby('SK_ID_CURR' ).mean()
print(avg_buro.head())
 del buro, buro_full
gc.collect()
print('Read prev')
 prev = pd.read_csv('.. /input/previous_application.csv')
 prev_cat_features = [
f_ for f_ in prev.columns if prev[f_].dtype == 'object'
]
print('Go to dummies')
 prev_dum = pd.DataFrame()
for f_ in prev_cat_features:
prev_dum = pd.concat([prev_dum, pd.get_dummies(prev[f_], prefix=f_ ).astype(np.uint8)], axis=1)
 prev = pd.concat([prev, prev_dum], axis=1)
 del prev_dum
gc.collect()
print('Counting number of Prevs')
 nb_prev_per_curr = prev[['SK_ID_CURR', 'SK_ID_PREV']].groupby('SK_ID_CURR' ).count()
prev['SK_ID_PREV'] = prev['SK_ID_CURR'].map(nb_prev_per_curr['SK_ID_PREV'])
 print('Averaging prev')
 avg_prev = prev.groupby('SK_ID_CURR' ).mean()
print(avg_prev.head())
 del prev
gc.collect()
print('Reading POS_CASH')
 pos = pd.read_csv('.. /input/POS_CASH_balance.csv')
 print('Go to dummies')
 pos = pd.concat([pos, pd.get_dummies(pos['NAME_CONTRACT_STATUS'])], axis=1)
 print('Compute nb of prevs per curr')
 nb_prevs = pos[['SK_ID_CURR', 'SK_ID_PREV']].groupby('SK_ID_CURR' ).count()
pos['SK_ID_PREV'] = pos['SK_ID_CURR'].map(nb_prevs['SK_ID_PREV'])
 print('Go to averages')
 avg_pos = pos.groupby('SK_ID_CURR' ).mean()
del pos, nb_prevs
gc.collect()
print('Reading CC balance')
 cc_bal = pd.read_csv('.. /input/credit_card_balance.csv')
 print('Go to dummies')
 cc_bal = pd.concat([cc_bal, pd.get_dummies(cc_bal['NAME_CONTRACT_STATUS'], prefix='cc_bal_status_')], axis=1)
 nb_prevs = cc_bal[['SK_ID_CURR', 'SK_ID_PREV']].groupby('SK_ID_CURR' ).count()
cc_bal['SK_ID_PREV'] = cc_bal['SK_ID_CURR'].map(nb_prevs['SK_ID_PREV'])
 print('Compute average')
 avg_cc_bal = cc_bal.groupby('SK_ID_CURR' ).mean()
avg_cc_bal.columns = ['cc_bal_' + f_ for f_ in avg_cc_bal.columns]
del cc_bal, nb_prevs
gc.collect()
print('Reading Installments')
 inst = pd.read_csv('.. /input/installments_payments.csv')
 nb_prevs = inst[['SK_ID_CURR', 'SK_ID_PREV']].groupby('SK_ID_CURR' ).count()
inst['SK_ID_PREV'] = inst['SK_ID_CURR'].map(nb_prevs['SK_ID_PREV'])
 avg_inst = inst.groupby('SK_ID_CURR' ).mean()
avg_inst.columns = ['inst_' + f_ for f_ in avg_inst.columns]
print('Read data and test')
 data = pd.read_csv('.. /input/application_train.csv')
 test = pd.read_csv('.. /input/application_test.csv')
 print('Shapes : ', data.shape, test.shape)
 print('Read data and test')
 train = pd.read_csv('.. /input/application_train.csv')
 test = pd.read_csv('.. /input/application_test.csv')
 print('Shapes : ', train.shape, train.shape)
 categorical_feats = [
f for f in data.columns if data[f].dtype == 'object'
]
categorical_feats
for f_ in categorical_feats:
train[f_], indexer = pd.factorize(train[f_])
 test[f_] = indexer.get_indexer(test[f_])
 train = train.merge(right=avg_buro.reset_index() , how='left', on='SK_ID_CURR')
 test = test.merge(right=avg_buro.reset_index() , how='left', on='SK_ID_CURR')
 train = train.merge(right=avg_prev.reset_index() , how='left', on='SK_ID_CURR')
 test = test.merge(right=avg_prev.reset_index() , how='left', on='SK_ID_CURR')
 train = train.merge(right=avg_pos.reset_index() , how='left', on='SK_ID_CURR')
 test = test.merge(right=avg_pos.reset_index() , how='left', on='SK_ID_CURR')
 train = train.merge(right=avg_cc_bal.reset_index() , how='left', on='SK_ID_CURR')
 test = test.merge(right=avg_cc_bal.reset_index() , how='left', on='SK_ID_CURR')
 train = train.merge(right=avg_inst.reset_index() , how='left', on='SK_ID_CURR')
 test = test.merge(right=avg_inst.reset_index() , how='left', on='SK_ID_CURR')
 del avg_buro, avg_prev
gc.collect()
ID = test.SK_ID_CURR
train.columns = train.columns.str.replace('[^A-Za-z0-9_]', '_')
 test.columns = test.columns.str.replace('[^A-Za-z0-9_]', '_' )<data_type_conversions>
def load_kaggle_csv(dataset: str)-> pd.DataFrame:
df = pd.read_csv(f"/kaggle/input/covid19-global-forecasting-week-2/{dataset}.csv", parse_dates=["Date"])
 df["Province_State"].fillna("", inplace=True)
 df["DayOfYear"] = df["Date"].dt.dayofyear
df["Date"] = df["Date"].dt.date
return df
floattypes = []
inttypes = []
stringtypes = []
for c in test.columns:
if(train[c].dtype=='object'):
train[c] = train[c].astype('str')
 stringtypes.append(c)
 elif(train[c].dtype=='int64'):
train[c] = train[c].astype('int32')
 inttypes.append(c)
 else:
train[c] = train[c].astype('float32')
 floattypes.append(c )<categorify>
train = load_kaggle_csv("train")
 test = load_kaggle_csv("test" )
kf = KFold(n_splits=5, shuffle=True, random_state=42)
 for col in stringtypes:
train['te_'+col] = 0.
 test['te_'+col] = 0.
 SMOOTHING = test[~test[col].isin(train[col])].shape[0]/test.shape[0]
_, test['te_'+col] = target_encode(train[col],
test[col],
target=train['TARGET'],
min_samples_leaf=100,
smoothing=SMOOTHING,
noise_level=0.0)
 for f,(vis_index, blind_index)in enumerate(kf.split(train)) :
_, train.loc[blind_index, 'te_'+col] = target_encode(train.loc[vis_index, col],
train.loc[blind_index, col],
target=train.loc[vis_index,'TARGET'],
min_samples_leaf=100,
smoothing=SMOOTHING,
noise_level=0.0 )<categorify>
def RMSLE(prediction, actual):
return np.sqrt(
 np.mean(np.power(np.log1p(prediction)- np.log1p(actual), 2)) 
)
 def logistic(x, x0, L, k):
return L /(1 + np.exp(-k *(x - x0)))
 def fit(function, x, y, maxfev):
p0 = [np.median(x), y[-1], 0.1]
pn0 = p0 *(np.random.random(len(p0)) + [0.5, 1.0, 0.5])
 try:
params, pcov = curve_fit(
 function,
x,
y,
p0=pn0,
maxfev=maxfev,
sigma=np.maximum(1, np.sqrt(y)) *(0.1 + 0.9 * np.random.random()),
bounds=([0, y[-1], 0.01], [200, 1e6, 1.5]), 
)
 pcov = pcov[np.triu_indices_from(pcov)]
except(RuntimeError, ValueError):
params = p0
pcov = np.zeros(len(p0)*(len(p0)- 1))
y_hat = function(x, *params)
 rmsle = RMSLE(y_hat, y)
 return(params, pcov, rmsle, y_hat)
 def fit_model(df: pd.DataFrame, n_samples=2, maxfev=1000):
def fit_one(function, ycol):
best_rmsle = None
best_params = None
for i in range(n_samples):
params, cov, rmsle, y_hat = fit(
 function,
df["DayOfYear"].to_numpy() ,
df[ycol].to_numpy() ,
maxfev=maxfev, 
)
 if rmsle >=(best_rmsle or rmsle):
best_rmsle = rmsle
best_params = params
result = {f"{ycol}_rmsle": best_rmsle}
result.update({f"{ycol}_p_{i}": p for i, p in enumerate(best_params)})
 return result
result = {}
result.update(fit_one(logistic, "ConfirmedCases"))
result.update(fit_one(logistic, "Fatalities"))
return pd.DataFrame([result] )
kf = KFold(n_splits=5, shuffle=True, random_state=42)
 for col in inttypes:
train['te_'+col] = 0.
 test['te_'+col] = 0.
 SMOOTHING = test[~test[col].isin(train[col])].shape[0]/test.shape[0]
_, test['te_'+col] = target_encode(train[col],
test[col],
target=train['TARGET'],
min_samples_leaf=100,
smoothing=SMOOTHING,
noise_level=0.0)
 for f,(vis_index, blind_index)in enumerate(kf.split(train.index)) :
_, train.loc[blind_index, 'te_'+col] = target_encode(train.loc[vis_index, col],
train.loc[blind_index, col],
target=train.loc[vis_index,'TARGET'],
min_samples_leaf=100,
smoothing=SMOOTHING,
noise_level=0.0)
<concatenate>
train_fit = train.groupby(
 ["Country_Region", "Province_State"], observed=True, sort=False
).apply(lambda x: fit_model(x, n_samples=20, maxfev=1000))
test.insert(1,'TARGET',-1)
 alldata = pd.concat([train,test])
 del train ,test
gc.collect()<data_type_conversions>
def predict(df):
def predict_one(col):
df[f"predict_{col}"] = logistic(
 df["DayOfYear"].to_numpy() ,
df[f"{col}_p_0"].to_numpy() ,
df[f"{col}_p_1"].to_numpy() ,
df[f"{col}_p_2"].to_numpy() , 
)
 predict_one("ConfirmedCases")
 predict_one("Fatalities" )
alldata[features] = alldata[features].astype('float32' )<feature_engineering>
test = pd.merge(
 test,
train_fit.reset_index() ,
on=["Country_Region", "Province_State"],
how="left", 
)
 predict(test )
alldata['nans'] = alldata.isnull().sum(axis=1 )<merge>
submission = test[["ForecastId", "predict_ConfirmedCases", "predict_Fatalities"]].rename(
 columns={"predict_ConfirmedCases": "ConfirmedCases", "predict_Fatalities": "Fatalities"})
 submission[["ConfirmedCases", "Fatalities"]] = submission[["ConfirmedCases", "Fatalities"]].round().astype(int)
 submission.to_csv('submission.csv', index=False)
 print(submission.head())
 print(submission.describe() )