Kaggle competition: Loan Default Prediction - Imperial College London
<SOS> metric: MAE Kaggle data source: loan-default-prediction-imperial-college-london<set_options>
%matplotlib inline
pd.set_option('display.max_columns', 100)
 warnings.filterwarnings('ignore' )<load_from_csv>
df = pd.read_csv('.. /input/loan-default-prediction/train_v2.csv.zip')
 test = pd.read_csv('.. /input/loan-default-prediction/test_v2.csv.zip' )
df_train = pd.read_csv(".. /input/covid19-global-forecasting-week-2/train.csv")
 print(df_train.shape)
 print(df_train.Date.min() , df_train.Date.max())
 df_train.head()<categorify>
for col in df.columns:
pct_missing = np.mean(df[col].isnull())
 if(round(pct_missing*100)) != 0.0:
print('{} - {}%'.format(col, round(pct_missing*100)) )
train_min_date, train_max_date = df_train.Date.min() , df_train.Date.max()
train_min_dayofyear, train_max_dayofyear =(pd.to_datetime(train_min_date)).dayofyear,(pd.to_datetime(train_max_date)).dayofyear
print(train_min_dayofyear, train_max_dayofyear )<define_variables>
for col in test.columns:
pct_missing = np.mean(test[col].isnull())
 if(round(pct_missing*100)) != 0.0:
print('{} - {}%'.format(col, round(pct_missing*100)) )
train_valid_cutoff_dayofyear = train_min_dayofyear +(train_max_dayofyear - train_min_dayofyear)// 3 * 2
train_valid_cutoff_dayofyear<load_from_csv>
df_numeric = df.select_dtypes(include=[np.number])
 numeric_cols = df_numeric.columns.values
for col in numeric_cols:
missing = df[col].isnull()
num_missing = np.sum(missing)
 if num_missing > 0:
df['{}_ismissing'.format(col)] = missing
med = df[col].median()
df[col] = df[col].fillna(med )
df_test = pd.read_csv(".. /input/covid19-global-forecasting-week-2/test.csv")
 print(df_test.shape)
 test_min_date, test_max_date = df_test.Date.min() , df_test.Date.max()
print(test_min_date, test_max_date)
 df_test.head()<concatenate>
test_numeric = test.select_dtypes(include=[np.number])
 test_numeric_cols = test_numeric.columns.values
for col in test_numeric_cols:
missing = test[col].isnull()
num_missing = np.sum(missing)
 if num_missing > 0:
test['{}_ismissing'.format(col)] = missing
med = test[col].median()
test[col] = test[col].fillna(med )
df_traintest = pd.concat([df_train, df_test])
 print(df_train.shape, df_test.shape, df_traintest.shape)
 df_traintest.head()<feature_engineering>
for col in df.columns:
pct_missing = np.mean(df[col].isnull())
 if(round(pct_missing*100)) != 0.0:
print('{} - {}%'.format(col, round(pct_missing*100)) )
def func(x):
try:
x_new = x['Country_Region'] + "/" + x['Province_State']
except:
x_new = x['Country_Region']
return x_new
df_traintest['place_id'] = df_traintest.apply(lambda x: func(x), axis=1)
 tmp = np.sort(df_traintest['place_id'].unique())
 print("num unique places: {}".format(len(tmp)))
 print(tmp[:10] )<data_type_conversions>
for col in test.columns:
pct_missing = np.mean(test[col].isnull())
 if(round(pct_missing*100)) != 0.0:
print('{} - {}%'.format(col, round(pct_missing*100)) )
df_traintest['Date'] = pd.to_datetime(df_traintest['Date'])
 time_cols = [
"month", "week", "day",
"dayofyear", "dayofweek",
]
for attr in time_cols:
dtype = np.int16 if attr == "year" else np.int8
df_traintest[attr] = getattr(df_traintest['Date'].dt, attr ).astype(dtype)
 df_traintest["is_weekend"] = df_traintest["dayofweek"].isin([5, 6] ).astype(np.int8)
 time_cols += ["is_weekend"]
print(time_cols)
 df_traintest.head(10 )<feature_engineering>
df.drop(df.iloc[:, 771:1284], inplace = True, axis = 1 )
df_traintest['cases/day'] = 0
df_traintest['fatal/day'] = 0
places = np.sort(df_traintest['place_id'].unique())
 for place in places:
tmp = df_traintest['ConfirmedCases'][df_traintest['place_id']==place].values
tmp[1:] -= tmp[:-1]
df_traintest['cases/day'][df_traintest['place_id']==place] = tmp
tmp = df_traintest['Fatalities'][df_traintest['place_id']==place].values
tmp[1:] -= tmp[:-1]
df_traintest['fatal/day'][df_traintest['place_id']==place] = tmp
df_traintest[df_traintest['place_id']=='China/Hubei']<feature_engineering>
test.drop(test.iloc[:, 770:1288], inplace = True, axis = 1 )
def do_aggregation(df, col, mean_range, method='mean'):
df_new = copy.deepcopy(df)
 col_new = '{}_{}_({}-{})'.format(col, method, mean_range[0], mean_range[1])
 df_new[col_new] = 0
if method=='mean':
tmp = df_new[col].rolling(mean_range[1]-mean_range[0]+1 ).mean()
elif method=='std':
tmp = df_new[col].rolling(mean_range[1]-mean_range[0]+1 ).std()
df_new[col_new][mean_range[0]:] = tmp[:-(mean_range[0])]
df_new[col_new][pd.isna(df_new[col_new])] = 0
return df_new[[col_new]].reset_index(drop=True)
 def do_aggregations(df):
for method in ['mean']:
df = pd.concat([df, do_aggregation(df, 'cases/day', [1,1], method ).reset_index(drop=True)], axis=1)
 df = pd.concat([df, do_aggregation(df, 'cases/day', [1,7], method ).reset_index(drop=True)], axis=1)
 df = pd.concat([df, do_aggregation(df, 'cases/day', [8,14], method ).reset_index(drop=True)], axis=1)
 df = pd.concat([df, do_aggregation(df, 'fatal/day', [1,1], method ).reset_index(drop=True)], axis=1)
 df = pd.concat([df, do_aggregation(df, 'fatal/day', [1,7], method ).reset_index(drop=True)], axis=1)
 df = pd.concat([df, do_aggregation(df, 'fatal/day', [8,14], method ).reset_index(drop=True)], axis=1)
 return df<concatenate>
upper = corr_matrix.where(np.triu(np.ones(corr_matrix.shape), k=1 ).astype(np.bool))
to_drop = [column for column in upper.columns if any(upper[column] > 0.95)]
df.drop(to_drop, axis=1, inplace=True)
 test.drop(to_drop, axis=1, inplace=True )
df_traintest2 = []
for place in places[:]:
df_tmp = df_traintest[df_traintest['place_id']==place].reset_index(drop=True)
 df_tmp = do_aggregations(df_tmp)
 df_traintest2.append(df_tmp)
 df_traintest2 = pd.concat(df_traintest2 ).reset_index(drop=True)
 df_traintest2[df_traintest2['place_id']=='China/Hubei'].head(20 )<load_from_csv>
df['loss_fact'] = df['loss']
loss_fact = df['loss']
for i in loss_fact:
if i != 0:
loss_fact = loss_fact.replace(i, 1)
 df['loss_fact'] = loss_fact
df_smoking = pd.read_csv(".. /input/shareofadultswhosmoke/adults-smoking-2000-2016.csv")
 print(np.sort(df_smoking['Entity'].unique())[:10])
 df_smoking.head()<merge>
df_traintest3 = pd.merge(df_traintest2, df_smoking_recent[['Country/Region', 'SmokingRate']], left_on='Country_Region', right_on='Country/Region', how='left')
 df_traintest3.head()<filter>
trainfloatloss = pd.concat([df.select_dtypes(include=[np.float64]), df['loss_fact']], axis=1 )
SmokingRate = df_smoking_recent['SmokingRate'][df_smoking_recent['Entity']=='World'].values[0]
print("Smoking rate of the world: {:.6f}".format(SmokingRate))
df_traintest3['SmokingRate'][pd.isna(df_traintest3['SmokingRate'])] = SmokingRate
df_traintest3.head()<merge>
float_columns = [c for c in trainfloatloss.columns if trainfloatloss[c].dtype.name == 'float64']
world_happiness_index = pd.read_csv(".. /input/world-bank-datasets/World_Happiness_Index.csv")
 world_happiness_grouped = world_happiness_index.groupby('Country name' ).nth(-1)
 world_happiness_grouped.head()
world_happiness_grouped.drop("Year", axis=1, inplace=True)
 df_traintest3 = pd.merge(left=df_traintest3, right=world_happiness_grouped, how='left', left_on='Country_Region', right_on='Country name' )<load_from_csv>
test_float_train = testfloat
test_float_train = test_float_train.drop('f5', axis = 1)
 test_float_train = test_float_train.drop('f531', axis = 1 )
malaria_world_health = pd.read_csv(".. /input/world-bank-datasets/Malaria_World_Health_Organization.csv")
 df_traintest3 = pd.merge(left=df_traintest3, right=malaria_world_health, how='left', left_on='Country_Region', right_on='Country')
 df_traintest3.drop("Country", axis=1, inplace=True)
 mwh_cols = [ col for col in malaria_world_health.columns.to_list() if col != "Country" ]
print(mwh_cols )<load_from_csv>
float_columns_test = [c for c in testfloat.columns if testfloat[c].dtype.name == 'float64']
float_columns_test.remove('f5')
 float_columns_test.remove('f531')
human_development_index = pd.read_csv(".. /input/world-bank-datasets/Human_Development_Index.csv")
 human_development_index.drop(["Gross national income(GNI)per capita 2018"], axis=1, inplace=True)
 df_traintest3 = pd.merge(left=df_traintest3, right=human_development_index, how='left', left_on='Country_Region', right_on='Country')
 df_traintest3.drop("Country", axis=1, inplace=True)
 hdi_cols = [ col for col in human_development_index.columns.to_list() if col != "Country" ]
print(hdi_cols )<load_from_csv>
for c in float_columns:
trainfloatloss[trainfloatloss[c].name + '_no_out'] = trainfloatloss[c]
loss_fact0_float = trainfloatloss[trainfloatloss['loss_fact'] == 0][c]
loss_fact1_float = trainfloatloss[trainfloatloss['loss_fact'] == 1][c]
loss_fact0_float_no_out = loss_fact0_float[~(loss_fact0_float > loss_fact0_float.mean() + 2*loss_fact0_float.std())]
loss_fact1_float_no_out = loss_fact1_float[~(loss_fact1_float > loss_fact1_float.mean() + 2*loss_fact1_float.std())]
trainfloatloss[trainfloatloss[c].name + '_no_out'] = loss_fact0_float_no_out.append(loss_fact1_float_no_out )
<load_from_csv>
trainfloatloss[['f756_no_out']].isna().values.sum()
<merge>
trainfloatlossdna = trainfloatloss.dropna(how='any' )
df_lat_long = pd.read_csv(".. /input/lat-long/lat_long.csv")
 df_lat_long['place_id'] = df_lat_long.apply(lambda x: func(x), axis=1)
 df_lat_long.drop(["Country_Region", 'Province_State'], axis=1, inplace=True)
 df_traintest3 = pd.merge(left=df_traintest3, right=df_lat_long, how='left', on='place_id' )<filter>
trainfloatlossdnanoout = pd.concat([trainfloatlossdna.filter(regex='no_out'), trainfloatlossdna['loss_fact']], axis=1 )
df_traintest3[df_traintest3['place_id']=='China/Hubei']<init_hyperparams>
X= trainfloatlossdnanoout.drop(['loss_fact'], axis=1)
 y= trainfloatlossdnanoout['loss_fact']
SEED = 42
params = {'num_leaves': 8,
'min_data_in_leaf': 5,
'objective': 'regression',
'max_depth': 8,
'learning_rate': 0.02,
'boosting': 'gbdt',
'bagging_freq': 5,
'bagging_fraction': 0.8,
'feature_fraction': 0.8201,
'bagging_seed': SEED,
'reg_alpha': 1,
'reg_lambda': 4.9847051755586085,
'random_state': SEED,
'metric': 'mse',
'verbosity': 100,
'min_gain_to_split': 0.02,
'min_child_weight': 5,
'num_threads': 6,
}<filter>
X_train, X_valid, y_train, y_valid = train_test_split(X, y, test_size=0.3, random_state=17 )
df_traintest3[df_traintest3.dayofyear == 72]<define_variables>
first_tree = DecisionTreeClassifier(random_state=17 )
col_target = 'fatal/day'
col_var = [
'Lat', 'Long',
'cases/day_mean_(1-1)', 'cases/day_mean_(1-7)', 'cases/day_mean_(8-14)',
'fatal/day_mean_(1-1)', 'fatal/day_mean_(1-7)', 'fatal/day_mean_(8-14)',
'SmokingRate',
]
col_var += time_cols
extra_cols = wh_cols + mwh_cols + hdi_cols
col_var += extra_cols
df_train = df_traintest3[(pd.isna(df_traintest3['ForecastId'])) &(df_traintest3['dayofyear']<train_valid_cutoff_dayofyear)]
df_valid = df_traintest3[(pd.isna(df_traintest3['ForecastId'])) &(df_traintest3['dayofyear']>=train_valid_cutoff_dayofyear)]
df_test = df_traintest3[pd.isna(df_traintest3['ForecastId'])==False]
X_train = df_train[col_var].values
X_valid = df_valid[col_var].values
y_train = df_train[col_target].values
y_valid = df_valid[col_target].values
train_data = lgb.Dataset(X_train, label=y_train)
 valid_data = lgb.Dataset(X_valid, label=y_valid)
 num_round = 15000
model = lgb.train(params, train_data, num_round, valid_sets=[train_data, valid_data],
verbose_eval=100,
early_stopping_rounds=150, )<train_model>
np.mean(cross_val_score(first_tree, X_train, y_train, cv=5))
X_train = df_train[col_var].values
X_valid = df_valid[col_var].values
y_train = df_train[col_target].values
y_valid = df_valid[col_target].values
train_data = lgb.Dataset(X_train, label=y_train)
 valid_data = lgb.Dataset(X_valid, label=y_valid)
 num_round = 15000
model = lgb.train(params, train_data, num_round, valid_sets=[train_data, valid_data],
verbose_eval=100,
early_stopping_rounds=150, )<define_variables>
first_knn = KNeighborsClassifier()
col_target2 = 'cases/day'
col_var2 = [
'Lat', 'Long',
'cases/day_mean_(1-1)', 'cases/day_mean_(1-7)', 'cases/day_mean_(8-14)',
'fatal/day_mean_(1-1)', 'fatal/day_mean_(1-7)', 'fatal/day_mean_(8-14)',
'SmokingRate',
]
col_var2 += time_cols
col_var2 += extra_cols
X_train = df_train[col_var2].values
X_valid = df_valid[col_var2].values
y_train = df_train[col_target2].values
y_valid = df_valid[col_target2].values
train_data = lgb.Dataset(X_train, label=y_train)
 valid_data = lgb.Dataset(X_valid, label=y_valid)
 num_round = 15000
model2 = lgb.train(params, train_data, num_round, valid_sets=[train_data, valid_data],
verbose_eval=100,
early_stopping_rounds=150, )<create_dataframe>
np.mean(cross_val_score(first_knn, X_train, y_train, cv=5))
tmp = pd.DataFrame()
tmp["feature"] = col_var2
tmp["importance"] = model2.feature_importance()
tmp = tmp.sort_values('importance', ascending=False)
 important_features = list(tmp[0:15]['feature'])
 col_var2 = important_features
tmp<train_model>
def bernoulli_naive_bayes(X, y):
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3)
 model = BernoulliNB(binarize = 0.01)
 classifier = model.fit(X_train,y_train)
 predict = classifier.predict(X_test)
 cm = confusion_matrix(predict,y_test)
 accuracy = cm.trace() /cm.sum()
acc.append(accuracy)
 ', confusion_matrix(predict,y_test))
', classification_report(predict,y_test))
print('Bernoulli Naive Bayes Classifier')
 acc = []
for i in range(20):
bernoulli_naive_bayes(X, y)
 acc_ = np.array(acc)
 print('Average accuracy in 20 iterations is:', np.average(acc_))
X_train = df_train[col_var2].values
X_valid = df_valid[col_var2].values
y_train = df_train[col_target2].values
y_valid = df_valid[col_target2].values
train_data = lgb.Dataset(X_train, label=y_train)
 valid_data = lgb.Dataset(X_valid, label=y_valid)
 num_round = 15000
model2 = lgb.train(params, train_data, num_round, valid_sets=[train_data, valid_data],
verbose_eval=100,
early_stopping_rounds=150, )<count_duplicates>
def logistic_regression(X,y):
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3)
 model = LogisticRegression(penalty='l2', solver='lbfgs', max_iter=1000)
 classifier = model.fit(X_train,y_train)
 predict = classifier.predict(X_test)
 cm = confusion_matrix(predict,y_test)
 accuracy = cm.trace() /cm.sum()
acc.append(accuracy)
 ', confusion_matrix(predict,y_test))
', classification_report(predict,y_test))
print('Logistic Regression')
 acc = []
for i in range(20):
logistic_regression(X,y)
 acc_ = np.array(acc)
 print('Average accuracy in 20 iterations is:', np.average(acc_))
df_traintest4 = copy.deepcopy(df_traintest3)
 df_traintest4['unique'] = df_traintest4.apply(lambda x: x['place_id'] + str(x['dayofyear']), axis=1)
 print(len(df_traintest4))
df_traintest4 = df_traintest4[df_traintest4['unique'].duplicated() ==False]
print(len(df_traintest4))
df_traintest4[(df_traintest4['place_id']=='China/Hubei')&(df_traintest4['dayofyear']>75)].head()<sort_values>
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3)
 model = LogisticRegression(penalty='l2', solver='lbfgs', max_iter=1000)
 classifier = model.fit(X_train,y_train)
 predict = classifier.predict(X_test)
 cm = confusion_matrix(predict,y_test)
 accuracy = cm.trace() /cm.sum()
df_tmp = df_traintest[pd.isna(df_traintest['Fatalities'])==False]
df_tmp = df_tmp[df_tmp['dayofyear']<61]
df_agg = df_tmp.groupby('place_id')['Fatalities'].agg('max' ).reset_index()
df_agg = df_agg.sort_values('Fatalities', ascending=False)
 df_agg.head()<data_type_conversions>
def write_to_submission_file(predicted_labels, out_file, train_num=105471,
target='loss', index_label="id"):
predicted_df = pd.DataFrame(predicted_labels,
index = np.arange(train_num + 1,
train_num + 1 +
predicted_labels.shape[0]),
columns=[target])
 predicted_df.to_csv(out_file, index_label=index_label )
place = 'China/Hubei'
df_interest_base = df_traintest4[df_traintest4['place_id']==place].reset_index(drop=True)
 df_interest = copy.deepcopy(df_interest_base)
 df_interest['cases/day'] = df_interest['cases/day'].astype(np.float)
 df_interest['fatal/day'] = df_interest['fatal/day'].astype(np.float)
 df_interest['cases/day'][df_interest['dayofyear']>=train_valid_cutoff_dayofyear] = -1
df_interest['fatal/day'][df_interest['dayofyear']>=train_valid_cutoff_dayofyear] = -1
len_known =(df_interest['cases/day']!=-1 ).sum()
len_unknown =(df_interest['cases/day']==-1 ).sum()
print("len train: {}, len prediction: {}".format(len_known, len_unknown))
for i in range(len_unknown):
X_valid = df_interest[col_var].iloc[i+len_known]
X_valid2 = df_interest[col_var2].iloc[i+len_known]
pred_f = model.predict(X_valid)
 pred_c = model2.predict(X_valid2)
 df_interest['fatal/day'][i+len_known] = pred_f
df_interest['cases/day'][i+len_known] = pred_c
df_interest = df_interest[['cases/day', 'fatal/day', 'Long', 'Lat', 'SmokingRate']+time_cols+extra_cols]
df_interest = do_aggregations(df_interest)
 tmp = df_interest_base['fatal/day'].values
tmp = np.cumsum(tmp)
 sns.lineplot(x=df_interest_base['dayofyear'][pd.isna(df_interest_base['Fatalities'])==False],
y=tmp[pd.isna(df_interest_base['Fatalities'])==False], label='true')
 tmp = df_interest['fatal/day'].values
tmp = np.cumsum(tmp)
 sns.lineplot(x=df_interest_base['dayofyear'], y=tmp, label='pred')
 plt.show()<data_type_conversions>
classifier.predict(test_float_train )
place = 'Iran'
df_interest_base = df_traintest4[df_traintest4['place_id']==place].reset_index(drop=True)
 df_interest = copy.deepcopy(df_interest_base)
 df_interest['cases/day'] = df_interest['cases/day'].astype(np.float)
 df_interest['fatal/day'] = df_interest['fatal/day'].astype(np.float)
 df_interest['cases/day'][df_interest['dayofyear']>=train_valid_cutoff_dayofyear] = -1
df_interest['fatal/day'][df_interest['dayofyear']>=train_valid_cutoff_dayofyear] = -1
len_known =(df_interest['cases/day']!=-1 ).sum()
len_unknown =(df_interest['cases/day']==-1 ).sum()
print("len train: {}, len prediction: {}".format(len_known, len_unknown))
for i in range(len_unknown):
X_valid = df_interest[col_var].iloc[i+len_known]
X_valid2 = df_interest[col_var2].iloc[i+len_known]
pred_f = model.predict(X_valid)
 pred_c = model2.predict(X_valid2)
 df_interest['fatal/day'][i+len_known] = pred_f
df_interest['cases/day'][i+len_known] = pred_c
df_interest = df_interest[['cases/day', 'fatal/day', 'Long', 'Lat', 'SmokingRate']+time_cols+extra_cols]
df_interest = do_aggregations(df_interest)
 tmp = df_interest_base['fatal/day'].values
tmp = np.cumsum(tmp)
 sns.lineplot(x=df_interest_base['dayofyear'][pd.isna(df_interest_base['Fatalities'])==False],
y=tmp[pd.isna(df_interest_base['Fatalities'])==False], label='true')
 tmp = df_interest['fatal/day'].values
tmp = np.cumsum(tmp)
 sns.lineplot(x=df_interest_base['dayofyear'], y=tmp, label='pred')
 plt.show()<data_type_conversions>
write_to_submission_file(classifier.predict(test_float_train), out_file="logistic_regression_loan_default_prediction.csv" )
place = 'Italy'
df_interest_base = df_traintest4[df_traintest4['place_id']==place].reset_index(drop=True)
 df_interest = copy.deepcopy(df_interest_base)
 df_interest['cases/day'] = df_interest['cases/day'].astype(np.float)
 df_interest['fatal/day'] = df_interest['fatal/day'].astype(np.float)
 df_interest['cases/day'][df_interest['dayofyear']>=train_valid_cutoff_dayofyear] = -1
df_interest['fatal/day'][df_interest['dayofyear']>=train_valid_cutoff_dayofyear] = -1
len_known =(df_interest['cases/day']!=-1 ).sum()
len_unknown =(df_interest['cases/day']==-1 ).sum()
print("len train: {}, len prediction: {}".format(len_known, len_unknown))
for i in range(len_unknown):
X_valid = df_interest[col_var].iloc[i+len_known]
X_valid2 = df_interest[col_var2].iloc[i+len_known]
pred_f = model.predict(X_valid)
 pred_c = model2.predict(X_valid2)
 df_interest['fatal/day'][i+len_known] = pred_f
df_interest['cases/day'][i+len_known] = pred_c
df_interest = df_interest[['cases/day', 'fatal/day', 'Long', 'Lat', 'SmokingRate']+time_cols+extra_cols]
df_interest = do_aggregations(df_interest)
 tmp = df_interest_base['fatal/day'].values
tmp = np.cumsum(tmp)
 sns.lineplot(x=df_interest_base['dayofyear'][pd.isna(df_interest_base['Fatalities'])==False],
y=tmp[pd.isna(df_interest_base['Fatalities'])==False], label='true')
 tmp = df_interest['fatal/day'].values
tmp = np.cumsum(tmp)
 sns.lineplot(x=df_interest_base['dayofyear'], y=tmp, label='pred')
 plt.show()<prepare_x_and_y>
df_train = df_traintest3[(pd.isna(df_traintest3['ForecastId'])) ]
X_train = df_train[col_var].values
X_valid = df_train[col_var].values
y_train = df_train[col_target].values
y_valid = df_train[col_target].values
train_data = lgb.Dataset(X_train, label=y_train)
 valid_data = lgb.Dataset(X_valid, label=y_valid)
 num_round = 575
model = lgb.train(params, train_data, num_round, valid_sets=[train_data, valid_data], verbose_eval=100, )<prepare_x_and_y>
X= df.select_dtypes(include=[np.int64] ).drop(['loss', 'loss_fact'], axis=1)
 y= df['loss_fact']
df_train = df_traintest3[(pd.isna(df_traintest3['ForecastId'])) ]
X_train = df_train[col_var2].values
X_valid = df_train[col_var2].values
y_train = df_train[col_target2].values
y_valid = df_train[col_target2].values
train_data = lgb.Dataset(X_train, label=y_train)
 valid_data = lgb.Dataset(X_valid, label=y_valid)
 num_round = 225
model2 = lgb.train(params, train_data, num_round, valid_sets=[train_data, valid_data], verbose_eval=100, )<filter>
X_train, X_valid, y_train, y_valid = train_test_split(X, y, test_size=0.3, random_state=17 )
df_traintest4[(df_traintest4['place_id']=='China/Hubei')&(df_traintest4['dayofyear']>=72)]<data_type_conversions>
def bernoulli_naive_bayes(X, y):
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3)
 model = BernoulliNB(binarize = 0.01)
 classifier = model.fit(X_train,y_train)
 predict = classifier.predict(X_test)
 cm = confusion_matrix(predict,y_test)
 accuracy = cm.trace() /cm.sum()
acc.append(accuracy)
 ', confusion_matrix(predict,y_test))
', classification_report(predict,y_test))
print('Bernoulli Naive Bayes Classifier')
 acc = []
for i in range(20):
bernoulli_naive_bayes(X, y)
 acc_ = np.array(acc)
 print('Average accuracy in 20 iterations is:', np.average(acc_))
df_preds = []
for i, place in enumerate(places[:]):
df_interest = copy.deepcopy(df_traintest4[df_traintest4['place_id']==place].reset_index(drop=True))
df_interest['cases/day'] = df_interest['cases/day'].astype(np.float)
 df_interest['fatal/day'] = df_interest['fatal/day'].astype(np.float)
 df_interest['cases/day'][(pd.isna(df_interest['ForecastId'])) ==False] = -1
df_interest['fatal/day'][(pd.isna(df_interest['ForecastId'])) ==False] = -1
len_known =(df_interest['cases/day']!=-1 ).sum()
len_unknown =(df_interest['cases/day']==-1 ).sum()
if(i+1)%10==0:
print("{:3d}/{} {}, len known: {}, len unknown: {}".format(i+1, len(places), place, len_known, len_unknown), df_interest.shape)
 for j in range(len_unknown):
X_valid = df_interest[col_var].iloc[j+len_known]
X_valid2 = df_interest[col_var2].iloc[j+len_known]
pred_f = model.predict(X_valid)
 pred_c = model2.predict(X_valid2)
 df_interest['fatal/day'][j+len_known] = pred_f
df_interest['cases/day'][j+len_known] = pred_c
df_interest = df_interest[['cases/day', 'fatal/day', 'Long', 'Lat', 'SmokingRate', 'ForecastId', 'place_id']+time_cols+extra_cols]
df_interest = do_aggregations(df_interest)
 df_interest['fatal_pred'] = np.cumsum(df_interest['fatal/day'].values)
 df_interest['cases_pred'] = np.cumsum(df_interest['cases/day'].values)
 df_preds.append(df_interest )<concatenate>
def logistic_regression(X,y):
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3)
 model = LogisticRegression(penalty='l2', solver='sag', max_iter=1000)
 classifier = model.fit(X_train,y_train)
 predict = classifier.predict(X_test)
 cm = confusion_matrix(predict,y_test)
 accuracy = cm.trace() /cm.sum()
acc.append(accuracy)
 ', confusion_matrix(predict,y_test))
', classification_report(predict,y_test))
print('Logistic Regression')
 acc = []
for i in range(20):
logistic_regression(X,y)
 acc_ = np.array(acc)
 print('Average accuracy in 20 iterations is:', np.average(acc_))
df_preds= pd.concat(df_preds)
 df_preds = df_preds.sort_values('dayofyear')
 col_tmp = ['place_id', 'ForecastId', 'dayofyear', 'cases/day', 'cases_pred', 'fatal/day', 'fatal_pred',]
df_preds[col_tmp][(df_preds['place_id']=='Afghanistan')&(df_preds['dayofyear']>75)].head(10 )<load_from_csv>
first_tree = DecisionTreeClassifier(random_state=17 )
df_sub = pd.read_csv(".. /input/covid19-global-forecasting-week-2/submission.csv")
 print(len(df_sub))
df_sub.head()<merge>
np.mean(cross_val_score(first_tree, X_train, y_train, cv=5))
df_sub = pd.merge(df_sub, df_traintest3[['ForecastId', 'place_id', 'dayofyear']])
 df_sub = pd.merge(df_sub, df_preds[['place_id', 'dayofyear', 'cases_pred', 'fatal_pred']], on=['place_id', 'dayofyear',], how='left')
 df_sub.head(10 )<save_to_csv>
first_knn = KNeighborsClassifier()
df_sub['ConfirmedCases'] = df_sub['cases_pred']
df_sub['Fatalities'] = df_sub['fatal_pred']
df_sub = df_sub[['ForecastId', 'ConfirmedCases', 'Fatalities']]
df_sub.to_csv("submission.csv", index=None)
 df_sub.head(10 )<load_from_csv>
np.mean(cross_val_score(first_knn, X_train, y_train, cv=5))
def get_nn_sub() :
df = pd.read_csv(".. /input/covid19-global-forecasting-week-2/train.csv")
 sub_df = pd.read_csv(".. /input/covid19-global-forecasting-week-2/test.csv")
 coo_df = pd.read_csv(".. /input/covid19week1/train.csv" ).rename(columns={"Country/Region": "Country_Region"})
 coo_df = coo_df.groupby("Country_Region")[["Lat", "Long"]].mean().reset_index()
coo_df = coo_df[coo_df["Country_Region"].notnull() ]
loc_group = ["Province_State", "Country_Region"]
def preprocess(df):
df["Date"] = df["Date"].astype("datetime64[ms]")
 df["days"] =(df["Date"] - pd.to_datetime("2020-01-01")).dt.days
df["weekend"] = df["Date"].dt.dayofweek//5
df = df.merge(coo_df, how="left", on="Country_Region")
 df["Lat"] =(df["Lat"] // 30 ).astype(np.float32 ).fillna(0)
 df["Long"] =(df["Long"] // 60 ).astype(np.float32 ).fillna(0)
 for col in loc_group:
df[col].fillna("none", inplace=True)
 return df
df = preprocess(df)
 sub_df = preprocess(sub_df)
 print(df.shape)
 TARGETS = ["ConfirmedCases", "Fatalities"]
for col in TARGETS:
df[col] = np.log1p(df[col])
 NUM_SHIFT = 5
features = ["Lat", "Long"]
for s in range(1, NUM_SHIFT+1):
for col in TARGETS:
df["prev_{}_{}".format(col, s)] = df.groupby(loc_group)[col].shift(s)
 features.append("prev_{}_{}".format(col, s))
df = df[df["Date"] >= df["Date"].min() + timedelta(days=NUM_SHIFT)].copy()
TEST_FIRST = sub_df["Date"].min()
TEST_DAYS =(df["Date"].max() - TEST_FIRST ).days + 1
dev_df, test_df = df[df["Date"] < TEST_FIRST].copy() , df[df["Date"] >= TEST_FIRST].copy()
def nn_block(input_layer, size, dropout_rate, activation):
out_layer = KL.Dense(size, activation=None )(input_layer)
 out_layer = KL.Activation(activation )(out_layer)
 out_layer = KL.Dropout(dropout_rate )(out_layer)
 return out_layer
def get_model() :
inp = KL.Input(shape=(len(features),))
hidden_layer = nn_block(inp, 64, 0.0, "relu")
 gate_layer = nn_block(hidden_layer, 32, 0.0, "sigmoid")
 hidden_layer = nn_block(hidden_layer, 32, 0.0, "relu")
 hidden_layer = KL.multiply([hidden_layer, gate_layer])
 out = KL.Dense(len(TARGETS), activation="linear" )(hidden_layer)
 model = tf.keras.models.Model(inputs=[inp], outputs=out)
 return model
get_model().summary()
def get_input(df):
return [df[features]]
NUM_MODELS = 10
def train_models(df, save=False):
models = []
for i in range(NUM_MODELS):
model = get_model()
model.compile(loss="mean_squared_error", optimizer=Nadam(lr=1e-4))
hist = model.fit(get_input(df), df[TARGETS],
batch_size=2048, epochs=500, verbose=0, shuffle=True)
 if save:
model.save_weights("model{}.h5".format(i))
models.append(model)
 return models
models = train_models(dev_df)
 prev_targets = ['prev_ConfirmedCases_1', 'prev_Fatalities_1']
def predict_one(df, models):
pred = np.zeros(( df.shape[0], 2))
for model in models:
pred += model.predict(get_input(df)) /len(models)
 pred = np.maximum(pred, df[prev_targets].values)
 pred[:, 0] = np.log1p(np.expm1(pred[:, 0])+ 0.1)
 pred[:, 1] = np.log1p(np.expm1(pred[:, 1])+ 0.01)
 return np.clip(pred, None, 15)
 print([mean_squared_error(dev_df[TARGETS[i]], predict_one(dev_df, models)[:, i])for i in range(len(TARGETS)) ])
 def rmse(y_true, y_pred):
return np.sqrt(mean_squared_error(y_true, y_pred))
def evaluate(df):
error = 0
for col in TARGETS:
error += rmse(df[col].values, df["pred_{}".format(col)].values)
 return np.round(error/len(TARGETS), 5)
 def predict(test_df, first_day, num_days, models, val=False):
temp_df = test_df.loc[test_df["Date"] == first_day].copy()
y_pred = predict_one(temp_df, models)
 for i, col in enumerate(TARGETS):
test_df["pred_{}".format(col)] = 0
test_df.loc[test_df["Date"] == first_day, "pred_{}".format(col)] = y_pred[:, i]
print(first_day, np.isnan(y_pred ).sum() , y_pred.min() , y_pred.max())
 if val:
print(evaluate(test_df[test_df["Date"] == first_day]))
y_prevs = [None]*NUM_SHIFT
for i in range(1, NUM_SHIFT):
y_prevs[i] = temp_df[['prev_ConfirmedCases_{}'.format(i), 'prev_Fatalities_{}'.format(i)]].values
for d in range(1, num_days):
date = first_day + timedelta(days=d)
 print(date, np.isnan(y_pred ).sum() , y_pred.min() , y_pred.max())
 temp_df = test_df.loc[test_df["Date"] == date].copy()
temp_df[prev_targets] = y_pred
for i in range(2, NUM_SHIFT+1):
temp_df[['prev_ConfirmedCases_{}'.format(i), 'prev_Fatalities_{}'.format(i)]] = y_prevs[i-1]
y_pred, y_prevs = predict_one(temp_df, models), [None, y_pred] + y_prevs[1:-1]
for i, col in enumerate(TARGETS):
test_df.loc[test_df["Date"] == date, "pred_{}".format(col)] = y_pred[:, i]
if val:
print(evaluate(test_df[test_df["Date"] == date]))
return test_df
test_df = predict(test_df, TEST_FIRST, TEST_DAYS, models, val=True)
 print(evaluate(test_df))
for col in TARGETS:
test_df[col] = np.expm1(test_df[col])
 test_df["pred_{}".format(col)] = np.expm1(test_df["pred_{}".format(col)])
 models = train_models(df, save=True)
 sub_df_public = sub_df[sub_df["Date"] <= df["Date"].max() ].copy()
sub_df_private = sub_df[sub_df["Date"] > df["Date"].max() ].copy()
pred_cols = ["pred_{}".format(col)for col in TARGETS]
sub_df_public = sub_df_public.merge(test_df[["Date"] + loc_group + TARGETS], how="left", on=["Date"] + loc_group)
 SUB_FIRST = sub_df_private["Date"].min()
SUB_DAYS =(sub_df_private["Date"].max() - sub_df_private["Date"].min() ).days + 1
sub_df_private = df.append(sub_df_private, sort=False)
 for s in range(1, NUM_SHIFT+1):
for col in TARGETS:
sub_df_private["prev_{}_{}".format(col, s)] = sub_df_private.groupby(loc_group)[col].shift(s)
 sub_df_private = sub_df_private[sub_df_private["Date"] >= SUB_FIRST].copy()
sub_df_private = predict(sub_df_private, SUB_FIRST, SUB_DAYS, models)
 for col in TARGETS:
sub_df_private[col] = np.expm1(sub_df_private["pred_{}".format(col)])
 sub_df = sub_df_public.append(sub_df_private, sort=False)
 sub_df["ForecastId"] = sub_df["ForecastId"].astype(np.int16)
 return sub_df[["ForecastId"] + TARGETS]<save_to_csv>
tree_params = {'max_depth': np.arange(1,11), 'max_features': [.5,.7,1]}
sub_df = get_nn_sub()
sub_df.to_csv("submission.csv", index=False )<import_modules>
tree_grid = GridSearchCV(first_tree, tree_params, cv=5, n_jobs=-1 )
import matplotlib.pyplot as plt
from typing import Tuple<define_variables>
%%time
tree_grid.fit(X_train, y_train )
fname_train = '.. /input/covid19-global-forecasting-week-2/train.csv'
fname_test = '.. /input/covid19-global-forecasting-week-2/test.csv'
fname_sub = '.. /input/covid19-global-forecasting-week-2/submission.csv'<load_from_csv>
tree_grid.best_score_
df_train = pd.read_csv(fname_train,parse_dates=True)
 df_test = pd.read_csv(fname_test, parse_dates=True)
 df_sub = pd.read_csv(fname_sub )<define_variables>
tree_grid.best_params_
START_TRAIN_DATE = df_train['Date'].min()
END_TRAIN_DATE = df_train['Date'].max()
START_PREDICTION_DATE = df_test['Date'].min()
END_PREDICTION_DATE = df_test['Date'].max()<load_from_csv>
knn_params = {'n_neighbors':range(10,30,1)}
fname_pop = '.. /input/population-by-country-2020/population_by_country_2020.csv'
df_pop = pd.read_csv(fname_pop )<define_variables>
knn_grid = GridSearchCV(first_knn, knn_params, cv=5 )
GLOBAL_FILL_STRING = 'not_provided'<define_variables>
%%time
knn_grid.fit(X_train, y_train )
prediction_scores = {}<compute_test_metric>
knn_grid.best_score_, knn_grid.best_params_
def rmsle(y, y_pred):
assert len(y)== len(y_pred)
 terms_to_sum = [(math.log(y_pred[i] + 1)- math.log(y[i] + 1)) ** 2.0 for i,pred in enumerate(y_pred)]
return(sum(terms_to_sum)*(1.0/len(y)))** 0.5<categorify>
tree_grid.best_estimator_
golden_data = {'DateTime':pd.to_datetime(['2020-06-01','2020-06-02','2020-06-03']),
'ConfirmedCases':[2,4,8],
'Fatalities':[0,0,1],
'Province_State':['PS_TBD','PS_TBD','PS_TBD'],
'Country_Region':['CR_TBD','CR_TBD','CR_TBD'],
'Id':[201,202,203],
'SimpleName':['TBD','TBD','TBD']}
predicted_data = {'DateTime':pd.to_datetime(['2020-06-01','2020-06-02','2020-06-03','2020-06-04','2020-06-05']),
'ConfirmedCases':[2,4,6,8,10],
'Fatalities':[0,0,0,1,2],
'SimpleName':['TBD','TBD','TBD','TBD','TBD']}
df_golden = pd.DataFrame.from_dict(golden_data)
 df_predicted = pd.DataFrame.from_dict(predicted_data)
 unit_c, unit_f = evaluate_predictions(df_golden,df_predicted,1,2,'unit test',False)
 assert(( unit_c == 0.1777)&(unit_f == 0.4901)) ,'Failed unit test for evaluate_predictions'<data_type_conversions>
tree_grid.predict(X_valid )
df_train[['Province_State','Country_Region']] = df_train[['Province_State','Country_Region']].fillna(GLOBAL_FILL_STRING)
 df_test[['Province_State','Country_Region']] = df_test[['Province_State','Country_Region']].fillna(GLOBAL_FILL_STRING )<feature_engineering>
tree_valid_pred = tree_grid.predict(X_valid )
df_train['SimpleName'] = df_train['Province_State'] + '__' + df_train['Country_Region']
df_test['SimpleName'] = df_test['Province_State'] + '__' + df_test['Country_Region']<data_type_conversions>
tree_grid.score(X_valid, y_valid )
df_train['DateTime'] = pd.to_datetime(df_train['Date'])
 df_test['DateTime'] = pd.to_datetime(df_test['Date'] )<count_unique_values>
1 - np.mean(y )
df_train['SimpleName'].nunique()<define_variables>
export_graphviz(tree_grid.best_estimator_, out_file='loan_tree.dot', feature_names=X.columns, filled=True )
<compute_test_metric><EOS> 
!dot -Tpng loan_tree.dot -o loan_tree.png