Kaggle competition: Loan Default Prediction - Imperial College London
<SOS> metric: MAE Kaggle data source: loan-default-prediction-imperial-college-london<create_dataframe>
warnings.filterwarnings("ignore")
parameter_data = create_parameter_database()
db_parameters = pd.DataFrame(parameter_data, index=sname_set,\
columns=['beta','gamma','ifx','rfx','population'] )<define_variables>
df = pd.read_csv('.. /input/loan-default-prediction/train_v2.csv.zip')
 df.head()
ctry_col = r'Country(or dependency)'
pop_col = r'Population(2020)'<filter>
df[df.duplicated() ].shape
df_pop[df_pop[ctry_col] == 'France'][pop_col].values[0]<filter>
df.dtypes.value_counts()
for index,row in db_parameters.iterrows() :
state, ctry = index.split('__')
 if(state == GLOBAL_FILL_STRING):
try:
pop = df_pop[df_pop[ctry_col] == ctry][pop_col].values[0]
except:
if(ctry == 'Czechia'):
pop = df_pop[df_pop[ctry_col] == r'Czech Republic(Czechia)'][pop_col].values[0]
elif(ctry == r'Congo(Kinshasa)'):
pop = df_pop[df_pop[ctry_col] == r'DR Congo'][pop_col].values[0]
elif(ctry == r'Congo(Brazzaville)'):
pop = df_pop[df_pop[ctry_col] == r'Congo'][pop_col].values[0]
elif(ctry == r'Saint Vincent and the Grenadines'):
pop = df_pop[df_pop[ctry_col] == r'St.Vincent & Grenadines'][pop_col].values[0]
elif(ctry == r'Korea, South'):
pop = df_pop[df_pop[ctry_col] == r'South Korea'][pop_col].values[0]
elif(ctry == r"Cote d'Ivoire"):
pop = df_pop[df_pop[ctry_col] == r"Côte d'Ivoire"][pop_col].values[0]
elif(ctry == r'Saint Kitts and Nevis'):
pop = df_pop[df_pop[ctry_col] == r'Saint Kitts & Nevis'][pop_col].values[0]
elif(ctry == r'Taiwan*'):
pop = df_pop[df_pop[ctry_col] == r'Taiwan'][pop_col].values[0]
elif(ctry == r'Diamond Princess'):
pop = 3711
else:
pop=500000
print("No population data available for {}, {}".format(state,ctry))
db_parameters.loc[index,'population'] = pop
else:
print("State missing population data: {}, {}".format(state,ctry))<feature_engineering>
df['num_missing'] = df.isnull().sum(axis = 1)/df.shape[1]
missing_row = df[df['num_missing'] > 0.20].index
df.drop(df.index[missing_row], inplace = True)
 df.shape
col_len = df_test.shape[0]
zeros_column = np.zeros(( col_len,))
df_test['ConfirmedCases'] = zeros_column
df_test['Fatalities'] = zeros_column
df_test['maxConfirmedCases'] = zeros_column<create_dataframe>
df.drop(columns = ['id','num_missing'], inplace = True )
df_train_preserve = df_train.copy()
df_test_preserve = df_test.copy()<categorify>
col_pct_miss = []
for col in df.columns:
percent_miss = np.mean(df[col].isnull())*100
if percent_miss > 0:
col_pct_miss.append([col, percent_miss])
 col_pct_miss_df = pd.DataFrame(col_pct_miss, columns = ['column_name','% of Missing'] ).sort_values(by = '% of Missing', ascending = False)
 col_pct_miss_df
def prepare_prediction_template(df_in,withhold=0,extend_to_date = END_PREDICTION_DATE,
verbose=False):

 withhold = abs(withhold)
 if(withhold != 0):
wth_str = str(withhold)+ 'D'
last_date = df_in['DateTime'].max() - pd.to_timedelta(wth_str)
 df_local = df_in[df_in['DateTime']<=last_date].copy()
if(verbose):
print("pptemp withhold={} so last_date to copy was {}".format(withhold,last_date))
else:
df_local = df_in.copy()
if(verbose):
print("pptemp withhold=0 so made complete copy before extension.")
 keep_ps = df_in.iloc[1]['Province_State']
keep_cr = df_in.iloc[1]['Country_Region']
keep_id = df_in.iloc[1]['Id']
keep_sn = df_in.iloc[1]['SimpleName']
max_date = df_local['DateTime'].max()
num_extensions =(pd.to_datetime(extend_to_date)- max_date ).days
if(verbose):
print("Max date was {} so we need {} extensions.".format(max_date,num_extensions))
next_date = max_date + pd.to_timedelta('1D')
 end_date_datetime = pd.to_datetime(extend_to_date)
 while(next_date <= end_date_datetime):
df_local = df_local.append({'Id':0,
'Province_State':keep_ps,
'Country_Region':keep_cr,
'DateTime':next_date,
'ConfirmedCases':0,
'maxConfirmedCases':0,
'Fatalities':0,
'SimpleName':keep_sn},
ignore_index=True)
 next_date = next_date + pd.to_timedelta('1D')
 return df_local,num_extensions<define_variables>
numeric_cols = df.select_dtypes(include=['number'] ).columns.values
for col in numeric_cols:
if col in list(col_pct_miss_df.column_name):
med = df[col].median()
df[col] = df[col].fillna(med )
df_1, num_e = prepare_prediction_template(df_golden,withhold=1,extend_to_date='2020-06-09',verbose=True)
 print("Output num_e={} and df is
{}".format(num_e,df_1))<define_variables>
not_numeric_cols = df.select_dtypes(exclude=['number'] ).columns.values
for col in not_numeric_cols:
if col in list(col_pct_miss_df.column_name):
mode = df[col].mode() [0]
df[col] = df[col].fillna(mode )
mdates = pd.date_range('2020-03-19', '2020-04-30', freq='D')
 date_list = list(mdates.strftime('%Y-%m-%d'))
date_list2 = ['2020-03-19','2020-03-20','2020-03-21','2020-03-22','2020-03-23','2020-03-24','2020-03-25',\
'2020-03-26','2020-03-27',\
'2020-03-28','2020-03-29','2020-03-30','2020-03-31','2020-04-01','2020-04-02','2020-04-03','2020-04-04',\
'2020-04-05','2020-04-06','2020-04-07','2020-04-08','2020-04-09','2020-04-10','2020-04-11','2020-04-12',\
'2020-04-13','2020-04-14','2020-04-15','2020-04-16','2020-04-17','2020-04-18','2020-04-19','2020-04-20',\
'2020-04-21','2020-04-22','2020-04-23','2020-04-24','2020-04-24','2020-04-25','2020-04-26','2020-04-27',\
'2020-04-28','2020-04-29','2020-04-30'
]
def copy_preditions_to_final_list(location, df_predictions, df_final_list,verbose=False):

 if(verbose):
print("Copying predictions for {}:".format(location))
assert(100==df_predictions.shape[0]),'Error 1 - Passed in an incorrect predictions DataFrame'
df_small = df_predictions[df_predictions['SimpleName'] == location]
assert(df_small.shape[0]==df_predictions.shape[0]),'Error 2 - Passed in an incorrect predictions DataFrame'
for date1 in date_list:
date1_dt = pd.to_datetime(date1)
 mask_location = df_final_list['SimpleName'] == location
mask_date = df_final_list['DateTime'] == date1_dt
idx_final = df_final_list[mask_location & mask_date].index[0]
mask_date2 = df_small['DateTime'] == date1_dt
idx_small = df_small[mask_date2].index[0]
if(verbose):
print('idx: final={} small={}'.format(idx_final, idx_small))
df_final_list.at[idx_final,'ConfirmedCases'] = df_small.at[idx_small,'ConfirmedCases']
df_final_list.at[idx_final,'maxConfirmedCases'] = df_small.at[idx_small,'maxConfirmedCases']
df_final_list.at[idx_final,'Fatalities'] = df_small.at[idx_small,'Fatalities'] 
<define_variables>
df.isnull().sum().value_counts()
def single_model(location, practice_withhold=0, verbose=False):
this_name = location
df_sname = df_train[df_train['SimpleName'] == this_name].copy()
df_sname['maxConfirmedCases'] = 0
df_sname_template,num_extended = prepare_prediction_template(df_sname,(-1*practice_withhold),verbose=verbose)
 df_sname.head()
this_beta, this_gamma, this_ifx, this_rfx, this_pop = db_parameters.loc[this_name]
df_SIR_predictions = sir_model(df_sname_template,N=this_pop,\
beta=this_beta,gamma=this_gamma,\
i_factor = this_ifx, r_factor = this_rfx)
 if(verbose):
starter = practice_withhold + num_extended
print("ends to compare are
{}
***AND***
{}".format(df_sname,
df_SIR_predictions.iloc[:(-1*num_extended)]))
rmsle_c, rmsle_f = evaluate_predictions(df_sname,df_SIR_predictions,
practice_withhold,num_extended,
'simple SIR',verbose)
 rmsle_c2, rmsle_f2 = evaluate_predictions2(df_sname,df_SIR_predictions,
practice_withhold,num_extended,
'simple SIR',verbose)
 plot_title = this_name + " v1 SIR plots"
print("Finished evaluating...{}: {} {}".format(this_name,rmsle_c,rmsle_f))
print("Finished evaluating 2...{}: {} {}".format(this_name,rmsle_c2,rmsle_f2))
copy_preditions_to_final_list(this_name, df_SIR_predictions, df_test)
 prediction_scores[this_name] = [rmsle_c2, rmsle_f2]
return df_SIR_predictions, df_sname, plot_title<feature_engineering>
corr_matrix = df.corr().abs()
upper = corr_matrix.where(np.triu(np.ones(corr_matrix.shape), k=1 ).astype(np.bool))
to_drop = [column for column in upper.columns if any(upper[column] > 0.95)]
df.drop(columns = to_drop, inplace = True )
practice_withhold = 7
verbose = True
this_name = 'Hubei__China'
df_sname = df_train[df_train['SimpleName'] == this_name].copy()
df_sname['maxConfirmedCases'] = 0
df_sname_template,num_extended = prepare_prediction_template(df_sname,(-1*practice_withhold),verbose=verbose)
 df_sname.head()
this_beta, this_gamma, this_ifx, this_rfx, this_pop = db_parameters.loc[this_name]
df_SIR_predictions = sir_model(df_sname_template,N=this_pop,\
beta=this_beta,gamma=this_gamma,\
i_factor = this_ifx, r_factor = this_rfx )<predict_on_test>
num_rows = df.shape[0]
rep_cols = []
for col in df.loc[:, df.columns != 'loss'].columns :
cnts = df[col].value_counts()
top_pct =(cnts/num_rows ).iloc[0]
if top_pct > 0.80:
rep_cols.append([col,top_pct])
 rep_col_df = pd.DataFrame(rep_cols, columns = ['column_name','% top repetitve value'] ).sort_values(by = '% top repetitve value', ascending = False ).reset_index(drop=True)
 rep_col_df
df.shape
rmsle_c, rmsle_f = evaluate_predictions(df_sname,df_SIR_predictions,
practice_withhold,num_extended,
'simple SIR',verbose)
 rmsle_c2, rmsle_f2 = evaluate_predictions2(df_sname,df_SIR_predictions,
practice_withhold,num_extended,
'simple SIR',verbose)
 plot_title = this_name + " v1 SIR plots"
print("Finished evaluating...{}: {} {}".format(this_name,rmsle_c,rmsle_f))
print("Finished evaluating 2...{}: {} {}".format(this_name,rmsle_c2,rmsle_f2))<filter>
cat_cols = df.select_dtypes(exclude=['number'] ).columns.values
drop_cols = []
keep_cols = []
for col in cat_cols:
if df[col].value_counts().count() > 20000 :
print('column {} has {} categories > drop'.format(col,df[col].value_counts().count()))
drop_cols.append(col)
 else :
print('column {} has {} categories > keep'.format(col,df[col].value_counts().count()))
keep_cols.append(col )
df_sname = df_train[df_train['SimpleName'] == 'not_provided__Germany']<define_variables>
encoder = ce.BinaryEncoder(cols = keep_cols)
 bi_enc_df = encoder.fit_transform(df[keep_cols])
 bi_col_name = bi_enc_df.columns
bi_enc_df.head()
df = pd.concat([df,bi_enc_df],axis = 1)
 df.head()
df_sname_template_1,num_extended = prepare_prediction_template(df_sname,-5 )<find_best_params>
df['loan_status'] = np.where(df['loss'] > 0, 1, 0)
 df.head()
df_SIR_practice = sir_model(df_sname_template_1 )<define_variables>
not_default = df[df['loan_status'] == 0]
default = df[df['loan_status'] == 1]
resample_df_d = resample(default,
replace = False,
n_samples = 9778,
random_state = 1234)
 resample_df_n = resample(not_default,
replace = False,
n_samples = 9778,
random_state = 1234)
 resample_df = pd.concat([resample_df_n, resample_df_d] )
selected_country_region = "US"
selected_province_state = "Washington"
sname = selected_province_state + '__' + selected_country_region
print("Sanity check for {}".format(sname))<filter>
X = resample_df.drop(columns = ['loss','loan_status'])
 Y = resample_df['loss']
X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size = 0.2, random_state = 1234, stratify = resample_df['loan_status'])
 print('training set = {} records, test set= {} records'.format(X_train.shape[0],X_test.shape[0]))
db_parameters.loc['Isle of Man__United Kingdom']<define_variables>
from sklearn.feature_selection import SelectPercentile , SelectKBest, f_regression , f_classif
sname_set
mini_set = ['not_provided__Togo',
'not_provided__Cambodia',
'Connecticut__US',
'Jilin__China',
'Greenland__Denmark',
'Maine__US',
'Pennsylvania__US',
'Quebec__Canada',
'New Jersey__US',
'Manitoba__Canada',
'not_provided__Haiti']<create_dataframe>
X_train_num = X_train.drop(columns = bi_col_name)
 selector = SelectKBest(score_func = f_regression, k = 170)
 selector.fit(X_train_num,Y_train)
 select_cols = selector.get_support(indices = True)
 select_num_cols = X_train_num.iloc[:,select_cols]
select_num_col_name = select_num_cols.columns
select_num_cols.head()
df_train = df_train_preserve.copy()
df_test = df_test_preserve.copy()<data_type_conversions>
selector = RFE(LogisticRegression() , n_features_to_select=150, step=1, verbose = 2)
 selector = selector.fit(select_num_cols, Y_train)
 select_cols = selector.get_support(indices = True)
 select_cols_df = select_num_cols.iloc[:,select_cols]
best_X_col_name = select_cols_df.columns
select_cols_df.head()
last_day = df_test[df_test['DateTime'] == pd.to_datetime('2020-04-30')]
print("Final prediction for 2020-04-30 has {} ConfirmedCases and {} Fatalities".format(last_day['ConfirmedCases'].sum() ,\
last_day['Fatalities'].sum()))<create_dataframe>
X_train_cat = X_train.select_dtypes(exclude = 'number' ).copy()
le = LabelEncoder()
X_train_cat = X_train_cat.apply(lambda col: le.fit_transform(col.astype(str)) , axis=0, result_type='expand')
 oe = OrdinalEncoder()
oe.fit(X_train_cat)
 X_train_cat_enc = oe.transform(X_train_cat)
 selector = SelectKBest(score_func = f_classif , k=5)
 selector.fit(X_train_cat_enc,Y_train)
 select_cols = selector.get_support(indices = True)
 select_cat_cols = X_train_cat.iloc[:,select_cols]
select_cat_col_name = select_cat_cols.columns
select_cat_cols.head()
df_scores = pd.DataFrame.from_dict(prediction_scores,orient='index', columns=['cc_score','f_score'] )<drop_column>
def filter_x_df(x):
df = x.copy()
all_filter_col = []
for keep in select_cat_col_name[select_cat_col_name.isin(keep_cols)]:
filter_col = [col for col in df.columns if col.startswith(str(keep)) ]
for col in filter_col :
if col not in keep_cols:
all_filter_col.append(col)
 drop_cat_df = df.drop(columns = cat_cols)
 new_df = pd.concat([drop_cat_df[best_X_col_name],drop_cat_df[all_filter_col]],axis = 1)
 return new_df
df_sub = df_test.drop(['Province_State','SimpleName','Country_Region','Date','DateTime'],axis=1 )<drop_column>
filter_X_train = filter_x_df(X_train)
 filter_X_test = filter_x_df(X_test )
df_sub_mod = df_sub.drop('ConfirmedCases', axis=1 )<rename_columns>
scaler = StandardScaler()
scaler.fit(filter_X_train)
 X_train_scal = scaler.fit_transform(filter_X_train)
 X_test_scal = scaler.fit_transform(filter_X_test )
df_sub_mod.columns = ['ForecastId','Fatalities','ConfirmedCases']
df_sub_mod.head()<prepare_output>
neigh = KNeighborsClassifier()
neigh.fit(X_train_scal, Y_train)
 Knn_y_pred_train = neigh.predict(X_train_scal)
 Knn_y_pred_test = neigh.predict(X_test_scal)
 scores_kn = cross_val_score(estimator = neigh, y = Y_train, X = X_train_scal, cv=5)
 print('Cross Validation Score:', np.mean(scores_kn))
df_sub = df_sub_mod<save_to_csv>
logisticRegr = LogisticRegression()
logisticRegr = logisticRegr.fit(X_train_scal, Y_train)
 Lr_y_pred_train = logisticRegr.predict(X_train_scal)
 Lr_y_pred_test = logisticRegr.predict(X_test_scal)
 scores_lr = cross_val_score(estimator = logisticRegr, y = Y_train, X = X_train_scal, cv=5)
 print('Cross Validation Score:', np.mean(scores_lr))
df_sub.to_csv('submission.csv',index=False)
<install_modules>
rf = RandomForestClassifier(max_depth = 70)
 rf.fit(filter_X_train, Y_train)
 rf_y_pred_train = rf.predict(filter_X_train)
 rf_y_pred_test = rf.predict(filter_X_test)
 scores_rf = cross_val_score(estimator = rf, y = Y_train, X = filter_X_train, cv=5)
 print('Cross Validation Score:', np.mean(scores_rf))
! pip install segmentation_models_pytorch albumentations
! pip install -U git+https://github.com/albu/albumentations --no-cache-dir 
<import_modules>
xgb = XGBClassifier(gamma=0, learning_rate=0.1, max_depth=100, n_estimators=100)
 xgb.fit(filter_X_train,Y_train)
 xgb_y_pred_train = xgb.predict(filter_X_train)
 xgb_y_pred_test = xgb.predict(filter_X_test)
 scores_xg = cross_val_score(estimator = rf, y = Y_train, X = filter_X_train, cv=5)
 print('Cross Validation Score:', np.mean(scores_xg))
import os
import csv
import re
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import itertools
import cv2
from tqdm.notebook import tqdm
from glob import glob
from PIL import Image
from skimage.transform import resize
from sklearn.model_selection import train_test_split, KFold
import shutil
import torch
from torch.utils.data import DataLoader,Dataset
import torch.nn as nn
import albumentations
import torchvision
from torchvision import transforms, models
import random
import segmentation_models_pytorch as smp
from PIL import Image<split>
df_sm = df.drop(columns = cat_cols)
 smote = SMOTE(random_state = 1234)
 X_sm, Y_sm = smote.fit_resample(df_sm.drop(columns = 'loan_status'), df_sm['loan_status'])
 print("After OverSampling, counts of label '1': {}".format(sum(Y_sm == 1)))
 print("After OverSampling, counts of label '0': {}".format(sum(Y_sm == 0)) )
ROOT = "/kaggle/input/ultrasound-nerve-segmentation/"
trainpath = "/kaggle/input/ultrasound-nerve-segmentation/train/"
testpath = "/kaggle/input/ultrasound-nerve-segmentation/test/"
masks = [os.path.join(trainpath,i)for i in os.listdir(trainpath)if "mask" in i]
imgs = [i.replace("_mask","")for i in masks]
df = pd.DataFrame({"image":imgs,"mask":masks})
 df_train, df_val = train_test_split(df,test_size = 0.15)
 print(df_train.values.shape)
 print(df_val.values.shape )<categorify>
Y = X_sm['loss']
X = X_sm.drop(columns = 'loss')
 X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size = 0.2, random_state = 1234, stratify = Y_sm)
 print('training set = {} records, test set= {} records'.format(X_train.shape[0],X_test.shape[0]))
def rle_encoding(x):
dots = np.where(x.T.flatten() ==1)[0]
run_lengths = []
prev = -2
for b in dots:
if(b>prev+1): run_lengths.extend(( b+1, 0))
run_lengths[-1] += 1
prev = b
return run_lengths
def convert_to_tensor(x,**kwargs):
return x.transpose(2,0,1 ).astype("float32")
 def func_for_preprocessing(preprocessing_fn=None):
transform = []
if preprocessing_fn:
transform.append(albumentations.Lambda(image=preprocessing_fn))
transform.append(albumentations.Lambda(image=convert_to_tensor))
return albumentations.Compose(transform)
 def trainaugs() :
transform = [
albumentations.Resize(height=224,width=224,interpolation=Image.BILINEAR),
albumentations.RandomBrightnessContrast(brightness_limit=0.15, contrast_limit=0),
albumentations.ShiftScaleRotate(rotate_limit=15, shift_limit=0.15, scale_limit=0.2),
albumentations.HorizontalFlip() ,
]
return albumentations.Compose(transform)
 def valaugs() :
transform = [
albumentations.Resize(height=224,width=224,interpolation=Image.BILINEAR),
]
return albumentations.Compose(transform)
 class GetDataset(Dataset):
def __init__(self,imagespath,maskspath,augment=None,preprocess=None):
self.imagespath = imagespath
self.maskspath = maskspath
self.augment = augment
self.preprocess = preprocess
def __len__(self):
return len(self.imagespath)
 def __getitem__(self,idx):
image = cv2.cvtColor(cv2.imread(self.imagespath[idx]),cv2.COLOR_BGR2RGB)
 mask = cv2.imread(self.maskspath[idx], cv2.IMREAD_GRAYSCALE)
 if self.augment:
sample = self.augment(image=image, mask=mask)
 image,mask = sample['image'],sample['mask']
if self.preprocess:
sample = self.preprocess(image=image,mask=mask)
 image,mask = sample['image'],sample['mask']
mask =(mask / 255 ).astype(np.float32)
 mask = np.expand_dims(mask, axis=0)
 return image,mask<categorify>
X_train_num = X_train.drop(columns = bi_col_name)
 selector = SelectKBest(score_func = f_regression, k = 170)
 selector.fit(X_train_num,Y_train)
 select_cols_sm = selector.get_support(indices = True)
 select_num_cols_sm = X_train_num.iloc[:,select_cols_sm]
select_num_col_name = select_num_cols_sm.columns
select_num_cols_sm.head()
encoder = "resnet34"
encoder_wts = "imagenet"
activation = "sigmoid"
model = smp.Unet(encoder_name=encoder,activation=activation,encoder_weights=encoder_wts)
 preprocess_func = smp.encoders.get_preprocessing_fn(encoder,encoder_wts)
 traindata = GetDataset(imagespath = df_train['image'].tolist() ,
maskspath = df_train['mask'].tolist() ,
augment = trainaugs() ,
preprocess = func_for_preprocessing(preprocess_func))
validationdata = GetDataset(imagespath = df_val['image'].tolist() ,
maskspath = df_val['mask'].tolist() ,
augment = valaugs() ,
preprocess = func_for_preprocessing(preprocess_func))
batch_size = 16
trainloader = DataLoader(traindata,batch_size = batch_size,shuffle=True)
 valloader = DataLoader(validationdata,batch_size=batch_size,shuffle=False)
<choose_model_class>
selector = RFE(LogisticRegression() , n_features_to_select=150, step=1, verbose = 2)
 selector = selector.fit(select_num_cols_sm, Y_train)
 select_cols_sm = selector.get_support(indices = True)
 select_cols_df_sm = select_num_cols_sm.iloc[:,select_cols_sm]
best_X_col_name_sm = select_cols_df_sm.columns
select_cols_df_sm.head()
trainmodel = True
epochs = 20
device = "cuda"
loss = smp.utils.losses.DiceLoss()
metrics = [ smp.utils.metrics.IoU(threshold=0.5)]
optimizer = torch.optim.Adam([dict(params=model.parameters() , lr=0.001)])
 lr_scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, epochs)
 trainepoch = smp.utils.train.TrainEpoch(model,loss=loss,optimizer=optimizer,metrics=metrics,device=device,verbose=True)
 validepoch = smp.utils.train.ValidEpoch(model,loss=loss,metrics=metrics,device=device,verbose=True)
<find_best_params>
def filter_x_df_sm(x):
df = x.copy()
all_filter_col = []
for keep in select_cat_col_name[select_cat_col_name.isin(keep_cols)]:
filter_col = [col for col in df.columns if col.startswith(str(keep)) ]
for col in filter_col :
if col not in keep_cols:
all_filter_col.append(col)
 new_df = pd.concat([df[best_X_col_name_sm],df[all_filter_col]],axis = 1)
 return new_df
best_iou_score = 0.0
train_logs_list, valid_logs_list = [], []
for i in range(0,epochs):
print('
Epoch: {}'.format(i))
trainlogs = trainepoch.run(trainloader)
 validlogs = validepoch.run(valloader)
 lr_scheduler.step()
train_logs_list.append(trainlogs)
 valid_logs_list.append(validlogs)
 if best_iou_score < validlogs['iou_score']:
best_iou_score = validlogs['iou_score']
torch.save(model, './best_model.pth' )<create_dataframe>
filter_X_train_sm = filter_x_df_sm(X_train)
 filter_X_test_sm = filter_x_df_sm(X_test )
train_logs_df = pd.DataFrame(train_logs_list)
 valid_logs_df = pd.DataFrame(valid_logs_list)
 train_logs_df.T<create_dataframe>
scaler = StandardScaler()
scaler.fit(filter_X_train_sm)
 X_train_scal_sm = scaler.fit_transform(filter_X_train_sm)
 X_test_scal_sm = scaler.fit_transform(filter_X_test_sm )
best_model = torch.load('./best_model.pth')
 test_dataset = GetDataset(imagespath = df_val['image'].tolist() ,
maskspath = df_val['mask'].tolist() ,
augment = valaugs() ,
preprocess = func_for_preprocessing(preprocess_func))
test_dataset_vis = GetDataset(imagespath = df_val['image'].tolist() ,
maskspath = df_val['mask'].tolist())
 def visualize(**images):
n = len(images)
 plt.figure(figsize=(16, 5))
for i,(name, image)in enumerate(images.items()):
plt.subplot(1, n, i + 1)
 plt.xticks([])
 plt.yticks([])
 plt.title(' '.join(name.split('_')).title())
 if image.shape[0] == 3:
image = image.transpose([1, 2, 0])
 plt.imshow(image)
 plt.show()
for i in range(25):
n = np.random.choice(len(test_dataset))
image_vis = test_dataset_vis[n][0].astype('uint8')
 mask_vis = test_dataset_vis[n][1].astype('uint8')
 image, gt_mask = test_dataset[n]
gt_mask = gt_mask.squeeze()
x_tensor = torch.from_numpy(image ).to(device ).unsqueeze(0)
 pr_mask = best_model.predict(x_tensor)
 pr_mask = pr_mask.squeeze().cpu().numpy().round()
kernel = np.ones(( 5,5),np.uint8)
 pr_mask_er = cv2.erode(pr_mask,kernel,iterations = 4)
 pr_mask_er = cv2.dilate(pr_mask_er,kernel,iterations = 4)
 pr_mask = cv2.resize(pr_mask,(580, 420))
pr_mask_er = cv2.resize(pr_mask_er,(580, 420))
mask_vis = mask_vis.squeeze()
visualize(
 image=image_vis,
ground_truth_mask=mask_vis,
predicted_mask=pr_mask,
predicted_mask_erosion_with_dilation=pr_mask_er
)<categorify>
logisticRegr_sm = LogisticRegression()
logisticRegr_sm = logisticRegr_sm.fit(X_train_scal_sm, Y_train)
 Lr_y_pred_train_sm = logisticRegr_sm.predict(X_train_scal_sm)
 Lr_y_pred_test_sm = logisticRegr_sm.predict(X_test_scal_sm)
 scores_lr_sm = cross_val_score(estimator = logisticRegr_sm, y = Y_train, X = X_train_scal_sm, cv=5)
 print('Cross Validation Score:', np.mean(scores_lr_sm))
imgs = [f for f in os.listdir(testpath)]
imgs = sorted(imgs, key=lambda s: int(s.split('.')[0]))
encodings = []
for m in tqdm(imgs):
x = cv2.imread(os.path.join(testpath, m))
x = valaugs()(image=x)['image']
x = func_for_preprocessing(preprocess_func )(image=x)['image']
x_tensor = torch.from_numpy(x ).to(device ).unsqueeze(0)
 pr_mask = best_model.predict(x_tensor)
 pr_mask = pr_mask.squeeze().cpu().numpy().round().astype(np.uint8)
 pr_mask = albumentations.Resize(height=420,width=580,interpolation=Image.NEAREST )(image=pr_mask)['image']
encodings.append(rle_encoding(pr_mask))<save_to_csv>
cat_cols = df.select_dtypes(exclude=['number'] ).columns.values
for col in cat_cols:
if df[col].value_counts().count() > 20000 :
print('Column {} has {} categories'.format(col,df[col].value_counts().count()))
df_submission = pd.DataFrame(columns=["img", "pixels"])
 for i, encoding in enumerate(encodings):
pixels = ' '.join(map(str, encoding))
df_submission.loc[i] = [str(i+1), pixels]
df_submission.to_csv('submission.csv', index=False )<install_modules>
featurename = filter_X_train.columns
importances = list(rf.feature_importances_)
 feature_importances = [(feature, round(importance, 3)) for feature, importance in zip(featurename, importances)]
feature_importances = sorted(feature_importances, key = lambda x: x[1], reverse = True)
 print('Top 50 Importance Features
')
 [print('Variable: {} Importance Score: {}'.format(*pair)) for pair in feature_importances[:50]];
!pip install git+https://github.com/Ritvik19/pyradox.git<import_modules>
test_df = pd.read_csv('.. /input/loan-default-prediction/test_v2.csv.zip')
 test_df.head()
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
from math import sin, cos, pi
import cv2, os
from tqdm.auto import tqdm
from tensorflow.keras import layers, callbacks, utils, applications, optimizers, Input
from tensorflow.keras.models import Sequential, Model, load_model
from pyradox import convnets<define_variables>
encoder = ce.BinaryEncoder(cols = keep_cols)
 bi_enc_df = encoder.fit_transform(test_df[keep_cols])
 bi_col_name = bi_enc_df.columns
test_df = pd.concat([test_df,bi_enc_df],axis = 1)
 test_df.head()
class config:
NUM_EPOCHS = 100
BATCH_SIZE = 64<load_pretrained>
select_test_df = pd.concat([test_df['id'],test_df[filter_X_train.columns]],axis = 1)
 select_test_df.head()
print("Contents of input/facial-keypoints-detection directory: ")
 !ls.. /input/facial-keypoints-detection/
print("
Extracting.zip dataset files to working directory...")
 !unzip -u.. /input/facial-keypoints-detection/test.zip
!unzip -u.. /input/facial-keypoints-detection/training.zip
print("
Current working directory:")
 !pwd
print("
Contents of working directory:")
 !ls<load_from_csv>
select_test_df.isnull().sum().value_counts()
%%time
train_file = 'training.csv'
test_file = 'test.csv'
idlookup_file = '.. /input/facial-keypoints-detection/IdLookupTable.csv'
train_data = pd.read_csv(train_file)
 test_data = pd.read_csv(test_file)
 idlookup_data = pd.read_csv(idlookup_file )<count_missing_values>
numeric_cols = select_test_df.select_dtypes(include=['number'] ).columns.values
for col in numeric_cols:
if col in list(col_pct_miss_df.column_name):
med = df[col].median()
select_test_df[col] = select_test_df[col].fillna(med)
 not_numeric_cols = select_test_df.select_dtypes(exclude=['number'] ).columns.values
for col in not_numeric_cols:
mode = df[col].mode()
select_test_df[col] = select_test_df[col].fillna(mode[0] )
train_data.isnull().sum()<categorify>
select_test_df.isnull().sum().value_counts()
%%time
def load_images(image_data):
images = []
for idx, sample in image_data.iterrows() :
image = np.array(sample['Image'].split(' '), dtype=int)
 image = np.reshape(image,(96,96,1))
images.append(image)
 images = np.array(images)/255.
 return images
def load_keypoints(keypoint_data):
keypoint_data = keypoint_data.drop('Image',axis = 1)
 keypoint_features = []
for idx, sample_keypoints in keypoint_data.iterrows() :
keypoint_features.append(sample_keypoints)
 keypoint_features = np.array(keypoint_features, dtype = 'float')
 return keypoint_features
clean_train_images = load_images(clean_train_data)
 print("Shape of clean_train_images:", np.shape(clean_train_images))
clean_train_keypoints = load_keypoints(clean_train_data)
 print("Shape of clean_train_keypoints:", np.shape(clean_train_keypoints))
test_images = load_images(test_data)
 print("Shape of test_images:", np.shape(test_images))
train_images = clean_train_images
train_keypoints = clean_train_keypoints
fig, axis = plt.subplots()
plot_sample(clean_train_images[19], clean_train_keypoints[19], axis, "Sample image & keypoints")
 unclean_train_images = load_images(unclean_train_data)
 print("Shape of unclean_train_images:", np.shape(unclean_train_images))
unclean_train_keypoints = load_keypoints(unclean_train_data)
 print("Shape of unclean_train_keypoints:", np.shape(unclean_train_keypoints))
train_images = np.concatenate(( train_images, unclean_train_images))
train_keypoints = np.concatenate(( train_keypoints, unclean_train_keypoints))<choose_model_class>
select_test_df[columns_with_nan] = select_test_df[columns_with_nan].fillna(0 )
inputs = Input(shape=(96, 96, 1))
x = layers.Convolution2D(3,(1, 1), padding='same' )(inputs)
 x = layers.LeakyReLU(alpha = 0.1 )(x)
 x = convnets.MobileNetV3(config='small' )(x)
 x = layers.GlobalAveragePooling2D()(x)
 x = layers.Dropout(0.1 )(x)
 outputs = layers.Dense(30 )(x)
 model = Model(inputs=inputs, outputs=outputs)
 model.summary()<train_model>
test_df_rf = test_df.copy()
test_df_rf['loss'] = rf.predict(select_test_df.loc[:,select_test_df.columns != 'id'])
 test_df_rf.head()
<train_model><EOS> 
sample_submission = test_df_rf[['id','loss']]
sample_submission.to_csv('sample_submission_rf.csv', index = False )