Kaggle competition: Home Credit Default Risk
<SOS> metric: AUC Kaggle data source: home-credit-default-risk<split>
warnings.simplefilter(action='ignore', category=FutureWarning )
train_df, valid_df = train_test_split(train_df,
test_size =.2,
shuffle=True ,
random_state=SEED)
<choose_model_class>
DATA_DIRECTORY = ".. /input/home-credit-default-risk"
train_generator = ImageDataGenerator(preprocessing_function=preprocess_input,
rotation_range=45,
shear_range=0.1,
zoom_range=0.2,
horizontal_flip=False,
width_shift_range=0.1,
height_shift_range=0.1, 
)
 train_generator = train_generator.flow_from_dataframe(
 train_df,
UNZIP_TRAIN,
x_col='filename',
y_col='class',
target_size=(IMG_SIZE,IMG_SIZE),
batch_size=32,
class_mode='binary' 
)
 validation_generator = ImageDataGenerator(preprocessing_function=preprocess_input, 
)
 validation_generator = validation_generator.flow_from_dataframe(
 valid_df,
UNZIP_TRAIN,
x_col = 'filename',
y_col = 'class',
target_size =(IMG_SIZE,IMG_SIZE),
batch_size=32,
class_mode='binary' 
)
<choose_model_class>
df_train = pd.read_csv(os.path.join(DATA_DIRECTORY, 'application_train.csv'))
df_test = pd.read_csv(os.path.join(DATA_DIRECTORY, 'application_test.csv'))
df = df_train.append(df_test)
 del df_train, df_test; gc.collect()
pre_trained_model = EfficientNetB0(input_shape =(IMG_SIZE, IMG_SIZE, 3),
include_top = False,
weights = 'imagenet')
 for layer in pre_trained_model.layers:
layer.trainable = False 
<choose_model_class>
df = df[df['AMT_INCOME_TOTAL'] < 20000000]
df = df[df['CODE_GENDER'] != 'XNA']
df['DAYS_EMPLOYED'].replace(365243, np.nan, inplace=True)
 df['DAYS_LAST_PHONE_CHANGE'].replace(0, np.nan, inplace=True )
last_layer = pre_trained_model.get_layer('top_activation')
 last_output=last_layer.output
def create_model(last_output):
x=keras.layers.GlobalAveragePooling2D()(last_output)
 x=keras.layers.BatchNormalization()(x)
 x=keras.layers.Dense(1, activation='sigmoid' )(x)
 model = Model(pre_trained_model.input, x)
 model.compile(optimizer = RMSprop(learning_rate=0.001),
loss = 'binary_crossentropy',
metrics = ['acc'])
 return model
model=create_model(last_output)
<init_hyperparams>
def get_age_group(days_birth):
age_years = -days_birth / 365
if age_years < 27: return 1
elif age_years < 40: return 2
elif age_years < 50: return 3
elif age_years < 65: return 4
elif age_years < 99: return 5
else: return 0
learning_rate_reduction = ReduceLROnPlateau(monitor='val_acc',
patience=2,
verbose=2,
factor=0.5,
min_delt=0.001,
min_lr=0.00001)
 early_stopping = EarlyStopping(
 monitor = "val_accuracy",
patience = 50,
verbose = 2,
mode = "max", 
)
<train_model>
docs = [f for f in df.columns if 'FLAG_DOC' in f]
df['DOCUMENT_COUNT'] = df[docs].sum(axis=1)
 df['NEW_DOC_KURT'] = df[docs].kurtosis(axis=1)
 df['AGE_RANGE'] = df['DAYS_BIRTH'].apply(lambda x: get_age_group(x))
history = model.fit(
 train_generator,
validation_data = validation_generator,
epochs = EPOCHS,
callbacks = [learning_rate_reduction, early_stopping],
)<save_model>
df['EXT_SOURCES_PROD'] = df['EXT_SOURCE_1'] * df['EXT_SOURCE_2'] * df['EXT_SOURCE_3']
df['EXT_SOURCES_WEIGHTED'] = df.EXT_SOURCE_1 * 2 + df.EXT_SOURCE_2 * 1 + df.EXT_SOURCE_3 * 3
np.warnings.filterwarnings('ignore', r'All-NaN(slice|axis)encountered')
 for function_name in ['min', 'max', 'mean', 'nanmedian', 'var']:
feature_name = 'EXT_SOURCES_{}'.format(function_name.upper())
 df[feature_name] = eval('np.{}'.format(function_name))(
 df[['EXT_SOURCE_1', 'EXT_SOURCE_2', 'EXT_SOURCE_3']], axis=1 )
model.save('./dog_cat_model' )<predict_on_test>
df['CREDIT_TO_ANNUITY_RATIO'] = df['AMT_CREDIT'] / df['AMT_ANNUITY']
df['CREDIT_TO_GOODS_RATIO'] = df['AMT_CREDIT'] / df['AMT_GOODS_PRICE']
df['ANNUITY_TO_INCOME_RATIO'] = df['AMT_ANNUITY'] / df['AMT_INCOME_TOTAL']
df['CREDIT_TO_INCOME_RATIO'] = df['AMT_CREDIT'] / df['AMT_INCOME_TOTAL']
df['INCOME_TO_EMPLOYED_RATIO'] = df['AMT_INCOME_TOTAL'] / df['DAYS_EMPLOYED']
df['INCOME_TO_BIRTH_RATIO'] = df['AMT_INCOME_TOTAL'] / df['DAYS_BIRTH']
df['EMPLOYED_TO_BIRTH_RATIO'] = df['DAYS_EMPLOYED'] / df['DAYS_BIRTH']
df['ID_TO_BIRTH_RATIO'] = df['DAYS_ID_PUBLISH'] / df['DAYS_BIRTH']
df['CAR_TO_BIRTH_RATIO'] = df['OWN_CAR_AGE'] / df['DAYS_BIRTH']
df['CAR_TO_EMPLOYED_RATIO'] = df['OWN_CAR_AGE'] / df['DAYS_EMPLOYED']
df['PHONE_TO_BIRTH_RATIO'] = df['DAYS_LAST_PHONE_CHANGE'] / df['DAYS_BIRTH']
test_gen = ImageDataGenerator(preprocessing_function=preprocess_input)
 test_generator = test_gen.flow_from_dataframe(
 test_df,
UNZIP_TEST,
x_col='filename',
class_mode= None,
target_size=(IMG_SIZE,IMG_SIZE),
batch_size=BATCH_SIZE,
shuffle=False 
)
 predict = model.predict(test_generator, verbose = 1 )<feature_engineering>
def do_mean(df, group_cols, counted, agg_name):
gp = df[group_cols + [counted]].groupby(group_cols)[counted].mean().reset_index().rename(
 columns={counted: agg_name})
 df = df.merge(gp, on=group_cols, how='left')
 del gp
gc.collect()
return df
test_df["predict"] = predict
test_df["label"] = test_df["predict"]
result = test_df[["id", "label"]]<save_to_csv>
def do_median(df, group_cols, counted, agg_name):
gp = df[group_cols + [counted]].groupby(group_cols)[counted].median().reset_index().rename(
 columns={counted: agg_name})
 df = df.merge(gp, on=group_cols, how='left')
 del gp
gc.collect()
return df
result.to_csv('submission.csv', index=False )<import_modules>
def do_std(df, group_cols, counted, agg_name):
gp = df[group_cols + [counted]].groupby(group_cols)[counted].std().reset_index().rename(
 columns={counted: agg_name})
 df = df.merge(gp, on=group_cols, how='left')
 del gp
gc.collect()
return df
for dirname, _, filenames in os.walk('/kaggle/input'):
for filename in filenames:
print(os.path.join(dirname, filename)) 
<load_from_csv>
def do_sum(df, group_cols, counted, agg_name):
gp = df[group_cols + [counted]].groupby(group_cols)[counted].sum().reset_index().rename(
 columns={counted: agg_name})
 df = df.merge(gp, on=group_cols, how='left')
 del gp
gc.collect()
return df
df_train = pd.read_csv("/kaggle/input/nlp-getting-started/train.csv")
 df_test = pd.read_csv("/kaggle/input/nlp-getting-started/test.csv" )<feature_engineering>
group = ['ORGANIZATION_TYPE', 'NAME_EDUCATION_TYPE', 'OCCUPATION_TYPE', 'AGE_RANGE', 'CODE_GENDER']
df = do_median(df, group, 'EXT_SOURCES_MEAN', 'GROUP_EXT_SOURCES_MEDIAN')
 df = do_std(df, group, 'EXT_SOURCES_MEAN', 'GROUP_EXT_SOURCES_STD')
 df = do_mean(df, group, 'AMT_INCOME_TOTAL', 'GROUP_INCOME_MEAN')
 df = do_std(df, group, 'AMT_INCOME_TOTAL', 'GROUP_INCOME_STD')
 df = do_mean(df, group, 'CREDIT_TO_ANNUITY_RATIO', 'GROUP_CREDIT_TO_ANNUITY_MEAN')
 df = do_std(df, group, 'CREDIT_TO_ANNUITY_RATIO', 'GROUP_CREDIT_TO_ANNUITY_STD')
 df = do_mean(df, group, 'AMT_CREDIT', 'GROUP_CREDIT_MEAN')
 df = do_mean(df, group, 'AMT_ANNUITY', 'GROUP_ANNUITY_MEAN')
 df = do_std(df, group, 'AMT_ANNUITY', 'GROUP_ANNUITY_STD' )
def format_keyword(df):
df["keyword"] = df["keyword"].fillna(".")
 df["keyword"] = df.keyword.str.replace("%20"," " )<count_values>
def label_encoder(df, categorical_columns=None):
if not categorical_columns:
categorical_columns = [col for col in df.columns if df[col].dtype == 'object']
for col in categorical_columns:
df[col], uniques = pd.factorize(df[col])
 return df, categorical_columns
df_train.loc[df_train.target==0]["keyword"].value_counts()<string_transform>
def drop_application_columns(df):
drop_list = [
'CNT_CHILDREN', 'CNT_FAM_MEMBERS', 'HOUR_APPR_PROCESS_START',
'FLAG_EMP_PHONE', 'FLAG_MOBIL', 'FLAG_CONT_MOBILE', 'FLAG_EMAIL', 'FLAG_PHONE',
'FLAG_OWN_REALTY', 'REG_REGION_NOT_LIVE_REGION', 'REG_REGION_NOT_WORK_REGION',
'REG_CITY_NOT_WORK_CITY', 'OBS_30_CNT_SOCIAL_CIRCLE', 'OBS_60_CNT_SOCIAL_CIRCLE',
'AMT_REQ_CREDIT_BUREAU_DAY', 'AMT_REQ_CREDIT_BUREAU_MON', 'AMT_REQ_CREDIT_BUREAU_YEAR',
'COMMONAREA_MODE', 'NONLIVINGAREA_MODE', 'ELEVATORS_MODE', 'NONLIVINGAREA_AVG',
'FLOORSMIN_MEDI', 'LANDAREA_MODE', 'NONLIVINGAREA_MEDI', 'LIVINGAPARTMENTS_MODE',
'FLOORSMIN_AVG', 'LANDAREA_AVG', 'FLOORSMIN_MODE', 'LANDAREA_MEDI',
'COMMONAREA_MEDI', 'YEARS_BUILD_AVG', 'COMMONAREA_AVG', 'BASEMENTAREA_AVG',
'BASEMENTAREA_MODE', 'NONLIVINGAPARTMENTS_MEDI', 'BASEMENTAREA_MEDI',
'LIVINGAPARTMENTS_AVG', 'ELEVATORS_AVG', 'YEARS_BUILD_MEDI', 'ENTRANCES_MODE',
'NONLIVINGAPARTMENTS_MODE', 'LIVINGAREA_MODE', 'LIVINGAPARTMENTS_MEDI',
'YEARS_BUILD_MODE', 'YEARS_BEGINEXPLUATATION_AVG', 'ELEVATORS_MEDI', 'LIVINGAREA_MEDI',
'YEARS_BEGINEXPLUATATION_MODE', 'NONLIVINGAPARTMENTS_AVG', 'HOUSETYPE_MODE',
'FONDKAPREMONT_MODE', 'EMERGENCYSTATE_MODE'
]
for doc_num in [2,4,5,6,7,9,10,11,12,13,14,15,16,17,19,20,21]:
drop_list.append('FLAG_DOCUMENT_{}'.format(doc_num))
df.drop(drop_list, axis=1, inplace=True)
 return df
df_count = df_train.text.str.split().str.len()
max(df_count )<categorify>
df, le_encoded_cols = label_encoder(df, None)
 df = drop_application_columns(df )
def process_text(text):
text=text.replace("
","")
 text = re.sub(r'@\S+','',text)
 text = re.sub(r'
text = re.sub(r'https?://\S+|www\.\S+|http?://\S+','',text)
 text = re.sub('[%s]' % re.escape (<feature_engineering>
df = pd.get_dummies(df )
df_train["text"] = df_train.text.transform(lambda x: process_text(x))
df_test["text"] = df_test.text.transform(lambda x: process_text(x))<categorify>
bureau = pd.read_csv(os.path.join(DATA_DIRECTORY, 'bureau.csv'))
df_train["appears"]=df_train.groupby("text" ).text.transform("count" )<feature_engineering>
bureau['CREDIT_DURATION'] = -bureau['DAYS_CREDIT'] + bureau['DAYS_CREDIT_ENDDATE']
bureau['ENDDATE_DIF'] = bureau['DAYS_CREDIT_ENDDATE'] - bureau['DAYS_ENDDATE_FACT']
bureau['DEBT_PERCENTAGE'] = bureau['AMT_CREDIT_SUM'] / bureau['AMT_CREDIT_SUM_DEBT']
bureau['DEBT_CREDIT_DIFF'] = bureau['AMT_CREDIT_SUM'] - bureau['AMT_CREDIT_SUM_DEBT']
bureau['CREDIT_TO_ANNUITY_RATIO'] = bureau['AMT_CREDIT_SUM'] / bureau['AMT_ANNUITY']
df_train["target_std"]=df_train.groupby("text" ).target.transform(np.std)
 df_train["target_mean"]=df_train.groupby("text" ).target.transform(np.mean )<sort_values>
def one_hot_encoder(df, categorical_columns=None, nan_as_category=True):
original_columns = list(df.columns)
 if not categorical_columns:
categorical_columns = [col for col in df.columns if df[col].dtype == 'object']
df = pd.get_dummies(df, columns=categorical_columns, dummy_na=nan_as_category)
 categorical_columns = [c for c in df.columns if c not in original_columns]
return df, categorical_columns
duplicate_ids = df_train.loc[df_train.target_std>0].sort_values(by=["appears","text"],ascending=False ).index<drop_column>
def group(df_to_agg, prefix, aggregations, aggregate_by= 'SK_ID_CURR'):
agg_df = df_to_agg.groupby(aggregate_by ).agg(aggregations)
 agg_df.columns = pd.Index(['{}{}_{}'.format(prefix, e[0], e[1].upper())
 for e in agg_df.columns.tolist() ])
 return agg_df.reset_index()
df_train = df_train.drop(index = duplicate_ids )<remove_duplicates>
def group_and_merge(df_to_agg, df_to_merge, prefix, aggregations, aggregate_by= 'SK_ID_CURR'):
agg_df = group(df_to_agg, prefix, aggregations, aggregate_by= aggregate_by)
 return df_to_merge.merge(agg_df, how='left', on= aggregate_by )
df_train = df_train.drop_duplicates(subset=["text"] )<drop_column>
def get_bureau_balance(path, num_rows= None):
bb = pd.read_csv(os.path.join(path, 'bureau_balance.csv'))
bb, categorical_cols = one_hot_encoder(bb, nan_as_category= False)
 bb_processed = bb.groupby('SK_ID_BUREAU')[categorical_cols].mean().reset_index()
agg = {'MONTHS_BALANCE': ['min', 'max', 'mean', 'size']}
bb_processed = group_and_merge(bb, bb_processed, '', agg, 'SK_ID_BUREAU')
 del bb; gc.collect()
return bb_processed
df_train.reset_index(drop=True,inplace=True)
 df_train<feature_engineering>
bureau, categorical_cols = one_hot_encoder(bureau, nan_as_category= False)
 bureau = bureau.merge(get_bureau_balance(DATA_DIRECTORY), how='left', on='SK_ID_BUREAU')
 bureau['STATUS_12345'] = 0
for i in range(1,6):
bureau['STATUS_12345'] += bureau['STATUS_{}'.format(i)]
nlp = spacy.load("en_core_web_lg")
 keyword_train = np.array([nlp(text ).vector for text in df_train.keyword])
 keyword_test = np.array([nlp(text ).vector for text in df_test.keyword] )<feature_engineering>
features = ['AMT_CREDIT_MAX_OVERDUE', 'AMT_CREDIT_SUM_OVERDUE', 'AMT_CREDIT_SUM',
'AMT_CREDIT_SUM_DEBT', 'DEBT_PERCENTAGE', 'DEBT_CREDIT_DIFF', 'STATUS_0', 'STATUS_12345']
agg_length = bureau.groupby('MONTHS_BALANCE_SIZE')[features].mean().reset_index()
agg_length.rename({feat: 'LL_' + feat for feat in features}, axis=1, inplace=True)
 bureau = bureau.merge(agg_length, how='left', on='MONTHS_BALANCE_SIZE')
 del agg_length; gc.collect()
def nlp_vectors(text):
res = []
doc = nlp(text)
 for token in doc:
if not token.is_space:
res.append(token.vector)
 return res
def build_nlp_vectors(df_text):
spacy_vectors =([nlp_vectors(text)for text in df_text])
 max_length = 0;
for vector in spacy_vectors:
max_length = max(max_length, len(vector))
print(f"Maximum Length:{ max_length}")
 for i in range(len(spacy_vectors)) :
while(len(spacy_vectors[i])<max_length):
spacy_vectors[i].append([0]*300)
 spacy_vectors = np.array(spacy_vectors)
 print(f"Shape of spacy vector:{spacy_vectors.shape}")
 return spacy_vectors 
<feature_engineering>
BUREAU_AGG = {
'SK_ID_BUREAU': ['nunique'],
'DAYS_CREDIT': ['min', 'max', 'mean'],
'DAYS_CREDIT_ENDDATE': ['min', 'max'],
'AMT_CREDIT_MAX_OVERDUE': ['max', 'mean'],
'AMT_CREDIT_SUM': ['max', 'mean', 'sum'],
'AMT_CREDIT_SUM_DEBT': ['max', 'mean', 'sum'],
'AMT_CREDIT_SUM_OVERDUE': ['max', 'mean', 'sum'],
'AMT_ANNUITY': ['mean'],
'DEBT_CREDIT_DIFF': ['mean', 'sum'],
'MONTHS_BALANCE_MEAN': ['mean', 'var'],
'MONTHS_BALANCE_SIZE': ['mean', 'sum'],
'STATUS_0': ['mean'],
'STATUS_1': ['mean'],
'STATUS_12345': ['mean'],
'STATUS_C': ['mean'],
'STATUS_X': ['mean'],
'CREDIT_ACTIVE_Active': ['mean'],
'CREDIT_ACTIVE_Closed': ['mean'],
'CREDIT_ACTIVE_Sold': ['mean'],
'CREDIT_TYPE_Consumer credit': ['mean'],
'CREDIT_TYPE_Credit card': ['mean'],
'CREDIT_TYPE_Car loan': ['mean'],
'CREDIT_TYPE_Mortgage': ['mean'],
'CREDIT_TYPE_Microloan': ['mean'],
'LL_AMT_CREDIT_SUM_OVERDUE': ['mean'],
'LL_DEBT_CREDIT_DIFF': ['mean'],
'LL_STATUS_12345': ['mean'],
}
BUREAU_ACTIVE_AGG = {
'DAYS_CREDIT': ['max', 'mean'],
'DAYS_CREDIT_ENDDATE': ['min', 'max'],
'AMT_CREDIT_MAX_OVERDUE': ['max', 'mean'],
'AMT_CREDIT_SUM': ['max', 'sum'],
'AMT_CREDIT_SUM_DEBT': ['mean', 'sum'],
'AMT_CREDIT_SUM_OVERDUE': ['max', 'mean'],
'DAYS_CREDIT_UPDATE': ['min', 'mean'],
'DEBT_PERCENTAGE': ['mean'],
'DEBT_CREDIT_DIFF': ['mean'],
'CREDIT_TO_ANNUITY_RATIO': ['mean'],
'MONTHS_BALANCE_MEAN': ['mean', 'var'],
'MONTHS_BALANCE_SIZE': ['mean', 'sum'],
}
BUREAU_CLOSED_AGG = {
'DAYS_CREDIT': ['max', 'var'],
'DAYS_CREDIT_ENDDATE': ['max'],
'AMT_CREDIT_MAX_OVERDUE': ['max', 'mean'],
'AMT_CREDIT_SUM_OVERDUE': ['mean'],
'AMT_CREDIT_SUM': ['max', 'mean', 'sum'],
'AMT_CREDIT_SUM_DEBT': ['max', 'sum'],
'DAYS_CREDIT_UPDATE': ['max'],
'ENDDATE_DIF': ['mean'],
'STATUS_12345': ['mean'],
}
BUREAU_LOAN_TYPE_AGG = {
'DAYS_CREDIT': ['mean', 'max'],
'AMT_CREDIT_MAX_OVERDUE': ['mean', 'max'],
'AMT_CREDIT_SUM': ['mean', 'max'],
'AMT_CREDIT_SUM_DEBT': ['mean', 'max'],
'DEBT_PERCENTAGE': ['mean'],
'DEBT_CREDIT_DIFF': ['mean'],
'DAYS_CREDIT_ENDDATE': ['max'],
}
BUREAU_TIME_AGG = {
'AMT_CREDIT_MAX_OVERDUE': ['max', 'mean'],
'AMT_CREDIT_SUM_OVERDUE': ['mean'],
'AMT_CREDIT_SUM': ['max', 'sum'],
'AMT_CREDIT_SUM_DEBT': ['mean', 'sum'],
'DEBT_PERCENTAGE': ['mean'],
'DEBT_CREDIT_DIFF': ['mean'],
'STATUS_0': ['mean'],
'STATUS_12345': ['mean'],
}
nlp_train = build_nlp_vectors(df_train.text )<load_pretrained>
agg_bureau = group(bureau, 'BUREAU_', BUREAU_AGG)
 active = bureau[bureau['CREDIT_ACTIVE_Active'] == 1]
agg_bureau = group_and_merge(active,agg_bureau,'BUREAU_ACTIVE_',BUREAU_ACTIVE_AGG)
 closed = bureau[bureau['CREDIT_ACTIVE_Closed'] == 1]
agg_bureau = group_and_merge(closed,agg_bureau,'BUREAU_CLOSED_',BUREAU_CLOSED_AGG)
 del active, closed; gc.collect()
for credit_type in ['Consumer credit', 'Credit card', 'Mortgage', 'Car loan', 'Microloan']:
type_df = bureau[bureau['CREDIT_TYPE_' + credit_type] == 1]
prefix = 'BUREAU_' + credit_type.split(' ')[0].upper() + '_'
agg_bureau = group_and_merge(type_df, agg_bureau, prefix, BUREAU_LOAN_TYPE_AGG)
 del type_df; gc.collect()
for time_frame in [6, 12]:
prefix = "BUREAU_LAST{}M_".format(time_frame)
 time_frame_df = bureau[bureau['DAYS_CREDIT'] >= -30*time_frame]
agg_bureau = group_and_merge(time_frame_df, agg_bureau, prefix, BUREAU_TIME_AGG)
 del time_frame_df; gc.collect()
tokenizer = ppb.DistilBertTokenizer.from_pretrained("distilbert-base-uncased")
 bert_model = ppb.DistilBertModel.from_pretrained("distilbert-base-uncased" )<define_variables>
sort_bureau = bureau.sort_values(by=['DAYS_CREDIT'])
 gr = sort_bureau.groupby('SK_ID_CURR')['AMT_CREDIT_MAX_OVERDUE'].last().reset_index()
gr.rename({'AMT_CREDIT_MAX_OVERDUE': 'BUREAU_LAST_LOAN_MAX_OVERDUE'}, inplace=True)
 agg_bureau = agg_bureau.merge(gr, on='SK_ID_CURR', how='left')
 agg_bureau['BUREAU_DEBT_OVER_CREDIT'] = \
agg_bureau['BUREAU_AMT_CREDIT_SUM_DEBT_SUM']/agg_bureau['BUREAU_AMT_CREDIT_SUM_SUM']
agg_bureau['BUREAU_ACTIVE_DEBT_OVER_CREDIT'] = \
agg_bureau['BUREAU_ACTIVE_AMT_CREDIT_SUM_DEBT_SUM']/agg_bureau['BUREAU_ACTIVE_AMT_CREDIT_SUM_SUM']
def process_data(df_text):
tokens = df_text.apply(lambda text: tokenizer.encode(text,add_special_tokens=True))
max_len = 0;
i = 0;
for token in tokens.values:
max_len = max(max_len,len(token))
print(f"Max Length: {max_len}")
 padded = np.array([i+[0]*(max_len-len(i)) for i in tokens.values])
 attention_mask = np.where(padded !=0, 1,0)
 input_ids = torch.tensor(padded)
 attention_mask = torch.tensor(attention_mask)
 with torch.no_grad() :
last_hidden_states = bert_model(input_ids,attention_mask=attention_mask)
 X = last_hidden_states[0][:,0,:].numpy()
print(X.shape)
 return X 
<prepare_x_and_y>
df = pd.merge(df, agg_bureau, on='SK_ID_CURR', how='left')
 del agg_bureau, bureau
gc.collect()
X_train = process_data(df_train.text )<prepare_x_and_y>
prev = pd.read_csv(os.path.join(DATA_DIRECTORY, 'previous_application.csv'))
pay = pd.read_csv(os.path.join(DATA_DIRECTORY, 'installments_payments.csv'))
y_train = df_train.target<split>
PREVIOUS_AGG = {
'SK_ID_PREV': ['nunique'],
'AMT_ANNUITY': ['min', 'max', 'mean'],
'AMT_DOWN_PAYMENT': ['max', 'mean'],
'HOUR_APPR_PROCESS_START': ['min', 'max', 'mean'],
'RATE_DOWN_PAYMENT': ['max', 'mean'],
'DAYS_DECISION': ['min', 'max', 'mean'],
'CNT_PAYMENT': ['max', 'mean'],
'DAYS_TERMINATION': ['max'],
'CREDIT_TO_ANNUITY_RATIO': ['mean', 'max'],
'APPLICATION_CREDIT_DIFF': ['min', 'max', 'mean'],
'APPLICATION_CREDIT_RATIO': ['min', 'max', 'mean', 'var'],
'DOWN_PAYMENT_TO_CREDIT': ['mean'],
}
PREVIOUS_ACTIVE_AGG = {
'SK_ID_PREV': ['nunique'],
'SIMPLE_INTERESTS': ['mean'],
'AMT_ANNUITY': ['max', 'sum'],
'AMT_APPLICATION': ['max', 'mean'],
'AMT_CREDIT': ['sum'],
'AMT_DOWN_PAYMENT': ['max', 'mean'],
'DAYS_DECISION': ['min', 'mean'],
'CNT_PAYMENT': ['mean', 'sum'],
'DAYS_LAST_DUE_1ST_VERSION': ['min', 'max', 'mean'],
'AMT_PAYMENT': ['sum'],
'INSTALMENT_PAYMENT_DIFF': ['mean', 'max'],
'REMAINING_DEBT': ['max', 'mean', 'sum'],
'REPAYMENT_RATIO': ['mean'],
}
PREVIOUS_LATE_PAYMENTS_AGG = {
'DAYS_DECISION': ['min', 'max', 'mean'],
'DAYS_LAST_DUE_1ST_VERSION': ['min', 'max', 'mean'],
'APPLICATION_CREDIT_DIFF': ['min'],
'NAME_CONTRACT_TYPE_Consumer loans': ['mean'],
'NAME_CONTRACT_TYPE_Cash loans': ['mean'],
'NAME_CONTRACT_TYPE_Revolving loans': ['mean'],
}
PREVIOUS_LOAN_TYPE_AGG = {
'AMT_CREDIT': ['sum'],
'AMT_ANNUITY': ['mean', 'max'],
'SIMPLE_INTERESTS': ['min', 'mean', 'max', 'var'],
'APPLICATION_CREDIT_DIFF': ['min', 'var'],
'APPLICATION_CREDIT_RATIO': ['min', 'max', 'mean'],
'DAYS_DECISION': ['max'],
'DAYS_LAST_DUE_1ST_VERSION': ['max', 'mean'],
'CNT_PAYMENT': ['mean'],
}
PREVIOUS_TIME_AGG = {
'AMT_CREDIT': ['sum'],
'AMT_ANNUITY': ['mean', 'max'],
'SIMPLE_INTERESTS': ['mean', 'max'],
'DAYS_DECISION': ['min', 'mean'],
'DAYS_LAST_DUE_1ST_VERSION': ['min', 'max', 'mean'],
'APPLICATION_CREDIT_DIFF': ['min'],
'APPLICATION_CREDIT_RATIO': ['min', 'max', 'mean'],
'NAME_CONTRACT_TYPE_Consumer loans': ['mean'],
'NAME_CONTRACT_TYPE_Cash loans': ['mean'],
'NAME_CONTRACT_TYPE_Revolving loans': ['mean'],
}
PREVIOUS_APPROVED_AGG = {
'SK_ID_PREV': ['nunique'],
'AMT_ANNUITY': ['min', 'max', 'mean'],
'AMT_CREDIT': ['min', 'max', 'mean'],
'AMT_DOWN_PAYMENT': ['max'],
'AMT_GOODS_PRICE': ['max'],
'HOUR_APPR_PROCESS_START': ['min', 'max'],
'DAYS_DECISION': ['min', 'mean'],
'CNT_PAYMENT': ['max', 'mean'],
'DAYS_TERMINATION': ['mean'],
'CREDIT_TO_ANNUITY_RATIO': ['mean', 'max'],
'APPLICATION_CREDIT_DIFF': ['max'],
'APPLICATION_CREDIT_RATIO': ['min', 'max', 'mean'],
'DAYS_FIRST_DRAWING': ['max', 'mean'],
'DAYS_FIRST_DUE': ['min', 'mean'],
'DAYS_LAST_DUE_1ST_VERSION': ['min', 'max', 'mean'],
'DAYS_LAST_DUE': ['max', 'mean'],
'DAYS_LAST_DUE_DIFF': ['min', 'max', 'mean'],
'SIMPLE_INTERESTS': ['min', 'max', 'mean'],
}
PREVIOUS_REFUSED_AGG = {
'AMT_APPLICATION': ['max', 'mean'],
'AMT_CREDIT': ['min', 'max'],
'DAYS_DECISION': ['min', 'max', 'mean'],
'CNT_PAYMENT': ['max', 'mean'],
'APPLICATION_CREDIT_DIFF': ['min', 'max', 'mean', 'var'],
'APPLICATION_CREDIT_RATIO': ['min', 'mean'],
'NAME_CONTRACT_TYPE_Consumer loans': ['mean'],
'NAME_CONTRACT_TYPE_Cash loans': ['mean'],
'NAME_CONTRACT_TYPE_Revolving loans': ['mean'],
}
X_tr, X_val, nlp_tr, nlp_val, kw_tr, kw_val, y_tr, y_val = train_test_split(X_train,nlp_train, keyword_train, y_train, test_size=0.25, train_size=0.75,shuffle=True )<choose_model_class>
ohe_columns = [
'NAME_CONTRACT_STATUS', 'NAME_CONTRACT_TYPE', 'CHANNEL_TYPE',
'NAME_TYPE_SUITE', 'NAME_YIELD_GROUP', 'PRODUCT_COMBINATION',
'NAME_PRODUCT_TYPE', 'NAME_CLIENT_TYPE']
prev, categorical_cols = one_hot_encoder(prev, ohe_columns, nan_as_category= False )
def build_nn() :
model = tf.keras.Sequential()
model.add(layers.Input(shape=(768,)))
 model.add(layers.Dense(128,activation='tanh'))
model.add(layers.Dropout(0.6))
model.add(layers.Dense(32,activation='tanh'))
model.add(layers.Dropout(0.6))
model.add(layers.Dense(8,activation='tanh'))
model.add(layers.Dense(1,activation='sigmoid'))
model.compile(loss=tf.keras.losses.BinaryCrossentropy(from_logits=True),
optimizer=tf.keras.optimizers.Adam(1e-4),
metrics=['accuracy'])
 return model<choose_model_class>
prev['APPLICATION_CREDIT_DIFF'] = prev['AMT_APPLICATION'] - prev['AMT_CREDIT']
prev['APPLICATION_CREDIT_RATIO'] = prev['AMT_APPLICATION'] / prev['AMT_CREDIT']
prev['CREDIT_TO_ANNUITY_RATIO'] = prev['AMT_CREDIT']/prev['AMT_ANNUITY']
prev['DOWN_PAYMENT_TO_CREDIT'] = prev['AMT_DOWN_PAYMENT'] / prev['AMT_CREDIT']
total_payment = prev['AMT_ANNUITY'] * prev['CNT_PAYMENT']
prev['SIMPLE_INTERESTS'] =(total_payment/prev['AMT_CREDIT'] - 1)/prev['CNT_PAYMENT']
kfold = KFold(n_splits=4, shuffle=True, random_state=1 )<compute_test_metric>
approved = prev[prev['NAME_CONTRACT_STATUS_Approved'] == 1]
active_df = approved[approved['DAYS_LAST_DUE'] == 365243]
active_pay = pay[pay['SK_ID_PREV'].isin(active_df['SK_ID_PREV'])]
active_pay_agg = active_pay.groupby('SK_ID_PREV')[['AMT_INSTALMENT', 'AMT_PAYMENT']].sum()
active_pay_agg.reset_index(inplace= True)
 active_pay_agg['INSTALMENT_PAYMENT_DIFF'] = active_pay_agg['AMT_INSTALMENT'] - active_pay_agg['AMT_PAYMENT']
active_df = active_df.merge(active_pay_agg, on= 'SK_ID_PREV', how= 'left')
 active_df['REMAINING_DEBT'] = active_df['AMT_CREDIT'] - active_df['AMT_PAYMENT']
active_df['REPAYMENT_RATIO'] = active_df['AMT_PAYMENT'] / active_df['AMT_CREDIT']
active_agg_df = group(active_df, 'PREV_ACTIVE_', PREVIOUS_ACTIVE_AGG)
 active_agg_df['TOTAL_REPAYMENT_RATIO'] = active_agg_df['PREV_ACTIVE_AMT_PAYMENT_SUM']/\
active_agg_df['PREV_ACTIVE_AMT_CREDIT_SUM']
del active_pay, active_pay_agg, active_df; gc.collect()
def eval_f1_score(X_val, y_val, model):
pred_val =(model.predict(X_val)>0.5)
 f1 = f1_score(y_val,pred_val)
 return f1<define_variables>
prev['DAYS_FIRST_DRAWING'].replace(365243, np.nan, inplace= True)
 prev['DAYS_FIRST_DUE'].replace(365243, np.nan, inplace= True)
 prev['DAYS_LAST_DUE_1ST_VERSION'].replace(365243, np.nan, inplace= True)
 prev['DAYS_LAST_DUE'].replace(365243, np.nan, inplace= True)
 prev['DAYS_TERMINATION'].replace(365243, np.nan, inplace= True )
EPOCHS = 100
BATCH_SIZE = 64<train_model>
prev['DAYS_LAST_DUE_DIFF'] = prev['DAYS_LAST_DUE_1ST_VERSION'] - prev['DAYS_LAST_DUE']
approved['DAYS_LAST_DUE_DIFF'] = approved['DAYS_LAST_DUE_1ST_VERSION'] - approved['DAYS_LAST_DUE']
fold = 0
history_by_fold = []
cv_results = []
for train,val in kfold.split(X_train,y_train):
nn_model = build_nn()
history = nn_model.fit(X_train[train],y_train[train],
validation_data=(X_train[val],y_train[val]),
epochs=EPOCHS, batch_size=BATCH_SIZE, verbose=0)
 scores = nn_model.evaluate(X_train[val],y_train[val],verbose=0)
 print(f"-- Fold {fold} -- ")
 print(f"{nn_model.metrics_names[0]}: {scores[0]}")
 print(f"{nn_model.metrics_names[1]}: {scores[1]}")
 print(f"F1 Score: {eval_f1_score(X_train[val],y_train[val],nn_model)}")
 cv_results.append(scores[1])
 history_by_fold.append(history)
 fold+=1
print(f"{np.mean(cv_results)} +\- {np.std(cv_results)}")
 plot_history(history )<train_model>
categorical_agg = {key: ['mean'] for key in categorical_cols}
nn_model = build_nn()
history = nn_model.fit(X_tr,y_tr, validation_data=(X_val,y_val),
epochs=EPOCHS, batch_size=BATCH_SIZE,verbose=0)
 scores= nn_model.evaluate(X_val,y_val,verbose=0)
 print(f"Accuracy: {scores[1]}")
 print(f"F1 Score: {eval_f1_score(X_val,y_val,nn_model)}" )<choose_model_class>
agg_prev = group(prev, 'PREV_', {**PREVIOUS_AGG, **categorical_agg})
 agg_prev = agg_prev.merge(active_agg_df, how='left', on='SK_ID_CURR')
 del active_agg_df; gc.collect()
def build_LSTM() :
lstm_model = tf.keras.Sequential()
lstm_model.add(layers.Input(shape=(None,300)))
 lstm_model.add(layers.LSTM(16))
lstm_model.add(layers.Dense(8, activation="tanh"))
lstm_model.add(layers.Dense(8, activation="tanh"))
lstm_model.add(layers.Dense(1,activation="sigmoid"))
lstm_model.compile(loss=tf.keras.losses.BinaryCrossentropy(from_logits=True),
optimizer=tf.keras.optimizers.Adam(5e-5),
metrics=['accuracy'])
 return lstm_model<define_variables>
agg_prev = group_and_merge(approved, agg_prev, 'APPROVED_', PREVIOUS_APPROVED_AGG)
 refused = prev[prev['NAME_CONTRACT_STATUS_Refused'] == 1]
agg_prev = group_and_merge(refused, agg_prev, 'REFUSED_', PREVIOUS_REFUSED_AGG)
 del approved, refused; gc.collect()
EPOCHS = 30;
BATCH_SIZE = 64;<choose_model_class>
for loan_type in ['Consumer loans', 'Cash loans']:
type_df = prev[prev['NAME_CONTRACT_TYPE_{}'.format(loan_type)] == 1]
prefix = 'PREV_' + loan_type.split(" ")[0] + '_'
agg_prev = group_and_merge(type_df, agg_prev, prefix, PREVIOUS_LOAN_TYPE_AGG)
 del type_df; gc.collect()
kfold = KFold(n_splits=4, shuffle=True, random_state=1 )<train_model>
pay['LATE_PAYMENT'] = pay['DAYS_ENTRY_PAYMENT'] - pay['DAYS_INSTALMENT']
pay['LATE_PAYMENT'] = pay['LATE_PAYMENT'].apply(lambda x: 1 if x > 0 else 0)
 dpd_id = pay[pay['LATE_PAYMENT'] > 0]['SK_ID_PREV'].unique()
fold = 0
history_by_fold = []
cv_results = []
for train, val in kfold.split(nlp_train,y_train):
lstm_model = build_LSTM()
history = lstm_model.fit(nlp_train[train],y_train[train],
validation_data=(nlp_train[val],y_train[val]),
epochs=EPOCHS,batch_size=BATCH_SIZE,verbose=0)
 scores = lstm_model.evaluate(nlp_train[val],y_train[val],verbose=0)
 print(f"-- Fold{fold} --")
 print(f"{lstm_model.metrics_names[0]}: {scores[0]}")
 print(f"{lstm_model.metrics_names[1]}: {scores[1]}")
 print(f"F1 Score: {eval_f1_score(nlp_train[val],y_train[val],lstm_model)}")
 cv_results.append(scores[1])
 history_by_fold.append(history)
 fold+=1
print(f"{np.mean(cv_results)} +\- {np.std(cv_results)}" )<train_model>
agg_dpd = group_and_merge(prev[prev['SK_ID_PREV'].isin(dpd_id)], agg_prev,
'PREV_LATE_', PREVIOUS_LATE_PAYMENTS_AGG)
 del agg_dpd, dpd_id; gc.collect()
lstm_model = build_LSTM()
history = lstm_model.fit(nlp_tr,y_tr,validation_data=(nlp_val,y_val), epochs=EPOCHS, batch_size=BATCH_SIZE )<predict_on_test>
df = pd.merge(df, agg_prev, on='SK_ID_CURR', how='left' )
valid_predict =(lstm_model.predict(nlp_val)> 0.5)
 f1 = f1_score(y_val, valid_predict)
 print(f" F1 Score: {f1}" )<compute_test_metric>
train = df[df['TARGET'].notnull() ]
test = df[df['TARGET'].isnull() ]
del df
del agg_prev
gc.collect()
lr_keywords = LogisticRegression(max_iter=500)
 lr_keywords.fit(kw_tr,y_tr)
 val_pred = lr_keywords.predict(kw_val)
 print(f"Accurcay: {accuracy_score(y_val, val_pred)}")
 print(f"F1 score: {f1_score(y_val,val_pred)}" )<predict_on_test>
labels = train['TARGET']
test_lebels=test['TARGET']
train = train.drop(columns=['TARGET'])
 test = test.drop(columns=['TARGET'] )
nn_tr_predict = nn_model.predict(X_tr)
 kw_tr_predict = lr_keywords.predict_proba(kw_tr)[:,1]
lstm_tr_predict = lstm_model.predict(nlp_tr)
 nn_val_predict = nn_model.predict(X_val)
 kw_val_predict = lr_keywords.predict_proba(kw_val)[:,1]
lstm_val_predict = lstm_model.predict(nlp_val)
 kw_tr_predict = kw_tr_predict.reshape(( kw_tr_predict.shape[0],1))
kw_val_predict = kw_val_predict.reshape(( kw_val_predict.shape[0],1))
concat_tr = np.concatenate(( nn_tr_predict, kw_tr_predict, lstm_tr_predict), axis=1)
 concat_val = np.concatenate(( nn_val_predict, kw_val_predict, lstm_val_predict), axis=1)
<compute_train_metric>
feature = list(train.columns)
 train.replace([np.inf, -np.inf], np.nan, inplace=True)
 test.replace([np.inf, -np.inf], np.nan, inplace=True)
 test_df = test.copy()
train_df = train.copy()
train_df['TARGET'] = labels
test_df['TARGET'] = test_lebels
lr = LogisticRegression()
lr.fit(concat_tr,y_tr)
 val_pred = lr.predict(concat_val)
 print(f"Accurcay: {accuracy_score(y_val, val_pred)}")
 print(f"F1 score: {f1_score(y_val,val_pred)}" )<prepare_x_and_y>
imputer = SimpleImputer(strategy = 'median')
 imputer.fit(train)
 imputer.fit(test)
 train1 = imputer.transform(train)
 test1 = imputer.transform(test)
 del train
del test
gc.collect()
X_test = process_data(df_test.text )<feature_engineering>
scaler = MinMaxScaler(feature_range =(0, 1))
scaler.fit(train1)
 scaler.fit(test1)
 train = scaler.transform(train1)
 test = scaler.transform(test1)
 del train1
del test1
gc.collect()
nlp_test = build_nlp_vectors(df_test.text )<predict_on_test>
from keras.models import Sequential
from keras.layers import Dense
df_test["nn_predict"]= nn_model.predict(X_test)
 df_test["lstm_predict"]= lstm_model.predict(nlp_test)
 df_test["keyword_predict"] = lr_keywords.predict_proba(keyword_test)[:,1]
features = ["nn_predict","keyword_predict","lstm_predict"]
test_features = df_test[features]
predict = lr.predict(test_features )<save_to_csv>
model_2 = Sequential([
Dense(1000, activation='relu', input_shape=(461,)) ,
Dense(1000, activation='relu'),
Dense(1000, activation='relu'),
Dense(1000, activation='relu'),
Dense(1, activation='sigmoid'),
])
 model_2.compile(optimizer='adam',
loss='binary_crossentropy',
metrics=['accuracy'])
 hist_2 = model_2.fit(train, labels,
batch_size=32, epochs=5 )
output = pd.DataFrame({"id":df_test.id, "target":predict})
 output.to_csv("submission.csv",index=False)
 output<set_options>
pred = model_2.predict_proba(test )
%matplotlib inline
InteractiveShell.ast_node_interactivity = 'all'
!pip install chart_studio
plotly.offline.init_notebook_mode(connected=True)
 cufflinks.go_offline()
cufflinks.set_config_file(world_readable=True, theme='pearl')
 warnings.filterwarnings('ignore' )<load_from_csv>
submit = test_df[['SK_ID_CURR']]
submit['TARGET'] = pred
submit.to_csv('NN.csv', index = False )