Kaggle competition: Home Credit Default Risk
model.train_model(shuffled_training,
acc=sklearn.metrics.accuracy_score,
f1=sklearn.metrics.f1_score )<predict_on_test>
%matplotlib inline
warnings.filterwarnings('ignore')
 pd.set_option('display.max_rows', 100)
 pd.set_option('display.max_columns', 200 )
result, model_outputs, wrong_predictions = model.eval_model(shuffled_training,
acc=sklearn.metrics.accuracy_score,
f1=sklearn.metrics.f1_score )<compute_test_metric>
app_train = pd.read_csv('.. /input/home-credit-default-risk/application_train.csv')
 app_test = pd.read_csv('.. /input/home-credit-default-risk/application_test.csv' )
result<predict_on_test>
app_train.isnull().sum()
predictions, raw_outputs = model.predict(test["text"].to_list() )<create_dataframe>
app_train['TARGET'].value_counts()
mypreds = pd.DataFrame(test[["id"]])
 mypreds["target"] = predictions<save_to_csv>
columns = ['AMT_INCOME_TOTAL','AMT_CREDIT', 'AMT_ANNUITY', 'AMT_GOODS_PRICE', 'DAYS_BIRTH', 'DAYS_EMPLOYED', 'DAYS_ID_PUBLISH',
'DAYS_REGISTRATION', 'DAYS_LAST_PHONE_CHANGE', 'CNT_FAM_MEMBERS', 'REGION_RATING_CLIENT', 'EXT_SOURCE_1',
'EXT_SOURCE_2', 'EXT_SOURCE_3', 'AMT_REQ_CREDIT_BUREAU_HOUR', 'AMT_REQ_CREDIT_BUREAU_DAY', 'AMT_REQ_CREDIT_BUREAU_WEEK',
'AMT_REQ_CREDIT_BUREAU_MON', 'AMT_REQ_CREDIT_BUREAU_QRT', 'AMT_REQ_CREDIT_BUREAU_YEAR']
show_hist_by_target(app_train, columns )
mypreds.to_csv("submission.csv", index=False )<install_modules>
app_train['DAYS_BIRTH']=abs(app_train['DAYS_BIRTH'])
 app_train['DAYS_BIRTH'].corr(app_train['TARGET'] )
! python -m pip install tf-models-nightly --no-deps -q
! python -m pip install tf-models-official==2.4.0 -q
! python -m pip install tensorflow-gpu==2.4.1 -q
! python -m pip install tensorflow-text==2.4.1 -q
! python -m spacy download en_core_web_sm -q
! python -m spacy validate<import_modules>
app_train[['EXT_SOURCE_1', 'EXT_SOURCE_2', 'EXT_SOURCE_3']].isnull().sum()
print(f'TensorFlow Version: {tf.__version__}')
 print(f'Python Version: {python_version() }' )<set_options>
app_train['EXT_SOURCE_3'].value_counts(dropna=False )
RANDOM_SEED = 123
nlp = spacy.load('en_core_web_sm')
 pd.set_option('display.max_colwidth', None)
 rcParams['figure.figsize'] =(10, 6)
 sns.set_theme(palette='muted', style='whitegrid' )<load_from_csv>
cond_1 =(app_train['TARGET'] == 1)
 cond_0 =(app_train['TARGET'] == 0)
 print(app_train['CODE_GENDER'].value_counts() /app_train.shape[0])
 print('
연체인 경우
',app_train[cond_1]['CODE_GENDER'].value_counts() /app_train[cond_1].shape[0])
 print('
연체가 아닌 경우
',app_train[cond_0]['CODE_GENDER'].value_counts() /app_train[cond_0].shape[0] )
path = '.. /input/nlp-getting-started/train.csv'
df = pd.read_csv(path)
 print(df.shape)
 df.head()<load_from_csv>
app_train = pd.read_csv('.. /input/home-credit-default-risk/application_train.csv')
 app_test = pd.read_csv('.. /input/home-credit-default-risk/application_test.csv' )
path_test = '.. /input/nlp-getting-started/test.csv'
df_test = pd.read_csv(path_test)
 print(df_test.shape)
 df_test.head()<count_duplicates>
apps = pd.concat([app_train, app_test])
 print(apps.shape )
duplicates = df[df.duplicated(['text', 'target'], keep=False)]
print(f'Train Duplicate Entries(text, target): {len(duplicates)}')
 duplicates.head()<remove_duplicates>
def get_apps_processed(apps):
apps['APPS_EXT_SOURCE_MEAN'] = apps[['EXT_SOURCE_1', 'EXT_SOURCE_2', 'EXT_SOURCE_3']].mean(axis=1)
 apps['APPS_EXT_SOURCE_STD'] = apps[['EXT_SOURCE_1', 'EXT_SOURCE_2', 'EXT_SOURCE_3']].std(axis=1)
 apps['APPS_EXT_SOURCE_STD'] = apps['APPS_EXT_SOURCE_STD'].fillna(apps['APPS_EXT_SOURCE_STD'].mean())
 apps['APPS_ANNUITY_CREDIT_RATIO'] = apps['AMT_ANNUITY']/apps['AMT_CREDIT']
apps['APPS_GOODS_CREDIT_RATIO'] = apps['AMT_GOODS_PRICE']/apps['AMT_CREDIT']
apps['APPS_ANNUITY_INCOME_RATIO'] = apps['AMT_ANNUITY']/apps['AMT_INCOME_TOTAL']
apps['APPS_CREDIT_INCOME_RATIO'] = apps['AMT_CREDIT']/apps['AMT_INCOME_TOTAL']
apps['APPS_GOODS_INCOME_RATIO'] = apps['AMT_GOODS_PRICE']/apps['AMT_INCOME_TOTAL']
apps['APPS_CNT_FAM_INCOME_RATIO'] = apps['AMT_INCOME_TOTAL']/apps['CNT_FAM_MEMBERS']
apps['APPS_EMPLOYED_BIRTH_RATIO'] = apps['DAYS_EMPLOYED']/apps['DAYS_BIRTH']
apps['APPS_INCOME_EMPLOYED_RATIO'] = apps['AMT_INCOME_TOTAL']/apps['DAYS_EMPLOYED']
apps['APPS_INCOME_BIRTH_RATIO'] = apps['AMT_INCOME_TOTAL']/apps['DAYS_BIRTH']
apps['APPS_CAR_BIRTH_RATIO'] = apps['OWN_CAR_AGE'] / apps['DAYS_BIRTH']
apps['APPS_CAR_EMPLOYED_RATIO'] = apps['OWN_CAR_AGE'] / apps['DAYS_EMPLOYED']
return apps
df.drop_duplicates(['text', 'target'], inplace=True, ignore_index=True)
 print(df.shape, df_test.shape )<count_duplicates>
prev_app = pd.read_csv('.. /input/home-credit-default-risk/previous_application.csv')
 print(prev_app.shape, apps.shape )
new_duplicates = df[df.duplicated(['keyword', 'text'], keep=False)]
print(f'Train Duplicate Entries(keyword, text): {len(new_duplicates)}')
 new_duplicates[['text', 'target']].sort_values(by='text' )<drop_column>
prev_app_outer = prev_app.merge(apps['SK_ID_CURR'], on='SK_ID_CURR', how='outer', indicator=True )
df.drop([4253, 4193, 2802, 4554, 4182, 3212, 4249, 4259, 6535, 4319, 4239, 606, 3936, 6018, 5573], inplace=True )<drop_column>
prev_app_outer['_merge'].value_counts()
df = df.reset_index(drop=True)
 df<count_values>
def missing_data(data):
total = data.isnull().sum().sort_values(ascending = False)
 percent =(data.isnull().sum() /data.isnull().count() *100 ).sort_values(ascending = False)
 return pd.concat([total, percent], axis=1, keys=['Total', 'Percent'] )
df['target'].value_counts() / len(df )<count_missing_values>
prev_app.groupby('SK_ID_CURR' ).count()
def null_table(data):
null_list = []
for i in data:
if data[i].notnull().any() :
null_list.append(data[i].notnull().value_counts())
 return pd.DataFrame(pd.concat(null_list, axis=1 ).T )<count_missing_values>
prev_app.groupby('SK_ID_CURR')['SK_ID_CURR'].count()
null_table(df )<count_missing_values>
app_prev_target = prev_app.merge(app_train[['SK_ID_CURR', 'TARGET']], on='SK_ID_CURR', how='left')
 app_prev_target.shape
null_table(df_test )<define_variables>
num_columns = [column for column in num_columns if column not in ['SK_ID_PREV', 'SK_ID_CURR', 'TARGET']]
num_columns
text = df['text']
target = df['target']
test_text = df_test['text']
for i in np.random.randint(500, size=5):
print(f'Tweet
' * 2 )<define_variables>
app_prev_target.TARGET.value_counts()
lookup_dict = {
'abt' : 'about',
'afaik' : 'as far as i know',
'bc' : 'because',
'bfn' : 'bye for now',
'bgd' : 'background',
'bh' : 'blockhead',
'br' : 'best regards',
'btw' : 'by the way',
'cc': 'carbon copy',
'chk' : 'check',
'dam' : 'do not annoy me',
'dd' : 'dear daughter',
'df': 'dear fiance',
'ds' : 'dear son',
'dyk' : 'did you know',
'em': 'email',
'ema' : 'email address',
'ftf' : 'face to face',
'fb' : 'facebook',
'ff' : 'follow friday',
'fotd' : 'find of the day',
'ftw': 'for the win',
'fwiw' : 'for what it is worth',
'gts' : 'guess the song',
'hagn' : 'have a good night',
'hand' : 'have a nice day',
'hotd' : 'headline of the day',
'ht' : 'heard through',
'hth' : 'hope that helps',
'ic' : 'i see',
'icymi' : 'in case you missed it',
'idk' : 'i do not know',
'ig': 'instagram',
'iirc' : 'if i remember correctly',
'imho' : 'in my humble opinion',
'imo' : 'in my opinion',
'irl' : 'in real life',
'iwsn' : 'i want sex now',
'jk' : 'just kidding',
'jsyk' : 'just so you know',
'jv' : 'joint venture',
'kk' : 'cool cool',
'kyso' : 'knock your socks off',
'lmao' : 'laugh my ass off',
'lmk' : 'let me know',
'lo' : 'little one',
'lol' : 'laugh out loud',
'mm' : 'music monday',
'mirl' : 'meet in real life',
'mrjn' : 'marijuana',
'nbd' : 'no big deal',
'nct' : 'nobody cares though',
'njoy' : 'enjoy',
'nsfw' : 'not safe for work',
'nts' : 'note to self',
'oh' : 'overheard',
'omg': 'oh my god',
'oomf' : 'one of my friends',
'orly' : 'oh really',
'plmk' : 'please let me know',
'pnp' : 'party and play',
'qotd' : 'quote of the day',
're' : 'in reply to in regards to',
'rtq' : 'read the question',
'rt' : 'retweet',
'sfw' : 'safe for work',
'smdh' : 'shaking my damn head',
'smh' : 'shaking my head',
'so' : 'significant other',
'srs' : 'serious',
'tftf' : 'thanks for the follow',
'tftt' : 'thanks for this tweet',
'tj' : 'tweetjack',
'tl' : 'timeline',
'tldr' : 'too long did not read',
'tmb' : 'tweet me back',
'tt' : 'trending topic',
'ty' : 'thank you',
'tyia' : 'thank you in advance',
'tyt' : 'take your time',
'tyvw' : 'thank you very much',
'w': 'with',
'wtv' : 'whatever',
'ygtr' : 'you got that right',
'ykwim' : 'you know what i mean',
'ykyat' : 'you know you are addicted to',
'ymmv' : 'your mileage may vary',
'yolo' : 'you only live once',
'yoyo' : 'you are on your own',
'yt': 'youtube',
'yw' : 'you are welcome',
'zomg' : 'oh my god to the maximum'
}<string_transform>
print(app_prev_target.groupby('TARGET' ).agg({'AMT_ANNUITY': ['mean', 'median', 'count']}))
print(app_prev_target.groupby('TARGET' ).agg({'AMT_APPLICATION': ['mean', 'median', 'count']}))
print(app_prev_target.groupby('TARGET' ).agg({'AMT_CREDIT': ['mean', 'median', 'count']}))
def lemmatize_text(text, nlp=nlp):
doc = nlp(text)
 lemma_sent = [i.lemma_ for i in doc if not i.is_stop]
return ' '.join(lemma_sent)
 def abbrev_conversion(text):
words = text.split()
abbrevs_removed = []
for i in words:
if i in lookup_dict:
i = lookup_dict[i]
abbrevs_removed.append(i)
 return ' '.join(abbrevs_removed)
 def standardize_text(text_data):
entity_pattern = re.compile(r'(@[A-Za-z0-9]+)|([^0-9A-Za-z \t])|(\w+:\/\/\S+)')
 url_pattern = re.compile(r'(?:\@|http?\://|https?\://|www)\S+')
 retweet_pattern = re.compile(r'^(RT|RT:)\s+')
 digit_pattern = re.compile(r'[\d]+')
 emoji_pattern = re.compile("["
u"\U0001F600-\U0001F64F"
u"\U0001F300-\U0001F5FF"
u"\U0001F680-\U0001F6FF"
u"\U0001F1E0-\U0001F1FF"
u"\U00002500-\U00002BEF"
u"\U00002702-\U000027B0"
u"\U00002702-\U000027B0"
u"\U000024C2-\U0001F251"
u"\U0001f926-\U0001f937"
u"\U00010000-\U0010ffff"
u"\u2640-\u2642"
u"\u2600-\u2B55"
u"\u200d"
u"\u23cf"
u"\u23e9"
u"\u231a"
u"\ufe0f"
u"\u3030"
"]+", flags=re.UNICODE)
 url_strip = text_data.apply(lambda x: re.sub(url_pattern, '', x)if pd.isna(x)!= True else x)
 html_parse = url_strip.apply(lambda x: bs(x, 'html.parser' ).get_text() if pd.isna(x)!= True else x)
 retweet_strip = html_parse.apply(lambda x: re.sub(retweet_pattern, '', x)if pd.isna(x)!= True else x)
 emoji_strip = retweet_strip.apply(lambda x: re.sub(emoji_pattern, '', x)if pd.isna(x)!= True else x)
 entity_strip = emoji_strip.apply(lambda x: re.sub(entity_pattern, '', x)if pd.isna(x)!= True else x)
 lowercase = entity_strip.apply(lambda x: str.lower(x)if pd.isna(x)!= True else x)
 punct_strip = lowercase.apply(lambda x: re.sub(f'[{re.escape(string.punctuation)}]', '', x)if pd.isna(x)!= True else x)
 abbrev_converted = punct_strip.apply(lambda x: abbrev_conversion(x)if pd.isna(x)!= True else x)
 digit_strip = abbrev_converted.apply(lambda x: re.sub(digit_pattern, '', x)if pd.isna(x)!= True else x)
 lemma_and_stop = digit_strip.apply(lambda x: lemmatize_text(x)if pd.isna(x)!= True else x)
 return lemma_and_stop<create_dataframe>
app_prev_target.groupby(['NAME_CONTRACT_TYPE', 'NAME_GOODS_CATEGORY'] ).agg({'AMT_ANNUITY': ['mean', 'median', 'count', 'max']} )
df['clean_text'] = pd.DataFrame(clean_text)
 df_test['clean_text'] = pd.DataFrame(test_clean_text )<feature_engineering>
prev_app.groupby('SK_ID_CURR' )
df['clean_text'] = df['clean_text'].apply(lambda x: re.sub(pattern_new, '', x)if pd.isna(x)!= True else x)
 df_test['clean_text'] = df_test['clean_text'].apply(lambda x: re.sub(pattern_new, '', x)if pd.isna(x)!= True else x )<count_values>
prev_group = prev_app.groupby('SK_ID_CURR')
 prev_group.head()
print('Training Counts of 'new': ', len(re.findall(pattern_new, ' '.join(df['clean_text']))))
print('Test Counts of 'new': ', len(re.findall(pattern_new, ' '.join(df_test['clean_text']))))<load_pretrained>
prev_agg = pd.DataFrame()
prev_agg['CNT'] = prev_group['SK_ID_CURR'].count()
prev_agg.head()
sentence_enc = hub.load('https://tfhub.dev/google/universal-sentence-encoder/4' )<string_transform>
prev_agg['AVG_CREDIT'] = prev_group['AMT_CREDIT'].mean()
prev_agg['MAX_CREDIT'] = prev_group['AMT_CREDIT'].max()
prev_agg['MIN_CREDIT'] = prev_group['AMT_CREDIT'].min()
prev_agg.head()
def extract_keywords(text, nlp=nlp):
potential_keywords = []
TOP_KEYWORD = -1
pos_tag = ['ADJ', 'NOUN', 'PROPN']
doc = nlp(text)
 for i in doc:
if i.pos_ in pos_tag:
potential_keywords.append(i.text)
 document_embed = sentence_enc([text])
 potential_embed = sentence_enc(potential_keywords)
 vector_distances = cosine_similarity(document_embed, potential_embed)
 keyword = [potential_keywords[i] for i in vector_distances.argsort() [0][TOP_KEYWORD:]]
return keyword
def keyword_filler(keyword, text):
if pd.isnull(keyword):
try:
keyword = extract_keywords(text)[0]
except:
keyword = ''
return keyword<data_type_conversions>
prev_group = prev_app.groupby('SK_ID_CURR')
 prev_agg1 = prev_group['AMT_CREDIT'].agg(['mean', 'max', 'min'])
 prev_agg2 = prev_group['AMT_ANNUITY'].agg(['mean', 'max', 'min'])
 prev_agg = prev_agg1.merge(prev_agg2, on='SK_ID_CURR', how='inner')
 prev_agg.head()
df['keyword_fill'] = pd.DataFrame(list(map(keyword_filler, df['keyword'], df['clean_text'])) ).astype(str)
 df_test['keyword_fill'] = pd.DataFrame(list(map(keyword_filler, df_test['keyword'], df_test['clean_text'])) ).astype(str)
 print('Null Training Keywords => ', df['keyword_fill'].isnull().any())
 print('Null Test Keywords => ', df_test['keyword_fill'].isnull().any() )<create_dataframe>
prev_app['PREV_CREDIT_DIFF'] = prev_app['AMT_APPLICATION'] - prev_app['AMT_CREDIT']
prev_app['PREV_GOODS_DIFF'] = prev_app['AMT_APPLICATION'] - prev_app['AMT_GOODS_PRICE']
prev_app['PREV_CREDIT_APPL_RATIO'] = prev_app['AMT_CREDIT']/prev_app['AMT_APPLICATION']
prev_app['PREV_ANNUITY_APPL_RATIO'] = prev_app['AMT_ANNUITY']/prev_app['AMT_APPLICATION']
prev_app['PREV_GOODS_APPL_RATIO'] = prev_app['AMT_GOODS_PRICE']/prev_app['AMT_APPLICATION']
df['keyword_fill'] = pd.DataFrame(standardize_text(df['keyword_fill']))
df_test['keyword_fill'] = pd.DataFrame(standardize_text(df_test['keyword_fill']))<count_values>
prev_app['DAYS_FIRST_DRAWING'].replace(365243, np.nan, inplace=True)
 prev_app['DAYS_FIRST_DUE'].replace(365243, np.nan, inplace= True)
 prev_app['DAYS_LAST_DUE_1ST_VERSION'].replace(365243, np.nan, inplace= True)
 prev_app['DAYS_LAST_DUE'].replace(365243, np.nan, inplace= True)
 prev_app['DAYS_TERMINATION'].replace(365243, np.nan, inplace= True)
 prev_app['PREV_DAYS_LAST_DUE_DIFF'] = prev_app['DAYS_LAST_DUE_1ST_VERSION'] - prev_app['DAYS_LAST_DUE']
keyword_count_0 = pd.DataFrame(df['keyword_fill'][df['target']==0].value_counts().reset_index())
 keyword_count_1 = pd.DataFrame(df['keyword_fill'][df['target']==1].value_counts().reset_index() )<define_variables>
all_pay = prev_app['AMT_ANNUITY'] * prev_app['CNT_PAYMENT']
prev_app['PREV_INTERESTS_RATE'] =(all_pay/prev_app['AMT_CREDIT'] - 1)/prev_app['CNT_PAYMENT']
train_features = df[['clean_text','keyword_fill']]
test_features = df_test[['clean_text', 'keyword_fill']]<split>
agg_dict = {
'SK_ID_CURR':['count'],
'AMT_CREDIT':['mean', 'max', 'sum'],
'AMT_ANNUITY':['mean', 'max', 'sum'],
'AMT_APPLICATION':['mean', 'max', 'sum'],
'AMT_DOWN_PAYMENT':['mean', 'max', 'sum'],
'AMT_GOODS_PRICE':['mean', 'max', 'sum'],
'RATE_DOWN_PAYMENT': ['min', 'max', 'mean'],
'DAYS_DECISION': ['min', 'max', 'mean'],
'CNT_PAYMENT': ['mean', 'sum'],
'PREV_CREDIT_DIFF':['mean', 'max', 'sum'],
'PREV_CREDIT_APPL_RATIO':['mean', 'max'],
'PREV_GOODS_DIFF':['mean', 'max', 'sum'],
'PREV_GOODS_APPL_RATIO':['mean', 'max'],
'PREV_DAYS_LAST_DUE_DIFF':['mean', 'max', 'sum'],
'PREV_INTERESTS_RATE':['mean', 'max']
}
train_x, val_x, train_y, val_y = train_test_split(
 train_features,
target,
test_size=0.2,
random_state=RANDOM_SEED, 
)
 print(train_x.shape)
 print(train_y.shape)
 print(val_x.shape)
 print(val_y.shape )<create_dataframe>
prev_group = prev_app.groupby('SK_ID_CURR')
 prev_amt_agg = prev_group.agg(agg_dict)
 prev_amt_agg.columns = ['PREV_'+('_' ).join(column ).upper() for column in prev_amt_agg.columns.ravel() ]
train_ds = tf.data.Dataset.from_tensor_slices(( dict(train_x), train_y))
val_ds = tf.data.Dataset.from_tensor_slices(( dict(val_x), val_y))
test_ds = tf.data.Dataset.from_tensor_slices(dict(test_features))<categorify>
prev_app['NAME_CONTRACT_STATUS'].value_counts()
AUTOTUNE = tf.data.experimental.AUTOTUNE
BUFFER_SIZE = 1000
BATCH_SIZE = 32
def configure_dataset(dataset, shuffle=False, test=False):
if shuffle:
dataset = dataset.cache() \ 
.shuffle(BUFFER_SIZE, seed=RANDOM_SEED, reshuffle_each_iteration=True)\ 
.batch(BATCH_SIZE, drop_remainder=True ).prefetch(AUTOTUNE)
 elif test:
dataset = dataset.cache() \ 
.batch(BATCH_SIZE, drop_remainder=False ).prefetch(AUTOTUNE)
 else:
dataset = dataset.cache() \ 
.batch(BATCH_SIZE, drop_remainder=True ).prefetch(AUTOTUNE)
 return dataset<prepare_x_and_y>
prev_refused_agg = prev_refused.groupby('SK_ID_CURR')['SK_ID_CURR'].count()
prev_refused_agg.shape, prev_amt_agg.shape
train_ds = configure_dataset(train_ds, shuffle=True)
 val_ds = configure_dataset(val_ds)
 test_ds = configure_dataset(test_ds, test=True )<categorify>
pd.DataFrame(prev_refused_agg )
bert_preprocessor = hub.KerasLayer('https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3', name='BERT_preprocesser')
 bert_encoder = hub.KerasLayer('https://tfhub.dev/tensorflow/bert_en_uncased_L-12_H-768_A-12/4', trainable=True, name='BERT_encoder')
 nnlm_embed = hub.KerasLayer('https://tfhub.dev/google/nnlm-en-dim50/2', name='embedding_layer' )<categorify>
prev_refused_agg.reset_index(name='PREV_REFUSED_COUNT' )
def build_model() :
text_input = layers.Input(shape=() , dtype=tf.string, name='clean_text')
 encoder_inputs = bert_preprocessor(text_input)
 encoder_outputs = bert_encoder(encoder_inputs)
 pooled_output = encoder_outputs["pooled_output"]
bert_dropout = layers.Dropout(0.1, name='BERT_dropout' )(pooled_output)
 key_input = layers.Input(shape=() , dtype=tf.string, name='keyword_fill')
 key_embed = nnlm_embed(key_input)
 key_flat = layers.Flatten()(key_embed)
 key_dense = layers.Dense(128, activation='elu', kernel_regularizer=regularizers.l2(1e-4))(key_flat)
 key_dropout = layers.Dropout(0.5, name='dense_dropout' )(key_dense)
 merge = layers.concatenate([bert_dropout, key_dropout])
 dense = layers.Dense(128, activation='elu', kernel_regularizer=regularizers.l2(1e-4))(merge)
 dropout = layers.Dropout(0.5, name='merged_dropout' )(dense)
 clf = layers.Dense(1, activation='sigmoid', name='classifier' )(dropout)
 return Model([text_input, key_input], clf, name='BERT_classifier' )<init_hyperparams>
prev_refused_agg = prev_refused_agg.reset_index(name='PREV_REFUSED_COUNT')
 prev_amt_agg = prev_amt_agg.reset_index()
prev_amt_refused_agg = prev_amt_agg.merge(prev_refused_agg, on='SK_ID_CURR', how='left')
 prev_amt_refused_agg.head(10 )
EPOCHS = 2
LEARNING_RATE = 5e-5
STEPS_PER_EPOCH = int(train_ds.unbatch().cardinality().numpy() / BATCH_SIZE)
 VAL_STEPS = int(val_ds.unbatch().cardinality().numpy() / BATCH_SIZE)
 TRAIN_STEPS = STEPS_PER_EPOCH * EPOCHS
WARMUP_STEPS = int(TRAIN_STEPS * 0.1)
 adamw_optimizer = create_optimizer(
 init_lr=LEARNING_RATE,
num_train_steps=TRAIN_STEPS,
num_warmup_steps=WARMUP_STEPS
)<train_model>
prev_amt_refused_agg['PREV_REFUSED_COUNT'].value_counts(dropna=False )
bert_classifier.compile(
 loss=BinaryCrossentropy(from_logits=True),
optimizer= adamw_optimizer,
metrics=[BinaryAccuracy(name='accuracy')] 
)
 history = bert_classifier.fit(
 train_ds,
epochs=EPOCHS,
steps_per_epoch=STEPS_PER_EPOCH,
validation_data= val_ds,
validation_steps=VAL_STEPS
)<find_best_params>
prev_amt_refused_agg = prev_amt_refused_agg.fillna(0)
 prev_amt_refused_agg['PREV_REFUSE_RATIO'] = prev_amt_refused_agg['PREV_REFUSED_COUNT'] / prev_amt_refused_agg['PREV_SK_ID_CURR_COUNT']
prev_amt_refused_agg.head(10 )
train_loss = history.history['loss']
val_loss = history.history['val_loss']
train_acc = history.history['accuracy']
val_acc = history.history['val_accuracy']<define_variables>
prev_refused_appr_group = prev_app[prev_app['NAME_CONTRACT_STATUS'].isin(['Approved', 'Refused'])].groupby(['SK_ID_CURR', 'NAME_CONTRACT_STATUS'])
 prev_refused_appr_agg = prev_refused_appr_group['SK_ID_CURR'].count().unstack()
prev_refused_appr_agg.head(10 )
val_target = np.asarray([i[1] for i in list(val_ds.unbatch().as_numpy_iterator())])
 print(val_target.shape)
 val_target[:5]<predict_on_test>
prev_refused_appr_agg = prev_refused_appr_agg.fillna(0)
 prev_refused_appr_agg.columns = ['PREV_APPROVED_COUNT', 'PREV_REFUSED_COUNT']
prev_refused_appr_agg = prev_refused_appr_agg.reset_index()
prev_refused_appr_agg.head(10 )
val_predict = bert_classifier.predict(val_ds )<predict_on_test>
prev_agg = prev_amt_agg.merge(prev_refused_appr_agg, on='SK_ID_CURR', how='left')
 prev_agg['PREV_REFUSED_RATIO'] = prev_agg['PREV_REFUSED_COUNT']/prev_agg['PREV_SK_ID_CURR_COUNT']
prev_agg['PREV_APPROVED_RATIO'] = prev_agg['PREV_APPROVED_COUNT']/prev_agg['PREV_SK_ID_CURR_COUNT']
prev_agg = prev_agg.drop(['PREV_REFUSED_COUNT', 'PREV_APPROVED_COUNT'], axis=1)
 prev_agg.head(30 )
predictions = bert_classifier.predict(test_ds)
 print(predictions.shape)
 print(predictions[:5] )<prepare_output>
apps_all = get_apps_processed(apps )
predictions = np.where(predictions > THRESHOLD, 1, 0)
 df_predictions = pd.DataFrame(predictions)
 df_predictions.columns = ['target']
print(df_predictions.shape)
 df_predictions.head()<save_to_csv>
print(apps_all.shape, prev_agg.shape)
 apps_all = apps_all.merge(prev_agg, on='SK_ID_CURR', how='left')
 print(apps_all.shape )
submission = pd.concat([df_test['id'], df_predictions], axis=1)
 submission.to_csv('submission.csv', index=False )<load_from_csv>
object_columns = apps_all.dtypes[apps_all.dtypes == 'object'].index.tolist()
for column in object_columns:
apps_all[column] = pd.factorize(apps_all[column])[0]
train_filepath = '/kaggle/input/nlp-getting-started/train.csv'
test_filepath = '/kaggle/input/nlp-getting-started/test.csv'
df_train = pd.read_csv(train_filepath)
 df_test = pd.read_csv(test_filepath)
 df_train.head() 
<feature_engineering>
apps_all_train = apps_all[~apps_all['TARGET'].isnull() ]
apps_all_test = apps_all[apps_all['TARGET'].isnull() ]
apps_all_test = apps_all_test.drop('TARGET', axis=1 )
lemmatizer = WordNetLemmatizer()
for i in range(0, len(df_train)) :
text = re.sub('[^a-zA-Z]', ' ', df_train['text'][i])
 text = text.lower()
text = re.sub(r'^https?:\/\/.*[\r
]*', '', text)
 text = text.split()
text = [lemmatizer.lemmatize(word)for word in text if word not in stopwords.words('english')]
text = ' '.join(text)
 df_train['text'][i] = text
df_train.head()<feature_engineering>
ftr_app = apps_all_train.drop(['SK_ID_CURR', 'TARGET'], axis=1)
 target_app = apps_all_train['TARGET']
train_x, valid_x, train_y, valid_y = train_test_split(ftr_app, target_app, test_size=0.3, random_state=2020)
 train_x.shape, valid_x.shape
for i in range(0, len(df_test)) :
text = re.sub('[^a-zA-Z]', ' ', df_test['text'][i])
 text = text.lower()
text = re.sub(r'^https?:\/\/.*[\r
]*', '', text)
 text = text.split()
text = [lemmatizer.lemmatize(word)for word in text if word not in stopwords.words('english')]
text = ' '.join(text)
 df_test['text'][i] = text
df_test.head()<save_to_csv>
clf = LGBMClassifier(
 n_jobs=-1,
n_estimators=1000,
learning_rate=0.02,
num_leaves=32,
subsample=0.8,
max_depth=12,
silent=-1,
verbose=-1 
)
 clf.fit(train_x, train_y, eval_set=[(train_x, train_y),(valid_x, valid_y)], eval_metric= 'auc', verbose= 100,
early_stopping_rounds= 50 )
train_data = df_train.drop(['id','keyword','location'], axis=1)
 train_data.to_csv('cleaned_train.csv', index=False)
 test_data = df_test.drop(['keyword','location'], axis=1)
 test_data.to_csv('cleaned_test.csv', index=False )<load_from_csv>
preds = clf.predict_proba(apps_all_test.drop('SK_ID_CURR', axis=1)) [:, 1 ]
apps_all_test['TARGET'] = preds
apps_all_test[['SK_ID_CURR', 'TARGET']].to_csv('prev_baseline_03.csv', index=False )