Kaggle competition: Natural Language Processing with Disaster Tweets
<SOS> metric: meanfscore Kaggle data source: natural-language-processing-with-disaster-tweets<load_from_csv>
import numpy as np
import pandas as pd
import seaborn as sns
pcb = pd.read_csv(path + "POS_CASH_balance.csv")
 pcb.head()<load_from_csv>
train = pd.read_csv('/kaggle/input/nlp-getting-started/train.csv')
 test = pd.read_csv('/kaggle/input/nlp-getting-started/test.csv')
 submission = pd.read_csv('/kaggle/input/tweet-sentiment-extraction/sample_submission.csv' )
previous_application = pd.read_csv(path + "previous_application.csv")
 previous_application.head()<load_from_csv>
def concat_df(train, test):
return pd.concat([train, test], sort=True ).reset_index(drop=True)
 df_all = concat_df(train, test)
 print(train.shape)
 print(test.shape)
 print(df_all.shape)
 df_all.head()
installments_payments = pd.read_csv(path + "installments_payments.csv")
 installments_payments.head()<prepare_x_and_y>
features = ['keyword','location']
for feat in features :
print("The number of missing values in "+ str(feat)+" is "+str(df_all[feat].isnull().sum())+ " for the combined dataset")
 print("The number of missing values in "+ str(feat)+" is "+str(train[feat].isnull().sum())+ " for the train dataset")
 print("The number of missing values in "+ str(feat)+" is "+str(test[feat].isnull().sum())+ " for the test dataset" )
app_test = pd.read_csv('.. /input/application_test.csv')
 app_test['is_test'] = 1
app_test['is_train'] = 0
app_train['is_test'] = 0
app_train['is_train'] = 1
Y = app_train['TARGET']
train_X = app_train.drop(['TARGET'], axis = 1)
 test_id = app_test['SK_ID_CURR']
test_X = app_test
data = pd.concat([train_X, test_X], axis=0 ).set_index('SK_ID_CURR' )<categorify>
sentences = train['text']
train_size = int(7613*0.8)
 train_sentences = sentences[:train_size]
train_labels = target[:train_size]
test_sentences = sentences[train_size:]
test_labels = target[train_size:]
vocab_size = 10000
embedding_dim = 16
max_length = 120
trunc_type='post'
oov_tok = "<OOV>"
tokenizer = Tokenizer(num_words = vocab_size, oov_token=oov_tok)
 tokenizer.fit_on_texts(train_sentences)
 word_index = tokenizer.word_index
sequences = tokenizer.texts_to_sequences(train_sentences)
 padded = pad_sequences(sequences,maxlen=max_length, truncating=trunc_type)
 testing_sequences = tokenizer.texts_to_sequences(test_sentences)
 testing_padded = pad_sequences(testing_sequences,maxlen=max_length)
def _get_categorical_features(df):
feats = [col for col in list(df.columns)if df[col].dtype == 'object']
return feats
def _factorize_categoricals(df, cats):
for col in cats:
df[col], _ = pd.factorize(df[col])
 return df
def _get_dummies(df, cats):
for col in cats:
df = pd.concat([df, pd.get_dummies(df[col], prefix=col)], axis=1)
 return df
data_cats = _get_categorical_features(data)
 prev_app_cats = _get_categorical_features(previous_application)
 bureau_cats = _get_categorical_features(bureau)
 pcb_cats = _get_categorical_features(pcb)
 ccbal_cats = _get_categorical_features(credit_card_balance)
 previous_application = _get_dummies(previous_application, prev_app_cats)
 bureau = _get_dummies(bureau, bureau_cats)
 pcb = _get_dummies(pcb, pcb_cats)
 credit_card_balance = _get_dummies(credit_card_balance, ccbal_cats)
 data = _factorize_categoricals(data, data_cats )<groupby>
model = tf.keras.Sequential([
tf.keras.layers.Embedding(vocab_size, embedding_dim, input_length=max_length),
tf.keras.layers.GlobalAveragePooling1D() ,
tf.keras.layers.Dense(14, activation='relu'),
tf.keras.layers.Dense(1, activation='sigmoid')
 ])
 model.compile(loss='binary_crossentropy',optimizer='adam',metrics=['accuracy'])
 model.summary()
prev_apps_count = previous_application[['SK_ID_CURR', 'SK_ID_PREV']].groupby('SK_ID_CURR' ).count()
previous_application['SK_ID_PREV'] = previous_application['SK_ID_CURR'].map(prev_apps_count['SK_ID_PREV'])
 prev_apps_avg = previous_application.groupby('SK_ID_CURR' ).mean()

 data = data.merge(right=prev_apps_avg.reset_index() , how='left', on='SK_ID_CURR')

data.head()<merge>
num_epochs = 10
history = model.fit(padded, train_labels, epochs=num_epochs, validation_data=(testing_padded, test_labels))
bjoined = bureau_balance.merge(right=bureau[['AMT_CREDIT_SUM','SK_ID_BUREAU','SK_ID_CURR']].reset_index() , how='inner', on='SK_ID_BUREAU')
 bjoined = bjoined.merge(right=app_train[['AMT_CREDIT','SK_ID_CURR']].reset_index() , how='inner', on='SK_ID_CURR')
 bjoined['AMT_WEIGHT'] =(bjoined['AMT_CREDIT_SUM'] / bjoined['AMT_CREDIT'] ).pow (.04 ).round(1)
 bpv = pd.pivot_table(bjoined[['SK_ID_CURR','AMT_WEIGHT','YEAR']][(bjoined.AMT_WEIGHT < 1.2)&(bjoined.AMT_WEIGHT >.8)],index=['SK_ID_CURR'], columns=['AMT_WEIGHT'], aggfunc=len, fill_value=0)
 bjoined.head() 
<create_dataframe>
tokenizer_1 = Tokenizer(num_words = vocab_size, oov_token=oov_tok)
 tokenizer_1.fit_on_texts(train['text'])
 word_index = tokenizer_1.word_index
sequences = tokenizer_1.texts_to_sequences(train['text'])
 padded = pad_sequences(sequences,maxlen=max_length, truncating=trunc_type)
 true_test_sentences = test['text']
testing_sequences = tokenizer_1.texts_to_sequences(true_test_sentences)
 testing_padded = pad_sequences(testing_sequences,maxlen=max_length )
bflattened = pd.DataFrame(bpv.to_records() ).set_index(['SK_ID_CURR'])
 bflattened.columns = ['year_weight_table_' + col for col in bflattened.columns ]
bflattened.head() 
<merge>
model_2 = tf.keras.Sequential([
tf.keras.layers.Embedding(vocab_size, embedding_dim, input_length=max_length),
tf.keras.layers.GlobalAveragePooling1D() ,
tf.keras.layers.Dense(24, activation='relu'),
tf.keras.layers.Dense(1, activation='sigmoid')
 ])
 model_2.compile(loss='binary_crossentropy',optimizer='adam',metrics=['accuracy'])
 model_2.summary()
num_epochs = 10
history = model_2.fit(padded, target, epochs=num_epochs, verbose=2 )
data = data.merge(right=bflattened.reset_index() , how='left', on='SK_ID_CURR')
 gc.collect()
data.head() 
<create_dataframe>
final_output = []
for val in pred_plot.target:
if val > 0.5:
final_output.append(1)
 else:
final_output.append(0 )
<merge>
submission['target'] = final_output
submission.to_csv("final.csv", index=False)
 submission.head()