Kaggle competition: Natural Language Processing with Disaster Tweets
now = datetime.now()
sub_file = 'submission_LGB_{:.4f}_{}.csv'.format(global_score, str(now.strftime('%Y-%m-%d-%H-%M')))
 sub_file_soft = 'submission_soft_LGB_{:.4f}_{}.csv'.format(global_score_soft, str(now.strftime('%Y-%m-%d-%H-%M')))
 sub_file_hard = 'submission_hard_LGB_{:.4f}_{}.csv'.format(global_score_hard, str(now.strftime('%Y-%m-%d-%H-%M')))
 sub_file_0forNonHeads = 'submission_0forNonHead_LGB_{:.4f}_{}.csv'.format(global_score, str(now.strftime('%Y-%m-%d-%H-%M')))
 y_subm.to_csv(sub_file, index=False)
 y_subm_soft.to_csv(sub_file_soft, index=False)
 y_subm_hard.to_csv(sub_file_hard, index=False )<set_options>
print("Installing Dependencies")
 !pip install --upgrade tensorflow_text tensorflow > logs.txt
!rm -rf logs.txt
print("Done" )
pd.set_option('display.max_columns', 300)
 %matplotlib inline
sns.set(style='white', context='notebook', palette='deep')
 mycols = ["
sns.set_palette(palette = mycols, n_colors = 4)
 print('My colors are set!')
 print('sklearn imported!')
 print('lightgbm imported!' )<load_from_csv>
train_set = pd.read_csv('.. /input/train.csv')
 test_set = pd.read_csv('.. /input/test.csv')
 print(f'train set has {train_set.shape[0]} rows, and {train_set.shape[1]} features')
 print(f'test set has {test_set.shape[0]} rows, and {test_set.shape[1]} features' )<count_values>
import re
import tensorflow as tf
import tensorflow_text
import string
import tensorflow_hub as hub
import matplotlib.pyplot as plt
target = train_set['Target']
target.value_counts(normalize=True )<feature_engineering>
puncts = re.compile('[%s]' % re.escape(string.punctuation))
def preprocess(df):
df['text'] = re.sub(r'&[a-zA-Z0-9]+;?', "", df['text'])
 df['text'] = re.sub("'ld", ' would', df['text'])
 df['text'] = re.sub("'d", " had", df['text'])
 df['text'] = re.sub("'ve", " have", df['text'])
 df['text'] = re.sub("'m", " am", df['text'])
 df['text'] = re.sub("n't", " not", df['text'])
 df['text'] = re.sub("won't", "would not", df['text'])
 df['text'] = re.sub(r'https?:\/\/(www\.) ?[-a-zA-Z0-9@:%._\+~
df['text'] = re.sub("@[A-Za-z0-9_]+","",df['text'])
 df['text'] = re.sub('
df['text'] = puncts.sub("", df['text'])
 df['text'] = df['text'].encode('ascii', 'ignore' ).decode('ascii')
 df['text'] = re.sub('[0-9]+[a-zA-Z]+', '<UNIT>', df['text'])
 df['text'] = re.sub('[0-9]+', '<NUMBER>', df['text'])
 df['text'] = df['text'].lower()
df['text'] = re.sub(' u ', ' you ', df['text'])
 return df
test_set.loc[test_set['rez_esc'] == 99.0 , 'rez_esc'] = 5<create_dataframe>
if not os.path.exists('/kaggle/input/albert-temp/final_version.h5'):
text_input = tf.keras.layers.Input(shape=() , dtype=tf.string)
 preprocessor = hub.KerasLayer(
 "http://tfhub.dev/tensorflow/albert_en_preprocess/2")
 encoder_inputs = preprocessor(text_input)
 encoder = hub.KerasLayer(
 "https://tfhub.dev/tensorflow/albert_en_base/2",
trainable=True)
 outputs = encoder(encoder_inputs)
 pooled_output = outputs["pooled_output"]
batch_norm = tf.keras.layers.BatchNormalization()(pooled_output)
 dense1 = tf.keras.layers.Dense(512, activation='relu' )(batch_norm)
 drop = tf.keras.layers.Dropout(0.1 )(dense1)
 final = tf.keras.layers.Dense(1, activation='sigmoid' )(drop)
 model = tf.keras.Model(inputs=text_input, outputs=final)
 model.compile(optimizer=tf.keras.optimizers.Adam(2e-5), loss='binary_crossentropy')
 model.summary()
else:
print('Loading from previous version')
 model = tf.keras.models.load_model('/kaggle/input/albert-temp/final_version.h5', custom_objects={'KerasLayer':hub.KerasLayer})
 model.summary()
data_na = train_set.isnull().sum().values / train_set.shape[0] *100
df_na = pd.DataFrame(data_na, index=train_set.columns, columns=['Count'])
 df_na = df_na.sort_values(by=['Count'], ascending=False)
 missing_value_count = df_na[df_na['Count']>0].shape[0]
print(f'We got {missing_value_count} rows which have missing value in train set ')
 df_na.head(6 )<create_dataframe>
data_na = test_set.isnull().sum().values / test_set.shape[0] *100
df_na = pd.DataFrame(data_na, index=test_set.columns, columns=['Count'])
 df_na = df_na.sort_values(by=['Count'], ascending=False)
 missing_value_count = df_na[df_na['Count']>0].shape[0]
print(f'We got {missing_value_count} rows which have missing value in test set ')
 df_na.head(6 )<feature_engineering>
test_data = pd.read_csv('/kaggle/input/nlp-getting-started/test.csv', usecols=['id','text'] )
def repalce_v18q1(x):
if x['v18q'] == 0:
return x['v18q']
else:
return x['v18q1']
train_set['v18q1'] = train_set.apply(lambda x : repalce_v18q1(x),axis=1)
 test_set['v18q1'] = test_set.apply(lambda x : repalce_v18q1(x),axis=1)
 train_set['v2a1'] = train_set['v2a1'].fillna(value=train_set['tipovivi3'])
 test_set['v2a1'] = test_set['v2a1'].fillna(value=test_set['tipovivi3'] )<data_type_conversions>
test_data = test_data.apply(preprocess, axis=1)
 test_data.head()
cols = ['edjefe', 'edjefa']
train_set[cols] = train_set[cols].replace({'no': 0, 'yes':1} ).astype(float)
 test_set[cols] = test_set[cols].replace({'no': 0, 'yes':1} ).astype(float )<categorify>
preds = model.predict(test_data['text'].tolist() )
train_set['roof_waste_material'] = np.nan
test_set['roof_waste_material'] = np.nan
train_set['electricity_other'] = np.nan
test_set['electricity_other'] = np.nan
def fill_roof_exception(x):
if(x['techozinc'] == 0)and(x['techoentrepiso'] == 0)and(x['techocane'] == 0)and(x['techootro'] == 0):
return 1
else:
return 0
def fill_no_electricity(x):
if(x['public'] == 0)and(x['planpri'] == 0)and(x['noelec'] == 0)and(x['coopele'] == 0):
return 1
else:
return 0
train_set['roof_waste_material'] = train_set.apply(lambda x : fill_roof_exception(x),axis=1)
 test_set['roof_waste_material'] = test_set.apply(lambda x : fill_roof_exception(x),axis=1)
 train_set['electricity_other'] = train_set.apply(lambda x : fill_no_electricity(x),axis=1)
 test_set['electricity_other'] = test_set.apply(lambda x : fill_no_electricity(x),axis=1 )<feature_engineering>
def apply_threshold(x, thresh=0.5):
ret = []
for i in x:
if i[0] >= thresh:
ret.append(1)
 else:
ret.append(0)
 return ret
def owner_is_adult(x):
if x['age'] <= 18:
return 0
else:
return 1
train_set['head<18'] = train_set.apply(lambda x : owner_is_adult(x),axis=1)
 test_set['head<18'] = test_set.apply(lambda x : owner_is_adult(x),axis=1 )<feature_engineering>
preds = apply_threshold(preds )
<feature_engineering><EOS> 
submission = pd.DataFrame({'id':test_data['id'].tolist() , 'target':preds})
 submission.to_csv('submission.csv', index=False )