Kaggle competition: Natural Language Processing with Disaster Tweets
<SOS> metric: meanfscore Kaggle data source: natural-language-processing-with-disaster-tweets<merge>
! pip install tf-models-official==2.4.0 -q
! pip install tensorflow-gpu==2.4.1 -q
! pip install tensorflow-text==2.4.1 -q
! python -m spacy download en_core_web_sm -q
! pip install dataprep | grep -v 'already satisfied'
merged = df_test.merge(site1, left_on=['building_id', 'meter', 'timestamp'],
right_on=['building_id', 'meter', 'timestamp'], how='left' )<prepare_x_and_y>
np.set_printoptions(precision=4)
 warnings.filterwarnings('ignore' )
ytrue = merged[~merged['meter_reading'].isna() ]['meter_reading']
pred = submission[~merged['meter_reading'].isna() ]['meter_reading']<compute_test_metric>
tf.__version__
del site1, merged
print(f'RMSLE of buildings 105-155: {rmsle(ytrue, pred):.4f}' )<load_from_disk>
random.seed(319)
 np.random.seed(319)
 tf.random.set_seed(319 )
site2 = pd.read_feather('.. /input/asu-feather/site2.ft')
 site2 = site2.query('timestamp >= 2017' )<merge>
train_full = pd.read_csv('/kaggle/input/nlp-getting-started/train.csv')
 test_full = pd.read_csv('/kaggle/input/nlp-getting-started/test.csv')
 print('Training Set Shape = {}'.format(train_full.shape))
print('Training Set Memory Usage = {:.2f}MB'.format(train_full.memory_usage().sum() /2**20))
print('Test Set Shape = {}'.format(test_full.shape))
print('Test Set Memory Usage = {:.2f}MB'.format(test_full.memory_usage().sum() /2**20))
merged = df_test.merge(site2, left_on=['building_id', 'meter', 'timestamp'],
right_on=['building_id', 'meter', 'timestamp'], how='left' )<prepare_x_and_y>
df_train = pd.read_csv("/kaggle/input/disastertweet-prepared2/train_prepared.csv")
 df_test = pd.read_csv("/kaggle/input/disastertweet-prepared2/test_prepared.csv" )
ytrue = merged[~merged['meter_reading'].isna() ]['meter_reading']
pred = submission[~merged['meter_reading'].isna() ]['meter_reading']<compute_test_metric>
train_full = clean_text(train_full,'keyword')
 test_full = clean_text(test_full, 'keyword' )
del site2, merged
print(f'RMSLE of buildings 156-290: {rmsle(ytrue, pred):.4f}' )<load_from_disk>
df_train['keyword'] = train_full['keyword']
df_test['keyword'] = test_full['keyword']
site4 = pd.read_feather('.. /input/ucb-feather/site4.ft')
 site4 = site4.query('timestamp >= 2017' )<merge>
nlp_spacy = spacy.load('en_core_web_sm')
 sentence_enc = hub.load('https://tfhub.dev/google/universal-sentence-encoder/4' )
merged = df_test.merge(site4, left_on=['building_id', 'timestamp'],
right_on=['building_id', 'timestamp'], how='left' )<prepare_x_and_y>
def extract_keywords(text):
potential_keywords = []
TOP_KEYWORD = -1
pos_tag = ['ADJ', 'NOUN', 'PROPN']
doc = nlp_spacy(text)
 for i in doc:
if i.pos_ in pos_tag:
potential_keywords.append(i.text)
 document_embed = sentence_enc([text])
 potential_embed = sentence_enc(potential_keywords)
 vector_distances = cosine_similarity(document_embed, potential_embed)
 keyword = [potential_keywords[i] for i in vector_distances.argsort() [0][TOP_KEYWORD:]]
return keyword
def keyword_filler(keyword, text):
if pd.isnull(keyword):
try:
keyword = extract_keywords(text)[0]
except:
keyword = ''
return keyword
ytrue = merged[~merged['meter_reading'].isna() ]['meter_reading']
pred = submission[~merged['meter_reading'].isna() ]['meter_reading']<compute_test_metric>
df_train.keyword = pd.DataFrame(list(map(keyword_filler, df_train.keyword, df_train.text)) ).astype(str)
 df_test.keyword = pd.DataFrame(list(map(keyword_filler, df_test.keyword, df_test.text)) ).astype(str)
 print('Null Training Keywords => ', df_train['keyword'].isnull().any())
 print('Null Test Keywords => ', df_test['keyword'].isnull().any() )
del site4, merged
print(f'RMSLE of 74/91 buildings : {rmsle(ytrue, pred):.4f}' )<load_from_disk>
X_train, X_val, y_train, y_val = train_test_split(df_train[['text','keyword']],
df_train.target,
test_size=0.2,
random_state=42)
 X_train.shape, X_val.shape
site15 = pd.read_feather('.. /input/cornell-feather/site15.ft')
 site15 = site15.query('timestamp >= 2017')
 site15 = site15.drop_duplicates()<merge>
train_ds = tf.data.Dataset.from_tensor_slices(( dict(X_train), y_train))
val_ds = tf.data.Dataset.from_tensor_slices(( dict(X_val), y_val))
test_ds = tf.data.Dataset.from_tensor_slices(dict(df_test[['text','keyword']]))
merged = df_test.merge(site15, left_on=['building_id', 'meter', 'timestamp'],
right_on=['building_id', 'meter', 'timestamp'], how='left' )<prepare_x_and_y>
AUTOTUNE = tf.data.experimental.AUTOTUNE
BUFFER_SIZE = 1000
BATCH_SIZE = 32
RANDOM_SEED = 319
def configure_dataset(dataset, shuffle=False, test=False):
if shuffle:
dataset = dataset.cache() \ 
.shuffle(BUFFER_SIZE, seed=RANDOM_SEED, reshuffle_each_iteration=True)\ 
.batch(BATCH_SIZE, drop_remainder=True)\ 
.prefetch(AUTOTUNE)
 elif test:
dataset = dataset.cache() \ 
.batch(BATCH_SIZE, drop_remainder=False)\ 
.prefetch(AUTOTUNE)
 else:
dataset = dataset.cache() \ 
.batch(BATCH_SIZE, drop_remainder=True)\ 
.prefetch(AUTOTUNE)
 return dataset
ytrue = merged[~merged['meter_reading'].isna() ]['meter_reading']
pred = submission[~merged['meter_reading'].isna() ]['meter_reading']<compute_test_metric>
a3 = configure_dataset(train_ds, shuffle=True)
 dict3 = []
for elem in a3:
dict3.append(elem[0]['text'][0])
 dict3[:10]
del site15, merged
print(f'RMSLE of buildings 1325-1448: {rmsle(ytrue, pred):.4f}' )<load_from_disk>
train_ds = configure_dataset(train_ds, shuffle=True)
 val_ds = configure_dataset(val_ds)
 test_ds = configure_dataset(test_ds, test=True )
site012 = pd.read_feather('.. /input/comb-leaked-dataset/site012.ft')
 site012 = site012.query('timestamp >= 2017' )<merge>
del X_train, X_val, y_train, y_val, df_train, df_test, train_full, test_full
merged = df_test.merge(site012, left_on=['building_id', 'meter', 'timestamp'],
right_on=['building_id', 'meter', 'timestamp'], how='left' )<prepare_x_and_y>
bert_encoder_path = "https://tfhub.dev/tensorflow/bert_en_uncased_L-12_H-768_A-12/4"
bert_preprocessor_path = "https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3"
keyword_embedding_path = "https://tfhub.dev/google/nnlm-en-dim128-with-normalization/2"
ytrue = merged[~merged['meter_reading'].isna() ]['meter_reading']
pred = submission[~merged['meter_reading'].isna() ]['meter_reading']<compute_test_metric>
bert_encoder = hub.KerasLayer(bert_encoder_path, trainable=True, name="BERT_Encoder")
 bert_preprocessor = hub.KerasLayer(bert_preprocessor_path, name="BERT_Preprocessor")
 nnlm_embed = hub.KerasLayer(keyword_embedding_path, name="NNLM_Embedding" )
del site012, merged
gc.collect()
print(f'RMSLE of buildings 0-290: {rmsle(ytrue, pred):.4f}' )<define_variables>
kernel_initializer = tf.keras.initializers.GlorotNormal(seed=319)
 def create_model() :
text_input = Input(shape=() , dtype=tf.string, name="text")
 encoder_inputs = bert_preprocessor(text_input)
 encoder_outputs = bert_encoder(encoder_inputs)
 pooled_output = encoder_outputs["pooled_output"]
bert_branch = Dropout(0.1,
seed=319,
name="BERT_Dropout" )(pooled_output)
 keyword_input = Input(shape=() , dtype=tf.string, name='keyword')
 keyword_embed = nnlm_embed(keyword_input)
 keyword_flat = Flatten(name="Keyword_Flatten" )(keyword_embed)
 keyword_dense1 = Dense(128,
activation='relu',
kernel_initializer=kernel_initializer,
kernel_regularizer=regularizers.l2(1e-4),
name="Keyword_Dense1"
)(keyword_flat)
 keyword_branch1 = Dropout(0.5,
seed=319,
name='Keyword_dropout1'
)(keyword_dense1)
 keyword_dense2 = Dense(128,
activation='relu',
kernel_initializer=kernel_initializer,
kernel_regularizer=regularizers.l2(1e-4),
name="Keyword_Dense2"
)(keyword_branch1)
 keyword_branch2 = Dropout(0.5,
seed=319,
name='Keyword_dropout2'
)(keyword_dense2)
 keyword_dense3 = Dense(128,
activation='relu',
kernel_initializer=kernel_initializer,
kernel_regularizer=regularizers.l2(1e-4),
name="Keyword_Dense3"
)(keyword_branch2)
 keyword_branch3 = Dropout(0.5,
seed=319,
name='Keyword_dropout3'
)(keyword_dense3)
 merge = concatenate([bert_branch, keyword_branch3], name="Concatenate")
 dense = Dense(128,
activation='relu',
kernel_initializer=kernel_initializer,
kernel_regularizer=regularizers.l2(1e-4),
name="Merged_Dense" )(merge)
 dropout = Dropout(0.5,
seed=319,
name="Merged_Dropout"
)(dense)
 clf = Dense(1,
activation="sigmoid",
kernel_initializer=kernel_initializer,
name="Classifier"
)(dropout)
 return Model([text_input, keyword_input],
clf,
name="BERT_Classifier" )
path_data = "/kaggle/input/ashrae-energy-prediction/"
path_train = path_data + "train.csv"
path_test = path_data + "test.csv"
path_building = path_data + "building_metadata.csv"
path_weather_train = path_data + "weather_train.csv"
path_weather_test = path_data + "weather_test.csv"
path_drops = path_data + ".. /lier-list/rows_to_drop.csv"
path_model = path_data + ".. /building-models/building_meter_models.pkl"
plt.style.use("seaborn")
 sns.set(font_scale=1)
 myfavouritenumber = 0
seed = myfavouritenumber
random.seed(seed )<load_from_csv>
EPOCHS = 3
LEARNING_RATE = 5e-5
STEPS_PER_EPOCH = int(train_ds.unbatch().cardinality().numpy() / BATCH_SIZE)
 VAL_STEPS = int(val_ds.unbatch().cardinality().numpy() / BATCH_SIZE)
 TRAIN_STEPS = STEPS_PER_EPOCH * EPOCHS
WARMUP_STEPS = int(TRAIN_STEPS * 0.1)
 adamw_optimizer = create_optimizer(
 init_lr=LEARNING_RATE,
num_train_steps=TRAIN_STEPS,
num_warmup_steps=WARMUP_STEPS,
optimizer_type='adamw'
)
df_train = pd.read_csv(path_train)
 building = pd.read_csv(path_building)
 le = LabelEncoder()
building.primary_use = le.fit_transform(building.primary_use)
 weather_train = pd.read_csv(path_weather_train)
 bad_rows = pd.read_csv(path_drops)
 df_train = df_train.drop(index=bad_rows["0"] )<set_options>
bert_classifier.compile(loss=BinaryCrossentropy(from_logits=True),
optimizer=adamw_optimizer,
metrics=[BinaryAccuracy(name="accuracy")] 
)
 history = bert_classifier.fit(train_ds,
epochs=EPOCHS,
steps_per_epoch=STEPS_PER_EPOCH,
validation_data=val_ds,
validation_steps=VAL_STEPS
)
def reduce_mem_usage(df, use_float16=False):

 start_mem = df.memory_usage().sum() / 1024**2
print("Memory usage of dataframe is {:.2f} MB".format(start_mem))
for col in df.columns:
if is_datetime(df[col])or is_categorical_dtype(df[col]):
continue
col_type = df[col].dtype
if col_type != object:
c_min = df[col].min()
c_max = df[col].max()
if str(col_type)[:3] == "int":
if c_min > np.iinfo(np.int8 ).min and c_max < np.iinfo(np.int8 ).max:
df[col] = df[col].astype(np.int8)
 elif c_min > np.iinfo(np.int16 ).min and c_max < np.iinfo(np.int16 ).max:
df[col] = df[col].astype(np.int16)
 elif c_min > np.iinfo(np.int32 ).min and c_max < np.iinfo(np.int32 ).max:
df[col] = df[col].astype(np.int32)
 elif c_min > np.iinfo(np.int64 ).min and c_max < np.iinfo(np.int64 ).max:
df[col] = df[col].astype(np.int64)
 else:
if use_float16 and c_min > np.finfo(np.float16 ).min and c_max < np.finfo(np.float16 ).max:
df[col] = df[col].astype(np.float16)
 elif c_min > np.finfo(np.float32 ).min and c_max < np.finfo(np.float32 ).max:
df[col] = df[col].astype(np.float32)
 else:
df[col] = df[col].astype(np.float64)
 else:
df[col] = df[col].astype("category")
 end_mem = df.memory_usage().sum() / 1024**2
print("Memory usage after optimization is: {:.2f} MB".format(end_mem))
print("Decreased by {:.1f}%".format(100 *(start_mem - end_mem)/ start_mem))
return df<categorify>
def submission(model, test):
sample_sub = pd.read_csv('/kaggle/input/nlp-getting-started/sample_submission.csv')
 predictions = model.predict(test)
 y_preds = [ int(i)for i in np.rint(predictions)]
sub = pd.DataFrame({'id':sample_sub['id'].values.tolist() ,'target':y_preds})
 sub.to_csv('submission.csv', index=False )
<merge><EOS> 
submission(bert_classifier, test_ds )