Kaggle competition: Natural Language Processing with Disaster Tweets
<SOS> metric: meanfscore Kaggle data source: natural-language-processing-with-disaster-tweets<set_options>
import numpy as np
import pandas as pd
torch.cuda.empty_cache()<set_options>
train=pd.read_csv('.. /input/nlp-getting-started/train.csv')
 test=pd.read_csv('.. /input/nlp-getting-started/test.csv' )
!nvidia-smi<load_pretrained>
nltk.download('punkt')
 nltk.download('stopwords')
 !pip install contractions
nltk.download('wordnet')
 !pip install pyspellchecker
test_loaders = {
"test": DataLoader(dataset=test_dataset,
batch_size=BATCH_SIZE,
shuffle=False)
 }<find_best_params>
stop_words=nltk.corpus.stopwords.words('english')
 i=0
wnl=WordNetLemmatizer()
stemmer=SnowballStemmer('english')
 for doc in train.text:
doc=re.sub(r'https?://\S+|www\.\S+','',doc)
 doc=re.sub(r'<.*?>','',doc)
 doc=re.sub(r'[^a-zA-Z\s]','',doc,re.I|re.A)
 doc=' '.join([wnl.lemmatize(i)for i in doc.lower().split() ])
 doc=contractions.fix(doc)
 tokens=nltk.word_tokenize(doc)
 filtered=[token for token in tokens if token not in stop_words]
doc=' '.join(filtered)
 train.text[i]=doc
i+=1
i=0
for doc in test.text:
doc=re.sub(r'https?://\S+|www\.\S+','',doc)
 doc=re.sub(r'<.*?>','',doc)
 doc=re.sub(r'[^a-zA-Z\s]','',doc,re.I|re.A)
 doc=' '.join([wnl.lemmatize(i)for i in doc.lower().split() ])
 doc=contractions.fix(doc)
 tokens=nltk.word_tokenize(doc)
 filtered=[token for token in tokens if token not in stop_words]
doc=' '.join(filtered)
 test.text[i]=doc
i+=1
%%time
runner.infer(
 model=model,
loaders=test_loaders,
callbacks=[
CheckpointCallback(
 resume=f"{LOG_DIR}/checkpoints/best.pth" 
),
InferCallback() ,
],
verbose=True
)<predict_on_test>
!pip install tensorflow_text
predicted_probs = runner.callbacks[0].predictions['logits']<load_from_csv>
bert_model_name = 'bert_en_uncased_L-12_H-768_A-12'
map_name_to_handle = {
'bert_en_uncased_L-12_H-768_A-12':
'https://tfhub.dev/tensorflow/bert_en_uncased_L-12_H-768_A-12/3',
'bert_en_cased_L-12_H-768_A-12':
'https://tfhub.dev/tensorflow/bert_en_cased_L-12_H-768_A-12/3',
'bert_multi_cased_L-12_H-768_A-12':
'https://tfhub.dev/tensorflow/bert_multi_cased_L-12_H-768_A-12/3',
'small_bert/bert_en_uncased_L-2_H-128_A-2':
'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-2_H-128_A-2/1',
'small_bert/bert_en_uncased_L-2_H-256_A-4':
'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-2_H-256_A-4/1',
'small_bert/bert_en_uncased_L-2_H-512_A-8':
'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-2_H-512_A-8/1',
'small_bert/bert_en_uncased_L-2_H-768_A-12':
'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-2_H-768_A-12/1',
'small_bert/bert_en_uncased_L-4_H-128_A-2':
'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-4_H-128_A-2/1',
'small_bert/bert_en_uncased_L-4_H-256_A-4':
'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-4_H-256_A-4/1',
'small_bert/bert_en_uncased_L-4_H-512_A-8':
'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-4_H-512_A-8/1',
'small_bert/bert_en_uncased_L-4_H-768_A-12':
'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-4_H-768_A-12/1',
'small_bert/bert_en_uncased_L-6_H-128_A-2':
'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-6_H-128_A-2/1',
'small_bert/bert_en_uncased_L-6_H-256_A-4':
'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-6_H-256_A-4/1',
'small_bert/bert_en_uncased_L-6_H-512_A-8':
'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-6_H-512_A-8/1',
'small_bert/bert_en_uncased_L-6_H-768_A-12':
'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-6_H-768_A-12/1',
'small_bert/bert_en_uncased_L-8_H-128_A-2':
'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-8_H-128_A-2/1',
'small_bert/bert_en_uncased_L-8_H-256_A-4':
'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-8_H-256_A-4/1',
'small_bert/bert_en_uncased_L-8_H-512_A-8':
'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-8_H-512_A-8/1',
'small_bert/bert_en_uncased_L-8_H-768_A-12':
'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-8_H-768_A-12/1',
'small_bert/bert_en_uncased_L-10_H-128_A-2':
'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-10_H-128_A-2/1',
'small_bert/bert_en_uncased_L-10_H-256_A-4':
'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-10_H-256_A-4/1',
'small_bert/bert_en_uncased_L-10_H-512_A-8':
'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-10_H-512_A-8/1',
'small_bert/bert_en_uncased_L-10_H-768_A-12':
'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-10_H-768_A-12/1',
'small_bert/bert_en_uncased_L-12_H-128_A-2':
'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-12_H-128_A-2/1',
'small_bert/bert_en_uncased_L-12_H-256_A-4':
'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-12_H-256_A-4/1',
'small_bert/bert_en_uncased_L-12_H-512_A-8':
'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-12_H-512_A-8/1',
'small_bert/bert_en_uncased_L-12_H-768_A-12':
'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-12_H-768_A-12/1',
'albert_en_base':
'https://tfhub.dev/tensorflow/albert_en_base/2',
'electra_small':
'https://tfhub.dev/google/electra_small/2',
'electra_base':
'https://tfhub.dev/google/electra_base/2',
'experts_pubmed':
'https://tfhub.dev/google/experts/bert/pubmed/2',
'experts_wiki_books':
'https://tfhub.dev/google/experts/bert/wiki_books/2',
'talking-heads_base':
'https://tfhub.dev/tensorflow/talkheads_ggelu_bert_en_base/1',
}
map_model_to_preprocess = {
'bert_en_uncased_L-12_H-768_A-12':
'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',
'bert_en_cased_L-12_H-768_A-12':
'https://tfhub.dev/tensorflow/bert_en_cased_preprocess/3',
'small_bert/bert_en_uncased_L-2_H-128_A-2':
'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',
'small_bert/bert_en_uncased_L-2_H-256_A-4':
'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',
'small_bert/bert_en_uncased_L-2_H-512_A-8':
'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',
'small_bert/bert_en_uncased_L-2_H-768_A-12':
'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',
'small_bert/bert_en_uncased_L-4_H-128_A-2':
'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',
'small_bert/bert_en_uncased_L-4_H-256_A-4':
'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',
'small_bert/bert_en_uncased_L-4_H-512_A-8':
'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',
'small_bert/bert_en_uncased_L-4_H-768_A-12':
'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',
'small_bert/bert_en_uncased_L-6_H-128_A-2':
'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',
'small_bert/bert_en_uncased_L-6_H-256_A-4':
'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',
'small_bert/bert_en_uncased_L-6_H-512_A-8':
'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',
'small_bert/bert_en_uncased_L-6_H-768_A-12':
'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',
'small_bert/bert_en_uncased_L-8_H-128_A-2':
'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',
'small_bert/bert_en_uncased_L-8_H-256_A-4':
'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',
'small_bert/bert_en_uncased_L-8_H-512_A-8':
'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',
'small_bert/bert_en_uncased_L-8_H-768_A-12':
'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',
'small_bert/bert_en_uncased_L-10_H-128_A-2':
'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',
'small_bert/bert_en_uncased_L-10_H-256_A-4':
'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',
'small_bert/bert_en_uncased_L-10_H-512_A-8':
'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',
'small_bert/bert_en_uncased_L-10_H-768_A-12':
'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',
'small_bert/bert_en_uncased_L-12_H-128_A-2':
'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',
'small_bert/bert_en_uncased_L-12_H-256_A-4':
'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',
'small_bert/bert_en_uncased_L-12_H-512_A-8':
'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',
'small_bert/bert_en_uncased_L-12_H-768_A-12':
'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',
'bert_multi_cased_L-12_H-768_A-12':
'https://tfhub.dev/tensorflow/bert_multi_cased_preprocess/3',
'albert_en_base':
'https://tfhub.dev/tensorflow/albert_en_preprocess/3',
'electra_small':
'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',
'electra_base':
'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',
'experts_pubmed':
'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',
'experts_wiki_books':
'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',
'talking-heads_base':
'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',
}
tfhub_handle_encoder = map_name_to_handle[bert_model_name]
tfhub_handle_preprocess = map_model_to_preprocess[bert_model_name]
print(f'BERT model selected : {tfhub_handle_encoder}')
 print(f'Preprocess model auto-selected: {tfhub_handle_preprocess}' )
sample_sub_df = pd.read_csv(PATH_TO_DATA + 'sample_submission.csv.zip',
index_col='id' )<prepare_output>
bert_preprocess_model = hub.KerasLayer(tfhub_handle_preprocess)
 text_test = ['this is such an amazing movie!']
text_preprocessed = bert_preprocess_model(text_test)
 print(f'Keys : {list(text_preprocessed.keys())}')
 print(f'Shape : {text_preprocessed["input_word_ids"].shape}')
 print(f'Word Ids : {text_preprocessed["input_word_ids"][0, :12]}')
 print(f'Input Mask : {text_preprocessed["input_mask"][0, :12]}')
 print(f'Type Ids : {text_preprocessed["input_type_ids"][0, :12]}' )
sample_sub_df[TARGET_FIELDS] = predicted_probs<save_to_csv>
bert_model = hub.KerasLayer(tfhub_handle_encoder)
 bert_results = bert_model(text_preprocessed)
 print(f'Loaded BERT: {tfhub_handle_encoder}')
 print(f'Pooled Outputs Shape:{bert_results["pooled_output"].shape}')
 print(f'Pooled Outputs Values:{bert_results["pooled_output"][0, :12]}')
 print(f'Sequence Outputs Shape:{bert_results["sequence_output"].shape}')
 print(f'Sequence Outputs Values:{bert_results["sequence_output"][0, :12]}' )
sample_sub_df.to_csv('submissions.csv' )<set_options>
def build_classifier_model() :
text_input = tf.keras.layers.Input(shape=() , dtype=tf.string, name='text')
 preprocessing_layer = hub.KerasLayer(tfhub_handle_preprocess, name='preprocessing')
 encoder_inputs = preprocessing_layer(text_input)
 encoder = hub.KerasLayer(tfhub_handle_encoder, trainable=True, name='BERT_encoder')
 outputs = encoder(encoder_inputs)
 net = outputs['pooled_output']
net = tf.keras.layers.Dropout(0.1 )(net)
 net = tf.keras.layers.Dense(1, activation='sigmoid', name='classifier' )(net)
 return tf.keras.Model(text_input, net)
 classifier_model = build_classifier_model()
classifier_model.compile(optimizer=tf.keras.optimizers.Adam(3e-5),
loss=tf.keras.losses.BinaryCrossentropy(from_logits=True),
metrics=tf.metrics.BinaryAccuracy())
 checkpoint=tf.keras.callbacks.ModelCheckpoint('model.h5',monitor='val_loss',save_best_only=True)
 es=tf.keras.callbacks.EarlyStopping(monitor='val_loss',patience=2,restore_best_weights=True)
 history = classifier_model.fit(train.text,train.target,validation_split=0.2,epochs=10,callbacks=[checkpoint,es],batch_size=8 )
%matplotlib inline
print(os.listdir(".. /input"))
warnings.filterwarnings('ignore' )<load_from_csv>
classifier_model.load_weights('./model.h5')
 pred=classifier_model.predict(test.text )
train = pd.read_csv(".. /input/jigsawtraintest/train_jigsaw.csv")
 test= pd.read_csv(".. /input/jigsawtraintest/test_jigsaw.csv")
 EMBEDDING_FILE = '.. /input/glove840b300dtxt/glove.840B.300d.txt'<prepare_x_and_y>
pd.DataFrame(np.where(pred>0.5,1,0)).value_counts()
train["comment_text"].fillna("fillna")
 test["comment_text"].fillna("fillna")
 X_train = train["comment_text"].str.lower()
y_train = train[["toxic", "severe_toxic", "obscene", "threat", "insult", "identity_hate"]].values
X_test = test["comment_text"].str.lower()<define_variables>
pd.DataFrame({
'id':test.id,
'target':np.where(pred>0.5,1,0)[:,0]
} ).to_csv('submission.csv',index=False )