Kaggle competition: Natural Language Processing with Disaster Tweets
<SOS> metric: meanfscore Kaggle data source: natural-language-processing-with-disaster-tweets<install_modules>
seed(1)
%%bash
apt-get install -y xarchiver || true
SAVED_MODEL_PATH="/kaggle/input/tse2020-roberta-pytorch-multi-tpu-10-skfd-d"
NEW_MODEL_PATH="/kaggle/working"
for model_file in $(ls $SAVED_MODEL_PATH/*.pth)
 do
just_filename=$(basename "${model_file%.*}")
 if [[ ! -e "$NEW_MODEL_PATH/$just_filename.pth" ]]; then
echo "Copying $model_file to $NEW_MODEL_PATH"
cp $model_file /tmp
cd /tmp/
mkdir -p $just_filename
echo 3 > $just_filename/version
echo ""
zip -u $just_filename.pth $just_filename/version
echo "Moving model file $just_filename.pth to $NEW_MODEL_PATH"
mv $just_filename.pth $NEW_MODEL_PATH/$just_filename.pth
fi
echo "(After)Contents of '$just_filename/version' in the model file '$model_file'"
unzip -p $NEW_MODEL_PATH/$(basename $model_file)$just_filename/version
done<set_options>
tweet = pd.read_csv('/kaggle/input/nlp-getting-started/train.csv')
 testset = pd.read_csv('/kaggle/input/nlp-getting-started/test.csv')
 submission = pd.read_csv(".. /input/nlp-getting-started/sample_submission.csv")
 tweet.head()
warnings.filterwarnings("ignore")
<define_variables>
def create_corpus() :
corpus=[]
for x in tweet['text'].str.split() :
for i in x:
corpus.append(i)
 return corpus
MAX_LEN = 128
TRAIN_BATCH_SIZE = 32
VALID_BATCH_SIZE = 16
EPOCHS = 5
ROBERTA_PATH = "/kaggle/input/roberta-base"
TOKENIZER = tokenizers.ByteLevelBPETokenizer(
 vocab_file=f"{ROBERTA_PATH}/vocab.json",
merges_file=f"{ROBERTA_PATH}/merges.txt",
lowercase=True,
add_prefix_space=True 
)
 SAVED_MODEL_PATH="/kaggle/working/"
NUM_OF_SAVED_MODELS = 10
cpu_count = os.cpu_count()<define_variables>
def get_top_tweet_bigrams(corpus, n=None):
vec = CountVectorizer(ngram_range=(2, 2)).fit(corpus)
 bag_of_words = vec.transform(corpus)
 sum_words = bag_of_words.sum(axis=0)
 words_freq = [(word, sum_words[0, idx])for word, idx in vec.vocabulary_.items() ]
words_freq =sorted(words_freq, key = lambda x: x[1], reverse=True)
 return words_freq[:n]
def seed_everything(seed_value):
random.seed(seed_value)
 np.random.seed(seed_value)
 torch.manual_seed(seed_value)
 os.environ['PYTHONHASHSEED'] = str(seed_value)
 seed = 42
seed_everything(seed )<feature_engineering>
def build_vocab(X):
tweets = X.apply(lambda s: s.split() ).values
vocab = {}
for tweet in tweets:
for word in tweet:
try:
vocab[word] += 1
except KeyError:
vocab[word] = 1
return vocab
def check_embeddings_coverage(X, embeddings):
vocab = build_vocab(X)
 covered = {}
oov = {}
n_covered = 0
n_oov = 0
for word in vocab:
try:
covered[word] = embeddings[word]
n_covered += vocab[word]
except:
oov[word] = vocab[word]
n_oov += vocab[word]
vocab_coverage = len(covered)/ len(vocab)
 text_coverage =(n_covered /(n_covered + n_oov))
sorted_oov = sorted(oov.items() , key=operator.itemgetter(1)) [::-1]
return sorted_oov, vocab_coverage, text_coverage
class TweetDataset(torch.utils.data.Dataset):
def __init__(self, df, max_len=96):
self.df = df
self.max_len = max_len
self.labeled = 'selected_text' in df
self.tokenizer = tokenizers.ByteLevelBPETokenizer(
 vocab_file='.. /input/roberta-base/vocab.json',
merges_file='.. /input/roberta-base/merges.txt',
lowercase=True,
add_prefix_space=True)
 def __getitem__(self, index):
data = {}
row = self.df.iloc[index]
ids, masks, tweet, offsets = self.get_input_data(row)
 data['ids'] = ids
data['masks'] = masks
data['tweet'] = tweet
data['offsets'] = offsets
if self.labeled:
start_idx, end_idx = self.get_target_idx(row, tweet, offsets)
 data['start_idx'] = start_idx
data['end_idx'] = end_idx
return data
def __len__(self):
return len(self.df)
 def get_input_data(self, row):
tweet = " " + " ".join(row.text.lower().split())
 encoding = self.tokenizer.encode(tweet)
 sentiment_id = self.tokenizer.encode(row.sentiment ).ids
ids = [0] + sentiment_id + [2, 2] + encoding.ids + [2]
offsets = [(0, 0)] * 4 + encoding.offsets + [(0, 0)]
pad_len = self.max_len - len(ids)
 if pad_len > 0:
ids += [1] * pad_len
offsets += [(0, 0)] * pad_len
ids = torch.tensor(ids)
 masks = torch.where(ids != 1, torch.tensor(1), torch.tensor(0))
offsets = torch.tensor(offsets)
 return ids, masks, tweet, offsets
def get_target_idx(self, row, tweet, offsets):
selected_text = " " + " ".join(row.selected_text.lower().split())
 len_st = len(selected_text)- 1
idx0 = None
idx1 = None
for ind in(i for i, e in enumerate(tweet)if e == selected_text[1]):
if " " + tweet[ind: ind+len_st] == selected_text:
idx0 = ind
idx1 = ind + len_st - 1
break
char_targets = [0] * len(tweet)
 if idx0 != None and idx1 != None:
for ct in range(idx0, idx1 + 1):
char_targets[ct] = 1
target_idx = []
for j,(offset1, offset2)in enumerate(offsets):
if sum(char_targets[offset1: offset2])> 0:
target_idx.append(j)
 start_idx = target_idx[0]
end_idx = target_idx[-1]
return start_idx, end_idx
def get_train_val_loaders(df, train_idx, val_idx, batch_size=8):
train_df = df.iloc[train_idx]
val_df = df.iloc[val_idx]
train_loader = torch.utils.data.DataLoader(
 TweetDataset(train_df),
batch_size=batch_size,
shuffle=True,
num_workers=2,
drop_last=True)
 val_loader = torch.utils.data.DataLoader(
 TweetDataset(val_df),
batch_size=batch_size,
shuffle=False,
num_workers=2)
 dataloaders_dict = {"train": train_loader, "val": val_loader}
return dataloaders_dict
def get_test_loader(df, batch_size=32):
loader = torch.utils.data.DataLoader(
 TweetDataset(df),
batch_size=batch_size,
shuffle=False,
num_workers=2)
 return loader<load_pretrained>
embedding_dict={}
with open('.. /input/glove-global-vectors-for-word-representation/glove.6B.100d.txt','r')as f:
for line in f:
values=line.split()
word=values[0]
vectors=np.asarray(values[1:],'float32')
 embedding_dict[word]=vectors
f.close()
class TweetModel(nn.Module):
def __init__(self):
super(TweetModel, self ).__init__()
config = RobertaConfig.from_pretrained(
 f'{ROBERTA_PATH}/config.json', output_hidden_states=True)
 config.output_hidden_states = True
self.roberta = RobertaModel.from_pretrained(
 f'{ROBERTA_PATH}/pytorch_model.bin', config=config)
 self.dropout = nn.Dropout(0.5)
 self.fc = nn.Linear(config.hidden_size, 2)
 nn.init.normal_(self.fc.weight, std=0.02)
 nn.init.normal_(self.fc.bias, 0)
 def forward(self, input_ids, attention_mask):
_, _, hs = self.roberta(input_ids, attention_mask)
 x = torch.stack([hs[-1], hs[-2], hs[-3]])
 x = torch.mean(x, 0)
 x = self.dropout(x)
 x = self.fc(x)
 start_logits, end_logits = x.split(1, dim=-1)
 start_logits = start_logits.squeeze(-1)
 end_logits = end_logits.squeeze(-1)
 return start_logits, end_logits<statistical_test>
train_glove_oov, train_glove_vocab_coverage, train_glove_text_coverage = check_embeddings_coverage(tweet["text"], embedding_dict)
 test_glove_oov, test_glove_vocab_coverage, test_glove_text_coverage = check_embeddings_coverage(testset['text'], embedding_dict)
 print('GloVe Embeddings cover {:.2%} of vocabulary and {:.2%} of text in Training Set'.format(train_glove_vocab_coverage, train_glove_text_coverage))
print('GloVe Embeddings cover {:.2%} of vocabulary and {:.2%} of text in Test Set'.format(test_glove_vocab_coverage, test_glove_text_coverage))
def get_selected_text(text, start_idx, end_idx, offsets):
selected_text = ""
for ix in range(start_idx, end_idx + 1):
selected_text += text[offsets[ix][0]: offsets[ix][1]]
if(ix + 1)< len(offsets)and offsets[ix][1] < offsets[ix + 1][0]:
selected_text += " "
return selected_text
def jaccard(str1, str2):
a = set(str1.lower().split())
 b = set(str2.lower().split())
 c = a.intersection(b)
 return float(len(c)) /(len(a)+ len(b)- len(c))
def compute_jaccard_score(text, start_idx, end_idx, start_logits, end_logits, offsets):
start_pred = np.argmax(start_logits)
 end_pred = np.argmax(end_logits)
 if start_pred > end_pred:
pred = text
else:
pred = get_selected_text(text, start_pred, end_pred, offsets)
 true = get_selected_text(text, start_idx, end_idx, offsets)
 return jaccard(true, pred )<define_variables>
def utils_preprocess_text(text):
text = re.sub(r'[^\w\s]', '', str(text ).lower().strip())
 url = re.compile(r'https?://\S+|www\.\S+')
 text = url.sub(r'',text)
 url = re.compile(r'http?://\S+|www\.\S+')
 text = url.sub(r'',text)
 html=re.compile(r'<.*?>')
 html.sub(r'',text)
 text = re.sub(r'mh370','flight crash',text)
 text = re.sub(r'û_','',text)
 text = re.sub(r'ûò','',text)
 text = re.sub(r'typhoondevastated','typhoon devastated',text)
 text = re.sub(r'irandeal','iran deal',text)
 text = re.sub(r'worldnews','world news',text)
 text = re.sub(r'animalrescue','animal rescue',text)
 text = re.sub(r'viralspell','viral spell',text)
 text = re.sub(r'griefûª','grief',text)
 text = re.sub(r'pantherattack','panther attack',text)
 text = re.sub(r'injuryi495','injury in 495',text)
 text = re.sub(r'explosionproof','explosion proof',text)
 text = re.sub(r'americaûªs','americans',text)
 return text
%env JOBLIB_TEMP_FOLDER=/tmp
%env JOBLIB_START_METHOD="forkserver"
%env TMPDIR=/tmp<load_pretrained>
df=pd.concat([tweet,testset])
 df.shape
device = torch.device("cuda")
 model_config = transformers.RobertaConfig.from_pretrained(ROBERTA_PATH)
 model_config.output_hidden_states = True<load_pretrained>
df["text_clean"] = df["text"].apply(lambda x: utils_preprocess_text(x))
df.head()
%%time
models = {}
for index in range(NUM_OF_SAVED_MODELS):
model_filename=f"{SAVED_MODEL_PATH}/roberta_fold{index}.pth"
print(f"Loading model {index} from {model_filename}")
 models[index] = TweetModel()
models[index].to(device)
 models[index].load_state_dict(torch.load(model_filename))
models[index].eval()<load_from_csv>
df_glove_oov, df_glove_vocab_coverage, df_glove_text_coverage = check_embeddings_coverage(df["text_clean"], embedding_dict)
 print('GloVe Embeddings cover {:.2%} of vocabulary and {:.2%} of text in Training Set'.format(df_glove_vocab_coverage, df_glove_text_coverage))
%%time
test_df = pd.read_csv('/kaggle/input/tweet-sentiment-extraction/test.csv')
 test_df['text'] = test_df['text'].astype(str)
 test_loader = get_test_loader(test_df)
 predictions = []
for data in test_loader:
print('Reading test data via the test loader...')
 ids = data['ids'].cuda()
masks = data['masks'].cuda()
tweet = data['tweet']
offsets = data['offsets'].numpy()
start_logits = []
end_logits = []
for model in models:
print(f'Processing model {model}...')
 with torch.no_grad() :
output = models[model](ids, masks)
 start_logits.append(torch.softmax(output[0], dim=1 ).cpu().detach().numpy())
 end_logits.append(torch.softmax(output[1], dim=1 ).cpu().detach().numpy())
 print('Preparing predictions...')
 start_logits = np.mean(start_logits, axis=0)
 end_logits = np.mean(end_logits, axis=0)
 for i in range(len(ids)) :
start_pred = np.argmax(start_logits[i])
 end_pred = np.argmax(end_logits[i])
 if start_pred > end_pred:
pred = tweet[i]
else:
pred = get_selected_text(tweet[i], start_pred, end_pred, offsets[i])
 predictions.append(pred )<save_to_csv>
corpus = []
corpus = df["text_clean"]
sample = pd.read_csv(".. /input/tweet-sentiment-extraction/test.csv")
 sample.loc[:, 'selected_text'] = predictions
sample[['textID','selected_text']].to_csv("submission.csv", index=False )<load_from_csv>
MAX_LEN=50
tokenizer_obj=Tokenizer()
tokenizer_obj.fit_on_texts(corpus)
 sequences=tokenizer_obj.texts_to_sequences(corpus)
 tweet_pad=pad_sequences(sequences,maxlen=MAX_LEN,truncating='post',padding='post' )
train_data = pd.read_csv('/kaggle/input/tweet-sentiment-extraction/train.csv')
 test_data = pd.read_csv('/kaggle/input/tweet-sentiment-extraction/test.csv' )<count_missing_values>
word_index=tokenizer_obj.word_index
print('Number of unique words:',len(word_index))
print(train_data.notnull().sum())
 print(train_data.isnull().sum() )<correct_missing_values>
num_words=len(word_index)+1
embedding_matrix=np.zeros(( num_words,100))
for word,i in tqdm(word_index.items()):
if i < num_words:
emb_vec=embedding_dict.get(word)
 if emb_vec is not None:
embedding_matrix[i]=emb_vec
train_data.dropna(axis = 0,inplace=True )<count_missing_values>
model=Sequential()
embedding=Embedding(num_words,100,embeddings_initializer=Constant(embedding_matrix),
input_length=MAX_LEN,trainable=False)
 model.add(embedding)
 model.add(SpatialDropout1D(0.2))
model.add(LSTM(100, dropout=0.2, recurrent_dropout=0.2))
model.add(Dense(1, activation='sigmoid'))
optimzer=Adam(learning_rate=3e-4)
 model.compile(loss='binary_crossentropy',optimizer=optimzer,metrics=['accuracy'] )
print(test_data.notnull().sum())
 print(test_data.isnull().sum() )<feature_engineering>
train=tweet_pad[:tweet.shape[0]]
test=tweet_pad[tweet.shape[0]:]
def remove_punctuation(text):
no_punct = "".join([c for c in text if c not in string.punctuation])
 return no_punct
train_data['s_text_clean'] = train_data['selected_text'].apply(str ).apply(lambda x: remove_punctuation(x.lower()))
train_data.head(20 )<feature_engineering>
X_train,X_test,y_train,y_test=train_test_split(train,tweet['target'].values,test_size=0.2)
 print('Shape of train',X_train.shape)
 print("Shape of Validation ",X_test.shape )
tokenizer = RegexpTokenizer(r'\w+')
 train_data['s_text_tokens'] = train_data['s_text_clean'].apply(str ).apply(lambda x: tokenizer.tokenize(x))
train_data.head(20 )<feature_engineering>
history=model.fit(X_train,y_train,batch_size=32,epochs=10,validation_data=(X_test,y_test),verbose=2 )
def remove_stopwords(text):
words = [w for w in text if(w not in stopwords.words('english')or w not in 'im')]
return words
train_data['s_text_tokens_NOTstop'] = train_data['s_text_tokens'].apply(lambda x: remove_stopwords(x))
train_data.head(20 )<feature_engineering>
train_pred_GloVe = model.predict(train)
 train_pred_GloVe_int = train_pred_GloVe.round().astype('int' )
<string_transform><EOS> 
test_pred_GloVe = model.predict(test)
 test_pred_GloVe_int = test_pred_GloVe.round().astype('int')
 submission['target'] = test_pred_GloVe_int
submission.head(10)
 submission.to_csv("submission.csv", index=False, header=True)