Kaggle competition: Natural Language Processing with Disaster Tweets
<SOS> metric: meanfscore Kaggle data source: natural-language-processing-with-disaster-tweets<split>
!pip install fairseq
!pip install fastBPE
!pip install transformers
folds = train.copy()
mskf = MultilabelStratifiedKFold(n_splits=5)
 for f,(t_idx, v_idx)in enumerate(mskf.split(X=train, y=target)) :
folds.loc[v_idx, 'kfold'] = int(f)
 folds['kfold'] = folds['kfold'].astype(int)
 folds<categorify>
tokenizer = TweetTokenizer()
def normalizeToken(token):
lowercased_token = token.lower()
if token.startswith("@"):
return "@USER"
elif lowercased_token.startswith("http")or lowercased_token.startswith("www"):
return "HTTPURL"
elif len(token)== 1:
return demojize(token)
 else:
if token == "’":
return "'"
elif token == "…":
return "..."
else:
return token
def normalizeTweet(tweet):
tokens = tokenizer.tokenize(tweet.replace("’", "'" ).replace("…", "..."))
normTweet = " ".join([normalizeToken(token)for token in tokens])
 normTweet = normTweet.replace("cannot ", "can not " ).replace("n't ", " n't " ).replace("n 't ", " n't " ).replace("ca n't", "can't" ).replace("ai n't", "ain't")
 normTweet = normTweet.replace("'m ", " 'm " ).replace("'re ", " 're " ).replace("'s ", " 's " ).replace("'ll ", " 'll " ).replace("'d ", " 'd " ).replace("'ve ", " 've ")
 normTweet = normTweet.replace(" p.m.", " p.m." ).replace(" p.m ", " p.m " ).replace(" a.m.", " a.m." ).replace(" a.m ", " a.m ")
 normTweet = re.sub(r",([0-9]{2,4}),([0-9]{2,4})", r",\1,\2", normTweet)
 normTweet = re.sub(r"([0-9]{1,3})/([0-9]{2,4})", r"\1/\2", normTweet)
 normTweet = re.sub(r"([0-9]{1,3})-([0-9]{2,4})", r"\1-\2", normTweet)
 return " ".join(normTweet.split() )
class MoADataset:
def __init__(self, features, targets):
self.features = features
self.targets = targets
def __len__(self):
return(self.features.shape[0])
 def __getitem__(self, idx):
dct = {
'x' : torch.tensor(self.features[idx, :], dtype=torch.float),
'y' : torch.tensor(self.targets[idx, :], dtype=torch.float)
 }
return dct
class TestDataset:
def __init__(self, features):
self.features = features
def __len__(self):
return(self.features.shape[0])
 def __getitem__(self, idx):
dct = {
'x' : torch.tensor(self.features[idx, :], dtype=torch.float)
 }
return dct 
<train_model>
class BERTweetTokenizer() :
def __init__(self,pretrained_path):
bpe_dir = os.path.join(pretrained_path,"bpe.codes")
 vocab_dir = os.path.join(pretrained_path,"dict.txt")
 self.bpe = fastBPE(SimpleNamespace(bpe_codes= bpe_dir))
self.vocab = Dictionary()
self.vocab.add_from_file(vocab_dir)
 self.cls_token_id = 0
self.pad_token_id = 1
self.sep_token_id = 2
self.pad_token = '<pad>'
self.cls_token = '
'
self.sep_token = '
'
def bpe_encode(self,text):
return self.bpe.encode(text)
 def encode(self,text,add_special_tokens=False):
subwords = self.bpe.encode(text)
 input_ids = self.vocab.encode_line(subwords, append_eos=False, add_if_not_exist=False ).long().tolist()
return input_ids
def tokenize(self,text):
return self.bpe_encode(text ).split()
def convert_tokens_to_ids(self,tokens):
input_ids = self.vocab.encode_line(' '.join(tokens), append_eos=False, add_if_not_exist=False ).long().tolist()
return input_ids
def decode_id(self,id):
return self.vocab.string(id, bpe_symbol = '@@')
 def decode_id_nospace(self,id):
return self.vocab.string(id, bpe_symbol = '@@ ')
 tokenizer = BERTweetTokenizer('/kaggle/input/bertweet-base-transformers' )
def train_fn(model, optimizer, scheduler, loss_fn, dataloader, device):
model.train()
final_loss = 0
for data in dataloader:
optimizer.zero_grad()
inputs, targets = data['x'].to(device), data['y'].to(device)
 outputs = model(inputs)
 loss = loss_fn(outputs, targets)
 loss.backward()
optimizer.step()
scheduler.step()
final_loss += loss.item()
final_loss /= len(dataloader)
 return final_loss
def valid_fn(model, loss_fn, dataloader, device):
model.eval()
final_loss = 0
valid_preds = []
for data in dataloader:
inputs, targets = data['x'].to(device), data['y'].to(device)
 outputs = model(inputs)
 loss = loss_fn(outputs, targets)
 final_loss += loss.item()
valid_preds.append(outputs.sigmoid().detach().cpu().numpy())
 final_loss /= len(dataloader)
 valid_preds = np.concatenate(valid_preds)
 return final_loss, valid_preds
def inference_fn(model, dataloader, device):
model.eval()
preds = []
for data in dataloader:
inputs = data['x'].to(device)
 with torch.no_grad() :
outputs = model(inputs)
 preds.append(outputs.sigmoid().detach().cpu().numpy())
 preds = np.concatenate(preds)
 return preds 
<normalization>
def read_train() :
train=pd.read_csv('/kaggle/input/nlp-getting-started/train.csv')
 train['text']=train['text'].astype(str)
 return train
def read_test() :
test=pd.read_csv('/kaggle/input/nlp-getting-started/test.csv')
 test['text']=test['text'].astype(str)
 return test
def read_submission() :
test=pd.read_csv('/kaggle/input/nlp-getting-started/sample_submission.csv')
 return test
train_df = read_train()
test_df = read_test()
submission_df = read_submission()
class SmoothBCEwLogits(_WeightedLoss):
def __init__(self, weight=None, reduction='mean', smoothing=0.0):
super().__init__(weight=weight, reduction=reduction)
 self.smoothing = smoothing
self.weight = weight
self.reduction = reduction
@staticmethod
def _smooth(targets:torch.Tensor, n_labels:int, smoothing=0.0):
assert 0 <= smoothing < 1
with torch.no_grad() :
targets = targets *(1.0 - smoothing)+ 0.5 * smoothing
return targets
def forward(self, inputs, targets):
targets = SmoothBCEwLogits._smooth(targets, inputs.size(-1),
self.smoothing)
 loss = F.binary_cross_entropy_with_logits(inputs, targets,self.weight)
 if self.reduction == 'sum':
loss = loss.sum()
elif self.reduction == 'mean':
loss = loss.mean()
return loss<choose_model_class>
def Data_encode(Dataset,MAX_LEN):
ct = Dataset.shape[0]
tokens = np.ones(( ct,MAX_LEN),dtype='int32')
 masks = np.zeros(( ct,MAX_LEN),dtype='int32')
 segs = np.zeros(( ct,MAX_LEN),dtype='int32')
 for k in range(ct):
text = normalizeTweet(Dataset.loc[k,'text'])
 enc = tokenizer.encode(text)
 if len(enc)<MAX_LEN-2:
tokens[k,:len(enc)+2] = [0] + enc + [2]
masks[k,:len(enc)+2] = 1
else:
tokens[k,:MAX_LEN] = [0] + enc[:MAX_LEN-2] + [2]
masks[k,:MAX_LEN] = 1
return tokens,masks,segs
train_tokens,train_masks,train_segs = Data_encode(train_df,128)
 test_tokens,test_masks,test_segs = Data_encode(test_df,128 )
class Model(nn.Module):
def __init__(self, num_features, num_targets, hidden_size):
super(Model, self ).__init__()
self.batch_norm1 = nn.BatchNorm1d(num_features)
 self.dense1 = nn.utils.weight_norm(nn.Linear(num_features, hidden_size))
self.batch_norm2 = nn.BatchNorm1d(hidden_size)
 self.dropout2 = nn.Dropout(0.4)
 self.dense2 = nn.utils.weight_norm(nn.Linear(hidden_size, hidden_size))
self.batch_norm3 = nn.BatchNorm1d(hidden_size)
 self.dropout3 = nn.Dropout(0.4)
 self.dense3 = nn.utils.weight_norm(nn.Linear(hidden_size, num_targets))
def forward(self, x):
x = self.batch_norm1(x)
 x = F.relu(self.dense1(x))
x = self.batch_norm2(x)
 x = self.dropout2(x)
 x = F.relu(self.dense2(x))
x = self.batch_norm3(x)
 x = self.dropout3(x)
 x = self.dense3(x)
 return x<categorify>
def f1(y_true,y_pred):
def recall(y_true,y_pred):
true_positives = K.sum(K.round(K.clip(y_true * y_pred,0,1)))
 possible_positives = K.sum(K.round(K.clip(y_true,0,1)))
 recall = true_positives /(possible_positives + K.epsilon())
 return recall
def precision(y_true,y_pred):
true_positives = K.sum(K.round(K.clip(y_true * y_pred,0,1)))
 prediction_positives = K.sum(K.round(K.clip(y_pred,0,1)))
 precision = true_positives /(prediction_positives + K.epsilon())
 return precision
precision = precision(y_true,y_pred)
 recall = recall(y_true,y_pred)
 return 2*(precision * recall)/(precision + recall + K.epsilon() )
def process_data(data):
data = pd.get_dummies(data, columns=['cp_time','cp_dose'])
 return data<define_variables>
def build_model(MAX_LEN,PATH = '/kaggle/input/bertweet-base-transformers/'):
ids = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 att = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 tok = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 config_dir = os.path.join(PATH,'config.json')
 model_dir = os.path.join(PATH,'model.bin')
 config = RobertaConfig.from_pretrained(config_dir)
 bert_model = TFRobertaModel.from_pretrained(model_dir,config=config,from_pt=True)
 x,_ = bert_model(ids,attention_mask=att,token_type_ids=tok)
 out=Dense(1,activation='sigmoid' )(x[:,0,:])
 model = tf.keras.models.Model(inputs=[ids, att, tok], outputs=out)
 optimizer = tf.keras.optimizers.Adam(learning_rate=1e-5)
 model.compile(loss='binary_crossentropy',
optimizer=optimizer,
metrics=['accuracy',f1])
 return model
model = build_model(128)
 model.summary()
feature_cols = [c for c in process_data(folds ).columns if c not in target_cols]
feature_cols = [c for c in feature_cols if c not in ['kfold','sig_id']]
len(feature_cols )<define_variables>
es = tf.keras.callbacks.EarlyStopping(monitor='val_loss', mode='min', patience=3, restore_best_weights=True, verbose=1 )
DEVICE =('cuda' if torch.cuda.is_available() else 'cpu')
 EPOCHS = 25
BATCH_SIZE = 128
LEARNING_RATE = 1e-3
WEIGHT_DECAY = 1e-5
NFOLDS = 5
EARLY_STOPPING_STEPS = 10
EARLY_STOP = False
num_features=len(feature_cols)
 num_targets=len(target_cols)
 hidden_size=2048 
<prepare_x_and_y>
train_inp = [train_tokens,train_masks,train_segs]
train_labels = train_df['target']
train_history = model.fit(
 train_inp, train_labels,
validation_split=0.2,
epochs=5,
batch_size=16,
verbose = 2,
callbacks = [es]
)
<categorify><EOS> 
test_inp = [test_tokens,test_masks,test_segs]
test_pred = model.predict(test_inp)
 submission_df['target']=test_pred.round().astype(int)
 submission_df.to_csv("submission.csv",index=False )