Kaggle competition: Tweet Sentiment Extraction
<SOS> metric: Jaccard Kaggle data source: tweet-sentiment-extraction<load_from_csv>
with warnings.catch_warnings() :
warnings.filterwarnings("ignore")
data = pd.read_csv('.. /input/titanic/train.csv',index_col = "PassengerId")
 test = pd.read_csv('.. /input/titanic/test.csv',index_col = "PassengerId")
<define_variables>
RUN_LOCAL = os.environ.get("PWD")!= "/kaggle/working"
RUN_LOCAL
indexs= test.index<prepare_x_and_y>
SEED = 42
if RUN_LOCAL:
NUM_EPOCHS = 1
HOME_DIR = "/workspaces/kaggle"
BATCH_SIZE = 1
SKF = StratifiedKFold(n_splits=2, shuffle=True, random_state=SEED)
 else:
NUM_EPOCHS = 3
HOME_DIR = "/kaggle"
BATCH_SIZE = 32
SKF = StratifiedKFold(n_splits=10, shuffle=True, random_state=SEED )
X = data.iloc[:,1:]
y = data.iloc[:,0]<set_options>
def seed_everything(seed_value: int)-> None:

 random.seed(seed_value)
 np.random.seed(seed_value)
 torch.manual_seed(seed_value)
 os.environ["PYTHONHASHSEED"] = str(seed_value)
 if torch.cuda.is_available() :
torch.cuda.manual_seed(seed_value)
 torch.cuda.manual_seed_all(seed_value)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = True
seed_everything(SEED )
X['Ticket'].mode<drop_column>
def _get_input_data(
 row: pd.Series,
tokenizer: tokenizers.ByteLevelBPETokenizer,
max_len: int,
bos_token_id: int,
eos_token_id: int, 
)-> Tuple[torch.Tensor, torch.Tensor, str, torch.Tensor]:

 tweet = " " + " ".join(row.text.lower().split())
 encoding = tokenizer.encode(tweet)
 sentiment_id = tokenizer.encode(row.sentiment ).ids
ids =(
 [bos_token_id]
+ sentiment_id
+ [eos_token_id, eos_token_id]
+ encoding.ids
+ [eos_token_id] 
)
 offsets = [(0, 0)] * 4 + encoding.offsets + [(0, 0)]
pad_len = max_len - len(ids)
 if pad_len > 0:
ids += [1] * pad_len
offsets += [(0, 0)] * pad_len
ids = torch.tensor(ids)
 masks = torch.where(ids != 1, torch.tensor(1), torch.tensor(0))
offsets = torch.tensor(offsets)
 assert ids.shape == torch.Size([96])
 assert masks.shape == torch.Size([96])
 assert offsets.shape == torch.Size([96, 2])
 return ids, masks, tweet, offsets
X =X.drop(columns =['Name'] )<categorify>
def _get_target_idx(row: pd.Series, tweet: str, offsets: torch.Tensor)-> Tuple[int, int]:

 assert offsets.shape == torch.Size([96, 2])
 selected_text = " " + " ".join(row.selected_text.lower().split())
 len_st = len(selected_text)- 1
idx0 = None
idx1 = None
for ind in(i for i, e in enumerate(tweet)if e == selected_text[1]):
if " " + tweet[ind : ind + len_st] == selected_text:
idx0 = ind
idx1 = ind + len_st
break
char_targets = [0] * len(tweet)
 if idx0 is not None and idx1 is not None:
for ct in range(idx0, idx1):
char_targets[ct] = 1
target_idx: List[int] = []
for j,(offset1, offset2)in enumerate(offsets):
left = offset1.item()
right = offset2.item()
assert isinstance(left, int)
 assert isinstance(right, int)
 if sum(char_targets[left:right])> 0:
target_idx.append(j)
 start_idx = target_idx[0]
end_idx = target_idx[-1]
return start_idx, end_idx
imputer_no = SimpleImputer(missing_values= np.nan ,strategy = 'mean')
 imputer_no.fit(X[['Pclass','Age','SibSp','Fare','Parch']])
 X[['Pclass','Age','SibSp','Fare','Parch']] = imputer_no.transform(X[['Pclass','Age','SibSp','Fare','Parch']])
<categorify>
def _process_row(
 row: pd.Series,
tokenizer: tokenizers.ByteLevelBPETokenizer,
max_len: int,
bos_token_id: int,
eos_token_id: int, 
)-> Dict[str, Union[torch.Tensor, str, int]]:

 data: Dict[str, Union[torch.Tensor, str, int]] = {}
ids, masks, tweet, offsets = _get_input_data(
 row, tokenizer, max_len, bos_token_id, eos_token_id 
)
 data.update({"ids": ids})
 data["masks"] = masks
data["tweet"] = tweet
data["offsets"] = offsets
if "selected_text" in row:
start_idx, end_idx = _get_target_idx(row, tweet, offsets)
 data["start_idx"] = start_idx
data["end_idx"] = end_idx
return data
imputer_cat = SimpleImputer(missing_values= np.nan ,strategy = 'most_frequent')
 imputer_cat.fit(X[['Sex','Cabin','Embarked','Ticket']])
 X[['Sex','Cabin','Embarked','Ticket']]=imputer_cat.transform(X[['Sex','Cabin','Embarked','Ticket']] )<import_modules>
class TweetDataset(Dataset):

 def __init__(self, df: pd.DataFrame, max_len: int = 96)-> None:

 self.df = df
self.max_len = max_len
self.labeled = "selected_text" in df
tokenizer = RobertaTokenizer.from_pretrained(f"{HOME_DIR}/input/roberta-base")
 tokenizer.save_vocabulary(".")
 self.bos_token_id = tokenizer.bos_token_id
self.eos_token_id = tokenizer.eos_token_id
self.tokenizer = tokenizers.ByteLevelBPETokenizer(
 vocab_file="./vocab.json",
merges_file="./merges.txt",
lowercase=True,
add_prefix_space=True, 
)
 def __getitem__(self, index: int)-> Dict[str, Union[torch.Tensor, str, int]]:

 row = self.df.iloc[index]
data = _process_row(
 row, self.tokenizer, self.max_len, self.bos_token_id, self.eos_token_id 
)
 return data
def __len__(self)-> int:

 return len(self.df)
 def get_train_val_loaders(
 df: pd.DataFrame, train_idx: np.array, val_idx: np.array, batch_size: int = 8 
)-> Dict[str, torch.utils.data.DataLoader]:

 train_df = df.iloc[train_idx]
val_df = df.iloc[val_idx]
train_loader = torch.utils.data.DataLoader(
 TweetDataset(train_df),
batch_size=batch_size,
shuffle=True,
num_workers=2,
drop_last=True, 
)
 val_loader = torch.utils.data.DataLoader(
 TweetDataset(val_df), batch_size=batch_size, shuffle=False, num_workers=2, 
)
 dataloaders_dict = {"train": train_loader, "val": val_loader}
return dataloaders_dict
def get_test_loader(df: pd.DataFrame, batch_size: int = 32)-> torch.utils.data.DataLoader:

 loader = torch.utils.data.DataLoader(
 TweetDataset(df), batch_size=batch_size, shuffle=False, num_workers=2 
)
 return loader
from sklearn.compose import ColumnTransformer
from sklearn.preprocessing import OneHotEncoder<categorify>
class TweetModel(nn.Module):

 def __init__(self)-> None:

 super(TweetModel, self ).__init__()
self.roberta = RobertaModel.from_pretrained(
 ".. /input/roberta-base", output_hidden_states=True 
)
 self.dropout = nn.Dropout(0.5)
 self.fc = nn.Linear(self.roberta.config.hidden_size, 2)
 nn.init.normal_(self.fc.weight, std=0.02)
 nn.init.normal_(self.fc.bias, 0)
 def forward(
 self, input_ids: torch.Tensor, attention_mask: torch.Tensor 
)-> Tuple[torch.Tensor, torch.Tensor]:

 assert input_ids.shape[1:] == torch.Size([96])
 assert attention_mask.shape[1:] == torch.Size([96])
 hs: Tuple[torch.Tensor,...]
_, _, hs = self.roberta(input_ids, attention_mask)
 assert len(hs)== 13
assert all(i.shape[1:] == torch.Size([96, 768])for i in hs)
 x = torch.stack([hs[-1], hs[-2], hs[-3], hs[-4]])
 assert x.shape[0] == 4
assert x.shape[2:] == torch.Size([96, 768])
 x = torch.mean(x, 0)
 assert x.shape[1:] == torch.Size([96, 768])
 x = self.dropout(x)
 x = self.fc(x)
 assert x.shape[1:] == torch.Size([96, 2])
 start_logits, end_logits = x.split(1, dim=-1)
 assert all(i.shape[1:] == torch.Size([96, 1])for i in [start_logits, end_logits])
 start_logits = start_logits.squeeze(-1)
 end_logits = end_logits.squeeze(-1)
 assert all(i.shape[1:] == torch.Size([96])for i in [start_logits, end_logits])
 return start_logits, end_logits
ct = ColumnTransformer(transformers= [('encoder',OneHotEncoder(handle_unknown='ignore'),[1,5,7,8])],remainder = 'passthrough')
 X= ct.fit_transform(X )<split>
def loss_fn(
 start_logits: torch.Tensor,
end_logits: torch.Tensor,
start_positions: torch.Tensor,
end_positions: torch.Tensor, 
)-> torch.Tensor:

 assert start_logits.shape[1:] == torch.Size([96])
 assert end_logits.shape[1:] == torch.Size([96])
 assert start_positions.shape[1:] == torch.Size([])
 assert end_positions.shape[1:] == torch.Size([])
 ce_loss = nn.CrossEntropyLoss()
start_loss = ce_loss(start_logits, start_positions)
 assert start_loss.shape == torch.Size([])
 end_loss = ce_loss(end_logits, end_positions)
 assert end_loss.shape == torch.Size([])
 total_loss = start_loss + end_loss
return total_loss
train_X,test_X,train_y,test_y = train_test_split(X,y )<train_model>
def get_selected_text(text: str, start_idx: int, end_idx: int, offsets: np.array)-> str:

 assert offsets.shape ==(96, 2)
 selected_text = ""
for ix in range(start_idx, end_idx + 1):
selected_text += text[offsets[ix][0] : offsets[ix][1]]
if(ix + 1)< len(offsets)and offsets[ix][1] < offsets[ix + 1][0]:
selected_text += " "
return selected_text
def jaccard(str1: str, str2: str)-> float:

 a = set(str1.lower().split())
 b = set(str2.lower().split())
 c = a.intersection(b)
 return float(len(c)) /(len(a)+ len(b)- len(c))
def compute_jaccard_score(
 text: str,
start_idx: int,
end_idx: int,
start_logits: np.array,
end_logits: np.array,
offsets: np.array, 
)-> float:

 assert start_logits.shape ==(96,)
 assert end_logits.shape ==(96,)
 assert offsets.shape ==(96, 2)
 start_pred = np.argmax(start_logits)
 end_pred = np.argmax(end_logits)
 if start_pred > end_pred:
pred = text
else:
pred = get_selected_text(text, start_pred, end_pred, offsets)
 true = get_selected_text(text, start_idx, end_idx, offsets)
 return jaccard(true, pred )
for i in range(10,300,10):
classifier = RandomForestClassifier(n_estimators= i,criterion='gini')
 classifier.fit(train_X, train_y)
 y_predict = classifier.predict(test_X)
 print('for {} estimators and {}'.format({i},{accuracy_score(y_true=test_y,y_pred=y_predict)})) 
<drop_column>
def loop_through_data_loader(
 model: TweetModel,
dataloaders_dict: Dict[str, torch.utils.data.DataLoader],
optimizer: optim.AdamW,
num_epochs: int,
epoch: int,
phase: str, 
)-> TweetModel:

 assert phase in ["train", "val"]
epoch_loss = 0.0
epoch_jaccard = 0.0
for data in dataloaders_dict[phase]:
ids = data["ids"].cuda()
masks = data["masks"].cuda()
tweet = data["tweet"]
offsets = data["offsets"].numpy()
start_idx = data["start_idx"].cuda()
end_idx = data["end_idx"].cuda()
optimizer.zero_grad()
with torch.set_grad_enabled(phase == "train"):
start_logits, end_logits = model(ids, masks)
 loss = loss_fn(start_logits, end_logits, start_idx, end_idx)
 if phase == "train":
loss.backward()
optimizer.step()
epoch_loss += loss.item() * len(ids)
 start_idx = start_idx.cpu().detach().numpy()
end_idx = end_idx.cpu().detach().numpy()
start_logits = torch.softmax(start_logits, dim=1 ).cpu().detach().numpy()
end_logits = torch.softmax(end_logits, dim=1 ).cpu().detach().numpy()
for i in range(len(ids)) :
jaccard_score = compute_jaccard_score(
 tweet[i],
start_idx[i],
end_idx[i],
start_logits[i],
end_logits[i],
offsets[i], 
)
 epoch_jaccard += jaccard_score
epoch_loss = epoch_loss / len(dataloaders_dict[phase].dataset)
 epoch_jaccard = epoch_jaccard / len(dataloaders_dict[phase].dataset)
 print(
 "Epoch {}/{} | {:^5} | Loss: {:.4f} | Jaccard: {:.4f}".format(
 epoch + 1, num_epochs, phase, epoch_loss, epoch_jaccard 
)
)
 return model
test =test.drop(columns =['Name'] )<categorify>
def train_one_epoch(
 model: TweetModel,
dataloaders_dict: Dict[str, torch.utils.data.DataLoader],
optimizer: optim.AdamW,
num_epochs: int,
epoch: int, 
)-> TweetModel:

 model.train()
model = loop_through_data_loader(
 model, dataloaders_dict, optimizer, num_epochs, epoch, "train", 
)
 model.eval()
model = loop_through_data_loader(
 model, dataloaders_dict, optimizer, num_epochs, epoch, "val" 
)
 return model
imputer_no.fit(test[['Pclass','Age','SibSp','Fare','Parch']])
 test[['Pclass','Age','SibSp','Fare','Parch']] = imputer_no.transform(test[['Pclass','Age','SibSp','Fare','Parch']])
 imputer_cat.fit(test[['Sex','Cabin','Embarked','Ticket']])
 test[['Sex','Cabin','Embarked','Ticket']]=imputer_cat.transform(test[['Sex','Cabin','Embarked','Ticket']])
 test= ct.transform(test)
<save_to_csv>
def train_model(
 model: TweetModel,
dataloaders_dict: Dict[str, torch.utils.data.DataLoader],
optimizer: optim.AdamW,
num_epochs: int,
filename: str, 
)-> None:

 model.cuda()
for epoch in range(num_epochs):
model = train_one_epoch(model, dataloaders_dict, optimizer, num_epochs, epoch)
 torch.save(model.state_dict() , filename )
classifier = RandomForestClassifier(n_estimators= 150,criterion='gini')
 classifier.fit(X, y)
 y_predict = classifier.predict(test)
 pd.DataFrame(y_predict,index=indexs,columns=['Survived'] ).to_csv('output.csv' )<import_modules>
%%time
if __name__ == "__main__":
if not RUN_LOCAL:
train_df = pd.read_csv(".. /input/tweet-sentiment-extraction/train.csv")
 else:
train_df = pd.read_csv(".. /input/tweet-sentiment-extraction/train.csv", nrows=10)
 train_df["text"] = train_df["text"].astype(str)
 train_df["selected_text"] = train_df["selected_text"].astype(str)
 for fold,(train_idx, val_idx)in enumerate(
 SKF.split(train_df, train_df.sentiment), start=1 
):
print(f"Fold: {fold}")
 model = TweetModel()
optimizer = optim.AdamW(model.parameters() , lr=3e-5, betas=(0.9, 0.999))
dataloaders_dict = get_train_val_loaders(train_df, train_idx, val_idx, BATCH_SIZE)
 train_model(
 model, dataloaders_dict, optimizer, NUM_EPOCHS, f"roberta_fold{fold}.pth",
)
<categorify>
%%time
if __name__ == "__main__":
if not RUN_LOCAL:
test_df = pd.read_csv(".. /input/tweet-sentiment-extraction/test.csv")
 else:
test_df = pd.read_csv(".. /input/tweet-sentiment-extraction/test.csv", nrows=1)
 test_df["text"] = test_df["text"].astype(str)
 test_loader = get_test_loader(test_df)
 predictions = []
models = []
for fold in range(SKF.n_splits):
model = TweetModel()
model.cuda()
model.load_state_dict(torch.load(f"./roberta_fold{fold+1}.pth"))
model.eval()
models.append(model)
 for data in test_loader:
ids = data["ids"].cuda()
masks = data["masks"].cuda()
tweet = data["tweet"]
offsets = data["offsets"].numpy()
start_logits = []
end_logits = []
for model in models:
with torch.no_grad() :
output = model(ids, masks)
 start_logits.append(torch.softmax(output[0], dim=1 ).cpu().detach().numpy())
 end_logits.append(torch.softmax(output[1], dim=1 ).cpu().detach().numpy())
 start_logits = np.mean(start_logits, axis=0)
 end_logits = np.mean(end_logits, axis=0)
 for i in range(len(ids)) :
start_pred = np.argmax(start_logits[i])
 end_pred = np.argmax(end_logits[i])
 if start_pred > end_pred:
pred = tweet[i]
else:
pred = get_selected_text(tweet[i], start_pred, end_pred, offsets[i])
 predictions.append(pred )
<set_options><EOS> 
if __name__ == "__main__":
if not RUN_LOCAL:
sub_df = pd.read_csv(".. /input/tweet-sentiment-extraction/sample_submission.csv")
 else:
sub_df = pd.read_csv(
 ".. /input/tweet-sentiment-extraction/sample_submission.csv", nrows=1 
)
 sub_df["selected_text"] = predictions
sub_df["selected_text"] = sub_df["selected_text"].apply(
 lambda x: x.replace("!!!!", "!")if len(x.split())== 1 else x 
)
 sub_df["selected_text"] = sub_df["selected_text"].apply(
 lambda x: x.replace(".. ", ".")if len(x.split())== 1 else x 
)
 sub_df["selected_text"] = sub_df["selected_text"].apply(
 lambda x: x.replace("...", ".")if len(x.split())== 1 else x 
)
 sub_df.to_csv("submission.csv", index=False)
 sub_df.head()