Kaggle competition: Tweet Sentiment Extraction
<SOS> metric: Jaccard Kaggle data source: tweet-sentiment-extraction<train_on_grid>
BertTokenizer,
AdamW,
BertModel,
BertForPreTraining,
BertConfig,
get_linear_schedule_with_warmup,
BertTokenizerFast,
RobertaModel,
RobertaTokenizerFast,
RobertaConfig 
)
 def to_list(tensor):
return tensor.detach().cpu().tolist()
forest = RandomForestClassifier(n_estimators=100, random_state=10)
 param_range = [5, 10, 15]
criterion = ["entropy", "gini", "error"]
param_grid = [{"n_estimators":param_range, "criterion":criterion}]
gs_forest = GridSearchCV(estimator=forest, param_grid=param_grid, scoring="accuracy", cv=10, n_jobs=-1)
 gs_forest = gs_forest.fit(X_train, y_train)
 print(gs_forest.best_score_.round(3))
print(gs_forest.best_params_ )<train_on_grid>
def process_with_offsets(args, tweet, sentiment, tokenizer):
encoded = tokenizer.encode_plus(
 sentiment,
tweet,
max_length=args.max_seq_len,
pad_to_max_length=True,
return_token_type_ids=True, 
)
 encoded["tweet"] = tweet
encoded["sentiment"] = sentiment
return encoded
xgbc = xgb.XGBClassifier(random_state=10)
 max_depth = [10, 15, 20, 25]
min_samples_leaf = [1,3,5]
min_samples_split = [1,2,4]
param_grid = [{"max_depth":max_depth,
"min_samples_leaf":min_samples_leaf, "min_samples_split":min_samples_split}]
gs_xgb = GridSearchCV(estimator=xgbc, param_grid=param_grid, scoring="accuracy", cv=5, n_jobs=-1)
 gs_xgb = gs_xgb.fit(X_train, y_train)
 print(gs_xgb.best_score_)
 print(gs_xgb.best_params_ )<train_on_grid>
class TweetDataset:
def __init__(self, args, tokenizer, df, mode="test"):
self.mode = mode
self.tweet = df.text.values
self.sentiment = df.sentiment.values
self.tokenizer = tokenizer
self.args = args
def __len__(self):
return len(self.tweet)
 def __getitem__(self, item):
tweet = str(self.tweet[item])
 sentiment = str(self.sentiment[item])
 features = process_with_offsets(
 args=self.args,
tweet=tweet,
sentiment=sentiment,
tokenizer=self.tokenizer 
)
 return {
"input_ids":torch.tensor(features["input_ids"], dtype=torch.long),
"token_type_ids":torch.tensor(features["token_type_ids"], dtype=torch.long),
"attention_mask":torch.tensor(features["attention_mask"], dtype=torch.long),
"tweet":features["tweet"],
"sentiment":features["sentiment"]
}
lgbm = lgb.LGBMClassifier()
max_depth = [3, 5, 10]
min_samples_leaf = [1,3,5,7]
min_samples_split = [2, 4, 6, 8]
param_grid = [{"max_depth":max_depth,
"min_samples_leaf":min_samples_leaf, "min_samples_split":min_samples_split}]
gs_lgbm = GridSearchCV(estimator=lgbm, param_grid=param_grid, scoring="accuracy", cv=5, n_jobs=-1)
 gs_lgbm = gs_lgbm.fit(X_train, y_train)
 print(gs_lgbm.best_score_.round(3))
print(gs_lgbm.best_params_ )<compute_train_metric>
class TweetModel(transformers.BertPreTrainedModel):
def __init__(self, model_path, conf):
super(TweetModel, self ).__init__(conf)
 self.xlmroberta = transformers.XLMRobertaModel.from_pretrained(model_path, config=conf)
 self.drop_out = nn.Dropout(0.1)
 self.l0 = nn.Linear(768 * 2, 2)
 torch.nn.init.normal_(self.l0.weight, std=0.02)
 def forward(self,input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None):
_,_, out = self.xlmroberta(
 input_ids,
attention_mask=attention_mask,
token_type_ids=token_type_ids 
)
 out = torch.cat(( out[-1], out[-2]), dim=-1)
 out = self.drop_out(out)
 logits = self.l0(out)
 start_logits, end_logits = logits.split(1, dim=-1)
 start_logits = start_logits.squeeze(-1)
 end_logits = end_logits.squeeze(-1)
 return start_logits, end_logits
print("-"*50)
 y_pred = gs_lr.best_estimator_.predict(X_test_std)
 print("Logistic Regression Result")
 print("confusion_matrix =
", confusion_matrix(y_true=y_test, y_pred=y_pred))
print("accuracy = %.3f" % accuracy_score(y_true=y_test, y_pred=y_pred))
print("precision = %.3f" % precision_score(y_true=y_test, y_pred=y_pred))
print("recall = %.3f" % recall_score(y_true=y_test, y_pred=y_pred))
print("f1_score = %.3f" % f1_score(y_true=y_test, y_pred=y_pred))
print("-"*50)
 y_pred = gs_svm.best_estimator_.predict(X_test_std)
 print("Support vector machine, SVM")
 print("confusion_matrix =
", confusion_matrix(y_true=y_test, y_pred=y_pred))
print("accuracy = %.3f" % accuracy_score(y_true=y_test, y_pred=y_pred))
print("precision = %.3f" % precision_score(y_true=y_test, y_pred=y_pred))
print("recall = %.3f" % recall_score(y_true=y_test, y_pred=y_pred))
print("f1_score = %.3f" % f1_score(y_true=y_test, y_pred=y_pred))
print("-"*50)
 y_pred = gs_knn.best_estimator_.predict(X_test_std)
 print("KNeithborsClassfier")
 print("confusion_matrix =
", confusion_matrix(y_true=y_test, y_pred=y_pred))
print("accuracy = %.3f" % accuracy_score(y_true=y_test, y_pred=y_pred))
print("precision = %.3f" % precision_score(y_true=y_test, y_pred=y_pred))
print("recall = %.3f" % recall_score(y_true=y_test, y_pred=y_pred))
print("f1_score = %.3f" % f1_score(y_true=y_test, y_pred=y_pred))
print("-"*50)
 y_pred = gs_tree.best_estimator_.predict(X_test)
 print("Decision tree")
 print("confusion_matrix =
", confusion_matrix(y_true=y_test, y_pred=y_pred))
print("accuracy = %.3f" % accuracy_score(y_true=y_test, y_pred=y_pred))
print("precision = %.3f" % precision_score(y_true=y_test, y_pred=y_pred))
print("recall = %.3f" % recall_score(y_true=y_test, y_pred=y_pred))
print("f1_score = %.3f" % f1_score(y_true=y_test, y_pred=y_pred))
print("-"*50)
 y_pred = gs_forest.best_estimator_.predict(X_test)
 print("Random Forest")
 print("confusion_matrix =
", confusion_matrix(y_true=y_test, y_pred=y_pred))
print("accuracy = %.3f" % accuracy_score(y_true=y_test, y_pred=y_pred))
print("precision = %.3f" % precision_score(y_true=y_test, y_pred=y_pred))
print("recall = %.3f" % recall_score(y_true=y_test, y_pred=y_pred))
print("f1_score = %.3f" % f1_score(y_true=y_test, y_pred=y_pred))
print("-"*50)
 y_pred = gs_xgb.best_estimator_.predict(X_test)
 print("XGB")
 print("confusion_matrix =
", confusion_matrix(y_true=y_test, y_pred=y_pred))
print("accuracy = %.3f" % accuracy_score(y_true=y_test, y_pred=y_pred))
print("precision = %.3f" % precision_score(y_true=y_test, y_pred=y_pred))
print("recall = %.3f" % recall_score(y_true=y_test, y_pred=y_pred))
print("f1_score = %.3f" % f1_score(y_true=y_test, y_pred=y_pred))
print("-"*50)
 y_pred = gs_lgbm.best_estimator_.predict(X_test)
 print("LGBM")
 print("confusion_matrix =
", confusion_matrix(y_true=y_test, y_pred=y_pred))
print("accuracy = %.3f" % accuracy_score(y_true=y_test, y_pred=y_pred))
print("precision = %.3f" % precision_score(y_true=y_test, y_pred=y_pred))
print("recall = %.3f" % recall_score(y_true=y_test, y_pred=y_pred))
print("f1_score = %.3f" % f1_score(y_true=y_test, y_pred=y_pred))
print("-"*50 )<predict_on_test>
def test(args, test_loader, model):
start_positions = []
end_positions = []
model.eval()
with torch.no_grad() :
t = tqdm(test_loader)
 for step, d in enumerate(t):
input_ids = d["input_ids"].to(args.device)
 attention_mask = d["attention_mask"].to(args.device)
 token_type_ids = d["token_type_ids"].to(args.device)
 logits1, logits2 = model(
 input_ids=input_ids,
attention_mask=attention_mask,
token_type_ids=token_type_ids,
position_ids=None,
head_mask=None 
)
 logits1 = F.softmax(logits1, dim=1 ).cpu().data.numpy()
logits2 = F.softmax(logits2, dim=1 ).cpu().data.numpy()
start_positions.append(logits1)
 end_positions.append(logits2)
 return start_positions, end_positions
test = df_test[['Pclass', 'SibSp', 'Parch', 'Sex_cate', 'Age_band', 'Fare_band', 'Embarked_flg']]
y_pred_test = gs_tree.best_estimator_.predict(test)
 submit = pd.DataFrame({"PassengerId":df_test["PassengerId"], "Survived":y_pred_test} )<save_to_csv>
class args:
max_seq_len = 128+64
batch_size = 16
args.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 args.no_cuda = False if torch.cuda.is_available() else True
args.n_gpu = 0 if args.no_cuda else torch.cuda.device_count()
submit.to_csv('my_submission.csv', index=False)
 print("Your submission was successfully saved!" )<predict_on_test>
model_path = ".. /input/xlm-roberta-base/"
config = transformers.XLMRobertaConfig.from_pretrained(model_path)
 config.output_hidden_states = True
tokenizer = transformers.XLMRobertaTokenizer.from_pretrained(model_path, do_lower_case=True)
 model = TweetModel(model_path, config)
 model.to(args.device)
 test_df = pd.read_csv(".. /input/tweet-sentiment-extraction/test.csv" )
y_pred_test_lgbm = gs_lgbm.best_estimator_.predict(test)
 y_pred_test_xgb = gs_xgb.best_estimator_.predict(test)
 y_pred_test_en =(y_pred_test*0.3 + y_pred_test_lgbm*0.4 + y_pred_test_xgb*0.3)
 submit_en = pd.DataFrame({"PassengerId":df_test["PassengerId"], "Survived":y_pred_test_en} ).round(0)
 submit_en["Survived"] = [int(i)for i in submit_en["Survived"]]<save_to_csv>
test_dataset = TweetDataset(
 args=args,
df=test_df,
mode="test",
tokenizer=tokenizer 
)
 test_loader = DataLoader(
 test_dataset,
batch_size=args.batch_size,
shuffle=False,
num_workers=4,
pin_memory=True,
drop_last=False
)
submit_en.to_csv('my_submission_en.csv', index=False)
 print("Your submission was successfully saved!" )<load_from_csv>
model_weights_list = [
".. /input/tweet-xlm-roberta-3fold-tpu/xlm-roberta-base/xlm-roberta-base/fold_0",
".. /input/tweet-xlm-roberta-3fold-tpu/xlm-roberta-base/xlm-roberta-base/fold_1",
".. /input/tweet-xlm-roberta-3fold-tpu/xlm-roberta-base/xlm-roberta-base/fold_2"
]
start_list = []
end_list = []
for path in model_weights_list:
model.load_state_dict(torch.load(path, map_location=args.device))
start_positions, end_positions = test(args, test_loader, model)
 start_positions = np.concatenate(start_positions)
 end_positions = np.concatenate(end_positions)
 start_list.append(start_positions)
 end_list.append(end_positions )
data_train = pd.read_csv("/kaggle/input/titanic/train.csv")
 data_train
data_train.info()<data_type_conversions>
start_list = np.array(start_list)
 end_list = np.array(end_list )
def set_missing_ages(df):
age_df = df[['Age','Fare', 'Parch', 'SibSp', 'Pclass']]
known_age = age_df[age_df.Age.notnull() ].as_matrix()
unknown_age = age_df[age_df.Age.isnull() ].as_matrix()
y = known_age[:, 0]
X = known_age[:, 1:]
rfr = RandomForestRegressor(random_state=0, n_estimators=2000, n_jobs=-1)
 rfr.fit(X, y)
 predictedAges = rfr.predict(unknown_age[:, 1::])
 df.loc[(df.Age.isnull()), 'Age' ] = predictedAges
return df, rfr
def set_Cabin_type(df):
df.loc[(df.Cabin.notnull()), 'Cabin' ] = "Yes"
df.loc[(df.Cabin.isnull()), 'Cabin' ] = "No"
return df
data_train, rfr = set_missing_ages(data_train)
 data_train = set_Cabin_type(data_train)
<categorify>
start_positions = np.argmax(np.mean(start_list, axis=0), axis=1)
 end_positions = np.argmax(np.mean(end_list, axis=0), axis=1 )
dummies_Cabin = pd.get_dummies(data_train['Cabin'], prefix= 'Cabin')
 dummies_Embarked = pd.get_dummies(data_train['Embarked'], prefix= 'Embarked')
 dummies_Sex = pd.get_dummies(data_train['Sex'], prefix= 'Sex')
 dummies_Pclass = pd.get_dummies(data_train['Pclass'], prefix= 'Pclass')
 df = pd.concat([data_train, dummies_Cabin, dummies_Embarked, dummies_Sex, dummies_Pclass], axis=1)
 df.drop(['Pclass', 'Name', 'Sex', 'Ticket', 'Cabin', 'Embarked'], axis=1, inplace=True)
 df 
<normalization>
sub_df = pd.read_csv(".. /input/tweet-sentiment-extraction/sample_submission.csv")
 for i,(text, sentiment)in enumerate(zip(test_df.text.values, test_df.sentiment.values)) :
idx_start = start_positions[i]
idx_end = end_positions[i]
encoded = tokenizer.encode_plus(
 sentiment,
text,
max_length=args.max_seq_len,
pad_to_max_length=True,
return_token_type_is=True, 
)
 input_id = encoded["input_ids"]
if idx_end < idx_start:
idx_end = idx_start
filtered_output = tokenizer.decode(input_id[idx_start:idx_end+1], skip_special_tokens=True)
 if sentiment == "neutral" or len(text.split())< 2:
filtered_output = text
filtered_output = filtered_output.strip()
sub_df.loc[i, "selected_text"] = filtered_output
<prepare_x_and_y><EOS> 
sub_df.to_csv("submission.csv", index=False )