Kaggle competition: Tweet Sentiment Extraction
test['Ticket_Grp'] = test[['Ticket_First']].apply(Ticket_Grp2, axis =1 )<groupby>
%%time
!python.. /input/tweet-inference-scripts/inference_distilbert.py
PclassXSex_med = test[['Sex','Age','Pclass']].groupby(['Sex','Pclass'] ).median()
PclassXSex_med<feature_engineering>
%%time
!python.. /input/tweet-inference-scripts/inference_bert_wwm.py
test['Age_PclXSex'] = test[['Age', 'Pclass', 'Sex']].apply(age_PclassSex, axis = 1 )<drop_column>
%%time
!python.. /input/tweet-inference-scripts/inference_albert.py
test.drop(['Cabin', 'Age'], axis =1 , inplace = True )<groupby>
%%time
!pip install /kaggle/input/bertweet-libs/sacrebleu-1.4.10-py3-none-any.whl
!cp -R /kaggle/input/bertweet-libs/fairseq-0.9.0/fairseq-0.9.0 /kaggle/working
!cp -R /kaggle/input/bertweet-libs/fastBPE-0.1.0/fastBPE-0.1.0/ /kaggle/working
!pip install /kaggle/working/fairseq-0.9.0/
!pip install /kaggle/working/fastBPE-0.1.0/
!python.. /input/tweet-inference-scripts/inference_bertweet.py
Fare_med = test[['Pclass','Fare','Sex', 'Embarked']].groupby(['Pclass','Sex', 'Embarked'] ).agg(['count', 'mean'])
 Fare_med<correct_missing_values>
%%time
!python.. /input/tweet-inference-scripts/inference_roberta_anton.py
test['Fare'].fillna(12.718, inplace = True )<categorify>
%%time
!python.. /input/tweet-inference-scripts/inference_roberta_large_hiki.py
Sex_Dumm = pd.get_dummies(test['Sex'], drop_first = True)
 Embarked_Dumm = pd.get_dummies(test['Embarked'], drop_first = True)
 Ticket_Grp = pd.get_dummies(test['Ticket_Grp'], drop_first = True, prefix = 'Ticket')
 Salute_Group = pd.get_dummies(test['Salute_Grp'], drop_first = True )<concatenate>
%%time
!python.. /input/tweet-inference-scripts/inference_roberta_hiki.py
test = pd.concat([test, Sex_Dumm, Embarked_Dumm, Ticket_Grp, Salute_Group], axis = 1)
 test.columns<prepare_x_and_y>
def string_from_preds_char_level(texts, preds):
selected_texts = []
n_models = len(preds)
 for idx in range(len(texts)) :
data = texts[idx]
start_probas = np.mean(
 [preds[i][0][idx] for i in range(n_models)], 0)
 end_probas = np.mean(
 [preds[i][1][idx] for i in range(n_models)], 0)
 start_idx = np.argmax(start_probas)
 end_idx = np.argmax(end_probas)
 if end_idx < start_idx:
selected_text = data
else:
selected_text = data[start_idx: end_idx]
selected_texts.append(selected_text.strip())
 return selected_texts
df_test = pd.read_csv(
 '.. /input/tweet-sentiment-extraction/test.csv' ).fillna('')
 df_test['selected_text'] = ''
sub = pd.read_csv(
 '.. /input/tweet-sentiment-extraction/sample_submission.csv')
 preds_1 = np.load('preds_char_test_cnn_2_2.npy')
 preds_2 = np.load('preds_char_test_wavenet_4_3.npy')
 preds_3 = np.load('preds_char_test_rnn_5_3.npy')
 preds_4 = np.load('preds_char_test_wavenet_0_3.npy')
 test_preds =(preds_1 + preds_2 + preds_3 + preds_4)/ 4
selected_texts = string_from_preds_char_level(
 df_test['text'].values, test_preds)
 sub['selected_text'] = selected_texts
df_test['selected_text'] = selected_texts
sub.to_csv('submission.csv', index=False )