Kaggle competition: Tweet Sentiment Extraction
<SOS> metric: Jaccard Kaggle data source: tweet-sentiment-extraction<import_modules>
from sklearn.ensemble import GradientBoostingClassifier
from sklearn.ensemble import RandomForestClassifier
from sklearn.neighbors import KNeighborsClassifier
from sklearn.svm import SVC
import xgboost as xgb
import lightgbm as lgb
from sklearn.linear_model import LogisticRegression
from sklearn.ensemble import VotingClassifier
from sklearn.model_selection import cross_val_score
from sklearn.model_selection import cross_val_predict
from sklearn.metrics import confusion_matrix
from sklearn.model_selection import GridSearchCV<train_on_grid>
!pip install /kaggle/input/fairseq-and-fastbpe/sacrebleu-1.4.9-py3-none-any.whl
!pip install /kaggle/input/fairseq-and-fastbpe/fairseq-0.9.0-cp37-cp37m-linux_x86_64.whl
!pip install /kaggle/input/fairseq-and-fastbpe/fastBPE-0.1.0-cp37-cp37m-linux_x86_64.whl
param_grid = [
{'n_neighbors':[2,3,4,5,6,7,8], 'weights':['uniform', 'distance'], 'n_jobs':[-1] }
]
neigh = KNeighborsClassifier()
grid_search = GridSearchCV(neigh, param_grid, cv=5, scoring='accuracy')
 grid_search.fit(X_train, y_train)
 grid_search.best_params_<train_on_grid>
warnings.filterwarnings("ignore" )
neigh = KNeighborsClassifier(n_neighbors = 5, weights = 'uniform')
 neigh.fit(X_train, y_train)
 scores = cross_val_score(neigh, X_train, y_train, scoring="accuracy", cv=10)
 print(scores.mean() )<train_on_grid>
def seed_everything(seed_value):
random.seed(seed_value)
 np.random.seed(seed_value)
 torch.manual_seed(seed_value)
 os.environ['PYTHONHASHSEED'] = str(seed_value)
 if torch.cuda.is_available() :
torch.cuda.manual_seed(seed_value)
 torch.cuda.manual_seed_all(seed_value)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = True
seed = 777
seed_everything(seed )
log_reg = LogisticRegression(C = 0.001, penalty='none')
 log_reg.fit(X_train, y_train)
 scores = cross_val_score(log_reg, X_train, y_train, scoring="accuracy", cv=10)
 print("Logistic Regression mean score : ", scores.mean())

forest_clf = RandomForestClassifier(max_depth = 6, min_samples_leaf=1, min_samples_split=2, n_estimators=100)
 forest_clf.fit(X_train, y_train)
 scores = cross_val_score(forest_clf, X_train, y_train, scoring="accuracy", cv=10)
 print("Random Forest mean score : ", scores.mean())

svc = SVC(C = 10, kernel='rbf', gamma='auto', probability=True)
 svc.fit(X_train, y_train)
 scores = cross_val_score(svc, X_train, y_train, scoring="accuracy", cv=10)
 print("SVC mean score: ", scores.mean())
 gbc = GradientBoostingClassifier(n_estimators=500, max_depth=3, learning_rate=0.01)
 gbc.fit(X_train, y_train)
 scores = cross_val_score(gbc, X_train, y_train, scoring="accuracy", cv=10)
 print("Gradient Boosting mean score : ", scores.mean())
 xgb_clf = xgb.XGBClassifier(n_estimators=500, max_depth=3, learning_rate=0.01)
 xgb_clf.fit(X_train, y_train)
 scores = cross_val_score(xgb_clf, X_train, y_train, scoring="accuracy", cv=10)
 print("XGB mean score : ", scores.mean())
<find_best_model_class>
class BERTweetTokenizer() :
def __init__(self, path):
self.bpe = fastBPE(SimpleNamespace(bpe_codes=os.path.join(path, 'bpe.codes')))
 self.vocab = Dictionary()
self.vocab.add_from_file(os.path.join(path, 'dict.txt'))
def encode(self, text):
tokens = self.tokenize(text)
 input_ids = self.convert_tokens_to_ids(tokens)
 offsets = []
curr_pos = 0
for idx, token in enumerate(tokens):
need_space = idx > 0 and(not tokens[idx-1].endswith('@@'))
curr_pos += need_space
offsets.append(( curr_pos, curr_pos + len(token.replace('@@', ''))))
curr_pos += len(token.replace('@@', ''))
return SimpleNamespace(ids=input_ids, tokens=tokens, offsets=offsets)
 def tokenize(self, text):
return self.bpe.encode(text ).split()
def convert_tokens_to_ids(self, tokens):
input_ids = self.vocab.encode_line(' '.join(tokens), append_eos=False, add_if_not_exist=False ).long().tolist()
return input_ids
def decode_tokens(self, tokens):
decoded = ' '.join(tokens ).replace('@@ ', '' ).strip()
return decoded
voting_clf = VotingClassifier(estimators=[('gbc', gbc),('xgb', xgb_clf),('forest', forest_clf),('svc', svc),('lrc', log_reg),('knc', neigh)],
voting='hard')
 voting_clf.fit(X_train, y_train)
 scores = cross_val_score(voting_clf, X_train, y_train, scoring="accuracy", cv=10)
 print(scores.mean() )<save_to_csv>
class config:
VERSION = 'v0-0-5-bertweet'
INPUT_DATASET = 'tweet-sentiment-extraction'
MODEL_DATASET = f'model-{VERSION}'
ROBERTA_DATASET = 'bertweet-base-transformers'
TRAIN = False
IS_KAGGLE = True
DATA_DIR = 'data' if not IS_KAGGLE else f'.. /input/{INPUT_DATASET}'
MODELS_DIR = os.path.join(
 'models' if not IS_KAGGLE else '.. /input',
MODEL_DATASET 
)
 ROBERTA_DIR = os.path.join(
 'transformers_data' if not IS_KAGGLE else '.. /input',
ROBERTA_DATASET 
)
 FOLD = None
N_FOLDS = 5
LEARNING_RATE = 3e-5
N_LAST_LAYERS = 3
MAX_LEN = 120
TRAIN_BATCH_SIZE = 32
VALID_BATCH_SIZE = 32
EPOCHS = 5
TRAINING_FILE = os.path.join(DATA_DIR, 'train.csv')
 TEST_FILE = os.path.join(DATA_DIR, 'test.csv')
 SAMPLE_FILE = os.path.join(DATA_DIR, 'sample_submission.csv')
 TOKENIZER = BERTweetTokenizer(ROBERTA_DIR)
 @classmethod
def get_model_paths(cls):
res = []
for filename in os.listdir(cls.MODELS_DIR):
if filename.endswith('.bin'):
res.append(os.path.join(cls.MODELS_DIR, filename))
return res
preds = voting_clf.predict(X_test)
 output = pd.DataFrame({'PassengerId': test_data.PassengerId, 'Survived': preds})
 output.to_csv("submission.csv", index=False)
 output.head()<set_options>
def jaccard(str1, str2):
a = set(str1.lower().split())
 b = set(str2.lower().split())
 c = a.intersection(b)
 return float(len(c)) /(len(a)+ len(b)- len(c))
class AverageMeter:

 def __init__(self):
self.reset()
def reset(self):
self.val = 0
self.avg = 0
self.sum = 0
self.count = 0
def update(self, val, n=1):
self.val = val
self.sum += val * n
self.count += n
self.avg = self.sum / self.count
def upload_kaggle_dataset(dir_path, id_, user='drack3800', title=None):
title = title or id_
metadata = {
"title": title,
"id": f"{user}/{id_}",
"licenses": [
{
"name": "CC0-1.0"
}
]
}
with open(os.path.join(dir_path, 'dataset-metadata.json'), 'w')as fout:
fout.write(json.dumps(metadata)+ '
')
 proc = subprocess.run(['kaggle', 'datasets', 'create', '-p' , dir_path], capture_output=True)
 print(proc.stdout.decode())
warnings.filterwarnings('ignore')
 pd.options.display.max_columns = 40<load_from_csv>
class TweetDataset(torch.utils.data.Dataset):
def __init__(self, df, max_len=config.MAX_LEN):
self.df = df
self.max_len = max_len
self.labeled = 'selected_text' in df
self.tokenizer = config.TOKENIZER
def __getitem__(self, index):
data = {}
row = self.df.iloc[index]
ids, masks, tweet, offsets, token_type_ids = self.get_input_data(row)
 data['ids'] = ids
data['mask'] = masks
data['tweet'] = tweet
data['offsets'] = offsets
data['token_type_ids'] = token_type_ids
data['sentiment'] = row.sentiment
if self.labeled:
start_idx, end_idx = self.get_target_idx(row, tweet, offsets)
 data['start_idx'] = start_idx
data['end_idx'] = end_idx
data['selected_tweet'] = " ".join(str(row.selected_text ).lower().split())
 return data
def __len__(self):
return len(self.df)
 def get_input_data(self, row):
tweet = " ".join(str(row.text ).lower().split())
 encoding = self.tokenizer.encode(tweet)
 sentiment_id = self.tokenizer.encode(row.sentiment ).ids
ids = [0] + sentiment_id + [2, 2] + encoding.ids + [2]
offsets = [(0, 0)] * 4 + encoding.offsets + [(0, 0)]
pad_len = self.max_len - len(ids)
 if pad_len > 0:
ids += [1] * pad_len
offsets += [(0, 0)] * pad_len
ids = torch.tensor(ids)
 masks = torch.where(ids != 1, torch.tensor(1), torch.tensor(0))
offsets = torch.tensor(offsets)
 token_type_ids = torch.zeros(len(ids), dtype=torch.int)
 return ids, masks, tweet, offsets, token_type_ids
def get_target_idx(self, row, tweet, offsets):
selected_text = " ".join(str(row.selected_text ).lower().split())
 len_st = len(selected_text)
 idx0 = None
idx1 = None
for ind in range(len(tweet)) :
if tweet[ind: ind + len_st] == selected_text:
idx0 = ind
idx1 = ind + len_st - 1
break
char_targets = [0] * len(tweet)
 if idx0 != None and idx1 != None:
for ct in range(idx0, idx1 + 1):
char_targets[ct] = 1
target_idx = []
for j,(offset1, offset2)in enumerate(offsets):
if sum(char_targets[offset1: offset2])> 0:
target_idx.append(j)
 start_idx = target_idx[0]
end_idx = target_idx[-1]
return start_idx, end_idx
train_data = pd.read_csv("/kaggle/input/titanic/train.csv")
 test_data = pd.read_csv("/kaggle/input/titanic/test.csv" )<concatenate>
class TweetModel(nn.Module):
def __init__(self):
super().__init__()
conf = transformers.RobertaConfig.from_pretrained(
 config.ROBERTA_DIR,
output_hidden_states=True, 
)
 self.roberta = transformers.RobertaModel.from_pretrained(
 os.path.join(config.ROBERTA_DIR, 'model.bin'),
config=conf 
)
 self.drop_out = nn.Dropout(0.5)
 self.l0 = nn.Linear(768, 2)
 nn.init.normal_(self.l0.weight, std=0.02)
 nn.init.normal_(self.l0.bias, 0)
 self.layer_weights = nn.Parameter(torch.ones(config.N_LAST_LAYERS)/ config.N_LAST_LAYERS)
 def forward(self, ids, mask, token_type_ids=None):
_, _, out = self.roberta(
 ids,
attention_mask=mask,
token_type_ids=token_type_ids 
)
 last_layers_out = torch.stack([out[-i] for i in range(1, config.N_LAST_LAYERS + 1)], dim=0)
 layer_weights = torch.softmax(self.layer_weights, dim=0)
 out = torch.sum(layer_weights[:, None, None, None] * last_layers_out, dim=0)
 out = self.drop_out(out)
 logits = self.l0(out)
 start_logits, end_logits = logits.split(1, dim=-1)
 start_logits = start_logits.squeeze(-1)
 end_logits = end_logits.squeeze(-1)
 return start_logits, end_logits
def build_model(state_dict_path=None, train=True, device=None):
model = TweetModel()
if device:
model.to(device)
 if state_dict_path:
print(f"Loaded model from {state_dict_path}")
 model.load_state_dict(torch.load(state_dict_path))
model.train(train)
 return model
full_dataset = pd.concat([train_data, test_data])
 full_dataset.head()<categorify>
def loss_fn(start_logits, end_logits, start_positions, end_positions):
loss_fct = nn.CrossEntropyLoss()
start_loss = loss_fct(start_logits, start_positions)
 end_loss = loss_fct(end_logits, end_positions)
 total_loss =(start_loss + end_loss)
 return total_loss
class KnownCabinTransformer(BaseEstimator, TransformerMixin):

 def __init__(self):
print("- KnownCabin transformer initiated -")
 def fit(self, X, y=None):
return self
def transform(self, X, y=None):
X.Cabin.fillna(0, inplace=True)
 X["KnownCabin"] = 0
X.loc[X.Cabin != 0, 'KnownCabin'] = 1
print("- KnownCabin transformer applied -")
 return X 
<categorify>
def train_fn(data_loader, model, optimizer, device, num_batches, scheduler=None):
ewma_loss = None
model.train()
tk0 = tqdm(data_loader, total=num_batches, desc="Training")
 for bi, d in enumerate(tk0):
ids = d["ids"]
token_type_ids = d["token_type_ids"]
mask = d["mask"]
targets_start = d['start_idx']
targets_end = d['end_idx']
ids = ids.to(device, dtype=torch.long)
 token_type_ids = token_type_ids.to(device, dtype=torch.long)
 mask = mask.to(device, dtype=torch.long)
 targets_start = targets_start.to(device, dtype=torch.long)
 targets_end = targets_end.to(device, dtype=torch.long)
 model.zero_grad()
outputs_start, outputs_end = model(
 ids=ids,
mask=mask,
token_type_ids=token_type_ids, 
)
 loss = loss_fn(outputs_start, outputs_end, targets_start, targets_end)
 loss.backward()
optimizer.step()
if scheduler is not None:
scheduler.step()
loss_item = loss.item()
if ewma_loss is None:
ewma_loss = loss_item
else:
ewma_loss = 0.8 * ewma_loss + 0.2 * loss_item
tk0.set_postfix(loss=loss_item, ewma=ewma_loss)
 return ewma_loss
class TitleTransformer(BaseEstimator, TransformerMixin):

 def __init__(self):
print("- Title transformer initiated -")
 def fit(self, X, y=None):
return self
def transform(self, X, y=None):
X['Title'] = X.Name.str.extract('([A-Za-z]+)\.', expand=False)
 X['Title'] = X['Title'].replace(['Lady', 'Countess','Capt', 'Col',
'Don', 'Dr', 'Major', 'Rev',
'Sir', 'Jonkheer', 'Dona'],
'Rare')
 X['Title'] = X['Title'].replace(['Mlle', 'Ms'], 'Miss')
 X['Title'] = X['Title'].replace('Mme', 'Mrs')
 X['Title'] = X['Title'].fillna('Rare')
 title_dummies = pd.get_dummies(X['Title'], prefix="Title")
 X = pd.concat([X, title_dummies], axis=1)
 X = X.drop('Title', axis=1)
 print("- Title transformer applied -")
 return X<categorify>
def run(model, df_train, df_valid, model_name, device='cuda'):
model = model.to(device)
 train_dataset = TweetDataset(df_train)
 train_data_loader = torch.utils.data.DataLoader(
 train_dataset,
batch_size=config.TRAIN_BATCH_SIZE,
shuffle=True,
drop_last=True,
num_workers=2 
)
 valid_dataset = TweetDataset(df_valid)
 valid_data_loader = torch.utils.data.DataLoader(
 valid_dataset,
batch_size=config.VALID_BATCH_SIZE,
shuffle=False,
drop_last=False,
num_workers=1 
)
 num_train_steps = int(len(df_train)/ config.TRAIN_BATCH_SIZE * config.EPOCHS)
 param_optimizer = list(model.named_parameters())
 optimizer = AdamW(
 model.parameters() ,
lr=config.LEARNING_RATE 
)
 scheduler = get_linear_schedule_with_warmup(
 optimizer,
num_warmup_steps=int(0.1 * num_train_steps),
num_training_steps=num_train_steps 
)
 best_jac = 0
num_batches = int(len(df_train)/ config.TRAIN_BATCH_SIZE)
 print("Training is Starting.... ")
 for epoch in range(config.EPOCHS):
ewma_loss = train_fn(
 train_data_loader,
model,
optimizer,
device,
num_batches,
scheduler 
)
 jac = eval_fn(
 valid_data_loader,
model,
device 
)
 print(f'Epoch={epoch}, Jaccard={jac:.4f}, Train Loss={ewma_loss:.4f}')
 if jac > best_jac:
model_path = os.path.join(
 config.MODELS_DIR,
f'{model_name}.bin' 
)
 os.makedirs(config.MODELS_DIR, exist_ok=True)
 print(f"Model Improved! Saving model to {model_path}")
 torch.save(model.state_dict() , model_path)
 best_jac = jac
return best_jac
def generate_folds(dfx):
skf = model_selection.StratifiedKFold(n_splits=config.N_FOLDS, random_state=777)
 dfx = dfx.copy()
dfx['kfold'] = 0
for n_fold,(_, val_idx)in enumerate(skf.split(dfx.values, dfx.sentiment.values)) :
dfx.loc[val_idx, 'kfold'] = n_fold
return dfx
class MissingFareTransformer(BaseEstimator, TransformerMixin):

 def __init__(self):
print("- MissingFare transformer initiated -")
 def fit(self, X, y=None):
return self
def transform(self, X, y=None):
X.loc[(X.Pclass == 1)&(X.Fare.isnull()), 'Fare'] = X.loc[X.Pclass == 1]["Fare"].median()
X.loc[(X.Pclass == 2)&(X.Fare.isnull()), 'Fare'] = X.loc[X.Pclass == 2]["Fare"].median()
X.loc[(X.Pclass == 3)&(X.Fare.isnull()), 'Fare'] = X.loc[X.Pclass == 3]["Fare"].median()
print("- MissingFare transformer applied -")
 return X
class EmbarkedTransformer(BaseEstimator, TransformerMixin):

 def __init__(self):
print("- MissingEmbarked transformer initiated -")
 def fit(self, X, y=None):
return self
def transform(self, X, y=None):
self.most_frequent = X["Embarked"].value_counts().index[0]
X["Embarked"].fillna(self.most_frequent, inplace=True)
 embarked_dummies = pd.get_dummies(X['Embarked'], prefix="Embarked")
 X = pd.concat([X, embarked_dummies], axis=1)
 X = X.drop('Embarked', axis=1)
 print("- MissingEmbarked transformer applied -")
 return X
class HypotheticalMissingsTransformer(BaseEstimator, TransformerMixin):

 def __init__(self):
print("- HypotheticalMissings transformer initiated -")
 def fit(self, X, y=None):
return self
def transform(self, X, y=None):
self.most_frequent_class = X["Pclass"].value_counts().index[0]
self.unknown_name = "unknown"
self.horizontal_family = 0
self.vertical_family = 0
X["Pclass"].fillna(self.most_frequent_class, inplace=True)
 X["Name"].fillna(self.unknown_name, inplace=True)
 X["SibSp"].fillna(self.horizontal_family, inplace=True)
 X["Parch"].fillna(self.vertical_family, inplace=True)
 X.loc[(X.Sex.isnull())&(X.Title_Master == 1), "Sex"] == "male"
X.loc[(X.Sex.isnull())&(X.Title_Mr == 1), "Sex"] == "male"
X.loc[(X.Sex.isnull())&(X.Title_Rare == 1), "Sex"] == "male"
X.loc[(X.Sex.isnull())&(X.Title_Miss == 1), "Sex"] == "female"
X.loc[(X.Sex.isnull())&(X.Title_Mrs == 1), "Sex"] == "female"
print("- HypotheticalMissings transformer applied -")
 return X
class GenderTransformer(BaseEstimator, TransformerMixin):

 def __init__(self):
print("- Gender transformer initiated -")
 def fit(self, X, y=None):
return self
def transform(self, X, y=None):
X['Sex'] = X['Sex'].map({'male': 0, 'female': 1})
 print("- Gender transformer applied -")
 return X
class PclassTransformer(BaseEstimator, TransformerMixin):

 def __init__(self):
print("- Pclass transformer initiated -")
 def fit(self, X, y=None):
return self
def transform(self, X, y=None):
pclass_dummies = pd.get_dummies(X['Pclass'], prefix='Pclass')
 X = pd.concat([X, pclass_dummies], axis=1)
 X = X.drop('Pclass', axis=1)
 print("- Pclass transformer applied -")
 return X
class FamilySizeTransformer(BaseEstimator, TransformerMixin):

 def __init__(self):
print("- FamilySize transformer initiated -")
 def fit(self, X, y=None):
return self
def transform(self, X, y=None):
X["FamilySize"] = X["SibSp"] + X["Parch"] + 1
print("- FamilySize transformer applied -")
 return X<categorify>
class AgeStatusTransformer(BaseEstimator, TransformerMixin):

 def __init__(self):
print("- AgeStatus transformer initiated -")
 def fit(self, X, y=None):
return self
def transform(self, X, y=None):
X["AgeStatus"] = "known"
X.loc[(X["Age"] > 1)&(( X["Age"]*2)%2 != 0), "AgeStatus"] = "estimated"
X.loc[X["Age"].isnull() , "AgeStatus"] = "guessed"
age_status_dummies = pd.get_dummies(X['AgeStatus'], prefix="AgeStatus")
 X = pd.concat([X, age_status_dummies], axis=1)
 X = X.drop('AgeStatus', axis=1)
 print("- AgeStatus transformer applied -")
 return X 
<categorify>
if config.TRAIN:
fold_metrics = []
dfx = pd.read_csv(config.TRAINING_FILE)
 dfx = generate_folds(dfx)
 if config.FOLD is not None:
folds = [config.FOLD]
print("Going to train in single-fold mode")
 else:
folds = list(range(config.N_FOLDS))
print("Going to train in all-folds mode")
 for n_fold in folds:
model_name = f'model_{n_fold}'
MX = build_model()
df_train = dfx[dfx.kfold != n_fold].reset_index(drop=True)
 df_valid = dfx[dfx.kfold == n_fold].reset_index(drop=True)
 print(f'
>>> Running training for fold={n_fold}, model_name={model_name}')
 jac = run(MX, df_train, df_valid, model_name, device='cuda')
 print(f'>>> Finished training for fold={n_fold}.Jaccard = {jac:.4f}')
 fold_metrics.append(jac)
 print('
*********** REPORT **********')
 print(f'CV Jaccard = {np.mean(fold_metrics):.4f} +- {np.std(fold_metrics):.4f}')
class TicketGroupingTransformer(BaseEstimator, TransformerMixin):

 def __init__(self):
print("- TicketGrouping transformer initiated -")
 def fit(self, X, y=None):
return self
def transform(self, X, y=None):
cat_feature = ['Ticket']
count_enc = ce.CountEncoder(cols=cat_feature)
 count_enc.fit(X[cat_feature])
 grouping_col = count_enc.transform(X[cat_feature] ).add_suffix("_grouping")
 X = pd.concat([X, grouping_col], axis=1)
 print("- TicketGrouping transformer applied -")
 return X<categorify>
if config.TRAIN:
upload_kaggle_dataset(config.MODELS_DIR, config.MODEL_DATASET )
class AgeGuessingTransformer(BaseEstimator, TransformerMixin):

 def __init__(self):
print("- AgeGuessing transformer initiated -")
 def fit(self, X, y=None):
return self
def transform(self, X, y=None):
X.loc[(X.Age.isnull() == True)&(X.Title_Master == 1), "Age"] = X.loc[X.Title_Master == 1]["Age"].median()
X.loc[(X.Age.isnull() == True)&(X.SibSp > 0)&(X.Title_Miss == 1), "Age"] = X.loc[(X.Title_Miss == 1)&(X.SibSp > 0)]["Age"].median()
X.loc[(X.Age.isnull() == True)&(X.Ticket_grouping == 1), "Age"] = X.loc[(X.Ticket_grouping == 1)]["Age"].median()
X.loc[(X.Age.isnull() == True)&(X.SibSp > 1), "Age"] = X.loc[(X.Age.isnull() == False)&(X.SibSp > 1)]["Age"].median()
X.loc[(X.Age.isnull()), "Age"] = X.loc[(X.Age.isnull() == False)&(X.Ticket_grouping > 1)]["Age"].median()
print("- AgeGuessing transformer applied -")
 return X<categorify>
df_test = pd.read_csv(config.TEST_FILE)
 device = torch.device('cuda' )
class AgeBinningTransformer(BaseEstimator, TransformerMixin):

 def __init__(self):
print("- AgeBinning transformer initiated -")
 def fit(self, X, y=None):
return self
def transform(self, X, y=None):
self.bins = [0, 2, 16, 25, 40, 50, np.inf]
self.age_cat = ["0_2", "2_16", "16_25", "25_40", "30_50", "50+"]
X["AgeBins"] = pd.cut(X["Age"], bins=self.bins, labels=self.age_cat)
 age_bins_dummies = pd.get_dummies(X['AgeBins'], prefix="AgeBins")
 X = pd.concat([X, age_bins_dummies], axis=1)
 print("- AgeBinning transformer applied -")
 return X<categorify>
final_output_per_model = []
test_dataset = TweetDataset(df_test)
 data_loader = torch.utils.data.DataLoader(
 test_dataset,
batch_size=config.VALID_BATCH_SIZE,
shuffle=False,
num_workers=1 
)
 for model_path in config.get_model_paths() :
model = build_model(model_path, train=False, device=device)
 final_output = []
with torch.no_grad() :
tk0 = tqdm(data_loader, total=len(data_loader))
for bi, d in enumerate(tk0):
ids = d["ids"]
token_type_ids = d["token_type_ids"]
mask = d["mask"]
ids = ids.to(device, dtype=torch.long)
 token_type_ids = token_type_ids.to(device, dtype=torch.long)
 mask = mask.to(device, dtype=torch.long)
 output_start, output_end = model(ids=ids, mask=mask, token_type_ids=token_type_ids)
 output_start = torch.softmax(output_start, dim=1 ).cpu().detach().numpy()
output_end = torch.softmax(output_end, dim=1 ).cpu().detach().numpy()
final_output.append((
output_start,
output_end, 
))
final_output_per_model.append(final_output )
custom_pipeline = Pipeline([ 
("cabin_trans", KnownCabinTransformer()), 
("title_trans", TitleTransformer()), 
("hypo_missing_trans", HypotheticalMissingsTransformer()), 
("missing_fare_trans", MissingFareTransformer()), 
("embarked_trans", EmbarkedTransformer()), 
("gender_trans", GenderTransformer()), 
("class_trans", PclassTransformer()), 
("family_size_trans", FamilySizeTransformer()), 
("ticket_grp_trans", TicketGroupingTransformer()), 
("age_status_trans", AgeStatusTransformer()), 
("age_guessing_trans", AgeGuessingTransformer()), 
("age_binning_trans", AgeBinningTransformer()),
])
 full_dataset = custom_pipeline.fit_transform(full_dataset )<feature_engineering>
final_output = []
with torch.no_grad() :
tk0 = tqdm(data_loader, total=len(data_loader))
for bi, d in enumerate(tk0):
ids = d["ids"]
token_type_ids = d["token_type_ids"]
mask = d["mask"]
sentiment = d["sentiment"]
tweet = d["tweet"]
offsets = d["offsets"].numpy()
ids = ids.to(device, dtype=torch.long)
 token_type_ids = token_type_ids.to(device, dtype=torch.long)
 mask = mask.to(device, dtype=torch.long)
 outputs_start = [x[bi][0] for x in final_output_per_model]
outputs_end = [x[bi][1] for x in final_output_per_model]
output_start = sum(outputs_start)/ len(outputs_start)
 output_end = sum(outputs_end)/ len(outputs_end)
 for i in range(len(ids)) :
start_pred = np.argmax(output_start[i])
 end_pred = np.argmax(output_end[i])
 if start_pred > end_pred or sentiment[i] == 'neural' or len(tweet[i].split())< 3:
pred = tweet[i]
else:
pred = get_selected_text(tweet[i], start_pred, end_pred, offsets[i])
 final_output.append(pred)
full_dataset["TS_Tragedy"] = "null"
i = 0
unique_tickets_subset = full_dataset.loc[full_dataset["Ticket_grouping"] > 2]["Ticket"].unique()
while i < len(unique_tickets_subset):
subset = full_dataset.loc[full_dataset.Ticket == unique_tickets_subset[i]]
try:
ratio = len(subset.loc[subset["Survived"] == 0])/ len(subset.loc[subset.Survived.isnull() == False])
 if ratio > 0.65 :
full_dataset.loc[full_dataset.Ticket == unique_tickets_subset[i], "TS_Tragedy"] = "yes"
else:
full_dataset.loc[full_dataset.Ticket == unique_tickets_subset[i], "TS_Tragedy"] = "no"
i += 1
except ZeroDivisionError:
i += 1
TST_dummies = pd.get_dummies(full_dataset['TS_Tragedy'], prefix="TST")
 full_dataset = pd.concat([full_dataset, TST_dummies], axis=1)
 full_dataset = full_dataset.drop('TS_Tragedy', axis=1)
 full_dataset.head(3)
<prepare_output>
sample = pd.read_csv(config.SAMPLE_FILE)
 sample.loc[:, 'selected_text'] = final_output
<prepare_x_and_y><EOS> 
if config.TRAIN:
subm_file = f'{config.VERSION}_submission.csv'
else:
subm_file = 'submission.csv'
sample.to_csv(subm_file, index=False)
 print("saved to", subm_file )